Add a custom test runner script (#355)

* Add a custom test runner script

At the moment it supports only tests that output warnings or errors (test
type = output_check). More types can be added in the future.
This commit is contained in:
Zeex 2018-08-10 23:43:02 +06:00 committed by GitHub
parent ae53783c0e
commit 29a6df8163
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 251 additions and 111 deletions

View File

@ -20,12 +20,9 @@ before_script:
script:
- make
- make test
- make pawncc_tests
- make package
after_failure:
- cat Testing/Temporary/LastTest.log
deploy:
provider: releases
api_key:

View File

@ -12,7 +12,7 @@ build_script:
- cmake --build . --config %CONFIGURATION% --target package
test_script:
- ctest --build-config %CONFIGURATION%
- cmake --build . --config %CONFIGURATION% --target pawncc_tests
on_failure:
- type Testing\Temporary\LastTest.log<Paste>

View File

@ -1,96 +1,9 @@
set(DEFAULT_COMPILER_OPTIONS
-i${CMAKE_SOURCE_DIR}/include
"-\;+"
"-(+")
find_package(PythonInterp 2.7 REQUIRED)
function(add_compiler_test test_name options)
add_test(NAME ${test_name}
COMMAND $<TARGET_FILE:pawncc> ${DEFAULT_COMPILER_OPTIONS} ${options})
set_tests_properties(${test_name} PROPERTIES
ENVIRONMENT PATH=$<TARGET_FILE_DIR:pawnc>)
endfunction()
# Compile tests
#
# These tests compare compile output against a regular expression and fail if the output
# doesn't match the expected pattern.
add_compiler_test(gh_217 ${CMAKE_CURRENT_SOURCE_DIR}/gh_217.pwn)
set_tests_properties(gh_217 PROPERTIES PASS_REGULAR_EXPRESSION ".*\\.pwn\\(11\\) : warning 237: user warning: this is warning 1
.*\\.pwn\\(13\\) : warning 237: user warning: this is warning 2
.*\\.pwn\\(15\\) : warning 237: user warning: this is warning 3
.*\\.pwn\\(17\\) : warning 237: user warning: this is warning 4
.*\\.pwn\\(28\\) : warning 234: function is deprecated \\(symbol \"f\"\\) don't use this function please
.*\\.pwn\\(33\\) : warning 234: function is deprecated \\(symbol \"f\"\\) don't use this function please
")
add_compiler_test(reset_errline_gh_230 ${CMAKE_CURRENT_SOURCE_DIR}/reset_errline_gh_230.pwn)
set_tests_properties(reset_errline_gh_230 PROPERTIES PASS_REGULAR_EXPRESSION
".*\\.pwn\\(2\\) : error 017: undefined symbol \\\"undefined\\\"
.*\\.pwn\\(2\\) : warning 215: expression has no effect
.*\\.pwn\\(7\\) : warning 204: symbol is assigned a value that is never used: \\\"y\\\"
.*\\.pwn\\(4\\) : warning 204: symbol is assigned a value that is never used: \\\"x\\\"
")
add_compiler_test(unused_symbol_line_gh_252 ${CMAKE_CURRENT_SOURCE_DIR}/unused_symbol_line_gh_252.pwn)
set_tests_properties(unused_symbol_line_gh_252 PROPERTIES PASS_REGULAR_EXPRESSION
".*\\.pwn\\(4\\) : warning 203: symbol is never used: \\\"y\\\"
.*\\.pwn\\(8\\) : warning 203: symbol is never used: \\\"z\\\"
.*\\.pwn\\(1\\) : warning 203: symbol is never used: \\\"x\\\"
")
add_compiler_test(gh_283 ${CMAKE_CURRENT_SOURCE_DIR}/gh_283.pwn)
set_tests_properties(gh_283 PROPERTIES PASS_REGULAR_EXPRESSION
".*\\.pwn\\(5\\) : warning 234: function is deprecated \\(symbol \"print\"\\)")
add_compiler_test(too_many_args_crash_gh_298 ${CMAKE_CURRENT_SOURCE_DIR}/too_many_args_crash_gh_298.pwn)
set_tests_properties(too_many_args_crash_gh_298 PROPERTIES PASS_REGULAR_EXPRESSION "too many function arguments")
add_compiler_test(meaningless_class_specifiers_gh_172 ${CMAKE_CURRENT_SOURCE_DIR}/meaningless_class_specifiers_gh_172.pwn)
set_tests_properties(meaningless_class_specifiers_gh_172 PROPERTIES PASS_REGULAR_EXPRESSION
".*\\.pwn\\(1\\) : warning 238: meaningless combination of class specifiers \\(const reference\\)
.*\\.pwn\\(1 \\-\\- 2\\) : warning 238: meaningless combination of class specifiers \\(const variable arguments\\)
")
add_compiler_test(constexpr_result_prop_gh_308 ${CMAKE_CURRENT_SOURCE_DIR}/constexpr_result_prop_gh_308.pwn)
set_tests_properties(constexpr_result_prop_gh_308 PROPERTIES PASS_REGULAR_EXPRESSION
".*\\.pwn\\(2\\) : warning 237: user warning: \\\"Test passed.\\\"
.*\\.pwn\\(6\\) : warning 237: user warning: \\\"Test passed.\\\"
.*\\.pwn\\(10\\) : warning 237: user warning: \\\"Test passed.\\\"
.*\\.pwn\\(14\\) : warning 237: user warning: \\\"Test passed.\\\"
")
add_compiler_test(const_array_args_and_literals_gh_276 ${CMAKE_CURRENT_SOURCE_DIR}/const_array_args_and_literals_gh_276.pwn)
set_tests_properties(const_array_args_and_literals_gh_276 PROPERTIES PASS_REGULAR_EXPRESSION
".*\\.pwn\\(13\\) : warning 214: possibly a \\\"const\\\" array argument was intended: \\\"arr\\\"
.*\\.pwn\\(18\\) : warning 214: possibly a \\\"const\\\" array argument was intended: \\\"arr\\\"
.*\\.pwn\\(30\\) : warning 214: possibly a \\\"const\\\" array argument was intended: \\\"arr\\\"
.*\\.pwn\\(39\\) : warning 239: literal array/string passed to a non-const parameter
.*\\.pwn\\(40\\) : warning 239: literal array/string passed to a non-const parameter
.*\\.pwn\\(41\\) : warning 239: literal array/string passed to a non-const parameter
")
add_compiler_test(md_array_size_chk_gh_314 ${CMAKE_CURRENT_SOURCE_DIR}/md_array_size_chk_gh_314.pwn)
set_tests_properties(md_array_size_chk_gh_314 PROPERTIES PASS_REGULAR_EXPRESSION
"*\\.pwn\\(1\\) : error 009: invalid array size \\(negative, zero or out of bounds\\)
.*\\.pwn\\(2\\) : error 009: invalid array size \\(negative, zero or out of bounds\\)
.*\\.pwn\\(3\\) : error 009: invalid array size \\(negative, zero or out of bounds\\)
.*\\.pwn\\(5\\) : error 009: invalid array size \\(negative, zero or out of bounds\\)
.*\\.pwn\\(30\\) : warning 224: indeterminate array size in \"sizeof\" expression \\(symbol \"\"\\)
")
set_tests_properties(md_array_size_chk_gh_314 PROPERTIES WILL_FAIL TRUE)
add_compiler_test(destructor_not_impl_gh_310 ${CMAKE_CURRENT_SOURCE_DIR}/destructor_not_impl_gh_310.pwn)
set_tests_properties(destructor_not_impl_gh_310 PROPERTIES PASS_REGULAR_EXPRESSION
".*\\.pwn\\(6\\) : error 004: function \"operator~(Error:)\" is not implemented
")
set_tests_properties(destructor_not_impl_gh_310 PROPERTIES WILL_FAIL TRUE)
# Crashers
#
# These tests simply check that the compiler doesn't crash.
#
# TODO: Probably need to support tests that exist with a non-zero code but don't crash?
# Right now this will cause a failure.
add_compiler_test(md_array_crash_gh_220 ${CMAKE_CURRENT_SOURCE_DIR}/md_array_crash_gh_220.pwn)
add_custom_target(pawncc_tests COMMAND
COMMAND ${PYTHON_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/run_tests.py
-c $<TARGET_FILE:pawncc>
-i ../../../include
DEPENDS pawncc
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})

View File

@ -0,0 +1,11 @@
{
'test_type': 'output_check',
'errors': """
const_array_args_and_literals_gh_276.pwn(13) : warning 214: possibly a "const" array argument was intended: "arr"
const_array_args_and_literals_gh_276.pwn(18) : warning 214: possibly a "const" array argument was intended: "arr"
const_array_args_and_literals_gh_276.pwn(30) : warning 214: possibly a "const" array argument was intended: "arr"
const_array_args_and_literals_gh_276.pwn(39) : warning 239: literal array/string passed to a non-const parameter
const_array_args_and_literals_gh_276.pwn(40) : warning 239: literal array/string passed to a non-const parameter
const_array_args_and_literals_gh_276.pwn(41) : warning 239: literal array/string passed to a non-const parameter
"""
}

View File

@ -0,0 +1,9 @@
{
'test_type': 'output_check',
'errors': """
constexpr_result_prop_gh_308.pwn(2) : warning 237: user warning: "Test passed."
constexpr_result_prop_gh_308.pwn(6) : warning 237: user warning: "Test passed."
constexpr_result_prop_gh_308.pwn(10) : warning 237: user warning: "Test passed."
constexpr_result_prop_gh_308.pwn(14) : warning 237: user warning: "Test passed."
"""
}

View File

@ -0,0 +1,6 @@
{
'test_type': 'output_check',
'errors': """
destructor_not_impl_gh_310.pwn(4) : error 004: function "operator~(Error:)" is not implemented
"""
}

View File

@ -0,0 +1,11 @@
{
'test_type': 'output_check',
'errors': """
gh_217.pwn(11) : warning 237: user warning: this is warning 1
gh_217.pwn(13) : warning 237: user warning: this is warning 2
gh_217.pwn(15) : warning 237: user warning: this is warning 3
gh_217.pwn(17) : warning 237: user warning: this is warning 4
gh_217.pwn(28) : warning 234: function is deprecated (symbol "f") don't use this function please
gh_217.pwn(33) : warning 234: function is deprecated (symbol "f") don't use this function please
"""
}

View File

@ -1,6 +1,6 @@
// TODO: Check that string literals are concatenated correctly
native print(const s[]);
#include <console>
#define d1\
print("ok")

View File

@ -0,0 +1,6 @@
{
'test_type': 'output_check',
'errors': """
gh_283.pwn(5) : warning 234: function is deprecated (symbol "f")
"""
}

View File

@ -1,6 +1,6 @@
#pragma deprecated
native print(const string[]);
native f();
main() {
print("Hello World");
f();
}

View File

@ -0,0 +1,7 @@
{
'test_type': 'output_check',
'errors': """
md_array_crash_gh_220.pwn(6) : fatal error 111: user error: OK
Compilation aborted.
"""
}

View File

@ -3,4 +3,5 @@ new b[2000][500] = { { 0, -1, ... }, ... };
main() {
a[0][0] = b[0][0];
}
#error OK
}

View File

@ -0,0 +1,10 @@
{
'test_type': 'output_check',
'errors': """
md_array_size_chk_gh_314.pwn(1) : error 009: invalid array size (negative, zero or out of bounds)
md_array_size_chk_gh_314.pwn(2) : error 009: invalid array size (negative, zero or out of bounds)
md_array_size_chk_gh_314.pwn(3) : error 009: invalid array size (negative, zero or out of bounds)
md_array_size_chk_gh_314.pwn(5) : error 009: invalid array size (negative, zero or out of bounds)
md_array_size_chk_gh_314.pwn(30) : warning 224: indeterminate array size in "sizeof" expression (symbol "")
"""
}

View File

@ -1,8 +1,8 @@
new arr1[] = {};
new arr1[];
new arr2[5][];
new arr3[5][][5];
new arr4[5][5];
new arr5[][]= { { } };
new arr5[][];
f1(arr[]) {
#pragma unused arr

View File

@ -0,0 +1,7 @@
{
'test_type': 'output_check',
'errors': """
meaningless_class_specifiers_gh_172.pwn(1) : warning 238: meaningless combination of class specifiers (const reference)
meaningless_class_specifiers_gh_172.pwn(4) : warning 238: meaningless combination of class specifiers (const variable arguments)
"""
}

View File

@ -1,4 +1,6 @@
f1(const &v) { }
f1(const &v) {
#pragma unused v
}
f2(const ...) { }
f3(const v) {
#pragma unused v
@ -7,7 +9,9 @@ f4(...) { }
f5(v) {
#pragma unused v
}
f6(&v) { }
f6(&v) {
#pragma unused v
}
main() {
new a;
@ -17,4 +21,4 @@ main() {
f4(a);
f5(a);
f6(a);
}
}

View File

@ -0,0 +1,9 @@
{
'test_type': 'output_check',
'errors': """
reset_errline_gh_230.pwn(2) : error 017: undefined symbol \"undefined\"
reset_errline_gh_230.pwn(2) : warning 215: expression has no effect
reset_errline_gh_230.pwn(7) : warning 204: symbol is assigned a value that is never used: "y"
reset_errline_gh_230.pwn(4) : warning 204: symbol is assigned a value that is never used: "x"
"""
}

View File

@ -0,0 +1,132 @@
#!/usr/bin/env python
import argparse
import glob
import os.path
import re
import subprocess
import sys
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--compiler',
required=True,
help='path to the pawncc executable')
parser.add_argument('-i', '--include',
dest='include_dirs',
action='append',
help='add specified directory to include path')
options = parser.parse_args(sys.argv[1:])
def run_compiler(args):
process_args = [';+', '-(+']
if options.include_dirs is not None:
for dir in options.include_dirs:
process_args.append('-i' + dir)
if args is not None:
process_args += args
return subprocess.Popen(executable=options.compiler,
args=process_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
class OutputCheckTest:
def __init__(self,
name,
source_file,
errors=None,
extra_args=None):
self.name = name
self.source_file = source_file
self.errors = errors
self.extra_args = extra_args
def run(self):
args = [self.source_file]
if self.extra_args is not None:
args += extra_args
process = run_compiler(args=args)
stdout, stderr = process.communicate()
result = True
if self.errors is None:
if process.returncode != 0:
result = False
self.fail_reason = """
No errors specified and process exited with non-zero status
"""
else:
errors = stderr.decode('utf-8').splitlines()
errors = [e.strip() for e in errors if e.strip()]
expected_errors = self.errors.splitlines()
expected_errors = [e.strip() for e in expected_errors if e.strip()]
if errors != expected_errors:
result = False
self.fail_reason = (
'Error output didn\'t match\n\nExpected errors:\n\n{}\n\n'
'Actual errors:\n\n{}'
).format(
'\n'.join(expected_errors).strip(' \t\r\n'),
'\n'.join(errors).strip(' \t\r\n')
)
return result
class CrashTest:
def __init__(self, name, source_file, extra_args=None):
self.name = name
self.source_file = source_file
self.extra_args = extra_args
def run(self):
# TODO: Check if the process crashed.
return True
test_types = {
'output_check': OutputCheckTest,
'crash': CrashTest
}
tests = []
num_tests_disabled = 0
for meta_file in glob.glob('*.meta'):
name = os.path.splitext(meta_file)[0]
metadata = eval(open(meta_file).read(), None, None)
if metadata.get('disabled'):
num_tests_disabled += 1
continue
test_type = metadata['test_type']
if test_type == 'output_check':
tests.append(OutputCheckTest(name=name,
source_file=name + '.pwn',
errors=metadata.get('errors'),
extra_args=metadata.get('extra_args')))
elif test_type == 'crash':
tests.append(CrashTest(name=name, source_file=name + '.pwn'))
else:
raise KeyError('Unknown test type: ' + test_type)
num_tests = len(tests)
sys.stdout.write('DISCOVERED {} TEST{}'.format(num_tests, '' if num_tests == 1 else 'S'))
if num_tests_disabled > 0:
sys.stdout.write(' ({} DISABLED)'.format(num_tests_disabled))
sys.stdout.write('\n\n')
num_tests_failed = 0
for test in tests:
sys.stdout.write('Running ' + test.name + '... ')
if not test.run():
sys.stdout.write('FAILED\n')
print('Test {} failed for the following reason: {}'.format(
test.name, test.fail_reason))
print('')
num_tests_failed += 1
else:
sys.stdout.write('PASSED\n')
num_tests_passed = len(tests) - num_tests_failed
if num_tests_failed > 0:
print('\n{} TEST{} PASSED, {} FAILED'.format(
num_tests_passed,
'' if num_tests_passed == 1 else 'S',
num_tests_failed))
sys.exit(1)
else:
print('\nALL TESTS PASSED')

View File

@ -0,0 +1,11 @@
{
'test_type': 'output_check',
'errors': """
too_many_args_crash_gh_298.pwn(2) : error 045: too many function arguments
too_many_args_crash_gh_298.pwn(2) : warning 215: expression has no effect
too_many_args_crash_gh_298.pwn(2) : error 001: expected token: ";", but found ")"
too_many_args_crash_gh_298.pwn(2) : error 029: invalid expression, assumed zero
too_many_args_crash_gh_298.pwn(2) : fatal error 107: too many error messages on one line
Compilation aborted.
"""
}

View File

@ -1,5 +1,3 @@
native printf(const format[], ...);
main() {
printf("", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
}

View File

@ -0,0 +1,8 @@
{
'test_type': 'output_check',
'errors': """
unused_symbol_line_gh_252.pwn(4) : warning 203: symbol is never used: "y"
unused_symbol_line_gh_252.pwn(8) : warning 203: symbol is never used: "z"
unused_symbol_line_gh_252.pwn(1) : warning 203: symbol is never used: "x"
"""
}