diff --git a/.travis.yml b/.travis.yml index 11160e4..2378ac5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,12 +20,9 @@ before_script: script: - make - - make test + - make pawncc_tests - make package -after_failure: - - cat Testing/Temporary/LastTest.log - deploy: provider: releases api_key: diff --git a/appveyor.yml b/appveyor.yml index ec678af..78713fb 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -12,7 +12,7 @@ build_script: - cmake --build . --config %CONFIGURATION% --target package test_script: - - ctest --build-config %CONFIGURATION% + - cmake --build . --config %CONFIGURATION% --target pawncc_tests on_failure: - type Testing\Temporary\LastTest.log diff --git a/source/compiler/tests/CMakeLists.txt b/source/compiler/tests/CMakeLists.txt index 02110b4..26d4d97 100644 --- a/source/compiler/tests/CMakeLists.txt +++ b/source/compiler/tests/CMakeLists.txt @@ -1,96 +1,9 @@ -set(DEFAULT_COMPILER_OPTIONS - -i${CMAKE_SOURCE_DIR}/include - "-\;+" - "-(+") +find_package(PythonInterp 2.7 REQUIRED) -function(add_compiler_test test_name options) - add_test(NAME ${test_name} - COMMAND $ ${DEFAULT_COMPILER_OPTIONS} ${options}) - set_tests_properties(${test_name} PROPERTIES - ENVIRONMENT PATH=$) -endfunction() - -# Compile tests -# -# These tests compare compile output against a regular expression and fail if the output -# doesn't match the expected pattern. - -add_compiler_test(gh_217 ${CMAKE_CURRENT_SOURCE_DIR}/gh_217.pwn) -set_tests_properties(gh_217 PROPERTIES PASS_REGULAR_EXPRESSION ".*\\.pwn\\(11\\) : warning 237: user warning: this is warning 1 -.*\\.pwn\\(13\\) : warning 237: user warning: this is warning 2 -.*\\.pwn\\(15\\) : warning 237: user warning: this is warning 3 -.*\\.pwn\\(17\\) : warning 237: user warning: this is warning 4 -.*\\.pwn\\(28\\) : warning 234: function is deprecated \\(symbol \"f\"\\) don't use this function please -.*\\.pwn\\(33\\) : warning 234: function is deprecated \\(symbol \"f\"\\) don't use this function please -") - -add_compiler_test(reset_errline_gh_230 ${CMAKE_CURRENT_SOURCE_DIR}/reset_errline_gh_230.pwn) -set_tests_properties(reset_errline_gh_230 PROPERTIES PASS_REGULAR_EXPRESSION -".*\\.pwn\\(2\\) : error 017: undefined symbol \\\"undefined\\\" -.*\\.pwn\\(2\\) : warning 215: expression has no effect -.*\\.pwn\\(7\\) : warning 204: symbol is assigned a value that is never used: \\\"y\\\" -.*\\.pwn\\(4\\) : warning 204: symbol is assigned a value that is never used: \\\"x\\\" -") - -add_compiler_test(unused_symbol_line_gh_252 ${CMAKE_CURRENT_SOURCE_DIR}/unused_symbol_line_gh_252.pwn) -set_tests_properties(unused_symbol_line_gh_252 PROPERTIES PASS_REGULAR_EXPRESSION -".*\\.pwn\\(4\\) : warning 203: symbol is never used: \\\"y\\\" -.*\\.pwn\\(8\\) : warning 203: symbol is never used: \\\"z\\\" -.*\\.pwn\\(1\\) : warning 203: symbol is never used: \\\"x\\\" -") - -add_compiler_test(gh_283 ${CMAKE_CURRENT_SOURCE_DIR}/gh_283.pwn) -set_tests_properties(gh_283 PROPERTIES PASS_REGULAR_EXPRESSION - ".*\\.pwn\\(5\\) : warning 234: function is deprecated \\(symbol \"print\"\\)") - -add_compiler_test(too_many_args_crash_gh_298 ${CMAKE_CURRENT_SOURCE_DIR}/too_many_args_crash_gh_298.pwn) -set_tests_properties(too_many_args_crash_gh_298 PROPERTIES PASS_REGULAR_EXPRESSION "too many function arguments") - -add_compiler_test(meaningless_class_specifiers_gh_172 ${CMAKE_CURRENT_SOURCE_DIR}/meaningless_class_specifiers_gh_172.pwn) -set_tests_properties(meaningless_class_specifiers_gh_172 PROPERTIES PASS_REGULAR_EXPRESSION -".*\\.pwn\\(1\\) : warning 238: meaningless combination of class specifiers \\(const reference\\) -.*\\.pwn\\(1 \\-\\- 2\\) : warning 238: meaningless combination of class specifiers \\(const variable arguments\\) -") - -add_compiler_test(constexpr_result_prop_gh_308 ${CMAKE_CURRENT_SOURCE_DIR}/constexpr_result_prop_gh_308.pwn) -set_tests_properties(constexpr_result_prop_gh_308 PROPERTIES PASS_REGULAR_EXPRESSION -".*\\.pwn\\(2\\) : warning 237: user warning: \\\"Test passed.\\\" -.*\\.pwn\\(6\\) : warning 237: user warning: \\\"Test passed.\\\" -.*\\.pwn\\(10\\) : warning 237: user warning: \\\"Test passed.\\\" -.*\\.pwn\\(14\\) : warning 237: user warning: \\\"Test passed.\\\" -") - -add_compiler_test(const_array_args_and_literals_gh_276 ${CMAKE_CURRENT_SOURCE_DIR}/const_array_args_and_literals_gh_276.pwn) -set_tests_properties(const_array_args_and_literals_gh_276 PROPERTIES PASS_REGULAR_EXPRESSION -".*\\.pwn\\(13\\) : warning 214: possibly a \\\"const\\\" array argument was intended: \\\"arr\\\" -.*\\.pwn\\(18\\) : warning 214: possibly a \\\"const\\\" array argument was intended: \\\"arr\\\" -.*\\.pwn\\(30\\) : warning 214: possibly a \\\"const\\\" array argument was intended: \\\"arr\\\" -.*\\.pwn\\(39\\) : warning 239: literal array/string passed to a non-const parameter -.*\\.pwn\\(40\\) : warning 239: literal array/string passed to a non-const parameter -.*\\.pwn\\(41\\) : warning 239: literal array/string passed to a non-const parameter -") - -add_compiler_test(md_array_size_chk_gh_314 ${CMAKE_CURRENT_SOURCE_DIR}/md_array_size_chk_gh_314.pwn) -set_tests_properties(md_array_size_chk_gh_314 PROPERTIES PASS_REGULAR_EXPRESSION -"*\\.pwn\\(1\\) : error 009: invalid array size \\(negative, zero or out of bounds\\) -.*\\.pwn\\(2\\) : error 009: invalid array size \\(negative, zero or out of bounds\\) -.*\\.pwn\\(3\\) : error 009: invalid array size \\(negative, zero or out of bounds\\) -.*\\.pwn\\(5\\) : error 009: invalid array size \\(negative, zero or out of bounds\\) -.*\\.pwn\\(30\\) : warning 224: indeterminate array size in \"sizeof\" expression \\(symbol \"\"\\) -") -set_tests_properties(md_array_size_chk_gh_314 PROPERTIES WILL_FAIL TRUE) - -add_compiler_test(destructor_not_impl_gh_310 ${CMAKE_CURRENT_SOURCE_DIR}/destructor_not_impl_gh_310.pwn) -set_tests_properties(destructor_not_impl_gh_310 PROPERTIES PASS_REGULAR_EXPRESSION -".*\\.pwn\\(6\\) : error 004: function \"operator~(Error:)\" is not implemented -") -set_tests_properties(destructor_not_impl_gh_310 PROPERTIES WILL_FAIL TRUE) - -# Crashers -# -# These tests simply check that the compiler doesn't crash. -# -# TODO: Probably need to support tests that exist with a non-zero code but don't crash? -# Right now this will cause a failure. - -add_compiler_test(md_array_crash_gh_220 ${CMAKE_CURRENT_SOURCE_DIR}/md_array_crash_gh_220.pwn) +add_custom_target(pawncc_tests COMMAND + COMMAND ${PYTHON_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/run_tests.py + -c $ + -i ../../../include + DEPENDS pawncc + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/source/compiler/tests/const_array_args_and_literals_gh_276.meta b/source/compiler/tests/const_array_args_and_literals_gh_276.meta new file mode 100644 index 0000000..861e659 --- /dev/null +++ b/source/compiler/tests/const_array_args_and_literals_gh_276.meta @@ -0,0 +1,11 @@ +{ + 'test_type': 'output_check', + 'errors': """ +const_array_args_and_literals_gh_276.pwn(13) : warning 214: possibly a "const" array argument was intended: "arr" +const_array_args_and_literals_gh_276.pwn(18) : warning 214: possibly a "const" array argument was intended: "arr" +const_array_args_and_literals_gh_276.pwn(30) : warning 214: possibly a "const" array argument was intended: "arr" +const_array_args_and_literals_gh_276.pwn(39) : warning 239: literal array/string passed to a non-const parameter +const_array_args_and_literals_gh_276.pwn(40) : warning 239: literal array/string passed to a non-const parameter +const_array_args_and_literals_gh_276.pwn(41) : warning 239: literal array/string passed to a non-const parameter +""" +} diff --git a/source/compiler/tests/constexpr_result_prop_gh_308.meta b/source/compiler/tests/constexpr_result_prop_gh_308.meta new file mode 100644 index 0000000..197d682 --- /dev/null +++ b/source/compiler/tests/constexpr_result_prop_gh_308.meta @@ -0,0 +1,9 @@ +{ + 'test_type': 'output_check', + 'errors': """ +constexpr_result_prop_gh_308.pwn(2) : warning 237: user warning: "Test passed." +constexpr_result_prop_gh_308.pwn(6) : warning 237: user warning: "Test passed." +constexpr_result_prop_gh_308.pwn(10) : warning 237: user warning: "Test passed." +constexpr_result_prop_gh_308.pwn(14) : warning 237: user warning: "Test passed." +""" +} diff --git a/source/compiler/tests/destructor_not_impl_gh_310.meta b/source/compiler/tests/destructor_not_impl_gh_310.meta new file mode 100644 index 0000000..3e1e1be --- /dev/null +++ b/source/compiler/tests/destructor_not_impl_gh_310.meta @@ -0,0 +1,6 @@ +{ + 'test_type': 'output_check', + 'errors': """ +destructor_not_impl_gh_310.pwn(4) : error 004: function "operator~(Error:)" is not implemented +""" +} diff --git a/source/compiler/tests/gh_217.meta b/source/compiler/tests/gh_217.meta new file mode 100644 index 0000000..f40a838 --- /dev/null +++ b/source/compiler/tests/gh_217.meta @@ -0,0 +1,11 @@ +{ + 'test_type': 'output_check', + 'errors': """ +gh_217.pwn(11) : warning 237: user warning: this is warning 1 +gh_217.pwn(13) : warning 237: user warning: this is warning 2 +gh_217.pwn(15) : warning 237: user warning: this is warning 3 +gh_217.pwn(17) : warning 237: user warning: this is warning 4 +gh_217.pwn(28) : warning 234: function is deprecated (symbol "f") don't use this function please +gh_217.pwn(33) : warning 234: function is deprecated (symbol "f") don't use this function please +""" +} diff --git a/source/compiler/tests/gh_217.pwn b/source/compiler/tests/gh_217.pwn index c84af2c..9b6fe6e 100644 --- a/source/compiler/tests/gh_217.pwn +++ b/source/compiler/tests/gh_217.pwn @@ -1,6 +1,6 @@ // TODO: Check that string literals are concatenated correctly -native print(const s[]); +#include #define d1\ print("ok") diff --git a/source/compiler/tests/gh_283.meta b/source/compiler/tests/gh_283.meta new file mode 100644 index 0000000..3fcfc5f --- /dev/null +++ b/source/compiler/tests/gh_283.meta @@ -0,0 +1,6 @@ +{ + 'test_type': 'output_check', + 'errors': """ + gh_283.pwn(5) : warning 234: function is deprecated (symbol "f") + """ +} diff --git a/source/compiler/tests/gh_283.pwn b/source/compiler/tests/gh_283.pwn index 87645ac..0cc57dd 100644 --- a/source/compiler/tests/gh_283.pwn +++ b/source/compiler/tests/gh_283.pwn @@ -1,6 +1,6 @@ #pragma deprecated -native print(const string[]); +native f(); main() { - print("Hello World"); + f(); } diff --git a/source/compiler/tests/md_array_crash_gh_220.meta b/source/compiler/tests/md_array_crash_gh_220.meta new file mode 100644 index 0000000..69deb64 --- /dev/null +++ b/source/compiler/tests/md_array_crash_gh_220.meta @@ -0,0 +1,7 @@ +{ + 'test_type': 'output_check', + 'errors': """ +md_array_crash_gh_220.pwn(6) : fatal error 111: user error: OK +Compilation aborted. +""" +} diff --git a/source/compiler/tests/md_array_crash_gh_220.pwn b/source/compiler/tests/md_array_crash_gh_220.pwn index 74f27c7..20f2a75 100644 --- a/source/compiler/tests/md_array_crash_gh_220.pwn +++ b/source/compiler/tests/md_array_crash_gh_220.pwn @@ -3,4 +3,5 @@ new b[2000][500] = { { 0, -1, ... }, ... }; main() { a[0][0] = b[0][0]; -} \ No newline at end of file + #error OK +} diff --git a/source/compiler/tests/md_array_size_chk_gh_314.meta b/source/compiler/tests/md_array_size_chk_gh_314.meta new file mode 100644 index 0000000..44b8595 --- /dev/null +++ b/source/compiler/tests/md_array_size_chk_gh_314.meta @@ -0,0 +1,10 @@ +{ + 'test_type': 'output_check', + 'errors': """ +md_array_size_chk_gh_314.pwn(1) : error 009: invalid array size (negative, zero or out of bounds) +md_array_size_chk_gh_314.pwn(2) : error 009: invalid array size (negative, zero or out of bounds) +md_array_size_chk_gh_314.pwn(3) : error 009: invalid array size (negative, zero or out of bounds) +md_array_size_chk_gh_314.pwn(5) : error 009: invalid array size (negative, zero or out of bounds) +md_array_size_chk_gh_314.pwn(30) : warning 224: indeterminate array size in "sizeof" expression (symbol "") +""" +} diff --git a/source/compiler/tests/md_array_size_chk_gh_314.pwn b/source/compiler/tests/md_array_size_chk_gh_314.pwn index 368833d..f926648 100644 --- a/source/compiler/tests/md_array_size_chk_gh_314.pwn +++ b/source/compiler/tests/md_array_size_chk_gh_314.pwn @@ -1,8 +1,8 @@ -new arr1[] = {}; +new arr1[]; new arr2[5][]; new arr3[5][][5]; new arr4[5][5]; -new arr5[][]= { { } }; +new arr5[][]; f1(arr[]) { #pragma unused arr diff --git a/source/compiler/tests/meaningless_class_specifiers_gh_172.meta b/source/compiler/tests/meaningless_class_specifiers_gh_172.meta new file mode 100644 index 0000000..d6ccf1a --- /dev/null +++ b/source/compiler/tests/meaningless_class_specifiers_gh_172.meta @@ -0,0 +1,7 @@ +{ + 'test_type': 'output_check', + 'errors': """ +meaningless_class_specifiers_gh_172.pwn(1) : warning 238: meaningless combination of class specifiers (const reference) +meaningless_class_specifiers_gh_172.pwn(4) : warning 238: meaningless combination of class specifiers (const variable arguments) +""" +} diff --git a/source/compiler/tests/meaningless_class_specifiers_gh_172.pwn b/source/compiler/tests/meaningless_class_specifiers_gh_172.pwn index 68bfd02..1fd4d91 100755 --- a/source/compiler/tests/meaningless_class_specifiers_gh_172.pwn +++ b/source/compiler/tests/meaningless_class_specifiers_gh_172.pwn @@ -1,4 +1,6 @@ -f1(const &v) { } +f1(const &v) { + #pragma unused v +} f2(const ...) { } f3(const v) { #pragma unused v @@ -7,7 +9,9 @@ f4(...) { } f5(v) { #pragma unused v } -f6(&v) { } +f6(&v) { + #pragma unused v +} main() { new a; @@ -17,4 +21,4 @@ main() { f4(a); f5(a); f6(a); -} \ No newline at end of file +} diff --git a/source/compiler/tests/reset_errline_gh_230.meta b/source/compiler/tests/reset_errline_gh_230.meta new file mode 100644 index 0000000..8b754d7 --- /dev/null +++ b/source/compiler/tests/reset_errline_gh_230.meta @@ -0,0 +1,9 @@ +{ + 'test_type': 'output_check', + 'errors': """ +reset_errline_gh_230.pwn(2) : error 017: undefined symbol \"undefined\" +reset_errline_gh_230.pwn(2) : warning 215: expression has no effect +reset_errline_gh_230.pwn(7) : warning 204: symbol is assigned a value that is never used: "y" +reset_errline_gh_230.pwn(4) : warning 204: symbol is assigned a value that is never used: "x" +""" +} diff --git a/source/compiler/tests/run_tests.py b/source/compiler/tests/run_tests.py new file mode 100644 index 0000000..2280c74 --- /dev/null +++ b/source/compiler/tests/run_tests.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python + +import argparse +import glob +import os.path +import re +import subprocess +import sys + +parser = argparse.ArgumentParser() +parser.add_argument('-c', '--compiler', + required=True, + help='path to the pawncc executable') +parser.add_argument('-i', '--include', + dest='include_dirs', + action='append', + help='add specified directory to include path') +options = parser.parse_args(sys.argv[1:]) + +def run_compiler(args): + process_args = [';+', '-(+'] + if options.include_dirs is not None: + for dir in options.include_dirs: + process_args.append('-i' + dir) + if args is not None: + process_args += args + return subprocess.Popen(executable=options.compiler, + args=process_args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + +class OutputCheckTest: + def __init__(self, + name, + source_file, + errors=None, + extra_args=None): + self.name = name + self.source_file = source_file + self.errors = errors + self.extra_args = extra_args + + def run(self): + args = [self.source_file] + if self.extra_args is not None: + args += extra_args + process = run_compiler(args=args) + stdout, stderr = process.communicate() + result = True + if self.errors is None: + if process.returncode != 0: + result = False + self.fail_reason = """ + No errors specified and process exited with non-zero status + """ + else: + errors = stderr.decode('utf-8').splitlines() + errors = [e.strip() for e in errors if e.strip()] + expected_errors = self.errors.splitlines() + expected_errors = [e.strip() for e in expected_errors if e.strip()] + if errors != expected_errors: + result = False + self.fail_reason = ( + 'Error output didn\'t match\n\nExpected errors:\n\n{}\n\n' + 'Actual errors:\n\n{}' + ).format( + '\n'.join(expected_errors).strip(' \t\r\n'), + '\n'.join(errors).strip(' \t\r\n') + ) + return result + +class CrashTest: + def __init__(self, name, source_file, extra_args=None): + self.name = name + self.source_file = source_file + self.extra_args = extra_args + + def run(self): + # TODO: Check if the process crashed. + return True + +test_types = { + 'output_check': OutputCheckTest, + 'crash': CrashTest +} + +tests = [] +num_tests_disabled = 0 +for meta_file in glob.glob('*.meta'): + name = os.path.splitext(meta_file)[0] + metadata = eval(open(meta_file).read(), None, None) + if metadata.get('disabled'): + num_tests_disabled += 1 + continue + test_type = metadata['test_type'] + if test_type == 'output_check': + tests.append(OutputCheckTest(name=name, + source_file=name + '.pwn', + errors=metadata.get('errors'), + extra_args=metadata.get('extra_args'))) + elif test_type == 'crash': + tests.append(CrashTest(name=name, source_file=name + '.pwn')) + else: + raise KeyError('Unknown test type: ' + test_type) + +num_tests = len(tests) +sys.stdout.write('DISCOVERED {} TEST{}'.format(num_tests, '' if num_tests == 1 else 'S')) +if num_tests_disabled > 0: + sys.stdout.write(' ({} DISABLED)'.format(num_tests_disabled)) +sys.stdout.write('\n\n') + +num_tests_failed = 0 +for test in tests: + sys.stdout.write('Running ' + test.name + '... ') + if not test.run(): + sys.stdout.write('FAILED\n') + print('Test {} failed for the following reason: {}'.format( + test.name, test.fail_reason)) + print('') + num_tests_failed += 1 + else: + sys.stdout.write('PASSED\n') + +num_tests_passed = len(tests) - num_tests_failed +if num_tests_failed > 0: + print('\n{} TEST{} PASSED, {} FAILED'.format( + num_tests_passed, + '' if num_tests_passed == 1 else 'S', + num_tests_failed)) + sys.exit(1) +else: + print('\nALL TESTS PASSED') diff --git a/source/compiler/tests/too_many_args_crash_gh_298.meta b/source/compiler/tests/too_many_args_crash_gh_298.meta new file mode 100644 index 0000000..131078f --- /dev/null +++ b/source/compiler/tests/too_many_args_crash_gh_298.meta @@ -0,0 +1,11 @@ +{ + 'test_type': 'output_check', + 'errors': """ +too_many_args_crash_gh_298.pwn(2) : error 045: too many function arguments +too_many_args_crash_gh_298.pwn(2) : warning 215: expression has no effect +too_many_args_crash_gh_298.pwn(2) : error 001: expected token: ";", but found ")" +too_many_args_crash_gh_298.pwn(2) : error 029: invalid expression, assumed zero +too_many_args_crash_gh_298.pwn(2) : fatal error 107: too many error messages on one line +Compilation aborted. +""" +} diff --git a/source/compiler/tests/too_many_args_crash_gh_298.pwn b/source/compiler/tests/too_many_args_crash_gh_298.pwn index 525936c..5c07367 100644 --- a/source/compiler/tests/too_many_args_crash_gh_298.pwn +++ b/source/compiler/tests/too_many_args_crash_gh_298.pwn @@ -1,5 +1,3 @@ -native printf(const format[], ...); - main() { printf("", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); } diff --git a/source/compiler/tests/unused_symbol_line_gh_252.meta b/source/compiler/tests/unused_symbol_line_gh_252.meta new file mode 100644 index 0000000..4cfffc8 --- /dev/null +++ b/source/compiler/tests/unused_symbol_line_gh_252.meta @@ -0,0 +1,8 @@ +{ + 'test_type': 'output_check', + 'errors': """ +unused_symbol_line_gh_252.pwn(4) : warning 203: symbol is never used: "y" +unused_symbol_line_gh_252.pwn(8) : warning 203: symbol is never used: "z" +unused_symbol_line_gh_252.pwn(1) : warning 203: symbol is never used: "x" +""" +}