Add support for runtime compiler tests

.travis.yml

@@ -29,8 +29,8 @@ deploy:
     secure: gDAuRNlF2uVhVHyZtJrX6MwNxnkfkQzohrC/6UcKAgqt+NKs4vZyq5FzTUceiDAkB0se70ZVx08e9ibAiXP/b7D1MPkAEiRxt9J6Vu3x6Bi1kPPuK5RfjFeT3gc1SbrULAP8Nz0NdU0chUhei6/V5NGhTegwp925DJOISq7+Ibw=
   file_glob: true
   file:
-    - 'pawnc-*-linux.tar.gz'
-    - 'pawnc-*-macos.zip'
+    - 'build/pawnc-*-linux.tar.gz'
+    - 'build/pawnc-*-macos.zip'
   draft: true
   skip_cleanup: true
   on:

CMakeLists.txt

@@ -1,8 +1,11 @@
 project(pawn C)
 cmake_minimum_required(VERSION 2.8)
 
+set(OUTPUT_DIR ${CMAKE_BINARY_DIR}/products)
+add_subdirectory(source/compiler ${OUTPUT_DIR})
+
 # For the compiler we only utilize headers from source/amx, i.e. we don't
 # actually build any of the AMX modules.
-# add_subdirectory(source/amx)
-
-add_subdirectory(source/compiler)
+if(BUILD_AMX)
+  add_subdirectory(source/amx ${OUTPUT_DIR})
+endif()

appveyor.yml

@@ -5,7 +5,8 @@ configuration:
   - RelWithDebInfo
 
 before_build:
-  - cmake . -G "Visual Studio 14 2015" -DCPACK_GENERATOR=ZIP -DCMAKE_GENERATOR_TOOLSET=v100
+  - mkdir build && cd build
+  - cmake ../source -G "Visual Studio 14 2015" -DCPACK_GENERATOR=ZIP -DCMAKE_GENERATOR_TOOLSET=v100
 
 build_script:
   - cmake --build . --config %CONFIGURATION%
@@ -15,7 +16,7 @@ test_script:
   - cmake --build . --config %CONFIGURATION% --target pawncc_tests
 
 artifacts:
-  - path: pawnc-*-windows.zip
+  - path: build/pawnc-*-windows.zip
     name: Binary package
 
 deploy:

source/compiler/CMakeLists.txt

@@ -139,6 +139,9 @@ target_link_libraries(pawncc pawnc)
 # The Pawn disassembler
 set(PAWNDISASM_SRCS
   pawndisasm.c
+  sc.h
+  ../amx/amx.h
+  ../amx/amxdbg.h
   ../amx/amxdbg.c
 )
 add_executable(pawndisasm ${PAWNDISASM_SRCS})
@@ -161,6 +164,15 @@ if(MSVC)
 endif()
 
 # Generate targets for running compiler tests
+set(PAWNRUNS_SRCS
+  pawnruns.c
+  ../amx/amx.c
+  ../amx/amx.h
+  ../amx/amxaux.c
+  ../amx/amxaux.h
+  ../amx/amxcons.c
+  ../amx/amxcore.c)
+add_executable(pawnruns ${PAWNRUNS_SRCS})
 add_subdirectory(tests)
 
 # Generate a binary package with CPack

source/compiler/pawnruns.c

@@ -0,0 +1,55 @@
+/*  A simpler runner based on source/amx/pawnrun/prun1.c.
+ *
+ *  Copyright (c) ITB CompuPhase, 2001-2005
+ *
+ *  This file may be freely used. No warranties of any kind.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>             /* for exit() */
+#include <signal.h>
+#include <string.h>             /* for memset() (on some compilers) */
+#include "../amx/amx.h"
+#include "../amx/amxaux.h"
+
+static void ErrorExit(AMX *amx, int errorcode)
+{
+  printf("Run time error %d: \"%s\"\n", errorcode, aux_StrError(errorcode));
+  exit(1);
+}
+
+static void PrintUsage(char *program)
+{
+  printf("Usage: %s <filename>\n<filename> is a compiled script.\n", program);
+  exit(1);
+}
+
+int main(int argc,char *argv[])
+{
+  extern AMX_NATIVE_INFO console_Natives[];
+  extern AMX_NATIVE_INFO core_Natives[];
+
+  AMX amx;
+  cell ret = 0;
+  int err;
+
+  if (argc != 2)
+    PrintUsage(argv[0]);
+
+  err = aux_LoadProgram(&amx, argv[1], NULL);
+  if (err != AMX_ERR_NONE)
+    ErrorExit(&amx, err);
+
+  amx_Register(&amx, console_Natives, -1);
+  err = amx_Register(&amx, core_Natives, -1);
+  if (err != AMX_ERR_NONE)
+    ErrorExit(&amx, err);
+
+  err = amx_Exec(&amx, &ret, AMX_EXEC_MAIN);
+  if (err != AMX_ERR_NONE)
+    ErrorExit(&amx, err);
+  printf("%s returns %ld\n", argv[1], (long)ret);
+
+  aux_FreeProgram(&amx);
+  return 0;
+}

source/compiler/tests/CMakeLists.txt

@@ -1,9 +1,14 @@
-find_package(PythonInterp 2.7 REQUIRED)
+find_package(PythonInterp 2.7)
 
-add_custom_target(pawncc_tests COMMAND
-  COMMAND ${PYTHON_EXECUTABLE}
-    ${CMAKE_CURRENT_SOURCE_DIR}/run_tests.py
-    -c $<TARGET_FILE:pawncc>
-    -i ../../../include
-  DEPENDS pawncc
-  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+if(PYTHONINTERP_FOUND)
+  add_custom_target(pawncc_tests COMMAND
+    COMMAND ${PYTHON_EXECUTABLE}
+      ${CMAKE_CURRENT_SOURCE_DIR}/run_tests.py
+      -c $<TARGET_FILE:pawncc>
+      -r $<TARGET_FILE:pawnruns>
+      -i ../../../include
+    DEPENDS pawncc
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+else()
+  message("Python was not found, you will not be able to run the compiler tests")
+endif()

source/compiler/tests/md_array_crash_gh_220.meta

@@ -2,6 +2,7 @@
   'test_type': 'output_check',
   'errors': """
 md_array_crash_gh_220.pwn(6) : fatal error 111: user error: OK
+
 Compilation aborted.
 """
 }

source/compiler/tests/run_tests.py

@@ -10,11 +10,14 @@ import sys
 parser = argparse.ArgumentParser()
 parser.add_argument('-c', '--compiler',
                     required=True,
-                    help='path to the pawncc executable')
+                    help='path to compiler executable (pawncc or pawncc.exe)')
 parser.add_argument('-i', '--include',
                     dest='include_dirs',
                     action='append',
-                    help='add specified directory to include path')
+                    help='add custom include directories for compile tests')
+parser.add_argument('-r', '--runner',
+                    required=True,
+                    help='path to runner executable (pawnrun or pawnrun.exe)')
 options = parser.parse_args(sys.argv[1:])
 
 def run_compiler(args):
@@ -30,11 +33,7 @@ def run_compiler(args):
                             stderr=subprocess.PIPE)
 
 class OutputCheckTest:
-  def __init__(self,
-               name,
-               source_file,
-               errors=None,
-               extra_args=None):
+  def __init__(self, name, source_file, errors=None, extra_args=None):
     self.name = name
     self.source_file = source_file
     self.errors = errors
@@ -54,19 +53,14 @@ class OutputCheckTest:
           No errors specified and process exited with non-zero status
         """
     else:
-      errors = stderr.decode('utf-8').splitlines()
-      errors = [e.strip() for e in errors if e.strip()]
-      expected_errors = self.errors.splitlines()
-      expected_errors = [e.strip() for e in expected_errors if e.strip()]
+      errors = stderr.decode('utf-8').strip(' \t\r\n').replace('\r', '')
+      expected_errors = self.errors.strip(' \t\r\n').replace('\r', '')
       if errors != expected_errors:
         result = False
         self.fail_reason = (
           'Error output didn\'t match\n\nExpected errors:\n\n{}\n\n'
           'Actual errors:\n\n{}'
-        ).format(
-          '\n'.join(expected_errors).strip(' \t\r\n'),
-          '\n'.join(errors).strip(' \t\r\n')
-        )
+        ).format(expected_errors, errors)
       return result
 
 class CrashTest:
@@ -79,10 +73,37 @@ class CrashTest:
     # TODO: Check if the process crashed.
     return True
 
-test_types = {
-  'output_check': OutputCheckTest,
-  'crash': CrashTest
-}
+class RuntimeTest:
+  def __init__(self, name, amx_file, output, should_fail):
+    self.name = name
+    self.amx_file = amx_file
+    self.output = output
+    self.should_fail = should_fail
+
+  def run(self):
+    process = subprocess.Popen([options.runner, self.amx_file],
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
+    stdout, stderr = process.communicate()
+    output = ''
+    if stdout is not None:
+      output = stdout.decode('utf-8')
+    if stderr is not None:
+      output += stderr.decode('utf-8')
+    output = output.strip(' \t\r\n').replace('\r', '')
+    expected_output = self.output.strip(' \t\r\n').replace('\r', '')
+    if not self.should_fail and process.returncode != 0:
+      self.fail_reason = (
+        'Runner exited with status {}\n\nOutput: {}'
+      ).format(process.returncode, output)
+      return False
+    if output != expected_output:
+      self.fail_reason = (
+        'Output didn\'t match\n\nExpected output:\n\n{}\n\n'
+        'Actual output:\n\n{}'
+      ).format(expected_output, output)
+      return False
+    return True
 
 tests = []
 num_tests_disabled = 0
@@ -100,6 +121,11 @@ for meta_file in glob.glob('*.meta'):
                                  extra_args=metadata.get('extra_args')))
   elif test_type == 'crash':
     tests.append(CrashTest(name=name, source_file=name + '.pwn'))
+  elif test_type == 'runtime':
+    tests.append(RuntimeTest(name=name,
+                             amx_file=name + '.amx',
+                             output=metadata.get('output'),
+                             should_fail=metadata.get('should_fail')))
   else:
     raise KeyError('Unknown test type: ' + test_type)
 

source/compiler/tests/runtime_test_example.meta

@@ -0,0 +1,7 @@
+{
+  'test_type': 'runtime',
+  'output': """
+    Run time error 4: "Array index out of bounds"
+  """,
+  'should_fail': True
+}

source/compiler/tests/runtime_test_example.pwn

@@ -0,0 +1,10 @@
+#include <console>
+
+main() {
+	new a[1] = {1};
+	new i = 1;
+
+	// When compiled with at least debug level 1 (default) the line below should produce:
+	// Run time error 4: "Array index out of bounds"
+	printf("%d", a[i]);
+}

source/compiler/tests/too_many_args_crash_gh_298.meta

@@ -6,6 +6,7 @@ too_many_args_crash_gh_298.pwn(2) : warning 215: expression has no effect
 too_many_args_crash_gh_298.pwn(2) : error 001: expected token: ";", but found ")"
 too_many_args_crash_gh_298.pwn(2) : error 029: invalid expression, assumed zero
 too_many_args_crash_gh_298.pwn(2) : fatal error 107: too many error messages on one line
+
 Compilation aborted.
 """
 }