Inform qt-testrunner about coin_vxworks_qemu_runner.sh

Since we now have quite a few "special" test wrappers, I took the
opportunity to refactor the code and add a couple of testcases too.

Change-Id: I20e1214351d71c1474be32f03d4218ae6bdd2277
Reviewed-by: Axel Spoerl <axel.spoerl@qt.io>
Reviewed-by: Toni Saario <toni.saario@qt.io>

util/testrunner/qt-testrunner.py

@@ -87,6 +87,13 @@ NON_XML_GENERATING_TESTS = {
     "tst_QDoc",                     # Some of QDoc's tests are using an external test framework (Catch) that does not support -o argument
     "tst_QDoc_Catch_Generators",    # Some of QDoc's tests are using an external test framework (Catch) that does not support -o argument
 }
+# These are scripts that are used to wrap test execution for special platforms.
+# They need special handling (most times just skipping the wrapper name in argv[]).
+TEST_RUNNER_WRAPPERS = {
+    "coin_qnx_qemu_runner.sh",
+    "coin_vxworks_qemu_runner.sh",
+    "androidtestrunner",                # extra special handling needed, see code below.
+}
 
 
 def parse_args():
@@ -144,12 +151,6 @@ Default flags: --max-repeats 5 --passes-needed 1
     if args.test_basename.endswith(".exe"):
         args.test_basename = args.test_basename[:-4]
 
-    # QNX test wrapper just needs to be skipped to figure out test_basename
-    if args.test_basename == "coin_qnx_qemu_runner.sh":
-        args.test_basename = os.path.basename(args.testargs[1])
-        L.info("Detected coin_qnx_qemu_runner, test will be handled specially. Detected test basename: %s",
-               args.test_basename)
-
     # On Android emulated platforms, "androidtestrunner" is invoked by CMake
     # to wrap the tests.  We have to append the test arguments to it after
     # "--". Besides that we have to detect the basename to avoid saving the
@@ -167,6 +168,9 @@ Default flags: --max-repeats 5 --passes-needed 1
                     break
         L.info("Detected androidtestrunner, test will be handled specially. Detected test basename: %s",
                args.test_basename)
+    # Test wrapper just needs to be skipped to figure out test_basename.
+    elif args.test_basename in TEST_RUNNER_WRAPPERS:
+        args.test_basename = os.path.basename(args.testargs[1])
 
     if args.test_basename in NON_XML_GENERATING_TESTS:
         L.info("Detected special test not able to generate XML log! Will not parse it and will not repeat individual testcases")

util/testrunner/tests/tst_testrunner.py

@@ -6,6 +6,7 @@
 import sys
 import os
 import re
+import glob
 import subprocess
 
 from subprocess import STDOUT, PIPE
@@ -52,9 +53,12 @@ def run(*args, **kwargs):
     return proc
 
 # Helper to run qt-testrunner.py with proper testing arguments.
-def run_testrunner(xml_filename=None, extra_args=None, env=None):
+def run_testrunner(xml_filename=None, wrapper_script=None, extra_args=None, env=None):
 
-    args = [ testrunner, mock_test ]
+    args = [ testrunner ]
+    if wrapper_script:
+        args += [ wrapper_script ]
+    args += [ mock_test ]
     if xml_filename:
         args += [ "--parse-xml-testlog", xml_filename ]
     if extra_args:
@@ -127,9 +131,11 @@ class Test_testrunner(unittest.TestCase):
         state_file = os.environ["QT_MOCK_TEST_STATE_FILE"]
         if os.path.exists(state_file):
             os.remove(state_file)
-        old_logfile = os.path.join(TEMPDIR.name, os.path.basename(mock_test) + ".xml")
+        # The mock_test honors only the XML output arguments, the rest are ignored.
-        if os.path.exists(old_logfile):
+        old_logfiles = glob.glob(os.path.basename(mock_test) + "*.xml",
-            os.remove(old_logfile)
+                                 root_dir=TEMPDIR.name)
+        for fname in old_logfiles:
+            os.remove(os.path.join(TEMPDIR.name, fname))
         self.env = dict()
         self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] = os.environ["QT_MOCK_TEST_XML_TEMPLATE_FILE"]
         self.env["QT_MOCK_TEST_STATE_FILE"]        = state_file
@@ -147,6 +153,14 @@ class Test_testrunner(unittest.TestCase):
         self.prepare_env(run_list=["always_pass"])
         proc = self.run2()
         self.assertEqual(proc.returncode, 0)
+    def test_output_files_are_generated(self):
+        proc = self.run2()
+        xml_output_files = glob.glob(os.path.basename(mock_test) + "-*[0-9].xml",
+                                     root_dir=TEMPDIR.name)
+        if DEBUG:
+            print("Output files found: ",
+                  xml_output_files)
+        self.assertEqual(len(xml_output_files), 1)
     def test_always_fail(self):
         self.prepare_env(run_list=["always_fail"])
         proc = self.run2()
@@ -215,6 +229,25 @@ class Test_testrunner(unittest.TestCase):
         proc = self.run2()
         self.assertEqual(proc.returncode, 3)
 
+    def create_wrapper(self, filename):
+        with open(os.path.join(TEMPDIR.name, filename), "w") as f:
+            f.write('#!/bin/sh\nexec "$@"\n')
+            self.wrapper_script = f.name
+        os.chmod(self.wrapper_script, 0o500)
+    # Test that qt-testrunner detects the correct executable name even if we
+    # use a special wrapper script, and that it uses that in the XML log filename.
+    def test_wrapper(self):
+        self.create_wrapper("coin_vxworks_qemu_runner.sh")
+        proc = run_testrunner(wrapper_script=self.wrapper_script,
+                              extra_args=["--log-dir",TEMPDIR.name],
+                              env=self.env)
+        self.assertEqual(proc.returncode, 0)
+        xml_output_files = glob.glob(os.path.basename(mock_test) + "-*[0-9].xml",
+                                     root_dir=TEMPDIR.name)
+        if DEBUG:
+            print("XML output files found: ", xml_output_files)
+        self.assertEqual(len(xml_output_files), 1)
+
 
 # Test qt-testrunner script with an existing XML log file:
 #   qt-testrunner.py qt_mock_test.py --parse-xml-testlog file.xml