From 0f35b55dbbb7d3cddd408a6110886f143b5f7abf Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Tue, 3 Sep 2024 19:06:20 +0200 Subject: [PATCH] Inform qt-testrunner about coin_vxworks_qemu_runner.sh Since we now have quite a few "special" test wrappers, I took the opportunity to refactor the code and add a couple of testcases too. Change-Id: I20e1214351d71c1474be32f03d4218ae6bdd2277 Reviewed-by: Axel Spoerl Reviewed-by: Toni Saario --- util/testrunner/qt-testrunner.py | 16 +++++---- util/testrunner/tests/tst_testrunner.py | 43 ++++++++++++++++++++++--- 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/util/testrunner/qt-testrunner.py b/util/testrunner/qt-testrunner.py index 2305e4b4fba..fa90af89215 100755 --- a/util/testrunner/qt-testrunner.py +++ b/util/testrunner/qt-testrunner.py @@ -87,6 +87,13 @@ NON_XML_GENERATING_TESTS = { "tst_QDoc", # Some of QDoc's tests are using an external test framework (Catch) that does not support -o argument "tst_QDoc_Catch_Generators", # Some of QDoc's tests are using an external test framework (Catch) that does not support -o argument } +# These are scripts that are used to wrap test execution for special platforms. +# They need special handling (most times just skipping the wrapper name in argv[]). +TEST_RUNNER_WRAPPERS = { + "coin_qnx_qemu_runner.sh", + "coin_vxworks_qemu_runner.sh", + "androidtestrunner", # extra special handling needed, see code below. +} def parse_args(): @@ -144,12 +151,6 @@ Default flags: --max-repeats 5 --passes-needed 1 if args.test_basename.endswith(".exe"): args.test_basename = args.test_basename[:-4] - # QNX test wrapper just needs to be skipped to figure out test_basename - if args.test_basename == "coin_qnx_qemu_runner.sh": - args.test_basename = os.path.basename(args.testargs[1]) - L.info("Detected coin_qnx_qemu_runner, test will be handled specially. Detected test basename: %s", - args.test_basename) - # On Android emulated platforms, "androidtestrunner" is invoked by CMake # to wrap the tests. We have to append the test arguments to it after # "--". Besides that we have to detect the basename to avoid saving the @@ -167,6 +168,9 @@ Default flags: --max-repeats 5 --passes-needed 1 break L.info("Detected androidtestrunner, test will be handled specially. Detected test basename: %s", args.test_basename) + # Test wrapper just needs to be skipped to figure out test_basename. + elif args.test_basename in TEST_RUNNER_WRAPPERS: + args.test_basename = os.path.basename(args.testargs[1]) if args.test_basename in NON_XML_GENERATING_TESTS: L.info("Detected special test not able to generate XML log! Will not parse it and will not repeat individual testcases") diff --git a/util/testrunner/tests/tst_testrunner.py b/util/testrunner/tests/tst_testrunner.py index b2c100714d2..89b541d3ca8 100755 --- a/util/testrunner/tests/tst_testrunner.py +++ b/util/testrunner/tests/tst_testrunner.py @@ -6,6 +6,7 @@ import sys import os import re +import glob import subprocess from subprocess import STDOUT, PIPE @@ -52,9 +53,12 @@ def run(*args, **kwargs): return proc # Helper to run qt-testrunner.py with proper testing arguments. -def run_testrunner(xml_filename=None, extra_args=None, env=None): +def run_testrunner(xml_filename=None, wrapper_script=None, extra_args=None, env=None): - args = [ testrunner, mock_test ] + args = [ testrunner ] + if wrapper_script: + args += [ wrapper_script ] + args += [ mock_test ] if xml_filename: args += [ "--parse-xml-testlog", xml_filename ] if extra_args: @@ -127,9 +131,11 @@ class Test_testrunner(unittest.TestCase): state_file = os.environ["QT_MOCK_TEST_STATE_FILE"] if os.path.exists(state_file): os.remove(state_file) - old_logfile = os.path.join(TEMPDIR.name, os.path.basename(mock_test) + ".xml") - if os.path.exists(old_logfile): - os.remove(old_logfile) + # The mock_test honors only the XML output arguments, the rest are ignored. + old_logfiles = glob.glob(os.path.basename(mock_test) + "*.xml", + root_dir=TEMPDIR.name) + for fname in old_logfiles: + os.remove(os.path.join(TEMPDIR.name, fname)) self.env = dict() self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] = os.environ["QT_MOCK_TEST_XML_TEMPLATE_FILE"] self.env["QT_MOCK_TEST_STATE_FILE"] = state_file @@ -147,6 +153,14 @@ class Test_testrunner(unittest.TestCase): self.prepare_env(run_list=["always_pass"]) proc = self.run2() self.assertEqual(proc.returncode, 0) + def test_output_files_are_generated(self): + proc = self.run2() + xml_output_files = glob.glob(os.path.basename(mock_test) + "-*[0-9].xml", + root_dir=TEMPDIR.name) + if DEBUG: + print("Output files found: ", + xml_output_files) + self.assertEqual(len(xml_output_files), 1) def test_always_fail(self): self.prepare_env(run_list=["always_fail"]) proc = self.run2() @@ -215,6 +229,25 @@ class Test_testrunner(unittest.TestCase): proc = self.run2() self.assertEqual(proc.returncode, 3) + def create_wrapper(self, filename): + with open(os.path.join(TEMPDIR.name, filename), "w") as f: + f.write('#!/bin/sh\nexec "$@"\n') + self.wrapper_script = f.name + os.chmod(self.wrapper_script, 0o500) + # Test that qt-testrunner detects the correct executable name even if we + # use a special wrapper script, and that it uses that in the XML log filename. + def test_wrapper(self): + self.create_wrapper("coin_vxworks_qemu_runner.sh") + proc = run_testrunner(wrapper_script=self.wrapper_script, + extra_args=["--log-dir",TEMPDIR.name], + env=self.env) + self.assertEqual(proc.returncode, 0) + xml_output_files = glob.glob(os.path.basename(mock_test) + "-*[0-9].xml", + root_dir=TEMPDIR.name) + if DEBUG: + print("XML output files found: ", xml_output_files) + self.assertEqual(len(xml_output_files), 1) + # Test qt-testrunner script with an existing XML log file: # qt-testrunner.py qt_mock_test.py --parse-xml-testlog file.xml