Inform qt-testrunner about coin_vxworks_qemu_runner.sh

Since we now have quite a few "special" test wrappers, I took the
opportunity to refactor the code and add a couple of testcases too.

Change-Id: I20e1214351d71c1474be32f03d4218ae6bdd2277
Reviewed-by: Axel Spoerl <axel.spoerl@qt.io>
Reviewed-by: Toni Saario <toni.saario@qt.io>
This commit is contained in:
Dimitrios Apostolou 2024-09-03 19:06:20 +02:00
parent 2253c48805
commit 0f35b55dbb
2 changed files with 48 additions and 11 deletions

View File

@ -87,6 +87,13 @@ NON_XML_GENERATING_TESTS = {
"tst_QDoc", # Some of QDoc's tests are using an external test framework (Catch) that does not support -o argument "tst_QDoc", # Some of QDoc's tests are using an external test framework (Catch) that does not support -o argument
"tst_QDoc_Catch_Generators", # Some of QDoc's tests are using an external test framework (Catch) that does not support -o argument "tst_QDoc_Catch_Generators", # Some of QDoc's tests are using an external test framework (Catch) that does not support -o argument
} }
# These are scripts that are used to wrap test execution for special platforms.
# They need special handling (most times just skipping the wrapper name in argv[]).
TEST_RUNNER_WRAPPERS = {
"coin_qnx_qemu_runner.sh",
"coin_vxworks_qemu_runner.sh",
"androidtestrunner", # extra special handling needed, see code below.
}
def parse_args(): def parse_args():
@ -144,12 +151,6 @@ Default flags: --max-repeats 5 --passes-needed 1
if args.test_basename.endswith(".exe"): if args.test_basename.endswith(".exe"):
args.test_basename = args.test_basename[:-4] args.test_basename = args.test_basename[:-4]
# QNX test wrapper just needs to be skipped to figure out test_basename
if args.test_basename == "coin_qnx_qemu_runner.sh":
args.test_basename = os.path.basename(args.testargs[1])
L.info("Detected coin_qnx_qemu_runner, test will be handled specially. Detected test basename: %s",
args.test_basename)
# On Android emulated platforms, "androidtestrunner" is invoked by CMake # On Android emulated platforms, "androidtestrunner" is invoked by CMake
# to wrap the tests. We have to append the test arguments to it after # to wrap the tests. We have to append the test arguments to it after
# "--". Besides that we have to detect the basename to avoid saving the # "--". Besides that we have to detect the basename to avoid saving the
@ -167,6 +168,9 @@ Default flags: --max-repeats 5 --passes-needed 1
break break
L.info("Detected androidtestrunner, test will be handled specially. Detected test basename: %s", L.info("Detected androidtestrunner, test will be handled specially. Detected test basename: %s",
args.test_basename) args.test_basename)
# Test wrapper just needs to be skipped to figure out test_basename.
elif args.test_basename in TEST_RUNNER_WRAPPERS:
args.test_basename = os.path.basename(args.testargs[1])
if args.test_basename in NON_XML_GENERATING_TESTS: if args.test_basename in NON_XML_GENERATING_TESTS:
L.info("Detected special test not able to generate XML log! Will not parse it and will not repeat individual testcases") L.info("Detected special test not able to generate XML log! Will not parse it and will not repeat individual testcases")

View File

@ -6,6 +6,7 @@
import sys import sys
import os import os
import re import re
import glob
import subprocess import subprocess
from subprocess import STDOUT, PIPE from subprocess import STDOUT, PIPE
@ -52,9 +53,12 @@ def run(*args, **kwargs):
return proc return proc
# Helper to run qt-testrunner.py with proper testing arguments. # Helper to run qt-testrunner.py with proper testing arguments.
def run_testrunner(xml_filename=None, extra_args=None, env=None): def run_testrunner(xml_filename=None, wrapper_script=None, extra_args=None, env=None):
args = [ testrunner, mock_test ] args = [ testrunner ]
if wrapper_script:
args += [ wrapper_script ]
args += [ mock_test ]
if xml_filename: if xml_filename:
args += [ "--parse-xml-testlog", xml_filename ] args += [ "--parse-xml-testlog", xml_filename ]
if extra_args: if extra_args:
@ -127,9 +131,11 @@ class Test_testrunner(unittest.TestCase):
state_file = os.environ["QT_MOCK_TEST_STATE_FILE"] state_file = os.environ["QT_MOCK_TEST_STATE_FILE"]
if os.path.exists(state_file): if os.path.exists(state_file):
os.remove(state_file) os.remove(state_file)
old_logfile = os.path.join(TEMPDIR.name, os.path.basename(mock_test) + ".xml") # The mock_test honors only the XML output arguments, the rest are ignored.
if os.path.exists(old_logfile): old_logfiles = glob.glob(os.path.basename(mock_test) + "*.xml",
os.remove(old_logfile) root_dir=TEMPDIR.name)
for fname in old_logfiles:
os.remove(os.path.join(TEMPDIR.name, fname))
self.env = dict() self.env = dict()
self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] = os.environ["QT_MOCK_TEST_XML_TEMPLATE_FILE"] self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] = os.environ["QT_MOCK_TEST_XML_TEMPLATE_FILE"]
self.env["QT_MOCK_TEST_STATE_FILE"] = state_file self.env["QT_MOCK_TEST_STATE_FILE"] = state_file
@ -147,6 +153,14 @@ class Test_testrunner(unittest.TestCase):
self.prepare_env(run_list=["always_pass"]) self.prepare_env(run_list=["always_pass"])
proc = self.run2() proc = self.run2()
self.assertEqual(proc.returncode, 0) self.assertEqual(proc.returncode, 0)
def test_output_files_are_generated(self):
proc = self.run2()
xml_output_files = glob.glob(os.path.basename(mock_test) + "-*[0-9].xml",
root_dir=TEMPDIR.name)
if DEBUG:
print("Output files found: ",
xml_output_files)
self.assertEqual(len(xml_output_files), 1)
def test_always_fail(self): def test_always_fail(self):
self.prepare_env(run_list=["always_fail"]) self.prepare_env(run_list=["always_fail"])
proc = self.run2() proc = self.run2()
@ -215,6 +229,25 @@ class Test_testrunner(unittest.TestCase):
proc = self.run2() proc = self.run2()
self.assertEqual(proc.returncode, 3) self.assertEqual(proc.returncode, 3)
def create_wrapper(self, filename):
with open(os.path.join(TEMPDIR.name, filename), "w") as f:
f.write('#!/bin/sh\nexec "$@"\n')
self.wrapper_script = f.name
os.chmod(self.wrapper_script, 0o500)
# Test that qt-testrunner detects the correct executable name even if we
# use a special wrapper script, and that it uses that in the XML log filename.
def test_wrapper(self):
self.create_wrapper("coin_vxworks_qemu_runner.sh")
proc = run_testrunner(wrapper_script=self.wrapper_script,
extra_args=["--log-dir",TEMPDIR.name],
env=self.env)
self.assertEqual(proc.returncode, 0)
xml_output_files = glob.glob(os.path.basename(mock_test) + "-*[0-9].xml",
root_dir=TEMPDIR.name)
if DEBUG:
print("XML output files found: ", xml_output_files)
self.assertEqual(len(xml_output_files), 1)
# Test qt-testrunner script with an existing XML log file: # Test qt-testrunner script with an existing XML log file:
# qt-testrunner.py qt_mock_test.py --parse-xml-testlog file.xml # qt-testrunner.py qt_mock_test.py --parse-xml-testlog file.xml