1berry/qtbase
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Inform qt-testrunner about coin_vxworks_qemu_runner.sh

Browse Source 
Since we now have quite a few "special" test wrappers, I took the
opportunity to refactor the code and add a couple of testcases too.

Change-Id: I20e1214351d71c1474be32f03d4218ae6bdd2277
Reviewed-by: Axel Spoerl <axel.spoerl@qt.io>
Reviewed-by: Toni Saario <toni.saario@qt.io>
This commit is contained in:
Dimitrios Apostolou 2024-09-03 19:06:20 +02:00
parent 2253c48805
commit 0f35b55dbb
2 changed files with 48 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
util/testrunner/qt-testrunner.py
View File

@ -87,6 +87,13 @@ NON_XML_GENERATING_TESTS = {
"tst_QDoc", # Some of QDoc's tests are using an external test framework (Catch) that does not support -o argument
"tst_QDoc_Catch_Generators", # Some of QDoc's tests are using an external test framework (Catch) that does not support -o argument
}
# These are scripts that are used to wrap test execution for special platforms.
# They need special handling (most times just skipping the wrapper name in argv[]).
TEST_RUNNER_WRAPPERS = {
"coin_qnx_qemu_runner.sh",
"coin_vxworks_qemu_runner.sh",
"androidtestrunner", # extra special handling needed, see code below.
}
def parse_args():
@ -144,12 +151,6 @@ Default flags: --max-repeats 5 --passes-needed 1
if args.test_basename.endswith(".exe"):
args.test_basename = args.test_basename[:-4]
# QNX test wrapper just needs to be skipped to figure out test_basename
if args.test_basename == "coin_qnx_qemu_runner.sh":
args.test_basename = os.path.basename(args.testargs[1])
L.info("Detected coin_qnx_qemu_runner, test will be handled specially. Detected test basename: %s",
args.test_basename)
# On Android emulated platforms, "androidtestrunner" is invoked by CMake
# to wrap the tests. We have to append the test arguments to it after
# "--". Besides that we have to detect the basename to avoid saving the
@ -167,6 +168,9 @@ Default flags: --max-repeats 5 --passes-needed 1
break
L.info("Detected androidtestrunner, test will be handled specially. Detected test basename: %s",
args.test_basename)
# Test wrapper just needs to be skipped to figure out test_basename.
elif args.test_basename in TEST_RUNNER_WRAPPERS:
args.test_basename = os.path.basename(args.testargs[1])
if args.test_basename in NON_XML_GENERATING_TESTS:
L.info("Detected special test not able to generate XML log! Will not parse it and will not repeat individual testcases")

43
util/testrunner/tests/tst_testrunner.py
View File

@ -6,6 +6,7 @@
import sys
import os
import re
import glob
import subprocess
from subprocess import STDOUT, PIPE
@ -52,9 +53,12 @@ def run(*args, **kwargs):
return proc
# Helper to run qt-testrunner.py with proper testing arguments.
def run_testrunner(xml_filename=None, extra_args=None, env=None):
def run_testrunner(xml_filename=None, wrapper_script=None, extra_args=None, env=None):
args = [ testrunner, mock_test ]
args = [ testrunner ]
if wrapper_script:
args += [ wrapper_script ]
args += [ mock_test ]
if xml_filename:
args += [ "--parse-xml-testlog", xml_filename ]
if extra_args:
@ -127,9 +131,11 @@ class Test_testrunner(unittest.TestCase):
state_file = os.environ["QT_MOCK_TEST_STATE_FILE"]
if os.path.exists(state_file):
os.remove(state_file)
old_logfile = os.path.join(TEMPDIR.name, os.path.basename(mock_test) + ".xml")
if os.path.exists(old_logfile):
os.remove(old_logfile)
# The mock_test honors only the XML output arguments, the rest are ignored.
old_logfiles = glob.glob(os.path.basename(mock_test) + "*.xml",
root_dir=TEMPDIR.name)
for fname in old_logfiles:
os.remove(os.path.join(TEMPDIR.name, fname))
self.env = dict()
self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] = os.environ["QT_MOCK_TEST_XML_TEMPLATE_FILE"]
self.env["QT_MOCK_TEST_STATE_FILE"] = state_file
@ -147,6 +153,14 @@ class Test_testrunner(unittest.TestCase):
self.prepare_env(run_list=["always_pass"])
proc = self.run2()
self.assertEqual(proc.returncode, 0)
def test_output_files_are_generated(self):
proc = self.run2()
xml_output_files = glob.glob(os.path.basename(mock_test) + "-*[0-9].xml",
root_dir=TEMPDIR.name)
if DEBUG:
print("Output files found: ",
xml_output_files)
self.assertEqual(len(xml_output_files), 1)
def test_always_fail(self):
self.prepare_env(run_list=["always_fail"])
proc = self.run2()
@ -215,6 +229,25 @@ class Test_testrunner(unittest.TestCase):
proc = self.run2()
self.assertEqual(proc.returncode, 3)
def create_wrapper(self, filename):
with open(os.path.join(TEMPDIR.name, filename), "w") as f:
f.write('#!/bin/sh\nexec "$@"\n')
self.wrapper_script = f.name
os.chmod(self.wrapper_script, 0o500)
# Test that qt-testrunner detects the correct executable name even if we
# use a special wrapper script, and that it uses that in the XML log filename.
def test_wrapper(self):
self.create_wrapper("coin_vxworks_qemu_runner.sh")
proc = run_testrunner(wrapper_script=self.wrapper_script,
extra_args=["--log-dir",TEMPDIR.name],
env=self.env)
self.assertEqual(proc.returncode, 0)
xml_output_files = glob.glob(os.path.basename(mock_test) + "-*[0-9].xml",
root_dir=TEMPDIR.name)
if DEBUG:
print("XML output files found: ", xml_output_files)
self.assertEqual(len(xml_output_files), 1)
# Test qt-testrunner script with an existing XML log file:
# qt-testrunner.py qt_mock_test.py --parse-xml-testlog file.xml