diff --git a/util/testrunner/qt-testrunner.py b/util/testrunner/qt-testrunner.py
index 0020bd5fdf1..d0d54f054ca 100755
--- a/util/testrunner/qt-testrunner.py
+++ b/util/testrunner/qt-testrunner.py
@@ -64,9 +64,9 @@
# 0: PASS. Either no test failed, or failed initially but passed
# in the re-runs (FLAKY PASS).
# 1: Some unexpected error of this script.
-# 2: FAIL! for at least one test, even after the re-runs.
+# 2: FAIL! for at least one test, even after the individual re-runs.
# 3: CRASH! for the test executable even after re-running it once.
-
+# Or when we can't re-run individual functions for any reason.
import sys
@@ -99,6 +99,10 @@ VERBOSE_ENV = {
"QT_LOGGING_RULES": "*=true",
"QT_MESSAGE_PATTERN": "[%{time process} %{if-debug}D%{endif}%{if-warning}W%{endif}%{if-critical}C%{endif}%{if-fatal}F%{endif}] %{category} %{file}:%{line} %{function}() - %{message}",
}
+# The following special function names can not re-run individually.
+NO_RERUN_FUNCTIONS = {
+ "initTestCase", "init", "cleanup", "cleanupTestCase"
+}
def parse_args():
@@ -300,39 +304,49 @@ def main():
args = parse_args()
n_full_runs = 1 if args.parse_xml_testlog else 2
- for i in range(n_full_runs):
+ for i in range(n_full_runs + 1):
+
+ if 0 < i < n_full_runs:
+ L.info("Will re-run the full test executable")
+ elif i == n_full_runs: # Failed on the final run
+ L.error("Full test run failed repeatedly, aborting!")
+ sys.exit(3)
+
try:
- if i != 0:
- L.info("Re-running the full test!")
- if args.parse_xml_testlog:
- retcode = 1 # pretend the test returned error
- results_file = args.parse_xml_testlog
- else:
+ failed_functions = []
+ if args.parse_xml_testlog: # do not run test, just parse file
+ failed_functions = parse_log(args.parse_xml_testlog)
+ # Pretend the test returned correct exit code
+ retcode = len(failed_functions)
+ else: # normal invocation, run test
(retcode, results_file) = \
run_full_test(args.test_basename, args.testargs, args.log_dir,
args.no_extra_args, args.dry_run, args.timeout,
args.specific_extra_args)
- if retcode == 0:
- sys.exit(0) # PASS
+ if retcode != 0 and results_file:
+ failed_functions = parse_log(results_file)
- failed_functions = parse_log(results_file)
+ if retcode == 0:
+ sys.exit(0) # PASS
- if not args.parse_xml_testlog:
- assert len(failed_functions) > 0, \
- "The XML test log should contain at least one failure!" \
- " Did the test CRASH right after all its testcases PASSed?"
+ if len(failed_functions) == 0:
+ L.info("No failures listed in the XML test log!"
+ " Did the test CRASH right after all its testcases PASSed?")
+ continue
- break # go to re-running individual failed testcases
+ cant_rerun = [ f.func for f in failed_functions if f.func in NO_RERUN_FUNCTIONS ]
+ if cant_rerun:
+ L.info(f"Failure detected in the special test function '{cant_rerun[0]}'"
+ " which can not be re-run individually")
+ continue
+
+ assert len(failed_functions) > 0 and retcode != 0
+ break # all is fine, goto re-running individual failed testcases
except Exception as e:
L.exception("The test executable CRASHed uncontrollably!"
" Details about where we caught the problem:",
exc_info=e)
- if i < n_full_runs - 1:
- L.info("Will re-run the full test executable")
- else: # Failed on the final run
- L.error("Full test run failed repeatedly, aborting!")
- sys.exit(3)
if args.max_repeats == 0:
sys.exit(2) # Some tests failed but no re-runs were asked
diff --git a/util/testrunner/tests/qt_mock_test-log.xml b/util/testrunner/tests/qt_mock_test-log.xml
index 0c316d71c38..62e93bb8dcc 100644
--- a/util/testrunner/tests/qt_mock_test-log.xml
+++ b/util/testrunner/tests/qt_mock_test-log.xml
@@ -5,6 +5,10 @@
MOCK
6.3.0
+
+
+
+
diff --git a/util/testrunner/tests/qt_mock_test.py b/util/testrunner/tests/qt_mock_test.py
index 23d87581908..d4e6eca1406 100755
--- a/util/testrunner/tests/qt_mock_test.py
+++ b/util/testrunner/tests/qt_mock_test.py
@@ -127,7 +127,9 @@ def log_test(testcase, result,
# Return the exit code
def run_test(testname):
- if testname == "always_pass":
+ if testname == "initTestCase":
+ exit_code = 1 # specifically test that initTestCase fails
+ elif testname == "always_pass":
exit_code = 0
elif testname == "always_fail":
exit_code = 1
diff --git a/util/testrunner/tests/tst_testrunner.py b/util/testrunner/tests/tst_testrunner.py
index e826ccb3053..6c16ef96129 100755
--- a/util/testrunner/tests/tst_testrunner.py
+++ b/util/testrunner/tests/tst_testrunner.py
@@ -196,6 +196,10 @@ class Test_testrunner(unittest.TestCase):
proc = self.run2()
# TODO verify that one func was re-run and passed but the other failed.
self.assertEqual(proc.returncode, 2)
+ def test_initTestCase_fail_crash(self):
+ self.prepare_env(run_list=["initTestCase,always_pass"])
+ proc = self.run2()
+ self.assertEqual(proc.returncode, 3)
# If no XML file is found by qt-testrunner, it is usually considered a
# CRASH and the whole test is re-run. But when the return code is zero, it
@@ -231,6 +235,8 @@ class Test_testrunner(unittest.TestCase):
# + The "always_crash" test has failed. qt-testrunner should exit(2).
# + The "fail_then_pass:2" test failed. qt-testrunner should exit(0).
# + The "fail_then_pass:5" test failed. qt-testrunner should exit(2).
+# + The "initTestCase" failed which is listed as NO_RERUN thus
+# qt-testrunner should exit(3).
class Test_testrunner_with_xml_logfile(unittest.TestCase):
# Runs before every single test function, creating a unique temp file.
def setUp(self):
@@ -287,6 +293,10 @@ class Test_testrunner_with_xml_logfile(unittest.TestCase):
matches = re.findall(r"(PASS|FAIL!).*\n.*Test process exited with code",
proc.stdout.decode())
self.assertEqual(len(matches), 4)
+ def test_initTestCase_fail_crash(self):
+ write_xml_log(self.xml_file, failure="initTestCase")
+ proc = run_testrunner(self.xml_file)
+ self.assertEqual(proc.returncode, 3)
if __name__ == "__main__":