1berry/qtbase
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

qt-testrunner: do not try to re-run initTestCase and others

Browse Source 
Some function names are special for qtestlib, in the sense that they can
not be specified as a command line argument to run individually.

In such cases qt-testrunner treats the failure specially and tries once
to re-run the full test executable.

Fixes: QTBUG-89011
Change-Id: I0cc25f91c57374e5ac65ade10e2e223fe969f211
Reviewed-by: Volker Hilsheimer <volker.hilsheimer@qt.io>
Reviewed-by: Daniel Smith <Daniel.Smith@qt.io>
This commit is contained in:
Dimitrios Apostolou 2022-01-27 20:12:10 +01:00
parent 1ae5b3628d
commit b4d9d5c89c
4 changed files with 53 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
util/testrunner/qt-testrunner.py
View File

@ -64,9 +64,9 @@
# 0: PASS. Either no test failed, or failed initially but passed
# in the re-runs (FLAKY PASS).
# 1: Some unexpected error of this script.
# 2: FAIL! for at least one test, even after the re-runs.
# 2: FAIL! for at least one test, even after the individual re-runs.
# 3: CRASH! for the test executable even after re-running it once.
# Or when we can't re-run individual functions for any reason.
import sys
@ -99,6 +99,10 @@ VERBOSE_ENV = {
"QT_LOGGING_RULES": "*=true",
"QT_MESSAGE_PATTERN": "[%{time process} %{if-debug}D%{endif}%{if-warning}W%{endif}%{if-critical}C%{endif}%{if-fatal}F%{endif}] %{category} %{file}:%{line} %{function}() - %{message}",
}
# The following special function names can not re-run individually.
NO_RERUN_FUNCTIONS = {
"initTestCase", "init", "cleanup", "cleanupTestCase"
}
def parse_args():
@ -300,39 +304,49 @@ def main():
args = parse_args()
n_full_runs = 1 if args.parse_xml_testlog else 2
for i in range(n_full_runs):
for i in range(n_full_runs + 1):
if 0 < i < n_full_runs:
L.info("Will re-run the full test executable")
elif i == n_full_runs: # Failed on the final run
L.error("Full test run failed repeatedly, aborting!")
sys.exit(3)
try:
if i != 0:
L.info("Re-running the full test!")
if args.parse_xml_testlog:
retcode = 1 # pretend the test returned error
results_file = args.parse_xml_testlog
else:
failed_functions = []
if args.parse_xml_testlog: # do not run test, just parse file
failed_functions = parse_log(args.parse_xml_testlog)
# Pretend the test returned correct exit code
retcode = len(failed_functions)
else: # normal invocation, run test
(retcode, results_file) = \
run_full_test(args.test_basename, args.testargs, args.log_dir,
args.no_extra_args, args.dry_run, args.timeout,
args.specific_extra_args)
if retcode == 0:
sys.exit(0) # PASS
if retcode != 0 and results_file:
failed_functions = parse_log(results_file)
failed_functions = parse_log(results_file)
if retcode == 0:
sys.exit(0) # PASS
if not args.parse_xml_testlog:
assert len(failed_functions) > 0, \
"The XML test log should contain at least one failure!" \
" Did the test CRASH right after all its testcases PASSed?"
if len(failed_functions) == 0:
L.info("No failures listed in the XML test log!"
" Did the test CRASH right after all its testcases PASSed?")
continue
break # go to re-running individual failed testcases
cant_rerun = [ f.func for f in failed_functions if f.func in NO_RERUN_FUNCTIONS ]
if cant_rerun:
L.info(f"Failure detected in the special test function '{cant_rerun[0]}'"
" which can not be re-run individually")
continue
assert len(failed_functions) > 0 and retcode != 0
break # all is fine, goto re-running individual failed testcases
except Exception as e:
L.exception("The test executable CRASHed uncontrollably!"
" Details about where we caught the problem:",
exc_info=e)
if i < n_full_runs - 1:
L.info("Will re-run the full test executable")
else: # Failed on the final run
L.error("Full test run failed repeatedly, aborting!")
sys.exit(3)
if args.max_repeats == 0:
sys.exit(2) # Some tests failed but no re-runs were asked

4
util/testrunner/tests/qt_mock_test-log.xml
View File

@ -5,6 +5,10 @@
<QtBuild>MOCK</QtBuild>
<QTestVersion>6.3.0</QTestVersion>
</Environment>
<TestFunction name="initTestCase">
<Incident type="{{initTestCase_result}}" file="" line="0" />
<Duration msecs="0.00004"/>
</TestFunction>
<TestFunction name="always_pass">
<Incident type="{{always_pass_result}}" file="" line="0" />
<Duration msecs="0.71704"/>

4
util/testrunner/tests/qt_mock_test.py
View File

@ -127,7 +127,9 @@ def log_test(testcase, result,
# Return the exit code
def run_test(testname):
if testname == "always_pass":
if testname == "initTestCase":
exit_code = 1 # specifically test that initTestCase fails
elif testname == "always_pass":
exit_code = 0
elif testname == "always_fail":
exit_code = 1

10
util/testrunner/tests/tst_testrunner.py
View File

@ -196,6 +196,10 @@ class Test_testrunner(unittest.TestCase):
proc = self.run2()
# TODO verify that one func was re-run and passed but the other failed.
self.assertEqual(proc.returncode, 2)
def test_initTestCase_fail_crash(self):
self.prepare_env(run_list=["initTestCase,always_pass"])
proc = self.run2()
self.assertEqual(proc.returncode, 3)
# If no XML file is found by qt-testrunner, it is usually considered a
# CRASH and the whole test is re-run. But when the return code is zero, it
@ -231,6 +235,8 @@ class Test_testrunner(unittest.TestCase):
# + The "always_crash" test has failed. qt-testrunner should exit(2).
# + The "fail_then_pass:2" test failed. qt-testrunner should exit(0).
# + The "fail_then_pass:5" test failed. qt-testrunner should exit(2).
# + The "initTestCase" failed which is listed as NO_RERUN thus
# qt-testrunner should exit(3).
class Test_testrunner_with_xml_logfile(unittest.TestCase):
# Runs before every single test function, creating a unique temp file.
def setUp(self):
@ -287,6 +293,10 @@ class Test_testrunner_with_xml_logfile(unittest.TestCase):
matches = re.findall(r"(PASS|FAIL!).*\n.*Test process exited with code",
proc.stdout.decode())
self.assertEqual(len(matches), 4)
def test_initTestCase_fail_crash(self):
write_xml_log(self.xml_file, failure="initTestCase")
proc = run_testrunner(self.xml_file)
self.assertEqual(proc.returncode, 3)
if __name__ == "__main__":