1berry/qtbase
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

qt-testrunner: be more strict if bad XML log files are written

Browse Source 
If a test returns 0 but writes an XML logfile that contains FAIL or a
corrupted XML file, then qt-testrunner considers it a CRASH and exits
with 3.

Previously any test execution returning 0 (success) was considered
a PASS. Changing this behavior with this patch finds a lot of test
crashes on Android (QTBUG-100470), because the tests are run indirectly
on the emulator and the test wrapper process could not detect the crash,
thus returning 0 to qt-testrunner. But the corrupt XML file is caught
now.

Likewise, if a test returns != 0 but the XML logfile contains only PASS,
qt-testrunner considers it a FAIL. This used to be the case but now
tests are added.

Finally changed logging for such cases from INFO to WARNING.

Task-number: QTBUG-100470
Change-Id: I404c9d2211c7de027bf776d1914519d37f513ca1
Reviewed-by: Assam Boudjelthia <assam.boudjelthia@qt.io>
This commit is contained in:
Dimitrios Apostolou 2022-01-27 21:00:45 +01:00 committed by Assam Boudjelthia
parent ade96f461d
commit 0b4446e16f
2 changed files with 34 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
util/testrunner/qt-testrunner.py
View File

@ -344,21 +344,25 @@ def main():
run_full_test(args.test_basename, args.testargs, args.log_dir,
args.no_extra_args, args.dry_run, args.timeout,
args.specific_extra_args)
if retcode != 0 and results_file:
if results_file:
failed_functions = parse_log(results_file)
if retcode == 0:
if failed_functions:
L.warning("The test executable returned success but the logfile"
f" contains FAIL for function: {failed_functions[0].func}")
continue
sys.exit(0) # PASS
if len(failed_functions) == 0:
L.info("No failures listed in the XML test log!"
" Did the test CRASH right after all its testcases PASSed?")
L.warning("No failures listed in the XML test log!"
" Did the test CRASH right after all its testcases PASSed?")
continue
cant_rerun = [ f.func for f in failed_functions if f.func in NO_RERUN_FUNCTIONS ]
if cant_rerun:
L.info(f"Failure detected in the special test function '{cant_rerun[0]}'"
" which can not be re-run individually")
L.warning(f"Failure detected in the special test function '{cant_rerun[0]}'"
" which can not be re-run individually")
continue
assert len(failed_functions) > 0 and retcode != 0

31
util/testrunner/tests/tst_testrunner.py
View File

@ -202,20 +202,21 @@ class Test_testrunner(unittest.TestCase):
self.assertEqual(proc.returncode, 3)
# If no XML file is found by qt-testrunner, it is usually considered a
# CRASH and the whole test is re-run. But when the return code is zero, it
# doesn't care about XML file and passes anyway.
def test_no_xml_log_written_pass(self):
# CRASH and the whole test is re-run. Even when the return code is zero.
# It is a PASS only if the test is not capable of XML output (see no_extra_args, TODO test it).
def test_no_xml_log_written_pass_crash(self):
del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"]
self.prepare_env(run_list=["always_pass"])
proc = self.run2()
self.assertEqual(proc.returncode, 0)
self.assertEqual(proc.returncode, 3)
# On the 2nd iteration of the full test, both of the tests pass.
def test_no_xml_log_written_fail_then_pass(self):
# Still it's a CRASH because no XML file was found.
def test_no_xml_log_written_fail_then_pass_crash(self):
del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"]
self.prepare_env(run_list=["always_pass,fail_then_pass:1"])
proc = self.run2()
# TODO verify that the whole test has run twice.
self.assertEqual(proc.returncode, 0)
self.assertEqual(proc.returncode, 3)
# Even after 2 iterations of the full test we still get failures but no XML file,
# and this is considered a CRASH.
def test_no_xml_log_written_crash(self):
@ -224,6 +225,24 @@ class Test_testrunner(unittest.TestCase):
proc = self.run2()
self.assertEqual(proc.returncode, 3)
# If a test returns success but XML contains failures, it's a CRASH.
def test_wrong_xml_log_written_1_crash(self):
logfile = os.path.join(TEMPDIR.name, os.path.basename(mock_test) + ".xml")
write_xml_log(logfile, failure="always_fail")
del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"]
self.prepare_env(run_list=["always_pass"])
proc = self.run2()
self.assertEqual(proc.returncode, 3)
# If a test returns failure but XML contains only pass, it's a CRASH.
def test_wrong_xml_log_written_2_crash(self):
logfile = os.path.join(TEMPDIR.name, os.path.basename(mock_test) + ".xml")
write_xml_log(logfile)
del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"]
self.prepare_env(run_list=["always_fail"])
proc = self.run2()
self.assertEqual(proc.returncode, 3)
# Test qt-testrunner script with an existing XML log file:
# qt-testrunner.py qt_mock_test.py --parse-xml-testlog file.xml
# qt-testrunner should repeat the testcases that are logged as