qt-testrunner: do not try to re-run initTestCase and others

Some function names are special for qtestlib, in the sense that they can
not be specified as a command line argument to run individually.

In such cases qt-testrunner treats the failure specially and tries once
to re-run the full test executable.

Fixes: QTBUG-89011
Change-Id: I0cc25f91c57374e5ac65ade10e2e223fe969f211
Reviewed-by: Volker Hilsheimer <volker.hilsheimer@qt.io>
Reviewed-by: Daniel Smith <Daniel.Smith@qt.io>

util/testrunner/qt-testrunner.py

@@ -64,9 +64,9 @@
 #   0: PASS. Either no test failed, or failed initially but passed
 #      in the re-runs (FLAKY PASS).
 #   1: Some unexpected error of this script.
-#   2: FAIL! for at least one test, even after the re-runs.
+#   2: FAIL! for at least one test, even after the individual re-runs.
 #   3: CRASH! for the test executable even after re-running it once.
-
+#        Or when we can't re-run individual functions for any reason.
 
 
 import sys
@@ -99,6 +99,10 @@ VERBOSE_ENV = {
     "QT_LOGGING_RULES": "*=true",
     "QT_MESSAGE_PATTERN": "[%{time process} %{if-debug}D%{endif}%{if-warning}W%{endif}%{if-critical}C%{endif}%{if-fatal}F%{endif}] %{category} %{file}:%{line} %{function}()  -  %{message}",
 }
+# The following special function names can not re-run individually.
+NO_RERUN_FUNCTIONS = {
+    "initTestCase", "init", "cleanup", "cleanupTestCase"
+}
 
 
 def parse_args():
@@ -300,39 +304,49 @@ def main():
     args = parse_args()
     n_full_runs = 1 if args.parse_xml_testlog else 2
 
-    for i in range(n_full_runs):
+    for i in range(n_full_runs + 1):
+
+        if 0 < i < n_full_runs:
+            L.info("Will re-run the full test executable")
+        elif i == n_full_runs:    # Failed on the final run
+            L.error("Full test run failed repeatedly, aborting!")
+            sys.exit(3)
+
         try:
-            if i != 0:
+            failed_functions = []
-                L.info("Re-running the full test!")
+            if args.parse_xml_testlog:      # do not run test, just parse file
-            if args.parse_xml_testlog:
+                failed_functions = parse_log(args.parse_xml_testlog)
-                retcode = 1    # pretend the test returned error
+                # Pretend the test returned correct exit code
-                results_file = args.parse_xml_testlog
+                retcode = len(failed_functions)
-            else:
+            else:                                # normal invocation, run test
                 (retcode, results_file) = \
                     run_full_test(args.test_basename, args.testargs, args.log_dir,
                                   args.no_extra_args, args.dry_run, args.timeout,
                                   args.specific_extra_args)
-                if retcode == 0:
+                if retcode != 0 and results_file:
-                    sys.exit(0)    # PASS
+                    failed_functions = parse_log(results_file)
 
-            failed_functions = parse_log(results_file)
+            if retcode == 0:
+                sys.exit(0)    # PASS
 
-            if not args.parse_xml_testlog:
+            if len(failed_functions) == 0:
-                assert len(failed_functions) > 0, \
+                L.info("No failures listed in the XML test log!"
-                    "The XML test log should contain at least one failure!" \
+                       " Did the test CRASH right after all its testcases PASSed?")
-                    " Did the test CRASH right after all its testcases PASSed?"
+                continue
 
-            break    # go to re-running individual failed testcases
+            cant_rerun = [ f.func for f in failed_functions if f.func in NO_RERUN_FUNCTIONS ]
+            if cant_rerun:
+                L.info(f"Failure detected in the special test function '{cant_rerun[0]}'"
+                       " which can not be re-run individually")
+                continue
+
+            assert len(failed_functions) > 0  and  retcode != 0
+            break    # all is fine, goto re-running individual failed testcases
 
         except Exception as e:
             L.exception("The test executable CRASHed uncontrollably!"
                         " Details about where we caught the problem:",
                         exc_info=e)
-            if i < n_full_runs - 1:
-                L.info("Will re-run the full test executable")
-            else:    # Failed on the final run
-                L.error("Full test run failed repeatedly, aborting!")
-                sys.exit(3)
 
     if args.max_repeats == 0:
         sys.exit(2)          # Some tests failed but no re-runs were asked

util/testrunner/tests/qt_mock_test-log.xml

@@ -5,6 +5,10 @@
     <QtBuild>MOCK</QtBuild>
     <QTestVersion>6.3.0</QTestVersion>
   </Environment>
+  <TestFunction name="initTestCase">
+    <Incident type="{{initTestCase_result}}" file="" line="0" />
+    <Duration msecs="0.00004"/>
+  </TestFunction>
   <TestFunction name="always_pass">
     <Incident type="{{always_pass_result}}" file="" line="0" />
     <Duration msecs="0.71704"/>

util/testrunner/tests/qt_mock_test.py

@@ -127,7 +127,9 @@ def log_test(testcase, result,
 
 # Return the exit code
 def run_test(testname):
-    if   testname == "always_pass":
+    if   testname == "initTestCase":
+        exit_code = 1              # specifically test that initTestCase fails
+    elif testname == "always_pass":
         exit_code = 0
     elif testname == "always_fail":
         exit_code = 1

util/testrunner/tests/tst_testrunner.py

@@ -196,6 +196,10 @@ class Test_testrunner(unittest.TestCase):
         proc = self.run2()
         # TODO verify that one func was re-run and passed but the other failed.
         self.assertEqual(proc.returncode, 2)
+    def test_initTestCase_fail_crash(self):
+        self.prepare_env(run_list=["initTestCase,always_pass"])
+        proc = self.run2()
+        self.assertEqual(proc.returncode, 3)
 
     # If no XML file is found by qt-testrunner, it is usually considered a
     # CRASH and the whole test is re-run. But when the return code is zero, it
@@ -231,6 +235,8 @@ class Test_testrunner(unittest.TestCase):
 #   + The "always_crash" test has failed. qt-testrunner should exit(2).
 #   + The "fail_then_pass:2" test failed. qt-testrunner should exit(0).
 #   + The "fail_then_pass:5" test failed. qt-testrunner should exit(2).
+#   + The "initTestCase" failed which is listed as NO_RERUN thus
+#                                         qt-testrunner should exit(3).
 class Test_testrunner_with_xml_logfile(unittest.TestCase):
     # Runs before every single test function, creating a unique temp file.
     def setUp(self):
@@ -287,6 +293,10 @@ class Test_testrunner_with_xml_logfile(unittest.TestCase):
         matches = re.findall(r"(PASS|FAIL!).*\n.*Test process exited with code",
                              proc.stdout.decode())
         self.assertEqual(len(matches), 4)
+    def test_initTestCase_fail_crash(self):
+        write_xml_log(self.xml_file, failure="initTestCase")
+        proc = run_testrunner(self.xml_file)
+        self.assertEqual(proc.returncode, 3)
 
 
 if __name__ == "__main__":