Add QTest option for repeating the entire test execution

Repeated test execution can be useful, under a debugger, to catch an
intermittent failure or, under memory instrumentation, to make memory
leaks easier to recognize.

The new -repeat flag allows running the entire test suite multiple times
within the same process. It works by executing all tests sequentially
before repeating the execution again.

This switch is a developer tool, and is not intended for CI. It can only
be used with the plain text logger.

Change-Id: I2439462c5c44d1c8aa3d3b5656de3eef44898c68
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>

src/testlib/doc/src/qttestlib-manual.qdoc

@@ -348,6 +348,10 @@
     Disables the crash handler on Unix platforms.
     On Windows, it re-enables the Windows Error Reporting dialog, which is
     turned off by default. This is useful for debugging crashes.
+    \li \c -repeat \e n \br
+    Run the testsuite n times or until the test fails. Useful for finding
+    flaky tests. If negative, the tests are repeated forever. This is intended
+    as a developer tool, and is only supported with the plain text logger.
 
     \li \c -platform \e name \br
     This command line argument applies to all Qt applications, but might be

src/testlib/qabstracttestlogger.cpp

@@ -147,6 +147,19 @@ QAbstractTestLogger::~QAbstractTestLogger()
     stream = nullptr;
 }
 
+/*!
+    Returns true if the logger supports repeated test runs.
+
+    Repetition of test runs is disabled by default, and can be enabled only for
+    test loggers that support it. Even if the logger may create syntactically
+    correct test reports, log-file analyzers may assume that test names are
+    unique within one report file.
+*/
+bool QAbstractTestLogger::isRepeatSupported() const
+{
+    return false;
+}
+
 /*!
     Returns true if the \c output stream is standard output.
 */

src/testlib/qabstracttestlogger_p.h

@@ -76,6 +76,8 @@ public:
     virtual void addMessage(MessageTypes type, const QString &message,
                             const char *file = nullptr, int line = 0) = 0;
 
+    virtual bool isRepeatSupported() const;
+
     bool isLoggingToStdout() const;
 
     void outputString(const char *msg);

src/testlib/qplaintestlogger.cpp

@@ -492,4 +492,13 @@ void QPlainTestLogger::addMessage(MessageTypes type, const QString &message,
     printMessage(MessageSource::Other, QTest::ptMessageType2String(type), qPrintable(message), file, line);
 }
 
+bool QPlainTestLogger::isRepeatSupported() const
+{
+    // The plain text logger creates unstructured reports. Such reports are not
+    // parser friendly, and are unlikely to be parsed by any test reporting
+    // tools. We can therefore allow repeated test runs with minimum risk that
+    // any parsers fails to handle repeated test names.
+    return true;
+}
+
 QT_END_NAMESPACE

src/testlib/qplaintestlogger_p.h

@@ -43,6 +43,8 @@ public:
     void addMessage(MessageTypes type, const QString &message,
                     const char *file = nullptr, int line = 0) override;
 
+    bool isRepeatSupported() const override;
+
 private:
     enum class MessageSource {
         Incident,

src/testlib/qtestcase.cpp

@@ -539,6 +539,8 @@ static int eventDelay = -1;
 static int timeout = -1;
 #endif
 static bool noCrashHandler = false;
+static int repetitions = 1;
+static bool repeatForever = false;
 
 /*! \internal
     Invoke a method of the object without generating warning if the method does not exist
@@ -710,6 +712,9 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool
     int logFormat = -1; // Not set
     const char *logFilename = nullptr;
 
+    repetitions = 1;
+    repeatForever = false;
+
     QTest::testFunctions.clear();
     QTest::testTags.clear();
 
@@ -764,6 +769,10 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool
          " -maxwarnings n      : Sets the maximum amount of messages to output.\n"
          "                       0 means unlimited, default: 2000\n"
          " -nocrashhandler     : Disables the crash handler. Useful for debugging crashes.\n"
+         " -repeat n           : Run the testsuite n times or until the test fails.\n"
+         "                       Useful for finding flaky tests. If negative, the tests are\n"
+         "                       repeated forever. This is intended as a developer tool, and\n"
+         "                       is only supported with the plain text logger.\n"
          "\n"
          " Benchmarking options:\n"
 #if QT_CONFIG(valgrind)
@@ -913,6 +922,14 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool
             } else {
                 QTestLog::setMaxWarnings(qToInt(argv[++i]));
             }
+        } else if (strcmp(argv[i], "-repeat") == 0) {
+            if (i + 1 >= argc) {
+                fprintf(stderr, "-repeat needs an extra parameter for the number of repetitions\n");
+                exit(1);
+            } else {
+                repetitions = qToInt(argv[++i]);
+                repeatForever = repetitions < 0;
+            }
         } else if (strcmp(argv[i], "-nocrashhandler") == 0) {
             QTest::noCrashHandler = true;
 #if QT_CONFIG(valgrind)
@@ -1066,6 +1083,11 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool
 
     if (addFallbackLogger)
         QTestLog::addLogger(QTestLog::Plain, logFilename);
+
+    if (repetitions != 1 && !QTestLog::isRepeatSupported()) {
+        fprintf(stderr, "-repeat is only supported with plain text logger\n");
+        exit(1);
+    }
 }
 
 // Temporary, backwards compatibility, until qtdeclarative's use of it is converted
@@ -2330,11 +2352,8 @@ void QTest::qInit(QObject *testObject, int argc, char **argv)
 #if QT_CONFIG(valgrind)
     if (QBenchmarkGlobalData::current->mode() != QBenchmarkGlobalData::CallgrindParentProcess)
 #endif
-    {
-        QTestTable::globalTestTable();
         QTestLog::startLogging();
 }
-}
 
 /*! \internal
 */
@@ -2398,7 +2417,12 @@ int QTest::qRun()
                 return 1;
         }
         TestMethods test(currentTestObject, std::move(commandLineMethods));
+
+        while (QTestLog::failCount() == 0 && (repeatForever || repetitions-- > 0)) {
+            QTestTable::globalTestTable();
             test.invokeTests(currentTestObject);
+            QTestTable::clearGlobalTestTable();
+        }
     }
 
 #ifndef QT_NO_EXCEPTIONS
@@ -2435,10 +2459,7 @@ void QTest::qCleanup()
 #if QT_CONFIG(valgrind)
     if (QBenchmarkGlobalData::current->mode() != QBenchmarkGlobalData::CallgrindParentProcess)
 #endif
-    {
         QTestLog::stopLogging();
-        QTestTable::clearGlobalTestTable();
-    }
 
     delete QBenchmarkGlobalData::current;
     QBenchmarkGlobalData::current = nullptr;

src/testlib/qtestlog.cpp

@@ -559,6 +559,21 @@ bool QTestLog::hasLoggers()
     return !QTest::loggers()->empty();
 }
 
+/*!
+    \internal
+
+    Returns true if all loggers support repeated test runs
+*/
+bool QTestLog::isRepeatSupported()
+{
+    FOREACH_TEST_LOGGER {
+        if (!logger->isRepeatSupported())
+            return false;
+    }
+
+    return true;
+}
+
 bool QTestLog::loggerUsingStdout()
 {
     FOREACH_TEST_LOGGER {

src/testlib/qtestlog_p.h

@@ -91,6 +91,7 @@ public:
     static void addLogger(QAbstractTestLogger *logger);
 
     static bool hasLoggers();
+    static bool isRepeatSupported();
     static bool loggerUsingStdout();
 
     static void setVerboseLevel(int level);

tests/auto/testlib/selftests/tst_selftests.cpp

@@ -1233,6 +1233,7 @@ SCENARIO("Exit code is as expected")
         { 0, "globaldata  testGlobal:global=true" },
         { 0, "globaldata  testGlobal:local=true" },
         { 0, "globaldata  testGlobal:global=true:local=true" },
+        { 0, "globaldata  testGlobal  -repeat  2" },
         { 1, "globaldata  testGlobal:local=true:global=true" },
         { 1, "globaldata  testGlobal:global=true:blah" },
         { 1, "globaldata  testGlobal:blah:local=true" },
@@ -1244,6 +1245,15 @@ SCENARIO("Exit code is as expected")
         { 1, "globaldata  testGlobal:blah         skipSingle:global=true:local=true" },
         { 1, "globaldata  testGlobal:global=true  skipSingle:blah" },
         { 2, "globaldata  testGlobal:blah         skipSingle:blue" },
+    // Passing -repeat argument
+        { 1, "pass  testNumber1  -repeat" },
+        { 0, "pass  testNumber1  -repeat  1" },
+        { 0, "pass  testNumber1  -repeat  1  -o  out.xml,xml" },
+        { 0, "pass  testNumber1  -repeat  2" },
+        { 0, "pass  testNumber1  -repeat  2  -o  -,txt" },
+        { 0, "pass  testNumber1  -repeat  2  -o  -,txt  -o  log.txt,txt" },
+        { 1, "pass  testNumber1  -repeat  2  -o  log.xml,xml" },
+        { 1, "pass  testNumber1  -repeat  2  -o  -,txt  -o  -,xml" },
     };
 
     size_t n_testCases = sizeof(testCases) / sizeof(*testCases);