Make counts of various types of test result add up correctly

Added tests for repeated skips and failures (from within void lambdas,
to simulate skips and failures from within event handlers). These
exhibit yet more ways to count more than one outcome for a test.  The
new QTest::failOnWarning() can also provoke more than one failure from
a single test, and several existing selftests exhibited various ways
for the Totals line's counts to add up to more than the number of
actual tests run.

Fixed counting so that only the first decisive incident is counted.
Tests can still report later failure or skipping, but only the first
is counted.

Added a currentTestState in qtestlog.cpp, by which it keeps track of
whether the test has resolved to a result, and clearCurrentTestState()
by which other code can reset that at the end of each test. This
brought to light various places where test-end clean-up was not being
handled - due to failure or skipping in a *_data() method or init, or
a skip in cleanup.

Fixes: QTBUG-95661
Change-Id: I5d24a37a53d3db225fa602649d8aad8f5ed6c1ad
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>

src/testlib/qtestcase.cpp

@@ -946,33 +946,38 @@ void TestMethods::invokeTestOnData(int index) const
         do {
             if (m_initMethod.isValid())
                 m_initMethod.invoke(QTest::currentTestObject, Qt::DirectConnection);
-            if (QTestResult::skipCurrentTest() || QTestResult::currentTestFailed())
-                break;
 
-            QBenchmarkTestMethodData::current->result = QBenchmarkResult();
-            QBenchmarkTestMethodData::current->resultAccepted = false;
+            const bool initQuit =
+                QTestResult::skipCurrentTest() || QTestResult::currentTestFailed();
+            if (!initQuit) {
+                QBenchmarkTestMethodData::current->result = QBenchmarkResult();
+                QBenchmarkTestMethodData::current->resultAccepted = false;
 
-            QBenchmarkGlobalData::current->context.tag =
-                QLatin1String(
-                    QTestResult::currentDataTag()
-                    ? QTestResult::currentDataTag() : "");
+                QBenchmarkGlobalData::current->context.tag = QLatin1String(
+                    QTestResult::currentDataTag() ? QTestResult::currentDataTag() : "");
 
-            invokeOk = m_methods[index].invoke(QTest::currentTestObject, Qt::DirectConnection);
-            if (!invokeOk)
-                QTestResult::addFailure("Unable to execute slot", __FILE__, __LINE__);
+                invokeOk = m_methods[index].invoke(QTest::currentTestObject, Qt::DirectConnection);
+                if (!invokeOk)
+                    QTestResult::addFailure("Unable to execute slot", __FILE__, __LINE__);
 
-            isBenchmark = QBenchmarkTestMethodData::current->isBenchmark();
+                isBenchmark = QBenchmarkTestMethodData::current->isBenchmark();
+            } else {
+                invokeOk = false;
+            }
 
             QTestResult::finishedCurrentTestData();
 
-            if (m_cleanupMethod.isValid())
-                m_cleanupMethod.invoke(QTest::currentTestObject, Qt::DirectConnection);
+            if (!initQuit) {
+                if (m_cleanupMethod.isValid())
+                    m_cleanupMethod.invoke(QTest::currentTestObject, Qt::DirectConnection);
 
-            // Process any deleteLater(), like event-loop based apps would do. Fixes memleak reports.
-            if (QCoreApplication::instance())
-                QCoreApplication::sendPostedEvents(nullptr, QEvent::DeferredDelete);
-
-            // If the test isn't a benchmark, finalize the result after cleanup() has finished.
+                // Process any deleteLater(), used by event-loop-based apps.
+                // Fixes memleak reports.
+                if (QCoreApplication::instance())
+                    QCoreApplication::sendPostedEvents(nullptr, QEvent::DeferredDelete);
+            }
+            // If the test isn't a benchmark, finalize the result after
+            // cleanup() has finished (or init has lead us to skip the test).
             if (!isBenchmark)
                 QTestResult::finishedCurrentTestDataCleanup();
 
@@ -1522,7 +1527,7 @@ void TestMethods::invokeTests(QObject *testObject) const
 
     QSignalDumper::startDump();
 
-    if (!QTestResult::skipCurrentTest() && !QTest::currentTestFailed()) {
+    if (!QTestResult::skipCurrentTest() && !QTestResult::currentTestFailed()) {
         if (m_initTestCaseMethod.isValid())
             m_initTestCaseMethod.invoke(testObject, Qt::DirectConnection);
 
@@ -1544,12 +1549,15 @@ void TestMethods::invokeTests(QObject *testObject) const
             }
         }
 
+        const bool wasSkipped = QTestResult::skipCurrentTest();
         QTestResult::setSkipCurrentTest(false);
         QTestResult::setBlacklistCurrentTest(false);
         QTestResult::setCurrentTestFunction("cleanupTestCase");
         if (m_cleanupTestCaseMethod.isValid())
             m_cleanupTestCaseMethod.invoke(testObject, Qt::DirectConnection);
         QTestResult::finishedCurrentTestData();
+        // Restore skip state as it affects decision on whether we passed:
+        QTestResult::setSkipCurrentTest(wasSkipped || QTestResult::skipCurrentTest());
         QTestResult::finishedCurrentTestDataCleanup();
     }
     QTestResult::finishedCurrentTestFunction();

src/testlib/qtestlog.cpp

@@ -111,6 +111,7 @@ namespace QTest {
     int passes = 0;
     int skips = 0;
     int blacklists = 0;
+    enum { Unresolved, Passed, Skipped, Suppressed, Failed } currentTestState;
 
     struct IgnoreResultList
     {
@@ -345,19 +346,27 @@ void QTestLog::clearIgnoreMessages()
     QTest::IgnoreResultList::clearList(QTest::ignoreResultList);
 }
 
+
 void QTestLog::clearFailOnWarnings()
 {
     QTest::failOnWarningList.clear();
 }
 
+void QTestLog::clearCurrentTestState()
+{
+    QTest::currentTestState = QTest::Unresolved;
+}
+
 void QTestLog::addPass(const char *msg)
 {
     if (printAvailableTags)
         return;
 
     QTEST_ASSERT(msg);
+    Q_ASSERT(QTest::currentTestState == QTest::Unresolved);
 
     ++QTest::passes;
+    QTest::currentTestState = QTest::Passed;
 
     FOREACH_TEST_LOGGER
         logger->addIncident(QAbstractTestLogger::Pass, msg);
@@ -367,8 +376,18 @@ void QTestLog::addFail(const char *msg, const char *file, int line)
 {
     QTEST_ASSERT(msg);
 
-    ++QTest::fails;
+    if (QTest::currentTestState == QTest::Unresolved) {
+        ++QTest::fails;
+    } else {
+        // After an XPASS/Continue, or fail or skip in a function the test
+        // calls, we can subsequently fail.
+        Q_ASSERT(QTest::currentTestState == QTest::Failed
+                 || QTest::currentTestState == QTest::Skipped);
+    }
+    // It is up to particular loggers to decide whether to report such
+    // subsequent failures; they may carry useful information.
 
+    QTest::currentTestState = QTest::Failed;
     FOREACH_TEST_LOGGER
         logger->addIncident(QAbstractTestLogger::Fail, msg, file, line);
 }
@@ -387,8 +406,16 @@ void QTestLog::addXPass(const char *msg, const char *file, int line)
 {
     QTEST_ASSERT(msg);
 
-    ++QTest::fails;
+    if (QTest::currentTestState == QTest::Unresolved) {
+        ++QTest::fails;
+    } else {
+        // After an XPASS/Continue, we can subsequently XPASS again.
+        // Likewise after a fail or skip in a function called by the test.
+        Q_ASSERT(QTest::currentTestState == QTest::Failed
+                 || QTest::currentTestState == QTest::Skipped);
+    }
 
+    QTest::currentTestState = QTest::Failed;
     FOREACH_TEST_LOGGER
         logger->addIncident(QAbstractTestLogger::XPass, msg, file, line);
 }
@@ -396,8 +423,10 @@ void QTestLog::addXPass(const char *msg, const char *file, int line)
 void QTestLog::addBPass(const char *msg)
 {
     QTEST_ASSERT(msg);
+    Q_ASSERT(QTest::currentTestState == QTest::Unresolved);
 
-    ++QTest::blacklists;
+    ++QTest::blacklists; // Not passes ?
+    QTest::currentTestState = QTest::Suppressed;
 
     FOREACH_TEST_LOGGER
         logger->addIncident(QAbstractTestLogger::BlacklistedPass, msg);
@@ -407,8 +436,16 @@ void QTestLog::addBFail(const char *msg, const char *file, int line)
 {
     QTEST_ASSERT(msg);
 
-    ++QTest::blacklists;
+    if (QTest::currentTestState == QTest::Unresolved) {
+        ++QTest::blacklists;
+    } else {
+        // After a BXPASS/Continue, we can subsequently fail.
+        // Likewise after a fail or skip in a function called by a test.
+        Q_ASSERT(QTest::currentTestState == QTest::Suppressed
+                 || QTest::currentTestState == QTest::Skipped);
+    }
 
+    QTest::currentTestState = QTest::Suppressed;
     FOREACH_TEST_LOGGER
         logger->addIncident(QAbstractTestLogger::BlacklistedFail, msg, file, line);
 }
@@ -417,8 +454,16 @@ void QTestLog::addBXPass(const char *msg, const char *file, int line)
 {
     QTEST_ASSERT(msg);
 
-    ++QTest::blacklists;
+    if (QTest::currentTestState == QTest::Unresolved) {
+        ++QTest::blacklists;
+    } else {
+        // After a BXPASS/Continue, we may BXPASS again.
+        // Likewise after a fail or skip in a function called by a test.
+        Q_ASSERT(QTest::currentTestState == QTest::Suppressed
+                 || QTest::currentTestState == QTest::Skipped);
+    }
 
+    QTest::currentTestState = QTest::Suppressed;
     FOREACH_TEST_LOGGER
         logger->addIncident(QAbstractTestLogger::BlacklistedXPass, msg, file, line);
 }
@@ -437,7 +482,18 @@ void QTestLog::addSkip(const char *msg, const char *file, int line)
 {
     QTEST_ASSERT(msg);
 
-    ++QTest::skips;
+    if (QTest::currentTestState == QTest::Unresolved) {
+        ++QTest::skips;
+        QTest::currentTestState = QTest::Skipped;
+    } else {
+        // After an B?XPASS/Continue, we might subsequently skip.
+        // Likewise after a skip in a function called by a test.
+        Q_ASSERT(QTest::currentTestState == QTest::Suppressed
+                 || QTest::currentTestState == QTest::Failed
+                 || QTest::currentTestState == QTest::Skipped);
+    }
+    // It is up to particular loggers to decide whether to report such
+    // subsequent skips; they may carry useful information.
 
     FOREACH_TEST_LOGGER
         logger->addIncident(QAbstractTestLogger::Skip, msg, file, line);

src/testlib/qtestlog_p.h

@@ -1,6 +1,6 @@
 /****************************************************************************
 **
-** Copyright (C) 2016 The Qt Company Ltd.
+** Copyright (C) 2021 The Qt Company Ltd.
 ** Contact: https://www.qt.io/licensing/
 **
 ** This file is part of the QtTest module of the Qt Toolkit.
@@ -113,6 +113,7 @@ public:
     static void printUnhandledIgnoreMessages();
     static void clearIgnoreMessages();
     static void clearFailOnWarnings();
+    static void clearCurrentTestState();
 
     static void warn(const char *msg, const char *file, int line);
     static void info(const char *msg, const char *file, int line);

src/testlib/qtestresult.cpp

@@ -219,6 +219,7 @@ void QTestResult::finishedCurrentTestDataCleanup()
             QTestLog::addPass("");
     }
 
+    QTestLog::clearCurrentTestState();
     QTest::resetFailed();
 }
 
@@ -232,6 +233,7 @@ void QTestResult::finishedCurrentTestDataCleanup()
 */
 void QTestResult::finishedCurrentTestFunction()
 {
+    QTestLog::clearCurrentTestState(); // Needed if _data() skipped.
     QTestLog::leaveTestFunction();
 
     QTest::currentTestFunc = nullptr;
@@ -296,6 +298,7 @@ static bool checkStatement(bool statement, const char *msg, const char *file, in
                 QTestLog::addXPass(msg, file, line);
 
             QTest::setFailed(true);
+            // Should B?XPass always (a) continue or (b) abort, regardless of mode ?
             bool doContinue = (QTest::expectFailMode == QTest::Continue);
             clearExpectFail();
             return doContinue;

tests/auto/testlib/selftests/blacklisted/BLACKLIST

@@ -9,6 +9,12 @@ obscure # no such platform; is ignored
 [fail]
 *
 
+[multiSkip]
+*
+
+[multiFail]
+*
+
 [xfail]
 *
 

tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp

@@ -41,6 +41,8 @@ private slots:
     void skip();
     void fail();
     void xfail();
+    void multiSkip();
+    void multiFail();
     void xfailContinueSkip();
     void xfailContinueFail();
     void xpass();
@@ -86,6 +88,23 @@ void tst_Blacklisted::fail()
     QVERIFY2(false, "This test should BFAIL");
 }
 
+void tst_Blacklisted::multiFail() // cf. ../subtest/'s similar tests
+{
+    ++blacklisted;
+    for (int i = 0; i < 10; ++i)
+        []() { QFAIL("This failure message should be repeated ten times"); }();
+    QFAIL("But this test should only contribute one to the blacklisted count");
+}
+
+void tst_Blacklisted::multiSkip()
+{
+    // Similar to multiFail()
+    ++skipped;
+    for (int i = 0; i < 10; ++i)
+        []() { QSKIP("This skip should be repeated ten times"); }();
+    QSKIP("But this test should only contribute one to the skip count");
+}
+
 void tst_Blacklisted::xfail()
 {
     ++blacklisted;
@@ -121,7 +140,6 @@ void tst_Blacklisted::xpassContinueSkip()
     ++blacklisted;
     QEXPECT_FAIL("", "This test should BXPASS then SKIP", Continue);
     QVERIFY2(true, "This test should BXPASS then SKIP");
-    // FIXME QTBUG-95661: skip gets counted
     QSKIP("This skip should be seen but not counted");
 }
 
@@ -130,7 +148,6 @@ void tst_Blacklisted::xpassContinueFail()
     ++blacklisted;
     QEXPECT_FAIL("", "This test should BXPASS then BFAIL", Continue);
     QVERIFY2(true, "This test should BXPASS then BFAIL");
-    // FIXME QTBUG-95661: gets double-counted
     QFAIL("This fail should be seen and not counted (due to prior XPASS)");
 }
 

tests/auto/testlib/selftests/expected_benchlibtickcounter.csv

@@ -1 +1 @@
-"threeBillionTicks","","CPUTicks",3000023453,3000023453,1
+"threeBillionTicks","","CPUTicks",3000012216,3000012216,1

tests/auto/testlib/selftests/expected_blacklisted.junitxml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite name="tst_Blacklisted" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="11" failures="0" errors="0" skipped="3" time="@TEST_DURATION@">
+<testsuite name="tst_Blacklisted" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="13" failures="0" errors="0" skipped="3" time="@TEST_DURATION@">
   <properties>
     <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
     <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
@@ -16,6 +16,20 @@
   </testcase>
   <testcase name="fail" classname="tst_Blacklisted" time="@TEST_DURATION@"/>
   <testcase name="xfail" classname="tst_Blacklisted" time="@TEST_DURATION@"/>
+  <testcase name="multiSkip" classname="tst_Blacklisted" time="@TEST_DURATION@">
+    <skipped message="This skip should be repeated ten times"/>
+    <skipped message="This skip should be repeated ten times"/>
+    <skipped message="This skip should be repeated ten times"/>
+    <skipped message="This skip should be repeated ten times"/>
+    <skipped message="This skip should be repeated ten times"/>
+    <skipped message="This skip should be repeated ten times"/>
+    <skipped message="This skip should be repeated ten times"/>
+    <skipped message="This skip should be repeated ten times"/>
+    <skipped message="This skip should be repeated ten times"/>
+    <skipped message="This skip should be repeated ten times"/>
+    <skipped message="But this test should only contribute one to the skip count"/>
+  </testcase>
+  <testcase name="multiFail" classname="tst_Blacklisted" time="@TEST_DURATION@"/>
   <testcase name="xfailContinueSkip" classname="tst_Blacklisted" time="@TEST_DURATION@">
     <skipped message="This skip should be seen and counted"/>
   </testcase>
@@ -27,7 +41,7 @@
   <testcase name="xpassContinueFail" classname="tst_Blacklisted" time="@TEST_DURATION@"/>
   <testcase name="cleanupTestCase" classname="tst_Blacklisted" time="@TEST_DURATION@">
     <system-out>
-      <![CDATA[Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted]]>
+      <![CDATA[Totals should add up to 13: 2 passed, 0 failed, 3 skipped, 8 blacklisted]]>
     </system-out>
   </testcase>
 </testsuite>

tests/auto/testlib/selftests/expected_blacklisted.lightxml

@@ -33,6 +33,78 @@
     <Incident type="bpass" file="" line="0" />
     <Duration msecs="0"/>
   </TestFunction>
+  <TestFunction name="multiSkip">
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[But this test should only contribute one to the skip count]]></Description>
+    </Incident>
+    <Duration msecs="0"/>
+  </TestFunction>
+  <TestFunction name="multiFail">
+    <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[But this test should only contribute one to the blacklisted count]]></Description>
+    </Incident>
+    <Duration msecs="0"/>
+  </TestFunction>
   <TestFunction name="xfailContinueSkip">
     <Incident type="bxfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
       <Description><![CDATA[This test should BXFAIL then SKIP]]></Description>
@@ -77,7 +149,7 @@
   </TestFunction>
   <TestFunction name="cleanupTestCase">
   <Message type="qdebug" file="" line="0">
-    <Description><![CDATA[Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted]]></Description>
+    <Description><![CDATA[Totals should add up to 13: 2 passed, 0 failed, 3 skipped, 8 blacklisted]]></Description>
   </Message>
     <Incident type="pass" file="" line="0" />
     <Duration msecs="0"/>

tests/auto/testlib/selftests/expected_blacklisted.tap

@@ -23,32 +23,120 @@ not ok 5 - xfail() # TODO This test should BXFAIL then BPASS
   file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
   line: 0
   ...
-not ok 6 - xfailContinueSkip() # TODO This test should BXFAIL then SKIP
+ok 6 - multiSkip() # SKIP This skip should be repeated ten times
+ok 6 - multiSkip() # SKIP This skip should be repeated ten times
+ok 6 - multiSkip() # SKIP This skip should be repeated ten times
+ok 6 - multiSkip() # SKIP This skip should be repeated ten times
+ok 6 - multiSkip() # SKIP This skip should be repeated ten times
+ok 6 - multiSkip() # SKIP This skip should be repeated ten times
+ok 6 - multiSkip() # SKIP This skip should be repeated ten times
+ok 6 - multiSkip() # SKIP This skip should be repeated ten times
+ok 6 - multiSkip() # SKIP This skip should be repeated ten times
+ok 6 - multiSkip() # SKIP This skip should be repeated ten times
+ok 6 - multiSkip() # SKIP But this test should only contribute one to the skip count
+not ok 7 - multiFail() # TODO This failure message should be repeated ten times
+  ---
+  # This failure message should be repeated ten times
+  at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
+  line: 0
+  ...
+not ok 7 - multiFail() # TODO This failure message should be repeated ten times
+  ---
+  # This failure message should be repeated ten times
+  at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
+  line: 0
+  ...
+not ok 7 - multiFail() # TODO This failure message should be repeated ten times
+  ---
+  # This failure message should be repeated ten times
+  at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
+  line: 0
+  ...
+not ok 7 - multiFail() # TODO This failure message should be repeated ten times
+  ---
+  # This failure message should be repeated ten times
+  at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
+  line: 0
+  ...
+not ok 7 - multiFail() # TODO This failure message should be repeated ten times
+  ---
+  # This failure message should be repeated ten times
+  at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
+  line: 0
+  ...
+not ok 7 - multiFail() # TODO This failure message should be repeated ten times
+  ---
+  # This failure message should be repeated ten times
+  at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
+  line: 0
+  ...
+not ok 7 - multiFail() # TODO This failure message should be repeated ten times
+  ---
+  # This failure message should be repeated ten times
+  at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
+  line: 0
+  ...
+not ok 7 - multiFail() # TODO This failure message should be repeated ten times
+  ---
+  # This failure message should be repeated ten times
+  at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
+  line: 0
+  ...
+not ok 7 - multiFail() # TODO This failure message should be repeated ten times
+  ---
+  # This failure message should be repeated ten times
+  at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
+  line: 0
+  ...
+not ok 7 - multiFail() # TODO This failure message should be repeated ten times
+  ---
+  # This failure message should be repeated ten times
+  at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
+  line: 0
+  ...
+not ok 7 - multiFail() # TODO But this test should only contribute one to the blacklisted count
+  ---
+  # But this test should only contribute one to the blacklisted count
+  at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
+  line: 0
+  ...
+not ok 8 - xfailContinueSkip() # TODO This test should BXFAIL then SKIP
   ---
   # This test should BXFAIL then SKIP
   at: tst_Blacklisted::xfailContinueSkip() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
   line: 0
   ...
-ok 6 - xfailContinueSkip() # SKIP This skip should be seen and counted
-not ok 7 - xfailContinueFail() # TODO This test should BXFAIL then BFAIL
+ok 8 - xfailContinueSkip() # SKIP This skip should be seen and counted
+not ok 9 - xfailContinueFail() # TODO This test should BXFAIL then BFAIL
   ---
   # This test should BXFAIL then BFAIL
   at: tst_Blacklisted::xfailContinueFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
   line: 0
   ...
-not ok 7 - xfailContinueFail() # TODO This fail should be seen and counted as blacklisted
+not ok 9 - xfailContinueFail() # TODO This fail should be seen and counted as blacklisted
   ---
   # This fail should be seen and counted as blacklisted
   at: tst_Blacklisted::xfailContinueFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
   line: 0
   ...
-ok 8 - xpass() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS)
-ok 9 - xpassContinueSkip() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS then SKIP)
-ok 10 - xpassContinueSkip() # SKIP This skip should be seen but not counted
-ok 11 - xpassContinueFail() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS then BFAIL)
+ok 10 - xpass() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS)
+ok 11 - xpassContinueSkip() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS then SKIP)
+ok 11 - xpassContinueSkip() # SKIP This skip should be seen but not counted
+ok 12 - xpassContinueFail() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS then BFAIL)
 not ok 12 - xpassContinueFail() # TODO This fail should be seen and not counted (due to prior XPASS)
   ---
   # This fail should be seen and not counted (due to prior XPASS)
@@ -56,7 +144,7 @@ not ok 12 - xpassContinueFail() # TODO This fail should be seen and not counted
   file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
   line: 0
   ...
-# Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted
+# Totals should add up to 13: 2 passed, 0 failed, 3 skipped, 8 blacklisted
 ok 13 - cleanupTestCase()
 1..13
 # tests 13

tests/auto/testlib/selftests/expected_blacklisted.teamcity

@@ -12,6 +12,41 @@
 ##teamcity[testStarted name='xfail()' flowId='tst_Blacklisted']
 ##teamcity[testFinished name='xfail()' flowId='tst_Blacklisted']
 ##teamcity[testFinished name='xfail()' flowId='tst_Blacklisted']
+##teamcity[testStarted name='multiSkip()' flowId='tst_Blacklisted']
+##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted']
+##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted']
+##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted']
+##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted']
+##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted']
+##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted']
+##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted']
+##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted']
+##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted']
+##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted']
+##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted']
+##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted']
+##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted']
+##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted']
+##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted']
+##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted']
+##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted']
+##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted']
+##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted']
+##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted']
+##teamcity[testIgnored name='multiSkip()' message='But this test should only contribute one to the skip count |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted']
+##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted']
+##teamcity[testStarted name='multiFail()' flowId='tst_Blacklisted']
+##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted']
+##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted']
+##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted']
+##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted']
+##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted']
+##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted']
+##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted']
+##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted']
+##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted']
+##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted']
+##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted']
 ##teamcity[testStarted name='xfailContinueSkip()' flowId='tst_Blacklisted']
 ##teamcity[testFinished name='xfailContinueSkip()' flowId='tst_Blacklisted']
 ##teamcity[testIgnored name='xfailContinueSkip()' message='This skip should be seen and counted |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted']
@@ -29,6 +64,6 @@
 ##teamcity[testFinished name='xpassContinueFail()' flowId='tst_Blacklisted']
 ##teamcity[testFinished name='xpassContinueFail()' flowId='tst_Blacklisted']
 ##teamcity[testStarted name='cleanupTestCase()' flowId='tst_Blacklisted']
-##teamcity[testStdOut name='cleanupTestCase()' out='QDEBUG: Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted' flowId='tst_Blacklisted']
+##teamcity[testStdOut name='cleanupTestCase()' out='QDEBUG: Totals should add up to 13: 2 passed, 0 failed, 3 skipped, 8 blacklisted' flowId='tst_Blacklisted']
 ##teamcity[testFinished name='cleanupTestCase()' flowId='tst_Blacklisted']
 ##teamcity[testSuiteFinished name='tst_Blacklisted' flowId='tst_Blacklisted']

tests/auto/testlib/selftests/expected_blacklisted.txt

@@ -10,6 +10,50 @@ BFAIL  : tst_Blacklisted::fail() 'false' returned FALSE. (This test should BFAIL
 BXFAIL : tst_Blacklisted::xfail() This test should BXFAIL then BPASS
    Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
 BPASS  : tst_Blacklisted::xfail()
+SKIP   : tst_Blacklisted::multiSkip() This skip should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+SKIP   : tst_Blacklisted::multiSkip() This skip should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+SKIP   : tst_Blacklisted::multiSkip() This skip should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+SKIP   : tst_Blacklisted::multiSkip() This skip should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+SKIP   : tst_Blacklisted::multiSkip() This skip should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+SKIP   : tst_Blacklisted::multiSkip() This skip should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+SKIP   : tst_Blacklisted::multiSkip() This skip should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+SKIP   : tst_Blacklisted::multiSkip() This skip should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+SKIP   : tst_Blacklisted::multiSkip() This skip should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+SKIP   : tst_Blacklisted::multiSkip() This skip should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+SKIP   : tst_Blacklisted::multiSkip() But this test should only contribute one to the skip count
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+BFAIL  : tst_Blacklisted::multiFail() This failure message should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+BFAIL  : tst_Blacklisted::multiFail() This failure message should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+BFAIL  : tst_Blacklisted::multiFail() This failure message should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+BFAIL  : tst_Blacklisted::multiFail() This failure message should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+BFAIL  : tst_Blacklisted::multiFail() This failure message should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+BFAIL  : tst_Blacklisted::multiFail() This failure message should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+BFAIL  : tst_Blacklisted::multiFail() This failure message should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+BFAIL  : tst_Blacklisted::multiFail() This failure message should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+BFAIL  : tst_Blacklisted::multiFail() This failure message should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+BFAIL  : tst_Blacklisted::multiFail() This failure message should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+BFAIL  : tst_Blacklisted::multiFail() But this test should only contribute one to the blacklisted count
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
 BXFAIL : tst_Blacklisted::xfailContinueSkip() This test should BXFAIL then SKIP
    Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
 SKIP   : tst_Blacklisted::xfailContinueSkip() This skip should be seen and counted
@@ -28,7 +72,7 @@ BXPASS : tst_Blacklisted::xpassContinueFail() 'true' returned TRUE unexpectedly.
    Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
 BFAIL  : tst_Blacklisted::xpassContinueFail() This fail should be seen and not counted (due to prior XPASS)
    Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
-QDEBUG : tst_Blacklisted::cleanupTestCase() Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted
+QDEBUG : tst_Blacklisted::cleanupTestCase() Totals should add up to 13: 2 passed, 0 failed, 3 skipped, 8 blacklisted
 PASS   : tst_Blacklisted::cleanupTestCase()
 Totals: 2 passed, 0 failed, 3 skipped, 8 blacklisted, 0ms
 ********* Finished testing of tst_Blacklisted *********

tests/auto/testlib/selftests/expected_blacklisted.xml

@@ -35,6 +35,78 @@
     <Incident type="bpass" file="" line="0" />
     <Duration msecs="0"/>
   </TestFunction>
+  <TestFunction name="multiSkip">
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[But this test should only contribute one to the skip count]]></Description>
+    </Incident>
+    <Duration msecs="0"/>
+  </TestFunction>
+  <TestFunction name="multiFail">
+    <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+      <Description><![CDATA[But this test should only contribute one to the blacklisted count]]></Description>
+    </Incident>
+    <Duration msecs="0"/>
+  </TestFunction>
   <TestFunction name="xfailContinueSkip">
     <Incident type="bxfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
       <Description><![CDATA[This test should BXFAIL then SKIP]]></Description>
@@ -79,7 +151,7 @@
   </TestFunction>
   <TestFunction name="cleanupTestCase">
   <Message type="qdebug" file="" line="0">
-    <Description><![CDATA[Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted]]></Description>
+    <Description><![CDATA[Totals should add up to 13: 2 passed, 0 failed, 3 skipped, 8 blacklisted]]></Description>
   </Message>
     <Incident type="pass" file="" line="0" />
     <Duration msecs="0"/>

tests/auto/testlib/selftests/expected_expectfail.junitxml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite name="tst_ExpectFail" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="44" failures="17" errors="0" skipped="5" time="@TEST_DURATION@">
+<testsuite name="tst_ExpectFail" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="44" failures="17" errors="0" skipped="4" time="@TEST_DURATION@">
   <properties>
     <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
     <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>

tests/auto/testlib/selftests/expected_expectfail.tap

@@ -145,29 +145,29 @@ ok 32 - xpassAbortXfailContinue() # TODO 'true' returned TRUE unexpectedly. ()
 ok 33 - xpassContinue() # TODO 'true' returned TRUE unexpectedly. ()
 # This should be reached
 ok 34 - xpassContinueSkip() # TODO 'true' returned TRUE unexpectedly. ()
-ok 35 - xpassContinueSkip() # SKIP This should be reached but not increment skip-count
-ok 36 - xpassContinueXfailAbort() # TODO 'true' returned TRUE unexpectedly. ()
-not ok 37 - xpassContinueXfailAbort() # TODO This test should xfail but not add to totals
+ok 34 - xpassContinueSkip() # SKIP This should be reached but not increment skip-count
+ok 35 - xpassContinueXfailAbort() # TODO 'true' returned TRUE unexpectedly. ()
+not ok 36 - xpassContinueXfailAbort() # TODO This test should xfail but not add to totals
   ---
   at: tst_ExpectFail::xpassContinueXfailAbort() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
   line: 0
   ...
-ok 37 - xpassAbortDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. ()
-ok 38 - xpassAbortDataDrivenWithQVerify(Pass)
-ok 39 - xpassContinueDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. ()
+ok 36 - xpassAbortDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. ()
+ok 37 - xpassAbortDataDrivenWithQVerify(Pass)
+ok 38 - xpassContinueDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. ()
 # Test should Continue past XPASS
 # Test should simply PASS
-ok 40 - xpassContinueDataDrivenWithQVerify(Pass)
-ok 41 - xpassAbortDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly.
-ok 42 - xpassAbortDataDrivenWithQCompare(Pass)
-ok 43 - xpassContinueDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly.
+ok 39 - xpassContinueDataDrivenWithQVerify(Pass)
+ok 40 - xpassAbortDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly.
+ok 41 - xpassAbortDataDrivenWithQCompare(Pass)
+ok 42 - xpassContinueDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly.
 # Test should Continue past XPASS
 # Test should simply PASS
-ok 44 - xpassContinueDataDrivenWithQCompare(Pass)
+ok 43 - xpassContinueDataDrivenWithQCompare(Pass)
 # Totals should add up to 44: 23 passed, 17 failed, 4 skipped
-ok 45 - cleanupTestCase()
-1..45
-# tests 45
+ok 44 - cleanupTestCase()
+1..44
+# tests 44
 # pass 23
 # fail 17

tests/auto/testlib/selftests/expected_expectfail.txt

@@ -110,5 +110,5 @@ QDEBUG : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(Pass) Test should s
 PASS   : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(Pass)
 QDEBUG : tst_ExpectFail::cleanupTestCase() Totals should add up to 44: 23 passed, 17 failed, 4 skipped
 PASS   : tst_ExpectFail::cleanupTestCase()
-Totals: 23 passed, 17 failed, 5 skipped, 0 blacklisted, 0ms
+Totals: 23 passed, 17 failed, 4 skipped, 0 blacklisted, 0ms
 ********* Finished testing of tst_ExpectFail *********

tests/auto/testlib/selftests/expected_skipinit.lightxml

@@ -10,7 +10,6 @@
     <Duration msecs="0"/>
   </TestFunction>
   <TestFunction name="cleanupTestCase">
-    <Incident type="pass" file="" line="0" />
     <Duration msecs="0"/>
   </TestFunction>
   <Duration msecs="0"/>

tests/auto/testlib/selftests/expected_skipinit.tap

@@ -1,8 +1,7 @@
 TAP version 13
 # tst_SkipInit
 ok 1 - initTestCase() # SKIP Skip inside initTestCase. This should skip all tests in the class.
-ok 2 - cleanupTestCase()
-1..2
-# tests 2
-# pass 1
+1..1
+# tests 1
+# pass 0
 # fail 0

tests/auto/testlib/selftests/expected_skipinit.teamcity

@@ -2,6 +2,4 @@
 ##teamcity[testStarted name='initTestCase()' flowId='tst_SkipInit']
 ##teamcity[testIgnored name='initTestCase()' message='Skip inside initTestCase. This should skip all tests in the class. |[Loc: qtbase/tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp(0)|]' flowId='tst_SkipInit']
 ##teamcity[testFinished name='initTestCase()' flowId='tst_SkipInit']
-##teamcity[testStarted name='cleanupTestCase()' flowId='tst_SkipInit']
-##teamcity[testFinished name='cleanupTestCase()' flowId='tst_SkipInit']
 ##teamcity[testSuiteFinished name='tst_SkipInit' flowId='tst_SkipInit']

tests/auto/testlib/selftests/expected_skipinit.txt

@@ -2,6 +2,5 @@
 Config: Using QtTest library
 SKIP   : tst_SkipInit::initTestCase() Skip inside initTestCase. This should skip all tests in the class.
    Loc: [qtbase/tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp(0)]
-PASS   : tst_SkipInit::cleanupTestCase()
-Totals: 1 passed, 0 failed, 1 skipped, 0 blacklisted, 0ms
+Totals: 0 passed, 0 failed, 1 skipped, 0 blacklisted, 0ms
 ********* Finished testing of tst_SkipInit *********

tests/auto/testlib/selftests/expected_skipinit.xml

@@ -12,7 +12,6 @@
     <Duration msecs="0"/>
   </TestFunction>
   <TestFunction name="cleanupTestCase">
-    <Incident type="pass" file="" line="0" />
     <Duration msecs="0"/>
   </TestFunction>
   <Duration msecs="0"/>

tests/auto/testlib/selftests/expected_subtest.junitxml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite name="tst_Subtest" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="10" failures="3" errors="0" skipped="0" time="@TEST_DURATION@">
+<testsuite name="tst_Subtest" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="11" failures="3" errors="0" skipped="1" time="@TEST_DURATION@">
   <properties>
     <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
     <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
@@ -82,6 +82,23 @@
       <![CDATA[cleanup multiFail (null)]]>
     </system-out>
   </testcase>
+  <testcase name="multiSkip" classname="tst_Subtest" time="@TEST_DURATION@">
+    <skipped message="This skip should be repeated ten times"/>
+    <skipped message="This skip should be repeated ten times"/>
+    <skipped message="This skip should be repeated ten times"/>
+    <skipped message="This skip should be repeated ten times"/>
+    <skipped message="This skip should be repeated ten times"/>
+    <skipped message="This skip should be repeated ten times"/>
+    <skipped message="This skip should be repeated ten times"/>
+    <skipped message="This skip should be repeated ten times"/>
+    <skipped message="This skip should be repeated ten times"/>
+    <skipped message="This skip should be repeated ten times"/>
+    <skipped message="But this test should only contribute one to the skip count"/>
+    <system-out>
+      <![CDATA[init multiSkip (null)]]>
+      <![CDATA[cleanup multiSkip (null)]]>
+    </system-out>
+  </testcase>
   <testcase name="cleanupTestCase" classname="tst_Subtest" time="@TEST_DURATION@">
     <system-out>
       <![CDATA[cleanupTestCase cleanupTestCase (null)]]>

tests/auto/testlib/selftests/expected_subtest.lightxml

@@ -195,6 +195,48 @@
   </Message>
     <Duration msecs="0"/>
   </TestFunction>
+  <TestFunction name="multiSkip">
+  <Message type="qdebug" file="" line="0">
+    <Description><![CDATA[init multiSkip (null)]]></Description>
+  </Message>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+      <Description><![CDATA[But this test should only contribute one to the skip count]]></Description>
+    </Incident>
+  <Message type="qdebug" file="" line="0">
+    <Description><![CDATA[cleanup multiSkip (null)]]></Description>
+  </Message>
+    <Duration msecs="0"/>
+  </TestFunction>
   <TestFunction name="cleanupTestCase">
   <Message type="qdebug" file="" line="0">
     <Description><![CDATA[cleanupTestCase cleanupTestCase (null)]]></Description>

tests/auto/testlib/selftests/expected_subtest.tap

@@ -68,70 +68,70 @@ not ok 9 - multiFail()
   file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
   line: 0
   ...
-not ok 10 - multiFail()
+not ok 9 - multiFail()
   ---
   # This failure message should be repeated ten times
   at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
   line: 0
   ...
-not ok 11 - multiFail()
+not ok 9 - multiFail()
   ---
   # This failure message should be repeated ten times
   at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
   line: 0
   ...
-not ok 12 - multiFail()
+not ok 9 - multiFail()
   ---
   # This failure message should be repeated ten times
   at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
   line: 0
   ...
-not ok 13 - multiFail()
+not ok 9 - multiFail()
   ---
   # This failure message should be repeated ten times
   at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
   line: 0
   ...
-not ok 14 - multiFail()
+not ok 9 - multiFail()
   ---
   # This failure message should be repeated ten times
   at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
   line: 0
   ...
-not ok 15 - multiFail()
+not ok 9 - multiFail()
   ---
   # This failure message should be repeated ten times
   at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
   line: 0
   ...
-not ok 16 - multiFail()
+not ok 9 - multiFail()
   ---
   # This failure message should be repeated ten times
   at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
   line: 0
   ...
-not ok 17 - multiFail()
+not ok 9 - multiFail()
   ---
   # This failure message should be repeated ten times
   at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
   line: 0
   ...
-not ok 18 - multiFail()
+not ok 9 - multiFail()
   ---
   # This failure message should be repeated ten times
   at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
   line: 0
   ...
-not ok 19 - multiFail()
+not ok 9 - multiFail()
   ---
   # But this test should only contribute one to the failure count
   at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
@@ -139,9 +139,22 @@ not ok 19 - multiFail()
   line: 0
   ...
 # cleanup multiFail (null)
+# init multiSkip (null)
+ok 10 - multiSkip() # SKIP This skip should be repeated ten times
+ok 10 - multiSkip() # SKIP This skip should be repeated ten times
+ok 10 - multiSkip() # SKIP This skip should be repeated ten times
+ok 10 - multiSkip() # SKIP This skip should be repeated ten times
+ok 10 - multiSkip() # SKIP This skip should be repeated ten times
+ok 10 - multiSkip() # SKIP This skip should be repeated ten times
+ok 10 - multiSkip() # SKIP This skip should be repeated ten times
+ok 10 - multiSkip() # SKIP This skip should be repeated ten times
+ok 10 - multiSkip() # SKIP This skip should be repeated ten times
+ok 10 - multiSkip() # SKIP This skip should be repeated ten times
+ok 10 - multiSkip() # SKIP But this test should only contribute one to the skip count
+# cleanup multiSkip (null)
 # cleanupTestCase cleanupTestCase (null)
-ok 20 - cleanupTestCase()
-1..20
-# tests 20
+ok 11 - cleanupTestCase()
+1..11
+# tests 11
 # pass 7
-# fail 13
+# fail 3

tests/auto/testlib/selftests/expected_subtest.teamcity

@@ -49,7 +49,31 @@
 ##teamcity[testFinished name='multiFail()' flowId='tst_Subtest']
 ##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='But this test should only contribute one to the failure count' flowId='tst_Subtest']
 ##teamcity[testFinished name='multiFail()' flowId='tst_Subtest']
+##teamcity[testStarted name='multiSkip()' flowId='tst_Subtest']
+##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest']
+##teamcity[testStdOut name='multiSkip()' out='QDEBUG: cleanup multiFail (null)|nQDEBUG: init multiSkip (null)' flowId='tst_Subtest']
+##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest']
+##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest']
+##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest']
+##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest']
+##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest']
+##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest']
+##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest']
+##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest']
+##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest']
+##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest']
+##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest']
+##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest']
+##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest']
+##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest']
+##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest']
+##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest']
+##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest']
+##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest']
+##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest']
+##teamcity[testIgnored name='multiSkip()' message='But this test should only contribute one to the skip count |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest']
+##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest']
 ##teamcity[testStarted name='cleanupTestCase()' flowId='tst_Subtest']
-##teamcity[testStdOut name='cleanupTestCase()' out='QDEBUG: cleanup multiFail (null)|nQDEBUG: cleanupTestCase cleanupTestCase (null)' flowId='tst_Subtest']
+##teamcity[testStdOut name='cleanupTestCase()' out='QDEBUG: cleanup multiSkip (null)|nQDEBUG: cleanupTestCase cleanupTestCase (null)' flowId='tst_Subtest']
 ##teamcity[testFinished name='cleanupTestCase()' flowId='tst_Subtest']
 ##teamcity[testSuiteFinished name='tst_Subtest' flowId='tst_Subtest']

tests/auto/testlib/selftests/expected_subtest.txt

@@ -68,7 +68,31 @@ FAIL!  : tst_Subtest::multiFail() This failure message should be repeated ten ti
 FAIL!  : tst_Subtest::multiFail() But this test should only contribute one to the failure count
    Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
 QDEBUG : tst_Subtest::multiFail() cleanup multiFail (null)
+QDEBUG : tst_Subtest::multiSkip() init multiSkip (null)
+SKIP   : tst_Subtest::multiSkip() This skip should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
+SKIP   : tst_Subtest::multiSkip() This skip should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
+SKIP   : tst_Subtest::multiSkip() This skip should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
+SKIP   : tst_Subtest::multiSkip() This skip should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
+SKIP   : tst_Subtest::multiSkip() This skip should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
+SKIP   : tst_Subtest::multiSkip() This skip should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
+SKIP   : tst_Subtest::multiSkip() This skip should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
+SKIP   : tst_Subtest::multiSkip() This skip should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
+SKIP   : tst_Subtest::multiSkip() This skip should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
+SKIP   : tst_Subtest::multiSkip() This skip should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
+SKIP   : tst_Subtest::multiSkip() But this test should only contribute one to the skip count
+   Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
+QDEBUG : tst_Subtest::multiSkip() cleanup multiSkip (null)
 QDEBUG : tst_Subtest::cleanupTestCase() cleanupTestCase cleanupTestCase (null)
 PASS   : tst_Subtest::cleanupTestCase()
-Totals: 7 passed, 13 failed, 0 skipped, 0 blacklisted, 0ms
+Totals: 7 passed, 3 failed, 1 skipped, 0 blacklisted, 0ms
 ********* Finished testing of tst_Subtest *********

tests/auto/testlib/selftests/expected_subtest.xml

@@ -197,6 +197,48 @@
   </Message>
     <Duration msecs="0"/>
   </TestFunction>
+  <TestFunction name="multiSkip">
+  <Message type="qdebug" file="" line="0">
+    <Description><![CDATA[init multiSkip (null)]]></Description>
+  </Message>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+      <Description><![CDATA[This skip should be repeated ten times]]></Description>
+    </Incident>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+      <Description><![CDATA[But this test should only contribute one to the skip count]]></Description>
+    </Incident>
+  <Message type="qdebug" file="" line="0">
+    <Description><![CDATA[cleanup multiSkip (null)]]></Description>
+  </Message>
+    <Duration msecs="0"/>
+  </TestFunction>
   <TestFunction name="cleanupTestCase">
   <Message type="qdebug" file="" line="0">
     <Description><![CDATA[cleanupTestCase cleanupTestCase (null)]]></Description>

tests/auto/testlib/selftests/expected_warnings.junitxml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite name="tst_Warnings" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="15" failures="10" errors="0" skipped="1" time="@TEST_DURATION@">
+<testsuite name="tst_Warnings" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="15" failures="10" errors="0" skipped="0" time="@TEST_DURATION@">
   <properties>
     <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
     <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>

tests/auto/testlib/selftests/expected_warnings.tap

@@ -47,7 +47,7 @@ Ran out of cabbage!
 # Ran out of tortillas!
 # Ran out of oil!
 # nope
-not ok 8 - testFailOnWarnings()
+not ok 7 - testFailOnWarnings()
   ---
   # Received a warning that resulted in a failure:
 Ran out of biscuits!
@@ -55,7 +55,7 @@ Ran out of biscuits!
   file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
   line: 0
   ...
-not ok 9 - testFailOnWarnings()
+not ok 7 - testFailOnWarnings()
   ---
   # Received a warning that resulted in a failure:
 Running low on toothpaste!
@@ -67,8 +67,8 @@ Running low on toothpaste!
 # Running low on toothpaste!
 # Running low on toothpaste!
 # Ran out of muffins!
-ok 10 - testFailOnWarningsCleared()
-not ok 11 - testFailOnWarningsWithData(warning1)
+ok 8 - testFailOnWarningsCleared()
+not ok 9 - testFailOnWarningsWithData(warning1)
   ---
   # Received a warning that resulted in a failure:
 warning1
@@ -79,7 +79,7 @@ warning1
 # warning2
 # warning3
 # warning1
-not ok 12 - testFailOnWarningsWithData(warning2)
+not ok 10 - testFailOnWarningsWithData(warning2)
   ---
   # Received a warning that resulted in a failure:
 warning2
@@ -90,7 +90,7 @@ warning2
 # warning3
 # warning1
 # warning2
-not ok 13 - testFailOnWarningsWithData(warning3)
+not ok 11 - testFailOnWarningsWithData(warning3)
   ---
   # Received a warning that resulted in a failure:
 warning3
@@ -98,14 +98,14 @@ warning3
   file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
   line: 0
   ...
-not ok 14 - testFailOnWarningsFailInHelper()
+not ok 12 - testFailOnWarningsFailInHelper()
   ---
   # This failure message should be printed but not cause the test to abort
   at: tst_Warnings::testFailOnWarningsFailInHelper() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
   line: 0
   ...
-not ok 15 - testFailOnWarningsFailInHelper()
+not ok 12 - testFailOnWarningsFailInHelper()
   ---
   # Received a warning that resulted in a failure:
 Ran out of cabbage!
@@ -113,14 +113,14 @@ Ran out of cabbage!
   file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
   line: 0
   ...
-not ok 16 - testFailOnWarningsFailInHelper()
+not ok 12 - testFailOnWarningsFailInHelper()
   ---
   # My cabbage! :(
   at: tst_Warnings::testFailOnWarningsFailInHelper() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
   line: 0
   ...
-not ok 17 - testFailOnWarningsThenSkip()
+not ok 13 - testFailOnWarningsThenSkip()
   ---
   # Received a warning that resulted in a failure:
 Ran out of cabbage!
@@ -128,10 +128,10 @@ Ran out of cabbage!
   file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
   line: 0
   ...
-ok 18 - testFailOnWarningsThenSkip() # SKIP My cabbage! :(
-ok 19 - testFailOnWarningsAndIgnoreWarnings()
-ok 20 - cleanupTestCase()
-1..20
-# tests 20
+ok 13 - testFailOnWarningsThenSkip() # SKIP My cabbage! :(
+ok 14 - testFailOnWarningsAndIgnoreWarnings()
+ok 15 - cleanupTestCase()
+1..15
+# tests 15
 # pass 5
-# fail 14
+# fail 10

tests/auto/testlib/selftests/expected_warnings.txt

@@ -70,5 +70,5 @@ SKIP   : tst_Warnings::testFailOnWarningsThenSkip() My cabbage! :(
    Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
 PASS   : tst_Warnings::testFailOnWarningsAndIgnoreWarnings()
 PASS   : tst_Warnings::cleanupTestCase()
-Totals: 5 passed, 14 failed, 1 skipped, 0 blacklisted, 0ms
+Totals: 5 passed, 10 failed, 0 skipped, 0 blacklisted, 0ms
 ********* Finished testing of tst_Warnings *********

tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp

@@ -347,7 +347,6 @@ void tst_ExpectFail::xpassContinueSkip() const
     ++failed; // and *not* ++skipped
     QEXPECT_FAIL("", "This test should xpass", Continue);
     QVERIFY(true);
-    // FIXME: QTBUG-95661 skip-count is incremented.
     QSKIP("This should be reached but not increment skip-count");
 }
 

tests/auto/testlib/selftests/subtest/tst_subtest.cpp

@@ -49,6 +49,7 @@ private slots:
     void test3();
 
     void multiFail();
+    void multiSkip();
 private:
     void logNames(const char *caller);
     void table_data();
@@ -136,13 +137,20 @@ void tst_Subtest::test3()
 void tst_Subtest::multiFail()
 {
     // Simulates tests which call a shared function that does common checks, or
-    // that do checks in code run asynchronously from a messae loop.
+    // that do checks in code run asynchronously from a message loop.
     for (int i = 0; i < 10; ++i)
         []() { QFAIL("This failure message should be repeated ten times"); }();
-    // FIXME QTBUG-95661: it gets counted as eleven failures, of course.
     QFAIL("But this test should only contribute one to the failure count");
 }
 
+void tst_Subtest::multiSkip()
+{
+    // Similar to multiFail()
+    for (int i = 0; i < 10; ++i)
+        []() { QSKIP("This skip should be repeated ten times"); }();
+    QSKIP("But this test should only contribute one to the skip count");
+}
+
 QTEST_MAIN(tst_Subtest)
 
 #include "tst_subtest.moc"