Include check for the Totals line in testlib's expectedfail selftest

Count how many we expect to pass, fail and be skipped, so we can
report that for comparison with the actual totals line: this reveals
some double-counting, marked with a FIXME.

Task-number: QTBUG-95661
Change-Id: I8b0f13ded5202ed476d8abfee70ed60f9e639bf9
Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>

tests/auto/testlib/selftests/expected_expectfail.junitxml

@@ -180,5 +180,9 @@
       <![CDATA[Test should simply PASS]]>
     </system-out>
   </testcase>
-  <testcase name="cleanupTestCase" classname="tst_ExpectFail" time="@TEST_DURATION@"/>
+  <testcase name="cleanupTestCase" classname="tst_ExpectFail" time="@TEST_DURATION@">
+    <system-out>
+      <![CDATA[Totals should add up to 44: 23 passed, 17 failed, 4 skipped]]>
+    </system-out>
+  </testcase>
 </testsuite>

tests/auto/testlib/selftests/expected_expectfail.lightxml

@@ -312,6 +312,9 @@
     <Duration msecs="0"/>
 </TestFunction>
 <TestFunction name="cleanupTestCase">
+<Message type="qdebug" file="" line="0">
+    <Description><![CDATA[Totals should add up to 44: 23 passed, 17 failed, 4 skipped]]></Description>
+</Message>
 <Incident type="pass" file="" line="0" />
     <Duration msecs="0"/>
 </TestFunction>

tests/auto/testlib/selftests/expected_expectfail.tap

@@ -177,6 +177,7 @@ ok 43 - xpassContinueDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returne
 # Test should Continue past XPASS
 # Test should simply PASS
 ok 44 - xpassContinueDataDrivenWithQCompare(Pass)
+# Totals should add up to 44: 23 passed, 17 failed, 4 skipped
 ok 45 - cleanupTestCase()
 1..45
 # tests 45

tests/auto/testlib/selftests/expected_expectfail.teamcity

@@ -122,5 +122,6 @@
 ##teamcity[testStdOut name='xpassContinueDataDrivenWithQCompare(Pass)' out='QDEBUG: Test should Continue past XPASS|nQDEBUG: Test should simply PASS' flowId='tst_ExpectFail']
 ##teamcity[testFinished name='xpassContinueDataDrivenWithQCompare(Pass)' flowId='tst_ExpectFail']
 ##teamcity[testStarted name='cleanupTestCase()' flowId='tst_ExpectFail']
+##teamcity[testStdOut name='cleanupTestCase()' out='QDEBUG: Totals should add up to 44: 23 passed, 17 failed, 4 skipped' flowId='tst_ExpectFail']
 ##teamcity[testFinished name='cleanupTestCase()' flowId='tst_ExpectFail']
 ##teamcity[testSuiteFinished name='tst_ExpectFail' flowId='tst_ExpectFail']

tests/auto/testlib/selftests/expected_expectfail.txt

@@ -108,6 +108,7 @@ XPASS  : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(XPass) QCOMPARE(1,
 QDEBUG : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(XPass) Test should Continue past XPASS
 QDEBUG : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(Pass) Test should simply PASS
 PASS   : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(Pass)
+QDEBUG : tst_ExpectFail::cleanupTestCase() Totals should add up to 44: 23 passed, 17 failed, 4 skipped
 PASS   : tst_ExpectFail::cleanupTestCase()
 Totals: 23 passed, 17 failed, 5 skipped, 0 blacklisted, 0ms
 ********* Finished testing of tst_ExpectFail *********

tests/auto/testlib/selftests/expected_expectfail.xml

@@ -314,6 +314,9 @@
     <Duration msecs="0"/>
 </TestFunction>
 <TestFunction name="cleanupTestCase">
+<Message type="qdebug" file="" line="0">
+    <Description><![CDATA[Totals should add up to 44: 23 passed, 17 failed, 4 skipped]]></Description>
+</Message>
 <Incident type="pass" file="" line="0" />
     <Duration msecs="0"/>
 </TestFunction>

tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp

@@ -37,6 +37,8 @@ class tst_ExpectFail: public QObject
     Q_OBJECT
 
 private slots:
+    void cleanupTestCase() const;
+    void init() const;
     void xfailAndContinue() const;
     void xfailAndAbort() const;
     void xfailContinueSkip() const;
@@ -76,9 +78,25 @@ private:
     void xfailDataDriven_data(bool failOnly) const;
     void xpassDataDriven_data() const;
 };
+static int casesTested = 2;
+// What the totals line's numbers *should* be:
+static int passed = 2, failed = 0, skipped = 0;
+// Total and passed get {init,cleanup}TestCase() in addition to the actual tests.
+
+void tst_ExpectFail::init() const
+{
+    ++casesTested;
+}
+
+void tst_ExpectFail::cleanupTestCase() const
+{
+    qDebug("Totals should add up to %d: %d passed, %d failed, %d skipped",
+           casesTested, passed, failed, skipped);
+}
 
 void tst_ExpectFail::xfailAndContinue() const
 {
+    ++passed;
     qDebug("begin");
     QEXPECT_FAIL("", "This should xfail", Continue);
     QVERIFY(false);
@@ -87,6 +105,7 @@ void tst_ExpectFail::xfailAndContinue() const
 
 void tst_ExpectFail::xfailAndAbort() const
 {
+    ++passed;
     qDebug("begin");
     QEXPECT_FAIL("", "This should xfail", Abort);
     QVERIFY(false);
@@ -97,6 +116,7 @@ void tst_ExpectFail::xfailAndAbort() const
 
 void tst_ExpectFail::xfailContinueSkip() const
 {
+    ++skipped;
     QEXPECT_FAIL("", "This should xfail then skip", Continue);
     QVERIFY(false);
     QSKIP("This skip should be reported and counted");
@@ -104,6 +124,7 @@ void tst_ExpectFail::xfailContinueSkip() const
 
 void tst_ExpectFail::xfailAbortSkip() const
 {
+    ++passed;
     QEXPECT_FAIL("", "This should xfail", Abort);
     QVERIFY(false);
 
@@ -113,6 +134,7 @@ void tst_ExpectFail::xfailAbortSkip() const
 
 void tst_ExpectFail::xfailTwice() const
 {
+    ++failed;
     QEXPECT_FAIL("", "Calling QEXPECT_FAIL once is fine", Continue);
     QEXPECT_FAIL("", "Calling QEXPECT_FAIL when already expecting a failure is "
                      "an error and should abort this test function", Continue);
@@ -123,6 +145,7 @@ void tst_ExpectFail::xfailTwice() const
 
 void tst_ExpectFail::xfailDataDrivenTwice() const
 {
+    ++failed;
     // Same with data-driven cases (twist semantics of unused shouldPass; we
     // have four combinations to test):
     QEXPECT_FAIL("Pass Abort", "Calling QEXPECT_FAIL once on a test-case is fine", Abort);
@@ -144,6 +167,7 @@ void tst_ExpectFail::xfailDataDrivenTwice() const
 
 void tst_ExpectFail::xfailWithQString() const
 {
+    ++passed;
     QEXPECT_FAIL("", QString("A string").toLatin1().constData(), Continue);
     QVERIFY(false);
 
@@ -157,6 +181,11 @@ void tst_ExpectFail::xfailDataDrivenWithQString() const
 {
     // This test does not (yet) distinguish the two Pass cases.
     QFETCH(bool, shouldPass);
+    QFETCH(QTest::TestFailMode, failMode);
+    if (shouldPass || failMode == QTest::Continue)
+        ++skipped;
+    else
+        ++passed;
 
     QEXPECT_FAIL("Fail Abort", QString("A string").toLatin1().constData(), Abort);
     QEXPECT_FAIL("Fail Continue", QString("A string").toLatin1().constData(), Continue);
@@ -176,6 +205,7 @@ void tst_ExpectFail::xfailDataDrivenWithQString() const
 void tst_ExpectFail::xfailDataDrivenWithQVerify() const
 {
     // This test does not (yet) distinguish the two Pass cases.
+    ++passed;
     QFETCH(bool, shouldPass);
     QFETCH(QTest::TestFailMode, failMode);
 
@@ -205,6 +235,7 @@ void tst_ExpectFail::xfailDataDriven_data(bool failOnly) const
 void tst_ExpectFail::xfailDataDrivenWithQCompare() const
 {
     // This test does not (yet) distinguish the two Pass cases.
+    ++passed;
     QFETCH(bool, shouldPass);
     QFETCH(QTest::TestFailMode, failMode);
 
@@ -221,6 +252,7 @@ void tst_ExpectFail::xfailDataDrivenWithQCompare() const
 
 void tst_ExpectFail::xfailOnWrongRow() const
 {
+    ++passed;
     qDebug("Should pass (*not* xpass), despite test-case name");
     // QEXPECT_FAIL for a row that does not exist should be ignored.
     // (It might be conditional data(), so exist in other circumstances.)
@@ -237,6 +269,7 @@ void tst_ExpectFail::xfailOnWrongRow() const
 
 void tst_ExpectFail::xfailOnAnyRow() const
 {
+    ++passed;
     // In a data-driven test, passing an empty first parameter to QEXPECT_FAIL
     // should mean that the failure is expected for all data rows.
     QFETCH(QTest::TestFailMode, failMode);
@@ -252,6 +285,7 @@ void tst_ExpectFail::xfailOnAnyRow() const
 
 void tst_ExpectFail::xfailWithoutCheck() const
 {
+    ++failed;
     qDebug("Should fail (*not* xfail), despite test-case name");
     QTEST(false, "shouldPass"); // _data skips the passing tests as pass/fail is irrelevant
     QEXPECT_FAIL("Fail Abort", "Calling QEXPECT_FAIL without any subsequent check is an error",
@@ -262,6 +296,7 @@ void tst_ExpectFail::xfailWithoutCheck() const
 
 void tst_ExpectFail::xpassAbort() const
 {
+    ++failed;
     QEXPECT_FAIL("", "This test should xpass", Abort);
     QVERIFY(true);
 
@@ -272,6 +307,7 @@ void tst_ExpectFail::xpassAbort() const
 
 void tst_ExpectFail::xpassAbortSkip() const
 {
+    ++failed;
     QEXPECT_FAIL("", "This test should xpass", Abort);
     QVERIFY(true);
 
@@ -280,6 +316,7 @@ void tst_ExpectFail::xpassAbortSkip() const
 
 void tst_ExpectFail::xpassAbortXfailContinue() const
 {
+    ++failed;
     QEXPECT_FAIL("", "This test should xpass", Abort);
     QVERIFY(true);
 
@@ -291,6 +328,7 @@ void tst_ExpectFail::xpassAbortXfailContinue() const
 
 void tst_ExpectFail::xpassContinue() const
 {
+    ++failed;
     QEXPECT_FAIL("", "This test should xpass", Continue);
     QVERIFY(true);
     qDebug("This should be reached");
@@ -306,13 +344,16 @@ void tst_ExpectFail::xpassDataDriven_data() const
 
 void tst_ExpectFail::xpassContinueSkip() const
 {
+    ++failed; // and *not* ++skipped
     QEXPECT_FAIL("", "This test should xpass", Continue);
     QVERIFY(true);
+    // FIXME: QTBUG-95661 skip-count is incremented.
     QSKIP("This should be reached but not increment skip-count");
 }
 
 void tst_ExpectFail::xpassContinueXfailAbort() const
 {
+    ++failed;
     QEXPECT_FAIL("", "This test should xpass", Continue);
     QVERIFY(true);
     QEXPECT_FAIL("", "This test should xfail but not add to totals", Abort);
@@ -323,6 +364,10 @@ void tst_ExpectFail::xpassContinueXfailAbort() const
 void tst_ExpectFail::xpassAbortDataDrivenWithQVerify() const
 {
     QFETCH(bool, shouldXPass);
+    if (shouldXPass)
+        ++failed;
+    else
+        ++passed;
 
     QEXPECT_FAIL("XPass", "This test-row should xpass", Abort);
     QVERIFY(true);
@@ -334,6 +379,10 @@ void tst_ExpectFail::xpassAbortDataDrivenWithQVerify() const
 void tst_ExpectFail::xpassContinueDataDrivenWithQVerify() const
 {
     QFETCH(bool, shouldXPass);
+    if (shouldXPass)
+        ++failed;
+    else
+        ++passed;
 
     QEXPECT_FAIL("XPass", "This test-row should xpass", Continue);
     QVERIFY(true);
@@ -344,6 +393,10 @@ void tst_ExpectFail::xpassContinueDataDrivenWithQVerify() const
 void tst_ExpectFail::xpassAbortDataDrivenWithQCompare() const
 {
     QFETCH(bool, shouldXPass);
+    if (shouldXPass)
+        ++failed;
+    else
+        ++passed;
 
     QEXPECT_FAIL("XPass", "This test should xpass", Abort);
     QCOMPARE(1, 1);
@@ -355,6 +408,10 @@ void tst_ExpectFail::xpassAbortDataDrivenWithQCompare() const
 void tst_ExpectFail::xpassContinueDataDrivenWithQCompare() const
 {
     QFETCH(bool, shouldXPass);
+    if (shouldXPass)
+        ++failed;
+    else
+        ++passed;
 
     QEXPECT_FAIL("XPass", "This test should xpass", Continue);
     QCOMPARE(1, 1);