testlib: Add BXPASS and BXFAIL

Prioritize blacklisting over QEXPECT_FAIL so that a test that is
blacklisted no longer fails if QEXPECT_FAIL returns true unexpectedly. To
reflect this state properly, the two values of BXPASS and BXFAIL were
added to testlib's output.

[ChangeLog][Important Behavior Changes][QtTestLib] Blacklisting of tests
will be taken into account for XPASS and XFAIL. A blacklisted test that
causes an XPASS will no longer be a fail.

Task-number: QTBUG-72928
Change-Id: Ia2232fdc714d405fa3fd9aea6c89eb2836bc5950
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>

src/testlib/qabstracttestlogger_p.h

@@ -69,7 +69,9 @@ public:
         Fail,
         XPass,
         BlacklistedPass,
-        BlacklistedFail
+        BlacklistedFail,
+        BlacklistedXPass,
+        BlacklistedXFail
     };
 
     enum MessageTypes {

src/testlib/qappletestlogger.cpp

@@ -101,6 +101,10 @@ static IncidentClassification incidentTypeToClassification(QAbstractTestLogger::
         return IncidentClassification(QtWarningMsg, "bpass");
     case QAbstractTestLogger::BlacklistedFail:
         return IncidentClassification(QtInfoMsg, "bfail");
+    case QAbstractTestLogger::BlacklistedXPass:
+        return IncidentClassification(QtWarningMsg, "bxpass");
+    case QAbstractTestLogger::BlacklistedXFail:
+        return IncidentClassification(QtInfoMsg, "bxfail");
     }
     return IncidentClassification(QtFatalMsg, nullptr);
 }

src/testlib/qplaintestlogger.cpp

@@ -89,6 +89,10 @@ namespace QTest {
             return "BPASS  ";
         case QAbstractTestLogger::BlacklistedFail:
             return "BFAIL  ";
+        case QAbstractTestLogger::BlacklistedXPass:
+            return "BXPASS ";
+        case QAbstractTestLogger::BlacklistedXFail:
+            return "BXFAIL ";
         }
         return "??????";
     }

src/testlib/qtaptestlogger.cpp

@@ -123,13 +123,15 @@ void QTapTestLogger::addIncident(IncidentTypes type, const char *description,
         return;
     }
 
-    bool ok = type == Pass || type == XPass || type == BlacklistedPass;
+    bool ok = type == Pass || type == XPass || type == BlacklistedPass || type == BlacklistedXPass;
 
     QTestCharBuffer directive;
-    if (type == XFail || type == XPass || type == BlacklistedFail || type == BlacklistedPass)
+    if (type == XFail || type == XPass || type == BlacklistedFail || type == BlacklistedPass
+            || type == BlacklistedXFail || type == BlacklistedXPass) {
         // We treat expected or blacklisted failures/passes as TODO-failures/passes,
         // which should be treated as soft issues by consumers. Not all do though :/
         QTest::qt_asprintf(&directive, " # TODO %s", description);
+    }
 
     int testNumber = QTestLog::totalCount();
     if (type == XFail) {

src/testlib/qteamcitylogger.cpp

@@ -66,6 +66,10 @@ namespace QTest {
             return "BPASS";
         case QAbstractTestLogger::BlacklistedFail:
             return "BFAIL";
+        case QAbstractTestLogger::BlacklistedXPass:
+            return "BXPASS";
+        case QAbstractTestLogger::BlacklistedXFail:
+            return "BXFAIL";
         }
         return "??????";
     }

src/testlib/qtestlog.cpp

@@ -455,6 +455,24 @@ void QTestLog::addBFail(const char *msg, const char *file, int line)
     QTest::TestLoggers::addIncident(QAbstractTestLogger::BlacklistedFail, msg, file, line);
 }
 
+void QTestLog::addBXPass(const char *msg, const char *file, int line)
+{
+    QTEST_ASSERT(msg);
+    QTEST_ASSERT(file);
+
+    ++QTest::blacklists;
+
+    QTest::TestLoggers::addIncident(QAbstractTestLogger::BlacklistedXPass, msg, file, line);
+}
+
+void QTestLog::addBXFail(const char *msg, const char *file, int line)
+{
+    QTEST_ASSERT(msg);
+    QTEST_ASSERT(file);
+
+    QTest::TestLoggers::addIncident(QAbstractTestLogger::BlacklistedXFail, msg, file, line);
+}
+
 void QTestLog::addSkip(const char *msg, const char *file, int line)
 {
     QTEST_ASSERT(msg);

src/testlib/qtestlog_p.h

@@ -80,6 +80,8 @@ public:
     static void addXPass(const char *msg, const char *file, int line);
     static void addBPass(const char *msg);
     static void addBFail(const char *msg, const char *file, int line);
+    static void addBXPass(const char *msg, const char *file, int line);
+    static void addBXFail(const char *msg, const char *file, int line);
     static void addSkip(const char *msg, const char *file, int line);
     static void addBenchmarkResult(const QBenchmarkResult &result);
 

src/testlib/qtestresult.cpp

@@ -218,16 +218,23 @@ static bool checkStatement(bool statement, const char *msg, const char *file, in
 {
     if (statement) {
         if (QTest::expectFailMode) {
+            if (QTest::blacklistCurrentTest)
+                QTestLog::addBXPass(msg, file, line);
+            else
                 QTestLog::addXPass(msg, file, line);
+
+            QTest::failed = true;
             bool doContinue = (QTest::expectFailMode == QTest::Continue);
             clearExpectFail();
-            QTest::failed = true;
             return doContinue;
         }
         return true;
     }
 
     if (QTest::expectFailMode) {
+        if (QTest::blacklistCurrentTest)
+            QTestLog::addBXFail(QTest::expectFailComment, file, line);
+        else
             QTestLog::addXFail(QTest::expectFailComment, file, line);
         bool doContinue = (QTest::expectFailMode == QTest::Continue);
         clearExpectFail();

src/testlib/qxmltestlogger.cpp

@@ -91,6 +91,10 @@ namespace QTest {
             return "bpass";
         case QAbstractTestLogger::BlacklistedFail:
             return "bfail";
+        case QAbstractTestLogger::BlacklistedXPass:
+            return "bxpass";
+        case QAbstractTestLogger::BlacklistedXFail:
+            return "bxfail";
         }
         return "??????";
     }

src/testlib/qxunittestlogger.cpp

@@ -180,6 +180,13 @@ void QXunitTestLogger::addIncident(IncidentTypes type, const char *description,
         ++failureCounter;
         typeBuf = "bfail";
         break;
+    case QAbstractTestLogger::BlacklistedXPass:
+        typeBuf = "bxpass";
+        break;
+    case QAbstractTestLogger::BlacklistedXFail:
+        ++failureCounter;
+        typeBuf = "bxfail";
+        break;
     default:
         typeBuf = "??????";
         break;
@@ -212,11 +219,11 @@ void QXunitTestLogger::addIncident(IncidentTypes type, const char *description,
         if (!strcmp(oldResult, "pass")) {
             overwrite = true;
         }
-        else if (!strcmp(oldResult, "bpass")) {
+        else if (!strcmp(oldResult, "bpass") || !strcmp(oldResult, "bxfail")) {
             overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail) || (type == QAbstractTestLogger::XFail)
-                    || (type == QAbstractTestLogger::BlacklistedFail);
+                    || (type == QAbstractTestLogger::BlacklistedFail) || (type == QAbstractTestLogger::BlacklistedXPass);
         }
-        else if (!strcmp(oldResult, "bfail")) {
+        else if (!strcmp(oldResult, "bfail") || !strcmp(oldResult, "bxpass")) {
             overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail) || (type == QAbstractTestLogger::XFail);
         }
         else if (!strcmp(oldResult, "xfail")) {

tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp

@@ -64,14 +64,14 @@ void tst_Blacklisted::fail()
 
 void tst_Blacklisted::xfail()
 {
-    QEXPECT_FAIL("", "This test should XFAIL then BFAIL", Abort);
+    QEXPECT_FAIL("", "This test should BXFAIL then BPASS", Abort);
     QVERIFY(false);
 }
 
 void tst_Blacklisted::xpass()
 {
-    QEXPECT_FAIL("", "This test should XPASS", Abort);
+    QEXPECT_FAIL("", "This test should BXPASS", Abort);
-    QVERIFY2(true, "This test should XPASS, blacklist ignored for XPASS");
+    QVERIFY2(true, "This test should BXPASS");
 }
 
 void tst_Blacklisted::messages()

tests/auto/testlib/selftests/expected_blacklisted.lightxml

@@ -24,15 +24,15 @@
     <Duration msecs="0"/>
 </TestFunction>
 <TestFunction name="xfail">
-<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+<Incident type="bxfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
-    <Description><![CDATA[This test should XFAIL then BFAIL]]></Description>
+    <Description><![CDATA[This test should BXFAIL then BPASS]]></Description>
 </Incident>
 <Incident type="bpass" file="" line="0" />
     <Duration msecs="0"/>
 </TestFunction>
 <TestFunction name="xpass">
-<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+<Incident type="bxpass" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
-    <Description><![CDATA['true' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)]]></Description>
+    <Description><![CDATA['true' returned TRUE unexpectedly. (This test should BXPASS)]]></Description>
 </Incident>
     <Duration msecs="0"/>
 </TestFunction>

tests/auto/testlib/selftests/expected_blacklisted.teamcity

@@ -7,10 +7,9 @@
 ##teamcity[testStarted name='fail()' flowId='tst_Blacklisted']
 ##teamcity[testFinished name='fail()' flowId='tst_Blacklisted']
 ##teamcity[testStarted name='xfail()' flowId='tst_Blacklisted']
-##teamcity[testStdOut name='xfail()' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]: This test should XFAIL then BFAIL' flowId='tst_Blacklisted']
+##teamcity[testFinished name='xfail()' flowId='tst_Blacklisted']
 ##teamcity[testFinished name='xfail()' flowId='tst_Blacklisted']
 ##teamcity[testStarted name='xpass()' flowId='tst_Blacklisted']
-##teamcity[testFailed name='xpass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)' flowId='tst_Blacklisted']
 ##teamcity[testFinished name='xpass()' flowId='tst_Blacklisted']
 ##teamcity[testStarted name='messages()' flowId='tst_Blacklisted']
 ##teamcity[testStdOut name='messages()' out='QWARN: This is a warning that should not appear in silent test output|nWARNING |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]: This is an internal testlib warning that should not appear in silent test output|nQDEBUG: This is a debug message that should not appear in silent test output|nQSYSTEM: This is a critical message that should not appear in silent test output|nQINFO: This is an info message that should not appear in silent test output|nINFO |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]: This is an internal testlib info message that should not appear in silent test output|nQFATAL: This is a fatal error message that should still appear in silent test output' flowId='tst_Blacklisted']

tests/auto/testlib/selftests/expected_blacklisted.txt

@@ -6,10 +6,10 @@ SKIP   : tst_Blacklisted::skip() This test should SKIP
    Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
 BFAIL  : tst_Blacklisted::fail() 'false' returned FALSE. (This test should BFAIL)
    Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
-XFAIL  : tst_Blacklisted::xfail() This test should XFAIL then BFAIL
+BXFAIL : tst_Blacklisted::xfail() This test should BXFAIL then BPASS
    Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
 BPASS  : tst_Blacklisted::xfail()
-XPASS  : tst_Blacklisted::xpass() 'true' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)
+BXPASS : tst_Blacklisted::xpass() 'true' returned TRUE unexpectedly. (This test should BXPASS)
    Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
 QWARN  : tst_Blacklisted::messages() This is a warning that should not appear in silent test output
 WARNING: tst_Blacklisted::messages() This is an internal testlib warning that should not appear in silent test output
@@ -22,5 +22,5 @@ INFO   : tst_Blacklisted::messages() This is an internal testlib info message th
 QFATAL : tst_Blacklisted::messages() This is a fatal error message that should still appear in silent test output
 BFAIL  : tst_Blacklisted::messages() Received a fatal error.
    Loc: [Unknown file(0)]
-Totals: 1 passed, 1 failed, 1 skipped, 4 blacklisted, 0ms
+Totals: 1 passed, 0 failed, 1 skipped, 5 blacklisted, 0ms
 ********* Finished testing of tst_Blacklisted *********

tests/auto/testlib/selftests/expected_blacklisted.xml

@@ -26,15 +26,15 @@
     <Duration msecs="0"/>
 </TestFunction>
 <TestFunction name="xfail">
-<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+<Incident type="bxfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
-    <Description><![CDATA[This test should XFAIL then BFAIL]]></Description>
+    <Description><![CDATA[This test should BXFAIL then BPASS]]></Description>
 </Incident>
 <Incident type="bpass" file="" line="0" />
     <Duration msecs="0"/>
 </TestFunction>
 <TestFunction name="xpass">
-<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+<Incident type="bxpass" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
-    <Description><![CDATA['true' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)]]></Description>
+    <Description><![CDATA['true' returned TRUE unexpectedly. (This test should BXPASS)]]></Description>
 </Incident>
     <Duration msecs="0"/>
 </TestFunction>

tests/auto/testlib/selftests/expected_blacklisted.xunitxml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite errors="9" failures="3" tests="7" name="tst_Blacklisted">
+<testsuite errors="8" failures="3" tests="7" name="tst_Blacklisted">
   <properties>
     <property value="@INSERT_QT_VERSION_HERE@" name="QTestVersion"/>
     <property value="@INSERT_QT_VERSION_HERE@" name="QtVersion"/>
@@ -11,12 +11,8 @@
     <!-- message="This test should SKIP" type="skip" -->
   </testcase>
   <testcase result="bfail" name="fail"/>
-  <testcase result="xfail" name="xfail">
+  <testcase result="bxfail" name="xfail"/>
-    <!-- message="This test should XFAIL then BFAIL" type="info" -->
+  <testcase result="bxpass" name="xpass"/>
-  </testcase>
-  <testcase result="xpass" name="xpass">
-    <failure message="&apos;true&apos; returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)" result="xpass"/>
-  </testcase>
   <testcase result="bfail" name="messages">
     <!-- message="This is a warning that should not appear in silent test output" type="qwarn" -->
     <!-- message="This is an internal testlib warning that should not appear in silent test output" type="warn" -->
@@ -28,7 +24,6 @@
   </testcase>
   <system-err>
 <![CDATA[This test should SKIP]]>
-<![CDATA[This test should XFAIL then BFAIL]]>
 <![CDATA[This is a warning that should not appear in silent test output]]>
 <![CDATA[This is an internal testlib warning that should not appear in silent test output]]>
 <![CDATA[This is a debug message that should not appear in silent test output]]>