diff --git a/src/testlib/qabstracttestlogger_p.h b/src/testlib/qabstracttestlogger_p.h
index 018361b81ef..a64e7ea96fc 100644
--- a/src/testlib/qabstracttestlogger_p.h
+++ b/src/testlib/qabstracttestlogger_p.h
@@ -69,7 +69,9 @@ public:
Fail,
XPass,
BlacklistedPass,
- BlacklistedFail
+ BlacklistedFail,
+ BlacklistedXPass,
+ BlacklistedXFail
};
enum MessageTypes {
diff --git a/src/testlib/qappletestlogger.cpp b/src/testlib/qappletestlogger.cpp
index 2c1005ad806..8e75da88f82 100644
--- a/src/testlib/qappletestlogger.cpp
+++ b/src/testlib/qappletestlogger.cpp
@@ -101,6 +101,10 @@ static IncidentClassification incidentTypeToClassification(QAbstractTestLogger::
return IncidentClassification(QtWarningMsg, "bpass");
case QAbstractTestLogger::BlacklistedFail:
return IncidentClassification(QtInfoMsg, "bfail");
+ case QAbstractTestLogger::BlacklistedXPass:
+ return IncidentClassification(QtWarningMsg, "bxpass");
+ case QAbstractTestLogger::BlacklistedXFail:
+ return IncidentClassification(QtInfoMsg, "bxfail");
}
return IncidentClassification(QtFatalMsg, nullptr);
}
diff --git a/src/testlib/qplaintestlogger.cpp b/src/testlib/qplaintestlogger.cpp
index 853515f2d92..ed53dcdde8d 100644
--- a/src/testlib/qplaintestlogger.cpp
+++ b/src/testlib/qplaintestlogger.cpp
@@ -89,6 +89,10 @@ namespace QTest {
return "BPASS ";
case QAbstractTestLogger::BlacklistedFail:
return "BFAIL ";
+ case QAbstractTestLogger::BlacklistedXPass:
+ return "BXPASS ";
+ case QAbstractTestLogger::BlacklistedXFail:
+ return "BXFAIL ";
}
return "??????";
}
diff --git a/src/testlib/qtaptestlogger.cpp b/src/testlib/qtaptestlogger.cpp
index a81d9d5c8b2..540b36e2735 100644
--- a/src/testlib/qtaptestlogger.cpp
+++ b/src/testlib/qtaptestlogger.cpp
@@ -123,13 +123,15 @@ void QTapTestLogger::addIncident(IncidentTypes type, const char *description,
return;
}
- bool ok = type == Pass || type == XPass || type == BlacklistedPass;
+ bool ok = type == Pass || type == XPass || type == BlacklistedPass || type == BlacklistedXPass;
QTestCharBuffer directive;
- if (type == XFail || type == XPass || type == BlacklistedFail || type == BlacklistedPass)
+ if (type == XFail || type == XPass || type == BlacklistedFail || type == BlacklistedPass
+ || type == BlacklistedXFail || type == BlacklistedXPass) {
// We treat expected or blacklisted failures/passes as TODO-failures/passes,
// which should be treated as soft issues by consumers. Not all do though :/
QTest::qt_asprintf(&directive, " # TODO %s", description);
+ }
int testNumber = QTestLog::totalCount();
if (type == XFail) {
diff --git a/src/testlib/qteamcitylogger.cpp b/src/testlib/qteamcitylogger.cpp
index 9cfbe92b7d2..577c8e70cd9 100644
--- a/src/testlib/qteamcitylogger.cpp
+++ b/src/testlib/qteamcitylogger.cpp
@@ -66,6 +66,10 @@ namespace QTest {
return "BPASS";
case QAbstractTestLogger::BlacklistedFail:
return "BFAIL";
+ case QAbstractTestLogger::BlacklistedXPass:
+ return "BXPASS";
+ case QAbstractTestLogger::BlacklistedXFail:
+ return "BXFAIL";
}
return "??????";
}
diff --git a/src/testlib/qtestlog.cpp b/src/testlib/qtestlog.cpp
index 1268730cc69..3285a6d8a7d 100644
--- a/src/testlib/qtestlog.cpp
+++ b/src/testlib/qtestlog.cpp
@@ -455,6 +455,24 @@ void QTestLog::addBFail(const char *msg, const char *file, int line)
QTest::TestLoggers::addIncident(QAbstractTestLogger::BlacklistedFail, msg, file, line);
}
+void QTestLog::addBXPass(const char *msg, const char *file, int line)
+{
+ QTEST_ASSERT(msg);
+ QTEST_ASSERT(file);
+
+ ++QTest::blacklists;
+
+ QTest::TestLoggers::addIncident(QAbstractTestLogger::BlacklistedXPass, msg, file, line);
+}
+
+void QTestLog::addBXFail(const char *msg, const char *file, int line)
+{
+ QTEST_ASSERT(msg);
+ QTEST_ASSERT(file);
+
+ QTest::TestLoggers::addIncident(QAbstractTestLogger::BlacklistedXFail, msg, file, line);
+}
+
void QTestLog::addSkip(const char *msg, const char *file, int line)
{
QTEST_ASSERT(msg);
diff --git a/src/testlib/qtestlog_p.h b/src/testlib/qtestlog_p.h
index 600c078ce20..0bdd6290e19 100644
--- a/src/testlib/qtestlog_p.h
+++ b/src/testlib/qtestlog_p.h
@@ -80,6 +80,8 @@ public:
static void addXPass(const char *msg, const char *file, int line);
static void addBPass(const char *msg);
static void addBFail(const char *msg, const char *file, int line);
+ static void addBXPass(const char *msg, const char *file, int line);
+ static void addBXFail(const char *msg, const char *file, int line);
static void addSkip(const char *msg, const char *file, int line);
static void addBenchmarkResult(const QBenchmarkResult &result);
diff --git a/src/testlib/qtestresult.cpp b/src/testlib/qtestresult.cpp
index 88e3407c903..a7a4807e065 100644
--- a/src/testlib/qtestresult.cpp
+++ b/src/testlib/qtestresult.cpp
@@ -218,17 +218,24 @@ static bool checkStatement(bool statement, const char *msg, const char *file, in
{
if (statement) {
if (QTest::expectFailMode) {
- QTestLog::addXPass(msg, file, line);
+ if (QTest::blacklistCurrentTest)
+ QTestLog::addBXPass(msg, file, line);
+ else
+ QTestLog::addXPass(msg, file, line);
+
+ QTest::failed = true;
bool doContinue = (QTest::expectFailMode == QTest::Continue);
clearExpectFail();
- QTest::failed = true;
return doContinue;
}
return true;
}
if (QTest::expectFailMode) {
- QTestLog::addXFail(QTest::expectFailComment, file, line);
+ if (QTest::blacklistCurrentTest)
+ QTestLog::addBXFail(QTest::expectFailComment, file, line);
+ else
+ QTestLog::addXFail(QTest::expectFailComment, file, line);
bool doContinue = (QTest::expectFailMode == QTest::Continue);
clearExpectFail();
return doContinue;
diff --git a/src/testlib/qxmltestlogger.cpp b/src/testlib/qxmltestlogger.cpp
index 7153c016c5e..c47042c3a00 100644
--- a/src/testlib/qxmltestlogger.cpp
+++ b/src/testlib/qxmltestlogger.cpp
@@ -91,6 +91,10 @@ namespace QTest {
return "bpass";
case QAbstractTestLogger::BlacklistedFail:
return "bfail";
+ case QAbstractTestLogger::BlacklistedXPass:
+ return "bxpass";
+ case QAbstractTestLogger::BlacklistedXFail:
+ return "bxfail";
}
return "??????";
}
diff --git a/src/testlib/qxunittestlogger.cpp b/src/testlib/qxunittestlogger.cpp
index ec33c29ae53..336edb59942 100644
--- a/src/testlib/qxunittestlogger.cpp
+++ b/src/testlib/qxunittestlogger.cpp
@@ -180,6 +180,13 @@ void QXunitTestLogger::addIncident(IncidentTypes type, const char *description,
++failureCounter;
typeBuf = "bfail";
break;
+ case QAbstractTestLogger::BlacklistedXPass:
+ typeBuf = "bxpass";
+ break;
+ case QAbstractTestLogger::BlacklistedXFail:
+ ++failureCounter;
+ typeBuf = "bxfail";
+ break;
default:
typeBuf = "??????";
break;
@@ -212,11 +219,11 @@ void QXunitTestLogger::addIncident(IncidentTypes type, const char *description,
if (!strcmp(oldResult, "pass")) {
overwrite = true;
}
- else if (!strcmp(oldResult, "bpass")) {
+ else if (!strcmp(oldResult, "bpass") || !strcmp(oldResult, "bxfail")) {
overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail) || (type == QAbstractTestLogger::XFail)
- || (type == QAbstractTestLogger::BlacklistedFail);
+ || (type == QAbstractTestLogger::BlacklistedFail) || (type == QAbstractTestLogger::BlacklistedXPass);
}
- else if (!strcmp(oldResult, "bfail")) {
+ else if (!strcmp(oldResult, "bfail") || !strcmp(oldResult, "bxpass")) {
overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail) || (type == QAbstractTestLogger::XFail);
}
else if (!strcmp(oldResult, "xfail")) {
diff --git a/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp b/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
index 8578752e22b..b25489ca00e 100644
--- a/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
+++ b/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
@@ -64,14 +64,14 @@ void tst_Blacklisted::fail()
void tst_Blacklisted::xfail()
{
- QEXPECT_FAIL("", "This test should XFAIL then BFAIL", Abort);
+ QEXPECT_FAIL("", "This test should BXFAIL then BPASS", Abort);
QVERIFY(false);
}
void tst_Blacklisted::xpass()
{
- QEXPECT_FAIL("", "This test should XPASS", Abort);
- QVERIFY2(true, "This test should XPASS, blacklist ignored for XPASS");
+ QEXPECT_FAIL("", "This test should BXPASS", Abort);
+ QVERIFY2(true, "This test should BXPASS");
}
void tst_Blacklisted::messages()
diff --git a/tests/auto/testlib/selftests/expected_blacklisted.lightxml b/tests/auto/testlib/selftests/expected_blacklisted.lightxml
index 4193628e7c8..5cf62ed1044 100644
--- a/tests/auto/testlib/selftests/expected_blacklisted.lightxml
+++ b/tests/auto/testlib/selftests/expected_blacklisted.lightxml
@@ -24,15 +24,15 @@
-
-
+
+
-
-
+
+
diff --git a/tests/auto/testlib/selftests/expected_blacklisted.teamcity b/tests/auto/testlib/selftests/expected_blacklisted.teamcity
index 8180a7ce763..42b75b752b8 100644
--- a/tests/auto/testlib/selftests/expected_blacklisted.teamcity
+++ b/tests/auto/testlib/selftests/expected_blacklisted.teamcity
@@ -7,10 +7,9 @@
##teamcity[testStarted name='fail()' flowId='tst_Blacklisted']
##teamcity[testFinished name='fail()' flowId='tst_Blacklisted']
##teamcity[testStarted name='xfail()' flowId='tst_Blacklisted']
-##teamcity[testStdOut name='xfail()' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]: This test should XFAIL then BFAIL' flowId='tst_Blacklisted']
+##teamcity[testFinished name='xfail()' flowId='tst_Blacklisted']
##teamcity[testFinished name='xfail()' flowId='tst_Blacklisted']
##teamcity[testStarted name='xpass()' flowId='tst_Blacklisted']
-##teamcity[testFailed name='xpass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)' flowId='tst_Blacklisted']
##teamcity[testFinished name='xpass()' flowId='tst_Blacklisted']
##teamcity[testStarted name='messages()' flowId='tst_Blacklisted']
##teamcity[testStdOut name='messages()' out='QWARN: This is a warning that should not appear in silent test output|nWARNING |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]: This is an internal testlib warning that should not appear in silent test output|nQDEBUG: This is a debug message that should not appear in silent test output|nQSYSTEM: This is a critical message that should not appear in silent test output|nQINFO: This is an info message that should not appear in silent test output|nINFO |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]: This is an internal testlib info message that should not appear in silent test output|nQFATAL: This is a fatal error message that should still appear in silent test output' flowId='tst_Blacklisted']
diff --git a/tests/auto/testlib/selftests/expected_blacklisted.txt b/tests/auto/testlib/selftests/expected_blacklisted.txt
index 6fa2403b597..fccaa7d8c34 100644
--- a/tests/auto/testlib/selftests/expected_blacklisted.txt
+++ b/tests/auto/testlib/selftests/expected_blacklisted.txt
@@ -6,10 +6,10 @@ SKIP : tst_Blacklisted::skip() This test should SKIP
Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
BFAIL : tst_Blacklisted::fail() 'false' returned FALSE. (This test should BFAIL)
Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
-XFAIL : tst_Blacklisted::xfail() This test should XFAIL then BFAIL
+BXFAIL : tst_Blacklisted::xfail() This test should BXFAIL then BPASS
Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
BPASS : tst_Blacklisted::xfail()
-XPASS : tst_Blacklisted::xpass() 'true' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)
+BXPASS : tst_Blacklisted::xpass() 'true' returned TRUE unexpectedly. (This test should BXPASS)
Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
QWARN : tst_Blacklisted::messages() This is a warning that should not appear in silent test output
WARNING: tst_Blacklisted::messages() This is an internal testlib warning that should not appear in silent test output
@@ -22,5 +22,5 @@ INFO : tst_Blacklisted::messages() This is an internal testlib info message th
QFATAL : tst_Blacklisted::messages() This is a fatal error message that should still appear in silent test output
BFAIL : tst_Blacklisted::messages() Received a fatal error.
Loc: [Unknown file(0)]
-Totals: 1 passed, 1 failed, 1 skipped, 4 blacklisted, 0ms
+Totals: 1 passed, 0 failed, 1 skipped, 5 blacklisted, 0ms
********* Finished testing of tst_Blacklisted *********
diff --git a/tests/auto/testlib/selftests/expected_blacklisted.xml b/tests/auto/testlib/selftests/expected_blacklisted.xml
index 443bc6b199f..04d7e6c8285 100644
--- a/tests/auto/testlib/selftests/expected_blacklisted.xml
+++ b/tests/auto/testlib/selftests/expected_blacklisted.xml
@@ -26,15 +26,15 @@
-
-
+
+
-
-
+
+
diff --git a/tests/auto/testlib/selftests/expected_blacklisted.xunitxml b/tests/auto/testlib/selftests/expected_blacklisted.xunitxml
index 2752bc18b40..6e192687fb8 100644
--- a/tests/auto/testlib/selftests/expected_blacklisted.xunitxml
+++ b/tests/auto/testlib/selftests/expected_blacklisted.xunitxml
@@ -1,5 +1,5 @@
-
+
@@ -11,12 +11,8 @@
-
-
-
-
-
-
+
+
@@ -28,7 +24,6 @@
-