diff --git a/src/testlib/doc/src/qttestlib-manual.qdoc b/src/testlib/doc/src/qttestlib-manual.qdoc
index 5fe2b116517..5c7b298fb9e 100644
--- a/src/testlib/doc/src/qttestlib-manual.qdoc
+++ b/src/testlib/doc/src/qttestlib-manual.qdoc
@@ -352,6 +352,12 @@
Run the testsuite n times or until the test fails. Useful for finding
flaky tests. If negative, the tests are repeated forever. This is intended
as a developer tool, and is only supported with the plain text logger.
+ \li \c -skipblacklisted \br
+ Skip the blacklisted tests. This option is intended to allow more accurate
+ measurement of test coverage by preventing blacklisted tests from inflating
+ coverage statistics. When not measuring test coverage, it is recommended to
+ execute blacklisted tests to reveal any changes in their results, such as
+ a new crash or the issue that caused blacklisting being resolved.
\li \c -platform \e name \br
This command line argument applies to all Qt applications, but might be
diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp
index d3153123633..fd9e6197170 100644
--- a/src/testlib/qtestcase.cpp
+++ b/src/testlib/qtestcase.cpp
@@ -541,6 +541,7 @@ static int timeout = -1;
static bool noCrashHandler = false;
static int repetitions = 1;
static bool repeatForever = false;
+static bool skipBlacklisted = false;
/*! \internal
Invoke a method of the object without generating warning if the method does not exist
@@ -773,6 +774,7 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool
" Useful for finding flaky tests. If negative, the tests are\n"
" repeated forever. This is intended as a developer tool, and\n"
" is only supported with the plain text logger.\n"
+ " -skipblacklisted : Skip blacklisted tests. Useful for measuring test coverage.\n"
"\n"
" Benchmarking options:\n"
#if QT_CONFIG(valgrind)
@@ -932,6 +934,8 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool
}
} else if (strcmp(argv[i], "-nocrashhandler") == 0) {
QTest::noCrashHandler = true;
+ } else if (strcmp(argv[i], "-skipblacklisted") == 0) {
+ QTest::skipBlacklisted = true;
#if QT_CONFIG(valgrind)
} else if (strcmp(argv[i], "-callgrind") == 0) {
if (!QBenchmarkValgrindUtils::haveValgrind()) {
@@ -1417,6 +1421,7 @@ bool TestMethods::invokeTest(int index, QLatin1StringView tag, WatchDog *watchDo
tag[global.size()] == ':';
};
bool foundFunction = false;
+ bool blacklisted = false;
/* For each entry in the global data table, do: */
do {
@@ -1443,20 +1448,28 @@ bool TestMethods::invokeTest(int index, QLatin1StringView tag, WatchDog *watchDo
if (dataTagMatches(tag, QLatin1StringView(dataTag(curDataIndex)),
QLatin1StringView(globalDataTag(curGlobalDataIndex)))) {
foundFunction = true;
- if (QTestPrivate::checkBlackLists(name.constData(), dataTag(curDataIndex),
- globalDataTag(curGlobalDataIndex))) {
+ blacklisted = QTestPrivate::checkBlackLists(name.constData(), dataTag(curDataIndex),
+ globalDataTag(curGlobalDataIndex));
+ if (blacklisted)
QTestResult::setBlacklistCurrentTest(true);
- }
- QTestDataSetter s(curDataIndex >= dataCount ? nullptr : table.testData(curDataIndex));
+ if (blacklisted && skipBlacklisted) {
+ QTest::qSkip("Skipping blacklisted test since -skipblacklisted option is set.",
+ NULL, 0);
+ QTestResult::finishedCurrentTestData();
+ QTestResult::finishedCurrentTestDataCleanup();
+ } else {
+ QTestDataSetter s(
+ curDataIndex >= dataCount ? nullptr : table.testData(curDataIndex));
- QTestPrivate::qtestMouseButtons = Qt::NoButton;
- if (watchDog)
- watchDog->beginTest();
- QTest::lastMouseTimestamp += 500; // Maintain at least 500ms mouse event timestamps between each test function call
- invokeTestOnData(index);
- if (watchDog)
- watchDog->testFinished();
+ QTestPrivate::qtestMouseButtons = Qt::NoButton;
+ if (watchDog)
+ watchDog->beginTest();
+ QTest::lastMouseTimestamp += 500; // Maintain at least 500ms mouse event timestamps between each test function call
+ invokeTestOnData(index);
+ if (watchDog)
+ watchDog->testFinished();
+ }
if (!tag.isEmpty() && !globalDataCount)
break;
diff --git a/tests/auto/testlib/selftests/CMakeLists.txt b/tests/auto/testlib/selftests/CMakeLists.txt
index 643647e6d66..9e17638a660 100644
--- a/tests/auto/testlib/selftests/CMakeLists.txt
+++ b/tests/auto/testlib/selftests/CMakeLists.txt
@@ -95,6 +95,7 @@ set(subprograms
signaldumper
singleskip
skip
+ skipblacklisted
skipcleanup
skipcleanuptestcase
skipinit
diff --git a/tests/auto/testlib/selftests/expected_skipblacklisted.junitxml b/tests/auto/testlib/selftests/expected_skipblacklisted.junitxml
new file mode 100644
index 00000000000..308a4ea9997
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_skipblacklisted.junitxml
@@ -0,0 +1,18 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/tests/auto/testlib/selftests/expected_skipblacklisted.lightxml b/tests/auto/testlib/selftests/expected_skipblacklisted.lightxml
new file mode 100644
index 00000000000..45d3476d6a4
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_skipblacklisted.lightxml
@@ -0,0 +1,36 @@
+
+ @INSERT_QT_VERSION_HERE@
+
+ @INSERT_QT_VERSION_HERE@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/tests/auto/testlib/selftests/expected_skipblacklisted.tap b/tests/auto/testlib/selftests/expected_skipblacklisted.tap
new file mode 100644
index 00000000000..f71524d85fc
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_skipblacklisted.tap
@@ -0,0 +1,13 @@
+TAP version 13
+# tst_SkipBlacklisted
+ok 1 - initTestCase()
+ok 2 - pass()
+ok 3 - blacklisted() # SKIP Skipping blacklisted test since -skipblacklisted option is set.
+ok 4 - blacklistedData() # SKIP Skipping blacklisted test since -skipblacklisted option is set.
+ok 5 - blacklistedData(should pass)
+ok 6 - blacklistedData() # SKIP Skipping blacklisted test since -skipblacklisted option is set.
+ok 7 - cleanupTestCase()
+1..7
+# tests 7
+# pass 4
+# fail 0
diff --git a/tests/auto/testlib/selftests/expected_skipblacklisted.teamcity b/tests/auto/testlib/selftests/expected_skipblacklisted.teamcity
new file mode 100644
index 00000000000..67a788901a6
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_skipblacklisted.teamcity
@@ -0,0 +1,19 @@
+##teamcity[testSuiteStarted name='tst_SkipBlacklisted' flowId='tst_SkipBlacklisted']
+##teamcity[testStarted name='initTestCase()' flowId='tst_SkipBlacklisted']
+##teamcity[testFinished name='initTestCase()' flowId='tst_SkipBlacklisted']
+##teamcity[testStarted name='pass()' flowId='tst_SkipBlacklisted']
+##teamcity[testFinished name='pass()' flowId='tst_SkipBlacklisted']
+##teamcity[testStarted name='blacklisted()' flowId='tst_SkipBlacklisted']
+##teamcity[testIgnored name='blacklisted()' message='Skipping blacklisted test since -skipblacklisted option is set.' flowId='tst_SkipBlacklisted']
+##teamcity[testFinished name='blacklisted()' flowId='tst_SkipBlacklisted']
+##teamcity[testStarted name='blacklistedData()' flowId='tst_SkipBlacklisted']
+##teamcity[testIgnored name='blacklistedData()' message='Skipping blacklisted test since -skipblacklisted option is set.' flowId='tst_SkipBlacklisted']
+##teamcity[testFinished name='blacklistedData()' flowId='tst_SkipBlacklisted']
+##teamcity[testStarted name='blacklistedData(should pass)' flowId='tst_SkipBlacklisted']
+##teamcity[testFinished name='blacklistedData(should pass)' flowId='tst_SkipBlacklisted']
+##teamcity[testStarted name='blacklistedData()' flowId='tst_SkipBlacklisted']
+##teamcity[testIgnored name='blacklistedData()' message='Skipping blacklisted test since -skipblacklisted option is set.' flowId='tst_SkipBlacklisted']
+##teamcity[testFinished name='blacklistedData()' flowId='tst_SkipBlacklisted']
+##teamcity[testStarted name='cleanupTestCase()' flowId='tst_SkipBlacklisted']
+##teamcity[testFinished name='cleanupTestCase()' flowId='tst_SkipBlacklisted']
+##teamcity[testSuiteFinished name='tst_SkipBlacklisted' flowId='tst_SkipBlacklisted']
diff --git a/tests/auto/testlib/selftests/expected_skipblacklisted.txt b/tests/auto/testlib/selftests/expected_skipblacklisted.txt
new file mode 100644
index 00000000000..0e3dfa04e81
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_skipblacklisted.txt
@@ -0,0 +1,11 @@
+********* Start testing of tst_SkipBlacklisted *********
+Config: Using QtTest library
+PASS : tst_SkipBlacklisted::initTestCase()
+PASS : tst_SkipBlacklisted::pass()
+SKIP : tst_SkipBlacklisted::blacklisted() Skipping blacklisted test since -skipblacklisted option is set.
+SKIP : tst_SkipBlacklisted::blacklistedData() Skipping blacklisted test since -skipblacklisted option is set.
+PASS : tst_SkipBlacklisted::blacklistedData(should pass)
+SKIP : tst_SkipBlacklisted::blacklistedData() Skipping blacklisted test since -skipblacklisted option is set.
+PASS : tst_SkipBlacklisted::cleanupTestCase()
+Totals: 4 passed, 0 failed, 3 skipped, 0 blacklisted, 0ms
+********* Finished testing of tst_SkipBlacklisted *********
diff --git a/tests/auto/testlib/selftests/expected_skipblacklisted.xml b/tests/auto/testlib/selftests/expected_skipblacklisted.xml
new file mode 100644
index 00000000000..100fededb3c
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_skipblacklisted.xml
@@ -0,0 +1,39 @@
+
+
+
+ @INSERT_QT_VERSION_HERE@
+
+ @INSERT_QT_VERSION_HERE@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/tests/auto/testlib/selftests/generate_expected_output.py b/tests/auto/testlib/selftests/generate_expected_output.py
index c39c00772a7..4f4f8b14f33 100755
--- a/tests/auto/testlib/selftests/generate_expected_output.py
+++ b/tests/auto/testlib/selftests/generate_expected_output.py
@@ -37,10 +37,11 @@ TESTS = ['assert', 'badxml', 'benchlibcallgrind', 'benchlibcounting',
'fetchbogus', 'findtestdata', 'float', 'globaldata', 'longstring',
'maxwarnings', 'mouse', 'multiexec', 'pairdiagnostics', 'pass',
'printdatatags', 'printdatatagswithglobaltags', 'qexecstringlist',
- 'signaldumper', 'silent', 'silent_fatal', 'singleskip', 'skip', 'skipcleanup',
- 'skipcleanuptestcase', 'skipinit', 'skipinitdata', 'sleep', 'strcmp',
- 'subtest', 'testlib', 'tuplediagnostics', 'verbose1', 'verbose2',
- 'verifyexceptionthrown', 'warnings', 'watchdog', 'junit', 'keyboard']
+ 'signaldumper', 'silent', 'silent_fatal', 'singleskip', 'skip',
+ 'skipblacklisted', 'skipcleanup', 'skipcleanuptestcase', 'skipinit',
+ 'skipinitdata', 'sleep', 'strcmp', 'subtest', 'testlib', 'tuplediagnostics',
+ 'verbose1', 'verbose2', 'verifyexceptionthrown', 'warnings', 'watchdog',
+ 'junit', 'keyboard']
class Fail (Exception): pass
diff --git a/tests/auto/testlib/selftests/skipblacklisted/BLACKLIST b/tests/auto/testlib/selftests/skipblacklisted/BLACKLIST
new file mode 100644
index 00000000000..77c13250cd5
--- /dev/null
+++ b/tests/auto/testlib/selftests/skipblacklisted/BLACKLIST
@@ -0,0 +1,6 @@
+[blacklisted]
+*
+[blacklistedData:blacklisted 1]
+*
+[blacklistedData:blacklisted 2]
+*
diff --git a/tests/auto/testlib/selftests/skipblacklisted/CMakeLists.txt b/tests/auto/testlib/selftests/skipblacklisted/CMakeLists.txt
new file mode 100644
index 00000000000..2186279dc0e
--- /dev/null
+++ b/tests/auto/testlib/selftests/skipblacklisted/CMakeLists.txt
@@ -0,0 +1,17 @@
+#####################################################################
+## skipblacklisted Binary:
+#####################################################################
+
+qt_internal_add_executable(skipblacklisted
+ NO_INSTALL
+ OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+ SOURCES
+ tst_skipblacklisted.cpp
+ LIBRARIES
+ Qt::Test
+)
+
+## Scopes:
+#####################################################################
+
+qt_internal_apply_testlib_coverage_options(skipblacklisted)
diff --git a/tests/auto/testlib/selftests/skipblacklisted/tst_skipblacklisted.cpp b/tests/auto/testlib/selftests/skipblacklisted/tst_skipblacklisted.cpp
new file mode 100644
index 00000000000..321b7f1ffcc
--- /dev/null
+++ b/tests/auto/testlib/selftests/skipblacklisted/tst_skipblacklisted.cpp
@@ -0,0 +1,51 @@
+// Copyright (C) 2023 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
+
+#include
+
+class tst_SkipBlacklisted : public QObject
+{
+ Q_OBJECT
+
+private slots:
+ void pass();
+ void blacklisted();
+ void blacklistedData();
+ void blacklistedData_data();
+};
+
+void tst_SkipBlacklisted::pass()
+{
+ QVERIFY(true);
+}
+
+// This test have been blacklisted in skipblacklisted/BLACKLIST
+void tst_SkipBlacklisted::blacklisted()
+{
+ QFAIL("this line should never be reached, since we skip all blacklisted test functions");
+}
+
+// blacklisted 1 and blacklisted 2 have been blacklisted in skipblacklisted/BLACKLIST
+void tst_SkipBlacklisted::blacklistedData()
+{
+ QFETCH(int, testdata);
+ QCOMPARE(testdata, 2);
+}
+
+void tst_SkipBlacklisted::blacklistedData_data()
+{
+ QTest::addColumn("testdata");
+
+ QTest::newRow("blacklisted 1") << 1;
+ QTest::newRow("should pass") << 2;
+ QTest::newRow("blacklisted 2") << 3;
+}
+
+QTEST_MAIN_WRAPPER(tst_SkipBlacklisted,
+ std::vector args(argv, argv + argc);
+ args.push_back("-skipblacklisted");
+ argc = int(args.size());
+ argv = const_cast(&args[0]);
+ QTEST_MAIN_SETUP())
+
+#include "tst_skipblacklisted.moc"