From a397247e2b57e2a553dd868aca408b40c253c76e Mon Sep 17 00:00:00 2001 From: Kalle Viironen Date: Tue, 21 Nov 2023 09:45:46 +0200 Subject: [PATCH] qtestlib: Add option to skip blacklisted tests Add a command-line option "-skipblacklisted" to testlib to skip blacklisted test cases. Currently, blacklisted test cases are run, but results are ignored. For test code coverage measurements, it's important to see the code coverage of the actually tested code in comparison to the code that was run but not actually tested. The default approach remains unchanged, meaning that blacklisted tests are run with the results ignored. Fixes: QTBUG-112793 Change-Id: I6fe0a6353cb1c021e0232c79bb4f404632fb0bce Reviewed-by: Jason McDonald --- src/testlib/doc/src/qttestlib-manual.qdoc | 6 +++ src/testlib/qtestcase.cpp | 35 +++++++++---- tests/auto/testlib/selftests/CMakeLists.txt | 1 + .../expected_skipblacklisted.junitxml | 18 +++++++ .../expected_skipblacklisted.lightxml | 36 +++++++++++++ .../selftests/expected_skipblacklisted.tap | 13 +++++ .../expected_skipblacklisted.teamcity | 19 +++++++ .../selftests/expected_skipblacklisted.txt | 11 ++++ .../selftests/expected_skipblacklisted.xml | 39 ++++++++++++++ .../selftests/generate_expected_output.py | 9 ++-- .../selftests/skipblacklisted/BLACKLIST | 6 +++ .../selftests/skipblacklisted/CMakeLists.txt | 17 +++++++ .../skipblacklisted/tst_skipblacklisted.cpp | 51 +++++++++++++++++++ 13 files changed, 246 insertions(+), 15 deletions(-) create mode 100644 tests/auto/testlib/selftests/expected_skipblacklisted.junitxml create mode 100644 tests/auto/testlib/selftests/expected_skipblacklisted.lightxml create mode 100644 tests/auto/testlib/selftests/expected_skipblacklisted.tap create mode 100644 tests/auto/testlib/selftests/expected_skipblacklisted.teamcity create mode 100644 tests/auto/testlib/selftests/expected_skipblacklisted.txt create mode 100644 tests/auto/testlib/selftests/expected_skipblacklisted.xml create mode 100644 tests/auto/testlib/selftests/skipblacklisted/BLACKLIST create mode 100644 tests/auto/testlib/selftests/skipblacklisted/CMakeLists.txt create mode 100644 tests/auto/testlib/selftests/skipblacklisted/tst_skipblacklisted.cpp diff --git a/src/testlib/doc/src/qttestlib-manual.qdoc b/src/testlib/doc/src/qttestlib-manual.qdoc index 5fe2b116517..5c7b298fb9e 100644 --- a/src/testlib/doc/src/qttestlib-manual.qdoc +++ b/src/testlib/doc/src/qttestlib-manual.qdoc @@ -352,6 +352,12 @@ Run the testsuite n times or until the test fails. Useful for finding flaky tests. If negative, the tests are repeated forever. This is intended as a developer tool, and is only supported with the plain text logger. + \li \c -skipblacklisted \br + Skip the blacklisted tests. This option is intended to allow more accurate + measurement of test coverage by preventing blacklisted tests from inflating + coverage statistics. When not measuring test coverage, it is recommended to + execute blacklisted tests to reveal any changes in their results, such as + a new crash or the issue that caused blacklisting being resolved. \li \c -platform \e name \br This command line argument applies to all Qt applications, but might be diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp index d3153123633..fd9e6197170 100644 --- a/src/testlib/qtestcase.cpp +++ b/src/testlib/qtestcase.cpp @@ -541,6 +541,7 @@ static int timeout = -1; static bool noCrashHandler = false; static int repetitions = 1; static bool repeatForever = false; +static bool skipBlacklisted = false; /*! \internal Invoke a method of the object without generating warning if the method does not exist @@ -773,6 +774,7 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool " Useful for finding flaky tests. If negative, the tests are\n" " repeated forever. This is intended as a developer tool, and\n" " is only supported with the plain text logger.\n" + " -skipblacklisted : Skip blacklisted tests. Useful for measuring test coverage.\n" "\n" " Benchmarking options:\n" #if QT_CONFIG(valgrind) @@ -932,6 +934,8 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool } } else if (strcmp(argv[i], "-nocrashhandler") == 0) { QTest::noCrashHandler = true; + } else if (strcmp(argv[i], "-skipblacklisted") == 0) { + QTest::skipBlacklisted = true; #if QT_CONFIG(valgrind) } else if (strcmp(argv[i], "-callgrind") == 0) { if (!QBenchmarkValgrindUtils::haveValgrind()) { @@ -1417,6 +1421,7 @@ bool TestMethods::invokeTest(int index, QLatin1StringView tag, WatchDog *watchDo tag[global.size()] == ':'; }; bool foundFunction = false; + bool blacklisted = false; /* For each entry in the global data table, do: */ do { @@ -1443,20 +1448,28 @@ bool TestMethods::invokeTest(int index, QLatin1StringView tag, WatchDog *watchDo if (dataTagMatches(tag, QLatin1StringView(dataTag(curDataIndex)), QLatin1StringView(globalDataTag(curGlobalDataIndex)))) { foundFunction = true; - if (QTestPrivate::checkBlackLists(name.constData(), dataTag(curDataIndex), - globalDataTag(curGlobalDataIndex))) { + blacklisted = QTestPrivate::checkBlackLists(name.constData(), dataTag(curDataIndex), + globalDataTag(curGlobalDataIndex)); + if (blacklisted) QTestResult::setBlacklistCurrentTest(true); - } - QTestDataSetter s(curDataIndex >= dataCount ? nullptr : table.testData(curDataIndex)); + if (blacklisted && skipBlacklisted) { + QTest::qSkip("Skipping blacklisted test since -skipblacklisted option is set.", + NULL, 0); + QTestResult::finishedCurrentTestData(); + QTestResult::finishedCurrentTestDataCleanup(); + } else { + QTestDataSetter s( + curDataIndex >= dataCount ? nullptr : table.testData(curDataIndex)); - QTestPrivate::qtestMouseButtons = Qt::NoButton; - if (watchDog) - watchDog->beginTest(); - QTest::lastMouseTimestamp += 500; // Maintain at least 500ms mouse event timestamps between each test function call - invokeTestOnData(index); - if (watchDog) - watchDog->testFinished(); + QTestPrivate::qtestMouseButtons = Qt::NoButton; + if (watchDog) + watchDog->beginTest(); + QTest::lastMouseTimestamp += 500; // Maintain at least 500ms mouse event timestamps between each test function call + invokeTestOnData(index); + if (watchDog) + watchDog->testFinished(); + } if (!tag.isEmpty() && !globalDataCount) break; diff --git a/tests/auto/testlib/selftests/CMakeLists.txt b/tests/auto/testlib/selftests/CMakeLists.txt index 643647e6d66..9e17638a660 100644 --- a/tests/auto/testlib/selftests/CMakeLists.txt +++ b/tests/auto/testlib/selftests/CMakeLists.txt @@ -95,6 +95,7 @@ set(subprograms signaldumper singleskip skip + skipblacklisted skipcleanup skipcleanuptestcase skipinit diff --git a/tests/auto/testlib/selftests/expected_skipblacklisted.junitxml b/tests/auto/testlib/selftests/expected_skipblacklisted.junitxml new file mode 100644 index 00000000000..308a4ea9997 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_skipblacklisted.junitxml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_skipblacklisted.lightxml b/tests/auto/testlib/selftests/expected_skipblacklisted.lightxml new file mode 100644 index 00000000000..45d3476d6a4 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_skipblacklisted.lightxml @@ -0,0 +1,36 @@ + + @INSERT_QT_VERSION_HERE@ + + @INSERT_QT_VERSION_HERE@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_skipblacklisted.tap b/tests/auto/testlib/selftests/expected_skipblacklisted.tap new file mode 100644 index 00000000000..f71524d85fc --- /dev/null +++ b/tests/auto/testlib/selftests/expected_skipblacklisted.tap @@ -0,0 +1,13 @@ +TAP version 13 +# tst_SkipBlacklisted +ok 1 - initTestCase() +ok 2 - pass() +ok 3 - blacklisted() # SKIP Skipping blacklisted test since -skipblacklisted option is set. +ok 4 - blacklistedData() # SKIP Skipping blacklisted test since -skipblacklisted option is set. +ok 5 - blacklistedData(should pass) +ok 6 - blacklistedData() # SKIP Skipping blacklisted test since -skipblacklisted option is set. +ok 7 - cleanupTestCase() +1..7 +# tests 7 +# pass 4 +# fail 0 diff --git a/tests/auto/testlib/selftests/expected_skipblacklisted.teamcity b/tests/auto/testlib/selftests/expected_skipblacklisted.teamcity new file mode 100644 index 00000000000..67a788901a6 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_skipblacklisted.teamcity @@ -0,0 +1,19 @@ +##teamcity[testSuiteStarted name='tst_SkipBlacklisted' flowId='tst_SkipBlacklisted'] +##teamcity[testStarted name='initTestCase()' flowId='tst_SkipBlacklisted'] +##teamcity[testFinished name='initTestCase()' flowId='tst_SkipBlacklisted'] +##teamcity[testStarted name='pass()' flowId='tst_SkipBlacklisted'] +##teamcity[testFinished name='pass()' flowId='tst_SkipBlacklisted'] +##teamcity[testStarted name='blacklisted()' flowId='tst_SkipBlacklisted'] +##teamcity[testIgnored name='blacklisted()' message='Skipping blacklisted test since -skipblacklisted option is set.' flowId='tst_SkipBlacklisted'] +##teamcity[testFinished name='blacklisted()' flowId='tst_SkipBlacklisted'] +##teamcity[testStarted name='blacklistedData()' flowId='tst_SkipBlacklisted'] +##teamcity[testIgnored name='blacklistedData()' message='Skipping blacklisted test since -skipblacklisted option is set.' flowId='tst_SkipBlacklisted'] +##teamcity[testFinished name='blacklistedData()' flowId='tst_SkipBlacklisted'] +##teamcity[testStarted name='blacklistedData(should pass)' flowId='tst_SkipBlacklisted'] +##teamcity[testFinished name='blacklistedData(should pass)' flowId='tst_SkipBlacklisted'] +##teamcity[testStarted name='blacklistedData()' flowId='tst_SkipBlacklisted'] +##teamcity[testIgnored name='blacklistedData()' message='Skipping blacklisted test since -skipblacklisted option is set.' flowId='tst_SkipBlacklisted'] +##teamcity[testFinished name='blacklistedData()' flowId='tst_SkipBlacklisted'] +##teamcity[testStarted name='cleanupTestCase()' flowId='tst_SkipBlacklisted'] +##teamcity[testFinished name='cleanupTestCase()' flowId='tst_SkipBlacklisted'] +##teamcity[testSuiteFinished name='tst_SkipBlacklisted' flowId='tst_SkipBlacklisted'] diff --git a/tests/auto/testlib/selftests/expected_skipblacklisted.txt b/tests/auto/testlib/selftests/expected_skipblacklisted.txt new file mode 100644 index 00000000000..0e3dfa04e81 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_skipblacklisted.txt @@ -0,0 +1,11 @@ +********* Start testing of tst_SkipBlacklisted ********* +Config: Using QtTest library +PASS : tst_SkipBlacklisted::initTestCase() +PASS : tst_SkipBlacklisted::pass() +SKIP : tst_SkipBlacklisted::blacklisted() Skipping blacklisted test since -skipblacklisted option is set. +SKIP : tst_SkipBlacklisted::blacklistedData() Skipping blacklisted test since -skipblacklisted option is set. +PASS : tst_SkipBlacklisted::blacklistedData(should pass) +SKIP : tst_SkipBlacklisted::blacklistedData() Skipping blacklisted test since -skipblacklisted option is set. +PASS : tst_SkipBlacklisted::cleanupTestCase() +Totals: 4 passed, 0 failed, 3 skipped, 0 blacklisted, 0ms +********* Finished testing of tst_SkipBlacklisted ********* diff --git a/tests/auto/testlib/selftests/expected_skipblacklisted.xml b/tests/auto/testlib/selftests/expected_skipblacklisted.xml new file mode 100644 index 00000000000..100fededb3c --- /dev/null +++ b/tests/auto/testlib/selftests/expected_skipblacklisted.xml @@ -0,0 +1,39 @@ + + + + @INSERT_QT_VERSION_HERE@ + + @INSERT_QT_VERSION_HERE@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/generate_expected_output.py b/tests/auto/testlib/selftests/generate_expected_output.py index c39c00772a7..4f4f8b14f33 100755 --- a/tests/auto/testlib/selftests/generate_expected_output.py +++ b/tests/auto/testlib/selftests/generate_expected_output.py @@ -37,10 +37,11 @@ TESTS = ['assert', 'badxml', 'benchlibcallgrind', 'benchlibcounting', 'fetchbogus', 'findtestdata', 'float', 'globaldata', 'longstring', 'maxwarnings', 'mouse', 'multiexec', 'pairdiagnostics', 'pass', 'printdatatags', 'printdatatagswithglobaltags', 'qexecstringlist', - 'signaldumper', 'silent', 'silent_fatal', 'singleskip', 'skip', 'skipcleanup', - 'skipcleanuptestcase', 'skipinit', 'skipinitdata', 'sleep', 'strcmp', - 'subtest', 'testlib', 'tuplediagnostics', 'verbose1', 'verbose2', - 'verifyexceptionthrown', 'warnings', 'watchdog', 'junit', 'keyboard'] + 'signaldumper', 'silent', 'silent_fatal', 'singleskip', 'skip', + 'skipblacklisted', 'skipcleanup', 'skipcleanuptestcase', 'skipinit', + 'skipinitdata', 'sleep', 'strcmp', 'subtest', 'testlib', 'tuplediagnostics', + 'verbose1', 'verbose2', 'verifyexceptionthrown', 'warnings', 'watchdog', + 'junit', 'keyboard'] class Fail (Exception): pass diff --git a/tests/auto/testlib/selftests/skipblacklisted/BLACKLIST b/tests/auto/testlib/selftests/skipblacklisted/BLACKLIST new file mode 100644 index 00000000000..77c13250cd5 --- /dev/null +++ b/tests/auto/testlib/selftests/skipblacklisted/BLACKLIST @@ -0,0 +1,6 @@ +[blacklisted] +* +[blacklistedData:blacklisted 1] +* +[blacklistedData:blacklisted 2] +* diff --git a/tests/auto/testlib/selftests/skipblacklisted/CMakeLists.txt b/tests/auto/testlib/selftests/skipblacklisted/CMakeLists.txt new file mode 100644 index 00000000000..2186279dc0e --- /dev/null +++ b/tests/auto/testlib/selftests/skipblacklisted/CMakeLists.txt @@ -0,0 +1,17 @@ +##################################################################### +## skipblacklisted Binary: +##################################################################### + +qt_internal_add_executable(skipblacklisted + NO_INSTALL + OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + SOURCES + tst_skipblacklisted.cpp + LIBRARIES + Qt::Test +) + +## Scopes: +##################################################################### + +qt_internal_apply_testlib_coverage_options(skipblacklisted) diff --git a/tests/auto/testlib/selftests/skipblacklisted/tst_skipblacklisted.cpp b/tests/auto/testlib/selftests/skipblacklisted/tst_skipblacklisted.cpp new file mode 100644 index 00000000000..321b7f1ffcc --- /dev/null +++ b/tests/auto/testlib/selftests/skipblacklisted/tst_skipblacklisted.cpp @@ -0,0 +1,51 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 + +#include + +class tst_SkipBlacklisted : public QObject +{ + Q_OBJECT + +private slots: + void pass(); + void blacklisted(); + void blacklistedData(); + void blacklistedData_data(); +}; + +void tst_SkipBlacklisted::pass() +{ + QVERIFY(true); +} + +// This test have been blacklisted in skipblacklisted/BLACKLIST +void tst_SkipBlacklisted::blacklisted() +{ + QFAIL("this line should never be reached, since we skip all blacklisted test functions"); +} + +// blacklisted 1 and blacklisted 2 have been blacklisted in skipblacklisted/BLACKLIST +void tst_SkipBlacklisted::blacklistedData() +{ + QFETCH(int, testdata); + QCOMPARE(testdata, 2); +} + +void tst_SkipBlacklisted::blacklistedData_data() +{ + QTest::addColumn("testdata"); + + QTest::newRow("blacklisted 1") << 1; + QTest::newRow("should pass") << 2; + QTest::newRow("blacklisted 2") << 3; +} + +QTEST_MAIN_WRAPPER(tst_SkipBlacklisted, + std::vector args(argv, argv + argc); + args.push_back("-skipblacklisted"); + argc = int(args.size()); + argv = const_cast(&args[0]); + QTEST_MAIN_SETUP()) + +#include "tst_skipblacklisted.moc"