From edba9cddbb7f2e68436ee256d4471909e33d288f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tor=20Arne=20Vestb=C3=B8?= Date: Wed, 28 Jul 2021 00:03:00 +0200 Subject: [PATCH] testlib: Accurately name JUnit test, and only run for JUnitXML reporter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The JUnit reporter was initially named xunit, but the naming was inaccurate and the reporter was renamed in 27db9e458cef512fca3a6b5c9ebbcda7a8172428. The corresponding test has now been renamed as well, and as an added bonus we only run it for that reporter. Pick-to: 6.2 Change-Id: I59cb7d949514cdf46a0199a53a7a3e39f833207c Reviewed-by: Mårten Nordheim Reviewed-by: Friedemann Kleint --- .gitignore | 2 +- tests/auto/testlib/selftests/CMakeLists.txt | 2 +- ...xunit.junitxml => expected_junit.junitxml} | 2 +- .../testlib/selftests/expected_xunit.lightxml | 64 ------------------ .../auto/testlib/selftests/expected_xunit.tap | 44 ------------ .../testlib/selftests/expected_xunit.teamcity | 26 ------- .../auto/testlib/selftests/expected_xunit.txt | 26 ------- .../auto/testlib/selftests/expected_xunit.xml | 67 ------------------- .../selftests/generate_expected_output.py | 4 +- .../testlib/selftests/junit/CMakeLists.txt | 11 +++ .../tst_xunit.cpp => junit/tst_junit.cpp} | 24 +++---- .../auto/testlib/selftests/tst_selftests.cpp | 5 +- .../testlib/selftests/xunit/CMakeLists.txt | 21 ------ 13 files changed, 33 insertions(+), 265 deletions(-) rename tests/auto/testlib/selftests/{expected_xunit.junitxml => expected_junit.junitxml} (96%) delete mode 100644 tests/auto/testlib/selftests/expected_xunit.lightxml delete mode 100644 tests/auto/testlib/selftests/expected_xunit.tap delete mode 100644 tests/auto/testlib/selftests/expected_xunit.teamcity delete mode 100644 tests/auto/testlib/selftests/expected_xunit.txt delete mode 100644 tests/auto/testlib/selftests/expected_xunit.xml create mode 100644 tests/auto/testlib/selftests/junit/CMakeLists.txt rename tests/auto/testlib/selftests/{xunit/tst_xunit.cpp => junit/tst_junit.cpp} (87%) delete mode 100644 tests/auto/testlib/selftests/xunit/CMakeLists.txt diff --git a/.gitignore b/.gitignore index b8092980289..6126a58ddf9 100644 --- a/.gitignore +++ b/.gitignore @@ -191,7 +191,7 @@ __pycache__ /tests/auto/testlib/selftests/verifyexceptionthrown/verifyexceptionthrown /tests/auto/testlib/selftests/waitwithoutgui/waitwithoutgui /tests/auto/testlib/selftests/warnings/warnings -/tests/auto/testlib/selftests/xunit/xunit +/tests/auto/testlib/selftests/junit/junit /tests/auto/widgets/kernel/qapplication/modal/modal QObject.log diff --git a/tests/auto/testlib/selftests/CMakeLists.txt b/tests/auto/testlib/selftests/CMakeLists.txt index 3a2def6de27..0564fdaaf71 100644 --- a/tests/auto/testlib/selftests/CMakeLists.txt +++ b/tests/auto/testlib/selftests/CMakeLists.txt @@ -91,6 +91,7 @@ set(subprograms findtestdata float globaldata + junit longstring maxwarnings multiexec @@ -116,7 +117,6 @@ set(subprograms verifyexceptionthrown warnings watchdog - xunit ) if(TARGET Qt::Gui) diff --git a/tests/auto/testlib/selftests/expected_xunit.junitxml b/tests/auto/testlib/selftests/expected_junit.junitxml similarity index 96% rename from tests/auto/testlib/selftests/expected_xunit.junitxml rename to tests/auto/testlib/selftests/expected_junit.junitxml index e1b21760f24..b3591a68682 100644 --- a/tests/auto/testlib/selftests/expected_xunit.junitxml +++ b/tests/auto/testlib/selftests/expected_junit.junitxml @@ -1,5 +1,5 @@ - + diff --git a/tests/auto/testlib/selftests/expected_xunit.lightxml b/tests/auto/testlib/selftests/expected_xunit.lightxml deleted file mode 100644 index 863501bddf7..00000000000 --- a/tests/auto/testlib/selftests/expected_xunit.lightxml +++ /dev/null @@ -1,64 +0,0 @@ - - @INSERT_QT_VERSION_HERE@ - - @INSERT_QT_VERSION_HERE@ - - - - - - - - - - - - - - - ]]> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/tests/auto/testlib/selftests/expected_xunit.tap b/tests/auto/testlib/selftests/expected_xunit.tap deleted file mode 100644 index e71a10853d5..00000000000 --- a/tests/auto/testlib/selftests/expected_xunit.tap +++ /dev/null @@ -1,44 +0,0 @@ -TAP version 13 -# tst_Xunit -ok 1 - initTestCase() -# just a QWARN() ! -ok 2 - testFunc1() -# a qDebug() call with comment-ending stuff --> -not ok 3 - testFunc2() - --- - type: QCOMPARE - message: Compared values are not the same - wanted: 3 (3) - found: 2 (2) - expected: 3 (3) - actual: 2 (2) - at: tst_Xunit::testFunc2() (qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp:0) - file: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp - line: 0 - ... -ok 4 - testFunc3() # SKIP skipping this function! -not ok 5 - testFunc4() - --- - # a forced failure! - at: tst_Xunit::testFunc4() (qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp:0) - file: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp - line: 0 - ... -not ok 6 - testFunc5() # TODO this failure is expected - --- - at: tst_Xunit::testFunc5() (qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp:0) - file: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp - line: 0 - ... -not ok 7 - testFunc6() # TODO this failure is also expected - --- - at: tst_Xunit::testFunc6() (qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp:0) - file: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp - line: 0 - ... -ok 8 - testFunc7() # TODO 'true' returned TRUE unexpectedly. () -ok 9 - cleanupTestCase() -1..9 -# tests 9 -# pass 5 -# fail 3 diff --git a/tests/auto/testlib/selftests/expected_xunit.teamcity b/tests/auto/testlib/selftests/expected_xunit.teamcity deleted file mode 100644 index 99696244137..00000000000 --- a/tests/auto/testlib/selftests/expected_xunit.teamcity +++ /dev/null @@ -1,26 +0,0 @@ -##teamcity[testSuiteStarted name='tst_Xunit' flowId='tst_Xunit'] -##teamcity[testStarted name='initTestCase()' flowId='tst_Xunit'] -##teamcity[testFinished name='initTestCase()' flowId='tst_Xunit'] -##teamcity[testStarted name='testFunc1()' flowId='tst_Xunit'] -##teamcity[testStdOut name='testFunc1()' out='WARNING |[Loc: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)|]: just a QWARN() !' flowId='tst_Xunit'] -##teamcity[testFinished name='testFunc1()' flowId='tst_Xunit'] -##teamcity[testStarted name='testFunc2()' flowId='tst_Xunit'] -##teamcity[testFailed name='testFunc2()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)|]' details='Compared values are not the same|n Actual (2): 2|n Expected (3): 3' flowId='tst_Xunit'] -##teamcity[testStdOut name='testFunc2()' out='QDEBUG: a qDebug() call with comment-ending stuff -->' flowId='tst_Xunit'] -##teamcity[testFinished name='testFunc2()' flowId='tst_Xunit'] -##teamcity[testIgnored name='testFunc3()' message='skipping this function! |[Loc: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)|]' flowId='tst_Xunit'] -##teamcity[testStarted name='testFunc4()' flowId='tst_Xunit'] -##teamcity[testFailed name='testFunc4()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)|]' details='a forced failure!' flowId='tst_Xunit'] -##teamcity[testFinished name='testFunc4()' flowId='tst_Xunit'] -##teamcity[testStarted name='testFunc5()' flowId='tst_Xunit'] -##teamcity[testStdOut name='testFunc5()' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)|]: this failure is expected' flowId='tst_Xunit'] -##teamcity[testFinished name='testFunc5()' flowId='tst_Xunit'] -##teamcity[testStarted name='testFunc6()' flowId='tst_Xunit'] -##teamcity[testStdOut name='testFunc6()' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)|]: this failure is also expected' flowId='tst_Xunit'] -##teamcity[testFinished name='testFunc6()' flowId='tst_Xunit'] -##teamcity[testStarted name='testFunc7()' flowId='tst_Xunit'] -##teamcity[testFailed name='testFunc7()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_Xunit'] -##teamcity[testFinished name='testFunc7()' flowId='tst_Xunit'] -##teamcity[testStarted name='cleanupTestCase()' flowId='tst_Xunit'] -##teamcity[testFinished name='cleanupTestCase()' flowId='tst_Xunit'] -##teamcity[testSuiteFinished name='tst_Xunit' flowId='tst_Xunit'] diff --git a/tests/auto/testlib/selftests/expected_xunit.txt b/tests/auto/testlib/selftests/expected_xunit.txt deleted file mode 100644 index bc6f50a58f9..00000000000 --- a/tests/auto/testlib/selftests/expected_xunit.txt +++ /dev/null @@ -1,26 +0,0 @@ -********* Start testing of tst_Xunit ********* -Config: Using QtTest library -PASS : tst_Xunit::initTestCase() -WARNING: tst_Xunit::testFunc1() just a QWARN() ! - Loc: [qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)] -PASS : tst_Xunit::testFunc1() -QDEBUG : tst_Xunit::testFunc2() a qDebug() call with comment-ending stuff --> -FAIL! : tst_Xunit::testFunc2() Compared values are not the same - Actual (2): 2 - Expected (3): 3 - Loc: [qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)] -SKIP : tst_Xunit::testFunc3() skipping this function! - Loc: [qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)] -FAIL! : tst_Xunit::testFunc4() a forced failure! - Loc: [qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)] -XFAIL : tst_Xunit::testFunc5() this failure is expected - Loc: [qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)] -PASS : tst_Xunit::testFunc5() -XFAIL : tst_Xunit::testFunc6() this failure is also expected - Loc: [qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)] -PASS : tst_Xunit::testFunc6() -XPASS : tst_Xunit::testFunc7() 'true' returned TRUE unexpectedly. () - Loc: [qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)] -PASS : tst_Xunit::cleanupTestCase() -Totals: 5 passed, 3 failed, 1 skipped, 0 blacklisted, 0ms -********* Finished testing of tst_Xunit ********* diff --git a/tests/auto/testlib/selftests/expected_xunit.xml b/tests/auto/testlib/selftests/expected_xunit.xml deleted file mode 100644 index 723ae6c9147..00000000000 --- a/tests/auto/testlib/selftests/expected_xunit.xml +++ /dev/null @@ -1,67 +0,0 @@ - - - - @INSERT_QT_VERSION_HERE@ - - @INSERT_QT_VERSION_HERE@ - - - - - - - - - - - - - - - ]]> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/tests/auto/testlib/selftests/generate_expected_output.py b/tests/auto/testlib/selftests/generate_expected_output.py index bb224e2a4f3..fb1b7ee96cb 100755 --- a/tests/auto/testlib/selftests/generate_expected_output.py +++ b/tests/auto/testlib/selftests/generate_expected_output.py @@ -64,7 +64,7 @@ TESTS = ['assert', 'badxml', 'benchlibcallgrind', 'benchlibcounting', 'signaldumper', 'silent', 'singleskip', 'skip', 'skipcleanup', 'skipinit', 'skipinitdata', 'sleep', 'strcmp', 'subtest', 'testlib', 'tuplediagnostics', 'verbose1', 'verbose2', 'verifyexceptionthrown', - 'warnings', 'watchdog', 'xunit', 'keyboard'] + 'warnings', 'watchdog', 'junit', 'keyboard'] class Fail (Exception): pass @@ -306,6 +306,8 @@ def generateTestData(test_path, expected_path, clean, formats): env = testEnv(testname) for format in formats: + if testname == "junit" and not format == "junitxml": + continue print(f' running {testname}/{format}') cmd = [path, f'-{format}'] expected_file = f'expected_{testname}.{format}' diff --git a/tests/auto/testlib/selftests/junit/CMakeLists.txt b/tests/auto/testlib/selftests/junit/CMakeLists.txt new file mode 100644 index 00000000000..94138b65771 --- /dev/null +++ b/tests/auto/testlib/selftests/junit/CMakeLists.txt @@ -0,0 +1,11 @@ + +qt_internal_add_executable(junit + NO_INSTALL # special case + OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} # special case + SOURCES + tst_junit.cpp + PUBLIC_LIBRARIES + Qt::Test +) + +qt_internal_apply_testlib_coverage_options(junit) diff --git a/tests/auto/testlib/selftests/xunit/tst_xunit.cpp b/tests/auto/testlib/selftests/junit/tst_junit.cpp similarity index 87% rename from tests/auto/testlib/selftests/xunit/tst_xunit.cpp rename to tests/auto/testlib/selftests/junit/tst_junit.cpp index d620784e416..77f369b7792 100644 --- a/tests/auto/testlib/selftests/xunit/tst_xunit.cpp +++ b/tests/auto/testlib/selftests/junit/tst_junit.cpp @@ -28,12 +28,12 @@ #include -class tst_Xunit : public QObject +class tst_JUnit : public QObject { Q_OBJECT public: - tst_Xunit(); + tst_JUnit(); private slots: void testFunc1(); @@ -45,28 +45,28 @@ private slots: void testFunc7(); }; -tst_Xunit::tst_Xunit() +tst_JUnit::tst_JUnit() { } -void tst_Xunit::testFunc1() +void tst_JUnit::testFunc1() { QWARN("just a QWARN() !"); QCOMPARE(1,1); } -void tst_Xunit::testFunc2() +void tst_JUnit::testFunc2() { qDebug("a qDebug() call with comment-ending stuff -->"); QCOMPARE(2, 3); } -void tst_Xunit::testFunc3() +void tst_JUnit::testFunc3() { QSKIP("skipping this function!"); } -void tst_Xunit::testFunc4() +void tst_JUnit::testFunc4() { QFAIL("a forced failure!"); } @@ -79,24 +79,24 @@ void tst_Xunit::testFunc4() count. */ -void tst_Xunit::testFunc5() +void tst_JUnit::testFunc5() { QEXPECT_FAIL("", "this failure is expected", Abort); QVERIFY(false); } -void tst_Xunit::testFunc6() +void tst_JUnit::testFunc6() { QEXPECT_FAIL("", "this failure is also expected", Abort); QFAIL("This is a deliberate failure"); } -void tst_Xunit::testFunc7() +void tst_JUnit::testFunc7() { QEXPECT_FAIL("", "this pass is unexpected", Abort); QVERIFY(true); } -QTEST_APPLESS_MAIN(tst_Xunit) -#include "tst_xunit.moc" +QTEST_APPLESS_MAIN(tst_JUnit) +#include "tst_junit.moc" diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp index 1b1ad50e9ba..786a79a2f35 100644 --- a/tests/auto/testlib/selftests/tst_selftests.cpp +++ b/tests/auto/testlib/selftests/tst_selftests.cpp @@ -748,6 +748,9 @@ bool TestLogger::shouldIgnoreTest(const QString &test) const if (logger == QTestLog::TeamCity && test.startsWith("benchlib")) return true; // Skip benchmark for TeamCity logger + if (logger != QTestLog::JUnitXML && test == "junit") + return true; + return false; } @@ -775,7 +778,7 @@ void checkErrorOutput(const QString &test, const QByteArray &errorOutput) || test == "cmptest" // QImage comparison requires QGuiApplication || test == "fetchbogus" || test == "watchdog" - || test == "xunit" + || test == "junit" || test == "benchlibcallgrind") return; diff --git a/tests/auto/testlib/selftests/xunit/CMakeLists.txt b/tests/auto/testlib/selftests/xunit/CMakeLists.txt deleted file mode 100644 index c153d085d08..00000000000 --- a/tests/auto/testlib/selftests/xunit/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# Generated from xunit.pro. - -##################################################################### -## xunit Binary: -##################################################################### - -qt_internal_add_executable(xunit - NO_INSTALL # special case - OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} # special case - SOURCES - tst_xunit.cpp - PUBLIC_LIBRARIES - Qt::Test -) - -## Scopes: -##################################################################### - -# special case begin -qt_internal_apply_testlib_coverage_options(xunit) -# special case end