From be72cb9cfdacf90c145bb02d34ab2e2cb79f3935 Mon Sep 17 00:00:00 2001 From: Edward Welbourne Date: Thu, 12 Aug 2021 14:56:46 +0200 Subject: [PATCH] Include check for the Totals line in testlib's expectedfail selftest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Count how many we expect to pass, fail and be skipped, so we can report that for comparison with the actual totals line: this reveals some double-counting, marked with a FIXME. Task-number: QTBUG-95661 Change-Id: I8b0f13ded5202ed476d8abfee70ed60f9e639bf9 Reviewed-by: Mårten Nordheim Reviewed-by: Tor Arne Vestbø --- .../selftests/expected_expectfail.junitxml | 6 +- .../selftests/expected_expectfail.lightxml | 3 + .../testlib/selftests/expected_expectfail.tap | 1 + .../selftests/expected_expectfail.teamcity | 1 + .../testlib/selftests/expected_expectfail.txt | 1 + .../testlib/selftests/expected_expectfail.xml | 3 + .../selftests/expectfail/tst_expectfail.cpp | 57 +++++++++++++++++++ 7 files changed, 71 insertions(+), 1 deletion(-) diff --git a/tests/auto/testlib/selftests/expected_expectfail.junitxml b/tests/auto/testlib/selftests/expected_expectfail.junitxml index 31521263eef..54f0416016b 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.junitxml +++ b/tests/auto/testlib/selftests/expected_expectfail.junitxml @@ -180,5 +180,9 @@ - + + + + + diff --git a/tests/auto/testlib/selftests/expected_expectfail.lightxml b/tests/auto/testlib/selftests/expected_expectfail.lightxml index 36636e13c4a..4c2beec6ca6 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.lightxml +++ b/tests/auto/testlib/selftests/expected_expectfail.lightxml @@ -312,6 +312,9 @@ + + + diff --git a/tests/auto/testlib/selftests/expected_expectfail.tap b/tests/auto/testlib/selftests/expected_expectfail.tap index 984815909a5..ef464e41b0d 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.tap +++ b/tests/auto/testlib/selftests/expected_expectfail.tap @@ -177,6 +177,7 @@ ok 43 - xpassContinueDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returne # Test should Continue past XPASS # Test should simply PASS ok 44 - xpassContinueDataDrivenWithQCompare(Pass) +# Totals should add up to 44: 23 passed, 17 failed, 4 skipped ok 45 - cleanupTestCase() 1..45 # tests 45 diff --git a/tests/auto/testlib/selftests/expected_expectfail.teamcity b/tests/auto/testlib/selftests/expected_expectfail.teamcity index 738ef664eca..774318bdb60 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.teamcity +++ b/tests/auto/testlib/selftests/expected_expectfail.teamcity @@ -122,5 +122,6 @@ ##teamcity[testStdOut name='xpassContinueDataDrivenWithQCompare(Pass)' out='QDEBUG: Test should Continue past XPASS|nQDEBUG: Test should simply PASS' flowId='tst_ExpectFail'] ##teamcity[testFinished name='xpassContinueDataDrivenWithQCompare(Pass)' flowId='tst_ExpectFail'] ##teamcity[testStarted name='cleanupTestCase()' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='cleanupTestCase()' out='QDEBUG: Totals should add up to 44: 23 passed, 17 failed, 4 skipped' flowId='tst_ExpectFail'] ##teamcity[testFinished name='cleanupTestCase()' flowId='tst_ExpectFail'] ##teamcity[testSuiteFinished name='tst_ExpectFail' flowId='tst_ExpectFail'] diff --git a/tests/auto/testlib/selftests/expected_expectfail.txt b/tests/auto/testlib/selftests/expected_expectfail.txt index 83f673f2f39..2dff4ad55f1 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.txt +++ b/tests/auto/testlib/selftests/expected_expectfail.txt @@ -108,6 +108,7 @@ XPASS : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(XPass) QCOMPARE(1, QDEBUG : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(XPass) Test should Continue past XPASS QDEBUG : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(Pass) Test should simply PASS PASS : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(Pass) +QDEBUG : tst_ExpectFail::cleanupTestCase() Totals should add up to 44: 23 passed, 17 failed, 4 skipped PASS : tst_ExpectFail::cleanupTestCase() Totals: 23 passed, 17 failed, 5 skipped, 0 blacklisted, 0ms ********* Finished testing of tst_ExpectFail ********* diff --git a/tests/auto/testlib/selftests/expected_expectfail.xml b/tests/auto/testlib/selftests/expected_expectfail.xml index 27aae6b75cd..9c25b289818 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.xml +++ b/tests/auto/testlib/selftests/expected_expectfail.xml @@ -314,6 +314,9 @@ + + + diff --git a/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp b/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp index b97244b4768..7b555b82933 100644 --- a/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp +++ b/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp @@ -37,6 +37,8 @@ class tst_ExpectFail: public QObject Q_OBJECT private slots: + void cleanupTestCase() const; + void init() const; void xfailAndContinue() const; void xfailAndAbort() const; void xfailContinueSkip() const; @@ -76,9 +78,25 @@ private: void xfailDataDriven_data(bool failOnly) const; void xpassDataDriven_data() const; }; +static int casesTested = 2; +// What the totals line's numbers *should* be: +static int passed = 2, failed = 0, skipped = 0; +// Total and passed get {init,cleanup}TestCase() in addition to the actual tests. + +void tst_ExpectFail::init() const +{ + ++casesTested; +} + +void tst_ExpectFail::cleanupTestCase() const +{ + qDebug("Totals should add up to %d: %d passed, %d failed, %d skipped", + casesTested, passed, failed, skipped); +} void tst_ExpectFail::xfailAndContinue() const { + ++passed; qDebug("begin"); QEXPECT_FAIL("", "This should xfail", Continue); QVERIFY(false); @@ -87,6 +105,7 @@ void tst_ExpectFail::xfailAndContinue() const void tst_ExpectFail::xfailAndAbort() const { + ++passed; qDebug("begin"); QEXPECT_FAIL("", "This should xfail", Abort); QVERIFY(false); @@ -97,6 +116,7 @@ void tst_ExpectFail::xfailAndAbort() const void tst_ExpectFail::xfailContinueSkip() const { + ++skipped; QEXPECT_FAIL("", "This should xfail then skip", Continue); QVERIFY(false); QSKIP("This skip should be reported and counted"); @@ -104,6 +124,7 @@ void tst_ExpectFail::xfailContinueSkip() const void tst_ExpectFail::xfailAbortSkip() const { + ++passed; QEXPECT_FAIL("", "This should xfail", Abort); QVERIFY(false); @@ -113,6 +134,7 @@ void tst_ExpectFail::xfailAbortSkip() const void tst_ExpectFail::xfailTwice() const { + ++failed; QEXPECT_FAIL("", "Calling QEXPECT_FAIL once is fine", Continue); QEXPECT_FAIL("", "Calling QEXPECT_FAIL when already expecting a failure is " "an error and should abort this test function", Continue); @@ -123,6 +145,7 @@ void tst_ExpectFail::xfailTwice() const void tst_ExpectFail::xfailDataDrivenTwice() const { + ++failed; // Same with data-driven cases (twist semantics of unused shouldPass; we // have four combinations to test): QEXPECT_FAIL("Pass Abort", "Calling QEXPECT_FAIL once on a test-case is fine", Abort); @@ -144,6 +167,7 @@ void tst_ExpectFail::xfailDataDrivenTwice() const void tst_ExpectFail::xfailWithQString() const { + ++passed; QEXPECT_FAIL("", QString("A string").toLatin1().constData(), Continue); QVERIFY(false); @@ -157,6 +181,11 @@ void tst_ExpectFail::xfailDataDrivenWithQString() const { // This test does not (yet) distinguish the two Pass cases. QFETCH(bool, shouldPass); + QFETCH(QTest::TestFailMode, failMode); + if (shouldPass || failMode == QTest::Continue) + ++skipped; + else + ++passed; QEXPECT_FAIL("Fail Abort", QString("A string").toLatin1().constData(), Abort); QEXPECT_FAIL("Fail Continue", QString("A string").toLatin1().constData(), Continue); @@ -176,6 +205,7 @@ void tst_ExpectFail::xfailDataDrivenWithQString() const void tst_ExpectFail::xfailDataDrivenWithQVerify() const { // This test does not (yet) distinguish the two Pass cases. + ++passed; QFETCH(bool, shouldPass); QFETCH(QTest::TestFailMode, failMode); @@ -205,6 +235,7 @@ void tst_ExpectFail::xfailDataDriven_data(bool failOnly) const void tst_ExpectFail::xfailDataDrivenWithQCompare() const { // This test does not (yet) distinguish the two Pass cases. + ++passed; QFETCH(bool, shouldPass); QFETCH(QTest::TestFailMode, failMode); @@ -221,6 +252,7 @@ void tst_ExpectFail::xfailDataDrivenWithQCompare() const void tst_ExpectFail::xfailOnWrongRow() const { + ++passed; qDebug("Should pass (*not* xpass), despite test-case name"); // QEXPECT_FAIL for a row that does not exist should be ignored. // (It might be conditional data(), so exist in other circumstances.) @@ -237,6 +269,7 @@ void tst_ExpectFail::xfailOnWrongRow() const void tst_ExpectFail::xfailOnAnyRow() const { + ++passed; // In a data-driven test, passing an empty first parameter to QEXPECT_FAIL // should mean that the failure is expected for all data rows. QFETCH(QTest::TestFailMode, failMode); @@ -252,6 +285,7 @@ void tst_ExpectFail::xfailOnAnyRow() const void tst_ExpectFail::xfailWithoutCheck() const { + ++failed; qDebug("Should fail (*not* xfail), despite test-case name"); QTEST(false, "shouldPass"); // _data skips the passing tests as pass/fail is irrelevant QEXPECT_FAIL("Fail Abort", "Calling QEXPECT_FAIL without any subsequent check is an error", @@ -262,6 +296,7 @@ void tst_ExpectFail::xfailWithoutCheck() const void tst_ExpectFail::xpassAbort() const { + ++failed; QEXPECT_FAIL("", "This test should xpass", Abort); QVERIFY(true); @@ -272,6 +307,7 @@ void tst_ExpectFail::xpassAbort() const void tst_ExpectFail::xpassAbortSkip() const { + ++failed; QEXPECT_FAIL("", "This test should xpass", Abort); QVERIFY(true); @@ -280,6 +316,7 @@ void tst_ExpectFail::xpassAbortSkip() const void tst_ExpectFail::xpassAbortXfailContinue() const { + ++failed; QEXPECT_FAIL("", "This test should xpass", Abort); QVERIFY(true); @@ -291,6 +328,7 @@ void tst_ExpectFail::xpassAbortXfailContinue() const void tst_ExpectFail::xpassContinue() const { + ++failed; QEXPECT_FAIL("", "This test should xpass", Continue); QVERIFY(true); qDebug("This should be reached"); @@ -306,13 +344,16 @@ void tst_ExpectFail::xpassDataDriven_data() const void tst_ExpectFail::xpassContinueSkip() const { + ++failed; // and *not* ++skipped QEXPECT_FAIL("", "This test should xpass", Continue); QVERIFY(true); + // FIXME: QTBUG-95661 skip-count is incremented. QSKIP("This should be reached but not increment skip-count"); } void tst_ExpectFail::xpassContinueXfailAbort() const { + ++failed; QEXPECT_FAIL("", "This test should xpass", Continue); QVERIFY(true); QEXPECT_FAIL("", "This test should xfail but not add to totals", Abort); @@ -323,6 +364,10 @@ void tst_ExpectFail::xpassContinueXfailAbort() const void tst_ExpectFail::xpassAbortDataDrivenWithQVerify() const { QFETCH(bool, shouldXPass); + if (shouldXPass) + ++failed; + else + ++passed; QEXPECT_FAIL("XPass", "This test-row should xpass", Abort); QVERIFY(true); @@ -334,6 +379,10 @@ void tst_ExpectFail::xpassAbortDataDrivenWithQVerify() const void tst_ExpectFail::xpassContinueDataDrivenWithQVerify() const { QFETCH(bool, shouldXPass); + if (shouldXPass) + ++failed; + else + ++passed; QEXPECT_FAIL("XPass", "This test-row should xpass", Continue); QVERIFY(true); @@ -344,6 +393,10 @@ void tst_ExpectFail::xpassContinueDataDrivenWithQVerify() const void tst_ExpectFail::xpassAbortDataDrivenWithQCompare() const { QFETCH(bool, shouldXPass); + if (shouldXPass) + ++failed; + else + ++passed; QEXPECT_FAIL("XPass", "This test should xpass", Abort); QCOMPARE(1, 1); @@ -355,6 +408,10 @@ void tst_ExpectFail::xpassAbortDataDrivenWithQCompare() const void tst_ExpectFail::xpassContinueDataDrivenWithQCompare() const { QFETCH(bool, shouldXPass); + if (shouldXPass) + ++failed; + else + ++passed; QEXPECT_FAIL("XPass", "This test should xpass", Continue); QCOMPARE(1, 1);