From da18ada6bf60e0a6724ee55abb619d367f8c4633 Mon Sep 17 00:00:00 2001 From: Jason McDonald Date: Wed, 1 Feb 2012 15:42:00 +1000 Subject: [PATCH] testlib: Clear expected failures after every data row. Previously, expected failures were only cleared at the end of each test function, i.e. after all data rows were finished. This meant that if a data-driven test function called QEXPECT_FAIL and then didn't perform any further verification steps to trigger the expected failure, the expected failure would be carried over to the next data row, probably causing the first verification step in the test function to XPASS (with a seemingly irrelevant error message) for the next data row. This commit adds the new function QTestResult::finishedCurrentTestData() to cleanup after each data row is executed. This function treats calls to QEXPECT_FAIL without subsequent verification steps as a test failure. This commit also adds a regression test to demonstrate that expected failures can no longer be carried over from one data row to another. If run against the previous version of testlib, the new test would report a pass instead of an error. Change-Id: Ida5c7f080815b0dca9531131fed582b0918334cb Reviewed-by: Rohan McGovern --- src/testlib/qtestcase.cpp | 2 ++ src/testlib/qtestresult.cpp | 9 ++++-- src/testlib/qtestresult_p.h | 1 + .../selftests/expected_expectfail.lightxml | 32 ++++++++++++------- .../testlib/selftests/expected_expectfail.txt | 26 ++++++++------- .../testlib/selftests/expected_expectfail.xml | 32 ++++++++++++------- .../selftests/expected_expectfail.xunitxml | 6 +++- .../selftests/expectfail/tst_expectfail.cpp | 16 ++++++++++ 8 files changed, 87 insertions(+), 37 deletions(-) diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp index b76ff39de4f..84b03b73235 100644 --- a/src/testlib/qtestcase.cpp +++ b/src/testlib/qtestcase.cpp @@ -1506,6 +1506,8 @@ static void qInvokeTestMethodDataEntry(char *slot) if (!invokeOk) QTestResult::addFailure("Unable to execute slot", __FILE__, __LINE__); + QTestResult::finishedCurrentTestData(); + QTestResult::setCurrentTestLocation(QTestResult::CleanupFunc); invokeMethod(QTest::currentTestObject, "cleanup()"); QTestResult::setCurrentTestLocation(QTestResult::NoWhere); diff --git a/src/testlib/qtestresult.cpp b/src/testlib/qtestresult.cpp index d6846f363b9..c3634eb01ee 100644 --- a/src/testlib/qtestresult.cpp +++ b/src/testlib/qtestresult.cpp @@ -127,6 +127,13 @@ static void clearExpectFail() QTest::expectFailComment = 0; } +void QTestResult::finishedCurrentTestData() +{ + if (QTest::expectFailMode) + addFailure("QEXPECT_FAIL was called without any subsequent verification statements", 0, 0); + clearExpectFail(); +} + void QTestResult::finishedCurrentTestFunction() { if (!QTest::failed && QTestLog::unhandledIgnoreMessages()) { @@ -143,8 +150,6 @@ void QTestResult::finishedCurrentTestFunction() QTest::location = NoWhere; QTestLog::leaveTestFunction(); - - clearExpectFail(); } const char *QTestResult::currentTestFunction() diff --git a/src/testlib/qtestresult_p.h b/src/testlib/qtestresult_p.h index 5bbb7e2d27b..81e11180ef2 100644 --- a/src/testlib/qtestresult_p.h +++ b/src/testlib/qtestresult_p.h @@ -73,6 +73,7 @@ public: static TestLocation currentTestLocation(); static const char *currentDataTag(); static const char *currentGlobalDataTag(); + static void finishedCurrentTestData(); static void finishedCurrentTestFunction(); static void reset(); diff --git a/tests/auto/testlib/selftests/expected_expectfail.lightxml b/tests/auto/testlib/selftests/expected_expectfail.lightxml index 02e4dde7d88..eab0bbbaa38 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.lightxml +++ b/tests/auto/testlib/selftests/expected_expectfail.lightxml @@ -9,7 +9,7 @@ - + @@ -21,31 +21,31 @@ - + - + - + - + - + - + @@ -55,23 +55,33 @@ - + - + + + + + + + + + + + - + - + diff --git a/tests/auto/testlib/selftests/expected_expectfail.txt b/tests/auto/testlib/selftests/expected_expectfail.txt index 1b98b6b561a..0286490372c 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.txt +++ b/tests/auto/testlib/selftests/expected_expectfail.txt @@ -3,35 +3,37 @@ Config: Using QTest library @INSERT_QT_VERSION_HERE@, Qt @INSERT_QT_VERSION_HERE PASS : tst_ExpectFail::initTestCase() QDEBUG : tst_ExpectFail::xfailAndContinue() begin XFAIL : tst_ExpectFail::xfailAndContinue() This should xfail - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(72)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(74)] QDEBUG : tst_ExpectFail::xfailAndContinue() after PASS : tst_ExpectFail::xfailAndContinue() QDEBUG : tst_ExpectFail::xfailAndAbort() begin XFAIL : tst_ExpectFail::xfailAndAbort() This should xfail - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(80)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(82)] PASS : tst_ExpectFail::xfailAndAbort() FAIL! : tst_ExpectFail::xfailTwice() Already expecting a fail - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(90)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(92)] XFAIL : tst_ExpectFail::xfailWithQString() A string - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(99)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(101)] XFAIL : tst_ExpectFail::xfailWithQString() Bug 5 (The message) - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(104)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(106)] PASS : tst_ExpectFail::xfailWithQString() XFAIL : tst_ExpectFail::xfailDataDriven(Abort) This test should xfail - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(133)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(135)] XFAIL : tst_ExpectFail::xfailDataDriven(Continue) This test should xfail - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(133)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(135)] PASS : tst_ExpectFail::xfailDataDriven() PASS : tst_ExpectFail::xfailOnWrongRow() XFAIL : tst_ExpectFail::xfailOnAnyRow(first row) This test should xfail - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(168)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(170)] XFAIL : tst_ExpectFail::xfailOnAnyRow(second row) This test should xfail - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(168)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(170)] PASS : tst_ExpectFail::xfailOnAnyRow() +FAIL! : tst_ExpectFail::xfailWithoutVerify(first row) QEXPECT_FAIL was called without any subsequent verification statements +FAIL! : tst_ExpectFail::xfailWithoutVerify(second row) QEXPECT_FAIL was called without any subsequent verification statements XPASS : tst_ExpectFail::xpass() 'true' returned FALSE. () - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(172)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(190)] XPASS : tst_ExpectFail::xpassDataDriven(XPass) 'true' returned FALSE. () - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(196)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(212)] PASS : tst_ExpectFail::cleanupTestCase() -Totals: 8 passed, 3 failed, 0 skipped +Totals: 8 passed, 5 failed, 0 skipped ********* Finished testing of tst_ExpectFail ********* diff --git a/tests/auto/testlib/selftests/expected_expectfail.xml b/tests/auto/testlib/selftests/expected_expectfail.xml index 74b11fb0e5e..3c8baf92712 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.xml +++ b/tests/auto/testlib/selftests/expected_expectfail.xml @@ -11,7 +11,7 @@ - + @@ -23,31 +23,31 @@ - + - + - + - + - + - + @@ -57,23 +57,33 @@ - + - + + + + + + + + + + + - + - + diff --git a/tests/auto/testlib/selftests/expected_expectfail.xunitxml b/tests/auto/testlib/selftests/expected_expectfail.xunitxml index 017e4501c19..d13cf6687fe 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.xunitxml +++ b/tests/auto/testlib/selftests/expected_expectfail.xunitxml @@ -1,5 +1,5 @@ - + @@ -30,6 +30,10 @@ + + + + diff --git a/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp b/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp index 3a000ba1826..56895b38015 100644 --- a/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp +++ b/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp @@ -60,6 +60,8 @@ private slots: void xfailOnWrongRow() const; void xfailOnAnyRow_data() const; void xfailOnAnyRow() const; + void xfailWithoutVerify_data() const; + void xfailWithoutVerify() const; void xpass() const; void xpassDataDriven_data() const; void xpassDataDriven() const; @@ -168,6 +170,20 @@ void tst_ExpectFail::xfailOnAnyRow() const QVERIFY(false); } +void tst_ExpectFail::xfailWithoutVerify_data() const +{ + QTest::addColumn("dummy"); + + QTest::newRow("first row") << 0; + QTest::newRow("second row") << 1; +} + +void tst_ExpectFail::xfailWithoutVerify() const +{ + QVERIFY(true); + QEXPECT_FAIL("", "This expected failure should be ignored", Abort); +} + void tst_ExpectFail::xpass() const { QEXPECT_FAIL("", "This test should xpass", Abort);