From 78a58903553fe2b6dc8620f849641ffc5324370d Mon Sep 17 00:00:00 2001 From: Edward Welbourne Date: Mon, 11 Jul 2022 17:46:38 +0200 Subject: [PATCH] Add some testing of QTestEventLoop Fairly minimal for now, just enough to verify a bug and serve as the sign of success when it's fixed. Tests fail in ways they shouldn't, for now; see expected_eventloop.* for details, notably "Earlier test failed to clean up" messages. Task-number: QTBUG-104441 Change-Id: I59be4aa5f21fed23b19a0593a8c2f6c9956507df Reviewed-by: Jason McDonald (cherry picked from commit 35ad157d88c7bfcb9b90b01111b0f43dd2e012d9) Reviewed-by: Qt Cherry-pick Bot --- tests/auto/testlib/selftests/CMakeLists.txt | 1 + .../selftests/eventloop/CMakeLists.txt | 16 ++++ .../selftests/eventloop/tst_eventloop.cpp | 96 +++++++++++++++++++ .../selftests/expected_eventloop.junitxml | 19 ++++ .../selftests/expected_eventloop.lightxml | 41 ++++++++ .../testlib/selftests/expected_eventloop.tap | 75 +++++++++++++++ .../selftests/expected_eventloop.teamcity | 21 ++++ .../testlib/selftests/expected_eventloop.txt | 18 ++++ .../testlib/selftests/expected_eventloop.xml | 44 +++++++++ .../selftests/generate_expected_output.py | 4 +- 10 files changed, 333 insertions(+), 2 deletions(-) create mode 100644 tests/auto/testlib/selftests/eventloop/CMakeLists.txt create mode 100644 tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp create mode 100644 tests/auto/testlib/selftests/expected_eventloop.junitxml create mode 100644 tests/auto/testlib/selftests/expected_eventloop.lightxml create mode 100644 tests/auto/testlib/selftests/expected_eventloop.tap create mode 100644 tests/auto/testlib/selftests/expected_eventloop.teamcity create mode 100644 tests/auto/testlib/selftests/expected_eventloop.txt create mode 100644 tests/auto/testlib/selftests/expected_eventloop.xml diff --git a/tests/auto/testlib/selftests/CMakeLists.txt b/tests/auto/testlib/selftests/CMakeLists.txt index ceea1226f8e..c3f428d1108 100644 --- a/tests/auto/testlib/selftests/CMakeLists.txt +++ b/tests/auto/testlib/selftests/CMakeLists.txt @@ -56,6 +56,7 @@ set(subprograms deleteLater deleteLater_noApp differentexec + eventloop exceptionthrow expectfail extendedcompare diff --git a/tests/auto/testlib/selftests/eventloop/CMakeLists.txt b/tests/auto/testlib/selftests/eventloop/CMakeLists.txt new file mode 100644 index 00000000000..f1fa9a788c2 --- /dev/null +++ b/tests/auto/testlib/selftests/eventloop/CMakeLists.txt @@ -0,0 +1,16 @@ +##################################################################### +## eventloop Binary: +##################################################################### + +qt_internal_add_executable(eventloop + NO_INSTALL + OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + SOURCES + tst_eventloop.cpp + PUBLIC_LIBRARIES + Qt::Test +) + +## Scopes: +##################################################################### +qt_internal_apply_testlib_coverage_options(eventloop) diff --git a/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp b/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp new file mode 100644 index 00000000000..b5e8efce82a --- /dev/null +++ b/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp @@ -0,0 +1,96 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 + +#include +#include +#include + +// Tests for QTestEventLoop (and some QTRY_* details) +class tst_EventLoop: public QObject +{ +Q_OBJECT + + bool m_inTestFunction = false; +private slots: + void cleanup(); + void fail(); + void skip(); + void pass(); +}; + +class DeferredFlag +{ + bool m_flag; +public: + explicit DeferredFlag(bool initial = false) : m_flag(initial) + { + if (!initial) + QTimer::singleShot(50, [this] { m_flag = true; }); + } + explicit operator bool() const { return m_flag; } + bool operator!() const { return !m_flag; } + friend bool operator==(DeferredFlag a, DeferredFlag b) { return bool(a) == bool(b); } +}; + +char *toString(DeferredFlag val) +{ + return qstrdup(bool(val) ? "DeferredFlag(true)" : "DeferredFlag(false)"); +} + +void tst_EventLoop::cleanup() +{ + // QTBUG-104441: looping didn't happen in cleanup() if test failed or skipped. + { + DeferredFlag flag; + auto &loop = QTestEventLoop::instance(); + loop.enterLoopMSecs(100); + QVERIFY2(loop.timeout(), "QTestEventLoop exited prematurely in cleanup()"); + QVERIFY(flag); + } + { + DeferredFlag flag; + QTRY_VERIFY2(flag, "QTRY_* loop exited prematurely in cleanup()"); + } + + m_inTestFunction = false; +} + +void tst_EventLoop::fail() +{ + QVERIFY2(!std::exchange(m_inTestFunction, true), "Earlier test failed to clean up"); + QFAIL("Failing test should still clean up"); +} + +void tst_EventLoop::skip() +{ + QVERIFY2(!std::exchange(m_inTestFunction, true), "Earlier test failed to clean up"); + QSKIP("Skipping test should still clean up"); +} + +void tst_EventLoop::pass() +{ + QVERIFY2(!std::exchange(m_inTestFunction, true), "Earlier test failed to clean up"); + { + DeferredFlag flag; + auto &loop = QTestEventLoop::instance(); + loop.enterLoopMSecs(100); + QVERIFY(loop.timeout()); + QVERIFY(flag); + } + { + DeferredFlag flag; + QTRY_VERIFY(flag); + } + DeferredFlag flag; + QTestEventLoop loop(this); + QVERIFY(!flag); + loop.enterLoopMSecs(1); + QVERIFY(loop.timeout()); + QVERIFY(!flag); + loop.enterLoopMSecs(100); + QVERIFY(loop.timeout()); + QVERIFY(flag); +} + +QTEST_MAIN(tst_EventLoop) +#include "tst_eventloop.moc" diff --git a/tests/auto/testlib/selftests/expected_eventloop.junitxml b/tests/auto/testlib/selftests/expected_eventloop.junitxml new file mode 100644 index 00000000000..15a21427c83 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_eventloop.junitxml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_eventloop.lightxml b/tests/auto/testlib/selftests/expected_eventloop.lightxml new file mode 100644 index 00000000000..0d67d73f752 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_eventloop.lightxml @@ -0,0 +1,41 @@ + + @INSERT_QT_VERSION_HERE@ + + @INSERT_QT_VERSION_HERE@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_eventloop.tap b/tests/auto/testlib/selftests/expected_eventloop.tap new file mode 100644 index 00000000000..85f48a13a16 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_eventloop.tap @@ -0,0 +1,75 @@ +TAP version 13 +# tst_EventLoop +ok 1 - initTestCase() +not ok 2 - fail() + --- + # Failing test should still clean up + at: tst_EventLoop::fail() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0) + file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp + line: 0 + ... +not ok 2 - fail() + --- + type: QVERIFY + message: ) + wanted: true (loop.timeout()) + found: false (loop.timeout()) + expected: true (loop.timeout()) + actual: false (loop.timeout()) + at: tst_EventLoop::fail() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0) + file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp + line: 0 + ... +not ok 3 - skip() + --- + type: QVERIFY + message: Earlier test failed to clean up + wanted: true (!std::exchange(m_inTestFunction, true)) + found: false (!std::exchange(m_inTestFunction, true)) + expected: true (!std::exchange(m_inTestFunction, true)) + actual: false (!std::exchange(m_inTestFunction, true)) + at: tst_EventLoop::skip() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0) + file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp + line: 0 + ... +not ok 3 - skip() + --- + type: QVERIFY + message: ) + wanted: true (loop.timeout()) + found: false (loop.timeout()) + expected: true (loop.timeout()) + actual: false (loop.timeout()) + at: tst_EventLoop::skip() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0) + file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp + line: 0 + ... +not ok 4 - pass() + --- + type: QVERIFY + message: Earlier test failed to clean up + wanted: true (!std::exchange(m_inTestFunction, true)) + found: false (!std::exchange(m_inTestFunction, true)) + expected: true (!std::exchange(m_inTestFunction, true)) + actual: false (!std::exchange(m_inTestFunction, true)) + at: tst_EventLoop::pass() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0) + file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp + line: 0 + ... +not ok 4 - pass() + --- + type: QVERIFY + message: ) + wanted: true (loop.timeout()) + found: false (loop.timeout()) + expected: true (loop.timeout()) + actual: false (loop.timeout()) + at: tst_EventLoop::pass() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0) + file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp + line: 0 + ... +ok 5 - cleanupTestCase() +1..5 +# tests 5 +# pass 2 +# fail 3 diff --git a/tests/auto/testlib/selftests/expected_eventloop.teamcity b/tests/auto/testlib/selftests/expected_eventloop.teamcity new file mode 100644 index 00000000000..4c832be52f7 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_eventloop.teamcity @@ -0,0 +1,21 @@ +##teamcity[testSuiteStarted name='tst_EventLoop' flowId='tst_EventLoop'] +##teamcity[testStarted name='initTestCase()' flowId='tst_EventLoop'] +##teamcity[testFinished name='initTestCase()' flowId='tst_EventLoop'] +##teamcity[testStarted name='fail()' flowId='tst_EventLoop'] +##teamcity[testFailed name='fail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='Failing test should still clean up' flowId='tst_EventLoop'] +##teamcity[testFinished name='fail()' flowId='tst_EventLoop'] +##teamcity[testFailed name='fail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='|'loop.timeout()|' returned FALSE. (QTestEventLoop exited prematurely in cleanup())' flowId='tst_EventLoop'] +##teamcity[testFinished name='fail()' flowId='tst_EventLoop'] +##teamcity[testStarted name='skip()' flowId='tst_EventLoop'] +##teamcity[testFailed name='skip()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='|'!std::exchange(m_inTestFunction, true)|' returned FALSE. (Earlier test failed to clean up)' flowId='tst_EventLoop'] +##teamcity[testFinished name='skip()' flowId='tst_EventLoop'] +##teamcity[testFailed name='skip()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='|'loop.timeout()|' returned FALSE. (QTestEventLoop exited prematurely in cleanup())' flowId='tst_EventLoop'] +##teamcity[testFinished name='skip()' flowId='tst_EventLoop'] +##teamcity[testStarted name='pass()' flowId='tst_EventLoop'] +##teamcity[testFailed name='pass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='|'!std::exchange(m_inTestFunction, true)|' returned FALSE. (Earlier test failed to clean up)' flowId='tst_EventLoop'] +##teamcity[testFinished name='pass()' flowId='tst_EventLoop'] +##teamcity[testFailed name='pass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='|'loop.timeout()|' returned FALSE. (QTestEventLoop exited prematurely in cleanup())' flowId='tst_EventLoop'] +##teamcity[testFinished name='pass()' flowId='tst_EventLoop'] +##teamcity[testStarted name='cleanupTestCase()' flowId='tst_EventLoop'] +##teamcity[testFinished name='cleanupTestCase()' flowId='tst_EventLoop'] +##teamcity[testSuiteFinished name='tst_EventLoop' flowId='tst_EventLoop'] diff --git a/tests/auto/testlib/selftests/expected_eventloop.txt b/tests/auto/testlib/selftests/expected_eventloop.txt new file mode 100644 index 00000000000..b614c876c6d --- /dev/null +++ b/tests/auto/testlib/selftests/expected_eventloop.txt @@ -0,0 +1,18 @@ +********* Start testing of tst_EventLoop ********* +Config: Using QtTest library +PASS : tst_EventLoop::initTestCase() +FAIL! : tst_EventLoop::fail() Failing test should still clean up + Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)] +FAIL! : tst_EventLoop::fail() 'loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup()) + Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)] +FAIL! : tst_EventLoop::skip() '!std::exchange(m_inTestFunction, true)' returned FALSE. (Earlier test failed to clean up) + Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)] +FAIL! : tst_EventLoop::skip() 'loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup()) + Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)] +FAIL! : tst_EventLoop::pass() '!std::exchange(m_inTestFunction, true)' returned FALSE. (Earlier test failed to clean up) + Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)] +FAIL! : tst_EventLoop::pass() 'loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup()) + Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)] +PASS : tst_EventLoop::cleanupTestCase() +Totals: 2 passed, 3 failed, 0 skipped, 0 blacklisted, 0ms +********* Finished testing of tst_EventLoop ********* diff --git a/tests/auto/testlib/selftests/expected_eventloop.xml b/tests/auto/testlib/selftests/expected_eventloop.xml new file mode 100644 index 00000000000..2613a734494 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_eventloop.xml @@ -0,0 +1,44 @@ + + + + @INSERT_QT_VERSION_HERE@ + + @INSERT_QT_VERSION_HERE@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/generate_expected_output.py b/tests/auto/testlib/selftests/generate_expected_output.py index 987a08b2c1c..8f6134725bf 100755 --- a/tests/auto/testlib/selftests/generate_expected_output.py +++ b/tests/auto/testlib/selftests/generate_expected_output.py @@ -31,8 +31,8 @@ TESTS = ['assert', 'badxml', 'benchlibcallgrind', 'benchlibcounting', 'benchlibeventcounter', 'benchliboptions', 'benchlibtickcounter', 'benchlibwalltime', 'blacklisted', 'cmptest', 'commandlinedata', 'counting', 'crashes', 'datatable', 'datetime', 'deleteLater', - 'deleteLater_noApp', 'differentexec', 'exceptionthrow', 'expectfail', - "extendedcompare", 'failcleanup', 'failcleanuptestcase', + 'deleteLater_noApp', 'differentexec', 'eventloop', 'exceptionthrow', + 'expectfail', "extendedcompare", 'failcleanup', 'failcleanuptestcase', 'faildatatype', 'failfetchtype', 'failinit', 'failinitdata', 'fetchbogus', 'findtestdata', 'float', 'globaldata', 'longstring', 'maxwarnings', 'mouse', 'multiexec', 'pairdiagnostics', 'pass',