1berry/qtbase
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add some testing of QTestEventLoop

Browse Source 
Fairly minimal for now, just enough to verify a bug and serve as the
sign of success when it's fixed. Tests fail in ways they shouldn't,
for now; see expected_eventloop.* for details, notably "Earlier test
failed to clean up" messages.

Task-number: QTBUG-104441
Change-Id: I59be4aa5f21fed23b19a0593a8c2f6c9956507df
Reviewed-by: Jason McDonald <macadder1@gmail.com>
(cherry picked from commit 35ad157d88c7bfcb9b90b01111b0f43dd2e012d9)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>
This commit is contained in:
Edward Welbourne 2022-07-11 17:46:38 +02:00 committed by Qt Cherry-pick Bot
parent 590d01c36c
commit 78a5890355
10 changed files with 333 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tests/auto/testlib/selftests/CMakeLists.txt
View File

@ -56,6 +56,7 @@ set(subprograms
deleteLater
deleteLater_noApp
differentexec
eventloop
exceptionthrow
expectfail
extendedcompare

16
tests/auto/testlib/selftests/eventloop/CMakeLists.txt Normal file
View File

@ -0,0 +1,16 @@
#####################################################################
## eventloop Binary:
#####################################################################
qt_internal_add_executable(eventloop
NO_INSTALL
OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
SOURCES
tst_eventloop.cpp
PUBLIC_LIBRARIES
Qt::Test
)
## Scopes:
#####################################################################
qt_internal_apply_testlib_coverage_options(eventloop)

96
tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp Normal file
View File

@ -0,0 +1,96 @@
// Copyright (C) 2022 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
#include <QTest>
#include <QTestEventLoop>
#include <QtCore/QTimer>
// Tests for QTestEventLoop (and some QTRY_* details)
class tst_EventLoop: public QObject
{
Q_OBJECT
bool m_inTestFunction = false;
private slots:
void cleanup();
void fail();
void skip();
void pass();
};
class DeferredFlag
{
bool m_flag;
public:
explicit DeferredFlag(bool initial = false) : m_flag(initial)
{
if (!initial)
QTimer::singleShot(50, [this] { m_flag = true; });
}
explicit operator bool() const { return m_flag; }
bool operator!() const { return !m_flag; }
friend bool operator==(DeferredFlag a, DeferredFlag b) { return bool(a) == bool(b); }
};
char *toString(DeferredFlag val)
{
return qstrdup(bool(val) ? "DeferredFlag(true)" : "DeferredFlag(false)");
}
void tst_EventLoop::cleanup()
{
// QTBUG-104441: looping didn't happen in cleanup() if test failed or skipped.
{
DeferredFlag flag;
auto &loop = QTestEventLoop::instance();
loop.enterLoopMSecs(100);
QVERIFY2(loop.timeout(), "QTestEventLoop exited prematurely in cleanup()");
QVERIFY(flag);
}
{
DeferredFlag flag;
QTRY_VERIFY2(flag, "QTRY_* loop exited prematurely in cleanup()");
}
m_inTestFunction = false;
}
void tst_EventLoop::fail()
{
QVERIFY2(!std::exchange(m_inTestFunction, true), "Earlier test failed to clean up");
QFAIL("Failing test should still clean up");
}
void tst_EventLoop::skip()
{
QVERIFY2(!std::exchange(m_inTestFunction, true), "Earlier test failed to clean up");
QSKIP("Skipping test should still clean up");
}
void tst_EventLoop::pass()
{
QVERIFY2(!std::exchange(m_inTestFunction, true), "Earlier test failed to clean up");
{
DeferredFlag flag;
auto &loop = QTestEventLoop::instance();
loop.enterLoopMSecs(100);
QVERIFY(loop.timeout());
QVERIFY(flag);
}
{
DeferredFlag flag;
QTRY_VERIFY(flag);
}
DeferredFlag flag;
QTestEventLoop loop(this);
QVERIFY(!flag);
loop.enterLoopMSecs(1);
QVERIFY(loop.timeout());
QVERIFY(!flag);
loop.enterLoopMSecs(100);
QVERIFY(loop.timeout());
QVERIFY(flag);
}
QTEST_MAIN(tst_EventLoop)
#include "tst_eventloop.moc"

19
tests/auto/testlib/selftests/expected_eventloop.junitxml Normal file
View File

@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8" ?>
<testsuite name="tst_EventLoop" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="5" failures="3" errors="0" skipped="0" time="@TEST_DURATION@">
<properties>
<property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
<property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
<property name="QtBuild" value=""/>
</properties>
<testcase name="initTestCase" classname="tst_EventLoop" time="@TEST_DURATION@"/>
<testcase name="fail" classname="tst_EventLoop" time="@TEST_DURATION@">
<failure type="fail" message="Failing test should still clean up"/>
</testcase>
<testcase name="skip" classname="tst_EventLoop" time="@TEST_DURATION@">
<failure type="fail" message="&apos;!std::exchange(m_inTestFunction, true)&apos; returned FALSE. (Earlier test failed to clean up)"/>
</testcase>
<testcase name="pass" classname="tst_EventLoop" time="@TEST_DURATION@">
<failure type="fail" message="&apos;!std::exchange(m_inTestFunction, true)&apos; returned FALSE. (Earlier test failed to clean up)"/>
</testcase>
<testcase name="cleanupTestCase" classname="tst_EventLoop" time="@TEST_DURATION@"/>
</testsuite>

41
tests/auto/testlib/selftests/expected_eventloop.lightxml Normal file
View File

@ -0,0 +1,41 @@
<Environment>
<QtVersion>@INSERT_QT_VERSION_HERE@</QtVersion>
<QtBuild/>
<QTestVersion>@INSERT_QT_VERSION_HERE@</QTestVersion>
</Environment>
<TestFunction name="initTestCase">
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="fail">
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
<Description><![CDATA[Failing test should still clean up]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
<Description><![CDATA['loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="skip">
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
<Description><![CDATA['!std::exchange(m_inTestFunction, true)' returned FALSE. (Earlier test failed to clean up)]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
<Description><![CDATA['loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="pass">
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
<Description><![CDATA['!std::exchange(m_inTestFunction, true)' returned FALSE. (Earlier test failed to clean up)]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
<Description><![CDATA['loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="cleanupTestCase">
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<Duration msecs="0"/>

75
tests/auto/testlib/selftests/expected_eventloop.tap Normal file
View File

@ -0,0 +1,75 @@
TAP version 13
# tst_EventLoop
ok 1 - initTestCase()
not ok 2 - fail()
---
# Failing test should still clean up
at: tst_EventLoop::fail() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0)
file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp
line: 0
...
not ok 2 - fail()
---
type: QVERIFY
message: )
wanted: true (loop.timeout())
found: false (loop.timeout())
expected: true (loop.timeout())
actual: false (loop.timeout())
at: tst_EventLoop::fail() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0)
file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp
line: 0
...
not ok 3 - skip()
---
type: QVERIFY
message: Earlier test failed to clean up
wanted: true (!std::exchange(m_inTestFunction, true))
found: false (!std::exchange(m_inTestFunction, true))
expected: true (!std::exchange(m_inTestFunction, true))
actual: false (!std::exchange(m_inTestFunction, true))
at: tst_EventLoop::skip() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0)
file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp
line: 0
...
not ok 3 - skip()
---
type: QVERIFY
message: )
wanted: true (loop.timeout())
found: false (loop.timeout())
expected: true (loop.timeout())
actual: false (loop.timeout())
at: tst_EventLoop::skip() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0)
file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp
line: 0
...
not ok 4 - pass()
---
type: QVERIFY
message: Earlier test failed to clean up
wanted: true (!std::exchange(m_inTestFunction, true))
found: false (!std::exchange(m_inTestFunction, true))
expected: true (!std::exchange(m_inTestFunction, true))
actual: false (!std::exchange(m_inTestFunction, true))
at: tst_EventLoop::pass() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0)
file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp
line: 0
...
not ok 4 - pass()
---
type: QVERIFY
message: )
wanted: true (loop.timeout())
found: false (loop.timeout())
expected: true (loop.timeout())
actual: false (loop.timeout())
at: tst_EventLoop::pass() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0)
file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp
line: 0
...
ok 5 - cleanupTestCase()
1..5
# tests 5
# pass 2
# fail 3

21
tests/auto/testlib/selftests/expected_eventloop.teamcity Normal file
View File

@ -0,0 +1,21 @@
##teamcity[testSuiteStarted name='tst_EventLoop' flowId='tst_EventLoop']
##teamcity[testStarted name='initTestCase()' flowId='tst_EventLoop']
##teamcity[testFinished name='initTestCase()' flowId='tst_EventLoop']
##teamcity[testStarted name='fail()' flowId='tst_EventLoop']
##teamcity[testFailed name='fail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='Failing test should still clean up' flowId='tst_EventLoop']
##teamcity[testFinished name='fail()' flowId='tst_EventLoop']
##teamcity[testFailed name='fail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='|'loop.timeout()|' returned FALSE. (QTestEventLoop exited prematurely in cleanup())' flowId='tst_EventLoop']
##teamcity[testFinished name='fail()' flowId='tst_EventLoop']
##teamcity[testStarted name='skip()' flowId='tst_EventLoop']
##teamcity[testFailed name='skip()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='|'!std::exchange(m_inTestFunction, true)|' returned FALSE. (Earlier test failed to clean up)' flowId='tst_EventLoop']
##teamcity[testFinished name='skip()' flowId='tst_EventLoop']
##teamcity[testFailed name='skip()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='|'loop.timeout()|' returned FALSE. (QTestEventLoop exited prematurely in cleanup())' flowId='tst_EventLoop']
##teamcity[testFinished name='skip()' flowId='tst_EventLoop']
##teamcity[testStarted name='pass()' flowId='tst_EventLoop']
##teamcity[testFailed name='pass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='|'!std::exchange(m_inTestFunction, true)|' returned FALSE. (Earlier test failed to clean up)' flowId='tst_EventLoop']
##teamcity[testFinished name='pass()' flowId='tst_EventLoop']
##teamcity[testFailed name='pass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='|'loop.timeout()|' returned FALSE. (QTestEventLoop exited prematurely in cleanup())' flowId='tst_EventLoop']
##teamcity[testFinished name='pass()' flowId='tst_EventLoop']
##teamcity[testStarted name='cleanupTestCase()' flowId='tst_EventLoop']
##teamcity[testFinished name='cleanupTestCase()' flowId='tst_EventLoop']
##teamcity[testSuiteFinished name='tst_EventLoop' flowId='tst_EventLoop']

18
tests/auto/testlib/selftests/expected_eventloop.txt Normal file
View File

@ -0,0 +1,18 @@
********* Start testing of tst_EventLoop *********
Config: Using QtTest library
PASS : tst_EventLoop::initTestCase()
FAIL! : tst_EventLoop::fail() Failing test should still clean up
Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)]
FAIL! : tst_EventLoop::fail() 'loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())
Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)]
FAIL! : tst_EventLoop::skip() '!std::exchange(m_inTestFunction, true)' returned FALSE. (Earlier test failed to clean up)
Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)]
FAIL! : tst_EventLoop::skip() 'loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())
Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)]
FAIL! : tst_EventLoop::pass() '!std::exchange(m_inTestFunction, true)' returned FALSE. (Earlier test failed to clean up)
Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)]
FAIL! : tst_EventLoop::pass() 'loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())
Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)]
PASS : tst_EventLoop::cleanupTestCase()
Totals: 2 passed, 3 failed, 0 skipped, 0 blacklisted, 0ms
********* Finished testing of tst_EventLoop *********

44
tests/auto/testlib/selftests/expected_eventloop.xml Normal file
View File

@ -0,0 +1,44 @@
<?xml version="1.0" encoding="UTF-8"?>
<TestCase name="tst_EventLoop">
<Environment>
<QtVersion>@INSERT_QT_VERSION_HERE@</QtVersion>
<QtBuild/>
<QTestVersion>@INSERT_QT_VERSION_HERE@</QTestVersion>
</Environment>
<TestFunction name="initTestCase">
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="fail">
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
<Description><![CDATA[Failing test should still clean up]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
<Description><![CDATA['loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="skip">
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
<Description><![CDATA['!std::exchange(m_inTestFunction, true)' returned FALSE. (Earlier test failed to clean up)]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
<Description><![CDATA['loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="pass">
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
<Description><![CDATA['!std::exchange(m_inTestFunction, true)' returned FALSE. (Earlier test failed to clean up)]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
<Description><![CDATA['loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="cleanupTestCase">
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<Duration msecs="0"/>
</TestCase>

4
tests/auto/testlib/selftests/generate_expected_output.py
View File

@ -31,8 +31,8 @@ TESTS = ['assert', 'badxml', 'benchlibcallgrind', 'benchlibcounting',
'benchlibeventcounter', 'benchliboptions', 'benchlibtickcounter',
'benchlibwalltime', 'blacklisted', 'cmptest', 'commandlinedata',
'counting', 'crashes', 'datatable', 'datetime', 'deleteLater',
'deleteLater_noApp', 'differentexec', 'exceptionthrow', 'expectfail',
"extendedcompare", 'failcleanup', 'failcleanuptestcase',
'deleteLater_noApp', 'differentexec', 'eventloop', 'exceptionthrow',
'expectfail', "extendedcompare", 'failcleanup', 'failcleanuptestcase',
'faildatatype', 'failfetchtype', 'failinit', 'failinitdata',
'fetchbogus', 'findtestdata', 'float', 'globaldata', 'longstring',
'maxwarnings', 'mouse', 'multiexec', 'pairdiagnostics', 'pass',