1berry/qtbase
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add QTest::failOnWarning

Browse Source 
This solves the long-standing problem of not being able to easily
fail a test when a certain warning is output.

[ChangeLog][QtTest] Added QTest::failOnWarning. When called in a test
function, any warning that matches the given pattern will cause a test
failure. The test will continue execution when a failure is added.
All patterns are cleared at the end of each test function.

Fixes: QTBUG-70029
Change-Id: I5763f8d4acf1cee8178be43a503619fbfb0f4f36
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>
This commit is contained in:
Mitch Curtis 2021-10-08 13:58:27 +02:00
parent 1edf153a6b
commit efb283fb7f
12 changed files with 692 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
src/testlib/qtestcase.cpp
View File

@ -2127,6 +2127,68 @@ void QTest::ignoreMessage(QtMsgType type, const QRegularExpression &messagePatte
} }
#endif // QT_CONFIG(regularexpression) #endif // QT_CONFIG(regularexpression)
/*!
\since 6.3
\overload failOnWarning()
Appends a test failure to the test log if the \a message is output.
\sa failOnWarning()
*/
void QTest::failOnWarning(const char *message)
{
return QTestLog::failOnWarning(message);
}
#if QT_CONFIG(regularexpression)
/*!
\since 6.3
Appends a test failure to the test log for each warning that matches
\a messagePattern.
The test function will continue execution when a failure is added. To abort
the test instead, you can check \l currentTestFailed() and return early if
it's \c true.
For each warning, the first pattern that matches will cause a failure,
and the remaining patterns will be ignored.
All patterns are cleared at the end of each test function.
\code
void FileTest::loadFiles()
{
QTest::failOnWarning(QRegularExpression("^Failed to load"));
// Each of these will cause a test failure:
qWarning() << "Failed to load image";
qWarning() << "Failed to load video";
}
\endcode
To fail every test that triggers a given warning, pass a suitable regular
expression to this function in \l {Creating a Test}{init()}:
\code
void FileTest::init()
{
QTest::failOnWarning(QRegularExpression(".?"));
}
\endcode
\note \l ignoreMessage() takes precedence over this function, so any
warnings that match a pattern given to both \c ignoreMessage() and
\c failOnWarning() will be ignored.
\sa {Qt Test Environment Variables}{QTEST_FATAL_FAIL}
*/
void QTest::failOnWarning(const QRegularExpression &messagePattern)
{
QTestLog::failOnWarning(messagePattern);
}
#endif // QT_CONFIG(regularexpression)
/*! \internal /*! \internal
*/ */

4
src/testlib/qtestcase.h
View File

@ -331,6 +331,10 @@ namespace QTest
#if QT_CONFIG(regularexpression) #if QT_CONFIG(regularexpression)
Q_TESTLIB_EXPORT void ignoreMessage(QtMsgType type, const QRegularExpression &messagePattern); Q_TESTLIB_EXPORT void ignoreMessage(QtMsgType type, const QRegularExpression &messagePattern);
#endif #endif
Q_TESTLIB_EXPORT void failOnWarning(const char *message);
#if QT_CONFIG(regularexpression)
Q_TESTLIB_EXPORT void failOnWarning(const QRegularExpression &messagePattern);
#endif
#if QT_CONFIG(temporaryfile) #if QT_CONFIG(temporaryfile)
Q_TESTLIB_EXPORT QSharedPointer<QTemporaryDir> qExtractTestData(const QString &dirName); Q_TESTLIB_EXPORT QSharedPointer<QTemporaryDir> qExtractTestData(const QString &dirName);

51
src/testlib/qtestlog.cpp
View File

@ -68,6 +68,7 @@
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include <limits.h> #include <limits.h>
#include <vector>
#include <vector> #include <vector>
#include <memory> #include <memory>
@ -171,6 +172,8 @@ namespace QTest {
static IgnoreResultList *ignoreResultList = nullptr; static IgnoreResultList *ignoreResultList = nullptr;
static std::vector<QVariant> failOnWarningList;
Q_GLOBAL_STATIC(std::vector<std::unique_ptr<QAbstractTestLogger>>, loggers) Q_GLOBAL_STATIC(std::vector<std::unique_ptr<QAbstractTestLogger>>, loggers)
static int verbosity = 0; static int verbosity = 0;
@ -205,6 +208,32 @@ namespace QTest {
return false; return false;
} }
static bool handleFailOnWarning(const QMessageLogContext &context, const QString &message)
{
// failOnWarnings can be called multiple times per test function, so let
// each call cause a failure if required.
for (const auto &pattern : failOnWarningList) {
if (pattern.metaType() == QMetaType::fromType<QString>()) {
if (message != pattern.toString())
continue;
}
#if QT_CONFIG(regularexpression)
else if (pattern.metaType() == QMetaType::fromType<QRegularExpression>()) {
if (!message.contains(pattern.toRegularExpression()))
continue;
}
#endif
const size_t maxMsgLen = 1024;
char msg[maxMsgLen] = {'\0'};
qsnprintf(msg, maxMsgLen, "Received a warning that resulted in a failure:\n%s",
qPrintable(message));
QTestResult::addFailure(msg, context.file, context.line);
return true;
}
return false;
}
static void messageHandler(QtMsgType type, const QMessageLogContext & context, const QString &message) static void messageHandler(QtMsgType type, const QMessageLogContext & context, const QString &message)
{ {
static QBasicAtomicInt counter = Q_BASIC_ATOMIC_INITIALIZER(QTest::maxWarnings); static QBasicAtomicInt counter = Q_BASIC_ATOMIC_INITIALIZER(QTest::maxWarnings);
@ -220,6 +249,9 @@ namespace QTest {
return; return;
} }
if (type == QtWarningMsg && handleFailOnWarning(context, message))
return;
if (type != QtFatalMsg) { if (type != QtFatalMsg) {
if (counter.loadRelaxed() <= 0) if (counter.loadRelaxed() <= 0)
return; return;
@ -313,6 +345,11 @@ void QTestLog::clearIgnoreMessages()
QTest::IgnoreResultList::clearList(QTest::ignoreResultList); QTest::IgnoreResultList::clearList(QTest::ignoreResultList);
} }
void QTestLog::clearFailOnWarnings()
{
QTest::failOnWarningList.clear();
}
void QTestLog::addPass(const char *msg) void QTestLog::addPass(const char *msg)
{ {
if (printAvailableTags) if (printAvailableTags)
@ -546,6 +583,20 @@ void QTestLog::ignoreMessage(QtMsgType type, const QRegularExpression &expressio
} }
#endif // QT_CONFIG(regularexpression) #endif // QT_CONFIG(regularexpression)
void QTestLog::failOnWarning(const char *msg)
{
QTest::failOnWarningList.push_back(QString::fromUtf8(msg));
}
#if QT_CONFIG(regularexpression)
void QTestLog::failOnWarning(const QRegularExpression &expression)
{
QTEST_ASSERT(expression.isValid());
QTest::failOnWarningList.push_back(QVariant::fromValue(expression));
}
#endif // QT_CONFIG(regularexpression)
void QTestLog::setMaxWarnings(int m) void QTestLog::setMaxWarnings(int m)
{ {
QTest::maxWarnings = m <= 0 ? INT_MAX : m + 2; QTest::maxWarnings = m <= 0 ? INT_MAX : m + 2;

5
src/testlib/qtestlog_p.h
View File

@ -104,10 +104,15 @@ public:
static void ignoreMessage(QtMsgType type, const char *msg); static void ignoreMessage(QtMsgType type, const char *msg);
#ifndef QT_NO_REGULAREXPRESSION #ifndef QT_NO_REGULAREXPRESSION
static void ignoreMessage(QtMsgType type, const QRegularExpression &expression); static void ignoreMessage(QtMsgType type, const QRegularExpression &expression);
#endif
static void failOnWarning(const char *msg);
#ifndef QT_NO_REGULAREXPRESSION
static void failOnWarning(const QRegularExpression &expression);
#endif #endif
static int unhandledIgnoreMessages(); static int unhandledIgnoreMessages();
static void printUnhandledIgnoreMessages(); static void printUnhandledIgnoreMessages();
static void clearIgnoreMessages(); static void clearIgnoreMessages();
static void clearFailOnWarnings();
static void warn(const char *msg, const char *file, int line); static void warn(const char *msg, const char *file, int line);
static void info(const char *msg, const char *file, int line); static void info(const char *msg, const char *file, int line);

1
src/testlib/qtestresult.cpp
View File

@ -191,6 +191,7 @@ void QTestResult::finishedCurrentTestData()
addFailure("Not all expected messages were received"); addFailure("Not all expected messages were received");
} }
QTestLog::clearIgnoreMessages(); QTestLog::clearIgnoreMessages();
QTestLog::clearFailOnWarnings();
} }
/*! /*!

60
tests/auto/testlib/selftests/expected_warnings.junitxml
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8" ?> <?xml version="1.0" encoding="UTF-8" ?>
<testsuite name="tst_Warnings" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="7" failures="4" errors="0" skipped="0" time="@TEST_DURATION@"> <testsuite name="tst_Warnings" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="15" failures="10" errors="0" skipped="1" time="@TEST_DURATION@">
<properties> <properties>
<property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/> <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
<property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/> <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
@ -49,5 +49,63 @@
<![CDATA[Did not receive message: "Warning1"]]> <![CDATA[Did not receive message: "Warning1"]]>
</system-out> </system-out>
</testcase> </testcase>
<testcase name="testFailOnWarnings" classname="tst_Warnings" time="@TEST_DURATION@">
<failure type="fail" message="Received a warning that resulted in a failure:">
<![CDATA[Ran out of cabbage!]]>
</failure>
<system-out>
<![CDATA[Ran out of tortillas!]]>
<![CDATA[Ran out of oil!]]>
<![CDATA[Running low on toothpaste!]]>
<![CDATA[Running low on toothpaste!]]>
</system-out>
<system-err>
<![CDATA[Ran out of space!]]>
<![CDATA[nope]]>
<![CDATA[Running low on flour!]]>
</system-err>
</testcase>
<testcase name="testFailOnWarningsCleared" classname="tst_Warnings" time="@TEST_DURATION@">
<system-err>
<![CDATA[Ran out of muffins!]]>
</system-err>
</testcase>
<testcase name="testFailOnWarningsWithData(warning1)" classname="tst_Warnings" time="@TEST_DURATION@">
<failure type="fail" message="Received a warning that resulted in a failure:">
<![CDATA[warning1]]>
</failure>
<system-err>
<![CDATA[warning2]]>
<![CDATA[warning3]]>
</system-err>
</testcase>
<testcase name="testFailOnWarningsWithData(warning2)" classname="tst_Warnings" time="@TEST_DURATION@">
<failure type="fail" message="Received a warning that resulted in a failure:">
<![CDATA[warning2]]>
</failure>
<system-err>
<![CDATA[warning1]]>
<![CDATA[warning3]]>
</system-err>
</testcase>
<testcase name="testFailOnWarningsWithData(warning3)" classname="tst_Warnings" time="@TEST_DURATION@">
<failure type="fail" message="Received a warning that resulted in a failure:">
<![CDATA[warning3]]>
</failure>
<system-err>
<![CDATA[warning1]]>
<![CDATA[warning2]]>
</system-err>
</testcase>
<testcase name="testFailOnWarningsFailInHelper" classname="tst_Warnings" time="@TEST_DURATION@">
<failure type="fail" message="This failure message should be printed but not cause the test to abort"/>
</testcase>
<testcase name="testFailOnWarningsThenSkip" classname="tst_Warnings" time="@TEST_DURATION@">
<failure type="fail" message="Received a warning that resulted in a failure:">
<![CDATA[Ran out of cabbage!]]>
</failure>
<skipped message="My cabbage! :("/>
</testcase>
<testcase name="testFailOnWarningsAndIgnoreWarnings" classname="tst_Warnings" time="@TEST_DURATION@"/>
<testcase name="cleanupTestCase" classname="tst_Warnings" time="@TEST_DURATION@"/> <testcase name="cleanupTestCase" classname="tst_Warnings" time="@TEST_DURATION@"/>
</testsuite> </testsuite>

112
tests/auto/testlib/selftests/expected_warnings.lightxml
View File

@ -89,6 +89,118 @@
</Incident> </Incident>
<Duration msecs="0"/> <Duration msecs="0"/>
</TestFunction> </TestFunction>
<TestFunction name="testFailOnWarnings">
<Message type="qwarn" file="" line="0">
<Description><![CDATA[Ran out of space!]]></Description>
</Message>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[Received a warning that resulted in a failure:
Ran out of cabbage!]]></Description>
</Incident>
<Message type="qdebug" file="" line="0">
<Description><![CDATA[Ran out of tortillas!]]></Description>
</Message>
<Message type="qinfo" file="" line="0">
<Description><![CDATA[Ran out of oil!]]></Description>
</Message>
<Message type="qwarn" file="" line="0">
<Description><![CDATA[nope]]></Description>
</Message>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[Received a warning that resulted in a failure:
Ran out of biscuits!]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[Received a warning that resulted in a failure:
Running low on toothpaste!]]></Description>
</Incident>
<Message type="qwarn" file="" line="0">
<Description><![CDATA[Running low on flour!]]></Description>
</Message>
<Message type="qdebug" file="" line="0">
<Description><![CDATA[Running low on toothpaste!]]></Description>
</Message>
<Message type="qinfo" file="" line="0">
<Description><![CDATA[Running low on toothpaste!]]></Description>
</Message>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFailOnWarningsCleared">
<Message type="qwarn" file="" line="0">
<Description><![CDATA[Ran out of muffins!]]></Description>
</Message>
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFailOnWarningsWithData">
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<DataTag><![CDATA[warning1]]></DataTag>
<Description><![CDATA[Received a warning that resulted in a failure:
warning1]]></Description>
</Incident>
<Message type="qwarn" file="" line="0">
<DataTag><![CDATA[warning1]]></DataTag>
<Description><![CDATA[warning2]]></Description>
</Message>
<Message type="qwarn" file="" line="0">
<DataTag><![CDATA[warning1]]></DataTag>
<Description><![CDATA[warning3]]></Description>
</Message>
<Message type="qwarn" file="" line="0">
<DataTag><![CDATA[warning2]]></DataTag>
<Description><![CDATA[warning1]]></Description>
</Message>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<DataTag><![CDATA[warning2]]></DataTag>
<Description><![CDATA[Received a warning that resulted in a failure:
warning2]]></Description>
</Incident>
<Message type="qwarn" file="" line="0">
<DataTag><![CDATA[warning2]]></DataTag>
<Description><![CDATA[warning3]]></Description>
</Message>
<Message type="qwarn" file="" line="0">
<DataTag><![CDATA[warning3]]></DataTag>
<Description><![CDATA[warning1]]></Description>
</Message>
<Message type="qwarn" file="" line="0">
<DataTag><![CDATA[warning3]]></DataTag>
<Description><![CDATA[warning2]]></Description>
</Message>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<DataTag><![CDATA[warning3]]></DataTag>
<Description><![CDATA[Received a warning that resulted in a failure:
warning3]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFailOnWarningsFailInHelper">
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[This failure message should be printed but not cause the test to abort]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[Received a warning that resulted in a failure:
Ran out of cabbage!]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[My cabbage! :(]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFailOnWarningsThenSkip">
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[Received a warning that resulted in a failure:
Ran out of cabbage!]]></Description>
</Incident>
<Message type="skip" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[My cabbage! :(]]></Description>
</Message>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFailOnWarningsAndIgnoreWarnings">
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="cleanupTestCase"> <TestFunction name="cleanupTestCase">
<Incident type="pass" file="" line="0" /> <Incident type="pass" file="" line="0" />
<Duration msecs="0"/> <Duration msecs="0"/>

105
tests/auto/testlib/selftests/expected_warnings.tap
View File

@ -35,8 +35,103 @@ not ok 6 - testMissingWarningsWithData(second row)
--- ---
# Not all expected messages were received # Not all expected messages were received
... ...
ok 7 - cleanupTestCase() # Ran out of space!
1..7 not ok 7 - testFailOnWarnings()
# tests 7 ---
# pass 3 # Received a warning that resulted in a failure:
# fail 4 Ran out of cabbage!
at: tst_Warnings::testFailOnWarnings() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
line: 0
...
# Ran out of tortillas!
# Ran out of oil!
# nope
not ok 8 - testFailOnWarnings()
---
# Received a warning that resulted in a failure:
Ran out of biscuits!
at: tst_Warnings::testFailOnWarnings() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
line: 0
...
not ok 9 - testFailOnWarnings()
---
# Received a warning that resulted in a failure:
Running low on toothpaste!
at: tst_Warnings::testFailOnWarnings() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
line: 0
...
# Running low on flour!
# Running low on toothpaste!
# Running low on toothpaste!
# Ran out of muffins!
ok 10 - testFailOnWarningsCleared()
not ok 11 - testFailOnWarningsWithData(warning1)
---
# Received a warning that resulted in a failure:
warning1
at: tst_Warnings::testFailOnWarningsWithData() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
line: 0
...
# warning2
# warning3
# warning1
not ok 12 - testFailOnWarningsWithData(warning2)
---
# Received a warning that resulted in a failure:
warning2
at: tst_Warnings::testFailOnWarningsWithData() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
line: 0
...
# warning3
# warning1
# warning2
not ok 13 - testFailOnWarningsWithData(warning3)
---
# Received a warning that resulted in a failure:
warning3
at: tst_Warnings::testFailOnWarningsWithData() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
line: 0
...
not ok 14 - testFailOnWarningsFailInHelper()
---
# This failure message should be printed but not cause the test to abort
at: tst_Warnings::testFailOnWarningsFailInHelper() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
line: 0
...
not ok 15 - testFailOnWarningsFailInHelper()
---
# Received a warning that resulted in a failure:
Ran out of cabbage!
at: tst_Warnings::testFailOnWarningsFailInHelper() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
line: 0
...
not ok 16 - testFailOnWarningsFailInHelper()
---
# My cabbage! :(
at: tst_Warnings::testFailOnWarningsFailInHelper() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
line: 0
...
not ok 17 - testFailOnWarningsThenSkip()
---
# Received a warning that resulted in a failure:
Ran out of cabbage!
at: tst_Warnings::testFailOnWarningsThenSkip() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
line: 0
...
ok 18 - testFailOnWarningsThenSkip() # SKIP My cabbage! :(
ok 19 - testFailOnWarningsAndIgnoreWarnings()
ok 20 - cleanupTestCase()
1..20
# tests 20
# pass 5
# fail 14

36
tests/auto/testlib/selftests/expected_warnings.teamcity
View File

@ -20,6 +20,42 @@
##teamcity[testFailed name='testMissingWarningsWithData(second row)' message='Failure!' details='Not all expected messages were received' flowId='tst_Warnings'] ##teamcity[testFailed name='testMissingWarningsWithData(second row)' message='Failure!' details='Not all expected messages were received' flowId='tst_Warnings']
##teamcity[testStdOut name='testMissingWarningsWithData(second row)' out='INFO: Did not receive message: "Warning0"|nINFO: Did not receive message: "Warning1"' flowId='tst_Warnings'] ##teamcity[testStdOut name='testMissingWarningsWithData(second row)' out='INFO: Did not receive message: "Warning0"|nINFO: Did not receive message: "Warning1"' flowId='tst_Warnings']
##teamcity[testFinished name='testMissingWarningsWithData(second row)' flowId='tst_Warnings'] ##teamcity[testFinished name='testMissingWarningsWithData(second row)' flowId='tst_Warnings']
##teamcity[testStarted name='testFailOnWarnings()' flowId='tst_Warnings']
##teamcity[testFailed name='testFailOnWarnings()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nRan out of cabbage!' flowId='tst_Warnings']
##teamcity[testStdOut name='testFailOnWarnings()' out='QWARN: Ran out of space!' flowId='tst_Warnings']
##teamcity[testFinished name='testFailOnWarnings()' flowId='tst_Warnings']
##teamcity[testFailed name='testFailOnWarnings()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nRan out of biscuits!' flowId='tst_Warnings']
##teamcity[testStdOut name='testFailOnWarnings()' out='QDEBUG: Ran out of tortillas!|nQINFO: Ran out of oil!|nQWARN: nope' flowId='tst_Warnings']
##teamcity[testFinished name='testFailOnWarnings()' flowId='tst_Warnings']
##teamcity[testFailed name='testFailOnWarnings()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nRunning low on toothpaste!' flowId='tst_Warnings']
##teamcity[testFinished name='testFailOnWarnings()' flowId='tst_Warnings']
##teamcity[testStarted name='testFailOnWarningsCleared()' flowId='tst_Warnings']
##teamcity[testStdOut name='testFailOnWarningsCleared()' out='QWARN: Running low on flour!|nQDEBUG: Running low on toothpaste!|nQINFO: Running low on toothpaste!|nQWARN: Ran out of muffins!' flowId='tst_Warnings']
##teamcity[testFinished name='testFailOnWarningsCleared()' flowId='tst_Warnings']
##teamcity[testStarted name='testFailOnWarningsWithData(warning1)' flowId='tst_Warnings']
##teamcity[testFailed name='testFailOnWarningsWithData(warning1)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nwarning1' flowId='tst_Warnings']
##teamcity[testFinished name='testFailOnWarningsWithData(warning1)' flowId='tst_Warnings']
##teamcity[testStarted name='testFailOnWarningsWithData(warning2)' flowId='tst_Warnings']
##teamcity[testFailed name='testFailOnWarningsWithData(warning2)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nwarning2' flowId='tst_Warnings']
##teamcity[testStdOut name='testFailOnWarningsWithData(warning2)' out='QWARN: warning2|nQWARN: warning3|nQWARN: warning1' flowId='tst_Warnings']
##teamcity[testFinished name='testFailOnWarningsWithData(warning2)' flowId='tst_Warnings']
##teamcity[testStarted name='testFailOnWarningsWithData(warning3)' flowId='tst_Warnings']
##teamcity[testFailed name='testFailOnWarningsWithData(warning3)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nwarning3' flowId='tst_Warnings']
##teamcity[testStdOut name='testFailOnWarningsWithData(warning3)' out='QWARN: warning3|nQWARN: warning1|nQWARN: warning2' flowId='tst_Warnings']
##teamcity[testFinished name='testFailOnWarningsWithData(warning3)' flowId='tst_Warnings']
##teamcity[testStarted name='testFailOnWarningsFailInHelper()' flowId='tst_Warnings']
##teamcity[testFailed name='testFailOnWarningsFailInHelper()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='This failure message should be printed but not cause the test to abort' flowId='tst_Warnings']
##teamcity[testFinished name='testFailOnWarningsFailInHelper()' flowId='tst_Warnings']
##teamcity[testFailed name='testFailOnWarningsFailInHelper()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nRan out of cabbage!' flowId='tst_Warnings']
##teamcity[testFinished name='testFailOnWarningsFailInHelper()' flowId='tst_Warnings']
##teamcity[testFailed name='testFailOnWarningsFailInHelper()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='My cabbage! :(' flowId='tst_Warnings']
##teamcity[testFinished name='testFailOnWarningsFailInHelper()' flowId='tst_Warnings']
##teamcity[testStarted name='testFailOnWarningsThenSkip()' flowId='tst_Warnings']
##teamcity[testFailed name='testFailOnWarningsThenSkip()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nRan out of cabbage!' flowId='tst_Warnings']
##teamcity[testFinished name='testFailOnWarningsThenSkip()' flowId='tst_Warnings']
##teamcity[testIgnored name='testFailOnWarningsThenSkip()' message='My cabbage! :( |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' flowId='tst_Warnings']
##teamcity[testStarted name='testFailOnWarningsAndIgnoreWarnings()' flowId='tst_Warnings']
##teamcity[testFinished name='testFailOnWarningsAndIgnoreWarnings()' flowId='tst_Warnings']
##teamcity[testStarted name='cleanupTestCase()' flowId='tst_Warnings'] ##teamcity[testStarted name='cleanupTestCase()' flowId='tst_Warnings']
##teamcity[testFinished name='cleanupTestCase()' flowId='tst_Warnings'] ##teamcity[testFinished name='cleanupTestCase()' flowId='tst_Warnings']
##teamcity[testSuiteFinished name='tst_Warnings' flowId='tst_Warnings'] ##teamcity[testSuiteFinished name='tst_Warnings' flowId='tst_Warnings']

48
tests/auto/testlib/selftests/expected_warnings.txt
View File

@ -23,6 +23,52 @@ FAIL! : tst_Warnings::testMissingWarningsWithData(first row) Not all expected m
INFO : tst_Warnings::testMissingWarningsWithData(second row) Did not receive message: "Warning0" INFO : tst_Warnings::testMissingWarningsWithData(second row) Did not receive message: "Warning0"
INFO : tst_Warnings::testMissingWarningsWithData(second row) Did not receive message: "Warning1" INFO : tst_Warnings::testMissingWarningsWithData(second row) Did not receive message: "Warning1"
FAIL! : tst_Warnings::testMissingWarningsWithData(second row) Not all expected messages were received FAIL! : tst_Warnings::testMissingWarningsWithData(second row) Not all expected messages were received
QWARN : tst_Warnings::testFailOnWarnings() Ran out of space!
FAIL! : tst_Warnings::testFailOnWarnings() Received a warning that resulted in a failure:
Ran out of cabbage!
Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
QDEBUG : tst_Warnings::testFailOnWarnings() Ran out of tortillas!
QINFO : tst_Warnings::testFailOnWarnings() Ran out of oil!
QWARN : tst_Warnings::testFailOnWarnings() nope
FAIL! : tst_Warnings::testFailOnWarnings() Received a warning that resulted in a failure:
Ran out of biscuits!
Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
FAIL! : tst_Warnings::testFailOnWarnings() Received a warning that resulted in a failure:
Running low on toothpaste!
Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
QWARN : tst_Warnings::testFailOnWarnings() Running low on flour!
QDEBUG : tst_Warnings::testFailOnWarnings() Running low on toothpaste!
QINFO : tst_Warnings::testFailOnWarnings() Running low on toothpaste!
QWARN : tst_Warnings::testFailOnWarningsCleared() Ran out of muffins!
PASS : tst_Warnings::testFailOnWarningsCleared()
FAIL! : tst_Warnings::testFailOnWarningsWithData(warning1) Received a warning that resulted in a failure:
warning1
Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
QWARN : tst_Warnings::testFailOnWarningsWithData(warning1) warning2
QWARN : tst_Warnings::testFailOnWarningsWithData(warning1) warning3
QWARN : tst_Warnings::testFailOnWarningsWithData(warning2) warning1
FAIL! : tst_Warnings::testFailOnWarningsWithData(warning2) Received a warning that resulted in a failure:
warning2
Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
QWARN : tst_Warnings::testFailOnWarningsWithData(warning2) warning3
QWARN : tst_Warnings::testFailOnWarningsWithData(warning3) warning1
QWARN : tst_Warnings::testFailOnWarningsWithData(warning3) warning2
FAIL! : tst_Warnings::testFailOnWarningsWithData(warning3) Received a warning that resulted in a failure:
warning3
Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
FAIL! : tst_Warnings::testFailOnWarningsFailInHelper() This failure message should be printed but not cause the test to abort
Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
FAIL! : tst_Warnings::testFailOnWarningsFailInHelper() Received a warning that resulted in a failure:
Ran out of cabbage!
Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
FAIL! : tst_Warnings::testFailOnWarningsFailInHelper() My cabbage! :(
Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
FAIL! : tst_Warnings::testFailOnWarningsThenSkip() Received a warning that resulted in a failure:
Ran out of cabbage!
Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
SKIP : tst_Warnings::testFailOnWarningsThenSkip() My cabbage! :(
Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
PASS : tst_Warnings::testFailOnWarningsAndIgnoreWarnings()
PASS : tst_Warnings::cleanupTestCase() PASS : tst_Warnings::cleanupTestCase()
Totals: 3 passed, 4 failed, 0 skipped, 0 blacklisted, 0ms Totals: 5 passed, 14 failed, 1 skipped, 0 blacklisted, 0ms
********* Finished testing of tst_Warnings ********* ********* Finished testing of tst_Warnings *********

112
tests/auto/testlib/selftests/expected_warnings.xml
View File

@ -91,6 +91,118 @@
</Incident> </Incident>
<Duration msecs="0"/> <Duration msecs="0"/>
</TestFunction> </TestFunction>
<TestFunction name="testFailOnWarnings">
<Message type="qwarn" file="" line="0">
<Description><![CDATA[Ran out of space!]]></Description>
</Message>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[Received a warning that resulted in a failure:
Ran out of cabbage!]]></Description>
</Incident>
<Message type="qdebug" file="" line="0">
<Description><![CDATA[Ran out of tortillas!]]></Description>
</Message>
<Message type="qinfo" file="" line="0">
<Description><![CDATA[Ran out of oil!]]></Description>
</Message>
<Message type="qwarn" file="" line="0">
<Description><![CDATA[nope]]></Description>
</Message>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[Received a warning that resulted in a failure:
Ran out of biscuits!]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[Received a warning that resulted in a failure:
Running low on toothpaste!]]></Description>
</Incident>
<Message type="qwarn" file="" line="0">
<Description><![CDATA[Running low on flour!]]></Description>
</Message>
<Message type="qdebug" file="" line="0">
<Description><![CDATA[Running low on toothpaste!]]></Description>
</Message>
<Message type="qinfo" file="" line="0">
<Description><![CDATA[Running low on toothpaste!]]></Description>
</Message>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFailOnWarningsCleared">
<Message type="qwarn" file="" line="0">
<Description><![CDATA[Ran out of muffins!]]></Description>
</Message>
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFailOnWarningsWithData">
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<DataTag><![CDATA[warning1]]></DataTag>
<Description><![CDATA[Received a warning that resulted in a failure:
warning1]]></Description>
</Incident>
<Message type="qwarn" file="" line="0">
<DataTag><![CDATA[warning1]]></DataTag>
<Description><![CDATA[warning2]]></Description>
</Message>
<Message type="qwarn" file="" line="0">
<DataTag><![CDATA[warning1]]></DataTag>
<Description><![CDATA[warning3]]></Description>
</Message>
<Message type="qwarn" file="" line="0">
<DataTag><![CDATA[warning2]]></DataTag>
<Description><![CDATA[warning1]]></Description>
</Message>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<DataTag><![CDATA[warning2]]></DataTag>
<Description><![CDATA[Received a warning that resulted in a failure:
warning2]]></Description>
</Incident>
<Message type="qwarn" file="" line="0">
<DataTag><![CDATA[warning2]]></DataTag>
<Description><![CDATA[warning3]]></Description>
</Message>
<Message type="qwarn" file="" line="0">
<DataTag><![CDATA[warning3]]></DataTag>
<Description><![CDATA[warning1]]></Description>
</Message>
<Message type="qwarn" file="" line="0">
<DataTag><![CDATA[warning3]]></DataTag>
<Description><![CDATA[warning2]]></Description>
</Message>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<DataTag><![CDATA[warning3]]></DataTag>
<Description><![CDATA[Received a warning that resulted in a failure:
warning3]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFailOnWarningsFailInHelper">
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[This failure message should be printed but not cause the test to abort]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[Received a warning that resulted in a failure:
Ran out of cabbage!]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[My cabbage! :(]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFailOnWarningsThenSkip">
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[Received a warning that resulted in a failure:
Ran out of cabbage!]]></Description>
</Incident>
<Message type="skip" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[My cabbage! :(]]></Description>
</Message>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFailOnWarningsAndIgnoreWarnings">
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="cleanupTestCase"> <TestFunction name="cleanupTestCase">
<Incident type="pass" file="" line="0" /> <Incident type="pass" file="" line="0" />
<Duration msecs="0"/> <Duration msecs="0"/>

103
tests/auto/testlib/selftests/warnings/tst_warnings.cpp
View File

@ -40,6 +40,16 @@ private slots:
void testMissingWarningsRegularExpression(); void testMissingWarningsRegularExpression();
void testMissingWarningsWithData_data(); void testMissingWarningsWithData_data();
void testMissingWarningsWithData(); void testMissingWarningsWithData();
void testFailOnWarnings();
void testFailOnWarningsCleared();
#if QT_CONFIG(regularexpression)
void testFailOnWarningsWithData_data();
void testFailOnWarningsWithData();
void testFailOnWarningsFailInHelper();
void testFailOnWarningsThenSkip();
#endif
void testFailOnWarningsAndIgnoreWarnings();
}; };
void tst_Warnings::testWarnings() void tst_Warnings::testWarnings()
@ -124,6 +134,99 @@ void tst_Warnings::testMissingWarningsWithData()
qWarning("Warning2"); qWarning("Warning2");
} }
void tst_Warnings::testFailOnWarnings()
{
// failOnWarnings() wasn't called yet; shouldn't fail;
qWarning("Ran out of space!");
#if QT_CONFIG(regularexpression)
const auto warnRegex = QRegularExpression("Ran out of .*!");
QTest::failOnWarning(warnRegex);
// Should now fail.
qWarning("Ran out of cabbage!");
// Should not fail; none of these are warnings.
qDebug("Ran out of tortillas!");
qInfo("Ran out of oil!");
// Should not fail; regex doesn't match.
qWarning("nope");
// Should fail; matches regex.
qWarning("Ran out of biscuits!");
#endif // QT_CONFIG(regularexpression)
QTest::failOnWarning("Running low on toothpaste!");
// Should fail; strings match.
qWarning("Running low on toothpaste!");
// Shouldn't fail; strings don't match.
qWarning("Running low on flour!");
// Should not fail; none of these are warnings.
qDebug("Running low on toothpaste!");
qInfo("Running low on toothpaste!");
}
void tst_Warnings::testFailOnWarningsCleared()
{
// The patterns passed to failOnWarnings() should be cleared at the end of
// each test function, so this shouldn't fail because of the failOnWarning() call in the previous function.
// Note that this test always needs to come after testFailOnWarnings for it to work.
qWarning("Ran out of muffins!");
}
#if QT_CONFIG(regularexpression)
void tst_Warnings::testFailOnWarningsWithData_data()
{
// The warning message that should cause a failure.
QTest::addColumn<QString>("warningMessage");
QTest::newRow("warning1") << "warning1";
QTest::newRow("warning2") << "warning2";
QTest::newRow("warning3") << "warning3";
}
void tst_Warnings::testFailOnWarningsWithData()
{
QFETCH(QString, warningMessage);
QTest::failOnWarning(QRegularExpression(warningMessage));
// Only one of these should fail, depending on warningMessage.
qWarning("warning1");
qWarning("warning2");
qWarning("warning3");
}
void tst_Warnings::testFailOnWarningsFailInHelper()
{
[](){ QFAIL("This failure message should be printed but not cause the test to abort"); }();
const auto warnRegex = QRegularExpression("Ran out of .*!");
QTest::failOnWarning(warnRegex);
qWarning("Ran out of cabbage!");
QFAIL("My cabbage! :(");
}
void tst_Warnings::testFailOnWarningsThenSkip()
{
const auto warnRegex = QRegularExpression("Ran out of .*!");
QTest::failOnWarning(warnRegex);
qWarning("Ran out of cabbage!");
QSKIP("My cabbage! :(");
}
#endif // QT_CONFIG(regularexpression)
void tst_Warnings::testFailOnWarningsAndIgnoreWarnings()
{
const auto warningStr = "Running low on toothpaste!";
QTest::failOnWarning(warningStr);
QTest::ignoreMessage(QtWarningMsg, warningStr);
// Shouldn't fail; we ignored it.
qWarning(warningStr);
}
QTEST_MAIN(tst_Warnings) QTEST_MAIN(tst_Warnings)
#include "tst_warnings.moc" #include "tst_warnings.moc"