QTestlib/JUnit XML: Log most messages to standard output instead of error output

Add an enumeration for system-out and alog element
for it. Redirect the messages types that are not warnings/errors
to this element. For compatibility, write it out only
if it is not empty. Rename enumerations and members accordingly.

[ChangeLog][QtTestLib] In JUnit XML, output that is
not a warning/error is now logged under <system-out>
instead of <system-err>.

Fixes: QTBUG-86540
Change-Id: I55598eafa7dafa486ac5a8221029c332ff47413b
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>
This commit is contained in:
Friedemann Kleint 2021-01-13 11:21:01 +01:00
parent 7f5d41e286
commit 539553a572
28 changed files with 133 additions and 79 deletions

View File

@ -74,8 +74,10 @@ void QJUnitTestLogger::startLogging()
QAbstractTestLogger::startLogging();
logFormatter = new QTestJUnitStreamer(this);
delete errorLogElement;
errorLogElement = new QTestElement(QTest::LET_SystemError);
delete systemOutputElement;
systemOutputElement = new QTestElement(QTest::LET_SystemOutput);
delete systemErrorElement;
systemErrorElement = new QTestElement(QTest::LET_SystemError);
Q_ASSERT(!currentTestSuite);
currentTestSuite = new QTestElement(QTest::LET_TestSuite);
@ -132,7 +134,9 @@ void QJUnitTestLogger::stopLogging()
testcase = testcase->nextElement();
}
currentTestSuite->addLogElement(errorLogElement);
if (systemOutputElement->childElements())
currentTestSuite->addLogElement(systemOutputElement);
currentTestSuite->addLogElement(systemErrorElement);
logFormatter->output(currentTestSuite);
@ -305,11 +309,13 @@ void QJUnitTestLogger::addTag(QTestElement* element)
void QJUnitTestLogger::addMessage(MessageTypes type, const QString &message, const char *file, int line)
{
QTestElement *errorElement = new QTestElement(QTest::LET_Error);
auto messageElement = new QTestElement(QTest::LET_Message);
auto systemLogElement = systemOutputElement;
const char *typeBuf = nullptr;
switch (type) {
case QAbstractTestLogger::Warn:
systemLogElement = systemErrorElement;
typeBuf = "warn";
break;
case QAbstractTestLogger::QSystem:
@ -322,9 +328,11 @@ void QJUnitTestLogger::addMessage(MessageTypes type, const QString &message, con
typeBuf = "qinfo";
break;
case QAbstractTestLogger::QWarning:
systemLogElement = systemErrorElement;
typeBuf = "qwarn";
break;
case QAbstractTestLogger::QFatal:
systemLogElement = systemErrorElement;
typeBuf = "qfatal";
break;
case QAbstractTestLogger::Skip:
@ -338,27 +346,27 @@ void QJUnitTestLogger::addMessage(MessageTypes type, const QString &message, con
break;
}
errorElement->addAttribute(QTest::AI_Type, typeBuf);
errorElement->addAttribute(QTest::AI_Description, message.toUtf8().constData());
addTag(errorElement);
messageElement->addAttribute(QTest::AI_Type, typeBuf);
messageElement->addAttribute(QTest::AI_Description, message.toUtf8().constData());
addTag(messageElement);
if (file)
errorElement->addAttribute(QTest::AI_File, file);
messageElement->addAttribute(QTest::AI_File, file);
else
errorElement->addAttribute(QTest::AI_File, "");
messageElement->addAttribute(QTest::AI_File, "");
char buf[100];
qsnprintf(buf, sizeof(buf), "%i", line);
errorElement->addAttribute(QTest::AI_Line, buf);
messageElement->addAttribute(QTest::AI_Line, buf);
currentLogElement->addLogElement(errorElement);
currentLogElement->addLogElement(messageElement);
++errorCounter;
// Also add the message to the system error log (i.e. stderr), if one exists
if (errorLogElement) {
QTestElement *systemErrorElement = new QTestElement(QTest::LET_Error);
systemErrorElement->addAttribute(QTest::AI_Description, message.toUtf8().constData());
errorLogElement->addLogElement(systemErrorElement);
// Also add the message to the system log (stdout/stderr), if one exists
if (systemLogElement) {
auto messageElement = new QTestElement(QTest::LET_Message);
messageElement->addAttribute(QTest::AI_Description, message.toUtf8().constData());
systemLogElement->addLogElement(messageElement);
}
}

View File

@ -82,7 +82,8 @@ class QJUnitTestLogger : public QAbstractTestLogger
QTestElement *currentTestSuite = nullptr;
QTestElement *listOfTestcases = nullptr;
QTestElement *currentLogElement = nullptr;
QTestElement *errorLogElement = nullptr;
QTestElement *systemOutputElement = nullptr;
QTestElement *systemErrorElement = nullptr;
QTestJUnitStreamer *logFormatter = nullptr;
int testCounter = 0;

View File

@ -139,7 +139,8 @@ const char *QTestCoreElement<ElementType>::elementName() const
"testcase",
"testsuite",
"benchmark",
"system-err"
"system-err",
"system-out"
};
if (type != QTest::LET_Undefined)

View File

@ -87,11 +87,12 @@ namespace QTest {
LET_Property = 0,
LET_Properties = 1,
LET_Failure = 2,
LET_Error = 3,
LET_Message = 3,
LET_TestCase = 4,
LET_TestSuite = 5,
LET_Benchmark = 6,
LET_SystemError = 7
LET_SystemError = 7,
LET_SystemOutput = 8
};
}

View File

@ -82,12 +82,17 @@ void QTestJUnitStreamer::formatStart(const QTestElement *element, QTestCharBuffe
char indent[20];
indentForElement(element, indent, sizeof(indent));
// Errors are written as CDATA within system-err, comments elsewhere
if (element->elementType() == QTest::LET_Error) {
if (element->parentElement()->elementType() == QTest::LET_SystemError) {
// Messages/errors are written as CDATA within system-out, system-err,
// respectively, comments elsewhere
if (element->elementType() == QTest::LET_Message) {
switch (element->parentElement()->elementType()) {
case QTest::LET_SystemOutput:
case QTest::LET_SystemError:
QTest::qt_asprintf(formatted, "<![CDATA[");
} else {
break;
default:
QTest::qt_asprintf(formatted, "%s<!--", indent);
break;
}
return;
}
@ -118,9 +123,11 @@ void QTestJUnitStreamer::formatAttributes(const QTestElement* element, const QTe
QTest::AttributeIndex attrindex = attribute->index();
// For errors within system-err, we only want to output `message'
if (element && element->elementType() == QTest::LET_Error
&& element->parentElement()->elementType() == QTest::LET_SystemError) {
// For messages/errors within system-out, system-err, respectively,
// we only want to output `message'
if (element && element->elementType() == QTest::LET_Message
&& (element->parentElement()->elementType() == QTest::LET_SystemOutput
|| element->parentElement()->elementType() == QTest::LET_SystemError)) {
if (attrindex != QTest::AI_Description) return;
@ -148,12 +155,17 @@ void QTestJUnitStreamer::formatAfterAttributes(const QTestElement *element, QTes
if (!element || !formatted )
return;
// Errors are written as CDATA within system-err, comments elsewhere
if (element->elementType() == QTest::LET_Error) {
if (element->parentElement()->elementType() == QTest::LET_SystemError) {
// Messages/errors are written as CDATA within system-out, system-err,
// respectively, comments elsewhere
if (element->elementType() == QTest::LET_Message) {
switch (element->parentElement()->elementType()) {
case QTest::LET_SystemOutput:
case QTest::LET_SystemError:
QTest::qt_asprintf(formatted, "]]>\n");
} else {
break;
default:
QTest::qt_asprintf(formatted, " -->\n");
break;
}
return;
}

View File

@ -33,7 +33,7 @@
<!-- type="qdebug" message="&quot;Ülrich Ümläut&quot;" -->
</testcase>
<testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/>
<system-err>
<system-out>
<![CDATA[a message]]>
<![CDATA[a message]]>
<![CDATA[a message]]>
@ -47,5 +47,6 @@
<![CDATA[xml close > open < tags < text]]>
<![CDATA[all > " mixed ]]]><![CDATA[]> up > " in < the ]]]><![CDATA[]> hopes < of triggering "< ]]]><![CDATA[]> bugs]]>
<![CDATA["Ülrich Ümläut"]]>
</system-err>
</system-out>
<system-err/>
</testsuite>

View File

@ -15,7 +15,8 @@
<failure result="fail" message="This is a failing benchmark"/>
</testcase>
<testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/>
<system-err>
<system-out>
<![CDATA[This is a skipping benchmark]]>
</system-err>
</system-out>
<system-err/>
</testsuite>

View File

@ -22,14 +22,16 @@
<!-- type="info" message="This is an internal testlib info message that should not appear in silent test output" -->
<!-- type="qfatal" message="This is a fatal error message that should still appear in silent test output" -->
</testcase>
<system-err>
<system-out>
<![CDATA[This test should SKIP]]>
<![CDATA[This is a warning that should not appear in silent test output]]>
<![CDATA[This is an internal testlib warning that should not appear in silent test output]]>
<![CDATA[This is a debug message that should not appear in silent test output]]>
<![CDATA[This is a critical message that should not appear in silent test output]]>
<![CDATA[This is an info message that should not appear in silent test output]]>
<![CDATA[This is an internal testlib info message that should not appear in silent test output]]>
</system-out>
<system-err>
<![CDATA[This is a warning that should not appear in silent test output]]>
<![CDATA[This is an internal testlib warning that should not appear in silent test output]]>
<![CDATA[This is a fatal error message that should still appear in silent test output]]>
</system-err>
</testsuite>

View File

@ -17,12 +17,13 @@
<!-- type="info" message="QVERIFY(test)" tag="fiveTablePasses_data1" -->
</testcase>
<testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/>
<system-err>
<system-out>
<![CDATA[QVERIFY(test)]]>
<![CDATA[QVERIFY(test)]]>
<![CDATA[QVERIFY(test)]]>
<![CDATA[QVERIFY(test)]]>
<![CDATA[QVERIFY(test)]]>
<![CDATA[QVERIFY(test)]]>
</system-err>
</system-out>
<system-err/>
</testsuite>

View File

@ -50,7 +50,7 @@
<!-- type="skip" message="Skip in cleanup()" tag="skip" -->
</testcase>
<testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/>
<system-err>
<system-out>
<![CDATA[Skipping]]>
<![CDATA[Skipping]]>
<![CDATA[Skipping]]>
@ -61,5 +61,6 @@
<![CDATA[Skip in init()]]>
<![CDATA[This test function should execute and then QSKIP in cleanup()]]>
<![CDATA[Skip in cleanup()]]>
</system-err>
</system-out>
<system-err/>
</testsuite>

View File

@ -49,7 +49,7 @@
<failure result="xpass" message="QCOMPARE(1, 1) returned TRUE unexpectedly." tag="XPass"/>
</testcase>
<testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/>
<system-err>
<system-out>
<![CDATA[begin]]>
<![CDATA[This should xfail]]>
<![CDATA[after]]>
@ -63,5 +63,6 @@
<![CDATA[This test should xfail]]>
<![CDATA[This test should xfail]]>
<![CDATA[This test should xfail]]>
</system-err>
</system-out>
<system-err/>
</testsuite>

View File

@ -11,8 +11,10 @@
<!-- type="qfatal" message="ASSERT: &quot;false&quot; in file qtbase/src/testlib/qtestdata.cpp, line 0" -->
<failure result="fail" message="Received a fatal error."/>
</testcase>
<system-err>
<system-out>
<![CDATA[expected data of type 'QString', got 'bool' for element 0 of data with tag 'bool-as-string']]>
</system-out>
<system-err>
<![CDATA[ASSERT: "false" in file qtbase/src/testlib/qtestdata.cpp, line 0]]>
</system-err>
</testsuite>

View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8" ?>
<testsuite name="tst_float" timestamp="@TEST_START_TIME@" tests="6" failures="96" errors="0" time="@TEST_DURATION@">
<testsuite name="tst_float" timestamp="@TEST_START_TIME@" tests="6" failures="97" errors="0" time="@TEST_DURATION@">
<properties>
<property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
<property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
@ -201,12 +201,15 @@
Actual (operandLeft) : 1
Expected (operandRight): 3" tag="should FAIL 1"/>
<failure result="fail" message="Compared qfloat16s are not the same (fuzzy compare)
Actual (operandLeft) : 0.000999
Actual (operandLeft) : 0.001
Expected (operandRight): 0.003" tag="should FAIL 2"/>
<failure result="fail" message="Compared qfloat16s are not the same (fuzzy compare)
Actual (operandLeft) : 98
Expected (operandRight): 99" tag="should FAIL 3"/>
<failure result="fail" message="Compared qfloat16s are not the same (fuzzy compare)
Actual (operandLeft) : 0.001
Expected (operandRight): &#x002D;0.001" tag="should PASS 3"/>
<failure result="fail" message="Compared qfloat16s are not the same (fuzzy compare)
Actual (operandLeft) : 0.00101
Expected (operandRight): 0.00099" tag="should FAIL 4"/>
<failure result="fail" message="Compared qfloat16s are not the same (fuzzy compare)

View File

@ -60,7 +60,7 @@
<testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@">
<!-- type="qdebug" message="cleanupTestCase cleanupTestCase (null)" -->
</testcase>
<system-err>
<system-out>
<![CDATA[initTestCase initTestCase (null)]]>
<![CDATA[init testGlobal local=false]]>
<![CDATA[global: false]]>
@ -104,5 +104,6 @@
<![CDATA[global: true local: true]]>
<![CDATA[cleanup skipSingle local=true]]>
<![CDATA[cleanupTestCase cleanupTestCase (null)]]>
</system-err>
</system-out>
<system-err/>
</testsuite>

View File

@ -2011,6 +2011,9 @@
<!-- type="system" message="Maximum amount of warnings exceeded. Use &#x002D;maxwarnings to override." -->
</testcase>
<testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/>
<system-out>
<![CDATA[Maximum amount of warnings exceeded. Use -maxwarnings to override.]]>
</system-out>
<system-err>
<![CDATA[0]]>
<![CDATA[1]]>
@ -4013,6 +4016,5 @@
<![CDATA[1998]]>
<![CDATA[1999]]>
<![CDATA[2000]]>
<![CDATA[Maximum amount of warnings exceeded. Use -maxwarnings to override.]]>
</system-err>
</testsuite>

View File

@ -153,7 +153,7 @@
<!-- type="info" message="Signal: SignalSlotClass(_POINTER_) signalWithoutParameters ()" -->
</testcase>
<testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/>
<system-err>
<system-out>
<![CDATA[Signal: SignalSlotClass(_POINTER_) signalWithoutParameters ()]]>
<![CDATA[Signal: SignalSlotClass(_POINTER_) signalWithParameters (int(242), char(m))]]>
<![CDATA[Signal: SignalSlotClass(_POINTER_) signalWithoutParameters ()]]>
@ -278,5 +278,6 @@
<![CDATA[Signal: SignalSlotClass(_POINTER_) qVariantSignal (QVariant())]]>
<![CDATA[Signal: SignalSlotClass(_POINTER_) qVariantSignal (QVariant())]]>
<![CDATA[Signal: SignalSlotClass(_POINTER_) signalWithoutParameters ()]]>
</system-err>
</system-out>
<system-err/>
</testsuite>

View File

@ -29,15 +29,17 @@
<!-- type="qfatal" message="This is a fatal error message that should still appear in silent test output" -->
<failure result="fail" message="Received a fatal error."/>
</testcase>
<system-err>
<system-out>
<![CDATA[This test should skip]]>
<![CDATA[This test should XFAIL]]>
<![CDATA[This is a warning that should not appear in silent test output]]>
<![CDATA[This is an internal testlib warning that should not appear in silent test output]]>
<![CDATA[This is a debug message that should not appear in silent test output]]>
<![CDATA[This is a critical message that should not appear in silent test output]]>
<![CDATA[This is an info message that should not appear in silent test output]]>
<![CDATA[This is an internal testlib info message that should not appear in silent test output]]>
</system-out>
<system-err>
<![CDATA[This is a warning that should not appear in silent test output]]>
<![CDATA[This is an internal testlib warning that should not appear in silent test output]]>
<![CDATA[This is a fatal error message that should still appear in silent test output]]>
</system-err>
</testsuite>

View File

@ -10,7 +10,8 @@
<!-- type="skip" message="skipping test" -->
</testcase>
<testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/>
<system-err>
<system-out>
<![CDATA[skipping test]]>
</system-err>
</system-out>
<system-err/>
</testsuite>

View File

@ -17,10 +17,11 @@
<!-- type="qdebug" message="this line should only be reached once (true)" tag="local 2" -->
</testcase>
<testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/>
<system-err>
<system-out>
<![CDATA[skipping all]]>
<![CDATA[skipping all]]>
<![CDATA[skipping one]]>
<![CDATA[this line should only be reached once (true)]]>
</system-err>
</system-out>
<system-err/>
</testsuite>

View File

@ -10,7 +10,8 @@
<testcase name="cleanupTestCase" time="@TEST_DURATION@">
<!-- type="skip" message="Skip inside cleanupTestCase." -->
</testcase>
<system-err>
<system-out>
<![CDATA[Skip inside cleanupTestCase.]]>
</system-err>
</system-out>
<system-err/>
</testsuite>

View File

@ -9,7 +9,8 @@
<!-- type="skip" message="Skip inside initTestCase. This should skip all tests in the class." -->
</testcase>
<testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/>
<system-err>
<system-out>
<![CDATA[Skip inside initTestCase. This should skip all tests in the class.]]>
</system-err>
</system-out>
<system-err/>
</testsuite>

View File

@ -8,7 +8,8 @@
<testcase name="initTestCase" time="@TEST_DURATION@">
<!-- type="skip" message="Skip inside initTestCase_data. This should skip all tests in the class." -->
</testcase>
<system-err>
<system-out>
<![CDATA[Skip inside initTestCase_data. This should skip all tests in the class.]]>
</system-err>
</system-out>
<system-err/>
</testsuite>

View File

@ -36,9 +36,10 @@
Expected (QByteArray(&quot;7&quot;)): &quot;7&quot;"/>
</testcase>
<testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/>
<system-err>
<system-out>
<![CDATA[Next test should fail]]>
<![CDATA[Next test should fail]]>
<![CDATA[Next test should fail]]>
</system-err>
</system-out>
<system-err/>
</testsuite>

View File

@ -52,7 +52,7 @@
<testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@">
<!-- type="qdebug" message="cleanupTestCase cleanupTestCase (null)" -->
</testcase>
<system-err>
<system-out>
<![CDATA[initTestCase initTestCase (null)]]>
<![CDATA[init test1 (null)]]>
<![CDATA[test1 test1 (null)]]>
@ -84,5 +84,6 @@
<![CDATA[test2 test3 data2]]>
<![CDATA[cleanup test3 data2]]>
<![CDATA[cleanupTestCase cleanupTestCase (null)]]>
</system-err>
</system-out>
<system-err/>
</testsuite>

View File

@ -50,7 +50,7 @@
<!-- type="skip" message="Skip in cleanup()" tag="skip" -->
</testcase>
<testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/>
<system-err>
<system-out>
<![CDATA[Skipping]]>
<![CDATA[Skipping]]>
<![CDATA[Skipping]]>
@ -61,5 +61,6 @@
<![CDATA[Skip in init()]]>
<![CDATA[This test function should execute and then QSKIP in cleanup()]]>
<![CDATA[Skip in cleanup()]]>
</system-err>
</system-out>
<system-err/>
</testsuite>

View File

@ -69,7 +69,7 @@
<!-- type="skip" message="Skip in cleanup()" tag="skip" -->
</testcase>
<testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/>
<system-err>
<system-out>
<![CDATA[QVERIFY(true)]]>
<![CDATA[QCOMPARE(2 + 1, 3)]]>
<![CDATA[QVERIFY(true)]]>
@ -98,5 +98,6 @@
<![CDATA[Skip in init()]]>
<![CDATA[This test function should execute and then QSKIP in cleanup()]]>
<![CDATA[Skip in cleanup()]]>
</system-err>
</system-out>
<system-err/>
</testsuite>

View File

@ -36,9 +36,7 @@
<failure result="fail" message="Not all expected messages were received" tag="second row"/>
</testcase>
<testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/>
<system-err>
<![CDATA[Warning]]>
<![CDATA[Warning]]>
<system-out>
<![CDATA[Debug]]>
<![CDATA[Debug]]>
<![CDATA[Info]]>
@ -46,7 +44,6 @@
<![CDATA[Baba]]>
<![CDATA[Baba]]>
<![CDATA[Bubublabla]]>
<![CDATA[Babablabla]]>
<![CDATA[Did not receive message: "Warning0"]]>
<![CDATA[Did not receive message: "Warning1"]]>
<![CDATA[Did not receive any message matching: "Warning\s\d"]]>
@ -54,5 +51,10 @@
<![CDATA[Did not receive message: "Warning1"]]>
<![CDATA[Did not receive message: "Warning0"]]>
<![CDATA[Did not receive message: "Warning1"]]>
</system-out>
<system-err>
<![CDATA[Warning]]>
<![CDATA[Warning]]>
<![CDATA[Babablabla]]>
</system-err>
</testsuite>

View File

@ -31,11 +31,13 @@
<failure result="xpass" message="&apos;true&apos; returned TRUE unexpectedly. ()"/>
</testcase>
<testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/>
<system-err>
<![CDATA[just a QWARN() !]]>
<system-out>
<![CDATA[a qDebug() call with comment-ending stuff -->]]>
<![CDATA[skipping this function!]]>
<![CDATA[this failure is expected]]>
<![CDATA[this failure is also expected]]>
</system-out>
<system-err>
<![CDATA[just a QWARN() !]]>
</system-err>
</testsuite>