testlib: Only generate expecations for silent test with plain logger
The -silent option to tests is only supported with the plain text logger, so we don't need to maintain expectation files for the others. Pick-to: 6.2 Change-Id: I0f42bfe90d82b7ce04f550c747d4a80e99621e74 Reviewed-by: Edward Welbourne <edward.welbourne@qt.io> Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>
This commit is contained in:
Tor Arne Vestbø
parent
825e4291cd
commit
f6c7eb5f0f
@ -1,42 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8" ?>
|
||||
<testsuite name="tst_Silent" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="7" failures="2" errors="1" skipped="1" time="@TEST_DURATION@">
|
||||
<properties>
|
||||
<property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
|
||||
<property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
|
||||
<property name="QtBuild" value=""/>
|
||||
</properties>
|
||||
<testcase name="initTestCase" classname="tst_Silent" time="@TEST_DURATION@"/>
|
||||
<testcase name="pass" classname="tst_Silent" time="@TEST_DURATION@"/>
|
||||
<testcase name="skip" classname="tst_Silent" time="@TEST_DURATION@">
|
||||
<skipped message="This test should skip"/>
|
||||
</testcase>
|
||||
<testcase name="fail" classname="tst_Silent" time="@TEST_DURATION@">
|
||||
<failure type="fail" message="'false' returned FALSE. (This test should fail)"/>
|
||||
</testcase>
|
||||
<testcase name="xfail" classname="tst_Silent" time="@TEST_DURATION@">
|
||||
<!-- type="info" message="This test should XFAIL" -->
|
||||
</testcase>
|
||||
<testcase name="xpass" classname="tst_Silent" time="@TEST_DURATION@">
|
||||
<failure type="xpass" message="'true' returned TRUE unexpectedly. (This test should XPASS)"/>
|
||||
</testcase>
|
||||
<testcase name="messages" classname="tst_Silent" time="@TEST_DURATION@">
|
||||
<!-- type="qwarn" message="This is a warning that should not appear in silent test output" -->
|
||||
<!-- type="warn" message="This is an internal testlib warning that should not appear in silent test output" -->
|
||||
<!-- type="qdebug" message="This is a debug message that should not appear in silent test output" -->
|
||||
<!-- type="system" message="This is a critical message that should not appear in silent test output" -->
|
||||
<!-- type="qinfo" message="This is an info message that should not appear in silent test output" -->
|
||||
<!-- type="info" message="This is an internal testlib info message that should not appear in silent test output" -->
|
||||
<error type="qfatal" message="This is a fatal error message that should still appear in silent test output"/>
|
||||
</testcase>
|
||||
<system-out>
|
||||
<![CDATA[This test should XFAIL]]>
|
||||
<![CDATA[This is a debug message that should not appear in silent test output]]>
|
||||
<![CDATA[This is a critical message that should not appear in silent test output]]>
|
||||
<![CDATA[This is an info message that should not appear in silent test output]]>
|
||||
<![CDATA[This is an internal testlib info message that should not appear in silent test output]]>
|
||||
</system-out>
|
||||
<system-err>
|
||||
<![CDATA[This is a warning that should not appear in silent test output]]>
|
||||
<![CDATA[This is an internal testlib warning that should not appear in silent test output]]>
|
||||
</system-err>
|
||||
</testsuite>
|
||||
@ -1,66 +0,0 @@
|
||||
<Environment>
|
||||
<QtVersion>@INSERT_QT_VERSION_HERE@</QtVersion>
|
||||
<QtBuild/>
|
||||
<QTestVersion>@INSERT_QT_VERSION_HERE@</QTestVersion>
|
||||
</Environment>
|
||||
<TestFunction name="initTestCase">
|
||||
<Incident type="pass" file="" line="0" />
|
||||
<Duration msecs="0"/>
|
||||
</TestFunction>
|
||||
<TestFunction name="pass">
|
||||
<Incident type="pass" file="" line="0" />
|
||||
<Duration msecs="0"/>
|
||||
</TestFunction>
|
||||
<TestFunction name="skip">
|
||||
<Message type="skip" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
||||
<Description><![CDATA[This test should skip]]></Description>
|
||||
</Message>
|
||||
<Duration msecs="0"/>
|
||||
</TestFunction>
|
||||
<TestFunction name="fail">
|
||||
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
||||
<Description><![CDATA['false' returned FALSE. (This test should fail)]]></Description>
|
||||
</Incident>
|
||||
<Duration msecs="0"/>
|
||||
</TestFunction>
|
||||
<TestFunction name="xfail">
|
||||
<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
||||
<Description><![CDATA[This test should XFAIL]]></Description>
|
||||
</Incident>
|
||||
<Incident type="pass" file="" line="0" />
|
||||
<Duration msecs="0"/>
|
||||
</TestFunction>
|
||||
<TestFunction name="xpass">
|
||||
<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
||||
<Description><![CDATA['true' returned TRUE unexpectedly. (This test should XPASS)]]></Description>
|
||||
</Incident>
|
||||
<Duration msecs="0"/>
|
||||
</TestFunction>
|
||||
<TestFunction name="messages">
|
||||
<Message type="qwarn" file="" line="0">
|
||||
<Description><![CDATA[This is a warning that should not appear in silent test output]]></Description>
|
||||
</Message>
|
||||
<Message type="warn" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
||||
<Description><![CDATA[This is an internal testlib warning that should not appear in silent test output]]></Description>
|
||||
</Message>
|
||||
<Message type="qdebug" file="" line="0">
|
||||
<Description><![CDATA[This is a debug message that should not appear in silent test output]]></Description>
|
||||
</Message>
|
||||
<Message type="system" file="" line="0">
|
||||
<Description><![CDATA[This is a critical message that should not appear in silent test output]]></Description>
|
||||
</Message>
|
||||
<Message type="qinfo" file="" line="0">
|
||||
<Description><![CDATA[This is an info message that should not appear in silent test output]]></Description>
|
||||
</Message>
|
||||
<Message type="info" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
||||
<Description><![CDATA[This is an internal testlib info message that should not appear in silent test output]]></Description>
|
||||
</Message>
|
||||
<Message type="qfatal" file="" line="0">
|
||||
<Description><![CDATA[This is a fatal error message that should still appear in silent test output]]></Description>
|
||||
</Message>
|
||||
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
||||
<Description><![CDATA[Received a fatal error.]]></Description>
|
||||
</Incident>
|
||||
<Duration msecs="0"/>
|
||||
</TestFunction>
|
||||
<Duration msecs="0"/>
|
||||
@ -1,42 +0,0 @@
|
||||
TAP version 13
|
||||
# tst_Silent
|
||||
ok 1 - initTestCase()
|
||||
ok 2 - pass()
|
||||
ok 3 - skip() # SKIP This test should skip
|
||||
not ok 4 - fail()
|
||||
---
|
||||
type: QVERIFY
|
||||
message: This test should fail
|
||||
wanted: true (false)
|
||||
found: false (false)
|
||||
expected: true (false)
|
||||
actual: false (false)
|
||||
at: tst_Silent::fail() (qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp:0)
|
||||
file: qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp
|
||||
line: 0
|
||||
...
|
||||
not ok 5 - xfail() # TODO This test should XFAIL
|
||||
---
|
||||
at: tst_Silent::xfail() (qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp:0)
|
||||
file: qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp
|
||||
line: 0
|
||||
...
|
||||
ok 6 - xpass() # TODO 'true' returned TRUE unexpectedly. (This test should XPASS)
|
||||
# This is a warning that should not appear in silent test output
|
||||
# This is an internal testlib warning that should not appear in silent test output
|
||||
# This is a debug message that should not appear in silent test output
|
||||
# This is a critical message that should not appear in silent test output
|
||||
# This is an info message that should not appear in silent test output
|
||||
# This is an internal testlib info message that should not appear in silent test output
|
||||
# This is a fatal error message that should still appear in silent test output
|
||||
not ok 7 - messages()
|
||||
---
|
||||
# Received a fatal error.
|
||||
at: tst_Silent::messages() (qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp:0)
|
||||
file: qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp
|
||||
line: 0
|
||||
...
|
||||
1..7
|
||||
# tests 7
|
||||
# pass 3
|
||||
# fail 3
|
||||
@ -1,12 +0,0 @@
|
||||
##teamcity[testSuiteStarted name='tst_Silent' flowId='tst_Silent']
|
||||
##teamcity[testStarted name='fail()' flowId='tst_Silent']
|
||||
##teamcity[testFailed name='fail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp(0)|]' details='|'false|' returned FALSE. (This test should fail)' flowId='tst_Silent']
|
||||
##teamcity[testFinished name='fail()' flowId='tst_Silent']
|
||||
##teamcity[testStarted name='xpass()' flowId='tst_Silent']
|
||||
##teamcity[testFailed name='xpass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. (This test should XPASS)' flowId='tst_Silent']
|
||||
##teamcity[testFinished name='xpass()' flowId='tst_Silent']
|
||||
##teamcity[testStarted name='messages()' flowId='tst_Silent']
|
||||
##teamcity[testFailed name='messages()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp(0)|]' details='Received a fatal error.' flowId='tst_Silent']
|
||||
##teamcity[testStdOut name='messages()' out='QFATAL: This is a fatal error message that should still appear in silent test output' flowId='tst_Silent']
|
||||
##teamcity[testFinished name='messages()' flowId='tst_Silent']
|
||||
##teamcity[testSuiteFinished name='tst_Silent' flowId='tst_Silent']
|
||||
@ -1,69 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<TestCase name="tst_Silent">
|
||||
<Environment>
|
||||
<QtVersion>@INSERT_QT_VERSION_HERE@</QtVersion>
|
||||
<QtBuild/>
|
||||
<QTestVersion>@INSERT_QT_VERSION_HERE@</QTestVersion>
|
||||
</Environment>
|
||||
<TestFunction name="initTestCase">
|
||||
<Incident type="pass" file="" line="0" />
|
||||
<Duration msecs="0"/>
|
||||
</TestFunction>
|
||||
<TestFunction name="pass">
|
||||
<Incident type="pass" file="" line="0" />
|
||||
<Duration msecs="0"/>
|
||||
</TestFunction>
|
||||
<TestFunction name="skip">
|
||||
<Message type="skip" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
||||
<Description><![CDATA[This test should skip]]></Description>
|
||||
</Message>
|
||||
<Duration msecs="0"/>
|
||||
</TestFunction>
|
||||
<TestFunction name="fail">
|
||||
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
||||
<Description><![CDATA['false' returned FALSE. (This test should fail)]]></Description>
|
||||
</Incident>
|
||||
<Duration msecs="0"/>
|
||||
</TestFunction>
|
||||
<TestFunction name="xfail">
|
||||
<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
||||
<Description><![CDATA[This test should XFAIL]]></Description>
|
||||
</Incident>
|
||||
<Incident type="pass" file="" line="0" />
|
||||
<Duration msecs="0"/>
|
||||
</TestFunction>
|
||||
<TestFunction name="xpass">
|
||||
<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
||||
<Description><![CDATA['true' returned TRUE unexpectedly. (This test should XPASS)]]></Description>
|
||||
</Incident>
|
||||
<Duration msecs="0"/>
|
||||
</TestFunction>
|
||||
<TestFunction name="messages">
|
||||
<Message type="qwarn" file="" line="0">
|
||||
<Description><![CDATA[This is a warning that should not appear in silent test output]]></Description>
|
||||
</Message>
|
||||
<Message type="warn" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
||||
<Description><![CDATA[This is an internal testlib warning that should not appear in silent test output]]></Description>
|
||||
</Message>
|
||||
<Message type="qdebug" file="" line="0">
|
||||
<Description><![CDATA[This is a debug message that should not appear in silent test output]]></Description>
|
||||
</Message>
|
||||
<Message type="system" file="" line="0">
|
||||
<Description><![CDATA[This is a critical message that should not appear in silent test output]]></Description>
|
||||
</Message>
|
||||
<Message type="qinfo" file="" line="0">
|
||||
<Description><![CDATA[This is an info message that should not appear in silent test output]]></Description>
|
||||
</Message>
|
||||
<Message type="info" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
||||
<Description><![CDATA[This is an internal testlib info message that should not appear in silent test output]]></Description>
|
||||
</Message>
|
||||
<Message type="qfatal" file="" line="0">
|
||||
<Description><![CDATA[This is a fatal error message that should still appear in silent test output]]></Description>
|
||||
</Message>
|
||||
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
||||
<Description><![CDATA[Received a fatal error.]]></Description>
|
||||
</Incident>
|
||||
<Duration msecs="0"/>
|
||||
</TestFunction>
|
||||
<Duration msecs="0"/>
|
||||
</TestCase>
|
||||
@ -281,6 +281,15 @@ def testEnv(testname,
|
||||
data.update(extraEnv[testname])
|
||||
return data
|
||||
|
||||
# See TestLogger::shouldIgnoreTest() in tst_selftest.cpp
|
||||
def shouldIgnoreTest(testname, format):
|
||||
if testname == "junit" and not format == "junitxml":
|
||||
return True
|
||||
if testname in ["float", "silent"] and not format == "txt":
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def generateTestData(test_path, expected_path, clean, formats):
|
||||
"""Run one test and save its cleaned results.
|
||||
|
||||
@ -296,18 +305,11 @@ def generateTestData(test_path, expected_path, clean, formats):
|
||||
print("Warning: directory", testname, "contains no test executable")
|
||||
return
|
||||
|
||||
# See TestLogger::shouldIgnoreTest() in tst_selftest.cpp for these
|
||||
# single-format tests:
|
||||
if testname == 'junit':
|
||||
formats = ( 'junitxml', ) if 'junitxml' in formats else ()
|
||||
elif testname == 'float':
|
||||
formats = ( 'txt', ) if 'txt' in formats else ()
|
||||
|
||||
# Prepare environment in which to run tests:
|
||||
env = testEnv(testname)
|
||||
|
||||
for format in formats:
|
||||
if testname == "junit" and not format == "junitxml":
|
||||
if shouldIgnoreTest(testname, format):
|
||||
continue
|
||||
print(f' running {testname}/{format}')
|
||||
cmd = [path, f'-{format}']
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user