testlib: Only generate expecations for silent test with plain logger
The -silent option to tests is only supported with the plain text logger, so we don't need to maintain expectation files for the others. Pick-to: 6.2 Change-Id: I0f42bfe90d82b7ce04f550c747d4a80e99621e74 Reviewed-by: Edward Welbourne <edward.welbourne@qt.io> Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>
This commit is contained in:
Tor Arne Vestbø
parent
825e4291cd
commit
f6c7eb5f0f
@ -1,42 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8" ?>
|
|
||||||
<testsuite name="tst_Silent" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="7" failures="2" errors="1" skipped="1" time="@TEST_DURATION@">
|
|
||||||
<properties>
|
|
||||||
<property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
|
|
||||||
<property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
|
|
||||||
<property name="QtBuild" value=""/>
|
|
||||||
</properties>
|
|
||||||
<testcase name="initTestCase" classname="tst_Silent" time="@TEST_DURATION@"/>
|
|
||||||
<testcase name="pass" classname="tst_Silent" time="@TEST_DURATION@"/>
|
|
||||||
<testcase name="skip" classname="tst_Silent" time="@TEST_DURATION@">
|
|
||||||
<skipped message="This test should skip"/>
|
|
||||||
</testcase>
|
|
||||||
<testcase name="fail" classname="tst_Silent" time="@TEST_DURATION@">
|
|
||||||
<failure type="fail" message="'false' returned FALSE. (This test should fail)"/>
|
|
||||||
</testcase>
|
|
||||||
<testcase name="xfail" classname="tst_Silent" time="@TEST_DURATION@">
|
|
||||||
<!-- type="info" message="This test should XFAIL" -->
|
|
||||||
</testcase>
|
|
||||||
<testcase name="xpass" classname="tst_Silent" time="@TEST_DURATION@">
|
|
||||||
<failure type="xpass" message="'true' returned TRUE unexpectedly. (This test should XPASS)"/>
|
|
||||||
</testcase>
|
|
||||||
<testcase name="messages" classname="tst_Silent" time="@TEST_DURATION@">
|
|
||||||
<!-- type="qwarn" message="This is a warning that should not appear in silent test output" -->
|
|
||||||
<!-- type="warn" message="This is an internal testlib warning that should not appear in silent test output" -->
|
|
||||||
<!-- type="qdebug" message="This is a debug message that should not appear in silent test output" -->
|
|
||||||
<!-- type="system" message="This is a critical message that should not appear in silent test output" -->
|
|
||||||
<!-- type="qinfo" message="This is an info message that should not appear in silent test output" -->
|
|
||||||
<!-- type="info" message="This is an internal testlib info message that should not appear in silent test output" -->
|
|
||||||
<error type="qfatal" message="This is a fatal error message that should still appear in silent test output"/>
|
|
||||||
</testcase>
|
|
||||||
<system-out>
|
|
||||||
<![CDATA[This test should XFAIL]]>
|
|
||||||
<![CDATA[This is a debug message that should not appear in silent test output]]>
|
|
||||||
<![CDATA[This is a critical message that should not appear in silent test output]]>
|
|
||||||
<![CDATA[This is an info message that should not appear in silent test output]]>
|
|
||||||
<![CDATA[This is an internal testlib info message that should not appear in silent test output]]>
|
|
||||||
</system-out>
|
|
||||||
<system-err>
|
|
||||||
<![CDATA[This is a warning that should not appear in silent test output]]>
|
|
||||||
<![CDATA[This is an internal testlib warning that should not appear in silent test output]]>
|
|
||||||
</system-err>
|
|
||||||
</testsuite>
|
|
||||||
@ -1,66 +0,0 @@
|
|||||||
<Environment>
|
|
||||||
<QtVersion>@INSERT_QT_VERSION_HERE@</QtVersion>
|
|
||||||
<QtBuild/>
|
|
||||||
<QTestVersion>@INSERT_QT_VERSION_HERE@</QTestVersion>
|
|
||||||
</Environment>
|
|
||||||
<TestFunction name="initTestCase">
|
|
||||||
<Incident type="pass" file="" line="0" />
|
|
||||||
<Duration msecs="0"/>
|
|
||||||
</TestFunction>
|
|
||||||
<TestFunction name="pass">
|
|
||||||
<Incident type="pass" file="" line="0" />
|
|
||||||
<Duration msecs="0"/>
|
|
||||||
</TestFunction>
|
|
||||||
<TestFunction name="skip">
|
|
||||||
<Message type="skip" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
|
||||||
<Description><![CDATA[This test should skip]]></Description>
|
|
||||||
</Message>
|
|
||||||
<Duration msecs="0"/>
|
|
||||||
</TestFunction>
|
|
||||||
<TestFunction name="fail">
|
|
||||||
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
|
||||||
<Description><![CDATA['false' returned FALSE. (This test should fail)]]></Description>
|
|
||||||
</Incident>
|
|
||||||
<Duration msecs="0"/>
|
|
||||||
</TestFunction>
|
|
||||||
<TestFunction name="xfail">
|
|
||||||
<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
|
||||||
<Description><![CDATA[This test should XFAIL]]></Description>
|
|
||||||
</Incident>
|
|
||||||
<Incident type="pass" file="" line="0" />
|
|
||||||
<Duration msecs="0"/>
|
|
||||||
</TestFunction>
|
|
||||||
<TestFunction name="xpass">
|
|
||||||
<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
|
||||||
<Description><![CDATA['true' returned TRUE unexpectedly. (This test should XPASS)]]></Description>
|
|
||||||
</Incident>
|
|
||||||
<Duration msecs="0"/>
|
|
||||||
</TestFunction>
|
|
||||||
<TestFunction name="messages">
|
|
||||||
<Message type="qwarn" file="" line="0">
|
|
||||||
<Description><![CDATA[This is a warning that should not appear in silent test output]]></Description>
|
|
||||||
</Message>
|
|
||||||
<Message type="warn" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
|
||||||
<Description><![CDATA[This is an internal testlib warning that should not appear in silent test output]]></Description>
|
|
||||||
</Message>
|
|
||||||
<Message type="qdebug" file="" line="0">
|
|
||||||
<Description><![CDATA[This is a debug message that should not appear in silent test output]]></Description>
|
|
||||||
</Message>
|
|
||||||
<Message type="system" file="" line="0">
|
|
||||||
<Description><![CDATA[This is a critical message that should not appear in silent test output]]></Description>
|
|
||||||
</Message>
|
|
||||||
<Message type="qinfo" file="" line="0">
|
|
||||||
<Description><![CDATA[This is an info message that should not appear in silent test output]]></Description>
|
|
||||||
</Message>
|
|
||||||
<Message type="info" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
|
||||||
<Description><![CDATA[This is an internal testlib info message that should not appear in silent test output]]></Description>
|
|
||||||
</Message>
|
|
||||||
<Message type="qfatal" file="" line="0">
|
|
||||||
<Description><![CDATA[This is a fatal error message that should still appear in silent test output]]></Description>
|
|
||||||
</Message>
|
|
||||||
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
|
||||||
<Description><![CDATA[Received a fatal error.]]></Description>
|
|
||||||
</Incident>
|
|
||||||
<Duration msecs="0"/>
|
|
||||||
</TestFunction>
|
|
||||||
<Duration msecs="0"/>
|
|
||||||
@ -1,42 +0,0 @@
|
|||||||
TAP version 13
|
|
||||||
# tst_Silent
|
|
||||||
ok 1 - initTestCase()
|
|
||||||
ok 2 - pass()
|
|
||||||
ok 3 - skip() # SKIP This test should skip
|
|
||||||
not ok 4 - fail()
|
|
||||||
---
|
|
||||||
type: QVERIFY
|
|
||||||
message: This test should fail
|
|
||||||
wanted: true (false)
|
|
||||||
found: false (false)
|
|
||||||
expected: true (false)
|
|
||||||
actual: false (false)
|
|
||||||
at: tst_Silent::fail() (qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp:0)
|
|
||||||
file: qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp
|
|
||||||
line: 0
|
|
||||||
...
|
|
||||||
not ok 5 - xfail() # TODO This test should XFAIL
|
|
||||||
---
|
|
||||||
at: tst_Silent::xfail() (qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp:0)
|
|
||||||
file: qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp
|
|
||||||
line: 0
|
|
||||||
...
|
|
||||||
ok 6 - xpass() # TODO 'true' returned TRUE unexpectedly. (This test should XPASS)
|
|
||||||
# This is a warning that should not appear in silent test output
|
|
||||||
# This is an internal testlib warning that should not appear in silent test output
|
|
||||||
# This is a debug message that should not appear in silent test output
|
|
||||||
# This is a critical message that should not appear in silent test output
|
|
||||||
# This is an info message that should not appear in silent test output
|
|
||||||
# This is an internal testlib info message that should not appear in silent test output
|
|
||||||
# This is a fatal error message that should still appear in silent test output
|
|
||||||
not ok 7 - messages()
|
|
||||||
---
|
|
||||||
# Received a fatal error.
|
|
||||||
at: tst_Silent::messages() (qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp:0)
|
|
||||||
file: qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp
|
|
||||||
line: 0
|
|
||||||
...
|
|
||||||
1..7
|
|
||||||
# tests 7
|
|
||||||
# pass 3
|
|
||||||
# fail 3
|
|
||||||
@ -1,12 +0,0 @@
|
|||||||
##teamcity[testSuiteStarted name='tst_Silent' flowId='tst_Silent']
|
|
||||||
##teamcity[testStarted name='fail()' flowId='tst_Silent']
|
|
||||||
##teamcity[testFailed name='fail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp(0)|]' details='|'false|' returned FALSE. (This test should fail)' flowId='tst_Silent']
|
|
||||||
##teamcity[testFinished name='fail()' flowId='tst_Silent']
|
|
||||||
##teamcity[testStarted name='xpass()' flowId='tst_Silent']
|
|
||||||
##teamcity[testFailed name='xpass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. (This test should XPASS)' flowId='tst_Silent']
|
|
||||||
##teamcity[testFinished name='xpass()' flowId='tst_Silent']
|
|
||||||
##teamcity[testStarted name='messages()' flowId='tst_Silent']
|
|
||||||
##teamcity[testFailed name='messages()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp(0)|]' details='Received a fatal error.' flowId='tst_Silent']
|
|
||||||
##teamcity[testStdOut name='messages()' out='QFATAL: This is a fatal error message that should still appear in silent test output' flowId='tst_Silent']
|
|
||||||
##teamcity[testFinished name='messages()' flowId='tst_Silent']
|
|
||||||
##teamcity[testSuiteFinished name='tst_Silent' flowId='tst_Silent']
|
|
||||||
@ -1,69 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<TestCase name="tst_Silent">
|
|
||||||
<Environment>
|
|
||||||
<QtVersion>@INSERT_QT_VERSION_HERE@</QtVersion>
|
|
||||||
<QtBuild/>
|
|
||||||
<QTestVersion>@INSERT_QT_VERSION_HERE@</QTestVersion>
|
|
||||||
</Environment>
|
|
||||||
<TestFunction name="initTestCase">
|
|
||||||
<Incident type="pass" file="" line="0" />
|
|
||||||
<Duration msecs="0"/>
|
|
||||||
</TestFunction>
|
|
||||||
<TestFunction name="pass">
|
|
||||||
<Incident type="pass" file="" line="0" />
|
|
||||||
<Duration msecs="0"/>
|
|
||||||
</TestFunction>
|
|
||||||
<TestFunction name="skip">
|
|
||||||
<Message type="skip" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
|
||||||
<Description><![CDATA[This test should skip]]></Description>
|
|
||||||
</Message>
|
|
||||||
<Duration msecs="0"/>
|
|
||||||
</TestFunction>
|
|
||||||
<TestFunction name="fail">
|
|
||||||
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
|
||||||
<Description><![CDATA['false' returned FALSE. (This test should fail)]]></Description>
|
|
||||||
</Incident>
|
|
||||||
<Duration msecs="0"/>
|
|
||||||
</TestFunction>
|
|
||||||
<TestFunction name="xfail">
|
|
||||||
<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
|
||||||
<Description><![CDATA[This test should XFAIL]]></Description>
|
|
||||||
</Incident>
|
|
||||||
<Incident type="pass" file="" line="0" />
|
|
||||||
<Duration msecs="0"/>
|
|
||||||
</TestFunction>
|
|
||||||
<TestFunction name="xpass">
|
|
||||||
<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
|
||||||
<Description><![CDATA['true' returned TRUE unexpectedly. (This test should XPASS)]]></Description>
|
|
||||||
</Incident>
|
|
||||||
<Duration msecs="0"/>
|
|
||||||
</TestFunction>
|
|
||||||
<TestFunction name="messages">
|
|
||||||
<Message type="qwarn" file="" line="0">
|
|
||||||
<Description><![CDATA[This is a warning that should not appear in silent test output]]></Description>
|
|
||||||
</Message>
|
|
||||||
<Message type="warn" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
|
||||||
<Description><![CDATA[This is an internal testlib warning that should not appear in silent test output]]></Description>
|
|
||||||
</Message>
|
|
||||||
<Message type="qdebug" file="" line="0">
|
|
||||||
<Description><![CDATA[This is a debug message that should not appear in silent test output]]></Description>
|
|
||||||
</Message>
|
|
||||||
<Message type="system" file="" line="0">
|
|
||||||
<Description><![CDATA[This is a critical message that should not appear in silent test output]]></Description>
|
|
||||||
</Message>
|
|
||||||
<Message type="qinfo" file="" line="0">
|
|
||||||
<Description><![CDATA[This is an info message that should not appear in silent test output]]></Description>
|
|
||||||
</Message>
|
|
||||||
<Message type="info" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
|
||||||
<Description><![CDATA[This is an internal testlib info message that should not appear in silent test output]]></Description>
|
|
||||||
</Message>
|
|
||||||
<Message type="qfatal" file="" line="0">
|
|
||||||
<Description><![CDATA[This is a fatal error message that should still appear in silent test output]]></Description>
|
|
||||||
</Message>
|
|
||||||
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
|
|
||||||
<Description><![CDATA[Received a fatal error.]]></Description>
|
|
||||||
</Incident>
|
|
||||||
<Duration msecs="0"/>
|
|
||||||
</TestFunction>
|
|
||||||
<Duration msecs="0"/>
|
|
||||||
</TestCase>
|
|
||||||
@ -281,6 +281,15 @@ def testEnv(testname,
|
|||||||
data.update(extraEnv[testname])
|
data.update(extraEnv[testname])
|
||||||
return data
|
return data
|
||||||
|
|
||||||
|
# See TestLogger::shouldIgnoreTest() in tst_selftest.cpp
|
||||||
|
def shouldIgnoreTest(testname, format):
|
||||||
|
if testname == "junit" and not format == "junitxml":
|
||||||
|
return True
|
||||||
|
if testname in ["float", "silent"] and not format == "txt":
|
||||||
|
return True
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
def generateTestData(test_path, expected_path, clean, formats):
|
def generateTestData(test_path, expected_path, clean, formats):
|
||||||
"""Run one test and save its cleaned results.
|
"""Run one test and save its cleaned results.
|
||||||
|
|
||||||
@ -296,18 +305,11 @@ def generateTestData(test_path, expected_path, clean, formats):
|
|||||||
print("Warning: directory", testname, "contains no test executable")
|
print("Warning: directory", testname, "contains no test executable")
|
||||||
return
|
return
|
||||||
|
|
||||||
# See TestLogger::shouldIgnoreTest() in tst_selftest.cpp for these
|
|
||||||
# single-format tests:
|
|
||||||
if testname == 'junit':
|
|
||||||
formats = ( 'junitxml', ) if 'junitxml' in formats else ()
|
|
||||||
elif testname == 'float':
|
|
||||||
formats = ( 'txt', ) if 'txt' in formats else ()
|
|
||||||
|
|
||||||
# Prepare environment in which to run tests:
|
# Prepare environment in which to run tests:
|
||||||
env = testEnv(testname)
|
env = testEnv(testname)
|
||||||
|
|
||||||
for format in formats:
|
for format in formats:
|
||||||
if testname == "junit" and not format == "junitxml":
|
if shouldIgnoreTest(testname, format):
|
||||||
continue
|
continue
|
||||||
print(f' running {testname}/{format}')
|
print(f' running {testname}/{format}')
|
||||||
cmd = [path, f'-{format}']
|
cmd = [path, f'-{format}']
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user