1berry/qtbase
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

generate_expected_output.py: match tst_selftest's test environments

Browse Source 
The testlib selftest sets various things in the environment for
crashing tests; the generator for its expected output should set the
same things, as they affect what output is produced.

Change-Id: Iec2ed59982ea1043582573530c33619d8e8ed08e
Reviewed-by: Jędrzej Nowacki <jedrzej.nowacki@qt.io>
This commit is contained in:
Edward Welbourne 2018-10-15 19:48:13 +02:00
parent 30101884a6
commit b22e50acda
2 changed files with 24 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
tests/auto/testlib/selftests/generate_expected_output.py
View File

@ -223,6 +223,7 @@ del re
def generateTestData(testname, clean, def generateTestData(testname, clean,
formats = ('xml', 'txt', 'xunitxml', 'lightxml', 'teamcity', 'tap'), formats = ('xml', 'txt', 'xunitxml', 'lightxml', 'teamcity', 'tap'),
# Make sure this matches tst_Selftests::runSubTest_data():
extraArgs = { extraArgs = {
"commandlinedata": "fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2", "commandlinedata": "fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2",
"benchlibcallgrind": "-callgrind", "benchlibcallgrind": "-callgrind",
@ -236,7 +237,15 @@ def generateTestData(testname, clean,
"silent": "-silent", "silent": "-silent",
"verbose1": "-v1", "verbose1": "-v1",
"verbose2": "-v2", "verbose2": "-v2",
}): },
# Make sure this matches tst_Selftests::doRunSubTest():
extraEnv = {
"crashes": { "QTEST_DISABLE_CORE_DUMP": "1", "QTEST_DISABLE_STACK_DUMP": "1" },
},
# These are actually *other* crashers, beside those in extraEnv;
# must match tst_Selftests::runSubTest_data():
crashers = ("assert", "blacklisted", "crashedterminate",
"exceptionthrow", "fetchbogus", "silent")):
"""Run one test and save its cleaned results. """Run one test and save its cleaned results.
Required arguments are the name of the test directory (the binary Required arguments are the name of the test directory (the binary
@ -248,6 +257,16 @@ def generateTestData(testname, clean,
if not os.path.isfile(path): if not os.path.isfile(path):
print("Warning: directory", testname, "contains no test executable") print("Warning: directory", testname, "contains no test executable")
return return
env = None
try:
env = extraEnv[testname]
except KeyError:
if env in crashers:
env = extraEnv["crashes"]
if env:
data = os.environ.copy()
data.update(env)
env = data
print(" running", testname) print(" running", testname)
for format in formats: for format in formats:
@ -255,7 +274,7 @@ def generateTestData(testname, clean,
if testname in extraArgs: if testname in extraArgs:
cmd += extraArgs[testname].split() cmd += extraArgs[testname].split()
data = subprocess.Popen(cmd, stdout=subprocess.PIPE, data = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env,
universal_newlines=True).communicate()[0] universal_newlines=True).communicate()[0]
with open('expected_' + testname + '.' + format, 'w') as out: with open('expected_' + testname + '.' + format, 'w') as out:
out.write('\n'.join(clean(data))) # write() appends a newline, too out.write('\n'.join(clean(data))) # write() appends a newline, too

3
tests/auto/testlib/selftests/tst_selftests.cpp
View File

@ -520,6 +520,7 @@ void tst_Selftests::runSubTest_data()
foreach (QString const& subtest, tests) { foreach (QString const& subtest, tests) {
QStringList arguments = loggerSet.arguments; QStringList arguments = loggerSet.arguments;
// Keep in sync with generateTestData()'s extraArgs in generate_expected_output.py:
if (subtest == "commandlinedata") { if (subtest == "commandlinedata") {
arguments << QString("fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2").split(' '); arguments << QString("fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2").split(' ');
} }
@ -612,6 +613,7 @@ void tst_Selftests::runSubTest_data()
if (loggerSet.name.contains("teamcity") && subtest.startsWith("benchlib")) if (loggerSet.name.contains("teamcity") && subtest.startsWith("benchlib"))
continue; // Skip benchmark for TeamCity logger continue; // Skip benchmark for TeamCity logger
// Keep in sync with generateTestData()'s crashers in generate_expected_output.py:
const bool crashes = subtest == QLatin1String("assert") || subtest == QLatin1String("exceptionthrow") const bool crashes = subtest == QLatin1String("assert") || subtest == QLatin1String("exceptionthrow")
|| subtest == QLatin1String("fetchbogus") || subtest == QLatin1String("crashedterminate") || subtest == QLatin1String("fetchbogus") || subtest == QLatin1String("crashedterminate")
|| subtest == QLatin1String("crashes") || subtest == QLatin1String("silent") || subtest == QLatin1String("crashes") || subtest == QLatin1String("silent")
@ -687,6 +689,7 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge
QProcess proc; QProcess proc;
QProcessEnvironment environment = processEnvironment(); QProcessEnvironment environment = processEnvironment();
// Keep in sync with generateTestData()'s extraEnv in generate_expected_output.py:
if (crashes) { if (crashes) {
environment.insert("QTEST_DISABLE_CORE_DUMP", "1"); environment.insert("QTEST_DISABLE_CORE_DUMP", "1");
environment.insert("QTEST_DISABLE_STACK_DUMP", "1"); environment.insert("QTEST_DISABLE_STACK_DUMP", "1");