1berry/qtbase
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enable testlib self-testing with multiple loggers

Browse Source 
Modify the selftest to be able to run each subtest with a list of one or
more test loggers.  The addition of tests that use this capability will
be part of a subsequent commit.

Task-number: QTBUG-20615
Change-Id: Iac3efe8220e8245aa7e5589348d2c86b8034dd28
Reviewed-on: http://codereview.qt-project.org/5292
Reviewed-by: Qt Sanity Bot <qt_sanity_bot@ovi.com>
Reviewed-by: Rohan McGovern <rohan.mcgovern@nokia.com>
This commit is contained in:
Jason McDonald 2011-09-21 11:39:07 +10:00 committed by Qt by Nokia
parent 91ec8261ab
commit 157fc88f90
1 changed files with 170 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

135
tests/auto/testlib/selftests/tst_selftests.cpp
View File

@ -53,7 +53,7 @@ private slots:
void cleanup(); void cleanup();
private: private:
void doRunSubTest(QString const& subdir, QString const& logger, QStringList const& arguments ); void doRunSubTest(QString const& subdir, QStringList const& loggers, QStringList const& arguments);
}; };
struct BenchmarkResult struct BenchmarkResult
@ -142,45 +142,73 @@ static QList<QByteArray> splitLines(QByteArray ba)
return out; return out;
} }
// Return the log format, e.g. for both "stdout txt" and "txt", return "txt'.
static inline QString logFormat(const QString &logger)
{
return (logger.startsWith("stdout") ? logger.mid(7) : logger);
}
// Return the log file name, or an empty string if the log goes to stdout.
static inline QString logName(const QString &logger)
{
return (logger.startsWith("stdout") ? "" : "test_output." + logger);
}
// Load the expected test output for the nominated test (subdir) and logger // Load the expected test output for the nominated test (subdir) and logger
// as an array of lines. If there is no expected output file, return an // as an array of lines. If there is no expected output file, return an
// empty array. // empty array.
static QList<QByteArray> expectedResult(const QString &subdir, const QString &logger) static QList<QByteArray> expectedResult(const QString &subdir, const QString &logger)
{ {
QFile file(":/expected_" + subdir + "." + logger); QFile file(":/expected_" + subdir + "." + logFormat(logger));
if (!file.open(QIODevice::ReadOnly)) if (!file.open(QIODevice::ReadOnly))
return QList<QByteArray>(); return QList<QByteArray>();
return splitLines(file.readAll()); return splitLines(file.readAll());
} }
struct Logger // Each test is run with a set of one or more test output loggers.
// This struct holds information about one such test.
struct LoggerSet
{ {
Logger(QString const&, QStringList const&); LoggerSet(QString const& _name, QStringList const& _loggers, QStringList const& _arguments)
: name(_name), loggers(_loggers), arguments(_arguments)
{ }
QString name; QString name;
QStringList loggers;
QStringList arguments; QStringList arguments;
}; };
Logger::Logger(QString const& _name, QStringList const& _arguments) // This function returns a list of all sets of loggers to be used for
: name(_name) // running each subtest.
, arguments(_arguments) static QList<LoggerSet> allLoggerSets()
{ {
} // For the plain text logger, we'll test logging to file and to standard
// output. For all other loggers (XML), we'll tell testlib to redirect to
static QList<Logger> allLoggers() // file. The reason is that tests are allowed to print to standard output,
{ // and that means the test log is no longer guaranteed to be valid XML.
return QList<Logger>() return QList<LoggerSet>()
<< Logger("txt", QStringList()) << LoggerSet("stdout txt",
<< Logger("xml", QStringList() << "-xml") QStringList() << "stdout txt",
<< Logger("xunitxml", QStringList() << "-xunitxml") QStringList())
<< Logger("lightxml", QStringList() << "-lightxml") << LoggerSet("txt",
QStringList() << "txt",
QStringList() << "-o" << logName("txt"))
<< LoggerSet("xml",
QStringList() << "xml",
QStringList() << "-xml" << "-o" << logName("xml"))
<< LoggerSet("xunitxml",
QStringList() << "xunitxml",
QStringList() << "-xunitxml" << "-o" << logName("xunitxml"))
<< LoggerSet("lightxml",
QStringList() << "lightxml",
QStringList() << "-lightxml" << "-o" << logName("lightxml"))
; ;
} }
void tst_Selftests::runSubTest_data() void tst_Selftests::runSubTest_data()
{ {
QTest::addColumn<QString>("subdir"); QTest::addColumn<QString>("subdir");
QTest::addColumn<QString>("logger"); QTest::addColumn<QStringList>("loggers");
QTest::addColumn<QStringList>("arguments"); QTest::addColumn<QStringList>("arguments");
QStringList tests = QStringList() QStringList tests = QStringList()
@ -243,14 +271,11 @@ void tst_Selftests::runSubTest_data()
<< "badxml" << "badxml"
; ;
foreach (Logger const& logger, allLoggers()) { foreach (LoggerSet const& loggerSet, allLoggerSets()) {
QString rowSuffix; QStringList loggers = loggerSet.loggers;
if (logger.name != "txt") {
rowSuffix = QString(" %1").arg(logger.name);
}
foreach (QString const& subtest, tests) { foreach (QString const& subtest, tests) {
QStringList arguments = logger.arguments; QStringList arguments = loggerSet.arguments;
if (subtest == "commandlinedata") { if (subtest == "commandlinedata") {
arguments << QString("fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2").split(' '); arguments << QString("fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2").split(' ');
} }
@ -274,7 +299,7 @@ void tst_Selftests::runSubTest_data()
// standard output, either because they execute multiple test // standard output, either because they execute multiple test
// objects or because they internally supply arguments to // objects or because they internally supply arguments to
// themselves. // themselves.
if (logger.name != "txt") { if (loggerSet.name != "stdout txt") {
if (subtest == "differentexec") { if (subtest == "differentexec") {
continue; continue;
} }
@ -298,41 +323,35 @@ void tst_Selftests::runSubTest_data()
} }
} }
QTest::newRow(qPrintable(QString("%1%2").arg(subtest).arg(rowSuffix))) QTest::newRow(qPrintable(QString("%1 %2").arg(subtest).arg(loggerSet.name)))
<< subtest << subtest
<< logger.name << loggers
<< arguments << arguments
; ;
} }
} }
} }
void tst_Selftests::doRunSubTest(QString const& subdir, QString const& logger, QStringList const& arguments ) void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& loggers, QStringList const& arguments)
{ {
// For the plain text logger, we'll read straight from standard output.
// For all other loggers (XML), we'll tell testlib to redirect to a file.
// The reason is that tests are allowed to print to standard output, and
// that means the test log is no longer guaranteed to be valid XML.
QStringList extraArguments;
QString logfile;
if (logger != "txt") {
logfile = "test_output";
extraArguments << "-o" << logfile;
}
QProcess proc; QProcess proc;
proc.setEnvironment(QStringList("")); proc.setEnvironment(QStringList(""));
proc.start(subdir + "/" + subdir, QStringList() << arguments << extraArguments); proc.start(subdir + "/" + subdir, arguments);
QVERIFY2(proc.waitForFinished(), qPrintable(proc.errorString())); QVERIFY2(proc.waitForFinished(), qPrintable(proc.errorString()));
QList<QByteArray> actualOutputs;
for (int i = 0; i < loggers.count(); ++i) {
QString logFile = logName(loggers[i]);
QByteArray out; QByteArray out;
if (logfile.isEmpty()) { if (logFile.isEmpty()) {
out = proc.readAllStandardOutput(); out = proc.readAllStandardOutput();
} else { } else {
QFile file(logfile); QFile file(logFile);
if (file.open(QIODevice::ReadOnly)) if (file.open(QIODevice::ReadOnly))
out = file.readAll(); out = file.readAll();
} }
actualOutputs << out;
}
const QByteArray err(proc.readAllStandardError()); const QByteArray err(proc.readAllStandardError());
@ -352,9 +371,14 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QString const& logger, Q
&& subdir != QLatin1String("benchlibcallgrind")) && subdir != QLatin1String("benchlibcallgrind"))
QVERIFY2(err.isEmpty(), err.constData()); QVERIFY2(err.isEmpty(), err.constData());
QList<QByteArray> res = splitLines(out); for (int n = 0; n < loggers.count(); ++n) {
QString logger = loggers[n];
QList<QByteArray> res = splitLines(actualOutputs[n]);
QList<QByteArray> exp = expectedResult(subdir, logger); QList<QByteArray> exp = expectedResult(subdir, logger);
// For the "crashes" test, there are multiple versions of the
// expected output. Load the one with the same line count as
// the actual output.
if (exp.count() == 0) { if (exp.count() == 0) {
QList<QList<QByteArray> > expArr; QList<QList<QByteArray> > expArr;
int i = 1; int i = 1;
@ -374,12 +398,13 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QString const& logger, Q
QCOMPARE(res.count(), exp.count()); QCOMPARE(res.count(), exp.count());
} }
if (logger == "xunitxml" || logger == "xml" || logger == "lightxml") { // For xml output formats, verify that the log is valid XML.
QByteArray xml(out); if (logFormat(logger) == "xunitxml" || logFormat(logger) == "xml" || logFormat(logger) == "lightxml") {
QByteArray xml(actualOutputs[n]);
// lightxml intentionally skips the root element, which technically makes it // lightxml intentionally skips the root element, which technically makes it
// not valid XML. // not valid XML.
// We'll add that ourselves for the purpose of validation. // We'll add that ourselves for the purpose of validation.
if (logger == "lightxml") { if (logFormat(logger) == "lightxml") {
xml.prepend("<root>"); xml.prepend("<root>");
xml.append("</root>"); xml.append("</root>");
} }
@ -396,6 +421,8 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QString const& logger, Q
)); ));
} }
// Verify that the actual output is an acceptable match for the
// expected output.
bool benchmark = false; bool benchmark = false;
for (int i = 0; i < res.count(); ++i) { for (int i = 0; i < res.count(); ++i) {
QByteArray line = res.at(i); QByteArray line = res.at(i);
@ -414,7 +441,8 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QString const& logger, Q
// __FILE__, while others do not. // __FILE__, while others do not.
if (line.contains("ASSERT") && output != expected) { if (line.contains("ASSERT") && output != expected) {
const char msg[] = "Q_ASSERT prints out the absolute path on this platform."; const char msg[] = "Q_ASSERT prints out the absolute path on this platform.";
QEXPECT_FAIL("assert", msg, Continue); QEXPECT_FAIL("assert stdout txt", msg, Continue);
QEXPECT_FAIL("assert txt", msg, Continue);
QEXPECT_FAIL("assert xml", msg, Continue); QEXPECT_FAIL("assert xml", msg, Continue);
QEXPECT_FAIL("assert lightxml", msg, Continue); QEXPECT_FAIL("assert lightxml", msg, Continue);
QEXPECT_FAIL("assert xunitxml", msg, Continue); QEXPECT_FAIL("assert xunitxml", msg, Continue);
@ -445,14 +473,15 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QString const& logger, Q
benchmark = line.startsWith("RESULT : "); benchmark = line.startsWith("RESULT : ");
} }
} }
}
void tst_Selftests::runSubTest() void tst_Selftests::runSubTest()
{ {
QFETCH(QString, subdir); QFETCH(QString, subdir);
QFETCH(QString, logger); QFETCH(QStringList, loggers);
QFETCH(QStringList, arguments); QFETCH(QStringList, arguments);
doRunSubTest(subdir, logger, arguments); doRunSubTest(subdir, loggers, arguments);
} }
// attribute must contain =" // attribute must contain ="
@ -607,8 +636,14 @@ BenchmarkResult BenchmarkResult::parse(QString const& line, QString* error)
void tst_Selftests::cleanup() void tst_Selftests::cleanup()
{ {
// Remove the test output file QFETCH(QStringList, loggers);
QFile::remove("test_output");
// Remove the test output files
for (int i = 0; i < loggers.count(); ++i) {
QString logFile = logName(loggers[i]);
if (!logFile.isEmpty())
QVERIFY(QFile::remove(logFile));
}
} }
QTEST_MAIN(tst_Selftests) QTEST_MAIN(tst_Selftests)