Allow tests to log to multiple destinations

Each destination and the format of output to write there is specified by
adding "-o filename,format" to the command-line.  The special filename
"-" indicates that the log output is written to the standard output
stream, though standard output can be used as a destination at most
once.

The old-style testlib output options are still supported, but can only
be used to specify one logging destination, as before.

If no logging options are given on the command-line, a plain text log
will go to the console, as before.

To log to the console in plain text and to the file "test_output" in
xunit format, one would invoke a test in the following way:

    tst_foo -o test_output,xunitxml -o -,txt

This commit also enhances the selftests to test with multiple loggers,
but negative tests (e.g. bad combinations of command-line options) are
left for future task QTBUG-21567.

Task-number: QTBUG-20615
Change-Id: If91e752bc7001657e15e427aba9d25ab0a29a0b0
Reviewed-on: http://codereview.qt-project.org/4125
Reviewed-by: Qt Sanity Bot <qt_sanity_bot@ovi.com>
Reviewed-by: Rohan McGovern <rohan.mcgovern@nokia.com>
This commit is contained in:
Jason McDonald 2011-09-02 19:08:21 +10:00 committed by Qt by Nokia
parent 157fc88f90
commit 51589e834e
4 changed files with 317 additions and 85 deletions

View File

@ -1004,11 +1004,28 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml)
const char *logFilename = 0;
const char *testOptions =
" Output options:\n"
" -xunitxml : Outputs results as XML XUnit document\n"
" -xml : Outputs results as XML document\n"
" -lightxml : Outputs results as stream of XML tags\n"
" -o filename : Writes all output into a file\n"
" New-style logging options:\n"
" -o filename,format : Output results to file in the specified format\n"
" Use - to output to stdout\n"
" Valid formats are:\n"
" txt : Plain text\n"
" xunitxml : XML XUnit document\n"
" xml : XML document\n"
" lightxml : A stream of XML tags\n"
"\n"
" *** Multiple loggers can be specified, but at most one can log to stdout.\n"
"\n"
" Old-style logging options:\n"
" -o filename : Write the output into file\n"
" -txt : Output results in Plain Text\n"
" -xunitxml : Output results as XML XUnit document\n"
" -xml : Output results as XML document\n"
" -lightxml : Output results as stream of XML tags\n"
"\n"
" *** If no output file is specified, stdout is assumed.\n"
" *** If no output format is specified, -txt is assumed.\n"
"\n"
" Detail options:\n"
" -silent : Only outputs warnings and failures\n"
" -v1 : Print enter messages for each testfunction\n"
" -v2 : Also print out each QVERIFY/QCOMPARE/QTEST\n"
@ -1065,6 +1082,8 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml)
qPrintTestSlots(stdout);
exit(0);
}
} else if (strcmp(argv[i], "-txt") == 0) {
logFormat = QTestLog::Plain;
} else if (strcmp(argv[i], "-xunitxml") == 0) {
logFormat = QTestLog::XunitXML;
} else if (strcmp(argv[i], "-xml") == 0) {
@ -1081,11 +1100,38 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml)
QSignalDumper::startDump();
} else if (strcmp(argv[i], "-o") == 0) {
if (i + 1 >= argc) {
fprintf(stderr, "-o needs an extra parameter specifying the filename\n");
fprintf(stderr, "-o needs an extra parameter specifying the filename and optional format\n");
exit(1);
} else {
logFilename = argv[++i];
}
++i;
// Do we have the old or new style -o option?
char *filename = new char[strlen(argv[i])+1];
char *format = new char[strlen(argv[i])+1];
if (sscanf(argv[i], "%[^,],%s", filename, format) == 1) {
// Old-style
logFilename = argv[i];
} else {
// New-style
if (strcmp(format, "txt") == 0)
logFormat = QTestLog::Plain;
else if (strcmp(format, "lightxml") == 0)
logFormat = QTestLog::LightXML;
else if (strcmp(format, "xml") == 0)
logFormat = QTestLog::XML;
else if (strcmp(format, "xunitxml") == 0)
logFormat = QTestLog::XunitXML;
else {
fprintf(stderr, "output format must be one of txt, lightxml, xml or xunitxml\n");
exit(1);
}
if (strcmp(filename, "-") == 0 && QTestLog::loggerUsingStdout()) {
fprintf(stderr, "only one logger can log to stdout\n");
exit(1);
}
QTestLog::addLogger(logFormat, filename);
}
delete [] filename;
delete [] format;
} else if (strcmp(argv[i], "-eventdelay") == 0) {
if (i + 1 >= argc) {
fprintf(stderr, "-eventdelay needs an extra parameter to indicate the delay(ms)\n");
@ -1248,8 +1294,11 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml)
}
}
// Create the logger
QTestLog::initLogger(logFormat, logFilename);
// If no loggers were created by the long version of the -o command-line
// option, create a logger using whatever filename and format were
// set using the old-style command-line options.
if (QTestLog::loggerCount() == 0)
QTestLog::addLogger(logFormat, logFilename);
}
QBenchmarkResult qMedian(const QList<QBenchmarkResult> &container)

View File

@ -82,12 +82,103 @@ namespace QTest {
static IgnoreResultList *ignoreResultList = 0;
static QTestLog::LogMode logMode = QTestLog::Plain;
struct LoggerList
{
QAbstractTestLogger *logger;
LoggerList *next;
};
class TestLoggers
{
public:
static void addLogger(QAbstractTestLogger *logger)
{
LoggerList *l = new LoggerList;
l->logger = logger;
l->next = loggers;
loggers = l;
}
static void destroyLoggers()
{
while (loggers) {
LoggerList *l = loggers;
loggers = loggers->next;
delete l->logger;
delete l;
}
}
#define FOREACH_LOGGER(operation) \
LoggerList *l = loggers; \
while (l) { \
QAbstractTestLogger *logger = l->logger; \
Q_UNUSED(logger); \
operation; \
l = l->next; \
}
static void startLogging()
{
FOREACH_LOGGER(logger->startLogging());
}
static void stopLogging()
{
FOREACH_LOGGER(logger->stopLogging());
}
static void enterTestFunction(const char *function)
{
FOREACH_LOGGER(logger->enterTestFunction(function));
}
static void leaveTestFunction()
{
FOREACH_LOGGER(logger->leaveTestFunction());
}
static void addIncident(QAbstractTestLogger::IncidentTypes type, const char *description,
const char *file = 0, int line = 0)
{
FOREACH_LOGGER(logger->addIncident(type, description, file, line));
}
static void addBenchmarkResult(const QBenchmarkResult &result)
{
FOREACH_LOGGER(logger->addBenchmarkResult(result));
}
static void addMessage(QAbstractTestLogger::MessageTypes type, const char *message,
const char *file = 0, int line = 0)
{
FOREACH_LOGGER(logger->addMessage(type, message, file, line));
}
static void outputString(const char *msg)
{
FOREACH_LOGGER(logger->outputString(msg));
}
static int loggerCount()
{
int count = 0;
FOREACH_LOGGER(++count);
return count;
}
private:
static LoggerList *loggers;
};
#undef FOREACH_LOGGER
LoggerList *TestLoggers::loggers = 0;
static bool loggerUsingStdout = false;
static int verbosity = 0;
static int maxWarnings = 2002;
static QAbstractTestLogger *testLogger = 0;
static QtMsgHandler oldMessageHandler;
static bool handleIgnoredMessage(QtMsgType type, const char *msg)
@ -118,11 +209,11 @@ namespace QTest {
{
static QBasicAtomicInt counter = Q_BASIC_ATOMIC_INITIALIZER(QTest::maxWarnings);
if (!msg || !QTest::testLogger) {
if (!msg || QTest::TestLoggers::loggerCount() == 0) {
// if this goes wrong, something is seriously broken.
qInstallMsgHandler(oldMessageHandler);
QTEST_ASSERT(msg);
QTEST_ASSERT(QTest::testLogger);
QTEST_ASSERT(QTest::TestLoggers::loggerCount() != 0);
}
if (handleIgnoredMessage(type, msg))
@ -134,7 +225,7 @@ namespace QTest {
return;
if (!counter.deref()) {
QTest::testLogger->addMessage(QAbstractTestLogger::QSystem,
QTest::TestLoggers::addMessage(QAbstractTestLogger::QSystem,
"Maximum amount of warnings exceeded. Use -maxwarnings to override.");
return;
}
@ -142,16 +233,16 @@ namespace QTest {
switch (type) {
case QtDebugMsg:
QTest::testLogger->addMessage(QAbstractTestLogger::QDebug, msg);
QTest::TestLoggers::addMessage(QAbstractTestLogger::QDebug, msg);
break;
case QtCriticalMsg:
QTest::testLogger->addMessage(QAbstractTestLogger::QSystem, msg);
QTest::TestLoggers::addMessage(QAbstractTestLogger::QSystem, msg);
break;
case QtWarningMsg:
QTest::testLogger->addMessage(QAbstractTestLogger::QWarning, msg);
QTest::TestLoggers::addMessage(QAbstractTestLogger::QWarning, msg);
break;
case QtFatalMsg:
QTest::testLogger->addMessage(QAbstractTestLogger::QFatal, msg);
QTest::TestLoggers::addMessage(QAbstractTestLogger::QFatal, msg);
/* Right now, we're inside the custom message handler and we're
* being qt_message_output in qglobal.cpp. After we return from
* this function, it will proceed with calling exit() and abort()
@ -167,10 +258,9 @@ namespace QTest {
void QTestLog::enterTestFunction(const char* function)
{
QTEST_ASSERT(QTest::testLogger);
QTEST_ASSERT(function);
QTest::testLogger->enterTestFunction(function);
QTest::TestLoggers::enterTestFunction(function);
}
int QTestLog::unhandledIgnoreMessages()
@ -186,21 +276,17 @@ int QTestLog::unhandledIgnoreMessages()
void QTestLog::leaveTestFunction()
{
QTEST_ASSERT(QTest::testLogger);
QTest::IgnoreResultList::clearList(QTest::ignoreResultList);
QTest::testLogger->leaveTestFunction();
QTest::TestLoggers::leaveTestFunction();
}
void QTestLog::printUnhandledIgnoreMessages()
{
QTEST_ASSERT(QTest::testLogger);
char msg[1024];
QTest::IgnoreResultList *list = QTest::ignoreResultList;
while (list) {
QTest::qt_snprintf(msg, 1024, "Did not receive message: \"%s\"", list->msg);
QTest::testLogger->addMessage(QAbstractTestLogger::Info, msg);
QTest::TestLoggers::addMessage(QAbstractTestLogger::Info, msg);
list = list->next;
}
@ -208,110 +294,110 @@ void QTestLog::printUnhandledIgnoreMessages()
void QTestLog::addPass(const char *msg)
{
QTEST_ASSERT(QTest::testLogger);
QTEST_ASSERT(msg);
QTest::testLogger->addIncident(QAbstractTestLogger::Pass, msg);
QTest::TestLoggers::addIncident(QAbstractTestLogger::Pass, msg);
}
void QTestLog::addFail(const char *msg, const char *file, int line)
{
QTEST_ASSERT(QTest::testLogger);
QTEST_ASSERT(msg);
QTest::testLogger->addIncident(QAbstractTestLogger::Fail, msg, file, line);
QTest::TestLoggers::addIncident(QAbstractTestLogger::Fail, msg, file, line);
}
void QTestLog::addXFail(const char *msg, const char *file, int line)
{
QTEST_ASSERT(QTest::testLogger);
QTEST_ASSERT(msg);
QTEST_ASSERT(file);
QTest::testLogger->addIncident(QAbstractTestLogger::XFail, msg, file, line);
QTest::TestLoggers::addIncident(QAbstractTestLogger::XFail, msg, file, line);
}
void QTestLog::addXPass(const char *msg, const char *file, int line)
{
QTEST_ASSERT(QTest::testLogger);
QTEST_ASSERT(msg);
QTEST_ASSERT(file);
QTest::testLogger->addIncident(QAbstractTestLogger::XPass, msg, file, line);
QTest::TestLoggers::addIncident(QAbstractTestLogger::XPass, msg, file, line);
}
void QTestLog::addSkip(const char *msg, const char *file, int line)
{
QTEST_ASSERT(QTest::testLogger);
QTEST_ASSERT(msg);
QTEST_ASSERT(file);
QTest::testLogger->addMessage(QAbstractTestLogger::Skip, msg, file, line);
QTest::TestLoggers::addMessage(QAbstractTestLogger::Skip, msg, file, line);
}
void QTestLog::addBenchmarkResult(const QBenchmarkResult &result)
{
QTEST_ASSERT(QTest::testLogger);
QTest::testLogger->addBenchmarkResult(result);
QTest::TestLoggers::addBenchmarkResult(result);
}
void QTestLog::startLogging()
{
QTEST_ASSERT(QTest::testLogger);
QTest::testLogger->startLogging();
QTest::TestLoggers::startLogging();
QTest::oldMessageHandler = qInstallMsgHandler(QTest::messageHandler);
}
void QTestLog::stopLogging()
{
qInstallMsgHandler(QTest::oldMessageHandler);
QTEST_ASSERT(QTest::testLogger);
QTest::testLogger->stopLogging();
delete QTest::testLogger;
QTest::testLogger = 0;
QTest::TestLoggers::stopLogging();
QTest::TestLoggers::destroyLoggers();
QTest::loggerUsingStdout = false;
}
void QTestLog::initLogger(LogMode mode, const char *filename)
void QTestLog::addLogger(LogMode mode, const char *filename)
{
QTest::logMode = mode;
if (filename && strcmp(filename, "-") == 0)
filename = 0;
if (!filename)
QTest::loggerUsingStdout = true;
QAbstractTestLogger *logger = 0;
switch (mode) {
case QTestLog::Plain:
QTest::testLogger = new QPlainTestLogger(filename);
logger = new QPlainTestLogger(filename);
break;
case QTestLog::XML:
QTest::testLogger = new QXmlTestLogger(QXmlTestLogger::Complete, filename);
logger = new QXmlTestLogger(QXmlTestLogger::Complete, filename);
break;
case QTestLog::LightXML:
QTest::testLogger = new QXmlTestLogger(QXmlTestLogger::Light, filename);
logger = new QXmlTestLogger(QXmlTestLogger::Light, filename);
break;
case QTestLog::XunitXML:
QTest::testLogger = new QXunitTestLogger(filename);
logger = new QXunitTestLogger(filename);
break;
}
QTEST_ASSERT(QTest::testLogger);
QTEST_ASSERT(logger);
QTest::TestLoggers::addLogger(logger);
}
int QTestLog::loggerCount()
{
return QTest::TestLoggers::loggerCount();
}
bool QTestLog::loggerUsingStdout()
{
return QTest::loggerUsingStdout;
}
void QTestLog::warn(const char *msg)
{
QTEST_ASSERT(QTest::testLogger);
QTEST_ASSERT(msg);
QTest::testLogger->addMessage(QAbstractTestLogger::Warn, msg);
if (QTest::TestLoggers::loggerCount() > 0)
QTest::TestLoggers::addMessage(QAbstractTestLogger::Warn, msg);
}
void QTestLog::info(const char *msg, const char *file, int line)
{
QTEST_ASSERT(msg);
if (QTest::testLogger)
QTest::testLogger->addMessage(QAbstractTestLogger::Info, msg, file, line);
}
QTestLog::LogMode QTestLog::logMode()
{
return QTest::logMode;
QTest::TestLoggers::addMessage(QAbstractTestLogger::Info, msg, file, line);
}
void QTestLog::setVerboseLevel(int level)

View File

@ -83,9 +83,10 @@ public:
static void startLogging();
static void stopLogging();
static void initLogger(LogMode mode, const char *filename);
static void addLogger(LogMode mode, const char *filename);
static LogMode logMode();
static int loggerCount();
static bool loggerUsingStdout();
static void setVerboseLevel(int level);
static int verboseLevel();

View File

@ -182,26 +182,106 @@ struct LoggerSet
// running each subtest.
static QList<LoggerSet> allLoggerSets()
{
// For the plain text logger, we'll test logging to file and to standard
// output. For all other loggers (XML), we'll tell testlib to redirect to
// file. The reason is that tests are allowed to print to standard output,
// and that means the test log is no longer guaranteed to be valid XML.
// Note that in order to test XML output to standard output, the subtests
// must not send output directly to stdout, bypassing Qt's output mechanisms
// (e.g. via printf), otherwise the output may not be well-formed XML.
return QList<LoggerSet>()
<< LoggerSet("stdout txt",
// Test with old-style options for a single logger
<< LoggerSet("old stdout txt",
QStringList() << "stdout txt",
QStringList())
<< LoggerSet("txt",
QStringList()
)
<< LoggerSet("old txt",
QStringList() << "txt",
QStringList() << "-o" << logName("txt"))
<< LoggerSet("xml",
QStringList() << "-o" << logName("txt")
)
<< LoggerSet("old stdout xml",
QStringList() << "stdout xml",
QStringList() << "-xml"
)
<< LoggerSet("old xml",
QStringList() << "xml",
QStringList() << "-xml" << "-o" << logName("xml"))
<< LoggerSet("xunitxml",
QStringList() << "-xml" << "-o" << logName("xml")
)
<< LoggerSet("old stdout xunitxml",
QStringList() << "stdout xunitxml",
QStringList() << "-xunitxml"
)
<< LoggerSet("old xunitxml",
QStringList() << "xunitxml",
QStringList() << "-xunitxml" << "-o" << logName("xunitxml"))
<< LoggerSet("lightxml",
QStringList() << "-xunitxml" << "-o" << logName("xunitxml")
)
<< LoggerSet("old stdout lightxml",
QStringList() << "stdout lightxml",
QStringList() << "-lightxml"
)
<< LoggerSet("old lightxml",
QStringList() << "lightxml",
QStringList() << "-lightxml" << "-o" << logName("lightxml"))
QStringList() << "-lightxml" << "-o" << logName("lightxml")
)
// Test with new-style options for a single logger
<< LoggerSet("new stdout txt",
QStringList() << "stdout txt",
QStringList() << "-o" << "-,txt"
)
<< LoggerSet("new txt",
QStringList() << "txt",
QStringList() << "-o" << logName("txt")+",txt"
)
<< LoggerSet("new stdout xml",
QStringList() << "stdout xml",
QStringList() << "-o" << "-,xml"
)
<< LoggerSet("new xml",
QStringList() << "xml",
QStringList() << "-o" << logName("xml")+",xml"
)
<< LoggerSet("new stdout xunitxml",
QStringList() << "stdout xunitxml",
QStringList() << "-o" << "-,xunitxml"
)
<< LoggerSet("new xunitxml",
QStringList() << "xunitxml",
QStringList() << "-o" << logName("xunitxml")+",xunitxml"
)
<< LoggerSet("new stdout lightxml",
QStringList() << "stdout lightxml",
QStringList() << "-o" << "-,lightxml"
)
<< LoggerSet("new lightxml",
QStringList() << "lightxml",
QStringList() << "-o" << logName("lightxml")+",lightxml"
)
// Test with two loggers (don't test all 32 combinations, just a sample)
<< LoggerSet("stdout txt + txt",
QStringList() << "stdout txt" << "txt",
QStringList() << "-o" << "-,txt"
<< "-o" << logName("txt")+",txt"
)
<< LoggerSet("xml + stdout txt",
QStringList() << "xml" << "stdout txt",
QStringList() << "-o" << logName("xml")+",xml"
<< "-o" << "-,txt"
)
<< LoggerSet("txt + xunitxml",
QStringList() << "txt" << "xunitxml",
QStringList() << "-o" << logName("txt")+",txt"
<< "-o" << logName("xunitxml")+",xunitxml"
)
<< LoggerSet("lightxml + stdout xunitxml",
QStringList() << "lightxml" << "stdout xunitxml",
QStringList() << "-o" << logName("lightxml")+",lightxml"
<< "-o" << "-,xunitxml"
)
// All loggers at the same time
<< LoggerSet("all loggers",
QStringList() << "txt" << "xml" << "lightxml" << "stdout txt" << "xunitxml",
QStringList() << "-o" << logName("txt")+",txt"
<< "-o" << logName("xml")+",xml"
<< "-o" << logName("lightxml")+",lightxml"
<< "-o" << "-,txt"
<< "-o" << logName("xunitxml")+",xunitxml"
)
;
}
@ -299,7 +379,7 @@ void tst_Selftests::runSubTest_data()
// standard output, either because they execute multiple test
// objects or because they internally supply arguments to
// themselves.
if (loggerSet.name != "stdout txt") {
if (loggerSet.name != "old stdout txt" && loggerSet.name != "new stdout txt") {
if (subtest == "differentexec") {
continue;
}
@ -441,11 +521,27 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge
// __FILE__, while others do not.
if (line.contains("ASSERT") && output != expected) {
const char msg[] = "Q_ASSERT prints out the absolute path on this platform.";
QEXPECT_FAIL("assert stdout txt", msg, Continue);
QEXPECT_FAIL("assert txt", msg, Continue);
QEXPECT_FAIL("assert xml", msg, Continue);
QEXPECT_FAIL("assert lightxml", msg, Continue);
QEXPECT_FAIL("assert xunitxml", msg, Continue);
QEXPECT_FAIL("assert old stdout txt", msg, Continue);
QEXPECT_FAIL("assert old txt", msg, Continue);
QEXPECT_FAIL("assert old stdout xml", msg, Continue);
QEXPECT_FAIL("assert old xml", msg, Continue);
QEXPECT_FAIL("assert old stdout lightxml", msg, Continue);
QEXPECT_FAIL("assert old lightxml", msg, Continue);
QEXPECT_FAIL("assert old stdout xunitxml", msg, Continue);
QEXPECT_FAIL("assert old xunitxml", msg, Continue);
QEXPECT_FAIL("assert new stdout txt", msg, Continue);
QEXPECT_FAIL("assert new txt", msg, Continue);
QEXPECT_FAIL("assert new stdout xml", msg, Continue);
QEXPECT_FAIL("assert new xml", msg, Continue);
QEXPECT_FAIL("assert new stdout lightxml", msg, Continue);
QEXPECT_FAIL("assert new lightxml", msg, Continue);
QEXPECT_FAIL("assert new stdout xunitxml", msg, Continue);
QEXPECT_FAIL("assert new xunitxml", msg, Continue);
QEXPECT_FAIL("assert stdout txt + txt", msg, Continue);
QEXPECT_FAIL("assert xml + stdout txt", msg, Continue);
QEXPECT_FAIL("assert txt + xunitxml", msg, Continue);
QEXPECT_FAIL("assert lightxml + stdout xunitxml", msg, Continue);
QEXPECT_FAIL("assert all loggers", msg, Continue);
}
if (expected.startsWith(QLatin1String("FAIL! : tst_Exception::throwException() Caught unhandled exce")) && expected != output)