diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp index 2ef7258d81f..a9c5d407345 100644 --- a/tests/auto/testlib/selftests/tst_selftests.cpp +++ b/tests/auto/testlib/selftests/tst_selftests.cpp @@ -62,11 +62,22 @@ private slots: private: void doRunSubTest(QString const& subdir, QStringList const& loggers, QStringList const& arguments, bool crashes); + bool compareOutput(const QString &logger, const QString &subdir, + const QByteArray &rawOutput, const QByteArrayList &actual, + const QByteArrayList &expected, + QString *errorMessage) const; + bool compareLine(const QString &logger, const QString &subdir, bool benchmark, + const QString &actualLine, const QString &expLine, + QString *errorMessage) const; + bool checkXml(const QString &logger, QByteArray rawOutput, + QString *errorMessage) const; + QString logName(const QString &logger) const; QList allLoggerSets() const; QTemporaryDir tempDir; QRegularExpression durationRegExp; + QRegularExpression teamcityLocRegExp; }; struct BenchmarkResult @@ -81,46 +92,50 @@ struct BenchmarkResult static BenchmarkResult parse(QString const&, QString*); }; -QT_BEGIN_NAMESPACE -namespace QTest +static QString msgMismatch(const QString &actual, const QString &expected) { -template <> -inline bool qCompare - (BenchmarkResult const &r1, BenchmarkResult const &r2, - const char* actual, const char* expected, const char* file, int line) + return QLatin1String("Mismatch:\n'") + actual + QLatin1String("'\n !=\n'") + + expected + QLatin1Char('\''); +} + +static bool compareBenchmarkResult(BenchmarkResult const &r1, BenchmarkResult const &r2, + QString *errorMessage) { // First make sure the iterations and unit match. if (r1.iterations != r2.iterations || r1.unit != r2.unit) { // Nope - compare whole string for best failure message - return qCompare(r1.toString(), r2.toString(), actual, expected, file, line); + *errorMessage = msgMismatch(r1.toString(), r2.toString()); + return false; } // Now check the value. Some variance is allowed, and how much depends on // the measured unit. qreal variance = 0.; - if (r1.unit == "msecs" || r1.unit == "WalltimeMilliseconds") { + if (r1.unit == QLatin1String("msecs") || r1.unit == QLatin1String("WalltimeMilliseconds")) variance = 0.1; - } - else if (r1.unit == "instruction reads") { + else if (r1.unit == QLatin1String("instruction reads")) variance = 0.001; - } - else if (r1.unit == "CPU ticks" || r1.unit == "CPUTicks") { + else if (r1.unit == QLatin1String("CPU ticks") || r1.unit == QLatin1String("CPUTicks")) variance = 0.001; - } + if (variance == 0.) { // No variance allowed - compare whole string - return qCompare(r1.toString(), r2.toString(), actual, expected, file, line); + const QString r1S = r1.toString(); + const QString r2S = r2.toString(); + if (r1S != r2S) { + *errorMessage = msgMismatch(r1S, r2S); + return false; + } + return true; } - if (qAbs(qreal(r1.total) - qreal(r2.total)) <= qreal(r1.total)*variance) { - return compare_helper(true, 0, 0, 0, actual, expected, file, line); + if (qAbs(qreal(r1.total) - qreal(r2.total)) > qreal(r1.total) * variance) { + // Whoops, didn't match. Compare the whole string for the most useful failure message. + *errorMessage = msgMismatch(r1.toString(), r2.toString()); + return false; } - - // Whoops, didn't match. Compare the whole string for the most useful failure message. - return qCompare(r1.toString(), r2.toString(), actual, expected, file, line); + return true; } -} -QT_END_NAMESPACE // Split the passed block of text into an array of lines, replacing any // filenames and line numbers with generic markers to avoid failing the test @@ -227,26 +242,6 @@ static QByteArray runDiff(const QByteArrayList &expected, const QByteArrayList & return result; } -// Print the difference preferably using 'diff', else just print lines -static void printDifference(const QByteArrayList &expected, const QByteArrayList &actual) -{ - QDebug info = qInfo(); - info.noquote(); - info.nospace(); - const QByteArray diff = runDiff(expected, actual); - if (diff.isEmpty()) { - info << "<<<<<<\n"; - for (const QByteArray &line : actual) - info << line << '\n'; - info << "======\n"; - for (const QByteArray &line : expected) - info << line << '\n'; - info << ">>>>>>\n"; - } else { - info << diff; - } -} - // Each test is run with a set of one or more test output loggers. // This struct holds information about one such test. struct LoggerSet @@ -399,6 +394,7 @@ QList tst_Selftests::allLoggerSets() const tst_Selftests::tst_Selftests() : tempDir(QDir::tempPath() + "/tst_selftests.XXXXXX") , durationRegExp("") + , teamcityLocRegExp("\\|\\[Loc: .*\\(\\d*\\)\\|\\]") {} void tst_Selftests::initTestCase() @@ -760,149 +756,198 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge } QList res = splitLines(actualOutputs[n]); - const QString expectedFileName = expectedFileNameFromTest(subdir, logger); - QList exp = expectedResult(expectedFileName); + QString errorMessage; + QString expectedFileName = expectedFileNameFromTest(subdir, logger); + if (QFileInfo::exists(expectedFileName)) { + QByteArrayList exp = expectedResult(expectedFileName); #ifdef Q_CC_MINGW - // MinGW formats double numbers differently - if (n == 0 && subdir == QStringLiteral("float")) { - for (int i = 0; i < exp.size(); ++i) { - exp[i].replace("e-07", "e-007"); - exp[i].replace("e+07", "e+007"); - } - } -#endif - - // For the "crashes" test, there are multiple versions of the - // expected output. Load the one with the same line count as - // the actual output. - if (exp.count() == 0) { - QList > expArr; - QList tmp; - int i = 1; - do { - tmp = expectedResult(expectedFileNameFromTest(subdir + QLatin1Char('_') + QString::number(i++), logger)); - if (tmp.count()) - expArr += tmp; - } while (tmp.count()); - - for (int j = 0; j < expArr.count(); ++j) { - if (res.count() == expArr.at(j).count()) { - exp = expArr.at(j); - break; + // MinGW formats double numbers differently (last verified with 7.1) + if (n == 0 && subdir == QStringLiteral("float")) { + for (int i = 0; i < exp.size(); ++i) { + exp[i].replace("e-07", "e-007"); + exp[i].replace("e+07", "e+007"); } } - - if (expArr.count()) { - QVERIFY2(exp.count(), - qPrintable(QString::fromLatin1("None of the expected output files for " - "%1 format has matching line count.") - .arg(loggers.at(n)))); +#endif + if (!compareOutput(logger, subdir, actualOutputs[n], res, exp, &errorMessage)) { + errorMessage.prepend(QLatin1Char('"') + logger + QLatin1String("\", ") + + expectedFileName + QLatin1Char(' ')); + errorMessage += QLatin1String("\nActual:\n") + QLatin1String(actualOutputs[n]); + const QByteArray diff = runDiff(exp, res); + if (!diff.isEmpty()) + errorMessage += QLatin1String("\nDiff:\n") + QLatin1String(diff); + QFAIL(qPrintable(errorMessage)); } } else { - if (res.count() != exp.count()) { - printDifference(exp, res); - QVERIFY2(res.count() == exp.count(), - qPrintable(QString::fromLatin1("Mismatch in line count: %1 != %2 (%3, %4).") - .arg(res.count()).arg(exp.count()).arg(loggers.at(n), expectedFileName))); + // For the "crashes" and other tests, there are multiple versions of the + // expected output. Loop until a matching one is found. + bool ok = false; + for (int i = 1; !ok; ++i) { + expectedFileName = expectedFileNameFromTest(subdir + QLatin1Char('_') + QString::number(i), logger); + const QByteArrayList exp = expectedResult(expectedFileName); + if (exp.isEmpty()) + break; + QString errorMessage2; + ok = compareOutput(logger, subdir, actualOutputs[n], res, exp, &errorMessage2); + if (!ok) + errorMessage += QLatin1Char('\n') + expectedFileName + QLatin1String(": ") + errorMessage2; } - } - - // By this point, we should have loaded a non-empty expected data file. - QVERIFY2(exp.count(), - qPrintable(QString::fromLatin1("Expected test data for %1 format is empty or not found.") - .arg(loggers.at(n)))); - - // For xml output formats, verify that the log is valid XML. - if (logFormat(logger) == "xunitxml" || logFormat(logger) == "xml" || logFormat(logger) == "lightxml") { - QByteArray xml(actualOutputs[n]); - // lightxml intentionally skips the root element, which technically makes it - // not valid XML. - // We'll add that ourselves for the purpose of validation. - if (logFormat(logger) == "lightxml") { - xml.prepend(""); - xml.append(""); + if (!ok) { // Use QDebug's quote mechanism to report potentially garbled output. + errorMessage.prepend(QLatin1String("Cannot find a matching file for ") + subdir); + errorMessage += QLatin1String("\nActual:\n"); + QDebug(&errorMessage) << actualOutputs[n]; + QFAIL(qPrintable(errorMessage)); } - - QXmlStreamReader reader(xml); - - while (!reader.atEnd()) - reader.readNext(); - - QVERIFY2(!reader.error(), qPrintable(QString("line %1, col %2: %3") - .arg(reader.lineNumber()) - .arg(reader.columnNumber()) - .arg(reader.errorString()) - )); - } - - // Verify that the actual output is an acceptable match for the - // expected output. - bool benchmark = false; - for (int i = 0; i < res.count(); ++i) { - QByteArray line = res.at(i); - // the __FILE__ __LINE__ output is compiler dependent, skip it - if (line.startsWith(" Loc: [") && line.endsWith(")]")) - continue; - if (line.endsWith(" : failure location")) - continue; - - if (line.startsWith("Config: Using QtTest library") // Text build string - || line.startsWith(" "))) { // XUNIT-XML build string - continue; - } - - QByteArray expLine = exp.at(i); - - // Special handling for ignoring _FILE_ and _LINE_ if logger is teamcity - if (logFormat(logger) == "teamcity") { - QRegularExpression teamcityLocRegExp("\\|\\[Loc: .*\\(\\d*\\)\\|\\]"); - line = QString(line).replace(teamcityLocRegExp, "|[Loc: _FILE_(_LINE_)|]").toLatin1(); - expLine = QString(expLine).replace(teamcityLocRegExp, "|[Loc: _FILE_(_LINE_)|]").toLatin1(); - } - - const QString output(QString::fromLatin1(line)); - const QString expected(QString::fromLatin1(expLine).replace("@INSERT_QT_VERSION_HERE@", QT_VERSION_STR)); - - if (subdir == "assert" && output.contains("ASSERT: ") && expected.contains("ASSERT: ") && output != expected) - // Q_ASSERT uses __FILE__, the exact contents of which are - // undefined. If we something that looks like a Q_ASSERT and we - // were expecting to see a Q_ASSERT, we'll skip the line. - continue; - else if (expected.startsWith(QLatin1String("FAIL! : tst_Exception::throwException() Caught unhandled exce")) && expected != output) - // On some platforms we compile without RTTI, and as a result we never throw an exception. - QCOMPARE(output.simplified(), QString::fromLatin1("tst_Exception::throwException()").simplified()); - else if (benchmark || line.startsWith(" 0) - line.truncate(lastCommaPos); // Plain text logger: strip time (", 2323dms"). - } else { - QVERIFY2(output == expected, - qPrintable(QString::fromLatin1("Mismatch at line %1 (%2, %3):\n'%4'\n !=\n'%5'") - .arg(i + 1).arg(loggers.at(n), expectedFileName, output, expected))); - } - - benchmark = line.startsWith("RESULT : "); } } } +static QString teamCityLocation() { return QStringLiteral("|[Loc: _FILE_(_LINE_)|]"); } +static QString qtVersionPlaceHolder() { return QStringLiteral("@INSERT_QT_VERSION_HERE@"); } + +bool tst_Selftests::compareOutput(const QString &logger, const QString &subdir, + const QByteArray &rawOutput, const QByteArrayList &actual, + const QByteArrayList &expected, + QString *errorMessage) const +{ + + if (actual.size() != expected.size()) { + *errorMessage = QString::fromLatin1("Mismatch in line count: %1 != %2.") + .arg(actual.size()).arg(expected.size()); + return false; + } + + // For xml output formats, verify that the log is valid XML. + if (logger.endsWith(QLatin1String("xml")) && !checkXml(logger, rawOutput, errorMessage)) + return false; + + // Verify that the actual output is an acceptable match for the + // expected output. + + const QString qtVersion = QLatin1String(QT_VERSION_STR); + bool benchmark = false; + for (int i = 0, size = actual.size(); i < size; ++i) { + const QByteArray &actualLineBA = actual.at(i); + // the __FILE__ __LINE__ output is compiler dependent, skip it + if (actualLineBA.startsWith(" Loc: [") && actualLineBA.endsWith(")]")) + continue; + if (actualLineBA.endsWith(" : failure location")) + continue; + + if (actualLineBA.startsWith("Config: Using QtTest library") // Text build string + || actualLineBA.startsWith(" "))) { // XUNIT-XML build string + continue; + } + + QString actualLine = QString::fromLatin1(actualLineBA); + QString expectedLine = QString::fromLatin1(expected.at(i)); + expectedLine.replace(qtVersionPlaceHolder(), qtVersion); + + // Special handling for ignoring _FILE_ and _LINE_ if logger is teamcity + if (logger.endsWith(QLatin1String("teamcity"))) { + actualLine.replace(teamcityLocRegExp, teamCityLocation()); + expectedLine.replace(teamcityLocRegExp, teamCityLocation()); + } + + if (!compareLine(logger, subdir, benchmark, actualLine, + expectedLine, errorMessage)) { + errorMessage->prepend(QLatin1String("Line ") + QString::number(i + 1) + + QLatin1String(": ")); + return false; + } + + benchmark = actualLineBA.startsWith("RESULT : "); + } + return true; +} + +bool tst_Selftests::compareLine(const QString &logger, const QString &subdir, + bool benchmark, + const QString &actualLine, const QString &expectedLine, + QString *errorMessage) const +{ + if (subdir == QLatin1String("assert") && actualLine.contains(QLatin1String("ASSERT: ")) + && expectedLine.contains(QLatin1String("ASSERT: ")) && actualLine != expectedLine) { + // Q_ASSERT uses __FILE__, the exact contents of which are + // undefined. If have we something that looks like a Q_ASSERT and we + // were expecting to see a Q_ASSERT, we'll skip the line. + return true; + } + + if (expectedLine.startsWith(QLatin1String("FAIL! : tst_Exception::throwException() Caught unhandled exce")) + && actualLine != expectedLine) { + // On some platforms we compile without RTTI, and as a result we never throw an exception + if (actualLine.simplified() != QLatin1String("tst_Exception::throwException()")) { + *errorMessage = QString::fromLatin1("'%1' != 'tst_Exception::throwException()'").arg(actualLine); + return false; + } + return true; + } + + if (benchmark || actualLine.startsWith(QLatin1String(""); + xml.append(""); + } + + QXmlStreamReader reader(xml); + while (!reader.atEnd()) + reader.readNext(); + + if (reader.hasError()) { + const int lineNumber = int(reader.lineNumber()); + const QByteArray line = xml.split('\n').value(lineNumber - 1); + *errorMessage = QString::fromLatin1("line %1, col %2 '%3': %4") + .arg(lineNumber).arg(reader.columnNumber()) + .arg(QString::fromLatin1(line), reader.errorString()); + return false; + } + return true; +} + #endif // QT_CONFIG(process) void tst_Selftests::runSubTest()