diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp
index 2ef7258d81f..a9c5d407345 100644
--- a/tests/auto/testlib/selftests/tst_selftests.cpp
+++ b/tests/auto/testlib/selftests/tst_selftests.cpp
@@ -62,11 +62,22 @@ private slots:
 
 private:
     void doRunSubTest(QString const& subdir, QStringList const& loggers, QStringList const& arguments, bool crashes);
+    bool compareOutput(const QString &logger, const QString &subdir,
+                       const QByteArray &rawOutput, const QByteArrayList &actual,
+                       const QByteArrayList &expected,
+                       QString *errorMessage) const;
+    bool compareLine(const QString &logger, const QString &subdir, bool benchmark,
+                     const QString &actualLine, const QString &expLine,
+                     QString *errorMessage) const;
+    bool checkXml(const QString &logger, QByteArray rawOutput,
+                  QString *errorMessage) const;
+
     QString logName(const QString &logger) const;
     QList<LoggerSet> allLoggerSets() const;
 
     QTemporaryDir tempDir;
     QRegularExpression durationRegExp;
+    QRegularExpression teamcityLocRegExp;
 };
 
 struct BenchmarkResult
@@ -81,46 +92,50 @@ struct BenchmarkResult
     static BenchmarkResult parse(QString const&, QString*);
 };
 
-QT_BEGIN_NAMESPACE
-namespace QTest
+static QString msgMismatch(const QString &actual, const QString &expected)
 {
-template <>
-inline bool qCompare
-    (BenchmarkResult const &r1, BenchmarkResult const &r2,
-     const char* actual, const char* expected, const char* file, int line)
+    return QLatin1String("Mismatch:\n'") + actual + QLatin1String("'\n !=\n'")
+        + expected + QLatin1Char('\'');
+}
+
+static bool compareBenchmarkResult(BenchmarkResult const &r1, BenchmarkResult const &r2,
+                                   QString *errorMessage)
 {
     // First make sure the iterations and unit match.
     if (r1.iterations != r2.iterations || r1.unit != r2.unit) {
         // Nope - compare whole string for best failure message
-        return qCompare(r1.toString(), r2.toString(), actual, expected, file, line);
+        *errorMessage = msgMismatch(r1.toString(), r2.toString());
+        return false;
     }
 
     // Now check the value.  Some variance is allowed, and how much depends on
     // the measured unit.
     qreal variance = 0.;
-    if (r1.unit == "msecs" || r1.unit == "WalltimeMilliseconds") {
+    if (r1.unit == QLatin1String("msecs") || r1.unit == QLatin1String("WalltimeMilliseconds"))
         variance = 0.1;
-    }
-    else if (r1.unit == "instruction reads") {
+    else if (r1.unit == QLatin1String("instruction reads"))
         variance = 0.001;
-    }
-    else if (r1.unit == "CPU ticks" || r1.unit == "CPUTicks") {
+    else if (r1.unit == QLatin1String("CPU ticks") || r1.unit == QLatin1String("CPUTicks"))
         variance = 0.001;
-    }
+
     if (variance == 0.) {
         // No variance allowed - compare whole string
-        return qCompare(r1.toString(), r2.toString(), actual, expected, file, line);
+        const QString r1S = r1.toString();
+        const QString r2S = r2.toString();
+        if (r1S != r2S) {
+            *errorMessage = msgMismatch(r1S, r2S);
+            return false;
+        }
+        return true;
     }
 
-    if (qAbs(qreal(r1.total) - qreal(r2.total)) <= qreal(r1.total)*variance) {
-        return compare_helper(true, 0, 0, 0, actual, expected, file, line);
+    if (qAbs(qreal(r1.total) - qreal(r2.total)) > qreal(r1.total) * variance) {
+        // Whoops, didn't match.  Compare the whole string for the most useful failure message.
+        *errorMessage = msgMismatch(r1.toString(), r2.toString());
+        return false;
     }
-
-    // Whoops, didn't match.  Compare the whole string for the most useful failure message.
-    return qCompare(r1.toString(), r2.toString(), actual, expected, file, line);
+    return true;
 }
-}
-QT_END_NAMESPACE
 
 // Split the passed block of text into an array of lines, replacing any
 // filenames and line numbers with generic markers to avoid failing the test
@@ -227,26 +242,6 @@ static QByteArray runDiff(const QByteArrayList &expected, const QByteArrayList &
     return result;
 }
 
-// Print the difference preferably using 'diff', else just print lines
-static void printDifference(const QByteArrayList &expected, const QByteArrayList &actual)
-{
-    QDebug info = qInfo();
-    info.noquote();
-    info.nospace();
-    const QByteArray diff = runDiff(expected, actual);
-    if (diff.isEmpty()) {
-        info << "<<<<<<\n";
-        for (const QByteArray &line : actual)
-            info << line << '\n';
-        info << "======\n";
-        for (const QByteArray &line : expected)
-            info << line << '\n';
-        info << ">>>>>>\n";
-    } else {
-        info << diff;
-    }
-}
-
 // Each test is run with a set of one or more test output loggers.
 // This struct holds information about one such test.
 struct LoggerSet
@@ -399,6 +394,7 @@ QList<LoggerSet> tst_Selftests::allLoggerSets() const
 tst_Selftests::tst_Selftests()
     : tempDir(QDir::tempPath() + "/tst_selftests.XXXXXX")
     , durationRegExp("<Duration msecs=\"[\\d\\.]+\"/>")
+    , teamcityLocRegExp("\\|\\[Loc: .*\\(\\d*\\)\\|\\]")
 {}
 
 void tst_Selftests::initTestCase()
@@ -760,149 +756,198 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge
         }
 
         QList<QByteArray> res = splitLines(actualOutputs[n]);
-        const QString expectedFileName = expectedFileNameFromTest(subdir, logger);
-        QList<QByteArray> exp = expectedResult(expectedFileName);
+        QString errorMessage;
+        QString expectedFileName = expectedFileNameFromTest(subdir, logger);
+        if (QFileInfo::exists(expectedFileName)) {
+            QByteArrayList exp = expectedResult(expectedFileName);
 #ifdef Q_CC_MINGW
-        // MinGW formats double numbers differently
-        if (n == 0 && subdir == QStringLiteral("float")) {
-            for (int i = 0; i < exp.size(); ++i) {
-                exp[i].replace("e-07", "e-007");
-                exp[i].replace("e+07", "e+007");
-            }
-        }
-#endif
-
-        // For the "crashes" test, there are multiple versions of the
-        // expected output.  Load the one with the same line count as
-        // the actual output.
-        if (exp.count() == 0) {
-            QList<QList<QByteArray> > expArr;
-            QList<QByteArray> tmp;
-            int i = 1;
-            do {
-                tmp = expectedResult(expectedFileNameFromTest(subdir + QLatin1Char('_') + QString::number(i++), logger));
-                if (tmp.count())
-                    expArr += tmp;
-            } while (tmp.count());
-
-            for (int j = 0; j < expArr.count(); ++j) {
-                if (res.count() == expArr.at(j).count()) {
-                    exp = expArr.at(j);
-                    break;
+            // MinGW formats double numbers differently (last verified with 7.1)
+            if (n == 0 && subdir == QStringLiteral("float")) {
+                for (int i = 0; i < exp.size(); ++i) {
+                    exp[i].replace("e-07", "e-007");
+                    exp[i].replace("e+07", "e+007");
                 }
             }
-
-            if (expArr.count()) {
-                QVERIFY2(exp.count(),
-                         qPrintable(QString::fromLatin1("None of the expected output files for "
-                                                        "%1 format has matching line count.")
-                                    .arg(loggers.at(n))));
+#endif
+            if (!compareOutput(logger, subdir, actualOutputs[n], res, exp, &errorMessage)) {
+                errorMessage.prepend(QLatin1Char('"') + logger + QLatin1String("\", ")
+                                     + expectedFileName + QLatin1Char(' '));
+                errorMessage += QLatin1String("\nActual:\n") + QLatin1String(actualOutputs[n]);
+                const QByteArray diff = runDiff(exp, res);
+                if (!diff.isEmpty())
+                    errorMessage += QLatin1String("\nDiff:\n") + QLatin1String(diff);
+                QFAIL(qPrintable(errorMessage));
             }
         } else {
-            if (res.count() != exp.count()) {
-                printDifference(exp, res);
-                QVERIFY2(res.count() == exp.count(),
-                     qPrintable(QString::fromLatin1("Mismatch in line count: %1 != %2 (%3, %4).")
-                                .arg(res.count()).arg(exp.count()).arg(loggers.at(n), expectedFileName)));
+            // For the "crashes" and other tests, there are multiple versions of the
+            // expected output. Loop until a matching one is found.
+            bool ok = false;
+            for (int i = 1; !ok; ++i) {
+                expectedFileName = expectedFileNameFromTest(subdir + QLatin1Char('_') + QString::number(i), logger);
+                const QByteArrayList exp = expectedResult(expectedFileName);
+                if (exp.isEmpty())
+                    break;
+                QString errorMessage2;
+                ok = compareOutput(logger, subdir, actualOutputs[n], res, exp, &errorMessage2);
+                if (!ok)
+                    errorMessage += QLatin1Char('\n') + expectedFileName + QLatin1String(": ") + errorMessage2;
             }
-        }
-
-        // By this point, we should have loaded a non-empty expected data file.
-        QVERIFY2(exp.count(),
-                 qPrintable(QString::fromLatin1("Expected test data for %1 format is empty or not found.")
-                            .arg(loggers.at(n))));
-
-        // For xml output formats, verify that the log is valid XML.
-        if (logFormat(logger) == "xunitxml" || logFormat(logger) == "xml" || logFormat(logger) == "lightxml") {
-            QByteArray xml(actualOutputs[n]);
-            // lightxml intentionally skips the root element, which technically makes it
-            // not valid XML.
-            // We'll add that ourselves for the purpose of validation.
-            if (logFormat(logger) == "lightxml") {
-                xml.prepend("<root>");
-                xml.append("</root>");
+            if (!ok) { // Use QDebug's quote mechanism to report potentially garbled output.
+                errorMessage.prepend(QLatin1String("Cannot find a matching file for ") + subdir);
+                errorMessage += QLatin1String("\nActual:\n");
+                QDebug(&errorMessage) << actualOutputs[n];
+                QFAIL(qPrintable(errorMessage));
             }
-
-            QXmlStreamReader reader(xml);
-
-            while (!reader.atEnd())
-                reader.readNext();
-
-            QVERIFY2(!reader.error(), qPrintable(QString("line %1, col %2: %3")
-                .arg(reader.lineNumber())
-                .arg(reader.columnNumber())
-                .arg(reader.errorString())
-            ));
-        }
-
-        // Verify that the actual output is an acceptable match for the
-        // expected output.
-        bool benchmark = false;
-        for (int i = 0; i < res.count(); ++i) {
-            QByteArray line = res.at(i);
-            // the __FILE__ __LINE__ output is compiler dependent, skip it
-            if (line.startsWith("   Loc: [") && line.endsWith(")]"))
-                continue;
-            if (line.endsWith(" : failure location"))
-                continue;
-
-            if (line.startsWith("Config: Using QtTest library") // Text build string
-                || line.startsWith("    <QtBuild") // XML, Light XML build string
-                || (line.startsWith("    <property value=") &&  line.endsWith("name=\"QtBuild\"/>"))) { // XUNIT-XML build string
-                    continue;
-            }
-
-            QByteArray expLine = exp.at(i);
-
-            // Special handling for ignoring _FILE_ and _LINE_ if logger is teamcity
-            if (logFormat(logger) == "teamcity") {
-                QRegularExpression teamcityLocRegExp("\\|\\[Loc: .*\\(\\d*\\)\\|\\]");
-                line = QString(line).replace(teamcityLocRegExp, "|[Loc: _FILE_(_LINE_)|]").toLatin1();
-                expLine = QString(expLine).replace(teamcityLocRegExp, "|[Loc: _FILE_(_LINE_)|]").toLatin1();
-            }
-
-            const QString output(QString::fromLatin1(line));
-            const QString expected(QString::fromLatin1(expLine).replace("@INSERT_QT_VERSION_HERE@", QT_VERSION_STR));
-
-            if (subdir == "assert" && output.contains("ASSERT: ") && expected.contains("ASSERT: ") && output != expected)
-                // Q_ASSERT uses __FILE__, the exact contents of which are
-                // undefined.  If we something that looks like a Q_ASSERT and we
-                // were expecting to see a Q_ASSERT, we'll skip the line.
-                continue;
-            else if (expected.startsWith(QLatin1String("FAIL!  : tst_Exception::throwException() Caught unhandled exce")) && expected != output)
-                // On some platforms we compile without RTTI, and as a result we never throw an exception.
-                QCOMPARE(output.simplified(), QString::fromLatin1("tst_Exception::throwException()").simplified());
-            else if (benchmark || line.startsWith("<BenchmarkResult") || (logFormat(logger) == "csv" && line.startsWith('"'))) {
-                // Don't do a literal comparison for benchmark results, since
-                // results have some natural variance.
-                QString error;
-
-                BenchmarkResult actualResult = BenchmarkResult::parse(output, &error);
-                QVERIFY2(error.isEmpty(), qPrintable(QString("Actual line didn't parse as benchmark result: %1\nLine: %2").arg(error).arg(output)));
-
-                BenchmarkResult expectedResult = BenchmarkResult::parse(expected, &error);
-                QVERIFY2(error.isEmpty(), qPrintable(QString("Expected line didn't parse as benchmark result: %1\nLine: %2").arg(error).arg(expected)));
-
-                QCOMPARE(actualResult, expectedResult);
-            } else if (line.startsWith("    <Duration msecs=") || line.startsWith("<Duration msecs=")) {
-                QRegularExpressionMatch match = durationRegExp.match(line);
-                QVERIFY2(match.hasMatch(), qPrintable(QString::fromLatin1("Invalid Duration tag at line %1 (%2): '%3'")
-                                                      .arg(i).arg(loggers.at(n), output)));
-            } else if (line.startsWith("Totals:")) {
-                const int lastCommaPos = line.lastIndexOf(',');
-                if (lastCommaPos > 0)
-                    line.truncate(lastCommaPos); // Plain text logger: strip time (", 2323dms").
-            } else {
-                QVERIFY2(output == expected,
-                         qPrintable(QString::fromLatin1("Mismatch at line %1 (%2, %3):\n'%4'\n !=\n'%5'")
-                                    .arg(i + 1).arg(loggers.at(n), expectedFileName, output, expected)));
-            }
-
-            benchmark = line.startsWith("RESULT : ");
         }
     }
 }
 
+static QString teamCityLocation() { return QStringLiteral("|[Loc: _FILE_(_LINE_)|]"); }
+static QString qtVersionPlaceHolder() { return QStringLiteral("@INSERT_QT_VERSION_HERE@"); }
+
+bool tst_Selftests::compareOutput(const QString &logger, const QString &subdir,
+                                  const QByteArray &rawOutput, const QByteArrayList &actual,
+                                  const QByteArrayList &expected,
+                                  QString *errorMessage) const
+{
+
+    if (actual.size() != expected.size()) {
+        *errorMessage = QString::fromLatin1("Mismatch in line count: %1 != %2.")
+                        .arg(actual.size()).arg(expected.size());
+        return false;
+    }
+
+    // For xml output formats, verify that the log is valid XML.
+    if (logger.endsWith(QLatin1String("xml")) && !checkXml(logger, rawOutput, errorMessage))
+        return false;
+
+    // Verify that the actual output is an acceptable match for the
+    // expected output.
+
+    const QString qtVersion = QLatin1String(QT_VERSION_STR);
+    bool benchmark = false;
+    for (int i = 0, size = actual.size(); i < size; ++i) {
+        const QByteArray &actualLineBA = actual.at(i);
+        // the __FILE__ __LINE__ output is compiler dependent, skip it
+        if (actualLineBA.startsWith("   Loc: [") && actualLineBA.endsWith(")]"))
+            continue;
+        if (actualLineBA.endsWith(" : failure location"))
+            continue;
+
+        if (actualLineBA.startsWith("Config: Using QtTest library") // Text build string
+            || actualLineBA.startsWith("    <QtBuild") // XML, Light XML build string
+            || (actualLineBA.startsWith("    <property value=") &&  actualLineBA.endsWith("name=\"QtBuild\"/>"))) { // XUNIT-XML build string
+            continue;
+        }
+
+        QString actualLine = QString::fromLatin1(actualLineBA);
+        QString expectedLine = QString::fromLatin1(expected.at(i));
+        expectedLine.replace(qtVersionPlaceHolder(), qtVersion);
+
+        // Special handling for ignoring _FILE_ and _LINE_ if logger is teamcity
+        if (logger.endsWith(QLatin1String("teamcity"))) {
+            actualLine.replace(teamcityLocRegExp, teamCityLocation());
+            expectedLine.replace(teamcityLocRegExp, teamCityLocation());
+        }
+
+        if (!compareLine(logger, subdir, benchmark, actualLine,
+                         expectedLine, errorMessage)) {
+            errorMessage->prepend(QLatin1String("Line ") + QString::number(i + 1)
+                                  + QLatin1String(": "));
+            return false;
+        }
+
+        benchmark = actualLineBA.startsWith("RESULT : ");
+    }
+    return true;
+}
+
+bool tst_Selftests::compareLine(const QString &logger, const QString &subdir,
+                                bool benchmark,
+                                const QString &actualLine, const QString &expectedLine,
+                                QString *errorMessage) const
+{
+    if (subdir == QLatin1String("assert") && actualLine.contains(QLatin1String("ASSERT: "))
+        && expectedLine.contains(QLatin1String("ASSERT: ")) && actualLine != expectedLine) {
+        // Q_ASSERT uses __FILE__, the exact contents of which are
+        // undefined. If have we something that looks like a Q_ASSERT and we
+        // were expecting to see a Q_ASSERT, we'll skip the line.
+        return true;
+    }
+
+    if (expectedLine.startsWith(QLatin1String("FAIL!  : tst_Exception::throwException() Caught unhandled exce"))
+        && actualLine != expectedLine) {
+        // On some platforms we compile without RTTI, and as a result we never throw an exception
+        if (actualLine.simplified() != QLatin1String("tst_Exception::throwException()")) {
+            *errorMessage = QString::fromLatin1("'%1' != 'tst_Exception::throwException()'").arg(actualLine);
+            return false;
+        }
+        return true;
+    }
+
+    if (benchmark || actualLine.startsWith(QLatin1String("<BenchmarkResult"))
+        || (logger == QLatin1String("csv") && actualLine.startsWith(QLatin1Char('"')))) {
+        // Don't do a literal comparison for benchmark results, since
+        // results have some natural variance.
+        QString error;
+        BenchmarkResult actualResult = BenchmarkResult::parse(actualLine, &error);
+        if (!error.isEmpty()) {
+            *errorMessage = QString::fromLatin1("Actual line didn't parse as benchmark result: %1\nLine: %2").arg(error, actualLine);
+            return false;
+        }
+        BenchmarkResult expectedResult = BenchmarkResult::parse(expectedLine, &error);
+        if (!error.isEmpty()) {
+            *errorMessage = QString::fromLatin1("Expected line didn't parse as benchmark result: %1\nLine: %2").arg(error, expectedLine);
+            return false;
+        }
+        return compareBenchmarkResult(actualResult, expectedResult, errorMessage);
+    }
+
+    if (actualLine.startsWith(QLatin1String("    <Duration msecs="))
+        || actualLine.startsWith(QLatin1String("<Duration msecs="))) {
+        QRegularExpressionMatch match = durationRegExp.match(actualLine);
+        if (match.hasMatch())
+            return true;
+        *errorMessage = QString::fromLatin1("Invalid Duration tag: '%1'").arg(actualLine);
+        return false;
+    }
+
+    if (actualLine.startsWith(QLatin1String("Totals:")) && expectedLine.startsWith(QLatin1String("Totals:")))
+        return true;
+
+    if (actualLine == expectedLine)
+        return true;
+
+    *errorMessage = msgMismatch(actualLine, expectedLine);
+    return false;
+}
+
+bool tst_Selftests::checkXml(const QString &logger, QByteArray xml,
+                             QString *errorMessage) const
+{
+    // lightxml intentionally skips the root element, which technically makes it
+    // not valid XML.
+    // We'll add that ourselves for the purpose of validation.
+    if (logger.endsWith(QLatin1String("lightxml"))) {
+        xml.prepend("<root>");
+        xml.append("</root>");
+    }
+
+    QXmlStreamReader reader(xml);
+    while (!reader.atEnd())
+        reader.readNext();
+
+    if (reader.hasError()) {
+        const int lineNumber = int(reader.lineNumber());
+        const QByteArray line = xml.split('\n').value(lineNumber - 1);
+        *errorMessage = QString::fromLatin1("line %1, col %2 '%3': %4")
+                        .arg(lineNumber).arg(reader.columnNumber())
+                        .arg(QString::fromLatin1(line), reader.errorString());
+        return false;
+    }
+    return true;
+}
+
 #endif // QT_CONFIG(process)
 
 void tst_Selftests::runSubTest()