tst_selftests: split the qFatal() test away from tst_Silent

This allows us to control whether to run this particular test in ASan
mode or with specific loggers or not. Adding the expected log output for
tst_silent for other loggers is left as an exercise to the reader.

Change-Id: Ifa1111900d6945ea8e05fffd177f1548c8c8e714
Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>

tests/auto/testlib/selftests/CMakeLists.txt

@@ -91,6 +91,7 @@ set(subprograms
     printdatatagswithglobaltags
     qexecstringlist
     silent
+    silent_fatal
     signaldumper
     singleskip
     skip

tests/auto/testlib/selftests/expected_silent.txt

@@ -5,7 +5,4 @@ FAIL!  : tst_Silent::fail() 'false' returned FALSE. (This test should fail)
    Loc: [qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp(0)]
 XPASS  : tst_Silent::xpass() 'true' returned TRUE unexpectedly. (This test should XPASS)
    Loc: [qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp(0)]
-QFATAL : tst_Silent::messages() This is a fatal error message that should still appear in silent test output
+Totals: 5 passed, 2 failed, 1 skipped, 0 blacklisted, 0ms
-FAIL!  : tst_Silent::messages() Received a fatal error.
-   Loc: [qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp(0)]
-Totals: 3 passed, 3 failed, 1 skipped, 0 blacklisted, 0ms

tests/auto/testlib/selftests/expected_silent_fatal.txt

@@ -0,0 +1,5 @@
+Testing tst_SilentFatal
+QFATAL : tst_SilentFatal::fatalmessages() This is a fatal error message that should still appear in silent test output
+FAIL!  : tst_SilentFatal::fatalmessages() Received a fatal error.
+   Loc: [qtbase/tests/auto/testlib/selftests/silent_fatal/tst_silent_fatal.cpp(0)]
+Totals: 1 passed, 1 failed, 0 skipped, 0 blacklisted, 0ms

tests/auto/testlib/selftests/generate_expected_output.py

@@ -37,7 +37,7 @@ TESTS = ['assert', 'badxml', 'benchlibcallgrind', 'benchlibcounting',
          'fetchbogus', 'findtestdata', 'float', 'globaldata', 'longstring',
          'maxwarnings', 'mouse', 'multiexec', 'pairdiagnostics', 'pass',
          'printdatatags', 'printdatatagswithglobaltags', 'qexecstringlist',
-         'signaldumper', 'silent', 'singleskip', 'skip', 'skipcleanup',
+         'signaldumper', 'silent', 'silent_fatal', 'singleskip', 'skip', 'skipcleanup',
          'skipcleanuptestcase', 'skipinit', 'skipinitdata', 'sleep', 'strcmp',
          'subtest', 'testlib', 'tuplediagnostics', 'verbose1', 'verbose2',
          'verifyexceptionthrown', 'warnings', 'watchdog', 'junit', 'keyboard']
@@ -254,7 +254,7 @@ def testEnv(testname,
             # Must match tst_Selftests::runSubTest_data():
             crashers = ("assert", "crashes", "crashedterminate",
                         "exceptionthrow", "faildatatype", "failfetchtype",
-                        "fetchbogus", "silent", "watchdog")):
+                        "fetchbogus", "silent_fatal", "watchdog")):
     """Determine the environment in which to run a test."""
     data = baseEnv()
     if testname in crashers:
@@ -281,6 +281,7 @@ def shouldIgnoreTest(testname, format):
                         "printdatatags",
                         "printdatatagswithglobaltags",
                         "silent",
+                        "silent_fatal",
                         "crashes",
                         "benchlibcallgrind",
                         "float",

tests/auto/testlib/selftests/silent/tst_silent.cpp

@@ -15,8 +15,6 @@ private slots:
     void fail();
     void xfail();
     void xpass();
-
-    // This test function must be last, as it calls qFatal().
     void messages();
 };
 
@@ -55,7 +53,6 @@ void tst_Silent::messages()
     qCritical("This is a critical message that should not appear in silent test output");
     qInfo("This is an info message that should not appear in silent test output");
     QTestLog::info("This is an internal testlib info message that should not appear in silent test output", __FILE__, __LINE__);
-    qFatal("This is a fatal error message that should still appear in silent test output");
 }
 
 QTEST_MAIN_WRAPPER(tst_Silent,

tests/auto/testlib/selftests/silent_fatal/CMakeLists.txt

@@ -0,0 +1,14 @@
+# Copyright (C) 2022 The Qt Company Ltd.
+# SPDX-License-Identifier: BSD-3-Clause
+
+qt_internal_add_executable(silent_fatal
+    NO_INSTALL
+    OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    SOURCES
+        tst_silent_fatal.cpp
+    LIBRARIES
+        Qt::TestPrivate
+)
+
+# No coverage because this crashes, making data collection difficult, if not
+# impossible.

tests/auto/testlib/selftests/silent_fatal/tst_silent_fatal.cpp

@@ -0,0 +1,28 @@
+// Copyright (C) 2021 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
+
+#include <QtCore/QCoreApplication>
+#include <QTest>
+#include <private/qtestlog_p.h>
+
+class tst_SilentFatal : public QObject
+{
+    Q_OBJECT
+
+private slots:
+    void fatalmessages();
+};
+void tst_SilentFatal::fatalmessages()
+{
+    qFatal("This is a fatal error message that should still appear in silent test output");
+}
+
+QTEST_MAIN_WRAPPER(tst_SilentFatal,
+    std::vector<const char*> args(argv, argv + argc);
+    args.push_back("-silent");
+    args.push_back("-nocrashhandler");
+    argc = int(args.size());
+    argv = const_cast<char**>(&args[0]);
+    QTEST_MAIN_SETUP())
+
+#include "tst_silent_fatal.moc"

tests/auto/testlib/selftests/tst_selftests.cpp

@@ -699,7 +699,8 @@ bool TestLogger::shouldIgnoreTest(const QString &test) const
             || test == "benchliboptions"
             || test == "printdatatags"
             || test == "printdatatagswithglobaltags"
-            || test == "silent")
+            || test == "silent"
+            || test == "silent_fatal")
             return true;
 
         // These tests produce variable output (callgrind because of #if-ery,
@@ -764,11 +765,6 @@ void checkErrorOutput(const QString &test, const QByteArray &errorOutput)
         || test == "benchlibcallgrind")
         return;
 
-#ifdef Q_CC_MINGW
-    if (test == "silent") // calls qFatal()
-        return;
-#endif
-
 #ifdef Q_OS_WIN
     if (test == "crashes")
         return; // Complains about uncaught exception
@@ -782,12 +778,17 @@ void checkErrorOutput(const QString &test, const QByteArray &errorOutput)
     return; // Outputs "Received signal 6 (SIGABRT)"
 #endif
 
-#ifdef Q_OS_LINUX
+    if (test == "silent_fatal") {
-    if (test == "silent") {
+#if defined(__SANITIZE_ADDRESS__) || __has_feature(address_sanitizer)
+        // Under ASan, this test is not silent
+        return;
+#elif defined(Q_CC_MINGW)
+        // Originally QTBUG-29014 (I can't reproduce this -Thiago)
+        return;
+#endif
         if (QTestPrivate::isRunningArmOnX86())
             return;         // QEMU outputs to stderr about uncaught signals
     }
-#endif
 
     INFO(errorOutput.toStdString());
     REQUIRE(errorOutput.isEmpty());
@@ -974,7 +975,7 @@ TestProcessResult runTestProcess(const QString &test, const QStringList &argumen
     const bool expectedCrash = test == "assert" || test == "exceptionthrow"
         || test == "fetchbogus" || test == "crashedterminate"
         || test == "faildatatype" || test == "failfetchtype"
-        || test == "crashes" || test == "silent" || test == "watchdog";
+        || test == "crashes" || test == "silent_fatal" || test == "watchdog";
 
     if (expectedCrash) {
         environment.insert("QTEST_DISABLE_CORE_DUMP", "1");