Add a feature to enable CodeCoverage analysis of testlib

Based on Asmo Saarela's advice (QTPM-686), adapted on advice from
FrogLogic support and converted to a feature so that the selftest and
testlib qmake config can be co-ordinated.

Task-number: QTPM-1385
Change-Id: Icd706f086009e1e08b3f8c5cd553f792402e28c0
Reviewed-by: Volker Hilsheimer <volker.hilsheimer@qt.io>

src/testlib/configure.json

@@ -5,6 +5,12 @@
     ],
 
     "features": {
+        "testlib_selfcover": {
+            "label": "Coverage testing of testlib itself",
+            "purpose": "Gauges how thoroughly testlib's selftest exercises testlib's code",
+            "autoDetect": false,
+            "output": [ "publicFeature" ]
+        },
         "itemmodeltester": {
             "label": "Tester for item models",
             "purpose": "Provides a utility to test item models.",

src/testlib/qtest.h

@@ -374,9 +374,30 @@ QT_END_NAMESPACE
 #  define QTEST_SET_MAIN_SOURCE_PATH  QTest::setMainSourcePath(__FILE__);
 #endif
 
+// Hooks for coverage-testing of QTestLib itself:
+#if QT_CONFIG(testlib_selfcover) && defined(__COVERAGESCANNER__)
+struct QtCoverageScanner
+{
+    QtCoverageScanner(const char *name)
+    {
+        __coveragescanner_clear();
+        __coveragescanner_testname(name);
+    }
+    ~QtCoverageScanner()
+    {
+        __coveragescanner_save();
+        __coveragescanner_testname("");
+    }
+};
+#define TESTLIB_SELFCOVERAGE_START(name) QtCoverageScanner _qtCoverageScanner(name);
+#else
+#define TESTLIB_SELFCOVERAGE_START(name)
+#endif
+
 #define QTEST_APPLESS_MAIN(TestObject) \
 int main(int argc, char *argv[]) \
 { \
+    TESTLIB_SELFCOVERAGE_START(TestObject) \
     TestObject tc; \
     QTEST_SET_MAIN_SOURCE_PATH \
     return QTest::qExec(&tc, argc, argv); \
@@ -406,6 +427,7 @@ int main(int argc, char *argv[]) \
 #define QTEST_MAIN(TestObject) \
 int main(int argc, char *argv[]) \
 { \
+    TESTLIB_SELFCOVERAGE_START(#TestObject) \
     QApplication app(argc, argv); \
     app.setAttribute(Qt::AA_Use96Dpi, true); \
     QTEST_DISABLE_KEYPAD_NAVIGATION \
@@ -421,6 +443,7 @@ int main(int argc, char *argv[]) \
 #define QTEST_MAIN(TestObject) \
 int main(int argc, char *argv[]) \
 { \
+    TESTLIB_SELFCOVERAGE_START(#TestObject) \
     QGuiApplication app(argc, argv); \
     app.setAttribute(Qt::AA_Use96Dpi, true); \
     TestObject tc; \
@@ -433,6 +456,7 @@ int main(int argc, char *argv[]) \
 #define QTEST_MAIN(TestObject) \
 int main(int argc, char *argv[]) \
 { \
+    TESTLIB_SELFCOVERAGE_START(#TestObject) \
     QCoreApplication app(argc, argv); \
     app.setAttribute(Qt::AA_Use96Dpi, true); \
     TestObject tc; \
@@ -445,6 +469,7 @@ int main(int argc, char *argv[]) \
 #define QTEST_GUILESS_MAIN(TestObject) \
 int main(int argc, char *argv[]) \
 { \
+    TESTLIB_SELFCOVERAGE_START(#TestObject) \
     QCoreApplication app(argc, argv); \
     app.setAttribute(Qt::AA_Use96Dpi, true); \
     TestObject tc; \

src/testlib/qtestcase.cpp

@@ -246,7 +246,7 @@ static void stackTrace()
 
 static bool installCoverageTool(const char * appname, const char * testname)
 {
-#ifdef __COVERAGESCANNER__
+#if defined(__COVERAGESCANNER__) && !QT_CONFIG(testlib_selfcover)
     if (!qEnvironmentVariableIsEmpty("QT_TESTCOCOON_ACTIVE"))
         return false;
     // Set environment variable QT_TESTCOCOON_ACTIVE to prevent an eventual subtest from

src/testlib/qtestlog.cpp

@@ -73,6 +73,10 @@ QT_BEGIN_NAMESPACE
 static void saveCoverageTool(const char * appname, bool testfailed, bool installedTestCoverage)
 {
 #ifdef __COVERAGESCANNER__
+#  if QT_CONFIG(testlib_selfcover)
+    __coveragescanner_teststate(QTestLog::failCount() > 0 ? "FAILED" :
+                                QTestLog::passCount() > 0 ? "PASSED" : "SKIPPED");
+#  else
     if (!installedTestCoverage)
         return;
     // install again to make sure the filename is correct.
@@ -83,6 +87,7 @@ static void saveCoverageTool(const char * appname, bool testfailed, bool install
     __coveragescanner_testname("");
     __coveragescanner_clear();
     unsetenv("QT_TESTCOCOON_ACTIVE");
+#  endif // testlib_selfcover
 #else
     Q_UNUSED(appname);
     Q_UNUSED(testfailed);

src/testlib/selfcover.pri

@@ -0,0 +1,28 @@
+# Configuration for testlib and its tests, to instrument with
+# FrogLogic's Squish CoCo (cf. testcocoon.prf, which handles similar
+# for general code; but testlib needs special handling).
+
+# Only for use when feature testlib_selfcover is enabled:
+!qtConfig(testlib_selfcover): return()
+
+# This enables verification that testlib itself is adequately tested,
+# as a grounds for trusting that testing with it is useful.
+# Exclude all non-testlib source from coverage instrumentation:
+COVERAGE_OPTIONS = --cs-exclude-file-abs-wildcard=$$QT_SOURCE_TREE/*
+COVERAGE_OPTIONS += --cs-include-file-abs-wildcard=*/src/testlib/*
+COVERAGE_OPTIONS += --cs-mcc # enable Multiple Condition Coverage
+COVERAGE_OPTIONS += --cs-mcdc # enable Multiple Condition / Decision Coverage
+# (recommended for ISO 26262 ASIL A, B and C -- highly recommended for ASIL D)
+# https://doc.froglogic.com/squish-coco/4.1/codecoverage.html#sec%3Amcdc
+
+QMAKE_CFLAGS += $$COVERAGE_OPTIONS
+QMAKE_CXXFLAGS += $$COVERAGE_OPTIONS
+QMAKE_LFLAGS += $$COVERAGE_OPTIONS
+
+# FIXME: relies on QMAKE_* being just the command-names, with no path prefix
+QMAKE_CC = cs$$QMAKE_CC
+QMAKE_CXX = cs$$QMAKE_CXX
+QMAKE_LINK = cs$$QMAKE_LINK
+QMAKE_LINK_SHLIB = cs$$QMAKE_LINK_SHLIB
+QMAKE_AR = cs$$QMAKE_AR
+QMAKE_LIB = cs$$QMAKE_LIB

src/testlib/testlib.pro

@@ -146,4 +146,5 @@ mac {
 
 !qtHaveModule(network): HEADERSCLEAN_EXCLUDE += qtest_network.h
 
+include(selfcover.pri)
 load(qt_module)

tests/auto/testlib/qabstractitemmodeltester/qabstractitemmodeltester.pro

@@ -11,3 +11,5 @@ SOURCES += \
 
 HEADERS += \
   $${mtdir}/dynamictreemodel.h
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/qsignalspy/qsignalspy.pro

@@ -2,3 +2,5 @@ CONFIG += testcase
 TARGET = tst_qsignalspy
 SOURCES  += tst_qsignalspy.cpp
 QT = core testlib
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/alive/alive.pro

@@ -6,3 +6,5 @@ CONFIG -= debug_and_release_target
 
 
 TARGET = alive
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/assert/assert.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = assert
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/badxml/badxml.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = badxml
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/benchlibcallgrind/benchlibcallgrind.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = benchlibcallgrind
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/benchlibcounting/benchlibcounting.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = benchlibcounting
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/benchlibeventcounter/benchlibeventcounter.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = benchlibeventcounter
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/benchliboptions/benchliboptions.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = benchliboptions
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/benchlibtickcounter/benchlibtickcounter.pro

@@ -6,3 +6,5 @@ CONFIG -= debug_and_release_target
 
 
 TARGET = benchlibtickcounter
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/benchlibwalltime/benchlibwalltime.pro

@@ -6,3 +6,5 @@ CONFIG -= debug_and_release_target
 
 
 TARGET = benchlibwalltime
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/blacklisted/blacklisted.pro

@@ -5,3 +5,5 @@ mac: CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = blacklisted
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/cmptest/cmptest.pro

@@ -6,3 +6,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = cmptest
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/commandlinedata/commandlinedata.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = commandlinedata
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/counting/counting.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = counting
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/crashes/crashes.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = crashes
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/datatable/datatable.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = datatable
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/datetime/datetime.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = datetime
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/deleteLater/deleteLater.pro

@@ -5,3 +5,5 @@ CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = tst_deleteLater
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/deleteLater_noApp/deleteLater_noApp.pro

@@ -5,3 +5,5 @@ CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = tst_deleteLater_noApp
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/differentexec/differentexec.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = differentexec
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/exceptionthrow/exceptionthrow.pro

@@ -6,3 +6,5 @@ CONFIG -= debug_and_release_target
 CONFIG += exceptions
 
 TARGET = exceptionthrow
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/expectfail/expectfail.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = expectfail
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/failcleanup/failcleanup.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = failcleanup
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/failinit/failinit.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = failinit
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/failinitdata/failinitdata.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = failinitdata
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/fetchbogus/fetchbogus.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = fetchbogus
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/findtestdata/findtestdata.pro

@@ -7,3 +7,5 @@ CONFIG -= debug_and_release_target
 RESOURCES = findtestdata.qrc
 
 TARGET = findtestdata
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/float/float.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = float
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/globaldata/globaldata.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = globaldata
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/keyboard/keyboard.pro

@@ -5,3 +5,5 @@ macos:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = keyboard
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/longstring/longstring.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = longstring
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/maxwarnings/maxwarnings.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = maxwarnings
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/mouse/mouse.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = mouse
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/multiexec/multiexec.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = multiexec
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/pairdiagnostics/pairdiagnostics.pro

@@ -4,3 +4,5 @@ QT = core testlib
 CONFIG -= app_bundle debug_and_release_target
 
 TARGET = pairdiagnostics
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/printdatatags/printdatatags.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = printdatatags
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/printdatatagswithglobaltags/printdatatagswithglobaltags.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = printdatatagswithglobaltags
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/qexecstringlist/qexecstringlist.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = qexecstringlist
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/silent/silent.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = silent
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/singleskip/singleskip.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = singleskip
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/skip/skip.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = skip
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/skipcleanup/skipcleanup.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = skipcleanup
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/skipinit/skipinit.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = skipinit
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/skipinitdata/skipinitdata.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = skipinitdata
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/sleep/sleep.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = sleep
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/strcmp/strcmp.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = strcmp
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/subtest/subtest.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = subtest
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/test/test.pro

@@ -19,3 +19,4 @@ RESOURCES += expected_files
 include(../selftests.pri)
 !android:!winrt: for(file, SUBPROGRAMS): TEST_HELPER_INSTALLS += "../$${file}/$${file}"
 
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/testlib/testlib.pro

@@ -5,3 +5,5 @@ darwin: CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = testlib
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/tuplediagnostics/tuplediagnostics.pro

@@ -4,3 +4,5 @@ QT = core testlib
 CONFIG -= app_bundle debug_and_release_target
 
 TARGET = tuplediagnostics
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/verbose1/verbose1.pro

@@ -8,3 +8,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = verbose1
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/verbose2/verbose2.pro

@@ -8,3 +8,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = verbose2
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/verifyexceptionthrown/verifyexceptionthrown.pro

@@ -6,3 +6,5 @@ CONFIG -= debug_and_release_target
 CONFIG += exceptions
 
 TARGET = verifyexceptionthrown
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/warnings/warnings.pro

@@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = warnings
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

tests/auto/testlib/selftests/xunit/xunit.pro

@@ -6,3 +6,5 @@ mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
 
 TARGET = xunit
+
+include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)