From 2cb78aaf6ddd61b5d4ab1cff7b3a9422782cc8b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tor=20Arne=20Vestb=C3=B8?= <tor.arne.vestbo@qt.io>
Date: Fri, 20 Sep 2024 13:48:43 +0200
Subject: [PATCH] ctest: Only output test output on failure
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The test output of passing tests is most of the time unnecessary
noise when investigating a test failure in a failing test. With
the size of the qtbase log being 25MB uncompressed, working with
these logs locally or in a (mobile) browser is also painful.

We now disable verbose test output in ctest, and only output test
output if the test fails. We still run our tests with -o foo.txt,txt
and these files are uploaded to coin's artifact storage, so the
full logs are available if closer inspections are needed. We
also store CMake's LastTest.log, which include verbose output
of all non-testlib tests.

The verboseness of the coin config can also be enabled via the
VerboseCTestOutput feature, either for specific configs, or
on demand when scheduling a check via coin.

Fixes: QTQAINFRA-6592
Fixes: QTBUG-109441
Change-Id: I5cdfb8839190b1598f12b7f9ea1d9a242a32a790
Reviewed-by:  Alexey Edelev <alexey.edelev@qt.io>
Reviewed-by: Daniel Smith <daniel.smith@qt.io>
Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>
Reviewed-by: Dimitrios Apostolou <jimis@qt.io>
---
 coin/instructions/cmake_run_ctest.yaml | 32 ++++++++++++++++++--------
 1 file changed, 23 insertions(+), 9 deletions(-)

diff --git a/coin/instructions/cmake_run_ctest.yaml b/coin/instructions/cmake_run_ctest.yaml
index 47105a9c9c7..67a5843ec73 100644
--- a/coin/instructions/cmake_run_ctest.yaml
+++ b/coin/instructions/cmake_run_ctest.yaml
@@ -61,22 +61,24 @@ instructions:
 
   - !include "{{qt/qtbase}}/vxworks_test_env_setup.yaml"
 
-    # Keep the testrun quiet for ASAN testruns, since there are FAILs happening all over the place...
-  - type: EnvironmentVariable
+  # Only output verbose test output for failures, unless
+  # the VerboseCTestOutput feature is enabled. The full
+  # test output for all tests are still stored and uploaded
+  # to coin for deeper investigations (see LastTest.log below)
+  - type: AppendToEnvironmentVariable
     variableName: CTEST_ARGS
-    variableValue: "-V"
+    variableValue: " --output-on-failure"
     disable_if:
       condition: property
       property: features
-      contains_value: UseAddressSanitizer
-    # ...and only print the output from a failing test, i.e. test with ASAN errors.
-  - type: EnvironmentVariable
-    variableName: CTEST_OUTPUT_ON_FAILURE
-    variableValue: "1"
+      contains_value: VerboseCTestOutput
+  - type: AppendToEnvironmentVariable
+    variableName: CTEST_ARGS
+    variableValue: " -V"
     enable_if:
       condition: property
       property: features
-      contains_value: UseAddressSanitizer
+      contains_value: VerboseCTestOutput
 
   - type: AppendToEnvironmentVariable
     variableName: CTEST_ARGS
@@ -150,6 +152,18 @@ instructions:
         userMessageOnFailure: >
           Failed to run tests.
 
+  - type: Group
+    instructions:
+      - type: ExecuteCommand
+        command: >
+          cmake -E copy
+            Testing{{.Env.CI_PATH_SEP}}Temporary{{.Env.CI_PATH_SEP}}LastTest.log
+            {{.Env.COIN_CTEST_RESULTSDIR}}{{.Env.CI_PATH_SEP}}CTest.log
+        executeCommandArgumentSplitingBehavior: SplitAfterVariableSubstitution
+        ignoreExitCode: false
+        userMessageOnFailure: >
+          Failed to copy LastTest.log to testresults directory.
+
   - type: Group
     instructions:
       # Qttestrunner does not work with bic tests.