coin: Decide whether to ignore CTest exit code based on environment var

This deduplicates some of the coin instruction files, and opens up the
possibility for other instructions to determine whether a test run
is significant or not.

The only module still using the coin_module_test_template_v2 template
is the QtMQTT module, so the behavior of ignore the test run has been
kept.

The static library opt out was a remnant from the cmake porting, and
can be removed.

Change-Id: I0f9fcbbd3a5a18bd9c8e0f545b0f0656e03bf388
Reviewed-by: Alexandru Croitor <alexandru.croitor@qt.io>

coin/instructions/cmake_regular_test_instructions.yaml

@@ -1,8 +1,15 @@
 type: Group
 instructions:
-  - !include "{{qt/qtbase}}/cmake_regular_test_instructions_common.yaml"
+  - type: MakeDirectory
-  - !include "{{qt/qtbase}}/cmake_run_ctest_ignore_exit_code.yaml"
+    directory: "{{.SourceDir}}_standalone_tests"
-disable_if:
+  - type: InstallTestBinaryArchive
-  condition: property
+    relativeStoragePath: "{{.Env.MODULE_ARTIFACTS_RELATIVE_STORAGE_PATH}}/tests.tar.gz"
-  property: configureArgs
+    directory: "{{.SourceDir}}_standalone_tests"
-  contains_value: "-DBUILD_SHARED_LIBS=OFF"
+    maxTimeInSeconds: 1200
+    maxTimeBetweenOutput: 1200
+    userMessageOnFailure: >
+      Failed to install tests archive.
+  - type: ChangeDirectory
+    directory: "{{.SourceDir}}_standalone_tests"
+
+  - !include "{{qt/qtbase}}/cmake_run_ctest.yaml"

coin/instructions/cmake_regular_test_instructions_common.yaml

@@ -1,13 +0,0 @@
-type: Group
-instructions:
-  - type: MakeDirectory
-    directory: "{{.SourceDir}}_standalone_tests"
-  - type: InstallTestBinaryArchive
-    relativeStoragePath: "{{.Env.MODULE_ARTIFACTS_RELATIVE_STORAGE_PATH}}/tests.tar.gz"
-    directory: "{{.SourceDir}}_standalone_tests"
-    maxTimeInSeconds: 1200
-    maxTimeBetweenOutput: 1200
-    userMessageOnFailure: >
-      Failed to install tests archive.
-  - type: ChangeDirectory
-    directory: "{{.SourceDir}}_standalone_tests"

coin/instructions/cmake_regular_test_instructions_enforced.yaml

@@ -1,4 +0,0 @@
-type: Group
-instructions:
-  - !include "{{qt/qtbase}}/cmake_regular_test_instructions_common.yaml"
-  - !include "{{qt/qtbase}}/cmake_run_ctest_enforce_exit_code.yaml"

coin/instructions/cmake_run_ctest.yaml

@@ -104,14 +104,37 @@ instructions:
           property: features
           contains_value: TestOnly
 
-  - type: ExecuteCommand
+  # Run CTest, with or without ignoring exit code
-    command: "{{.Env.TESTS_ENV_PREFIX}} ctest {{.Env.CTEST_ARGS}}"
+  # FIXME: De-duplicate the CTest instruction when
-    executeCommandArgumentSplitingBehavior: SplitAfterVariableSubstitution
+  # ignoreExitCode expands environment variables.
-    ignoreExitCode: false
+  - type: Group
-    maxTimeInSeconds: 10800
+    enable_if:
-    maxTimeBetweenOutput: "{{.Env.COIN_COMMAND_OUTPUT_TIMEOUT}}"
+      condition: runtime
-    userMessageOnFailure: >
+      env_var: COIN_CTEST_IGNORE_EXIT_CODE
-      Failed to run tests.
+      not_equals_value: "1"
+    instructions:
+      - type: ExecuteCommand
+        command: "{{.Env.TESTS_ENV_PREFIX}} ctest {{.Env.CTEST_ARGS}}"
+        executeCommandArgumentSplitingBehavior: SplitAfterVariableSubstitution
+        ignoreExitCode: false
+        maxTimeInSeconds: 10800
+        maxTimeBetweenOutput: "{{.Env.COIN_COMMAND_OUTPUT_TIMEOUT}}"
+        userMessageOnFailure: >
+          Failed to run tests.
+  - type: Group
+    enable_if:
+      condition: runtime
+      env_var: COIN_CTEST_IGNORE_EXIT_CODE
+      equals_value: "1"
+    instructions:
+      - type: ExecuteCommand
+        command: "{{.Env.TESTS_ENV_PREFIX}} ctest {{.Env.CTEST_ARGS}}"
+        executeCommandArgumentSplitingBehavior: SplitAfterVariableSubstitution
+        ignoreExitCode: true
+        maxTimeInSeconds: 10800
+        maxTimeBetweenOutput: "{{.Env.COIN_COMMAND_OUTPUT_TIMEOUT}}"
+        userMessageOnFailure: >
+          Failed to run tests.
 
   - type: Group
     instructions:

coin/instructions/cmake_run_ctest_ignore_exit_code.yaml

@@ -1,20 +0,0 @@
-type: Group
-instructions:
-  - type: EnvironmentVariable
-    variableName: CTEST_ARGS
-    variableValue: "-V --rerun-failed --force-new-ctest-process"
-  - type: AppendToEnvironmentVariable
-    variableName: CTEST_ARGS
-    variableValue: " --stop-on-failure"
-    enable_if:
-      condition: property
-      property: features
-      contains_value: AbortTestingOnFirstFailure
-  - type: ExecuteCommand
-    command: "{{.Env.TESTS_ENV_PREFIX}} ctest {{.Env.CTEST_ARGS}}"
-    executeCommandArgumentSplitingBehavior: SplitAfterVariableSubstitution
-    ignoreExitCode: true
-    maxTimeInSeconds: 10800
-    maxTimeBetweenOutput: 900
-    userMessageOnFailure: >
-      Failed to run tests.

coin/instructions/coin_module_test_template_v2.yaml

@@ -5,6 +5,9 @@ instructions:
       - !include "{{qt/qtbase}}/coin_module_test_template_common.yaml"
       - type: Group
         instructions:
+          - type: EnvironmentVariable
+            variableName: COIN_CTEST_IGNORE_EXIT_CODE
+            variableValue: "1"
           - !include "{{qt/qtbase}}/cmake_regular_test_instructions.yaml"
         enable_if:
           condition: or

coin/instructions/coin_module_test_template_v3.yaml

@@ -5,7 +5,7 @@ instructions:
       - !include "{{qt/qtbase}}/coin_module_test_template_common.yaml"
       - type: Group
         instructions:
-          - !include "{{qt/qtbase}}/cmake_regular_test_instructions_enforced.yaml"
+          - !include "{{qt/qtbase}}/cmake_regular_test_instructions.yaml"
         enable_if:
           condition: or
           conditions: