diff --git a/.github/actions/launchable/setup/action.yml b/.github/actions/launchable/setup/action.yml index 2d92cb7d99..6ab8763df9 100644 --- a/.github/actions/launchable/setup/action.yml +++ b/.github/actions/launchable/setup/action.yml @@ -3,11 +3,6 @@ description: >- Install the required dependencies and execute the necessary Launchable commands for test recording inputs: - report-path: - default: launchable_reports.json - required: true - description: The file path of the test report for uploading to Launchable - os: required: true description: The operating system that CI runs on. This value is used in Launchable flavor. @@ -38,18 +33,27 @@ inputs: Directory to (re-)checkout source codes. Launchable retrieves the commit information from the directory. - launchable-workspace: - required: true - default: ${{ github.event.repository.name }} - description: >- - A workspace name in Launchable - test-task: - required: true + required: false default: ${{ matrix.test_task }} description: >- - A test task that determine which tests are executed. + Specifies a single test task to be executed. This value is used in the Launchable flavor. + Either 'test-task' or 'multi-test-tasks' must be configured. + + test-tasks: + required: false + default: '[]' + description: >- + Specifies an array of multiple test tasks to be executed. + For example: '["test", "test-all"]'. + If you want to run a single test task, use the 'test-task' input instead. + + is-yjit: + required: false + default: 'false' + description: >- + Whether this workflow is executed on YJIT. runs: using: composite @@ -61,11 +65,14 @@ runs: shell: bash if: >- ${{ - (github.repository == 'ruby/ruby' || - (github.repository != 'ruby/ruby' && env.LAUNCHABLE_TOKEN)) && - (inputs.test-task == 'check' || - inputs.test-task == 'test-all' || - inputs.test-task == 'test') + (github.repository == 'ruby/ruby' + || (github.repository != 'ruby/ruby' + && env.LAUNCHABLE_TOKEN)) + && (inputs.test-task == 'check' + || inputs.test-task == 'test-all' + || inputs.test-task == 'test' + || contains(fromJSON(inputs.test-tasks), 'test-all') + || contains(fromJSON(inputs.test-tasks), 'test')) }} # Launchable CLI requires Python and Java. @@ -83,6 +90,20 @@ runs: java-version: '17' if: steps.enable-launchable.outputs.enable-launchable + - name: Set global vars + id: global + shell: bash + run: | + test_all_enabled="${{ inputs.test-task == 'check' || inputs.test-task == 'test-all' || contains(fromJSON(inputs.test-tasks), 'test-all') }}" + btest_enabled="${{ inputs.test-task == 'check' || inputs.test-task == 'test' || contains(fromJSON(inputs.test-tasks), 'test') }}" + echo test_all_enabled="${test_all_enabled}" >> $GITHUB_OUTPUT + echo btest_enabled="${btest_enabled}" >> $GITHUB_OUTPUT + echo test_all_session_file='launchable_test_all_session.txt' >> $GITHUB_OUTPUT + echo btest_session_file='launchable_btest_session.txt' >> $GITHUB_OUTPUT + echo test_all_report_file='launchable_test_all_report.json' >> $GITHUB_OUTPUT + echo btest_report_file='launchable_btest_report.json' >> $GITHUB_OUTPUT + if: steps.enable-launchable.outputs.enable-launchable + - name: Set environment variables for Launchable shell: bash run: | @@ -92,7 +113,7 @@ runs: : # The following envs are necessary in Launchable tokenless authentication. : # https://github.com/launchableinc/cli/blob/v1.80.1/launchable/utils/authentication.py#L20 echo "LAUNCHABLE_ORGANIZATION=${{ github.repository_owner }}" >> $GITHUB_ENV - echo "LAUNCHABLE_WORKSPACE=${{ inputs.launchable-workspace }}" >> $GITHUB_ENV + echo "LAUNCHABLE_WORKSPACE=${{ github.event.repository.name }}" >> $GITHUB_ENV : # https://github.com/launchableinc/cli/blob/v1.80.1/launchable/utils/authentication.py#L71 echo "GITHUB_PR_HEAD_SHA=${{ github.event.pull_request.head.sha || github.sha }}" >> $GITHUB_ENV echo "LAUNCHABLE_TOKEN=${{ inputs.launchable-token }}" >> $GITHUB_ENV @@ -121,20 +142,51 @@ runs: : # : # FIXME: Need to fix `WARNING: Failed to process a change to a file`. : # https://github.com/launchableinc/cli/issues/786 - launchable record build --name ${github_ref}_${GITHUB_PR_HEAD_SHA} - echo "TESTS=${TESTS} --launchable-test-reports=${{ inputs.report-path }}" >> $GITHUB_ENV + build_name="${github_ref}_${GITHUB_PR_HEAD_SHA}" + test_opts="${{ inputs.test-opts }}" + test_opts="${test_opts// /}" + test_opts="${test_opts//=/:}" + test_all_test_suite='test-all' + btest_test_suite='btest' + if [ "${{ inputs.is-yjit }}" = "true" ]; then + test_all_test_suite="yjit-${test_all_test_suite}" + btest_test_suite="yjit-${btest_test_suite}" + fi + launchable record build --name "${build_name}" + if [ "${test_all_enabled}" = "true" ]; then + launchable record session \ + --build "${build_name}" \ + --flavor os=${{ inputs.os }} \ + --flavor test_task=${{ inputs.test-task }} \ + --flavor test_opts=${test_opts} \ + --test-suite ${test_all_test_suite} \ + > "${test_all_session_file}" + echo "TESTS=${TESTS} --launchable-test-reports=${test_all_report_file}" >> $GITHUB_ENV + fi + if [ "${btest_enabled}" = "true" ]; then + launchable record session \ + --build "${build_name}" \ + --flavor os=${{ inputs.os }} \ + --flavor test_task=${{ inputs.test-task }} \ + --flavor test_opts=${test_opts} \ + --test-suite ${btest_test_suite} \ + > "${btest_session_file}" + echo "BTESTS=${BTESTS} --launchable-test-reports=${btest_report_file}" >> $GITHUB_ENV + fi if: steps.enable-launchable.outputs.enable-launchable + env: + test_all_enabled: ${{ steps.global.outputs.test_all_enabled }} + btest_enabled: ${{ steps.global.outputs.btest_enabled }} + test_all_session_file: ${{ steps.global.outputs.test_all_session_file }} + btest_session_file: ${{ steps.global.outputs.btest_session_file }} + test_all_report_file: ${{ steps.global.outputs.test_all_report_file }} + btest_report_file: ${{ steps.global.outputs.btest_report_file }} - name: Variables to report Launchable id: variables shell: bash run: | set -x - : # flavor - test_opts="${{ inputs.test-opts }}" - test_opts="${test_opts// /}" - test_opts="${test_opts//=/:}" - echo test-opts="$test_opts" >> $GITHUB_OUTPUT : # report-path from srcdir if [ "${srcdir}" = "${{ github.workspace }}" ]; then dir= @@ -142,13 +194,22 @@ runs: # srcdir must be equal to or under workspace dir=$(echo ${srcdir:+${srcdir}/} | sed 's:[^/][^/]*/:../:g') fi - report_path="${dir}${builddir:+${builddir}/}${report_path}" - echo report-path="${report_path}" >> $GITHUB_OUTPUT + if [ "${test_all_enabled}" = "true" ]; then + test_report_path="${dir}${builddir:+${builddir}/}${test_all_report_file}" + echo test_report_path="${test_report_path}" >> $GITHUB_OUTPUT + fi + if [ "${btest_enabled}" = "true" ]; then + btest_report_path="${dir}${builddir:+${builddir}/}${btest_report_file}" + echo btest_report_path="${btest_report_path}" >> $GITHUB_OUTPUT + fi if: steps.enable-launchable.outputs.enable-launchable env: srcdir: ${{ inputs.srcdir }} builddir: ${{ inputs.builddir }} - report_path: ${{ inputs.report-path }} + test_all_enabled: ${{ steps.global.outputs.test_all_enabled }} + btest_enabled: ${{ steps.global.outputs.btest_enabled }} + test_all_report_file: ${{ steps.global.outputs.test_all_report_file }} + btest_report_file: ${{ steps.global.outputs.btest_report_file }} - name: Record test results in Launchable uses: gacts/run-and-post-run@674528335da98a7afc80915ff2b4b860a0b3553a # v1.4.0 @@ -156,10 +217,25 @@ runs: shell: bash working-directory: ${{ inputs.srcdir }} post: | - : # record - launchable record tests --flavor os=${{ inputs.os }} --flavor test_task=${{ inputs.test-task }} --flavor test_opts=${test_opts} raw ${report_path} - rm -f ${report_path} + [[ "${test_all_enabled}" = "true" ]] && \ + launchable record tests \ + --session "$(cat "${test_all_session_file}")" \ + raw "${test_report_path}" || true + + [[ "${btest_enabled}" = "true" ]] && \ + launchable record tests \ + --session "$(cat "${btest_session_file}")" \ + raw "${btest_report_path}" || true + + rm -f "${test_all_session_file}" + rm -f "${btest_session_file}" + rm -f "${test_report_path}" + rm -f "${btest_report_path}" if: ${{ always() && steps.enable-launchable.outputs.enable-launchable }} env: - test_opts: ${{ steps.variables.outputs.test-opts }} - report_path: ${{ steps.variables.outputs.report-path }} + test_report_path: ${{ steps.variables.outputs.test_report_path }} + btest_report_path: ${{ steps.variables.outputs.btest_report_path }} + test_all_enabled: ${{ steps.global.outputs.test_all_enabled }} + btest_enabled: ${{ steps.global.outputs.btest_enabled }} + test_all_session_file: ${{ steps.global.outputs.test_all_session_file }} + btest_session_file: ${{ steps.global.outputs.btest_session_file }} diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml index 37cbfa5f38..e03a8d2838 100644 --- a/.github/workflows/mingw.yml +++ b/.github/workflows/mingw.yml @@ -123,12 +123,10 @@ jobs: uses: ./.github/actions/launchable/setup with: os: windows-2022 - # If we support new test task, we need to change this test-opts. - test-opts: --retry --job-status=normal --show-skip --timeout-scale=1.5 - ${{ matrix.test-all-opts }} launchable-token: ${{ secrets.LAUNCHABLE_TOKEN }} builddir: build srcdir: src + test-tasks: '["test", "test-all"]' continue-on-error: true - name: test diff --git a/.github/workflows/rjit.yml b/.github/workflows/rjit.yml index 12e6b94b53..320534d99c 100644 --- a/.github/workflows/rjit.yml +++ b/.github/workflows/rjit.yml @@ -83,12 +83,11 @@ jobs: uses: ./.github/actions/launchable/setup with: os: ubuntu-22.04 - test-task: test launchable-token: ${{ secrets.LAUNCHABLE_TOKEN }} builddir: build srcdir: src - launchable-workspace: ruby-make-btest test-opts: ${{ matrix.run_opts }} + test-tasks: '["test", "test-all"]' continue-on-error: true - name: make test @@ -103,8 +102,10 @@ jobs: RUN_OPTS: ${{ matrix.run_opts }} - name: make test-all - run: | - $SETARCH make -s test-all RUN_OPTS="$RUN_OPTS" + run: >- + $SETARCH make -s test-all + RUN_OPTS="$RUN_OPTS" + ${TESTS:+TESTS="$TESTS"} timeout-minutes: 60 env: GNUMAKEFLAGS: '' diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 619f9c00af..90c8af82d6 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -181,6 +181,7 @@ jobs: launchable-token: ${{ secrets.LAUNCHABLE_TOKEN }} builddir: build srcdir: src + test-task: ${{ matrix.test_task || 'check' }} continue-on-error: true if: ${{ matrix.test_task != 'test-bundled-gems' }} diff --git a/.github/workflows/yjit-macos.yml b/.github/workflows/yjit-macos.yml index e93981f379..b322abbfa1 100644 --- a/.github/workflows/yjit-macos.yml +++ b/.github/workflows/yjit-macos.yml @@ -124,6 +124,7 @@ jobs: launchable-token: ${{ secrets.LAUNCHABLE_TOKEN }} builddir: build srcdir: src + is-yjit: true continue-on-error: true - name: make ${{ matrix.test_task }} diff --git a/.github/workflows/yjit-ubuntu.yml b/.github/workflows/yjit-ubuntu.yml index 7b7ec644a2..f1c62c6c77 100644 --- a/.github/workflows/yjit-ubuntu.yml +++ b/.github/workflows/yjit-ubuntu.yml @@ -177,6 +177,7 @@ jobs: launchable-token: ${{ secrets.LAUNCHABLE_TOKEN }} builddir: build srcdir: src + is-yjit: true continue-on-error: true - name: make ${{ matrix.test_task }}