Launchable: Start recording test-spec results (#12302)

Merged-By: ono-max <onoto1998@gmail.com>

.github/actions/launchable/setup/action.yml

@@ -96,12 +96,16 @@ runs:
       run: |
         test_all_enabled="${{ inputs.test-task == 'check' ||  inputs.test-task == 'test-all' || contains(fromJSON(inputs.test-tasks), 'test-all') }}"
         btest_enabled="${{ inputs.test-task == 'check' ||  inputs.test-task == 'test' || contains(fromJSON(inputs.test-tasks), 'test') }}"
+        test_spec_enabled="${{ inputs.test-task == 'check' ||  inputs.test-task == 'test-spec' || contains(fromJSON(inputs.test-tasks), 'test-spec') }}"
         echo test_all_enabled="${test_all_enabled}" >> $GITHUB_OUTPUT
         echo btest_enabled="${btest_enabled}" >> $GITHUB_OUTPUT
+        echo test_spec_enabled="${test_spec_enabled}" >> $GITHUB_OUTPUT
         echo test_all_session_file='launchable_test_all_session.txt' >> $GITHUB_OUTPUT
         echo btest_session_file='launchable_btest_session.txt' >> $GITHUB_OUTPUT
+        echo test_spec_session_file='launchable_test_spec_session.txt' >> $GITHUB_OUTPUT
         echo test_all_report_file='launchable_test_all_report.json' >> $GITHUB_OUTPUT
         echo btest_report_file='launchable_btest_report.json' >> $GITHUB_OUTPUT
+        echo test_spec_report_dir='launchable_test_spec_report' >> $GITHUB_OUTPUT
       if: steps.enable-launchable.outputs.enable-launchable
 
     - name: Set environment variables for Launchable
@@ -144,9 +148,11 @@ runs:
         test_opts="${test_opts//=/:}"
         test_all_test_suite='test-all'
         btest_test_suite='btest'
+        test_spec_test_suite='test-spec'
         if [ "${{ inputs.is-yjit }}" = "true" ]; then
           test_all_test_suite="yjit-${test_all_test_suite}"
           btest_test_suite="yjit-${btest_test_suite}"
+          test_spec_test_suite="yjit-${test_spec_test_suite}"
         fi
         launchable record build --name "${build_name}"
         if [ "${test_all_enabled}" = "true" ]; then
@@ -169,18 +175,32 @@ runs:
             > "${btest_session_file}"
           echo "BTESTS=${BTESTS} --launchable-test-reports=${btest_report_file}" >> $GITHUB_ENV
         fi
+        if [ "${test_spec_enabled}" = "true" ]; then
+          launchable record session \
+            --build "${build_name}" \
+            --flavor os=${{ inputs.os }} \
+            --flavor test_task=${{ inputs.test-task }} \
+            --flavor test_opts=${test_opts} \
+            --test-suite ${test_spec_test_suite} \
+            > "${test_spec_session_file}"
+          echo "SPECOPTS=${SPECOPTS} --launchable-test-reports=${test_spec_report_dir}" >> $GITHUB_ENV
+        fi
       if: steps.enable-launchable.outputs.enable-launchable
       env:
         test_all_enabled: ${{ steps.global.outputs.test_all_enabled }}
         btest_enabled: ${{ steps.global.outputs.btest_enabled }}
+        test_spec_enabled: ${{ steps.global.outputs.test_spec_enabled }}
         test_all_session_file: ${{ steps.global.outputs.test_all_session_file }}
         btest_session_file: ${{ steps.global.outputs.btest_session_file }}
+        test_spec_session_file: ${{ steps.global.outputs.test_spec_session_file }}
         test_all_report_file: ${{ steps.global.outputs.test_all_report_file }}
         btest_report_file: ${{ steps.global.outputs.btest_report_file }}
+        test_spec_report_dir: ${{ steps.global.outputs.test_spec_report_dir }}
 
     - name: Variables to report Launchable
       id: variables
       shell: bash
+      working-directory: ${{ inputs.srcdir }}
       run: |
         set -x
         : # report-path from srcdir
@@ -198,14 +218,21 @@ runs:
           btest_report_path="${dir}${builddir:+${builddir}/}${btest_report_file}"
           echo btest_report_path="${btest_report_path}" >> $GITHUB_OUTPUT
         fi
+        if [ "${test_spec_enabled}" = "true" ]; then
+          test_spec_report_path="${dir}${builddir:+${builddir}/}${test_spec_report_dir}"
+          mkdir "${test_spec_report_path}"
+          echo test_spec_report_path="${test_spec_report_path}" >> $GITHUB_OUTPUT
+        fi
       if: steps.enable-launchable.outputs.enable-launchable
       env:
         srcdir: ${{ inputs.srcdir }}
         builddir: ${{ inputs.builddir }}
         test_all_enabled: ${{ steps.global.outputs.test_all_enabled }}
         btest_enabled: ${{ steps.global.outputs.btest_enabled }}
+        test_spec_enabled: ${{ steps.global.outputs.test_spec_enabled }}
         test_all_report_file: ${{ steps.global.outputs.test_all_report_file }}
         btest_report_file: ${{ steps.global.outputs.btest_report_file }}
+        test_spec_report_dir: ${{ steps.global.outputs.test_spec_report_dir }}
 
     - name: Record test results in Launchable
       uses: gacts/run-and-post-run@674528335da98a7afc80915ff2b4b860a0b3553a # v1.4.0
@@ -223,15 +250,25 @@ runs:
               --session "$(cat "${btest_session_file}")" \
               raw "${btest_report_path}" || true
 
+          [[ "${test_spec_enabled}" = "true" ]] && \
+            launchable record tests \
+              --session "$(cat "${test_spec_session_file}")" \
+              raw ${test_spec_report_path}/* || true
+
           rm -f "${test_all_session_file}"
           rm -f "${btest_session_file}"
+          rm -f "${test_spec_session_file}"
           rm -f "${test_report_path}"
           rm -f "${btest_report_path}"
+          rm -fr "${test_spec_report_path}"
       if: ${{ always() && steps.enable-launchable.outputs.enable-launchable }}
       env:
         test_report_path: ${{ steps.variables.outputs.test_report_path }}
         btest_report_path: ${{ steps.variables.outputs.btest_report_path }}
+        test_spec_report_path: ${{ steps.variables.outputs.test_spec_report_path }}
         test_all_enabled: ${{ steps.global.outputs.test_all_enabled }}
         btest_enabled: ${{ steps.global.outputs.btest_enabled }}
+        test_spec_enabled: ${{ steps.global.outputs.test_spec_enabled }}
         test_all_session_file: ${{ steps.global.outputs.test_all_session_file }}
         btest_session_file: ${{ steps.global.outputs.btest_session_file }}
+        test_spec_session_file: ${{ steps.global.outputs.test_spec_session_file }}

.github/workflows/mingw.yml

@@ -126,7 +126,7 @@ jobs:
           launchable-token: ${{ secrets.LAUNCHABLE_TOKEN }}
           builddir: build
           srcdir: src
-          test-tasks: '["test", "test-all"]'
+          test-tasks: '["test", "test-all", "test-spec"]'
         continue-on-error: true
 
       - name: test

.github/workflows/rjit.yml

@@ -87,7 +87,7 @@ jobs:
           builddir: build
           srcdir: src
           test-opts: ${{ matrix.run_opts }}
-          test-tasks: '["test", "test-all"]'
+          test-tasks: '["test", "test-all", "test-spec"]'
         continue-on-error: true
 
       - name: make test

spec/mspec/lib/mspec/commands/mspec-run.rb

@@ -53,6 +53,9 @@ class MSpecRun < MSpecScript
     options.doc "\n When to perform it"
     options.action_filters
 
+    options.doc "\n Launchable"
+    options.launchable
+
     options.doc "\n Help!"
     options.debug
     options.version MSpec::VERSION

spec/mspec/lib/mspec/runner/formatters/launchable.rb

@@ -0,0 +1,88 @@
+module LaunchableFormatter
+  def self.extend_object(obj)
+    super
+    obj.init
+  end
+
+  def self.setDir(dir)
+    @@path = File.join(dir, "#{rand.to_s}.json")
+    self
+  end
+
+  def init
+    @timer = nil
+    @tests = []
+  end
+
+  def before(state = nil)
+    super
+    @timer = TimerAction.new
+    @timer.start
+  end
+
+  def after(state = nil)
+    super
+    @timer.finish
+    file = MSpec.file
+    return if file.nil? || state&.example.nil? || exception?
+
+    @tests << {:test => state, :file => file, :exception => false, duration: @timer.elapsed}
+  end
+
+  def exception(exception)
+    super
+    @timer.finish
+    file = MSpec.file
+    return if file.nil?
+
+    @tests << {:test => exception, :file => file, :exception => true, duration: @timer.elapsed}
+  end
+
+  def finish
+    super
+
+    require_relative '../../../../../../tool/lib/launchable'
+
+    @writer = writer = Launchable::JsonStreamWriter.new(@@path)
+    @writer.write_array('testCases')
+    at_exit {
+      @writer.close
+    }
+
+    repo_path = File.expand_path("#{__dir__}/../../../../../../")
+
+    @tests.each do |t|
+      testcase = t[:test].description
+      relative_path = t[:file].delete_prefix("#{repo_path}/")
+      # The test path is a URL-encoded representation.
+      # https://github.com/launchableinc/cli/blob/v1.81.0/launchable/testpath.py#L18
+      test_path = {file: relative_path, testcase: testcase}.map{|key, val|
+        "#{encode_test_path_component(key)}=#{encode_test_path_component(val)}"
+      }.join('#')
+
+      status = 'TEST_PASSED'
+      if t[:exception]
+        message = t[:test].message
+        backtrace = t[:test].backtrace
+        e = "#{message}\n#{backtrace}"
+        status = 'TEST_FAILED'
+      end
+
+      @writer.write_object(
+        {
+          testPath: test_path,
+          status: status,
+          duration: t[:duration],
+          createdAt: Time.now.to_s,
+          stderr: e,
+          stdout: nil
+        }
+      )
+    end
+  end
+
+  private
+  def encode_test_path_component component
+    component.to_s.gsub('%', '%25').gsub('=', '%3D').gsub('#', '%23').gsub('&', '%26').tr("\x00-\x08", "")
+  end
+end

spec/mspec/lib/mspec/utils/options.rb

@@ -489,6 +489,14 @@ class MSpecOptions
     end
   end
 
+  def launchable
+    on("--launchable-test-reports", "DIR",
+       "DIR The directory for reporting test results in Launchable JSON format") do |o|
+      require 'mspec/runner/formatters/launchable'
+      config[:launchable] = LaunchableFormatter.setDir(o)
+    end
+  end
+
   def all
     configure {}
     env
@@ -508,5 +516,6 @@ class MSpecOptions
     action_filters
     actions
     debug
+    launchable
   end
 end

spec/mspec/lib/mspec/utils/script.rb

@@ -159,8 +159,14 @@ class MSpecScript
     end
 
     if config[:formatter]
-      config[:formatter].new(config[:output])
+      config[:formatter] = config[:formatter].new(config[:output])
     end
+
+    if config[:launchable]
+      config[:formatter].extend config[:launchable]
+    end
+
+    config[:formatter]
   end
 
   # Callback for enabling custom actions, etc. This version is a