Improve test results publishing in PRs and commits

.github/workflows/ci-build.yml

@@ -238,14 +238,14 @@ jobs:
         if: ${{ matrix.run_tests && (success() || failure()) }}
         with:
           name: MethaneKit_${{ matrix.name }}_TestResults_${{ env.METHANE_VERSION_MAJOR }}.${{ env.METHANE_VERSION_MINOR }}.${{ env.METHANE_VERSION_PATCH }}.${{ env.METHANE_VERSION_BUILD }}
-          path: ${{ env.INSTALL_DIR }}/Tests/Results/*.xml
+          path: ${{ env.INSTALL_DIR }}/Tests/Results/junit/*.xml
 
       - name: Upload Test Results and Build Log to TestSpace server
         if: ${{ github.repository == env.ORIGIN_REPOSITORY && matrix.run_tests && (success() || failure()) }}
         shell: bash
         run: |
           testspace \
-            "[ ${{ matrix.name }} ]${{ env.INSTALL_DIR }}/Tests/Results/*.xml" \
+            "[ ${{ matrix.name }} ]${{ env.INSTALL_DIR }}/Tests/Results/junit/*.xml" \
             "[ ${{ matrix.name }} ]${{ env.BUILD_LOG_FILE }}"
 
       - name: Add README and Build-Info files
@@ -304,3 +304,13 @@ jobs:
       - name: Windows disk space report
         if: ${{ runner.os == 'Windows' && always() }}
         run: Get-PSDrive
+
+  event_file:
+    name: "Event File"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Upload Event File
+        uses: actions/upload-artifact@v3
+        with:
+          name: EventFile
+          path: ${{ github.event_path }}

.github/workflows/ci-sonar-scan.yml

@@ -116,28 +116,18 @@ jobs:
       COMPILE_COMMANDS_FILE: Build/Output/${{ matrix.config_preset }}/Build/compile_commands.json
 
     steps:
-      - name: Checkout origin repository branch
+      - name: Checkout pull-request merge commit from origin
         if: ${{ github.event_name != 'pull_request_target' }}
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 2 # 2 commits are necessary for CodeCov diff
 
-      - name: Checkout fork repository branch
+      - name: Checkout pull-request merge commit from fork
         if: ${{ github.event_name == 'pull_request_target' }}
         uses: actions/checkout@v3
         with:
-          repository: ${{ github.event.pull_request.head.repo.full_name }}
-          ref: ${{ github.event.pull_request.head.ref }}
-          fetch-depth: 0
-
-      - name: Checkout base branch
-        if: ${{ github.event_name == 'pull_request_target' }}
-        run: |
-          git remote add upstream ${{ github.server_url }}/${{ github.event.pull_request.base.repo.full_name }}
-          git fetch upstream
-          git checkout -B ${{ github.event.pull_request.base.ref }} upstream/${{ github.event.pull_request.base.ref }}
-          git checkout ${{ github.event.pull_request.head.ref }}
-          git clean -ffdx && git reset --hard HEAD
+          ref: "${{ github.event.pull_request.merge_commit_sha }}"
+          fetch-depth: 2 # 2 commits are necessary for CodeCov diff
 
       - name: Install Linux prerequisites
         if: ${{ runner.os == 'Linux' }}
@@ -204,14 +194,14 @@ jobs:
         shell: cmd
         working-directory: 'Build\Output\${{ matrix.config_preset }}\Install\Tests'
         run: |
-          ${{ github.workspace }}\Build\Windows\CI\RunUnitTestsWithCoverage.bat sonarqube cobertura ^
+          ${{ github.workspace }}\Build\Windows\CI\RunUnitTestsWithCoverage.bat cobertura sonarqube junit ^
             "${{ github.workspace }}\Modules" ^
-            "Build/Output/${{ matrix.config_preset }}/Install/Tests"
+            Build/Output/${{ matrix.config_preset }}/Install/Tests
 
-      - name: Run all unit-tests to collect SonarQube test results on Linux
+      - name: Run all unit-tests to collect SonarQube and JUnit test results on Linux
         if: ${{ runner.os == 'Linux' }}
         working-directory: 'Build/Output/${{ matrix.config_preset }}/Install/Tests'
-        run: ${{ github.workspace }}/Build/Unix/CI/RunUnitTests.sh sonarqube
+        run: ${{ github.workspace }}/Build/Unix/CI/RunUnitTests.sh sonarqube junit
 
       - name: Run all unit-tests with code coverage using CTest and GCov on Linux
         if: ${{ runner.os == 'Linux' && (success() || failure()) }}
@@ -220,7 +210,7 @@ jobs:
       - name: Run all unit-tests with LCov code coverage on MacOS
         if: ${{ runner.os == 'macOS' }}
         working-directory: 'Build/Output/${{ matrix.config_preset }}/Install/Tests'
-        run: ${{ github.workspace }}/Build/Unix/CI/RunUnitTestsWithCoverage.sh sonarqube lcov
+        run: ${{ github.workspace }}/Build/Unix/CI/RunUnitTestsWithCoverage.sh lcov sonarqube junit
 
       - name: Generate Code Coverage Reports
         if: ${{ success() || failure() }}
@@ -247,14 +237,34 @@ jobs:
             "[ ${{ matrix.name }} ]Build/Output/${{ matrix.config_preset }}/Install/Tests/Coverage/Report/Cobertura.xml" \
             "[ ${{ matrix.name }} ]${{ env.BUILD_LOG_FILE }}"
 
-      - name: Upload Code Coverage to CodeCov server
-        if: ${{ success() || failure() }}
+      - name: Publish Code Coverage to CodeCov server
+        if: ${{ github.event_name != 'pull_request_target' && (success() || failure()) }}
         uses: codecov/codecov-action@v3
         with:
           files: Build/Output/${{ matrix.config_preset }}/Install/Tests/Coverage/Report/Cobertura.xml
           flags: unittests,${{ runner.os }}
           name: ${{ matrix.name }}
 
+      - name: Publish Code Coverage to CodeCov server (from PR target)
+        if: ${{ github.event_name == 'pull_request_target' && (success() || failure()) }}
+        uses: codecov/codecov-action@v3
+        with:
+          override_pr: ${{ github.event.pull_request.number }}
+          override_commit: ${{ github.event.pull_request.head.sha }}
+          files: Build/Output/${{ matrix.config_preset }}/Install/Tests/Coverage/Report/Cobertura.xml
+          flags: unittests,${{ runner.os }}
+          name: ${{ matrix.name }}
+
+      - name: Publish Test Results to GitHub
+        if: ${{ success() || failure() }}
+        uses: EnricoMi/publish-unit-test-result-action/composite@v2
+        with:
+          check_name: ${{ matrix.name }} Test Results
+          comment_title: ${{ matrix.name }} Test Results
+          files: Build/Output/${{ matrix.config_preset }}/Install/Tests/Results/junit/*Test.xml
+          action_fail: true
+          action_fail_on_inconclusive: true
+
       - name: Run Sonar Scanner
         if: ${{ success() || failure() }}
         shell: bash

.github/workflows/ci-tests-report.yml

@@ -12,6 +12,12 @@ on:
 jobs:
   test-report:
     name: ${{ matrix.name }} Test Report
+    if: ${{ github.event.workflow_run.conclusion != 'skipped' }}
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      pull-requests: write
+      actions: read
 
     strategy:
       fail-fast: false
@@ -25,43 +31,32 @@ jobs:
           - name: "MacOS_VK_Release"
           - name: "MacOS_MTL_Release"
 
-    runs-on: ubuntu-latest
-
     steps:
-      - name: Generate Test Results Report
-        id: test-reporter
-        uses: dorny/test-reporter@v1
+      - name: Download Event File
+        uses: dawidd6/action-download-artifact@246dbf436b23d7c49e21a7ab8204ca9ecd1fe615
         with:
-          artifact:  /MethaneKit_${{ matrix.name }}_TestResults_(.*)/
-          name: ${{ matrix.name }} Tests
-          path: '*Test.xml'
-          reporter: java-junit
-          fail-on-error: 'true'
+          run_id: ${{ github.event.workflow_run.id }}
+          name: EventFile
+          path: EventFile
 
-      - name: Add PR Comment with Test Results
-        uses: marocchino/sticky-pull-request-comment@v2
-        if: ${{ github.event.workflow_run.event == 'pull_request' && steps.test-reporter.outputs.conclusion == 'success' && (success() || failure()) }}
+      - name: Download Test Results
+        uses: dawidd6/action-download-artifact@v3
         with:
-          number: ${{ github.event.workflow_run.pull_requests[0].number }}
-          header: ${{ matrix.name }} Test Results
-          recreate: true
-          message: |
-            # ${{ matrix.name }} Test Results
-            
-            - :white_check_mark: ${{ steps.test-reporter.outputs.passed }} tests **passed**
-            - :x: ${{ steps.test-reporter.outputs.failed }} tests **failed**
-            - :warning: ${{ steps.test-reporter.outputs.skipped }} tests **skipped**
-            - :stopwatch: ${{ steps.test-reporter.outputs.time }} ms. run duration
-
-      - name: Add Commit Comment with Test Results
-        if: ${{ github.event.workflow_run.event == 'push' && steps.test-reporter.outputs.conclusion == 'success' && (success() || failure()) }}
-        uses: peter-evans/commit-comment@v2
+          run_id: ${{ github.event.workflow_run.id }}
+          name: ^MethaneKit_${{ matrix.name }}_TestResults_.*$
+          name_is_regexp: true
+          path: TestResults
+          allow_forks: true
+
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action/composite@v2
+        if: ${{ success() || failure() }}
         with:
-          sha: ${{ github.event.workflow_run.head_sha }}
-          body: |
-            # ${{ matrix.name }} Test Results
-
-            - :white_check_mark: ${{ steps.test-reporter.outputs.passed }} tests **passed**
-            - :x: ${{ steps.test-reporter.outputs.failed }} tests **failed**
-            - :warning: ${{ steps.test-reporter.outputs.skipped }} tests **skipped**
-            - :stopwatch: ${{ steps.test-reporter.outputs.time }} ms. run duration
+          check_name: ${{ matrix.name }} Test Results
+          comment_title: ${{ matrix.name }} Test Results
+          files: TestResults/**/*Test.xml
+          event_file: EventFile/event.json
+          commit: ${{ github.event.workflow_run.head_sha }}
+          event_name: ${{ github.event.workflow_run.event }}
+          action_fail: true
+          action_fail_on_inconclusive: true

Build/Unix/CI/RunUnitTests.sh

@@ -2,21 +2,33 @@
 # CI script to run all unit-test executables "*Test" from current directory
 set +e
 test_format="${1}"
+extra_test_format="${2}"
 result_ext='.xml'
 test_results=''
 result_error_level=0
 echo Running unit-tests in directory $PWD
 mkdir Results
+mkdir Results/${test_format}
+if [ ! -z "$extra_test_format" ]; then
+    mkdir Results/${extra_test_format}
+fi
 for test_exe in *Test
 do
-  ./$test_exe -r "${test_format}" -o "Results/$test_exe$result_ext"
+  out_test_result=Results/${test_format}/$test_exe$result_ext
+  if [ -z "$extra_test_format" ]; then
+    ./$test_exe -r "${test_format}" -o "${out_test_result}"
+  else
+    out_extra_test_result=Results/${extra_test_format}/${test_exe}${result_ext}
+    ./$test_exe -r "${test_format}::out=${out_test_result}" \
+                -r "${extra_test_format}::out=${out_extra_test_result}"
+  fi
   last_error_level=$?
   echo  - $test_exe - completed with $last_error_level exit status
   if [ $last_error_level != 0 ]; then
     result_error_level=$last_error_level
   fi
-  if [ -f "$PWD/Results/$test_exe$result_ext" ]; then
-    test_results+="$PWD/Results/$test_exe$result_ext,"
+  if [ -f "$PWD/${out_test_result}" ]; then
+    test_results+="$PWD/${out_test_result},"
   fi
 done
 echo "Test Result Files: $test_results"

Build/Unix/CI/RunUnitTestsWithCoverage.sh

@@ -1,8 +1,9 @@
 #!/bin/bash
 # CI script to run all unit-test executables "*Test" from current directory with code-coverage data collection
 set +e
-test_format="${1}"
-coverage_format="${2}"
+coverage_format="${1}"
+test_format="${2}"
+extra_test_format="${3}"
 result_error_level=0
 result_ext='.xml'
 prof_data_ext='.profdata'
@@ -11,17 +12,22 @@ lcov_ext='.lcov'
 test_results=''
 echo Running unit-tests and Converting LLVM code coverage data to lcov text format in directory $PWD
 mkdir Results
+mkdir Results/${test_format}
+mkdir Results/${extra_test_format}
 mkdir Coverage
 for test_exe in *Test
 do
-  ./$test_exe -r "${test_format}" -o "Results/$test_exe$result_ext"
+  out_test_result=Results/${test_format}/${test_exe}${result_ext}
+  out_extra_test_result=Results/${extra_test_format}/${test_exe}${result_ext}
+  ./$test_exe -r "${test_format}::out=${out_test_result}" \
+              -r "${extra_test_format}::out=${out_extra_test_result}"
   last_error_level=$?
   echo  - $test_exe - completed with $last_error_level exit status
   if [ $last_error_level != 0 ]; then
     result_error_level=$last_error_level
   fi
-  if [ -f "$PWD/Results/$test_exe$result_ext" ]; then
-    test_results+="$PWD/Results/$test_exe$result_ext,"
+  if [ -f "$PWD/${out_test_result}" ]; then
+    test_results+="$PWD/${out_test_result},"
   fi
   if [ ! -f default.profraw ]; then
     continue

Build/Windows/CI/RunUnitTests.bat

@@ -4,8 +4,9 @@ set test_format=%1
 echo Running unit-tests in directory "%cd%"
 set /A result_error_level=0
 mkdir Results
+mkdir Results\%test_format%
 for /r "." %%a in (*Test.exe) do (
-  "%%~fa" -r %test_format% -o "%%~dpa\Results\%%~na.xml"
+  "%%~fa" -r %test_format% -o "%%~dpa\Results\%test_format%\%%~na.xml"
   echo  - %%~na - completed with !errorlevel! exit status
   if not !errorlevel!==0 (
     set /A result_error_level=!errorlevel!