Merge pull request #41 from diegov/add_test_job_summary_to_action

Add test results summary to GH action

.github/workflows/rust.yml

@@ -79,14 +79,14 @@ jobs:
     - name: Run integration tests
       if: github.event_name != 'schedule'
       # CI run, test against the versions we specify in the firefox_versions file
-      run: ./run_tests.sh -f origin "$XPI_FILE" 2>>test.log
+      run: ./run_tests.sh -o "$GITHUB_STEP_SUMMARY" -f origin "$XPI_FILE" 2>>test.log
       working-directory: integration_tests
       env:
         VIRTUALENV_DIR: ${{ env.XDG_CACHE_HOME }}/virtualenv
     - name: Run integration tests against latest FF version
       if: github.event_name == 'schedule'
       # Scheduled job, test against the latest FF developer edition (the `-l` option)
-      run: ./run_tests.sh -l -f origin "$XPI_FILE" 2>>test.log
+      run: ./run_tests.sh -o "$GITHUB_STEP_SUMMARY" -l -f origin "$XPI_FILE" 2>>test.log
       working-directory: integration_tests
       env:
         VIRTUALENV_DIR: ${{ env.XDG_CACHE_HOME }}/virtualenv

integration_tests/firefox_versions

@@ -1,2 +1,2 @@
-106.0b9
 124.0b3
+126.0b9

integration_tests/requirements.in

@@ -0,0 +1,8 @@
+beautifulsoup4>=4.10.0,<5.0.0
+marionette-driver>=3.4.0,<4.0.0
+mozprofile>=2.5.0,<3.0.0
+packaging>=21.3,<22.0
+PyHamcrest>=2.0.3,<3.0.0
+requests>=2.27.1,<3.0.0
+snakemd>=2.2.0,<3.0.0
+types-requests>=2.27.1,<3.0.0

integration_tests/requirements.txt

@@ -1,7 +1,51 @@
-marionette-driver==3.1.0
-requests==2.27.1
-types-requests==2.27.1
-mozprofile==2.5.0
-PyHamcrest==2.0.3
+beautifulsoup4==4.12.3
+blessed==1.20.0
+certifi==2024.6.2
+charset-normalizer==3.3.2
+distro==1.9.0
+idna==3.7
+marionette-driver==3.4.0
+mozdevice==4.1.2
+mozfile==3.0.0
+mozinfo==1.2.3
+mozlog==8.0.0
+mozprocess==1.3.1
+mozprofile==2.6.1
+mozrunner==8.3.0
+mozterm==1.0.0
+mozversion==2.4.0
 packaging==21.3
-beautifulsoup4==4.10.0
+PyHamcrest==2.1.0
+pyparsing==3.1.2
+requests==2.32.3
+six==1.16.0
+soupsieve==2.5
+types-requests==2.32.0.20240602
+urllib3==2.2.1
+wcwidth==0.2.13
+beautifulsoup4==4.12.3
+blessed==1.20.0
+certifi==2024.6.2
+charset-normalizer==3.3.2
+distro==1.9.0
+idna==3.7
+marionette-driver==3.4.0
+mozdevice==4.1.2
+mozfile==3.0.0
+mozinfo==1.2.3
+mozlog==8.0.0
+mozprocess==1.3.1
+mozprofile==2.6.1
+mozrunner==8.3.0
+mozterm==1.0.0
+mozversion==2.4.0
+packaging==21.3
+PyHamcrest==2.1.0
+pyparsing==3.1.2
+requests==2.32.3
+six==1.16.0
+snakemd==2.2.0
+soupsieve==2.5
+types-requests==2.32.0.20240602
+urllib3==2.2.1
+wcwidth==0.2.13

integration_tests/run_tests.sh

@@ -1,25 +1,29 @@
 #!/usr/bin/env bash
 
-set -e
-set -o pipefail
+set -eEo pipefail
 
 function usage() {
     echo "Usage: $0 [-f <remote>] [-l] <xpi_file>" 1>&2
     echo "    xpi_file    path of the XPI extension to test" 1>&2
-    echo "    -f          Fetch tag from specified remote for backwards compatibility tests" 1>&2
+    echo "    -f REMOTE   Fetch tag from specified REMOTE for backwards compatibility tests" 1>&2
+    echo "    -o FILE     Write results as markdown to FILE" 1>&2
     echo "    -l          Test against latest available FF version. If not provided, the versions" 1>&2
     echo "                are read from the firefox_versions file." 1>&2
     exit "$1"
 }
 
 FETCH_REMOTE=
+REPORT_MD_FILE=
 FF_USE_LATEST=no
 
-while getopts "f:lh" o; do
+while getopts "f:o:lh" o; do
     case "${o}" in
         f)
             FETCH_REMOTE=${OPTARG}
             ;;
+        o)
+            REPORT_MD_FILE="${OPTARG}"
+            ;;
         l)
             FF_USE_LATEST=yes
             ;;
@@ -89,10 +93,36 @@ export HOME="$TMP_HOME"
 
 export PATH="$TMP_HOME"/.local/bin:"$PATH"
 
+
+function run-suite() {
+    local ff_version="$1"
+    local xpi_file="$2"
+    local host_target_version="$3"
+
+    local invocation=( dbus-launch "$VIRTUALENV_DIR"/bin/python3 tabreport_tests.py "$ff_version" "$xpi_file" )
+    tmpfile=
+    if [ "$REPORT_MD_FILE" != "" ]; then
+        tmpfile="$(mktemp)"
+        invocation=( "${invocation[@]}" -o "$tmpfile" )
+    fi
+
+    if [ "$host_target_version" != "" ]; then
+        echo "Running integration tests with Firefox $ff_version and native host version ${host_target_version}" >&2
+        echo "" >&2
+        HOST_TARGET_VERSION="$host_target_version" "${invocation[@]}"
+    else
+        echo "Running integration tests with Firefox $ff_version" >&2
+        echo "" >&2
+        "${invocation[@]}"
+    fi
+
+    if [ "$REPORT_MD_FILE" != "" ]; then
+        cat "$tmpfile" >> "$REPORT_MD_FILE"
+    fi
+}
+
 while read -r ff_version; do
-    echo "Running integration tests with Firefox $ff_version" >&2
-    echo "" >&2
-    dbus-launch "$VIRTUALENV_DIR"/bin/python3 tabreport_tests.py "$ff_version" "$xpi_file"
+    run-suite "$ff_version" "$xpi_file"
 done < <(get-ff-versions)
 
 # Backwards compatibility test, since extensions are likely updated automatically
@@ -115,10 +145,6 @@ HOME="$ORIGINAL_HOME" ./install_native.sh "$TMP_HOME"
 popd || exit 1
 
 while read -r ff_version; do
-    if [ "$ff_version" != "" ]; then
-        echo "Running integration tests with Firefox $ff_version and native host version ${HOST_TARGET_VERSION}" >&2
-        echo "" >&2
-        HOST_TARGET_VERSION="$HOST_TARGET_VERSION" dbus-launch "$VIRTUALENV_DIR"/bin/python3 tabreport_tests.py "$ff_version" "$xpi_file"
-    fi
+    run-suite "$ff_version" "$xpi_file" "$HOST_TARGET_VERSION"
     # Execute against latest FF version only
 done < <(get-ff-versions | tail -n 1)

integration_tests/tabreport_tests.py

@@ -3,6 +3,7 @@
 import sys
 import os
 import os.path
+import argparse
 import subprocess
 import time
 import json
@@ -12,7 +13,7 @@
 import unittest
 from hamcrest import assert_that, has_length, starts_with
 from packaging import version
-from packaging.version import Version
+import snakemd
 
 
 SLEEP_TIME = 0.25
@@ -26,21 +27,9 @@ def get_tabs() -> List[Dict[str, Any]]:
     return json.loads(result)  # type: ignore
 
 
-FF_VERSION = sys.argv[1]
-EXTENSION_PATH = os.path.realpath(sys.argv[2])
-sys.argv = sys.argv[2:]
-
-
-host_version_string = os.environ.get("HOST_TARGET_VERSION")
-if host_version_string:
-    HOST_TARGET_VERSION = version.parse(host_version_string)
-else:
-    HOST_TARGET_VERSION = None
-
-
-print(f"Running tests for extension {EXTENSION_PATH} using Firefox {FF_VERSION}")
-if not os.path.isfile(EXTENSION_PATH):
-    raise FileNotFoundError(EXTENSION_PATH)
+FF_VERSION: str = None  # type: ignore
+EXTENSION_PATH: str = None  # type: ignore
+HOST_TARGET_VERSION: version.Version | version.LegacyVersion | None = None
 
 
 class IntegrationTests(unittest.TestCase):
@@ -320,5 +309,105 @@ def test_activate_invalid_tab(self):
             self.assertEqual(url, "http://127.0.7.1:9919/four.html")
 
 
+def main():
+    global FF_VERSION, EXTENSION_PATH, HOST_TARGET_VERSION
+
+    parser = argparse.ArgumentParser(
+        prog="tabreport_tests.py", description="Run tabreport integration tests"
+    )
+
+    parser.add_argument(
+        "firefox_version", action="store", help="Version of Firefox to test against"
+    )
+    parser.add_argument(
+        "extension_path", action="store", help="Full path to the built extension"
+    )
+    parser.add_argument(
+        "-o",
+        "--output",
+        help="Write test output to OUTPUT, in JUnit XML format",
+        action="store",
+    )
+
+    cli_args, remaining = parser.parse_known_args()
+    sys.argv = [sys.argv[0], *remaining]
+
+    FF_VERSION = cli_args.firefox_version
+    EXTENSION_PATH = cli_args.extension_path
+
+    host_version_string = os.environ.get("HOST_TARGET_VERSION")
+    if host_version_string:
+        HOST_TARGET_VERSION = version.parse(host_version_string)
+    else:
+        HOST_TARGET_VERSION = None
+
+    print(f"Running tests for extension {EXTENSION_PATH} using Firefox {FF_VERSION}")
+    if not os.path.isfile(EXTENSION_PATH):
+        raise FileNotFoundError(EXTENSION_PATH)
+
+    runner = None
+    if cli_args.output:
+        runner = unittest.TextTestRunner(resultclass=_make_result(cli_args.output))
+
+    unittest.main(testRunner=runner, verbosity=3)
+
+
+class MarkdownResult(unittest.result.TestResult):
+    def __init__(self, output_file: str, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.output_file = output_file
+
+    def stopTestRun(self):
+        doc = snakemd.Document()
+
+        title = f"Test Results - FF version {FF_VERSION}"
+        if HOST_TARGET_VERSION:
+            title += f", native host version {HOST_TARGET_VERSION}"
+
+        doc.add_heading(title, level=2)
+        doc.add_heading("Summary", level=3)
+
+        headers = ["Tests", "Failures", "Errors", "Skipped"]
+        align = [
+            snakemd.Table.Align.RIGHT,
+            snakemd.Table.Align.RIGHT,
+            snakemd.Table.Align.RIGHT,
+            snakemd.Table.Align.RIGHT,
+        ]
+        rows = [
+            [
+                str(self.testsRun),
+                str(len(self.failures)),
+                str(len(self.errors)),
+                str(len(self.skipped)),
+            ]
+        ]
+
+        doc.add_table(headers, rows, align)
+
+        def _render_unsuccessful(data: list[tuple[unittest.TestCase, str]], title: str):
+            if data:
+                doc.add_heading(title, level=3)
+                for testresult, msg in data:
+                    doc.add_block(
+                        snakemd.Heading(snakemd.Inline(f"`{testresult.id()}`"), level=5)
+                    )
+                    doc.add_code(msg, lang="generic")
+
+        _render_unsuccessful(self.failures, "Failures")
+        _render_unsuccessful(self.errors, "Errors")
+
+        with open(self.output_file, "w") as f:
+            f.write(str(doc))
+            f.write("\n\n")
+
+
+def _make_result(output_file: str):
+    def make_result(*args, **kwargs):
+        return MarkdownResult(output_file, *args, **kwargs)
+
+    return make_result
+
+
 if __name__ == "__main__":
-    unittest.main()
+    main()