Merge pull request #237 from Lombiq/issue/OSOE-471

OSOE-471: In GitHub Actions workflows, surface log message if a UI test is retried
Lombiq · Nov 27, 2022 · 1d95a4f · 1d95a4f
2 parents 900828a + 2027bb1
commit 1d95a4f
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 5 deletions.
diff --git a/Lombiq.Tests.UI/Services/GitHub/GitHubActionsOutputConfiguration.cs b/Lombiq.Tests.UI/Services/GitHub/GitHubActionsOutputConfiguration.cs
@@ -15,4 +15,11 @@ public class GitHubActionsOutputConfiguration
     /// in the GitHub Actions output. This only takes effect when tests are executed from a GitHub Actions workflow.
     /// </summary>
     public bool EnableErrorAnnotations { get; set; } = true;
+
+    /// <summary>
+    /// Gets or sets a value indicating whether test retries should be surfaced as warning annotations (<see
+    /// href="https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-a-warning-message"/>)
+    /// in the GitHub Actions output. This only takes effect when tests are executed from a GitHub Actions workflow.
+    /// </summary>
+    public bool EnableTestRetryWarningAnnotations { get; set; } = true;
 }
diff --git a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs
@@ -6,6 +6,7 @@
 using Lombiq.Tests.UI.Extensions;
 using Lombiq.Tests.UI.Helpers;
 using Lombiq.Tests.UI.Models;
+using Lombiq.Tests.UI.Services.GitHub;
 using Microsoft.VisualBasic.FileIO;
 using Mono.Unix;
 using Newtonsoft.Json.Linq;
@@ -135,11 +136,7 @@ public async Task<bool> ExecuteAsync(int retryCount, string dumpRootPath)
                 throw;
             }
 
-            _testOutputHelper.WriteLineTimestampedAndDebug(
-                "The test was attempted {0} time(s). {1} more attempt(s) will be made after waiting {2}.",
-                retryCount + 1,
-                _configuration.MaxRetryCount - retryCount,
-                _configuration.RetryInterval);
+            LogRetry(retryCount);
 
             await Task.Delay(_configuration.RetryInterval);
         }
@@ -428,6 +425,27 @@ private void CaptureMarkupValidationResults(Exception ex, string debugInformatio
         }
     }
 
+    private void LogRetry(int retryCount)
+    {
+        _testOutputHelper.WriteLineTimestampedAndDebug(
+            "The test was attempted {0} time(s). {1} more attempt(s) will be made after waiting {2}.",
+            retryCount + 1,
+            _configuration.MaxRetryCount - retryCount,
+            _configuration.RetryInterval);
+
+        if (_configuration.ExtendGitHubActionsOutput &&
+            _configuration.GitHubActionsOutputConfiguration.EnableTestRetryWarningAnnotations &&
+            GitHubHelper.IsGitHubEnvironment)
+        {
+            new GitHubAnnotationWriter(_testOutputHelper).Annotate(
+                Microsoft.Extensions.Logging.LogLevel.Warning,
+                "UI test may be flaky",
+                $"The {_testManifest.Name} test failed {(retryCount + 1).ToTechnicalString()} time(s) and will be " +
+                    "retried. This may indicate it being flaky.",
+                string.Empty);
+        }
+    }
+
     private async Task SetupAsync()
     {
         var setupConfiguration = _configuration.SetupConfiguration;