GOV-29: Adding the ability to do UI testing on a remote environment

.github/workflows/publish-nuget.yml

@@ -7,6 +7,6 @@ on:
 
 jobs:
   publish-nuget:
-    uses: Lombiq/GitHub-Actions/.github/workflows/publish-nuget.yml@dev
+    uses: Lombiq/GitHub-Actions/.github/workflows/publish-nuget.yml@issue/GOV-29
     secrets:
       API_KEY: ${{ secrets.DEFAULT_NUGET_PUBLISH_API_KEY }}

Lombiq.Tests.UI.Samples/Readme.md

@@ -29,6 +29,7 @@ For general details about and on using the Toolbox see the [root Readme](../Read
 - [Testing in tenants](Tests/TenantTests.cs)
 - [Interactive mode](Tests/InteractiveModeTests.cs)
 - [Security scanning](Tests/SecurityScanningTests.cs)
+- [Testing remote apps](Tests/RemoteTests.cs)
 
 ## Adding new tutorials
 

Lombiq.Tests.UI.Samples/Tests/RemoteTests.cs

@@ -0,0 +1,45 @@
+using Lombiq.Tests.UI.Extensions;
+using OpenQA.Selenium;
+using Shouldly;
+using System;
+using System.Threading.Tasks;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Lombiq.Tests.UI.Samples.Tests;
+
+// We recommend always running UI tests on the latest code of your app as part of your CI workflow. So, if anything got
+// broken by a pull request, it should be readily visible. Such tests are self-contained and should not even need access
+// to the internet.
+
+// However, sometimes in addition to this, you also want to test remote apps available online, like running rudimentary
+// smoke tests on your production app (e.g.: Can people still log in? Are payments still working?). The UI Testing
+// Toolbox also supports this. Check out the example below!
+
+// Note how the test derives from RemoteUITestBase this time, not UITestBase.
+public class RemoteTests : RemoteUITestBase
+{
+    public RemoteTests(ITestOutputHelper testOutputHelper)
+        : base(testOutputHelper)
+    {
+    }
+
+    // The test itself is largely the same as all the local ones, but you need to provide a base URI.
+    [Fact]
+    public Task ExampleDotComShouldWork() =>
+        ExecuteTestAsync(
+            new Uri("https://example.com/"),
+            context =>
+            {
+                // Assertions work as usual. Implicit assertions like HTML validation and accessibility checks work too,
+                // and upon a failing assertion a failure dump is generated as you'd expect it.
+                context.Get(By.CssSelector("h1")).Text.ShouldBe("Example Domain");
+                context.Exists(By.LinkText("More information..."));
+
+                // Note that due to a remote app not being under our control, some things are not supported. E.g., you
+                // can't access the Orchard Core logs, or use shortcuts (the *Directly methods).
+            },
+            configuration => configuration.HtmlValidationConfiguration.RunHtmlValidationAssertionOnAllPageChanges = false);
+}
+
+// END OF TRAINING SECTION: Remote tests.

Lombiq.Tests.UI.Samples/Tests/SecurityScanningTests.cs

@@ -164,3 +164,4 @@ protected override Task ExecuteTestAfterSetupAsync(
 }
 
 // END OF TRAINING SECTION: Security scanning.
+// NEXT STATION: Head over to Tests/RemoteTests.cs.

Lombiq.Tests.UI/OrchardCoreUITestBase.cs

@@ -4,7 +4,6 @@
 using Lombiq.Tests.UI.Helpers;
 using Lombiq.Tests.UI.Models;
 using Lombiq.Tests.UI.Services;
-using Lombiq.Tests.UI.Services.GitHub;
 using SixLabors.ImageSharp;
 using System;
 using System.Threading.Tasks;
@@ -31,19 +30,18 @@ public delegate Task ExecuteTestAfterSetupAsync(
         Browser browser,
         Func<OrchardCoreUITestExecutorConfiguration, Task> changeConfigurationAsync);
 
-public abstract class OrchardCoreUITestBase<TEntryPoint>
+public abstract class OrchardCoreUITestBase<TEntryPoint> : UITestBase
      where TEntryPoint : class
 {
     private const string AppFolder = nameof(AppFolder);
 
-    protected ITestOutputHelper _testOutputHelper;
-
     protected virtual Size StandardBrowserSize => CommonDisplayResolutions.Standard;
     protected virtual Size MobileBrowserSize => CommonDisplayResolutions.NhdPortrait; // #spell-check-ignore-line
 
-    static OrchardCoreUITestBase() => AtataFactory.SetupShellCliCommandFactory();
-
-    protected OrchardCoreUITestBase(ITestOutputHelper testOutputHelper) => _testOutputHelper = testOutputHelper;
+    protected OrchardCoreUITestBase(ITestOutputHelper testOutputHelper)
+        : base(testOutputHelper)
+    {
+    }
 
     // If you change this, then also change the corresponding delegate above.
     protected abstract Task ExecuteTestAfterSetupAsync(
@@ -335,28 +333,10 @@ protected virtual async Task ExecuteTestAsync(
 
         if (changeConfigurationAsync != null) await changeConfigurationAsync(configuration);
 
-        var originalTestOutputHelper = _testOutputHelper;
-        Action afterTest = null;
-        if (configuration.ExtendGitHubActionsOutput &&
-            configuration.GitHubActionsOutputConfiguration.EnablePerTestOutputGrouping &&
-            GitHubHelper.IsGitHubEnvironment)
-        {
-            (_testOutputHelper, afterTest) =
-                GitHubActionsGroupingTestOutputHelper.CreateDecorator(_testOutputHelper, testManifest);
-            configuration.TestOutputHelper = _testOutputHelper;
-        }
-
-        try
-        {
-            await UITestExecutor.ExecuteOrchardCoreTestAsync<TEntryPoint>(testManifest, configuration);
-        }
-        finally
-        {
-            _testOutputHelper = originalTestOutputHelper;
-            // This warning is a false positive as it is not considering the evaluation of the if statement above.
-#pragma warning disable S2583 // Conditionally executed code should be reachable
-            afterTest?.Invoke();
-#pragma warning restore S2583 // Conditionally executed code should be reachable
-        }
+        await ExecuteOrchardCoreTestAsync(
+            (configuration, contextId) =>
+                new OrchardCoreInstance<TEntryPoint>(configuration.OrchardCoreConfiguration, contextId, configuration.TestOutputHelper),
+            testManifest,
+            configuration);
     }
 }

Lombiq.Tests.UI/RemoteUITestBase.cs

@@ -0,0 +1,76 @@
+using Lombiq.Tests.UI.Extensions;
+using Lombiq.Tests.UI.Models;
+using Lombiq.Tests.UI.Services;
+using System;
+using System.Threading.Tasks;
+using Xunit.Abstractions;
+
+namespace Lombiq.Tests.UI;
+
+public abstract class RemoteUITestBase : UITestBase
+{
+    protected RemoteUITestBase(ITestOutputHelper testOutputHelper)
+    : base(testOutputHelper)
+    {
+    }
+
+    /// <summary>
+    /// Executes the given UI test on a remote (i.e. not locally running) app.
+    /// </summary>
+    protected virtual Task ExecuteTestAsync(
+        Uri baseUri,
+        Action<UITestContext> testAsync,
+        Action<OrchardCoreUITestExecutorConfiguration> changeConfiguration = null) =>
+        ExecuteTestAsync(baseUri, testAsync.AsCompletedTask(), changeConfiguration.AsCompletedTask());
+
+    /// <summary>
+    /// Executes the given UI test on a remote (i.e. not locally running) app.
+    /// </summary>
+    protected virtual Task ExecuteTestAsync(
+        Uri baseUri,
+        Func<UITestContext, Task> testAsync,
+        Action<OrchardCoreUITestExecutorConfiguration> changeConfiguration = null) =>
+        ExecuteTestAsync(baseUri, testAsync, changeConfiguration.AsCompletedTask());
+
+    /// <summary>
+    /// Executes the given UI test on a remote (i.e. not locally running) app.
+    /// </summary>
+    protected virtual Task ExecuteTestAsync(
+        Uri baseUri,
+        Func<UITestContext, Task> testAsync,
+        Func<OrchardCoreUITestExecutorConfiguration, Task> changeConfigurationAsync) =>
+        ExecuteTestAsync(baseUri, testAsync, default, changeConfigurationAsync);
+
+    /// <summary>
+    /// Executes the given UI test on a remote (i.e. not locally running) app.
+    /// </summary>
+    protected virtual async Task ExecuteTestAsync(
+        Uri baseUri,
+        Func<UITestContext, Task> testAsync,
+        Browser browser,
+        Func<OrchardCoreUITestExecutorConfiguration, Task> changeConfigurationAsync)
+    {
+        async Task BaseUriVisitingTest(UITestContext context)
+        {
+            await context.GoToAbsoluteUrlAsync(baseUri);
+            await testAsync(context);
+        }
+
+        var testManifest = new UITestManifest(_testOutputHelper) { TestAsync = BaseUriVisitingTest };
+
+        var configuration = new OrchardCoreUITestExecutorConfiguration
+        {
+            OrchardCoreConfiguration = new OrchardCoreConfiguration(),
+            TestOutputHelper = _testOutputHelper,
+            BrowserConfiguration = { Browser = browser },
+        };
+
+        configuration.HtmlValidationConfiguration.HtmlValidationAndAssertionOnPageChangeRule = (_) => true;
+        configuration.AccessibilityCheckingConfiguration.AccessibilityCheckingAndAssertionOnPageChangeRule = (_) => true;
+        configuration.FailureDumpConfiguration.CaptureAppSnapshot = false;
+
+        if (changeConfigurationAsync != null) await changeConfigurationAsync(configuration);
+
+        await ExecuteOrchardCoreTestAsync((_, _) => new RemoteInstance(baseUri), testManifest, configuration);
+    }
+}

Lombiq.Tests.UI/Services/RemoteInstance.cs

@@ -0,0 +1,26 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Lombiq.Tests.UI.Services;
+
+public sealed class RemoteInstance : IWebApplicationInstance
+{
+    public IServiceProvider Services => throw new NotSupportedException();
+
+    private readonly Uri _baseUri;
+
+    public RemoteInstance(Uri baseUri) => _baseUri = baseUri;
+
+    public Task<Uri> StartUpAsync() => Task.FromResult(_baseUri);
+
+    public IEnumerable<IApplicationLog> GetLogs(CancellationToken cancellationToken = default) => Enumerable.Empty<IApplicationLog>();
+    public TService GetRequiredService<TService>() => throw new NotSupportedException();
+    public Task PauseAsync() => throw new NotSupportedException();
+    public Task ResumeAsync() => throw new NotSupportedException();
+    public Task TakeSnapshotAsync(string snapshotDirectoryPath) => throw new NotSupportedException();
+
+    public ValueTask DisposeAsync() => ValueTask.CompletedTask;
+}

Lombiq.Tests.UI/Services/UITestExecutionSession.cs

@@ -24,9 +24,9 @@
 
 namespace Lombiq.Tests.UI.Services;
 
-internal sealed class UITestExecutionSession<TEntryPoint> : IAsyncDisposable
-    where TEntryPoint : class
+internal sealed class UITestExecutionSession : IAsyncDisposable
 {
+    private readonly WebApplicationInstanceFactory _webApplicationInstanceFactory;
     private readonly UITestManifest _testManifest;
     private readonly OrchardCoreUITestExecutorConfiguration _configuration;
     private readonly UITestExecutorFailureDumpConfiguration _dumpConfiguration;
@@ -45,8 +45,12 @@ internal sealed class UITestExecutionSession<TEntryPoint> : IAsyncDisposable
     private UITestContext _context;
     private DockerConfiguration _dockerConfiguration;
 
-    public UITestExecutionSession(UITestManifest testManifest, OrchardCoreUITestExecutorConfiguration configuration)
+    public UITestExecutionSession(
+        WebApplicationInstanceFactory webApplicationInstanceFactory,
+        UITestManifest testManifest,
+        OrchardCoreUITestExecutorConfiguration configuration)
     {
+        _webApplicationInstanceFactory = webApplicationInstanceFactory;
         _testManifest = testManifest;
         _configuration = configuration;
         _dumpConfiguration = configuration.FailureDumpConfiguration;
@@ -98,7 +102,7 @@ public async Task<bool> ExecuteAsync(int retryCount, string dumpRootPath)
             }
 
             // In some cases, there is a temporary setup snapshot directory path but no setup operation. For example,
-            // when calling the "ExecuteTestAsync()" method without setup operation.
+            // when calling the "ExecuteTestAsync()" method without a setup operation.
             else if (_setupSnapshotDirectoryContainsApp)
             {
                 _configuration.OrchardCoreConfiguration.SnapshotDirectoryPath = setupConfiguration.SetupSnapshotDirectoryPath;
@@ -626,10 +630,7 @@ Task UITestingBeforeAppStartHandlerAsync(string contentRootPath, InstanceCommand
             _configuration.OrchardCoreConfiguration.BeforeAppStart.RemoveAll(UITestingBeforeAppStartHandlerAsync);
         _configuration.OrchardCoreConfiguration.BeforeAppStart += UITestingBeforeAppStartHandlerAsync;
 
-        _applicationInstance = new OrchardCoreInstance<TEntryPoint>(
-            _configuration.OrchardCoreConfiguration,
-            contextId,
-            _testOutputHelper);
+        _applicationInstance = _webApplicationInstanceFactory(_configuration, contextId);
         var uri = await _applicationInstance.StartUpAsync();
 
         _configuration.SetUpEvents();

Lombiq.Tests.UI/Services/UITestExecutor.cs

@@ -10,6 +10,10 @@
 
 namespace Lombiq.Tests.UI.Services;
 
+public delegate IWebApplicationInstance WebApplicationInstanceFactory(
+    OrchardCoreUITestExecutorConfiguration configuration,
+    string contextId);
+
 public static class UITestExecutor
 {
     private static readonly object _numberOfTestsLimitLock = new();
@@ -18,10 +22,10 @@ public static class UITestExecutor
     /// <summary>
     /// Executes a test on a new Orchard Core web app instance within a newly created Atata scope.
     /// </summary>
-    public static Task ExecuteOrchardCoreTestAsync<TEntryPoint>(
+    public static Task ExecuteOrchardCoreTestAsync(
+        WebApplicationInstanceFactory webApplicationInstanceFactory,
         UITestManifest testManifest,
         OrchardCoreUITestExecutorConfiguration configuration)
-        where TEntryPoint : class
     {
         if (string.IsNullOrEmpty(testManifest.Name))
         {
@@ -59,14 +63,14 @@ public static Task ExecuteOrchardCoreTestAsync<TEntryPoint>(
             }
         }
 
-        return ExecuteOrchardCoreTestInnerAsync<TEntryPoint>(testManifest, configuration, dumpRootPath);
+        return ExecuteOrchardCoreTestInnerAsync(webApplicationInstanceFactory, testManifest, configuration, dumpRootPath);
     }
 
-    private static async Task ExecuteOrchardCoreTestInnerAsync<TEntryPoint>(
+    private static async Task ExecuteOrchardCoreTestInnerAsync(
+        WebApplicationInstanceFactory webApplicationInstanceFactory,
         UITestManifest testManifest,
         OrchardCoreUITestExecutorConfiguration configuration,
         string dumpRootPath)
-        where TEntryPoint : class
     {
         var retryCount = 0;
         var passed = false;
@@ -79,7 +83,7 @@ private static async Task ExecuteOrchardCoreTestInnerAsync<TEntryPoint>(
                     await _numberOfTestsLimit.WaitAsync();
                 }
 
-                await using var instance = new UITestExecutionSession<TEntryPoint>(testManifest, configuration);
+                await using var instance = new UITestExecutionSession(webApplicationInstanceFactory, testManifest, configuration);
                 passed = await instance.ExecuteAsync(retryCount, dumpRootPath);
             }
             catch (Exception ex) when (retryCount < configuration.MaxRetryCount)

Lombiq.Tests.UI/UITestBase.cs

@@ -0,0 +1,47 @@
+using Lombiq.Tests.UI.Models;
+using Lombiq.Tests.UI.Services;
+using Lombiq.Tests.UI.Services.GitHub;
+using System;
+using System.Threading.Tasks;
+using Xunit.Abstractions;
+
+namespace Lombiq.Tests.UI;
+
+public abstract class UITestBase
+{
+    protected ITestOutputHelper _testOutputHelper;
+
+    static UITestBase() => AtataFactory.SetupShellCliCommandFactory();
+
+    protected UITestBase(ITestOutputHelper testOutputHelper) => _testOutputHelper = testOutputHelper;
+
+    protected async Task ExecuteOrchardCoreTestAsync(
+        WebApplicationInstanceFactory webApplicationInstanceFactory,
+        UITestManifest testManifest,
+        OrchardCoreUITestExecutorConfiguration configuration)
+    {
+        var originalTestOutputHelper = _testOutputHelper;
+        Action afterTest = null;
+        if (configuration.ExtendGitHubActionsOutput &&
+            configuration.GitHubActionsOutputConfiguration.EnablePerTestOutputGrouping &&
+            GitHubHelper.IsGitHubEnvironment)
+        {
+            (_testOutputHelper, afterTest) =
+                GitHubActionsGroupingTestOutputHelper.CreateDecorator(_testOutputHelper, testManifest);
+            configuration.TestOutputHelper = _testOutputHelper;
+        }
+
+        try
+        {
+            await UITestExecutor.ExecuteOrchardCoreTestAsync(webApplicationInstanceFactory, testManifest, configuration);
+        }
+        finally
+        {
+            _testOutputHelper = originalTestOutputHelper;
+            // This warning is a false positive as it is not considering the evaluation of the if statement above.
+#pragma warning disable S2583 // Conditionally executed code should be reachable
+            afterTest?.Invoke();
+#pragma warning restore S2583 // Conditionally executed code should be reachable
+        }
+    }
+}

Readme.md

@@ -9,7 +9,7 @@ Web UI testing toolbox mostly for Orchard Core applications. Everything you need
 Highlights:
 
 - Builds on proven libraries like Selenium, Atata, and xUnit. See all the tools we use [here](Lombiq.Tests.UI/Docs/Tools.md).
-- Execute fully self-contained, repeatable, parallelizable automated UI tests on Orchard Core apps.
+- Execute fully self-contained, repeatable, parallelizable automated UI tests on Orchard Core apps, running locally or remotely.
 - Do cross-browser testing with all current browsers, both in normal and headless modes.
 - Check the HTML structure and behavior of the app, check for errors in the Orchard logs and browser logs. Start troubleshooting from the detailed full application dumps and test logs if a test fails.
 - Start tests with a setup using recipes, start with an existing Orchard Core app or take snapshots in between tests and resume from there. Use SQLite or SQL Server database snapshots (but you can also use PostgreSQL or MySQL too without snapshots, see [this proof of concept](https://github.com/OrchardCMS/OrchardCore/pull/11194/files)).