OSOE-365: Support for running tests without a browser

Lombiq.Tests.UI.Samples/Tests/SecurityScanningTests.cs

@@ -48,7 +48,10 @@ public SecurityScanningTests(ITestOutputHelper testOutputHelper)
     // will fail the scan, but don't worry! You'll get a nice report about the findings in the failure dump.
     [Fact]
     public Task BasicSecurityScanShouldPass() =>
-        ExecuteTestAfterSetupAsync(
+        // Note how we use a method that doesn't launch a browser. Security scanning happens fully in ZAP, and doesn't
+        // use the browser launched by the UI Testing Toolbox. However, you can use the browser to prepare the app for
+        // security scanning if necessary.
+        ExecuteTestAfterBrowserSetupWithoutBrowserAsync(
             context => context.RunAndAssertBaselineSecurityScanAsync(),
             // You should configure the assertion that checks the app logs to accept some common cases that only should
             // appear during security scanning. If you launch a full scan, this is automatically configured by the
@@ -74,7 +77,7 @@ public Task BasicSecurityScanShouldPass() =>
     //   are only present to illustrate the type of adjustments you may want for your own site.
     [Fact]
     public Task SecurityScanWithCustomConfigurationShouldPass() =>
-        ExecuteTestAfterSetupAsync(
+        ExecuteTestAfterBrowserSetupWithoutBrowserAsync(
             context => context.RunAndAssertBaselineSecurityScanAsync(
                 configuration => configuration
                     ////.UseAjaxSpider() // This is quite slow so just showing you here but not running it.
@@ -105,7 +108,7 @@ public Task SecurityScanWithCustomConfigurationShouldPass() =>
     // customize them if something you need is not surfaced as configuration.
     [Fact]
     public Task SecurityScanWithCustomAutomationFrameworkPlanShouldPass() =>
-        ExecuteTestAfterSetupAsync(
+        ExecuteTestAfterBrowserSetupWithoutBrowserAsync(
             context => context.RunAndAssertSecurityScanAsync(
                 "Tests/CustomZapAutomationFrameworkPlan.yml",
                 configuration => configuration

Lombiq.Tests.UI.Tests.UI/TestCases/TimeoutTestCases.cs

@@ -7,7 +7,7 @@ public static class TimeoutTestCases
 {
     public static Task TestRunTimeoutShouldThrowAsync(
         ExecuteTestAfterSetupAsync executeTestAfterSetupAsync,
-        Browser browser = default) =>
+        Browser browser = Browser.None) =>
         Should.ThrowAsync(
             async () => await executeTestAfterSetupAsync(
                 context => Task.Delay(TimeSpan.FromSeconds(1)),

Lombiq.Tests.UI/Extensions/IWebApplicationInstanceExtensions.cs

@@ -39,6 +39,15 @@ public static async Task UsingScopeAsync(
 
         try
         {
+            // If there's no Default shell settings then the shell host hasn't been initialized yet. This can happen if
+            // no request hit the app yet.
+            var defaultShellSettingExist = shellHost.TryGetSettings("Default", out var _);
+
+            if (!defaultShellSettingExist)
+            {
+                await shellHost.InitializeAsync();
+            }
+
             // Injecting a fake HttpContext is required for many things, but it needs to happen before UsingAsync()
             // below to avoid NullReferenceExceptions in
             // OrchardCore.Recipes.Services.RecipeEnvironmentFeatureProvider.PopulateEnvironmentAsync. Migrations

Lombiq.Tests.UI/Extensions/VerificationUITestContextExtensions.cs

@@ -20,7 +20,7 @@ public static async Task AssertLogsAsync(this UITestContext context)
         var configuration = context.Configuration;
         var testOutputHelper = configuration.TestOutputHelper;
 
-        await context.UpdateHistoricBrowserLogAsync();
+        if (context.IsBrowserRunning) await context.UpdateHistoricBrowserLogAsync();
 
         try
         {
@@ -34,16 +34,19 @@ public static async Task AssertLogsAsync(this UITestContext context)
             throw;
         }
 
-        try
-        {
-            configuration.AssertBrowserLog?.Invoke(context.HistoricBrowserLog);
-        }
-        catch (Exception)
+        if (context.IsBrowserRunning)
         {
-            testOutputHelper.WriteLine("Browser logs: " + Environment.NewLine);
-            testOutputHelper.WriteLine(context.HistoricBrowserLog.ToFormattedString());
-
-            throw;
+            try
+            {
+                configuration.AssertBrowserLog?.Invoke(context.HistoricBrowserLog);
+            }
+            catch (Exception)
+            {
+                testOutputHelper.WriteLine("Browser logs: " + Environment.NewLine);
+                testOutputHelper.WriteLine(context.HistoricBrowserLog.ToFormattedString());
+
+                throw;
+            }
         }
     }
 }

Lombiq.Tests.UI/OrchardCoreUITestBase.cs

@@ -148,6 +148,53 @@ protected virtual Task ExecuteMultiSizeTestAfterSetupAsync(
             browser,
             changeConfigurationAsync);
 
+    protected virtual Task ExecuteTestAfterBrowserSetupWithoutBrowserAsync(
+        Func<UITestContext, Task> testAsync,
+        Action<OrchardCoreUITestExecutorConfiguration> changeConfiguration = null) =>
+        ExecuteTestAfterBrowserSetupWithoutBrowserAsync(testAsync, default, changeConfiguration);
+
+    protected Task ExecuteTestAfterBrowserSetupWithoutBrowserAsync(
+        Func<UITestContext, Task> testAsync,
+        Func<OrchardCoreUITestExecutorConfiguration, Task> changeConfigurationAsync) =>
+        ExecuteTestAfterBrowserSetupWithoutBrowserAsync(testAsync, default, changeConfigurationAsync);
+
+    protected virtual Task ExecuteTestAfterBrowserSetupWithoutBrowserAsync(
+        Func<UITestContext, Task> testAsync,
+        Browser browser,
+        Action<OrchardCoreUITestExecutorConfiguration> changeConfiguration = null) =>
+        ExecuteTestAfterBrowserSetupWithoutBrowserAsync(testAsync, changeConfiguration.AsCompletedTask());
+
+    protected Task ExecuteTestAfterBrowserSetupWithoutBrowserAsync(
+        Func<UITestContext, Task> testAsync,
+        Browser browser,
+        Func<OrchardCoreUITestExecutorConfiguration, Task> changeConfigurationAsync) =>
+        ExecuteTestAfterSetupWithoutBrowserAsync(testAsync, async configuration =>
+        {
+            configuration.SetupConfiguration.BeforeSetup = configuration =>
+            {
+                configuration.BrowserConfiguration.Browser = browser;
+                return Task.CompletedTask;
+            };
+
+            configuration.SetupConfiguration.AfterSetup = configuration =>
+            {
+                configuration.BrowserConfiguration.Browser = Browser.None;
+                return Task.CompletedTask;
+            };
+
+            if (changeConfigurationAsync != null) await changeConfigurationAsync(configuration);
+        });
+
+    protected virtual Task ExecuteTestAfterSetupWithoutBrowserAsync(
+        Func<UITestContext, Task> testAsync,
+        Action<OrchardCoreUITestExecutorConfiguration> changeConfiguration = null) =>
+        ExecuteTestAfterSetupWithoutBrowserAsync(testAsync, changeConfiguration.AsCompletedTask());
+
+    protected Task ExecuteTestAfterSetupWithoutBrowserAsync(
+        Func<UITestContext, Task> testAsync,
+        Func<OrchardCoreUITestExecutorConfiguration, Task> changeConfigurationAsync) =>
+        ExecuteTestAfterSetupAsync(testAsync, Browser.None, changeConfigurationAsync);
+
     protected virtual Task ExecuteTestAfterSetupAsync(
         Action<UITestContext> test,
         Action<OrchardCoreUITestExecutorConfiguration> changeConfiguration = null) =>

Lombiq.Tests.UI/SecurityScanning/SecurityScanningUITestContextExtensions.cs

@@ -236,7 +236,7 @@ public static Task<SecurityScanResult> RunSecurityScanAsync(
         Action<SecurityScanConfiguration> configure = null)
     {
         var configuration = new SecurityScanConfiguration()
-            .StartAtUri(context.GetCurrentUri());
+            .StartAtUri(context.IsBrowserRunning ? context.GetCurrentUri() : context.Scope.BaseUri);
 
         // By default ignore /vendor/ or /vendors/ URLs. This is case-insensitive. We have no control over them, and
         // they may contain several false positives (e.g. in font-awesome).

Lombiq.Tests.UI/Services/AtataFactory.cs

@@ -33,7 +33,6 @@ public static async Task<AtataScope> StartAtataScopeAsync(
         var browserConfiguration = configuration.BrowserConfiguration;
 
         var builder = AtataContext.Configure()
-            .UseDriver(await CreateDriverAsync(browserConfiguration, timeoutConfiguration, testOutputHelper))
             .UseBaseUrl(baseUri.ToString())
             .UseCulture(browserConfiguration.AcceptLanguage.ToString())
             .UseTestName(configuration.AtataConfiguration.TestName)
@@ -42,6 +41,17 @@ public static async Task<AtataScope> StartAtataScopeAsync(
             .PageSnapshots.UseCdpOrPageSourceStrategy() // #spell-check-ignore-line
             .UseArtifactsPathTemplate(contextId); // Necessary to prevent long paths, an issue under Windows.
 
+        if (configuration.BrowserConfiguration.Browser != Browser.None)
+        {
+            builder
+                .UseDriverInitializationStage(AtataContextDriverInitializationStage.OnDemand)
+                .UseDriver(await CreateDriverFactoryAsync(browserConfiguration, timeoutConfiguration, testOutputHelper));
+        }
+        else
+        {
+            builder.UseDriverInitializationStage(AtataContextDriverInitializationStage.None);
+        }
+
         builder.LogConsumers.AddDebug();
         builder.LogConsumers.Add(new TestOutputLogConsumer(testOutputHelper));
 
@@ -55,15 +65,11 @@ public static void SetupShellCliCommandFactory() =>
             .UseCmdForWindows()
             .UseForOtherOS(new BashShellCliCommandFactory("-login"));
 
-    private static async Task<IWebDriver> CreateDriverAsync(
+    private static async Task<Func<IWebDriver>> CreateDriverFactoryAsync(
         BrowserConfiguration browserConfiguration,
         TimeoutConfiguration timeoutConfiguration,
         ITestOutputHelper testOutputHelper)
     {
-        Task<T> FromAsync<T>(Func<BrowserConfiguration, TimeSpan, Task<T>> factory)
-            where T : IWebDriver =>
-            factory(browserConfiguration, timeoutConfiguration.PageLoadTimeout);
-
         // Driver creation can fail with "Cannot start the driver service on http://localhost:56686/" exceptions if the
         // machine is under load. Retrying it here so not the whole test needs to be re-run.
         const int maxTryCount = 3;
@@ -79,12 +85,14 @@ Task<T> FromAsync<T>(Func<BrowserConfiguration, TimeSpan, Task<T>> factory)
         {
             try
             {
+                var pageLoadTimeout = timeoutConfiguration.PageLoadTimeout;
+
                 return browserConfiguration.Browser switch
                 {
-                    Browser.Chrome => await FromAsync(WebDriverFactory.CreateChromeDriverAsync),
-                    Browser.Edge => await FromAsync(WebDriverFactory.CreateEdgeDriverAsync),
-                    Browser.Firefox => await FromAsync(WebDriverFactory.CreateFirefoxDriverAsync),
-                    Browser.InternetExplorer => await FromAsync(WebDriverFactory.CreateInternetExplorerDriverAsync),
+                    Browser.Chrome => await WebDriverFactory.CreateChromeDriverAsync(browserConfiguration, pageLoadTimeout),
+                    Browser.Edge => await WebDriverFactory.CreateEdgeDriverAsync(browserConfiguration, pageLoadTimeout),
+                    Browser.Firefox => await WebDriverFactory.CreateFirefoxDriverAsync(browserConfiguration, pageLoadTimeout),
+                    Browser.InternetExplorer => await WebDriverFactory.CreateInternetExplorerDriverAsync(browserConfiguration, pageLoadTimeout),
                     _ => throw new InvalidOperationException($"Unknown browser: {browserConfiguration.Browser}."),
                 };
             }

Lombiq.Tests.UI/Services/AtataScope.cs

@@ -13,7 +13,18 @@ public sealed class AtataScope : IDisposable
     private Uri _baseUri;
 
     public AtataContext AtataContext { get; }
-    public IWebDriver Driver => AtataContext.Driver;
+
+    public IWebDriver Driver
+    {
+        get
+        {
+            var driver = AtataContext.Driver;
+            if (driver != null) IsBrowserRunning = true;
+            return driver;
+        }
+    }
+
+    public bool IsBrowserRunning { get; private set; }
 
     public Uri BaseUri
     {

Lombiq.Tests.UI/Services/OrchardCoreSetupConfiguration.cs

@@ -6,6 +6,7 @@
 namespace Lombiq.Tests.UI.Services;
 
 public delegate Task BeforeSetupHandler(OrchardCoreUITestExecutorConfiguration configuration);
+public delegate Task AfterSetupHandler(OrchardCoreUITestExecutorConfiguration configuration);
 
 /// <summary>
 /// Configuration for the initial setup of an Orchard Core app.
@@ -35,4 +36,5 @@ public class OrchardCoreSetupConfiguration
         Path.Combine(DirectoryPaths.Temp, DirectoryPaths.SetupSnapshot);
 
     public BeforeSetupHandler BeforeSetup { get; set; }
+    public AfterSetupHandler AfterSetup { get; set; }
 }

Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs

@@ -20,6 +20,12 @@ public enum Browser
     Edge,
     Firefox,
     InternetExplorer,
+
+    /// <summary>
+    /// No browser will be used. Useful for testing things that don't require a browser, like API endpoints or running
+    /// security scans.
+    /// </summary>
+    None,
 }
 
 public class OrchardCoreUITestExecutorConfiguration

Lombiq.Tests.UI/Services/UITestContext.cs

@@ -62,6 +62,22 @@ public class UITestContext
     /// </summary>
     public IWebDriver Driver => Scope.Driver;
 
+    /// <summary>
+    /// Gets a value indicating whether a browser is currently running for the test. <see langword="false"/> means that
+    /// no browser was launched. Note that since the browser is only started on demand, with the first operation
+    /// requiring it, a browser might not be currently running even if <see cref="IsBrowserConfigured"/> suggests it
+    /// should.
+    /// </summary>
+    public bool IsBrowserRunning => Scope.IsBrowserRunning;
+
+    /// <summary>
+    /// Gets a value indicating whether a browser is configured to be used for the test. <see langword="false"/> means
+    /// that no browser will be launched. Note that since the browser is only started on demand, with the first
+    /// operation requiring it, a browser might not be currently running even if this suggests it should. Check <see
+    /// cref="IsBrowserRunning"/>" to check for that.
+    /// </summary>
+    public bool IsBrowserConfigured => Configuration.BrowserConfiguration.Browser != Browser.None;
+
     /// <summary>
     /// Gets the context for the SMTP service running for the test, if it was requested.
     /// </summary>

Lombiq.Tests.UI/Services/UITestExecutionSession.cs

@@ -77,35 +77,12 @@ public async Task<bool> ExecuteAsync(int retryCount, string dumpRootPath)
 
             if (_hasSetupOperation)
             {
-                var snapshotSubdirectory = "SQLite";
-                if (_configuration.UseSqlServer)
-                {
-                    snapshotSubdirectory = _configuration.UseAzureBlobStorage
-                        ? "SqlServer-AzureBlob"
-                        : "SqlServer";
-                }
-                else if (_configuration.UseAzureBlobStorage)
-                {
-                    snapshotSubdirectory = "SQLite-AzureBlob";
-                }
-
-                snapshotSubdirectory += "-" + setupConfiguration.SetupOperation!.GetHashCode().ToTechnicalString();
-
-                _snapshotDirectoryPath = Path.Combine(setupConfiguration.SetupSnapshotDirectoryPath, snapshotSubdirectory);
-
-                _configuration.OrchardCoreConfiguration.SnapshotDirectoryPath = _snapshotDirectoryPath;
-
-                _currentSetupSnapshotManager = UITestExecutionSessionsMeta.SetupSnapshotManagers.GetOrAdd(
-                    _snapshotDirectoryPath,
-                    path => new SynchronizingWebApplicationSnapshotManager(path));
-
                 await SetupAsync();
             }
-
-            // In some cases, there is a temporary setup snapshot directory path but no setup operation. For example,
-            // when calling the "ExecuteTestAsync()" method without a setup operation.
             else if (_setupSnapshotDirectoryContainsApp)
             {
+                // In some cases, there is a temporary setup snapshot directory path but no setup operation. For
+                // example, when calling the "ExecuteTestAsync()" method without a setup operation.
                 _configuration.OrchardCoreConfiguration.SnapshotDirectoryPath = setupConfiguration.SetupSnapshotDirectoryPath;
             }
 
@@ -116,7 +93,7 @@ public async Task<bool> ExecuteAsync(int retryCount, string dumpRootPath)
             _context.FailureDumpContainer.Clear();
             failureDumpContainer = _context.FailureDumpContainer;
 
-            _context.SetDefaultBrowserSize();
+            if (_context.IsBrowserConfigured) _context.SetDefaultBrowserSize();
 
             await _testManifest.TestAsync(_context);
 
@@ -479,6 +456,28 @@ private async Task SetupAsync()
     {
         var setupConfiguration = _configuration.SetupConfiguration;
 
+        var snapshotSubdirectory = "SQLite";
+        if (_configuration.UseSqlServer)
+        {
+            snapshotSubdirectory = _configuration.UseAzureBlobStorage
+                ? "SqlServer-AzureBlob"
+                : "SqlServer";
+        }
+        else if (_configuration.UseAzureBlobStorage)
+        {
+            snapshotSubdirectory = "SQLite-AzureBlob";
+        }
+
+        snapshotSubdirectory += "-" + setupConfiguration.SetupOperation!.GetHashCode().ToTechnicalString();
+
+        _snapshotDirectoryPath = Path.Combine(setupConfiguration.SetupSnapshotDirectoryPath, snapshotSubdirectory);
+
+        _configuration.OrchardCoreConfiguration.SnapshotDirectoryPath = _snapshotDirectoryPath;
+
+        _currentSetupSnapshotManager = UITestExecutionSessionsMeta.SetupSnapshotManagers.GetOrAdd(
+            _snapshotDirectoryPath,
+            path => new SynchronizingWebApplicationSnapshotManager(path));
+
         try
         {
             _testOutputHelper.WriteLineTimestampedAndDebug("Starting waiting for the setup operation.");
@@ -505,11 +504,14 @@ private async Task SetupAsync()
                 SetupSqlServerSnapshot();
                 SetupAzureBlobStorageSnapshot();
 
-                _context.SetDefaultBrowserSize();
+                if (_context.IsBrowserConfigured) _context.SetDefaultBrowserSize();
 
                 var result = (_context, await setupConfiguration.SetupOperation(_context));
 
                 await _context.AssertLogsAsync();
+
+                await setupConfiguration.AfterSetup.InvokeAsync<AfterSetupHandler>(handler => handler(_configuration));
+
                 _testOutputHelper.WriteLineTimestampedAndDebug("Finished setup operation.");
 
                 return result;
@@ -526,7 +528,7 @@ private async Task SetupAsync()
 
             _context = await CreateContextAsync();
 
-            await _context.GoToRelativeUrlAsync(resultUri.PathAndQuery);
+            if (_context.IsBrowserConfigured) await _context.GoToRelativeUrlAsync(resultUri.PathAndQuery);
         }
         catch (Exception ex) when (ex is not SetupFailedFastException)
         {