Merge pull request #421 from Lombiq/issue/OSOE-904

OSOE-904: Create an example of using FrontendServer and context.ExecuteJavascriptTestAsync

Lombiq.Tests.UI.Samples/FrontendUITestBase.cs

@@ -0,0 +1,100 @@
+using Lombiq.Tests.UI.Extensions;
+using Lombiq.Tests.UI.Services;
+using System;
+using System.Diagnostics.CodeAnalysis;
+using System.Threading.Tasks;
+using Xunit.Abstractions;
+
+namespace Lombiq.Tests.UI.Samples;
+
+// Sometimes Orchard Core is used in a headless manner, as a web API server that the visitors reach through a web
+// frontend (for example a Vue or React single page application). In this scenario you can test the API directly and
+// test the frontend with a dummy backend separately, but that will only get you so far. You'll want some tests that
+// ensure the two work together. To make this happen, you need some custom logic to initialize the frontend process
+// after setup, with a unique port number to avoid clashes. Also the frontend may need the backend URL, whose port is
+// already randomized with every test.
+// In this base class we define a custom "setup and test" method. Creating such a method is a good practice so you don't
+// have to configure the FrontendServer over and over again in every test. We placed this method into its own
+// intermediate abstract class for better organization, but you can also put it into your UITestBase as well.
+public abstract class FrontendUITestBase : UITestBase
+{
+    protected FrontendUITestBase(ITestOutputHelper testOutputHelper)
+        : base(testOutputHelper)
+    {
+    }
+
+    /// <summary>
+    /// Executes a UI test where the frontend is served by a separate process.
+    /// </summary>
+    [SuppressMessage("Style", "IDE0055:Fix formatting", Justification = "Needed for more readable comments.")]
+    [SuppressMessage("StyleCop.CSharp.ReadabilityRules", "SA1114:Parameter list should follow declaration", Justification = "Same.")]
+    protected Task ExecuteFrontendTestAfterSetupAsync(
+        Func<UITestContext, Task> testAsync,
+        Browser browser,
+        Func<OrchardCoreUITestExecutorConfiguration, Task> changeConfigurationAsync = null) =>
+        ExecuteTestAfterSetupAsync(
+            async context =>
+            {
+                // Before executing provided test, we switch to the frontend URL, as if we switched to a different
+                // tenant, then actually navigate to the frontend so the tests won't start in the backend home page
+                // that's probably not used anyway.
+                context.SwitchToFrontend();
+                await context.GoToHomePageAsync();
+
+                await testAsync(context);
+            },
+            browser,
+            configuration =>
+            {
+                // The FrontendServer instance manages the test configuration by registering event handlers for the
+                // application startup and stop. It also initializes the default frontend and backend URLs. Below the
+                // "name" is the label you will find in the test application logs in front of all lines that come from
+                // this process. The process's arguments can be set with the arguments array, but likely you'll want to
+                // set pass in the frontend and backend ports too which you can do in the "configureCommand" function.
+                new FrontendServer(name: "http-server (test frontend)", configuration, _testOutputHelper)
+                    .Configure(
+                        // Here we use NPX which executes an NPM package without locally installing it, to avoid having
+                        // to maintain a separate binary or script just for this sample.
+                        program: "npx",
+                        arguments: null,
+                        // This function lets you edit the command dynamically before the process is created. This is
+                        // needed to add the frontend and backend URLs or port numbers to the process, e.g. as arguments
+                        // or environment variables. Since we set the arguments here, the parameter above is left null.
+                        // If necessary, this is also where you'd set the program's working directory.
+                        configureCommand: (frontendCommand, context) =>
+                        {
+                            // You can also get this from the configuration using configuration
+                            // .GetFrontendAndBackendUris().FrontendUri.Port. The values in the configuration have been
+                            // initialized shortly before this function is called, but it's not worth it unless you need
+                            // both frontend and backend URLS. The frontend port is a unique, reserved number that's
+                            // guaranteed to be available during this test just as much as any Orchard Core instance,
+                            // because it's coming from the same pool of numbers.
+                            var port = context.FrontendPort.ToTechnicalString();
+
+                            // Here we configure NPX to automatically download http-server without prompting (--yes) and
+                            // use the provided port number. Since this server uses HTTP instead of HTTPS, you have to
+                            // set the frontend URL too. The backend URL is not changed, so pass null to leave it as-is.
+                            configuration.SetFrontendAndBackendUris(
+                                frontendUrl: $"http://localhost:" + port,
+                                backendUrl: null);
+                            return frontendCommand.WithArguments(["--yes", "http-server", "--port", port]);
+                        },
+                        // When this function is not null, test setup will call it on each output line and wait for it
+                        // to return true. This can be used to look for an output that only appears when the frontend
+                        // server is done with its own startup.
+                        checkProgramReady: (line, _) => line?.Contains("Hit CTRL-C to stop the server") == true,
+                        // You can use this async function to perform additional tasks right before the FrontendServer's
+                        // BeforeAppStart event handler is finished. You can also edit the Orchard Core instance's
+                        // startup command here. We don't need it right now.
+                        thenAsync: _ => Task.CompletedTask,
+                        // Don't start up the server during setup and snapshot restore. You'd rarely want that, so we
+                        // have prepared a static method to generate a callback for this parameter.
+                        skipStartup: FrontendServer.SkipDuringSetupAndRestore(configuration),
+                        // Since we call NPX, the process may start with downloading from the network. So we allow some
+                        // grace period here, but it's unlikely that it would take too long unless the program is stuck
+                        // or frozen somehow. In that case it's better to fail than wait forever.
+                        startupTimeout: TimeSpan.FromMinutes(3));
+
+                return changeConfigurationAsync.InvokeFuncAsync(configuration);
+            });
+}

Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj

@@ -9,6 +9,7 @@
     <None Remove=".htmlvalidate.json" />
     <None Remove="Tests\CustomZapAutomationFrameworkPlan.yml" />
     <None Remove="xunit.runner.json" />
+    <None Update="Tests\*.mjs" CopyToOutputDirectory="PreserveNewest" />
   </ItemGroup>
 
   <ItemGroup>

Lombiq.Tests.UI.Samples/Readme.md

@@ -31,6 +31,8 @@ For general details about and on using the Toolbox see the [root Readme](../Read
 - [Security scanning](Tests/SecurityScanningTests.cs)
 - [Testing remote apps](Tests/RemoteTests.cs)
 - [Testing time-dependent functionality](Tests/ShiftTimeTests.cs)
+- [Test headless Orchard Core with a frontend subprocess](FrontendUITestBase.cs)
+- [Executing tests written in JavaScript](Tests/JavaScriptTests.cs)
 
 ## Adding new tutorials
 

Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs

@@ -0,0 +1,45 @@
+using Lombiq.Tests.UI.Extensions;
+using OpenQA.Selenium;
+using System.Threading.Tasks;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Lombiq.Tests.UI.Samples.Tests;
+
+public class FrontendTests : FrontendUITestBase
+{
+    public FrontendTests(ITestOutputHelper testOutputHelper)
+        : base(testOutputHelper)
+    {
+    }
+
+    // The interesting details are in FrontendUITestBase, here we just show that you can freely interact with the pages
+    // served by frontend server the same way as usual. In this case we have an HTTP file server, so you can navigate
+    // these directories and files. If we had a large client application that interacts with the headless OC in the
+    // backend, we would be able to do something more interesting but that's outside the scope of this demo.
+    [Fact]
+    public Task FrontendServerShouldStartWithTest() =>
+        ExecuteFrontendTestAfterSetupAsync(
+            async context =>
+            {
+                // Don't forget that if you want to interact with the frontend manually, you can use the interactive
+                // mode extension method.
+                //// await context.SwitchToInteractiveAsync();
+
+                await context.ClickReliablyOnAsync(By.LinkText("App_Data/"));
+                await context.ClickReliablyOnAsync(By.LinkText("Sites/"));
+                await context.ClickReliablyOnAsync(By.LinkText("Default/"));
+                await context.ClickReliablyOnAsync(By.LinkText("DataProtection-Keys/"));
+                await context.ClickReliablyOnAsync(By.XPath("//td/a[contains(@href, '.xml')]"));
+            },
+            browser: default,
+            configuration =>
+            {
+                // Since this server is not our code, we should disable HTML validation.
+                configuration.HtmlValidationConfiguration.RunHtmlValidationAssertionOnAllPageChanges = false;
+                return Task.CompletedTask;
+            });
+}
+
+// END OF TRAINING SECTION: Test headless Orchard Core with a frontend subprocess.
+// NEXT STATION: Head over to Tests/JavaScriptTests.cs.

Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.cs

@@ -0,0 +1,50 @@
+using Lombiq.Tests.UI.Extensions;
+using System.IO;
+using System.Threading.Tasks;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Lombiq.Tests.UI.Samples.Tests;
+
+// Let's suppose you want to write UI tests in JavaScript. Why would you want to do that? Unlikely if you are an Orchard
+// Core developer, but what if the person responsible for writing the tests is not? In the previous training section we
+// discussed using a separate frontend server, with mention of technologies using Node.js. In that case the frontend
+// developers may be more familiar with JavaScript so it makes sense to write and debug the tests in Node.js so they
+// don't have to learn different tools and tech stacks just to create some UI tests.
+public class JavaScriptTests : UITestBase
+{
+    public JavaScriptTests(ITestOutputHelper testOutputHelper)
+        : base(testOutputHelper)
+    {
+    }
+
+    // Using this approach you only have to write minimal C# boilerplate, which you can see below.
+    [Fact]
+    public Task ExampleJavaScriptTestShouldWork() =>
+        ExecuteTestAfterSetupAsync(context =>
+        {
+            // Don't forget to mark the script files as "Copy if newer", so they are available to the test. If you
+            // include something like the following in your csproj file, then you only have to do this once:
+            // <None Update="Tests\*.mjs" CopyToOutputDirectory="PreserveNewest" />
+            var scriptPath = Path.Join("Tests", "JavaScriptTests.mjs");
+
+            // Set up the JS dependencies in the test's temp directory to ensure there are no clashes, then run the
+            // script. This method has an additional parameter to list further NPM dependencies beyond
+            // "selenium-webdriver", if the script requires it. We will check out this script file in the next station.
+            return context.SetupSeleniumAndExecuteJavaScriptTestAsync(_testOutputHelper, scriptPath);
+        });
+
+    // To best debug the JavaScript code, you may want to set up the site and then invoke node manually. This is not a
+    // real test, but it sets up the site in interactive mode (see Tests/InteractiveModeTests.cs for more) with
+    // information on how to start up test scripts from your GUI. It's an example of some tooling that can improve the
+    // test developer's workflow.
+    [Fact]
+    public Task Sandbox() =>
+        OpenSandboxAfterSetupAsync(async context =>
+        {
+            await context.SetupNodeSeleniumAsync(_testOutputHelper);
+            await context.SwitchToInteractiveWithJavaScriptTestInfoAsync(Path.Join("Tests", "JavaScriptTests.mjs"));
+        });
+}
+
+// NEXT STATION: Head over to Tests/JavaScriptTests.mjs.

Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.mjs

@@ -0,0 +1,26 @@
+import { By, until } from 'selenium-webdriver';
+
+// This dependency is copied into the build directory by Lombiq.Tests.UI.
+import { runTest, shouldContainText, navigate } from '../ui-testing-toolkit.mjs';
+
+// This function automatically handles the command line arguments and sets up a Chrome driver.
+await runTest(async (driver, startUrl) => {
+    // Inside you can use all normal Selenium JavaScript code, e.g.:
+    // - https://www.selenium.dev/selenium/docs/api/javascript/WebDriver.html
+    // - https://www.selenium.dev/selenium/docs/api/javascript/By.html
+    await driver.findElement(By.xpath("//a[@href = '/blog/post-1']")).click();
+
+    // We also included a shortcut function to safely check text content.
+    await shouldContainText(
+        await driver.findElement(By.tagName("h1")),
+        "Man must explore, and this is exploration at its greatest");
+    await shouldContainText(
+        await driver.findElement(By.className("field-name-blog-post-subtitle")),
+        "Problems look mighty small from 150 miles up");
+
+    // And another one to navigate and safely wait for the page to load.
+    await navigate(driver, startUrl);
+    await driver.findElement(By.xpath("id('footer')//a[@href='https://lombiq.com/']"));
+});
+
+// END OF TRAINING SECTION: Executing tests written in JavaScript.

Lombiq.Tests.UI.Samples/Tests/ShiftTimeTests.cs

@@ -61,3 +61,4 @@ private static async Task<DateTime> GetNowAsync(UITestContext context)
 }
 
 // END OF TRAINING SECTION: Testing time-dependent functionality.
+// NEXT STATION: Head over to FrontendUITestBase.cs.

Lombiq.Tests.UI.Shortcuts/Views/InteractiveMode/Index.cshtml

@@ -27,6 +27,11 @@
         width: 30vw;
     }
 
+    #messages {
+        z-index: 999999999;
+        position: fixed;
+    }
+
     .message {
         width: 100%;
     }

Lombiq.Tests.UI/Constants/DirectoryPaths.cs

@@ -1,3 +1,4 @@
+using Lombiq.Tests.UI.Services;
 using System;
 using System.IO;
 
@@ -10,14 +11,13 @@ public static class DirectoryPaths
     public const string Screenshots = nameof(Screenshots);
 
     public static string GetTempDirectoryPath(params string[] subDirectoryNames) =>
-        Path.Combine(
-            Path.Combine(Environment.CurrentDirectory, Temp),
-            Path.Combine(subDirectoryNames));
+        Path.Combine([Environment.CurrentDirectory, Temp, .. subDirectoryNames]);
 
+    [Obsolete($"Use {nameof(UITestContext)}.{nameof(UITestContext.GetTempSubDirectoryPath)}() instead.")]
     public static string GetTempSubDirectoryPath(string contextId, params string[] subDirectoryNames) =>
-        GetTempDirectoryPath(
-            Path.Combine(contextId, Path.Combine(subDirectoryNames)));
+        GetTempDirectoryPath([contextId, .. subDirectoryNames]);
 
+    [Obsolete($"Use {nameof(UITestContext)}.{nameof(UITestContext.ScreenshotsDirectoryPath)} instead.")]
     public static string GetScreenshotsDirectoryPath(string contextId) =>
         GetTempSubDirectoryPath(contextId, Screenshots);
 }

Lombiq.Tests.UI/Extensions/AccessibilityCheckingUITestContextExtensions.cs

@@ -1,6 +1,5 @@
 using Deque.AxeCore.Commons;
 using Deque.AxeCore.Selenium;
-using Lombiq.Tests.UI.Constants;
 using Lombiq.Tests.UI.Exceptions;
 using Lombiq.Tests.UI.Helpers;
 using Lombiq.Tests.UI.Services;
@@ -47,7 +46,7 @@ public static void AssertAccessibility(
         if (accessibilityConfiguration.CreateReportAlways)
         {
             var reportDirectoryPath = DirectoryHelper.CreateEnumeratedDirectory(
-                DirectoryPaths.GetTempSubDirectoryPath(context.Id, "AxeHtmlReport"));
+                context.GetTempSubDirectoryPath("AxeHtmlReport"));
 
             var reportPath = Path.Combine(
                     reportDirectoryPath,

Lombiq.Tests.UI/Extensions/FrontendOrchardCoreUITestExecutorConfigurationExtensions.cs

@@ -0,0 +1,36 @@
+using Lombiq.Tests.UI.Services;
+using System;
+using System.Collections.Generic;
+
+namespace Lombiq.Tests.UI.Extensions;
+
+public static class FrontendOrchardCoreUITestExecutorConfigurationExtensions
+{
+    private const string BackendUri = nameof(BackendUri);
+    private const string FrontendUri = nameof(FrontendUri);
+
+    /// <summary>
+    /// Returns the start URLs for the frontend and the Orchard Core backend from the <see
+    /// cref="OrchardCoreUITestExecutorConfiguration.CustomConfiguration"/>.
+    /// </summary>
+    public static (Uri FrontendUri, Uri BackendUri) GetFrontendAndBackendUris(
+        this OrchardCoreUITestExecutorConfiguration configuration) =>
+    (
+        configuration.CustomConfiguration.GetMaybe(FrontendUri) as Uri,
+        configuration.CustomConfiguration.GetMaybe(BackendUri) as Uri
+    );
+
+    /// <summary>
+    /// Updates the <see cref="OrchardCoreUITestExecutorConfiguration.CustomConfiguration"/> by storing the frontend and
+    /// the Orchard Core backend URLs as <see cref="Uri"/> instances. If either parameter is <see langword="null"/>,
+    /// that value is not changed.
+    /// </summary>
+    public static void SetFrontendAndBackendUris(
+        this OrchardCoreUITestExecutorConfiguration configuration,
+        string frontendUrl,
+        string backendUrl)
+    {
+        if (frontendUrl != null) configuration.CustomConfiguration[FrontendUri] = new Uri(frontendUrl);
+        if (backendUrl != null) configuration.CustomConfiguration[BackendUri] = new Uri(backendUrl);
+    }
+}