Merge pull request #119 from Lombiq/issue/OSOE-49

OSOE-49: Replace Windows-only code in UI Testing Toolbox, upgrade to latest Atata/Selenium

.gitattributes

@@ -1 +1,5 @@
 * text=auto
+
+# Enforce Windows newlines for C# files to avoid false positives with IDE0055 warning.
+# See https://github.com/Lombiq/Open-Source-Orchard-Core-Extensions/issues/106 for more information.
+*.cs text eol=crlf

Lombiq.Tests.UI.Samples/Extensions/UITestContextExtensions.cs

@@ -0,0 +1,22 @@
+using Lombiq.Tests.UI.Extensions;
+using Lombiq.Tests.UI.Services;
+using OpenQA.Selenium;
+using Shouldly;
+using System.Threading.Tasks;
+
+namespace Lombiq.Tests.UI.Samples.Extensions;
+
+public static class UITestContextExtensions
+{
+    public static async Task CheckIfAnonymousHomePageExistsAsync(this UITestContext context)
+    {
+        // Is the title correct?
+        context
+            .Get(By.ClassName("navbar-brand"))
+            .Text
+            .ShouldBe("Lombiq's OSOCE - UI Testing");
+
+        // Are we logged out?
+        (await context.GetCurrentUserNameAsync()).ShouldBeNullOrEmpty();
+    }
+}

Lombiq.Tests.UI.Samples/Helpers/SetupHelpers.cs

@@ -1,9 +1,12 @@
+using Atata;
 using Lombiq.Tests.UI.Constants;
 using Lombiq.Tests.UI.Extensions;
 using Lombiq.Tests.UI.Pages;
 using Lombiq.Tests.UI.Services;
 using OpenQA.Selenium;
 using System;
+using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
 
 namespace Lombiq.Tests.UI.Samples.Helpers;
@@ -65,5 +68,20 @@ public static async Task<Uri> RunAutoSetupAsync(UITestContext context)
         return context.GetCurrentUri();
     }
 
-    private static void AssertSetupSuccessful(UITestContext context) => context.Exists(By.Id("navbar"));
+    private static void AssertSetupSuccessful(UITestContext context)
+    {
+        try
+        {
+            context.Exists(By.Id("navbar"));
+        }
+        catch (NoSuchElementException)
+        {
+            var validationErrors = context.GetAll(By.ClassName("field-validation-error"));
+
+            if (!validationErrors.Any()) throw;
+
+            var errors = "\n- " + validationErrors.Select(element => element.Text.Trim()).Join("\n- ");
+            throw new AssertionException($"Setup has failed with the following validation errors:{errors}");
+        }
+    }
 }

Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj

@@ -23,8 +23,8 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.2.0" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

Lombiq.Tests.UI.Samples/Tests/AzureBlobStorageTests.cs

@@ -1,8 +1,7 @@
 using Lombiq.Tests.UI.Attributes;
 using Lombiq.Tests.UI.Extensions;
+using Lombiq.Tests.UI.Samples.Extensions;
 using Lombiq.Tests.UI.Services;
-using OpenQA.Selenium;
-using Shouldly;
 using System.Linq;
 using System.Threading.Tasks;
 using Xunit;
@@ -25,10 +24,7 @@ public AzureBlobStorageTests(ITestOutputHelper testOutputHelper)
     [Theory, Chrome]
     public Task AnonymousHomePageShouldExistWithAzureBlobStorage(Browser browser) =>
         ExecuteTestAfterSetupAsync(
-            context => context
-                .Get(By.ClassName("navbar-brand"))
-                .Text
-                .ShouldBe("Lombiq's OSOCE - UI Testing"),
+            context => context.CheckIfAnonymousHomePageExistsAsync(),
             browser,
             // Note the configuration! We could also set this globally in UITestBase. You'll need an accessible Azure
             // Blob Storage account. For testing we recommend the Azurite emulator

Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs

@@ -2,7 +2,9 @@
 using Lombiq.Tests.UI.Exceptions;
 using Lombiq.Tests.UI.Extensions;
 using Lombiq.Tests.UI.Services;
+using Shouldly;
 using System;
+using System.Linq;
 using System.Threading.Tasks;
 using Xunit;
 using Xunit.Abstractions;
@@ -64,6 +66,40 @@ public Task ClientSideErrorOnLoadedPageShouldHaltTest(Browser browser) =>
                 }
             },
             browser);
+
+    // To be able to trust the test above, we have to be sure that the browser logs survive the navigation events and
+    // all get collected into the historic browser log.
+    [Theory, Chrome]
+    public Task BrowserLogsShouldPersist(Browser browser) =>
+        ExecuteTestAfterSetupAsync(
+            async context =>
+            {
+                const string testLog = "--test log--";
+                void WriteConsoleLog() => context.ExecuteScript($"console.info('{testLog}');");
+
+                await context.SignInDirectlyAndGoToHomepageAsync();
+
+                WriteConsoleLog();
+                WriteConsoleLog();
+
+                await context.GoToDashboardAsync();
+
+                WriteConsoleLog();
+
+                await context.GoToHomePageAsync();
+
+                WriteConsoleLog();
+                WriteConsoleLog();
+                WriteConsoleLog();
+
+                await context.UpdateHistoricBrowserLogAsync();
+
+                context
+                    .HistoricBrowserLog
+                    .Count(entry => entry.Message.Contains(testLog))
+                    .ShouldBe(6);
+            },
+            browser);
 }
 
 // END OF TRAINING SECTION: Error handling.

Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs

@@ -3,6 +3,7 @@
 using Lombiq.Tests.UI.MonkeyTesting;
 using Lombiq.Tests.UI.MonkeyTesting.UrlFilters;
 using Lombiq.Tests.UI.Services;
+using OpenQA.Selenium;
 using Shouldly;
 using System;
 using System.Linq;
@@ -95,11 +96,10 @@ private static MonkeyTestingOptions CreateMonkeyTestingOptions() =>
             PageTestTime = TimeSpan.FromSeconds(10),
         };
 
-    private static bool IsValidAdminBrowserLogMessage(BrowserLogMessage message) =>
+    private static bool IsValidAdminBrowserLogMessage(LogEntry message) =>
         OrchardCoreUITestExecutorConfiguration.IsValidBrowserLogMessage(message) &&
-        !(message.Source == BrowserLogMessage.Sources.Intervention &&
-            message.Message.ContainsOrdinalIgnoreCase(
-                "Blocked attempt to show a 'beforeunload' confirmation panel for a frame that never had a user gesture since its load."));
+        !message.Message.ContainsOrdinalIgnoreCase(
+            "Blocked attempt to show a 'beforeunload' confirmation panel for a frame that never had a user gesture since its load.");
 }
 
 // END OF TRAINING SECTION: Monkey tests.

Lombiq.Tests.UI.Samples/Tests/SqlServerTests.cs

@@ -1,8 +1,7 @@
 using Lombiq.Tests.UI.Attributes;
 using Lombiq.Tests.UI.Extensions;
+using Lombiq.Tests.UI.Samples.Extensions;
 using Lombiq.Tests.UI.Services;
-using OpenQA.Selenium;
-using Shouldly;
 using System.Linq;
 using System.Threading.Tasks;
 using Xunit;
@@ -27,10 +26,7 @@ public SqlServerTests(ITestOutputHelper testOutputHelper)
     [Theory, Chrome]
     public Task AnonymousHomePageShouldExistWithSqlServer(Browser browser) =>
         ExecuteTestAfterSetupAsync(
-            context => context
-                .Get(By.ClassName("navbar-brand"))
-                .Text
-                .ShouldBe("Lombiq's OSOCE - UI Testing"),
+            context => context.CheckIfAnonymousHomePageExistsAsync(),
             browser,
             // Note the configuration! We could also set this globally in UITestBase.
             configuration => configuration.UseSqlServer = true);

Lombiq.Tests.UI/.config/dotnet-tools.json

@@ -3,7 +3,7 @@
   "isRoot": true,
   "tools": {
     "rnwood.smtp4dev": {
-      "version": "3.1.2-ci20201001101",
+      "version": "3.1.4",
       "commands": [
         "smtp4dev"
       ]

Lombiq.Tests.UI/Components/AlertMessage.cs

@@ -11,8 +11,8 @@ public class AlertMessage<TOwner> : Control<TOwner>
     [GetsContentFromSource(ContentSource.FirstChildTextNode)]
     public Text<TOwner> Text { get; private set; }
 
-    public DataProvider<bool, TOwner> IsSuccess =>
-        GetOrCreateDataProvider("success state", GetIsSuccess);
+    public ValueProvider<bool, TOwner> IsSuccess =>
+        CreateValueProvider("success state", GetIsSuccess);
 
     private bool GetIsSuccess() =>
         Attributes.Class.Value.Contains("message-success");

Lombiq.Tests.UI/Docs/Configuration.md

@@ -49,15 +49,17 @@ Here's a full *TestConfiguration.json* file example, something appropriate durin
     }
   }
 }
-
 ```
 
 Note that this will execute tests in headless mode, so no browser windows will be opened (for browsers that support it). If you want to troubleshoot a failing test then disable headless mode.
 
 We encourage you to experiment with a `RetryTimeoutSeconds` value suitable for your hardware. Higher, paradoxically, is usually less safe.
 
+If you have several UI test projects it can be cumbersome to maintain a *TestConfiguration.json* file for each. Instead you can set the value of the `LOMBIQ_UI_TESTING_TOOLBOX_SHARED_TEST_CONFIGURATION` environment variable to the absolute path of a central configuration file and then each project will look it up. If you place an individual *TestConfiguration.json* into a test directory it will still take precedence in case you need special configuration for just that one.
+
 `MaxParallelTests` sets how many UI tests should run at the same time. It is an important property if you want to run your UI tests in parallel, check out the inline documentation in [`OrchardCoreUITestExecutorConfiguration`](../Services/OrchardCoreUITestExecutorConfiguration.cs).
 
+
 ## <a name="multi-process"></a>Multi-process test execution
 
 UI tests are executed in parallel by default for the given test execution process (see the [xUnit documentation](https://xunit.net/docs/running-tests-in-parallel.html)). However, if you'd like multiple processes to execute tests like when multiple build agents run tests for separate branches on the same build machine then you'll need to tell each process which build agent they are on. This is so clashes on e.g. network port numbers can be prevented.
@@ -73,27 +75,48 @@ If you have multiple UI test projects in a single solution and you're executing
 
 You can learn more about the *microsoft-mssql-server* container [here](https://hub.docker.com/_/microsoft-mssql-server). You have to mount a local volume that can be shared between the host and the container. Update the values of `device` and `SA_PASSWORD` in the code below and execute it.
 
+#### On Windows
+
 ```powershell
 docker pull mcr.microsoft.com/mssql/server
-docker volume create --driver local -o o=bind -o type=none -o device="C:\docker\data" data
-docker run --name sql2019 -e "ACCEPT_EULA=Y" -e "SA_PASSWORD=yourStrong(!)Password" -v data:/data -p 1433:1433 -d mcr.microsoft.com/mssql/server:2019-latest
+docker run --name sql2019 -e "ACCEPT_EULA=Y" -e "SA_PASSWORD=Password1!" -p 1433:1433 -d mcr.microsoft.com/mssql/server:2019-latest
+docker exec -u 0 sql2019 bash -c "mkdir /data; chmod 777 /data -R; chown mssql:root /data"
+```
+
+#### On Linux
+
+You need to put the shared directory inside your _$HOME_, in this example _~/.local/docker/mssql/data_:
+
+```shell
+docker pull mcr.microsoft.com/mssql/server
+docker run --name sql2019 -e 'ACCEPT_EULA=Y' -e 'SA_PASSWORD=Password1!' -p 1433:1433 -d 'mcr.microsoft.com/mssql/server:2019-latest'
+docker exec -u 0 sql2019 bash -c 'mkdir /data; chmod 777 /data -R; chown mssql:root /data'
 ```
 
-Now log in as root using `docker exec -u 0 -it sql2019 bash` and give access to the data directory with `chown 'mssql:root' /data`.
+If you haven't yet, add your user to the `docker` group.
+
+If you get a `PlatformNotSupportedException`, that's a known problem with _Microsoft.Data.SqlClient_ on .Net 5 and above. As a workaround, temporarily set the project's runtime identifier to `linux-x64` - either [on the terminal](https://github.com/dotnet/SqlClient/issues/1423#issuecomment-1093430430), or by adding `<RuntimeIdentifier>linux-x64</RuntimeIdentifier>` to the project file.   
+
+#### On Both
 
-You can use Docker Desktop to stop or start the container going forward. 
+If you want to test it out, type `docker exec -u 0 -it sql2019 /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'Password1!'` to access the SQL console.
 
+You can use [Docker Desktop](https://www.docker.com/products/docker-desktop) or [Portainer](https://www.portainer.io) to stop or start the container going forward. 
 
 ### Extending TestConfiguration.json
 
 SQL Server on Linux only has SQL Authentication and you still have to tell the toolbox about your backup paths. Add the following properties to your `Lombiq_Tests_UI`. Adjust the Password field of the connection string and `HostSnapshotPath` property as needed.
 
 ```json
-"SqlServerDatabaseConfiguration": {
-    "ConnectionStringTemplate": "Server=.;Database=LombiqUITestingToolbox_{{id}};User Id=sa;Password=yourStrong(!)Password;MultipleActiveResultSets=True;Connection Timeout=60;ConnectRetryCount=15;ConnectRetryInterval=5"
-},
-"DockerConfiguration": {
-    "ContainerSnapshotPath": "/data",
-    "HostSnapshotPath": "C:\\docker\\data"
+{
+  "SqlServerDatabaseConfiguration": {
+    "ConnectionStringTemplate": "Server=.;Database=LombiqUITestingToolbox_{{id}};User Id=sa;Password=Password1!;MultipleActiveResultSets=True;Connection Timeout=60;ConnectRetryCount=15;ConnectRetryInterval=5"
+  },
+  "DockerConfiguration": {
+    "ContainerSnapshotPath": "/data/Snapshots",
+    "ContainerName": "sql2019"
+  }
 }
 ```
+
+The default value of `ContainerSnapshotPath` is `"/data/Snapshots"` so you can omit that.

Lombiq.Tests.UI/Docs/Linux.md

@@ -0,0 +1,25 @@
+# Linux-specific considerations
+
+
+
+## Global NPM vs Userspace NPM via Node Version Manager
+
+
+As Linux has a stricter access policy you may want to install NPM in the userspace so you can still install global packages (e.g. html-validate) without sudoing. The easiest way to do this is via NVM. If you don't have NVM yet, [follow the guide here](https://github.com/Lombiq/NPM-Targets/tree/dev#global-npm-vs-userspace-npm-via-node-version-manager-on-linux).
+
+This library configures processes launched by Atata (via `Atata.Cli.ProgramCli`) to use Bash as login shell on non-Windows systems by default, like this:
+
+```csharp
+ProgramCli.DefaultShellCliCommandFactory = OSDependentShellCliCommandFactory
+    .UseCmdForWindows()
+    .UseForOtherOS(new BashShellCliCommandFactory("-login"));
+```
+
+If your project has different requirements, you can change it in your `OrchardCoreUITestBase.ExecuteTestAfterSetupAsync` implementation. Set a new value in the configuration function you pass to `base.ExecuteTestAsync`.
+
+
+## SQL Server Usage
+
+Since 2017, Microsoft SQL Server is available on [RHEL](https://redhat.com/rhel/), [SUSE](https://www.suse.com/products/server/) and [Ubuntu](https://ubuntu.com/) as well as a [Linux-based Docker image](https://hub.docker.com/_/microsoft-mssql-server) that you can run on any OS.
+
+We suggest using the Docker image even on those OSes. It reduces the number of unknowns and moving parts in your setup, it's easier to reset if something goes wrong, and that's what we support. We have a guide for setting up SQL Server for Linux on Docker [here](Configuration.md#using-sql-server-from-a-docker-container).

Lombiq.Tests.UI/Exceptions/SetupFailedFastException.cs

@@ -8,9 +8,10 @@ public class SetupFailedFastException : Exception
 {
     public int FailureCount { get; }
 
-    public SetupFailedFastException(int failureCount) => FailureCount = failureCount;
-
-    public override string Message =>
-         $"The given setup operation failed {FailureCount.ToTechnicalString()} times and won't be retried any " +
-        $"more. All tests using this operation for setup will instantly fail.";
+    public SetupFailedFastException(int failureCount, Exception latestException)
+        : base(
+            $"The given setup operation failed {failureCount.ToTechnicalString()} times and won't be retried any " +
+            $"more. All tests using this operation for setup will instantly fail.",
+            latestException) =>
+        FailureCount = failureCount;
 }

Lombiq.Tests.UI/Extensions/ElementRetrievalUITestContextExtensions.cs

@@ -1,7 +1,6 @@
 using Atata;
 using Lombiq.Tests.UI.Services;
 using OpenQA.Selenium;
-using OpenQA.Selenium.Remote;
 using Shouldly;
 using System;
 using System.Collections.Generic;
@@ -142,7 +141,7 @@ public static void VerifyElementTexts(this UITestContext context, By by, params
     public static void VerifyElementTexts(this UITestContext context, By by, IEnumerable<object> toMatch) =>
         VerifyElementTexts(context, by, toMatch is object[] array ? array : toMatch.ToArray());
 
-    private static ExtendedSearchContext<RemoteWebDriver> CreateSearchContext(this UITestContext context) =>
+    private static ExtendedSearchContext<IWebDriver> CreateSearchContext(this UITestContext context) =>
         new(
             context.Driver,
             context.Configuration.TimeoutConfiguration.RetryTimeout,