OSOE-838: Updating Atata v3.1.0, and other dependencies to latest too, reliability improvements

Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj

@@ -3,19 +3,15 @@
   <PropertyGroup>
     <TargetFramework>net8.0</TargetFramework>
     <IsPackable>false</IsPackable>
-    <!-- Eliminates problem with dotnet publish caused by duplicate .htmlvalidate.json file. See
-         https://docs.microsoft.com/en-us/dotnet/core/compatibility/sdk/6.0/duplicate-files-in-output. -->
-    <ErrorOnDuplicatePublishOutputFiles>false</ErrorOnDuplicatePublishOutputFiles>
   </PropertyGroup>
 
   <ItemGroup>
-    <None Remove=".htmlvalidate.json" />
     <None Remove="Tests\CustomZapAutomationFrameworkPlan.yml" />
     <None Remove="xunit.runner.json" />
   </ItemGroup>
 
   <ItemGroup>
-    <Content Include=".htmlvalidate.json">
+    <Content Include="samples.htmlvalidate.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </Content>
     <Content Include="Tests\CustomZapAutomationFrameworkPlan.yml">
@@ -28,7 +24,7 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.7.2" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.5.1">
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.8.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

Lombiq.Tests.UI.Samples/Tests/InteractiveModeTests.cs

@@ -44,8 +44,8 @@ public Task SampleTest() =>
             });
 
     // This test checks if interactive mode works by opening it in one thread, and then clicking it away in a different
-    // thread. This ensures that the new tab correctly appears with the clickable "Continue Test" button, and then
-    // disappears once it's clicked.
+    // thread. Two threads are necessary because interactive mode stops test execution on its current thread, so we
+    // wouldn't be able to end it from within a test.
     [Fact]
     public Task EnteringInteractiveModeShouldWait() =>
         ExecuteTestAfterSetupAsync(
@@ -59,7 +59,7 @@ await Task.WhenAll(
                     {
                         // Ensure that the interactive mode polls for status at least once, so the arbitrary waiting
                         // actually works in a real testing scenario.
-                        await Task.Delay(1000);
+                        await Task.Delay(5000);
 
                         await context.ClickReliablyOnAsync(By.ClassName("interactive__continue"));
                     }));

Lombiq.Tests.UI.Samples/UITestBase.cs

@@ -4,6 +4,7 @@
 using Lombiq.Tests.UI.Services;
 using Shouldly;
 using System;
+using System.IO;
 using System.Threading.Tasks;
 using Xunit.Abstractions;
 
@@ -79,10 +80,17 @@ protected override Task ExecuteTestAsync(
                 // disable it with the below config). With this, you can make sure that the HTML markup the app
                 // generates (also from content items) is valid. While the default settings for HTML validation are most
                 // possibly suitable for your projects, check out the HtmlValidationConfiguration class for what else
-                // you can configure. We've also added a .htmlvalidate.json file (note the Content Build
+                // you can configure. We've also added a custom .htmlvalidate.json file (note the Content Build
                 // Action) to further configure it.
                 ////configuration.HtmlValidationConfiguration.RunHtmlValidationAssertionOnAllPageChanges = false;
 
+                // This is to make sure that the HTML validation config file in this project takes effect. Only needed
+                // until we fix https://github.com/Lombiq/UI-Testing-Toolbox/issues/359.
+                configuration.HtmlValidationConfiguration.HtmlValidationOptions =
+                    configuration.HtmlValidationConfiguration.HtmlValidationOptions
+                        .CloneWith(validationOptions => validationOptions.ConfigPath =
+                            Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "samples.htmlvalidate.json")); // #spell-check-ignore-line
+
                 // The UI Testing Toolbox can run several checks for the app even if you don't add explicit
                 // assertions: By default, the Orchard logs and the browser logs (where e.g. JavaScript errors show
                 // up) are checked and if there are any errors, the test will fail. You can also enable the checking of

Lombiq.Tests.UI.Samples/samples.htmlvalidate.json

@@ -0,0 +1,11 @@
+{
+  "extends": [
+    "./.htmlvalidate.json"
+  ],
+
+  "rules": {
+    "long-title": "off",
+    "no-dup-class": "off"
+  },
+  "root": true
+}

Lombiq.Tests.UI/Docs/Configuration.md

@@ -59,26 +59,17 @@ Recommendations and notes for such configuration:
 
 ### HTML validation configuration
 
-If you want to change some HTML validation rules from only a few specific tests, you can create a custom _.htmlvalidate.json_ file (e.g. _TestName.htmlvalidate.json_). For example:
+If you want to change some HTML validation rules from only a few specific tests, you can create a custom _.htmlvalidate.json_ file (e.g. _TestName.htmlvalidate.json_). It can also extend the default one, only overriding the configuration necessary. For example:
 
 ```json
 {
   "extends": [
-    "html-validate:recommended"
+    "./.htmlvalidate.json"
   ],
 
   "rules": {
-    "attribute-boolean-style": "off",
     "element-required-attributes": "off",
-    "no-trailing-whitespace": "off",
-    "no-inline-style": "off",
-    "no-implicit-button-type": "off",
-    "wcag/h30": "off",
-    "wcag/h32": "off",
-    "wcag/h36": "off",
-    "wcag/h37": "off",
-    "wcag/h67": "off",
-    "wcag/h71": "off"
+    "no-implicit-button-type": "off"
   },
 
   "root":  true
@@ -88,10 +79,8 @@ If you want to change some HTML validation rules from only a few specific tests,
 Then you can change the configuration to use that:
 
 ```cs
- changeConfiguration: configuration => configuration.HtmlValidationConfiguration.HtmlValidationOptions =
-                configuration.HtmlValidationConfiguration.HtmlValidationOptions
-                    .CloneWith(validationOptions => validationOptions.ConfigPath =
-                        Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "TestName.htmlvalidate.json")));
+ changeConfiguration: configuration => 
+    configuration.HtmlValidationConfiguration.HtmlValidationOptions.SetLocalConfigFile("TestName.htmlvalidate.json");
 ```
 
 Make sure to also include the `root` attribute and set it to `true` inside the custom _.htmlvalidate.json_ file and include it in the test project like this:

Lombiq.Tests.UI/Extensions/ExtendedLoggingExtensions.cs

@@ -148,15 +148,15 @@ private static async Task<TResult> ExecuteSectionAsync<TResult>(
             var notLast = i < StabilityRetryCount - 1;
             try
             {
-                // This is somewhat risky. ILogManager is not thread-safe and uses as stack to keep track of sections, so if
-                // multiple sections are started in concurrent threads, the result will be incorrect. This shouldn't be too much
-                // of an issue for now though since tests, while async, are single-threaded.
-                context.Scope.AtataContext.Log.Start(section);
-                context.Scope.AtataContext.Log.Info("Log section {0} started.", section.Message);
-                var result = await functionAsync();
-                context.Scope.AtataContext.Log.Info("Log section {0} ended.", section.Message);
-                context.Scope.AtataContext.Log.EndSection();
-                return result;
+                return await context.Scope.AtataContext.Log.ExecuteSectionAsync(
+                    section,
+                    async () =>
+                    {
+                        context.Scope.AtataContext.Log.Info($"Log section {section.Message} started.");
+                        var result = await functionAsync();
+                        context.Scope.AtataContext.Log.Info($"Log section {section.Message} ended.");
+                        return result;
+                    });
             }
             catch (StaleElementReferenceException) when (notLast)
             {
@@ -171,7 +171,5 @@ private static async Task<TResult> ExecuteSectionAsync<TResult>(
     private static void LogStaleElementReferenceExceptionRetry(UITestContext context, int tryIndex) =>
         context.Scope.AtataContext.Log.Info(
             "The operation in the log section failed with StaleElementReferenceException but will be retried. This " +
-            "is try number {0} out of {1}.",
-            tryIndex + 1,
-            StabilityRetryCount);
+            $"is try number {(tryIndex + 1).ToTechnicalString()} out of {StabilityRetryCount}.");
 }

Lombiq.Tests.UI/Extensions/FormUITestContextExtensions.cs

@@ -108,6 +108,9 @@ public static void FillInMonacoEditor(
         string editorId,
         string text)
     {
+        // Waiting for the editor to load.
+        context.Get(By.CssSelector($"#{editorId} .monaco-editor"));
+
         var script = $@"
             monaco.editor.getEditors().find((element) =>
                 element.getContainerDomNode().id == {JsonConvert.SerializeObject(editorId)}).setValue({JsonConvert.SerializeObject(text)});";

Lombiq.Tests.UI/Extensions/HtmlValidationOptionsExtensions.cs

@@ -0,0 +1,15 @@
+using System;
+using System.IO;
+
+namespace Atata.HtmlValidation;
+
+public static class HtmlValidationOptionsExtensions
+{
+    /// <summary>
+    /// Sets the path of an HTML-validate config file local to the current app as <see
+    /// cref="HtmlValidationOptions.ConfigPath"/>.
+    /// </summary>
+    /// <param name="configFileRelativePath">The relative path to the HTML-validate config file.</param>
+    public static void SetLocalConfigFile(this HtmlValidationOptions htmlValidationOptions, string configFileRelativePath) =>
+        htmlValidationOptions.ConfigPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, configFileRelativePath);
+}

Lombiq.Tests.UI/Lombiq.Tests.UI.csproj

@@ -59,31 +59,30 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Atata" Version="2.9.0" />
-    <PackageReference Include="Atata.Bootstrap" Version="2.1.0" />
-    <PackageReference Include="Atata.HtmlValidation" Version="2.4.0" />
-    <PackageReference Include="Atata.WebDriverExtras" Version="2.2.0" />
+    <PackageReference Include="Atata" Version="3.1.0" />
+    <PackageReference Include="Atata.Bootstrap" Version="3.0.0" />
+    <PackageReference Include="Atata.HtmlValidation" Version="3.0.0" />
+    <PackageReference Include="Atata.WebDriverExtras" Version="3.0.0" />
     <PackageReference Include="Atata.WebDriverSetup" Version="2.10.0" />
-    <PackageReference Include="Azure.Storage.Blobs" Version="12.18.0" />
+    <PackageReference Include="Azure.Storage.Blobs" Version="12.19.1" />
     <PackageReference Include="Ben.Demystifier" Version="0.4.1" />
-    <PackageReference Include="Codeuctivity.ImageSharpCompare" Version="3.0.183" />
+    <PackageReference Include="Codeuctivity.ImageSharpCompare" Version="4.0.258" />
     <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="8.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
-    <PackageReference Include="Microsoft.SqlServer.DACFx" Version="162.0.52" />
-    <PackageReference Include="Microsoft.SqlServer.SqlManagementObjects" Version="170.18.0" />
+    <PackageReference Include="Microsoft.SqlServer.DACFx" Version="162.2.111" />
+    <PackageReference Include="Microsoft.SqlServer.SqlManagementObjects" Version="171.30.0" />
     <PackageReference Include="Mono.Posix.NETStandard" Version="1.0.0" />
     <PackageReference Include="OrchardCore.ContentFields" Version="1.8.2" />
     <PackageReference Include="OrchardCore.Logging.NLog" Version="1.8.2" />
     <PackageReference Include="OrchardCore.Abstractions" Version="1.8.2" />
     <PackageReference Include="OrchardCore.Recipes.Core" Version="1.8.2" />
     <PackageReference Include="Refit.HttpClientFactory" Version="7.0.0" />
-    <PackageReference Include="Sarif.Sdk" Version="4.3.7" />
-    <PackageReference Include="Selenium.Axe" Version="4.0.14" />
+    <PackageReference Include="Sarif.Sdk" Version="4.5.4" />
+    <PackageReference Include="Selenium.Axe" Version="4.0.17" />
     <PackageReference Include="Shouldly" Version="4.2.1" />
     <PackageReference Include="System.Linq.Async" Version="6.0.1" />
-    <PackageReference Include="WebDriverManager" Version="2.16.2" />
-    <PackageReference Include="xunit" Version="2.5.1" />
-    <PackageReference Include="YamlDotNet" Version="13.7.1" />
+    <PackageReference Include="xunit" Version="2.8.0" />
+    <PackageReference Include="YamlDotNet" Version="15.1.2" />
   </ItemGroup>
 
   <ItemGroup>

Lombiq.Tests.UI/MonkeyTesting/MonkeyTester.cs

@@ -26,61 +26,49 @@ internal MonkeyTester(UITestContext context, MonkeyTestingOptions options = null
         _randomizer = new NonSecurityRandomizer(_options.BaseRandomSeed);
     }
 
-    internal async Task TestOnePageAsync(int? randomSeed = null)
-    {
-        Log.Start(new LogSection("Execute monkey testing against one page"));
-
-        try
-        {
-            WriteOptionsToLog();
-
-            var pageTestInfo = GetCurrentPageTestInfo();
-
-            if (randomSeed is null) await TestCurrentPageAsync(pageTestInfo);
-            else await TestCurrentPageWithRandomSeedAsync(pageTestInfo, randomSeed.Value);
-        }
-        finally
-        {
-            Log.EndSection();
-        }
-    }
+    internal Task TestOnePageAsync(int? randomSeed = null) =>
+        Log.ExecuteSectionAsync(
+            new LogSection("Execute monkey testing against one page"),
+            async () =>
+            {
+                WriteOptionsToLog();
 
-    internal async Task TestRecursivelyAsync()
-    {
-        Log.Start(new LogSection($"Execute monkey testing recursively"));
+                var pageTestInfo = GetCurrentPageTestInfo();
 
-        try
-        {
-            WriteOptionsToLog();
-
-            var pageTestInfo = GetCurrentPageTestInfo();
-            await TestCurrentPageAsync(pageTestInfo);
+                if (randomSeed is null) await TestCurrentPageAsync(pageTestInfo);
+                else await TestCurrentPageWithRandomSeedAsync(pageTestInfo, randomSeed.Value);
+            });
 
-            while (true)
+    internal Task TestRecursivelyAsync() =>
+        Log.ExecuteSectionAsync(
+            new LogSection($"Execute monkey testing recursively"),
+            async () =>
             {
-                pageTestInfo = GetCurrentPageTestInfo();
+                WriteOptionsToLog();
 
-                if (CanTestPage(pageTestInfo))
-                {
-                    await TestCurrentPageAsync(pageTestInfo);
-                }
-                else if (TryGetAvailablePageToTest(out var availablePageToTest))
-                {
-                    await _context.GoToAbsoluteUrlAsync(availablePageToTest.Url);
+                var pageTestInfo = GetCurrentPageTestInfo();
+                await TestCurrentPageAsync(pageTestInfo);
 
-                    await TestCurrentPageAsync(availablePageToTest);
-                }
-                else
+                while (true)
                 {
-                    return;
+                    pageTestInfo = GetCurrentPageTestInfo();
+
+                    if (CanTestPage(pageTestInfo))
+                    {
+                        await TestCurrentPageAsync(pageTestInfo);
+                    }
+                    else if (TryGetAvailablePageToTest(out var availablePageToTest))
+                    {
+                        await _context.GoToAbsoluteUrlAsync(availablePageToTest.Url);
+
+                        await TestCurrentPageAsync(availablePageToTest);
+                    }
+                    else
+                    {
+                        return;
+                    }
                 }
-            }
-        }
-        finally
-        {
-            Log.EndSection();
-        }
-    }
+            });
 
     private void WriteOptionsToLog() =>
         Log.Trace(@$"Monkey testing options:

Lombiq.Tests.UI/Pages/OrchardCoreAdminPage.cs

@@ -12,9 +12,9 @@ public abstract class OrchardCoreAdminPage<TOwner> : Page<TOwner>
 
     public ControlList<AlertMessage<TOwner>, TOwner> AlertMessages { get; private set; }
 
-    public TOwner ShouldStayOnAdminPage() => AdminMenu.Should.Exist();
+    public TOwner ShouldStayOnAdminPage() => AdminMenu.Should.BePresent();
 
-    public TOwner ShouldLeaveAdminPage() => AdminMenu.Should.Not.Exist();
+    public TOwner ShouldLeaveAdminPage() => AdminMenu.Should.Not.BePresent();
 
     protected override void OnVerify()
     {

Lombiq.Tests.UI/Services/AtataFactory.cs

@@ -17,11 +17,13 @@ public class AtataConfiguration
 public static class AtataFactory
 {
     public static async Task<AtataScope> StartAtataScopeAsync(
+        string contextId,
         ITestOutputHelper testOutputHelper,
         Uri baseUri,
         OrchardCoreUITestExecutorConfiguration configuration)
     {
-        AtataContext.ModeOfCurrent = AtataContextModeOfCurrent.AsyncLocal;
+        AtataContext.GlobalProperties.ModeOfCurrent = AtataContextModeOfCurrent.AsyncLocal;
+        AtataContext.GlobalProperties.UseUtcTimeZone();
 
         // Since Atata 2.0 the default visibility option is Visibility.Any, these lines restore it to the 1.x behavior.
         AtataContext.GlobalConfiguration.UseDefaultControlVisibility(Visibility.Visible);
@@ -37,8 +39,8 @@ public static async Task<AtataScope> StartAtataScopeAsync(
             .UseTestName(configuration.AtataConfiguration.TestName)
             .UseBaseRetryTimeout(timeoutConfiguration.RetryTimeout)
             .UseBaseRetryInterval(timeoutConfiguration.RetryInterval)
-            .UseUtcTimeZone()
-            .PageSnapshots.UseCdpOrPageSourceStrategy(); // #spell-check-ignore-line
+            .PageSnapshots.UseCdpOrPageSourceStrategy() // #spell-check-ignore-line
+            .UseArtifactsPathTemplate(contextId); // Necessary to prevent long paths, an issue under Windows.
 
         builder.LogConsumers.AddDebug();
         builder.LogConsumers.Add(new TestOutputLogConsumer(testOutputHelper));

Lombiq.Tests.UI/Services/UITestExecutionSession.cs

@@ -656,7 +656,7 @@ Task UITestingBeforeAppStartHandlerAsync(string contentRootPath, InstanceCommand
             _configuration.Events.AfterPageChange += TakeScreenshotIfEnabledAsync;
         }
 
-        var atataScope = await AtataFactory.StartAtataScopeAsync(_testOutputHelper, uri, _configuration);
+        var atataScope = await AtataFactory.StartAtataScopeAsync(contextId, _testOutputHelper, uri, _configuration);
 
         return new UITestContext(
             contextId,