OSOE-97: Maximizing concurrent tests running

Lombiq.Tests.UI/Docs/Configuration.md

@@ -41,7 +41,8 @@ Here's a full *TestConfiguration.json* file example, something appropriate durin
     },
     "OrchardCoreUITestExecutorConfiguration": {
       "MaxRetryCount": 0,
-      "RetryIntervalSeconds": 0
+      "RetryIntervalSeconds": 0,
+      "MaxRunningConcurrentTests": 0
     },
     "BrowserConfiguration": {
       "Headless": true
@@ -55,6 +56,7 @@ Note that this will execute tests in headless mode, so no browser windows will b
 
 We encourage you to experiment with a `RetryTimeoutSeconds` value suitable for your hardware. Higher, paradoxically, is usually less safe.
 
+`MaxRunningConcurrentTests` sets how many tests should run at the same time. Use a value of `0` to indicate that you would like the default behavior. Use a value of `-1` to indicate that you do not wish to limit the number of tests running at the same time. The default behaviour and `0` uses the [System.Environment.ProcessorCount](https://docs.microsoft.com/en-us/dotnet/api/system.environment.processorcount?view=net-6.0#remarks) property. Set any other positive integer to limit to the exact number.
 
 ## <a name="multi-process"></a>Multi-process test execution
 

Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs

@@ -25,10 +25,20 @@ public class OrchardCoreUITestExecutorConfiguration
     public OrchardCoreConfiguration OrchardCoreConfiguration { get; set; }
 
     public int MaxRetryCount { get; set; } =
-        TestConfigurationManager.GetIntConfiguration("OrchardCoreUITestExecutorConfiguration:MaxRetryCount", 2);
+        TestConfigurationManager.GetIntConfiguration(
+            $"{nameof(OrchardCoreUITestExecutorConfiguration)}:{nameof(MaxRetryCount)}",
+            2);
 
     public TimeSpan RetryInterval { get; set; } =
-        TimeSpan.FromSeconds(TestConfigurationManager.GetIntConfiguration("OrchardCoreUITestExecutorConfiguration:RetryIntervalSeconds", 0));
+        TimeSpan.FromSeconds(TestConfigurationManager.GetIntConfiguration(
+            $"{nameof(OrchardCoreUITestExecutorConfiguration)}:RetryIntervalSeconds",
+            0));
+
+    public int MaxRunningConcurrentTests { get; set; } =
+            TestConfigurationManager.GetIntConfiguration(
+                $"{nameof(OrchardCoreUITestExecutorConfiguration)}:{nameof(MaxRunningConcurrentTests)}") is not { } intValue || intValue == 0
+                ? Environment.ProcessorCount
+                : intValue;
 
     public Func<IWebApplicationInstance, Task> AssertAppLogsAsync { get; set; } = AssertAppLogsCanContainWarningsAsync;
     public Action<IEnumerable<BrowserLogMessage>> AssertBrowserLog { get; set; } = AssertBrowserLogIsEmpty;

Lombiq.Tests.UI/Services/UITestExecutor.cs

@@ -4,13 +4,17 @@
 using Lombiq.Tests.UI.Models;
 using System;
 using System.IO;
+using System.Threading;
 using System.Threading.Tasks;
 using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Services;
 
 public static class UITestExecutor
 {
+    private static readonly object _initLock = new();
+    private static SemaphoreSlim _semaphoreSlim;
+
     /// <summary>
     /// Executes a test on a new Orchard Core web app instance within a newly created Atata scope.
     /// </summary>
@@ -51,12 +55,25 @@ private static async Task ExecuteOrchardCoreTestInnerAsync(UITestManifest testMa
 
         configuration.TestOutputHelper.WriteLineTimestampedAndDebug("Finished preparation for {0}.", testManifest.Name);
 
+        if (_semaphoreSlim == null && configuration.MaxRunningConcurrentTests > 0)
+        {
+            lock (_initLock)
+            {
+                _semaphoreSlim ??= new SemaphoreSlim(configuration.MaxRunningConcurrentTests);
+            }
+        }
+
         var retryCount = 0;
         var passed = false;
         while (!passed)
         {
             try
             {
+                if (_semaphoreSlim != null)
+                {
+                    await _semaphoreSlim.WaitAsync();
+                }
+
                 await using var instance = new UITestExecutionSession(testManifest, configuration);
                 passed = await instance.ExecuteAsync(retryCount, dumpRootPath);
             }
@@ -71,6 +88,8 @@ private static async Task ExecuteOrchardCoreTestInnerAsync(UITestManifest testMa
                 {
                     TeamCityMetadataReporter.ReportInt(testManifest, "TryCount", retryCount + 1);
                 }
+
+                _semaphoreSlim?.Release();
             }
 
             retryCount++;