Merge pull request #152 from Lombiq/issue/OSOE-98

OSOE-98: Save per-page screenshots to files directly to conserve memory

Lombiq.Tests.UI/Constants/DirectoryPaths.cs

@@ -0,0 +1,18 @@
+using System;
+using System.IO;
+using System.Linq;
+
+namespace Lombiq.Tests.UI.Constants;
+
+public static class DirectoryPaths
+{
+    public const string SetupSnapshot = nameof(SetupSnapshot);
+    public const string Temp = nameof(Temp);
+    public const string Screenshots = nameof(Screenshots);
+
+    public static string GetTempSubDirectoryPath(string contextId, params string[] subDirectoryNames) =>
+        Path.Combine(new[] { Environment.CurrentDirectory, Temp, contextId }.Concat(subDirectoryNames).ToArray());
+
+    public static string GetScreenshotsDirectoryPath(string contextId) =>
+        GetTempSubDirectoryPath(contextId, Screenshots);
+}

Lombiq.Tests.UI/Models/RunningContextContainer.cs

@@ -0,0 +1,20 @@
+using Lombiq.Tests.UI.Services;
+
+namespace Lombiq.Tests.UI.Models;
+
+public class RunningContextContainer
+{
+    public SqlServerRunningContext SqlServerRunningContext { get; }
+    public SmtpServiceRunningContext SmtpServiceRunningContext { get; }
+    public AzureBlobStorageRunningContext AzureBlobStorageRunningContext { get; }
+
+    public RunningContextContainer(
+        SqlServerRunningContext sqlServerContext,
+        SmtpServiceRunningContext smtpContext,
+        AzureBlobStorageRunningContext blobStorageContext)
+    {
+        SqlServerRunningContext = sqlServerContext;
+        SmtpServiceRunningContext = smtpContext;
+        AzureBlobStorageRunningContext = blobStorageContext;
+    }
+}

Lombiq.Tests.UI/Services/OrchardCoreInstance.cs

@@ -1,5 +1,6 @@
 using CliWrap;
 using CliWrap.Builders;
+using Lombiq.Tests.UI.Constants;
 using Lombiq.Tests.UI.Helpers;
 using Microsoft.VisualBasic.FileIO;
 using System;
@@ -54,6 +55,7 @@ public sealed class OrchardCoreInstance : IWebApplicationInstance
     private static readonly object _exeCopyLock = new();
 
     private readonly OrchardCoreConfiguration _configuration;
+    private readonly string _contextId;
     private readonly ITestOutputHelper _testOutputHelper;
     private Command _command;
     private CancellationTokenSource _cancellationTokenSource;
@@ -74,9 +76,10 @@ static OrchardCoreInstance()
         _portLeaseManager = new PortLeaseManager(9000 + agentIndexTimesHundred, 9099 + agentIndexTimesHundred);
     }
 
-    public OrchardCoreInstance(OrchardCoreConfiguration configuration, ITestOutputHelper testOutputHelper)
+    public OrchardCoreInstance(OrchardCoreConfiguration configuration, string contextId, ITestOutputHelper testOutputHelper)
     {
         _configuration = configuration;
+        _contextId = contextId;
         _testOutputHelper = testOutputHelper;
     }
 
@@ -215,7 +218,7 @@ public async ValueTask DisposeAsync()
 
     private void CreateContentRootFolder()
     {
-        _contentRootPath = Path.Combine(Environment.CurrentDirectory, Guid.NewGuid().ToString());
+        _contentRootPath = DirectoryPaths.GetTempSubDirectoryPath(_contextId, "App");
         Directory.CreateDirectory(_contentRootPath);
         _testOutputHelper.WriteLineTimestampedAndDebug("Content root path was created: {0}", _contentRootPath);
     }

Lombiq.Tests.UI/Services/OrchardCoreSetupConfiguration.cs

@@ -30,7 +30,7 @@ public class OrchardCoreSetupConfiguration
     /// </summary>
     public bool FastFailSetup { get; set; } = true;
 
-    public string SetupSnapshotDirectoryPath { get; set; } = Snapshots.DefaultSetupSnapshotDirectoryPath;
+    public string SetupSnapshotDirectoryPath { get; set; } = DirectoryPaths.SetupSnapshot;
 
     public BeforeSetupHandler BeforeSetup { get; set; }
 }

Lombiq.Tests.UI/Services/UITestContext.cs

@@ -16,6 +16,12 @@ public class UITestContext
 {
     private readonly List<BrowserLogMessage> _historicBrowserLog = new();
 
+    /// <summary>
+    /// Gets the globally unique ID of this context. You can use this ID to refer to the current text execution in
+    /// external systems, or in file names.
+    /// </summary>
+    public string Id { get; }
+
     /// <summary>
     /// Gets data about the currently executing test.
     /// </summary>
@@ -79,21 +85,21 @@ public class UITestContext
     public string TenantName { get; set; } = "Default";
 
     public UITestContext(
+        string id,
         UITestManifest testManifest,
         OrchardCoreUITestExecutorConfiguration configuration,
-        SqlServerRunningContext sqlServerContext,
         IWebApplicationInstance application,
         AtataScope scope,
-        SmtpServiceRunningContext smtpContext,
-        AzureBlobStorageRunningContext blobStorageContext)
+        RunningContextContainer runningContextContainer)
     {
+        Id = id;
         TestManifest = testManifest;
         Configuration = configuration;
-        SqlServerRunningContext = sqlServerContext;
+        SqlServerRunningContext = runningContextContainer.SqlServerRunningContext;
         Application = application;
         Scope = scope;
-        SmtpServiceRunningContext = smtpContext;
-        AzureBlobStorageRunningContext = blobStorageContext;
+        SmtpServiceRunningContext = runningContextContainer.SmtpServiceRunningContext;
+        AzureBlobStorageRunningContext = runningContextContainer.AzureBlobStorageRunningContext;
     }
 
     /// <summary>

Lombiq.Tests.UI/Services/UITestExecutionSession.cs

@@ -1,11 +1,13 @@
 using Atata.HtmlValidation;
 using CliWrap.Builders;
+using Lombiq.HelpfulLibraries.Common.Utilities;
 using Lombiq.Tests.UI.Constants;
 using Lombiq.Tests.UI.Exceptions;
 using Lombiq.Tests.UI.Extensions;
+using Lombiq.Tests.UI.Helpers;
 using Lombiq.Tests.UI.Models;
+using Microsoft.VisualBasic.FileIO;
 using Newtonsoft.Json.Linq;
-using OpenQA.Selenium;
 using Selenium.Axe;
 using System;
 using System.Collections.Concurrent;
@@ -25,7 +27,6 @@ internal sealed class UITestExecutionSession : IAsyncDisposable
     private readonly OrchardCoreUITestExecutorConfiguration _configuration;
     private readonly UITestExecutorFailureDumpConfiguration _dumpConfiguration;
     private readonly ITestOutputHelper _testOutputHelper;
-    private readonly List<Screenshot> _screenshots = new();
 
     // We need to have different snapshots based on whether the test uses the defaults, SQL Server and/or Azure Blob.
     private static readonly ConcurrentDictionary<string, SynchronizingWebApplicationSnapshotManager> _setupSnapshotManagers = new();
@@ -34,6 +35,8 @@ internal sealed class UITestExecutionSession : IAsyncDisposable
 
     private static bool _dockerIsSetup;
 
+    private int _screenshotCount;
+
     private SynchronizingWebApplicationSnapshotManager _currentSetupSnapshotManager;
     private string _snapshotDirectoryPath;
     private SqlServerManager _sqlServerManager;
@@ -130,6 +133,8 @@ public async Task<bool> ExecuteAsync(int retryCount, string dumpRootPath)
         }
         finally
         {
+            await ShutdownAsync();
+
             _testOutputHelper.WriteLineTimestampedAndDebug(
                 "Finishing execution of {0}, total time: {1}", _testManifest.Name, DateTime.UtcNow - startTime);
         }
@@ -147,13 +152,21 @@ private async ValueTask ShutdownAsync()
 
         if (_applicationInstance != null) await _applicationInstance.DisposeAsync();
 
+        if (_context != null)
+        {
+            _context.Scope?.Dispose();
+
+            DirectoryHelper.SafelyDeleteDirectoryIfExists(DirectoryPaths.GetTempSubDirectoryPath(_context.Id));
+        }
+
         _sqlServerManager?.Dispose();
-        _context?.Scope?.Dispose();
 
         if (_smtpService != null) await _smtpService.DisposeAsync();
         if (_azureBlobStorageManager != null) await _azureBlobStorageManager.DisposeAsync();
 
-        if (_dumpConfiguration.CaptureScreenshots) _screenshots.Clear();
+        _screenshotCount = 0;
+
+        _context = null;
     }
 
     private Exception PrepareAndLogException(Exception ex)
@@ -201,25 +214,7 @@ private async Task CreateFailureDumpAsync(Exception ex, string dumpRootPath, int
 
             // Saving the failure screenshot and HTML output should be as early after the test fail as possible so they
             // show an accurate state. Otherwise, e.g. the UI can change, resources can load in the meantime.
-            if (_dumpConfiguration.CaptureScreenshots)
-            {
-                await TakeScreenshotAsync(_context);
-
-                var pageScreenshotsPath = Path.Combine(debugInformationPath, "Screenshots");
-                Directory.CreateDirectory(pageScreenshotsPath);
-                var digitCount = _screenshots.Count.DigitCount();
-
-                string GetScreenshotPath(int index) =>
-                    Path.Combine(pageScreenshotsPath, index.PadZeroes(digitCount) + ".png");
-
-                for (int i = 0; i < _screenshots.Count; i++) _screenshots[i].SaveAsFile(GetScreenshotPath(i));
-
-                if (_configuration.ReportTeamCityMetadata)
-                {
-                    TeamCityMetadataReporter.ReportImage(
-                        _testManifest, "FailureScreenshot", GetScreenshotPath(_screenshots.Count - 1));
-                }
-            }
+            if (_dumpConfiguration.CaptureScreenshots) await CreateScreenshotsDumpAsync(debugInformationPath);
 
             if (_dumpConfiguration.CaptureHtmlSource)
             {
@@ -441,8 +436,8 @@ private void SetupDocker()
         // We add this subdirectory to ensure the HostSnapshotPath isn't set to the mounted volume's directory itself
         // (which would be logical). Removing the volume directory instantly severs the connection between host and the
         // container so that should be avoided at all costs.
-        docker.ContainerSnapshotPath += '/' + Snapshots.DefaultSetupSnapshotDirectoryPath; // Always a Unix path.
-        docker.HostSnapshotPath = Path.Combine(docker.HostSnapshotPath, Snapshots.DefaultSetupSnapshotDirectoryPath);
+        docker.ContainerSnapshotPath += '/' + DirectoryPaths.SetupSnapshot; // Always a Unix path.
+        docker.HostSnapshotPath = Path.Combine(docker.HostSnapshotPath, DirectoryPaths.SetupSnapshot);
 
         _dockerConfiguration = docker;
 
@@ -507,6 +502,10 @@ Task AzureBlobStorageManagerBeforeTakeSnapshotHandlerAsync(string contentRootPat
 
     private async Task<UITestContext> CreateContextAsync()
     {
+        var contextId = Guid.NewGuid().ToString();
+
+        FileSystemHelper.EnsureDirectoryExists(DirectoryPaths.GetTempSubDirectoryPath(contextId));
+
         SqlServerRunningContext sqlServerContext = null;
         AzureBlobStorageRunningContext azureBlobStorageContext = null;
         SmtpServiceRunningContext smtpContext = null;
@@ -533,7 +532,7 @@ Task UITestingBeforeAppStartHandlerAsync(string contentRootPath, ArgumentsBuilde
             _configuration.OrchardCoreConfiguration.BeforeAppStart.RemoveAll(UITestingBeforeAppStartHandlerAsync);
         _configuration.OrchardCoreConfiguration.BeforeAppStart += UITestingBeforeAppStartHandlerAsync;
 
-        _applicationInstance = new OrchardCoreInstance(_configuration.OrchardCoreConfiguration, _testOutputHelper);
+        _applicationInstance = new OrchardCoreInstance(_configuration.OrchardCoreConfiguration, contextId, _testOutputHelper);
         var uri = await _applicationInstance.StartUpAsync();
 
         _configuration.SetUpEvents();
@@ -556,8 +555,8 @@ Task UITestingBeforeAppStartHandlerAsync(string contentRootPath, ArgumentsBuilde
 
         if (_dumpConfiguration.CaptureScreenshots)
         {
-            _configuration.Events.AfterPageChange -= TakeScreenshotAsync;
-            _configuration.Events.AfterPageChange += TakeScreenshotAsync;
+            _configuration.Events.AfterPageChange -= TakeScreenshotIfEnabledAsync;
+            _configuration.Events.AfterPageChange += TakeScreenshotIfEnabledAsync;
         }
 
         var atataScope = AtataFactory.StartAtataScope(
@@ -566,13 +565,12 @@ Task UITestingBeforeAppStartHandlerAsync(string contentRootPath, ArgumentsBuilde
             _configuration);
 
         return new UITestContext(
+            contextId,
             _testManifest,
             _configuration,
-            sqlServerContext,
             _applicationInstance,
             atataScope,
-            smtpContext,
-            azureBlobStorageContext);
+            new RunningContextContainer(sqlServerContext, smtpContext, azureBlobStorageContext));
     }
 
     private string GetSetupHashCode() =>
@@ -703,9 +701,53 @@ Task SmtpServiceBeforeAppStartHandlerAsync(string contentRootPath, ArgumentsBuil
         return smtpContext;
     }
 
-    private Task TakeScreenshotAsync(UITestContext context)
+    private Task TakeScreenshotIfEnabledAsync(UITestContext context)
     {
-        _screenshots.Add(context.TakeScreenshot());
+        if (_context == null || !_dumpConfiguration.CaptureScreenshots) return Task.CompletedTask;
+
+        var screnshotsPath = DirectoryPaths.GetScreenshotsDirectoryPath(_context.Id);
+        FileSystemHelper.EnsureDirectoryExists(screnshotsPath);
+
+        try
+        {
+            context
+                .TakeScreenshot()
+                .SaveAsFile(GetScreenshotPath(screnshotsPath, _screenshotCount));
+        }
+        catch (FormatException ex) when (ex.Message.Contains("The input is not a valid Base-64 string"))
+        {
+            // Random "The input is not a valid Base-64 string as it contains a non-base 64 character, more than two
+            // padding characters, or an illegal character among the padding characters." exceptions can happen.
+
+            _testOutputHelper.WriteLineTimestampedAndDebug(
+                $"Taking the screenshot #{_screenshotCount.ToTechnicalString()} failed with the following exception: {ex}");
+        }
+
+        _screenshotCount++;
+
         return Task.CompletedTask;
     }
+
+    private async Task CreateScreenshotsDumpAsync(string debugInformationPath)
+    {
+        await TakeScreenshotIfEnabledAsync(_context);
+
+        var screenshotsSourcePath = DirectoryPaths.GetScreenshotsDirectoryPath(_context.Id);
+        if (Directory.Exists(screenshotsSourcePath))
+        {
+            var screenshotsDestinationPath = Path.Combine(debugInformationPath, DirectoryPaths.Screenshots);
+            FileSystem.CopyDirectory(screenshotsSourcePath, screenshotsDestinationPath);
+
+            if (_configuration.ReportTeamCityMetadata)
+            {
+                TeamCityMetadataReporter.ReportImage(
+                    _testManifest,
+                    "FailureScreenshot",
+                    GetScreenshotPath(screenshotsDestinationPath, _screenshotCount - 1));
+            }
+        }
+    }
+
+    private static string GetScreenshotPath(string parentDirectoryPath, int index) =>
+        Path.Combine(parentDirectoryPath, index.ToTechnicalString() + ".png");
 }