diff --git a/Lombiq.Tests.UI/Constants/DirectoryPaths.cs b/Lombiq.Tests.UI/Constants/DirectoryPaths.cs new file mode 100644 index 000000000..f73b6db7e --- /dev/null +++ b/Lombiq.Tests.UI/Constants/DirectoryPaths.cs @@ -0,0 +1,18 @@ +using System; +using System.IO; +using System.Linq; + +namespace Lombiq.Tests.UI.Constants; + +public static class DirectoryPaths +{ + public const string SetupSnapshot = nameof(SetupSnapshot); + public const string Temp = nameof(Temp); + public const string Screenshots = nameof(Screenshots); + + public static string GetTempSubDirectoryPath(string contextId, params string[] subDirectoryNames) => + Path.Combine(new[] { Environment.CurrentDirectory, Temp, contextId }.Concat(subDirectoryNames).ToArray()); + + public static string GetScreenshotsDirectoryPath(string contextId) => + GetTempSubDirectoryPath(contextId, Screenshots); +} diff --git a/Lombiq.Tests.UI/Constants/Snapshots.cs b/Lombiq.Tests.UI/Constants/Snapshots.cs deleted file mode 100644 index 69882c8d7..000000000 --- a/Lombiq.Tests.UI/Constants/Snapshots.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace Lombiq.Tests.UI.Constants; - -public static class Snapshots -{ - public const string DefaultSetupSnapshotDirectoryPath = "SetupSnapshot"; -} diff --git a/Lombiq.Tests.UI/Models/RunningContextContainer.cs b/Lombiq.Tests.UI/Models/RunningContextContainer.cs new file mode 100644 index 000000000..e5ff6f1b3 --- /dev/null +++ b/Lombiq.Tests.UI/Models/RunningContextContainer.cs @@ -0,0 +1,20 @@ +using Lombiq.Tests.UI.Services; + +namespace Lombiq.Tests.UI.Models; + +public class RunningContextContainer +{ + public SqlServerRunningContext SqlServerRunningContext { get; } + public SmtpServiceRunningContext SmtpServiceRunningContext { get; } + public AzureBlobStorageRunningContext AzureBlobStorageRunningContext { get; } + + public RunningContextContainer( + SqlServerRunningContext sqlServerContext, + SmtpServiceRunningContext smtpContext, + AzureBlobStorageRunningContext blobStorageContext) + { + SqlServerRunningContext = sqlServerContext; + SmtpServiceRunningContext = smtpContext; + AzureBlobStorageRunningContext = blobStorageContext; + } +} diff --git a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs index 26430e5d0..758ba3895 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs @@ -1,5 +1,6 @@ using CliWrap; using CliWrap.Builders; +using Lombiq.Tests.UI.Constants; using Lombiq.Tests.UI.Helpers; using Microsoft.VisualBasic.FileIO; using System; @@ -54,6 +55,7 @@ public sealed class OrchardCoreInstance : IWebApplicationInstance private static readonly object _exeCopyLock = new(); private readonly OrchardCoreConfiguration _configuration; + private readonly string _contextId; private readonly ITestOutputHelper _testOutputHelper; private Command _command; private CancellationTokenSource _cancellationTokenSource; @@ -74,9 +76,10 @@ static OrchardCoreInstance() _portLeaseManager = new PortLeaseManager(9000 + agentIndexTimesHundred, 9099 + agentIndexTimesHundred); } - public OrchardCoreInstance(OrchardCoreConfiguration configuration, ITestOutputHelper testOutputHelper) + public OrchardCoreInstance(OrchardCoreConfiguration configuration, string contextId, ITestOutputHelper testOutputHelper) { _configuration = configuration; + _contextId = contextId; _testOutputHelper = testOutputHelper; } @@ -215,7 +218,7 @@ public async ValueTask DisposeAsync() private void CreateContentRootFolder() { - _contentRootPath = Path.Combine(Environment.CurrentDirectory, Guid.NewGuid().ToString()); + _contentRootPath = DirectoryPaths.GetTempSubDirectoryPath(_contextId, "App"); Directory.CreateDirectory(_contentRootPath); _testOutputHelper.WriteLineTimestampedAndDebug("Content root path was created: {0}", _contentRootPath); } diff --git a/Lombiq.Tests.UI/Services/OrchardCoreSetupConfiguration.cs b/Lombiq.Tests.UI/Services/OrchardCoreSetupConfiguration.cs index 6d2da5249..f90a28164 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreSetupConfiguration.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreSetupConfiguration.cs @@ -30,7 +30,7 @@ public class OrchardCoreSetupConfiguration /// public bool FastFailSetup { get; set; } = true; - public string SetupSnapshotDirectoryPath { get; set; } = Snapshots.DefaultSetupSnapshotDirectoryPath; + public string SetupSnapshotDirectoryPath { get; set; } = DirectoryPaths.SetupSnapshot; public BeforeSetupHandler BeforeSetup { get; set; } } diff --git a/Lombiq.Tests.UI/Services/UITestContext.cs b/Lombiq.Tests.UI/Services/UITestContext.cs index 767ab7dda..68127905f 100644 --- a/Lombiq.Tests.UI/Services/UITestContext.cs +++ b/Lombiq.Tests.UI/Services/UITestContext.cs @@ -16,6 +16,12 @@ public class UITestContext { private readonly List _historicBrowserLog = new(); + /// + /// Gets the globally unique ID of this context. You can use this ID to refer to the current text execution in + /// external systems, or in file names. + /// + public string Id { get; } + /// /// Gets data about the currently executing test. /// @@ -79,21 +85,21 @@ public class UITestContext public string TenantName { get; set; } = "Default"; public UITestContext( + string id, UITestManifest testManifest, OrchardCoreUITestExecutorConfiguration configuration, - SqlServerRunningContext sqlServerContext, IWebApplicationInstance application, AtataScope scope, - SmtpServiceRunningContext smtpContext, - AzureBlobStorageRunningContext blobStorageContext) + RunningContextContainer runningContextContainer) { + Id = id; TestManifest = testManifest; Configuration = configuration; - SqlServerRunningContext = sqlServerContext; + SqlServerRunningContext = runningContextContainer.SqlServerRunningContext; Application = application; Scope = scope; - SmtpServiceRunningContext = smtpContext; - AzureBlobStorageRunningContext = blobStorageContext; + SmtpServiceRunningContext = runningContextContainer.SmtpServiceRunningContext; + AzureBlobStorageRunningContext = runningContextContainer.AzureBlobStorageRunningContext; } /// diff --git a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs index a6d1f81d2..4f81b4bc9 100644 --- a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs +++ b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs @@ -1,11 +1,13 @@ using Atata.HtmlValidation; using CliWrap.Builders; +using Lombiq.HelpfulLibraries.Common.Utilities; using Lombiq.Tests.UI.Constants; using Lombiq.Tests.UI.Exceptions; using Lombiq.Tests.UI.Extensions; +using Lombiq.Tests.UI.Helpers; using Lombiq.Tests.UI.Models; +using Microsoft.VisualBasic.FileIO; using Newtonsoft.Json.Linq; -using OpenQA.Selenium; using Selenium.Axe; using System; using System.Collections.Concurrent; @@ -25,7 +27,6 @@ internal sealed class UITestExecutionSession : IAsyncDisposable private readonly OrchardCoreUITestExecutorConfiguration _configuration; private readonly UITestExecutorFailureDumpConfiguration _dumpConfiguration; private readonly ITestOutputHelper _testOutputHelper; - private readonly List _screenshots = new(); // We need to have different snapshots based on whether the test uses the defaults, SQL Server and/or Azure Blob. private static readonly ConcurrentDictionary _setupSnapshotManagers = new(); @@ -34,6 +35,8 @@ internal sealed class UITestExecutionSession : IAsyncDisposable private static bool _dockerIsSetup; + private int _screenshotCount; + private SynchronizingWebApplicationSnapshotManager _currentSetupSnapshotManager; private string _snapshotDirectoryPath; private SqlServerManager _sqlServerManager; @@ -130,6 +133,8 @@ public async Task ExecuteAsync(int retryCount, string dumpRootPath) } finally { + await ShutdownAsync(); + _testOutputHelper.WriteLineTimestampedAndDebug( "Finishing execution of {0}, total time: {1}", _testManifest.Name, DateTime.UtcNow - startTime); } @@ -147,13 +152,21 @@ private async ValueTask ShutdownAsync() if (_applicationInstance != null) await _applicationInstance.DisposeAsync(); + if (_context != null) + { + _context.Scope?.Dispose(); + + DirectoryHelper.SafelyDeleteDirectoryIfExists(DirectoryPaths.GetTempSubDirectoryPath(_context.Id)); + } + _sqlServerManager?.Dispose(); - _context?.Scope?.Dispose(); if (_smtpService != null) await _smtpService.DisposeAsync(); if (_azureBlobStorageManager != null) await _azureBlobStorageManager.DisposeAsync(); - if (_dumpConfiguration.CaptureScreenshots) _screenshots.Clear(); + _screenshotCount = 0; + + _context = null; } private Exception PrepareAndLogException(Exception ex) @@ -201,25 +214,7 @@ private async Task CreateFailureDumpAsync(Exception ex, string dumpRootPath, int // Saving the failure screenshot and HTML output should be as early after the test fail as possible so they // show an accurate state. Otherwise, e.g. the UI can change, resources can load in the meantime. - if (_dumpConfiguration.CaptureScreenshots) - { - await TakeScreenshotAsync(_context); - - var pageScreenshotsPath = Path.Combine(debugInformationPath, "Screenshots"); - Directory.CreateDirectory(pageScreenshotsPath); - var digitCount = _screenshots.Count.DigitCount(); - - string GetScreenshotPath(int index) => - Path.Combine(pageScreenshotsPath, index.PadZeroes(digitCount) + ".png"); - - for (int i = 0; i < _screenshots.Count; i++) _screenshots[i].SaveAsFile(GetScreenshotPath(i)); - - if (_configuration.ReportTeamCityMetadata) - { - TeamCityMetadataReporter.ReportImage( - _testManifest, "FailureScreenshot", GetScreenshotPath(_screenshots.Count - 1)); - } - } + if (_dumpConfiguration.CaptureScreenshots) await CreateScreenshotsDumpAsync(debugInformationPath); if (_dumpConfiguration.CaptureHtmlSource) { @@ -441,8 +436,8 @@ private void SetupDocker() // We add this subdirectory to ensure the HostSnapshotPath isn't set to the mounted volume's directory itself // (which would be logical). Removing the volume directory instantly severs the connection between host and the // container so that should be avoided at all costs. - docker.ContainerSnapshotPath += '/' + Snapshots.DefaultSetupSnapshotDirectoryPath; // Always a Unix path. - docker.HostSnapshotPath = Path.Combine(docker.HostSnapshotPath, Snapshots.DefaultSetupSnapshotDirectoryPath); + docker.ContainerSnapshotPath += '/' + DirectoryPaths.SetupSnapshot; // Always a Unix path. + docker.HostSnapshotPath = Path.Combine(docker.HostSnapshotPath, DirectoryPaths.SetupSnapshot); _dockerConfiguration = docker; @@ -507,6 +502,10 @@ Task AzureBlobStorageManagerBeforeTakeSnapshotHandlerAsync(string contentRootPat private async Task CreateContextAsync() { + var contextId = Guid.NewGuid().ToString(); + + FileSystemHelper.EnsureDirectoryExists(DirectoryPaths.GetTempSubDirectoryPath(contextId)); + SqlServerRunningContext sqlServerContext = null; AzureBlobStorageRunningContext azureBlobStorageContext = null; SmtpServiceRunningContext smtpContext = null; @@ -533,7 +532,7 @@ Task UITestingBeforeAppStartHandlerAsync(string contentRootPath, ArgumentsBuilde _configuration.OrchardCoreConfiguration.BeforeAppStart.RemoveAll(UITestingBeforeAppStartHandlerAsync); _configuration.OrchardCoreConfiguration.BeforeAppStart += UITestingBeforeAppStartHandlerAsync; - _applicationInstance = new OrchardCoreInstance(_configuration.OrchardCoreConfiguration, _testOutputHelper); + _applicationInstance = new OrchardCoreInstance(_configuration.OrchardCoreConfiguration, contextId, _testOutputHelper); var uri = await _applicationInstance.StartUpAsync(); _configuration.SetUpEvents(); @@ -556,8 +555,8 @@ Task UITestingBeforeAppStartHandlerAsync(string contentRootPath, ArgumentsBuilde if (_dumpConfiguration.CaptureScreenshots) { - _configuration.Events.AfterPageChange -= TakeScreenshotAsync; - _configuration.Events.AfterPageChange += TakeScreenshotAsync; + _configuration.Events.AfterPageChange -= TakeScreenshotIfEnabledAsync; + _configuration.Events.AfterPageChange += TakeScreenshotIfEnabledAsync; } var atataScope = AtataFactory.StartAtataScope( @@ -566,13 +565,12 @@ Task UITestingBeforeAppStartHandlerAsync(string contentRootPath, ArgumentsBuilde _configuration); return new UITestContext( + contextId, _testManifest, _configuration, - sqlServerContext, _applicationInstance, atataScope, - smtpContext, - azureBlobStorageContext); + new RunningContextContainer(sqlServerContext, smtpContext, azureBlobStorageContext)); } private string GetSetupHashCode() => @@ -703,9 +701,53 @@ Task SmtpServiceBeforeAppStartHandlerAsync(string contentRootPath, ArgumentsBuil return smtpContext; } - private Task TakeScreenshotAsync(UITestContext context) + private Task TakeScreenshotIfEnabledAsync(UITestContext context) { - _screenshots.Add(context.TakeScreenshot()); + if (_context == null || !_dumpConfiguration.CaptureScreenshots) return Task.CompletedTask; + + var screnshotsPath = DirectoryPaths.GetScreenshotsDirectoryPath(_context.Id); + FileSystemHelper.EnsureDirectoryExists(screnshotsPath); + + try + { + context + .TakeScreenshot() + .SaveAsFile(GetScreenshotPath(screnshotsPath, _screenshotCount)); + } + catch (FormatException ex) when (ex.Message.Contains("The input is not a valid Base-64 string")) + { + // Random "The input is not a valid Base-64 string as it contains a non-base 64 character, more than two + // padding characters, or an illegal character among the padding characters." exceptions can happen. + + _testOutputHelper.WriteLineTimestampedAndDebug( + $"Taking the screenshot #{_screenshotCount.ToTechnicalString()} failed with the following exception: {ex}"); + } + + _screenshotCount++; + return Task.CompletedTask; } + + private async Task CreateScreenshotsDumpAsync(string debugInformationPath) + { + await TakeScreenshotIfEnabledAsync(_context); + + var screenshotsSourcePath = DirectoryPaths.GetScreenshotsDirectoryPath(_context.Id); + if (Directory.Exists(screenshotsSourcePath)) + { + var screenshotsDestinationPath = Path.Combine(debugInformationPath, DirectoryPaths.Screenshots); + FileSystem.CopyDirectory(screenshotsSourcePath, screenshotsDestinationPath); + + if (_configuration.ReportTeamCityMetadata) + { + TeamCityMetadataReporter.ReportImage( + _testManifest, + "FailureScreenshot", + GetScreenshotPath(screenshotsDestinationPath, _screenshotCount - 1)); + } + } + } + + private static string GetScreenshotPath(string parentDirectoryPath, int index) => + Path.Combine(parentDirectoryPath, index.ToTechnicalString() + ".png"); }