From c0f9ce6c38e0f7d0bc3f192eda137003aa94804f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Thu, 7 Apr 2022 20:17:57 +0200 Subject: [PATCH 01/12] Running the OC app from a dedicated Temp folder --- Lombiq.Tests.UI/Constants/{Snapshots.cs => Paths.cs} | 3 ++- Lombiq.Tests.UI/Services/OrchardCoreInstance.cs | 3 ++- Lombiq.Tests.UI/Services/OrchardCoreSetupConfiguration.cs | 2 +- Lombiq.Tests.UI/Services/UITestExecutionSession.cs | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) rename Lombiq.Tests.UI/Constants/{Snapshots.cs => Paths.cs} (61%) diff --git a/Lombiq.Tests.UI/Constants/Snapshots.cs b/Lombiq.Tests.UI/Constants/Paths.cs similarity index 61% rename from Lombiq.Tests.UI/Constants/Snapshots.cs rename to Lombiq.Tests.UI/Constants/Paths.cs index 69882c8d7..0a9e0763e 100644 --- a/Lombiq.Tests.UI/Constants/Snapshots.cs +++ b/Lombiq.Tests.UI/Constants/Paths.cs @@ -1,6 +1,7 @@ namespace Lombiq.Tests.UI.Constants; -public static class Snapshots +public static class Paths { public const string DefaultSetupSnapshotDirectoryPath = "SetupSnapshot"; + public const string TempFolderPath = "Temp"; } diff --git a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs index 26430e5d0..a4f92d9c4 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs @@ -1,5 +1,6 @@ using CliWrap; using CliWrap.Builders; +using Lombiq.Tests.UI.Constants; using Lombiq.Tests.UI.Helpers; using Microsoft.VisualBasic.FileIO; using System; @@ -215,7 +216,7 @@ public async ValueTask DisposeAsync() private void CreateContentRootFolder() { - _contentRootPath = Path.Combine(Environment.CurrentDirectory, Guid.NewGuid().ToString()); + _contentRootPath = Path.Combine(Environment.CurrentDirectory, Paths.TempFolderPath, Guid.NewGuid().ToString()); Directory.CreateDirectory(_contentRootPath); _testOutputHelper.WriteLineTimestampedAndDebug("Content root path was created: {0}", _contentRootPath); } diff --git a/Lombiq.Tests.UI/Services/OrchardCoreSetupConfiguration.cs b/Lombiq.Tests.UI/Services/OrchardCoreSetupConfiguration.cs index 6d2da5249..534b61028 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreSetupConfiguration.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreSetupConfiguration.cs @@ -30,7 +30,7 @@ public class OrchardCoreSetupConfiguration /// public bool FastFailSetup { get; set; } = true; - public string SetupSnapshotDirectoryPath { get; set; } = Snapshots.DefaultSetupSnapshotDirectoryPath; + public string SetupSnapshotDirectoryPath { get; set; } = Paths.DefaultSetupSnapshotDirectoryPath; public BeforeSetupHandler BeforeSetup { get; set; } } diff --git a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs index a6d1f81d2..103df54fc 100644 --- a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs +++ b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs @@ -441,8 +441,8 @@ private void SetupDocker() // We add this subdirectory to ensure the HostSnapshotPath isn't set to the mounted volume's directory itself // (which would be logical). Removing the volume directory instantly severs the connection between host and the // container so that should be avoided at all costs. - docker.ContainerSnapshotPath += '/' + Snapshots.DefaultSetupSnapshotDirectoryPath; // Always a Unix path. - docker.HostSnapshotPath = Path.Combine(docker.HostSnapshotPath, Snapshots.DefaultSetupSnapshotDirectoryPath); + docker.ContainerSnapshotPath += '/' + Paths.DefaultSetupSnapshotDirectoryPath; // Always a Unix path. + docker.HostSnapshotPath = Path.Combine(docker.HostSnapshotPath, Paths.DefaultSetupSnapshotDirectoryPath); _dockerConfiguration = docker; From b5b652b912c39ceac97bdb3854cff78dc06383c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Thu, 7 Apr 2022 20:55:25 +0200 Subject: [PATCH 02/12] Central subfolder management for temp folders --- Lombiq.Tests.UI/Constants/Paths.cs | 6 ++++++ .../Models/RunningContextContainer.cs | 20 +++++++++++++++++++ .../Services/OrchardCoreInstance.cs | 6 ++++-- Lombiq.Tests.UI/Services/UITestContext.cs | 18 +++++++++++------ .../Services/UITestExecutionSession.cs | 18 +++++++++++++---- 5 files changed, 56 insertions(+), 12 deletions(-) create mode 100644 Lombiq.Tests.UI/Models/RunningContextContainer.cs diff --git a/Lombiq.Tests.UI/Constants/Paths.cs b/Lombiq.Tests.UI/Constants/Paths.cs index 0a9e0763e..35d26c5af 100644 --- a/Lombiq.Tests.UI/Constants/Paths.cs +++ b/Lombiq.Tests.UI/Constants/Paths.cs @@ -1,7 +1,13 @@ +using System; +using System.IO; + namespace Lombiq.Tests.UI.Constants; public static class Paths { public const string DefaultSetupSnapshotDirectoryPath = "SetupSnapshot"; public const string TempFolderPath = "Temp"; + + public static string GetTempSubDirectoryPath(string contextId, string subDirectoryName) => + Path.Combine(Environment.CurrentDirectory, TempFolderPath, contextId, subDirectoryName); } diff --git a/Lombiq.Tests.UI/Models/RunningContextContainer.cs b/Lombiq.Tests.UI/Models/RunningContextContainer.cs new file mode 100644 index 000000000..e5ff6f1b3 --- /dev/null +++ b/Lombiq.Tests.UI/Models/RunningContextContainer.cs @@ -0,0 +1,20 @@ +using Lombiq.Tests.UI.Services; + +namespace Lombiq.Tests.UI.Models; + +public class RunningContextContainer +{ + public SqlServerRunningContext SqlServerRunningContext { get; } + public SmtpServiceRunningContext SmtpServiceRunningContext { get; } + public AzureBlobStorageRunningContext AzureBlobStorageRunningContext { get; } + + public RunningContextContainer( + SqlServerRunningContext sqlServerContext, + SmtpServiceRunningContext smtpContext, + AzureBlobStorageRunningContext blobStorageContext) + { + SqlServerRunningContext = sqlServerContext; + SmtpServiceRunningContext = smtpContext; + AzureBlobStorageRunningContext = blobStorageContext; + } +} diff --git a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs index a4f92d9c4..53a1c50b7 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs @@ -55,6 +55,7 @@ public sealed class OrchardCoreInstance : IWebApplicationInstance private static readonly object _exeCopyLock = new(); private readonly OrchardCoreConfiguration _configuration; + private readonly string _contextId; private readonly ITestOutputHelper _testOutputHelper; private Command _command; private CancellationTokenSource _cancellationTokenSource; @@ -75,9 +76,10 @@ static OrchardCoreInstance() _portLeaseManager = new PortLeaseManager(9000 + agentIndexTimesHundred, 9099 + agentIndexTimesHundred); } - public OrchardCoreInstance(OrchardCoreConfiguration configuration, ITestOutputHelper testOutputHelper) + public OrchardCoreInstance(OrchardCoreConfiguration configuration, string contextId, ITestOutputHelper testOutputHelper) { _configuration = configuration; + _contextId = contextId; _testOutputHelper = testOutputHelper; } @@ -216,7 +218,7 @@ public async ValueTask DisposeAsync() private void CreateContentRootFolder() { - _contentRootPath = Path.Combine(Environment.CurrentDirectory, Paths.TempFolderPath, Guid.NewGuid().ToString()); + _contentRootPath = Paths.GetTempSubDirectoryPath(_contextId, "App"); Directory.CreateDirectory(_contentRootPath); _testOutputHelper.WriteLineTimestampedAndDebug("Content root path was created: {0}", _contentRootPath); } diff --git a/Lombiq.Tests.UI/Services/UITestContext.cs b/Lombiq.Tests.UI/Services/UITestContext.cs index 767ab7dda..68127905f 100644 --- a/Lombiq.Tests.UI/Services/UITestContext.cs +++ b/Lombiq.Tests.UI/Services/UITestContext.cs @@ -16,6 +16,12 @@ public class UITestContext { private readonly List _historicBrowserLog = new(); + /// + /// Gets the globally unique ID of this context. You can use this ID to refer to the current text execution in + /// external systems, or in file names. + /// + public string Id { get; } + /// /// Gets data about the currently executing test. /// @@ -79,21 +85,21 @@ public class UITestContext public string TenantName { get; set; } = "Default"; public UITestContext( + string id, UITestManifest testManifest, OrchardCoreUITestExecutorConfiguration configuration, - SqlServerRunningContext sqlServerContext, IWebApplicationInstance application, AtataScope scope, - SmtpServiceRunningContext smtpContext, - AzureBlobStorageRunningContext blobStorageContext) + RunningContextContainer runningContextContainer) { + Id = id; TestManifest = testManifest; Configuration = configuration; - SqlServerRunningContext = sqlServerContext; + SqlServerRunningContext = runningContextContainer.SqlServerRunningContext; Application = application; Scope = scope; - SmtpServiceRunningContext = smtpContext; - AzureBlobStorageRunningContext = blobStorageContext; + SmtpServiceRunningContext = runningContextContainer.SmtpServiceRunningContext; + AzureBlobStorageRunningContext = runningContextContainer.AzureBlobStorageRunningContext; } /// diff --git a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs index 103df54fc..332ed01dc 100644 --- a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs +++ b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs @@ -1,8 +1,10 @@ using Atata.HtmlValidation; using CliWrap.Builders; +using Lombiq.HelpfulLibraries.Common.Utilities; using Lombiq.Tests.UI.Constants; using Lombiq.Tests.UI.Exceptions; using Lombiq.Tests.UI.Extensions; +using Lombiq.Tests.UI.Helpers; using Lombiq.Tests.UI.Models; using Newtonsoft.Json.Linq; using OpenQA.Selenium; @@ -154,6 +156,11 @@ private async ValueTask ShutdownAsync() if (_azureBlobStorageManager != null) await _azureBlobStorageManager.DisposeAsync(); if (_dumpConfiguration.CaptureScreenshots) _screenshots.Clear(); + + if (_context != null) + { + DirectoryHelper.SafelyDeleteDirectoryIfExists(Paths.GetTempSubDirectoryPath(_context.Id, string.Empty)); + } } private Exception PrepareAndLogException(Exception ex) @@ -507,6 +514,10 @@ Task AzureBlobStorageManagerBeforeTakeSnapshotHandlerAsync(string contentRootPat private async Task CreateContextAsync() { + var contextId = Guid.NewGuid().ToString(); + + FileSystemHelper.EnsureDirectoryExists(Paths.GetTempSubDirectoryPath(contextId, string.Empty)); + SqlServerRunningContext sqlServerContext = null; AzureBlobStorageRunningContext azureBlobStorageContext = null; SmtpServiceRunningContext smtpContext = null; @@ -533,7 +544,7 @@ Task UITestingBeforeAppStartHandlerAsync(string contentRootPath, ArgumentsBuilde _configuration.OrchardCoreConfiguration.BeforeAppStart.RemoveAll(UITestingBeforeAppStartHandlerAsync); _configuration.OrchardCoreConfiguration.BeforeAppStart += UITestingBeforeAppStartHandlerAsync; - _applicationInstance = new OrchardCoreInstance(_configuration.OrchardCoreConfiguration, _testOutputHelper); + _applicationInstance = new OrchardCoreInstance(_configuration.OrchardCoreConfiguration, contextId, _testOutputHelper); var uri = await _applicationInstance.StartUpAsync(); _configuration.SetUpEvents(); @@ -566,13 +577,12 @@ Task UITestingBeforeAppStartHandlerAsync(string contentRootPath, ArgumentsBuilde _configuration); return new UITestContext( + contextId, _testManifest, _configuration, - sqlServerContext, _applicationInstance, atataScope, - smtpContext, - azureBlobStorageContext); + new RunningContextContainer(sqlServerContext, smtpContext, azureBlobStorageContext)); } private string GetSetupHashCode() => From a6278cc3e7201007aebd420cc7f2e63471f6da11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Thu, 7 Apr 2022 22:35:38 +0200 Subject: [PATCH 03/12] Saving screenshots immediately as files --- .../Services/UITestExecutionSession.cs | 41 +++++++++++-------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs index 332ed01dc..7244e1aaa 100644 --- a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs +++ b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs @@ -6,8 +6,8 @@ using Lombiq.Tests.UI.Extensions; using Lombiq.Tests.UI.Helpers; using Lombiq.Tests.UI.Models; +using Microsoft.VisualBasic.FileIO; using Newtonsoft.Json.Linq; -using OpenQA.Selenium; using Selenium.Axe; using System; using System.Collections.Concurrent; @@ -27,7 +27,6 @@ internal sealed class UITestExecutionSession : IAsyncDisposable private readonly OrchardCoreUITestExecutorConfiguration _configuration; private readonly UITestExecutorFailureDumpConfiguration _dumpConfiguration; private readonly ITestOutputHelper _testOutputHelper; - private readonly List _screenshots = new(); // We need to have different snapshots based on whether the test uses the defaults, SQL Server and/or Azure Blob. private static readonly ConcurrentDictionary _setupSnapshotManagers = new(); @@ -36,6 +35,8 @@ internal sealed class UITestExecutionSession : IAsyncDisposable private static bool _dockerIsSetup; + private int _screenshotCount; + private SynchronizingWebApplicationSnapshotManager _currentSetupSnapshotManager; private string _snapshotDirectoryPath; private SqlServerManager _sqlServerManager; @@ -155,7 +156,7 @@ private async ValueTask ShutdownAsync() if (_smtpService != null) await _smtpService.DisposeAsync(); if (_azureBlobStorageManager != null) await _azureBlobStorageManager.DisposeAsync(); - if (_dumpConfiguration.CaptureScreenshots) _screenshots.Clear(); + _screenshotCount = 0; if (_context != null) { @@ -212,19 +213,19 @@ private async Task CreateFailureDumpAsync(Exception ex, string dumpRootPath, int { await TakeScreenshotAsync(_context); - var pageScreenshotsPath = Path.Combine(debugInformationPath, "Screenshots"); - Directory.CreateDirectory(pageScreenshotsPath); - var digitCount = _screenshots.Count.DigitCount(); - - string GetScreenshotPath(int index) => - Path.Combine(pageScreenshotsPath, index.PadZeroes(digitCount) + ".png"); - - for (int i = 0; i < _screenshots.Count; i++) _screenshots[i].SaveAsFile(GetScreenshotPath(i)); - - if (_configuration.ReportTeamCityMetadata) + var screenshotsSourcePath = Paths.GetTempSubDirectoryPath(_context.Id, "Screenshots"); + if (Directory.Exists(screenshotsSourcePath)) { - TeamCityMetadataReporter.ReportImage( - _testManifest, "FailureScreenshot", GetScreenshotPath(_screenshots.Count - 1)); + var screenshotsDestinationPath = Path.Combine(debugInformationPath, "Screenshots"); + FileSystem.CopyDirectory(screenshotsSourcePath, screenshotsDestinationPath); + + if (_configuration.ReportTeamCityMetadata) + { + TeamCityMetadataReporter.ReportImage( + _testManifest, + "FailureScreenshot", + Path.Combine(screenshotsDestinationPath, (_screenshotCount - 1).ToTechnicalString() + ".png")); + } } } @@ -715,7 +716,15 @@ Task SmtpServiceBeforeAppStartHandlerAsync(string contentRootPath, ArgumentsBuil private Task TakeScreenshotAsync(UITestContext context) { - _screenshots.Add(context.TakeScreenshot()); + var screnshotsPath = Paths.GetTempSubDirectoryPath(context.Id, "Screenshots"); + FileSystemHelper.EnsureDirectoryExists(screnshotsPath); + + context + .TakeScreenshot() + .SaveAsFile(Path.Combine(screnshotsPath, _screenshotCount.ToTechnicalString() + ".png")); + + _screenshotCount++; + return Task.CompletedTask; } } From 69cac8f08d237245c369cbc96bd25ce676ed8b07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Thu, 7 Apr 2022 22:41:29 +0200 Subject: [PATCH 04/12] Refactoring --- .../Services/UITestExecutionSession.cs | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs index 7244e1aaa..7dc40bb11 100644 --- a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs +++ b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs @@ -209,25 +209,7 @@ private async Task CreateFailureDumpAsync(Exception ex, string dumpRootPath, int // Saving the failure screenshot and HTML output should be as early after the test fail as possible so they // show an accurate state. Otherwise, e.g. the UI can change, resources can load in the meantime. - if (_dumpConfiguration.CaptureScreenshots) - { - await TakeScreenshotAsync(_context); - - var screenshotsSourcePath = Paths.GetTempSubDirectoryPath(_context.Id, "Screenshots"); - if (Directory.Exists(screenshotsSourcePath)) - { - var screenshotsDestinationPath = Path.Combine(debugInformationPath, "Screenshots"); - FileSystem.CopyDirectory(screenshotsSourcePath, screenshotsDestinationPath); - - if (_configuration.ReportTeamCityMetadata) - { - TeamCityMetadataReporter.ReportImage( - _testManifest, - "FailureScreenshot", - Path.Combine(screenshotsDestinationPath, (_screenshotCount - 1).ToTechnicalString() + ".png")); - } - } - } + if (_dumpConfiguration.CaptureScreenshots) await CreateScreenshotsDumpAsync(debugInformationPath); if (_dumpConfiguration.CaptureHtmlSource) { @@ -727,4 +709,24 @@ private Task TakeScreenshotAsync(UITestContext context) return Task.CompletedTask; } + + private async Task CreateScreenshotsDumpAsync(string debugInformationPath) + { + await TakeScreenshotAsync(_context); + + var screenshotsSourcePath = Paths.GetTempSubDirectoryPath(_context.Id, "Screenshots"); + if (Directory.Exists(screenshotsSourcePath)) + { + var screenshotsDestinationPath = Path.Combine(debugInformationPath, "Screenshots"); + FileSystem.CopyDirectory(screenshotsSourcePath, screenshotsDestinationPath); + + if (_configuration.ReportTeamCityMetadata) + { + TeamCityMetadataReporter.ReportImage( + _testManifest, + "FailureScreenshot", + Path.Combine(screenshotsDestinationPath, (_screenshotCount - 1).ToTechnicalString() + ".png")); + } + } + } } From a053514fa9dfbfa7e146259d1a70cf6b0a09b359 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Thu, 7 Apr 2022 22:59:32 +0200 Subject: [PATCH 05/12] Deliberately failing test --- Lombiq.Tests.UI.Samples/Tests/BasicTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI.Samples/Tests/BasicTests.cs b/Lombiq.Tests.UI.Samples/Tests/BasicTests.cs index 1acbbdf84..76d50a067 100644 --- a/Lombiq.Tests.UI.Samples/Tests/BasicTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/BasicTests.cs @@ -30,7 +30,7 @@ public Task AnonymousHomePageShouldExist(Browser browser) => { // Is the title correct? context - .Get(By.ClassName("navbar-brand")) + .Get(By.ClassName("navbar-brandadf")) .Text .ShouldBe("Lombiq's OSOCE - UI Testing"); From d5fc2b803e7d831b68b30261faf3d76aa315c643 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Thu, 7 Apr 2022 23:31:35 +0200 Subject: [PATCH 06/12] Refactoring, fixing that not all UI testing processes were disposed if tests failed --- Lombiq.Tests.UI/Constants/Paths.cs | 8 +++- .../Services/UITestExecutionSession.cs | 38 ++++++++++++------- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/Lombiq.Tests.UI/Constants/Paths.cs b/Lombiq.Tests.UI/Constants/Paths.cs index 35d26c5af..0590eb988 100644 --- a/Lombiq.Tests.UI/Constants/Paths.cs +++ b/Lombiq.Tests.UI/Constants/Paths.cs @@ -6,8 +6,12 @@ namespace Lombiq.Tests.UI.Constants; public static class Paths { public const string DefaultSetupSnapshotDirectoryPath = "SetupSnapshot"; - public const string TempFolderPath = "Temp"; + public const string TempDirectoryPath = "Temp"; + public const string ScreenshotsDirectoryName = "Screenshots"; public static string GetTempSubDirectoryPath(string contextId, string subDirectoryName) => - Path.Combine(Environment.CurrentDirectory, TempFolderPath, contextId, subDirectoryName); + Path.Combine(Environment.CurrentDirectory, TempDirectoryPath, contextId, subDirectoryName); + + public static string GetScreenshotsDirectoryPath(string contextId) => + GetTempSubDirectoryPath(contextId, ScreenshotsDirectoryName); } diff --git a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs index 7dc40bb11..dbc11e41a 100644 --- a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs +++ b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs @@ -133,6 +133,8 @@ public async Task ExecuteAsync(int retryCount, string dumpRootPath) } finally { + await ShutdownAsync(); + _testOutputHelper.WriteLineTimestampedAndDebug( "Finishing execution of {0}, total time: {1}", _testManifest.Name, DateTime.UtcNow - startTime); } @@ -150,18 +152,21 @@ private async ValueTask ShutdownAsync() if (_applicationInstance != null) await _applicationInstance.DisposeAsync(); + if (_context != null) + { + _context.Scope?.Dispose(); + + DirectoryHelper.SafelyDeleteDirectoryIfExists(Paths.GetTempSubDirectoryPath(_context.Id, string.Empty)); + } + _sqlServerManager?.Dispose(); - _context?.Scope?.Dispose(); if (_smtpService != null) await _smtpService.DisposeAsync(); if (_azureBlobStorageManager != null) await _azureBlobStorageManager.DisposeAsync(); _screenshotCount = 0; - if (_context != null) - { - DirectoryHelper.SafelyDeleteDirectoryIfExists(Paths.GetTempSubDirectoryPath(_context.Id, string.Empty)); - } + _context = null; } private Exception PrepareAndLogException(Exception ex) @@ -550,8 +555,8 @@ Task UITestingBeforeAppStartHandlerAsync(string contentRootPath, ArgumentsBuilde if (_dumpConfiguration.CaptureScreenshots) { - _configuration.Events.AfterPageChange -= TakeScreenshotAsync; - _configuration.Events.AfterPageChange += TakeScreenshotAsync; + _configuration.Events.AfterPageChange -= TakeScreenshotIfEnabledAsync; + _configuration.Events.AfterPageChange += TakeScreenshotIfEnabledAsync; } var atataScope = AtataFactory.StartAtataScope( @@ -696,14 +701,16 @@ Task SmtpServiceBeforeAppStartHandlerAsync(string contentRootPath, ArgumentsBuil return smtpContext; } - private Task TakeScreenshotAsync(UITestContext context) + private Task TakeScreenshotIfEnabledAsync(UITestContext context) { - var screnshotsPath = Paths.GetTempSubDirectoryPath(context.Id, "Screenshots"); + if (!_dumpConfiguration.CaptureScreenshots) return Task.CompletedTask; + + var screnshotsPath = Paths.GetScreenshotsDirectoryPath(_context.Id); FileSystemHelper.EnsureDirectoryExists(screnshotsPath); context .TakeScreenshot() - .SaveAsFile(Path.Combine(screnshotsPath, _screenshotCount.ToTechnicalString() + ".png")); + .SaveAsFile(GetScreenshotPath(screnshotsPath, _screenshotCount)); _screenshotCount++; @@ -712,12 +719,12 @@ private Task TakeScreenshotAsync(UITestContext context) private async Task CreateScreenshotsDumpAsync(string debugInformationPath) { - await TakeScreenshotAsync(_context); + await TakeScreenshotIfEnabledAsync(_context); - var screenshotsSourcePath = Paths.GetTempSubDirectoryPath(_context.Id, "Screenshots"); + var screenshotsSourcePath = Paths.GetScreenshotsDirectoryPath(_context.Id); if (Directory.Exists(screenshotsSourcePath)) { - var screenshotsDestinationPath = Path.Combine(debugInformationPath, "Screenshots"); + var screenshotsDestinationPath = Path.Combine(debugInformationPath, Paths.ScreenshotsDirectoryName); FileSystem.CopyDirectory(screenshotsSourcePath, screenshotsDestinationPath); if (_configuration.ReportTeamCityMetadata) @@ -725,8 +732,11 @@ private async Task CreateScreenshotsDumpAsync(string debugInformationPath) TeamCityMetadataReporter.ReportImage( _testManifest, "FailureScreenshot", - Path.Combine(screenshotsDestinationPath, (_screenshotCount - 1).ToTechnicalString() + ".png")); + GetScreenshotPath(screenshotsDestinationPath, _screenshotCount - 1)); } } } + + private static string GetScreenshotPath(string parentDirectoryPath, int index) => + Path.Combine(parentDirectoryPath, index.ToTechnicalString() + ".png"); } From ce171acfc6fba54e2d516c06b4db2b962e4f32dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Thu, 7 Apr 2022 23:54:48 +0200 Subject: [PATCH 07/12] Fixing NRE --- Lombiq.Tests.UI/Services/UITestExecutionSession.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs index dbc11e41a..35d510e18 100644 --- a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs +++ b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs @@ -703,7 +703,7 @@ Task SmtpServiceBeforeAppStartHandlerAsync(string contentRootPath, ArgumentsBuil private Task TakeScreenshotIfEnabledAsync(UITestContext context) { - if (!_dumpConfiguration.CaptureScreenshots) return Task.CompletedTask; + if (_context == null || !_dumpConfiguration.CaptureScreenshots) return Task.CompletedTask; var screnshotsPath = Paths.GetScreenshotsDirectoryPath(_context.Id); FileSystemHelper.EnsureDirectoryExists(screnshotsPath); From 99d50472a7546815db1877ab8952372bf4b34d7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Fri, 8 Apr 2022 00:18:12 +0200 Subject: [PATCH 08/12] Making page screenshots safer to take --- .../Services/UITestExecutionSession.cs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs index 35d510e18..cdadcdbc2 100644 --- a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs +++ b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs @@ -708,9 +708,20 @@ private Task TakeScreenshotIfEnabledAsync(UITestContext context) var screnshotsPath = Paths.GetScreenshotsDirectoryPath(_context.Id); FileSystemHelper.EnsureDirectoryExists(screnshotsPath); - context - .TakeScreenshot() - .SaveAsFile(GetScreenshotPath(screnshotsPath, _screenshotCount)); + try + { + context + .TakeScreenshot() + .SaveAsFile(GetScreenshotPath(screnshotsPath, _screenshotCount)); + } + catch (FormatException ex) when (ex.Message.Contains("The input is not a valid Base-64 string")) + { + // Random "The input is not a valid Base-64 string as it contains a non-base 64 character, more than two + // padding characters, or an illegal character among the padding characters." exceptions can happen. + + _testOutputHelper.WriteLineTimestampedAndDebug( + $"Taking the screenshot #{_screenshotCount.ToTechnicalString()} failed with the following exception: {ex}"); + } _screenshotCount++; From 7809466972f968afc3d5df2079f0bd1842b9a8ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Fri, 8 Apr 2022 00:18:20 +0200 Subject: [PATCH 09/12] Revert "Deliberately failing test" This reverts commit a053514fa9dfbfa7e146259d1a70cf6b0a09b359. --- Lombiq.Tests.UI.Samples/Tests/BasicTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI.Samples/Tests/BasicTests.cs b/Lombiq.Tests.UI.Samples/Tests/BasicTests.cs index 76d50a067..1acbbdf84 100644 --- a/Lombiq.Tests.UI.Samples/Tests/BasicTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/BasicTests.cs @@ -30,7 +30,7 @@ public Task AnonymousHomePageShouldExist(Browser browser) => { // Is the title correct? context - .Get(By.ClassName("navbar-brandadf")) + .Get(By.ClassName("navbar-brand")) .Text .ShouldBe("Lombiq's OSOCE - UI Testing"); From fac99d7dffbbd73516e95cf8a4fc91b64b876729 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Fri, 8 Apr 2022 16:05:47 +0200 Subject: [PATCH 10/12] Renaming path management members --- Lombiq.Tests.UI/Constants/DirectoryPaths.cs | 17 +++++++++++++++++ Lombiq.Tests.UI/Constants/Paths.cs | 17 ----------------- Lombiq.Tests.UI/Services/OrchardCoreInstance.cs | 2 +- .../Services/OrchardCoreSetupConfiguration.cs | 2 +- .../Services/UITestExecutionSession.cs | 14 +++++++------- 5 files changed, 26 insertions(+), 26 deletions(-) create mode 100644 Lombiq.Tests.UI/Constants/DirectoryPaths.cs delete mode 100644 Lombiq.Tests.UI/Constants/Paths.cs diff --git a/Lombiq.Tests.UI/Constants/DirectoryPaths.cs b/Lombiq.Tests.UI/Constants/DirectoryPaths.cs new file mode 100644 index 000000000..24b6844c6 --- /dev/null +++ b/Lombiq.Tests.UI/Constants/DirectoryPaths.cs @@ -0,0 +1,17 @@ +using System; +using System.IO; + +namespace Lombiq.Tests.UI.Constants; + +public static class DirectoryPaths +{ + public const string SetupSnapshot = nameof(SetupSnapshot); + public const string Temp = nameof(Temp); + public const string Screenshots = nameof(Screenshots); + + public static string GetTempSubDirectoryPath(string contextId, string subDirectoryName) => + Path.Combine(Environment.CurrentDirectory, Temp, contextId, subDirectoryName); + + public static string GetScreenshotsDirectoryPath(string contextId) => + GetTempSubDirectoryPath(contextId, Screenshots); +} diff --git a/Lombiq.Tests.UI/Constants/Paths.cs b/Lombiq.Tests.UI/Constants/Paths.cs deleted file mode 100644 index 0590eb988..000000000 --- a/Lombiq.Tests.UI/Constants/Paths.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System; -using System.IO; - -namespace Lombiq.Tests.UI.Constants; - -public static class Paths -{ - public const string DefaultSetupSnapshotDirectoryPath = "SetupSnapshot"; - public const string TempDirectoryPath = "Temp"; - public const string ScreenshotsDirectoryName = "Screenshots"; - - public static string GetTempSubDirectoryPath(string contextId, string subDirectoryName) => - Path.Combine(Environment.CurrentDirectory, TempDirectoryPath, contextId, subDirectoryName); - - public static string GetScreenshotsDirectoryPath(string contextId) => - GetTempSubDirectoryPath(contextId, ScreenshotsDirectoryName); -} diff --git a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs index 53a1c50b7..758ba3895 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs @@ -218,7 +218,7 @@ public async ValueTask DisposeAsync() private void CreateContentRootFolder() { - _contentRootPath = Paths.GetTempSubDirectoryPath(_contextId, "App"); + _contentRootPath = DirectoryPaths.GetTempSubDirectoryPath(_contextId, "App"); Directory.CreateDirectory(_contentRootPath); _testOutputHelper.WriteLineTimestampedAndDebug("Content root path was created: {0}", _contentRootPath); } diff --git a/Lombiq.Tests.UI/Services/OrchardCoreSetupConfiguration.cs b/Lombiq.Tests.UI/Services/OrchardCoreSetupConfiguration.cs index 534b61028..f90a28164 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreSetupConfiguration.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreSetupConfiguration.cs @@ -30,7 +30,7 @@ public class OrchardCoreSetupConfiguration /// public bool FastFailSetup { get; set; } = true; - public string SetupSnapshotDirectoryPath { get; set; } = Paths.DefaultSetupSnapshotDirectoryPath; + public string SetupSnapshotDirectoryPath { get; set; } = DirectoryPaths.SetupSnapshot; public BeforeSetupHandler BeforeSetup { get; set; } } diff --git a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs index cdadcdbc2..e9c839c66 100644 --- a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs +++ b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs @@ -156,7 +156,7 @@ private async ValueTask ShutdownAsync() { _context.Scope?.Dispose(); - DirectoryHelper.SafelyDeleteDirectoryIfExists(Paths.GetTempSubDirectoryPath(_context.Id, string.Empty)); + DirectoryHelper.SafelyDeleteDirectoryIfExists(DirectoryPaths.GetTempSubDirectoryPath(_context.Id, string.Empty)); } _sqlServerManager?.Dispose(); @@ -436,8 +436,8 @@ private void SetupDocker() // We add this subdirectory to ensure the HostSnapshotPath isn't set to the mounted volume's directory itself // (which would be logical). Removing the volume directory instantly severs the connection between host and the // container so that should be avoided at all costs. - docker.ContainerSnapshotPath += '/' + Paths.DefaultSetupSnapshotDirectoryPath; // Always a Unix path. - docker.HostSnapshotPath = Path.Combine(docker.HostSnapshotPath, Paths.DefaultSetupSnapshotDirectoryPath); + docker.ContainerSnapshotPath += '/' + DirectoryPaths.SetupSnapshot; // Always a Unix path. + docker.HostSnapshotPath = Path.Combine(docker.HostSnapshotPath, DirectoryPaths.SetupSnapshot); _dockerConfiguration = docker; @@ -504,7 +504,7 @@ private async Task CreateContextAsync() { var contextId = Guid.NewGuid().ToString(); - FileSystemHelper.EnsureDirectoryExists(Paths.GetTempSubDirectoryPath(contextId, string.Empty)); + FileSystemHelper.EnsureDirectoryExists(DirectoryPaths.GetTempSubDirectoryPath(contextId, string.Empty)); SqlServerRunningContext sqlServerContext = null; AzureBlobStorageRunningContext azureBlobStorageContext = null; @@ -705,7 +705,7 @@ private Task TakeScreenshotIfEnabledAsync(UITestContext context) { if (_context == null || !_dumpConfiguration.CaptureScreenshots) return Task.CompletedTask; - var screnshotsPath = Paths.GetScreenshotsDirectoryPath(_context.Id); + var screnshotsPath = DirectoryPaths.GetScreenshotsDirectoryPath(_context.Id); FileSystemHelper.EnsureDirectoryExists(screnshotsPath); try @@ -732,10 +732,10 @@ private async Task CreateScreenshotsDumpAsync(string debugInformationPath) { await TakeScreenshotIfEnabledAsync(_context); - var screenshotsSourcePath = Paths.GetScreenshotsDirectoryPath(_context.Id); + var screenshotsSourcePath = DirectoryPaths.GetScreenshotsDirectoryPath(_context.Id); if (Directory.Exists(screenshotsSourcePath)) { - var screenshotsDestinationPath = Path.Combine(debugInformationPath, Paths.ScreenshotsDirectoryName); + var screenshotsDestinationPath = Path.Combine(debugInformationPath, DirectoryPaths.Screenshots); FileSystem.CopyDirectory(screenshotsSourcePath, screenshotsDestinationPath); if (_configuration.ReportTeamCityMetadata) From 6304e9a7d90ae53a41249a5e09f7825092cd6e09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Fri, 8 Apr 2022 16:10:15 +0200 Subject: [PATCH 11/12] More generic GetTempSubDirectoryPath() --- Lombiq.Tests.UI/Constants/DirectoryPaths.cs | 5 +++-- Lombiq.Tests.UI/Services/UITestExecutionSession.cs | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Lombiq.Tests.UI/Constants/DirectoryPaths.cs b/Lombiq.Tests.UI/Constants/DirectoryPaths.cs index 24b6844c6..2e0b425df 100644 --- a/Lombiq.Tests.UI/Constants/DirectoryPaths.cs +++ b/Lombiq.Tests.UI/Constants/DirectoryPaths.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.Linq; namespace Lombiq.Tests.UI.Constants; @@ -9,8 +10,8 @@ public static class DirectoryPaths public const string Temp = nameof(Temp); public const string Screenshots = nameof(Screenshots); - public static string GetTempSubDirectoryPath(string contextId, string subDirectoryName) => - Path.Combine(Environment.CurrentDirectory, Temp, contextId, subDirectoryName); + public static string GetTempSubDirectoryPath(string contextId, params string[] subDirectoryNames) => + Path.Combine(new[] { Environment.CurrentDirectory, Temp, contextId }.Union(subDirectoryNames).ToArray()); public static string GetScreenshotsDirectoryPath(string contextId) => GetTempSubDirectoryPath(contextId, Screenshots); diff --git a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs index e9c839c66..4f81b4bc9 100644 --- a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs +++ b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs @@ -156,7 +156,7 @@ private async ValueTask ShutdownAsync() { _context.Scope?.Dispose(); - DirectoryHelper.SafelyDeleteDirectoryIfExists(DirectoryPaths.GetTempSubDirectoryPath(_context.Id, string.Empty)); + DirectoryHelper.SafelyDeleteDirectoryIfExists(DirectoryPaths.GetTempSubDirectoryPath(_context.Id)); } _sqlServerManager?.Dispose(); @@ -504,7 +504,7 @@ private async Task CreateContextAsync() { var contextId = Guid.NewGuid().ToString(); - FileSystemHelper.EnsureDirectoryExists(DirectoryPaths.GetTempSubDirectoryPath(contextId, string.Empty)); + FileSystemHelper.EnsureDirectoryExists(DirectoryPaths.GetTempSubDirectoryPath(contextId)); SqlServerRunningContext sqlServerContext = null; AzureBlobStorageRunningContext azureBlobStorageContext = null; From d1a3bcf0d402c000db8f32842b8826594ed818b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Fri, 8 Apr 2022 17:06:27 +0200 Subject: [PATCH 12/12] Fixing path building --- Lombiq.Tests.UI/Constants/DirectoryPaths.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Constants/DirectoryPaths.cs b/Lombiq.Tests.UI/Constants/DirectoryPaths.cs index 2e0b425df..f73b6db7e 100644 --- a/Lombiq.Tests.UI/Constants/DirectoryPaths.cs +++ b/Lombiq.Tests.UI/Constants/DirectoryPaths.cs @@ -11,7 +11,7 @@ public static class DirectoryPaths public const string Screenshots = nameof(Screenshots); public static string GetTempSubDirectoryPath(string contextId, params string[] subDirectoryNames) => - Path.Combine(new[] { Environment.CurrentDirectory, Temp, contextId }.Union(subDirectoryNames).ToArray()); + Path.Combine(new[] { Environment.CurrentDirectory, Temp, contextId }.Concat(subDirectoryNames).ToArray()); public static string GetScreenshotsDirectoryPath(string contextId) => GetTempSubDirectoryPath(contextId, Screenshots);