From 980caa7528b8774f6dac900fb296d123b529d076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Wed, 6 Nov 2024 01:15:48 +0100 Subject: [PATCH 01/33] Formatting --- .../Services/OrchardCoreHosting/OrchardApplicationFactory.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Services/OrchardCoreHosting/OrchardApplicationFactory.cs b/Lombiq.Tests.UI/Services/OrchardCoreHosting/OrchardApplicationFactory.cs index 5f90ad820..f91e3a347 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreHosting/OrchardApplicationFactory.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreHosting/OrchardApplicationFactory.cs @@ -71,7 +71,8 @@ protected override IHost CreateHost(IHostBuilder builder) protected override void ConfigureWebHost(IWebHostBuilder builder) { - builder.ConfigureTestServices(ConfigureTestServices) + builder + .ConfigureTestServices(ConfigureTestServices) .ConfigureLogging((context, loggingBuilder) => { var environment = context.HostingEnvironment; From 2b73625c73b886db4032611d6eb42eec418723f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Wed, 6 Nov 2024 01:47:30 +0100 Subject: [PATCH 02/33] Basics of FakeLogger logging --- Lombiq.Tests.UI/Lombiq.Tests.UI.csproj | 1 + .../OrchardApplicationFactory.cs | 18 +++--------------- .../Services/OrchardCoreInstance.cs | 12 +++++++++++- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj index 3fc27d706..df2db5492 100644 --- a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj +++ b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj @@ -74,6 +74,7 @@ + diff --git a/Lombiq.Tests.UI/Services/OrchardCoreHosting/OrchardApplicationFactory.cs b/Lombiq.Tests.UI/Services/OrchardCoreHosting/OrchardApplicationFactory.cs index f91e3a347..876312fe0 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreHosting/OrchardApplicationFactory.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreHosting/OrchardApplicationFactory.cs @@ -9,10 +9,8 @@ using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Hosting; using NLog; -using NLog.Web; using System; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Threading.Tasks; using YesSql; @@ -73,19 +71,9 @@ protected override void ConfigureWebHost(IWebHostBuilder builder) { builder .ConfigureTestServices(ConfigureTestServices) - .ConfigureLogging((context, loggingBuilder) => - { - var environment = context.HostingEnvironment; - var nLogConfig = Path.Combine(environment.ContentRootPath, "NLog.config"); - var factory = new LogFactory() - .Setup() - .LoadConfigurationFromFile(nLogConfig) - .LogFactory; - - factory.Configuration.Variables["configDir"] = environment.ContentRootPath; - - loggingBuilder.AddNLogWeb(factory, new NLogAspNetCoreOptions { ReplaceLoggerFactory = true }); - }); + // NLog, if used, will put log files into configDir. Not setting this would use the default, which would be + // App_Data/App_Data/logs. + .ConfigureLogging((context, _) => LogManager.Configuration.Variables["configDir"] = context.HostingEnvironment.ContentRootPath); _configuration?.Invoke(builder); } diff --git a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs index e8f856a80..01a5d8cc4 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs @@ -8,6 +8,8 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Testing; using Microsoft.VisualBasic.FileIO; using System; using System.Collections.Generic; @@ -56,6 +58,7 @@ public sealed class OrchardCoreInstance : IWebApplicationInstance private readonly OrchardCoreConfiguration _configuration; private readonly string _contextId; private readonly ITestOutputHelper _testOutputHelper; + private string _contentRootPath; private bool _isDisposed; private OrchardApplicationFactory _orchardApplication; @@ -172,7 +175,14 @@ await _configuration.BeforeAppStart builder => builder .UseContentRoot(_contentRootPath) .UseWebRoot(Path.Combine(_contentRootPath, "wwwroot")) - .UseEnvironment(Environments.Development), + .UseEnvironment(Environments.Development) + .ConfigureLogging(loggingBuilder => loggingBuilder.AddFakeLogging(options => + { + options.CollectRecordsForDisabledLogLevels = false; + options.FilteredLevels.Add(LogLevel.Warning); + options.FilteredLevels.Add(LogLevel.Error); + options.OutputSink += message => _testOutputHelper.WriteLine(message); + })), (configuration, orchardBuilder) => orchardBuilder .ConfigureUITesting(configuration, enableShortcutsDuringUITesting: true)); From c8639d25a9b3bf510a77cd62f72454906a907690 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Thu, 7 Nov 2024 18:20:59 +0100 Subject: [PATCH 03/33] Unusing --- Lombiq.Tests.UI/Services/OrchardCoreInstance.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs index 01a5d8cc4..5b7ace81f 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs @@ -9,7 +9,6 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Testing; using Microsoft.VisualBasic.FileIO; using System; using System.Collections.Generic; From e53821e5bbe1cae583b1c58fe18fb127ce672491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Thu, 7 Nov 2024 18:26:07 +0100 Subject: [PATCH 04/33] Removing superfluous GetRequiredService() from IWebApplicationInstance --- .../Extensions/WebApplicationInstanceExtensions.cs | 12 ++++++++++++ Lombiq.Tests.UI/Services/IWebApplicationInstance.cs | 10 ---------- Lombiq.Tests.UI/Services/OrchardCoreInstance.cs | 3 --- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs b/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs index df5775113..faab8f95f 100644 --- a/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs @@ -1,4 +1,5 @@ using Lombiq.Tests.UI.Services; +using Microsoft.Extensions.DependencyInjection; using Shouldly; using System; using System.Collections.Generic; @@ -62,4 +63,15 @@ public static async Task GetLogOutputAsync( Environment.NewLine + Environment.NewLine, await webApplicationInstance.GetLogs(cancellationToken).ToFormattedStringAsync()); } + + /// + /// Get service of type . + /// + /// The type of service object to get. + /// A service object of type . + /// + /// There is no service of type . + /// + public static TService GetRequiredService(this IWebApplicationInstance webApplicationInstance) => + webApplicationInstance.Services.GetRequiredService(); } diff --git a/Lombiq.Tests.UI/Services/IWebApplicationInstance.cs b/Lombiq.Tests.UI/Services/IWebApplicationInstance.cs index 994d970e3..273102b18 100644 --- a/Lombiq.Tests.UI/Services/IWebApplicationInstance.cs +++ b/Lombiq.Tests.UI/Services/IWebApplicationInstance.cs @@ -44,16 +44,6 @@ public interface IWebApplicationInstance : IAsyncDisposable /// /// The collection of log names and their contents. IEnumerable GetLogs(CancellationToken cancellationToken = default); - - /// - /// Get service of type . - /// - /// The type of service object to get. - /// A service object of type . - /// - /// There is no service of type . - /// - TService GetRequiredService(); } /// diff --git a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs index 5b7ace81f..c4e53fb21 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs @@ -130,9 +130,6 @@ public IEnumerable GetLogs(CancellationToken cancellationToken : []; } - public TService GetRequiredService() => - _orchardApplication.Services.GetRequiredService(); - public async ValueTask DisposeAsync() { if (_isDisposed) return; From 38ab51b9e0a1632904967b03f3a91d9b780b9c5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Fri, 8 Nov 2024 19:13:50 +0100 Subject: [PATCH 05/33] Better exception messages --- .../Services/OrchardCoreHosting/OrchardApplicationFactory.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lombiq.Tests.UI/Services/OrchardCoreHosting/OrchardApplicationFactory.cs b/Lombiq.Tests.UI/Services/OrchardCoreHosting/OrchardApplicationFactory.cs index 876312fe0..d0cf6b5bd 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreHosting/OrchardApplicationFactory.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreHosting/OrchardApplicationFactory.cs @@ -84,12 +84,13 @@ private void ConfigureTestServices(IServiceCollection services) .LastOrDefault(descriptor => descriptor.ServiceType == typeof(OrchardCoreBuilder))? .ImplementationInstance as OrchardCoreBuilder ?? throw new InvalidOperationException( - "Please call WebApplicationBuilder.Services.AddOrchardCms() in your Program.cs!"); + "Please call WebApplicationBuilder.Services.AddOrchardCms() in your Program.cs."); var configuration = services .LastOrDefault(descriptor => descriptor.ServiceType == typeof(ConfigurationManager))? .ImplementationInstance as ConfigurationManager ?? throw new InvalidOperationException( - $"Please add {nameof(ConfigurationManager)} instance to WebApplicationBuilder.Services in your Program.cs!"); + $"Please register the {nameof(ConfigurationManager)} instance in the Service Collection in your " + + "Program.cs, following the documentation."); _configureOrchard?.Invoke(configuration, builder); From 7b9547dfff2b1cfa7fce74c9cdf4c95051ae1caf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Fri, 8 Nov 2024 21:14:54 +0100 Subject: [PATCH 06/33] Surfacing structured log objects from IWebApplicationInstance --- .../ApplicationLogEnumerableExtensions.cs | 7 +- .../WebApplicationInstanceExtensions.cs | 6 ++ Lombiq.Tests.UI/Services/IApplicationLog.cs | 69 +++++++++++++++++++ .../Services/IWebApplicationInstance.cs | 22 ------ .../Services/OrchardCoreInstance.cs | 68 +++++++++++------- 5 files changed, 125 insertions(+), 47 deletions(-) create mode 100644 Lombiq.Tests.UI/Services/IApplicationLog.cs diff --git a/Lombiq.Tests.UI/Extensions/ApplicationLogEnumerableExtensions.cs b/Lombiq.Tests.UI/Extensions/ApplicationLogEnumerableExtensions.cs index 5e063fe7b..de8457a7d 100644 --- a/Lombiq.Tests.UI/Extensions/ApplicationLogEnumerableExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/ApplicationLogEnumerableExtensions.cs @@ -11,5 +11,10 @@ public static class ApplicationLogEnumerableExtensions public static async Task ToFormattedStringAsync(this IEnumerable logs) => string.Join( Environment.NewLine + Environment.NewLine, - await Task.WhenAll(logs.Select(async log => log.Name + Environment.NewLine + Environment.NewLine + await log.GetContentAsync()))); + await Task.WhenAll( + logs.Select(async log => + log.Name + + Environment.NewLine + + Environment.NewLine + + (await log.GetContentAsync()).Select(logMessage => logMessage.ToString())))); } diff --git a/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs b/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs index faab8f95f..7b0ff68d5 100644 --- a/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs @@ -53,6 +53,12 @@ public static async Task LogsShouldBeEmptyAsync( /// /// Retrieves all the logs and concatenates them into a single formatted string. /// + /// + /// + /// If you want to inspect the logs in a more structured way, message by message, consider using directly instead. + /// + /// public static async Task GetLogOutputAsync( this IWebApplicationInstance webApplicationInstance, CancellationToken cancellationToken = default) diff --git a/Lombiq.Tests.UI/Services/IApplicationLog.cs b/Lombiq.Tests.UI/Services/IApplicationLog.cs new file mode 100644 index 000000000..c4ad0725b --- /dev/null +++ b/Lombiq.Tests.UI/Services/IApplicationLog.cs @@ -0,0 +1,69 @@ +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace Lombiq.Tests.UI.Services; + +/// +/// An abstraction over a log, be it in the form of a file or something else. +/// +public interface IApplicationLog +{ + /// + /// Gets the name of the log, such as the file name. + /// + string Name { get; } + + /// + /// Gets the number of messages in the log. + /// + int MessageCount { get; } + + /// + /// Returns the content of the log, in case of log files reads the file contents. + /// + /// The contents. + Task> GetContentAsync(); + + /// + /// Removes the log if possible. + /// + void Remove(); +} + +/// +/// An abstraction over a log message. +/// +public interface IApplicationLogMessage +{ + /// + /// Gets the level of the log message, like or . + /// + LogLevel Level { get; } + + /// + /// Gets the ID that uniquely identifies the log message. + /// + EventId Id { get; } + + /// + /// Gets the exception associated with the log message, if any. + /// + Exception Exception { get; } + + /// + /// Gets the human-readable formatted log message. + /// + string Message { get; } + + /// + /// Gets the category of the log message. This is the type parameter of . + /// + string Category { get; } + + /// + /// Gets the timestamp of when the log message was created. + /// + DateTimeOffset Timestamp { get; } +} diff --git a/Lombiq.Tests.UI/Services/IWebApplicationInstance.cs b/Lombiq.Tests.UI/Services/IWebApplicationInstance.cs index 273102b18..3a891db1b 100644 --- a/Lombiq.Tests.UI/Services/IWebApplicationInstance.cs +++ b/Lombiq.Tests.UI/Services/IWebApplicationInstance.cs @@ -45,25 +45,3 @@ public interface IWebApplicationInstance : IAsyncDisposable /// The collection of log names and their contents. IEnumerable GetLogs(CancellationToken cancellationToken = default); } - -/// -/// An abstraction over a log, be it in the form of a file or something else. -/// -public interface IApplicationLog -{ - /// - /// Gets the name of the log, such as the file name. - /// - string Name { get; } - - /// - /// Returns the content of the log, in case of log files reads the file contents. - /// - /// The contents. - Task GetContentAsync(); - - /// - /// Removes the log if possible. - /// - void Remove(); -} diff --git a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs index c4e53fb21..689c79e80 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs @@ -9,6 +9,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Testing; using Microsoft.VisualBasic.FileIO; using System; using System.Collections.Generic; @@ -117,17 +118,17 @@ public Task TakeSnapshotAsync(string snapshotDirectoryPath) public IEnumerable GetLogs(CancellationToken cancellationToken = default) { - var logFolderPath = Path.Combine(_contentRootPath, "App_Data", "logs"); - return Directory.Exists(logFolderPath) - ? Directory - .EnumerateFiles(logFolderPath, "*.log") - .Select(filePath => (IApplicationLog)new ApplicationLog + if (Services.GetService() is { } logCollector) + { + return [ + new ApplicationLog { - Name = Path.GetFileName(filePath), - FullName = Path.GetFullPath(filePath), - ContentLoader = () => GetFileContentAsync(filePath, cancellationToken), - }) - : []; + LogCollector = logCollector, + }, + ]; + } + + return []; } public async ValueTask DisposeAsync() @@ -206,13 +207,6 @@ await _configuration.AfterAppStop .InvokeAsync(handler => handler(CreateAppStartContext())); } - private static async Task GetFileContentAsync(string filePath, CancellationToken cancellationToken) - { - await using var fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); - using var streamReader = new StreamReader(fileStream); - return await streamReader.ReadToEndAsync(cancellationToken); - } - private async Task TakeSnapshotInnerAsync(string snapshotDirectoryPath) { await PauseAsync(); @@ -232,15 +226,41 @@ private OrchardCoreAppStartContext CreateAppStartContext() => private sealed class ApplicationLog : IApplicationLog { - public string Name { get; init; } - public string FullName { get; init; } - public Func> ContentLoader { get; init; } + public string Name => "FakeLog"; + public FakeLogCollector LogCollector { get; init; } + public int MessageCount => LogCollector.Count; - public Task GetContentAsync() => ContentLoader(); - - public void Remove() + public Task> GetContentAsync() { - if (File.Exists(FullName)) File.Delete(FullName); + var records = LogCollector.GetSnapshot(); + + return Task.FromResult(records.Select(record => (IApplicationLogMessage)new ApplicationLogMessage + { + Level = record.Level, + Id = record.Id, + Exception = record.Exception, + Message = record.Message, + Category = record.Category, + Timestamp = record.Timestamp, + LogRecord = record, + })); } + + public void Remove() => LogCollector.Clear(); + } + + private sealed class ApplicationLogMessage : IApplicationLogMessage + { + public LogLevel Level { get; init; } + public EventId Id { get; init; } + public Exception Exception { get; init; } + public string Message { get; init; } + public string Category { get; init; } + public DateTimeOffset Timestamp { get; init; } + public FakeLogRecord LogRecord { get; init; } + + public override string ToString() => + $"{Timestamp:yyyy-MM-dd HH:mm:ss.fff} [{Level}] {Category}: {Message}" + + (Exception != null ? Exception.ToString() : string.Empty); } } From 0fdb6b473c74b62ec74eab999b5a47329c7588a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Fri, 8 Nov 2024 21:28:21 +0100 Subject: [PATCH 07/33] Accepting breaking changes --- Lombiq.Tests.UI/CompatibilitySuppressions.xml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 Lombiq.Tests.UI/CompatibilitySuppressions.xml diff --git a/Lombiq.Tests.UI/CompatibilitySuppressions.xml b/Lombiq.Tests.UI/CompatibilitySuppressions.xml new file mode 100644 index 000000000..58604759e --- /dev/null +++ b/Lombiq.Tests.UI/CompatibilitySuppressions.xml @@ -0,0 +1,39 @@ + + + + + CP0002 + M:Lombiq.Tests.UI.Services.IApplicationLog.GetContentAsync + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.IWebApplicationInstance.GetRequiredService``1 + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.OrchardCoreInstance`1.GetRequiredService``1 + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0006 + M:Lombiq.Tests.UI.Services.IApplicationLog.GetContentAsync + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0006 + P:Lombiq.Tests.UI.Services.IApplicationLog.MessageCount + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + \ No newline at end of file From e66df37a5c342eb526e72ba64a22340d5087380d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Fri, 8 Nov 2024 22:11:16 +0100 Subject: [PATCH 08/33] Betters LogsShouldBe* assertions --- .../WebApplicationInstanceExtensions.cs | 61 +++++++++---------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs b/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs index 7b0ff68d5..e447366bf 100644 --- a/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs @@ -1,10 +1,8 @@ using Lombiq.Tests.UI.Services; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Shouldly; using System; -using System.Collections.Generic; -using System.Linq; -using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; @@ -16,37 +14,40 @@ public static class WebApplicationInstanceExtensions /// Asserting that the logs should be empty. When they aren't the Shouldly exception will contain the logs' /// contents. /// - /// - /// If not or empty, each line is split and any lines containing |ERROR| will be - /// ignored if they regex match any string from this collection (case-insensitive). - /// + /// + /// + /// If you want to inspect the logs in a more structured way, message by message, consider using directly instead. + /// + /// public static async Task LogsShouldBeEmptyAsync( this IWebApplicationInstance webApplicationInstance, - bool canContainWarnings = false, - ICollection permittedErrorLinePatterns = null, CancellationToken cancellationToken = default) { - permittedErrorLinePatterns ??= []; - - var logOutput = await webApplicationInstance.GetLogOutputAsync(cancellationToken); - - logOutput.ShouldNotContain("|FATAL|"); - - var lines = logOutput.SplitByNewLines(); - - var errorLines = lines.Where(line => line.Contains("|ERROR|")); - - if (permittedErrorLinePatterns.Count != 0) - { - errorLines = errorLines.Where(line => - !permittedErrorLinePatterns.Any(pattern => Regex.IsMatch(line, pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled))); - } + var logs = await webApplicationInstance.GetLogsAsync(cancellationToken); + logs.ShouldNotContain(log => log.MessageCount > 0, await logs.ToFormattedStringAsync()); + } - errorLines.ShouldBeEmpty(); + /// + /// Asserting that the logs should not contain messages with the greater than or equal to + /// . When they do, the Shouldly exception will contain the logs' contents. + /// + /// + /// + /// If you want to inspect the logs in a more structured way, message by message, consider using directly instead. + /// + /// + public static async Task LogsShouldNotContainErrorsAsync( + this IWebApplicationInstance webApplicationInstance, + CancellationToken cancellationToken = default) + { + var logs = await webApplicationInstance.GetLogsAsync(cancellationToken); - if (!canContainWarnings) + foreach (var log in logs) { - lines.Where(line => line.Contains("|WARNING|")).ShouldBeEmpty(); + (await log.GetContentAsync()) + .ShouldNotContain(logMessage => logMessage.Level > LogLevel.Warning, await logs.ToFormattedStringAsync()); } } @@ -56,7 +57,7 @@ public static async Task LogsShouldBeEmptyAsync( /// /// /// If you want to inspect the logs in a more structured way, message by message, consider using directly instead. + /// cref="IWebApplicationInstance.GetLogsAsync(CancellationToken)"/> directly instead. /// /// public static async Task GetLogOutputAsync( @@ -65,9 +66,7 @@ public static async Task GetLogOutputAsync( { if (cancellationToken == default) cancellationToken = CancellationToken.None; - return string.Join( - Environment.NewLine + Environment.NewLine, - await webApplicationInstance.GetLogs(cancellationToken).ToFormattedStringAsync()); + return await (await webApplicationInstance.GetLogsAsync(cancellationToken)).ToFormattedStringAsync(); } /// From eb1af552bc9a3187fcc58b20568243d0b3dc2f29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sat, 9 Nov 2024 21:40:51 +0100 Subject: [PATCH 09/33] Adapting log retrieval and assertion --- .../Tests/ErrorHandlingTests.cs | 2 +- Lombiq.Tests.UI.Samples/UITestBase.cs | 12 +-- .../TestCases/SecurityShortcutsTestCases.cs | 4 +- .../ApplicationLogEnumerableExtensions.cs | 4 +- .../BrowserUITestContextExtensions.cs | 4 +- .../WebApplicationInstanceExtensions.cs | 18 +---- .../Models/FakeLoggerLogApplicationLog.cs | 53 +++++++++++++ ...reUITestExecutorConfigurationExtensions.cs | 72 ++++++++++++++++++ ...SecurityScanningUITestContextExtensions.cs | 6 +- Lombiq.Tests.UI/Services/IApplicationLog.cs | 6 +- .../Services/IWebApplicationInstance.cs | 2 +- .../Services/OrchardCoreInstance.cs | 57 +++----------- .../OrchardCoreUITestExecutorConfiguration.cs | 76 +++---------------- Lombiq.Tests.UI/Services/RemoteInstance.cs | 4 +- Lombiq.Tests.UI/Services/UITestContext.cs | 4 +- 15 files changed, 168 insertions(+), 156 deletions(-) create mode 100644 Lombiq.Tests.UI/Models/FakeLoggerLogApplicationLog.cs create mode 100644 Lombiq.Tests.UI/SecurityScanning/OrchardCoreUITestExecutorConfigurationExtensions.cs diff --git a/Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs b/Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs index 5c57a0d5f..dd0df24c5 100644 --- a/Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs @@ -38,7 +38,7 @@ public Task ServerSideErrorOnLoadedPageShouldHaltTest() => catch (PageChangeAssertionException) { // Remove all logs to have a clean slate. - context.ClearLogs(); + await context.ClearLogsAsync(); } }); diff --git a/Lombiq.Tests.UI.Samples/UITestBase.cs b/Lombiq.Tests.UI.Samples/UITestBase.cs index e8af21d8a..a154a23ee 100644 --- a/Lombiq.Tests.UI.Samples/UITestBase.cs +++ b/Lombiq.Tests.UI.Samples/UITestBase.cs @@ -87,17 +87,9 @@ protected override Task ExecuteTestAsync( // up) are checked and if there are any errors, the test will fail. You can also enable the checking of // accessibility rules as we'll see later. Maybe not all of the default checks are suitable for you. // Then it's simple to override them; here we change which log entries cause the tests to fail, and - // allow warnings and certain errors. - // Note that this is just for demonstration; you could use - // OrchardCoreUITestExecutorConfiguration.AssertAppLogsCanContainWarningsAndCacheFolderErrorsAsync which - // provides this configuration built-in. + // allow certain log entries. configuration.AssertAppLogsAsync = webApplicationInstance => - webApplicationInstance.LogsShouldBeEmptyAsync( - canContainWarnings: true, - permittedErrorLinePatterns: - [ - "OrchardCore.Media.Core.DefaultMediaFileStoreCacheFileProvider|ERROR|Error deleting cache folder", - ]); + webApplicationInstance.LogsShouldNotContainAsync(logEntry => logEntry.Message != "My permitted message."); // Strictly speaking this is not necessary here, because we always use the same static method for setup. // However, if you used a dynamic setup operation (e.g. `context => SetupHelpers.RunSetupAsync(context, diff --git a/Lombiq.Tests.UI.Tests.UI/TestCases/SecurityShortcutsTestCases.cs b/Lombiq.Tests.UI.Tests.UI/TestCases/SecurityShortcutsTestCases.cs index 12b0cccf3..548f650e6 100644 --- a/Lombiq.Tests.UI.Tests.UI/TestCases/SecurityShortcutsTestCases.cs +++ b/Lombiq.Tests.UI.Tests.UI/TestCases/SecurityShortcutsTestCases.cs @@ -48,7 +48,7 @@ public static Task AddUserToFakeRoleShouldThrowAsync( await context.CreateUserAsync(UserUserName, DefaultUser.Password, UserEmail); await context.AddUserToRoleAsync(UserUserName, FakeRole).ShouldThrowAsync(); - context.ClearLogs(); + await context.ClearLogsAsync(); }, browser, ConfigurationHelper.DisableHtmlValidation); @@ -61,7 +61,7 @@ public static Task AllowFakePermissionToRoleShouldThrowAsync( await context.AddPermissionToRoleAsync(FakePermission, AuthorRole) .ShouldThrowAsync(); - context.ClearLogs(); + await context.ClearLogsAsync(); }, browser, ConfigurationHelper.DisableHtmlValidation); diff --git a/Lombiq.Tests.UI/Extensions/ApplicationLogEnumerableExtensions.cs b/Lombiq.Tests.UI/Extensions/ApplicationLogEnumerableExtensions.cs index de8457a7d..0abb49992 100644 --- a/Lombiq.Tests.UI/Extensions/ApplicationLogEnumerableExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/ApplicationLogEnumerableExtensions.cs @@ -13,8 +13,8 @@ public static async Task ToFormattedStringAsync(this IEnumerable - log.Name + + $"# Log name: {log.Name}" + Environment.NewLine + Environment.NewLine + - (await log.GetContentAsync()).Select(logMessage => logMessage.ToString())))); + string.Join(Environment.NewLine, (await log.GetEntriesAsync()).Select(logEntry => logEntry.ToString()))))); } diff --git a/Lombiq.Tests.UI/Extensions/BrowserUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/BrowserUITestContextExtensions.cs index 9e43cec34..f8e8f89e4 100644 --- a/Lombiq.Tests.UI/Extensions/BrowserUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/BrowserUITestContextExtensions.cs @@ -1,4 +1,4 @@ -using Lombiq.Tests.UI.Extensions; +using Lombiq.Tests.UI.Extensions; using Lombiq.Tests.UI.Services; using System.Diagnostics.CodeAnalysis; using System.Net.Http; @@ -87,7 +87,7 @@ public static async Task DoWithoutAppLogAssertionAsync(this UITestContext } catch { - context.ClearLogs(); + await context.ClearLogsAsync(); return true; } diff --git a/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs b/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs index e447366bf..3e1b658a2 100644 --- a/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs @@ -1,8 +1,8 @@ using Lombiq.Tests.UI.Services; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; using Shouldly; using System; +using System.Linq.Expressions; using System.Threading; using System.Threading.Tasks; @@ -28,26 +28,16 @@ public static async Task LogsShouldBeEmptyAsync( logs.ShouldNotContain(log => log.MessageCount > 0, await logs.ToFormattedStringAsync()); } - /// - /// Asserting that the logs should not contain messages with the greater than or equal to - /// . When they do, the Shouldly exception will contain the logs' contents. - /// - /// - /// - /// If you want to inspect the logs in a more structured way, message by message, consider using directly instead. - /// - /// - public static async Task LogsShouldNotContainErrorsAsync( + public static async Task LogsShouldNotContainAsync( this IWebApplicationInstance webApplicationInstance, + Expression> logEntryPredicate, CancellationToken cancellationToken = default) { var logs = await webApplicationInstance.GetLogsAsync(cancellationToken); foreach (var log in logs) { - (await log.GetContentAsync()) - .ShouldNotContain(logMessage => logMessage.Level > LogLevel.Warning, await logs.ToFormattedStringAsync()); + (await log.GetEntriesAsync()).ShouldNotContain(logEntryPredicate, await logs.ToFormattedStringAsync()); } } diff --git a/Lombiq.Tests.UI/Models/FakeLoggerLogApplicationLog.cs b/Lombiq.Tests.UI/Models/FakeLoggerLogApplicationLog.cs new file mode 100644 index 000000000..8bea85f78 --- /dev/null +++ b/Lombiq.Tests.UI/Models/FakeLoggerLogApplicationLog.cs @@ -0,0 +1,53 @@ +using Lombiq.Tests.UI.Services; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Testing; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Lombiq.Tests.UI.Models; + +public sealed class FakeLoggerLogApplicationLog : IApplicationLog +{ + public string Name => "FakeLog"; + public FakeLogCollector LogCollector { get; init; } + public int MessageCount => LogCollector.Count; + + public Task> GetEntriesAsync() + { + var records = LogCollector.GetSnapshot(); + + return Task.FromResult(records.Select(record => (IApplicationLogEntry)new FakeLoggerApplicationLogEntry + { + Level = record.Level, + Id = record.Id, + Exception = record.Exception, + Message = record.Message, + Category = record.Category, + Timestamp = record.Timestamp, + LogRecord = record, + })); + } + + public Task RemoveAsync() + { + LogCollector.Clear(); + return Task.CompletedTask; + } +} + +public sealed class FakeLoggerApplicationLogEntry : IApplicationLogEntry +{ + public LogLevel Level { get; init; } + public EventId Id { get; init; } + public Exception Exception { get; init; } + public string Message { get; init; } + public string Category { get; init; } + public DateTimeOffset Timestamp { get; init; } + public FakeLogRecord LogRecord { get; init; } + + public override string ToString() => + $"{Timestamp:yyyy-MM-dd HH:mm:ss.fff} [{Level}] {Category}: {Message}" + + (Exception != null ? Exception.ToString() : string.Empty); +} diff --git a/Lombiq.Tests.UI/SecurityScanning/OrchardCoreUITestExecutorConfigurationExtensions.cs b/Lombiq.Tests.UI/SecurityScanning/OrchardCoreUITestExecutorConfigurationExtensions.cs new file mode 100644 index 000000000..b0483e825 --- /dev/null +++ b/Lombiq.Tests.UI/SecurityScanning/OrchardCoreUITestExecutorConfigurationExtensions.cs @@ -0,0 +1,72 @@ +using Lombiq.Tests.UI.Extensions; +using Lombiq.Tests.UI.Services; +using Lombiq.Tests.UI.Shortcuts.Controllers; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace Lombiq.Tests.UI.SecurityScanning; + +public static class OrchardCoreUITestExecutorConfigurationExtensions +{ + /// + /// Sets the to the output of so it accepts errors in the log caused by the security scanning. + /// + public static OrchardCoreUITestExecutorConfiguration UseAssertAppLogsForSecurityScan( + this OrchardCoreUITestExecutorConfiguration configuration, + params string[] additionalPermittedErrorLinePatterns) + { + configuration.AssertAppLogsAsync = CreateAppLogAssertionForSecurityScan(additionalPermittedErrorLinePatterns); + + return configuration; + } + + /// + /// Similar to , + /// but also permits certain log messages which represent correct reactions to + /// incorrect or malicious user behavior during a security scan. + /// + public static Func CreateAppLogAssertionForSecurityScan(params string[] additionalPermittedErrorLinePatterns) + { + var permittedErrorLinePatterns = new List + { + // The model binding will throw FormatException exception with this text during ZAP active scan, when the + // bot tries to send malicious query strings or POST data that doesn't fit the types expected by the model. + // This is correct, safe behavior and should be logged in production. + "is not a valid value for Boolean", + "An unhandled exception has occurred while executing the request. System.FormatException: any", + "System.FormatException: The input string '[\\S\\s]+' was not in a correct format.", + "System.FormatException: The input string 'any", + // Happens when the static file middleware tries to access a path that doesn't exist or access a file as a + // directory. Presumably this is an attempt to access protected files using source path manipulation. This + // is handled by ASP.NET Core and there is nothing for us to worry about. + "System.IO.IOException: Not a directory", + "System.IO.IOException: The filename, directory name, or volume label syntax is incorrect", + "System.IO.DirectoryNotFoundException: Could not find a part of the path", + // This happens when a request's model contains a dictionary and a key is missing. While this can be a + // legitimate application error, during a security scan it's more likely the result of an incomplete + // artificially constructed request. So the means the ASP.NET Core model binding is working as intended. + "An unhandled exception has occurred while executing the request. System.ArgumentNullException: Value cannot be null. (Parameter 'key')", + // One way to verify correct error handling is to navigate to ~/Lombiq.Tests.UI.Shortcuts/Error/Index, which + // always throws an exception. This also gets logged but it's expected, so it should be ignored. + ErrorController.ExceptionMessage, + // Thrown from Microsoft.AspNetCore.Authentication.AuthenticationService.ChallengeAsync() when ZAP sends + // invalid authentication challenges. + "System.InvalidOperationException: No authentication handler is registered for the scheme", + // These errors frequently happen during UI testing when using Azure Blob Storage for media storage. They're + // harmless, though. + "Error deleting cache folder", + }; + + permittedErrorLinePatterns.AddRange(additionalPermittedErrorLinePatterns); + + return app => + app.LogsShouldNotContainAsync(logEntry => + !permittedErrorLinePatterns.Any(pattern => + Regex.IsMatch(logEntry.ToString(), pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled))); + } +} diff --git a/Lombiq.Tests.UI/SecurityScanning/SecurityScanningUITestContextExtensions.cs b/Lombiq.Tests.UI/SecurityScanning/SecurityScanningUITestContextExtensions.cs index 3af26f6a2..947686bcf 100644 --- a/Lombiq.Tests.UI/SecurityScanning/SecurityScanningUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/SecurityScanning/SecurityScanningUITestContextExtensions.cs @@ -57,9 +57,9 @@ public static Task RunAndAssertFullSecurityScanAsync( /// This extension method makes changes to the normal configuration of the test to be more suited for CI operation. /// It changes the to not do any retries because this is a long running /// test. It also replaces the app log assertion logic with the specialized version for security scans, . The scan is configured t - /// ignore the admin dashboard, optionally log in as admin, and use the provided time limits for the "active scan" - /// portion of the security scan. + /// cref="OrchardCoreUITestExecutorConfigurationExtensions.UseAssertAppLogsForSecurityScan"/>. The scan is + /// configured to ignore the admin dashboard, optionally log in as admin, and use the provided time limits for the + /// "active scan" portion of the security scan. /// public static Task RunAndConfigureAndAssertFullSecurityScanForContinuousIntegrationAsync( this UITestContext context, diff --git a/Lombiq.Tests.UI/Services/IApplicationLog.cs b/Lombiq.Tests.UI/Services/IApplicationLog.cs index c4ad0725b..e67912100 100644 --- a/Lombiq.Tests.UI/Services/IApplicationLog.cs +++ b/Lombiq.Tests.UI/Services/IApplicationLog.cs @@ -24,18 +24,18 @@ public interface IApplicationLog /// Returns the content of the log, in case of log files reads the file contents. /// /// The contents. - Task> GetContentAsync(); + Task> GetEntriesAsync(); /// /// Removes the log if possible. /// - void Remove(); + Task RemoveAsync(); } /// /// An abstraction over a log message. /// -public interface IApplicationLogMessage +public interface IApplicationLogEntry { /// /// Gets the level of the log message, like or . diff --git a/Lombiq.Tests.UI/Services/IWebApplicationInstance.cs b/Lombiq.Tests.UI/Services/IWebApplicationInstance.cs index 3a891db1b..0aee1d96a 100644 --- a/Lombiq.Tests.UI/Services/IWebApplicationInstance.cs +++ b/Lombiq.Tests.UI/Services/IWebApplicationInstance.cs @@ -43,5 +43,5 @@ public interface IWebApplicationInstance : IAsyncDisposable /// Reads all the application logs. /// /// The collection of log names and their contents. - IEnumerable GetLogs(CancellationToken cancellationToken = default); + Task> GetLogsAsync(CancellationToken cancellationToken = default); } diff --git a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs index 689c79e80..f7809f62c 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs @@ -116,19 +116,21 @@ public Task TakeSnapshotAsync(string snapshotDirectoryPath) return TakeSnapshotInnerAsync(snapshotDirectoryPath); } - public IEnumerable GetLogs(CancellationToken cancellationToken = default) + public Task> GetLogsAsync(CancellationToken cancellationToken = default) { if (Services.GetService() is { } logCollector) { - return [ - new ApplicationLog + return Task.FromResult( + new IApplicationLog[] { - LogCollector = logCollector, - }, - ]; + new FakeLoggerLogApplicationLog + { + LogCollector = logCollector, + }, + }.AsEnumerable()); } - return []; + return Task.FromResult(Enumerable.Empty()); } public async ValueTask DisposeAsync() @@ -176,7 +178,6 @@ await _configuration.BeforeAppStart .ConfigureLogging(loggingBuilder => loggingBuilder.AddFakeLogging(options => { options.CollectRecordsForDisabledLogLevels = false; - options.FilteredLevels.Add(LogLevel.Warning); options.FilteredLevels.Add(LogLevel.Error); options.OutputSink += message => _testOutputHelper.WriteLine(message); })), @@ -223,44 +224,4 @@ await _configuration.BeforeTakeSnapshot private OrchardCoreAppStartContext CreateAppStartContext() => new(_contentRootPath, _url, OrchardCoreInstanceCounter.PortLeases); - - private sealed class ApplicationLog : IApplicationLog - { - public string Name => "FakeLog"; - public FakeLogCollector LogCollector { get; init; } - public int MessageCount => LogCollector.Count; - - public Task> GetContentAsync() - { - var records = LogCollector.GetSnapshot(); - - return Task.FromResult(records.Select(record => (IApplicationLogMessage)new ApplicationLogMessage - { - Level = record.Level, - Id = record.Id, - Exception = record.Exception, - Message = record.Message, - Category = record.Category, - Timestamp = record.Timestamp, - LogRecord = record, - })); - } - - public void Remove() => LogCollector.Clear(); - } - - private sealed class ApplicationLogMessage : IApplicationLogMessage - { - public LogLevel Level { get; init; } - public EventId Id { get; init; } - public Exception Exception { get; init; } - public string Message { get; init; } - public string Category { get; init; } - public DateTimeOffset Timestamp { get; init; } - public FakeLogRecord LogRecord { get; init; } - - public override string ToString() => - $"{Timestamp:yyyy-MM-dd HH:mm:ss.fff} [{Level}] {Category}: {Message}" + - (Exception != null ? Exception.ToString() : string.Empty); - } } diff --git a/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs b/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs index 060685017..32abb13fa 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs @@ -1,7 +1,6 @@ using Lombiq.Tests.UI.Extensions; using Lombiq.Tests.UI.SecurityScanning; using Lombiq.Tests.UI.Services.GitHub; -using Lombiq.Tests.UI.Shortcuts.Controllers; using OpenQA.Selenium; using Shouldly; using System; @@ -30,21 +29,15 @@ public enum Browser public class OrchardCoreUITestExecutorConfiguration { - // These errors frequently happen during UI testing when using Azure Blob Storage for media storage. - // They're harmless, though. - private const string CacheFolderErrorPattern = - "OrchardCore.Media.Core.DefaultMediaFileStoreCacheFileProvider|ERROR|Error deleting cache folder"; - public static readonly Func AssertAppLogsAreEmptyAsync = app => app.LogsShouldBeEmptyAsync(); - public static readonly Func AssertAppLogsCanContainWarningsAsync = - app => app.LogsShouldBeEmptyAsync(canContainWarnings: true); - - public static readonly Func AssertAppLogsCanContainWarningsAndCacheFolderErrorsAsync = - app => app.LogsShouldBeEmptyAsync( - canContainWarnings: true, - permittedErrorLinePatterns: [CacheFolderErrorPattern]); + public static readonly Func AssertAppLogsCanContainCacheFolderErrorsAsync = + app => app.LogsShouldNotContainAsync(logEntry => + // These errors frequently happen during UI testing when using Azure Blob Storage for media storage. They're + // harmless, though. + logEntry.Category == "OrchardCore.Media.Core.DefaultMediaFileStoreCacheFileProvider" && + logEntry.Message.StartsWithOrdinalIgnoreCase("Error deleting cache folder")); public static readonly Action> AssertBrowserLogIsEmpty = logEntries => logEntries.ShouldNotContain( @@ -117,7 +110,7 @@ public class OrchardCoreUITestExecutorConfiguration ? intValue : Environment.ProcessorCount; - public Func AssertAppLogsAsync { get; set; } = AssertAppLogsCanContainWarningsAsync; + public Func AssertAppLogsAsync { get; set; } = AssertAppLogsCanContainCacheFolderErrorsAsync; public Action> AssertBrowserLog { get; set; } = AssertBrowserLogIsEmpty; public ITestOutputHelper TestOutputHelper { get; set; } @@ -128,9 +121,8 @@ public class OrchardCoreUITestExecutorConfiguration /// /// /// - /// For this to properly work the build artifacts should be configured to contain the TestDumps folder (it can - /// also contain other folders but it must contain a folder called "TestDumps", e.g.: +:TestDumps => - /// TestDumps. + /// For this to properly work the build artifacts should be configured to contain the TestDumps folder (it can also + /// contain other folders but it must contain a folder called "TestDumps", e.g.: +:TestDumps => TestDumps. /// /// public bool ReportTeamCityMetadata { get; set; } = @@ -230,54 +222,4 @@ public void AssertBrowserLogMaybe(IList browserLogs, Action lo throw; } } - - /// - /// Sets the to the output of so - /// it accepts errors in the log caused by the security scanning. - /// - public OrchardCoreUITestExecutorConfiguration UseAssertAppLogsForSecurityScan(params string[] additionalPermittedErrorLinePatterns) - { - AssertAppLogsAsync = CreateAppLogAssertionForSecurityScan(additionalPermittedErrorLinePatterns); - - return this; - } - - /// - /// Similar to , but also permits certain |ERROR log - /// entries which represent correct reactions to incorrect or malicious user behavior during a security scan. - /// - public static Func CreateAppLogAssertionForSecurityScan(params string[] additionalPermittedErrorLinePatterns) - { - var permittedErrorLinePatterns = new List - { - // The model binding will throw FormatException exception with this text during ZAP active scan, when - // the bot tries to send malicious query strings or POST data that doesn't fit the types expected by the - // model. This is correct, safe behavior and should be logged in production. - "is not a valid value for Boolean", - "An unhandled exception has occurred while executing the request. System.FormatException: any", - "System.FormatException: The input string '[\\S\\s]+' was not in a correct format.", - "System.FormatException: The input string 'any", - // Happens when the static file middleware tries to access a path that doesn't exist or access a file as - // a directory. Presumably this is an attempt to access protected files using source path manipulation. - // This is handled by ASP.NET Core and there is nothing for us to worry about. - "System.IO.IOException: Not a directory", - "System.IO.IOException: The filename, directory name, or volume label syntax is incorrect", - "System.IO.DirectoryNotFoundException: Could not find a part of the path", - // This happens when a request's model contains a dictionary and a key is missing. While this can be a - // legitimate application error, during a security scan it's more likely the result of an incomplete - // artificially constructed request. So the means the ASP.NET Core model binding is working as intended. - "An unhandled exception has occurred while executing the request. System.ArgumentNullException: Value cannot be null. (Parameter 'key')", - // One way to verify correct error handling is to navigate to ~/Lombiq.Tests.UI.Shortcuts/Error/Index, which - // always throws an exception. This also gets logged but it's expected, so it should be ignored. - ErrorController.ExceptionMessage, - // Thrown from Microsoft.AspNetCore.Authentication.AuthenticationService.ChallengeAsync() when ZAP sends - // invalid authentication challenges. - "System.InvalidOperationException: No authentication handler is registered for the scheme", - CacheFolderErrorPattern, - }; - - permittedErrorLinePatterns.AddRange(additionalPermittedErrorLinePatterns); - - return app => app.LogsShouldBeEmptyAsync(canContainWarnings: true, permittedErrorLinePatterns); - } } diff --git a/Lombiq.Tests.UI/Services/RemoteInstance.cs b/Lombiq.Tests.UI/Services/RemoteInstance.cs index 30c75ec22..69a147ce8 100644 --- a/Lombiq.Tests.UI/Services/RemoteInstance.cs +++ b/Lombiq.Tests.UI/Services/RemoteInstance.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -15,7 +16,8 @@ public sealed class RemoteInstance : IWebApplicationInstance public Task StartUpAsync() => Task.FromResult(_baseUri); - public IEnumerable GetLogs(CancellationToken cancellationToken = default) => []; + public Task> GetLogsAsync(CancellationToken cancellationToken = default) => + Task.FromResult(Enumerable.Empty()); public TService GetRequiredService() => throw new NotSupportedException(); public Task PauseAsync() => throw new NotSupportedException(); public Task ResumeAsync() => throw new NotSupportedException(); diff --git a/Lombiq.Tests.UI/Services/UITestContext.cs b/Lombiq.Tests.UI/Services/UITestContext.cs index f5ba24a8a..8f4ae5d3a 100644 --- a/Lombiq.Tests.UI/Services/UITestContext.cs +++ b/Lombiq.Tests.UI/Services/UITestContext.cs @@ -219,9 +219,9 @@ public Task AssertCurrentBrowserLogAsync() /// Clears the application and historic browser logs. /// /// Optional cancellation token for reading the application logs. - public void ClearLogs(CancellationToken cancellationToken = default) + public async Task ClearLogsAsync(CancellationToken cancellationToken = default) { - foreach (var log in Application.GetLogs(cancellationToken)) log.Remove(); + foreach (var log in await Application.GetLogsAsync(cancellationToken)) await log.RemoveAsync(); ClearHistoricBrowserLog(); } From 66c80f5c7fc0e1405b762660e1413c158b5c994f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sat, 9 Nov 2024 23:08:32 +0100 Subject: [PATCH 10/33] Formatting --- Lombiq.Tests.UI.Samples/Tests/ShiftTimeTests.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lombiq.Tests.UI.Samples/Tests/ShiftTimeTests.cs b/Lombiq.Tests.UI.Samples/Tests/ShiftTimeTests.cs index 45905c91d..226b5bbc7 100644 --- a/Lombiq.Tests.UI.Samples/Tests/ShiftTimeTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/ShiftTimeTests.cs @@ -10,11 +10,11 @@ namespace Lombiq.Tests.UI.Samples.Tests; -// When you enable the "Shift Time - Shortcuts - Lombiq UI Testing Toolbox" feature, it replaces OC's stock ICLock -// implementation with the custom ShiftTimeClock class. You can use the ~/Lombiq.Tests.UI.Samples/ShiftTime/Set?days=... -// action to update the ShiftTimeClock.Shift property for the current tenant, which will trick any service that uses -// IClock into thinking you are in the future. This can be used to test features that can expire, such as a limited-time -// product discount in a web store. +// When you enable the "Shift Time - Shortcuts - Lombiq UI Testing Toolbox" feature, it replaces Orchard Core's stock +// ICLock implementation with the custom ShiftTimeClock class. You can use the +// ~/Lombiq.Tests.UI.Samples/ShiftTime/Set?days=... action to update the ShiftTimeClock.Shift property for the current +// tenant, which will trick any service that uses IClock into thinking you are in the future. This can be used to test +// features that can expire, such as a limited-time product discount in a web store, without having to wait. public class ShiftTimeTests : UITestBase { public ShiftTimeTests(ITestOutputHelper testOutputHelper) From 241866a304557036da12774ab17e038cea41642d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sat, 9 Nov 2024 23:09:34 +0100 Subject: [PATCH 11/33] Copy-paste error --- Lombiq.Tests.UI/Extensions/ShortcutsUITestContextExtensions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Extensions/ShortcutsUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/ShortcutsUITestContextExtensions.cs index e82597740..ba21f77ba 100644 --- a/Lombiq.Tests.UI/Extensions/ShortcutsUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/ShortcutsUITestContextExtensions.cs @@ -733,5 +733,5 @@ public static Task SetShiftTimeAsync(this UITestContext context, double days = 0 /// provided, then the values for both are added. Negative values are supported as well. /// public static Task AddShiftTimeAsync(this UITestContext context, double days = 0, double seconds = 0) => - context.GoToAsync(controller => controller.Set(days, seconds)); + context.GoToAsync(controller => controller.Add(days, seconds)); } From 612ecdbb428ecb628886668426d523bb3cfa67fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sat, 9 Nov 2024 23:10:15 +0100 Subject: [PATCH 12/33] Refactoring, LogsShouldContainAsync() --- .../VerificationUITestContextExtensions.cs | 2 +- .../WebApplicationInstanceExtensions.cs | 74 +++++++++++++++---- .../Models/FakeLoggerLogApplicationLog.cs | 2 +- Lombiq.Tests.UI/Services/IApplicationLog.cs | 16 ++-- .../OrchardCoreUITestExecutorConfiguration.cs | 2 +- 5 files changed, 70 insertions(+), 26 deletions(-) diff --git a/Lombiq.Tests.UI/Extensions/VerificationUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/VerificationUITestContextExtensions.cs index 121ae1b03..ab2a509a3 100644 --- a/Lombiq.Tests.UI/Extensions/VerificationUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/VerificationUITestContextExtensions.cs @@ -29,7 +29,7 @@ public static async Task AssertLogsAsync(this UITestContext context) catch (Exception) { testOutputHelper.WriteLine("Application logs: " + Environment.NewLine); - testOutputHelper.WriteLine(await context.Application.GetLogOutputAsync()); + testOutputHelper.WriteLine(await context.Application.GetLogContentsAsync()); throw; } diff --git a/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs b/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs index 3e1b658a2..2f9c00e9d 100644 --- a/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.DependencyInjection; using Shouldly; using System; +using System.Collections.Generic; using System.Linq.Expressions; using System.Threading; using System.Threading.Tasks; @@ -11,8 +12,8 @@ namespace Lombiq.Tests.UI.Extensions; public static class WebApplicationInstanceExtensions { /// - /// Asserting that the logs should be empty. When they aren't the Shouldly exception will contain the logs' - /// contents. + /// Asserts that the logs should be empty, i.e. contain no entries. If the assertion fails, the Shouldly exception + /// will contain all log entries. /// /// /// @@ -20,26 +21,54 @@ public static class WebApplicationInstanceExtensions /// cref="IWebApplicationInstance.GetLogsAsync(CancellationToken)"/> directly instead. /// /// + /// A that can cancel the log retrieval. public static async Task LogsShouldBeEmptyAsync( this IWebApplicationInstance webApplicationInstance, CancellationToken cancellationToken = default) { var logs = await webApplicationInstance.GetLogsAsync(cancellationToken); - logs.ShouldNotContain(log => log.MessageCount > 0, await logs.ToFormattedStringAsync()); + logs.ShouldNotContain(log => log.EntryCount > 0, await logs.ToFormattedStringAsync()); } - public static async Task LogsShouldNotContainAsync( + /// + /// Asserts that the logs should contain any entry matching the given predicate. If the assertion fails, the + /// Shouldly exception will contain all log entries. + /// + /// + /// + /// If you want to inspect the logs in a more structured way, message by message, consider using directly instead. + /// + /// + /// A that can cancel the log retrieval. + /// + /// A predicate that when returns , the assertion will fail. + /// + public static Task LogsShouldContainAsync( this IWebApplicationInstance webApplicationInstance, Expression> logEntryPredicate, - CancellationToken cancellationToken = default) - { - var logs = await webApplicationInstance.GetLogsAsync(cancellationToken); + CancellationToken cancellationToken = default) => + AssertLogsAsyn(webApplicationInstance, logEntryPredicate, ShouldBeEnumerableTestExtensions.ShouldContain, cancellationToken); - foreach (var log in logs) - { - (await log.GetEntriesAsync()).ShouldNotContain(logEntryPredicate, await logs.ToFormattedStringAsync()); - } - } + /// + /// Asserts that the logs should NOT contain any entries matching the given predicate. If the assertion fails, the + /// Shouldly exception will contain all log entries. + /// + /// + /// + /// If you want to inspect the logs in a more structured way, message by message, consider using directly instead. + /// + /// + /// A that can cancel the log retrieval. + /// + /// A predicate that when returns , the assertion will fail. + /// + public static Task LogsShouldNotContainAsync( + this IWebApplicationInstance webApplicationInstance, + Expression> logEntryPredicate, + CancellationToken cancellationToken = default) => + AssertLogsAsyn(webApplicationInstance, logEntryPredicate, ShouldBeEnumerableTestExtensions.ShouldNotContain, cancellationToken); /// /// Retrieves all the logs and concatenates them into a single formatted string. @@ -50,7 +79,8 @@ public static async Task LogsShouldNotContainAsync( /// cref="IWebApplicationInstance.GetLogsAsync(CancellationToken)"/> directly instead. /// /// - public static async Task GetLogOutputAsync( + /// A that can cancel the log retrieval. + public static async Task GetLogContentsAsync( this IWebApplicationInstance webApplicationInstance, CancellationToken cancellationToken = default) { @@ -62,11 +92,25 @@ public static async Task GetLogOutputAsync( /// /// Get service of type . /// - /// The type of service object to get. - /// A service object of type . + /// The type of service service to get. + /// An instance of the service of type . /// /// There is no service of type . /// public static TService GetRequiredService(this IWebApplicationInstance webApplicationInstance) => webApplicationInstance.Services.GetRequiredService(); + + private static async Task AssertLogsAsyn( + IWebApplicationInstance webApplicationInstance, + Expression> logEntryPredicate, + Action, Expression>, string> shouldlyMethod, + CancellationToken cancellationToken = default) + { + var logs = await webApplicationInstance.GetLogsAsync(cancellationToken); + + foreach (var log in logs) + { + shouldlyMethod(await log.GetEntriesAsync(), logEntryPredicate, await logs.ToFormattedStringAsync()); + } + } } diff --git a/Lombiq.Tests.UI/Models/FakeLoggerLogApplicationLog.cs b/Lombiq.Tests.UI/Models/FakeLoggerLogApplicationLog.cs index 8bea85f78..c3c79a872 100644 --- a/Lombiq.Tests.UI/Models/FakeLoggerLogApplicationLog.cs +++ b/Lombiq.Tests.UI/Models/FakeLoggerLogApplicationLog.cs @@ -12,7 +12,7 @@ public sealed class FakeLoggerLogApplicationLog : IApplicationLog { public string Name => "FakeLog"; public FakeLogCollector LogCollector { get; init; } - public int MessageCount => LogCollector.Count; + public int EntryCount => LogCollector.Count; public Task> GetEntriesAsync() { diff --git a/Lombiq.Tests.UI/Services/IApplicationLog.cs b/Lombiq.Tests.UI/Services/IApplicationLog.cs index e67912100..36f1f9731 100644 --- a/Lombiq.Tests.UI/Services/IApplicationLog.cs +++ b/Lombiq.Tests.UI/Services/IApplicationLog.cs @@ -16,9 +16,9 @@ public interface IApplicationLog string Name { get; } /// - /// Gets the number of messages in the log. + /// Gets the number of log entries in the log. /// - int MessageCount { get; } + int EntryCount { get; } /// /// Returns the content of the log, in case of log files reads the file contents. @@ -33,22 +33,22 @@ public interface IApplicationLog } /// -/// An abstraction over a log message. +/// An abstraction over a log entries. /// public interface IApplicationLogEntry { /// - /// Gets the level of the log message, like or . + /// Gets the level of the log entry, like or . /// LogLevel Level { get; } /// - /// Gets the ID that uniquely identifies the log message. + /// Gets the ID that uniquely identifies the log entry. /// EventId Id { get; } /// - /// Gets the exception associated with the log message, if any. + /// Gets the exception associated with the log entry, if any. /// Exception Exception { get; } @@ -58,12 +58,12 @@ public interface IApplicationLogEntry string Message { get; } /// - /// Gets the category of the log message. This is the type parameter of . + /// Gets the category of the log entry. This is the type parameter of . /// string Category { get; } /// - /// Gets the timestamp of when the log message was created. + /// Gets the timestamp of when the log entry was created. /// DateTimeOffset Timestamp { get; } } diff --git a/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs b/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs index 32abb13fa..24542c038 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs @@ -200,7 +200,7 @@ public async Task AssertAppLogsMaybeAsync(IWebApplicationInstance instance, Acti catch (Exception) { log("Application logs: " + Environment.NewLine); - log(await instance.GetLogOutputAsync()); + log(await instance.GetLogContentsAsync()); throw; } From 84416c678f865e427719578df46bfe523eab9cdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sat, 9 Nov 2024 23:13:23 +0100 Subject: [PATCH 13/33] Locking down ShiftTimeController to only testing --- Lombiq.Tests.UI.Shortcuts/Controllers/ShiftTimeController.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lombiq.Tests.UI.Shortcuts/Controllers/ShiftTimeController.cs b/Lombiq.Tests.UI.Shortcuts/Controllers/ShiftTimeController.cs index e8bb22b1c..ad682a802 100644 --- a/Lombiq.Tests.UI.Shortcuts/Controllers/ShiftTimeController.cs +++ b/Lombiq.Tests.UI.Shortcuts/Controllers/ShiftTimeController.cs @@ -1,3 +1,4 @@ +using Lombiq.HelpfulLibraries.AspNetCore.Mvc; using Lombiq.Tests.UI.Shortcuts.Services; using Microsoft.AspNetCore.Mvc; using OrchardCore.Modules; @@ -10,6 +11,7 @@ namespace Lombiq.Tests.UI.Shortcuts.Controllers; "Major Code Smell", "S6967:ModelState.IsValid should be called in controller actions", Justification = "Not relevant in a test-only controller.")] +[DevelopmentAndLocalhostOnly] public class ShiftTimeController : Controller { private readonly IClock _clock; From 794a14c8eef6934a223c282dafbfeeb9326a61d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sat, 9 Nov 2024 23:14:16 +0100 Subject: [PATCH 14/33] Typo --- .../Extensions/WebApplicationInstanceExtensions.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs b/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs index 2f9c00e9d..f6567460e 100644 --- a/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs @@ -48,7 +48,7 @@ public static Task LogsShouldContainAsync( this IWebApplicationInstance webApplicationInstance, Expression> logEntryPredicate, CancellationToken cancellationToken = default) => - AssertLogsAsyn(webApplicationInstance, logEntryPredicate, ShouldBeEnumerableTestExtensions.ShouldContain, cancellationToken); + AssertLogsAsync(webApplicationInstance, logEntryPredicate, ShouldBeEnumerableTestExtensions.ShouldContain, cancellationToken); /// /// Asserts that the logs should NOT contain any entries matching the given predicate. If the assertion fails, the @@ -68,7 +68,7 @@ public static Task LogsShouldNotContainAsync( this IWebApplicationInstance webApplicationInstance, Expression> logEntryPredicate, CancellationToken cancellationToken = default) => - AssertLogsAsyn(webApplicationInstance, logEntryPredicate, ShouldBeEnumerableTestExtensions.ShouldNotContain, cancellationToken); + AssertLogsAsync(webApplicationInstance, logEntryPredicate, ShouldBeEnumerableTestExtensions.ShouldNotContain, cancellationToken); /// /// Retrieves all the logs and concatenates them into a single formatted string. @@ -100,7 +100,7 @@ public static async Task GetLogContentsAsync( public static TService GetRequiredService(this IWebApplicationInstance webApplicationInstance) => webApplicationInstance.Services.GetRequiredService(); - private static async Task AssertLogsAsyn( + private static async Task AssertLogsAsync( IWebApplicationInstance webApplicationInstance, Expression> logEntryPredicate, Action, Expression>, string> shouldlyMethod, From 619cf9bb0fd6c62658a81686c15a82f492c27629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 10 Nov 2024 01:05:57 +0100 Subject: [PATCH 15/33] Adding extension point to configure logging --- Lombiq.Tests.UI/Services/OrchardCoreInstance.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs index f7809f62c..aa038a53c 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs @@ -22,14 +22,15 @@ namespace Lombiq.Tests.UI.Services; public delegate Task BeforeAppStartHandler(OrchardCoreAppStartContext context, InstanceCommandLineArgumentsBuilder arguments); +public delegate void AfterFakeLoggingConfigurationHandler(OrchardCoreAppStartContext context, FakeLogCollectorOptions fakeLogCollectorOptions); public delegate Task AfterAppStopHandler(OrchardCoreAppStartContext context); - public delegate Task BeforeTakeSnapshotHandler(OrchardCoreAppStartContext context, string snapshotDirectoryPath); public class OrchardCoreConfiguration { public string SnapshotDirectoryPath { get; set; } public BeforeAppStartHandler BeforeAppStart { get; set; } + public AfterFakeLoggingConfigurationHandler AfterFakeLoggingConfiguration { get; set; } public AfterAppStopHandler AfterAppStop { get; set; } public BeforeTakeSnapshotHandler BeforeTakeSnapshot { get; set; } public int StartCount { get; internal set; } @@ -180,6 +181,8 @@ await _configuration.BeforeAppStart options.CollectRecordsForDisabledLogLevels = false; options.FilteredLevels.Add(LogLevel.Error); options.OutputSink += message => _testOutputHelper.WriteLine(message); + + _configuration.AfterFakeLoggingConfiguration?.Invoke(CreateAppStartContext(), options); })), (configuration, orchardBuilder) => orchardBuilder .ConfigureUITesting(configuration, enableShortcutsDuringUITesting: true)); From 5f4aa957219c7db8ff05ef19957eddaf33e23686 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 10 Nov 2024 01:13:08 +0100 Subject: [PATCH 16/33] Spelling --- .../Extensions/WebApplicationInstanceExtensions.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs b/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs index f6567460e..6b7f15a26 100644 --- a/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs @@ -103,14 +103,14 @@ public static TService GetRequiredService(this IWebApplicationInstance private static async Task AssertLogsAsync( IWebApplicationInstance webApplicationInstance, Expression> logEntryPredicate, - Action, Expression>, string> shouldlyMethod, + Action, Expression>, string> shouldlyMethod, // #spell-check-ignore-line CancellationToken cancellationToken = default) { var logs = await webApplicationInstance.GetLogsAsync(cancellationToken); foreach (var log in logs) { - shouldlyMethod(await log.GetEntriesAsync(), logEntryPredicate, await logs.ToFormattedStringAsync()); + shouldlyMethod(await log.GetEntriesAsync(), logEntryPredicate, await logs.ToFormattedStringAsync()); // #spell-check-ignore-line } } } From 84bb073b8abe55270931bf1c713cd4343b404143 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 10 Nov 2024 01:34:36 +0100 Subject: [PATCH 17/33] Removing unneeded suppression --- Lombiq.Tests.UI.Shortcuts/Controllers/ShiftTimeController.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Lombiq.Tests.UI.Shortcuts/Controllers/ShiftTimeController.cs b/Lombiq.Tests.UI.Shortcuts/Controllers/ShiftTimeController.cs index ad682a802..631b524b1 100644 --- a/Lombiq.Tests.UI.Shortcuts/Controllers/ShiftTimeController.cs +++ b/Lombiq.Tests.UI.Shortcuts/Controllers/ShiftTimeController.cs @@ -3,14 +3,9 @@ using Microsoft.AspNetCore.Mvc; using OrchardCore.Modules; using System; -using System.Diagnostics.CodeAnalysis; namespace Lombiq.Tests.UI.Shortcuts.Controllers; -[SuppressMessage( - "Major Code Smell", - "S6967:ModelState.IsValid should be called in controller actions", - Justification = "Not relevant in a test-only controller.")] [DevelopmentAndLocalhostOnly] public class ShiftTimeController : Controller { From 166865c39e25ef26e35ea96ba810ffb5d094b06e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 10 Nov 2024 01:37:11 +0100 Subject: [PATCH 18/33] Fixing that the app wasn't resumed after taking a snapshot --- Lombiq.Tests.UI/Services/OrchardCoreInstance.cs | 2 ++ .../Services/SynchronizingWebApplicationSnapshotManager.cs | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs index aa038a53c..f141f7752 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs @@ -223,6 +223,8 @@ await _configuration.BeforeTakeSnapshot .InvokeAsync(handler => handler(CreateAppStartContext(), snapshotDirectoryPath)); FileSystem.CopyDirectory(_contentRootPath, snapshotDirectoryPath, overwrite: true); + + await ResumeAsync(); } private OrchardCoreAppStartContext CreateAppStartContext() => diff --git a/Lombiq.Tests.UI/Services/SynchronizingWebApplicationSnapshotManager.cs b/Lombiq.Tests.UI/Services/SynchronizingWebApplicationSnapshotManager.cs index 02aec3a93..1c9a89484 100644 --- a/Lombiq.Tests.UI/Services/SynchronizingWebApplicationSnapshotManager.cs +++ b/Lombiq.Tests.UI/Services/SynchronizingWebApplicationSnapshotManager.cs @@ -46,7 +46,6 @@ public async Task RunOperationAndSnapshotIfNewAsync(AppInitializer appIniti var result = await appInitializer(); await result.Context.Application.TakeSnapshotAsync(_snapshotDirectoryPath); - await result.Context.Application.ResumeAsync(); DebugHelper.WriteLineTimestamped("Finished snapshot."); From db5eed317936bf730383045164dd137b64015fbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 10 Nov 2024 01:38:09 +0100 Subject: [PATCH 19/33] Fixing DatabaseSnapshotTests --- .../Tests/DatabaseSnapshotTests.cs | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Lombiq.Tests.UI.Samples/Tests/DatabaseSnapshotTests.cs b/Lombiq.Tests.UI.Samples/Tests/DatabaseSnapshotTests.cs index a57dffc69..42f49b154 100644 --- a/Lombiq.Tests.UI.Samples/Tests/DatabaseSnapshotTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/DatabaseSnapshotTests.cs @@ -23,19 +23,19 @@ public DatabaseSnapshotTests(ITestOutputHelper testOutputHelper) // "ExecuteTestFromExistingDBAsync()" to run the test on that. Finally, we test the basic Orchard features to check // that the application was set up correctly. [Fact] - public Task BasicOrchardFeaturesShouldWorkWithExistingDatabase() => - ExecuteTestAsync( - async context => - { - var appForDatabaseTestFolder = Path.Combine("Temp", "AppForDatabaseTest"); + public async Task BasicOrchardFeaturesShouldWorkWithExistingDatabase() + { + var appForDatabaseTestFolder = Path.Combine("Temp", "AppForDatabaseTest"); - await context.GoToSetupPageAndSetupOrchardCoreAsync(RecipeIds.BasicOrchardFeaturesTests); - await context.Application.TakeSnapshotAsync(appForDatabaseTestFolder); + await ExecuteTestAsync( + async context => + { + await context.GoToSetupPageAndSetupOrchardCoreAsync(RecipeIds.BasicOrchardFeaturesTests); + await context.Application.TakeSnapshotAsync(appForDatabaseTestFolder); + }); - await ExecuteTestFromExistingDBAsync( - async context => await context.TestBasicOrchardFeaturesExceptSetupAsync(), - appForDatabaseTestFolder); - }); + await ExecuteTestFromExistingDBAsync(context => context.TestBasicOrchardFeaturesExceptSetupAsync(), appForDatabaseTestFolder); + } } // END OF TRAINING SECTION: Database snapshot tests. From 86eb457e1bc07756aff0d03aa89d1b1dd74c805c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 10 Nov 2024 02:03:33 +0100 Subject: [PATCH 20/33] Fixing log assertion --- .../Services/OrchardCoreUITestExecutorConfiguration.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs b/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs index 24542c038..f1bcd56a8 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs @@ -36,8 +36,8 @@ public class OrchardCoreUITestExecutorConfiguration app => app.LogsShouldNotContainAsync(logEntry => // These errors frequently happen during UI testing when using Azure Blob Storage for media storage. They're // harmless, though. - logEntry.Category == "OrchardCore.Media.Core.DefaultMediaFileStoreCacheFileProvider" && - logEntry.Message.StartsWithOrdinalIgnoreCase("Error deleting cache folder")); + logEntry.Category != "OrchardCore.Media.Core.DefaultMediaFileStoreCacheFileProvider" || + !logEntry.Message.StartsWithOrdinalIgnoreCase("Error deleting cache folder")); public static readonly Action> AssertBrowserLogIsEmpty = logEntries => logEntries.ShouldNotContain( From 88a9e0435f32c3ad3011ccd45092a806c8050a22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 10 Nov 2024 02:23:29 +0100 Subject: [PATCH 21/33] DRY "Error deleting cache folder" log entry ignore --- .../Helpers/AppLogAssertionHelper.cs | 23 +++++++++++++++++++ ...reUITestExecutorConfigurationExtensions.cs | 7 +++--- .../OrchardCoreUITestExecutorConfiguration.cs | 7 ++---- 3 files changed, 28 insertions(+), 9 deletions(-) create mode 100644 Lombiq.Tests.UI/Helpers/AppLogAssertionHelper.cs diff --git a/Lombiq.Tests.UI/Helpers/AppLogAssertionHelper.cs b/Lombiq.Tests.UI/Helpers/AppLogAssertionHelper.cs new file mode 100644 index 000000000..5ffc5b682 --- /dev/null +++ b/Lombiq.Tests.UI/Helpers/AppLogAssertionHelper.cs @@ -0,0 +1,23 @@ +using Lombiq.Tests.UI.Services; +using System; +using System.Linq.Expressions; + +namespace Lombiq.Tests.UI.Helpers; + +public static class AppLogAssertionHelper +{ + /// + /// An wrapping . + /// + public static readonly Expression> NotMediaCacheEntriesPredicate = + logEntry => NotMediaCacheEntries(logEntry); + + /// + /// Creates a predicate that can be used to filter out "Error deleting cache folder" log entries coming from + /// DefaultMediaFileStoreCacheFileProvider. These errors frequently happen during UI testing when using Azure + /// Blob Storage for media storage. They're harmless, though. + /// + public static bool NotMediaCacheEntries(IApplicationLogEntry logEntry) => + logEntry.Category != "OrchardCore.Media.Core.DefaultMediaFileStoreCacheFileProvider" || + !logEntry.Message.StartsWithOrdinalIgnoreCase("Error deleting cache folder"); +} diff --git a/Lombiq.Tests.UI/SecurityScanning/OrchardCoreUITestExecutorConfigurationExtensions.cs b/Lombiq.Tests.UI/SecurityScanning/OrchardCoreUITestExecutorConfigurationExtensions.cs index b0483e825..1e497e673 100644 --- a/Lombiq.Tests.UI/SecurityScanning/OrchardCoreUITestExecutorConfigurationExtensions.cs +++ b/Lombiq.Tests.UI/SecurityScanning/OrchardCoreUITestExecutorConfigurationExtensions.cs @@ -1,4 +1,5 @@ using Lombiq.Tests.UI.Extensions; +using Lombiq.Tests.UI.Helpers; using Lombiq.Tests.UI.Services; using Lombiq.Tests.UI.Shortcuts.Controllers; using Microsoft.Extensions.Logging; @@ -57,9 +58,6 @@ public static Func CreateAppLogAssertionForSecuri // Thrown from Microsoft.AspNetCore.Authentication.AuthenticationService.ChallengeAsync() when ZAP sends // invalid authentication challenges. "System.InvalidOperationException: No authentication handler is registered for the scheme", - // These errors frequently happen during UI testing when using Azure Blob Storage for media storage. They're - // harmless, though. - "Error deleting cache folder", }; permittedErrorLinePatterns.AddRange(additionalPermittedErrorLinePatterns); @@ -67,6 +65,7 @@ public static Func CreateAppLogAssertionForSecuri return app => app.LogsShouldNotContainAsync(logEntry => !permittedErrorLinePatterns.Any(pattern => - Regex.IsMatch(logEntry.ToString(), pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled))); + Regex.IsMatch(logEntry.ToString(), pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled)) && + AppLogAssertionHelper.NotMediaCacheEntries(logEntry)); } } diff --git a/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs b/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs index f1bcd56a8..c4abbe2c5 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs @@ -1,4 +1,5 @@ using Lombiq.Tests.UI.Extensions; +using Lombiq.Tests.UI.Helpers; using Lombiq.Tests.UI.SecurityScanning; using Lombiq.Tests.UI.Services.GitHub; using OpenQA.Selenium; @@ -33,11 +34,7 @@ public class OrchardCoreUITestExecutorConfiguration app.LogsShouldBeEmptyAsync(); public static readonly Func AssertAppLogsCanContainCacheFolderErrorsAsync = - app => app.LogsShouldNotContainAsync(logEntry => - // These errors frequently happen during UI testing when using Azure Blob Storage for media storage. They're - // harmless, though. - logEntry.Category != "OrchardCore.Media.Core.DefaultMediaFileStoreCacheFileProvider" || - !logEntry.Message.StartsWithOrdinalIgnoreCase("Error deleting cache folder")); + app => app.LogsShouldNotContainAsync(AppLogAssertionHelper.NotMediaCacheEntriesPredicate); public static readonly Action> AssertBrowserLogIsEmpty = logEntries => logEntries.ShouldNotContain( From 2109b1d852afaef7f6c3185419f3d59a85238a17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 10 Nov 2024 02:28:27 +0100 Subject: [PATCH 22/33] Removing unused and confusing Asser*LogsMaybeAsync methods --- .../OrchardCoreUITestExecutorConfiguration.cs | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs b/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs index c4abbe2c5..54cff1c0a 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs @@ -185,38 +185,4 @@ public class OrchardCoreUITestExecutorConfiguration /// enabled in the app for these to work. /// public ShortcutsConfiguration ShortcutsConfiguration { get; set; } = new(); - - public async Task AssertAppLogsMaybeAsync(IWebApplicationInstance instance, Action log) - { - if (instance == null || AssertAppLogsAsync == null) return; - - try - { - await AssertAppLogsAsync(instance); - } - catch (Exception) - { - log("Application logs: " + Environment.NewLine); - log(await instance.GetLogContentsAsync()); - - throw; - } - } - - public void AssertBrowserLogMaybe(IList browserLogs, Action log) - { - if (AssertBrowserLog == null) return; - - try - { - AssertBrowserLog(browserLogs); - } - catch (Exception) - { - log("Browser logs: " + Environment.NewLine); - log(browserLogs.ToFormattedString()); - - throw; - } - } } From 238ece1d30521194ceff3f7b5b9a4dcb9113de56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 10 Nov 2024 19:05:21 +0100 Subject: [PATCH 23/33] Docs --- Lombiq.Tests.UI.Samples/UITestBase.cs | 45 ++++++++++++++++++++----- Lombiq.Tests.UI/Docs/Troubleshooting.md | 2 +- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/Lombiq.Tests.UI.Samples/UITestBase.cs b/Lombiq.Tests.UI.Samples/UITestBase.cs index a154a23ee..8d4d2cc63 100644 --- a/Lombiq.Tests.UI.Samples/UITestBase.cs +++ b/Lombiq.Tests.UI.Samples/UITestBase.cs @@ -2,6 +2,7 @@ using Lombiq.Tests.UI.Extensions; using Lombiq.Tests.UI.Samples.Helpers; using Lombiq.Tests.UI.Services; +using Microsoft.Extensions.Logging; using System; using System.Threading.Tasks; using Xunit.Abstractions; @@ -64,7 +65,8 @@ protected override Task ExecuteTestAsync( // https://docs.orchardcore.net/en/latest/docs/reference/core/Configuration/. We can set e.g. Orchard's // AdminUrlPrefix like below, but this is just setting the default, so it's only an example. A more // useful example is enabling offline operation of the Lombiq Hosting - Azure Application Insights for - // Orchard Core module (see https://github.com/Lombiq/Orchard-Azure-Application-Insights). + // Orchard Core module (see the UI test project in + // https://github.com/Lombiq/Orchard-Azure-Application-Insights). configuration.OrchardCoreConfiguration.BeforeAppStart += (_, argumentsBuilder) => { @@ -82,14 +84,41 @@ protected override Task ExecuteTestAsync( // Action) to further configure it. ////configuration.HtmlValidationConfiguration.RunHtmlValidationAssertionOnAllPageChanges = false; - // The UI Testing Toolbox can run several checks for the app even if you don't add explicit - // assertions: By default, the Orchard logs and the browser logs (where e.g. JavaScript errors show - // up) are checked and if there are any errors, the test will fail. You can also enable the checking of - // accessibility rules as we'll see later. Maybe not all of the default checks are suitable for you. - // Then it's simple to override them; here we change which log entries cause the tests to fail, and - // allow certain log entries. + // For locally running apps, the UI Testing Toolbox configures Fake Logging (see + // https://learn.microsoft.com/en-us/dotnet/api/microsoft.extensions.logging.testing). This provides an + // in-memory log for assertions, regardless of the logging framework your app uses otherwise. By + // default, it'll only collect log entries with the Error level and up. However, you can change this, + // and the usual Logging app settings still work. + configuration.OrchardCoreConfiguration.BeforeAppStart += + (_, argumentsBuilder) => + { + // This is how you can configure logging to be on the >=Error level in general, while still + // allowing info-level entries for ShellHost. + argumentsBuilder + .AddWithValue("Logging:LogLevel:Default", "Error") + .AddWithValue("Logging:LogLevel:System", "Error") + .AddWithValue("Logging:LogLevel:Microsoft", "Error") + .AddWithValue("Logging:LogLevel:OrchardCore.Environment.Shell.ShellHost", "Information"); + + return Task.CompletedTask; + }; + + // Enabling FakeLogger to collect the info-level ShellHost log entries configured above. These will show + // up in the test's output like "[Information] OrchardCore.Environment.Shell.ShellHost: Start creation + // of shells". + configuration.OrchardCoreConfiguration.AfterFakeLoggingConfiguration = + (_, fakeLogCollectorOptions) => fakeLogCollectorOptions.FilteredLevels.Add(LogLevel.Information); + + // Logging is important because the UI Testing Toolbox can run several checks for the app even if you + // don't add explicit assertions: By default, the Orchard logs and the browser logs (where e.g. + // JavaScript errors show up) are checked and if there are any errors, the test will fail. You can also + // enable the checking of accessibility rules as we'll see later. Maybe not all of the default checks + // are suitable for you. Then it's simple to override them; here we change which log entries cause the + // tests to fail, and allow certain log entries. configuration.AssertAppLogsAsync = webApplicationInstance => - webApplicationInstance.LogsShouldNotContainAsync(logEntry => logEntry.Message != "My permitted message."); + webApplicationInstance.LogsShouldNotContainAsync(logEntry => + // Allowing info-level log entries for ShellHost, see above. + logEntry.Message != "My permitted message." && logEntry.Level != LogLevel.Information); // Strictly speaking this is not necessary here, because we always use the same static method for setup. // However, if you used a dynamic setup operation (e.g. `context => SetupHelpers.RunSetupAsync(context, diff --git a/Lombiq.Tests.UI/Docs/Troubleshooting.md b/Lombiq.Tests.UI/Docs/Troubleshooting.md index 8d6006bad..c944e8525 100644 --- a/Lombiq.Tests.UI/Docs/Troubleshooting.md +++ b/Lombiq.Tests.UI/Docs/Troubleshooting.md @@ -7,7 +7,7 @@ - Browser logs, i.e. the developer console output. - Screenshots of each page in order the test visited them, as well as when the test failed (Windows Photo Viewer won't be able to open it though, use something else like the Windows 10 Photos app). - The HTML output on the page the test failed. - - Any direct output of the test (like the exception thrown) as well as a log of the operations it completed. + - Any direct output of the test (like the exception thrown) as well as a log of the operations it completed (including browser interactions and corresponding Orchard Core log entries). - If accessibility was checked and asserting it failed then an accessibility report will be included too. - If HTML validation was done and asserting it failed then an HTML validation report will be included too. - Run tests with the debugger attached to stop where the test fails. This way you can take a good look at what's in the driven browser window so you can examine the web page and you can debug the web app under test at the same time. From a79adf272a5fb23e2c6ff8a454035376f811bd1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 10 Nov 2024 19:05:38 +0100 Subject: [PATCH 24/33] Better log formatting --- .../Models/FakeLoggerLogApplicationLog.cs | 27 +++++++++---------- .../Services/OrchardCoreInstance.cs | 2 ++ 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/Lombiq.Tests.UI/Models/FakeLoggerLogApplicationLog.cs b/Lombiq.Tests.UI/Models/FakeLoggerLogApplicationLog.cs index c3c79a872..b2251579d 100644 --- a/Lombiq.Tests.UI/Models/FakeLoggerLogApplicationLog.cs +++ b/Lombiq.Tests.UI/Models/FakeLoggerLogApplicationLog.cs @@ -1,3 +1,4 @@ +using Lombiq.HelpfulLibraries.Common.Utilities; using Lombiq.Tests.UI.Services; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Testing; @@ -20,12 +21,6 @@ public Task> GetEntriesAsync() return Task.FromResult(records.Select(record => (IApplicationLogEntry)new FakeLoggerApplicationLogEntry { - Level = record.Level, - Id = record.Id, - Exception = record.Exception, - Message = record.Message, - Category = record.Category, - Timestamp = record.Timestamp, LogRecord = record, })); } @@ -39,15 +34,17 @@ public Task RemoveAsync() public sealed class FakeLoggerApplicationLogEntry : IApplicationLogEntry { - public LogLevel Level { get; init; } - public EventId Id { get; init; } - public Exception Exception { get; init; } - public string Message { get; init; } - public string Category { get; init; } - public DateTimeOffset Timestamp { get; init; } + public LogLevel Level => LogRecord.Level; + public EventId Id => LogRecord.Id; + public Exception Exception => LogRecord.Exception; + public string Message => LogRecord.Message; + public string Category => LogRecord.Category; + public DateTimeOffset Timestamp => LogRecord.Timestamp; public FakeLogRecord LogRecord { get; init; } - public override string ToString() => - $"{Timestamp:yyyy-MM-dd HH:mm:ss.fff} [{Level}] {Category}: {Message}" + - (Exception != null ? Exception.ToString() : string.Empty); + public override string ToString() => FormatLogRecord(LogRecord); + + public static string FormatLogRecord(FakeLogRecord record) => + StringHelper.CreateInvariant($"{record.Timestamp:yyyy-MM-dd HH:mm:ss.fff} [{record.Level}] {record.Category}: {record.Message}") + + (record.Exception != null ? record.Exception.ToString() : string.Empty); } diff --git a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs index f141f7752..c7d981b95 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs @@ -180,6 +180,8 @@ await _configuration.BeforeAppStart { options.CollectRecordsForDisabledLogLevels = false; options.FilteredLevels.Add(LogLevel.Error); + options.FilteredLevels.Add(LogLevel.Critical); + options.OutputFormatter = FakeLoggerApplicationLogEntry.FormatLogRecord; options.OutputSink += message => _testOutputHelper.WriteLine(message); _configuration.AfterFakeLoggingConfiguration?.Invoke(CreateAppStartContext(), options); From 709882c084caf0e0fa203bac0aaa9964848c9530 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 10 Nov 2024 19:08:42 +0100 Subject: [PATCH 25/33] Docs --- .../WebApplicationInstanceExtensions.cs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs b/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs index 6b7f15a26..2e699330f 100644 --- a/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs @@ -18,7 +18,9 @@ public static class WebApplicationInstanceExtensions /// /// /// If you want to inspect the logs in a more structured way, message by message, consider using directly instead. + /// cref="IWebApplicationInstance.GetLogsAsync(CancellationToken)"/> directly instead. Alternatively, set log + /// filtering options to not log unwanted messages in first place with the standard Logging:LogLevel app + /// configuration (see the samples). /// /// /// A that can cancel the log retrieval. @@ -37,7 +39,9 @@ public static async Task LogsShouldBeEmptyAsync( /// /// /// If you want to inspect the logs in a more structured way, message by message, consider using directly instead. + /// cref="IWebApplicationInstance.GetLogsAsync(CancellationToken)"/> directly instead. Alternatively, set log + /// filtering options to not log unwanted messages in first place with the standard Logging:LogLevel app + /// configuration (see the samples). /// /// /// A that can cancel the log retrieval. @@ -57,7 +61,9 @@ public static Task LogsShouldContainAsync( /// /// /// If you want to inspect the logs in a more structured way, message by message, consider using directly instead. + /// cref="IWebApplicationInstance.GetLogsAsync(CancellationToken)"/> directly instead. Alternatively, set log + /// filtering options to not log unwanted messages in first place with the standard Logging:LogLevel app + /// configuration (see the samples). /// /// /// A that can cancel the log retrieval. @@ -76,7 +82,9 @@ public static Task LogsShouldNotContainAsync( /// /// /// If you want to inspect the logs in a more structured way, message by message, consider using directly instead. + /// cref="IWebApplicationInstance.GetLogsAsync(CancellationToken)"/> directly instead. Alternatively, set log + /// filtering options to not log unwanted messages in first place with the standard Logging:LogLevel app + /// configuration (see the samples). /// /// /// A that can cancel the log retrieval. From 1e9a5d28a54826f3bedbbf909404ac3f38ed757c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 10 Nov 2024 19:08:56 +0100 Subject: [PATCH 26/33] Breaking changes suppression --- Lombiq.Tests.UI/CompatibilitySuppressions.xml | 109 +++++++++++++++++- 1 file changed, 107 insertions(+), 2 deletions(-) diff --git a/Lombiq.Tests.UI/CompatibilitySuppressions.xml b/Lombiq.Tests.UI/CompatibilitySuppressions.xml index 58604759e..991a8ef7a 100644 --- a/Lombiq.Tests.UI/CompatibilitySuppressions.xml +++ b/Lombiq.Tests.UI/CompatibilitySuppressions.xml @@ -1,6 +1,34 @@  + + CP0002 + F:Lombiq.Tests.UI.Services.OrchardCoreUITestExecutorConfiguration.AssertAppLogsCanContainWarningsAndCacheFolderErrorsAsync + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + F:Lombiq.Tests.UI.Services.OrchardCoreUITestExecutorConfiguration.AssertAppLogsCanContainWarningsAsync + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Extensions.WebApplicationInstanceExtensions.GetLogOutputAsync(Lombiq.Tests.UI.Services.IWebApplicationInstance,System.Threading.CancellationToken) + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Extensions.WebApplicationInstanceExtensions.LogsShouldBeEmptyAsync(Lombiq.Tests.UI.Services.IWebApplicationInstance,System.Boolean,System.Collections.Generic.ICollection{System.String},System.Threading.CancellationToken) + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + CP0002 M:Lombiq.Tests.UI.Services.IApplicationLog.GetContentAsync @@ -8,6 +36,20 @@ lib/net8.0/Lombiq.Tests.UI.dll true + + CP0002 + M:Lombiq.Tests.UI.Services.IApplicationLog.Remove + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.IWebApplicationInstance.GetLogs(System.Threading.CancellationToken) + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + CP0002 M:Lombiq.Tests.UI.Services.IWebApplicationInstance.GetRequiredService``1 @@ -15,6 +57,13 @@ lib/net8.0/Lombiq.Tests.UI.dll true + + CP0002 + M:Lombiq.Tests.UI.Services.OrchardCoreInstance`1.GetLogs(System.Threading.CancellationToken) + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + CP0002 M:Lombiq.Tests.UI.Services.OrchardCoreInstance`1.GetRequiredService``1 @@ -22,16 +71,72 @@ lib/net8.0/Lombiq.Tests.UI.dll true + + CP0002 + M:Lombiq.Tests.UI.Services.OrchardCoreUITestExecutorConfiguration.AssertAppLogsMaybeAsync(Lombiq.Tests.UI.Services.IWebApplicationInstance,System.Action{System.String}) + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.OrchardCoreUITestExecutorConfiguration.AssertBrowserLogMaybe(System.Collections.Generic.IList{OpenQA.Selenium.LogEntry},System.Action{System.String}) + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.OrchardCoreUITestExecutorConfiguration.CreateAppLogAssertionForSecurityScan(System.String[]) + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.OrchardCoreUITestExecutorConfiguration.UseAssertAppLogsForSecurityScan(System.String[]) + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.RemoteInstance.GetLogs(System.Threading.CancellationToken) + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.UITestContext.ClearLogs(System.Threading.CancellationToken) + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + CP0006 - M:Lombiq.Tests.UI.Services.IApplicationLog.GetContentAsync + M:Lombiq.Tests.UI.Services.IApplicationLog.GetEntriesAsync + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0006 + M:Lombiq.Tests.UI.Services.IApplicationLog.RemoveAsync + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0006 + M:Lombiq.Tests.UI.Services.IWebApplicationInstance.GetLogsAsync(System.Threading.CancellationToken) lib/net8.0/Lombiq.Tests.UI.dll lib/net8.0/Lombiq.Tests.UI.dll true CP0006 - P:Lombiq.Tests.UI.Services.IApplicationLog.MessageCount + P:Lombiq.Tests.UI.Services.IApplicationLog.EntryCount lib/net8.0/Lombiq.Tests.UI.dll lib/net8.0/Lombiq.Tests.UI.dll true From 1fbb8c55a0c1eadab8d26b7cc975dafda8b1d36c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 10 Nov 2024 19:47:05 +0100 Subject: [PATCH 27/33] Security scanning log assertion should only care about errors --- .../OrchardCoreUITestExecutorConfigurationExtensions.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/SecurityScanning/OrchardCoreUITestExecutorConfigurationExtensions.cs b/Lombiq.Tests.UI/SecurityScanning/OrchardCoreUITestExecutorConfigurationExtensions.cs index 1e497e673..9a56dc7b3 100644 --- a/Lombiq.Tests.UI/SecurityScanning/OrchardCoreUITestExecutorConfigurationExtensions.cs +++ b/Lombiq.Tests.UI/SecurityScanning/OrchardCoreUITestExecutorConfigurationExtensions.cs @@ -66,6 +66,7 @@ public static Func CreateAppLogAssertionForSecuri app.LogsShouldNotContainAsync(logEntry => !permittedErrorLinePatterns.Any(pattern => Regex.IsMatch(logEntry.ToString(), pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled)) && - AppLogAssertionHelper.NotMediaCacheEntries(logEntry)); + AppLogAssertionHelper.NotMediaCacheEntries(logEntry) && + logEntry.Level >= LogLevel.Error); } } From 264d6ce93889689b92ef7654dc1ed53effda24af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 10 Nov 2024 20:51:37 +0100 Subject: [PATCH 28/33] Adding LogsShouldNotContainErrorsAsync() --- .../WebApplicationInstanceExtensions.cs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs b/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs index 2e699330f..2dda322b2 100644 --- a/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs @@ -1,5 +1,6 @@ using Lombiq.Tests.UI.Services; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Shouldly; using System; using System.Collections.Generic; @@ -54,6 +55,28 @@ public static Task LogsShouldContainAsync( CancellationToken cancellationToken = default) => AssertLogsAsync(webApplicationInstance, logEntryPredicate, ShouldBeEnumerableTestExtensions.ShouldContain, cancellationToken); + /// + /// Asserts that the logs should NOT contain any entries with and above. If the + /// assertion fails, the Shouldly exception will contain all log entries. + /// + /// + /// + /// If you want to inspect the logs in a more structured way, message by message, consider using directly instead. Alternatively, set log + /// filtering options to not log unwanted messages in first place with the standard Logging:LogLevel app + /// configuration (see the samples). + /// + /// + /// A that can cancel the log retrieval. + public static Task LogsShouldNotContainErrorsAsync( + this IWebApplicationInstance webApplicationInstance, + CancellationToken cancellationToken = default) => + AssertLogsAsync( + webApplicationInstance, + logEntry => logEntry.Level > LogLevel.Error, + ShouldBeEnumerableTestExtensions.ShouldNotContain, + cancellationToken); + /// /// Asserts that the logs should NOT contain any entries matching the given predicate. If the assertion fails, the /// Shouldly exception will contain all log entries. From 8393dd8edd7f6afe7d7a3dbb44c1503e6cbcf3f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 10 Nov 2024 21:03:27 +0100 Subject: [PATCH 29/33] Not printing the log name for single logs in test outputs --- .../ApplicationLogEnumerableExtensions.cs | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/Lombiq.Tests.UI/Extensions/ApplicationLogEnumerableExtensions.cs b/Lombiq.Tests.UI/Extensions/ApplicationLogEnumerableExtensions.cs index 0abb49992..c3aa76ecc 100644 --- a/Lombiq.Tests.UI/Extensions/ApplicationLogEnumerableExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/ApplicationLogEnumerableExtensions.cs @@ -8,13 +8,25 @@ namespace Lombiq.Tests.UI.Extensions; public static class ApplicationLogEnumerableExtensions { - public static async Task ToFormattedStringAsync(this IEnumerable logs) => - string.Join( + public static async Task ToFormattedStringAsync(this IEnumerable logs) + { + var logsArray = logs.ToArray(); + + if (logsArray.Length == 1) + { + return Environment.NewLine + await LogLinesToFormattedStringAsync(logsArray[0]); + } + + return string.Join( Environment.NewLine + Environment.NewLine, await Task.WhenAll( - logs.Select(async log => + logsArray.Select(async log => $"# Log name: {log.Name}" + Environment.NewLine + Environment.NewLine + - string.Join(Environment.NewLine, (await log.GetEntriesAsync()).Select(logEntry => logEntry.ToString()))))); + await LogLinesToFormattedStringAsync(log)))); + } + + private static async Task LogLinesToFormattedStringAsync(IApplicationLog log) => + string.Join(Environment.NewLine, (await log.GetEntriesAsync()).Select(logEntry => logEntry.ToString())); } From a08987dbb7870036632123eac7b1777a75829dd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Mon, 11 Nov 2024 00:49:44 +0100 Subject: [PATCH 30/33] Pointing GHA to issue branch --- .github/workflows/publish-nuget.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-nuget.yml b/.github/workflows/publish-nuget.yml index 44a74afd4..62279b4e2 100644 --- a/.github/workflows/publish-nuget.yml +++ b/.github/workflows/publish-nuget.yml @@ -9,6 +9,6 @@ jobs: publish-nuget: name: Publish to NuGet if: ${{ !contains(github.ref_name, '-preview.') }} - uses: Lombiq/GitHub-Actions/.github/workflows/publish-nuget.yml@dev + uses: Lombiq/GitHub-Actions/.github/workflows/publish-nuget.yml@issue/OSOE-402 secrets: API_KEY: ${{ secrets.DEFAULT_NUGET_PUBLISH_API_KEY }} From 42221b14e445a05e8a85bc7dea0a0d15cf21a91d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Mon, 11 Nov 2024 01:14:25 +0100 Subject: [PATCH 31/33] Pointing GHA to issue branch --- .github/workflows/validate-nuget-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-nuget-publish.yml b/.github/workflows/validate-nuget-publish.yml index 9f8979c5d..42549fccf 100644 --- a/.github/workflows/validate-nuget-publish.yml +++ b/.github/workflows/validate-nuget-publish.yml @@ -9,4 +9,4 @@ on: jobs: validate-nuget-publish: name: Validate NuGet Publish - uses: Lombiq/GitHub-Actions/.github/workflows/validate-nuget-publish.yml@dev + uses: Lombiq/GitHub-Actions/.github/workflows/validate-nuget-publish.yml@issue/OSOE-402 From b5f69a51f6dcf937c86122146a84326386cc1f8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sat, 16 Nov 2024 22:49:45 +0100 Subject: [PATCH 32/33] Preserving the order of logs --- .../ApplicationLogEnumerableExtensions.cs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Lombiq.Tests.UI/Extensions/ApplicationLogEnumerableExtensions.cs b/Lombiq.Tests.UI/Extensions/ApplicationLogEnumerableExtensions.cs index c3aa76ecc..ebfea111c 100644 --- a/Lombiq.Tests.UI/Extensions/ApplicationLogEnumerableExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/ApplicationLogEnumerableExtensions.cs @@ -17,14 +17,12 @@ public static async Task ToFormattedStringAsync(this IEnumerable - $"# Log name: {log.Name}" + - Environment.NewLine + - Environment.NewLine + - await LogLinesToFormattedStringAsync(log)))); + // Parallelization with Task.WhenAll() isn't really necessary for performance here but would potentially change + // the order of the logs in the output. + var logContents = logsArray.AwaitEachAsync(async log => + $"# Log name: {log.Name}" + Environment.NewLine + Environment.NewLine + await LogLinesToFormattedStringAsync(log)); + + return string.Join(Environment.NewLine + Environment.NewLine, logContents); } private static async Task LogLinesToFormattedStringAsync(IApplicationLog log) => From 4609fa9969a8eda8af022ea9200c4e944734c47a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sat, 16 Nov 2024 22:53:40 +0100 Subject: [PATCH 33/33] Turning FakeLoggerApplicationLogEntry into a record --- Lombiq.Tests.UI/Models/FakeLoggerLogApplicationLog.cs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Lombiq.Tests.UI/Models/FakeLoggerLogApplicationLog.cs b/Lombiq.Tests.UI/Models/FakeLoggerLogApplicationLog.cs index b2251579d..d6093839f 100644 --- a/Lombiq.Tests.UI/Models/FakeLoggerLogApplicationLog.cs +++ b/Lombiq.Tests.UI/Models/FakeLoggerLogApplicationLog.cs @@ -19,10 +19,7 @@ public Task> GetEntriesAsync() { var records = LogCollector.GetSnapshot(); - return Task.FromResult(records.Select(record => (IApplicationLogEntry)new FakeLoggerApplicationLogEntry - { - LogRecord = record, - })); + return Task.FromResult(records.Select(record => (IApplicationLogEntry)new FakeLoggerApplicationLogEntry(record))); } public Task RemoveAsync() @@ -32,7 +29,7 @@ public Task RemoveAsync() } } -public sealed class FakeLoggerApplicationLogEntry : IApplicationLogEntry +public record FakeLoggerApplicationLogEntry(FakeLogRecord LogRecord) : IApplicationLogEntry { public LogLevel Level => LogRecord.Level; public EventId Id => LogRecord.Id; @@ -40,7 +37,6 @@ public sealed class FakeLoggerApplicationLogEntry : IApplicationLogEntry public string Message => LogRecord.Message; public string Category => LogRecord.Category; public DateTimeOffset Timestamp => LogRecord.Timestamp; - public FakeLogRecord LogRecord { get; init; } public override string ToString() => FormatLogRecord(LogRecord);