diff --git a/.github/workflows/publish-nuget.yml b/.github/workflows/publish-nuget.yml index 44a74afd4..62279b4e2 100644 --- a/.github/workflows/publish-nuget.yml +++ b/.github/workflows/publish-nuget.yml @@ -9,6 +9,6 @@ jobs: publish-nuget: name: Publish to NuGet if: ${{ !contains(github.ref_name, '-preview.') }} - uses: Lombiq/GitHub-Actions/.github/workflows/publish-nuget.yml@dev + uses: Lombiq/GitHub-Actions/.github/workflows/publish-nuget.yml@issue/OSOE-402 secrets: API_KEY: ${{ secrets.DEFAULT_NUGET_PUBLISH_API_KEY }} diff --git a/.github/workflows/validate-nuget-publish.yml b/.github/workflows/validate-nuget-publish.yml index 9f8979c5d..42549fccf 100644 --- a/.github/workflows/validate-nuget-publish.yml +++ b/.github/workflows/validate-nuget-publish.yml @@ -9,4 +9,4 @@ on: jobs: validate-nuget-publish: name: Validate NuGet Publish - uses: Lombiq/GitHub-Actions/.github/workflows/validate-nuget-publish.yml@dev + uses: Lombiq/GitHub-Actions/.github/workflows/validate-nuget-publish.yml@issue/OSOE-402 diff --git a/Lombiq.Tests.UI.Samples/Tests/DatabaseSnapshotTests.cs b/Lombiq.Tests.UI.Samples/Tests/DatabaseSnapshotTests.cs index a57dffc69..42f49b154 100644 --- a/Lombiq.Tests.UI.Samples/Tests/DatabaseSnapshotTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/DatabaseSnapshotTests.cs @@ -23,19 +23,19 @@ public DatabaseSnapshotTests(ITestOutputHelper testOutputHelper) // "ExecuteTestFromExistingDBAsync()" to run the test on that. Finally, we test the basic Orchard features to check // that the application was set up correctly. [Fact] - public Task BasicOrchardFeaturesShouldWorkWithExistingDatabase() => - ExecuteTestAsync( - async context => - { - var appForDatabaseTestFolder = Path.Combine("Temp", "AppForDatabaseTest"); + public async Task BasicOrchardFeaturesShouldWorkWithExistingDatabase() + { + var appForDatabaseTestFolder = Path.Combine("Temp", "AppForDatabaseTest"); - await context.GoToSetupPageAndSetupOrchardCoreAsync(RecipeIds.BasicOrchardFeaturesTests); - await context.Application.TakeSnapshotAsync(appForDatabaseTestFolder); + await ExecuteTestAsync( + async context => + { + await context.GoToSetupPageAndSetupOrchardCoreAsync(RecipeIds.BasicOrchardFeaturesTests); + await context.Application.TakeSnapshotAsync(appForDatabaseTestFolder); + }); - await ExecuteTestFromExistingDBAsync( - async context => await context.TestBasicOrchardFeaturesExceptSetupAsync(), - appForDatabaseTestFolder); - }); + await ExecuteTestFromExistingDBAsync(context => context.TestBasicOrchardFeaturesExceptSetupAsync(), appForDatabaseTestFolder); + } } // END OF TRAINING SECTION: Database snapshot tests. diff --git a/Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs b/Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs index 5c57a0d5f..dd0df24c5 100644 --- a/Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs @@ -38,7 +38,7 @@ public Task ServerSideErrorOnLoadedPageShouldHaltTest() => catch (PageChangeAssertionException) { // Remove all logs to have a clean slate. - context.ClearLogs(); + await context.ClearLogsAsync(); } }); diff --git a/Lombiq.Tests.UI.Samples/Tests/ShiftTimeTests.cs b/Lombiq.Tests.UI.Samples/Tests/ShiftTimeTests.cs index 45905c91d..226b5bbc7 100644 --- a/Lombiq.Tests.UI.Samples/Tests/ShiftTimeTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/ShiftTimeTests.cs @@ -10,11 +10,11 @@ namespace Lombiq.Tests.UI.Samples.Tests; -// When you enable the "Shift Time - Shortcuts - Lombiq UI Testing Toolbox" feature, it replaces OC's stock ICLock -// implementation with the custom ShiftTimeClock class. You can use the ~/Lombiq.Tests.UI.Samples/ShiftTime/Set?days=... -// action to update the ShiftTimeClock.Shift property for the current tenant, which will trick any service that uses -// IClock into thinking you are in the future. This can be used to test features that can expire, such as a limited-time -// product discount in a web store. +// When you enable the "Shift Time - Shortcuts - Lombiq UI Testing Toolbox" feature, it replaces Orchard Core's stock +// ICLock implementation with the custom ShiftTimeClock class. You can use the +// ~/Lombiq.Tests.UI.Samples/ShiftTime/Set?days=... action to update the ShiftTimeClock.Shift property for the current +// tenant, which will trick any service that uses IClock into thinking you are in the future. This can be used to test +// features that can expire, such as a limited-time product discount in a web store, without having to wait. public class ShiftTimeTests : UITestBase { public ShiftTimeTests(ITestOutputHelper testOutputHelper) diff --git a/Lombiq.Tests.UI.Samples/UITestBase.cs b/Lombiq.Tests.UI.Samples/UITestBase.cs index e8af21d8a..8d4d2cc63 100644 --- a/Lombiq.Tests.UI.Samples/UITestBase.cs +++ b/Lombiq.Tests.UI.Samples/UITestBase.cs @@ -2,6 +2,7 @@ using Lombiq.Tests.UI.Extensions; using Lombiq.Tests.UI.Samples.Helpers; using Lombiq.Tests.UI.Services; +using Microsoft.Extensions.Logging; using System; using System.Threading.Tasks; using Xunit.Abstractions; @@ -64,7 +65,8 @@ protected override Task ExecuteTestAsync( // https://docs.orchardcore.net/en/latest/docs/reference/core/Configuration/. We can set e.g. Orchard's // AdminUrlPrefix like below, but this is just setting the default, so it's only an example. A more // useful example is enabling offline operation of the Lombiq Hosting - Azure Application Insights for - // Orchard Core module (see https://github.com/Lombiq/Orchard-Azure-Application-Insights). + // Orchard Core module (see the UI test project in + // https://github.com/Lombiq/Orchard-Azure-Application-Insights). configuration.OrchardCoreConfiguration.BeforeAppStart += (_, argumentsBuilder) => { @@ -82,22 +84,41 @@ protected override Task ExecuteTestAsync( // Action) to further configure it. ////configuration.HtmlValidationConfiguration.RunHtmlValidationAssertionOnAllPageChanges = false; - // The UI Testing Toolbox can run several checks for the app even if you don't add explicit - // assertions: By default, the Orchard logs and the browser logs (where e.g. JavaScript errors show - // up) are checked and if there are any errors, the test will fail. You can also enable the checking of - // accessibility rules as we'll see later. Maybe not all of the default checks are suitable for you. - // Then it's simple to override them; here we change which log entries cause the tests to fail, and - // allow warnings and certain errors. - // Note that this is just for demonstration; you could use - // OrchardCoreUITestExecutorConfiguration.AssertAppLogsCanContainWarningsAndCacheFolderErrorsAsync which - // provides this configuration built-in. + // For locally running apps, the UI Testing Toolbox configures Fake Logging (see + // https://learn.microsoft.com/en-us/dotnet/api/microsoft.extensions.logging.testing). This provides an + // in-memory log for assertions, regardless of the logging framework your app uses otherwise. By + // default, it'll only collect log entries with the Error level and up. However, you can change this, + // and the usual Logging app settings still work. + configuration.OrchardCoreConfiguration.BeforeAppStart += + (_, argumentsBuilder) => + { + // This is how you can configure logging to be on the >=Error level in general, while still + // allowing info-level entries for ShellHost. + argumentsBuilder + .AddWithValue("Logging:LogLevel:Default", "Error") + .AddWithValue("Logging:LogLevel:System", "Error") + .AddWithValue("Logging:LogLevel:Microsoft", "Error") + .AddWithValue("Logging:LogLevel:OrchardCore.Environment.Shell.ShellHost", "Information"); + + return Task.CompletedTask; + }; + + // Enabling FakeLogger to collect the info-level ShellHost log entries configured above. These will show + // up in the test's output like "[Information] OrchardCore.Environment.Shell.ShellHost: Start creation + // of shells". + configuration.OrchardCoreConfiguration.AfterFakeLoggingConfiguration = + (_, fakeLogCollectorOptions) => fakeLogCollectorOptions.FilteredLevels.Add(LogLevel.Information); + + // Logging is important because the UI Testing Toolbox can run several checks for the app even if you + // don't add explicit assertions: By default, the Orchard logs and the browser logs (where e.g. + // JavaScript errors show up) are checked and if there are any errors, the test will fail. You can also + // enable the checking of accessibility rules as we'll see later. Maybe not all of the default checks + // are suitable for you. Then it's simple to override them; here we change which log entries cause the + // tests to fail, and allow certain log entries. configuration.AssertAppLogsAsync = webApplicationInstance => - webApplicationInstance.LogsShouldBeEmptyAsync( - canContainWarnings: true, - permittedErrorLinePatterns: - [ - "OrchardCore.Media.Core.DefaultMediaFileStoreCacheFileProvider|ERROR|Error deleting cache folder", - ]); + webApplicationInstance.LogsShouldNotContainAsync(logEntry => + // Allowing info-level log entries for ShellHost, see above. + logEntry.Message != "My permitted message." && logEntry.Level != LogLevel.Information); // Strictly speaking this is not necessary here, because we always use the same static method for setup. // However, if you used a dynamic setup operation (e.g. `context => SetupHelpers.RunSetupAsync(context, diff --git a/Lombiq.Tests.UI.Shortcuts/Controllers/ShiftTimeController.cs b/Lombiq.Tests.UI.Shortcuts/Controllers/ShiftTimeController.cs index e8bb22b1c..631b524b1 100644 --- a/Lombiq.Tests.UI.Shortcuts/Controllers/ShiftTimeController.cs +++ b/Lombiq.Tests.UI.Shortcuts/Controllers/ShiftTimeController.cs @@ -1,15 +1,12 @@ +using Lombiq.HelpfulLibraries.AspNetCore.Mvc; using Lombiq.Tests.UI.Shortcuts.Services; using Microsoft.AspNetCore.Mvc; using OrchardCore.Modules; using System; -using System.Diagnostics.CodeAnalysis; namespace Lombiq.Tests.UI.Shortcuts.Controllers; -[SuppressMessage( - "Major Code Smell", - "S6967:ModelState.IsValid should be called in controller actions", - Justification = "Not relevant in a test-only controller.")] +[DevelopmentAndLocalhostOnly] public class ShiftTimeController : Controller { private readonly IClock _clock; diff --git a/Lombiq.Tests.UI.Tests.UI/TestCases/SecurityShortcutsTestCases.cs b/Lombiq.Tests.UI.Tests.UI/TestCases/SecurityShortcutsTestCases.cs index 12b0cccf3..548f650e6 100644 --- a/Lombiq.Tests.UI.Tests.UI/TestCases/SecurityShortcutsTestCases.cs +++ b/Lombiq.Tests.UI.Tests.UI/TestCases/SecurityShortcutsTestCases.cs @@ -48,7 +48,7 @@ public static Task AddUserToFakeRoleShouldThrowAsync( await context.CreateUserAsync(UserUserName, DefaultUser.Password, UserEmail); await context.AddUserToRoleAsync(UserUserName, FakeRole).ShouldThrowAsync(); - context.ClearLogs(); + await context.ClearLogsAsync(); }, browser, ConfigurationHelper.DisableHtmlValidation); @@ -61,7 +61,7 @@ public static Task AllowFakePermissionToRoleShouldThrowAsync( await context.AddPermissionToRoleAsync(FakePermission, AuthorRole) .ShouldThrowAsync(); - context.ClearLogs(); + await context.ClearLogsAsync(); }, browser, ConfigurationHelper.DisableHtmlValidation); diff --git a/Lombiq.Tests.UI/CompatibilitySuppressions.xml b/Lombiq.Tests.UI/CompatibilitySuppressions.xml new file mode 100644 index 000000000..991a8ef7a --- /dev/null +++ b/Lombiq.Tests.UI/CompatibilitySuppressions.xml @@ -0,0 +1,144 @@ + + + + + CP0002 + F:Lombiq.Tests.UI.Services.OrchardCoreUITestExecutorConfiguration.AssertAppLogsCanContainWarningsAndCacheFolderErrorsAsync + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + F:Lombiq.Tests.UI.Services.OrchardCoreUITestExecutorConfiguration.AssertAppLogsCanContainWarningsAsync + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Extensions.WebApplicationInstanceExtensions.GetLogOutputAsync(Lombiq.Tests.UI.Services.IWebApplicationInstance,System.Threading.CancellationToken) + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Extensions.WebApplicationInstanceExtensions.LogsShouldBeEmptyAsync(Lombiq.Tests.UI.Services.IWebApplicationInstance,System.Boolean,System.Collections.Generic.ICollection{System.String},System.Threading.CancellationToken) + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.IApplicationLog.GetContentAsync + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.IApplicationLog.Remove + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.IWebApplicationInstance.GetLogs(System.Threading.CancellationToken) + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.IWebApplicationInstance.GetRequiredService``1 + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.OrchardCoreInstance`1.GetLogs(System.Threading.CancellationToken) + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.OrchardCoreInstance`1.GetRequiredService``1 + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.OrchardCoreUITestExecutorConfiguration.AssertAppLogsMaybeAsync(Lombiq.Tests.UI.Services.IWebApplicationInstance,System.Action{System.String}) + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.OrchardCoreUITestExecutorConfiguration.AssertBrowserLogMaybe(System.Collections.Generic.IList{OpenQA.Selenium.LogEntry},System.Action{System.String}) + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.OrchardCoreUITestExecutorConfiguration.CreateAppLogAssertionForSecurityScan(System.String[]) + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.OrchardCoreUITestExecutorConfiguration.UseAssertAppLogsForSecurityScan(System.String[]) + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.RemoteInstance.GetLogs(System.Threading.CancellationToken) + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.UITestContext.ClearLogs(System.Threading.CancellationToken) + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0006 + M:Lombiq.Tests.UI.Services.IApplicationLog.GetEntriesAsync + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0006 + M:Lombiq.Tests.UI.Services.IApplicationLog.RemoveAsync + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0006 + M:Lombiq.Tests.UI.Services.IWebApplicationInstance.GetLogsAsync(System.Threading.CancellationToken) + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0006 + P:Lombiq.Tests.UI.Services.IApplicationLog.EntryCount + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + \ No newline at end of file diff --git a/Lombiq.Tests.UI/Docs/Troubleshooting.md b/Lombiq.Tests.UI/Docs/Troubleshooting.md index 8d6006bad..c944e8525 100644 --- a/Lombiq.Tests.UI/Docs/Troubleshooting.md +++ b/Lombiq.Tests.UI/Docs/Troubleshooting.md @@ -7,7 +7,7 @@ - Browser logs, i.e. the developer console output. - Screenshots of each page in order the test visited them, as well as when the test failed (Windows Photo Viewer won't be able to open it though, use something else like the Windows 10 Photos app). - The HTML output on the page the test failed. - - Any direct output of the test (like the exception thrown) as well as a log of the operations it completed. + - Any direct output of the test (like the exception thrown) as well as a log of the operations it completed (including browser interactions and corresponding Orchard Core log entries). - If accessibility was checked and asserting it failed then an accessibility report will be included too. - If HTML validation was done and asserting it failed then an HTML validation report will be included too. - Run tests with the debugger attached to stop where the test fails. This way you can take a good look at what's in the driven browser window so you can examine the web page and you can debug the web app under test at the same time. diff --git a/Lombiq.Tests.UI/Extensions/ApplicationLogEnumerableExtensions.cs b/Lombiq.Tests.UI/Extensions/ApplicationLogEnumerableExtensions.cs index 5e063fe7b..ebfea111c 100644 --- a/Lombiq.Tests.UI/Extensions/ApplicationLogEnumerableExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/ApplicationLogEnumerableExtensions.cs @@ -8,8 +8,23 @@ namespace Lombiq.Tests.UI.Extensions; public static class ApplicationLogEnumerableExtensions { - public static async Task ToFormattedStringAsync(this IEnumerable logs) => - string.Join( - Environment.NewLine + Environment.NewLine, - await Task.WhenAll(logs.Select(async log => log.Name + Environment.NewLine + Environment.NewLine + await log.GetContentAsync()))); + public static async Task ToFormattedStringAsync(this IEnumerable logs) + { + var logsArray = logs.ToArray(); + + if (logsArray.Length == 1) + { + return Environment.NewLine + await LogLinesToFormattedStringAsync(logsArray[0]); + } + + // Parallelization with Task.WhenAll() isn't really necessary for performance here but would potentially change + // the order of the logs in the output. + var logContents = logsArray.AwaitEachAsync(async log => + $"# Log name: {log.Name}" + Environment.NewLine + Environment.NewLine + await LogLinesToFormattedStringAsync(log)); + + return string.Join(Environment.NewLine + Environment.NewLine, logContents); + } + + private static async Task LogLinesToFormattedStringAsync(IApplicationLog log) => + string.Join(Environment.NewLine, (await log.GetEntriesAsync()).Select(logEntry => logEntry.ToString())); } diff --git a/Lombiq.Tests.UI/Extensions/BrowserUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/BrowserUITestContextExtensions.cs index 9e43cec34..f8e8f89e4 100644 --- a/Lombiq.Tests.UI/Extensions/BrowserUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/BrowserUITestContextExtensions.cs @@ -1,4 +1,4 @@ -using Lombiq.Tests.UI.Extensions; +using Lombiq.Tests.UI.Extensions; using Lombiq.Tests.UI.Services; using System.Diagnostics.CodeAnalysis; using System.Net.Http; @@ -87,7 +87,7 @@ public static async Task DoWithoutAppLogAssertionAsync(this UITestContext } catch { - context.ClearLogs(); + await context.ClearLogsAsync(); return true; } diff --git a/Lombiq.Tests.UI/Extensions/ShortcutsUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/ShortcutsUITestContextExtensions.cs index e82597740..ba21f77ba 100644 --- a/Lombiq.Tests.UI/Extensions/ShortcutsUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/ShortcutsUITestContextExtensions.cs @@ -733,5 +733,5 @@ public static Task SetShiftTimeAsync(this UITestContext context, double days = 0 /// provided, then the values for both are added. Negative values are supported as well. /// public static Task AddShiftTimeAsync(this UITestContext context, double days = 0, double seconds = 0) => - context.GoToAsync(controller => controller.Set(days, seconds)); + context.GoToAsync(controller => controller.Add(days, seconds)); } diff --git a/Lombiq.Tests.UI/Extensions/VerificationUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/VerificationUITestContextExtensions.cs index 121ae1b03..ab2a509a3 100644 --- a/Lombiq.Tests.UI/Extensions/VerificationUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/VerificationUITestContextExtensions.cs @@ -29,7 +29,7 @@ public static async Task AssertLogsAsync(this UITestContext context) catch (Exception) { testOutputHelper.WriteLine("Application logs: " + Environment.NewLine); - testOutputHelper.WriteLine(await context.Application.GetLogOutputAsync()); + testOutputHelper.WriteLine(await context.Application.GetLogContentsAsync()); throw; } diff --git a/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs b/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs index df5775113..2dda322b2 100644 --- a/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs @@ -1,9 +1,10 @@ using Lombiq.Tests.UI.Services; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Shouldly; using System; using System.Collections.Generic; -using System.Linq; -using System.Text.RegularExpressions; +using System.Linq.Expressions; using System.Threading; using System.Threading.Tasks; @@ -12,54 +13,135 @@ namespace Lombiq.Tests.UI.Extensions; public static class WebApplicationInstanceExtensions { /// - /// Asserting that the logs should be empty. When they aren't the Shouldly exception will contain the logs' - /// contents. + /// Asserts that the logs should be empty, i.e. contain no entries. If the assertion fails, the Shouldly exception + /// will contain all log entries. /// - /// - /// If not or empty, each line is split and any lines containing |ERROR| will be - /// ignored if they regex match any string from this collection (case-insensitive). - /// + /// + /// + /// If you want to inspect the logs in a more structured way, message by message, consider using directly instead. Alternatively, set log + /// filtering options to not log unwanted messages in first place with the standard Logging:LogLevel app + /// configuration (see the samples). + /// + /// + /// A that can cancel the log retrieval. public static async Task LogsShouldBeEmptyAsync( this IWebApplicationInstance webApplicationInstance, - bool canContainWarnings = false, - ICollection permittedErrorLinePatterns = null, CancellationToken cancellationToken = default) { - permittedErrorLinePatterns ??= []; - - var logOutput = await webApplicationInstance.GetLogOutputAsync(cancellationToken); - - logOutput.ShouldNotContain("|FATAL|"); - - var lines = logOutput.SplitByNewLines(); - - var errorLines = lines.Where(line => line.Contains("|ERROR|")); + var logs = await webApplicationInstance.GetLogsAsync(cancellationToken); + logs.ShouldNotContain(log => log.EntryCount > 0, await logs.ToFormattedStringAsync()); + } - if (permittedErrorLinePatterns.Count != 0) - { - errorLines = errorLines.Where(line => - !permittedErrorLinePatterns.Any(pattern => Regex.IsMatch(line, pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled))); - } + /// + /// Asserts that the logs should contain any entry matching the given predicate. If the assertion fails, the + /// Shouldly exception will contain all log entries. + /// + /// + /// + /// If you want to inspect the logs in a more structured way, message by message, consider using directly instead. Alternatively, set log + /// filtering options to not log unwanted messages in first place with the standard Logging:LogLevel app + /// configuration (see the samples). + /// + /// + /// A that can cancel the log retrieval. + /// + /// A predicate that when returns , the assertion will fail. + /// + public static Task LogsShouldContainAsync( + this IWebApplicationInstance webApplicationInstance, + Expression> logEntryPredicate, + CancellationToken cancellationToken = default) => + AssertLogsAsync(webApplicationInstance, logEntryPredicate, ShouldBeEnumerableTestExtensions.ShouldContain, cancellationToken); - errorLines.ShouldBeEmpty(); + /// + /// Asserts that the logs should NOT contain any entries with and above. If the + /// assertion fails, the Shouldly exception will contain all log entries. + /// + /// + /// + /// If you want to inspect the logs in a more structured way, message by message, consider using directly instead. Alternatively, set log + /// filtering options to not log unwanted messages in first place with the standard Logging:LogLevel app + /// configuration (see the samples). + /// + /// + /// A that can cancel the log retrieval. + public static Task LogsShouldNotContainErrorsAsync( + this IWebApplicationInstance webApplicationInstance, + CancellationToken cancellationToken = default) => + AssertLogsAsync( + webApplicationInstance, + logEntry => logEntry.Level > LogLevel.Error, + ShouldBeEnumerableTestExtensions.ShouldNotContain, + cancellationToken); - if (!canContainWarnings) - { - lines.Where(line => line.Contains("|WARNING|")).ShouldBeEmpty(); - } - } + /// + /// Asserts that the logs should NOT contain any entries matching the given predicate. If the assertion fails, the + /// Shouldly exception will contain all log entries. + /// + /// + /// + /// If you want to inspect the logs in a more structured way, message by message, consider using directly instead. Alternatively, set log + /// filtering options to not log unwanted messages in first place with the standard Logging:LogLevel app + /// configuration (see the samples). + /// + /// + /// A that can cancel the log retrieval. + /// + /// A predicate that when returns , the assertion will fail. + /// + public static Task LogsShouldNotContainAsync( + this IWebApplicationInstance webApplicationInstance, + Expression> logEntryPredicate, + CancellationToken cancellationToken = default) => + AssertLogsAsync(webApplicationInstance, logEntryPredicate, ShouldBeEnumerableTestExtensions.ShouldNotContain, cancellationToken); /// /// Retrieves all the logs and concatenates them into a single formatted string. /// - public static async Task GetLogOutputAsync( + /// + /// + /// If you want to inspect the logs in a more structured way, message by message, consider using directly instead. Alternatively, set log + /// filtering options to not log unwanted messages in first place with the standard Logging:LogLevel app + /// configuration (see the samples). + /// + /// + /// A that can cancel the log retrieval. + public static async Task GetLogContentsAsync( this IWebApplicationInstance webApplicationInstance, CancellationToken cancellationToken = default) { if (cancellationToken == default) cancellationToken = CancellationToken.None; - return string.Join( - Environment.NewLine + Environment.NewLine, - await webApplicationInstance.GetLogs(cancellationToken).ToFormattedStringAsync()); + return await (await webApplicationInstance.GetLogsAsync(cancellationToken)).ToFormattedStringAsync(); + } + + /// + /// Get service of type . + /// + /// The type of service service to get. + /// An instance of the service of type . + /// + /// There is no service of type . + /// + public static TService GetRequiredService(this IWebApplicationInstance webApplicationInstance) => + webApplicationInstance.Services.GetRequiredService(); + + private static async Task AssertLogsAsync( + IWebApplicationInstance webApplicationInstance, + Expression> logEntryPredicate, + Action, Expression>, string> shouldlyMethod, // #spell-check-ignore-line + CancellationToken cancellationToken = default) + { + var logs = await webApplicationInstance.GetLogsAsync(cancellationToken); + + foreach (var log in logs) + { + shouldlyMethod(await log.GetEntriesAsync(), logEntryPredicate, await logs.ToFormattedStringAsync()); // #spell-check-ignore-line + } } } diff --git a/Lombiq.Tests.UI/Helpers/AppLogAssertionHelper.cs b/Lombiq.Tests.UI/Helpers/AppLogAssertionHelper.cs new file mode 100644 index 000000000..5ffc5b682 --- /dev/null +++ b/Lombiq.Tests.UI/Helpers/AppLogAssertionHelper.cs @@ -0,0 +1,23 @@ +using Lombiq.Tests.UI.Services; +using System; +using System.Linq.Expressions; + +namespace Lombiq.Tests.UI.Helpers; + +public static class AppLogAssertionHelper +{ + /// + /// An wrapping . + /// + public static readonly Expression> NotMediaCacheEntriesPredicate = + logEntry => NotMediaCacheEntries(logEntry); + + /// + /// Creates a predicate that can be used to filter out "Error deleting cache folder" log entries coming from + /// DefaultMediaFileStoreCacheFileProvider. These errors frequently happen during UI testing when using Azure + /// Blob Storage for media storage. They're harmless, though. + /// + public static bool NotMediaCacheEntries(IApplicationLogEntry logEntry) => + logEntry.Category != "OrchardCore.Media.Core.DefaultMediaFileStoreCacheFileProvider" || + !logEntry.Message.StartsWithOrdinalIgnoreCase("Error deleting cache folder"); +} diff --git a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj index 3fc27d706..df2db5492 100644 --- a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj +++ b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj @@ -74,6 +74,7 @@ + diff --git a/Lombiq.Tests.UI/Models/FakeLoggerLogApplicationLog.cs b/Lombiq.Tests.UI/Models/FakeLoggerLogApplicationLog.cs new file mode 100644 index 000000000..d6093839f --- /dev/null +++ b/Lombiq.Tests.UI/Models/FakeLoggerLogApplicationLog.cs @@ -0,0 +1,46 @@ +using Lombiq.HelpfulLibraries.Common.Utilities; +using Lombiq.Tests.UI.Services; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Testing; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Lombiq.Tests.UI.Models; + +public sealed class FakeLoggerLogApplicationLog : IApplicationLog +{ + public string Name => "FakeLog"; + public FakeLogCollector LogCollector { get; init; } + public int EntryCount => LogCollector.Count; + + public Task> GetEntriesAsync() + { + var records = LogCollector.GetSnapshot(); + + return Task.FromResult(records.Select(record => (IApplicationLogEntry)new FakeLoggerApplicationLogEntry(record))); + } + + public Task RemoveAsync() + { + LogCollector.Clear(); + return Task.CompletedTask; + } +} + +public record FakeLoggerApplicationLogEntry(FakeLogRecord LogRecord) : IApplicationLogEntry +{ + public LogLevel Level => LogRecord.Level; + public EventId Id => LogRecord.Id; + public Exception Exception => LogRecord.Exception; + public string Message => LogRecord.Message; + public string Category => LogRecord.Category; + public DateTimeOffset Timestamp => LogRecord.Timestamp; + + public override string ToString() => FormatLogRecord(LogRecord); + + public static string FormatLogRecord(FakeLogRecord record) => + StringHelper.CreateInvariant($"{record.Timestamp:yyyy-MM-dd HH:mm:ss.fff} [{record.Level}] {record.Category}: {record.Message}") + + (record.Exception != null ? record.Exception.ToString() : string.Empty); +} diff --git a/Lombiq.Tests.UI/SecurityScanning/OrchardCoreUITestExecutorConfigurationExtensions.cs b/Lombiq.Tests.UI/SecurityScanning/OrchardCoreUITestExecutorConfigurationExtensions.cs new file mode 100644 index 000000000..9a56dc7b3 --- /dev/null +++ b/Lombiq.Tests.UI/SecurityScanning/OrchardCoreUITestExecutorConfigurationExtensions.cs @@ -0,0 +1,72 @@ +using Lombiq.Tests.UI.Extensions; +using Lombiq.Tests.UI.Helpers; +using Lombiq.Tests.UI.Services; +using Lombiq.Tests.UI.Shortcuts.Controllers; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace Lombiq.Tests.UI.SecurityScanning; + +public static class OrchardCoreUITestExecutorConfigurationExtensions +{ + /// + /// Sets the to the output of so it accepts errors in the log caused by the security scanning. + /// + public static OrchardCoreUITestExecutorConfiguration UseAssertAppLogsForSecurityScan( + this OrchardCoreUITestExecutorConfiguration configuration, + params string[] additionalPermittedErrorLinePatterns) + { + configuration.AssertAppLogsAsync = CreateAppLogAssertionForSecurityScan(additionalPermittedErrorLinePatterns); + + return configuration; + } + + /// + /// Similar to , + /// but also permits certain log messages which represent correct reactions to + /// incorrect or malicious user behavior during a security scan. + /// + public static Func CreateAppLogAssertionForSecurityScan(params string[] additionalPermittedErrorLinePatterns) + { + var permittedErrorLinePatterns = new List + { + // The model binding will throw FormatException exception with this text during ZAP active scan, when the + // bot tries to send malicious query strings or POST data that doesn't fit the types expected by the model. + // This is correct, safe behavior and should be logged in production. + "is not a valid value for Boolean", + "An unhandled exception has occurred while executing the request. System.FormatException: any", + "System.FormatException: The input string '[\\S\\s]+' was not in a correct format.", + "System.FormatException: The input string 'any", + // Happens when the static file middleware tries to access a path that doesn't exist or access a file as a + // directory. Presumably this is an attempt to access protected files using source path manipulation. This + // is handled by ASP.NET Core and there is nothing for us to worry about. + "System.IO.IOException: Not a directory", + "System.IO.IOException: The filename, directory name, or volume label syntax is incorrect", + "System.IO.DirectoryNotFoundException: Could not find a part of the path", + // This happens when a request's model contains a dictionary and a key is missing. While this can be a + // legitimate application error, during a security scan it's more likely the result of an incomplete + // artificially constructed request. So the means the ASP.NET Core model binding is working as intended. + "An unhandled exception has occurred while executing the request. System.ArgumentNullException: Value cannot be null. (Parameter 'key')", + // One way to verify correct error handling is to navigate to ~/Lombiq.Tests.UI.Shortcuts/Error/Index, which + // always throws an exception. This also gets logged but it's expected, so it should be ignored. + ErrorController.ExceptionMessage, + // Thrown from Microsoft.AspNetCore.Authentication.AuthenticationService.ChallengeAsync() when ZAP sends + // invalid authentication challenges. + "System.InvalidOperationException: No authentication handler is registered for the scheme", + }; + + permittedErrorLinePatterns.AddRange(additionalPermittedErrorLinePatterns); + + return app => + app.LogsShouldNotContainAsync(logEntry => + !permittedErrorLinePatterns.Any(pattern => + Regex.IsMatch(logEntry.ToString(), pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled)) && + AppLogAssertionHelper.NotMediaCacheEntries(logEntry) && + logEntry.Level >= LogLevel.Error); + } +} diff --git a/Lombiq.Tests.UI/SecurityScanning/SecurityScanningUITestContextExtensions.cs b/Lombiq.Tests.UI/SecurityScanning/SecurityScanningUITestContextExtensions.cs index 3af26f6a2..947686bcf 100644 --- a/Lombiq.Tests.UI/SecurityScanning/SecurityScanningUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/SecurityScanning/SecurityScanningUITestContextExtensions.cs @@ -57,9 +57,9 @@ public static Task RunAndAssertFullSecurityScanAsync( /// This extension method makes changes to the normal configuration of the test to be more suited for CI operation. /// It changes the to not do any retries because this is a long running /// test. It also replaces the app log assertion logic with the specialized version for security scans, . The scan is configured t - /// ignore the admin dashboard, optionally log in as admin, and use the provided time limits for the "active scan" - /// portion of the security scan. + /// cref="OrchardCoreUITestExecutorConfigurationExtensions.UseAssertAppLogsForSecurityScan"/>. The scan is + /// configured to ignore the admin dashboard, optionally log in as admin, and use the provided time limits for the + /// "active scan" portion of the security scan. /// public static Task RunAndConfigureAndAssertFullSecurityScanForContinuousIntegrationAsync( this UITestContext context, diff --git a/Lombiq.Tests.UI/Services/IApplicationLog.cs b/Lombiq.Tests.UI/Services/IApplicationLog.cs new file mode 100644 index 000000000..36f1f9731 --- /dev/null +++ b/Lombiq.Tests.UI/Services/IApplicationLog.cs @@ -0,0 +1,69 @@ +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace Lombiq.Tests.UI.Services; + +/// +/// An abstraction over a log, be it in the form of a file or something else. +/// +public interface IApplicationLog +{ + /// + /// Gets the name of the log, such as the file name. + /// + string Name { get; } + + /// + /// Gets the number of log entries in the log. + /// + int EntryCount { get; } + + /// + /// Returns the content of the log, in case of log files reads the file contents. + /// + /// The contents. + Task> GetEntriesAsync(); + + /// + /// Removes the log if possible. + /// + Task RemoveAsync(); +} + +/// +/// An abstraction over a log entries. +/// +public interface IApplicationLogEntry +{ + /// + /// Gets the level of the log entry, like or . + /// + LogLevel Level { get; } + + /// + /// Gets the ID that uniquely identifies the log entry. + /// + EventId Id { get; } + + /// + /// Gets the exception associated with the log entry, if any. + /// + Exception Exception { get; } + + /// + /// Gets the human-readable formatted log message. + /// + string Message { get; } + + /// + /// Gets the category of the log entry. This is the type parameter of . + /// + string Category { get; } + + /// + /// Gets the timestamp of when the log entry was created. + /// + DateTimeOffset Timestamp { get; } +} diff --git a/Lombiq.Tests.UI/Services/IWebApplicationInstance.cs b/Lombiq.Tests.UI/Services/IWebApplicationInstance.cs index 994d970e3..0aee1d96a 100644 --- a/Lombiq.Tests.UI/Services/IWebApplicationInstance.cs +++ b/Lombiq.Tests.UI/Services/IWebApplicationInstance.cs @@ -43,37 +43,5 @@ public interface IWebApplicationInstance : IAsyncDisposable /// Reads all the application logs. /// /// The collection of log names and their contents. - IEnumerable GetLogs(CancellationToken cancellationToken = default); - - /// - /// Get service of type . - /// - /// The type of service object to get. - /// A service object of type . - /// - /// There is no service of type . - /// - TService GetRequiredService(); -} - -/// -/// An abstraction over a log, be it in the form of a file or something else. -/// -public interface IApplicationLog -{ - /// - /// Gets the name of the log, such as the file name. - /// - string Name { get; } - - /// - /// Returns the content of the log, in case of log files reads the file contents. - /// - /// The contents. - Task GetContentAsync(); - - /// - /// Removes the log if possible. - /// - void Remove(); + Task> GetLogsAsync(CancellationToken cancellationToken = default); } diff --git a/Lombiq.Tests.UI/Services/OrchardCoreHosting/OrchardApplicationFactory.cs b/Lombiq.Tests.UI/Services/OrchardCoreHosting/OrchardApplicationFactory.cs index 5f90ad820..d0cf6b5bd 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreHosting/OrchardApplicationFactory.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreHosting/OrchardApplicationFactory.cs @@ -9,10 +9,8 @@ using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Hosting; using NLog; -using NLog.Web; using System; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Threading.Tasks; using YesSql; @@ -71,20 +69,11 @@ protected override IHost CreateHost(IHostBuilder builder) protected override void ConfigureWebHost(IWebHostBuilder builder) { - builder.ConfigureTestServices(ConfigureTestServices) - .ConfigureLogging((context, loggingBuilder) => - { - var environment = context.HostingEnvironment; - var nLogConfig = Path.Combine(environment.ContentRootPath, "NLog.config"); - var factory = new LogFactory() - .Setup() - .LoadConfigurationFromFile(nLogConfig) - .LogFactory; - - factory.Configuration.Variables["configDir"] = environment.ContentRootPath; - - loggingBuilder.AddNLogWeb(factory, new NLogAspNetCoreOptions { ReplaceLoggerFactory = true }); - }); + builder + .ConfigureTestServices(ConfigureTestServices) + // NLog, if used, will put log files into configDir. Not setting this would use the default, which would be + // App_Data/App_Data/logs. + .ConfigureLogging((context, _) => LogManager.Configuration.Variables["configDir"] = context.HostingEnvironment.ContentRootPath); _configuration?.Invoke(builder); } @@ -95,12 +84,13 @@ private void ConfigureTestServices(IServiceCollection services) .LastOrDefault(descriptor => descriptor.ServiceType == typeof(OrchardCoreBuilder))? .ImplementationInstance as OrchardCoreBuilder ?? throw new InvalidOperationException( - "Please call WebApplicationBuilder.Services.AddOrchardCms() in your Program.cs!"); + "Please call WebApplicationBuilder.Services.AddOrchardCms() in your Program.cs."); var configuration = services .LastOrDefault(descriptor => descriptor.ServiceType == typeof(ConfigurationManager))? .ImplementationInstance as ConfigurationManager ?? throw new InvalidOperationException( - $"Please add {nameof(ConfigurationManager)} instance to WebApplicationBuilder.Services in your Program.cs!"); + $"Please register the {nameof(ConfigurationManager)} instance in the Service Collection in your " + + "Program.cs, following the documentation."); _configureOrchard?.Invoke(configuration, builder); diff --git a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs index e8f856a80..c7d981b95 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs @@ -8,6 +8,8 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Testing; using Microsoft.VisualBasic.FileIO; using System; using System.Collections.Generic; @@ -20,14 +22,15 @@ namespace Lombiq.Tests.UI.Services; public delegate Task BeforeAppStartHandler(OrchardCoreAppStartContext context, InstanceCommandLineArgumentsBuilder arguments); +public delegate void AfterFakeLoggingConfigurationHandler(OrchardCoreAppStartContext context, FakeLogCollectorOptions fakeLogCollectorOptions); public delegate Task AfterAppStopHandler(OrchardCoreAppStartContext context); - public delegate Task BeforeTakeSnapshotHandler(OrchardCoreAppStartContext context, string snapshotDirectoryPath); public class OrchardCoreConfiguration { public string SnapshotDirectoryPath { get; set; } public BeforeAppStartHandler BeforeAppStart { get; set; } + public AfterFakeLoggingConfigurationHandler AfterFakeLoggingConfiguration { get; set; } public AfterAppStopHandler AfterAppStop { get; set; } public BeforeTakeSnapshotHandler BeforeTakeSnapshot { get; set; } public int StartCount { get; internal set; } @@ -56,6 +59,7 @@ public sealed class OrchardCoreInstance : IWebApplicationInstance private readonly OrchardCoreConfiguration _configuration; private readonly string _contextId; private readonly ITestOutputHelper _testOutputHelper; + private string _contentRootPath; private bool _isDisposed; private OrchardApplicationFactory _orchardApplication; @@ -113,23 +117,22 @@ public Task TakeSnapshotAsync(string snapshotDirectoryPath) return TakeSnapshotInnerAsync(snapshotDirectoryPath); } - public IEnumerable GetLogs(CancellationToken cancellationToken = default) + public Task> GetLogsAsync(CancellationToken cancellationToken = default) { - var logFolderPath = Path.Combine(_contentRootPath, "App_Data", "logs"); - return Directory.Exists(logFolderPath) - ? Directory - .EnumerateFiles(logFolderPath, "*.log") - .Select(filePath => (IApplicationLog)new ApplicationLog + if (Services.GetService() is { } logCollector) + { + return Task.FromResult( + new IApplicationLog[] { - Name = Path.GetFileName(filePath), - FullName = Path.GetFullPath(filePath), - ContentLoader = () => GetFileContentAsync(filePath, cancellationToken), - }) - : []; - } + new FakeLoggerLogApplicationLog + { + LogCollector = logCollector, + }, + }.AsEnumerable()); + } - public TService GetRequiredService() => - _orchardApplication.Services.GetRequiredService(); + return Task.FromResult(Enumerable.Empty()); + } public async ValueTask DisposeAsync() { @@ -172,7 +175,17 @@ await _configuration.BeforeAppStart builder => builder .UseContentRoot(_contentRootPath) .UseWebRoot(Path.Combine(_contentRootPath, "wwwroot")) - .UseEnvironment(Environments.Development), + .UseEnvironment(Environments.Development) + .ConfigureLogging(loggingBuilder => loggingBuilder.AddFakeLogging(options => + { + options.CollectRecordsForDisabledLogLevels = false; + options.FilteredLevels.Add(LogLevel.Error); + options.FilteredLevels.Add(LogLevel.Critical); + options.OutputFormatter = FakeLoggerApplicationLogEntry.FormatLogRecord; + options.OutputSink += message => _testOutputHelper.WriteLine(message); + + _configuration.AfterFakeLoggingConfiguration?.Invoke(CreateAppStartContext(), options); + })), (configuration, orchardBuilder) => orchardBuilder .ConfigureUITesting(configuration, enableShortcutsDuringUITesting: true)); @@ -200,13 +213,6 @@ await _configuration.AfterAppStop .InvokeAsync(handler => handler(CreateAppStartContext())); } - private static async Task GetFileContentAsync(string filePath, CancellationToken cancellationToken) - { - await using var fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); - using var streamReader = new StreamReader(fileStream); - return await streamReader.ReadToEndAsync(cancellationToken); - } - private async Task TakeSnapshotInnerAsync(string snapshotDirectoryPath) { await PauseAsync(); @@ -219,22 +225,10 @@ await _configuration.BeforeTakeSnapshot .InvokeAsync(handler => handler(CreateAppStartContext(), snapshotDirectoryPath)); FileSystem.CopyDirectory(_contentRootPath, snapshotDirectoryPath, overwrite: true); + + await ResumeAsync(); } private OrchardCoreAppStartContext CreateAppStartContext() => new(_contentRootPath, _url, OrchardCoreInstanceCounter.PortLeases); - - private sealed class ApplicationLog : IApplicationLog - { - public string Name { get; init; } - public string FullName { get; init; } - public Func> ContentLoader { get; init; } - - public Task GetContentAsync() => ContentLoader(); - - public void Remove() - { - if (File.Exists(FullName)) File.Delete(FullName); - } - } } diff --git a/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs b/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs index 060685017..54cff1c0a 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs @@ -1,7 +1,7 @@ using Lombiq.Tests.UI.Extensions; +using Lombiq.Tests.UI.Helpers; using Lombiq.Tests.UI.SecurityScanning; using Lombiq.Tests.UI.Services.GitHub; -using Lombiq.Tests.UI.Shortcuts.Controllers; using OpenQA.Selenium; using Shouldly; using System; @@ -30,21 +30,11 @@ public enum Browser public class OrchardCoreUITestExecutorConfiguration { - // These errors frequently happen during UI testing when using Azure Blob Storage for media storage. - // They're harmless, though. - private const string CacheFolderErrorPattern = - "OrchardCore.Media.Core.DefaultMediaFileStoreCacheFileProvider|ERROR|Error deleting cache folder"; - public static readonly Func AssertAppLogsAreEmptyAsync = app => app.LogsShouldBeEmptyAsync(); - public static readonly Func AssertAppLogsCanContainWarningsAsync = - app => app.LogsShouldBeEmptyAsync(canContainWarnings: true); - - public static readonly Func AssertAppLogsCanContainWarningsAndCacheFolderErrorsAsync = - app => app.LogsShouldBeEmptyAsync( - canContainWarnings: true, - permittedErrorLinePatterns: [CacheFolderErrorPattern]); + public static readonly Func AssertAppLogsCanContainCacheFolderErrorsAsync = + app => app.LogsShouldNotContainAsync(AppLogAssertionHelper.NotMediaCacheEntriesPredicate); public static readonly Action> AssertBrowserLogIsEmpty = logEntries => logEntries.ShouldNotContain( @@ -117,7 +107,7 @@ public class OrchardCoreUITestExecutorConfiguration ? intValue : Environment.ProcessorCount; - public Func AssertAppLogsAsync { get; set; } = AssertAppLogsCanContainWarningsAsync; + public Func AssertAppLogsAsync { get; set; } = AssertAppLogsCanContainCacheFolderErrorsAsync; public Action> AssertBrowserLog { get; set; } = AssertBrowserLogIsEmpty; public ITestOutputHelper TestOutputHelper { get; set; } @@ -128,9 +118,8 @@ public class OrchardCoreUITestExecutorConfiguration /// /// /// - /// For this to properly work the build artifacts should be configured to contain the TestDumps folder (it can - /// also contain other folders but it must contain a folder called "TestDumps", e.g.: +:TestDumps => - /// TestDumps. + /// For this to properly work the build artifacts should be configured to contain the TestDumps folder (it can also + /// contain other folders but it must contain a folder called "TestDumps", e.g.: +:TestDumps => TestDumps. /// /// public bool ReportTeamCityMetadata { get; set; } = @@ -196,88 +185,4 @@ public class OrchardCoreUITestExecutorConfiguration /// enabled in the app for these to work. /// public ShortcutsConfiguration ShortcutsConfiguration { get; set; } = new(); - - public async Task AssertAppLogsMaybeAsync(IWebApplicationInstance instance, Action log) - { - if (instance == null || AssertAppLogsAsync == null) return; - - try - { - await AssertAppLogsAsync(instance); - } - catch (Exception) - { - log("Application logs: " + Environment.NewLine); - log(await instance.GetLogOutputAsync()); - - throw; - } - } - - public void AssertBrowserLogMaybe(IList browserLogs, Action log) - { - if (AssertBrowserLog == null) return; - - try - { - AssertBrowserLog(browserLogs); - } - catch (Exception) - { - log("Browser logs: " + Environment.NewLine); - log(browserLogs.ToFormattedString()); - - throw; - } - } - - /// - /// Sets the to the output of so - /// it accepts errors in the log caused by the security scanning. - /// - public OrchardCoreUITestExecutorConfiguration UseAssertAppLogsForSecurityScan(params string[] additionalPermittedErrorLinePatterns) - { - AssertAppLogsAsync = CreateAppLogAssertionForSecurityScan(additionalPermittedErrorLinePatterns); - - return this; - } - - /// - /// Similar to , but also permits certain |ERROR log - /// entries which represent correct reactions to incorrect or malicious user behavior during a security scan. - /// - public static Func CreateAppLogAssertionForSecurityScan(params string[] additionalPermittedErrorLinePatterns) - { - var permittedErrorLinePatterns = new List - { - // The model binding will throw FormatException exception with this text during ZAP active scan, when - // the bot tries to send malicious query strings or POST data that doesn't fit the types expected by the - // model. This is correct, safe behavior and should be logged in production. - "is not a valid value for Boolean", - "An unhandled exception has occurred while executing the request. System.FormatException: any", - "System.FormatException: The input string '[\\S\\s]+' was not in a correct format.", - "System.FormatException: The input string 'any", - // Happens when the static file middleware tries to access a path that doesn't exist or access a file as - // a directory. Presumably this is an attempt to access protected files using source path manipulation. - // This is handled by ASP.NET Core and there is nothing for us to worry about. - "System.IO.IOException: Not a directory", - "System.IO.IOException: The filename, directory name, or volume label syntax is incorrect", - "System.IO.DirectoryNotFoundException: Could not find a part of the path", - // This happens when a request's model contains a dictionary and a key is missing. While this can be a - // legitimate application error, during a security scan it's more likely the result of an incomplete - // artificially constructed request. So the means the ASP.NET Core model binding is working as intended. - "An unhandled exception has occurred while executing the request. System.ArgumentNullException: Value cannot be null. (Parameter 'key')", - // One way to verify correct error handling is to navigate to ~/Lombiq.Tests.UI.Shortcuts/Error/Index, which - // always throws an exception. This also gets logged but it's expected, so it should be ignored. - ErrorController.ExceptionMessage, - // Thrown from Microsoft.AspNetCore.Authentication.AuthenticationService.ChallengeAsync() when ZAP sends - // invalid authentication challenges. - "System.InvalidOperationException: No authentication handler is registered for the scheme", - CacheFolderErrorPattern, - }; - - permittedErrorLinePatterns.AddRange(additionalPermittedErrorLinePatterns); - - return app => app.LogsShouldBeEmptyAsync(canContainWarnings: true, permittedErrorLinePatterns); - } } diff --git a/Lombiq.Tests.UI/Services/RemoteInstance.cs b/Lombiq.Tests.UI/Services/RemoteInstance.cs index 30c75ec22..69a147ce8 100644 --- a/Lombiq.Tests.UI/Services/RemoteInstance.cs +++ b/Lombiq.Tests.UI/Services/RemoteInstance.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -15,7 +16,8 @@ public sealed class RemoteInstance : IWebApplicationInstance public Task StartUpAsync() => Task.FromResult(_baseUri); - public IEnumerable GetLogs(CancellationToken cancellationToken = default) => []; + public Task> GetLogsAsync(CancellationToken cancellationToken = default) => + Task.FromResult(Enumerable.Empty()); public TService GetRequiredService() => throw new NotSupportedException(); public Task PauseAsync() => throw new NotSupportedException(); public Task ResumeAsync() => throw new NotSupportedException(); diff --git a/Lombiq.Tests.UI/Services/SynchronizingWebApplicationSnapshotManager.cs b/Lombiq.Tests.UI/Services/SynchronizingWebApplicationSnapshotManager.cs index 02aec3a93..1c9a89484 100644 --- a/Lombiq.Tests.UI/Services/SynchronizingWebApplicationSnapshotManager.cs +++ b/Lombiq.Tests.UI/Services/SynchronizingWebApplicationSnapshotManager.cs @@ -46,7 +46,6 @@ public async Task RunOperationAndSnapshotIfNewAsync(AppInitializer appIniti var result = await appInitializer(); await result.Context.Application.TakeSnapshotAsync(_snapshotDirectoryPath); - await result.Context.Application.ResumeAsync(); DebugHelper.WriteLineTimestamped("Finished snapshot."); diff --git a/Lombiq.Tests.UI/Services/UITestContext.cs b/Lombiq.Tests.UI/Services/UITestContext.cs index f5ba24a8a..8f4ae5d3a 100644 --- a/Lombiq.Tests.UI/Services/UITestContext.cs +++ b/Lombiq.Tests.UI/Services/UITestContext.cs @@ -219,9 +219,9 @@ public Task AssertCurrentBrowserLogAsync() /// Clears the application and historic browser logs. /// /// Optional cancellation token for reading the application logs. - public void ClearLogs(CancellationToken cancellationToken = default) + public async Task ClearLogsAsync(CancellationToken cancellationToken = default) { - foreach (var log in Application.GetLogs(cancellationToken)) log.Remove(); + foreach (var log in await Application.GetLogsAsync(cancellationToken)) await log.RemoveAsync(); ClearHistoricBrowserLog(); }