OSOE-402: Custom logging instead of NLog-based one

.github/workflows/publish-nuget.yml

@@ -9,6 +9,6 @@ jobs:
   publish-nuget:
     name: Publish to NuGet
     if: ${{ !contains(github.ref_name, '-preview.') }}
-    uses: Lombiq/GitHub-Actions/.github/workflows/publish-nuget.yml@dev
+    uses: Lombiq/GitHub-Actions/.github/workflows/publish-nuget.yml@issue/OSOE-402
     secrets:
       API_KEY: ${{ secrets.DEFAULT_NUGET_PUBLISH_API_KEY }}

.github/workflows/validate-nuget-publish.yml

@@ -9,4 +9,4 @@ on:
 jobs:
   validate-nuget-publish:
     name: Validate NuGet Publish
-    uses: Lombiq/GitHub-Actions/.github/workflows/validate-nuget-publish.yml@dev
+    uses: Lombiq/GitHub-Actions/.github/workflows/validate-nuget-publish.yml@issue/OSOE-402

Lombiq.Tests.UI.Samples/Tests/DatabaseSnapshotTests.cs

@@ -23,19 +23,19 @@ public DatabaseSnapshotTests(ITestOutputHelper testOutputHelper)
     // "ExecuteTestFromExistingDBAsync()" to run the test on that. Finally, we test the basic Orchard features to check
     // that the application was set up correctly.
     [Fact]
-    public Task BasicOrchardFeaturesShouldWorkWithExistingDatabase() =>
-         ExecuteTestAsync(
-                async context =>
-                {
-                    var appForDatabaseTestFolder = Path.Combine("Temp", "AppForDatabaseTest");
+    public async Task BasicOrchardFeaturesShouldWorkWithExistingDatabase()
+    {
+        var appForDatabaseTestFolder = Path.Combine("Temp", "AppForDatabaseTest");
 
-                    await context.GoToSetupPageAndSetupOrchardCoreAsync(RecipeIds.BasicOrchardFeaturesTests);
-                    await context.Application.TakeSnapshotAsync(appForDatabaseTestFolder);
+        await ExecuteTestAsync(
+            async context =>
+            {
+                await context.GoToSetupPageAndSetupOrchardCoreAsync(RecipeIds.BasicOrchardFeaturesTests);
+                await context.Application.TakeSnapshotAsync(appForDatabaseTestFolder);
+            });
 
-                    await ExecuteTestFromExistingDBAsync(
-                         async context => await context.TestBasicOrchardFeaturesExceptSetupAsync(),
-                         appForDatabaseTestFolder);
-                });
+        await ExecuteTestFromExistingDBAsync(context => context.TestBasicOrchardFeaturesExceptSetupAsync(), appForDatabaseTestFolder);
+    }
 }
 
 // END OF TRAINING SECTION: Database snapshot tests.

Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs

@@ -38,7 +38,7 @@ public Task ServerSideErrorOnLoadedPageShouldHaltTest() =>
                 catch (PageChangeAssertionException)
                 {
                     // Remove all logs to have a clean slate.
-                    context.ClearLogs();
+                    await context.ClearLogsAsync();
                 }
             });
 

Lombiq.Tests.UI.Samples/Tests/ShiftTimeTests.cs

@@ -10,11 +10,11 @@
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 
-// When you enable the "Shift Time - Shortcuts - Lombiq UI Testing Toolbox" feature, it replaces OC's stock ICLock
-// implementation with the custom ShiftTimeClock class. You can use the ~/Lombiq.Tests.UI.Samples/ShiftTime/Set?days=...
-// action to update the ShiftTimeClock.Shift property for the current tenant, which will trick any service that uses
-// IClock into thinking you are in the future. This can be used to test features that can expire, such as a limited-time
-// product discount in a web store.
+// When you enable the "Shift Time - Shortcuts - Lombiq UI Testing Toolbox" feature, it replaces Orchard Core's stock
+// ICLock implementation with the custom ShiftTimeClock class. You can use the
+// ~/Lombiq.Tests.UI.Samples/ShiftTime/Set?days=... action to update the ShiftTimeClock.Shift property for the current
+// tenant, which will trick any service that uses IClock into thinking you are in the future. This can be used to test
+// features that can expire, such as a limited-time product discount in a web store, without having to wait.
 public class ShiftTimeTests : UITestBase
 {
     public ShiftTimeTests(ITestOutputHelper testOutputHelper)

Lombiq.Tests.UI.Samples/UITestBase.cs

@@ -2,6 +2,7 @@
 using Lombiq.Tests.UI.Extensions;
 using Lombiq.Tests.UI.Samples.Helpers;
 using Lombiq.Tests.UI.Services;
+using Microsoft.Extensions.Logging;
 using System;
 using System.Threading.Tasks;
 using Xunit.Abstractions;
@@ -64,7 +65,8 @@ protected override Task ExecuteTestAsync(
                 // https://docs.orchardcore.net/en/latest/docs/reference/core/Configuration/. We can set e.g. Orchard's
                 // AdminUrlPrefix like below, but this is just setting the default, so it's only an example. A more
                 // useful example is enabling offline operation of the Lombiq Hosting - Azure Application Insights for
-                // Orchard Core module (see https://github.com/Lombiq/Orchard-Azure-Application-Insights).
+                // Orchard Core module (see the UI test project in
+                // https://github.com/Lombiq/Orchard-Azure-Application-Insights).
                 configuration.OrchardCoreConfiguration.BeforeAppStart +=
                     (_, argumentsBuilder) =>
                     {
@@ -82,22 +84,41 @@ protected override Task ExecuteTestAsync(
                 // Action) to further configure it.
                 ////configuration.HtmlValidationConfiguration.RunHtmlValidationAssertionOnAllPageChanges = false;
 
-                // The UI Testing Toolbox can run several checks for the app even if you don't add explicit
-                // assertions: By default, the Orchard logs and the browser logs (where e.g. JavaScript errors show
-                // up) are checked and if there are any errors, the test will fail. You can also enable the checking of
-                // accessibility rules as we'll see later. Maybe not all of the default checks are suitable for you.
-                // Then it's simple to override them; here we change which log entries cause the tests to fail, and
-                // allow warnings and certain errors.
-                // Note that this is just for demonstration; you could use
-                // OrchardCoreUITestExecutorConfiguration.AssertAppLogsCanContainWarningsAndCacheFolderErrorsAsync which
-                // provides this configuration built-in.
+                // For locally running apps, the UI Testing Toolbox configures Fake Logging (see
+                // https://learn.microsoft.com/en-us/dotnet/api/microsoft.extensions.logging.testing). This provides an
+                // in-memory log for assertions, regardless of the logging framework your app uses otherwise. By
+                // default, it'll only collect log entries with the Error level and up. However, you can change this,
+                // and the usual Logging app settings still work.
+                configuration.OrchardCoreConfiguration.BeforeAppStart +=
+                    (_, argumentsBuilder) =>
+                    {
+                        // This is how you can configure logging to be on the >=Error level in general, while still
+                        // allowing info-level entries for ShellHost.
+                        argumentsBuilder
+                            .AddWithValue("Logging:LogLevel:Default", "Error")
+                            .AddWithValue("Logging:LogLevel:System", "Error")
+                            .AddWithValue("Logging:LogLevel:Microsoft", "Error")
+                            .AddWithValue("Logging:LogLevel:OrchardCore.Environment.Shell.ShellHost", "Information");
+
+                        return Task.CompletedTask;
+                    };
+
+                // Enabling FakeLogger to collect the info-level ShellHost log entries configured above. These will show
+                // up in the test's output like "[Information] OrchardCore.Environment.Shell.ShellHost: Start creation
+                // of shells".
+                configuration.OrchardCoreConfiguration.AfterFakeLoggingConfiguration =
+                    (_, fakeLogCollectorOptions) => fakeLogCollectorOptions.FilteredLevels.Add(LogLevel.Information);
+
+                // Logging is important because the UI Testing Toolbox can run several checks for the app even if you
+                // don't add explicit assertions: By default, the Orchard logs and the browser logs (where e.g.
+                // JavaScript errors show up) are checked and if there are any errors, the test will fail. You can also
+                // enable the checking of accessibility rules as we'll see later. Maybe not all of the default checks
+                // are suitable for you. Then it's simple to override them; here we change which log entries cause the
+                // tests to fail, and allow certain log entries.
                 configuration.AssertAppLogsAsync = webApplicationInstance =>
-                    webApplicationInstance.LogsShouldBeEmptyAsync(
-                        canContainWarnings: true,
-                        permittedErrorLinePatterns:
-                        [
-                            "OrchardCore.Media.Core.DefaultMediaFileStoreCacheFileProvider|ERROR|Error deleting cache folder",
-                        ]);
+                    webApplicationInstance.LogsShouldNotContainAsync(logEntry =>
+                    // Allowing info-level log entries for ShellHost, see above.
+                    logEntry.Message != "My permitted message." && logEntry.Level != LogLevel.Information);
 
                 // Strictly speaking this is not necessary here, because we always use the same static method for setup.
                 // However, if you used a dynamic setup operation (e.g. `context => SetupHelpers.RunSetupAsync(context,

Lombiq.Tests.UI.Shortcuts/Controllers/ShiftTimeController.cs

@@ -1,15 +1,12 @@
+using Lombiq.HelpfulLibraries.AspNetCore.Mvc;
 using Lombiq.Tests.UI.Shortcuts.Services;
 using Microsoft.AspNetCore.Mvc;
 using OrchardCore.Modules;
 using System;
-using System.Diagnostics.CodeAnalysis;
 
 namespace Lombiq.Tests.UI.Shortcuts.Controllers;
 
-[SuppressMessage(
-    "Major Code Smell",
-    "S6967:ModelState.IsValid should be called in controller actions",
-    Justification = "Not relevant in a test-only controller.")]
+[DevelopmentAndLocalhostOnly]
 public class ShiftTimeController : Controller
 {
     private readonly IClock _clock;

Lombiq.Tests.UI.Tests.UI/TestCases/SecurityShortcutsTestCases.cs

@@ -48,7 +48,7 @@ public static Task AddUserToFakeRoleShouldThrowAsync(
                 await context.CreateUserAsync(UserUserName, DefaultUser.Password, UserEmail);
                 await context.AddUserToRoleAsync(UserUserName, FakeRole).ShouldThrowAsync<RoleNotFoundException>();
 
-                context.ClearLogs();
+                await context.ClearLogsAsync();
             },
             browser,
             ConfigurationHelper.DisableHtmlValidation);
@@ -61,7 +61,7 @@ public static Task AllowFakePermissionToRoleShouldThrowAsync(
                 await context.AddPermissionToRoleAsync(FakePermission, AuthorRole)
                     .ShouldThrowAsync<PermissionNotFoundException>();
 
-                context.ClearLogs();
+                await context.ClearLogsAsync();
             },
             browser,
             ConfigurationHelper.DisableHtmlValidation);

Lombiq.Tests.UI/CompatibilitySuppressions.xml

@@ -0,0 +1,144 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- https://learn.microsoft.com/en-us/dotnet/fundamentals/package-validation/diagnostic-ids -->
+<Suppressions xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
+  <Suppression>
+    <DiagnosticId>CP0002</DiagnosticId>
+    <Target>F:Lombiq.Tests.UI.Services.OrchardCoreUITestExecutorConfiguration.AssertAppLogsCanContainWarningsAndCacheFolderErrorsAsync</Target>
+    <Left>lib/net8.0/Lombiq.Tests.UI.dll</Left>
+    <Right>lib/net8.0/Lombiq.Tests.UI.dll</Right>
+    <IsBaselineSuppression>true</IsBaselineSuppression>
+  </Suppression>
+  <Suppression>
+    <DiagnosticId>CP0002</DiagnosticId>
+    <Target>F:Lombiq.Tests.UI.Services.OrchardCoreUITestExecutorConfiguration.AssertAppLogsCanContainWarningsAsync</Target>
+    <Left>lib/net8.0/Lombiq.Tests.UI.dll</Left>
+    <Right>lib/net8.0/Lombiq.Tests.UI.dll</Right>
+    <IsBaselineSuppression>true</IsBaselineSuppression>
+  </Suppression>
+  <Suppression>
+    <DiagnosticId>CP0002</DiagnosticId>
+    <Target>M:Lombiq.Tests.UI.Extensions.WebApplicationInstanceExtensions.GetLogOutputAsync(Lombiq.Tests.UI.Services.IWebApplicationInstance,System.Threading.CancellationToken)</Target>
+    <Left>lib/net8.0/Lombiq.Tests.UI.dll</Left>
+    <Right>lib/net8.0/Lombiq.Tests.UI.dll</Right>
+    <IsBaselineSuppression>true</IsBaselineSuppression>
+  </Suppression>
+  <Suppression>
+    <DiagnosticId>CP0002</DiagnosticId>
+    <Target>M:Lombiq.Tests.UI.Extensions.WebApplicationInstanceExtensions.LogsShouldBeEmptyAsync(Lombiq.Tests.UI.Services.IWebApplicationInstance,System.Boolean,System.Collections.Generic.ICollection{System.String},System.Threading.CancellationToken)</Target>
+    <Left>lib/net8.0/Lombiq.Tests.UI.dll</Left>
+    <Right>lib/net8.0/Lombiq.Tests.UI.dll</Right>
+    <IsBaselineSuppression>true</IsBaselineSuppression>
+  </Suppression>
+  <Suppression>
+    <DiagnosticId>CP0002</DiagnosticId>
+    <Target>M:Lombiq.Tests.UI.Services.IApplicationLog.GetContentAsync</Target>
+    <Left>lib/net8.0/Lombiq.Tests.UI.dll</Left>
+    <Right>lib/net8.0/Lombiq.Tests.UI.dll</Right>
+    <IsBaselineSuppression>true</IsBaselineSuppression>
+  </Suppression>
+  <Suppression>
+    <DiagnosticId>CP0002</DiagnosticId>
+    <Target>M:Lombiq.Tests.UI.Services.IApplicationLog.Remove</Target>
+    <Left>lib/net8.0/Lombiq.Tests.UI.dll</Left>
+    <Right>lib/net8.0/Lombiq.Tests.UI.dll</Right>
+    <IsBaselineSuppression>true</IsBaselineSuppression>
+  </Suppression>
+  <Suppression>
+    <DiagnosticId>CP0002</DiagnosticId>
+    <Target>M:Lombiq.Tests.UI.Services.IWebApplicationInstance.GetLogs(System.Threading.CancellationToken)</Target>
+    <Left>lib/net8.0/Lombiq.Tests.UI.dll</Left>
+    <Right>lib/net8.0/Lombiq.Tests.UI.dll</Right>
+    <IsBaselineSuppression>true</IsBaselineSuppression>
+  </Suppression>
+  <Suppression>
+    <DiagnosticId>CP0002</DiagnosticId>
+    <Target>M:Lombiq.Tests.UI.Services.IWebApplicationInstance.GetRequiredService``1</Target>
+    <Left>lib/net8.0/Lombiq.Tests.UI.dll</Left>
+    <Right>lib/net8.0/Lombiq.Tests.UI.dll</Right>
+    <IsBaselineSuppression>true</IsBaselineSuppression>
+  </Suppression>
+  <Suppression>
+    <DiagnosticId>CP0002</DiagnosticId>
+    <Target>M:Lombiq.Tests.UI.Services.OrchardCoreInstance`1.GetLogs(System.Threading.CancellationToken)</Target>
+    <Left>lib/net8.0/Lombiq.Tests.UI.dll</Left>
+    <Right>lib/net8.0/Lombiq.Tests.UI.dll</Right>
+    <IsBaselineSuppression>true</IsBaselineSuppression>
+  </Suppression>
+  <Suppression>
+    <DiagnosticId>CP0002</DiagnosticId>
+    <Target>M:Lombiq.Tests.UI.Services.OrchardCoreInstance`1.GetRequiredService``1</Target>
+    <Left>lib/net8.0/Lombiq.Tests.UI.dll</Left>
+    <Right>lib/net8.0/Lombiq.Tests.UI.dll</Right>
+    <IsBaselineSuppression>true</IsBaselineSuppression>
+  </Suppression>
+  <Suppression>
+    <DiagnosticId>CP0002</DiagnosticId>
+    <Target>M:Lombiq.Tests.UI.Services.OrchardCoreUITestExecutorConfiguration.AssertAppLogsMaybeAsync(Lombiq.Tests.UI.Services.IWebApplicationInstance,System.Action{System.String})</Target>
+    <Left>lib/net8.0/Lombiq.Tests.UI.dll</Left>
+    <Right>lib/net8.0/Lombiq.Tests.UI.dll</Right>
+    <IsBaselineSuppression>true</IsBaselineSuppression>
+  </Suppression>
+  <Suppression>
+    <DiagnosticId>CP0002</DiagnosticId>
+    <Target>M:Lombiq.Tests.UI.Services.OrchardCoreUITestExecutorConfiguration.AssertBrowserLogMaybe(System.Collections.Generic.IList{OpenQA.Selenium.LogEntry},System.Action{System.String})</Target>
+    <Left>lib/net8.0/Lombiq.Tests.UI.dll</Left>
+    <Right>lib/net8.0/Lombiq.Tests.UI.dll</Right>
+    <IsBaselineSuppression>true</IsBaselineSuppression>
+  </Suppression>
+  <Suppression>
+    <DiagnosticId>CP0002</DiagnosticId>
+    <Target>M:Lombiq.Tests.UI.Services.OrchardCoreUITestExecutorConfiguration.CreateAppLogAssertionForSecurityScan(System.String[])</Target>
+    <Left>lib/net8.0/Lombiq.Tests.UI.dll</Left>
+    <Right>lib/net8.0/Lombiq.Tests.UI.dll</Right>
+    <IsBaselineSuppression>true</IsBaselineSuppression>
+  </Suppression>
+  <Suppression>
+    <DiagnosticId>CP0002</DiagnosticId>
+    <Target>M:Lombiq.Tests.UI.Services.OrchardCoreUITestExecutorConfiguration.UseAssertAppLogsForSecurityScan(System.String[])</Target>
+    <Left>lib/net8.0/Lombiq.Tests.UI.dll</Left>
+    <Right>lib/net8.0/Lombiq.Tests.UI.dll</Right>
+    <IsBaselineSuppression>true</IsBaselineSuppression>
+  </Suppression>
+  <Suppression>
+    <DiagnosticId>CP0002</DiagnosticId>
+    <Target>M:Lombiq.Tests.UI.Services.RemoteInstance.GetLogs(System.Threading.CancellationToken)</Target>
+    <Left>lib/net8.0/Lombiq.Tests.UI.dll</Left>
+    <Right>lib/net8.0/Lombiq.Tests.UI.dll</Right>
+    <IsBaselineSuppression>true</IsBaselineSuppression>
+  </Suppression>
+  <Suppression>
+    <DiagnosticId>CP0002</DiagnosticId>
+    <Target>M:Lombiq.Tests.UI.Services.UITestContext.ClearLogs(System.Threading.CancellationToken)</Target>
+    <Left>lib/net8.0/Lombiq.Tests.UI.dll</Left>
+    <Right>lib/net8.0/Lombiq.Tests.UI.dll</Right>
+    <IsBaselineSuppression>true</IsBaselineSuppression>
+  </Suppression>
+  <Suppression>
+    <DiagnosticId>CP0006</DiagnosticId>
+    <Target>M:Lombiq.Tests.UI.Services.IApplicationLog.GetEntriesAsync</Target>
+    <Left>lib/net8.0/Lombiq.Tests.UI.dll</Left>
+    <Right>lib/net8.0/Lombiq.Tests.UI.dll</Right>
+    <IsBaselineSuppression>true</IsBaselineSuppression>
+  </Suppression>
+  <Suppression>
+    <DiagnosticId>CP0006</DiagnosticId>
+    <Target>M:Lombiq.Tests.UI.Services.IApplicationLog.RemoveAsync</Target>
+    <Left>lib/net8.0/Lombiq.Tests.UI.dll</Left>
+    <Right>lib/net8.0/Lombiq.Tests.UI.dll</Right>
+    <IsBaselineSuppression>true</IsBaselineSuppression>
+  </Suppression>
+  <Suppression>
+    <DiagnosticId>CP0006</DiagnosticId>
+    <Target>M:Lombiq.Tests.UI.Services.IWebApplicationInstance.GetLogsAsync(System.Threading.CancellationToken)</Target>
+    <Left>lib/net8.0/Lombiq.Tests.UI.dll</Left>
+    <Right>lib/net8.0/Lombiq.Tests.UI.dll</Right>
+    <IsBaselineSuppression>true</IsBaselineSuppression>
+  </Suppression>
+  <Suppression>
+    <DiagnosticId>CP0006</DiagnosticId>
+    <Target>P:Lombiq.Tests.UI.Services.IApplicationLog.EntryCount</Target>
+    <Left>lib/net8.0/Lombiq.Tests.UI.dll</Left>
+    <Right>lib/net8.0/Lombiq.Tests.UI.dll</Right>
+    <IsBaselineSuppression>true</IsBaselineSuppression>
+  </Suppression>
+</Suppressions>

Lombiq.Tests.UI/Docs/Troubleshooting.md

@@ -7,7 +7,7 @@
   - Browser logs, i.e. the developer console output.
   - Screenshots of each page in order the test visited them, as well as when the test failed (Windows Photo Viewer won't be able to open it though, use something else like the Windows 10 Photos app).
   - The HTML output on the page the test failed.
-  - Any direct output of the test (like the exception thrown) as well as a log of the operations it completed.
+  - Any direct output of the test (like the exception thrown) as well as a log of the operations it completed (including browser interactions and corresponding Orchard Core log entries).
   - If accessibility was checked and asserting it failed then an accessibility report will be included too.
   - If HTML validation was done and asserting it failed then an HTML validation report will be included too.
 - Run tests with the debugger attached to stop where the test fails. This way you can take a good look at what's in the driven browser window so you can examine the web page and you can debug the web app under test at the same time.