Merge pull request #166 from Lombiq/issue/OSOE-111

OSOE-111: Detect visual errors (visual verification testing) in the UI Testing Toolbox

Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>net6.0</TargetFramework>
@@ -12,7 +12,7 @@
     <None Remove=".htmlvalidate.json" />
     <None Remove="xunit.runner.json" />
   </ItemGroup>
-
+  
   <ItemGroup>
     <Content Include=".htmlvalidate.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
@@ -21,6 +21,18 @@
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
   </ItemGroup>
+
+  <ItemGroup>
+    <EmbeddedResource Include="Tests\BasicVisualVerificationTests_VerifyBlogImage_By_ClassName[Contains]_-field-name-blog-image.png" />
+    <EmbeddedResource Include="Tests\BasicVisualVerificationTests_VerifyHomePage_By_TagName_-body_Unix_chrome.png" />
+    <EmbeddedResource Include="Tests\BasicVisualVerificationTests_VerifyHomePage_By_TagName_-body_Unix_msedge.png" />
+    <EmbeddedResource Include="Tests\BasicVisualVerificationTests_VerifyHomePage_By_TagName_-body_Win32NT_chrome.png" />
+    <EmbeddedResource Include="Tests\BasicVisualVerificationTests_VerifyHomePage_By_TagName_-body_Win32NT_msedge.png" />
+    <EmbeddedResource Include="Tests\BasicVisualVerificationTests_VerifyNavbar_By_ClassName[Contains]_-navbar-brand_Unix_chrome.png" />
+    <EmbeddedResource Include="Tests\BasicVisualVerificationTests_VerifyNavbar_By_ClassName[Contains]_-navbar-brand_Unix_msedge.png" />
+    <EmbeddedResource Include="Tests\BasicVisualVerificationTests_VerifyNavbar_By_ClassName[Contains]_-navbar-brand_Win32NT_chrome.png" />
+    <EmbeddedResource Include="Tests\BasicVisualVerificationTests_VerifyNavbar_By_ClassName[Contains]_-navbar-brand_Win32NT_msedge.png" />
+  </ItemGroup>
 
   <ItemGroup>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.2.0" />

Lombiq.Tests.UI.Samples/Readme.md

@@ -28,6 +28,7 @@ For general details about and on using the Toolbox see the [root Readme](../Read
 - [Error handling](Tests/ErrorHandlingtests.cs)
 - [Monkey tests](Tests/MonkeyTests.cs)
 - [Database snapshot tests](Tests/DatabaseSnapshotTests.cs)
+- [Basic visual verification tests](Tests/BasicVisualVerificationTests.cs)
 
 
 ## Adding new tutorials

Lombiq.Tests.UI.Samples/Tests/BasicVisualVerificationTests.cs

@@ -0,0 +1,101 @@
+using Lombiq.Tests.UI.Attributes;
+using Lombiq.Tests.UI.Constants;
+using Lombiq.Tests.UI.Extensions;
+using Lombiq.Tests.UI.Services;
+using OpenQA.Selenium;
+using System.Threading.Tasks;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Lombiq.Tests.UI.Samples.Tests;
+
+// In this basic test we will check visually the rendered content.
+public class BasicVisualVerificationTests : UITestBase
+{
+    public BasicVisualVerificationTests(ITestOutputHelper testOutputHelper)
+        : base(testOutputHelper)
+    {
+    }
+
+    // This is a very basic sample to check that the header image is what we expect and looks as we expect. For this
+    // magic we are using the ImageSharp.Compare package. You can find more info about it here:
+    // https://github.com/Codeuctivity/ImageSharp.Compare. This looks really simple, but there is some trap to comparing
+    // block containers containing images like this. Take attention to reproducing the geometries, because the image
+    // fits the container and the container size depends on the client area, so if the geometries are not exactly the
+    // same, the test will fail. One more trap is the changes between browser versions, e.g. there was a change between
+    // the Chrome version 67 and 68 in the image rendering. This caused that the rendered image looked similar, but
+    // comparing pixel-by-pixel was different. You can investigate this or similar failure using the captured and
+    // generated diff images under the path FailureDumps/<test-name>/Attempt <n>/DebugInformation/VisualVerification.
+    [Theory, Chrome]
+    public Task VerifyBlogImage(Browser browser) =>
+        ExecuteTestAfterSetupAsync(
+            context =>
+            {
+                // Instead of using context.SetBrowserSize(...), which sets the outer size of the browser's window to
+                // the given size, we are using context.SetViewportSize(...) here. This is because the window borders,
+                // toolbars, tabs, and scroll bars usually have different sizes on different platforms/browsers, but we
+                // want the same geometries of rendered content on all platforms.
+                context.SetViewportSize(CommonDisplayResolutions.HdPlus);
+                // Here we hide the scrollbars to get more control over the size of the client area. If there are more
+                // operations after visual verification and the scrollbars are required, then don't forget to restore
+                // with context.RestoreHiddenScrollbar().
+                context.HideScrollbar();
+
+                var blogImageElementSelector = By.ClassName("field-name-blog-image");
+
+                // Here we check that the rendered content visually equals the baseline image within a given error
+                // percentage. You can read more about this in the AssertVisualVerificationApproved method documentation.
+                context.AssertVisualVerificationApproved(blogImageElementSelector, 0);
+            },
+            browser);
+
+    // Checking that everything is OK with the branding of the navbar on the homepage. If you want to visually validate
+    // text content on different platforms (like Windows or Linux) or browsers, it can cause surprises too. The reason
+    // is the different rendering of text on each platform, but it can occur between different Linux distributions too.
+    // Here: https://pandasauce.org/post/linux-fonts/ you can find a good summary about this from 2019, but still valid
+    // in 2022.
+    [Theory, Chrome, Edge]
+    public Task VerifyNavbar(Browser browser) =>
+        ExecuteTestAfterSetupAsync(
+            context =>
+            {
+                context.SetViewportSize(CommonDisplayResolutions.HdPlus);
+
+                var navbarElementSelector = By.ClassName("navbar-brand");
+
+                // Here we check that the rendered content visually equals the baseline image within a given error
+                // percentage using different baseline image on each platform and browser. You can read more about
+                // this in the AssertVisualVerificationApproved method documentation.
+                context.AssertVisualVerificationApproved(
+                    navbarElementSelector,
+                    0,
+                    configurator: configuration =>
+                        configuration
+                            // These configurations below are to generate/use different baseline images on each
+                            // platform/browser.
+                            .WithUsePlatformAsSuffix()
+                            .WithUseBrowserNameAsSuffix());
+            },
+            browser);
+
+    // Checking that everything is OK with the homepage, just for fun.
+    [Theory, Chrome, Edge]
+    public Task VerifyHomePage(Browser browser) =>
+        ExecuteTestAfterSetupAsync(
+            context =>
+            {
+                context.SetViewportSize(CommonDisplayResolutions.HdPlus);
+                context.HideScrollbar();
+
+                // Here we don't need any element selector to validate the whole page.
+                context.AssertVisualVerificationApproved(0, configurator: configuration =>
+                    configuration
+                        // These configurations below are to generate/use different baseline images on each
+                        // platform/browser.
+                        .WithUsePlatformAsSuffix()
+                        .WithUseBrowserNameAsSuffix());
+            },
+            browser);
+}
+
+// END OF TRAINING SECTION: Basic visual verification tests.

Lombiq.Tests.UI.Samples/Tests/DatabaseSnapshotTests.cs

@@ -41,3 +41,4 @@ await ExecuteTestFromExistingDBAsync(
 }
 
 // END OF TRAINING SECTION: Database snapshot tests.
+// NEXT STATION: Head over to Tests/BasicVisualVerificationTests.cs.

Lombiq.Tests.UI/Attributes/VisualVerificationApprovedMethodAttribute.cs

@@ -0,0 +1,12 @@
+using System;
+
+namespace Lombiq.Tests.UI.Attributes;
+
+/// <summary>
+/// This attribute is used to annotate the VisualVerificationApproved methods in the call stack to get the consumer
+/// method.
+/// </summary>
+[AttributeUsage(AttributeTargets.Method, AllowMultiple = false, Inherited = true)]
+public sealed class VisualVerificationApprovedMethodAttribute : Attribute
+{
+}

Lombiq.Tests.UI/Constants/VisualVerificationMatchNames.cs

@@ -0,0 +1,13 @@
+namespace Lombiq.Tests.UI.Constants;
+
+public static class VisualVerificationMatchNames
+{
+    public const string DumpFolderName = "VisualVerification";
+    public const string FullScreenImageFileName = "FullScreen.png";
+    public const string ElementImageFileName = "Element.png";
+    public const string BaselineImageFileName = "Baseline.png";
+    public const string CroppedElementImageFileName = "Element-cropped.png";
+    public const string CroppedBaselineImageFileName = "Baseline-cropped.png";
+    public const string DiffImageFileName = "Diff.png";
+    public const string DiffLogFileName = "Diff.log";
+}

Lombiq.Tests.UI/Docs/Tools.md

@@ -13,3 +13,5 @@
 - Accessibility checking can be done with [axe](https://github.com/dequelabs/axe-core) via [Selenium.Axe for .NET](https://github.com/TroyWalshProf/SeleniumAxeDotnet).
 - HTML markup validation can be done with [html-validate](https://gitlab.com/html-validate/html-validate) via [Atata.HtmlValidation](https://github.com/atata-framework/atata-htmlvalidation).
 - Monkey testing is implemented using [Gremlins.js](https://github.com/marmelab/gremlins.js/) library.
+- Visual verification is implemented using [ImageSharpCompare](https://github.com/Codeuctivity/ImageSharp.Compare).
+- [Ben.Demystifier](https://github.com/benaadams/Ben.Demystifier) is used to simplify stack traces, mainly around async methods.

Lombiq.Tests.UI/Exceptions/FailureDumpItemAlreadyExistsException.cs

@@ -0,0 +1,33 @@
+using System;
+
+namespace Lombiq.Tests.UI.Exceptions;
+
+// Here we only need fileName and customMessage.
+#pragma warning disable CA1032 // Implement standard exception constructors
+public class FailureDumpItemAlreadyExistsException : Exception
+#pragma warning restore CA1032 // Implement standard exception constructors
+{
+    public FailureDumpItemAlreadyExistsException(string fileName)
+        : this(fileName, customMessage: null, innerException: null)
+    {
+    }
+
+    public FailureDumpItemAlreadyExistsException(string fileName, Exception innerException)
+        : this(fileName, customMessage: null, innerException)
+    {
+    }
+
+    public FailureDumpItemAlreadyExistsException(string fileName, string customMessage)
+        : this(fileName, customMessage, innerException: null)
+    {
+    }
+
+    public FailureDumpItemAlreadyExistsException(string fileName, string customMessage, Exception innerException)
+        : base(
+            $"A failure dump item with the same file name already exists. fileName: {fileName}"
+            + Environment.NewLine
+            + (customMessage ?? string.Empty),
+            innerException)
+    {
+    }
+}

Lombiq.Tests.UI/Exceptions/VisualVerificationAssertionException.cs

@@ -0,0 +1,20 @@
+using System;
+
+namespace Lombiq.Tests.UI.Exceptions;
+
+public class VisualVerificationAssertionException : Exception
+{
+    public VisualVerificationAssertionException(string message)
+        : base(message)
+    {
+    }
+
+    public VisualVerificationAssertionException(string message, Exception innerException)
+        : base(message, innerException)
+    {
+    }
+
+    public VisualVerificationAssertionException()
+    {
+    }
+}

Lombiq.Tests.UI/Exceptions/VisualVerificationBaselineImageNotFoundException.cs

@@ -0,0 +1,25 @@
+using System;
+
+namespace Lombiq.Tests.UI.Exceptions;
+
+// Here we only need path instead of message.
+#pragma warning disable CA1032 // Implement standard exception constructors
+public class VisualVerificationBaselineImageNotFoundException : Exception
+#pragma warning restore CA1032 // Implement standard exception constructors
+{
+    public VisualVerificationBaselineImageNotFoundException(string path)
+        : this(path, innerException: null)
+    {
+    }
+
+    public VisualVerificationBaselineImageNotFoundException(string path, Exception innerException)
+        : base(
+            $"Baseline image file not found, thus it was created automatically under the path {path}."
+            + " Please set its \"Build action\" to \"Embedded resource\" if you want to deploy a self-contained"
+            + " (like a NuGet package) UI testing assembly. If you run the test again, this newly created verification"
+            + " file will be asserted against and the assertion will pass (unless the display of the app changed in the"
+            + " meantime).",
+            innerException)
+    {
+    }
+}

Lombiq.Tests.UI/Exceptions/VisualVerificationCallerMethodNotFoundException.cs

@@ -0,0 +1,26 @@
+using System;
+
+namespace Lombiq.Tests.UI.Exceptions;
+
+public class VisualVerificationCallerMethodNotFoundException : Exception
+{
+    public VisualVerificationCallerMethodNotFoundException(string message)
+        : base(message)
+    {
+    }
+
+    public VisualVerificationCallerMethodNotFoundException(string message, Exception innerException)
+        : base(message, innerException)
+    {
+    }
+
+    public VisualVerificationCallerMethodNotFoundException()
+        : this(innerException: null)
+    {
+    }
+
+    public VisualVerificationCallerMethodNotFoundException(Exception innerException)
+        : this("Caller method not found", innerException)
+    {
+    }
+}

Lombiq.Tests.UI/Exceptions/VisualVerificationSourceInformationNotAvailableException.cs

@@ -0,0 +1,20 @@
+using System;
+
+namespace Lombiq.Tests.UI.Exceptions;
+
+public class VisualVerificationSourceInformationNotAvailableException : Exception
+{
+    public VisualVerificationSourceInformationNotAvailableException(string message)
+        : base(message)
+    {
+    }
+
+    public VisualVerificationSourceInformationNotAvailableException(string message, Exception innerException)
+        : base(message, innerException)
+    {
+    }
+
+    public VisualVerificationSourceInformationNotAvailableException()
+    {
+    }
+}

Lombiq.Tests.UI/Extensions/AssemblyResourceExtensions.cs

@@ -0,0 +1,39 @@
+using SixLabors.ImageSharp;
+using System.Reflection;
+
+using DrawingBitmap = System.Drawing.Bitmap;
+using DrawingImage = System.Drawing.Image;
+
+namespace Lombiq.Tests.UI.Extensions;
+
+public static class AssemblyResourceExtensions
+{
+    /// <summary>
+    /// Loads resource specified by name from the given assembly.
+    /// </summary>
+    /// <param name="name">Resource name.</param>
+    /// <returns><see cref="Image"/> instance.</returns>
+    public static Image GetResourceImageSharpImage(this Assembly assembly, string name)
+    {
+        using var resourceStream = assembly.GetManifestResourceStream(name);
+
+        return Image.Load(resourceStream);
+    }
+
+    /// <summary>
+    /// Loads resource specified by name from the given assembly.
+    /// </summary>
+    /// <param name="name">Resource name.</param>
+    /// <returns><see cref="DrawingBitmap"/> instance.</returns>
+    public static DrawingBitmap TryGetResourceBitmap(this Assembly assembly, string name)
+    {
+        var resourceStream = assembly.GetManifestResourceStream(name);
+
+        if (resourceStream == null)
+        {
+            return null;
+        }
+
+        return (DrawingBitmap)DrawingImage.FromStream(resourceStream);
+    }
+}