From e469a8b5970fdb3fd7040ef228e5940f9f5362ba Mon Sep 17 00:00:00 2001 From: Aydin Erdas Date: Fri, 3 May 2024 19:37:53 +0200 Subject: [PATCH 01/15] default JSON html validation --- Lombiq.Tests.UI/Docs/Configuration.md | 57 +++++-------------- .../HtmlValidationResultExtensions.cs | 34 +++++++++++ .../Models/JsonHtmlValidationError.cs | 28 +++++++++ .../Services/HtmlValidationConfiguration.cs | 18 +++++- 4 files changed, 93 insertions(+), 44 deletions(-) create mode 100644 Lombiq.Tests.UI/Models/JsonHtmlValidationError.cs diff --git a/Lombiq.Tests.UI/Docs/Configuration.md b/Lombiq.Tests.UI/Docs/Configuration.md index b97ec21b0..f580ff8be 100644 --- a/Lombiq.Tests.UI/Docs/Configuration.md +++ b/Lombiq.Tests.UI/Docs/Configuration.md @@ -59,51 +59,22 @@ Recommendations and notes for such configuration: ### HTML validation configuration -If you want to change some HTML validation rules from only a few specific tests, you can create a custom _.htmlvalidate.json_ file (e.g. _TestName.htmlvalidate.json_). For example: - -```json -{ - "extends": [ - "html-validate:recommended" - ], - - "rules": { - "attribute-boolean-style": "off", - "element-required-attributes": "off", - "no-trailing-whitespace": "off", - "no-inline-style": "off", - "no-implicit-button-type": "off", - "wcag/h30": "off", - "wcag/h32": "off", - "wcag/h36": "off", - "wcag/h37": "off", - "wcag/h67": "off", - "wcag/h71": "off" - }, - - "root": true -} +If you want to change some HTML validation rules for only a few specific tests you can exclude them from the results. +For example if you want to exclude the `prefer-native-element` rule from the results you can do it by doing the following: + +```c# +configuration => configuration.HtmlValidationConfiguration.AssertHtmlValidationResultAsync = + validationResult => + { + var errors = validationResult.GetParsedErrors() + .Where(error => error.RuleId is not "prefer-native-element"); + errors.ShouldBeEmpty(string.Join('\n', errors.Select(error => error.Message))); + return Task.CompletedTask; + }); ``` -Then you can change the configuration to use that: - -```cs - changeConfiguration: configuration => configuration.HtmlValidationConfiguration.HtmlValidationOptions = - configuration.HtmlValidationConfiguration.HtmlValidationOptions - .CloneWith(validationOptions => validationOptions.ConfigPath = - Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "TestName.htmlvalidate.json"))); -``` - -Make sure to also include the `root` attribute and set it to `true` inside the custom _.htmlvalidate.json_ file and include it in the test project like this: - -```xml - - - PreserveNewest - true - - - ``` +Note that the `RuleId` is the identifier of the rule that you want to exclude from the results. +The custom string formatter in the call to `errors.ShouldBeEmpty` is used to display the errors in a more readable way and is not strictly necessary. ## Multi-process test execution diff --git a/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs b/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs index 3ff2b7715..61c9bcc35 100644 --- a/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs @@ -1,8 +1,10 @@ using Atata.HtmlValidation; +using Lombiq.Tests.UI.Models; using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text.Json; using System.Threading.Tasks; namespace Lombiq.Tests.UI.Extensions; @@ -22,4 +24,36 @@ public static async Task> GetErrorsAsync(this HtmlValidation .Select(error => (error.StartsWith("error:", StringComparison.OrdinalIgnoreCase) ? string.Empty : "error:") + error); } + + /// + /// Gets the parsed errors from the HTML validation result. + /// Can only be used if the output formatter is set to JSON. + /// + public static IEnumerable GetParsedErrors(this HtmlValidationResult result) => ParseOutput(result.Output); + + private static IEnumerable ParseOutput(string output) + { + output = output.Trim(); + if ((!output.StartsWith('{') || !output.EndsWith('}')) && + (!output.StartsWith('[') || !output.EndsWith(']'))) + { + throw new JsonException($"Invalid JSON, make sure to set the OutputFormatter to JSON. Output: {output}"); + } + + try + { + var document = JsonDocument.Parse(output); + return document.RootElement.EnumerateArray() + .SelectMany(element => element.GetProperty("messages").EnumerateArray()) + .Select(message => + { + var rawMessageText = message.GetRawText(); + return JsonSerializer.Deserialize(rawMessageText); + }); + } + catch (JsonException) + { + throw new JsonException("Unable to parse output, did you set the OutputFormatter to JSON?"); + } + } } diff --git a/Lombiq.Tests.UI/Models/JsonHtmlValidationError.cs b/Lombiq.Tests.UI/Models/JsonHtmlValidationError.cs new file mode 100644 index 000000000..f5892f8c6 --- /dev/null +++ b/Lombiq.Tests.UI/Models/JsonHtmlValidationError.cs @@ -0,0 +1,28 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Lombiq.Tests.UI.Models; + +public class JsonHtmlValidationError +{ + [JsonPropertyName("ruleId")] + public string RuleId { get; set; } + [JsonPropertyName("severity")] + public int Severity { get; set; } + [JsonPropertyName("message")] + public string Message { get; set; } + [JsonPropertyName("offset")] + public int Offset { get; set; } + [JsonPropertyName("line")] + public int Line { get; set; } + [JsonPropertyName("column")] + public int Column { get; set; } + [JsonPropertyName("size")] + public int Size { get; set; } + [JsonPropertyName("selector")] + public string Selector { get; set; } + [JsonPropertyName("ruleUrl")] + public string RuleUrl { get; set; } + [JsonPropertyName("context")] + public JsonElement Context { get; set; } +} diff --git a/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs b/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs index f2bc8762c..77afe5204 100644 --- a/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs +++ b/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs @@ -1,7 +1,10 @@ +using Atata.Cli.HtmlValidate; using Atata.HtmlValidation; +using Lombiq.Tests.UI.Extensions; using Lombiq.Tests.UI.Helpers; using Shouldly; using System; +using System.Linq; using System.Threading.Tasks; namespace Lombiq.Tests.UI.Services; @@ -28,6 +31,8 @@ public class HtmlValidationConfiguration /// public HtmlValidationOptions HtmlValidationOptions { get; set; } = new() { + ResultFileFormatter = HtmlValidateFormatter.Names.Text, + OutputFormatter = HtmlValidateFormatter.Names.Json, SaveHtmlToFile = HtmlSaveCondition.Never, SaveResultToFile = true, // This is necessary so no long folder names will be generated, see: @@ -64,7 +69,18 @@ public class HtmlValidationConfiguration public static readonly Func AssertHtmlValidationOutputIsEmptyAsync = validationResult => { - validationResult.Output.ShouldBeEmpty(); + // Keep supporting cases where output format is not set to JSON. + if (validationResult.Output.Trim().StartsWith('[') || + validationResult.Output.Trim().StartsWith('{')) + { + var errors = validationResult.GetParsedErrors(); + errors.ShouldBeEmpty(string.Join('\n', errors.Select(error => error.Message))); + } + else + { + validationResult.Output.ShouldBeEmpty(); + } + return Task.CompletedTask; }; From 13fa417ee5ca9cebe619d9d6c404fddf9b6e0262 Mon Sep 17 00:00:00 2001 From: Aydin Erdas Date: Fri, 3 May 2024 19:52:31 +0200 Subject: [PATCH 02/15] Update Configuration.md --- Lombiq.Tests.UI/Docs/Configuration.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Lombiq.Tests.UI/Docs/Configuration.md b/Lombiq.Tests.UI/Docs/Configuration.md index f580ff8be..b786631e1 100644 --- a/Lombiq.Tests.UI/Docs/Configuration.md +++ b/Lombiq.Tests.UI/Docs/Configuration.md @@ -59,8 +59,7 @@ Recommendations and notes for such configuration: ### HTML validation configuration -If you want to change some HTML validation rules for only a few specific tests you can exclude them from the results. -For example if you want to exclude the `prefer-native-element` rule from the results you can do it by doing the following: +If you want to change some HTML validation rules for only a few specific tests you can exclude them from the results. For example if you want to exclude the `prefer-native-element` rule from the results you can do it by doing the following: ```c# configuration => configuration.HtmlValidationConfiguration.AssertHtmlValidationResultAsync = @@ -73,8 +72,7 @@ configuration => configuration.HtmlValidationConfiguration.AssertHtmlValidationR }); ``` -Note that the `RuleId` is the identifier of the rule that you want to exclude from the results. -The custom string formatter in the call to `errors.ShouldBeEmpty` is used to display the errors in a more readable way and is not strictly necessary. +Note that the `RuleId` is the identifier of the rule that you want to exclude from the results. The custom string formatter in the call to `errors.ShouldBeEmpty` is used to display the errors in a more readable way and is not strictly necessary. ## Multi-process test execution From 6c37fd24ffd7c7f0ac4ca31705f72650bdd815b2 Mon Sep 17 00:00:00 2001 From: Aydin Erdas Date: Fri, 3 May 2024 21:33:59 +0200 Subject: [PATCH 03/15] Update HtmlValidationResultExtensions.cs --- .../Extensions/HtmlValidationResultExtensions.cs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs b/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs index 61c9bcc35..c134f1a6a 100644 --- a/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs @@ -33,13 +33,6 @@ public static async Task> GetErrorsAsync(this HtmlValidation private static IEnumerable ParseOutput(string output) { - output = output.Trim(); - if ((!output.StartsWith('{') || !output.EndsWith('}')) && - (!output.StartsWith('[') || !output.EndsWith(']'))) - { - throw new JsonException($"Invalid JSON, make sure to set the OutputFormatter to JSON. Output: {output}"); - } - try { var document = JsonDocument.Parse(output); @@ -51,9 +44,9 @@ private static IEnumerable ParseOutput(string output) return JsonSerializer.Deserialize(rawMessageText); }); } - catch (JsonException) + catch (JsonException exception) { - throw new JsonException("Unable to parse output, did you set the OutputFormatter to JSON?"); + throw new JsonException($"Unable to parse output, did you set the OutputFormatter to JSON? Output: {output}", exception); } } } From aa48a7d08213856820bf6b7e417a2e1a1834d0eb Mon Sep 17 00:00:00 2001 From: Aydin Erdas Date: Mon, 6 May 2024 09:11:29 +0200 Subject: [PATCH 04/15] Use output file instead of output --- .../Extensions/HtmlValidationResultExtensions.cs | 12 +++++++++++- .../Services/HtmlValidationConfiguration.cs | 10 ++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs b/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs index c134f1a6a..b05c62fd9 100644 --- a/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs @@ -29,7 +29,17 @@ public static async Task> GetErrorsAsync(this HtmlValidation /// Gets the parsed errors from the HTML validation result. /// Can only be used if the output formatter is set to JSON. /// - public static IEnumerable GetParsedErrors(this HtmlValidationResult result) => ParseOutput(result.Output); + public static async Task> GetParsedErrorsAsync(this HtmlValidationResult result) + { + if (string.IsNullOrEmpty(result.ResultFilePath) || !File.Exists(result.ResultFilePath)) + { + return new List(); + } + + var fullOutput = await File.ReadAllTextAsync(result.ResultFilePath); + + return ParseOutput(fullOutput); + } private static IEnumerable ParseOutput(string output) { diff --git a/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs b/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs index 77afe5204..5623a0394 100644 --- a/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs +++ b/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs @@ -31,8 +31,8 @@ public class HtmlValidationConfiguration /// public HtmlValidationOptions HtmlValidationOptions { get; set; } = new() { - ResultFileFormatter = HtmlValidateFormatter.Names.Text, - OutputFormatter = HtmlValidateFormatter.Names.Json, + ResultFileFormatter = HtmlValidateFormatter.Names.Json, + OutputFormatter = HtmlValidateFormatter.Names.Stylish, SaveHtmlToFile = HtmlSaveCondition.Never, SaveResultToFile = true, // This is necessary so no long folder names will be generated, see: @@ -67,21 +67,19 @@ public class HtmlValidationConfiguration EnableOnValidatablePagesHtmlValidationAndAssertionOnPageChangeRule; public static readonly Func AssertHtmlValidationOutputIsEmptyAsync = - validationResult => + async validationResult => { // Keep supporting cases where output format is not set to JSON. if (validationResult.Output.Trim().StartsWith('[') || validationResult.Output.Trim().StartsWith('{')) { - var errors = validationResult.GetParsedErrors(); + var errors = await validationResult.GetParsedErrorsAsync(); errors.ShouldBeEmpty(string.Join('\n', errors.Select(error => error.Message))); } else { validationResult.Output.ShouldBeEmpty(); } - - return Task.CompletedTask; }; public static readonly Predicate EnableOnValidatablePagesHtmlValidationAndAssertionOnPageChangeRule = From 2c5c21b02c18d1ab449f3bbcb2fc66d2022e466e Mon Sep 17 00:00:00 2001 From: Aydin Erdas Date: Mon, 6 May 2024 09:51:51 +0200 Subject: [PATCH 05/15] Logging --- Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs b/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs index b05c62fd9..c4cd93e11 100644 --- a/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs @@ -56,7 +56,9 @@ private static IEnumerable ParseOutput(string output) } catch (JsonException exception) { - throw new JsonException($"Unable to parse output, did you set the OutputFormatter to JSON? Output: {output}", exception); + throw new JsonException( + $"Unable to parse output, was OutputFormatter set to JSON? Length: {output.Length} Output: {output[0]} - {output[^1]}", + exception); } } } From 6e5e040166b66189d26a389fdac9c34723c96e4b Mon Sep 17 00:00:00 2001 From: Aydin Erdas Date: Mon, 6 May 2024 09:55:15 +0200 Subject: [PATCH 06/15] Update HtmlValidationResultExtensions.cs --- Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs b/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs index c4cd93e11..5bb2da51f 100644 --- a/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs @@ -57,7 +57,8 @@ private static IEnumerable ParseOutput(string output) catch (JsonException exception) { throw new JsonException( - $"Unable to parse output, was OutputFormatter set to JSON? Length: {output.Length} Output: {output[0]} - {output[^1]}", + $"Unable to parse output, was OutputFormatter set to JSON? Length: {output.Length} " + + $"Output: {output[..Math.Min(10, output.Length)]} - {output[Math.Max(0, output.Length - 10)..]}", exception); } } From 68623e47b59ee5b0791a76ad6b44380e18e5fcb1 Mon Sep 17 00:00:00 2001 From: Aydin Erdas Date: Mon, 6 May 2024 14:22:16 +0200 Subject: [PATCH 07/15] Update HtmlValidationResultExtensions.cs --- .../Extensions/HtmlValidationResultExtensions.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs b/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs index 5bb2da51f..fb879f10f 100644 --- a/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs @@ -45,6 +45,12 @@ private static IEnumerable ParseOutput(string output) { try { + // In some cases the output is too large and is not a valid JSON anymore. In this case we need to fix it. + if (output.Trim().StartsWith('[') && !output.Trim().EndsWith(']')) + { + output += "\"}]"; + } + var document = JsonDocument.Parse(output); return document.RootElement.EnumerateArray() .SelectMany(element => element.GetProperty("messages").EnumerateArray()) From 9e23b39ec9462209c777aebf8febb76326fba798 Mon Sep 17 00:00:00 2001 From: Aydin Erdas Date: Mon, 6 May 2024 14:49:38 +0200 Subject: [PATCH 08/15] Update HtmlValidationResultExtensions.cs --- Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs b/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs index fb879f10f..e27358ae9 100644 --- a/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs @@ -64,7 +64,7 @@ private static IEnumerable ParseOutput(string output) { throw new JsonException( $"Unable to parse output, was OutputFormatter set to JSON? Length: {output.Length} " + - $"Output: {output[..Math.Min(10, output.Length)]} - {output[Math.Max(0, output.Length - 10)..]}", + $"Output: {output}", exception); } } From db4ae9917d2de79af16fee45a32fc31e235806ef Mon Sep 17 00:00:00 2001 From: Aydin Erdas Date: Tue, 7 May 2024 10:59:30 +0200 Subject: [PATCH 09/15] Use output directly instead of file --- .../Extensions/HtmlValidationResultExtensions.cs | 12 +----------- .../Services/HtmlValidationConfiguration.cs | 3 +++ 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs b/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs index e27358ae9..2faa4c78a 100644 --- a/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs @@ -29,17 +29,7 @@ public static async Task> GetErrorsAsync(this HtmlValidation /// Gets the parsed errors from the HTML validation result. /// Can only be used if the output formatter is set to JSON. /// - public static async Task> GetParsedErrorsAsync(this HtmlValidationResult result) - { - if (string.IsNullOrEmpty(result.ResultFilePath) || !File.Exists(result.ResultFilePath)) - { - return new List(); - } - - var fullOutput = await File.ReadAllTextAsync(result.ResultFilePath); - - return ParseOutput(fullOutput); - } + public static IEnumerable GetParsedErrors(this HtmlValidationResult result) => ParseOutput(result.Output); private static IEnumerable ParseOutput(string output) { diff --git a/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs b/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs index e85b16f7d..4f9601803 100644 --- a/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs +++ b/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs @@ -1,3 +1,4 @@ +using Atata.Cli.HtmlValidate; using Atata.HtmlValidation; using Lombiq.Tests.UI.Helpers; using Shouldly; @@ -29,6 +30,8 @@ public class HtmlValidationConfiguration /// public HtmlValidationOptions HtmlValidationOptions { get; set; } = new() { + OutputFormatter = HtmlValidateFormatter.Names.Json, + ResultFileFormatter = HtmlValidateFormatter.Names.Codeframe, SaveHtmlToFile = HtmlSaveCondition.Never, SaveResultToFile = true, // This is necessary so no long folder names will be generated, see: From 57732e1d20d3f5f02cd13410469795627ac14c5b Mon Sep 17 00:00:00 2001 From: Aydin Erdas Date: Tue, 7 May 2024 11:01:55 +0200 Subject: [PATCH 10/15] Update HtmlValidationConfiguration.cs --- Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs b/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs index 4f9601803..05ceebb57 100644 --- a/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs +++ b/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs @@ -31,7 +31,6 @@ public class HtmlValidationConfiguration public HtmlValidationOptions HtmlValidationOptions { get; set; } = new() { OutputFormatter = HtmlValidateFormatter.Names.Json, - ResultFileFormatter = HtmlValidateFormatter.Names.Codeframe, SaveHtmlToFile = HtmlSaveCondition.Never, SaveResultToFile = true, // This is necessary so no long folder names will be generated, see: From b47339f294b779b820c0d3ac6ec7d90700a41228 Mon Sep 17 00:00:00 2001 From: Aydin Erdas Date: Tue, 7 May 2024 14:37:27 +0200 Subject: [PATCH 11/15] Update Configuration.md --- Lombiq.Tests.UI/Docs/Configuration.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Docs/Configuration.md b/Lombiq.Tests.UI/Docs/Configuration.md index 4ed02a4bd..f3707b632 100644 --- a/Lombiq.Tests.UI/Docs/Configuration.md +++ b/Lombiq.Tests.UI/Docs/Configuration.md @@ -59,7 +59,22 @@ Recommendations and notes for such configuration: ### HTML validation configuration -If you want to change some HTML validation rules from only a few specific tests, you can create a custom _.htmlvalidate.json_ file (e.g. _TestName.htmlvalidate.json_). This should extend the [default.htmlvalidate.json](../default.htmlvalidate.json) file (which is always copied into the build directory) by setting the value of `"extends"` to a relative path pointing to it and declaring `"root": true`. For example: +If you want to filter out certain html validation errors for a specific test you can simply filter them out of the error results by their rule ID. For example: + +```c# +configuration => configuration.HtmlValidationConfiguration.AssertHtmlValidationResultAsync = + validationResult => + { + var errors = validationResult.GetParsedErrors() + .Where(error => error.RuleId is not "prefer-native-element"); + errors.ShouldBeEmpty(string.Join('\n', errors.Select(error => error.Message))); + return Task.CompletedTask; + }); +``` + +Note that the `RuleId` is the identifier of the rule that you want to exclude from the results. The custom string formatter in the call to `errors.ShouldBeEmpty` is used to display the errors in a more readable way and is not strictly necessary. + +If you want to change some HTML validation rules for multiple tests, you can also create a custom _.htmlvalidate.json_ file (e.g. _TestName.htmlvalidate.json_). This should extend the [default.htmlvalidate.json](../default.htmlvalidate.json) file (which is always copied into the build directory) by setting the value of `"extends"` to a relative path pointing to it and declaring `"root": true`. For example: ```json { From 416b1a7420f5abc29f39a43592aac7717fd554e0 Mon Sep 17 00:00:00 2001 From: Aydin Erdas Date: Tue, 7 May 2024 15:14:16 +0200 Subject: [PATCH 12/15] Update HtmlValidationConfiguration.cs --- .../Services/HtmlValidationConfiguration.cs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs b/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs index 05ceebb57..23df627e3 100644 --- a/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs +++ b/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs @@ -1,9 +1,11 @@ using Atata.Cli.HtmlValidate; using Atata.HtmlValidation; +using Lombiq.Tests.UI.Extensions; using Lombiq.Tests.UI.Helpers; using Shouldly; using System; using System.IO; +using System.Linq; using System.Threading.Tasks; namespace Lombiq.Tests.UI.Services; @@ -90,7 +92,18 @@ public HtmlValidationConfiguration WithRelativeConfigPath(params string[] pathSe public static readonly Func AssertHtmlValidationOutputIsEmptyAsync = validationResult => { - validationResult.Output.ShouldBeEmpty(); + // Keep supporting cases where output format is not set to JSON. + if (validationResult.Output.Trim().StartsWith('[') || + validationResult.Output.Trim().StartsWith('{')) + { + var errors = validationResult.GetParsedErrors(); + errors.ShouldBeEmpty(string.Join('\n', errors.Select(error => error.Message))); + } + else + { + validationResult.Output.ShouldBeEmpty(); + } + return Task.CompletedTask; }; From 274567d3f020b497abeeac500b029c199fdeefca Mon Sep 17 00:00:00 2001 From: Aydin Erdas Date: Wed, 8 May 2024 10:33:08 +0200 Subject: [PATCH 13/15] Track JSON formatter github issue --- Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs b/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs index 2faa4c78a..ff4102167 100644 --- a/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs @@ -36,6 +36,7 @@ private static IEnumerable ParseOutput(string output) try { // In some cases the output is too large and is not a valid JSON anymore. In this case we need to fix it. + // tracking issue: https://github.com/atata-framework/atata-htmlvalidation/issues/9 if (output.Trim().StartsWith('[') && !output.Trim().EndsWith(']')) { output += "\"}]"; From fb4483e2469bf368b8d4c5f00623f1b1b4841d7d Mon Sep 17 00:00:00 2001 From: Aydin Erdas Date: Fri, 10 May 2024 10:59:25 +0200 Subject: [PATCH 14/15] Delete .editorconfig --- .editorconfig | 45 --------------------------------------------- 1 file changed, 45 deletions(-) delete mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig deleted file mode 100644 index 574b0e97d..000000000 --- a/.editorconfig +++ /dev/null @@ -1,45 +0,0 @@ -# WARNING: Only edit this file in the Lombiq .NET Analyzers repository's "Lombiq.Analyzers" folder. A copy of this file -# anywhere else will be overwritten. - -# All files -[*] - -# Basics -charset = utf-8 -guidelines = 120 1px solid a0ffc000, 150 1px solid 80ff0000 - -# Indentation and spacing -indent_size = 4 -indent_style = space -trim_trailing_whitespace = true - -# New line preferences -end_of_line = crlf -insert_final_newline = true -tab_width = 4 - -# Various config files -[*.{config,csproj,json,props,targets}] - -indent_size = 2 - -# Markdown files -[*.md] - -trim_trailing_whitespace = false - -# JavaScript files -[*.js] - -# Placeholder, no unique rules for JS files at the moment. - - -# SCSS files -[*.scss] - -# Placeholder, no unique rules for SCSS files at the moment. - -# PowerShell files -[*.ps1] - -# Placeholder, no unique rules for PS files at the moment. From e3da0f686b7cf43fe173bc7e9e11eb9070f09f32 Mon Sep 17 00:00:00 2001 From: Aydin Erdas Date: Fri, 10 May 2024 11:26:17 +0200 Subject: [PATCH 15/15] Create method for building error messages --- Lombiq.Tests.UI/Docs/Configuration.md | 2 +- .../Extensions/HtmlValidationResultExtensions.cs | 8 ++++++++ Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs | 3 +-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Lombiq.Tests.UI/Docs/Configuration.md b/Lombiq.Tests.UI/Docs/Configuration.md index f3707b632..44327b7fc 100644 --- a/Lombiq.Tests.UI/Docs/Configuration.md +++ b/Lombiq.Tests.UI/Docs/Configuration.md @@ -67,7 +67,7 @@ configuration => configuration.HtmlValidationConfiguration.AssertHtmlValidationR { var errors = validationResult.GetParsedErrors() .Where(error => error.RuleId is not "prefer-native-element"); - errors.ShouldBeEmpty(string.Join('\n', errors.Select(error => error.Message))); + errors.ShouldBeEmpty(HtmlValidationResultExtensions.GetParsedErrorMessageString(errors)); return Task.CompletedTask; }); ``` diff --git a/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs b/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs index ff4102167..2ccc67d25 100644 --- a/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs @@ -2,6 +2,7 @@ using Lombiq.Tests.UI.Models; using System; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; using System.Text.Json; @@ -31,6 +32,13 @@ public static async Task> GetErrorsAsync(this HtmlValidation /// public static IEnumerable GetParsedErrors(this HtmlValidationResult result) => ParseOutput(result.Output); + public static string GetParsedErrorMessageString(IEnumerable errors) => + string.Join( + '\n', errors.Select(error => + $"{error.Line.ToString(CultureInfo.InvariantCulture)}:{error.Column.ToString(CultureInfo.InvariantCulture)} - " + + $"{error.Message} - " + + $"{error.RuleId}")); + private static IEnumerable ParseOutput(string output) { try diff --git a/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs b/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs index 23df627e3..5aa04d86c 100644 --- a/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs +++ b/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs @@ -5,7 +5,6 @@ using Shouldly; using System; using System.IO; -using System.Linq; using System.Threading.Tasks; namespace Lombiq.Tests.UI.Services; @@ -97,7 +96,7 @@ public HtmlValidationConfiguration WithRelativeConfigPath(params string[] pathSe validationResult.Output.Trim().StartsWith('{')) { var errors = validationResult.GetParsedErrors(); - errors.ShouldBeEmpty(string.Join('\n', errors.Select(error => error.Message))); + errors.ShouldBeEmpty(HtmlValidationResultExtensions.GetParsedErrorMessageString(errors)); } else {