Skip to content

Commit

Permalink
Merge pull request #362 from Lombiq/issue/OSOE-838
Browse files Browse the repository at this point in the history
OSOE-838: Updating Atata v3.1.0, and other dependencies to latest too, reliability improvements
  • Loading branch information
Piedone authored May 15, 2024
2 parents 7bb8d63 + e548468 commit 003d6f5
Show file tree
Hide file tree
Showing 16 changed files with 112 additions and 100 deletions.
7 changes: 2 additions & 5 deletions Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,6 @@
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<IsPackable>false</IsPackable>
<!-- Eliminates problem with dotnet publish caused by duplicate .htmlvalidate.json file. See
https://docs.microsoft.com/en-us/dotnet/core/compatibility/sdk/6.0/duplicate-files-in-output. -->
<ErrorOnDuplicatePublishOutputFiles>false</ErrorOnDuplicatePublishOutputFiles>
</PropertyGroup>

<ItemGroup>
Expand All @@ -16,7 +13,7 @@

<ItemGroup>
<Content Include=".htmlvalidate.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="Tests\CustomZapAutomationFrameworkPlan.yml">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
Expand All @@ -28,7 +25,7 @@

<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.7.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.5.1">
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.0">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
Expand Down
6 changes: 3 additions & 3 deletions Lombiq.Tests.UI.Samples/Tests/InteractiveModeTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,8 @@ public Task SampleTest() =>
});

// This test checks if interactive mode works by opening it in one thread, and then clicking it away in a different
// thread. This ensures that the new tab correctly appears with the clickable "Continue Test" button, and then
// disappears once it's clicked.
// thread. Two threads are necessary because interactive mode stops test execution on its current thread, so we
// wouldn't be able to end it from within a test.
[Fact]
public Task EnteringInteractiveModeShouldWait() =>
ExecuteTestAfterSetupAsync(
Expand All @@ -59,7 +59,7 @@ await Task.WhenAll(
{
// Ensure that the interactive mode polls for status at least once, so the arbitrary waiting
// actually works in a real testing scenario.
await Task.Delay(1000);
await Task.Delay(5000);
await context.ClickReliablyOnAsync(By.ClassName("interactive__continue"));
}));
Expand Down
2 changes: 1 addition & 1 deletion Lombiq.Tests.UI.Samples/UITestBase.cs
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ protected override Task ExecuteTestAsync(
// disable it with the below config). With this, you can make sure that the HTML markup the app
// generates (also from content items) is valid. While the default settings for HTML validation are most
// possibly suitable for your projects, check out the HtmlValidationConfiguration class for what else
// you can configure. We've also added a .htmlvalidate.json file (note the Content Build
// you can configure. We've also added a custom .htmlvalidate.json file (note the Content Build
// Action) to further configure it.
////configuration.HtmlValidationConfiguration.RunHtmlValidationAssertionOnAllPageChanges = false;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
namespace Lombiq.Tests.UI.Extensions;

/// <summary>
/// Extension methods to retrieve elements using Atata helpers. See the Atata docs ( <see
/// Extension methods to retrieve elements using Atata helpers. See the Atata docs (<see
/// href="https://github.com/atata-framework/atata-webdriverextras#usage"/>) for more information on what you can do
/// with these.
/// </summary>
Expand Down Expand Up @@ -146,5 +146,5 @@ private static ExtendedSearchContext<IWebDriver> CreateSearchContext(this UITest
new(
context.Driver,
context.Configuration.TimeoutConfiguration.RetryTimeout,
context.Configuration.TimeoutConfiguration.RetryTimeout);
context.Configuration.TimeoutConfiguration.RetryInterval);
}
22 changes: 10 additions & 12 deletions Lombiq.Tests.UI/Extensions/ExtendedLoggingExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -148,15 +148,15 @@ private static async Task<TResult> ExecuteSectionAsync<TResult>(
var notLast = i < StabilityRetryCount - 1;
try
{
// This is somewhat risky. ILogManager is not thread-safe and uses as stack to keep track of sections, so if
// multiple sections are started in concurrent threads, the result will be incorrect. This shouldn't be too much
// of an issue for now though since tests, while async, are single-threaded.
context.Scope.AtataContext.Log.Start(section);
context.Scope.AtataContext.Log.Info("Log section {0} started.", section.Message);
var result = await functionAsync();
context.Scope.AtataContext.Log.Info("Log section {0} ended.", section.Message);
context.Scope.AtataContext.Log.EndSection();
return result;
return await context.Scope.AtataContext.Log.ExecuteSectionAsync(
section,
async () =>
{
context.Scope.AtataContext.Log.Info($"Log section {section.Message} started.");
var result = await functionAsync();
context.Scope.AtataContext.Log.Info($"Log section {section.Message} ended.");
return result;
});
}
catch (StaleElementReferenceException) when (notLast)
{
Expand All @@ -171,7 +171,5 @@ private static async Task<TResult> ExecuteSectionAsync<TResult>(
private static void LogStaleElementReferenceExceptionRetry(UITestContext context, int tryIndex) =>
context.Scope.AtataContext.Log.Info(
"The operation in the log section failed with StaleElementReferenceException but will be retried. This " +
"is try number {0} out of {1}.",
tryIndex + 1,
StabilityRetryCount);
$"is try number {(tryIndex + 1).ToTechnicalString()} out of {StabilityRetryCount}.");
}
7 changes: 7 additions & 0 deletions Lombiq.Tests.UI/Extensions/FormUITestContextExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,8 @@ public static void FillInMonacoEditor(
string editorId,
string text)
{
WaitForMonacoEditor(context, editorId);

var script = $@"
monaco.editor.getEditors().find((element) =>
element.getContainerDomNode().id == {JsonConvert.SerializeObject(editorId)}).setValue({JsonConvert.SerializeObject(text)});";
Expand All @@ -122,6 +124,8 @@ public static string GetMonacoEditorText(
this UITestContext context,
string editorId)
{
WaitForMonacoEditor(context, editorId);

var script = $@"
return monaco.editor.getEditors().find((element) =>
element.getContainerDomNode().id == {JsonConvert.SerializeObject(editorId)}).getValue();";
Expand Down Expand Up @@ -375,4 +379,7 @@ private static IWebElement TryFillElement(UITestContext context, By by, string t

return context.Driver.TryFillElement(element, text);
}

private static void WaitForMonacoEditor(UITestContext context, string editorId) =>
context.Get(By.CssSelector($"#{editorId} .monaco-editor"));
}
3 changes: 2 additions & 1 deletion Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ public static async Task<IEnumerable<string>> GetErrorsAsync(this HtmlValidation

public static string GetParsedErrorMessageString(IEnumerable<JsonHtmlValidationError> errors) =>
string.Join(
'\n', errors.Select(error =>
'\n',
errors.Select(error =>
$"{error.Line.ToString(CultureInfo.InvariantCulture)}:{error.Column.ToString(CultureInfo.InvariantCulture)} - " +
$"{error.Message} - " +
$"{error.RuleId}"));
Expand Down
25 changes: 12 additions & 13 deletions Lombiq.Tests.UI/Lombiq.Tests.UI.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -59,31 +59,30 @@
</ItemGroup>

<ItemGroup>
<PackageReference Include="Atata" Version="2.9.0" />
<PackageReference Include="Atata.Bootstrap" Version="2.1.0" />
<PackageReference Include="Atata.HtmlValidation" Version="2.4.0" />
<PackageReference Include="Atata.WebDriverExtras" Version="2.2.0" />
<PackageReference Include="Atata" Version="3.1.0" />
<PackageReference Include="Atata.Bootstrap" Version="3.0.0" />
<PackageReference Include="Atata.HtmlValidation" Version="3.0.0" />
<PackageReference Include="Atata.WebDriverExtras" Version="3.0.0" />
<PackageReference Include="Atata.WebDriverSetup" Version="2.10.0" />
<PackageReference Include="Azure.Storage.Blobs" Version="12.18.0" />
<PackageReference Include="Azure.Storage.Blobs" Version="12.19.1" />
<PackageReference Include="Ben.Demystifier" Version="0.4.1" />
<PackageReference Include="Codeuctivity.ImageSharpCompare" Version="3.0.183" />
<PackageReference Include="Codeuctivity.ImageSharpCompare" Version="4.0.258" />
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
<PackageReference Include="Microsoft.SqlServer.DACFx" Version="162.0.52" />
<PackageReference Include="Microsoft.SqlServer.SqlManagementObjects" Version="170.18.0" />
<PackageReference Include="Microsoft.SqlServer.DACFx" Version="162.2.111" />
<PackageReference Include="Microsoft.SqlServer.SqlManagementObjects" Version="171.30.0" />
<PackageReference Include="Mono.Posix.NETStandard" Version="1.0.0" />
<PackageReference Include="OrchardCore.ContentFields" Version="1.8.2" />
<PackageReference Include="OrchardCore.Logging.NLog" Version="1.8.2" />
<PackageReference Include="OrchardCore.Abstractions" Version="1.8.2" />
<PackageReference Include="OrchardCore.Recipes.Core" Version="1.8.2" />
<PackageReference Include="Refit.HttpClientFactory" Version="7.0.0" />
<PackageReference Include="Sarif.Sdk" Version="4.3.7" />
<PackageReference Include="Selenium.Axe" Version="4.0.14" />
<PackageReference Include="Sarif.Sdk" Version="4.5.4" />
<PackageReference Include="Selenium.Axe" Version="4.0.17" />
<PackageReference Include="Shouldly" Version="4.2.1" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
<PackageReference Include="WebDriverManager" Version="2.16.2" />
<PackageReference Include="xunit" Version="2.5.1" />
<PackageReference Include="YamlDotNet" Version="13.7.1" />
<PackageReference Include="xunit" Version="2.8.0" />
<PackageReference Include="YamlDotNet" Version="15.1.4" />
</ItemGroup>

<ItemGroup>
Expand Down
82 changes: 35 additions & 47 deletions Lombiq.Tests.UI/MonkeyTesting/MonkeyTester.cs
Original file line number Diff line number Diff line change
Expand Up @@ -26,61 +26,49 @@ internal MonkeyTester(UITestContext context, MonkeyTestingOptions options = null
_randomizer = new NonSecurityRandomizer(_options.BaseRandomSeed);
}

internal async Task TestOnePageAsync(int? randomSeed = null)
{
Log.Start(new LogSection("Execute monkey testing against one page"));

try
{
WriteOptionsToLog();

var pageTestInfo = GetCurrentPageTestInfo();

if (randomSeed is null) await TestCurrentPageAsync(pageTestInfo);
else await TestCurrentPageWithRandomSeedAsync(pageTestInfo, randomSeed.Value);
}
finally
{
Log.EndSection();
}
}
internal Task TestOnePageAsync(int? randomSeed = null) =>
Log.ExecuteSectionAsync(
new LogSection("Execute monkey testing against one page"),
async () =>
{
WriteOptionsToLog();
internal async Task TestRecursivelyAsync()
{
Log.Start(new LogSection($"Execute monkey testing recursively"));
var pageTestInfo = GetCurrentPageTestInfo();
try
{
WriteOptionsToLog();

var pageTestInfo = GetCurrentPageTestInfo();
await TestCurrentPageAsync(pageTestInfo);
if (randomSeed is null) await TestCurrentPageAsync(pageTestInfo);
else await TestCurrentPageWithRandomSeedAsync(pageTestInfo, randomSeed.Value);
});

while (true)
internal Task TestRecursivelyAsync() =>
Log.ExecuteSectionAsync(
new LogSection($"Execute monkey testing recursively"),
async () =>
{
pageTestInfo = GetCurrentPageTestInfo();
WriteOptionsToLog();
if (CanTestPage(pageTestInfo))
{
await TestCurrentPageAsync(pageTestInfo);
}
else if (TryGetAvailablePageToTest(out var availablePageToTest))
{
await _context.GoToAbsoluteUrlAsync(availablePageToTest.Url);
var pageTestInfo = GetCurrentPageTestInfo();
await TestCurrentPageAsync(pageTestInfo);
await TestCurrentPageAsync(availablePageToTest);
}
else
while (true)
{
return;
pageTestInfo = GetCurrentPageTestInfo();
if (CanTestPage(pageTestInfo))
{
await TestCurrentPageAsync(pageTestInfo);
}
else if (TryGetAvailablePageToTest(out var availablePageToTest))
{
await _context.GoToAbsoluteUrlAsync(availablePageToTest.Url);
await TestCurrentPageAsync(availablePageToTest);
}
else
{
return;
}
}
}
}
finally
{
Log.EndSection();
}
}
});

private void WriteOptionsToLog() =>
Log.Trace(@$"Monkey testing options:
Expand Down
4 changes: 2 additions & 2 deletions Lombiq.Tests.UI/Pages/OrchardCoreAdminPage.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@ public abstract class OrchardCoreAdminPage<TOwner> : Page<TOwner>

public ControlList<AlertMessage<TOwner>, TOwner> AlertMessages { get; private set; }

public TOwner ShouldStayOnAdminPage() => AdminMenu.Should.Exist();
public TOwner ShouldStayOnAdminPage() => AdminMenu.Should.BePresent();

public TOwner ShouldLeaveAdminPage() => AdminMenu.Should.Not.Exist();
public TOwner ShouldLeaveAdminPage() => AdminMenu.Should.Not.BePresent();

protected override void OnVerify()
{
Expand Down
8 changes: 5 additions & 3 deletions Lombiq.Tests.UI/Services/AtataFactory.cs
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,13 @@ public class AtataConfiguration
public static class AtataFactory
{
public static async Task<AtataScope> StartAtataScopeAsync(
string contextId,
ITestOutputHelper testOutputHelper,
Uri baseUri,
OrchardCoreUITestExecutorConfiguration configuration)
{
AtataContext.ModeOfCurrent = AtataContextModeOfCurrent.AsyncLocal;
AtataContext.GlobalProperties.ModeOfCurrent = AtataContextModeOfCurrent.AsyncLocal;
AtataContext.GlobalProperties.UseUtcTimeZone();

// Since Atata 2.0 the default visibility option is Visibility.Any, these lines restore it to the 1.x behavior.
AtataContext.GlobalConfiguration.UseDefaultControlVisibility(Visibility.Visible);
Expand All @@ -37,8 +39,8 @@ public static async Task<AtataScope> StartAtataScopeAsync(
.UseTestName(configuration.AtataConfiguration.TestName)
.UseBaseRetryTimeout(timeoutConfiguration.RetryTimeout)
.UseBaseRetryInterval(timeoutConfiguration.RetryInterval)
.UseUtcTimeZone()
.PageSnapshots.UseCdpOrPageSourceStrategy(); // #spell-check-ignore-line
.PageSnapshots.UseCdpOrPageSourceStrategy() // #spell-check-ignore-line
.UseArtifactsPathTemplate(contextId); // Necessary to prevent long paths, an issue under Windows.

builder.LogConsumers.AddDebug();
builder.LogConsumers.Add(new TestOutputLogConsumer(testOutputHelper));
Expand Down
3 changes: 2 additions & 1 deletion Lombiq.Tests.UI/Services/OrchardCoreInstance.cs
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,8 @@ await _configuration.BeforeAppStart

private async Task StopOrchardAppAsync()
{
_reverseProxy.DetachConnectionProvider();
_reverseProxy?.DetachConnectionProvider();

if (_orchardApplication == null) return;

_testOutputHelper.WriteLineTimestampedAndDebug("Attempting to stop the Orchard Core instance.");
Expand Down
2 changes: 1 addition & 1 deletion Lombiq.Tests.UI/Services/UITestExecutionSession.cs
Original file line number Diff line number Diff line change
Expand Up @@ -656,7 +656,7 @@ Task UITestingBeforeAppStartHandlerAsync(string contentRootPath, InstanceCommand
_configuration.Events.AfterPageChange += TakeScreenshotIfEnabledAsync;
}

var atataScope = await AtataFactory.StartAtataScopeAsync(_testOutputHelper, uri, _configuration);
var atataScope = await AtataFactory.StartAtataScopeAsync(contextId, _testOutputHelper, uri, _configuration);

return new UITestContext(
contextId,
Expand Down
30 changes: 24 additions & 6 deletions Lombiq.Tests.UI/Services/UITestExecutor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -125,11 +125,26 @@ private static string PrepareDumpFolder(
{
var dumpConfiguration = configuration.FailureDumpConfiguration;
var dumpFolderNameBase = testManifest.Name;
if (dumpConfiguration.UseShortNames && dumpFolderNameBase.Contains('(', StringComparison.Ordinal))
if (dumpConfiguration.UseShortNames)
{
var dumpFolderNameBeginningIndex =
dumpFolderNameBase[..dumpFolderNameBase.IndexOf('(', StringComparison.Ordinal)].LastIndexOf('.') + 1;
dumpFolderNameBase = dumpFolderNameBase[dumpFolderNameBeginningIndex..];
if (dumpFolderNameBase.Contains('(', StringComparison.Ordinal))
{
// The test uses parameters and is thus in the
// "Lombiq.Tests.UI.Samples.Tests.BasicTests.AnonymousHomePageShouldExist(browser: Chrome)" format.
var dumpFolderNameBeginningIndex =
dumpFolderNameBase[..dumpFolderNameBase.IndexOf('(', StringComparison.Ordinal)].LastIndexOf('.') + 1;
dumpFolderNameBase = dumpFolderNameBase[dumpFolderNameBeginningIndex..];
}
else
{
// The test doesn't use parameters and is thus in the
// "Lombiq.Tests.UI.Samples.Tests.BasicTests.AnonymousHomePageShouldExist" format.
var dumpFolderNameBeginningIndex = dumpFolderNameBase.LastIndexOf('.') + 1;
dumpFolderNameBase = dumpFolderNameBase[dumpFolderNameBeginningIndex..];
}

// Can't use string.GetHasCode() because that varies between executions.
dumpFolderNameBase += "-" + Sha256Helper.ComputeHash(testManifest.Name);
}

dumpFolderNameBase = dumpFolderNameBase.MakeFileSystemFriendly();
Expand Down Expand Up @@ -159,9 +174,12 @@ private static string PrepareDumpFolder(
var openingBracketIndex = dumpFolderNameBase.IndexOf('(', StringComparison.Ordinal);
var closingBracketIndex = dumpFolderNameBase.LastIndexOf(')');

// Only adding a hash of the parameters if the hash of the test's full name is not already there due to
// path shortening above.
// Can't use string.GetHasCode() because that varies between executions.
var hashedParameters = Sha256Helper
.ComputeHash(dumpFolderNameBase[(openingBracketIndex + 1)..(closingBracketIndex + 1)]);
var hashedParameters = dumpConfiguration.UseShortNames
? string.Empty
: Sha256Helper.ComputeHash(dumpFolderNameBase[(openingBracketIndex + 1)..(closingBracketIndex + 1)]);

dumpFolderNameBase =
dumpFolderNameBase[0..(openingBracketIndex + 1)] +
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,9 @@ public class UITestExecutorFailureDumpConfiguration
{
/// <summary>
/// Gets or sets a value indicating whether the subfolder of each test's dumps will use a shortened name, only
/// containing the name of the test method, without the name of the test class and its namespace. This is to
/// overcome the 260 character path length limitations on Windows. Defaults to <see langword="true"/> on Windows.
/// containing the name of the test method suffixed with the test name's hash to make it unique, without the name of
/// the test class and its namespace. This is to overcome the 260 character path length limitations on Windows.
/// Defaults to <see langword="true"/> on Windows.
/// </summary>
public bool UseShortNames { get; set; } = OperatingSystem.IsWindows();

Expand Down
Loading

0 comments on commit 003d6f5

Please sign in to comment.