Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

TDEAL-38: Save suggested image if it's the final try regardless of retry count #389

Merged
merged 3 commits into from
Jul 9, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -9,18 +9,18 @@ public class VisualVerificationBaselineImageNotFoundException : Exception
{
public VisualVerificationBaselineImageNotFoundException(
string path,
int maxRetryCount,
bool isFinalTry,
Exception innerException = null)
: base(GetExceptionMessage(path, maxRetryCount), innerException)
: base(GetExceptionMessage(path, isFinalTry), innerException)
{
}

private static string GetExceptionMessage(string path, int maxRetryCount) =>
maxRetryCount == 0 ? $"Baseline image file not found, thus it was created automatically under the path {path}."
+ " Please set its \"Build action\" to \"Embedded resource\" if you want to deploy a self-contained"
+ " (like a NuGet package) UI testing assembly. If you run the test again, this newly created verification"
+ " file will be asserted against and the assertion will pass (unless the display of the app changed in the"
+ " meantime)."
: $"Baseline image file was not found under the path {path} and maxRetryCount is set to "
+ $"{maxRetryCount.ToTechnicalString()}, so it won't be generated. Set maxRetryCount to 0 to generate images.";
private static string GetExceptionMessage(string path, bool isFinalTry) =>
isFinalTry
? $"Baseline image file not found, thus it was created automatically under the path {path}. Please set " +
$"its \"Build action\" to \"Embedded resource\" if you want to deploy a self-contained (like a NuGet " +
$"package) UI testing assembly. If you run the test again, this newly created verification file will " +
$"be asserted against and the assertion will pass (unless the display of the app changed in the " +
$"meantime)."
: $"Baseline image file was not found under the path {path} and this isn't the final try.";
}
Original file line number Diff line number Diff line change
Expand Up @@ -423,7 +423,7 @@ private static void AssertVisualVerificationApproved(

if (!File.Exists(approvedContext.BaselineImagePath))
{
if (context.Configuration.MaxRetryCount == 0)
if (context.IsFinalTry)
{
context.SaveSuggestedImage(
element,
Expand All @@ -432,7 +432,7 @@ private static void AssertVisualVerificationApproved(
}

throw new VisualVerificationBaselineImageNotFoundException(
approvedContext.BaselineImagePath, context.Configuration.MaxRetryCount);
approvedContext.BaselineImagePath, context.IsFinalTry);
}

baselineImage = Image.Load(approvedContext.BaselineImagePath);
Expand Down
10 changes: 10 additions & 0 deletions Lombiq.Tests.UI/Services/UITestContext.cs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,16 @@ public class UITestContext
/// </summary>
public string Id { get; }

/// <summary>
/// Gets or sets the current retry count. This should only be edited from <c>UITestExecutionSession</c>.
/// </summary>
internal int RetryCount { get; set; }

/// <summary>
/// Gets a value indicating whether there may be further retries for this test.
/// </summary>
public bool IsFinalTry => RetryCount >= Configuration?.MaxRetryCount;

/// <summary>
/// Gets data about the currently executing test.
/// </summary>
Expand Down
6 changes: 5 additions & 1 deletion Lombiq.Tests.UI/Services/UITestExecutionSession.cs
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ public async Task<bool> ExecuteAsync(int retryCount, string dumpRootPath)
{
var startTime = DateTime.UtcNow;
IDictionary<string, IFailureDumpItem> failureDumpContainer = null;
// At this point _context may not exist yet.
if (_context != null) _context.RetryCount = retryCount;

_testOutputHelper.WriteLineTimestampedAndDebug("Starting execution of {0}.", _testManifest.Name);

Expand Down Expand Up @@ -108,6 +110,8 @@ public async Task<bool> ExecuteAsync(int retryCount, string dumpRootPath)
}

_context ??= await CreateContextAsync();
// At this point _context definitely exists, so ensure that RetryCount is set.
_context.RetryCount = retryCount;

_context.FailureDumpContainer.Clear();
failureDumpContainer = _context.FailureDumpContainer;
Expand All @@ -128,7 +132,7 @@ public async Task<bool> ExecuteAsync(int retryCount, string dumpRootPath)

await CreateFailureDumpAsync(ex, dumpRootPath, retryCount, failureDumpContainer);

if (retryCount == _configuration.MaxRetryCount)
if (_context?.IsFinalTry == true || retryCount >= _configuration.MaxRetryCount)
{
var dumpFolderAbsolutePath = Path.Combine(AppContext.BaseDirectory, dumpRootPath);

Expand Down