From 49258d304cf41bb28e77a3af2d8e215480bf7e75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Fri, 23 Aug 2024 15:13:25 +0200 Subject: [PATCH 1/6] More logging for CloudflareHelper --- Lombiq.Tests.UI/CloudflareRemoteUITestBase.cs | 3 ++- Lombiq.Tests.UI/Helpers/CloudflareHelper.cs | 14 +++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Lombiq.Tests.UI/CloudflareRemoteUITestBase.cs b/Lombiq.Tests.UI/CloudflareRemoteUITestBase.cs index 07d2d31a8..b63fb29a2 100644 --- a/Lombiq.Tests.UI/CloudflareRemoteUITestBase.cs +++ b/Lombiq.Tests.UI/CloudflareRemoteUITestBase.cs @@ -75,6 +75,7 @@ async Task ChangeConfigurationForCloudflareAsync(OrchardCoreUITestExecutorConfig return CloudflareHelper.ExecuteWrappedInIpAccessRuleManagementAsync( () => base.ExecuteTestAsync(baseUri, testAsync, browser, ChangeConfigurationForCloudflareAsync), CloudflareAccountId, - CloudflareApiToken); + CloudflareApiToken, + _testOutputHelper); } } diff --git a/Lombiq.Tests.UI/Helpers/CloudflareHelper.cs b/Lombiq.Tests.UI/Helpers/CloudflareHelper.cs index bbba5b095..7420d4e13 100644 --- a/Lombiq.Tests.UI/Helpers/CloudflareHelper.cs +++ b/Lombiq.Tests.UI/Helpers/CloudflareHelper.cs @@ -8,6 +8,7 @@ using System.Net.Http; using System.Threading; using System.Threading.Tasks; +using Xunit.Abstractions; namespace Lombiq.Tests.UI.Helpers; @@ -23,13 +24,15 @@ internal static class CloudflareHelper public static async Task ExecuteWrappedInIpAccessRuleManagementAsync( Func testAsync, string cloudflareAccountId, - string cloudflareApiToken) + string cloudflareApiToken, + ITestOutputHelper testOutputHelper) { + testOutputHelper.WriteLineTimestampedAndDebug( + "Current Cloudflare IP Access Rule reference count before the start of the test: {0}.", _referenceCount); + await _semaphore.WaitAsync(); Interlocked.Increment(ref _referenceCount); - Debug.WriteLine("Current reference count at the start of the test: {0}.", _referenceCount); - try { _cloudflareApi ??= RestService.For("https://api.cloudflare.com/client/v4", new RefitSettings @@ -41,7 +44,7 @@ public static async Task ExecuteWrappedInIpAccessRuleManagementAsync( if (_ipAccessRuleId == null) { - Debug.WriteLine("Creating an IP Access Rule for the IP {0}.", (object)_currentIp); + testOutputHelper.WriteLineTimestampedAndDebug("Creating a Cloudflare IP Access Rule for the IP {0}.", _currentIp); // Delete any pre-existing rules for the current IP first. string preexistingRuleId = null; @@ -104,7 +107,8 @@ await ReliabilityHelper.DoWithRetriesAndCatchesAsync( // Clean up the IP access rule. if (_ipAccessRuleId != null && Interlocked.Decrement(ref _referenceCount) == 0) { - Debug.WriteLine("Removing the IP Access Rule. Current reference count: {0}.", _referenceCount); + testOutputHelper.WriteLineTimestampedAndDebug( + "Removing the Cloudflare IP Access Rule for the IP {0} since the current reference count is 0.", _currentIp); var deleteSucceededResult = await DeleteIpAccessRuleWithRetriesAsync(cloudflareAccountId, _ipAccessRuleId); From efdd986feab6074819dc9a7f256795f92d6912aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Fri, 23 Aug 2024 15:45:37 +0200 Subject: [PATCH 2/6] Even more logging --- Lombiq.Tests.UI/Helpers/CloudflareHelper.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Helpers/CloudflareHelper.cs b/Lombiq.Tests.UI/Helpers/CloudflareHelper.cs index 7420d4e13..7d07232a5 100644 --- a/Lombiq.Tests.UI/Helpers/CloudflareHelper.cs +++ b/Lombiq.Tests.UI/Helpers/CloudflareHelper.cs @@ -104,11 +104,14 @@ await ReliabilityHelper.DoWithRetriesAndCatchesAsync( } finally { + testOutputHelper.WriteLineTimestampedAndDebug( + "Current Cloudflare IP Access Rule reference count after the test (including this test): {0}.", _referenceCount); + // Clean up the IP access rule. if (_ipAccessRuleId != null && Interlocked.Decrement(ref _referenceCount) == 0) { testOutputHelper.WriteLineTimestampedAndDebug( - "Removing the Cloudflare IP Access Rule for the IP {0} since the current reference count is 0.", _currentIp); + "Removing the Cloudflare IP Access Rule for the IP {0} since this test had the last reference to it.", _currentIp); var deleteSucceededResult = await DeleteIpAccessRuleWithRetriesAsync(cloudflareAccountId, _ipAccessRuleId); @@ -116,6 +119,11 @@ await ReliabilityHelper.DoWithRetriesAndCatchesAsync( ThrowIfNotSuccess(deleteSucceededResult, _currentIp, "couldn't be deleted"); } + else + { + testOutputHelper.WriteLineTimestampedAndDebug( + "Not removing the Cloudflare IP Access Rule for the IP {0} since the current reference count is NOT 0.", _currentIp); + } } } From 9e37611d4b8a63f782394a4d5918c314a2a7f875 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Fri, 23 Aug 2024 15:50:50 +0200 Subject: [PATCH 3/6] And more --- Lombiq.Tests.UI/Helpers/CloudflareHelper.cs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/Lombiq.Tests.UI/Helpers/CloudflareHelper.cs b/Lombiq.Tests.UI/Helpers/CloudflareHelper.cs index 7d07232a5..91242fe47 100644 --- a/Lombiq.Tests.UI/Helpers/CloudflareHelper.cs +++ b/Lombiq.Tests.UI/Helpers/CloudflareHelper.cs @@ -91,6 +91,9 @@ await ReliabilityHelper.DoWithRetriesAndCatchesAsync( }); ThrowIfNotSuccess(ruleCheckRequestResult, _currentIp, "didn't get activated"); + + testOutputHelper.WriteLineTimestampedAndDebug( + "Created a Cloudflare IP Access Rule for the IP {0} (Rule ID: {1}).", _currentIp, _ipAccessRuleId); } } finally @@ -111,18 +114,29 @@ await ReliabilityHelper.DoWithRetriesAndCatchesAsync( if (_ipAccessRuleId != null && Interlocked.Decrement(ref _referenceCount) == 0) { testOutputHelper.WriteLineTimestampedAndDebug( - "Removing the Cloudflare IP Access Rule for the IP {0} since this test had the last reference to it.", _currentIp); + "Removing the Cloudflare IP Access Rule for the IP {0} (Rule ID: {1}) since this test has the last reference to it.", + _currentIp, + _ipAccessRuleId); + + var oldIpAccessRuleId = _ipAccessRuleId; var deleteSucceededResult = await DeleteIpAccessRuleWithRetriesAsync(cloudflareAccountId, _ipAccessRuleId); if (deleteSucceededResult.IsSuccess) _ipAccessRuleId = null; ThrowIfNotSuccess(deleteSucceededResult, _currentIp, "couldn't be deleted"); + + testOutputHelper.WriteLineTimestampedAndDebug( + "Removed the Cloudflare IP Access Rule for the IP {0} (Rule ID: {1}) since this test had the last reference to it.", + _currentIp, + oldIpAccessRuleId); } else { testOutputHelper.WriteLineTimestampedAndDebug( - "Not removing the Cloudflare IP Access Rule for the IP {0} since the current reference count is NOT 0.", _currentIp); + "Not removing the Cloudflare IP Access Rule for the IP {0} (Rule ID: {1}) since the current reference count is NOT 0.", + _currentIp, + _ipAccessRuleId); } } } From f758d079895241055b94395e2d7b874cb3e707da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Fri, 23 Aug 2024 16:07:00 +0200 Subject: [PATCH 4/6] And more --- Lombiq.Tests.UI/Helpers/CloudflareHelper.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Helpers/CloudflareHelper.cs b/Lombiq.Tests.UI/Helpers/CloudflareHelper.cs index 91242fe47..f8cc341eb 100644 --- a/Lombiq.Tests.UI/Helpers/CloudflareHelper.cs +++ b/Lombiq.Tests.UI/Helpers/CloudflareHelper.cs @@ -28,11 +28,14 @@ public static async Task ExecuteWrappedInIpAccessRuleManagementAsync( ITestOutputHelper testOutputHelper) { testOutputHelper.WriteLineTimestampedAndDebug( - "Current Cloudflare IP Access Rule reference count before the start of the test: {0}.", _referenceCount); + "Current Cloudflare IP Access Rule reference count before entering semaphore: {0}.", _referenceCount); await _semaphore.WaitAsync(); Interlocked.Increment(ref _referenceCount); + testOutputHelper.WriteLineTimestampedAndDebug( + "Current Cloudflare IP Access Rule reference count after entering semaphore: {0}.", _referenceCount); + try { _cloudflareApi ??= RestService.For("https://api.cloudflare.com/client/v4", new RefitSettings From 28dc6b919a72bb60d3d3eceaff7fc39a19971842 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Fri, 23 Aug 2024 18:46:57 +0200 Subject: [PATCH 5/6] Adding Edge verification test file with new browser name --- ...ontains]_-navbar-brand_Unix_MicrosoftEdge.png | Bin 0 -> 3749 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 Lombiq.Tests.UI.Samples/Tests/BasicVisualVerificationTests_VerifyNavbar_By_ClassName[Contains]_-navbar-brand_Unix_MicrosoftEdge.png diff --git a/Lombiq.Tests.UI.Samples/Tests/BasicVisualVerificationTests_VerifyNavbar_By_ClassName[Contains]_-navbar-brand_Unix_MicrosoftEdge.png b/Lombiq.Tests.UI.Samples/Tests/BasicVisualVerificationTests_VerifyNavbar_By_ClassName[Contains]_-navbar-brand_Unix_MicrosoftEdge.png new file mode 100644 index 0000000000000000000000000000000000000000..351f87485824c32b125461443f640149bd54a7d8 GIT binary patch literal 3749 zcmZt}Wmwct6XB5KsG}Q>E=9VVqr0Re4v`e3OF%ejk2pe*MnFPZ8bJ|Gq!o^q5)hPb z@QC;K|Neg1nVo(1+1crtL}NoOG6)j{4-bz_M;mU6hj#}I>{-Bf0a);r<12uOeYCCp z@$ks||66ymgvgnJjinA;)jTAB_a|W*4~B~HJ3d`-aB%$R#l^*f&x^zXz5o?`{#nrn zfVa0*7m_CR6}WD6+vmNrqug{k$){>>|v+J%e)>8Yd9w*a!U>S^id$gF`%MONw>Q5%m z;mG!_x39j0ZP(JDEj;_waC?J`zo8W$l`y;bo%ifZnmYO1?sE#|tAmCCw&yXn-W@#m zTCk>G%o)N*QZ%N~AUc>dp4S5e|);2DP( z5g~^d`o6a1NM=nI-+qx%iOCq+hgKXBLN+|Fx%(cR|BIC}dKj%I2t))@lL)&r8-t8gbM}vHu9Yn`Tk?pVMQGd-W?mA3T*yU6-Lk3_MT+h z(P=V!H8%jt3(B_*S&hxBLv$U~TS6D_5CxxrP(*R;kr%DWn}907?`xWLLsp`RejU@> zxRr>k#!$xk=N3+pa128orTb8Am2H3+s`l`~g)brCb7xeX;buu2b-a?T&0wUz?O!=-0_lh;$uZ zV!#C=`ectUe)^ANLR$-Ek6PU5L3>#bZK|SS!X4}B2f>a}oC!-XO8CDy+pEjXBG&bW zX&cYj3`02ql-LkI+00Mh0anCr%M3a5vyZ+gXaWq;wirYLIFNe#@t8a_DFKw^w0H2? z$HT9ckfQYid$)q~q(PHs+Ej>O$(Qt03jGg3J0M|ADBlGu8+SPV!$kpxqZ%++T; z@CO5XNjmX|$uM$Ma;D!i2yp9$lL@_GcyYFUSXD-ie%jj{TaV}+Bbk*bf?qNk`HR8J z-IEb7$)v2z*-W9wJ2K}#{pBO~gI`=7YH>$#%0``boKD-8QvCy@fBet<%#J|3j6sV% zB0zs(Ib~(I}<>ZF#dDeH+c|AFuvZT;r@r9pEf+AJDKHLJD6&{gUL!l4KGfg!;d@nb$ z_mx!B>a5Y01cZ>u$ThtC z!Bwp^j%^L+bX$O zhBdu1HBFZNol`m)kuZiE-;GT*5!(irzW zY$NtWM*6wydt@Oj$JFtRLuCxevOiX*s0#tfd;qh@i(B`^ z^4zKjYsVQ_c?xj-GD?$ck{!>AW6JZ_p)B7F%!9#=40(?RUr}A<`M#k^VQ0U25h`S} z3`8+&%9kjO$Rmcw<-I<<;~kg3^W-T(erdRJu-eY_LH+ZCjzVNAEjvC?_Fvw7Jqa}XDc&S^`QYT?(iRX|&1xmRbtdX<+5wE{Q zcDeg}iJ9xUHQ!3%^SwmnR}*91yacKi9Dzh-^c|Bhy5(6?=YN(C5w!BOc2#Z_YL@nh z>}@jw&Xmeu2iMxOv*<^K_ArFo6r?W9wewST=>)B|Vs5VunVV!1ivajAIsCpV-})e5 z5o$~z+0kJKo|{7>ZTvl@P%XCrNAsyvt}G5&-&!5gB$JHqd)GfDx4sXhs)Yn}j zV#D{!o8G+iXY-@A=H_?X&^9zvD9SfwvxHlZJwS@f9aWArSS%qXboh2p{dR&omj664&~*?N5$cg4a`cT$`YN~$%(>C3fv>H-xzLdfQ$?32!1122LlQwl;20_o!i=oyYr#I6 zcqtAoi&-1=f5@@a0oK^NGVx!G#E}Q`AXP7`13{O~N=UmYKcQgzZtH>*E*NNfQul=jn$GlV zZAJE_Y^$u!cNe5JO9T^aNL5|=%Nc&_l2?0|a%Is!^-oBjV7vT4XDh@rU61}N^0sw4 z)cUWGn!2@gWx@9dalWKc%r#|DFK1fQ2{tDp{v|8FA=gc$22`M>g7a98uFNS=t*wh; zo^w8(SMhK$D<7a%5{-Sw#xjO=aPMhYHRbPOP zfx1l~uOWQ9)fG1I&@{!vN92))&02QQx(X>OI%A$B-SNgWK6?G0I8C)C=-auDPnu9& zJLfwM&i9^2NZKbW!DHf7r&%lsFR6Fi*HR9dThRVICH3`t9Au4@+eN%Y#uTNMHPlWg z2BG)klecnZDIKBr2Dv9-+TbkI zVWfDm^0EMOKnCLQyR?lHo2{)VcA{jc=PV?qpBj8dX=MJ``5NCQg~eb~-2pVJCzhd2ULkzvA#u_+MP~26Xf@8_9cSs zXpf86N@aM|bWHq0;<1>V+S*Y`?T4d-I2v8XuAYcF&3oLNH!clc9oCffT@7^80(0-q zIMU6|DMtzMP4c>%=YLMymf5|Vkmw)Ps*#j07HmE=%k}P_vc*;fluxiQR|M)5NDq;_ zZ>4P+n+ykSo!#9Te3T=7?0j#@OQG}enHRSN6htSiivqp;08X;b#Q*b@-hLkO4XW`j z{Mb-F?TDMCUfV5wBkxFOe`ebK?p+~dCA*`RAXZN6EBgY$>v!vpB(`j)@jRf#DNJiXRIz9J{YbZRg;c+bkD#AQ8KLpDyD1O(pWFKuX^686n+2Nc549vnu7s30&^#x Mj)oz;PR$YdKjqXgIRF3v literal 0 HcmV?d00001 From 9fe37ae7a1a454258be3601ebe50a6bf33fafad3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Fri, 23 Aug 2024 18:47:21 +0200 Subject: [PATCH 6/6] Temporarily stopOnFail = false --- Lombiq.Tests.UI.Samples/xunit.runner.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lombiq.Tests.UI.Samples/xunit.runner.json b/Lombiq.Tests.UI.Samples/xunit.runner.json index 3df5bfd1a..07f35a639 100644 --- a/Lombiq.Tests.UI.Samples/xunit.runner.json +++ b/Lombiq.Tests.UI.Samples/xunit.runner.json @@ -2,6 +2,5 @@ "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", "parallelizeAssembly": false, "parallelizeTestCollections": true, - "maxParallelThreads": 3, - "stopOnFail": true + "maxParallelThreads": 3 }