ci: Add scripts to retry windows builds on spurious errors

Similar to cerbero, we run meson commands inside a powershell script that will examine the output for spurious errors and re-run that particular command. https://gitlab.freedesktop.org/slomo/gstreamer/-/jobs/65265526 https://gitlab.freedesktop.org/slomo/gstreamer/-/jobs/65265524 https://gitlab.freedesktop.org/nirbheek/gstreamer/-/jobs/65331410 https://gitlab.freedesktop.org/jcowgill/gstreamer/-/jobs/65489856 rust-lang/rust#127883 (comment) Co-Authored-by: L. E. Segovia <[email protected]> Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/7680>
intel-media-ci · Oct 23, 2024 · 15b0b49 · 15b0b49
1 parent 339e6e0
commit 15b0b49
Show file tree

Hide file tree

Showing 2 changed files with 65 additions and 5 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -527,8 +527,8 @@ build vs2022 amd64:
   extends: '.build windows'
   script:
     - !reference [".build windows", "script"]
-    - meson setup --vsenv build $env:MESON_ARGS $env:GTK_ARGS
-    - meson compile -C build --jobs $env:FDO_CI_CONCURRENT
+    - .\ci\run_retry.ps1 meson setup --vsenv build $env:MESON_ARGS $env:GTK_ARGS
+    - .\ci\run_retry.ps1 meson compile -C build --jobs $env:FDO_CI_CONCURRENT
     - .\gst-env.py gst-inspect-1.0.exe --version
     - .\gst-env.py gst-inspect-1.0.exe
     - mkdir destdir
@@ -543,7 +543,7 @@ build vs2022 amd64 full-static:
   script:
     - !reference [".build windows", "script"]
     - cmd.exe /C "meson setup --vsenv build --default-library=static $env:MESON_ARGS"
-    - meson compile -C build --jobs $env:FDO_CI_CONCURRENT
+    - .\ci\run_retry.ps1 meson compile -C build --jobs $env:FDO_CI_CONCURRENT
     - .\gst-env.py gst-inspect-1.0.exe --version
     - .\gst-env.py gst-inspect-1.0.exe
   variables:
@@ -561,8 +561,10 @@ build vs2022 amd64 full-static:
       # Setting up a cross build with MSVC is still non-trivial because
       # the --vsenv argument cannot be used to set it up
     - echo $env:MESON_CROSS_ARGS
-    - cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=$env:ARCH && meson setup build $env:MESON_ARGS $env:GTK_ARGS $env:MESON_CROSS_ARGS"
-    - cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=$env:ARCH && meson compile -C build --jobs $env:FDO_CI_CONCURRENT"
+    - $env:JOB="meson setup build $env:MESON_ARGS $env:GTK_ARGS $env:MESON_CROSS_ARGS"
+    - cmd.exe /C """C:\BuildTools\VC\Auxiliary\Build\vcvarsamd64_$env:ARCH.bat""  && powershell.exe -ExecutionPolicy Bypass -File .\ci\run_retry.ps1"
+    - $env:JOB="meson compile -C build --jobs $env:FDO_CI_CONCURRENT"
+    - cmd.exe /C """C:\BuildTools\VC\Auxiliary\Build\vcvarsamd64_$env:ARCH.bat"" && powershell.exe -ExecutionPolicy Bypass -File .\ci\run_retry.ps1"
 
 build vs2022 x86:
   extends: '.build windows cross'

diff --git a/ci/run_retry.ps1 b/ci/run_retry.ps1
@@ -0,0 +1,58 @@
+# Set error action preference to stop on errors
+$ErrorActionPreference = "Stop"
+
+# Define error patterns to catch
+$ERRORS = @(
+    'fatal error C1060: compiler is out of heap space'
+    'cc1plus.exe: out of memory allocating'
+    'cl : Command line error D8027 : cannot execute'
+    'Access is denied (os error 5)'
+    'LINK : fatal error LNK1104: cannot open file'
+    'PermissionError: [Errno 13] Permission denied:'
+    'c1: fatal error C1356: unable to find mspdbcore.dll'
+)
+$RETRIES = 3
+$LOGFILE = [System.IO.Path]::GetTempPath() + "logfile.txt"
+
+while ($true) {
+    $spurious_error = ""
+
+    # Execute command and capture output to log file while displaying it
+    if ($env:JOB) {
+        # Bypass argument parsing -- https://github.com/PowerShell/PowerShell/issues/19451
+        & iex $env:JOB | Tee-Object -FilePath $LOGFILE *>&1
+    } else {
+        & $args[0] $args[1..($args.Count-1)] | Tee-Object -FilePath $LOGFILE *>&1
+    }
+    $ret = $LASTEXITCODE
+
+    if ($ret -eq 0) {
+        break
+    }
+
+    # Read log file and check for known errors
+    foreach ($line in Get-Content $LOGFILE) {
+        foreach ($e in $ERRORS) {
+            if ($line -match [regex]::Escape($e)) {
+                $spurious_error = $line
+                break
+            }
+        }
+        if ($spurious_error) {
+            break
+        }
+    }
+
+    # Clean up log file
+    if (Test-Path $LOGFILE) {
+        Remove-Item -Force $LOGFILE
+    }
+
+    # Exit if no spurious error found or no retries left
+    if ([string]::IsNullOrEmpty($spurious_error) -or $RETRIES -eq 0) {
+        exit $ret
+    }
+
+    $RETRIES--
+    Write-Host "`nRetrying, caught spurious failure: $spurious_error`n"
+}