ci: retry known infra failures once, by signature

.github/workflows/test-run-android.yml

@@ -32,6 +32,10 @@ jobs:
       HOMEBREW_NO_INSTALL_CLEANUP: 1
       SENTRY_DSN: ${{ secrets.SENTRY_TEST_DSN }}
       SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+      EMULATOR_TARGET: google_apis
+      EMULATOR_CHANNEL: stable
+      REDROID_IMAGE: redroid/redroid:13.0.0_64only-latest
+      REDROID_BOOT_ARGS: androidboot.redroid_gpu_mode=guest
 
     steps:
       - name: Checkout
@@ -84,15 +88,38 @@ jobs:
           mkdir -p $HOME/.android/avd
           touch $HOME/.android/repositories.cfg
 
+      # android-emulator-runner calls sdkmanager once, without retrying a corrupt download.
+      - name: Install Android emulator and system image
+        if: ${{ steps.device.outputs.device == 'emulator' }}
+        shell: bash
+        run: |
+          sdkmanager="$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager"
+          packages=(emulator "system-images;android-${{ inputs.api-level }};$EMULATOR_TARGET;${{ steps.device.outputs.arch }}")
+          channel=$(case "$EMULATOR_CHANNEL" in (stable) echo 0;; (beta) echo 1;; (dev) echo 2;; (canary) echo 3;; (*) exit 1;; esac)
+          yes | "$sdkmanager" --licenses > /dev/null || true
+          for try in 1 2 3; do
+            status=0
+            "$sdkmanager" --install "${packages[@]}" --channel="$channel" > sdkmanager.log 2>&1 || status=$?
+            if ! grep -q 'Error on ZipFile' sdkmanager.log; then
+              [ "$status" -eq 0 ] || { cat sdkmanager.log; exit "$status"; }
+              exit 0
+            fi
+            grep 'Error on ZipFile' sdkmanager.log
+            [ "$try" -lt 3 ] || exit 1
+            echo "::warning::sdkmanager download was corrupt, retrying ($try/2) (signature: sdkmanager-corrupt-zip)"
+            rm -rf "$ANDROID_HOME/.temp" "$ANDROID_HOME/.downloadIntermediates"
+            sleep $((try * 20))
+          done
+
       - name: Run Android Integration Tests (emulator)
         if: ${{ steps.device.outputs.device == 'emulator' }}
         uses: reactivecircus/android-emulator-runner@0a638108440efd5c7f980e6ba145dbcdd8f32009 # v2.37.0
         id: integration-test
         timeout-minutes: 30
         with:
           api-level: ${{ inputs.api-level }}
-          target: 'google_apis'
-          channel: 'stable'
+          target: ${{ env.EMULATOR_TARGET }}
+          channel: ${{ env.EMULATOR_CHANNEL }}
           force-avd-creation: true
           disable-animations: true
           disable-spellchecker: true
@@ -144,8 +171,8 @@ jobs:
           docker run -d --rm --privileged `
             --name redroid `
             -p 5555:5555 `
-            redroid/redroid:13.0.0_64only-latest `
-            androidboot.redroid_gpu_mode=guest
+            $env:REDROID_IMAGE `
+            $env:REDROID_BOOT_ARGS
 
       - name: Run Android Integration Tests (redroid)
         if: ${{ steps.device.outputs.device == 'redroid' }}
@@ -165,6 +192,14 @@ jobs:
             adb connect localhost:5555 *> $null
             $booted = (adb -s localhost:5555 shell getprop sys.boot_completed 2>$null | Out-String).Trim()
             if ($booted -eq "1") { Write-Host "Redroid booted after $($i * 5)s"; break }
+            if ($i -eq 30) {
+              Write-Host "::warning::Redroid did not boot in 150s, restarting the container once (signature: redroid-boot-timeout)"
+              docker logs redroid 2>&1 | Select-Object -Last 40
+              docker rm -f redroid
+              docker run -d --rm --privileged --name redroid -p 5555:5555 $env:REDROID_IMAGE $env:REDROID_BOOT_ARGS
+              if ($LASTEXITCODE -ne 0) { exit 1 }
+              adb disconnect localhost:5555 *> $null
+            }
             if ($i -eq 60) {
               Write-Host "Redroid did not boot in time"
               adb devices -l

Directory.Build.targets

@@ -18,13 +18,16 @@
   <Target Name="FindUnity">
     <Message Text="Unity Version: $(UnityVersion)" Importance="Normal" />
 
-    <Exec
-      Command="pwsh -NoProfile -File &quot;$(RepoRoot)scripts/find-unity.ps1&quot; -UnityVersion &quot;$(UnityVersion)&quot; -HubInstallDir &quot;$(HubInstallDir)&quot; -HubDefaultEditor &quot;$(HubDefaultEditor)&quot;"
-      ConsoleToMSBuild="true">
-      <Output TaskParameter="ConsoleOutput" PropertyName="_FindUnityOutput" />
-    </Exec>
+    <MSBuild
+      Projects="$(RepoRoot)build/find-unity.proj"
+      Targets="FindUnity"
+      Properties="UnityVersion=$(UnityVersion);HubInstallDir=$(HubInstallDir);HubDefaultEditor=$(HubDefaultEditor)"
+      RemoveProperties="TargetFramework;RuntimeIdentifier">
+      <Output TaskParameter="TargetOutputs" ItemName="_FindUnityOutputLines" />
+    </MSBuild>
 
     <PropertyGroup>
+      <_FindUnityOutput>@(_FindUnityOutputLines)</_FindUnityOutput>
       <UnityRoot>$([System.Text.RegularExpressions.Regex]::Match($(_FindUnityOutput), 'UnityRoot=([^;\r\n]+)').Groups[1].Value)</UnityRoot>
       <UnityDataPath>$([System.Text.RegularExpressions.Regex]::Match($(_FindUnityOutput), 'UnityDataPath=([^;\r\n]+)').Groups[1].Value)</UnityDataPath>
       <UnityManagedPath>$([System.Text.RegularExpressions.Regex]::Match($(_FindUnityOutput), 'UnityManagedPath=([^;\r\n]+)').Groups[1].Value)</UnityManagedPath>

build/find-unity.proj

@@ -0,0 +1,10 @@
+<Project>
+  <!-- Invoked through the MSBuild task so that MSBuild caches the result and runs find-unity.ps1 once per build, not once per project. -->
+  <Target Name="FindUnity" Returns="@(_FindUnityOutput)">
+    <Exec
+      Command="pwsh -NoProfile -File &quot;$(MSBuildThisFileDirectory)../scripts/find-unity.ps1&quot; -UnityVersion &quot;$(UnityVersion)&quot; -HubInstallDir &quot;$(HubInstallDir)&quot; -HubDefaultEditor &quot;$(HubDefaultEditor)&quot;"
+      ConsoleToMSBuild="true">
+      <Output TaskParameter="ConsoleOutput" ItemName="_FindUnityOutput" />
+    </Exec>
+  </Target>
+</Project>

scripts/setup-unity.ps1

@@ -29,13 +29,24 @@ if (-not $cliFile) { throw "No Unity CLI binary pinned for $os-$arch" }
 $cliDir = Join-Path ($env:RUNNER_TEMP ?? [System.IO.Path]::GetTempPath()) 'unity-cli'
 New-Item -ItemType Directory -Force -Path $cliDir | Out-Null
 $cli = Join-Path $cliDir ($IsWindows ? 'unity.exe' : 'unity')
-Invoke-WebRequest -Uri "https://public-cdn.cloud.unity3d.com/hub/prod/cli/$CliVersion/$cliFile" -OutFile $cli
+Invoke-WebRequest -Uri "https://public-cdn.cloud.unity3d.com/hub/prod/cli/$CliVersion/$cliFile" -OutFile $cli -MaximumRetryCount 3 -RetryIntervalSec 10
 if ((Get-FileHash $cli -Algorithm SHA256).Hash -ne $cliSha) { throw "Unity CLI checksum mismatch for $cliFile" }
 if (-not $IsWindows) { chmod +x $cli }
 
 $installArgs = @('install', $unity.version, '--changeset', $unity.changeset, '--child-modules', '--accept-eula', '--yes')
 if ($moduleList) { $installArgs += @('--module') + $moduleList }
-& $cli @installArgs
+$retryDelays = @(30, 90)
+for ($try = 1; ; $try++) {
+    $PSNativeCommandUseErrorActionPreference = $false
+    & $cli @installArgs
+    $installExitCode = $LASTEXITCODE
+    $PSNativeCommandUseErrorActionPreference = $true
+    if ($installExitCode -eq 0) { break }
+    if ($try -gt $retryDelays.Count) { throw "unity install failed with exit code $installExitCode after $try tries" }
+    $delay = $retryDelays[$try - 1]
+    Write-Host "::warning::unity install failed with exit code $installExitCode, retrying in $delay s ($try/$($retryDelays.Count)) (signature: editor-install-failed)"
+    Start-Sleep -Seconds $delay
+}
 
 $editor = (& $cli editors --installed --verbose --json | ConvertFrom-Json).data | Where-Object version -EQ $unity.version | Select-Object -First 1
 if (-not $editor) { throw "Unity $($unity.version) not listed after install" }

scripts/unity-utils.ps1

@@ -2,6 +2,53 @@ $RunUnityLicenseRetryTimeoutSeconds = 1800
 $RunUnityLicenseRetryIntervalMinSeconds = 20
 $RunUnityLicenseRetryIntervalMaxSeconds = 90
 $ClearUnityLogTimeoutSeconds = 30
+$RunUnityRateLimitRetrySleepSeconds = 90
+
+# Returns the name of a known infra failure worth one retry, or $null.
+function GetUnityRetrySignature([string] $unityPath, [string[]] $arguments, [int] $exitCode, [string[]] $stdout)
+{
+    if ($exitCode -eq 0)
+    {
+        return $null
+    }
+
+    # Unity 6.6 on Linux intermittently gets SIGKILLed during startup.
+    if ($exitCode -eq 137)
+    {
+        return 'killed'
+    }
+
+    # xvfb-run exits 5 instead of 137 when it fails to clean up after the kill. Unity never got to log its exit.
+    if ($exitCode -eq 5 -and $unityPath -in @('docker', 'xvfb-run') `
+            -and -not ($stdout -match 'Cleanup mono|Exiting batchmode|Aborting batchmode|Application will (exit|terminate) with return code|Caught fatal signal'))
+    {
+        return 'killed'
+    }
+
+    # A crash while running tests may be an SDK bug.
+    if ($arguments -contains '-runTests')
+    {
+        return $null
+    }
+
+    if ($stdout -match 'Received status code 429 from server')
+    {
+        return 'gradle-429'
+    }
+
+    # Bee reports a segfaulting build tool, e.g. il2cpp, as its exit code.
+    if ($stdout -match '\| 139 \(Segmentation fault\)')
+    {
+        return 'tool-segfault'
+    }
+
+    if (($stdout -match 'Batchmode quit successfully invoked') -and ($stdout -match 'Caught fatal signal'))
+    {
+        return 'shutdown-crash'
+    }
+
+    return $null
+}
 
 function RunUnity([string] $unityPath, [string[]] $arguments, [switch] $ReturnLogOutput)
 {
@@ -44,7 +91,8 @@ function RunUnity([string] $unityPath, [string[]] $arguments, [switch] $ReturnLo
     $stopwatch = [System.Diagnostics.Stopwatch]::new()
     $stdout = ""
     $try = 1
-    $retriedKilled = $false
+    $licenseTry = 1
+    $retried = @{}
     do
     {
         $stopwatch.Restart()
@@ -77,19 +125,23 @@ function RunUnity([string] $unityPath, [string[]] $arguments, [switch] $ReturnLo
                 Write-Host -ForegroundColor Yellow "$msg Sleeping for $timeToSleep seconds before retrying. Total time remaining: $timeRemaining seconds."
                 Start-Sleep -Seconds $timeToSleep
                 $try = $try + 1
+                $licenseTry = $licenseTry + 1
             }
             else
             {
-                $msg = "Unity couldn't acquire a license after $try tries in $RunUnityLicenseRetryTimeoutSeconds seconds. The floating license entitlements are likely exhausted (no free Unity license seat)."
+                $msg = "Unity couldn't acquire a license after $licenseTry tries in $RunUnityLicenseRetryTimeoutSeconds seconds. The floating license entitlements are likely exhausted (no free Unity license seat)."
                 Write-Host "::error::$msg"
                 Throw $msg
             }
         }
-        ElseIf ($process.ExitCode -eq 137 -and -not $retriedKilled)
+        ElseIf (($signature = GetUnityRetrySignature $unityPath $arguments $process.ExitCode $stdout) -and -not $retried[$signature])
         {
-            # Unity 6.6 on Linux intermittently SIGKILLs itself during startup.
-            Write-Host "::warning::Unity was killed (exit code 137). Retrying once."
-            $retriedKilled = $true
+            Write-Host "::warning::Unity failed with exit code $($process.ExitCode) (signature: $signature). Retrying once."
+            $retried[$signature] = $true
+            if ($signature -eq 'gradle-429')
+            {
+                Start-Sleep -Seconds $RunUnityRateLimitRetrySleepSeconds
+            }
             $try = $try + 1
         }
         else
@@ -105,9 +157,9 @@ function RunUnity([string] $unityPath, [string[]] $arguments, [switch] $ReturnLo
 
     $timeTaken = "Time taken: $($stopwatch.Elapsed.ToString('hh\:mm\:ss\.fff'))"
     Write-Host -ForegroundColor Green "Unity finished successfully. $timeTaken"
-    if ($try -gt 3)
+    if ($licenseTry -gt 3)
     {
-        Write-Host "::$($try -gt 10 ? 'warning' : 'notice')::It took $try tries to successfully acquire a Unity license. Total time taken: $($stopwatchTotal.Elapsed.ToString('hh\:mm\:ss\.fff'))"
+        Write-Host "::$($licenseTry -gt 10 ? 'warning' : 'notice')::It took $licenseTry tries to successfully acquire a Unity license. Total time taken: $($stopwatchTotal.Elapsed.ToString('hh\:mm\:ss\.fff'))"
     }
     return $ReturnLogOutput ? $stdout : $null
 }