diff --git a/.github/workflows/test-run-android.yml b/.github/workflows/test-run-android.yml index 6400e761d..630484d23 100644 --- a/.github/workflows/test-run-android.yml +++ b/.github/workflows/test-run-android.yml @@ -32,6 +32,10 @@ jobs: HOMEBREW_NO_INSTALL_CLEANUP: 1 SENTRY_DSN: ${{ secrets.SENTRY_TEST_DSN }} SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + EMULATOR_TARGET: google_apis + EMULATOR_CHANNEL: stable + REDROID_IMAGE: redroid/redroid:13.0.0_64only-latest + REDROID_BOOT_ARGS: androidboot.redroid_gpu_mode=guest steps: - name: Checkout @@ -84,6 +88,29 @@ jobs: mkdir -p $HOME/.android/avd touch $HOME/.android/repositories.cfg + # android-emulator-runner calls sdkmanager once, without retrying a corrupt download. + - name: Install Android emulator and system image + if: ${{ steps.device.outputs.device == 'emulator' }} + shell: bash + run: | + sdkmanager="$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" + packages=(emulator "system-images;android-${{ inputs.api-level }};$EMULATOR_TARGET;${{ steps.device.outputs.arch }}") + channel=$(case "$EMULATOR_CHANNEL" in (stable) echo 0;; (beta) echo 1;; (dev) echo 2;; (canary) echo 3;; (*) exit 1;; esac) + yes | "$sdkmanager" --licenses > /dev/null || true + for try in 1 2 3; do + status=0 + "$sdkmanager" --install "${packages[@]}" --channel="$channel" > sdkmanager.log 2>&1 || status=$? + if ! grep -q 'Error on ZipFile' sdkmanager.log; then + [ "$status" -eq 0 ] || { cat sdkmanager.log; exit "$status"; } + exit 0 + fi + grep 'Error on ZipFile' sdkmanager.log + [ "$try" -lt 3 ] || exit 1 + echo "::warning::sdkmanager download was corrupt, retrying ($try/2) (signature: sdkmanager-corrupt-zip)" + rm -rf "$ANDROID_HOME/.temp" "$ANDROID_HOME/.downloadIntermediates" + sleep $((try * 20)) + done + - name: Run Android Integration Tests (emulator) if: ${{ steps.device.outputs.device == 'emulator' }} uses: reactivecircus/android-emulator-runner@0a638108440efd5c7f980e6ba145dbcdd8f32009 # v2.37.0 @@ -91,8 +118,8 @@ jobs: timeout-minutes: 30 with: api-level: ${{ inputs.api-level }} - target: 'google_apis' - channel: 'stable' + target: ${{ env.EMULATOR_TARGET }} + channel: ${{ env.EMULATOR_CHANNEL }} force-avd-creation: true disable-animations: true disable-spellchecker: true @@ -144,8 +171,8 @@ jobs: docker run -d --rm --privileged ` --name redroid ` -p 5555:5555 ` - redroid/redroid:13.0.0_64only-latest ` - androidboot.redroid_gpu_mode=guest + $env:REDROID_IMAGE ` + $env:REDROID_BOOT_ARGS - name: Run Android Integration Tests (redroid) if: ${{ steps.device.outputs.device == 'redroid' }} @@ -165,6 +192,14 @@ jobs: adb connect localhost:5555 *> $null $booted = (adb -s localhost:5555 shell getprop sys.boot_completed 2>$null | Out-String).Trim() if ($booted -eq "1") { Write-Host "Redroid booted after $($i * 5)s"; break } + if ($i -eq 30) { + Write-Host "::warning::Redroid did not boot in 150s, restarting the container once (signature: redroid-boot-timeout)" + docker logs redroid 2>&1 | Select-Object -Last 40 + docker rm -f redroid + docker run -d --rm --privileged --name redroid -p 5555:5555 $env:REDROID_IMAGE $env:REDROID_BOOT_ARGS + if ($LASTEXITCODE -ne 0) { exit 1 } + adb disconnect localhost:5555 *> $null + } if ($i -eq 60) { Write-Host "Redroid did not boot in time" adb devices -l diff --git a/Directory.Build.targets b/Directory.Build.targets index 66b81368c..801ee528a 100644 --- a/Directory.Build.targets +++ b/Directory.Build.targets @@ -18,13 +18,16 @@ - - - + + + + <_FindUnityOutput>@(_FindUnityOutputLines) $([System.Text.RegularExpressions.Regex]::Match($(_FindUnityOutput), 'UnityRoot=([^;\r\n]+)').Groups[1].Value) $([System.Text.RegularExpressions.Regex]::Match($(_FindUnityOutput), 'UnityDataPath=([^;\r\n]+)').Groups[1].Value) $([System.Text.RegularExpressions.Regex]::Match($(_FindUnityOutput), 'UnityManagedPath=([^;\r\n]+)').Groups[1].Value) diff --git a/build/find-unity.proj b/build/find-unity.proj new file mode 100644 index 000000000..482b511b0 --- /dev/null +++ b/build/find-unity.proj @@ -0,0 +1,10 @@ + + + + + + + + diff --git a/scripts/setup-unity.ps1 b/scripts/setup-unity.ps1 index 4a875311a..a8b697ae8 100644 --- a/scripts/setup-unity.ps1 +++ b/scripts/setup-unity.ps1 @@ -29,13 +29,24 @@ if (-not $cliFile) { throw "No Unity CLI binary pinned for $os-$arch" } $cliDir = Join-Path ($env:RUNNER_TEMP ?? [System.IO.Path]::GetTempPath()) 'unity-cli' New-Item -ItemType Directory -Force -Path $cliDir | Out-Null $cli = Join-Path $cliDir ($IsWindows ? 'unity.exe' : 'unity') -Invoke-WebRequest -Uri "https://public-cdn.cloud.unity3d.com/hub/prod/cli/$CliVersion/$cliFile" -OutFile $cli +Invoke-WebRequest -Uri "https://public-cdn.cloud.unity3d.com/hub/prod/cli/$CliVersion/$cliFile" -OutFile $cli -MaximumRetryCount 3 -RetryIntervalSec 10 if ((Get-FileHash $cli -Algorithm SHA256).Hash -ne $cliSha) { throw "Unity CLI checksum mismatch for $cliFile" } if (-not $IsWindows) { chmod +x $cli } $installArgs = @('install', $unity.version, '--changeset', $unity.changeset, '--child-modules', '--accept-eula', '--yes') if ($moduleList) { $installArgs += @('--module') + $moduleList } -& $cli @installArgs +$retryDelays = @(30, 90) +for ($try = 1; ; $try++) { + $PSNativeCommandUseErrorActionPreference = $false + & $cli @installArgs + $installExitCode = $LASTEXITCODE + $PSNativeCommandUseErrorActionPreference = $true + if ($installExitCode -eq 0) { break } + if ($try -gt $retryDelays.Count) { throw "unity install failed with exit code $installExitCode after $try tries" } + $delay = $retryDelays[$try - 1] + Write-Host "::warning::unity install failed with exit code $installExitCode, retrying in $delay s ($try/$($retryDelays.Count)) (signature: editor-install-failed)" + Start-Sleep -Seconds $delay +} $editor = (& $cli editors --installed --verbose --json | ConvertFrom-Json).data | Where-Object version -EQ $unity.version | Select-Object -First 1 if (-not $editor) { throw "Unity $($unity.version) not listed after install" } diff --git a/scripts/unity-utils.ps1 b/scripts/unity-utils.ps1 index 31eb08e09..0573b0525 100644 --- a/scripts/unity-utils.ps1 +++ b/scripts/unity-utils.ps1 @@ -2,6 +2,53 @@ $RunUnityLicenseRetryTimeoutSeconds = 1800 $RunUnityLicenseRetryIntervalMinSeconds = 20 $RunUnityLicenseRetryIntervalMaxSeconds = 90 $ClearUnityLogTimeoutSeconds = 30 +$RunUnityRateLimitRetrySleepSeconds = 90 + +# Returns the name of a known infra failure worth one retry, or $null. +function GetUnityRetrySignature([string] $unityPath, [string[]] $arguments, [int] $exitCode, [string[]] $stdout) +{ + if ($exitCode -eq 0) + { + return $null + } + + # Unity 6.6 on Linux intermittently gets SIGKILLed during startup. + if ($exitCode -eq 137) + { + return 'killed' + } + + # xvfb-run exits 5 instead of 137 when it fails to clean up after the kill. Unity never got to log its exit. + if ($exitCode -eq 5 -and $unityPath -in @('docker', 'xvfb-run') ` + -and -not ($stdout -match 'Cleanup mono|Exiting batchmode|Aborting batchmode|Application will (exit|terminate) with return code|Caught fatal signal')) + { + return 'killed' + } + + # A crash while running tests may be an SDK bug. + if ($arguments -contains '-runTests') + { + return $null + } + + if ($stdout -match 'Received status code 429 from server') + { + return 'gradle-429' + } + + # Bee reports a segfaulting build tool, e.g. il2cpp, as its exit code. + if ($stdout -match '\| 139 \(Segmentation fault\)') + { + return 'tool-segfault' + } + + if (($stdout -match 'Batchmode quit successfully invoked') -and ($stdout -match 'Caught fatal signal')) + { + return 'shutdown-crash' + } + + return $null +} function RunUnity([string] $unityPath, [string[]] $arguments, [switch] $ReturnLogOutput) { @@ -44,7 +91,8 @@ function RunUnity([string] $unityPath, [string[]] $arguments, [switch] $ReturnLo $stopwatch = [System.Diagnostics.Stopwatch]::new() $stdout = "" $try = 1 - $retriedKilled = $false + $licenseTry = 1 + $retried = @{} do { $stopwatch.Restart() @@ -77,19 +125,23 @@ function RunUnity([string] $unityPath, [string[]] $arguments, [switch] $ReturnLo Write-Host -ForegroundColor Yellow "$msg Sleeping for $timeToSleep seconds before retrying. Total time remaining: $timeRemaining seconds." Start-Sleep -Seconds $timeToSleep $try = $try + 1 + $licenseTry = $licenseTry + 1 } else { - $msg = "Unity couldn't acquire a license after $try tries in $RunUnityLicenseRetryTimeoutSeconds seconds. The floating license entitlements are likely exhausted (no free Unity license seat)." + $msg = "Unity couldn't acquire a license after $licenseTry tries in $RunUnityLicenseRetryTimeoutSeconds seconds. The floating license entitlements are likely exhausted (no free Unity license seat)." Write-Host "::error::$msg" Throw $msg } } - ElseIf ($process.ExitCode -eq 137 -and -not $retriedKilled) + ElseIf (($signature = GetUnityRetrySignature $unityPath $arguments $process.ExitCode $stdout) -and -not $retried[$signature]) { - # Unity 6.6 on Linux intermittently SIGKILLs itself during startup. - Write-Host "::warning::Unity was killed (exit code 137). Retrying once." - $retriedKilled = $true + Write-Host "::warning::Unity failed with exit code $($process.ExitCode) (signature: $signature). Retrying once." + $retried[$signature] = $true + if ($signature -eq 'gradle-429') + { + Start-Sleep -Seconds $RunUnityRateLimitRetrySleepSeconds + } $try = $try + 1 } else @@ -105,9 +157,9 @@ function RunUnity([string] $unityPath, [string[]] $arguments, [switch] $ReturnLo $timeTaken = "Time taken: $($stopwatch.Elapsed.ToString('hh\:mm\:ss\.fff'))" Write-Host -ForegroundColor Green "Unity finished successfully. $timeTaken" - if ($try -gt 3) + if ($licenseTry -gt 3) { - Write-Host "::$($try -gt 10 ? 'warning' : 'notice')::It took $try tries to successfully acquire a Unity license. Total time taken: $($stopwatchTotal.Elapsed.ToString('hh\:mm\:ss\.fff'))" + Write-Host "::$($licenseTry -gt 10 ? 'warning' : 'notice')::It took $licenseTry tries to successfully acquire a Unity license. Total time taken: $($stopwatchTotal.Elapsed.ToString('hh\:mm\:ss\.fff'))" } return $ReturnLogOutput ? $stdout : $null }