From 37d4633cbc0a497d02045387b80cb532e593ad5d Mon Sep 17 00:00:00 2001
From: Mike Park <mike@flexcompute.com>
Date: Tue, 9 Jun 2026 13:33:06 +0000
Subject: [PATCH] download.sh: retry transient MIT fetch failures, fail fast on
 expected 404

The ESP 1.29.2 release build failed when both `curl --head` probes to
acdl.mit.edu timed out (~134s each) and download.sh fell through to its
not-found branch. MIT was reachable -- it was a transient runner->MIT blip.

Route the probe and download through mit_curl(): --retry / --retry-connrefused
ride out transient errors (timeouts, 5xx, refused connections) and
--connect-timeout 30 keeps a hung endpoint from stalling the job for minutes.
A 404 is deliberately NOT retried (curl --retry skips it) -- a given version
lives in exactly one of PreBuilts/ or archive/, so a miss in the other is
expected and should fail over immediately. Also drops the redundant
`-O ... -o` (which logged "Got more output options than URLs").

Verified: ./download.sh 129 completes end-to-end (probe -> download -> all
three 129 patches, exit 0); the expected 404 on archive/ fails fast (~0s).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 download.sh | 27 ++++++++++++++++++++-------
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/download.sh b/download.sh
index 6803c815..6d6fbb62 100755
--- a/download.sh
+++ b/download.sh
@@ -7,20 +7,33 @@ set -x # echo commands
 version=${1:-129}
 tar=ESP${version}-linux-x86_64.tgz
 
-# MIT rotates older releases from PreBuilts/ to archive/
+# A given ESP version lives in exactly ONE of PreBuilts/ or archive/: MIT moves
+# a release from PreBuilts/ to archive/ once it ages out, so a 404 from the
+# other directory is EXPECTED and must not abort the build. Reaching
+# acdl.mit.edu can also fail transiently (connection timeouts, 5xx) -- that is
+# what we retry. curl's --retry covers those transient errors but never a 404,
+# and --connect-timeout keeps a hung endpoint from stalling the job for minutes.
 prebuilts_url=https://acdl.mit.edu/ESP/PreBuilts/${tar}
 archive_url=https://acdl.mit.edu/ESP/archive/${tar}
 
-if curl --head --silent --fail "$prebuilts_url" > /dev/null 2>&1; then
-  curl -O "$prebuilts_url" -o ${tar}
-elif curl --head --silent --fail "$archive_url" > /dev/null 2>&1; then
-  echo "Not found in PreBuilts, downloading from archive"
-  curl -O "$archive_url" -o ${tar}
+mit_curl() {
+  curl --fail --location --connect-timeout 30 \
+       --retry 5 --retry-delay 5 --retry-connrefused "$@"
+}
+
+if mit_curl --silent --head --max-time 120 "$prebuilts_url" > /dev/null; then
+  url=$prebuilts_url
+  echo "Found ESP${version} in PreBuilts"
+elif mit_curl --silent --head --max-time 120 "$archive_url" > /dev/null; then
+  url=$archive_url
+  echo "Not in PreBuilts (expected); found ESP${version} in archive"
 else
-  echo "ERROR: ESP${version} not found in PreBuilts or archive" >&2
+  echo "ERROR: ESP${version} unreachable in PreBuilts or archive after retries" >&2
   exit 1
 fi
 
+mit_curl --silent --show-error --max-time 1800 -o "${tar}" "$url"
+
 rm -rf ESP ESP${version}
 tar xzf ${tar}
 mv ESP${version} ESP