Skip to content

Commit 6cd3354

Browse files
CI artifact wait: keep polling through transient GitHub API errors
A single 502 from the artifacts API failed master's OpenCode 2 Docker job while its producer was healthy. 5xx responses, dropped connections and timeouts now count as transient: the wait keeps polling until its deadline and names the last error if the deadline passes. 4xx responses still fail at once. Co-authored-by: Alfonso <alfonso-magic-context@users.noreply.github.com>
1 parent 8b5e0cb commit 6cd3354

1 file changed

Lines changed: 77 additions & 17 deletions

File tree

‎scripts/wait-ci-artifact.py‎

Lines changed: 77 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -6,35 +6,58 @@
66
import os
77
import time
88
import unittest
9+
from urllib.error import HTTPError, URLError
910
from urllib.request import Request, urlopen
1011

1112

13+
class TransientApiError(RuntimeError):
14+
"""An API failure that can clear on its own: a 5xx, a dropped connection or a timeout."""
15+
16+
1217
def wait_for_artifact(fetch, producer, artifact, timeout, clock=time.monotonic, pause=time.sleep):
1318
deadline = clock() + timeout
19+
last_transient = None
1420
while True:
1521
if clock() >= deadline:
16-
raise RuntimeError(f"{producer}: artifact wait timed out after {timeout:g}s")
17-
jobs = fetch("jobs")["jobs"]
18-
matches = [job for job in jobs if job["name"] == producer]
19-
if len(matches) > 1:
20-
raise RuntimeError(f"{producer}: ambiguous producer jobs")
21-
job = matches[0] if matches else None
22-
if job and job["status"] == "completed" and job["conclusion"] != "success":
23-
raise RuntimeError(f"{producer}: producer ended {job['conclusion']} without a usable artifact")
24-
artifacts = fetch("artifacts")["artifacts"]
25-
ready = [item for item in artifacts if item["name"] == artifact and not item["expired"]]
26-
if len(ready) == 1:
27-
return ready[0]["id"]
28-
if len(ready) > 1:
29-
raise RuntimeError(f"{producer}: ambiguous artifact {artifact}")
30-
if job and job["status"] == "completed":
31-
raise RuntimeError(f"{producer}: producer succeeded without artifact {artifact}")
22+
detail = f"; last API error: {last_transient}" if last_transient else ""
23+
raise RuntimeError(f"{producer}: artifact wait timed out after {timeout:g}s{detail}")
24+
try:
25+
found = poll_once(fetch, producer, artifact)
26+
except TransientApiError as error:
27+
# GitHub's API returns the occasional 502 or drops a connection; one
28+
# such blip must not fail a job whose producer is healthy. Keep polling
29+
# until the deadline, and name the last error if the deadline passes.
30+
last_transient = error
31+
found = None
32+
if found is not None:
33+
return found
3234
remaining = deadline - clock()
3335
if remaining <= 0:
34-
raise RuntimeError(f"{producer}: artifact wait timed out after {timeout:g}s")
36+
detail = f"; last API error: {last_transient}" if last_transient else ""
37+
raise RuntimeError(f"{producer}: artifact wait timed out after {timeout:g}s{detail}")
3538
pause(min(10, remaining))
3639

3740

41+
def poll_once(fetch, producer, artifact):
42+
"""One look at the producer job and the artifact list: the artifact id, or None to keep waiting."""
43+
jobs = fetch("jobs")["jobs"]
44+
matches = [job for job in jobs if job["name"] == producer]
45+
if len(matches) > 1:
46+
raise RuntimeError(f"{producer}: ambiguous producer jobs")
47+
job = matches[0] if matches else None
48+
if job and job["status"] == "completed" and job["conclusion"] != "success":
49+
raise RuntimeError(f"{producer}: producer ended {job['conclusion']} without a usable artifact")
50+
artifacts = fetch("artifacts")["artifacts"]
51+
ready = [item for item in artifacts if item["name"] == artifact and not item["expired"]]
52+
if len(ready) == 1:
53+
return ready[0]["id"]
54+
if len(ready) > 1:
55+
raise RuntimeError(f"{producer}: ambiguous artifact {artifact}")
56+
if job and job["status"] == "completed":
57+
raise RuntimeError(f"{producer}: producer succeeded without artifact {artifact}")
58+
return None
59+
60+
3861
def producer_for_attempt(current, history, producer):
3962
matches = [job for job in current if job["name"] == producer]
4063
if matches:
@@ -108,6 +131,36 @@ def test_wait_timeout(self):
108131
wait_for_artifact(fetch, "producer", "artifact", 23, lambda: now[0], lambda seconds: now.__setitem__(0, now[0] + seconds))
109132
self.assertEqual(now[0], 23)
110133

134+
def test_transient_api_error_keeps_waiting(self):
135+
now = [0]
136+
failures = [TransientApiError("artifact API query failed: HTTP Error 502: Bad Gateway")]
137+
138+
def fetch(kind):
139+
if failures:
140+
raise failures.pop()
141+
if kind == "jobs":
142+
return {"jobs": [{"name": "producer", "status": "in_progress", "conclusion": None}]}
143+
return {"artifacts": [{"name": "artifact", "expired": False, "id": 7}]}
144+
145+
found = wait_for_artifact(fetch, "producer", "artifact", 60, lambda: now[0], lambda s: now.__setitem__(0, now[0] + s))
146+
self.assertEqual((found, now[0]), (7, 10))
147+
148+
def test_transient_api_errors_until_deadline_name_the_last_error(self):
149+
now = [0]
150+
151+
def fetch(kind):
152+
raise TransientApiError("artifact API query failed: HTTP Error 502: Bad Gateway")
153+
154+
with self.assertRaisesRegex(RuntimeError, "timed out after 25s; last API error: .*502"):
155+
wait_for_artifact(fetch, "producer", "artifact", 25, lambda: now[0], lambda s: now.__setitem__(0, now[0] + s))
156+
157+
def test_permanent_api_error_fails_at_once(self):
158+
def fetch(kind):
159+
raise RuntimeError("artifact API query failed: HTTP Error 404: Not Found")
160+
161+
with self.assertRaisesRegex(RuntimeError, "404"):
162+
wait_for_artifact(fetch, "producer", "artifact", 60, lambda: 0, lambda s: None)
163+
111164
def test_failed_only_rerun_reuses_the_successful_producers_attempt(self):
112165
previous = {"id": 1, "name": "producer", "status": "completed", "conclusion": "success", "run_attempt": 1}
113166
self.assertEqual(producer_for_attempt([], [previous], "producer"), [previous])
@@ -150,6 +203,13 @@ def request_page(route):
150203
try:
151204
with urlopen(request, timeout=max(0.001, min(30, deadline - time.monotonic()))) as response:
152205
result = json.load(response)
206+
except HTTPError as error:
207+
# A 5xx is GitHub's side and may clear; anything else (401, 403, 404)
208+
# will not, so it still fails the job at once.
209+
kind = TransientApiError if error.code >= 500 else RuntimeError
210+
raise kind(f"{args.producer}: artifact API query failed: {error}") from error
211+
except (URLError, TimeoutError, ConnectionError) as error:
212+
raise TransientApiError(f"{args.producer}: artifact API query failed: {error}") from error
153213
except Exception as error:
154214
raise RuntimeError(f"{args.producer}: artifact API query failed: {error}") from error
155215
return result

0 commit comments

Comments
 (0)