|
6 | 6 | import os |
7 | 7 | import time |
8 | 8 | import unittest |
| 9 | +from urllib.error import HTTPError, URLError |
9 | 10 | from urllib.request import Request, urlopen |
10 | 11 |
|
11 | 12 |
|
| 13 | +class TransientApiError(RuntimeError): |
| 14 | + """An API failure that can clear on its own: a 5xx, a dropped connection or a timeout.""" |
| 15 | + |
| 16 | + |
12 | 17 | def wait_for_artifact(fetch, producer, artifact, timeout, clock=time.monotonic, pause=time.sleep): |
13 | 18 | deadline = clock() + timeout |
| 19 | + last_transient = None |
14 | 20 | while True: |
15 | 21 | if clock() >= deadline: |
16 | | - raise RuntimeError(f"{producer}: artifact wait timed out after {timeout:g}s") |
17 | | - jobs = fetch("jobs")["jobs"] |
18 | | - matches = [job for job in jobs if job["name"] == producer] |
19 | | - if len(matches) > 1: |
20 | | - raise RuntimeError(f"{producer}: ambiguous producer jobs") |
21 | | - job = matches[0] if matches else None |
22 | | - if job and job["status"] == "completed" and job["conclusion"] != "success": |
23 | | - raise RuntimeError(f"{producer}: producer ended {job['conclusion']} without a usable artifact") |
24 | | - artifacts = fetch("artifacts")["artifacts"] |
25 | | - ready = [item for item in artifacts if item["name"] == artifact and not item["expired"]] |
26 | | - if len(ready) == 1: |
27 | | - return ready[0]["id"] |
28 | | - if len(ready) > 1: |
29 | | - raise RuntimeError(f"{producer}: ambiguous artifact {artifact}") |
30 | | - if job and job["status"] == "completed": |
31 | | - raise RuntimeError(f"{producer}: producer succeeded without artifact {artifact}") |
| 22 | + detail = f"; last API error: {last_transient}" if last_transient else "" |
| 23 | + raise RuntimeError(f"{producer}: artifact wait timed out after {timeout:g}s{detail}") |
| 24 | + try: |
| 25 | + found = poll_once(fetch, producer, artifact) |
| 26 | + except TransientApiError as error: |
| 27 | + # GitHub's API returns the occasional 502 or drops a connection; one |
| 28 | + # such blip must not fail a job whose producer is healthy. Keep polling |
| 29 | + # until the deadline, and name the last error if the deadline passes. |
| 30 | + last_transient = error |
| 31 | + found = None |
| 32 | + if found is not None: |
| 33 | + return found |
32 | 34 | remaining = deadline - clock() |
33 | 35 | if remaining <= 0: |
34 | | - raise RuntimeError(f"{producer}: artifact wait timed out after {timeout:g}s") |
| 36 | + detail = f"; last API error: {last_transient}" if last_transient else "" |
| 37 | + raise RuntimeError(f"{producer}: artifact wait timed out after {timeout:g}s{detail}") |
35 | 38 | pause(min(10, remaining)) |
36 | 39 |
|
37 | 40 |
|
| 41 | +def poll_once(fetch, producer, artifact): |
| 42 | + """One look at the producer job and the artifact list: the artifact id, or None to keep waiting.""" |
| 43 | + jobs = fetch("jobs")["jobs"] |
| 44 | + matches = [job for job in jobs if job["name"] == producer] |
| 45 | + if len(matches) > 1: |
| 46 | + raise RuntimeError(f"{producer}: ambiguous producer jobs") |
| 47 | + job = matches[0] if matches else None |
| 48 | + if job and job["status"] == "completed" and job["conclusion"] != "success": |
| 49 | + raise RuntimeError(f"{producer}: producer ended {job['conclusion']} without a usable artifact") |
| 50 | + artifacts = fetch("artifacts")["artifacts"] |
| 51 | + ready = [item for item in artifacts if item["name"] == artifact and not item["expired"]] |
| 52 | + if len(ready) == 1: |
| 53 | + return ready[0]["id"] |
| 54 | + if len(ready) > 1: |
| 55 | + raise RuntimeError(f"{producer}: ambiguous artifact {artifact}") |
| 56 | + if job and job["status"] == "completed": |
| 57 | + raise RuntimeError(f"{producer}: producer succeeded without artifact {artifact}") |
| 58 | + return None |
| 59 | + |
| 60 | + |
38 | 61 | def producer_for_attempt(current, history, producer): |
39 | 62 | matches = [job for job in current if job["name"] == producer] |
40 | 63 | if matches: |
@@ -108,6 +131,36 @@ def test_wait_timeout(self): |
108 | 131 | wait_for_artifact(fetch, "producer", "artifact", 23, lambda: now[0], lambda seconds: now.__setitem__(0, now[0] + seconds)) |
109 | 132 | self.assertEqual(now[0], 23) |
110 | 133 |
|
| 134 | + def test_transient_api_error_keeps_waiting(self): |
| 135 | + now = [0] |
| 136 | + failures = [TransientApiError("artifact API query failed: HTTP Error 502: Bad Gateway")] |
| 137 | + |
| 138 | + def fetch(kind): |
| 139 | + if failures: |
| 140 | + raise failures.pop() |
| 141 | + if kind == "jobs": |
| 142 | + return {"jobs": [{"name": "producer", "status": "in_progress", "conclusion": None}]} |
| 143 | + return {"artifacts": [{"name": "artifact", "expired": False, "id": 7}]} |
| 144 | + |
| 145 | + found = wait_for_artifact(fetch, "producer", "artifact", 60, lambda: now[0], lambda s: now.__setitem__(0, now[0] + s)) |
| 146 | + self.assertEqual((found, now[0]), (7, 10)) |
| 147 | + |
| 148 | + def test_transient_api_errors_until_deadline_name_the_last_error(self): |
| 149 | + now = [0] |
| 150 | + |
| 151 | + def fetch(kind): |
| 152 | + raise TransientApiError("artifact API query failed: HTTP Error 502: Bad Gateway") |
| 153 | + |
| 154 | + with self.assertRaisesRegex(RuntimeError, "timed out after 25s; last API error: .*502"): |
| 155 | + wait_for_artifact(fetch, "producer", "artifact", 25, lambda: now[0], lambda s: now.__setitem__(0, now[0] + s)) |
| 156 | + |
| 157 | + def test_permanent_api_error_fails_at_once(self): |
| 158 | + def fetch(kind): |
| 159 | + raise RuntimeError("artifact API query failed: HTTP Error 404: Not Found") |
| 160 | + |
| 161 | + with self.assertRaisesRegex(RuntimeError, "404"): |
| 162 | + wait_for_artifact(fetch, "producer", "artifact", 60, lambda: 0, lambda s: None) |
| 163 | + |
111 | 164 | def test_failed_only_rerun_reuses_the_successful_producers_attempt(self): |
112 | 165 | previous = {"id": 1, "name": "producer", "status": "completed", "conclusion": "success", "run_attempt": 1} |
113 | 166 | self.assertEqual(producer_for_attempt([], [previous], "producer"), [previous]) |
@@ -150,6 +203,13 @@ def request_page(route): |
150 | 203 | try: |
151 | 204 | with urlopen(request, timeout=max(0.001, min(30, deadline - time.monotonic()))) as response: |
152 | 205 | result = json.load(response) |
| 206 | + except HTTPError as error: |
| 207 | + # A 5xx is GitHub's side and may clear; anything else (401, 403, 404) |
| 208 | + # will not, so it still fails the job at once. |
| 209 | + kind = TransientApiError if error.code >= 500 else RuntimeError |
| 210 | + raise kind(f"{args.producer}: artifact API query failed: {error}") from error |
| 211 | + except (URLError, TimeoutError, ConnectionError) as error: |
| 212 | + raise TransientApiError(f"{args.producer}: artifact API query failed: {error}") from error |
153 | 213 | except Exception as error: |
154 | 214 | raise RuntimeError(f"{args.producer}: artifact API query failed: {error}") from error |
155 | 215 | return result |
|
0 commit comments