diff --git a/docs/roadmaps/entries/PMAT-4715.yaml b/docs/roadmaps/entries/PMAT-4715.yaml new file mode 100644 index 0000000000..398c99140b --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4715.yaml @@ -0,0 +1,19 @@ +- id: PMAT-4715 + github_issue: 4715 + item_type: task + title: 'B2: root-cause fixes for red nightlies on main' + status: inprogress + priority: critical + assigned_to: aprender-wmainred + created: 2026-10-04T00:00:00Z + updated: 2026-10-04T00:00:00Z + acceptance_criteria: + - Each red or cancelled scheduled workflow on main has a root cause named in its fix PR, one PR per defect, listed on issue #4715 + - Each fix PR changes no gate's acceptance; a threshold or acceptance change goes to the train lead's sign-off batch instead + - After each fix merges, the next scheduled or dispatched run of that workflow on main gets past the step that failed + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:ci + notes: 'Parent #4678 (0.70.2), bar B2: no check red on main for more than 24 h. Proposed by aprender-a7, minted by cop aprender-14.' diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index f65016f6ce..6d12cec68b 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -24060,3 +24060,22 @@ Not done, pending the ruling: moving the 31 fixture files to an obviously synthe labels: - kind:docs notes: 'Operator C197, 2026-10-01 12:40Z: re-date via the spec''s own procedure; check unchanged; not a waiver.' +- id: PMAT-4715 + github_issue: 4715 + item_type: task + title: 'B2: root-cause fixes for red nightlies on main' + status: inprogress + priority: critical + assigned_to: aprender-wmainred + created: 2026-10-04T00:00:00Z + updated: 2026-10-04T00:00:00Z + acceptance_criteria: + - Each red or cancelled scheduled workflow on main has a root cause named in its fix PR, one PR per defect, listed on issue #4715 + - Each fix PR changes no gate's acceptance; a threshold or acceptance change goes to the train lead's sign-off batch instead + - After each fix merges, the next scheduled or dispatched run of that workflow on main gets past the step that failed + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:ci + notes: 'Parent #4678 (0.70.2), bar B2: no check red on main for more than 24 h. Proposed by aprender-a7, minted by cop aprender-14.' diff --git a/scripts/nightly_manifest.py b/scripts/nightly_manifest.py index 206b97fce7..05c06cd215 100755 --- a/scripts/nightly_manifest.py +++ b/scripts/nightly_manifest.py @@ -327,6 +327,19 @@ def publish_plan(manifest, sha, dist_files): return names + [MANIFEST_ASSET] +def release_assets(A, rel_id, token, get=api): + """{name: id} for EVERY asset on the release, all pages. GitHub caps a page at 100; + page one alone missed 134 of 234 assets, so the staged leftovers past it were never + deleted and the next upload of the same name died 422 already_exists (run 37171118510).""" + out, page = {}, 1 + while True: + batch = get("GET", f"{A}/releases/{rel_id}/assets?per_page=100&page={page}", token) + out.update({a["name"]: a["id"] for a in batch}) + if len(batch) < 100: + return out + page += 1 + + def publish(manifest_path, dist, sha, repo, token): with open(manifest_path) as f: man = json.load(f) @@ -349,7 +362,7 @@ def publish(manifest_path, dist, sha, repo, token): rel = api("POST", f"{A}/releases", token, json.dumps({ "tag_name": "nightly", "target_commitish": sha, "name": "Nightly Build", "prerelease": True, "make_latest": "false"}).encode()) - existing = {a["name"]: a["id"] for a in api("GET", f"{A}/releases/{rel['id']}/assets?per_page=100", token)} + existing = release_assets(A, rel["id"], token) for n, i in existing.items(): # a staged upload an earlier run died holding if n.startswith(STAGED): api("DELETE", f"{A}/releases/assets/{i}", token) @@ -379,6 +392,18 @@ def publish(manifest_path, dist, sha, repo, token): # ---------------------------------------------------------------- self-test +def _self_test_release_assets(check): + print("release assets:") + rows = [{"name": f"staged.a{i}" if i % 2 else f"a{i}", "id": i} for i in range(234)] + + def pages(of): + return lambda m, url, tok: of[(int(url.rsplit("page=", 1)[1]) - 1) * 100:][:100] + got = release_assets("A", 1, None, pages(rows)) + check("every page is read: 234 assets over 3 pages, all 117 staged leftovers seen", + (len(got), sum(n.startswith(STAGED) for n in got)), (234, 117)) + check("an exact multiple of 100 ends on the empty page", len(release_assets("A", 1, None, pages(rows[:200]))), 200) + + def self_test(): fails = [] @@ -608,6 +633,7 @@ def pkg(name, bins, root=False, feats=None): except ValueError: got = "error" check("a manifest naming an absent asset refuses to publish", got, "error") + _self_test_release_assets(check) print(f"\n{'PASS' if not fails else 'FAIL'}: {len(fails)} failing row(s)") return 1 if fails else 0