From 94804c4c577f93ac1c41a8403f7f41c6cf5478dd Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 19 Oct 2026 13:18:11 +0200 Subject: [PATCH 1/2] aprender#5751: keep crates.io and the build-base fetch off the merge path Registry was keyed per PR x shard, so every new PR and merge-queue ref cold-fetched all crates from crates.io; pin_build_base re-fetched a base the checkout could already hold. Seed a cold registry from a host-local seed (published by green shards); pin reads a local base first and checkout depth is 2 in the guard jobs. brownout_test.sh is RED without the fix. Pmat-Ticket: PMAT-5751 Co-Authored-By: Claude Opus 5.5 --- ci/sections.yml | 11 ++++- docs/roadmaps/entries/PMAT-5751.yaml | 19 +++++++++ scripts/ci/pin_build_base.sh | 12 +++++- scripts/ci/seed_registry.sh | 62 +++++++++++++++++++++++++++ scripts/tests/brownout_test.sh | 64 ++++++++++++++++++++++++++++ 5 files changed, 164 insertions(+), 4 deletions(-) create mode 100644 docs/roadmaps/entries/PMAT-5751.yaml create mode 100644 scripts/ci/seed_registry.sh create mode 100644 scripts/tests/brownout_test.sh diff --git a/ci/sections.yml b/ci/sections.yml index e2bf1475da..e8173d7ef5 100644 --- a/ci/sections.yml +++ b/ci/sections.yml @@ -310,6 +310,9 @@ jobs: fi heal_path /mnt/nvme-raid0/targets/aprender-ci "${parent}/run-${GITHUB_RUN_ID}" heal_path /mnt/nvme-raid0/cargo-ci/registry "$reg" + # aprender#5751: a cold per-PR registry is seeded from the host-local seed, so a crates.io + # brownout cannot fail the job before it compiles. Never fails: no seed = cold start. + bash scripts/ci/seed_registry.sh seed "$reg" /mnt/nvme-raid0/cargo-ci/registry/_seed ls -ld "$parent" "${parent}/run-${GITHUB_RUN_ID}" "$reg" - name: Pre-build chown — fix per-RUN root ownership # Root cause (five-whys): @@ -1106,6 +1109,10 @@ jobs: -w /workspace \ "$IMAGE" \ setsid --wait bash scripts/check_format_sovereignty.sh + - name: Publish this shard's warmed registry as the host-local seed (aprender#5751) + # Best effort and never red: the seed only ever makes the NEXT cold registry warm. + if: success() + run: bash scripts/ci/seed_registry.sh publish "/mnt/nvme-raid0/cargo-ci/registry/${PR_OR_REF}" /mnt/nvme-raid0/cargo-ci/registry/_seed || echo "::warning::seed publish skipped (the seed is a cache, not a check)" - name: Fix file ownership (container runs as root, runner as noah:1000) if: always() run: | @@ -1217,7 +1224,7 @@ jobs: fi - uses: actions/checkout@v7 with: - fetch-depth: 1 + fetch-depth: 2 # THE COMPARAND, FETCHED ONCE, BEFORE ANY GUARD THAT NEEDS IT. # # Every shrink-only baseline in scripts/ is now compared against a ref a @@ -1772,7 +1779,7 @@ jobs: fi - uses: actions/checkout@v7 with: - fetch-depth: 1 + fetch-depth: 2 # THE COMPARAND, FETCHED ONCE, BEFORE ANY GUARD THAT NEEDS IT. # # Every shrink-only baseline in scripts/ is now compared against a ref a diff --git a/docs/roadmaps/entries/PMAT-5751.yaml b/docs/roadmaps/entries/PMAT-5751.yaml new file mode 100644 index 0000000000..8206b43552 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-5751.yaml @@ -0,0 +1,19 @@ +- id: PMAT-5751 + github_issue: 5751 + item_type: task + title: 'aprender#5751: brownout-proof merge path: seed per-PR cargo registry from host-local seed; pin_build_base local-first' + status: planned + priority: high + assigned_to: null + created: 2026-10-19T11:16:46Z + updated: 2026-10-19T11:16:46Z + spec: null + acceptance_criteria: + - Registry keyed per PR x shard so every new ref cold-fetches crates.io; pin_build_base refetches a base the checkout can hold. + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 5510 diff --git a/scripts/ci/pin_build_base.sh b/scripts/ci/pin_build_base.sh index 92cf6ff85f..87a304198b 100755 --- a/scripts/ci/pin_build_base.sh +++ b/scripts/ci/pin_build_base.sh @@ -31,7 +31,7 @@ ZERO=0000000000000000000000000000000000000000 # #4936 (P6, operator C343 #9): every fetch below goes through fetch_p6.sh, as the sections' fetches # do. A read that did not answer (curl 92, curl 56, a stall) is read again, up to 3 reads in all; a # read that answered, an object origin does not have among them, is never read again. -P6="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/fetch_p6.sh" +P6="${P6_OVERRIDE:-$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/fetch_p6.sh}" fetch() { # fetch : git fetch in , read under P6 local dir="$1"; shift @@ -71,7 +71,15 @@ pin() { # pin : pins origin/main there for $GITHUB_EVENT_NAME return 0 ;; esac - fetch "$dir" -q --no-tags --depth=1 origin "$base" || refuse "cannot fetch the build base $base" || return 1 + # aprender#5751: the build base is already in the checkout when it holds the merge commit's + # parents (fetch-depth >= 2, or a warm clone). Reading it from github.com again put a WAN + # brownout on the merge path (run 38363869044: "cannot fetch the build base"). Fetch only + # an object this checkout does not have. + if git -C "$dir" cat-file -e "${base}^{commit}" 2>/dev/null; then + printf 'pin_build_base: build base %s already in the checkout; no fetch\n' "$base" + else + fetch "$dir" -q --no-tags --depth=1 origin "$base" || refuse "cannot fetch the build base $base" || return 1 + fi git -C "$dir" update-ref refs/remotes/origin/main "$base" [ "$(git -C "$dir" rev-parse 'origin/main^{commit}')" = "$base" ] || refuse "origin/main does not read back as $base" || return 1 printf 'comparand: %s pinned as origin/main (%s; event %s)\n' "$base" "$how" "$ev" diff --git a/scripts/ci/seed_registry.sh b/scripts/ci/seed_registry.sh new file mode 100644 index 0000000000..289ddd0620 --- /dev/null +++ b/scripts/ci/seed_registry.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# seed_registry.sh -- keep crates.io off the merge path (aprender#5751, PMAT-5751). +# +# WHY: the cargo registry mounted into a job is keyed per PR (x shard). A new PR, and every +# merge-queue ref, starts with an EMPTY registry, so each shard downloads its whole dependency +# set from crates.io before it compiles a line. A WAN brownout then fails a merge job +# ("spurious network error: Timeout was reached ... linux-raw-sys / libsqlite3-sys", run +# 38363869044). Retrying hides the dependency; this removes it: a cold registry is seeded from a +# warm host-local copy, so cargo finds the crates on disk and only a genuinely NEW crate version +# touches the network. +# +# seed_registry.sh seed cold registry <- seed. Never fails the job. +# seed_registry.sh publish warm registry -> seed (atomic rename). +# +# Only index/ and cache/ travel (src/ is re-extracted by cargo). A registry is COLD when it holds +# no cached .crate. seed never overwrites a warm registry. publish refuses a registry with no +# cached crate, so an empty one cannot replace a good seed. A missing seed is a cold start, exit 0. +set -uo pipefail + +crate_count() { find "$1/cache" -name '*.crate' 2>/dev/null | head -n 1000 | wc -l; } + +cmd_seed() { + local reg="$1" seed="$2" + if [ "$(crate_count "$reg")" -gt 0 ]; then + printf 'seed_registry: %s is warm; left alone\n' "$reg"; return 0 + fi + if [ "$(crate_count "$seed")" -eq 0 ]; then + printf 'seed_registry: no seed at %s; cold start (crates.io will be read)\n' "$seed"; return 0 + fi + mkdir -p "$reg" || return 0 + local d + for d in index cache; do + [ -d "$seed/$d" ] || continue + cp -a --reflink=auto "$seed/$d" "$reg/" 2>/dev/null \ + || printf 'seed_registry: warning: could not copy %s/%s\n' "$seed" "$d" + done + printf 'seed_registry: seeded %s from %s (%s crates)\n' "$reg" "$seed" "$(crate_count "$reg")" + return 0 +} + +cmd_publish() { + local reg="$1" seed="$2" n tmp + n=$(crate_count "$reg") + if [ "$n" -eq 0 ]; then + printf 'seed_registry: refusing to publish %s: no cached crates\n' "$reg" >&2; return 1 + fi + if [ "$n" -lt "$(crate_count "$seed")" ]; then + printf 'seed_registry: %s has fewer crates than the seed; seed kept\n' "$reg"; return 0 + fi + mkdir -p "$(dirname -- "$seed")" || return 1 + tmp="${seed}.tmp.$$" + mkdir -p "$tmp" && cp -a --reflink=auto "$reg/index" "$reg/cache" "$tmp/" || { rm -rf "${tmp:?}"; return 1; } + rm -rf "${seed:?}.old" && { [ ! -d "$seed" ] || mv "$seed" "${seed}.old"; } && mv "$tmp" "$seed" || return 1 + rm -rf "${seed:?}.old" + printf 'seed_registry: published %s crates to %s\n' "$n" "$seed" +} + +case "${1:-}" in + seed) [ "$#" -eq 3 ] || { echo 'usage: seed ' >&2; exit 64; }; cmd_seed "$2" "$3" ;; + publish) [ "$#" -eq 3 ] || { echo 'usage: publish ' >&2; exit 64; }; cmd_publish "$2" "$3" ;; + *) echo 'usage: seed_registry.sh seed|publish ' >&2; exit 64 ;; +esac diff --git a/scripts/tests/brownout_test.sh b/scripts/tests/brownout_test.sh new file mode 100644 index 0000000000..6c4f4f711d --- /dev/null +++ b/scripts/tests/brownout_test.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# brownout_test.sh -- falsifier for aprender#5751 (PMAT-5751): a WAN brownout must not fail a merge job. +# R1 a cold per-PR registry is seeded from the host-local seed: crates are on disk before cargo runs +# R2 a warm registry is left alone (never clobbered by the seed) +# R3 no seed present: exit 0, registry stays cold (a missing seed is a cold start, not a failure) +# R4 publish refuses a registry with no cached crates (an empty one must not poison the seed) +# R5 publish makes a cold registry seedable (round trip) +# P1 pin_build_base never touches the network when the build base is already in the checkout +# P2 pin_build_base still fetches when the base is absent (fallback unchanged) +# Usage: bash scripts/tests/brownout_test.sh exit 0 all green, 1 any red +set -uo pipefail +ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" +SEED="$ROOT/scripts/ci/seed_registry.sh" +PIN="$ROOT/scripts/ci/pin_build_base.sh" +TD=$(mktemp -d); trap 'rm -rf "${TD:?}"' EXIT +ok=0; bad=0 +row() { if [ "$2" = "$3" ]; then ok=$((ok+1)); printf 'ok %s\n' "$1"; else bad=$((bad+1)); printf 'RED %s (got %s, want %s)\n' "$1" "$2" "$3"; fi; } +[ -f "$SEED" ] || { printf 'RED %s is missing\n' "$SEED"; bad=$((bad+1)); } + +mkreg() { mkdir -p "$1/cache/idx" "$1/index/idx"; printf 'crate\n' > "$1/cache/idx/linux-raw-sys-0.4.14.crate"; printf 'i\n' > "$1/index/idx/.cache"; } +S="$TD/seed"; mkreg "$S" + +R1="$TD/r1"; mkdir -p "$R1" +bash "$SEED" seed "$R1" "$S" >/dev/null 2>&1; rc=$? +row "R1 cold registry gets the crate" "$rc $([ -f "$R1/cache/idx/linux-raw-sys-0.4.14.crate" ] && echo present || echo absent)" "0 present" + +R2="$TD/r2"; mkreg "$R2"; printf 'mine\n' > "$R2/cache/idx/own.crate" +bash "$SEED" seed "$R2" "$S" >/dev/null 2>&1 +row "R2 warm registry untouched" "$([ -f "$R2/cache/idx/own.crate" ] && echo kept || echo clobbered)" kept + +R3="$TD/r3"; mkdir -p "$R3" +bash "$SEED" seed "$R3" "$TD/no-such-seed" >/dev/null 2>&1; rc=$? +row "R3 missing seed is a cold start" "$rc $(ls -A "$R3" | wc -l)" "0 0" + +E="$TD/empty"; mkdir -p "$E" +bash "$SEED" publish "$E" "$TD/seed2" >/dev/null 2>&1; rc=$? +row "R4 empty registry is not published" "$([ "$rc" -ne 0 ] && [ ! -d "$TD/seed2" ] && echo refused || echo published)" refused + +bash "$SEED" publish "$R2" "$TD/seed2" >/dev/null 2>&1; rc=$? +R5="$TD/r5"; mkdir -p "$R5"; bash "$SEED" seed "$R5" "$TD/seed2" >/dev/null 2>&1 +row "R5 publish then seed round trip" "$rc $([ -f "$R5/cache/idx/own.crate" ] && echo present || echo absent)" "0 present" + +# --- pin_build_base: a PR merge commit whose first parent is the base, checkout deep enough to hold it. +o="$TD/origin.git"; w="$TD/w" +git init -q --template= --bare -b main "$o"; git init -q --template= -b main "$w" +g() { git -C "$w" -c user.name=t -c user.email=t@example.invalid -c commit.gpgsign=false "$@"; } +printf a > "$w/a"; g add -A; g commit -q -m c0; c0=$(g rev-parse HEAD) +g checkout -q -b pr; printf b > "$w/b"; g add -A; g commit -q -m pr; prh=$(g rev-parse HEAD) +g checkout -q --detach "$c0"; g merge -q --no-ff --no-edit pr; m=$(g rev-parse HEAD) +# Network is DOWN: a fetch_p6 stand-in that fails and records that it was called. +printf '#!/usr/bin/env bash\necho called >> "%s/net-calls"\nexit 1\n' "$TD" > "$TD/dead_fetch.sh" +pinrun() { ( cd "$1" && GITHUB_EVENT_NAME=pull_request PR_HEAD_SHA="$prh" P6_OVERRIDE="$TD/dead_fetch.sh" bash "$PIN" ) >"$TD/pin.log" 2>&1; } +rc=0; pinrun "$w" || rc=$? +row "P1 base already local: no network, pinned" "$rc $(git -C "$w" rev-parse origin/main 2>/dev/null | cut -c1-7) $(ls "$TD/net-calls" 2>/dev/null | wc -l)" "0 $(printf %s "$c0" | cut -c1-7) 0" + +# P2: a depth-1 checkout of the merge commit holds no parent. +d="$TD/shallow"; git init -q --template= "$d"; git -C "$d" remote add origin "file://$w" +git -C "$d" fetch -q --depth=1 origin "$m" && git -C "$d" checkout -q --detach FETCH_HEAD +rm -f "${TD:?}/net-calls" +rc=0; pinrun "$d" || rc=$? +row "P2 base absent: fetch attempted, refuses when net is down" "$rc $(ls "$TD/net-calls" 2>/dev/null | wc -l)" "1 1" + +printf 'brownout_test: %s ok, %s red\n' "$ok" "$bad" +[ "$bad" -eq 0 ] From 00641d9026b0870107bd560f0edfe14727c78ab1 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 19 Oct 2026 20:26:13 +0200 Subject: [PATCH 2/2] fix(ci): brownout_test.sh confines its depth fetch (GIT_CEILING_DIRECTORIES, #5025) (Refs #5751) Agent: aprender-w5018 Co-Authored-By: Claude Sonnet 5.5 --- scripts/tests/brownout_test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/tests/brownout_test.sh b/scripts/tests/brownout_test.sh index 6c4f4f711d..0cfde72316 100644 --- a/scripts/tests/brownout_test.sh +++ b/scripts/tests/brownout_test.sh @@ -13,6 +13,7 @@ ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" SEED="$ROOT/scripts/ci/seed_registry.sh" PIN="$ROOT/scripts/ci/pin_build_base.sh" TD=$(mktemp -d); trap 'rm -rf "${TD:?}"' EXIT +export GIT_CEILING_DIRECTORIES="$TD" # #5025: a slipped -C must fail, not write into the shared checkout ok=0; bad=0 row() { if [ "$2" = "$3" ]; then ok=$((ok+1)); printf 'ok %s\n' "$1"; else bad=$((bad+1)); printf 'RED %s (got %s, want %s)\n' "$1" "$2" "$3"; fi; } [ -f "$SEED" ] || { printf 'RED %s is missing\n' "$SEED"; bad=$((bad+1)); }