aprender#5751: keep crates.io and the build-base fetch off the merge path (seeded registry, local build base)

ci/sections.yml

@@ -313,6 +313,9 @@ jobs:
           fi
           heal_path /mnt/nvme-raid0/targets/aprender-ci "${parent}/run-${GITHUB_RUN_ID}"
           heal_path /mnt/nvme-raid0/cargo-ci/registry "$reg"
+          # aprender#5751: a cold per-PR registry is seeded from the host-local seed, so a crates.io
+          # brownout cannot fail the job before it compiles. Never fails: no seed = cold start.
+          bash scripts/ci/seed_registry.sh seed "$reg" /mnt/nvme-raid0/cargo-ci/registry/_seed
           ls -ld "$parent" "${parent}/run-${GITHUB_RUN_ID}" "$reg"
       - name: Pre-build chown — fix per-RUN root ownership
         # Root cause (five-whys):
@@ -1134,6 +1137,10 @@ jobs:
             -w /workspace \
             "$IMAGE" \
             setsid --wait bash scripts/check_format_sovereignty.sh
+      - name: Publish this shard's warmed registry as the host-local seed (aprender#5751)
+        # Best effort and never red: the seed only ever makes the NEXT cold registry warm.
+        if: success()
+        run: bash scripts/ci/seed_registry.sh publish "/mnt/nvme-raid0/cargo-ci/registry/${PR_OR_REF}" /mnt/nvme-raid0/cargo-ci/registry/_seed || echo "::warning::seed publish skipped (the seed is a cache, not a check)"
       - name: Fix file ownership (container runs as root, runner as noah:1000)
         if: always()
         run: |
@@ -1245,7 +1252,7 @@ jobs:
           fi
       - uses: actions/checkout@v7
         with:
-          fetch-depth: 1
+          fetch-depth: 2
       # THE COMPARAND, FETCHED ONCE, BEFORE ANY GUARD THAT NEEDS IT.
       #
       # Every shrink-only baseline in scripts/ is now compared against a ref a
@@ -1800,7 +1807,7 @@ jobs:
           fi
       - uses: actions/checkout@v7
         with:
-          fetch-depth: 1
+          fetch-depth: 2
       # THE COMPARAND, FETCHED ONCE, BEFORE ANY GUARD THAT NEEDS IT.
       #
       # Every shrink-only baseline in scripts/ is now compared against a ref a

docs/roadmaps/entries/PMAT-5751.yaml

@@ -0,0 +1,19 @@
+- id: PMAT-5751
+  github_issue: 5751
+  item_type: task
+  title: 'aprender#5751: brownout-proof merge path: seed per-PR cargo registry from host-local seed; pin_build_base local-first'
+  status: planned
+  priority: high
+  assigned_to: null
+  created: 2026-10-19T11:16:46Z
+  updated: 2026-10-19T11:16:46Z
+  spec: null
+  acceptance_criteria:
+  - Registry keyed per PR x shard so every new ref cold-fetches crates.io; pin_build_base refetches a base the checkout can hold.
+  phases: []
+  subtasks: []
+  estimated_effort: null
+  labels:
+  - kind:code
+  notes: null
+  epic: 5510

scripts/ci/pin_build_base.sh

@@ -31,7 +31,7 @@ ZERO=0000000000000000000000000000000000000000
 # #4936 (P6, operator C343 #9): every fetch below goes through fetch_p6.sh, as the sections' fetches
 # do. A read that did not answer (curl 92, curl 56, a stall) is read again, up to 3 reads in all; a
 # read that answered, an object origin does not have among them, is never read again.
-P6="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/fetch_p6.sh"
+P6="${P6_OVERRIDE:-$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/fetch_p6.sh}"
 
 fetch() { # fetch <repo dir> <git fetch args...>: git fetch in <repo dir>, read under P6
     local dir="$1"; shift
@@ -71,7 +71,15 @@ pin() { # pin <repo dir>: pins origin/main there for $GITHUB_EVENT_NAME
             return 0
             ;;
     esac
-    fetch "$dir" -q --no-tags --depth=1 origin "$base" || refuse "cannot fetch the build base $base" || return 1
+    # aprender#5751: the build base is already in the checkout when it holds the merge commit's
+    # parents (fetch-depth >= 2, or a warm clone). Reading it from github.com again put a WAN
+    # brownout on the merge path (run 38363869044: "cannot fetch the build base"). Fetch only
+    # an object this checkout does not have.
+    if git -C "$dir" cat-file -e "${base}^{commit}" 2>/dev/null; then
+        printf 'pin_build_base: build base %s already in the checkout; no fetch\n' "$base"
+    else
+        fetch "$dir" -q --no-tags --depth=1 origin "$base" || refuse "cannot fetch the build base $base" || return 1
+    fi
     git -C "$dir" update-ref refs/remotes/origin/main "$base"
     [ "$(git -C "$dir" rev-parse 'origin/main^{commit}')" = "$base" ] || refuse "origin/main does not read back as $base" || return 1
     printf 'comparand: %s pinned as origin/main (%s; event %s)\n' "$base" "$how" "$ev"

scripts/ci/seed_registry.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# seed_registry.sh -- keep crates.io off the merge path (aprender#5751, PMAT-5751).
+#
+# WHY: the cargo registry mounted into a job is keyed per PR (x shard). A new PR, and every
+# merge-queue ref, starts with an EMPTY registry, so each shard downloads its whole dependency
+# set from crates.io before it compiles a line. A WAN brownout then fails a merge job
+# ("spurious network error: Timeout was reached ... linux-raw-sys / libsqlite3-sys", run
+# 38363869044). Retrying hides the dependency; this removes it: a cold registry is seeded from a
+# warm host-local copy, so cargo finds the crates on disk and only a genuinely NEW crate version
+# touches the network.
+#
+#   seed_registry.sh seed    <registry dir> <seed dir>   cold registry <- seed. Never fails the job.
+#   seed_registry.sh publish <registry dir> <seed dir>   warm registry -> seed (atomic rename).
+#
+# Only index/ and cache/ travel (src/ is re-extracted by cargo). A registry is COLD when it holds
+# no cached .crate. seed never overwrites a warm registry. publish refuses a registry with no
+# cached crate, so an empty one cannot replace a good seed. A missing seed is a cold start, exit 0.
+set -uo pipefail
+
+crate_count() { find "$1/cache" -name '*.crate' 2>/dev/null | head -n 1000 | wc -l; }
+
+cmd_seed() {
+    local reg="$1" seed="$2"
+    if [ "$(crate_count "$reg")" -gt 0 ]; then
+        printf 'seed_registry: %s is warm; left alone\n' "$reg"; return 0
+    fi
+    if [ "$(crate_count "$seed")" -eq 0 ]; then
+        printf 'seed_registry: no seed at %s; cold start (crates.io will be read)\n' "$seed"; return 0
+    fi
+    mkdir -p "$reg" || return 0
+    local d
+    for d in index cache; do
+        [ -d "$seed/$d" ] || continue
+        cp -a --reflink=auto "$seed/$d" "$reg/" 2>/dev/null \
+            || printf 'seed_registry: warning: could not copy %s/%s\n' "$seed" "$d"
+    done
+    printf 'seed_registry: seeded %s from %s (%s crates)\n' "$reg" "$seed" "$(crate_count "$reg")"
+    return 0
+}
+
+cmd_publish() {
+    local reg="$1" seed="$2" n tmp
+    n=$(crate_count "$reg")
+    if [ "$n" -eq 0 ]; then
+        printf 'seed_registry: refusing to publish %s: no cached crates\n' "$reg" >&2; return 1
+    fi
+    if [ "$n" -lt "$(crate_count "$seed")" ]; then
+        printf 'seed_registry: %s has fewer crates than the seed; seed kept\n' "$reg"; return 0
+    fi
+    mkdir -p "$(dirname -- "$seed")" || return 1
+    tmp="${seed}.tmp.$$"
+    mkdir -p "$tmp" && cp -a --reflink=auto "$reg/index" "$reg/cache" "$tmp/" || { rm -rf "${tmp:?}"; return 1; }
+    rm -rf "${seed:?}.old" && { [ ! -d "$seed" ] || mv "$seed" "${seed}.old"; } && mv "$tmp" "$seed" || return 1
+    rm -rf "${seed:?}.old"
+    printf 'seed_registry: published %s crates to %s\n' "$n" "$seed"
+}
+
+case "${1:-}" in
+    seed)    [ "$#" -eq 3 ] || { echo 'usage: seed <registry> <seed>' >&2; exit 64; }; cmd_seed "$2" "$3" ;;
+    publish) [ "$#" -eq 3 ] || { echo 'usage: publish <registry> <seed>' >&2; exit 64; }; cmd_publish "$2" "$3" ;;
+    *) echo 'usage: seed_registry.sh seed|publish <registry> <seed>' >&2; exit 64 ;;
+esac

scripts/tests/brownout_test.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# brownout_test.sh -- falsifier for aprender#5751 (PMAT-5751): a WAN brownout must not fail a merge job.
+#   R1  a cold per-PR registry is seeded from the host-local seed: crates are on disk before cargo runs
+#   R2  a warm registry is left alone (never clobbered by the seed)
+#   R3  no seed present: exit 0, registry stays cold (a missing seed is a cold start, not a failure)
+#   R4  publish refuses a registry with no cached crates (an empty one must not poison the seed)
+#   R5  publish makes a cold registry seedable (round trip)
+#   P1  pin_build_base never touches the network when the build base is already in the checkout
+#   P2  pin_build_base still fetches when the base is absent (fallback unchanged)
+# Usage: bash scripts/tests/brownout_test.sh   exit 0 all green, 1 any red
+set -uo pipefail
+ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)"
+SEED="$ROOT/scripts/ci/seed_registry.sh"
+PIN="$ROOT/scripts/ci/pin_build_base.sh"
+TD=$(mktemp -d); trap 'rm -rf "${TD:?}"' EXIT
+export GIT_CEILING_DIRECTORIES="$TD"   # #5025: a slipped -C must fail, not write into the shared checkout
+ok=0; bad=0
+row() { if [ "$2" = "$3" ]; then ok=$((ok+1)); printf 'ok    %s\n' "$1"; else bad=$((bad+1)); printf 'RED   %s (got %s, want %s)\n' "$1" "$2" "$3"; fi; }
+[ -f "$SEED" ] || { printf 'RED   %s is missing\n' "$SEED"; bad=$((bad+1)); }
+
+mkreg() { mkdir -p "$1/cache/idx" "$1/index/idx"; printf 'crate\n' > "$1/cache/idx/linux-raw-sys-0.4.14.crate"; printf 'i\n' > "$1/index/idx/.cache"; }
+S="$TD/seed"; mkreg "$S"
+
+R1="$TD/r1"; mkdir -p "$R1"
+bash "$SEED" seed "$R1" "$S" >/dev/null 2>&1; rc=$?
+row "R1 cold registry gets the crate" "$rc $([ -f "$R1/cache/idx/linux-raw-sys-0.4.14.crate" ] && echo present || echo absent)" "0 present"
+
+R2="$TD/r2"; mkreg "$R2"; printf 'mine\n' > "$R2/cache/idx/own.crate"
+bash "$SEED" seed "$R2" "$S" >/dev/null 2>&1
+row "R2 warm registry untouched" "$([ -f "$R2/cache/idx/own.crate" ] && echo kept || echo clobbered)" kept
+
+R3="$TD/r3"; mkdir -p "$R3"
+bash "$SEED" seed "$R3" "$TD/no-such-seed" >/dev/null 2>&1; rc=$?
+row "R3 missing seed is a cold start" "$rc $(ls -A "$R3" | wc -l)" "0 0"
+
+E="$TD/empty"; mkdir -p "$E"
+bash "$SEED" publish "$E" "$TD/seed2" >/dev/null 2>&1; rc=$?
+row "R4 empty registry is not published" "$([ "$rc" -ne 0 ] && [ ! -d "$TD/seed2" ] && echo refused || echo published)" refused
+
+bash "$SEED" publish "$R2" "$TD/seed2" >/dev/null 2>&1; rc=$?
+R5="$TD/r5"; mkdir -p "$R5"; bash "$SEED" seed "$R5" "$TD/seed2" >/dev/null 2>&1
+row "R5 publish then seed round trip" "$rc $([ -f "$R5/cache/idx/own.crate" ] && echo present || echo absent)" "0 present"
+
+# --- pin_build_base: a PR merge commit whose first parent is the base, checkout deep enough to hold it.
+o="$TD/origin.git"; w="$TD/w"
+git init -q --template= --bare -b main "$o"; git init -q --template= -b main "$w"
+g() { git -C "$w" -c user.name=t -c user.email=t@example.invalid -c commit.gpgsign=false "$@"; }
+printf a > "$w/a"; g add -A; g commit -q -m c0; c0=$(g rev-parse HEAD)
+g checkout -q -b pr; printf b > "$w/b"; g add -A; g commit -q -m pr; prh=$(g rev-parse HEAD)
+g checkout -q --detach "$c0"; g merge -q --no-ff --no-edit pr; m=$(g rev-parse HEAD)
+# Network is DOWN: a fetch_p6 stand-in that fails and records that it was called.
+printf '#!/usr/bin/env bash\necho called >> "%s/net-calls"\nexit 1\n' "$TD" > "$TD/dead_fetch.sh"
+pinrun() { ( cd "$1" && GITHUB_EVENT_NAME=pull_request PR_HEAD_SHA="$prh" P6_OVERRIDE="$TD/dead_fetch.sh" bash "$PIN" ) >"$TD/pin.log" 2>&1; }
+rc=0; pinrun "$w" || rc=$?
+row "P1 base already local: no network, pinned" "$rc $(git -C "$w" rev-parse origin/main 2>/dev/null | cut -c1-7) $(ls "$TD/net-calls" 2>/dev/null | wc -l)" "0 $(printf %s "$c0" | cut -c1-7) 0"
+
+# P2: a depth-1 checkout of the merge commit holds no parent.
+d="$TD/shallow"; git init -q --template= "$d"; git -C "$d" remote add origin "file://$w"
+git -C "$d" fetch -q --depth=1 origin "$m" && git -C "$d" checkout -q --detach FETCH_HEAD
+rm -f "${TD:?}/net-calls"
+rc=0; pinrun "$d" || rc=$?
+row "P2 base absent: fetch attempted, refuses when net is down" "$rc $(ls "$TD/net-calls" 2>/dev/null | wc -l)" "1 1"
+
+printf 'brownout_test: %s ok, %s red\n' "$ok" "$bad"
+[ "$bad" -eq 0 ]