diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6440a069fe..273178a2b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -458,6 +458,8 @@ jobs: run: setsid --wait bash scripts/ci/archive_list_equal.sh --self-test - name: archive_bind_check case table must still turn RED run: setsid --wait bash scripts/ci/archive_bind_check.sh --self-test + - name: archive_fetch (LAN cache) case table must still turn RED + run: setsid --wait bash scripts/ci/archive_fetch.sh --self-test - name: Sections id: run run: >- diff --git a/ci/sections.yml b/ci/sections.yml index 9ceae5527a..b6ef39b975 100644 --- a/ci/sections.yml +++ b/ci/sections.yml @@ -939,6 +939,15 @@ jobs: # by name into the section workspace. Required in the quick tier. - uses: actions/download-artifact@v7 if: steps.tier.outputs.tier == 'quick' + with: + name: ws-archive-ptr + path: ws-archive-ptr + - name: Fetch the archive over the LAN (a miss falls back to the artifact below) + id: lan + if: steps.tier.outputs.tier == 'quick' + run: bash scripts/ci/archive_fetch.sh fetch ws-archive-ptr/ptr.json ws-archive + - uses: actions/download-artifact@v7 + if: steps.tier.outputs.tier == 'quick' && steps.lan.outputs.hit != '1' with: name: ws-archive path: ws-archive @@ -1557,6 +1566,21 @@ jobs: --arg i "$rustc_h" \ '{tier:"quick",sha:$s,expr:$e,flags:$f,rustc:$i,build_s:($b|tonumber),list_s:($l|tonumber)}' > "$out/bind.json" ls -l "$out" + # LAN cache (archive_fetch.sh): the same bytes on this host, served over http to the shards; the artifact + # below stays as the fallback. WS_ARCHIVE_CACHE_ROOT unset = disabled (pointer says so). + - name: Publish the archive to the LAN cache + env: + CACHE_ROOT: ${{ vars.WS_ARCHIVE_CACHE_ROOT }} + CACHE_PORT: ${{ vars.WS_ARCHIVE_CACHE_PORT }} + run: | + set -euo pipefail + bash scripts/ci/archive_fetch.sh publish "${CACHE_ROOT:-}" "${CACHE_PORT:-8080}" "${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" ws-archive-out ws-archive-ptr + - uses: actions/upload-artifact@v7 + with: + name: ws-archive-ptr + path: ws-archive-ptr/ + if-no-files-found: error + overwrite: true - uses: actions/upload-artifact@v7 with: name: ws-archive diff --git a/scripts/ci/archive_fetch.sh b/scripts/ci/archive_fetch.sh new file mode 100755 index 0000000000..140f78a903 --- /dev/null +++ b/scripts/ci/archive_fetch.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# archive_fetch.sh -- LAN cache for the FLOW-010 P1 nextest archive (ws-archive). Transport only. +# +# WHY. Every workspace-test-shard downloaded the ~541 MB ws-archive artifact from GitHub's blob store. Run 39595316228: +# kfocus shard 9 spent 14:19-14:43Z on it (0.4 MB/s, load 0.03) while intel shards finished in 6-7 min. +# +# publish ROOT PORT KEY OUTDIR PTRDIR archive job: copy OUTDIR/{ws.tar.zst,bind.json} to ROOT/KEY/ (atomic rename), write +# SHA256SUM there and PTRDIR/ptr.json {cache_url,sha256,key}. ROOT empty = disabled: +# ptr.json carries cache_url "" so every shard falls back. Always rc 0 on a disabled cache. +# fetch PTR.json DEST shard: curl the files from cache_url, verify sha256 of ws.tar.zst against the POINTER's +# sha256 (which travels as a GitHub artifact, not from the cache host). Prints +# `hit=1` / `hit=0 reason=...` and appends it to $GITHUB_OUTPUT. Always rc 0: a miss means +# "download the GitHub artifact", the same bytes. +# +# NOT a second verdict path: archive_bind_check.sh still runs on DEST whichever way the bytes arrived (sha, expr, flags, +# toolchain), so a stale or wrong cache file is RED exactly as before. No continue-on-error. +# Usage: bash scripts/ci/archive_fetch.sh publish|fetch ... | --self-test +set -euo pipefail + +emit() { # hit=1|hit=0 reason=... + printf 'ws-archive %s\n' "$*" + case "$1" in hit=1) h=1 ;; *) h=0 ;; esac + [ -z "${GITHUB_OUTPUT:-}" ] || printf 'hit=%s\n' "$h" >> "$GITHUB_OUTPUT" +} + +publish() { + local root="$1" port="$2" key="$3" out="$4" ptr="$5" tmp sum host + mkdir -p "$ptr" + if [ -z "$root" ]; then + jq -n '{cache_url:"",sha256:"",key:""}' > "$ptr/ptr.json"; echo "ws-archive publish: WS_ARCHIVE_CACHE_ROOT unset, cache disabled"; return 0 + fi + mkdir -p "$root" + tmp="$root/.tmp-$key-$$"; rm -rf "${tmp:?}"; mkdir -p "$tmp" + cp "$out/ws.tar.zst" "$out/bind.json" "$tmp/" + sum="$(sha256sum "$tmp/ws.tar.zst" | cut -d' ' -f1)" + printf '%s ws.tar.zst\n' "$sum" > "$tmp/SHA256SUM" + rm -rf "${root:?}/$key"; mv "$tmp" "$root/$key" + host="$(hostname -f 2> /dev/null || hostname)" + jq -n --arg u "http://$host:$port/$key" --arg s "$sum" --arg k "$key" '{cache_url:$u,sha256:$s,key:$k}' > "$ptr/ptr.json" + echo "ws-archive publish: $root/$key sha256=$sum" +} + +fetch() { + local ptr="$1" dest="$2" url sum got t0 + [ -f "$ptr" ] || { emit hit=0 reason=no-pointer; return 0; } + url="$(jq -r '.cache_url // ""' "$ptr")"; sum="$(jq -r '.sha256 // ""' "$ptr")" + [ -n "$url" ] && [ -n "$sum" ] || { emit hit=0 reason=cache-disabled; return 0; } + mkdir -p "$dest"; t0="$(date +%s)" + if ! curl -fsS --connect-timeout 5 --max-time "${WS_ARCHIVE_FETCH_MAX_S:-300}" -o "$dest/ws.tar.zst" "$url/ws.tar.zst" 2> "$dest/.curl.err" \ + || ! curl -fsS --connect-timeout 5 --max-time 60 -o "$dest/bind.json" "$url/bind.json" 2>> "$dest/.curl.err"; then + emit hit=0 "reason=unreachable url=$url"; rm -f "$dest/ws.tar.zst" "$dest/bind.json" "$dest/.curl.err"; return 0 + fi + rm -f "$dest/.curl.err" + if [ "${AF_MUTANT:-}" != nosum ]; then # test-only mutant: skip the checksum + got="$(sha256sum "$dest/ws.tar.zst" | cut -d' ' -f1)" + if [ "$got" != "$sum" ]; then emit hit=0 "reason=sha256-mismatch url=$url"; rm -f "$dest/ws.tar.zst" "$dest/bind.json"; return 0; fi + fi + emit hit=1 "source=lan url=$url seconds=$(( $(date +%s) - t0 ))" +} + +self_test() { + local td fails=0 out + td="$(mktemp -d)"; trap 'rm -rf "${td:?}"' RETURN + mkdir -p "$td/out"; head -c 100000 /dev/urandom > "$td/out/ws.tar.zst"; echo '{"tier":"quick"}' > "$td/out/bind.json" + want() { # LABEL EXPECT(hit=1|hit=0) -- reads `fetch` output on stdin + local o; o="$(cat)" + if printf '%s' "$o" | /usr/bin/grep -q "^ws-archive $2"; then echo "ok $1"; else echo "FAIL $1: wanted $2, got: $o"; fails=1; fi + } + publish "$td/cache" 8080 K1 "$td/out" "$td/ptr" > /dev/null + [ "$(jq -r .sha256 "$td/ptr/ptr.json")" = "$(sha256sum "$td/out/ws.tar.zst" | cut -d' ' -f1)" ] && echo "ok publish writes the archive's sha256 into the pointer" || { echo "FAIL pointer sha256"; fails=1; } + # file:// stands in for http://host:port; curl treats both the same way + jq --arg u "file://$td/cache/K1" '.cache_url=$u' "$td/ptr/ptr.json" > "$td/ptr/good.json" + fetch "$td/ptr/good.json" "$td/d1" | want "hit: intact cache file verifies" "hit=1" + cmp -s "$td/d1/ws.tar.zst" "$td/out/ws.tar.zst" && echo "ok hit: bytes identical to the archive" || { echo "FAIL hit bytes differ"; fails=1; } + printf 'X' | dd of="$td/cache/K1/ws.tar.zst" bs=1 seek=500 conv=notrunc 2> /dev/null + fetch "$td/ptr/good.json" "$td/d2" | want "miss: one corrupted byte is a sha256 mismatch" "hit=0 reason=sha256-mismatch" + [ ! -e "$td/d2/ws.tar.zst" ] && echo "ok miss: the corrupt file is removed" || { echo "FAIL corrupt file kept"; fails=1; } + jq '.cache_url="http://127.0.0.1:1/x"' "$td/ptr/good.json" > "$td/ptr/dead.json" + fetch "$td/ptr/dead.json" "$td/d3" | want "miss: unreachable host" "hit=0 reason=unreachable" + publish "$td/cache" 8080 K1 "$td/out" "$td/ptr" > /dev/null; jq --arg u "file://$td/cache/K1" '.cache_url=$u|.sha256="00"' "$td/ptr/ptr.json" > "$td/ptr/wrong.json" + fetch "$td/ptr/wrong.json" "$td/d4" | want "miss: pointer sha256 differs from the cache file" "hit=0 reason=sha256-mismatch" + publish "" 8080 K1 "$td/out" "$td/ptr0" > /dev/null + fetch "$td/ptr0/ptr.json" "$td/d5" | want "miss: cache disabled (var unset)" "hit=0 reason=cache-disabled" + fetch "$td/nope.json" "$td/d6" | want "miss: no pointer" "hit=0 reason=no-pointer" + # MUTANT: with the checksum skipped, the corrupt case must NOT be a clean miss -> this self-test goes RED + if [ -z "${AF_MUTANT:-}" ]; then + if AF_MUTANT=nosum bash "$0" --self-test > /dev/null 2>&1; then echo "FAIL mutant: --self-test stayed green with the checksum skipped"; fails=1; else echo "ok red MUTANT: skipping the checksum turns --self-test RED"; fi + fi + return "$fails" +} + +case "${1:-}" in + publish) [ "$#" -eq 6 ] || { echo "usage: $0 publish ROOT PORT KEY OUTDIR PTRDIR" >&2; exit 2; }; publish "$2" "$3" "$4" "$5" "$6" ;; + fetch) [ "$#" -eq 3 ] || { echo "usage: $0 fetch PTR.json DEST" >&2; exit 2; }; fetch "$2" "$3" ;; + --self-test) self_test; exit $? ;; + *) echo "usage: $0 publish|fetch|--self-test" >&2; exit 2 ;; +esac