diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de2f17321f..7e1453b166 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -205,6 +205,8 @@ jobs: setsid --wait bash scripts/check_hardcoded_paths.sh --self-test setsid --wait bash scripts/check_pr_review_wiring.sh --self-test setsid --wait bash scripts/ci/fixture_quiet.sh --self-test + setsid --wait bash scripts/ci/fetch_p6.sh --self-test + setsid --wait bash scripts/ci/fetch_p6.sh --check-wiring # The driver decides what every section's if:/matrix/needs means and whether # the vendored workflow is the upstream one; its case table must go RED on # each planted defect before it is trusted to report a section green. diff --git a/ci/sections.yml b/ci/sections.yml index 1dfd1089dc..4a27927189 100644 --- a/ci/sections.yml +++ b/ci/sections.yml @@ -370,8 +370,8 @@ jobs: # warning below says so instead of the run silently paying the hour. case "${GITHUB_EVENT_NAME}" in merge_group|push) - git fetch --no-tags --depth=2 origin "${GITHUB_SHA}" 2>/dev/null \ - || git fetch --no-tags --deepen=1 origin 2>/dev/null \ + bash scripts/ci/fetch_p6.sh --no-tags --depth=2 origin "${GITHUB_SHA}" 2>/dev/null \ + || bash scripts/ci/fetch_p6.sh --no-tags --deepen=1 origin 2>/dev/null \ || true if git rev-parse -q --verify 'HEAD^1' >/dev/null 2>&1; then printf 'first parent available: %s\n' "$(git rev-parse HEAD^1)" @@ -382,7 +382,7 @@ jobs: esac case "${GITHUB_EVENT_NAME}" in pull_request) - git fetch --no-tags --depth=1 origin "+refs/heads/${GITHUB_BASE_REF}:refs/remotes/origin/${GITHUB_BASE_REF}" + bash scripts/ci/fetch_p6.sh --no-tags --depth=1 origin "+refs/heads/${GITHUB_BASE_REF}:refs/remotes/origin/${GITHUB_BASE_REF}" # --no-renames (#3664): a rename must list BOTH paths, or a source # moved into docs/ reads as docs-only and runs zero tests. git diff --no-renames --name-only "origin/${GITHUB_BASE_REF}" HEAD > "$RUNNER_TEMP/touched.txt" @@ -393,7 +393,7 @@ jobs: pr=$(printf '%s' "${GITHUB_REF_NAME}" | sed -n 's|^gh-readonly-queue/[^/]*/pr-\([0-9]*\)-.*|\1|p') if [ -n "$pr" ]; then head=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${pr}" --jq .head.sha) - git fetch --no-tags --depth=1 origin "$head" + bash scripts/ci/fetch_p6.sh --no-tags --depth=1 origin "$head" concl=$(gh api "repos/${GITHUB_REPOSITORY}/commits/${head}/check-runs?check_name=workspace-test&per_page=20" --jq '[.check_runs[]|select(.status=="completed")]|sort_by(.completed_at)|last|.conclusion // "none"') args+=(--pr-head "$head" --pr-head-conclusion "${concl:-none}") fi @@ -439,7 +439,7 @@ jobs: merge_group) base="${MG_BASE_SHA:-}" ;; push) base="${PUSH_BEFORE:-}" ;; workflow_dispatch) - git fetch --no-tags --depth=1 origin +refs/heads/main:refs/remotes/origin/main + bash scripts/ci/fetch_p6.sh --no-tags --depth=1 origin +refs/heads/main:refs/remotes/origin/main printf 'refinement base: origin/main tip %s (workflow_dispatch has no event base)\n' "$(git rev-parse origin/main)" exit 0 ;; @@ -449,7 +449,7 @@ jobs: printf '::error::refinement base: %s event carries no base sha (got "%s")\n' "${GITHUB_EVENT_NAME}" "$base" exit 1 fi - git fetch --no-tags --depth=1 origin "$base" + bash scripts/ci/fetch_p6.sh --no-tags --depth=1 origin "$base" git update-ref refs/remotes/origin/main "$base" printf 'refinement base: %s base %s pinned as origin/main\n' "${GITHUB_EVENT_NAME}" "$(git rev-parse 'origin/main^{commit}')" - name: Workspace lib tests (25,300+) @@ -3054,7 +3054,7 @@ jobs: # PR's diff, which is the safe direction for a selector — it can only # run the GPU jobs when they were not needed, never skip them when # they were. - git fetch --no-tags --depth=1 origin \ + bash scripts/ci/fetch_p6.sh --no-tags --depth=1 origin \ "+refs/heads/${GITHUB_BASE_REF}:refs/remotes/origin/${GITHUB_BASE_REF}" git diff --name-only "origin/${GITHUB_BASE_REF}" HEAD > "$RUNNER_TEMP/touched.txt" printf 'touched (tip vs tip): %s path(s)\n' \ diff --git a/docs/roadmaps/entries/PMAT-4936.yaml b/docs/roadmaps/entries/PMAT-4936.yaml new file mode 100644 index 0000000000..564c1640c5 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4936.yaml @@ -0,0 +1,18 @@ +- id: PMAT-4936 + github_issue: 4936 + item_type: task + title: 'x86-main: fence the fat_driver case-table annotations; section git fetch reads up to 3 times (P6)' + status: inprogress + priority: high + assigned_to: null + created: 2026-10-08T14:17:11Z + updated: 2026-10-08T14:17:16.352731207+00:00 + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 3998 diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index d1b846cf90..e545faf47d 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -24553,6 +24553,24 @@ Not done, pending the ruling: moving the 31 fixture files to an obviously synthe - kind:code notes: null epic: 3998 +- id: PMAT-4936 + github_issue: 4936 + item_type: task + title: 'x86-main: fence the fat_driver case-table annotations; section git fetch reads up to 3 times (P6)' + status: inprogress + priority: high + assigned_to: null + created: 2026-10-08T14:17:11Z + updated: 2026-10-08T14:17:16.352731207+00:00 + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 3998 - id: PMAT-4948 github_issue: 4948 item_type: task diff --git a/scripts/check_roadmap_diff_additive.sh b/scripts/check_roadmap_diff_additive.sh index 67c03af079..0434184309 100755 --- a/scripts/check_roadmap_diff_additive.sh +++ b/scripts/check_roadmap_diff_additive.sh @@ -419,6 +419,50 @@ EOF *"is not a merge commit nor a commit on the origin/main tip"*) printf 'ok row %-2s stacked entry with deepening disabled (mutation): refused by name\n' "$row" ;; *) printf 'FAIL row %-2s stacked entry with deepening disabled: refused for the wrong reason (rc=%s): %s\n' "$row" "$rc7" "$err7"; fails=1 ;; esac + # #4936 (P6): the same stacked entry with the deepen's first read dead (curl 92). fetch_p6.sh reads the + # deepen again and the base still resolves. A planted git logs every fetch read; both must be the + # deepen, so a deepen that is one bare read (it dies, and the by-sha fetch after it answers) is RED. + # It runs from a cwd outside REPO_ROOT (the shim's own dir), as check_ont_ratchet.sh may (it never cds), + # and git finds no repository above that cwd: a deepen read that lost its -C "$REPO_ROOT" fails, the row is + # RED. The deepen answers on its second read, so the by-sha read never runs here; the next row covers it. + row=$((row + 1)) + rm -rf "${Q:?}.clone"; git clone -q --depth=1 -b main "file://$Q" "$Q.clone" 2>/dev/null; git -C "$Q.clone" fetch -q --depth=1 origin '+queue-base:refs/remotes/origin/main' 2>/dev/null + S="$TD/p6-shim"; rm -rf "${S:?}"; mkdir -p "$S" + printf '%s\n' '#!/usr/bin/env bash' \ + 'case " $* " in *" fetch "*)' \ + ' printf "%s\n" "$*" >> "$P6_SHIM_LOG"' \ + ' if [ "$(wc -l < "$P6_SHIM_LOG")" -eq 1 ]; then' \ + " printf '%s\\n' 'error: RPC failed; curl 92 HTTP/2 stream 5 was not closed cleanly: CANCEL (err 8)' 'fatal: early EOF' >&2; exit 128" \ + ' fi ;;' \ + 'esac' \ + 'exec "$P6_SHIM_REAL" "$@"' > "$S/git" + printf '%s\n' '#!/usr/bin/env bash' 'exit 0' > "$S/sleep"; chmod +x "$S/git" "$S/sleep"; : > "$S/reads" + got8=$( PATH="$S:$PATH" P6_SHIM_REAL="$(command -v git)" P6_SHIM_LOG="$S/reads" GITHUB_EVENT_NAME=merge_group GIT_CEILING_DIRECTORIES="$TD" bash -c 'cd "$2" || exit 2; if git rev-parse --git-dir >/dev/null 2>&1; then printf "cwd is a repo|"; else printf "no repo|"; fi; . "$0" --lib-only; REPO_ROOT="$1"; resolve_base HEAD && printf "%s|%s" "$BASE_REF" "$BASE_HOW"' "$SELF" "$Q.clone" "$S" 2>/dev/null ) || true + reads8="$(grep -c -e '--deepen=1' "$S/reads") of $(wc -l < "$S/reads" | tr -d ' ')" + case "$reads8|$got8" in "2 of 2|no repo|$want6|single parent (stacked merge_group entry"*) printf 'ok row %-2s stacked entry, the deepen read dead once (curl 92), from a cwd that is no repo: read again under P6 in REPO_ROOT, base = the previous entry squash\n' "$row" ;; + *) printf 'FAIL row %-2s stacked entry, deepen read dead once: wanted 2 of 2 deepen reads, a cwd that is no repo and base %s; got %s deepen reads, %s\n' "$row" "$want6" "$reads8" "$got8"; fails=1 ;; esac + # #4936 (P6): the same entry with all 3 deepen reads dead, then the by-sha read dead once. P6 reads the by-sha + # fetch again and the base still resolves. Same cwd and ceiling as the row above: a by-sha read that lost its + # -C "$REPO_ROOT" runs in no repo, and one that skips P6 is never read again; either way the row is RED. + # Wanted: 3 deepen reads, then 2 by-sha reads. + row=$((row + 1)) + rm -rf "${Q:?}.clone"; git clone -q --depth=1 -b main "file://$Q" "$Q.clone" 2>/dev/null; git -C "$Q.clone" fetch -q --depth=1 origin '+queue-base:refs/remotes/origin/main' 2>/dev/null + printf '%s\n' '#!/usr/bin/env bash' \ + 'case " $* " in *" fetch "*)' \ + ' printf "%s\n" "$*" >> "$P6_SHIM_LOG"' \ + ' dead=; case " $* " in *" --deepen=1 "*) dead=1 ;; *) [ "$(grep -vc -e --deepen=1 "$P6_SHIM_LOG")" -eq 1 ] && dead=1 ;; esac' \ + ' if [ -n "$dead" ]; then' \ + " printf '%s\\n' 'error: RPC failed; curl 92 HTTP/2 stream 5 was not closed cleanly: CANCEL (err 8)' 'fatal: early EOF' >&2; exit 128" \ + ' fi ;;' \ + 'esac' \ + 'exec "$P6_SHIM_REAL" "$@"' > "$S/git" + : > "$S/reads" + got9=$( PATH="$S:$PATH" P6_SHIM_REAL="$(command -v git)" P6_SHIM_LOG="$S/reads" GITHUB_EVENT_NAME=merge_group GIT_CEILING_DIRECTORIES="$TD" bash -c 'cd "$2" || exit 2; if git rev-parse --git-dir >/dev/null 2>&1; then printf "cwd is a repo|"; else printf "no repo|"; fi; . "$0" --lib-only; REPO_ROOT="$1"; resolve_base HEAD && printf "%s|%s" "$BASE_REF" "$BASE_HOW"' "$SELF" "$Q.clone" "$S" 2>/dev/null ) || true + reads9="$(grep -c -e '--deepen=1' "$S/reads") of $(wc -l < "$S/reads" | tr -d ' ')" + case "$reads9|$got9" in "3 of 5|no repo|$want6|single parent (stacked merge_group entry"*) printf 'ok row %-2s stacked entry, all 3 deepen reads dead and the by-sha read dead once, from a cwd that is no repo: by-sha read again under P6 in REPO_ROOT, base = the previous entry squash\n' "$row" ;; + *) printf 'FAIL row %-2s stacked entry, all 3 deepen reads dead and the by-sha read dead once: wanted 3 of 5 deepen reads, a cwd that is no repo and base %s; got %s deepen reads, %s\n' "$row" "$want6" "$reads9" "$got9"; fails=1 ;; esac + rm -rf "${S:?}" + rm -rf "${Q:?}" "${Q:?}.clone" # Rows 19-22: --staged, the pre-commit entry point (B4, #3047). A scratch diff --git a/scripts/ci/fetch_p6.sh b/scripts/ci/fetch_p6.sh new file mode 100644 index 0000000000..73c2f38ff3 --- /dev/null +++ b/scripts/ci/fetch_p6.sh @@ -0,0 +1,309 @@ +#!/usr/bin/env bash +# fetch_p6.sh -- `git fetch` for the CI sections, read up to 3 times under P6 +# (#4936, operator C343 #9). +# +# WHY THIS EXISTS +# --------------- +# Shard, determinism and x86-main sections died on `git fetch` with +# "RPC failed; curl 92 HTTP/2 stream 5 was not closed cleanly: CANCEL (err 8)" +# or "curl 56 Recv failure: Connection reset by peer", on every host alike. +# The tier step's depth-1 fetch of main ran 376 s and died, although main's +# objects were already in the section clone: the read did not answer, it was +# not too big. Every section fetch was a single read, so one stalled +# connection was a red section. +# +# P6: a read that did not answer (non-zero exit, HTTP 5xx, timeout, empty +# body) is read at most 3 times in all. A read that answered is never re-read: +# a missing ref, an unadvertised object, a 401/403/404 and a usage error are +# answers. Each read that did not answer prints one status line. A read that +# stalls is cut (under LOW_SPEED_LIMIT bytes/s for LOW_SPEED_TIME s), so "did +# not answer" never hangs a section. +# +# bash scripts/ci/fetch_p6.sh # in place of `git fetch` +# bash scripts/ci/fetch_p6.sh -C # in place of `git -C fetch` +# bash scripts/ci/fetch_p6.sh --check-wiring # no bare `git fetch` in ci/sections.yml +# bash scripts/ci/fetch_p6.sh --self-test +set -uo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +SECTIONS_REL="ci/sections.yml" +HELPER_REL="scripts/ci/fetch_p6.sh" +P6_READS=3 +LOW_SPEED_LIMIT=1000 +LOW_SPEED_TIME=60 +# Seconds before re-read n is (n * SLEEP_S). The case table plants a `sleep` +# that records its argument and returns at once. +SLEEP_S=10 + +# answered -> 0 when the read answered and a re-read cannot +# change the answer. git runs under LC_ALL=C, so these are git's own texts. +# rc 129 is git's usage error: the read never left this host. +# "could not read Username" is a 401 with no credentials to offer. +answered() { + [ "$1" -eq 129 ] && return 0 + grep -qiE "couldn't find remote ref|not our ref|unadvertised object|Repository not found|does not appear to be a git repository|not a git repository|Authentication failed|could not read Username|The requested URL returned error: 40[134]" "$2" +} + +# p6_fetch [-C ] -> git fetch (in when -C names one), +# read up to P6_READS times. -C with no dir, or an empty one, is a usage error (rc 129): an +# answer. `git -C ""` keeps the cwd, so a caller's unset dir would read in whatever repo the cwd is. +# rc = the last read's rc. git's stderr reaches the caller's stderr. +p6_fetch() { + local err n rc why at=() + if [ "${1:-}" = -C ]; then + if [ -z "${2:-}" ]; then printf 'fetch_p6: -C needs a directory\n' >&2; return 129; fi + at=(-C "$2"); shift 2 + fi + err="$(mktemp)" || return 2 + for ((n = 1; n <= P6_READS; n++)); do + LC_ALL=C git "${at[@]}" -c "http.lowSpeedLimit=$LOW_SPEED_LIMIT" -c "http.lowSpeedTime=$LOW_SPEED_TIME" \ + fetch "$@" 2> "$err" + rc=$? + cat "$err" >&2 + if [ "$rc" -eq 0 ] || answered "$rc" "$err"; then + rm -f -- "${err:?}" + return "$rc" + fi + why="$(grep -E '^(error|fatal):' "$err" | head -n 1)" + if [ "$n" -lt "$P6_READS" ]; then + printf 'fetch_p6: read %d/%d did not answer (rc=%d: %s); reading again\n' \ + "$n" "$P6_READS" "$rc" "${why:-no error line}" + sleep "$((n * SLEEP_S))" + else + printf 'fetch_p6: read %d/%d did not answer (rc=%d: %s); no answer under P6\n' \ + "$n" "$P6_READS" "$rc" "${why:-no error line}" + fi + done + rm -f -- "${err:?}" + return "$rc" +} + +# A git command whose subcommand is fetch. git's global options may sit +# between them (-C , -c , --git-dir=, --no-pager), quoted or not, +# and any run of blanks separates words. The self-test's must-match and +# must-not-match forms are this pattern's case table. +FETCH_RE='(^|[^[:alnum:]_./-])"?git"?([[:space:]]+(-[cC][[:space:]]+("[^"]*"|[^[:space:]]+)|--?[^[:space:]]+))*[[:space:]]+fetch([[:space:];&|)]|$)' + +# check_wiring -> one FAIL line per bare `git fetch` outside a +# comment. rc 0 = every fetch goes through the helper; 1 = a bare fetch; +# 2 = unreadable, or no helper call found (a check that read nothing). +check_wiring() { + [ -r "$1" ] || return 2 + local bad=0 line + while IFS= read -r line; do + printf 'FAIL: %s fetches without P6 re-reads -- use bash %s: %s\n' \ + "$SECTIONS_REL" "$HELPER_REL" "$line" + bad=1 + done < <(grep -nE "$FETCH_RE" "$1" | grep -vE '^[0-9]+:[[:space:]]*#') + grep -qF "bash $HELPER_REL " "$1" || return 2 + return "$bad" +} + +self_test() { + printf '=== case table: fetch_p6.sh ===\n' + local tmp fails=0 rc real_git + tmp="$(mktemp -d)" || return 1 + trap 'rm -rf "${tmp:?}"' RETURN + # #4936: git finds no repository above tmp, so a read that lost its -C never reaches a checkout. + export GIT_CEILING_DIRECTORIES="$tmp" + real_git="$(command -v git)" + row() { # row