From 2e082ae7914a6cad52ce81fb48df0ffeb1a12c90 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 8 Oct 2026 16:24:22 +0200 Subject: [PATCH 01/11] ci(x86-main): fence the fat_driver case table so planted rows raise no annotations The "external job 'workspace-test': not completed within 12000s" error on x86-main never came from a wait. x86-main runs no --external-job. The line, and "section a: failure" / "section a: cancelled" beside it, are ::error:: workflow commands printed by planted rows of `fat_driver.py self-test`, which passed 60/60 about a minute into the job. GitHub turned them into job error annotations. When the job went red later on another section (#4813 pr-review-sign, #4913 guard-tree at its 3600 s budget), the annotation was read as the cause. scripts/ci/fence_workflow_commands.sh runs a case table between `::stop-commands::` and `::::` (fresh token, never exported, stderr merged into the one ordered pipe). The step's verdict is the table's rc, and a failing table gets one live ::error:: naming it. Both fat_driver self-test steps (x86-main, workspace-test-shard) go through it. `fat_driver.py run`/`wait` stay unfenced, so a real deadline still annotates. The script's wiring check is RED on an unfenced self-test step and on a fenced real run. It runs, with its 19-row case table, in x86-main's cheap-checks step. No gate loosened: same commands, same exit codes, one guard added. Refs #4936 Agent: aprender-4936 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 8 +- docs/roadmaps/entries/PMAT-4936.yaml | 18 +++ docs/roadmaps/roadmap.yaml | 18 +++ scripts/ci/fence_workflow_commands.sh | 200 ++++++++++++++++++++++++++ 4 files changed, 242 insertions(+), 2 deletions(-) create mode 100644 docs/roadmaps/entries/PMAT-4936.yaml create mode 100644 scripts/ci/fence_workflow_commands.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index db2067e8ab..c0632a0d64 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -204,11 +204,15 @@ jobs: setsid --wait bash scripts/check_no_claim_literals.sh --selftest setsid --wait bash scripts/check_hardcoded_paths.sh --self-test setsid --wait bash scripts/check_pr_review_wiring.sh --self-test + setsid --wait bash scripts/ci/fence_workflow_commands.sh --self-test + setsid --wait bash scripts/ci/fence_workflow_commands.sh # The driver decides what every section's if:/matrix/needs means and whether # the vendored workflow is the upstream one; its case table must go RED on # each planted defect before it is trusted to report a section green. + # #4936: fenced, so the planted rows' ::error:: lines ("external job + # 'workspace-test': not completed within 12000s") are not job annotations. - name: fat_driver case table must still turn RED - run: python3 scripts/ci/fat_driver.py self-test + run: bash scripts/ci/fence_workflow_commands.sh -- python3 scripts/ci/fat_driver.py self-test # PMAT-4510: every secrets. a section reads must be plumbed above as # FAT_SECRET_ under its full name; #4441 cut one short and the # receipt signer ran with an empty key on every PR. Text-only, no build. @@ -500,7 +504,7 @@ jobs: with: fetch-depth: 0 - name: fat_driver case table must still turn RED - run: python3 scripts/ci/fat_driver.py self-test + run: bash scripts/ci/fence_workflow_commands.sh -- python3 scripts/ci/fat_driver.py self-test # `?` for the brackets and the comma: --sections splits on commas. - name: Sections id: run diff --git a/docs/roadmaps/entries/PMAT-4936.yaml b/docs/roadmaps/entries/PMAT-4936.yaml new file mode 100644 index 0000000000..852b202067 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4936.yaml @@ -0,0 +1,18 @@ +- id: PMAT-4936 + github_issue: 4936 + item_type: task + title: 'x86-main: fat_driver self-test prints live ::error:: annotations (false 12000s red)' + status: inprogress + priority: high + assigned_to: null + created: 2026-10-08T14:17:11Z + updated: 2026-10-08T14:17:16.352731207+00:00 + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 3998 diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index 568507b532..c54e4f88a2 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -24294,3 +24294,21 @@ Not done, pending the ruling: moving the 31 fixture files to an obviously synthe labels: - kind:code notes: 'Cause: the test writes a script then execs it while sibling test threads fork; a child forked in that window holds the write fd, so exec fails with ETXTBSY (run 37403017673, workspace-test shard 3). Passed on the same PR head in run 37408602581.' +- id: PMAT-4936 + github_issue: 4936 + item_type: task + title: 'x86-main: fat_driver self-test prints live ::error:: annotations (false 12000s red)' + status: inprogress + priority: high + assigned_to: null + created: 2026-10-08T14:17:11Z + updated: 2026-10-08T14:17:16.352731207+00:00 + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 3998 diff --git a/scripts/ci/fence_workflow_commands.sh b/scripts/ci/fence_workflow_commands.sh new file mode 100644 index 0000000000..d68a67a536 --- /dev/null +++ b/scripts/ci/fence_workflow_commands.sh @@ -0,0 +1,200 @@ +#!/usr/bin/env bash +# fence_workflow_commands.sh -- run a case table with GitHub's workflow-command +# processing stopped for its output, so a planted row cannot raise a live +# annotation on the job (#4936). +# +# WHY THIS EXISTS +# --------------- +# `fat_driver.py self-test` drives the real code paths with planted failures, +# and those paths print real `::error::` workflow commands: "section a: +# failure", "section a: cancelled" and "external job 'workspace-test': not +# completed within 12000s". The table passed 60/60, yet every x86-main and +# workspace-test shard job carried those three as error annotations about a +# minute in. x86-main runs no --external-job at all, so its 12000s line could +# only ever come from the table; when the job went red half an hour later on +# another section, the annotation was read as the cause (#4813, #4913). +# +# The fence is GitHub's own `::stop-commands::` ... `::::`. The +# token is drawn fresh per call and never exported, so the child cannot end +# the fence early. The child's stderr is merged into stdout so every line +# crosses one pipe in order and none lands after the resume line. The verdict +# is untouched: the child's exit status is the step's, and a failing table gets +# one live ::error:: naming it, after the fence. +# +# Only case tables are fenced. `fat_driver.py run` and `wait` stay unfenced: +# their deadline ::error:: is the real one, and the wiring check refuses a +# fenced one. +# +# bash scripts/ci/fence_workflow_commands.sh -- python3 scripts/ci/fat_driver.py self-test +# bash scripts/ci/fence_workflow_commands.sh # wiring check of ci.yml +# bash scripts/ci/fence_workflow_commands.sh --self-test +# bash scripts/ci/fence_workflow_commands.sh --live # live annotation count +set -uo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +WORKFLOW_REL=".github/workflows/ci.yml" +FENCE_REL="scripts/ci/fence_workflow_commands.sh" + +# fenced -> the child's output between a stop line and its resume +# line. rc = the child's rc; 2 when no command is given or no token is drawn. +fenced() { + if [ "$#" -eq 0 ]; then + printf '::error::%s: no command to fence\n' "$FENCE_REL" + return 2 + fi + local tok rc + tok="$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n')" + if [ "${#tok}" -ne 32 ]; then + printf '::error::%s: no stop-commands token drawn\n' "$FENCE_REL" + return 2 + fi + printf '::stop-commands::%s\n' "$tok" + "$@" 2>&1 + rc=$? + printf '::%s::\n' "$tok" + if [ "$rc" -ne 0 ]; then + printf '::error::case table failed (rc=%s): %s\n' "$rc" "$*" + fi + return "$rc" +} + +# live_commands -> how many annotation commands the runner would act on: +# ::error/::warning/::notice lines outside every stop..resume region. +live_commands() { + awk ' + stop != "" { if ($0 == "::" stop "::") stop = ""; next } + /^::stop-commands::/ { stop = substr($0, 18); next } + /^::(error|warning|notice)( [^:]*)?::/ { n++ } + END { print n + 0 }' "$1" +} + +# check_wiring -> one FAIL line per defect. rc 0 = every +# `fat_driver.py self-test` runs through the fence and no `fat_driver.py run` +# or `wait` does; 1 = a defect; 2 = unreadable, or no self-test step found. +check_wiring() { + [ -r "$1" ] || return 2 + local bad=0 seen=0 line + while IFS= read -r line; do + seen=$((seen + 1)) + case "$line" in + *"$FENCE_REL -- python3 scripts/ci/fat_driver.py self-test"*) ;; + *) + printf 'FAIL: %s runs the fat_driver case table unfenced -- its planted rows raise live ::error:: annotations: %s\n' \ + "$WORKFLOW_REL" "$(printf '%s' "$line" | sed -E 's/^[[:space:]]+//')" + bad=1 ;; + esac + done < <(grep -E 'fat_driver\.py self-test' "$1") + while IFS= read -r line; do + printf 'FAIL: %s fences a real fat_driver run -- its deadline ::error:: must stay live: %s\n' \ + "$WORKFLOW_REL" "$(printf '%s' "$line" | sed -E 's/^[[:space:]]+//')" + bad=1 + done < <(grep -E "$FENCE_REL.*fat_driver\\.py (run|wait)" "$1") + [ "$seen" -gt 0 ] || return 2 + return "$bad" +} + +self_test() { + printf '=== case table: fence_workflow_commands.sh ===\n' + local tmp fails=0 rc t1 t2 good + tmp="$(mktemp -d)" + trap 'rm -rf "${tmp:?}"' RETURN + row() { # row