ci: merge-group fail-fast cancel + per-step budgets (refs #5394)

.github/workflows/ci.yml

@@ -158,7 +158,7 @@ jobs:
     if: ${{ !cancelled() && needs.mg-reuse.outputs.x86 != '1' }}
     # pull_request/push stay on the shared clean-room pool; a merge_group entry takes the mq lane (custom label exactly mq)
     runs-on: ${{ github.event_name == 'merge_group' && fromJSON('["self-hosted","Linux","X64","mq"]') || fromJSON('["self-hosted","Linux","X64","clean-room"]') }}
-    timeout-minutes: 240
+    timeout-minutes: ${{ github.event_name == 'merge_group' && 75 || 240 }}
     # The sections carry their own permissions blocks (the sov provenance section
     # needs id-token + attestations); one job token must cover all of them.
     permissions: write-all
@@ -264,6 +264,14 @@ jobs:
         # the teardown below keeps always() and still reaps the containers.
         if: ${{ !cancelled() }}
         run: python3 scripts/ci/fat_driver.py wait --results "$RUNNER_TEMP/fat/results.json"
+      # Operator 2026-10-19: root cause is no gate time budget. A merge_group entry that is already RED must
+      # not keep the other jobs and shards holding mq runners for another ~45 min: the first failed job cancels
+      # the run. merge_group only (a PR run keeps every verdict); the teardown below is always() and still reaps.
+      - name: Fail fast, cancel this merge-group run
+        if: ${{ failure() && github.event_name == 'merge_group' }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: gh run cancel "$GITHUB_RUN_ID" --repo "$GITHUB_REPOSITORY" || echo "::warning::fail-fast cancel did not take"
       # #4948: a cancelled run never reaches a section's own container removal,
       # and its fat-<run>-<section> container then sits on `tail -f /dev/null`
       # for days. always() runs on cancel too; on the passing path the driver
@@ -442,7 +450,10 @@ jobs:
   workspace-test-shard:
     # same lanes as x86-main: clean-room for pull_request/push, mq for a merge_group entry
     runs-on: ${{ github.event_name == 'merge_group' && fromJSON('["self-hosted","Linux","X64","mq"]') || fromJSON('["self-hosted","Linux","X64","clean-room"]') }}
-    timeout-minutes: 180
+    timeout-minutes: ${{ github.event_name == 'merge_group' && 75 || 180 }}
+    permissions:
+      contents: read
+      actions: write   # the fail-fast step cancels this run
     strategy:
       fail-fast: false
       matrix:
@@ -466,6 +477,14 @@ jobs:
           python3 scripts/ci/fat_driver.py run
           --sections 'workspace-test-shard?${{ matrix.shard }}?3?'
           --results "$RUNNER_TEMP/fat/results.json"
+      # Operator 2026-10-19: root cause is no gate time budget. A merge_group entry that is already RED must
+      # not keep the other jobs and shards holding mq runners for another ~45 min: the first failed job cancels
+      # the run. merge_group only (a PR run keeps every verdict); the teardown below is always() and still reaps.
+      - name: Fail fast, cancel this merge-group run
+        if: ${{ failure() && github.event_name == 'merge_group' }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: gh run cancel "$GITHUB_RUN_ID" --repo "$GITHUB_REPOSITORY" || echo "::warning::fail-fast cancel did not take"
       # #4948: a cancelled run never reaches a section's own container removal,
       # and its fat-<run>-<section> container then sits on `tail -f /dev/null`
       # for days. always() runs on cancel too; on the passing path the driver

ci/step-budgets.tsv

@@ -0,0 +1,6 @@
+# job<TAB>step<TAB>budget_seconds. A step over 1.5x its budget is flagged by scripts/ci/step_budget_check.sh.
+# Seeded 2026-10-19 from MG run 39205769238 (shard norm ~24 min) and the guard-cargo green time after #5712 (~45 min x86-main). The owner tunes them from data.
+x86-main	Sections (wait for every one to finish)	2700
+workspace-test-shard (1)	Sections	1440
+workspace-test-shard (2)	Sections	1440
+workspace-test-shard (3)	Sections	1440

scripts/check_ci_failfast.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# check_ci_failfast.sh -- a RED merge_group job must cancel its run (operator 2026-10-19, refs #5394).
+# x86-main and workspace-test-shard each carry a `gh run cancel` step guarded by
+# `failure() && github.event_name == 'merge_group'`, and the shard job holds `actions: write`.
+# Also runs scripts/ci/step_budget_check.sh --self-test and checks every budget row has 3 fields.
+#   bash scripts/check_ci_failfast.sh [workflow] | --self-test
+set -uo pipefail
+cd "$(dirname "${BASH_SOURCE[0]}")/.." || exit 2
+
+# job_has_failfast <workflow> <job>: 0 when the job block holds the guarded cancel step
+job_has_failfast() {
+  awk -v job="$2" '
+    $0 ~ "^  " job ":" { on = 1; next }
+    on && /^  [a-z0-9-]+:/ { on = 0 }
+    on && /if: .*failure\(\).*github\.event_name == .merge_group./ { g = 1 }
+    on && g && /gh run cancel "\$GITHUB_RUN_ID"/ { ok = 1 }
+    END { exit ok ? 0 : 1 }' "$1"
+}
+
+check() {
+  local wf=$1 rc=0 j
+  for j in x86-main workspace-test-shard; do
+    job_has_failfast "$wf" "$j" || { echo "FAIL: job $j has no merge_group failure() -> gh run cancel step"; rc=1; }
+  done
+  awk '/^  workspace-test-shard:/{on=1;next} on&&/^  [a-z0-9-]+:/{on=0} on&&/actions: write/{f=1} END{exit f?0:1}' "$wf" \
+    || { echo "FAIL: workspace-test-shard lacks 'actions: write' (gh run cancel would 403)"; rc=1; }
+  return $rc
+}
+
+case "${1:-}" in -h|--help) echo "usage: check_ci_failfast.sh [workflow] | --self-test"; exit 0 ;; esac
+
+if [ "${1:-}" = "--self-test" ]; then
+  check .github/workflows/ci.yml || { echo "FAIL: the real workflow is not fail-fast"; exit 1; }
+  t=$(mktemp -d); trap 'rm -rf "${t:?}"' EXIT
+  grep -v 'gh run cancel' .github/workflows/ci.yml > "$t/mut.yml"
+  check "$t/mut.yml" > /dev/null && { echo "FAIL: a workflow with the cancel step removed still passed"; exit 1; }
+  bash scripts/ci/step_budget_check.sh --self-test || exit 1
+  awk -F'\t' '!/^#/ && NF != 3 { bad = 1 } END { exit bad }' ci/step-budgets.tsv || { echo "FAIL: ci/step-budgets.tsv row without 3 fields"; exit 1; }
+  echo "ok: check_ci_failfast case table"; exit 0
+fi
+check "${1:-.github/workflows/ci.yml}" && echo "ok: merge_group failures cancel the run"

scripts/ci/step_budget_check.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# step_budget_check.sh -- flag any CI step that ran longer than 1.5x its budget (ci/step-budgets.tsv).
+# Operator 2026-10-19: "per-step budgets, with CI flagging any step >1.5x its budget". Reads the run's jobs
+# API (GraphQL-free REST, one call). Prints one line per breach; exit 0 always on a measured run, 2 when the
+# run cannot be read (not_measured, never a pass). A LAB signal: run on demand or by the nightly LAB lane, never in a PR or merge job (C324 rule 5/6).
+#   bash scripts/ci/step_budget_check.sh <run_id>      bash scripts/ci/step_budget_check.sh --self-test
+set -uo pipefail
+BUD=${STEP_BUDGETS:-ci/step-budgets.tsv}
+
+# breaches: stdin = "job<TAB>step<TAB>seconds" rows; $BUD rows; prints "BREACH job | step | Ns > 1.5 x Bs"
+breaches() {
+  awk -F'\t' 'NR==FNR { if ($0 !~ /^#/ && NF >= 3) b[$1 FS $2] = $3; next }
+    ($1 FS $2) in b { lim = b[$1 FS $2] * 1.5; if ($3 > lim) printf "BREACH %s | %s | %ds > 1.5 x %ds\n", $1, $2, $3, b[$1 FS $2] }' "$BUD" -
+}
+
+if [ "${1:-}" = "--self-test" ]; then
+  t=$(mktemp -d); trap 'rm -rf "${t:?}"' EXIT
+  printf 'j\ts\t100\n' > "$t/b.tsv"
+  r1=$(printf 'j\ts\t151\n' | BUD="$t/b.tsv" breaches)
+  r2=$(printf 'j\ts\t150\n' | BUD="$t/b.tsv" breaches)
+  r3=$(printf 'other\ts\t999\n' | BUD="$t/b.tsv" breaches)
+  [ -n "$r1" ] && [ -z "$r2" ] && [ -z "$r3" ] || { echo "FAIL: 151>1.5x100 must breach; 150 and an unbudgeted step must not"; exit 1; }
+  echo "ok: step_budget_check case table"; exit 0
+fi
+
+run=${1:?usage: step_budget_check.sh <run_id>|--self-test}
+rows=$(gh api "repos/${GITHUB_REPOSITORY:-paiml/aprender}/actions/runs/$run/jobs?per_page=100" --jq \
+  '.jobs[] | .name as $j | .steps[]? | select(.started_at and .completed_at) | [$j, .name, ((.completed_at|fromdate) - (.started_at|fromdate))] | @tsv') \
+  || { echo "NOT_MEASURED: jobs API read failed for run $run"; exit 2; }
+out=$(printf '%s\n' "$rows" | breaches)
+if [ -n "$out" ]; then printf '%s\n' "$out" | while read -r l; do echo "::error::$l"; done; else echo "no step over 1.5x budget (run $run)"; fi