From c96f737a08e20d7513eb5b4f0dcda52fa48e1fda Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 19 Oct 2026 09:15:44 +0200 Subject: [PATCH 1/3] ci: merge-group fail-fast cancel and per-step budgets (refs #5394) A RED merge_group job cancels its run; ci/step-budgets.tsv plus step_budget_check.sh flag any step over 1.5x budget (push to main only, LAB). MG job timeout-minutes 35 is deliberately NOT changed: x86-main takes ~57 min today. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 27 +++++++++++++++++++++++ ci/step-budgets.tsv | 6 +++++ scripts/check_ci_failfast.sh | 39 +++++++++++++++++++++++++++++++++ scripts/ci/step_budget_check.sh | 31 ++++++++++++++++++++++++++ 4 files changed, 103 insertions(+) create mode 100644 ci/step-budgets.tsv create mode 100755 scripts/check_ci_failfast.sh create mode 100755 scripts/ci/step_budget_check.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0f3850566..d4e3a414b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -264,6 +264,14 @@ jobs: # the teardown below keeps always() and still reaps the containers. if: ${{ !cancelled() }} run: python3 scripts/ci/fat_driver.py wait --results "$RUNNER_TEMP/fat/results.json" + # Operator 2026-10-19: root cause is no gate time budget. A merge_group entry that is already RED must + # not keep the other jobs and shards holding mq runners for another ~45 min: the first failed job cancels + # the run. merge_group only (a PR run keeps every verdict); the teardown below is always() and still reaps. + - name: Fail fast, cancel this merge-group run + if: ${{ failure() && github.event_name == 'merge_group' }} + env: + GH_TOKEN: ${{ github.token }} + run: gh run cancel "$GITHUB_RUN_ID" --repo "$GITHUB_REPOSITORY" || echo "::warning::fail-fast cancel did not take" # #4948: a cancelled run never reaches a section's own container removal, # and its fat--
container then sits on `tail -f /dev/null` # for days. always() runs on cancel too; on the passing path the driver @@ -300,6 +308,14 @@ jobs: continue-on-error: true with: subject-path: ${{ runner.temp }}/fat/sov.provenance/ws/${{ github.event.repository.name }}/Cargo.toml + # Operator 2026-10-19: CI flags any step over 1.5x its budget (ci/step-budgets.tsv). On push to main + # only, outside the merge path (C324: a LAB signal, so it can never block a merge). + - name: Flag steps over 1.5x their budget (push to main, LAB) + if: ${{ always() && github.event_name == 'push' }} + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + run: bash scripts/ci/step_budget_check.sh "$GITHUB_RUN_ID" # #5397: free this job's own target dir (`/run-`), never the PR parent. # Last step, so nothing after it reads the dir; always() so a failed or # cancelled run frees it too. @@ -443,6 +459,9 @@ jobs: # same lanes as x86-main: clean-room for pull_request/push, mq for a merge_group entry runs-on: ${{ github.event_name == 'merge_group' && fromJSON('["self-hosted","Linux","X64","mq"]') || fromJSON('["self-hosted","Linux","X64","clean-room"]') }} timeout-minutes: 180 + permissions: + contents: read + actions: write # the fail-fast step cancels this run strategy: fail-fast: false matrix: @@ -466,6 +485,14 @@ jobs: python3 scripts/ci/fat_driver.py run --sections 'workspace-test-shard?${{ matrix.shard }}?3?' --results "$RUNNER_TEMP/fat/results.json" + # Operator 2026-10-19: root cause is no gate time budget. A merge_group entry that is already RED must + # not keep the other jobs and shards holding mq runners for another ~45 min: the first failed job cancels + # the run. merge_group only (a PR run keeps every verdict); the teardown below is always() and still reaps. + - name: Fail fast, cancel this merge-group run + if: ${{ failure() && github.event_name == 'merge_group' }} + env: + GH_TOKEN: ${{ github.token }} + run: gh run cancel "$GITHUB_RUN_ID" --repo "$GITHUB_REPOSITORY" || echo "::warning::fail-fast cancel did not take" # #4948: a cancelled run never reaches a section's own container removal, # and its fat--
container then sits on `tail -f /dev/null` # for days. always() runs on cancel too; on the passing path the driver diff --git a/ci/step-budgets.tsv b/ci/step-budgets.tsv new file mode 100644 index 0000000000..75120c1ac2 --- /dev/null +++ b/ci/step-budgets.tsv @@ -0,0 +1,6 @@ +# jobstepbudget_seconds. A step over 1.5x its budget is flagged by scripts/ci/step_budget_check.sh. +# Seeded 2026-10-19 from MG run 39205769238 (shard norm ~24 min) and the guard-cargo green time after #5712 (~45 min x86-main). The owner tunes them from data. +x86-main Sections (wait for every one to finish) 2700 +workspace-test-shard (1) Sections 1440 +workspace-test-shard (2) Sections 1440 +workspace-test-shard (3) Sections 1440 diff --git a/scripts/check_ci_failfast.sh b/scripts/check_ci_failfast.sh new file mode 100755 index 0000000000..60dfb7d049 --- /dev/null +++ b/scripts/check_ci_failfast.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# check_ci_failfast.sh -- a RED merge_group job must cancel its run (operator 2026-10-19, refs #5394). +# x86-main and workspace-test-shard each carry a `gh run cancel` step guarded by +# `failure() && github.event_name == 'merge_group'`, and the shard job holds `actions: write`. +# Also runs scripts/ci/step_budget_check.sh --self-test and checks every budget row has 3 fields. +# bash scripts/check_ci_failfast.sh [workflow] | --self-test +set -uo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." || exit 2 + +# job_has_failfast : 0 when the job block holds the guarded cancel step +job_has_failfast() { + awk -v job="$2" ' + $0 ~ "^ " job ":" { on = 1; next } + on && /^ [a-z0-9-]+:/ { on = 0 } + on && /if: .*failure\(\).*github\.event_name == .merge_group./ { g = 1 } + on && g && /gh run cancel "\$GITHUB_RUN_ID"/ { ok = 1 } + END { exit ok ? 0 : 1 }' "$1" +} + +check() { + local wf=$1 rc=0 j + for j in x86-main workspace-test-shard; do + job_has_failfast "$wf" "$j" || { echo "FAIL: job $j has no merge_group failure() -> gh run cancel step"; rc=1; } + done + awk '/^ workspace-test-shard:/{on=1;next} on&&/^ [a-z0-9-]+:/{on=0} on&&/actions: write/{f=1} END{exit f?0:1}' "$wf" \ + || { echo "FAIL: workspace-test-shard lacks 'actions: write' (gh run cancel would 403)"; rc=1; } + return $rc +} + +if [ "${1:-}" = "--self-test" ]; then + check .github/workflows/ci.yml || { echo "FAIL: the real workflow is not fail-fast"; exit 1; } + t=$(mktemp -d); trap 'rm -rf "${t:?}"' EXIT + grep -v 'gh run cancel' .github/workflows/ci.yml > "$t/mut.yml" + check "$t/mut.yml" > /dev/null && { echo "FAIL: a workflow with the cancel step removed still passed"; exit 1; } + bash scripts/ci/step_budget_check.sh --self-test || exit 1 + awk -F'\t' '!/^#/ && NF != 3 { bad = 1 } END { exit bad }' ci/step-budgets.tsv || { echo "FAIL: ci/step-budgets.tsv row without 3 fields"; exit 1; } + echo "ok: check_ci_failfast case table"; exit 0 +fi +check "${1:-.github/workflows/ci.yml}" && echo "ok: merge_group failures cancel the run" diff --git a/scripts/ci/step_budget_check.sh b/scripts/ci/step_budget_check.sh new file mode 100755 index 0000000000..bc1aba785c --- /dev/null +++ b/scripts/ci/step_budget_check.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# step_budget_check.sh -- flag any CI step that ran longer than 1.5x its budget (ci/step-budgets.tsv). +# Operator 2026-10-19: "per-step budgets, with CI flagging any step >1.5x its budget". Reads the run's jobs +# API (GraphQL-free REST, one call). Prints one line per breach; exit 0 always on a measured run, 2 when the +# run cannot be read (not_measured, never a pass). Runs on push to main only: a LAB signal, never on the merge path. +# bash scripts/ci/step_budget_check.sh bash scripts/ci/step_budget_check.sh --self-test +set -uo pipefail +BUD=${STEP_BUDGETS:-ci/step-budgets.tsv} + +# breaches: stdin = "jobstepseconds" rows; $BUD rows; prints "BREACH job | step | Ns > 1.5 x Bs" +breaches() { + awk -F'\t' 'NR==FNR { if ($0 !~ /^#/ && NF >= 3) b[$1 FS $2] = $3; next } + ($1 FS $2) in b { lim = b[$1 FS $2] * 1.5; if ($3 > lim) printf "BREACH %s | %s | %ds > 1.5 x %ds\n", $1, $2, $3, b[$1 FS $2] }' "$BUD" - +} + +if [ "${1:-}" = "--self-test" ]; then + t=$(mktemp -d); trap 'rm -rf "${t:?}"' EXIT + printf 'j\ts\t100\n' > "$t/b.tsv" + r1=$(printf 'j\ts\t151\n' | BUD="$t/b.tsv" breaches) + r2=$(printf 'j\ts\t150\n' | BUD="$t/b.tsv" breaches) + r3=$(printf 'other\ts\t999\n' | BUD="$t/b.tsv" breaches) + [ -n "$r1" ] && [ -z "$r2" ] && [ -z "$r3" ] || { echo "FAIL: 151>1.5x100 must breach; 150 and an unbudgeted step must not"; exit 1; } + echo "ok: step_budget_check case table"; exit 0 +fi + +run=${1:?usage: step_budget_check.sh |--self-test} +rows=$(gh api "repos/${GITHUB_REPOSITORY:-paiml/aprender}/actions/runs/$run/jobs?per_page=100" --jq \ + '.jobs[] | .name as $j | .steps[]? | select(.started_at and .completed_at) | [$j, .name, ((.completed_at|fromdate) - (.started_at|fromdate))] | @tsv') \ + || { echo "NOT_MEASURED: jobs API read failed for run $run"; exit 2; } +out=$(printf '%s\n' "$rows" | breaches) +if [ -n "$out" ]; then printf '%s\n' "$out" | while read -r l; do echo "::error::$l"; done; else echo "no step over 1.5x budget (run $run)"; fi From 7eef48051a7fcac370214ee7ea1b9b4ce8009b05 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 19 Oct 2026 10:11:45 +0200 Subject: [PATCH 2/3] ci: drop the continue-on-error budget step from the PR path; make check_ci_failfast.sh advertise --self-test so guard_tree runs it (refs #5394) Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 8 -------- scripts/check_ci_failfast.sh | 2 ++ scripts/ci/step_budget_check.sh | 2 +- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4e3a414b3..433a089c4f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -308,14 +308,6 @@ jobs: continue-on-error: true with: subject-path: ${{ runner.temp }}/fat/sov.provenance/ws/${{ github.event.repository.name }}/Cargo.toml - # Operator 2026-10-19: CI flags any step over 1.5x its budget (ci/step-budgets.tsv). On push to main - # only, outside the merge path (C324: a LAB signal, so it can never block a merge). - - name: Flag steps over 1.5x their budget (push to main, LAB) - if: ${{ always() && github.event_name == 'push' }} - continue-on-error: true - env: - GH_TOKEN: ${{ github.token }} - run: bash scripts/ci/step_budget_check.sh "$GITHUB_RUN_ID" # #5397: free this job's own target dir (`/run-`), never the PR parent. # Last step, so nothing after it reads the dir; always() so a failed or # cancelled run frees it too. diff --git a/scripts/check_ci_failfast.sh b/scripts/check_ci_failfast.sh index 60dfb7d049..a185000244 100755 --- a/scripts/check_ci_failfast.sh +++ b/scripts/check_ci_failfast.sh @@ -27,6 +27,8 @@ check() { return $rc } +case "${1:-}" in -h|--help) echo "usage: check_ci_failfast.sh [workflow] | --self-test"; exit 0 ;; esac + if [ "${1:-}" = "--self-test" ]; then check .github/workflows/ci.yml || { echo "FAIL: the real workflow is not fail-fast"; exit 1; } t=$(mktemp -d); trap 'rm -rf "${t:?}"' EXIT diff --git a/scripts/ci/step_budget_check.sh b/scripts/ci/step_budget_check.sh index bc1aba785c..547032b300 100755 --- a/scripts/ci/step_budget_check.sh +++ b/scripts/ci/step_budget_check.sh @@ -2,7 +2,7 @@ # step_budget_check.sh -- flag any CI step that ran longer than 1.5x its budget (ci/step-budgets.tsv). # Operator 2026-10-19: "per-step budgets, with CI flagging any step >1.5x its budget". Reads the run's jobs # API (GraphQL-free REST, one call). Prints one line per breach; exit 0 always on a measured run, 2 when the -# run cannot be read (not_measured, never a pass). Runs on push to main only: a LAB signal, never on the merge path. +# run cannot be read (not_measured, never a pass). A LAB signal: run on demand or by the nightly LAB lane, never in a PR or merge job (C324 rule 5/6). # bash scripts/ci/step_budget_check.sh bash scripts/ci/step_budget_check.sh --self-test set -uo pipefail BUD=${STEP_BUDGETS:-ci/step-budgets.tsv} From 995d23b1880a20dde1281ba1f1f59e9280219eae Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 19 Oct 2026 12:35:46 +0200 Subject: [PATCH 3/3] ci: merge_group job timeout 75 min on x86-main and workspace-test-shard (operator order, refs #5394) PR runs keep 240/180. Measured p95 over 20 MG runs: x86-main 65 min, shards 74 min. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 433a089c4f..0042ac60f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -158,7 +158,7 @@ jobs: if: ${{ !cancelled() && needs.mg-reuse.outputs.x86 != '1' }} # pull_request/push stay on the shared clean-room pool; a merge_group entry takes the mq lane (custom label exactly mq) runs-on: ${{ github.event_name == 'merge_group' && fromJSON('["self-hosted","Linux","X64","mq"]') || fromJSON('["self-hosted","Linux","X64","clean-room"]') }} - timeout-minutes: 240 + timeout-minutes: ${{ github.event_name == 'merge_group' && 75 || 240 }} # The sections carry their own permissions blocks (the sov provenance section # needs id-token + attestations); one job token must cover all of them. permissions: write-all @@ -450,7 +450,7 @@ jobs: workspace-test-shard: # same lanes as x86-main: clean-room for pull_request/push, mq for a merge_group entry runs-on: ${{ github.event_name == 'merge_group' && fromJSON('["self-hosted","Linux","X64","mq"]') || fromJSON('["self-hosted","Linux","X64","clean-room"]') }} - timeout-minutes: 180 + timeout-minutes: ${{ github.event_name == 'merge_group' && 75 || 180 }} permissions: contents: read actions: write # the fail-fast step cancels this run