diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 905ae731c2..7f3c75c62e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,7 +160,7 @@ jobs: if: ${{ !cancelled() && needs.mg-reuse.outputs.x86 != '1' && github.event.pull_request.draft != true }} # pull_request/push stay on the shared clean-room pool; a merge_group entry takes the mq lane (custom label exactly mq) runs-on: ${{ github.event_name == 'merge_group' && fromJSON('["self-hosted","Linux","X64","mq"]') || fromJSON('["self-hosted","Linux","X64","clean-room"]') }} - timeout-minutes: 240 + timeout-minutes: ${{ github.event_name == 'merge_group' && 75 || 240 }} # The sections carry their own permissions blocks (the sov provenance section # needs id-token + attestations); one job token must cover all of them. permissions: write-all @@ -266,6 +266,14 @@ jobs: # the teardown below keeps always() and still reaps the containers. if: ${{ !cancelled() }} run: python3 scripts/ci/fat_driver.py wait --results "$RUNNER_TEMP/fat/results.json" + # Operator 2026-10-19: root cause is no gate time budget. A merge_group entry that is already RED must + # not keep the other jobs and shards holding mq runners for another ~45 min: the first failed job cancels + # the run. merge_group only (a PR run keeps every verdict); the teardown below is always() and still reaps. + - name: Fail fast, cancel this merge-group run + if: ${{ failure() && github.event_name == 'merge_group' }} + env: + GH_TOKEN: ${{ github.token }} + run: gh run cancel "$GITHUB_RUN_ID" --repo "$GITHUB_REPOSITORY" || echo "::warning::fail-fast cancel did not take" # #4948: a cancelled run never reaches a section's own container removal, # and its fat--
container then sits on `tail -f /dev/null` # for days. always() runs on cancel too; on the passing path the driver @@ -445,7 +453,10 @@ jobs: if: ${{ github.event.pull_request.draft != true }} # same lanes as x86-main: clean-room for pull_request/push, mq for a merge_group entry runs-on: ${{ github.event_name == 'merge_group' && fromJSON('["self-hosted","Linux","X64","mq"]') || fromJSON('["self-hosted","Linux","X64","clean-room"]') }} - timeout-minutes: 180 + timeout-minutes: ${{ github.event_name == 'merge_group' && 75 || 180 }} + permissions: + contents: read + actions: write # the fail-fast step cancels this run strategy: fail-fast: false matrix: @@ -470,6 +481,14 @@ jobs: python3 scripts/ci/fat_driver.py run --sections 'workspace-test-shard?${{ matrix.shard }}?${{ matrix.shards }}?' --results "$RUNNER_TEMP/fat/results.json" + # Operator 2026-10-19: root cause is no gate time budget. A merge_group entry that is already RED must + # not keep the other jobs and shards holding mq runners for another ~45 min: the first failed job cancels + # the run. merge_group only (a PR run keeps every verdict); the teardown below is always() and still reaps. + - name: Fail fast, cancel this merge-group run + if: ${{ failure() && github.event_name == 'merge_group' }} + env: + GH_TOKEN: ${{ github.token }} + run: gh run cancel "$GITHUB_RUN_ID" --repo "$GITHUB_REPOSITORY" || echo "::warning::fail-fast cancel did not take" # #4948: a cancelled run never reaches a section's own container removal, # and its fat--
container then sits on `tail -f /dev/null` # for days. always() runs on cancel too; on the passing path the driver diff --git a/ci/step-budgets.tsv b/ci/step-budgets.tsv new file mode 100644 index 0000000000..75120c1ac2 --- /dev/null +++ b/ci/step-budgets.tsv @@ -0,0 +1,6 @@ +# jobstepbudget_seconds. A step over 1.5x its budget is flagged by scripts/ci/step_budget_check.sh. +# Seeded 2026-10-19 from MG run 39205769238 (shard norm ~24 min) and the guard-cargo green time after #5712 (~45 min x86-main). The owner tunes them from data. +x86-main Sections (wait for every one to finish) 2700 +workspace-test-shard (1) Sections 1440 +workspace-test-shard (2) Sections 1440 +workspace-test-shard (3) Sections 1440 diff --git a/scripts/check_ci_failfast.sh b/scripts/check_ci_failfast.sh new file mode 100755 index 0000000000..a185000244 --- /dev/null +++ b/scripts/check_ci_failfast.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# check_ci_failfast.sh -- a RED merge_group job must cancel its run (operator 2026-10-19, refs #5394). +# x86-main and workspace-test-shard each carry a `gh run cancel` step guarded by +# `failure() && github.event_name == 'merge_group'`, and the shard job holds `actions: write`. +# Also runs scripts/ci/step_budget_check.sh --self-test and checks every budget row has 3 fields. +# bash scripts/check_ci_failfast.sh [workflow] | --self-test +set -uo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." || exit 2 + +# job_has_failfast : 0 when the job block holds the guarded cancel step +job_has_failfast() { + awk -v job="$2" ' + $0 ~ "^ " job ":" { on = 1; next } + on && /^ [a-z0-9-]+:/ { on = 0 } + on && /if: .*failure\(\).*github\.event_name == .merge_group./ { g = 1 } + on && g && /gh run cancel "\$GITHUB_RUN_ID"/ { ok = 1 } + END { exit ok ? 0 : 1 }' "$1" +} + +check() { + local wf=$1 rc=0 j + for j in x86-main workspace-test-shard; do + job_has_failfast "$wf" "$j" || { echo "FAIL: job $j has no merge_group failure() -> gh run cancel step"; rc=1; } + done + awk '/^ workspace-test-shard:/{on=1;next} on&&/^ [a-z0-9-]+:/{on=0} on&&/actions: write/{f=1} END{exit f?0:1}' "$wf" \ + || { echo "FAIL: workspace-test-shard lacks 'actions: write' (gh run cancel would 403)"; rc=1; } + return $rc +} + +case "${1:-}" in -h|--help) echo "usage: check_ci_failfast.sh [workflow] | --self-test"; exit 0 ;; esac + +if [ "${1:-}" = "--self-test" ]; then + check .github/workflows/ci.yml || { echo "FAIL: the real workflow is not fail-fast"; exit 1; } + t=$(mktemp -d); trap 'rm -rf "${t:?}"' EXIT + grep -v 'gh run cancel' .github/workflows/ci.yml > "$t/mut.yml" + check "$t/mut.yml" > /dev/null && { echo "FAIL: a workflow with the cancel step removed still passed"; exit 1; } + bash scripts/ci/step_budget_check.sh --self-test || exit 1 + awk -F'\t' '!/^#/ && NF != 3 { bad = 1 } END { exit bad }' ci/step-budgets.tsv || { echo "FAIL: ci/step-budgets.tsv row without 3 fields"; exit 1; } + echo "ok: check_ci_failfast case table"; exit 0 +fi +check "${1:-.github/workflows/ci.yml}" && echo "ok: merge_group failures cancel the run" diff --git a/scripts/ci/step_budget_check.sh b/scripts/ci/step_budget_check.sh new file mode 100755 index 0000000000..547032b300 --- /dev/null +++ b/scripts/ci/step_budget_check.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# step_budget_check.sh -- flag any CI step that ran longer than 1.5x its budget (ci/step-budgets.tsv). +# Operator 2026-10-19: "per-step budgets, with CI flagging any step >1.5x its budget". Reads the run's jobs +# API (GraphQL-free REST, one call). Prints one line per breach; exit 0 always on a measured run, 2 when the +# run cannot be read (not_measured, never a pass). A LAB signal: run on demand or by the nightly LAB lane, never in a PR or merge job (C324 rule 5/6). +# bash scripts/ci/step_budget_check.sh bash scripts/ci/step_budget_check.sh --self-test +set -uo pipefail +BUD=${STEP_BUDGETS:-ci/step-budgets.tsv} + +# breaches: stdin = "jobstepseconds" rows; $BUD rows; prints "BREACH job | step | Ns > 1.5 x Bs" +breaches() { + awk -F'\t' 'NR==FNR { if ($0 !~ /^#/ && NF >= 3) b[$1 FS $2] = $3; next } + ($1 FS $2) in b { lim = b[$1 FS $2] * 1.5; if ($3 > lim) printf "BREACH %s | %s | %ds > 1.5 x %ds\n", $1, $2, $3, b[$1 FS $2] }' "$BUD" - +} + +if [ "${1:-}" = "--self-test" ]; then + t=$(mktemp -d); trap 'rm -rf "${t:?}"' EXIT + printf 'j\ts\t100\n' > "$t/b.tsv" + r1=$(printf 'j\ts\t151\n' | BUD="$t/b.tsv" breaches) + r2=$(printf 'j\ts\t150\n' | BUD="$t/b.tsv" breaches) + r3=$(printf 'other\ts\t999\n' | BUD="$t/b.tsv" breaches) + [ -n "$r1" ] && [ -z "$r2" ] && [ -z "$r3" ] || { echo "FAIL: 151>1.5x100 must breach; 150 and an unbudgeted step must not"; exit 1; } + echo "ok: step_budget_check case table"; exit 0 +fi + +run=${1:?usage: step_budget_check.sh |--self-test} +rows=$(gh api "repos/${GITHUB_REPOSITORY:-paiml/aprender}/actions/runs/$run/jobs?per_page=100" --jq \ + '.jobs[] | .name as $j | .steps[]? | select(.started_at and .completed_at) | [$j, .name, ((.completed_at|fromdate) - (.started_at|fromdate))] | @tsv') \ + || { echo "NOT_MEASURED: jobs API read failed for run $run"; exit 2; } +out=$(printf '%s\n' "$rows" | breaches) +if [ -n "$out" ]; then printf '%s\n' "$out" | while read -r l; do echo "::error::$l"; done; else echo "no step over 1.5x budget (run $run)"; fi