diff --git a/scripts/check_ci_failfast.sh b/scripts/check_ci_failfast.sh new file mode 100755 index 0000000000..a185000244 --- /dev/null +++ b/scripts/check_ci_failfast.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# check_ci_failfast.sh -- a RED merge_group job must cancel its run (operator 2026-10-19, refs #5394). +# x86-main and workspace-test-shard each carry a `gh run cancel` step guarded by +# `failure() && github.event_name == 'merge_group'`, and the shard job holds `actions: write`. +# Also runs scripts/ci/step_budget_check.sh --self-test and checks every budget row has 3 fields. +# bash scripts/check_ci_failfast.sh [workflow] | --self-test +set -uo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." || exit 2 + +# job_has_failfast : 0 when the job block holds the guarded cancel step +job_has_failfast() { + awk -v job="$2" ' + $0 ~ "^ " job ":" { on = 1; next } + on && /^ [a-z0-9-]+:/ { on = 0 } + on && /if: .*failure\(\).*github\.event_name == .merge_group./ { g = 1 } + on && g && /gh run cancel "\$GITHUB_RUN_ID"/ { ok = 1 } + END { exit ok ? 0 : 1 }' "$1" +} + +check() { + local wf=$1 rc=0 j + for j in x86-main workspace-test-shard; do + job_has_failfast "$wf" "$j" || { echo "FAIL: job $j has no merge_group failure() -> gh run cancel step"; rc=1; } + done + awk '/^ workspace-test-shard:/{on=1;next} on&&/^ [a-z0-9-]+:/{on=0} on&&/actions: write/{f=1} END{exit f?0:1}' "$wf" \ + || { echo "FAIL: workspace-test-shard lacks 'actions: write' (gh run cancel would 403)"; rc=1; } + return $rc +} + +case "${1:-}" in -h|--help) echo "usage: check_ci_failfast.sh [workflow] | --self-test"; exit 0 ;; esac + +if [ "${1:-}" = "--self-test" ]; then + check .github/workflows/ci.yml || { echo "FAIL: the real workflow is not fail-fast"; exit 1; } + t=$(mktemp -d); trap 'rm -rf "${t:?}"' EXIT + grep -v 'gh run cancel' .github/workflows/ci.yml > "$t/mut.yml" + check "$t/mut.yml" > /dev/null && { echo "FAIL: a workflow with the cancel step removed still passed"; exit 1; } + bash scripts/ci/step_budget_check.sh --self-test || exit 1 + awk -F'\t' '!/^#/ && NF != 3 { bad = 1 } END { exit bad }' ci/step-budgets.tsv || { echo "FAIL: ci/step-budgets.tsv row without 3 fields"; exit 1; } + echo "ok: check_ci_failfast case table"; exit 0 +fi +check "${1:-.github/workflows/ci.yml}" && echo "ok: merge_group failures cancel the run" diff --git a/scripts/check_lab_timeouts.sh b/scripts/check_lab_timeouts.sh new file mode 100755 index 0000000000..42d5990ec4 --- /dev/null +++ b/scripts/check_lab_timeouts.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# check_lab_timeouts.sh -- PMAT-5847. LAB/nightly (schedule:) workflows are bounded: every job has timeout-minutes, +# and the cron start plus the longest job timeout ends by 05:00Z (a start before 05:00Z ends by 05:00Z that day, +# a later start by 05:00Z the next day). Operator, verbatim, 11:04Z: "Lab/nightly workflows get timeout-minutes +# and must end by 05:00Z". A LAB check: run on demand or at night, never inside a PR or release job (C324). +# A job that only calls a reusable workflow (uses:) cannot carry timeout-minutes; the callee is scanned itself. +# Usage: check_lab_timeouts.sh [workflows-dir] | --self-test Exit: 0 green, 1 RED +set -euo pipefail +LIMIT=300 DAY=1440 +DAY_FROM=300 DAY_TO=1080 # a start in 05:00-18:00Z runs in the working day +# Named exception (reason in the ticket): rehearsal starts after nightly-train has read the night's producers; its 300 min cannot end by 05:00Z. Decision owed by the operator/cop. +EXCEPT="${LAB_TIMEOUT_EXCEPT-release-rehearsal-nightly.yml}" +PATH_AWK=$(cat <<'AWK' +function rec(j, k, best, v, n, a, i) { + if (j in memo) return memo[j] + best = 0; n = split(need[j], a, " ") + for (i = 1; i <= n; i++) if (a[i] in tm) { v = rec(a[i]); if (v > best) best = v } + return memo[j] = best + tm[j] +} +/^jobs:/ { in_jobs = 1; next } +in_jobs && /^ [A-Za-z0-9_-]+:[[:space:]]*(#.*)?$/ { name = $1; sub(/:$/, "", name); tm[name] = 0; need[name] = ""; innd = 0; next } +in_jobs && name && /^ timeout-minutes:/ { tm[name] = $2 + 0; innd = 0; next } +in_jobs && name && /^ needs:/ { s = $0; sub(/^ needs:[[:space:]]*/, "", s); gsub(/[\[\],]/, " ", s); need[name] = s; innd = (s ~ /^[[:space:]]*$/); next } +in_jobs && name && innd && /^ - / { s = $0; sub(/^ - /, "", s); need[name] = need[name] " " s; next } +in_jobs && name && /^ [A-Za-z]/ { innd = 0 } +END { m = 0; for (j in tm) { v = rec(j); if (v > m) m = v } print m } +AWK +) +scan() { # one workflow file -> RED lines + local f="$1" j longest m h s end + grep -qE '^[[:space:]]+schedule:' "$f" || return 0 + case " $EXCEPT " in *" $(basename "$f") "*) return 0 ;; esac + for j in $(awk '/^jobs:/{in_jobs=1;next} in_jobs&&/^ [A-Za-z0-9_-]+:[[:space:]]*(#.*)?$/{if(name&&runs&&!seen)print name; name=$1; runs=0; seen=0; next} in_jobs&&/^ runs-on:/{runs=1} in_jobs&&/^ timeout-minutes:/{seen=1} END{if(name&&runs&&!seen)print name}' "$f"); do + echo "RED $f job ${j%:}: no timeout-minutes" + done + longest=$(awk "$PATH_AWK" "$f") # longest needs: chain, not the longest single job + grep -E '^[[:space:]]+- cron:' "$f" | sed -E "s/.*cron:[[:space:]]*['\"]?([^'\"#]*).*/\1/" | while read -r m h _; do + case "$m$h" in *[!0-9]*) continue ;; esac # */10 or a range is not one nightly start + s=$((h * 60 + m)); end=$((s + longest)) + if [ "$s" -ge "$DAY_FROM" ] && [ "$s" -lt "$DAY_TO" ]; then echo "RED $f: cron $m $h starts in the working day (05:00-18:00Z)"; continue; fi + if [ "$s" -lt "$LIMIT" ]; then [ "$end" -le "$LIMIT" ] || echo "RED $f: cron $m $h + ${longest}m ends after 05:00Z" + else [ "$end" -le $((DAY + LIMIT)) ] || echo "RED $f: cron $m $h + ${longest}m ends after 05:00Z"; fi + done +} +check() { local bad=0 out f; for f in "$1"/*.yml; do out=$(scan "$f"); [ -z "$out" ] || { echo "$out"; bad=1; }; done; return "$bad"; } +if [ "${1:-}" = --self-test ]; then + t=$(mktemp -d); trap 'rm -rf -- "${t:?}"' EXIT; fail=0 + w() { rm -f "${t:?}"/*.yml; printf 'on:\n schedule:\n - cron: "%s"\njobs:\n a:\n runs-on: x\n%s steps: []\n' "$1" "$2" > "$t/w.yml"; } + w '5 23 * * *' $' timeout-minutes: 60\n'; check "$t" > /dev/null || { echo "FAIL bounded row is RED"; fail=1; } + w '5 23 * * *' ''; check "$t" > /dev/null && { echo "FAIL no-timeout row not RED"; fail=1; } + w '5 23 * * *' $' timeout-minutes: 420\n'; check "$t" > /dev/null && { echo "FAIL ends-after-05Z row not RED"; fail=1; } + w '5 23 * * *' $' timeout-minutes: 355\n'; check "$t" > /dev/null || { echo "FAIL ends-exactly-05:00 row is RED"; fail=1; } + w '20 3 * * *' $' timeout-minutes: 120\n'; check "$t" > /dev/null && { echo "FAIL 03:20+120m row not RED"; fail=1; } + rm -f "${t:?}"/*.yml; printf 'on:\n schedule:\n - cron: "5 22 * * *"\njobs:\n a:\n runs-on: x\n timeout-minutes: 200\n b:\n needs: a\n runs-on: x\n timeout-minutes: 200\n c:\n needs: [a, b]\n runs-on: x\n timeout-minutes: 100\n' > "$t/chain.yml" + check "$t" > /dev/null && { echo "FAIL chain overrun (a+b+c=500) not RED"; fail=1; } + w '0 12 * * *' $' timeout-minutes: 10\n'; check "$t" > /dev/null && { echo "FAIL daytime start not RED"; fail=1; } + rm -f "${t:?}"/*.yml; printf 'on:\n pull_request:\njobs:\n a:\n runs-on: x\n steps: []\n' > "$t/pr.yml" + check "$t" > /dev/null || { echo "FAIL non-schedule workflow flagged"; fail=1; } + [ "$fail" -eq 0 ] && echo "ok: self-test (8 rows)"; exit "$fail" +fi +check "${1:-.github/workflows}"