diff --git a/.config/nextest.toml b/.config/nextest.toml index b0c188a42a..ba48acda18 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -16,7 +16,16 @@ # now fails on any key nextest ignores. [profile.ci] retries = 2 -fail-fast = true +# false, not true (PMAT-3587). fail-fast is a verdict-discarding default: on the first +# failing test nextest cancels every test still queued and reports the one failure with +# no count of what it did not measure. Measured on a 6-test probe whose 2nd test fails: +# true -> "3/6 tests run ... Cancelling due to test failure"; false -> "6 tests run: +# 5 passed, 1 failed" (still rc 100 -- a failure is still a failure, it is just no +# longer the only thing measured). An ABSENT key is the same as true: nextest's default +# is ON, so scripts/check_nextest_ci_profile_no_fail_fast.sh requires this line to be +# present and literally false. The tree paid for the old value once: "nextest fail-fast +# hid every other dark failure: 7 rounds for 4 tests." +fail-fast = false # Measured on an idle 48-core box, 80806 tests, 353s wall: slowest single test # is 202.7s (aprender-orchestrate bug_hunter::test_bh_mod_001_hunt_all_modes), # only 2 tests exceed 120s and NONE exceeds 300s. terminate-after = 20 periods diff --git a/docs/audits/quorum-PMAT-3587.json b/docs/audits/quorum-PMAT-3587.json new file mode 100644 index 0000000000..6e4adb26e8 --- /dev/null +++ b/docs/audits/quorum-PMAT-3587.json @@ -0,0 +1,212 @@ +{ + "ticket": "PMAT-3587", + "base": "origin/main", + "base_resolved": "origin/main", + "base_note": "no origin/origin/main exists; judged against the local ref", + "head": "3df1c64aac8c2efb04e3fba6c36667707d13960e", + "diff_sha256": "e54d3397340ac956739ce78c9cef906d56644eb982ff52db619882149eae3942", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 31749, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff accurately implements the PMAT-3587 ticket requirements by setting `fail-fast = false` under `[profile.ci]` in `.config/nextest.toml`, effectively resolving the issue of nextest prematurely discarding test verdicts upon first failure. This prevents false metrics (like '1 failure + 15 Unknown') when executing CI tests. Additionally, the introduced script `scripts/check_nextest_ci_profile_no_fail_fast.sh` correctly establishes a robust mechanism to guarantee the property structurally, bypassing brittle substring checks. The script's 30-case `--self-test` suite has been executed and evaluated; it covers syntax variants and gracefully exits correctly when checking all edge cases. The required tracking items (docs/roadmaps/entries/PMAT-3587.yaml and docs/roadmaps/roadmap.yaml) are present. There are no violations, weakened gates, or instances of non-compliance with the ticket parameters. The outcome is PASS.", + "findings": [ + { + "claim": "The diff correctly updates .config/nextest.toml to explicitly set fail-fast = false under [profile.ci], which meets the PMAT-3587 requirement to prevent discarding of verdicts after a failure.", + "file": ".config/nextest.toml", + "grounding": "cited", + "line": 28 + }, + { + "claim": "The guard script scripts/check_nextest_ci_profile_no_fail_fast.sh implements anti-vacuous structural TOML validation that correctly asserts fail-fast = false is set under [profile.ci] and passes its comprehensive 30-row self-test table covering true/false/absent cases and malformed inputs.", + "command": "./scripts/check_nextest_ci_profile_no_fail_fast.sh --self-test", + "file": "scripts/check_nextest_ci_profile_no_fail_fast.sh", + "grounding": "measured", + "line": 204, + "output": "=== nextest [profile.ci] fail-fast guard: case table ===\\nok row 1 rc=0 [library] fail-fast = false under [profile.ci] -> PASS\\nok row 2 rc=1 [library] fail-fast = true under [profile.ci] -> RED\\nok row 3 rc=1 [library] key ABSENT under [profile.ci] -> RED (nextest default is ON)\\nok row 4 rc=1 [library] false under a DIFFERENT profile only -> RED\\nok row 5 rc=1 [library] no [profile.ci] at all -> RED\\nok row 6 rc=1 [library] false in a COMMENT, true in the key -> RED (not a substring test)\\nok row 7 rc=1 [library] the string \"false\" (a string, not a bool) -> RED\\nok row 8 rc=1 [library] false under [[profile.ci.overrides]] only -> RED (an override entry is not the profile)\\nok row 9 rc=0 [library] false after an inline table and before a sub-table -> PASS (the real file's shape)\\nok row 10 rc=2 [library] [[profile.ci]] (an array of tables, not the profile) -> ENV rc=2, never a pass\\nok row 11 rc=2 [library] unparseable TOML -> ENV rc=2, never a pass\\nok row 12 rc=0 [library] the real .config/nextest.toml is GREEN\\nok row 13 rc=0 [fallback] fail-fast = false under [profile.ci] -> PASS\\nok row 14 rc=1 [fallback] fail-fast = true under [profile.ci] -> RED\\nok row 15 rc=1 [fallback] key ABSENT under [profile.ci] -> RED (nextest default is ON)\\nok row 16 rc=1 [fallback] false under a DIFFERENT profile only -> RED\\nok row 17 rc=1 [fallback] no [profile.ci] at all -> RED\\nok row 18 rc=1 [fallback] false in a COMMENT, true in the key -> RED (not a substring test)\\nok row 19 rc=1 [fallback] the string \"false\" (a string, not a bool) -> RED\\nok row 20 rc=1 [fallback] false under [[profile.ci.overrides]] only -> RED (an override entry is not the profile)\\nok row 21 rc=0 [fallback] false after an inline table and before a sub-table -> PASS (the real file's shape)\\nok row 22 rc=2 [fallback] [[profile.ci]] (an array of tables, not the profile) -> ENV rc=2, never a pass\\nok row 23 rc=2 [fallback] unparseable TOML -> ENV rc=2, never a pass\\nok row 24 rc=0 [fallback] the real .config/nextest.toml is GREEN\\nok row 25 rc=2 [fallback] a multi-line string hiding a [profile.ci] header -> ENV, never a guess (a line reader that skipped it would say PASS)\\nok row 26 rc=2 [fallback] a value continuing past its line -> ENV, never a guess\\nok row 27 rc=2 [fallback] fail-fast = fals (not a value it can type) -> ENV, never a guess\\nok row 28 rc=2 interpreter exits 1 with no verdict -> ENV rc=2, never 1\\nok row 29 rc=2 no interpreter at all (rc=127) -> ENV rc=2\\nok row 30 rc=2 missing file -> ENV rc=2, never a pass\\nSELF-TEST PASSED: 30 rows" + }, + { + "claim": "docs/roadmaps/entries/PMAT-3587.yaml appropriately tracks the PMAT-3587 ticket and establishes its roadmap entry with correctly specified acceptance criteria for preventing verdict discards.", + "file": "docs/roadmaps/entries/PMAT-3587.yaml", + "grounding": "cited", + "line": 1 + } + ], + "raw_bytes": 11924, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff implements the required changes for PMAT-3587 by configuring `fail-fast = false` in `.config/nextest.toml` under `[profile.ci]`. It also includes a robust structural validation script with a `--self-test` to prevent regressions, satisfying the anti-vacuity and self-testing requirements. No gates are weakened and the changes directly fulfill the stated acceptance criteria.", + "findings": [ + { + "claim": "The diff sets fail-fast to false under [profile.ci] to ensure nextest does not cancel remaining tests on failure.", + "file": ".config/nextest.toml", + "grounding": "cited", + "line": 27 + }, + { + "claim": "A guard script is added to structurally enforce that [profile.ci].fail-fast is false, with a comprehensive --self-test.", + "file": "scripts/check_nextest_ci_profile_no_fail_fast.sh", + "grounding": "cited", + "line": 2 + }, + { + "claim": "The ticket is properly added to the roadmap to track the completion of the nextest [profile.ci] requirement.", + "file": "docs/roadmaps/entries/PMAT-3587.yaml", + "grounding": "cited", + "line": 1 + } + ], + "raw_bytes": 4155, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "As Lane 3 reviewer, I evaluated the diff against ticket PMAT-3587 and attempted to refute it.\n\nReview Analysis:\n1. `.config/nextest.toml`: Updated to set `fail-fast = false` under `[profile.ci]`, ensuring nextest does not cancel remaining queued tests on failure or discard verdicts.\n2. `scripts/check_nextest_ci_profile_no_fail_fast.sh`: Added an anti-vacuous guard script using Python (`tomllib`/`tomli` or fallback reader) to enforce `fail-fast = false` under `[profile.ci]`. Measured and verified that `--self-test` passes all 30 test cases covering boolean types, string values, comment tricks, section scoping, unparseable TOML, missing files, and interpreter failure handling (rc=2 ENV exit).\n3. Roadmap tracking (`docs/roadmaps/entries/PMAT-3587.yaml` and `docs/roadmaps/roadmap.yaml`): Added PMAT-3587 tracking metadata and acceptance criteria.\n4. Quorum audit record (`docs/audits/quorum-PMAT-3587.json`): Added audit quorum records.\n\nNo gates were weakened, no assertions violate the ticket requirements, and no unauthorized modifications were introduced. Verdict is PASS.", + "findings": [ + { + "claim": "The guard script scripts/check_nextest_ci_profile_no_fail_fast.sh implements structural TOML validation ensuring [profile.ci].fail-fast = false and passes its 30-row self-test matrix covering all polarities, missing keys, comment tricks, string types, invalid TOML, missing files, and interpreter execution failure modes.", + "command": "NEXTEST_CONF_OVERRIDE=/tmp/test_pmat_3587/.config/nextest.toml /tmp/test_pmat_3587/check_nextest_ci_profile_no_fail_fast.sh --self-test", + "file": "scripts/check_nextest_ci_profile_no_fail_fast.sh", + "grounding": "measured", + "line": 57, + "output": "=== nextest [profile.ci] fail-fast guard: case table ===\nok row 1 rc=0 [library] fail-fast = false under [profile.ci] -> PASS\nok row 2 rc=1 [library] fail-fast = true under [profile.ci] -> RED\nok row 3 rc=1 [library] key ABSENT under [profile.ci] -> RED (nextest default is ON)\n...\nSELF-TEST PASSED: 30 rows" + }, + { + "claim": ".config/nextest.toml explicitly sets fail-fast = false under [profile.ci], satisfying the nextest profile configuration requirement of ticket PMAT-3587 so test executions are not cancelled early on failure.", + "file": ".config/nextest.toml", + "grounding": "cited", + "line": 28 + }, + { + "claim": "docs/roadmaps/entries/PMAT-3587.yaml and docs/roadmaps/roadmap.yaml properly add the tracking entry and acceptance notes for ticket PMAT-3587.", + "file": "docs/roadmaps/entries/PMAT-3587.yaml", + "grounding": "cited", + "line": 1 + } + ], + "raw_bytes": 7282, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.6-flash-high", + "model_measured": "gemini-3.6-flash-high", + "model_source": "measured", + "family": "gemini" + } + ], + "dissent": [], + "dedup": [ + { + "file": ".config/nextest.toml", + "line": 27, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "The diff sets fail-fast to false under [profile.ci] to ensure nextest does not cancel remaining tests on failure." + ] + }, + { + "file": ".config/nextest.toml", + "line": 28, + "lanes_agreeing": [ + 1, + 3 + ], + "claims": [ + ".config/nextest.toml explicitly sets fail-fast = false under [profile.ci], satisfying the nextest profile configuration requirement of ticket PMAT-3587 so test executions are not cancelled early on failure.", + "The diff correctly updates .config/nextest.toml to explicitly set fail-fast = false under [profile.ci], which meets the PMAT-3587 requirement to prevent discarding of verdicts after a failure." + ] + }, + { + "file": "docs/roadmaps/entries/PMAT-3587.yaml", + "line": 1, + "lanes_agreeing": [ + 1, + 2, + 3 + ], + "claims": [ + "The ticket is properly added to the roadmap to track the completion of the nextest [profile.ci] requirement.", + "docs/roadmaps/entries/PMAT-3587.yaml and docs/roadmaps/roadmap.yaml properly add the tracking entry and acceptance notes for ticket PMAT-3587.", + "docs/roadmaps/entries/PMAT-3587.yaml appropriately tracks the PMAT-3587 ticket and establishes its roadmap entry with correctly specified acceptance criteria for preventing verdict discards." + ] + }, + { + "file": "scripts/check_nextest_ci_profile_no_fail_fast.sh", + "line": 2, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "A guard script is added to structurally enforce that [profile.ci].fail-fast is false, with a comprehensive --self-test." + ] + }, + { + "file": "scripts/check_nextest_ci_profile_no_fail_fast.sh", + "line": 57, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "The guard script scripts/check_nextest_ci_profile_no_fail_fast.sh implements structural TOML validation ensuring [profile.ci].fail-fast = false and passes its 30-row self-test matrix covering all polarities, missing keys, comment tricks, string types, invalid TOML, missing files, and interpreter execution failure modes." + ] + }, + { + "file": "scripts/check_nextest_ci_profile_no_fail_fast.sh", + "line": 204, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "The guard script scripts/check_nextest_ci_profile_no_fail_fast.sh implements anti-vacuous structural TOML validation that correctly asserts fail-fast = false is set under [profile.ci] and passes its comprehensive 30-row self-test table covering true/false/absent cases and malformed inputs." + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [], + "auto_merge": { + "checked": true, + "was_armed": false, + "disarmed": false, + "note": "auto-merge not armed" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/roadmaps/entries/PMAT-3587.yaml b/docs/roadmaps/entries/PMAT-3587.yaml new file mode 100644 index 0000000000..5d77bde6a0 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3587.yaml @@ -0,0 +1,17 @@ +- id: PMAT-3587 + github_issue: 3587 + item_type: task + title: 'nextest fail-fast discards verdicts: one failure hid 15 unmeasured commands and the shard read as ''1 failure'', not ''1 failure + 15 Unknown''' + status: planned + priority: critical + assigned_to: null + created: 2026-09-20T12:37:48Z + updated: 2026-09-20T12:37:48Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (hand-entered from issue #3587 "What lands"; minted on the PR branch so the AD-04 quorum can run -- pmat work status refuses a missing item). Two halves: the explicit-command runner (#3616, merged) and the nextest [profile.ci] key (#3626). Lands: 1. **--no-fail-fast on the shard.** The cost is runner minutes on an already-red shard; the benefit is that a red run still measures everything else it was going to measure — which is exactly when you most want the data. 2. **The reducer reports ran / skipped / total** in the shard receipt. The skipped count is a **number**, never an absence. 3. **A receipt with skipped > 0 is Unknown, not Fail and not Pass** — it names what was not measured, per doctrine 4. ' diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index 12cdaa22ce..6c7d059051 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -18869,6 +18869,23 @@ roadmap: labels: - kind:code notes: 'ACCEPTANCE (hand-entered; pmat work add derived neither spec: nor acceptance_criteria: — defect paiml-mcp-agent-toolkit#1414). Spec: docs/specifications/ruling-receipts-under-contract.md (operator ruling 2026-09-20). Done when: contracts/parity-receipt-v1.yaml with its shape on origin/main; extract:parity-receipt implemented; pv lint --gate shapes --path evidence/ in the required check; PR body shows 7 older receipts RED before back-fill and 0 after, plant violation = 1, mutation RED, pc_shape fired; check_parity_receipt.sh folded into the shape or its remainder listed under not_expressible; ONT-4c3 bound with the parity receipt as focus node; verdict rendered on a fleet host once the pv pin lands, else checkout-only: true. Sigma parent bound: json. Back-fill denominator: 7. STOP: subset-insufficient; shared-file-touched without the guard label; any threshold typed into the shape instead of resolved from thresholds.yaml.' +- id: PMAT-3587 + github_issue: 3587 + item_type: task + title: 'nextest fail-fast discards verdicts: one failure hid 15 unmeasured commands and the shard read as ''1 failure'', not ''1 failure + 15 Unknown''' + status: planned + priority: critical + assigned_to: null + created: 2026-09-20T12:37:48Z + updated: 2026-09-20T12:37:48Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (hand-entered from issue #3587 "What lands"; minted on the PR branch so the AD-04 quorum can run -- pmat work status refuses a missing item). Two halves: the explicit-command runner (#3616, merged) and the nextest [profile.ci] key (#3626). Lands: 1. **--no-fail-fast on the shard.** The cost is runner minutes on an already-red shard; the benefit is that a red run still measures everything else it was going to measure — which is exactly when you most want the data. 2. **The reducer reports ran / skipped / total** in the shard receipt. The skipped count is a **number**, never an absence. 3. **A receipt with skipped > 0 is Unknown, not Fail and not Pass** — it names what was not measured, per doctrine 4. ' - id: PMAT-3627 github_issue: 3627 item_type: task diff --git a/scripts/check_nextest_ci_profile_no_fail_fast.sh b/scripts/check_nextest_ci_profile_no_fail_fast.sh new file mode 100755 index 0000000000..406b360b2c --- /dev/null +++ b/scripts/check_nextest_ci_profile_no_fail_fast.sh @@ -0,0 +1,244 @@ +#!/usr/bin/env bash +# check_nextest_ci_profile_no_fail_fast.sh -- nextest's [profile.ci] must declare +# fail-fast = false, explicitly (PMAT-3587, the nextest half). +# +# THE DEFECT. fail-fast is a verdict-discarding default: on the first failing test +# nextest cancels every test still queued, and the shard reports the one failure with +# no count of what it did not measure. Measured on a 6-test probe whose 2nd test fails: +# fail-fast = true Summary 3/6 tests run: 2 passed, 1 failed +# Cancelling due to test failure: 1 test still running +# fail-fast = false Summary 6 tests run: 5 passed, 1 failed +# (key absent) Summary 3/6 tests run -- nextest's DEFAULT is fail-fast ON +# The tree paid for this once already: "nextest fail-fast hid every other dark failure: +# 7 rounds for 4 tests." A red run that still measures everything else it was going +# to measure is exactly the run whose data is most wanted. +# +# WHY THIS GUARD READS A TOML KEY AND DOES NOT RUN NEXTEST. The property IS the config +# value, read structurally, under the profile CI actually uses -- not a substring +# anywhere in the file. A guard that invoked the build tool would match guard_tree.sh's +# CARGO_RE, be excluded from its --no-cargo run, and need its own ci.yml step; the +# behavioural proof above was run once and recorded in the PR that landed this. +# +# ENV IS NEVER 1. The first cut imported tomllib (python 3.11+) at module level. On +# intel-clean-room-6 (#3626, job 106169486109) python3 is older, the import traceback +# exited 1, and the guard reported "the real .config/nextest.toml is RED" -- an +# environment death read as a code defect; rows 2-7 of the self-test even "passed", +# because they wanted rc=1 anyway. Now: tomllib, else tomli, else a purpose-built reader +# for the one key this guard judges; the shell side accepts a verdict only when the +# judge SAID one (a VERDICT line), and anything else -- a traceback, a dead +# interpreter -- is ENV rc=2 naming the runner. --self-test runs the whole case table +# twice, once per reader, plus the death rows. +# +# check_nextest_ci_profile_no_fail_fast.sh judge .config/nextest.toml +# check_nextest_ci_profile_no_fail_fast.sh --self-test the case table (fixtures, no cargo) +set -uo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" || exit 2 +CONF="${NEXTEST_CONF_OVERRIDE:-$ROOT/.config/nextest.toml}" +# test seams (read per call, so a self-test row can set them): NEXTEST_GUARD_PYTHON, the +# interpreter (a dead one reproduces the intel shape); NEXTEST_GUARD_FORCE_FALLBACK=1, a +# real import failure of both TOML libraries (reproduces an old python on a new one). + +# judge -> 0 when [profile.ci].fail-fast is literally false; 1 otherwise; 2 ENV. +judge() { + local f=$1 out rc=0 PYTHON="${NEXTEST_GUARD_PYTHON:-python3}" + [ -r "$f" ] || { printf 'ENV %s: not readable -- cannot judge, not a pass\n' "$f" >&2; return 2; } + out=$(NEXTEST_GUARD_RUNNER="${RUNNER_NAME:-unknown}" "$PYTHON" - "$f" 2>&1 <<'PY' +import os, re, sys +p = sys.argv[1] +runner = os.environ.get("NEXTEST_GUARD_RUNNER", "unknown") +if os.environ.get("NEXTEST_GUARD_FORCE_FALLBACK") == "1": + # make `import tomllib` / `import tomli` raise for real, so the fallback is the + # path an old interpreter takes and not a flag that skips around it + sys.modules["tomllib"] = None + sys.modules["tomli"] = None + +def verdict(code, tag, msg): + print("VERDICT %s %s: %s" % (tag, p, msg)) + sys.exit(code) + +def load_with_lib(path): + """(profile.ci table or None, reader name) via tomllib/tomli; None,None when neither imports.""" + for name in ("tomllib", "tomli"): + try: + mod = __import__(name) + except ImportError: + continue + with open(path, "rb") as fh: + d = mod.load(fh) + ci = d.get("profile", {}).get("ci") + if ci is not None and not isinstance(ci, dict): + raise ValueError("profile.ci is a %s, not a table" % type(ci).__name__) + return ci, name + return None, None + +_QUOTED = re.compile(r'"(?:[^"\\]|\\.)*"|\'[^\']*\'') +_HEADER = re.compile(r'^\[(\[)?\s*([A-Za-z0-9_.\-"\' ]+?)\s*\](\])?$') +_KEY = re.compile(r'^([A-Za-z0-9_\-]+|"[^"]+"|\'[^\']+\')\s*=\s*(.*)$') + +def load_minimal(path): + """Purpose-built reader for ONE question: under the table [profile.ci], what is the + value of fail-fast? Table headers set the current table; every other line must be a + single-line `key = value` that is balanced and opens no multi-line construct -- such + lines are only SKIPPED unless they are the judged key. Anything it does not + understand raises, and the caller reports ENV: it never guesses. Returns + (None, ...) when no [profile.ci] table exists, else the dict of judged keys.""" + tables = {} + cur = None + with open(path, "r", encoding="utf-8") as fh: + for n, raw in enumerate(fh, 1): + line = raw.strip() + if not line or line.startswith("#"): + continue + m = _HEADER.match(line) + if m: + aot = bool(m.group(1)) or bool(m.group(3)) + if bool(m.group(1)) != bool(m.group(3)): + raise ValueError("line %d: unbalanced table header" % n) + name = m.group(2).replace('"', "").replace("'", "") + if aot and name == "profile.ci": + raise ValueError("line %d: [[profile.ci]] is an array of tables, not the profile" % n) + cur = name + ("[]" if aot else "") + if not aot and cur in tables: + raise ValueError("line %d: duplicate table [%s]" % (n, name)) + tables.setdefault(cur, {}) + continue + m = _KEY.match(line) + if not m: + raise ValueError("line %d: not a table header or a `key = value` line" % n) + key, val = m.group(1).strip('"\''), m.group(2) + if "." in key: + raise ValueError("line %d: dotted key %r is outside this reader's scope" % (n, key)) + if val.startswith('"""') or val.startswith("'''"): + raise ValueError("line %d: multi-line string is outside this reader's scope" % n) + bare = _QUOTED.sub("", val) + if bare.count("[") != bare.count("]") or bare.count("{") != bare.count("}"): + raise ValueError("line %d: value continues past the line -- outside this reader's scope" % n) + if bare.count('"') % 2 or bare.count("'") % 2: + raise ValueError("line %d: unterminated string" % n) + if cur is None: + raise ValueError("line %d: top-level key %r is outside this reader's scope" % (n, key)) + if cur != "profile.ci": + continue + v = bare.split("#", 1)[0].strip() if not val.lstrip().startswith(('"', "'")) else val.strip() + if v == "true": + v = True + elif v == "false": + v = False + elif re.match(r'^-?\d+$', v): + v = int(v) + elif len(v) >= 2 and v[0] == v[-1] and v[0] in "\"'": + v = v[1:-1] + elif key == "fail-fast": + raise ValueError("line %d: fail-fast = %r is not a value this reader can type" % (n, v)) + if key in tables[cur]: + raise ValueError("line %d: duplicate key %r" % (n, key)) + tables[cur][key] = v + return tables.get("profile.ci") + +try: + try: + prof, how = load_with_lib(p) + if how is None: + prof, how = load_minimal(p), "purpose-built reader (no tomllib/tomli on this interpreter)" + except Exception as e: + verdict(2, "ENV", "cannot be parsed on runner=%s (%s: %s) -- cannot judge, not a pass and not a fail" + % (runner, type(e).__name__, e)) + if prof is None: + verdict(1, "FAIL", "no [profile.ci] -- CI runs --profile ci, so the run would take nextest's default, " + "which is fail-fast ON (reader=%s)" % how) + v = prof.get("fail-fast") + if v is None: + verdict(1, "FAIL", "[profile.ci] does not set fail-fast -- nextest's default is ON (measured: 3/6 tests " + "run on a 6-test probe) (reader=%s)" % how) + if v is not False: + verdict(1, "FAIL", "[profile.ci].fail-fast = %r -- a failing test cancels every test still queued and " + "their verdicts are discarded (reader=%s)" % (v, how)) + verdict(0, "ok", "[profile.ci].fail-fast = false -- a red run still measures everything else " + "(reader=%s, runner=%s)" % (how, runner)) +except SystemExit: + raise +except BaseException as e: # the judge itself died: say so, as ENV + verdict(2, "ENV", "judge crashed on runner=%s (%s: %s) -- cannot judge, not a pass and not a fail" + % (runner, type(e).__name__, e)) +PY + ) || rc=$? + # A verdict is something the judge SAID. rc=1 with no VERDICT line is the interpreter + # dying (a traceback before the first statement, a SyntaxError on an old python, no + # python3 at all) -- the exact shape that read as "RED" on intel-clean-room-6. + case "$out" in + *"VERDICT ok "*|*"VERDICT FAIL "*|*"VERDICT ENV "*) ;; + *) printf 'ENV %s: the judge did not judge (interpreter=%s rc=%s runner=%s) -- not a pass and not a fail\n%s\n' \ + "$f" "$PYTHON" "$rc" "${RUNNER_NAME:-unknown}" "$out" >&2; return 2 ;; + esac + case "$rc" in + 0) printf '%s\n' "${out#VERDICT }" ;; + 1|2) printf '%s\n' "${out#VERDICT }" >&2 ;; + *) printf 'ENV %s: judge rc=%s is not a verdict (runner=%s)\n%s\n' "$f" "$rc" "${RUNNER_NAME:-unknown}" "$out" >&2; return 2 ;; + esac + return "$rc" +} + +case "${1:-}" in -h|--help) sed -n '2,33p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; esac + +if [ "${1:-}" = "--self-test" ]; then + echo "=== nextest [profile.ci] fail-fast guard: case table ===" + d=$(mktemp -d) || exit 2 + rmtree() { case "${1:-}" in ''|/) return 0 ;; *) [ -d "$1" ] && rm -rf -- "$1" ;; esac; return 0; } + trap 'rmtree "${d:-}"' EXIT + bad=0; n=0 + row() { # row WANT_RC LABEL TOML-BODY (reader named by $READER, set by table()) + local want=$1 label=$2 body=$3 rc=0; n=$((n + 1)) + printf '%s' "$body" > "$d/c.toml" + judge "$d/c.toml" > /dev/null 2>&1 || rc=$? + if [ "$rc" = "$want" ]; then printf 'ok row %-2s rc=%s [%s] %s\n' "$n" "$rc" "$READER" "$label" + else printf 'FAIL row %-2s rc=%s (wanted %s) [%s] %s\n' "$n" "$rc" "$want" "$READER" "$label" >&2; bad=1; fi + } + table() { # the same table under BOTH readers; an old-python runner must judge the same property + row 0 "fail-fast = false under [profile.ci] -> PASS" $'[profile.ci]\nretries = 2\nfail-fast = false\n' + row 1 "fail-fast = true under [profile.ci] -> RED" $'[profile.ci]\nretries = 2\nfail-fast = true\n' + row 1 "key ABSENT under [profile.ci] -> RED (nextest default is ON)" $'[profile.ci]\nretries = 2\n' + row 1 "false under a DIFFERENT profile only -> RED" $'[profile.default]\nfail-fast = false\n[profile.ci]\nretries = 2\n' + row 1 "no [profile.ci] at all -> RED" $'[profile.default]\nfail-fast = false\n' + row 1 "false in a COMMENT, true in the key -> RED (not a substring test)" $'[profile.ci]\n# fail-fast = false\nfail-fast = true\n' + row 1 "the string \"false\" (a string, not a bool) -> RED" $'[profile.ci]\nfail-fast = "false"\n' + row 1 "false under [[profile.ci.overrides]] only -> RED (an override entry is not the profile)" \ + $'[profile.ci]\nretries = 2\n[[profile.ci.overrides]]\nfilter = "test(/x/)"\nfail-fast = false\n' + row 0 "false after an inline table and before a sub-table -> PASS (the real file's shape)" \ + $'[profile.ci]\nretries = 2\nfail-fast = false\nslow-timeout = { period = "60s", terminate-after = 20 }\nstatus-level = "slow"\n[profile.ci.junit]\npath = "junit.xml"\n' + row 2 "[[profile.ci]] (an array of tables, not the profile) -> ENV rc=2, never a pass" $'[[profile.ci]]\nfail-fast = false\n' + row 2 "unparseable TOML -> ENV rc=2, never a pass" $'[profile.ci\nfail-fast = false\n' + # and the real config, so a red tree cannot hide behind green fixtures + n=$((n + 1)); rc=0; judge "$CONF" > /dev/null 2>&1 || rc=$? + [ "$rc" -eq 0 ] && printf 'ok row %-2s rc=0 [%s] the real %s is GREEN\n' "$n" "$READER" "${CONF#"$ROOT/"}" \ + || { printf 'FAIL row %-2s rc=%s [%s] the real %s is RED\n' "$n" "$rc" "$READER" "${CONF#"$ROOT/"}" >&2; bad=1; } + } + READER=library; table + READER=fallback; NEXTEST_GUARD_FORCE_FALLBACK=1 table + # what the purpose-built reader must REFUSE rather than guess (ENV, never a pass) + READER=fallback + NEXTEST_GUARD_FORCE_FALLBACK=1 row 2 "a multi-line string hiding a [profile.ci] header -> ENV, never a guess (a line reader that skipped it would say PASS)" \ + $'[profile.default]\nnote = """title"\n[profile.ci]\nfail-fast = false\nx = """"\n' + NEXTEST_GUARD_FORCE_FALLBACK=1 row 2 "a value continuing past its line -> ENV, never a guess" \ + $'[profile.ci]\nfail-fast = false\nslow-timeout = { period = "60s",\n terminate-after = 20 }\n' + NEXTEST_GUARD_FORCE_FALLBACK=1 row 2 "fail-fast = fals (not a value it can type) -> ENV, never a guess" \ + $'[profile.ci]\nfail-fast = fals\n' + # the death rows: the shape that read as RED on intel-clean-room-6 + n=$((n + 1)); rc=0; printf '[profile.ci]\nfail-fast = false\n' > "$d/c.toml" + NEXTEST_GUARD_PYTHON=/bin/false judge "$d/c.toml" > /dev/null 2>&1 || rc=$? + [ "$rc" -eq 2 ] && printf 'ok row %-2s rc=2 interpreter exits 1 with no verdict -> ENV rc=2, never 1\n' "$n" \ + || { printf 'FAIL row %-2s rc=%s (wanted 2) interpreter exits 1 with no verdict must be ENV, not RED\n' "$n" "$rc" >&2; bad=1; } + n=$((n + 1)); rc=0 + NEXTEST_GUARD_PYTHON="$d/no-such-interpreter" judge "$d/c.toml" > /dev/null 2>&1 || rc=$? + [ "$rc" -eq 2 ] && printf 'ok row %-2s rc=2 no interpreter at all (rc=127) -> ENV rc=2\n' "$n" \ + || { printf 'FAIL row %-2s rc=%s (wanted 2) no interpreter must be ENV\n' "$n" "$rc" >&2; bad=1; } + n=$((n + 1)); rc=0; judge "$d/absent.toml" > /dev/null 2>&1 || rc=$? + [ "$rc" -eq 2 ] && printf 'ok row %-2s rc=2 missing file -> ENV rc=2, never a pass\n' "$n" \ + || { printf 'FAIL row %-2s rc=%s (wanted 2) missing file -> ENV\n' "$n" "$rc" >&2; bad=1; } + [ "$bad" -eq 0 ] && { printf 'SELF-TEST PASSED: %s rows\n' "$n"; exit 0; } + printf 'SELF-TEST FAILED\n' >&2; exit 1 +fi + +echo "=== nextest [profile.ci] must not discard verdicts on the first failure (check_nextest_ci_profile_no_fail_fast.sh) ===" +judge "$CONF"; rc=$? +[ "$rc" -eq 0 ] && echo "PASS" || echo "FAIL (rc=$rc)" >&2 +exit "$rc"