From 326a6d0756b8f505c7e14d1a41c187459c4292a8 Mon Sep 17 00:00:00 2001 From: Raj Madhivanan Date: Fri, 11 Sep 2026 10:56:42 -0700 Subject: [PATCH 01/12] feat: add safety traceability linter Co-authored-by: OpenCode --- .github/workflows/safety-lint.yml | 22 + .../change-0001-safety-traceability-linter.md | 585 ++++++++++++++++++ docs/safety/TRACEABILITY.md | 11 +- docs/safety/lint-baseline.json | 76 +++ tools/safety_lint/__init__.py | 3 + tools/safety_lint/__main__.py | 112 ++++ tools/safety_lint/checks.py | 277 +++++++++ tools/safety_lint/coverage.py | 45 ++ tools/safety_lint/fixtures/repository/code.c | 3 + .../fixtures/repository/docs/safety/FMEA.md | 6 + .../fixtures/repository/docs/safety/HARA.md | 6 + .../docs/safety/SAFETY_REQUIREMENTS.md | 15 + .../docs/safety/SYSTEM_DEFINITION.md | 12 + .../repository/docs/safety/TRACEABILITY.md | 29 + .../repository/docs/safety/lint-baseline.json | 5 + .../firmware/test/test_estop_verdict.c | 3 + .../src/pstop/requirements/req_2_02_test.c | 3 + .../src/pstop/requirements/req_2_03_test.c | 3 + .../test/test_json_lite.cpp | 3 + .../repository/tests/test_unique_probe.py | 3 + .../repository/tools/hil/test_10_button.py | 3 + .../tools/hil/test_20_discordance.py | 3 + .../tools/hil/test_30_power_cycle.py | 3 + .../tools/pstop_multi_remote_test.py | 3 + tools/safety_lint/model.py | 88 +++ tools/safety_lint/parse_srs.py | 138 +++++ tools/safety_lint/parse_system_definition.py | 32 + tools/safety_lint/parse_traceability.py | 283 +++++++++ tools/safety_lint/render.py | 57 ++ tools/safety_lint/runner.py | 35 ++ tools/safety_lint/self_test.py | 468 ++++++++++++++ 31 files changed, 2334 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/safety-lint.yml create mode 100644 changes/change-0001-safety-traceability-linter.md create mode 100644 docs/safety/lint-baseline.json create mode 100644 tools/safety_lint/__init__.py create mode 100644 tools/safety_lint/__main__.py create mode 100644 tools/safety_lint/checks.py create mode 100644 tools/safety_lint/coverage.py create mode 100644 tools/safety_lint/fixtures/repository/code.c create mode 100644 tools/safety_lint/fixtures/repository/docs/safety/FMEA.md create mode 100644 tools/safety_lint/fixtures/repository/docs/safety/HARA.md create mode 100644 tools/safety_lint/fixtures/repository/docs/safety/SAFETY_REQUIREMENTS.md create mode 100644 tools/safety_lint/fixtures/repository/docs/safety/SYSTEM_DEFINITION.md create mode 100644 tools/safety_lint/fixtures/repository/docs/safety/TRACEABILITY.md create mode 100644 tools/safety_lint/fixtures/repository/docs/safety/lint-baseline.json create mode 100644 tools/safety_lint/fixtures/repository/firmware/test/test_estop_verdict.c create mode 100644 tools/safety_lint/fixtures/repository/pstop_c/pstop/test/src/pstop/requirements/req_2_02_test.c create mode 100644 tools/safety_lint/fixtures/repository/pstop_c/pstop/test/src/pstop/requirements/req_2_03_test.c create mode 100644 tools/safety_lint/fixtures/repository/ros2/protective_stop_machine/test/test_json_lite.cpp create mode 100644 tools/safety_lint/fixtures/repository/tests/test_unique_probe.py create mode 100644 tools/safety_lint/fixtures/repository/tools/hil/test_10_button.py create mode 100644 tools/safety_lint/fixtures/repository/tools/hil/test_20_discordance.py create mode 100644 tools/safety_lint/fixtures/repository/tools/hil/test_30_power_cycle.py create mode 100644 tools/safety_lint/fixtures/repository/tools/pstop_multi_remote_test.py create mode 100644 tools/safety_lint/model.py create mode 100644 tools/safety_lint/parse_srs.py create mode 100644 tools/safety_lint/parse_system_definition.py create mode 100644 tools/safety_lint/parse_traceability.py create mode 100644 tools/safety_lint/render.py create mode 100644 tools/safety_lint/runner.py create mode 100755 tools/safety_lint/self_test.py diff --git a/.github/workflows/safety-lint.yml b/.github/workflows/safety-lint.yml new file mode 100644 index 00000000..2c99c27e --- /dev/null +++ b/.github/workflows/safety-lint.yml @@ -0,0 +1,22 @@ +--- +name: Safety traceability lint + +on: + push: + branches: [main] + pull_request: + +jobs: + safety_lint: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + submodules: recursive + + - name: Safety linter self-tests + run: python3 tools/safety_lint/self_test.py + + - name: Safety traceability check + run: python3 -m tools.safety_lint --check diff --git a/changes/change-0001-safety-traceability-linter.md b/changes/change-0001-safety-traceability-linter.md new file mode 100644 index 00000000..5d00bf54 --- /dev/null +++ b/changes/change-0001-safety-traceability-linter.md @@ -0,0 +1,585 @@ +# change-0001 — Safety traceability linter: machine-checked requirements coverage + +**Implements:** No ADR. Authority is the safety chain itself — +`docs/safety/SAFETY_REQUIREMENTS.md`, `docs/safety/TRACEABILITY.md`, +`docs/safety/SYSTEM_DEFINITION.md`. +**Branch:** `change-0001-safety-traceability-linter` +**Base:** `main` +**Status:** implementation complete; PR CI pending +**Safety class:** B — assurance tooling and CI only; no runtime safety-path, +interface, timing, state-machine, protocol, or requirement change. +**Authorization:** one authorizer approved implementation with the exact instruction +`proceed` on 2026-09-11. + +Read `docs/safety/TRACEABILITY.md` §1 (Method) and §3 (coverage summary) in full +before starting, then `docs/safety/SAFETY_REQUIREMENTS.md` §1 (numbering and +conventions). Those documents are the authority; this document is the work +breakdown. Where the two disagree, the safety documents win — stop and flag the +conflict rather than choosing. + +The repository is the source of truth for the safety case. Where any Notion page +disagrees with a file under `docs/safety/`, the file wins +(`docs/safety/RECONCILIATION.md` establishes this). + +The design decisions in §1 and §1a are settled and are not open for +reinterpretation during implementation. + +--- + +## 1. What this change delivers + +A checked-in linter that parses the existing safety markdown, asserts +bidirectional traceability consistency across `SAFETY_REQUIREMENTS.md`, +`TRACEABILITY.md` and `SYSTEM_DEFINITION.md`, computes the requirements-coverage +numbers, regenerates the §3 summary block in `TRACEABILITY.md`, and fails CI when +a change breaks traceability or silently drops a requirement's verification. + +**Settled decision 1 — markdown is canonical; the linter parses it.** The SR +tables carry hand-authored safety argument in the Status column (reconciliation +dates, commit hashes, named residuals). Extracting them into YAML and generating +the markdown back would flatten that prose into quoted strings and make review +worse, not better. The linter reads the tables and never rewrites them, with one +exception: the generated block in §4d. + +**Settled decision 2 — stdlib only.** No `pyyaml`, no `pytest`, no new package +manager. The repo has no Python packaging (`pyproject.toml` absent) and its +existing Python tools are standalone `python3` scripts. Baseline and output files +are JSON, which the `polymath-json` pre-commit hook already covers. + +**Settled decision 3 — ratchet, not gate.** The tree does not pass these checks +today: five functions carry no requirement, and `OPEN_ITEMS.md` and +`TRACEABILITY.md` disagree on the requirement count. A baseline file records every +pre-existing violation with a reason. CI fails on new violations only, and fails +when a baselined violation is fixed but not removed from the baseline. + +### 1a. Ratified implementation clarifications (2026-09-11) + +1. Define the SRS status vocabulary in `checks.py` as `Satisfied`, + `Partially satisfied`, `Gap`, and `Residual-accepted`, longest-match-first. + Do not derive it from the conventions prose and do not edit that prose. Emit an + informational finding when a status in use is absent from the conventions list. +2. The CI workflow has no path filter. It runs on every pull request and every push + to `main` so moved or deleted evidence cannot escape the check. +3. Baseline the broken `test_timing_floors` citation with owner `raj`. Git history + establishes that `test_timing_floors.cpp` was added by `1f226be` and deleted by + `9a28d4a` during the ROS 2 convention restoration. Record those facts without + deciding whether SR-M-01 remains Verified. +4. Do not rearrange `TRACEABILITY.md`. Use multiple named generated marker pairs + around purely numeric regions. A bullet mixing numbers with hand-authored prose + remains outside markers and is checked rather than rewritten. If markers cannot + meet that rule, stop and quote the resisting text. +5. Evidence resolves only through the documented shorthand, an explicit repository + path, a repository-wide unique filename or test-source stem, or an existing HIL + or evidence report that names the cited SR. An ambiguous stem is a finding. Never + choose the nearest match and never infer evidence from prose. +6. The linter measures citations, not passing execution. Generated output is labelled + "SRs with at least one cited verifying test" and includes a footnote that citation + resolution, not test execution, is checked. Existing "passing" wording is not + edited. If the generated label conflicts with surrounding prose, stop and report. + If computed citation coverage differs from committed 32/40, report both values and + the per-SR delta and change nothing. + +--- + +## 2. Scope + +### IN scope + +1. `tools/safety_lint/` — parsers for the three safety documents. +2. Nine consistency checks (§4c), each with an error/warning severity. +3. Coverage computation reproducing the §3.1 and §3.2 numbers from the parsed + data. +4. Generated-block rendering into `TRACEABILITY.md` §3 between HTML markers, with + a `--check` mode that fails when the committed block is stale. +5. `docs/safety/lint-baseline.json` — recorded pre-existing violations with + reasons and owners. +6. `.github/workflows/safety-lint.yml` — runs the linter on every PR and on every + push to `main`, without path filters. +7. `tools/safety_lint/self_test.py` — the linter's own test suite, stdlib + `unittest`, run by the same workflow. + +### OUT of scope — do NOT build + +- **Any edit to the safety argument itself.** Do not add, remove, reword or + re-status a requirement. Do not change a Status cell to make a check pass. If a + check fails on real content, baseline it and report it. +- Any file under `docs/safety/` other than approved generated/check-only markers in + `TRACEABILITY.md` §3 and the new `lint-baseline.json`. In particular, do not edit + the SRS conventions section. +- Anything under `pstop_c/` — certified library, separate upstream track, + excluded from pre-commit for that reason. +- Anything under `ros2/` — governed by ament linters. +- Line-number verification of `file:line` code citations. Files are checked to + exist; line numbers drift constantly and are explicitly documented as "±a few + lines as the tree evolves" (`SAFETY_REQUIREMENTS.md` §1). Do not assert on them. +- Parsing `FMEA.md`, `HARA.md` or `FMEDA.md` beyond extracting bare ID sets for + check C7. +- Structural-coverage integration. `COVERAGE.md` and `docs/safety/coverage/host-summary.json` + stay out; this change measures requirements coverage only. +- The modification procedure migration from Notion. Separate change. +- Any CI gate on `main` branch protection. The workflow is added; enabling it as a + required check is a repo-admin action, not this change. + +--- + +## 3. Integration facts (do not rediscover these) + +| Fact | Value | +|---|---| +| Requirements spec | `docs/safety/SAFETY_REQUIREMENTS.md` | +| Traceability matrix | `docs/safety/TRACEABILITY.md` | +| Function decomposition | `docs/safety/SYSTEM_DEFINITION.md` §4 (`## 4. Function / item decomposition`, line 95) | +| FMEA (DU register) | `docs/safety/FMEA.md` §3 | +| HARA (safety goals SG-1..6, hazards H-nn) | `docs/safety/HARA.md` §5 | +| Reconciliation record ("code wins") | `docs/safety/RECONCILIATION.md` | +| Open items register | `docs/safety/OPEN_ITEMS.md` | +| SR ID grammar | `SR--`; areas `SYS`, `R`, `H`, `M`, `I` (`SAFETY_REQUIREMENTS.md` §1) | +| SR table columns | `ID · Requirement (shall) · Derived from · Allocated to · Integrity · Verify · Status` | +| SR section headings | `## 2.` SR-SYS, `## 3.` SR-R (subsections `### 3.1`, `### 3.2`), `## 4.` SR-H, `## 5.` SR-M, `## 6.` SR-I | +| SRS status vocabulary | `Satisfied`, `Partially satisfied`, `Gap`, `Residual-accepted` — bold-wrapped, often followed by parenthetical prose | +| Traceability table columns | `SR · Alloc F-xx · Code (file:line) · Verifying test(s) · Method · Status` | +| Traceability section headings | `## 2. Traceability matrix` with `### 2.1`–`### 2.5`; `## 3. Requirements coverage summary`; `## 4. Test-gap register`; `## 5. Function → SR reverse map` with `### 5.1`–`### 5.x` | +| Traceability status vocabulary | `Verified`, `Partially-verified`, `Unverified-gap`, `Residual-accepted` (defined in a table under `## 1. Method`) | +| No-test marker | The literal string `NO TEST`, bold-wrapped, in the Verifying test(s) cell | +| Test-file shorthand legend | `TRACEABILITY.md` §2 preamble: `EV`, `MR`, `HIL10/20/30`, `JL`, `REQ n_nn` — each maps to a real path | +| `EV` | `firmware/test/test_estop_verdict.c` | +| `MR` | `tools/pstop_multi_remote_test.py` | +| `HIL10/20/30` | `tools/hil/test_10_button.py`, `test_20_discordance.py`, `test_30_power_cycle.py` | +| `JL` | `ros2/protective_stop_machine/test/test_json_lite.cpp` | +| `REQ n_nn` | `pstop_c/pstop/test/src/pstop/requirements/req_n_nn_test.c` | +| Existing CI guard script pattern | `scripts/check_estop_diversity.sh` — SPDX header, comment block naming the SR and DU it guards, `set -uo pipefail`, exit 2 for "cannot run", exit 1 for "check failed" | +| How a guard is wired to CI | `.github/workflows/firmware-build.yml:46-52` | +| Existing standalone Python test pattern | `tools/test_config_floor.py` — `#!/usr/bin/env python3`, SPDX header, module docstring naming the SR and DU | +| Existing workflows | `pre-commit.yml`, `firmware-build.yml`, `host-check.yml`, `ros2_build.yml`, `pstop_c_build.yml`, `pstop_c_coverage.yml`, `coverage.yml` | +| Checkout action version in use | `actions/checkout@v7` | +| Pre-commit config | `.pre-commit-config.yaml`, `polymath_code_standard` v2.2.0; hooks include `polymath-python`, `polymath-json`, `polymath-markdown`, `polymath-copyright` | +| Pre-commit exclusions | `pstop_c/`, `ros2/`, `archive/`, vendored wireguard/x25519, `hardware/` binaries. `tools/` and `docs/` are NOT excluded — new files must satisfy the hooks | +| Required file header | SPDX two-line header, Apache-2.0, matching `tools/test_config_floor.py` and `scripts/check_estop_diversity.sh` | +| Markdown hook caveat | `OPEN_ITEMS.md` §8 records that the markdown hook rewrites `-` list markers to `+` and that repo-wide normalization is deliberately deferred. Do not let the hook reformat `TRACEABILITY.md` wholesale — only the generated block changes | + +### The requirement count disagreement + +`TRACEABILITY.md` §3.1 states 40 safety requirements. `OPEN_ITEMS.md` §5 states +39 with a different coverage fraction, dated earlier. **Do not resolve this by +editing either document.** Piece 4 computes the count from the parsed SR tables. +Report the computed number. If it is neither 39 nor 40, stop and raise before +proceeding — the parser is wrong, not the documents. + +--- + +## 4. Pieces, in dependency order + +Each piece is committable on its own. Pieces 1–3 add no CI surface and cannot +break the build. Piece 5 is the first piece that can fail a PR. + +--- + +### Piece 1 — SR parser + +**Files:** `tools/safety_lint/__init__.py`, `tools/safety_lint/parse_srs.py`, +`tools/safety_lint/model.py`. + +**1a. Record types** in `model.py`, frozen dataclasses, stdlib only: + +```python +@dataclass(frozen=True) +class SafetyRequirement: + sr_id: str # "SR-R-03" + area: str # "SYS" | "R" | "H" | "M" | "I" + number: int # 3 + shall_text: str # raw cell, markdown intact + derived_from: tuple[str, ...] # ("SG-1", "H-09", "DU-3") + allocated_to: tuple[str, ...] # ("F-R-02",) + integrity: str # raw cell + verify_methods: tuple[str, ...] # ("Fault-injection", "Test") + status: str # normalized token from the SRS vocabulary + status_prose: str # everything after the token + source_line: int # 1-based line in SAFETY_REQUIREMENTS.md +``` + +**1b. Table discovery.** Walk the file line by line. A requirements table is a +markdown pipe table whose header row's first cell is `ID` and which contains a +cell `Derived from`. Do not hardcode section numbers — `SAFETY_REQUIREMENTS.md` +has subsections (`### 3.1`, `### 3.2`) and more may be added. Ignore every other +pipe table in the file, including the ID-grammar table in §1. + +**1c. Cell splitting.** Cells are separated by unescaped `|`. Cells contain inline +code spans with pipes inside them in at least one row — split on `|` only when not +inside a backtick span. Trim, then strip surrounding `**`. + +**1d. ID parse.** `SR-(SYS|R|H|M|I)-(\d{2})`, taken from the bolded first cell. +An ID that does not match is an error, not a skip. + +**1e. Reference extraction.** From `derived_from`, extract every token matching +`SG-\d`, `H-\d{2}`, `DU-\d`, and FMEA cross-refs of the form `[A-Z]\d{2}-\d`. +From `allocated_to`, extract every `F-[A-Z]-\d{2}`. Expand the range form +`F-R-01..05` into its members. Expand the slash form `F-M-03/04` into +`F-M-03`, `F-M-04`. Both forms occur. + +**1f. Status normalization.** Match the leading bolded token against the SRS +vocabulary, case-insensitively, longest match first so `Partially satisfied` is +not read as `Satisfied`. Note that `Partially satisfied — feedback limb +DESCOPED 2026-08` occurs; everything after the token is `status_prose`. A cell +whose leading token is outside the vocabulary is an error naming the file, line +and cell. + +**Tests** (`self_test.py`, stdlib `unittest`): +- `test_parses_all_sr_areas` — parse the real `SAFETY_REQUIREMENTS.md`; assert at + least one SR in each of the five areas. +- `test_sr_ids_unique` — no duplicate `sr_id`. +- `test_allocated_to_range_expansion` — `"F-R-01..05, F-H-03"` yields six + functions including `F-R-03`. +- `test_allocated_to_slash_expansion` — `"F-M-03/04"` yields two. +- `test_status_longest_match_wins` — a cell beginning `**Partially satisfied**` + normalizes to `Partially satisfied`, never `Satisfied`. Drift-verify: reorder + the vocabulary so the short token matches first, confirm the test fails, revert. +- `test_pipe_inside_code_span_does_not_split_cell` — synthetic row containing + `` `a|b` `` parses as one cell. + +--- + +### Piece 2 — Traceability and system-definition parsers + +**Files:** `tools/safety_lint/parse_traceability.py`, +`tools/safety_lint/parse_system_definition.py`, `tools/safety_lint/model.py`. + +**2a. `TraceRow`** — `sr_id`, `allocated_to`, `code_refs`, `test_refs`, +`methods`, `status`, `status_prose`, `has_no_test_marker`, `source_line`. + +**2b. Matrix discovery.** A traceability table's header row's first cell is `SR` +and it contains a cell beginning `Verifying test`. Same subsection caveat as 1b. + +**2c. Code-reference extraction.** From the Code cell, extract tokens shaped +`path:line` or `path:line-line` or a bare backticked filename. Resolve a bare +filename against the paths named in `SYSTEM_DEFINITION.md` §4 where possible; +where not, record it as unresolved rather than failing. + +**2d. Test-reference extraction.** Expand the shorthand legend from §3 above into +real paths. Handle the group-letter form `MR[B/D/E/F]` — the path is the same, the +bracket is a group selector, so record the path once. Handle `REQ 2_02/2_03` as +two `req_*_test.c` paths. Also resolve explicit repository paths and a bare filename +or test-source stem only when it is unique across the whole repository. Ambiguity is +a finding, never a nearest-match choice. An HIL/evidence report counts only when the +file exists and names the row's SR. Never infer evidence from prose. Record the +literal `NO TEST` marker separately from test paths; a cell may contain both a +`NO TEST` phrase and real tests, and it does in several rows. + +**2e. Reverse-map parser.** Parse §5's tables into `{F-xx: (function_name, +tuple_of_sr_ids)}`. A cell whose SR list is an em-dash-led "none" phrase yields an +empty tuple plus a `declared_non_safety` flag when the phrase contains +`non-safety`. + +**2f. System-definition parser.** Parse `SYSTEM_DEFINITION.md` §4's tables into +the authoritative `F-xx` set with names. This is the spine both other documents +are checked against. + +**Tests:** +- `test_every_trace_row_has_sr_id` +- `test_test_shorthand_expands_to_existing_paths` — every expanded shorthand path + exists on disk. +- `test_mr_group_selector_yields_single_path` — `MR[B/D/E/F]` yields one path. +- `test_req_slash_form_yields_two_paths` +- `test_no_test_marker_detected_alongside_real_tests` — a cell with both records + `has_no_test_marker=True` and a non-empty `test_refs`. +- `test_reverse_map_flags_declared_non_safety` — `F-R-08` parses with an empty SR + tuple and `declared_non_safety=True`. +- `test_system_definition_function_set_nonempty` + +--- + +### Piece 3 — Consistency checks + +**Files:** `tools/safety_lint/checks.py`. + +Each check returns zero or more `Finding(check_id, severity, subject, message, +file, line)`. `severity` is `error` or `warning`. + +**C1 — SR set parity.** Every SR in `SAFETY_REQUIREMENTS.md` appears exactly once +in the traceability matrix, and vice versa. Severity: error. Subject: the SR ID. + +**C2 — Status vocabulary closed.** Every SRS status normalizes to the SRS +vocabulary; every traceability status normalizes to the traceability vocabulary. +Severity: error. + +**C3 — Status and evidence agree.** A traceability row with status `Verified` or +`Partially-verified` names at least one resolvable test path. A row with status +`Unverified-gap` carries the `NO TEST` marker and names no test path outside a +`NO TEST` clause. Severity: error. This is the check that catches a requirement +losing its test silently. + +**C4 — Cited test files exist.** Every resolved test path exists in the working +tree. Severity: error. Paths under `pstop_c/` are checked for existence but never +opened. + +**C5 — Cited code files exist.** Every resolved code path exists. Severity: +error for a missing file. Line numbers are never checked — see §2 OUT of scope. +An unresolved bare filename is severity `warning`. + +**C6 — Function allocation is real.** Every `F-xx` in any `Allocated to` cell +exists in the `SYSTEM_DEFINITION.md` §4 set. Severity: error. + +**C7 — Upstream references resolve.** Every `SG-n` appears in `HARA.md`, every +`H-nn` appears in `HARA.md`, every `DU-n` appears in `FMEA.md`. Extract bare ID +sets by regex from those files; do not parse their structure. Severity: warning — +those documents restructure and a false positive here must not block a PR. + +**C8 — Reverse map completeness.** Every `F-xx` in the system-definition set +appears in the reverse map, and the reverse map's SR list for each function equals +the set of SRs whose `Allocated to` names it. Severity: error for a disagreement, +warning for a function with no SRs at all — the latter is a real known state +(`F-H-04`, `F-M-01`, `F-M-06`) and belongs in the baseline. + +**C9 — SRS and traceability agree on allocation.** For each SR, the `Allocated +to` set in the SRS equals the `Alloc F-xx` set in the matrix. Severity: error. + +**Baseline handling.** `docs/safety/lint-baseline.json`: + +```json +{ + "generated": "2026-09-10", + "note": "Pre-existing findings accepted at linter introduction. Each entry needs an exact finding message, owner, and reason. Removing a fixed entry is required; CI fails on a stale entry.", + "findings": [ + { + "check_id": "C8", + "subject": "F-H-04", + "finding": "function has no allocated safety requirement", + "reason": "Robot status output + logging carries no SR. Known requirements-coverage hole, TRACEABILITY.md §5.2.", + "owner": "raj" + } + ] +} +``` + +A baseline entry matches exactly one finding by `(check_id, subject, finding)`, +where `finding` is the complete stable finding message. That exact finding is +downgraded to informational and reported in a separate section. A new finding on +the same check and subject remains active. Duplicate exact entries are invalid, +and a baselined entry with no exact matching finding is itself an error — this is +the ratchet. + +**Tests:** +- One `test__flags_` per check, driven by small synthetic + markdown fixtures under `tools/safety_lint/fixtures/`, not by the real + documents. The real documents are exercised in Piece 4's integration test. +- `test_baseline_suppresses_exact_matching_finding` +- `test_same_subject_new_finding_is_not_suppressed` +- `test_stale_exact_baseline_entry_is_an_error` — a baselined exact finding with + no corresponding finding produces an error. +- `test_duplicate_exact_baseline_entries_are_rejected` +- Drift-verify C3 and C8: break the guarded condition in a fixture, confirm the + test fails, revert. Report that you did it. + +--- + +### Piece 4 — Coverage computation and generated summary + +**Files:** `tools/safety_lint/coverage.py`, `tools/safety_lint/render.py`, +`docs/safety/TRACEABILITY.md` (generated block only). + +**4a. Counts.** From the parsed traceability rows, compute per area and in total: +count, and counts of `Verified`, `Partially-verified`, `Unverified-gap`, +`Residual-accepted`. + +**4b. The two headline fractions**, matching the definitions in §3.1 verbatim: +- **(a) SRs with at least one passing verifying test** = rows whose `test_refs` is + non-empty, irrespective of status. Note that this currently includes one + `Residual-accepted` row that has a test (`SR-M-06`) and excludes the other + (`SR-M-04`, inspection only). Do not special-case by status; count by evidence. +- **(b) Functions traced to at least one SR** = functions in the system-definition + set with a non-empty SR list. Report both the raw fraction and the fraction + excluding functions flagged `declared_non_safety`. +- **Strict, fully-verified** = rows with status exactly `Verified`. + +**4c. Reproduce before you replace.** Run the computation against the current +tree and compare with the committed §3.1 and §3.2 numbers. They should match. If +any figure differs, **stop and report the discrepancy with both numbers** before +touching the document. A mismatch means either the parser is wrong or the +committed summary is stale; both need a human decision, and the second is a real +finding worth surfacing. + +**4d. Generated regions.** Use multiple named marker pairs around purely numeric +regions only: + +``` + +... + +``` + +The surrounding prose in §3 — mixed numeric/prose bullets, the "Reading:" +paragraph, the `†` footnote, and bracketed reconciliation notes — is hand-authored +and stays outside the markers without being moved. Mixed bullets are checked, not +rewritten. Generated coverage is explicitly labelled "SRs with at least one cited +verifying test" and states that the linter checks citation resolution, not test +execution. Rendering replaces only marker contents. `--check` mode renders and +diffs without writing, exiting non-zero on a difference. + +**Tests:** +- `test_coverage_matches_committed_summary` — integration against the real + documents. If Piece 4c found a legitimate discrepancy, this test asserts the + computed value and carries a comment naming the discrepancy and its resolution. +- `test_generated_block_is_idempotent` — render twice, byte-identical. +- `test_check_mode_detects_stale_block` — mutate a number inside the markers, + assert `--check` exits non-zero. +- `test_render_does_not_touch_prose_outside_markers` — assert the "Reading:" + paragraph is byte-identical before and after a render. + +--- + +### Piece 5 — CLI and CI wiring + +**Files:** `tools/safety_lint/__main__.py`, `.github/workflows/safety-lint.yml`. + +**5a. CLI.** `python3 -m tools.safety_lint [--check] [--write] [--json]`, run from +the repository root. +- Default: run all checks, print a human-readable report, exit 1 on any + non-baselined error, 0 otherwise. +- `--check`: also verify the generated block is current; exit 1 if stale. +- `--write`: regenerate the block in place. +- `--json`: emit findings as JSON to stdout for future tooling. + +Exit codes follow `scripts/check_estop_diversity.sh`: `0` pass, `1` check failed, +`2` cannot run (a required document missing or unparseable). + +**5b. Report format.** Errors first, then warnings, then a baselined section, then +the coverage summary. Every finding prints `file:line: [check_id] subject — +message`. The coverage summary prints the same numbers that go into the generated +block, so a reviewer reading CI output does not need to open the diff. + +**5c. Workflow.** `.github/workflows/safety-lint.yml`, modelled on the existing +workflows: `actions/checkout@v7`, `ubuntu-latest`, triggers on every push to `main` +and every `pull_request`, with no path filters. Two steps: run `self_test.py`, then +run the linter with `--check`. + +**5d. Do not add this to `.pre-commit-config.yaml`.** The linter reads several +files and computes cross-document state; pre-commit's per-file model fits it +badly and the markdown hook's list-marker rewrite is a known landmine +(`OPEN_ITEMS.md` §8). CI only. + +**Tests:** +- `test_cli_exit_code_zero_on_clean_tree` — with the baseline in place, the real + tree exits 0. +- `test_cli_exit_code_one_on_injected_error` — copy the safety docs to a temp + directory, delete a traceability row, assert exit 1 and a C1 finding. +- `test_cli_exit_code_two_on_missing_document` +- Workflow: confirm it runs and passes on the PR that introduces it. Paste the + run URL. + +--- + +## 5. Definition of Done + +Non-negotiable. A piece is not done until all of these are true and reported. + +1. **Tests written first** (red → green) and exercising the real parsing path — no + monkeypatching the parser under test, no asserting on hand-built record objects + where the point is that the parser produced them. +2. **Drift-verify each guard test** named above: break the guarded thing, confirm + the test fails, revert. Report that you did it. C3, C8 and the status + longest-match test are the three that matter most. +3. **Run the real gate yourself and paste actual output:** + `python3 tools/safety_lint/self_test.py` and + `python3 -m tools.safety_lint --check`. +4. **Run `pre-commit run --all-files`** and paste the result. New files under + `tools/` and `docs/` are in scope for the hooks; the SPDX header and + `polymath-python` formatting are enforced. +5. **Confirm `git diff docs/safety/` touches only the generated block and + `lint-baseline.json`.** Paste the diff. Any other change to a safety document + is a scope violation and must be reverted. +6. **The baseline is justified line by line.** Every entry carries a reason + naming the document section it comes from, and an owner. A baseline entry with + a reason of "pre-existing" is not acceptable. +7. **Report two separate sections,** and do not merge them: + - **"Out of scope, confirmed not built"** + - **"In scope, required, not done"** — must be empty. Anything in it is + blocking. +8. **Report the computed requirement count** and state whether it is 39, 40, or + something else, with the per-area breakdown. + +--- + +## 6. Acceptance criteria + +- **AC-1 — parity.** C1 reports zero non-baselined findings against the current + tree. Every SR in the spec appears exactly once in the matrix and vice versa. +- **AC-2 — count resolved.** The linter reports a single authoritative requirement + count with a per-area breakdown, and the report states whether it agrees with + `TRACEABILITY.md` §3.1, `OPEN_ITEMS.md` §5, both, or neither. +- **AC-3 — coverage reproduces.** The computed headline fractions match the + committed §3.1 numbers, or every difference is reported with both values and an + explanation before any document is modified. +- **AC-4 — evidence check bites.** Delete `tools/hil/test_10_button.py` in a + scratch copy; the linter reports a C4 error naming `SR-SYS-03` among the + affected requirements. Restore. Record the output. +- **AC-5 — silent-drop check bites.** In a scratch copy, change one + `**Verified**` row's Verifying test cell to `**NO TEST**` while leaving the + status as `Verified`; the linter reports a C3 error. Restore. Record the output. +- **AC-6 — ratchet works.** Remove one entry from `lint-baseline.json` without + fixing the underlying issue; the linter reports the finding as a new error. + Re-add it, then fix the underlying issue in a scratch copy without removing the + baseline entry; the linter reports the stale entry as an error. +- **AC-7 — generated block is safe.** Running `--write` twice produces no diff on + the second run, and the "Reading:" paragraph and the `†` footnote in §3 are + byte-identical before and after. +- **AC-8 — pre-commit clean.** `pre-commit run --all-files` passes, and the run + did not reformat any file outside this change's scope. +- **AC-9 — CI green.** `safety-lint.yml` passes on the introducing PR. Paste the + run URL. + +--- + +## 7. Workspace hygiene + +- `git status` before you start. If the tree has changes you did not make, stop + and report — do not stage, revert, or build over them. +- Work on `change-0001-safety-traceability-linter`, branched from `main`. Never + commit to `main` and never `git push origin main`. +- Touch only the files this plan names. If another file must change, say so and + why before changing it. +- Every change lands via this branch and a PR targeting `main`. +- Note that `origin/pstop` → `main` may still be open for review + (`OPEN_ITEMS.md` §8). Confirm `docs/safety/` is present on `main` before + branching; if it is not, stop and raise. + +--- + +## 8. Known traps + +1. **Do not fix the safety documents to make the linter pass.** The linter exists + to surface disagreements, and the disagreements it surfaces are safety + findings owned by a human. Editing a Status cell, adding a test reference, or + silently correcting a count converts a finding into a lie. Baseline it and + report it. +2. **`Partially satisfied` starts with the substring `Satisfied` is false, but + `Residual-accepted` and `Residual-with-test` both start with `Residual`.** + Longest-match-first on status normalization, always. A short-match bug scores + partials as fully verified and inflates the headline number — the single most + damaging failure mode this tool can have. +3. **Cells contain pipes inside backtick spans.** A naive `line.split("|")` + shreds rows and the damage is silent: you get a parsed row with shifted columns + rather than an exception. +4. **`Allocated to` uses three notations in the same document** — plain + (`F-R-02`), range (`F-R-01..05`), and slash (`F-M-03/04`). All three appear in + §2's SR-SYS table alone. +5. **A `NO TEST` marker can coexist with real test references in one cell.** + `SR-SYS-01`, `SR-SYS-02`, `SR-SYS-05`, `SR-SYS-08`, `SR-SYS-09` and `SR-R-13` + all name real tests and then state that a specific leg has none. Treating the + marker as "this row has no tests" mis-scores six rows. +6. **`OPEN_ITEMS.md` and `TRACEABILITY.md` disagree on the requirement count and + the coverage fractions.** They were written at different dates. Neither is + authoritative for this change; the parsed tables are. +7. **The markdown pre-commit hook rewrites `-` list markers to `+`.** + `OPEN_ITEMS.md` §8 records that repo-wide normalization is deliberately + deferred and that per-file application creates inconsistency. If the hook + rewrites list markers across `TRACEABILITY.md`, revert and report — the + generated block must not drag a repo-wide reformat in with it. +8. **`pstop_c/` is excluded from pre-commit and is on a separate upstream track.** + Test paths under it are checked for existence and never opened, never + formatted, never modified. +9. **The five functions with no SR are real, not parser bugs.** `F-R-08` and + `F-R-10` are declared non-safety; `F-H-04`, `F-M-01` and `F-M-06` are genuine + holes recorded in `TRACEABILITY.md` §5. All five belong in the baseline with + that distinction preserved. +10. **Line numbers in `file:line` citations drift by design.** + `SAFETY_REQUIREMENTS.md` §1 says so explicitly. A linter that asserts on them + fails on every unrelated commit and will be disabled within a week. diff --git a/docs/safety/TRACEABILITY.md b/docs/safety/TRACEABILITY.md index eacf8df7..5b9b5734 100644 --- a/docs/safety/TRACEABILITY.md +++ b/docs/safety/TRACEABILITY.md @@ -129,6 +129,13 @@ Test-file shorthand: ### 3.1 Headline numbers + +- **SRs with at least one cited verifying test: 32 / 40 = 80.0 %**‡ +- **Strict, fully-verified only: 17 / 40 = 42.5 %** + +‡ Citation resolution, not test execution or passing state, is checked by the linter. + + - **(a) SRs with ≥1 passing verifying test: 32 / 40 = 80.0 %** [Reconciled 2026-08-07: +4 as DU-1/2/3/4 closures gained tests — SR-H-03/SR-H-04 now Verified, SR-R-03/SR-R-09 now Partially-verified]. @@ -148,7 +155,8 @@ Test-file shorthand: ### 3.2 Breakdown by area -| Area | Count | Verified | Partially-verified | Unverified-gap | Residual-accepted | ≥1-test % | Fully-verified % | + +| Area | Count | Verified | Partially-verified | Unverified-gap | Residual-accepted | >=1 cited test % | Fully-verified % | |---|---|---|---|---|---|---|---| | SR-SYS | 9 | 2 | 6 | 1 | 0 | 88.9 % | 22.2 % | | SR-R | 15 | 6 | 3 | 6 | 0 | 60.0 % | 40.0 % | @@ -156,6 +164,7 @@ Test-file shorthand: | SR-M | 6 | 3 | 1 | 0 | 2 | 83.3 %† | 50.0 % | | SR-I | 4 | 1 | 3 | 0 | 0 | 100 % | 25.0 % | | **Total** | **40** | **17** | **14** | **7** | **2** | **80.0 %** | **42.5 %** | + † SR-M ≥1-test counts SR-M-01/03/05 (Verified) + SR-M-02 (Partial) + SR-M-06 (Residual-with-test) = 5/6 = 83.3 % (SR-M-01/03 verified 2026-08-02). diff --git a/docs/safety/lint-baseline.json b/docs/safety/lint-baseline.json new file mode 100644 index 00000000..2e29b797 --- /dev/null +++ b/docs/safety/lint-baseline.json @@ -0,0 +1,76 @@ +{ + "generated": "2026-09-11", + "note": "Pre-existing findings accepted at linter introduction. Each entry needs an exact finding message, owner, and reason. Removing a fixed entry is required; CI fails on a stale entry.", + "findings": [ + { + "check_id": "C4", + "subject": "SR-M-01", + "finding": "test_timing_floors: cited evidence file does not exist", + "reason": "TRACEABILITY.md section 2.4 cites test_timing_floors, but test_timing_floors.cpp was added by 1f226be and deleted by 9a28d4a during ROS2 convention restoration. This records broken citation resolution only and does not decide SR-M-01 verification status.", + "owner": "raj" + }, + { + "check_id": "C8", + "subject": "F-H-04", + "finding": "function has no allocated safety requirement", + "reason": "TRACEABILITY.md section 5.2 declares Robot status output + logging has no SR and identifies it as an undeclared requirements-coverage hole.", + "owner": "raj" + }, + { + "check_id": "C8", + "subject": "F-M-01", + "finding": "function has no allocated safety requirement", + "reason": "TRACEABILITY.md section 5.3 declares Lifecycle node management has no SR and identifies it as an undeclared requirements-coverage hole.", + "owner": "raj" + }, + { + "check_id": "C8", + "subject": "F-M-06", + "finding": "function has no allocated safety requirement", + "reason": "TRACEABILITY.md section 5.3 declares ROS topic publication has no SR and identifies it as an undeclared requirements-coverage hole.", + "owner": "raj" + }, + { + "check_id": "C8", + "subject": "F-P-04", + "finding": "reverse map ['SR-I-04', 'SR-R-14', 'SR-SYS-02', 'SR-SYS-04', 'SR-SYS-08'] != allocated SRs ['SR-I-04', 'SR-SYS-02', 'SR-SYS-04', 'SR-SYS-08']", + "reason": "TRACEABILITY.md section 5.4 includes SR-R-14 via F-R-06 in the F-P-04 reverse map although the SRS allocation for SR-R-14 names only F-R-06/07.", + "owner": "raj" + }, + { + "check_id": "C8", + "subject": "F-R-02", + "finding": "reverse map ['SR-R-02', 'SR-R-03', 'SR-R-04', 'SR-SYS-02'] != allocated SRs ['SR-R-02', 'SR-R-03', 'SR-R-04', 'SR-SYS-01', 'SR-SYS-02']", + "reason": "TRACEABILITY.md section 5.1 omits SR-SYS-01 from F-R-02 although the SRS range allocation F-R-01..05 includes it.", + "owner": "raj" + }, + { + "check_id": "C8", + "subject": "F-R-03", + "finding": "reverse map ['SR-R-05', 'SR-R-06', 'SR-SYS-03', 'SR-SYS-06'] != allocated SRs ['SR-R-05', 'SR-R-06', 'SR-SYS-01', 'SR-SYS-03', 'SR-SYS-06']", + "reason": "TRACEABILITY.md section 5.1 omits SR-SYS-01 from F-R-03 although the SRS range allocation F-R-01..05 includes it.", + "owner": "raj" + }, + { + "check_id": "C8", + "subject": "F-R-04", + "finding": "reverse map ['SR-R-07', 'SR-R-08'] != allocated SRs ['SR-R-07', 'SR-R-08', 'SR-SYS-01']", + "reason": "TRACEABILITY.md section 5.1 omits SR-SYS-01 from F-R-04 although the SRS range allocation F-R-01..05 includes it.", + "owner": "raj" + }, + { + "check_id": "C8", + "subject": "F-R-08", + "finding": "function has no allocated safety requirement", + "reason": "TRACEABILITY.md section 5.1 explicitly declares Status LED ring non-safety with no allocated SR.", + "owner": "raj" + }, + { + "check_id": "C8", + "subject": "F-R-10", + "finding": "function has no allocated safety requirement", + "reason": "TRACEABILITY.md sections 5.1 and 5.5 explicitly declare Admin/web non-safety with no allocated SR while retaining a separate non-interference obligation gap.", + "owner": "raj" + } + ] +} diff --git a/tools/safety_lint/__init__.py b/tools/safety_lint/__init__.py new file mode 100644 index 00000000..ff6eb706 --- /dev/null +++ b/tools/safety_lint/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Safety traceability linting package.""" diff --git a/tools/safety_lint/__main__.py b/tools/safety_lint/__main__.py new file mode 100644 index 00000000..a4a3f3e9 --- /dev/null +++ b/tools/safety_lint/__main__.py @@ -0,0 +1,112 @@ +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Command-line entry point for the safety traceability linter.""" + +import argparse +import json +import sys +from pathlib import Path + +from .checks import apply_baseline, check_summary_prose, run_checks +from .coverage import compute_coverage +from .model import LintError +from .render import render_traceability +from .runner import analyze + + +def _load_baseline(path): + if not path.is_file(): + raise LintError(f'required document missing: {path}') + try: + document = json.loads(path.read_text(encoding='utf-8')) + entries = document['findings'] + baseline = {} + for entry in entries: + if not entry.get('owner') or not entry.get('reason') or not entry.get('finding'): + raise LintError(f'baseline entry needs owner, reason, and exact finding: {entry}') + key = (entry['check_id'], entry['subject'], entry['finding']) + if key in baseline: + raise LintError(f'duplicate exact baseline entry: {key}') + baseline[key] = entry + return baseline + except (KeyError, TypeError, json.JSONDecodeError) as error: + raise LintError(f'invalid baseline {path}: {error}') from error + + +def _coverage_dict(coverage): + return { + 'total': coverage.total, + 'cited_tests': coverage.cited_tests, + 'verified': coverage.verified, + 'areas': coverage.areas, + 'functions': { + 'traced': coverage.functions_traced, + 'total': coverage.functions_total, + 'safety_total': coverage.safety_functions_total, + }, + } + + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument('--check', action='store_true') + parser.add_argument('--write', action='store_true') + parser.add_argument('--json', action='store_true') + parser.add_argument('--root', default='.', help=argparse.SUPPRESS) + args = parser.parse_args(argv) + if args.check and args.write: + parser.error('--check and --write are mutually exclusive') + try: + root = Path(args.root).resolve() + analysis = analyze(root) + baseline = _load_baseline(root / 'docs/safety/lint-baseline.json') + coverage = compute_coverage(analysis) + trace_path = root / 'docs/safety/TRACEABILITY.md' + original = trace_path.read_text(encoding='utf-8') + active, suppressed = apply_baseline(run_checks(analysis) + check_summary_prose(original, coverage), baseline) + rendered = render_traceability(original, coverage) + stale = rendered != original + active_errors = [finding for finding in active if finding.severity == 'error'] + if args.write and not active_errors: + trace_path.write_text(rendered, encoding='utf-8') + stale = False + if args.json: + print( + json.dumps( + { + 'findings': [finding.__dict__ for finding in active], + 'baselined': [finding.__dict__ for finding in suppressed], + 'coverage': _coverage_dict(coverage), + 'stale': stale, + }, + indent=2, + sort_keys=True, + ) + ) + else: + for severity in ('error', 'warning', 'info'): + for finding in active: + if finding.severity == severity: + print( + f'{finding.file}:{finding.line}: [{finding.check_id}] {finding.subject} — {finding.message}' + ) + print('Baselined findings:') + for finding in suppressed: + print(f'{finding.file}:{finding.line}: [{finding.check_id}] {finding.subject} — {finding.message}') + print( + f'Coverage: cited tests {coverage.cited_tests}/{coverage.total}; strict Verified {coverage.verified}/{coverage.total}; functions {coverage.functions_traced}/{coverage.functions_total} ({coverage.functions_traced}/{coverage.safety_functions_total} excluding declared non-safety)' + ) + print('Citation limitation: resolution does not verify test execution or passing state.') + for area, data in coverage.areas.items(): + print(f' SR-{area}: {data["count"]} total, {data["cited"]} cited, {data["Verified"]} Verified') + if args.check and stale: + print('docs/safety/TRACEABILITY.md: generated regions are stale') + failed = bool(active_errors) or (args.check and stale) + return 1 if failed else 0 + except (LintError, OSError) as error: + print(f'safety-lint: cannot run: {error}', file=sys.stderr) + return 2 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/safety_lint/checks.py b/tools/safety_lint/checks.py new file mode 100644 index 00000000..876dc471 --- /dev/null +++ b/tools/safety_lint/checks.py @@ -0,0 +1,277 @@ +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Bidirectional consistency checks and baseline ratchet.""" + +import re +from collections import Counter, defaultdict + +from .model import Finding + +SRS_STATUSES = ('Partially satisfied', 'Residual-accepted', 'Satisfied', 'Gap') +TRACE_STATUSES = ('Partially-verified', 'Residual-accepted', 'Unverified-gap', 'Verified') + + +def _finding(check, severity, subject, message, file, line=1): + return Finding(check, severity, subject, message, file, line) + + +def run_checks(analysis): + findings = [] + srs_counts = Counter(row.sr_id for row in analysis.srs) + trace_counts = Counter(row.sr_id for row in analysis.trace) + for sr_id in sorted(set(srs_counts) | set(trace_counts)): + if srs_counts[sr_id] != 1 or trace_counts[sr_id] != 1: + findings.append( + _finding( + 'C1', + 'error', + sr_id, + f'SRS count {srs_counts[sr_id]}, matrix count {trace_counts[sr_id]}', + 'docs/safety/TRACEABILITY.md', + ) + ) + + used_statuses = {row.status for row in analysis.srs} + for status in sorted(used_statuses - analysis.conventions_statuses): + findings.append( + _finding( + 'C2', + 'info', + status, + 'valid status is in actual use but absent from SRS conventions section 1', + 'docs/safety/SAFETY_REQUIREMENTS.md', + ) + ) + for row in analysis.srs: + if row.status not in SRS_STATUSES: + findings.append( + _finding( + 'C2', + 'error', + row.sr_id, + f'invalid SRS status {row.status}', + 'docs/safety/SAFETY_REQUIREMENTS.md', + row.source_line, + ) + ) + for row in analysis.trace: + if row.status not in TRACE_STATUSES: + findings.append( + _finding( + 'C2', + 'error', + row.sr_id, + f'invalid trace status {row.status}', + 'docs/safety/TRACEABILITY.md', + row.source_line, + ) + ) + + for row in analysis.trace: + if row.status in ('Verified', 'Partially-verified') and not row.test_refs: + findings.append( + _finding( + 'C3', + 'error', + row.sr_id, + f'{row.status} has no resolvable test citation', + 'docs/safety/TRACEABILITY.md', + row.source_line, + ) + ) + if row.status == 'Unverified-gap' and (not row.has_no_test_marker or row.test_refs): + findings.append( + _finding( + 'C3', + 'error', + row.sr_id, + 'Unverified-gap must carry NO TEST and no resolvable test citation', + 'docs/safety/TRACEABILITY.md', + row.source_line, + ) + ) + + for issue in analysis.issues: + if issue.category == 'test': + findings.append( + _finding( + 'C4', + 'error', + issue.sr_id, + f'{issue.literal}: {issue.message}', + 'docs/safety/TRACEABILITY.md', + issue.source_line, + ) + ) + else: + severity = 'warning' if issue.kind in ('unresolved', 'ambiguous') else 'error' + findings.append( + _finding( + 'C5', + severity, + issue.sr_id, + f'{issue.literal}: {issue.message}', + 'docs/safety/TRACEABILITY.md', + issue.source_line, + ) + ) + + for row in analysis.srs: + for function_id in row.allocated_to: + if function_id not in analysis.functions: + findings.append( + _finding( + 'C6', + 'error', + function_id, + f'allocated by {row.sr_id} but absent from system definition', + 'docs/safety/SAFETY_REQUIREMENTS.md', + row.source_line, + ) + ) + for row in analysis.trace: + for function_id in row.allocated_to: + if function_id not in analysis.functions: + findings.append( + _finding( + 'C6', + 'error', + function_id, + f'allocated by {row.sr_id} but absent from system definition', + 'docs/safety/TRACEABILITY.md', + row.source_line, + ) + ) + + for row in analysis.srs: + for reference in row.derived_from: + if not reference.startswith(('SG-', 'H-', 'DU-')): + continue + known = reference in (analysis.fmea_ids if reference.startswith('DU-') else analysis.hara_ids) + if not known: + findings.append( + _finding( + 'C7', + 'warning', + reference, + f'upstream reference from {row.sr_id} not found', + 'docs/safety/SAFETY_REQUIREMENTS.md', + row.source_line, + ) + ) + + allocated = defaultdict(set) + for row in analysis.srs: + for function_id in row.allocated_to: + allocated[function_id].add(row.sr_id) + for function_id in sorted(set(analysis.functions) | set(analysis.reverse)): + entry = analysis.reverse.get(function_id) + if entry is None: + findings.append( + _finding( + 'C8', 'error', function_id, 'system function absent from reverse map', 'docs/safety/TRACEABILITY.md' + ) + ) + continue + expected = allocated.get(function_id, set()) + actual = set(entry.sr_ids) + if expected != actual: + findings.append( + _finding( + 'C8', + 'error', + function_id, + f'reverse map {sorted(actual)} != allocated SRs {sorted(expected)}', + 'docs/safety/TRACEABILITY.md', + entry.source_line, + ) + ) + if not expected: + findings.append( + _finding( + 'C8', + 'warning', + function_id, + 'function has no allocated safety requirement', + 'docs/safety/TRACEABILITY.md', + entry.source_line, + ) + ) + + srs_by_id = {row.sr_id: row for row in analysis.srs} + for row in analysis.trace: + if row.sr_id in srs_by_id and set(row.allocated_to) != set(srs_by_id[row.sr_id].allocated_to): + findings.append( + _finding( + 'C9', + 'error', + row.sr_id, + f'matrix allocation {sorted(row.allocated_to)} != SRS allocation {sorted(srs_by_id[row.sr_id].allocated_to)}', + 'docs/safety/TRACEABILITY.md', + row.source_line, + ) + ) + return tuple(findings) + + +def check_summary_prose(text, coverage): + """Check numeric claims embedded in hand-authored prose without rewriting it.""" + expected = ( + ( + r'SRs with ≥1 passing verifying test:\s*(\d+)\s*/\s*(\d+)', + (coverage.cited_tests, coverage.total), + 'passing-test headline', + ), + (r'Strict, fully-verified only:\s*(\d+)\s*/\s*(\d+)', (coverage.verified, coverage.total), 'strict headline'), + ( + r'Safety functions F-xx traced to ≥1 SR:\s*(\d+)\s*/\s*(\d+)', + (coverage.functions_traced, coverage.functions_total), + 'function headline', + ), + ( + r'Excluding the two declared-non-safety functions:\s*(\d+)\s*/\s*(\d+)', + (coverage.functions_traced, coverage.safety_functions_total), + 'safety-function headline', + ), + ) + findings = [] + for pattern, wanted, subject in expected: + match = re.search(pattern, text, re.IGNORECASE) + actual = tuple(map(int, match.groups())) if match else None + if actual != wanted: + findings.append( + _finding( + 'SUMMARY', + 'error', + subject, + f'committed prose value {actual} != citation-derived {wanted}; agreement does not verify execution', + 'docs/safety/TRACEABILITY.md', + ) + ) + return tuple(findings) + + +def apply_baseline(findings, baseline): + active = [] + suppressed = [] + matched = set() + for finding in findings: + key = (finding.check_id, finding.subject, finding.message) + if key in baseline and key not in matched and finding.severity != 'info': + suppressed.append( + Finding(finding.check_id, 'info', finding.subject, finding.message, finding.file, finding.line) + ) + matched.add(key) + else: + active.append(finding) + for key in sorted(set(baseline) - matched): + active.append( + _finding( + 'BASELINE', + 'error', + f'{key[0]}:{key[1]}', + f'stale baseline entry has no exact current finding: {key[2]}', + 'docs/safety/lint-baseline.json', + ) + ) + return tuple(active), tuple(suppressed) diff --git a/tools/safety_lint/coverage.py b/tools/safety_lint/coverage.py new file mode 100644 index 00000000..540168b1 --- /dev/null +++ b/tools/safety_lint/coverage.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Compute citation, status, and function coverage from parsed records.""" + +from collections import Counter +from dataclasses import dataclass + + +@dataclass(frozen=True) +class Coverage: + total: int + cited_tests: int + verified: int + areas: dict[str, dict[str, int]] + functions_total: int + functions_traced: int + safety_functions_total: int + + +def compute_coverage(analysis): + areas = {} + for area in ('SYS', 'R', 'H', 'M', 'I'): + rows = [row for row in analysis.trace if row.sr_id.split('-')[1] == area] + counts = Counter(row.status for row in rows) + areas[area] = { + 'count': len(rows), + 'Verified': counts['Verified'], + 'Partially-verified': counts['Partially-verified'], + 'Unverified-gap': counts['Unverified-gap'], + 'Residual-accepted': counts['Residual-accepted'], + 'cited': sum(bool(row.test_refs) for row in rows), + } + traced = sum(bool(entry.sr_ids) for entry in analysis.reverse.values() if entry.function_id in analysis.functions) + non_safety = sum( + entry.declared_non_safety for entry in analysis.reverse.values() if entry.function_id in analysis.functions + ) + return Coverage( + len(analysis.trace), + sum(bool(row.test_refs) for row in analysis.trace), + sum(row.status == 'Verified' for row in analysis.trace), + areas, + len(analysis.functions), + traced, + len(analysis.functions) - non_safety, + ) diff --git a/tools/safety_lint/fixtures/repository/code.c b/tools/safety_lint/fixtures/repository/code.c new file mode 100644 index 00000000..70661b08 --- /dev/null +++ b/tools/safety_lint/fixtures/repository/code.c @@ -0,0 +1,3 @@ +// SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +// SPDX-License-Identifier: Apache-2.0 +/* fixture */ diff --git a/tools/safety_lint/fixtures/repository/docs/safety/FMEA.md b/tools/safety_lint/fixtures/repository/docs/safety/FMEA.md new file mode 100644 index 00000000..a711cca1 --- /dev/null +++ b/tools/safety_lint/fixtures/repository/docs/safety/FMEA.md @@ -0,0 +1,6 @@ + + + +# FMEA + +DU-1 diff --git a/tools/safety_lint/fixtures/repository/docs/safety/HARA.md b/tools/safety_lint/fixtures/repository/docs/safety/HARA.md new file mode 100644 index 00000000..ccad338b --- /dev/null +++ b/tools/safety_lint/fixtures/repository/docs/safety/HARA.md @@ -0,0 +1,6 @@ + + + +# HARA + +SG-1 H-01 diff --git a/tools/safety_lint/fixtures/repository/docs/safety/SAFETY_REQUIREMENTS.md b/tools/safety_lint/fixtures/repository/docs/safety/SAFETY_REQUIREMENTS.md new file mode 100644 index 00000000..3afe2d55 --- /dev/null +++ b/tools/safety_lint/fixtures/repository/docs/safety/SAFETY_REQUIREMENTS.md @@ -0,0 +1,15 @@ + + + +# SRS + +## 1. Conventions + +Statuses: **Satisfied**, **Gap**, **Residual-accepted**. + +## 2. Requirements + +| ID | Requirement (shall) | Derived from | Allocated to | Integrity | Verify | Status | +|---|---|---|---|---|---|---| +| **SR-SYS-01** | Stop safely. | SG-1, H-01, DU-1 | F-R-01 | SIL 3 | Test | **Satisfied** | +| **SR-R-01** | Remain fresh with `a|b`. | SG-1 | F-R-01 | SIL 3 | Test | **Partially satisfied** pending integration | diff --git a/tools/safety_lint/fixtures/repository/docs/safety/SYSTEM_DEFINITION.md b/tools/safety_lint/fixtures/repository/docs/safety/SYSTEM_DEFINITION.md new file mode 100644 index 00000000..93a9a922 --- /dev/null +++ b/tools/safety_lint/fixtures/repository/docs/safety/SYSTEM_DEFINITION.md @@ -0,0 +1,12 @@ + + + +# System + +## 4. Function / item decomposition + +| ID | Function | Location | +|---|---|---| +| F-R-01 | Sense | code.c | + +## 5. End diff --git a/tools/safety_lint/fixtures/repository/docs/safety/TRACEABILITY.md b/tools/safety_lint/fixtures/repository/docs/safety/TRACEABILITY.md new file mode 100644 index 00000000..98e152f6 --- /dev/null +++ b/tools/safety_lint/fixtures/repository/docs/safety/TRACEABILITY.md @@ -0,0 +1,29 @@ + + + +# Traceability + +Test-file shorthand: `EV` = `tests/ev.c`; `MR` = `tools/pstop_multi_remote_test.py`; `HIL10` = `tools/hil/test_10_button.py`; `HIL20` = `tools/hil/test_20_discordance.py`; `HIL30` = `tools/hil/test_30_power_cycle.py`; `JL` = `tests/json_lite.cpp`; `REQ n_nn` = `pstop_c/pstop/test/src/pstop/requirements/req_n_nn_test.c`. + +| SR | Alloc F-xx | Code (file:line) | Verifying test(s) | Method | Status | +|---|---|---|---|---|---| +| SR-SYS-01 | F-R-01 | code.c:1 | MR[B/D/E/F], REQ 2_02/2_03, **NO TEST** for latency | Test | **Partially-verified** | +| SR-R-01 | F-R-01 | code.c:1 | test_unique_probe.py | Test | **Verified** | + + +stale + + +stale + + +- **(a) SRs with ≥1 passing verifying test: 2 / 2 = 100 %** +- **Strict, fully-verified only: 1 / 2 = 50.0 %.** +- **(b) Safety functions F-xx traced to ≥1 SR: 1 / 1 = 100 %.** + Excluding the two declared-non-safety functions: 1 / 1 = 100 %. + +**Reading:** hand-authored prose. + +| F-xx | Function | SRs touching it | +|---|---|---| +| F-R-01 | Sense | SR-SYS-01, SR-R-01 | diff --git a/tools/safety_lint/fixtures/repository/docs/safety/lint-baseline.json b/tools/safety_lint/fixtures/repository/docs/safety/lint-baseline.json new file mode 100644 index 00000000..416d8259 --- /dev/null +++ b/tools/safety_lint/fixtures/repository/docs/safety/lint-baseline.json @@ -0,0 +1,5 @@ +{ + "generated": "2026-09-11", + "note": "Fixture baseline.", + "findings": [] +} diff --git a/tools/safety_lint/fixtures/repository/firmware/test/test_estop_verdict.c b/tools/safety_lint/fixtures/repository/firmware/test/test_estop_verdict.c new file mode 100644 index 00000000..29473825 --- /dev/null +++ b/tools/safety_lint/fixtures/repository/firmware/test/test_estop_verdict.c @@ -0,0 +1,3 @@ +// SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +// SPDX-License-Identifier: Apache-2.0 +/* fixture test */ diff --git a/tools/safety_lint/fixtures/repository/pstop_c/pstop/test/src/pstop/requirements/req_2_02_test.c b/tools/safety_lint/fixtures/repository/pstop_c/pstop/test/src/pstop/requirements/req_2_02_test.c new file mode 100644 index 00000000..29473825 --- /dev/null +++ b/tools/safety_lint/fixtures/repository/pstop_c/pstop/test/src/pstop/requirements/req_2_02_test.c @@ -0,0 +1,3 @@ +// SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +// SPDX-License-Identifier: Apache-2.0 +/* fixture test */ diff --git a/tools/safety_lint/fixtures/repository/pstop_c/pstop/test/src/pstop/requirements/req_2_03_test.c b/tools/safety_lint/fixtures/repository/pstop_c/pstop/test/src/pstop/requirements/req_2_03_test.c new file mode 100644 index 00000000..29473825 --- /dev/null +++ b/tools/safety_lint/fixtures/repository/pstop_c/pstop/test/src/pstop/requirements/req_2_03_test.c @@ -0,0 +1,3 @@ +// SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +// SPDX-License-Identifier: Apache-2.0 +/* fixture test */ diff --git a/tools/safety_lint/fixtures/repository/ros2/protective_stop_machine/test/test_json_lite.cpp b/tools/safety_lint/fixtures/repository/ros2/protective_stop_machine/test/test_json_lite.cpp new file mode 100644 index 00000000..2535f05e --- /dev/null +++ b/tools/safety_lint/fixtures/repository/ros2/protective_stop_machine/test/test_json_lite.cpp @@ -0,0 +1,3 @@ +// SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +// SPDX-License-Identifier: Apache-2.0 +// fixture test diff --git a/tools/safety_lint/fixtures/repository/tests/test_unique_probe.py b/tools/safety_lint/fixtures/repository/tests/test_unique_probe.py new file mode 100644 index 00000000..0bb231a0 --- /dev/null +++ b/tools/safety_lint/fixtures/repository/tests/test_unique_probe.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +# fixture test diff --git a/tools/safety_lint/fixtures/repository/tools/hil/test_10_button.py b/tools/safety_lint/fixtures/repository/tools/hil/test_10_button.py new file mode 100644 index 00000000..0bb231a0 --- /dev/null +++ b/tools/safety_lint/fixtures/repository/tools/hil/test_10_button.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +# fixture test diff --git a/tools/safety_lint/fixtures/repository/tools/hil/test_20_discordance.py b/tools/safety_lint/fixtures/repository/tools/hil/test_20_discordance.py new file mode 100644 index 00000000..0bb231a0 --- /dev/null +++ b/tools/safety_lint/fixtures/repository/tools/hil/test_20_discordance.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +# fixture test diff --git a/tools/safety_lint/fixtures/repository/tools/hil/test_30_power_cycle.py b/tools/safety_lint/fixtures/repository/tools/hil/test_30_power_cycle.py new file mode 100644 index 00000000..0bb231a0 --- /dev/null +++ b/tools/safety_lint/fixtures/repository/tools/hil/test_30_power_cycle.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +# fixture test diff --git a/tools/safety_lint/fixtures/repository/tools/pstop_multi_remote_test.py b/tools/safety_lint/fixtures/repository/tools/pstop_multi_remote_test.py new file mode 100644 index 00000000..0bb231a0 --- /dev/null +++ b/tools/safety_lint/fixtures/repository/tools/pstop_multi_remote_test.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +# fixture test diff --git a/tools/safety_lint/model.py b/tools/safety_lint/model.py new file mode 100644 index 00000000..53fa04eb --- /dev/null +++ b/tools/safety_lint/model.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Immutable records shared by the safety traceability linter.""" + +from dataclasses import dataclass +from pathlib import Path + + +class LintError(Exception): + """A required input is absent or structurally unparseable.""" + + +@dataclass(frozen=True) +class SafetyRequirement: + sr_id: str + area: str + number: int + shall_text: str + derived_from: tuple[str, ...] + allocated_to: tuple[str, ...] + integrity: str + verify_methods: tuple[str, ...] + status: str + status_prose: str + source_line: int + + +@dataclass(frozen=True) +class TraceRow: + sr_id: str + allocated_to: tuple[str, ...] + code_refs: tuple[str, ...] + test_refs: tuple[str, ...] + methods: tuple[str, ...] + status: str + status_prose: str + has_no_test_marker: bool + source_line: int + raw_test_cell: str + + +@dataclass(frozen=True) +class Function: + function_id: str + name: str + source_line: int + + +@dataclass(frozen=True) +class ReverseEntry: + function_id: str + name: str + sr_ids: tuple[str, ...] + declared_non_safety: bool + source_line: int + + +@dataclass(frozen=True) +class ResolutionIssue: + kind: str + sr_id: str + literal: str + message: str + source_line: int + category: str + + +@dataclass(frozen=True) +class Finding: + check_id: str + severity: str + subject: str + message: str + file: str + line: int + + +@dataclass(frozen=True) +class Analysis: + root: Path + srs: tuple[SafetyRequirement, ...] + trace: tuple[TraceRow, ...] + functions: dict[str, Function] + reverse: dict[str, ReverseEntry] + issues: tuple[ResolutionIssue, ...] + conventions_statuses: frozenset[str] + hara_ids: frozenset[str] + fmea_ids: frozenset[str] diff --git a/tools/safety_lint/parse_srs.py b/tools/safety_lint/parse_srs.py new file mode 100644 index 00000000..507f9686 --- /dev/null +++ b/tools/safety_lint/parse_srs.py @@ -0,0 +1,138 @@ +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Parse canonical safety requirements markdown without rewriting it.""" + +import re +from pathlib import Path + +from .checks import SRS_STATUSES +from .model import LintError, SafetyRequirement + +SR_RE = re.compile(r'^SR-(SYS|R|H|M|I)-(\d{2})$') + + +def split_row(line): + """Split unescaped markdown pipes while preserving pipes in code spans.""" + cells = [] + current = [] + in_code = False + escaped = False + for char in line.strip(): + if escaped: + current.append(char) + escaped = False + elif char == '\\': + current.append(char) + escaped = True + elif char == '`': + in_code = not in_code + current.append(char) + elif char == '|' and not in_code: + cells.append(''.join(current).strip()) + current = [] + else: + current.append(char) + cells.append(''.join(current).strip()) + if cells and not cells[0]: + cells.pop(0) + if cells and not cells[-1]: + cells.pop() + return cells + + +def _strip_bold(value): + value = value.strip() + if value.startswith('**') and value.endswith('**') and len(value) >= 4: + return value[2:-2].strip() + return value + + +def normalize_status(cell, vocabulary, path, line): + """Return a canonical leading status token and untouched trailing prose.""" + value = cell.strip() + if value.startswith('**'): + close = value.find('**', 2) + if close != -1: + value = value[2:close] + value[close + 2 :] + value = value.strip() + for token in vocabulary: + if re.match(re.escape(token) + r'(?:\b|\s|$|[—(\[])', value, re.IGNORECASE): + return token, value[len(token) :].strip() + raise LintError(f'{path}:{line}: unknown status cell {cell!r}') + + +def expand_allocations(cell): + """Expand F-X-01..03 and F-X-01/02 notation into complete IDs.""" + result = [] + occupied = [] + pattern = re.compile(r'F-([A-Z])-([0-9]{2})(?:\.\.([0-9]{2})|/([0-9]{2}))?') + for match in pattern.finditer(cell): + area, first, end, alternate = match.groups() + occupied.append(match.span()) + if end: + result.extend(f'F-{area}-{number:02d}' for number in range(int(first), int(end) + 1)) + else: + result.append(f'F-{area}-{first}') + if alternate: + result.append(f'F-{area}-{alternate}') + return tuple(dict.fromkeys(result)) + + +def parse_srs(path): + """Parse all requirement tables discovered by their semantic headers.""" + path = Path(path) + if not path.is_file(): + raise LintError(f'required document missing: {path}') + lines = path.read_text(encoding='utf-8').splitlines() + rows = [] + index = 0 + while index < len(lines): + if not lines[index].lstrip().startswith('|'): + index += 1 + continue + header = split_row(lines[index]) + if not header or header[0] != 'ID' or 'Derived from' not in header: + index += 1 + continue + columns = {name: position for position, name in enumerate(header)} + index += 2 + while index < len(lines) and lines[index].lstrip().startswith('|'): + line_number = index + 1 + cells = split_row(lines[index]) + if len(cells) != len(header): + raise LintError(f'{path}:{line_number}: expected {len(header)} cells, got {len(cells)}') + sr_id = _strip_bold(cells[columns['ID']]) + match = SR_RE.fullmatch(sr_id) + if not match: + raise LintError(f'{path}:{line_number}: invalid requirement ID {sr_id!r}') + status, prose = normalize_status(cells[columns['Status']], SRS_STATUSES, path, line_number) + derived = tuple( + dict.fromkeys(re.findall(r'SG-\d+|H-\d{2}|DU-\d+|\b[A-Z]\d{2}-\d\b', cells[columns['Derived from']])) + ) + verify = tuple(part.strip() for part in re.split(r'\s*\+\s*', cells[columns['Verify']]) if part.strip()) + rows.append( + SafetyRequirement( + sr_id, + match.group(1), + int(match.group(2)), + cells[columns['Requirement (shall)']], + derived, + expand_allocations(cells[columns['Allocated to']]), + cells[columns['Integrity']], + verify, + status, + prose, + line_number, + ) + ) + index += 1 + if not rows: + raise LintError(f'{path}: no requirements table found') + return tuple(rows) + + +def convention_statuses(path): + """Extract only statuses explicitly listed in conventions section 1.""" + text = Path(path).read_text(encoding='utf-8') + section = text.split('## 1.', 1)[1].split('\n## 2.', 1)[0] if '## 1.' in text else text + return frozenset(status for status in SRS_STATUSES if re.search(rf'\*\*{re.escape(status)}\*\*', section)) diff --git a/tools/safety_lint/parse_system_definition.py b/tools/safety_lint/parse_system_definition.py new file mode 100644 index 00000000..4450ad05 --- /dev/null +++ b/tools/safety_lint/parse_system_definition.py @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Parse the authoritative function decomposition.""" + +import re +from pathlib import Path + +from .model import Function, LintError +from .parse_srs import split_row + + +def parse_system_definition(path): + path = Path(path) + if not path.is_file(): + raise LintError(f'required document missing: {path}') + lines = path.read_text(encoding='utf-8').splitlines() + in_section = False + functions = {} + for index, line in enumerate(lines): + if line.startswith('## 4.'): + in_section = True + elif in_section and line.startswith('## '): + break + if not in_section or not line.lstrip().startswith('|'): + continue + cells = split_row(line) + if len(cells) >= 2 and re.fullmatch(r'F-[A-Z]-\d{2}', cells[0].strip('*')): + function_id = cells[0].strip('*') + functions[function_id] = Function(function_id, cells[1], index + 1) + if not functions: + raise LintError(f'{path}: no functions found in section 4') + return functions diff --git a/tools/safety_lint/parse_traceability.py b/tools/safety_lint/parse_traceability.py new file mode 100644 index 00000000..b883441e --- /dev/null +++ b/tools/safety_lint/parse_traceability.py @@ -0,0 +1,283 @@ +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Parse trace rows, reverse maps, and repository evidence citations.""" + +import os +import re +from collections import defaultdict +from pathlib import Path + +from .checks import TRACE_STATUSES +from .model import LintError, ResolutionIssue, ReverseEntry, TraceRow +from .parse_srs import SR_RE, expand_allocations, normalize_status, split_row + +SHORTHAND = { + 'EV': 'firmware/test/test_estop_verdict.c', + 'MR': 'tools/pstop_multi_remote_test.py', + 'HIL10': 'tools/hil/test_10_button.py', + 'HIL20': 'tools/hil/test_20_discordance.py', + 'HIL30': 'tools/hil/test_30_power_cycle.py', + 'JL': 'ros2/protective_stop_machine/test/test_json_lite.cpp', +} + + +def _repo_index(root): + by_name = defaultdict(list) + by_stem = defaultdict(list) + all_paths = set() + for directory, names, files in os.walk(root): + relative_directory = Path(directory).relative_to(root).as_posix() + if relative_directory == 'tools/safety_lint/fixtures': + names[:] = [] + continue + names[:] = [name for name in names if name not in {'.git', '.venv', '__pycache__', 'build', 'install', 'log'}] + for filename in files: + relative = Path(directory, filename).relative_to(root).as_posix() + all_paths.add(relative) + by_name[filename].append(relative) + by_stem[Path(filename).stem].append(relative) + return all_paths, by_name, by_stem + + +def _expand_sr_list(cell): + result = [] + for match in re.finditer(r'SR-(SYS|R|H|M|I)-(\d{2}(?:/\d{2})*)', cell): + area, numbers = match.groups() + result.extend(f'SR-{area}-{number}' for number in numbers.split('/')) + return tuple(dict.fromkeys(result)) + + +def _test_refs(root, sr_id, cell, line, index): + all_paths, by_name, by_stem = index + # Text following NO TEST describes an uncovered leg, not verifying evidence. + evidence_cell = cell.split('NO TEST', 1)[0] + refs = [] + issues = [] + attached_symbols = set( + re.findall( + r'`[^`]+\.(?:c|cc|cpp|py):\d+(?:-\d+)?`\s*\(`(test_[A-Za-z0-9_]+)`\)', + evidence_cell, + ) + ) + + for match in re.finditer(r'\bHIL(10|20|30)((?:/(?:10|20|30))*)', evidence_cell): + numbers = (match.group(1), *match.group(2).lstrip('/').split('/')) + for number in filter(None, numbers): + token = f'HIL{number}' + path = SHORTHAND[token] + if path in all_paths: + refs.append(path) + else: + issues.append( + ResolutionIssue( + 'missing-shorthand', sr_id, token, f'shorthand resolves to missing {path}', line, 'test' + ) + ) + for token, path in SHORTHAND.items(): + if token.startswith('HIL'): + continue + if re.search(rf'\b{token}(?:\[[A-Z/]+\])?\b', evidence_cell): + if path in all_paths: + refs.append(path) + else: + issues.append( + ResolutionIssue( + 'missing-shorthand', sr_id, token, f'shorthand resolves to missing {path}', line, 'test' + ) + ) + for match in re.finditer(r'\bREQ\s+(\d+_\d+(?:/\d+_\d+)*)', evidence_cell): + for number in match.group(1).split('/'): + path = f'pstop_c/pstop/test/src/pstop/requirements/req_{number}_test.c' + if path in all_paths: + refs.append(path) + else: + issues.append( + ResolutionIssue( + 'missing-shorthand', sr_id, f'REQ {number}', f'REQ resolves to missing {path}', line, 'test' + ) + ) + + literals = [literal for literal in re.findall(r'`([^`]+)`', evidence_cell) if _looks_like_test_citation(literal)] + literals += re.findall( + r'(? 1: + issues.append( + ResolutionIssue( + 'ambiguous', sr_id, literal, f'ambiguous evidence citation: {", ".join(candidates)}', line, 'test' + ) + ) + elif explicit or _is_test_literal(literal): + issues.append( + ResolutionIssue('missing', sr_id, literal, 'cited evidence file does not exist', line, 'test') + ) + return tuple(dict.fromkeys(refs)), tuple(issues) + + +def _is_test_artifact(path): + """Return whether a bare candidate is independently recognizable as test evidence.""" + candidate = Path(path) + lower_parts = {part.lower() for part in candidate.parts} + stem = candidate.stem.lower() + if candidate.suffix.lower() == '.md': + return True + return stem.startswith('test_') or stem.endswith('_test') or bool(lower_parts & {'test', 'tests', 'requirements'}) + + +def _looks_like_test_citation(literal): + without_line = re.sub(r':\d+(?:-\d+)?$', '', literal) + return ( + ('/' in without_line and not without_line.startswith('/') and '.' in Path(without_line).name) + or without_line.startswith('test_') + or without_line.endswith(('.c', '.cc', '.cpp', '.py', '.md')) + ) + + +def _is_test_literal(literal): + stem = Path(literal).stem.lower() + return stem.startswith('test_') or stem.endswith('_test') + + +def _code_refs(sr_id, cell, line, index): + all_paths, by_name, _ = index + refs = [] + issues = [] + documented_locations = { + 'main.c': 'firmware/main/main.c', + 'machine_app_runner.c': 'host/machine_app_runner.c', + 'machine.c': 'pstop_c/pstop/src/pstop/machine.c', + } + citations = re.findall( + r'(? 1: + issues.append( + ResolutionIssue( + 'ambiguous', sr_id, citation, 'unresolved ambiguous bare code filename', line, 'code' + ) + ) + else: + issues.append( + ResolutionIssue('unresolved', sr_id, citation, 'unresolved bare code filename', line, 'code') + ) + return tuple(dict.fromkeys(refs)), tuple(issues) + + +def parse_traceability(root): + root = Path(root) + path = root / 'docs/safety/TRACEABILITY.md' + if not path.is_file(): + raise LintError(f'required document missing: {path}') + lines = path.read_text(encoding='utf-8').splitlines() + index_data = _repo_index(root) + rows = [] + reverse = {} + issues = [] + index = 0 + while index < len(lines): + if not lines[index].lstrip().startswith('|'): + index += 1 + continue + header = split_row(lines[index]) + is_trace = header and header[0] == 'SR' and any(cell.startswith('Verifying test') for cell in header) + is_reverse = header and header[0] == 'F-xx' and 'SRs touching it' in header + if not is_trace and not is_reverse: + index += 1 + continue + columns = {name: position for position, name in enumerate(header)} + index += 2 + while index < len(lines) and lines[index].lstrip().startswith('|'): + line_number = index + 1 + cells = split_row(lines[index]) + if len(cells) != len(header): + raise LintError(f'{path}:{line_number}: malformed table row') + if is_trace: + sr_id = cells[0].strip('*') + if not SR_RE.fullmatch(sr_id): + raise LintError(f'{path}:{line_number}: invalid trace SR ID {sr_id!r}') + test_cell = cells[next(pos for name, pos in columns.items() if name.startswith('Verifying test'))] + test_refs, test_issues = _test_refs(root, sr_id, test_cell, line_number, index_data) + code_refs, code_issues = _code_refs(sr_id, cells[2], line_number, index_data) + status, prose = normalize_status(cells[-1], TRACE_STATUSES, path, line_number) + rows.append( + TraceRow( + sr_id, + expand_allocations(cells[1]), + code_refs, + test_refs, + tuple(part.strip() for part in cells[-2].split('+') if part.strip()), + status, + prose, + 'NO TEST' in test_cell, + line_number, + test_cell, + ) + ) + issues.extend(test_issues + code_issues) + else: + function_id = cells[0].strip('*') + sr_cell = cells[2] + reverse[function_id] = ReverseEntry( + function_id, + cells[1], + _expand_sr_list(sr_cell), + 'non-safety' in sr_cell.lower(), + line_number, + ) + index += 1 + if not rows: + raise LintError(f'{path}: no traceability rows found') + return tuple(rows), reverse, tuple(issues) diff --git a/tools/safety_lint/render.py b/tools/safety_lint/render.py new file mode 100644 index 00000000..71858e28 --- /dev/null +++ b/tools/safety_lint/render.py @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Render named numeric regions without touching safety-argument prose.""" + +import re + +from .model import LintError + + +def _percent(numerator, denominator): + if denominator == 0: + return '0 %' + value = 100 * numerator / denominator + return '100 %' if value == 100 else f'{value:.1f} %' + + +def _replace(text, name, content): + pattern = re.compile( + rf'(\n).*?(\n)', + re.DOTALL, + ) + if len(pattern.findall(text)) != 1: + raise LintError(f'TRACEABILITY.md requires exactly one generated marker pair named {name}') + return pattern.sub(lambda match: match.group(1) + content + match.group(2), text) + + +def render_traceability(text, coverage): + headline = ( + f'- **SRs with at least one cited verifying test: {coverage.cited_tests} / {coverage.total} = ' + f'{_percent(coverage.cited_tests, coverage.total)}**‡\n' + f'- **Strict, fully-verified only: {coverage.verified} / {coverage.total} = ' + f'{_percent(coverage.verified, coverage.total)}**\n' + '\n' + '‡ Citation resolution, not test execution or passing state, is checked by the linter.' + ) + lines = [ + '| Area | Count | Verified | Partially-verified | Unverified-gap | Residual-accepted | >=1 cited test % | Fully-verified % |', + '|---|---|---|---|---|---|---|---|', + ] + for area in ('SYS', 'R', 'H', 'M', 'I'): + data = coverage.areas[area] + cited_percent = _percent(data['cited'], data['count']) + if area == 'M': + cited_percent += '†' + lines.append( + f'| SR-{area} | {data["count"]} | {data["Verified"]} | {data["Partially-verified"]} | ' + f'{data["Unverified-gap"]} | {data["Residual-accepted"]} | {cited_percent} | ' + f'{_percent(data["Verified"], data["count"])} |' + ) + lines.append( + f'| **Total** | **{coverage.total}** | **{sum(v["Verified"] for v in coverage.areas.values())}** | ' + f'**{sum(v["Partially-verified"] for v in coverage.areas.values())}** | ' + f'**{sum(v["Unverified-gap"] for v in coverage.areas.values())}** | ' + f'**{sum(v["Residual-accepted"] for v in coverage.areas.values())}** | ' + f'**{_percent(coverage.cited_tests, coverage.total)}** | **{_percent(coverage.verified, coverage.total)}** |' + ) + return _replace(_replace(text, 'headline', headline), 'areas', '\n'.join(lines)) diff --git a/tools/safety_lint/runner.py b/tools/safety_lint/runner.py new file mode 100644 index 00000000..b3bc43aa --- /dev/null +++ b/tools/safety_lint/runner.py @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Load all canonical linter inputs as one analysis snapshot.""" + +import re +from pathlib import Path + +from .model import Analysis, LintError +from .parse_srs import convention_statuses, parse_srs +from .parse_system_definition import parse_system_definition +from .parse_traceability import parse_traceability + + +def _ids(path, pattern): + if not path.is_file(): + raise LintError(f'required document missing: {path}') + return frozenset(re.findall(pattern, path.read_text(encoding='utf-8'))) + + +def analyze(root): + root = Path(root).resolve() + safety = root / 'docs/safety' + srs_path = safety / 'SAFETY_REQUIREMENTS.md' + trace, reverse, issues = parse_traceability(root) + return Analysis( + root, + parse_srs(srs_path), + trace, + parse_system_definition(safety / 'SYSTEM_DEFINITION.md'), + reverse, + issues, + convention_statuses(srs_path), + _ids(safety / 'HARA.md', r'\b(?:SG-\d+|H-\d{2})\b'), + _ids(safety / 'FMEA.md', r'\bDU-\d+\b'), + ) diff --git a/tools/safety_lint/self_test.py b/tools/safety_lint/self_test.py new file mode 100755 index 00000000..2590de57 --- /dev/null +++ b/tools/safety_lint/self_test.py @@ -0,0 +1,468 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Specification tests for the checked-in safety traceability linter.""" + +import json +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +FIXTURE = Path(__file__).with_name('fixtures') / 'repository' + +sys.path.insert(0, str(REPO)) + +from tools.safety_lint.__main__ import _load_baseline # noqa: E402 +from tools.safety_lint.checks import ( # noqa: E402 + SRS_STATUSES, + TRACE_STATUSES, + apply_baseline, + check_summary_prose, + run_checks, +) +from tools.safety_lint.coverage import compute_coverage # noqa: E402 +from tools.safety_lint.model import LintError # noqa: E402 +from tools.safety_lint.parse_srs import expand_allocations, parse_srs, split_row # noqa: E402 +from tools.safety_lint.parse_system_definition import parse_system_definition # noqa: E402 +from tools.safety_lint.parse_traceability import parse_traceability # noqa: E402 +from tools.safety_lint.render import render_traceability # noqa: E402 +from tools.safety_lint.runner import analyze # noqa: E402 + + +class FixtureRepo(unittest.TestCase): + """Copy a caller-visible markdown repository for each adversarial test.""" + + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + shutil.copytree(FIXTURE, self.root, dirs_exist_ok=True) + + def tearDown(self): + self.temp.cleanup() + + def replace(self, relative, old, new): + path = self.root / relative + path.write_text(path.read_text(encoding='utf-8').replace(old, new), encoding='utf-8') + + def findings(self): + return run_checks(analyze(self.root)) + + def assert_check(self, check_id, subject=None): + matches = [f for f in self.findings() if f.check_id == check_id] + if subject is not None: + matches = [f for f in matches if f.subject == subject] + self.assertTrue(matches, f'expected {check_id} {subject or ""}') + + +class ParserTests(FixtureRepo): + def test_parses_all_sr_areas(self): + """The canonical SRS exposes at least one requirement in every declared area.""" + areas = {row.area for row in parse_srs(REPO / 'docs/safety/SAFETY_REQUIREMENTS.md')} + self.assertEqual(areas, {'SYS', 'R', 'H', 'M', 'I'}) + + def test_sr_ids_unique(self): + """Every canonical requirement ID identifies exactly one SRS row.""" + rows = parse_srs(REPO / 'docs/safety/SAFETY_REQUIREMENTS.md') + self.assertEqual(len({row.sr_id for row in rows}), len(rows)) + + def test_allocated_to_range_expansion(self): + """Compact function ranges expand to every member without dropping other allocations.""" + self.assertEqual( + set(expand_allocations('F-R-01..05, F-H-03')), + {'F-R-01', 'F-R-02', 'F-R-03', 'F-R-04', 'F-R-05', 'F-H-03'}, + ) + + def test_allocated_to_slash_expansion(self): + """Compact slash allocations expand to complete function IDs.""" + self.assertEqual(expand_allocations('F-M-03/04'), ('F-M-03', 'F-M-04')) + + def test_status_longest_match_wins(self): + """A partial SRS status is never inflated to a fully satisfied status.""" + rows = parse_srs(self.root / 'docs/safety/SAFETY_REQUIREMENTS.md') + self.assertEqual(rows[1].status, 'Partially satisfied') + + def test_status_vocabularies_are_defined_longest_match_first(self): + """Checks defines both closed status vocabularies in longest-match-first order.""" + self.assertEqual(SRS_STATUSES, ('Partially satisfied', 'Residual-accepted', 'Satisfied', 'Gap')) + self.assertEqual(TRACE_STATUSES, ('Partially-verified', 'Residual-accepted', 'Unverified-gap', 'Verified')) + + def test_pipe_inside_code_span_does_not_split_cell(self): + """A pipe inside an inline code span remains part of its markdown cell.""" + self.assertEqual(split_row('| a | `b|c` | d |'), ['a', '`b|c`', 'd']) + + def test_every_trace_row_has_sr_id(self): + """Malformed matrix rows cannot silently disappear as non-requirement content.""" + self.replace('docs/safety/TRACEABILITY.md', '| SR-R-01 |', '| BAD-ID |') + with self.assertRaises(LintError): + parse_traceability(self.root) + + def test_test_shorthand_expands_to_existing_paths(self): + """Every documented shorthand expands to its exact existing repository path.""" + self.replace( + 'docs/safety/TRACEABILITY.md', + 'test_unique_probe.py', + 'EV, MR[B/D], HIL10, HIL20, HIL30, JL, REQ 2_02/2_03', + ) + rows, _, issues = parse_traceability(self.root) + self.assertEqual( + set(rows[1].test_refs), + { + 'firmware/test/test_estop_verdict.c', + 'tools/pstop_multi_remote_test.py', + 'tools/hil/test_10_button.py', + 'tools/hil/test_20_discordance.py', + 'tools/hil/test_30_power_cycle.py', + 'ros2/protective_stop_machine/test/test_json_lite.cpp', + 'pstop_c/pstop/test/src/pstop/requirements/req_2_02_test.c', + 'pstop_c/pstop/test/src/pstop/requirements/req_2_03_test.c', + }, + ) + self.assertFalse([i for i in issues if i.kind == 'missing-shorthand']) + + def test_compact_hil_slash_form_resolves_both_paths(self): + """Compact HIL20/30 notation resolves both independently existing HIL sources.""" + self.replace('docs/safety/TRACEABILITY.md', 'test_unique_probe.py', 'HIL20/30') + rows, _, _ = parse_traceability(self.root) + self.assertEqual( + rows[1].test_refs, + ('tools/hil/test_20_discordance.py', 'tools/hil/test_30_power_cycle.py'), + ) + + def test_mr_group_selector_yields_single_path(self): + """MR group selectors identify one test source, not one source per group letter.""" + rows, _, _ = parse_traceability(self.root) + self.assertEqual(rows[0].test_refs.count('tools/pstop_multi_remote_test.py'), 1) + + def test_req_slash_form_yields_two_paths(self): + """A slash-separated REQ citation resolves each independent requirement test.""" + rows, _, _ = parse_traceability(self.root) + self.assertEqual(len([p for p in rows[0].test_refs if 'req_' in p]), 2) + + def test_no_test_marker_detected_alongside_real_tests(self): + """A scoped NO TEST note does not erase real evidence cited in the same cell.""" + rows, _, _ = parse_traceability(self.root) + self.assertTrue(rows[0].has_no_test_marker and rows[0].test_refs) + + def test_reverse_map_flags_declared_non_safety(self): + """An explicit non-safety reverse-map declaration remains distinguishable from a hole.""" + _, reverse, _ = parse_traceability(REPO) + self.assertTrue(reverse['F-R-08'].declared_non_safety) + + def test_system_definition_function_set_nonempty(self): + """The authoritative function decomposition yields a nonempty function set.""" + self.assertTrue(parse_system_definition(REPO / 'docs/safety/SYSTEM_DEFINITION.md')) + + def test_unique_repository_wide_stem_resolves(self): + """A repository-wide unique test stem resolves without directory guessing.""" + decoy = self.root / 'tools/safety_lint/fixtures/repository/tests/test_unique_probe.py' + decoy.parent.mkdir(parents=True) + decoy.write_text('# synthetic fixture, not project evidence\n', encoding='utf-8') + self.replace('docs/safety/TRACEABILITY.md', 'test_unique_probe.py', 'test_unique_probe') + rows, _, issues = parse_traceability(self.root) + self.assertIn('tests/test_unique_probe.py', rows[1].test_refs) + self.assertFalse([i for i in issues if i.literal == 'test_unique_probe']) + + def test_missing_bare_test_stem_is_not_resolved(self): + """A missing bare test stem produces an issue but never a resolved evidence path.""" + self.replace('docs/safety/TRACEABILITY.md', 'test_unique_probe.py', 'test_deleted_source') + rows, _, issues = parse_traceability(self.root) + self.assertFalse(rows[1].test_refs) + self.assertTrue([issue for issue in issues if issue.literal == 'test_deleted_source']) + + def test_ambiguous_repository_wide_stem_is_finding(self): + """An ambiguous test stem is reported instead of selecting the nearest file.""" + (self.root / 'other').mkdir() + (self.root / 'other/test_unique_probe.py').write_text('# duplicate\n', encoding='utf-8') + self.replace('docs/safety/TRACEABILITY.md', 'test_unique_probe.py', 'test_unique_probe') + _, _, issues = parse_traceability(self.root) + self.assertTrue([i for i in issues if i.kind == 'ambiguous' and i.literal == 'test_unique_probe']) + + def test_hil_report_must_name_sr(self): + """An evidence report counts only when its content names the cited requirement.""" + (self.root / 'docs/evidence.md').write_text('# HIL evidence for another SR\n', encoding='utf-8') + self.replace('docs/safety/TRACEABILITY.md', 'test_unique_probe.py', 'docs/evidence.md') + _, _, issues = parse_traceability(self.root) + self.assertTrue([i for i in issues if i.kind == 'report-does-not-name-sr']) + + def test_prose_does_not_infer_evidence(self): + """Words describing a successful test never become a test-file citation.""" + self.replace('docs/safety/TRACEABILITY.md', 'test_unique_probe.py', 'bench test passed 8/8') + rows, _, _ = parse_traceability(self.root) + self.assertFalse(rows[1].test_refs) + + def test_unique_production_source_basename_is_not_test_evidence(self): + """A unique production source basename cannot satisfy a verifying-test citation.""" + self.replace('docs/safety/TRACEABILITY.md', 'test_unique_probe.py', 'code.c') + rows, _, _ = parse_traceability(self.root) + self.assertFalse(rows[1].test_refs) + + def test_bare_test_filename_with_line_suffix_resolves_file(self): + """A bare test filename citation may carry a line suffix without changing its target.""" + self.replace('docs/safety/TRACEABILITY.md', 'test_unique_probe.py', 'test_unique_probe.py:10') + rows, _, _ = parse_traceability(self.root) + self.assertEqual(rows[1].test_refs, ('tests/test_unique_probe.py',)) + + def test_explicit_test_path_with_line_range_resolves_file(self): + """An explicit test path may carry a line range without changing its target.""" + self.replace('docs/safety/TRACEABILITY.md', 'test_unique_probe.py', 'tests/test_unique_probe.py:10-20') + rows, _, _ = parse_traceability(self.root) + self.assertEqual(rows[1].test_refs, ('tests/test_unique_probe.py',)) + + def test_attached_test_symbol_uses_preceding_file_citation(self): + """A parenthesized test symbol attached to file:line does not become a missing stem.""" + self.replace( + 'docs/safety/TRACEABILITY.md', + 'test_unique_probe.py', + '`test_unique_probe.py:10` (`test_named_case`)', + ) + rows, _, issues = parse_traceability(self.root) + self.assertEqual(rows[1].test_refs, ('tests/test_unique_probe.py',)) + self.assertFalse([issue for issue in issues if issue.literal == 'test_named_case']) + + def test_real_machine_test_line_citations_resolve_without_opening_symbols(self): + """SR-SYS-09 file:line citations resolve while their attached symbols add no findings.""" + rows, _, issues = parse_traceability(REPO) + row = next(row for row in rows if row.sr_id == 'SR-SYS-09') + self.assertIn('pstop_c/pstop/test/src/pstop/machine_test.c', row.test_refs) + self.assertFalse([ + issue + for issue in issues + if issue.sr_id == 'SR-SYS-09' + and issue.literal in {'test_bond_stop_ok_stop_only_operator', 'test_2_clients_stop_only_stop'} + ]) + + +class ConsistencyTests(FixtureRepo): + def test_c1_flags_sr_set_mismatch(self): + """C1 reports an SRS requirement omitted from the matrix.""" + self.replace( + 'docs/safety/TRACEABILITY.md', + '| SR-R-01 | F-R-01 | code.c:1 | test_unique_probe.py | Test | **Verified** |\n', + '', + ) + self.assert_check('C1', 'SR-R-01') + + def test_c2_flags_unknown_trace_status(self): + """C2 rejects matrix statuses outside the documented closed vocabulary.""" + self.replace('docs/safety/TRACEABILITY.md', '**Verified** |', '**Complete** |') + with self.assertRaises(LintError): + analyze(self.root) + + def test_c3_flags_verified_without_evidence(self): + """C3 rejects a Verified row whose evidence cell resolves no test path.""" + self.replace('docs/safety/TRACEABILITY.md', 'test_unique_probe.py', 'NO TEST') + self.assert_check('C3', 'SR-R-01') + + def test_c4_flags_missing_cited_test(self): + """C4 identifies every SR affected by a cited test file removed from the tree.""" + (self.root / 'tests/test_unique_probe.py').unlink() + self.assert_check('C4', 'SR-R-01') + + def test_missing_shorthand_removes_ref_reduces_coverage_and_triggers_c3_c4(self): + """Deleting sole shorthand evidence removes coverage and reports both evidence checks.""" + self.replace('docs/safety/TRACEABILITY.md', 'test_unique_probe.py', 'HIL10') + before = compute_coverage(analyze(self.root)).cited_tests + (self.root / 'tools/hil/test_10_button.py').unlink() + analysis = analyze(self.root) + row = next(row for row in analysis.trace if row.sr_id == 'SR-R-01') + findings = run_checks(analysis) + self.assertEqual((row.test_refs, compute_coverage(analysis).cited_tests), ((), before - 1)) + self.assertTrue({finding.check_id for finding in findings if finding.subject == 'SR-R-01'} >= {'C3', 'C4'}) + + def test_missing_req_removes_ref_and_triggers_c3_c4(self): + """Deleting sole REQ evidence leaves a Partial row unresolved and reports C3 plus C4.""" + self.replace('docs/safety/TRACEABILITY.md', 'test_unique_probe.py', 'REQ 2_02') + (self.root / 'pstop_c/pstop/test/src/pstop/requirements/req_2_02_test.c').unlink() + analysis = analyze(self.root) + row = next(row for row in analysis.trace if row.sr_id == 'SR-R-01') + findings = run_checks(analysis) + self.assertFalse(row.test_refs) + self.assertTrue({finding.check_id for finding in findings if finding.subject == 'SR-R-01'} >= {'C3', 'C4'}) + + def test_c5_flags_missing_code_file(self): + """C5 rejects an explicit code citation whose repository file is absent.""" + self.replace('docs/safety/TRACEABILITY.md', 'code.c:1', 'missing.c:1') + self.assert_check('C5', 'SR-SYS-01') + + def test_c6_flags_unknown_function(self): + """C6 rejects allocations outside the authoritative function decomposition.""" + self.replace('docs/safety/SAFETY_REQUIREMENTS.md', 'F-R-01 |', 'F-X-99 |') + self.assert_check('C6', 'F-X-99') + + def test_c7_flags_unknown_upstream_reference(self): + """C7 warns when a requirement cites an absent hazard, goal, or DU identifier.""" + self.replace('docs/safety/SAFETY_REQUIREMENTS.md', 'SG-1, H-01, DU-1', 'SG-9, H-99, DU-9') + self.assert_check('C7', 'SG-9') + + def test_c8_flags_reverse_map_disagreement(self): + """C8 rejects a reverse map that disagrees with forward requirement allocation.""" + self.replace( + 'docs/safety/TRACEABILITY.md', '| F-R-01 | Sense | SR-SYS-01, SR-R-01 |', '| F-R-01 | Sense | SR-SYS-01 |' + ) + self.assert_check('C8', 'F-R-01') + + def test_c9_flags_srs_matrix_allocation_disagreement(self): + """C9 rejects differing SRS and matrix allocations for the same requirement.""" + self.replace('docs/safety/TRACEABILITY.md', '| SR-R-01 | F-R-01 |', '| SR-R-01 | F-R-02 |') + self.assert_check('C9', 'SR-R-01') + + def test_baseline_suppresses_exact_matching_finding(self): + """A justified baseline entry suppresses exactly one matching finding message.""" + self.replace('docs/safety/TRACEABILITY.md', 'test_unique_probe.py', 'NO TEST') + finding = next(finding for finding in self.findings() if finding.check_id == 'C3') + active, suppressed = apply_baseline( + self.findings(), + {('C3', 'SR-R-01', finding.message): {'reason': 'fixture', 'owner': 'test'}}, + ) + self.assertFalse([f for f in active if f.check_id == 'C3']) + self.assertTrue([f for f in suppressed if f.check_id == 'C3']) + + def test_same_subject_new_finding_is_not_suppressed(self): + """A new message on an already-baselined subject remains an active violation.""" + self.replace( + 'docs/safety/TRACEABILITY.md', + 'test_unique_probe.py', + 'test_missing_one, test_missing_two', + ) + findings = self.findings() + existing = next(finding for finding in findings if 'test_missing_one' in finding.message) + active, _ = apply_baseline( + findings, + {('C4', 'SR-R-01', existing.message): {'reason': 'fixture', 'owner': 'test'}}, + ) + self.assertFalse([finding for finding in active if finding.check_id == 'BASELINE']) + self.assertTrue([ + finding + for finding in active + if finding.check_id == 'C4' and finding.subject == 'SR-R-01' and 'test_missing_two' in finding.message + ]) + + def test_stale_exact_baseline_entry_is_an_error(self): + """An exact baseline entry with no current finding fails the ratchet as stale.""" + active, _ = apply_baseline( + self.findings(), {('C8', 'absent', 'old exact finding'): {'reason': 'fixture', 'owner': 'test'}} + ) + self.assertTrue([f for f in active if f.check_id == 'BASELINE' and f.severity == 'error']) + + def test_duplicate_exact_baseline_entries_are_rejected(self): + """The baseline loader rejects duplicate exact finding discriminators.""" + entry = { + 'check_id': 'C4', + 'subject': 'SR-R-01', + 'finding': 'missing fixture', + 'reason': 'fixture reason', + 'owner': 'test', + } + path = self.root / 'duplicate-baseline.json' + path.write_text(json.dumps({'findings': [entry, entry]}), encoding='utf-8') + with self.assertRaises(LintError): + _load_baseline(path) + + def test_undocumented_srs_status_is_informational(self): + """A valid status used by requirements but omitted from conventions is informationally visible.""" + findings = self.findings() + self.assertTrue([f for f in findings if f.check_id == 'C2' and f.severity == 'info']) + + +class CoverageRenderCliTests(FixtureRepo): + def test_coverage_matches_committed_summary(self): + """Real citation and status counts reproduce the ratified committed summary.""" + result = analyze(REPO) + coverage = compute_coverage(result) + self.assertEqual((coverage.total, coverage.cited_tests, coverage.verified), (40, 32, 17)) + + def test_generated_block_is_idempotent(self): + """Rendering an already rendered traceability document is byte-idempotent.""" + result = analyze(self.root) + original = (self.root / 'docs/safety/TRACEABILITY.md').read_text(encoding='utf-8') + once = render_traceability(original, compute_coverage(result)) + self.assertEqual(render_traceability(once, compute_coverage(result)), once) + + def test_check_mode_detects_stale_block(self): + """Check mode returns one when a generated numeric region is stale.""" + self.assertEqual(self.run_cli('--write').returncode, 0) + self.replace( + 'docs/safety/TRACEABILITY.md', + 'SRs with at least one cited verifying test: 2 / 2', + 'SRs with at least one cited verifying test: 1 / 2', + ) + proc = self.run_cli('--check') + self.assertEqual(proc.returncode, 1) + + def test_render_does_not_touch_prose_outside_markers(self): + """Rendering preserves hand-authored Reading prose byte-for-byte.""" + path = self.root / 'docs/safety/TRACEABILITY.md' + before = path.read_text(encoding='utf-8').split('**Reading:**', 1)[1] + after = render_traceability(path.read_text(encoding='utf-8'), compute_coverage(analyze(self.root))).split( + '**Reading:**', 1 + )[1] + self.assertEqual(after, before) + + def test_mixed_numeric_prose_is_checked_not_generated(self): + """A stale number outside generated markers remains a check failure.""" + result = analyze(REPO) + text = (REPO / 'docs/safety/TRACEABILITY.md').read_text(encoding='utf-8') + stale = text.replace('22 / 27 = 81.5 %', '21 / 27 = 77.8 %') + self.assertTrue(check_summary_prose(stale, compute_coverage(result))) + + def test_cli_exit_code_zero_on_clean_tree(self): + """The real repository passes with its checked-in baseline and current generated regions.""" + proc = subprocess.run( + [sys.executable, '-m', 'tools.safety_lint'], cwd=REPO, capture_output=True, text=True, check=False + ) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + + def test_cli_exit_code_one_on_injected_error(self): + """A parseable traceability parity defect returns the check-failed exit code.""" + self.replace( + 'docs/safety/TRACEABILITY.md', + '| SR-R-01 | F-R-01 | code.c:1 | test_unique_probe.py | Test | **Verified** |\n', + '', + ) + self.assertEqual(self.run_cli().returncode, 1) + + def test_write_refuses_active_error_without_touching_traceability(self): + """Write mode returns one and preserves bytes when consistency errors are active.""" + self.replace( + 'docs/safety/TRACEABILITY.md', + '| SR-R-01 | F-R-01 | code.c:1 | test_unique_probe.py | Test | **Verified** |\n', + '', + ) + path = self.root / 'docs/safety/TRACEABILITY.md' + before = path.read_bytes() + proc = self.run_cli('--write') + self.assertEqual((proc.returncode, path.read_bytes()), (1, before)) + + def test_normal_cli_reports_citation_execution_limitation(self): + """Normal CLI output explicitly says citation resolution does not verify execution.""" + proc = self.run_cli() + self.assertIn('Citation limitation: resolution does not verify test execution or passing state.', proc.stdout) + + def test_cli_exit_code_two_on_missing_document(self): + """A missing required safety document returns the cannot-run exit code.""" + (self.root / 'docs/safety/SAFETY_REQUIREMENTS.md').unlink() + self.assertEqual(self.run_cli().returncode, 2) + + def test_workflow_has_no_path_filters(self): + """CI runs on every pull request and every main push without path filtering.""" + text = (REPO / '.github/workflows/safety-lint.yml').read_text(encoding='utf-8') + self.assertNotIn('paths:', text) + self.assertIn('pull_request:', text) + self.assertIn('branches: [main]', text) + + def run_cli(self, *args): + return subprocess.run( + [sys.executable, '-m', 'tools.safety_lint', '--root', str(self.root), *args], + cwd=REPO, + capture_output=True, + text=True, + check=False, + ) + + +if __name__ == '__main__': + unittest.main(verbosity=2) From b8cef25ae784a0a1d7832362f98a6e27f633ef4d Mon Sep 17 00:00:00 2001 From: Raj Madhivanan Date: Fri, 11 Sep 2026 17:07:41 -0700 Subject: [PATCH 02/12] feat: enforce modification procedure records Co-authored-by: OpenCode --- .github/CODEOWNERS | 3 + .github/ISSUE_TEMPLATE/change-request.yml | 130 +++ .github/ISSUE_TEMPLATE/config.yml | 6 + .github/PULL_REQUEST_TEMPLATE.md | 13 + .github/workflows/change-control.yml | 32 + .github/workflows/coverage-delta.yml | 30 + .github/workflows/wire-break.yml | 47 + CONTRIBUTING.md | 7 + SECURITY.md | 2 - ...0002-modification-procedure-enforcement.md | 620 +++++++++++++ docs/process/BRANCH_PROTECTION.md | 16 + docs/process/EXTERNAL_CONTRIBUTIONS.md | 7 + docs/process/MODIFICATION_PROCEDURE.md | 337 +++++++ docs/process/NOTION_MIGRATION_MAP.md | 12 + docs/process/enforcement-mode | 1 + docs/process/templates/IMPACT_ANALYSIS.md | 93 ++ docs/process/templates/RELEASE_RECORD.md | 82 ++ scripts/check_wire_format.sh | 25 + scripts/sync_labels.sh | 35 + tools/change_control/__init__.py | 3 + tools/change_control/__main__.py | 139 +++ tools/change_control/checks.py | 365 ++++++++ tools/change_control/coverage_delta.py | 148 ++++ tools/change_control/fixtures/fake_gh.py | 46 + tools/change_control/issue_form.py | 120 +++ tools/change_control/labels.json | 18 + tools/change_control/self_test.py | 831 ++++++++++++++++++ tools/change_control/wire_format.py | 160 ++++ tools/change_control/wire_format.sha256 | 13 + 29 files changed, 3339 insertions(+), 2 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 .github/ISSUE_TEMPLATE/change-request.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/workflows/change-control.yml create mode 100644 .github/workflows/coverage-delta.yml create mode 100644 .github/workflows/wire-break.yml create mode 100644 changes/change-0002-modification-procedure-enforcement.md create mode 100644 docs/process/BRANCH_PROTECTION.md create mode 100644 docs/process/EXTERNAL_CONTRIBUTIONS.md create mode 100644 docs/process/MODIFICATION_PROCEDURE.md create mode 100644 docs/process/NOTION_MIGRATION_MAP.md create mode 100644 docs/process/enforcement-mode create mode 100644 docs/process/templates/IMPACT_ANALYSIS.md create mode 100644 docs/process/templates/RELEASE_RECORD.md create mode 100755 scripts/check_wire_format.sh create mode 100755 scripts/sync_labels.sh create mode 100644 tools/change_control/__init__.py create mode 100644 tools/change_control/__main__.py create mode 100644 tools/change_control/checks.py create mode 100644 tools/change_control/coverage_delta.py create mode 100755 tools/change_control/fixtures/fake_gh.py create mode 100644 tools/change_control/issue_form.py create mode 100644 tools/change_control/labels.json create mode 100755 tools/change_control/self_test.py create mode 100644 tools/change_control/wire_format.py create mode 100644 tools/change_control/wire_format.sha256 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..545a63da --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +* @iliabaranov @rajasimman-madhivanan @davidt315 diff --git a/.github/ISSUE_TEMPLATE/change-request.yml b/.github/ISSUE_TEMPLATE/change-request.yml new file mode 100644 index 00000000..44a309b3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/change-request.yml @@ -0,0 +1,130 @@ +--- +name: Change Request +description: Request an assessed change to Protective Stop +title: '[CR] ' +labels: [change-request, status:proposed] +body: + - type: textarea + id: reason + attributes: + label: Reason for the change + description: Why this change is needed. A defect, a new requirement, a dependency update, a corrective action. Link to the source if there is one. + placeholder: State the reason and source. + validations: + required: true + - type: textarea + id: hazards + attributes: + label: Hazards that may be affected + description: Which identified hazards this change could bear on. Reference the HARA entries. "None identified, because…" is a valid answer; blank is not. + placeholder: None identified, because… + validations: + required: true + - type: textarea + id: description + attributes: + label: Description of the proposed change + description: What specifically is proposed, covering both hardware and software. Enough detail that someone other than the requester can assess it. + validations: + required: true + - type: textarea + id: baseline + attributes: + label: Baseline affected + description: Firmware, host and hardware versions. + validations: + required: true + - type: input + id: requester + attributes: + label: Requester + description: Name and date. + validations: + required: false + - type: textarea + id: impact-analysis + attributes: + label: Impact Analysis + description: Link to or paste the completed Impact Analysis. Required before authorization. + validations: + required: false + - type: dropdown + id: class + attributes: + label: Proposed class + description: The Impact Analyst proposes and the Authorizer confirms the class. + options: + - A + - B + - C + default: 2 + validations: + required: true + - type: textarea + id: authorization + attributes: + label: Authorization + description: Authorizer, decision, date and basis. Class C requires two distinct Authorizers. + validations: + required: false + - type: textarea + id: implementation + attributes: + label: Implementation + description: Implementer; competency basis; pull requests covering software, hardware and tests; documentation updated; and upstream pstop_c change required - Yes / No, with link. + validations: + required: false + - type: textarea + id: gate-0 + attributes: + label: Gate 0 - Build Acceptance Test + description: Version under test, CI run, date and result. + validations: + required: false + - type: textarea + id: gate-1 + attributes: + label: Gate 1 - Merge qualification + description: Scope per the change class and Impact Analysis. Record test, version under test, Run by, date and result. For Class C record Forward - affected safety requirements to verification performed; Backward - verification performed to requirements + covered. + validations: + required: false + - type: textarea + id: review + attributes: + label: Review + description: Reviewer, who is not the Implementer; pull request review link; date; and outcome. + validations: + required: false + - type: textarea + id: deviations + attributes: + label: Deviations + description: Any deviation from this procedure, with justification. Includes the case where the Authorizer and Implementer are the same person. + validations: + required: false + - type: input + id: release + attributes: + label: Release Record + description: Release is handled separately in the Release Record. Record only the link here once this change ships. + validations: + required: false + - type: dropdown + id: status + attributes: + label: Status + description: Current Change Request lifecycle status. + options: + - Proposed + - Under Analysis + - Authorized + - Rejected + - In Implementation + - In Verification + - Merged + - Released + - Closed + default: 0 + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..cb179ead --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,6 @@ +--- +blank_issues_enabled: true +contact_links: + - name: Private safety and security reports + url: https://github.com/polymathrobotics/protective-stop/security/policy + about: Read SECURITY.md and report suspected safety or security defects privately. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..94270dbe --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,13 @@ +## Change control + +- Change Request: Closes #NN or Refs #NN +- Class: A / B / C +- Impact Analysis: link + +## Completion + +- [ ] Authorization preceded implementation +- [ ] Named tests and affected interfaces were exercised +- [ ] Documentation identified by the Impact Analysis is updated +- [ ] Review is by someone other than the implementer +- [ ] Gate evidence is attached to the Change Request diff --git a/.github/workflows/change-control.yml b/.github/workflows/change-control.yml new file mode 100644 index 00000000..0ce2e9b8 --- /dev/null +++ b/.github/workflows/change-control.yml @@ -0,0 +1,32 @@ +--- +name: Change control + +on: + pull_request: + types: [opened, synchronize, reopened, labeled, unlabeled, edited] + pull_request_review: + types: [submitted, dismissed] + +permissions: + contents: read + issues: write + pull-requests: read + checks: read + actions: read + +jobs: + change-control: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Run tests + run: python3 tools/change_control/self_test.py + - name: Check modification records + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + CAN_COMMENT: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + run: |- + args=() + [ "$CAN_COMMENT" = 'true' ] || args+=(--no-comment) + python3 -m tools.change_control --repository '${{ github.repository }}' --pr "$PR_NUMBER" "${args[@]}" diff --git a/.github/workflows/coverage-delta.yml b/.github/workflows/coverage-delta.yml new file mode 100644 index 00000000..45cf700d --- /dev/null +++ b/.github/workflows/coverage-delta.yml @@ -0,0 +1,30 @@ +--- +name: Coverage delta + +on: + pull_request: + +permissions: + contents: read + issues: write + +jobs: + coverage-delta: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: Compare safety-linter citations + env: + GH_TOKEN: ${{ github.token }} + CAN_COMMENT: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + run: |- + args=() + [ "$CAN_COMMENT" = 'true' ] || args+=(--no-comment) + python3 -m tools.change_control.coverage_delta \ + --base '${{ github.event.pull_request.base.sha }}' \ + --head '${{ github.event.pull_request.head.sha }}' \ + --repository '${{ github.repository }}' \ + --pr '${{ github.event.pull_request.number }}' \ + "${args[@]}" diff --git a/.github/workflows/wire-break.yml b/.github/workflows/wire-break.yml new file mode 100644 index 00000000..afb1fde6 --- /dev/null +++ b/.github/workflows/wire-break.yml @@ -0,0 +1,47 @@ +--- +name: Wire break + +on: + pull_request: + +permissions: + contents: read + issues: write + +jobs: + wire-break: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: Detect watched changes and apply wire-break label + env: + GH_TOKEN: ${{ github.token }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + CAN_LABEL: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + run: | + set -eu + signature_changed=false + expectation_preexisted=false + if git cat-file -e "$BASE_SHA:tools/change_control/wire_format.sha256" 2>/dev/null; then + expectation_preexisted=true + fi + if { [ "$expectation_preexisted" = true ] && \ + git diff --name-only "$BASE_SHA"...HEAD | grep -Eq '^tools/change_control/wire_format\.sha256$'; } || \ + { [ "$expectation_preexisted" = false ] && \ + git diff --name-only "$BASE_SHA"...HEAD | grep -Eq '^pstop_c/pstop/include/pstop/'; }; then + signature_changed=true + elif ! python3 -m tools.change_control.wire_format check --root . >/dev/null; then + signature_changed=true + fi + if [ "$signature_changed" = true ] && [ "$CAN_LABEL" = true ]; then + gh api --method POST 'repos/${{ github.repository }}/issues/'"$PR_NUMBER"'/labels' -f 'labels[]=wire-break' >/dev/null + fi + labels=$(gh api 'repos/${{ github.repository }}/issues/'"$PR_NUMBER" --jq '[.labels[].name] | join(",")') + printf 'PSTOP_PR_LABELS=%s\n' "$labels" >> "$GITHUB_ENV" + - name: Enforce wire-format declaration + env: + PSTOP_BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: scripts/check_wire_format.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f584d605..d7eeefc0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -90,6 +90,13 @@ Note `pstop_c/` is intentionally excluded from the C/C++ hooks. ## Pull request expectations +### Maintainer-owned change records + +You do not need to understand or write the project's safety Change Request or +Impact Analysis before contributing. If a pull request arrives without one, a +maintainer follows [`docs/process/EXTERNAL_CONTRIBUTIONS.md`](docs/process/EXTERNAL_CONTRIBUTIONS.md), +opens and assesses the record, and then starts review. + - **CI green.** Firmware build, host build, `pstop_c` build + tests, and pre-commit must all pass. - **Pre-commit clean.** Run it locally before pushing; do not disable diff --git a/SECURITY.md b/SECURITY.md index 14d30522..cde2228b 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -11,8 +11,6 @@ Please report suspected vulnerabilities **privately**. Do not open a public issue, pull request, or discussion for a security problem. - Email: **security@polymathrobotics.com** - _(placeholder — please confirm the correct security contact before this - policy is published)_ Include, where possible: diff --git a/changes/change-0002-modification-procedure-enforcement.md b/changes/change-0002-modification-procedure-enforcement.md new file mode 100644 index 00000000..3327f287 --- /dev/null +++ b/changes/change-0002-modification-procedure-enforcement.md @@ -0,0 +1,620 @@ +# change-0002 — Modification procedure in-repo, and automated change-record enforcement + +**Implements:** The Protective-Stop Modification Procedure, currently held in Notion +and migrated to the repository by this change. Authority for the safety argument +remains `docs/safety/`. +**Branch:** `change-0002-modification-procedure-enforcement` +**Base:** `main` +**Depends on:** change-0001 (safety traceability linter). Piece 6 consumes +`tools/safety_lint`. Pieces 1–5 and 7 do not. +**Status:** implementation and label synchronization complete; PR verification pending +**Safety class:** B — process governance, assurance tooling, and CI guards only; +no runtime safety-path or wire-format change. +**Authorization:** one authorizer approved implementation with the exact instruction +`proceed` on 2026-09-11. +**Stacked development base:** change-0001 commit `326a6d0` while PR #119 remains +open; the change-0002 PR still targets `main` and depends on #119 landing first. +**Security contact authorization:** on 2026-09-11 the authorizer confirmed +`security@polymathrobotics.com` as the private reporting route, authorizing removal +of its placeholder warning in `SECURITY.md` and use by the issue chooser. + +Read `changes/change-0001-safety-traceability-linter.md` §3 and §8 before starting — +the integration facts and traps there apply here unchanged. Read the four Notion +pages named in Piece 1 in full before migrating them; they are the source text and +this document is the work breakdown. Where the two disagree, the Notion pages win for +*content* and this document wins for *destination and format*. + +**Everything this change adds is inert until switched on.** A single file sets warn +or enforce mode. Land it in warn, read what it would have blocked over two or three +weeks of real changes, then flip. One exception, stated in Piece 5 and non-negotiable: +the wire-break check enforces from day one. + +--- + +## 1. What this change delivers + +The modification procedure and its three record templates move from Notion into the +repository. The Change Request becomes a real GitHub issue form. Labels, CODEOWNERS +and branch protection give the procedure's approval rules a mechanism. A CI job checks +that each PR has an authorized change request with a completed impact analysis behind +it, that the class label matches what the diff actually touches, and that the tests the +impact analysis named were the tests that ran. A separate check refuses an unannounced +wire-format change. The Claude review bot posts an advisory comment on requirement +coverage; it gates nothing and never writes to a safety document. + +**Settled decision 1 — the mode switch is a file, not a repository variable.** Flipping +enforcement is itself a change to how the safety process operates and belongs in the +git history with an author and a reviewer. + +**Settled decision 2 — CI enforces that an artifact exists, is complete, and arrived in +the right order. It never judges whether the artifact is correct.** Classification +accuracy, impact-analysis quality, and verification sufficiency are human judgements. +The bot may advise on all three; it decides none of them. + +**Settled decision 3 — the bot's output never reaches a certification number.** The +coverage figure in `docs/safety/TRACEABILITY.md` comes only from the deterministic +linter. A model's judgement is not reproducible, and wiring one into a safety gate +would oblige a tool-qualification argument under IEC 61508-3 §7.4.4 that nothing else +in this change requires. + +--- + +## 2. Scope + +### IN scope + +1. `docs/process/MODIFICATION_PROCEDURE.md` plus three templates, migrated from Notion. +2. `.github/ISSUE_TEMPLATE/change-request.yml` — the Change Request as a GitHub issue + form. +3. Label set, `CODEOWNERS`, and a written branch-protection configuration. +4. `docs/process/enforcement-mode` — the one-line file that sets warn or enforce. +5. `.github/workflows/change-control.yml` and `tools/change_control/` — the CR, impact + analysis, classification-floor and IA-versus-CI checks. +6. `.github/workflows/wire-break.yml` and `scripts/check_wire_format.sh` — enforcing + from day one. +7. `.github/workflows/coverage-delta.yml` — runs the change-0001 linter on base and + head and posts the delta. +8. Claude review bot configuration for the advisory sufficiency comment. +9. `docs/process/EXTERNAL_CONTRIBUTIONS.md` — the path for a PR that arrives without a + change request. +10. `SECURITY.md` — remove the now-resolved placeholder warning for the confirmed + private reporting address. + +### OUT of scope — do NOT build + +- **Any edit to `docs/safety/`.** Same prohibition as change-0001 and for the same + reason. If a check fails on safety-document content, report it; do not fix it. +- **Deleting or archiving the Notion pages.** Migration is copy-then-verify. Raj + retires them once exida has the URL map from Piece 1e. +- Enabling enforce mode. This change lands in warn. +- Enabling branch protection. Piece 4 writes the configuration down and the repo admin + applies it out of band; an agent does not change repository settings. +- The Gate 2 release-qualification workflow. That is change-0003. +- Any change to `tools/safety_lint/` beyond calling it. If it needs a new output mode, + stop and raise rather than editing it here. +- `pstop_c/` contents. Piece 5 reads its headers and never modifies them. +- Any automated status transition in `TRACEABILITY.md`. + +--- + +## 3. Integration facts (do not rediscover these) + +| Fact | Value | +|---|---| +| Repository | `polymathrobotics/protective-stop`, public | +| Existing workflows | `pre-commit.yml`, `firmware-build.yml`, `host-check.yml`, `ros2_build.yml`, `pstop_c_build.yml`, `pstop_c_coverage.yml`, `coverage.yml` | +| Existing `.github` contents | the seven workflows above plus `dependabot.yml`. No `CODEOWNERS`, no `ISSUE_TEMPLATE`, no `PULL_REQUEST_TEMPLATE` | +| Checkout action version in use | `actions/checkout@v7` | +| Authorizers | Ilia Baranov, Raj, David Tarazi — all three are code owners of everything and all three may authorize | +| Protocol version constant | `pstop_c/pstop/include/pstop/config.h` — `PSTOP_VERSION 0x02U`, `PSTOP_MESSAGE_SIZE 48U` | +| Wire-format headers | `pstop_c/pstop/include/pstop/` — `config.h`, `constants.h`, `protocol.h`, `protocol_data.h`, `pstop_msg.h`, `checksum.h`, `device_id.h`, `endian.h` | +| `pstop_c` is vendored in-tree | A directory, NOT an ESP-IDF managed component. It does not appear in `firmware/dependencies.lock`. A "version bump" is a directory update, so the check compares header content, not a lockfile line | +| Linter entry point (change-0001) | `python3 -m tools.safety_lint --json` | +| Pre-commit exclusions | `pstop_c/`, `ros2/`, `archive/`, vendored wireguard and x25519, `hardware/` binaries. `tools/`, `docs/` and `.github/` are in scope | +| Required file header | SPDX two-line, Apache-2.0. Copy `tools/test_config_floor.py` or `scripts/check_estop_diversity.sh` | +| Guard-script convention | `set -uo pipefail`, repo-root `cd`, header comment naming the SR and DU it guards, exit `0` pass / `1` check failed / `2` cannot run | +| Notion — Modification Procedure | `3d6c0b1ac5fa8105b456e862e939f051` | +| Notion — Impact Analysis Template | `3d6c0b1ac5fa81838a09f22388095c33` | +| Notion — Change Request Template | `3d6c0b1ac5fa8167927af2e753be67c5` | +| Notion — Release Record Template | `3d6c0b1ac5fa81708a8cff41150d2b86` | + +### Two things the plan author could not determine + +This plan was written without authenticated GitHub access. **Piece 0 discovers both +before anything else is built.** Do not assume either answer. + +1. **Which branch-protection and CODEOWNERS features this repository actually has.** + Public repositories get most of them free, but required-reviewer behaviour and + rulesets differ by plan. +2. **How the Claude review bot is installed and configured.** There is no + `.github/claude.yml` and no reference to `claude` or `anthropic` anywhere under + `.github/`, so it is an org-level GitHub App or configured outside the repo. + +### Piece 0 discovery result — 2026-09-11 + +1. The repository is public on GitHub Team. `main` reports `protected: true`, but + this non-admin account receives 404 from the classic protection-details endpoint; + repository and branch rules endpoints expose no applicable rules. Existing + protection details therefore remain unknown and Piece 4 documents the intended + settings without applying them. +2. Required reviews from CODEOWNERS are available for this public Team repository. + Verified write-capable handles are `@iliabaranov`, + `@rajasimman-madhivanan`, and `@davidt315`. The originally suggested `@dtarazi` + resolves but has read-only repository access and must not be used as a code owner. +3. Check-run metadata identifies GitHub App `claude`, owned by `anthropics`, as the + review bot. Neither this repository nor the organization `.github` repository + contains visible Claude configuration, and installation metadata requires + unavailable `admin:org` access. Per Piece 8e, build deterministic coverage delta + only; do not build or guess bot configuration. +4. Existing labels are only GitHub's defaults: `bug`, `dependencies`, + `documentation`, `duplicate`, `enhancement`, `github_actions`, `good first issue`, + `help wanted`, `invalid`, `question`, and `wontfix`. None collides with Piece 3. + +--- + +## 4. Pieces, in dependency order + +Piece 0 is discovery and must complete first. Pieces 1–4 add no CI surface. Piece 5 is +the only thing in this change that can block a merge on the day it lands. + +--- + +### Piece 0 — Discovery + +**Files:** none. Output is a report. + +**0a.** Run, and record the output verbatim: + +``` +gh api repos/polymathrobotics/protective-stop --jq '{visibility,default_branch,allow_auto_merge,delete_branch_on_merge}' +gh api repos/polymathrobotics/protective-stop/branches/main/protection 2>&1 | head -40 +gh api repos/polymathrobotics/protective-stop/rulesets 2>&1 | head -40 +gh api orgs/polymathrobotics/installations --jq '.installations[].app_slug' 2>&1 +gh api repos/polymathrobotics/protective-stop/labels --jq '.[].name' +``` + +**0b. Report four answers before writing any code:** +- Is branch protection already configured on `main`, and with what? +- Are required reviewers via CODEOWNERS available on this plan? +- Which app provides the Claude review bot, and where does it read configuration from? + If you cannot determine this from the API, say so and check for an org-level + `.github` repository holding shared workflow or app configuration. +- Which labels already exist, so Piece 3 adds rather than collides. + +**0c.** If required reviewers are unavailable on this plan, **stop and raise.** The +two-authorizer rule for Class C then has no mechanism and Piece 4 needs redesigning as +a CI check against PR review state rather than a CODEOWNERS rule. Do not silently +substitute one for the other. + +--- + +### Piece 1 — Migrate the procedure and templates out of Notion + +**Files:** `docs/process/MODIFICATION_PROCEDURE.md`, +`docs/process/templates/IMPACT_ANALYSIS.md`, +`docs/process/templates/RELEASE_RECORD.md`, +`docs/process/NOTION_MIGRATION_MAP.md`. + +**1a.** Read all four Notion pages in full using the Notion MCP tools. Bare page UUIDs +work; full URLs are less reliable. + +**1b. Migrate content unchanged.** Preserve section numbering, the gate model, the +clause index, the declared-gaps table and the callout warning that the procedure has +not yet been exercised. **Do not improve the text.** Three corrections are authorized +and no others: +- Rewrite Notion-relative references as repo-relative links. +- Update the Roles section for three authorizers. The deviation clause covering + "where team size makes this impossible" stays in the document and is annotated as + not currently applicable. +- Where the procedure names a template, link the migrated file rather than the Notion + page. + +**1c. The Change Request template does NOT become a markdown file.** It becomes the +issue form in Piece 2. `docs/process/` holds the procedure, the impact analysis +template and the release record template only. + +**1d. Convert Notion callouts and tables to plain markdown** that the `polymath-markdown` +pre-commit hook accepts. Watch the list-marker rewrite recorded in +`docs/safety/OPEN_ITEMS.md` §8 — if the hook reformats files outside `docs/process/`, +revert and report. + +**1e. `NOTION_MIGRATION_MAP.md`** — a table mapping each Notion page URL to its new +repository path. Required for the certification evidence chain: the assessment +workbook cites Notion URLs as evidence and those citations need a forwarding address. +Include the four pages in this change and note that other Notion citations +(`FSM-1`, `FSM-11`, `CM-2`, `CM-5`, `VTP-8`) are not yet migrated. + +**Tests:** none — this piece is content migration. Verification is Piece 1f. + +**1f. Verify the migration by diff, not by reading.** For each of the three migrated +pages, produce a normalized text comparison between the Notion source and the +committed markdown, and report any sentence present in one and absent from the other. +Attach the comparison to the PR. A migration that silently drops a paragraph is the +failure mode here, and re-reading your own output will not catch it. + +--- + +### Piece 2 — Change Request as a GitHub issue form + +**Files:** `.github/ISSUE_TEMPLATE/change-request.yml`, +`.github/ISSUE_TEMPLATE/config.yml`, `.github/PULL_REQUEST_TEMPLATE.md`. + +**2a.** Build `change-request.yml` as a GitHub issue form carrying every field from the +Notion Change Request Template. Fields marked required in the form: +- Reason for the change +- **Hazards that may be affected** — required, with the placeholder making clear that + "none identified, because…" is a valid answer and blank is not +- Description, covering both hardware and software +- Baseline affected — firmware, host and hardware versions +- Proposed class — dropdown A / B / C, **defaulting to C** + +Non-required at creation because they are completed later: impact analysis link, +authorization, implementation, gate evidence, review, deviations, release record. + +**2b. Default to C.** A requester who does not know the class produces a Class C +request, which gets reviewed. The reverse default produces silent under-classification. + +**2c. `config.yml`** — `blank_issues_enabled: false` is **not** set. A public repository +needs a path for bug reports and questions that are not change requests. Add a contact +link to `SECURITY.md` for safety-defect reports. + +**Implementation blocker (2026-09-11):** `SECURITY.md` explicitly marks +`security@polymathrobotics.com` as a placeholder requiring confirmation, and GitHub +issue-form contact links require a URL rather than a repository-relative file. No +truthful private reporting destination is available in the repository, so +`.github/ISSUE_TEMPLATE/config.yml` is intentionally not created. An authorizer must +confirm a private destination before Piece 2c can be completed; a public issue route +must not be presented as suitable for safety-defect reports. + +**2d. PR template** — a short form with: linked change request (`Closes #NN` or +`Refs #NN`), class, impact analysis link, and a checklist mirroring the definition of +done. Kept short; a long PR template gets ticked without reading. + +**2e. The form is the single source of truth for its own field list.** Piece 6's +checker reads the required-field names from this YAML rather than holding a private +copy. A checker with its own idea of the format is a second source of truth. + +**Tests:** +- `test_issue_form_is_valid_yaml_and_parses` +- `test_required_fields_present` — the five fields in 2a are marked required. +- `test_class_dropdown_defaults_to_c` — drift-verify: change the default to A, confirm + the test fails, revert. + +--- + +### Piece 3 — Labels + +**Files:** `tools/change_control/labels.json`, `scripts/sync_labels.sh`. + +**3a.** Define the label set as data, and a script that creates or updates them via +`gh label`. Do not create them by hand — the set has to be reproducible. + +| Label | Purpose | +|---|---| +| `change-request` | Marks an issue as a CR. Applied automatically by the issue form | +| `class-a`, `class-b`, `class-c` | Safety classification | +| `emergency` | Compressed-timeline path | +| `safety-defect` | Defect in a released baseline affecting safety | +| `wire-break` | Applied automatically by Piece 5 | +| `needs-change-request` | External PR awaiting a maintainer-opened CR | +| `status:proposed`, `status:under-analysis`, `status:authorized`, `status:rejected`, `status:in-implementation`, `status:in-verification`, `status:merged`, `status:released` | CR lifecycle | + +**3b.** Use whatever labels Piece 0b found already present rather than creating a +near-duplicate. Report any collision instead of resolving it yourself. + +**Tests:** `test_labels_json_has_no_duplicate_names`; +`test_every_label_referenced_in_the_procedure_exists_in_labels_json` — parse the +migrated procedure for backticked label names and assert each is defined. This catches +the procedure and the tooling drifting apart. + +--- + +### Piece 4 — CODEOWNERS and branch protection + +**Files:** `.github/CODEOWNERS`, `docs/process/BRANCH_PROTECTION.md`. + +**4a. CODEOWNERS — everyone owns everything**, per the Director's instruction: + +``` +* @iliabaranov @rajasimman-madhivanan @davidt315 +``` + +Resolve the real GitHub handles with `gh api users/...` or from the repo's contributor +list; do not guess them. A CODEOWNERS file with an unresolvable handle silently matches +nothing, which is the worst failure mode available here — it looks configured and +enforces nothing. **Verify each handle resolves and report the verification.** + +**4b. `BRANCH_PROTECTION.md`** — the intended configuration, written down for a repo +admin to apply. It is documentation in this change, not an applied setting. + +| Setting | Value | Rationale | +|---|---|---| +| Require a pull request before merging | on | The procedure's merge gate | +| Required approvals | 1 | Class C's second approval comes from the Piece 6 check, so that the rule tracks the class rather than applying to every change | +| Dismiss stale approvals on new commits | on | An approval is of a diff, not of a branch | +| Require review from Code Owners | on | Makes one of the three authorizers a required reviewer | +| Require status checks | `pre-commit`, `host-check`, `firmware-build`, `ros2_build`, `pstop_c_build`, `wire-break` | Gate 0. `change-control` joins this list when enforce mode is switched on, not before | +| Require branches up to date | on | Two changes each green alone can be red together | +| Allow force pushes | off | Public repo; published history stands | +| Allow deletions | off | | + +**4c. Self-approval.** GitHub already refuses a review from the PR author, so +"authorizer ≠ implementer" holds for the first approval without extra machinery. The +second Class C approval is checked in Piece 6, which must assert two distinct +approving reviewers neither of whom is the author. + +**Tests:** `test_codeowners_parses_and_covers_root`; +`test_codeowners_handles_resolve` — network-gated, skipped without `GH_TOKEN`, and its +skip must be visible in the output rather than silent. + +--- + +### Piece 5 — Wire-break check (enforces immediately) + +**Files:** `scripts/check_wire_format.sh`, `.github/workflows/wire-break.yml`, +`tools/change_control/wire_format.sha256`. + +**5a. What this guards.** `pstop_c` is vendored in-tree. A change to the message +layout alters the checksum computed over it, and a remote and a machine on different +layouts reject each other's messages entirely. The machine's heartbeat times out and +it fail-safes to STOP — safe, and permanently stopped until both ends are updated +together. The build stays green throughout. This check makes that impossible to ship +unannounced. + +**5b. The signature.** A SHA-256 over the normalized content of the wire-format headers +listed in §3, plus the literal values of `PSTOP_VERSION` and `PSTOP_MESSAGE_SIZE`. +Normalize by stripping comments and collapsing whitespace so a comment edit does not +trip it. Store the expected value in `wire_format.sha256` with a comment recording the +`PSTOP_VERSION` it corresponds to. + +**5c. Behaviour on mismatch.** Fail the PR with a message naming which headers changed +and stating that remote and machine must be released and deployed together. Apply the +`wire-break` label and require the `class-c` label. The fix path is to update +`wire_format.sha256` in the same PR, which makes the change explicit in the diff and +reviewable — the point is not to prevent wire changes, it is to prevent *silent* ones. + +**5d. This check ignores `docs/process/enforcement-mode` and always enforces.** It has +no judgement in it and no false positives — the headers either changed or they did not +— and the failure mode is a field outage rather than a process complaint. Hardcode +this; do not make it configurable. + +**Tests:** +- `test_signature_stable_across_comment_only_change` — add a comment to `protocol.h` + in a scratch copy, assert the signature is unchanged. +- `test_signature_changes_on_field_addition` — add a field to a struct in a scratch + copy, assert it changes. +- `test_signature_changes_on_message_size_change` +- `test_check_exits_one_on_mismatch_and_names_the_headers` +- Drift-verify: corrupt `wire_format.sha256`, confirm CI fails, restore. Paste the run. + +--- + +### Piece 6 — Change-control checks + +**Files:** `tools/change_control/__main__.py`, `tools/change_control/checks.py`, +`.github/workflows/change-control.yml`, `docs/process/enforcement-mode`. + +Stdlib only, same constraint as change-0001. GitHub state comes from `gh api` called +via `subprocess`, not from a Python GitHub library. + +**6a. The mode file.** `docs/process/enforcement-mode` contains exactly one word, +`warn` or `enforce`. Land it as `warn`. Every check prints the active mode in its +output so no result is ever ambiguous about which mode produced it. In `warn`, findings +print and the job exits 0. In `enforce`, findings print and the job exits 1. + +**6b — E1. A change request exists and is authorized.** The PR body links an issue +carrying `change-request`. That issue has the `status:authorized` label and an +authorization comment from one of the three authorizers. A PR labelled +`needs-change-request` is exempt and reported as pending — see Piece 7. + +**6c — E2. An impact analysis exists and is complete.** The linked CR has a comment +containing the impact analysis. Every section heading from +`docs/process/templates/IMPACT_ANALYSIS.md` is present **and has non-empty content +beneath it**. Presence alone is not completeness — an IA with every heading and "N/A" +under each passes a heading check perfectly, which is why this check reads content. + +**This check cannot tell whether the content is true.** It catches wholesale omission +only. Say so in the output so nobody reads a green E2 as an endorsement. + +**6d — E3. Cited requirement IDs are real.** Every `SR--` in the impact +analysis parses and exists in `docs/safety/SAFETY_REQUIREMENTS.md`. Reuse +`tools.safety_lint.parse_srs` rather than writing a second parser. + +**6e — E4. Classification floor.** Path and content rules that force a minimum class +regardless of the applied label: + +| Trigger | Minimum | +|---|---| +| Any file under `pstop_c/` | C | +| `firmware/main/main.c` or `machn/main/main.c` | C | +| Wire signature changed (Piece 5) | C | +| Any file under `docs/safety/` | C | +| Any file under `components/` or `common/` | B | +| `sdkconfig.defaults` in `firmware/` or `machn/` | B | +| Any `.github/workflows/**` or `scripts/**` guard | B | + +A label below the floor is a finding. **A label above the floor is never a finding** — +over-classification is always allowed. The floor cannot catch a Class C change in an +unlisted path; that is what the bot's advisory comment in Piece 8 is for. + +**6f — E5. Two distinct approvals for Class C.** Two approving reviews from two +different accounts, neither of whom is the PR author, and both among the three +authorizers. Read review state with `gh api`. + +**6g — E6. The impact analysis named tests; those tests ran.** Parse test names and +paths from the IA's verification-plan section. Check each against the workflow runs on +the head commit. A test named in the IA with no corresponding run is a finding. + +**This is the strongest check in the change.** It is the only one that catches the +plan and the execution diverging, which is the failure nobody notices today. Expect it +to be noisy at first — that noise is information about how IAs are actually written, +and it is the main thing warn mode exists to surface. + +**6h — E7. Emergency path.** A PR labelled `emergency` requires both authorizer +approvals regardless of class, and the CR must carry a short-form IA. Report the +five-working-day retrospective deadline in the output; do not attempt to enforce a +deadline in CI. + +**6i. Output.** A single PR comment, updated in place rather than appended, listing +each check with pass / fail / not-applicable, the active mode, and a one-line +explanation per failure. Never more than one comment per PR. + +**Tests:** one per check against synthetic fixtures under +`tools/change_control/fixtures/`, not against live GitHub. Plus: +- `test_warn_mode_exits_zero_with_findings` +- `test_enforce_mode_exits_one_with_findings` +- `test_mode_file_rejects_unknown_value` — exit 2, not a silent default to warn. +- `test_e4_over_classification_is_not_a_finding` +- `test_e2_rejects_heading_present_but_empty` — drift-verify this one; it is the + difference between a real check and a decorative one. + +--- + +### Piece 7 — External contributions + +**Files:** `docs/process/EXTERNAL_CONTRIBUTIONS.md`, edit to `CONTRIBUTING.md`. + +**7a.** The path, per the Director's ruling: a PR arriving without a change request is +labelled `needs-change-request` and review does not begin. A maintainer opens the CR on +the contributor's behalf, classifies it, and completes the impact analysis. The +contributor is not asked to write one — they cannot, since they do not have the safety +context. + +**7b.** Once the CR exists and is authorized, the label is removed and the normal +checks apply. + +**7c.** Add a short section to `CONTRIBUTING.md` pointing at this document, framed so a +contributor understands the CR is maintainer work rather than a barrier to them. + +**Tests:** `test_needs_change_request_label_exempts_e1` — a PR with that label produces +a pending result on E1, not a failure. + +--- + +### Piece 8 — Advisory review + +**Files:** `.github/workflows/coverage-delta.yml`, Claude bot configuration at +whatever path Piece 0 established. + +**8a. Coverage delta is deterministic and comes from the linter.** Run +`python3 -m tools.safety_lint --json` at the merge base and at the head, diff the +results, post: coverage before and after, any requirement that gained or lost a +citation, any newly unresolvable citation. Mechanical, reproducible, no model +involved. + +**8b. Sufficiency is advisory and comes from the bot.** Its input: the diff, the +coverage delta from 8a, and the full text of every requirement the change touched. Its +question: *are the tests in this change adequate for what changed?* + +**8c. Three instructions the bot needs, because these are its predictable failure +modes:** +- **Flat coverage is not a pass.** A change can add a whole code path under an + already-cited requirement and move no number at all. Reason about the change, not + the metric. +- **Report what you could not assess.** A comment with an empty "could not confirm" + section means the review was shallow, not that everything is fine. +- **Propose a classification, and flag under-classification the path rules would + miss.** The Piece 6e floor is mechanical and cannot catch a Class C change in an + unlisted path. + +**8d. Hard boundary.** The bot's output is a PR comment. It never writes to any file +under `docs/safety/`, never proposes a status transition, never feeds a published +number, and blocks nothing. State this in its configuration, not only here. + +**8e.** If Piece 0 could not determine how the bot is configured, **build 8a and stop.** +Report what you found and leave 8b–8d unbuilt. The deterministic half is the half that +matters; do not guess at an app's configuration format. + +**Tests:** `test_coverage_delta_detects_lost_citation` — integration: in a scratch +clone, delete a cited test, assert the delta names the affected requirement. + +--- + +## 5. Definition of Done + +1. **Piece 0's four answers reported before any other piece was built.** +2. **Tests written first** (red → green), driving the real path. +3. **Drift-verify** the three named guards — the class default in 2c, the wire + signature in 5e, and the empty-section check in 6e. Break it, confirm failure, + revert, report that you did it. +4. **Run the gates yourself and paste actual output:** each new workflow's run, the + check suites, and `pre-commit run --all-files`. +5. **`git diff docs/safety/` must be empty.** Paste it. Any change there is a scope + violation. +6. **The migration comparison from 1f is attached to the PR.** +7. **`docs/process/enforcement-mode` says `warn`.** Confirm explicitly. +8. **The wire-break check is live and enforcing**, with the drift-verify run pasted. +9. **CODEOWNERS handles verified to resolve**, with the verification output. +10. **Two separate sections, not merged:** *"out of scope, confirmed not built"* and + *"in scope, required, not done"* — the second must be empty. + +--- + +## 6. Acceptance criteria + +- **AC-1 — nothing new blocks a merge except the wire check.** Open a trivial + docs-only PR with no change request. The `change-control` job reports findings and + passes. The `wire-break` job passes. Paste both. +- **AC-2 — the wire check bites.** In a scratch branch, add a field to a struct in a + wire-format header. `wire-break` fails, names the header, and applies the label. + Restore. Paste the run. +- **AC-3 — the wire check does not false-positive.** Add a comment to `protocol.h`. + The check passes. Paste the run. +- **AC-4 — enforce mode works.** Flip the mode file to `enforce` in a scratch branch, + re-run against the AC-1 PR, confirm the job now fails with the same findings. Revert + to `warn`. Paste both runs. +- **AC-5 — the empty-IA check is real.** A fixture IA with every heading present and no + content under them produces an E2 finding. +- **AC-6 — over-classification is permitted.** A docs-only PR labelled `class-c` + produces no E4 finding. +- **AC-7 — the IA-versus-CI check works.** A fixture IA naming a test that did not run + produces an E6 finding naming that test. +- **AC-8 — migration is complete.** The 1f comparison shows no sentence present in + Notion and absent from the repository. +- **AC-9 — the label set is reproducible.** Run `scripts/sync_labels.sh` twice; the + second run makes no changes. +- **AC-10 — coverage delta works.** Paste the bot comment from a PR that touches a + cited test. + +--- + +## 7. Workspace hygiene + +- `git status` before you start. Foreign changes: stop and report. +- Branch `change-0002-modification-procedure-enforcement` from `main`. Never commit to + `main`. +- Touch only the files this plan names. +- **Do not change repository settings.** Branch protection, label creation on the live + repo beyond `sync_labels.sh`, and app installation are human actions. Write the + configuration down; do not apply it. +- **Do not modify or delete anything in Notion.** + +--- + +## 8. Known traps + +1. **A CODEOWNERS file with an unresolvable handle matches nothing and reports + nothing.** It looks configured and enforces zero. Verify every handle resolves and + paste the verification. This is the highest-consequence silent failure in the + change. +2. **`pstop_c` is vendored in-tree, not a managed component.** It is absent from + `firmware/dependencies.lock`. A check written against the lockfile will never fire. + Compare header content. +3. **An impact analysis with every heading and "N/A" under each passes a formatting + check perfectly.** That is why E2 reads content. A decorative check next to real + ones is worse than no check, because it lends them its own emptiness. +4. **Warn mode that nobody reads is the same as no mode.** The output of warn mode is + the deliverable of this change, not a side effect. E6 in particular will be noisy, + and that noise is the finding. +5. **The markdown pre-commit hook rewrites `-` list markers to `+`** + (`docs/safety/OPEN_ITEMS.md` §8). If it reformats files outside `docs/process/`, + revert and report rather than dragging a repo-wide reformat in. +6. **Issue forms are YAML with a strict schema.** An invalid form does not error — it + silently falls back to a blank issue, and nobody notices the hazards field stopped + being required. Validate it and assert on the parse. +7. **`docs/safety/` is out of scope, and this change touches the process that governs + it.** The temptation to fix one small inconsistency while in there is exactly what + the prohibition exists to stop. +8. **Do not enable enforce mode, and do not enable branch protection.** Both are + Director decisions after the warn period. An agent that "finishes the job" by + switching them on has shipped an unreviewed process change. diff --git a/docs/process/BRANCH_PROTECTION.md b/docs/process/BRANCH_PROTECTION.md new file mode 100644 index 00000000..cae3af3b --- /dev/null +++ b/docs/process/BRANCH_PROTECTION.md @@ -0,0 +1,16 @@ +# Intended Branch Protection + +These are intended settings for a repository administrator to apply to `main`. This document does not assert that the settings are currently active; protection details were not readable by the implementing account. + +| Setting | Value | Rationale | +|---|---|---| +| Require a pull request before merging | On | Modification Procedure merge gate | +| Required approvals | 1 | Class C's second approval is checked by `change-control`, so the rule follows class rather than burdening every change | +| Dismiss stale approvals on new commits | On | An approval applies to a diff, not a branch name | +| Require review from Code Owners | On | Requires one of the three authorizers | +| Required status checks | `pre-commit`, `host-check`, `firmware-build`, `ros2_build`, `pstop_c_build`, `wire-break` | Gate 0 subset; add `change-control` only after enforce mode is authorized | +| Require branches up to date | On | Independently green changes can fail together | +| Allow force pushes | Off | Published history stands | +| Allow deletions | Off | Preserve the protected baseline | + +The repository reports `main` as protected, but the classic protection-details endpoint returned 404 and no applicable rules were visible. These intended settings must be compared with live settings by an administrator before application. diff --git a/docs/process/EXTERNAL_CONTRIBUTIONS.md b/docs/process/EXTERNAL_CONTRIBUTIONS.md new file mode 100644 index 00000000..c9085d54 --- /dev/null +++ b/docs/process/EXTERNAL_CONTRIBUTIONS.md @@ -0,0 +1,7 @@ +# External Contributions + +An external pull request that arrives without a Change Request is labelled `needs-change-request`, and review does not begin yet. This is maintainer work, not a requirement placed on the contributor. + +A maintainer opens the Change Request on the contributor's behalf, classifies the change, and completes the Impact Analysis using the project's safety context. Once the Change Request exists and is authorized, the maintainer removes `needs-change-request` and normal checks and review apply. + +Contributors should describe the intended change and provide tests where possible. They are not expected to author the project's safety Impact Analysis. diff --git a/docs/process/MODIFICATION_PROCEDURE.md b/docs/process/MODIFICATION_PROCEDURE.md new file mode 100644 index 00000000..2f901a50 --- /dev/null +++ b/docs/process/MODIFICATION_PROCEDURE.md @@ -0,0 +1,337 @@ +# MOD — Modification Procedure + +> **Warning:** This procedure is **written but not yet exercised**. No change has been processed through it as of the date below. Treat every claim here as a statement of intent until the first completed change record exists. + +# 1. Purpose and scope + +This procedure defines how a change to the Protective Stop (PSTOP) is requested, assessed, authorized, implemented, re-verified and released, so that functional safety is preserved across the life of the product. + +**Integrity target: SIL 3 (IEC 61508) with an equivalent PL e (ISO 13849) track.** The obligations in section 7 are set to the SIL 3 level. + +It applies to every change, after first release, to: + +- The `pstop_c` protocol and machine-safety library +- Remote firmware, host machine wrapper, components, tools and scripts +- Hardware: schematics, PCB layout, BOM, mechanical parts, enclosure CAD +- The safety requirements, HARA, architecture, design, FMEDA, test plans, traceability matrix and safety manual +- Build toolchain, compiler flags, MISRA configuration and CI definition +- Third-party or open source components integrated into PSTOP + +It does not apply to changes made before the baseline of the first released version, which are governed by the development process, nor to documentation with no bearing on the safety argument. + +Changes to protocol or safety behaviour belong upstream in `pstop_c` and are never made from the shell repository. A `pstop_c` bump that changes the CRC is a wire break and is always Class C. + +# 2. Definitions + +| Term | Meaning | +|---|---| +| Change Request (CR) | A GitHub Issue carrying the `change-request` label. The single entry point for all changes in scope. | +| Impact Analysis (IA) | The recorded assessment of what a proposed change affects, and what must be re-done as a result. | +| Change Record | The completed CR issue plus its linked IA, pull requests, test evidence and approvals. Covers one change. | +| Release Record | A GitHub Release plus its linked Gate 2 evidence and approval. Covers every change since the previous release. | +| BAT | Build Acceptance Test. A fast smoke suite, Gate 0. Fails the build early so that no expensive testing is wasted on a broken candidate. It is a filter, not a qualification. | +| Safety module | Any module that implements or can influence a safety requirement. Includes all of `pstop_c`, the lockstep comparator, the arming policy, and the heartbeat and timeout paths. | +| Baseline | A released, tagged configuration of hardware and software, recorded as a GitHub Release and in Approved Versions For Deployment. | + +# 3. Roles + +| Role | Held by | Responsibility | +|---|---|---| +| Requester | Anyone, including external contributors | Raises the CR with reason, description and affected hazards. | +| Impact Analyst | Assigned per CR, not the sole implementer | Completes the Impact Analysis and proposes the change class and re-verification scope. | +| Authorizer | `@iliabaranov`, `@rajasimman-madhivanan`, or `@davidt315` | Approves or rejects on the basis of the IA. Two distinct Authorizers are required for Class C. Records the decision on the CR. | +| Implementer | Assigned per CR | Makes the change by pull request. | +| Reviewer | A competent person other than the Implementer | Reviews against the coding standard and the IA before merge. | +| Release Approver | `@iliabaranov`, `@rajasimman-madhivanan`, or `@davidt315` | Confirms Gate 2 evidence is complete and passing before a baseline is tagged and published. | + +The Authorizer for a given CR is not also its Implementer. Where team size makes this impossible, the deviation is recorded on the CR with a justification. This team-size deviation is retained but is currently not applicable because three Authorizers are available. The competency of the staff assigned to a modification is identified on the CR. + +# 4. When this procedure is triggered + +A CR is raised whenever any of the following occurs: + +1. A defect is found in a released baseline, whether by the team, a user, or in the field. +2. A new or amended safety requirement is proposed. +3. A change to hardware, software, toolchain or third-party component is proposed for a released baseline. +4. A corrective action from a review, audit or incident requires a product change. +5. A dependency of PSTOP publishes a version change that the team intends to adopt. +6. A configuration default that affects safety behaviour is changed, including heartbeat interval, missed-heartbeat count, minimum stop hold, or the operator allowlist policy. + +Work that bypasses this procedure is not merged to `main`. + +# 5. Change classification + +The Impact Analyst proposes a class; the Authorizer confirms it. The class determines the Gate 1 scope in section 7. + +| Class | Definition | Approval | +|---|---|---| +| A — No safety impact | Comments, formatting, non-shipped tooling, documentation with no bearing on the safety argument. No change to executable behaviour or hardware. | One Authorizer | +| B — Indirect safety impact | Change to a non-safety module, or to a safety module with no change to its interface, timing, state machine, or the requirements it implements. | One Authorizer | +| C — Direct safety impact | Change to a safety requirement, a safety module interface, the wire protocol or CRC, a timing budget, the bond or arming state machine, the lockstep comparator, diagnostic coverage, or any hardware in the stop path. | Two distinct Authorizers | + +When the class is uncertain, the higher class applies. + +# 6. The procedure + +## Step 1 — Raise the Change Request + +The Requester opens a GitHub Issue using the [Change Request issue form](../../.github/ISSUE_TEMPLATE/change-request.yml) and applies the `change-request` label. The CR states the reason for the change, a detailed description of what is proposed covering both hardware and software, and the identified hazards that may be affected. It is assigned a status of `Proposed`. + +CR status values, tracked by GitHub label: `Proposed`, `Under Analysis`, `Authorized`, `Rejected`, `In Implementation`, `In Verification`, `Merged`, `Released`, `Closed`. + +## Step 2 — Impact Analysis + +The Impact Analyst completes the [Impact Analysis Template](templates/IMPACT_ANALYSIS.md) as a comment on the CR, or as a linked document where the analysis is long. The IA must identify: + +- Which modules are changed, and which modules depend on them +- Which hardware items are changed +- Which safety requirements are affected +- Which HARA entries, architecture, design, FMEDA, test plan, traceability matrix and safety manual sections require update +- The proposed change class +- The earliest lifecycle phase the change must return to (Step 3) +- The specific tests that must be run to validate the change, and the specific tests that must be re-run to confirm nothing else regressed +- Whether any deviation from normal operating conditions is involved +- The effect on human interaction with the machine, and on the operating environment +- Any other modification currently in flight that could interact with this one +- Whether functional safety is preserved *during* the modification, as well as after it +- Whether the change affects a fielded unit and therefore triggers section 10 + +An IA that does not name specific tests is incomplete. The IA is documented on the CR before authorization. + +## Step 3 — Determine the return-to-phase point + +The IA states the earliest lifecycle phase the change re-enters. All subsequent phases are then executed under the normal development process. + +| Nature of change | Return to | +|---|---| +| Hazard newly identified or reassessed | HARA | +| Safety requirement added, removed or altered | Safety Requirements Specification | +| Module interface, wire protocol, timing budget or state machine altered | Architecture / High Level Design | +| Internal logic of a module altered, interface unchanged | Detailed Design | +| Defect fix with no design consequence | Implementation | +| Hardware component, layout, or enclosure altered | System Architecture Design, plus FMEDA and diagnostic coverage review | + +## Step 4 — Authorize + +The Authorizer reviews the IA and records `Authorized` or `Rejected` on the CR with a dated comment stating the basis for the decision. Authorization rests on the assessment of the impact analysis and on the systematic capability claimed for the affected element, not on the description of the change alone. Class C requires two distinct Authorizers to comment. + +Implementation does not begin before this step completes. + +## Step 5 — Implement + +The Implementer makes the change by pull request against the [protective-stop repository](https://github.com/polymathrobotics/protective-stop), with the CR issue number in the branch name and the PR title, and the CR linked from the PR body. Software, hardware and test changes all follow this same path; commit and PR history is the revision record for all three. + +Changes to protocol or machine-safety logic are made upstream in `pstop_c` and consumed here, never edited in place. + +All documentation identified in the IA is updated in the same change, not deferred. This includes any change to system procedures. + +## Step 6 — Gate 0 and Gate 1 + +Executed per section 7. Results are attached to the CR before review. + +## Step 7 — Review and approve + +A Reviewer who is not the Implementer reviews the change against the coding standard and confirms that everything the IA required has been done. The GitHub pull request review is the review record, for hardware as well as software. + +Merge is blocked until review passes and Gate 1 is green. On merge the CR moves to `Merged`. + +## Step 8 — Close the change + +The CR moves to `Closed` when it has been merged and its Gate 1 evidence is attached. A CR is never closed with outstanding IA actions. Release happens separately, in section 8. + +# 7. Verification gates + +Three gates. Each has a different scope and answers a different question. + +## 7.1 Gate 0 — Build Acceptance Test + +A fast smoke suite that runs on every push and on every release candidate, for every change class including Class A. Purpose is to fail early so that no expensive testing is spent on a broken candidate. + +Contents: clean build of firmware, host and library; `pre-commit` clean; MISRA C:2012 pass with no new findings; the full unit test suite; one end-to-end protocol round trip including a bond, an arm, a stop, and a stop-on-silence. + +A red BAT stops all downstream testing. Nothing merges or releases on a red BAT. + +BAT is a filter. It does not qualify a change and it does not qualify a release. + +## 7.2 Gate 1 — Merge qualification + +Scope is tailored to the change, per the class and the IA. This answers "is this change correct and did it break its neighbours." + +| Class | Required at Gate 1, after a green Gate 0 | +|---|---| +| A | Nothing further. | +| B | Unit tests for the changed module and for every module that directly depends on it. Integration tests covering the affected interfaces. | +| C | The above, plus every validation test that exercises an affected safety requirement, plus the traceability re-check in both directions: forward from the affected safety requirements to the re-verification and re-validation performed, and backward from that work to the requirements it covers. | +| Hardware change | The above for any coupled software, plus re-execution of environmental, EMC and fault injection testing where the IA finds the change could affect those results, plus FMEDA and diagnostic coverage review. | + +Any change that alters a timing budget, the wire protocol or CRC, the bond or arming state machine, the lockstep comparator, or diagnostic coverage is Class C regardless of how small the code delta is. + +## 7.3 Gate 2 — Release qualification + +Scope is the whole system, untailored, run against the release candidate. This answers "does the assembled baseline still meet every safety requirement, including where two independently-merged changes interact." + +Gate 2 covers every change since the previous release, not one CR. It runs in full regardless of how small the changes were. + +Contents: every test that verifies a safety requirement, as listed in the traceability matrix. The Gate 2 suite is derived from the requirements, not from any change, which is what makes it an invariant. Today that means the arming-policy suite over the real wire protocol, the chaos ladder, the netem ladder, per-transport soaks across Ethernet, USB-NCM and WiFi, multi-remote and multi-machine validation, the two-site failover scenario, lockstep and fail-safe-silence checks, stop-on-silence timing, and full traceability closure across every CR in the release. + +The suite changes only when the safety requirements change, and it changes through this procedure like anything else: a CR that adds or alters a requirement also adds or alters the Gate 2 entry that verifies it. It never varies because of what a particular change touched. + +Type tests are the exception. EMC, environmental, fault injection and relay endurance are too costly to run every release, so they sit outside Gate 2 and are triggered by the Impact Analysis instead. When the IA says a change could affect them, they run before that release ships. + +Gate 2 is honest about its own coverage. Where a safety requirement has no verifying test, the release record states that rather than passing silently. + +At SIL 3, revalidation of the complete system is a highly recommended technique, and Gate 2 is how this procedure satisfies it. Regression validation at Gate 1 is the tailored alternative used per change; it does not replace Gate 2. + +## 7.4 Scope of module re-verification + +At SIL 3, re-verification covers the changed module and all modules affected by the change, as determined by the dependency analysis in the IA. Narrowing to the changed module alone is not permitted at this integrity target. + +## 7.5 Recording results + +Results at every gate are attached to the CR, or to the [Release Record](templates/RELEASE_RECORD.md) for Gate 2, with the date, the person or CI job that ran them, the software and hardware versions under test, and the pass or fail outcome per test. A summary line without underlying results is not evidence. + +Gate results are analysed as a body, not only read individually. A rising failure rate in one area is a signal about the design, not just about the change that tripped it. + +# 8. Release + +A release aggregates every change merged since the previous baseline. It has its own record. + +1. The Release Approver opens a [Release Record](templates/RELEASE_RECORD.md): a draft GitHub Release listing every CR included. +2. Gate 0 runs against the release candidate. +3. Gate 2 runs in full against the release candidate. Evidence is attached to the Release Record. +4. Traceability is closed across the whole release: every affected safety requirement traces forward to the verification performed and backward from that verification to the requirement. +5. The Release Approver confirms the evidence is complete and passing. +6. The baseline is tagged and published. Approved Versions For Deployment is updated with the new firmware, host and hardware versions. +7. Release notes are written using the Release Notes Template and reference every CR in the release. +8. The safety manual is reissued if any IA in the release required it. +9. Every CR in the release moves to `Released`. + +A `pstop_c` version bump that changes the CRC is a wire break. Remote and machine are released and deployed together, and the release notes say so explicitly. + +# 9. Emergency changes + +An emergency change is one where a defect creates an immediate safety risk to a fielded unit, or blocks safe operation, and the normal timeline cannot be met. + +The emergency path compresses the schedule. It does not remove steps. + +1. The Requester raises the CR with the `emergency` label and notifies two Authorizers directly. +2. A short-form IA is completed covering, at minimum: what changed, what it could affect, and which tests will be run. It is recorded on the CR before implementation. +3. Two distinct Authorizers approve, in writing on the CR, regardless of class. +4. The change is implemented, reviewed by a second person, and Gate 0 is run. Gate 0 is never waived. +5. Gate 2 may be reduced to the subset the IA justifies, and the reduction is recorded on the Release Record with the reasoning and the Release Approver's name. +6. Release proceeds with the release notes marked `emergency`. +7. Within five working days of release, the full Impact Analysis is completed retrospectively, the omitted Gate 2 coverage is executed, and both records are updated. The CR remains open until this is done. + +Emergency changes are reviewed as a group at the periodic safety review. A rising count is treated as a signal that the normal path is too slow. + +# 10. Notifying users of a safety-affecting defect + +Where a defect in a released baseline affects safety, the following applies in addition to the change process: + +1. The defect is recorded on the CR with the `safety-defect` label at the point it is identified, before a fix exists. +2. The affected baselines and, where known, the affected deployments are identified. +3. A notice is published stating the defect, the affected versions, the interim mitigation, and the expected remedy. This is published as a GitHub security advisory on the repository and repeated in the release notes. +4. The notice is issued on identification of a safety-affecting defect, not deferred until a fix ships. + +There is currently no register recording where PSTOP builds are deployed, so step 2 cannot be completed for external users. See gap G-2. + +# 11. Records and where they live + +| Record | Location | +|---|---| +| Change Request and status | [GitHub Issues](https://github.com/polymathrobotics/protective-stop/issues), `change-request` label | +| Impact Analysis | Comment or linked document on the CR issue | +| Authorization decision | Dated comment on the CR issue | +| Revision history, all of software, hardware and tests | Git commit and [pull request](https://github.com/polymathrobotics/protective-stop/pulls) history | +| Review record, all of software, hardware and tests | GitHub pull request review | +| Gate 0 and automated Gate 1 results | [GitHub Actions](https://github.com/polymathrobotics/protective-stop/actions) run linked from the CR | +| Manual Gate 1 and Gate 2 results | Attached to the CR or the Release Record; reports under [`docs/`](../) | +| Release Record | GitHub Release, tagged | +| Released baseline | Approved Versions For Deployment | +| Release notes | GitHub Release body, per the Release Notes Template | +| Safety-defect notice | GitHub security advisory, per [`SECURITY.md`](../../SECURITY.md) | + +# 12. Equivalent rigor + +Modification activities are planned, performed and documented with at least the same level of expertise, automated tooling, planning and management as the original development. The same coding standard applies, the same MISRA configuration and thresholds apply, the same review requirements apply, the same competency expectations apply to the people involved, and the same toolchain version is used unless a toolchain change is itself the subject of the CR. + +A change is not a licence to work to a lower standard because it is small. + +# 13. Referenced documents + +| Document | Location | Status | +|---|---|---| +| System definition | [`docs/safety/SYSTEM_DEFINITION.md`](../safety/SYSTEM_DEFINITION.md) | Exists | +| HARA | [`docs/safety/HARA.md`](../safety/HARA.md) | Exists | +| **Safety requirements — authoritative baseline** | [`docs/safety/SAFETY_REQUIREMENTS.md`](../safety/SAFETY_REQUIREMENTS.md) | Exists. This is the baseline the Impact Analysis traces against. | +| FMEA | [`docs/safety/FMEA.md`](../safety/FMEA.md) | Exists | +| FMEDA and firm-up playbook | [`docs/safety/FMEDA.md`](../safety/FMEDA.md), [`docs/safety/FMEDA_FIRMUP_GUIDE.md`](../safety/FMEDA_FIRMUP_GUIDE.md) | Exists, unquantified | +| Traceability matrix | [`docs/safety/TRACEABILITY.md`](../safety/TRACEABILITY.md) | Exists. Defines the Gate 2 suite — see 7.3. | +| Structural coverage baselines and tool policy | [`docs/safety/COVERAGE.md`](../safety/COVERAGE.md) | Exists | +| Object-code diversity argument | [`docs/safety/DU3_OBJECT_CODE_DIVERSITY.md`](../safety/DU3_OBJECT_CODE_DIVERSITY.md) | Exists | +| Clock guard verification | [`docs/safety/CLOCK_GUARD_AND_GPIO_REVERIFY.md`](../safety/CLOCK_GUARD_AND_GPIO_REVERIFY.md), [`docs/safety/MACHN_CLOCK_GUARD_HIL.md`](../safety/MACHN_CLOCK_GUARD_HIL.md) | Exists | +| Document reconciliation record | [`docs/safety/RECONCILIATION.md`](../safety/RECONCILIATION.md) | Exists | +| Open items against a full quantified claim | [`docs/safety/OPEN_ITEMS.md`](../safety/OPEN_ITEMS.md) | Exists. Authoritative register for safety-case gaps; section 15 here covers process gaps only. | +| Test harness and validation approach | [`docs/TESTING.md`](../TESTING.md) | Exists | +| Connectivity soak procedure | [`docs/CONNECTIVITY_SOAK.md`](../CONNECTIVITY_SOAK.md) | Exists | +| Safety chain and recovery | [`docs/SAFETY_CHAIN.md`](../SAFETY_CHAIN.md), [`docs/RECOVERY_PLAYBOOK.md`](../RECOVERY_PLAYBOOK.md) | Exists | +| MISRA compliance and deviation register | [`docs/MISRA_COMPLIANCE_2026-07-21.md`](../MISRA_COMPLIANCE_2026-07-21.md) | Exists; excludes `pstop_c`, which is on its own track | +| Failover and arming design | [`docs/FAILOVER_AND_ARMING_DESIGN_2026-07-21.md`](../FAILOVER_AND_ARMING_DESIGN_2026-07-21.md) | Exists | +| Multi-remote validation and operation | [`docs/MULTI_REMOTE_VALIDATION_2026-07-22.md`](../MULTI_REMOTE_VALIDATION_2026-07-22.md), [`docs/MULTI_REMOTE_MULTI_MACHINE.md`](../MULTI_REMOTE_MULTI_MACHINE.md) | Exists | +| Two-site failover report | [`docs/TWO_SITE_FAILOVER_2026-07-21.md`](../TWO_SITE_FAILOVER_2026-07-21.md) | Exists | +| Contribution rules and CI gate | [`CONTRIBUTING.md`](../../CONTRIBUTING.md) | Exists | +| Security and defect reporting | [`SECURITY.md`](../../SECURITY.md) | Exists | +| Hardware design and per-file licence manifest | [`hardware/README.md`](../../hardware/README.md) | Exists, work in progress | +| Coding Standard | Notion: CS_D060 | Exists | +| High Level Software Design Specification | Notion: SWA_D049 | Exists | +| Validation Test Plan | Notion: VTP_D069 | Exists, in progress | +| Safety Requirements Specification (Notion SRS_D040) | Notion | **Superseded.** Retained as working notes only. `docs/safety/SAFETY_REQUIREMENTS.md` is authoritative. | +| Approved Versions For Deployment | Notion: CM-2 | Exists | +| Release Notes Template | Notion: CM-5 | Exists | +| Impact Analysis Template | [`templates/IMPACT_ANALYSIS.md`](templates/IMPACT_ANALYSIS.md) | Exists | +| Change Request Template | [`.github/ISSUE_TEMPLATE/change-request.yml`](../../.github/ISSUE_TEMPLATE/change-request.yml) | Exists | +| Integration Test Plan | — | Not yet written | +| Verification Plan | — | Not yet written | +| Non-Conformance Reporting Procedure | — | Not yet written | +| Corrective Action Procedure | — | Not yet written | +| Safety Manual | — | Not yet written | + +# 14. Clause index + +Clause numbers refer to IEC 61508:2010. The standard's text is not reproduced here; the numbers are given so that a reader holding the standard can check the mapping. + +| Section here | IEC 61508:2010 clauses | +|---|---| +| 1 Scope — modification procedures exist before any change | Part 1 §7.16.2.1; Part 3 §7.8.2.1 | +| 4, 6 Step 1 — Raise the CR | Part 1 §7.16.2.2; Part 3 §6.2.3 d, §7.8.2.2 | +| 6 Step 2 — Impact Analysis | Part 1 §7.16.2.3; Part 2 §7.8.2.1 b; Part 3 §7.1.2.9, §7.8.2.3, Annex A.8.1 | +| 6 Step 2 — Impact Analysis is documented | Part 1 §7.16.2.4; Part 3 §7.8.2.4 | +| 6 Step 3 — Return to earlier phase | Part 1 §7.16.2.6; Part 3 §7.1.2.9, §7.8.2.3 b, §7.8.2.5 | +| 6 Step 4 — Authorize | Part 1 §7.16.2.5; Part 2 §7.8.2.1 c; Part 3 §6.2.3 d, §7.8.2.10 | +| 6 Step 5 — Implement as planned | Part 3 §7.8.2.7 | +| 6 Step 5 — Revision history and configuration management | Part 2 §7.8.2.1 f; Part 3 §6.2.3 c, §7.8.2.8 c, Annex A.8.5 | +| 7.1, 7.2, 7.3 Re-verification and re-validation after modification | Part 2 §7.8.2.4; Part 3 §7.8.2.6 d | +| 7.2 Regression validation, tailored per change | Part 3 Annex A.8.4 b | +| 7.3 Gate 2 — revalidation of the complete system | Part 3 Annex A.8.4 a | +| 7.2, 8 Traceability re-check, both directions | Part 3 Annex A.8.7, A.8.8 | +| 7.4 Scope of module re-verification | Part 3 Annex A.8.2 (changed module), Annex A.8.3 (affected modules) | +| 7.5 Recording and analysing results | Part 1 §7.16.2.7, §7.18.2.1 to §7.18.2.4; Part 2 §7.8.2.1 e; Part 3 §7.8.2.9, Annex A.8.6 | +| 7, 9 Verification planning for the modification | Part 3 §7.8.2.6 c | +| 8 Documentation and procedure updates released with the change | Part 2 §7.8.2.1 h, §7.8.2.1 i; Part 3 §7.8.2.8 e | +| 10 User notification of a safety-affecting defect | Part 2 §7.8.2.2 | +| 11 Records | Part 1 §7.16.2.7; Part 2 §7.8.2.1 a, c, d, e, f, g, h, i; Part 3 §7.8.2.8 a to e | +| 3, 12 Competency, tooling, planning and management equal to original development | Part 2 §7.8.2.3; Part 3 §7.8.2.5, §7.8.2.6 a, §7.8.2.7 | + +# 15. Declared gaps + +These are open. They are listed so that the limits of this procedure are stated rather than implied. + +| ID | Gap | Consequence | Owner | +|---|---|---|---| +| G-1 | Resolved. `docs/safety/SAFETY_REQUIREMENTS.md` is the authoritative safety requirements baseline; Notion SRS_D040 is superseded working notes. | None. Retained for the record. | — | +| G-2 | No deployment register exists recording where PSTOP builds are running. | Section 10 step 2 cannot be executed for external users. Safety-defect notification is best-effort only. | Raj | +| G-3 | The Gate 0 BAT suite is defined in principle but not yet enumerated as a fixed, named set of tests with pass criteria. | Section 7.1 is not executable as written. | Ilia / John | +| G-4 | The Gate 2 suite is derivable from the traceability matrix, but that matrix currently shows a minority of safety requirements strictly verified. | A passing Gate 2 today certifies less than it appears to. The release record must state the unverified requirements explicitly. | Ilia / John | +| G-5 | No Verification Plan, Integration Test Plan, Non-Conformance Reporting procedure or Corrective Action procedure exists. | Steps 4 and 6 reference processes that are not yet documented. | Raj | +| G-6 | The stop-on-silence timing in the requirement baseline and in the shipped configuration do not agree. | Traceability cannot be closed on the affected safety requirement. Deferred by decision; recorded here so it is not lost. | Ilia | +| G-7 | No change has yet been processed through this procedure. | There are no change records to demonstrate the procedure is followed in practice. | Raj | diff --git a/docs/process/NOTION_MIGRATION_MAP.md b/docs/process/NOTION_MIGRATION_MAP.md new file mode 100644 index 00000000..68ba6316 --- /dev/null +++ b/docs/process/NOTION_MIGRATION_MAP.md @@ -0,0 +1,12 @@ +# Notion Migration Map + +This map preserves forwarding addresses for certification evidence citations. The source pages remain in Notion until their owner retires them. + +| Notion source | Repository destination | +|---|---| +| [Modification Procedure](https://app.notion.com/p/3d6c0b1ac5fa8105b456e862e939f051) | [`MODIFICATION_PROCEDURE.md`](MODIFICATION_PROCEDURE.md) | +| [Impact Analysis Template](https://app.notion.com/p/3d6c0b1ac5fa81838a09f22388095c33) | [`templates/IMPACT_ANALYSIS.md`](templates/IMPACT_ANALYSIS.md) | +| [Change Request Template](https://app.notion.com/p/3d6c0b1ac5fa8167927af2e753be67c5) | [`.github/ISSUE_TEMPLATE/change-request.yml`](../../.github/ISSUE_TEMPLATE/change-request.yml) | +| [Release Record Template](https://app.notion.com/p/3d6c0b1ac5fa81708a8cff41150d2b86) | [`templates/RELEASE_RECORD.md`](templates/RELEASE_RECORD.md) | + +Other Notion citations `FSM-1`, `FSM-11`, `CM-2`, `CM-5`, and `VTP-8` are not yet migrated. diff --git a/docs/process/enforcement-mode b/docs/process/enforcement-mode new file mode 100644 index 00000000..1ef71804 --- /dev/null +++ b/docs/process/enforcement-mode @@ -0,0 +1 @@ +warn diff --git a/docs/process/templates/IMPACT_ANALYSIS.md b/docs/process/templates/IMPACT_ANALYSIS.md new file mode 100644 index 00000000..d8829b7b --- /dev/null +++ b/docs/process/templates/IMPACT_ANALYSIS.md @@ -0,0 +1,93 @@ +> Copy this into a comment on the Change Request issue, or into a linked document where the analysis is long. Every field is answered. "None" is a valid answer; blank is not. + +# Identification + +| Field | Value | +|---|---| +| Change Request | GitHub issue number and link | +| Analyst | Name | +| Date | YYYY-MM-DD | +| Baseline affected | Software version and hardware revision this change applies to | + +# 1. What is changing + +One paragraph in plain language. What is being changed and why. + +# 2. Affected software + +- Modules changed: +- Modules that directly depend on the changed modules: +- Is any changed module a safety module (implements or influences Safety Requirements Specification section 5)? Yes / No, and which. +- Does the change alter any of the following? Answer each Yes / No. + - Public interface or API signature + - Message protocol or wire format + - Timing budget, including the bond-loss stop latency + - Bond / stop / OK / unbond state machine + - Diagnostic or fault-detection behaviour + - Memory allocation behaviour + +# 3. Affected hardware + +- Items changed (schematic, PCB, BOM line, mechanical part, enclosure CAD): +- Is any changed item in the stop signal path? Yes / No. +- Could the change plausibly affect environmental performance, EMC behaviour, or fault-injection results? Yes / No, with reasoning. +- Does the change affect the FMEDA, the diagnostic coverage argument, or the common-cause argument? Yes / No, with reasoning. + +# 3b. Wider effects + +- Effect on human interaction with the machine: +- Effect on the operating environment, or assumptions about it: +- Other modifications currently in flight that could interact with this one: +- Is functional safety preserved *during* the modification, as well as after it? State how. + +# 4. Affected requirements and documents + +| Artifact | Affected? | What must change | +|---|---|---| +| Safety requirements baseline (state which — see gap G-1) | | | +| HARA | | | +| FMEDA / diagnostic coverage / common cause | | | +| System Architecture Design | | | +| High Level Software Design | | | +| Detailed Software Design | | | +| Coding Standard | | | +| Validation Test Plan | | | +| Integration Test Plan | | | +| Traceability matrix | | | +| Safety Manual | | | +| Release notes | | | + +# 5. Proposed change class + +A, B or C, with one sentence of justification. Where the class is uncertain, the higher class is proposed. + +# 6. Return-to-phase point + +The earliest lifecycle phase this change re-enters, and why. All later phases are then executed under the normal development process. + +# 7. Verification plan for this change + +Name the tests. A plan that says "run the relevant tests" is not complete. This defines Gate 1 scope only; Gate 2 release qualification runs in full regardless of what is entered here. + +| Purpose | Specific tests | +|---|---| +| Tests that validate the change itself | | +| Tests re-run to confirm nothing else regressed | | +| New tests that must be written | | +| Hardware tests to re-run, if any | | + +A complete Gate 0 Build Acceptance Test runs regardless of what is entered above, and a red Gate 0 stops all of it. + +# 8. Deviations + +Any deviation from normal operating conditions, normal process, or normal roles involved in this change, and the justification for it. Includes the case where the Authorizer and Implementer are the same person. + +# 9. Fielded units + +- Does this change relate to a defect present in a released baseline? Yes / No. +- Does that defect affect safety? Yes / No. +- If yes to both, section 10 of the Modification Procedure applies. Record the affected baselines here. + +# 10. Analyst conclusion + +One paragraph. What this change touches, what could go wrong if the analysis is incomplete, and the recommendation to the Authorizer. diff --git a/docs/process/templates/RELEASE_RECORD.md b/docs/process/templates/RELEASE_RECORD.md new file mode 100644 index 00000000..a9ea82f0 --- /dev/null +++ b/docs/process/templates/RELEASE_RECORD.md @@ -0,0 +1,82 @@ +> One Release Record per release, covering every change merged since the previous baseline. This is the body of the GitHub Release. Gate 2 evidence attaches here, not to any individual Change Request. + +# Identification + +| Field | Value | +|---|---| +| Tag | | +| Previous baseline | | +| Firmware version | | +| Host / machine wrapper version | | +| `pstop_c` version | | +| Hardware revision | | +| Release Approver | | +| Date | | + +# 1. Changes included + +Every Change Request merged since the previous baseline. A release with an unlisted merged change is incomplete. + +| CR | Class | Summary | Gate 1 passed | +|---|---|---|---| +| | | | | + +# 2. Interaction assessment + +Where two or more changes in this release touch related areas, state what could interact and what was done about it. This is the question Gate 1 cannot answer, because each change was tested alone. + +# 3. Wire compatibility + +- Did `pstop_c` change in a way that alters the CRC or wire format? Yes / No. +- If yes, this is a wire break. Remote and machine must be updated together. State that plainly in the release notes. + +# 4. Gate 0 — Build Acceptance Test on the release candidate + +| Version under test | CI run | Date | Result | +|---|---|---|---| +| | | | | + +# 5. Gate 2 — Release qualification + +Run in full against the release candidate, regardless of how small the changes were. Any omission is a deviation and is recorded in section 8. + +| Suite | Version under test | Run by | Date | Result | +|---|---|---|---|---| +| Arming policy suite over the real wire protocol | | | | | +| Chaos ladder — loss, delay, duplication, corruption | | | | | +| Netem ladder — underlay impairment | | | | | +| Soak — Ethernet | | | | | +| Soak — USB-NCM | | | | | +| Soak — WiFi | | | | | +| Multi-remote and multi-machine validation | | | | | +| Two-site failover | | | | | +| Lockstep mismatch and fail-safe silence | | | | | +| Stop-on-silence timing | | | | | + +This list is provisional until the Validation Test Plan fixes it as a named set with pass criteria. See gap G-4. + +# 6. Traceability closure + +Across every change in this release, not per change. + +- [ ] Forward: every affected safety requirement traces to the verification performed +- [ ] Backward: every piece of verification performed traces to the requirement it covers +- [ ] Traceability matrix updated and committed + +# 7. Documentation + +- [ ] Release notes written, referencing every CR +- [ ] Approved Versions For Deployment updated +- [ ] Safety manual reissued, or recorded as not required +- [ ] Any system procedure changes published +- [ ] Open items register updated + +# 8. Deviations + +Anything omitted from Gate 2, the reasoning, and the Release Approver who accepted it. An emergency release records its reduced scope and the date by which the omitted coverage will be run. + +# 9. Approval + +| Release Approver | Date | Statement | +|---|---|---| +| | | Gate 2 evidence is complete and passing; this baseline is approved for release. | diff --git a/scripts/check_wire_format.sh b/scripts/check_wire_format.sh new file mode 100755 index 00000000..df9b94bc --- /dev/null +++ b/scripts/check_wire_format.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +# +# Process guard for unannounced pstop_c wire breaks. No SR or DU is claimed: +# this checks coordinated release process, not runtime safety behavior. +set -uo pipefail + +ROOT=$(cd "$(dirname "$0")/.." && pwd) +cd "$ROOT" || exit 2 + +args=(check --root "$ROOT" --labels "${PSTOP_PR_LABELS:-}") +if [ -n "${PSTOP_BASE_SHA:-}" ]; then + if ! git cat-file -e "$PSTOP_BASE_SHA:tools/change_control/wire_format.sha256" 2>/dev/null; then + args+=(--initial-expectation) + fi + if ! changed=$(git diff --name-only "$PSTOP_BASE_SHA"...HEAD 2>/dev/null); then + echo "wire-format: cannot run: unable to compare PSTOP_BASE_SHA" >&2 + exit 2 + fi + while IFS= read -r path; do + [ -z "$path" ] || args+=(--changed-file "$path") + done <<< "$changed" +fi +python3 -m tools.change_control.wire_format "${args[@]}" diff --git a/scripts/sync_labels.sh b/scripts/sync_labels.sh new file mode 100755 index 00000000..fb7a69d2 --- /dev/null +++ b/scripts/sync_labels.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +# +# Process configuration helper; it guards no SR or DU and changes labels only +# when explicitly run by an authorized repository administrator. +set -uo pipefail + +ROOT=$(cd "$(dirname "$0")/.." && pwd) +cd "$ROOT" || exit 2 +REPOSITORY="${PSTOP_REPOSITORY:-polymathrobotics/protective-stop}" + +if [ "$#" -gt 1 ] || { [ "$#" -eq 1 ] && [ "$1" != "--dry-run" ]; }; then + echo "usage: scripts/sync_labels.sh [--dry-run]" >&2 + exit 2 +fi +dry_run=false +[ "${1:-}" = "--dry-run" ] && dry_run=true +if [ "$dry_run" = false ]; then + command -v gh >/dev/null 2>&1 || { echo "sync-labels: gh not found" >&2; exit 2; } +fi + +python3 - "$ROOT/tools/change_control/labels.json" <<'PY' | while IFS=$'\t' read -r name color description; do +import json +import sys + +for label in json.load(open(sys.argv[1], encoding='utf-8')): + print(label['name'], label['color'], label['description'], sep='\t') +PY + if [ "$dry_run" = true ]; then + printf 'would sync label: %s (%s)\n' "$name" "$color" + else + gh label create "$name" --repo "$REPOSITORY" --color "$color" --description "$description" --force || exit 1 + fi +done diff --git a/tools/change_control/__init__.py b/tools/change_control/__init__.py new file mode 100644 index 00000000..7a7d8e90 --- /dev/null +++ b/tools/change_control/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Repository change-control checks.""" diff --git a/tools/change_control/__main__.py b/tools/change_control/__main__.py new file mode 100644 index 00000000..5302a3a4 --- /dev/null +++ b/tools/change_control/__main__.py @@ -0,0 +1,139 @@ +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Run change-control checks against pull-request state obtained through gh api.""" + +import argparse +import json +import shlex +import subprocess +import sys +from pathlib import Path + +from tools.safety_lint.model import LintError + +from .checks import evaluate, load_mode, render_report, upsert_comment + + +class GhApi: + """Small fail-closed subprocess adapter around gh api.""" + + def __init__(self, command, repository): + self.command = shlex.split(command) + self.repository = repository + + def __call__(self, method, path, body=None, paginate=False, collection_key=None): + endpoint = path.replace('{repo}', self.repository) + command = [*self.command, 'api', endpoint] + if paginate: + command.extend(['--paginate', '--slurp']) + if method != 'GET': + command.extend(['--method', method]) + if body: + for key, value in body.items(): + command.extend(['--field', f'{key}={value}']) + result = subprocess.run(command, check=False, capture_output=True, text=True) + if result.returncode: + raise RuntimeError(result.stderr.strip() or f'gh api failed for {endpoint}') + try: + response = json.loads(result.stdout or '{}') + except json.JSONDecodeError as error: + raise RuntimeError(f'gh api returned invalid JSON for {endpoint}') from error + if not paginate: + return response + if not isinstance(response, list): + raise RuntimeError(f'paginated gh api response is not a page list for {endpoint}') + if collection_key: + merged = [] + for page in response: + if not isinstance(page, dict) or not isinstance(page.get(collection_key), list): + raise RuntimeError(f'paginated gh api response lacks {collection_key} for {endpoint}') + merged.extend(page[collection_key]) + return {collection_key: merged} + if not all(isinstance(page, list) for page in response): + raise RuntimeError(f'paginated gh api response contains a non-list page for {endpoint}') + return [item for page in response for item in page] + + +def _require(mapping, path): + value = mapping + for key in path: + if not isinstance(value, dict) or key not in value: + raise RuntimeError(f'partial GitHub response missing {".".join(path)}') + value = value[key] + return value + + +def _cr_number(body): + import re + + values = set( + re.findall( + r'(?im)^\s*(?:closes|refs)\s+(?:(?:https://github\.com/[^/]+/[^/]+/issues/)?#?)(\d+)\s*$', + body or '', + ) + ) + return int(next(iter(values))) if len(values) == 1 else None + + +def collect(api, repository, pr_number): + """Collect the complete GitHub snapshot used by pure policy evaluation.""" + prefix = f'repos/{repository}' + pr = api('GET', f'{prefix}/pulls/{pr_number}') + _require(pr, ('user', 'login')) + head = _require(pr, ('head', 'sha')) + if 'body' not in pr or 'labels' not in pr: + raise RuntimeError('partial GitHub response missing PR body or labels') + cr = _cr_number(pr['body']) + issue = api('GET', f'{prefix}/issues/{cr}') if cr else {'labels': [], 'body': ''} + comments = api('GET', f'{prefix}/issues/{cr}/comments', paginate=True) if cr else [] + reviews = api('GET', f'{prefix}/pulls/{pr_number}/reviews', paginate=True) + files = api('GET', f'{prefix}/pulls/{pr_number}/files', paginate=True) + checks = api('GET', f'{prefix}/commits/{head}/check-runs', paginate=True, collection_key='check_runs') + workflows = api( + 'GET', + f'{prefix}/actions/runs?head_sha={head}', + paginate=True, + collection_key='workflow_runs', + ) + pr_comments = api('GET', f'{prefix}/issues/{pr_number}/comments', paginate=True) + for name, value in (('files', files), ('comments', comments), ('reviews', reviews), ('PR comments', pr_comments)): + if not isinstance(value, list): + raise RuntimeError(f'partial GitHub response: {name} is not a list') + return { + 'pr': pr, + 'issue': issue, + 'issue_comments': comments, + 'reviews': reviews, + 'files': files, + 'check_runs': _require(checks, ('check_runs',)), + 'workflow_runs': _require(workflows, ('workflow_runs',)), + 'pr_comments': pr_comments, + } + + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument('--root', default='.') + parser.add_argument('--repository', required=True) + parser.add_argument('--pr', required=True, type=int) + parser.add_argument('--gh', default='gh') + parser.add_argument('--no-comment', action='store_true') + args = parser.parse_args(argv) + try: + mode = load_mode(args.root) + api = GhApi(args.gh, args.repository) + data = collect(api, args.repository, args.pr) + results = evaluate(Path(args.root), data) + report = render_report(mode, results) + print(report) + if not args.no_comment: + upsert_comment(api, args.pr, report, data['pr_comments']) + findings = any(item.status == 'fail' for item in results) + return 1 if findings and mode == 'enforce' else 0 + except (OSError, RuntimeError, ValueError, KeyError, LintError) as error: + print(f'change-control: cannot run: {error}', file=sys.stderr) + return 2 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/change_control/checks.py b/tools/change_control/checks.py new file mode 100644 index 00000000..6f360d7b --- /dev/null +++ b/tools/change_control/checks.py @@ -0,0 +1,365 @@ +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Evaluate existence and ordering of modification records without judging adequacy.""" + +import re +from dataclasses import dataclass +from pathlib import Path + +from tools.safety_lint.parse_srs import parse_srs + +from .issue_form import parse_issue_form + +AUTHORIZERS = ('iliabaranov', 'rajasimman-madhivanan', 'davidt315') +CLASS_RANK = {'A': 1, 'B': 2, 'C': 3} +COMMENT_MARKER = '' + + +@dataclass(frozen=True) +class CheckResult: + """One caller-visible policy result.""" + + check_id: str + status: str + message: str + + +def load_mode(root): + """Read the auditable mode switch and reject every value except warn or enforce.""" + path = Path(root) / 'docs/process/enforcement-mode' + try: + value = path.read_text(encoding='utf-8') + except OSError as error: + raise RuntimeError(f'cannot read enforcement mode: {error}') from error + if value not in ('warn\n', 'enforce\n'): + raise RuntimeError('enforcement-mode must contain exactly warn or enforce followed by a newline') + return value.strip() + + +def _headings(path): + return [ + line.strip() for line in Path(path).read_text(encoding='utf-8').splitlines() if re.match(r'^#(?:\s|\d)', line) + ] + + +def impact_analysis_complete(text, headings): + """Check that headings occur in order and each has non-whitespace content beneath it.""" + positions = [] + cursor = 0 + for heading in headings: + match = re.search(rf'(?m)^{re.escape(heading)}\s*$', text[cursor:]) + if not match: + return False, [heading] + start = cursor + match.start() + end = cursor + match.end() + positions.append((heading, start, end)) + cursor = end + missing = [] + for index, (heading, _, end) in enumerate(positions): + next_start = positions[index + 1][1] if index + 1 < len(positions) else len(text) + if not text[end:next_start].strip(): + missing.append(heading) + return not missing, missing + + +def minimum_class(paths, wire_changed=False): + """Return the mechanical classification floor for changed repository paths.""" + floor = 'C' if wire_changed else 'A' + for path in paths: + if ( + path.startswith('pstop_c/') + or path in ('firmware/main/main.c', 'machn/main/main.c') + or path.startswith('docs/safety/') + ): + candidate = 'C' + elif path.startswith(('components/', 'common/')) or path in ( + 'firmware/sdkconfig.defaults', + 'machn/sdkconfig.defaults', + ): + candidate = 'B' + elif path.startswith('.github/workflows/') or path.startswith('scripts/'): + candidate = 'B' + else: + candidate = 'A' + if CLASS_RANK[candidate] > CLASS_RANK[floor]: + floor = candidate + return floor + + +def _labels(entity): + return {label['name'] if isinstance(label, dict) else label for label in entity.get('labels', [])} + + +def _cr_number(body): + matches = re.findall( + r'(?im)^\s*(?:closes|refs)\s+(?:(?:https://github\.com/[^/]+/[^/]+/issues/)?#?)(\d+)\s*$', + body or '', + ) + return int(matches[0]) if len(set(matches)) == 1 else None + + +def _issue_fields_complete(root, body): + form = parse_issue_form(Path(root) / '.github/ISSUE_TEMPLATE/change-request.yml') + missing = [] + for field in form['body']: + if not field['required']: + continue + label = field['label'] + match = re.search(rf'(?ms)^###\s+{re.escape(label)}\s*$\n(.*?)(?=^###\s|\Z)', body or '') + if not match or not match.group(1).strip() or match.group(1).strip() == '_No response_': + missing.append(field['id']) + return missing + + +def _find_ia(comments, headings): + for comment in comments: + body = comment.get('body', '') + if headings and headings[0] in body: + return body + return '' + + +def _find_short_ia(comments): + for comment in comments: + body = comment.get('body', '') + if all(re.search(phrase, body, re.IGNORECASE) for phrase in ('what changed', 'what it could affect', 'tests')): + return body + return '' + + +def _cited_sr_tokens(text): + return set(re.findall(r'\bSR-[A-Z]+-[0-9A-Za-z]+(?:-[0-9A-Za-z]+)*\b', text)) + + +def _is_authorization(comment, author=''): + login = comment.get('user', {}).get('login', '').lower() + return ( + login in AUTHORIZERS + and login != author.lower() + and bool(re.search(r'(?im)^\s*(?:decision:\s*)?authori[sz]ed\b', comment.get('body', ''))) + ) + + +def _named_tests(text): + sections = re.findall(r'(?ms)^# 7\. Verification plan for this change\s*$\n(.*?)(?=^# 8\.|\Z)', text) + names = [] + for section in sections: + names.extend(re.findall(r'`([^`]+)`', section)) + for line in section.splitlines(): + if line.lstrip().startswith(('-', '+')) and ':' in line: + value = line.split(':', 1)[1].strip() + if value and value.lower() not in ('none', 'n/a'): + names.append(value) + if line.strip().startswith('|'): + cells = [cell.strip() for cell in line.strip().strip('|').split('|')] + if len(cells) >= 2 and cells[0] not in ('Purpose', '---'): + value = cells[1] + if value and value not in ('---', 'None', 'N/A'): + names.extend(part.strip() for part in re.split(r'|,', value) if part.strip()) + if not sections: + match = re.search(r'(?im)^\s*(?:which\s+)?tests(?:\s+will\s+be\s+run)?\s*:\s*(.+)$', text) + if match: + names.extend(part.strip(' `') for part in match.group(1).split(',') if part.strip(' `')) + return list(dict.fromkeys(name.strip() for name in names if name.strip())) + + +def _evidence_matches(name, evidence_name): + """Match an IA entry to one complete check or workflow name, never a substring.""" + planned = ' '.join(name.casefold().split()) + observed = ' '.join(evidence_name.casefold().split()) + return bool(planned and observed and planned == observed) + + +def _approvers(data, require_head=True): + head = data['pr']['head']['sha'] + author = data['pr']['user']['login'].lower() + latest = {} + for review in data.get('reviews', []): + login = review.get('user', {}).get('login', '').lower() + if login: + latest[login] = review + return { + login + for login, review in latest.items() + if login in AUTHORIZERS + and login != author + and review.get('state') == 'APPROVED' + and (not require_head or review.get('commit_id') == head) + } + + +def evaluate(root, data): + """Evaluate E1-E7 against a complete, synthetic-or-live GitHub state snapshot.""" + root = Path(root) + pr = data['pr'] + labels = _labels(pr) + cr_number = _cr_number(pr.get('body', '')) + issue = data.get('issue', {}) + issue_labels = _labels(issue) + comments = data.get('issue_comments', []) + headings = _headings(root / 'docs/process/templates/IMPACT_ANALYSIS.md') + ia = _find_ia(comments, headings) + short_ia = _find_short_ia(comments) + emergency = 'emergency' in labels + if emergency and not ia: + ia = short_ia + results = [] + + if 'needs-change-request' in labels: + results.append(CheckResult('E1', 'pending', 'maintainer Change Request required before review begins')) + else: + author = pr['user']['login'] + required_authorizers = 2 if 'class-c' in labels or emergency else 1 + authorization_comments = {} + for comment in comments: + if _is_authorization(comment, author): + authorization_comments[comment['user']['login'].lower()] = comment + authorized_comment = len(authorization_comments) >= required_authorizers + authorization_times = [ + comment.get('created_at') for comment in authorization_comments.values() if comment.get('created_at') + ] + ia_times = [ + comment.get('created_at') + for comment in comments + if comment.get('body', '') == ia and comment.get('created_at') + ] + before_implementation = not pr.get('created_at') or ( + len(authorization_times) >= required_authorizers and max(authorization_times) <= pr['created_at'] + ) + after_analysis = not ia_times or ( + len(authorization_times) >= required_authorizers and max(ia_times) <= min(authorization_times) + ) + ordered = before_implementation and after_analysis + missing_fields = _issue_fields_complete(root, issue.get('body', '')) if cr_number else ['change-request-link'] + okay = ( + cr_number is not None + and 'change-request' in issue_labels + and 'status:authorized' in issue_labels + and authorized_comment + and ordered + and not missing_fields + ) + detail = ( + f'authorized Change Request has {required_authorizers} distinct pre-implementation authorizer(s)' + if okay + else f'Change Request missing, ambiguous, incomplete, or lacks {required_authorizers} distinct pre-implementation authorizer(s) ({", ".join(missing_fields)})' + ) + results.append(CheckResult('E1', 'pass' if okay else 'fail', detail)) + + if emergency and ia == short_ia and short_ia: + complete, empty = True, [] + else: + complete, empty = impact_analysis_complete(ia, headings) if ia else (False, ['Impact Analysis']) + results.append( + CheckResult( + 'E2', + 'pass' if complete else 'fail', + 'all IA sections exist and are nonblank; content truth and adequacy are not assessed' + if complete + else f'IA sections missing or blank: {", ".join(empty)}; content truth and adequacy are not assessed', + ) + ) + + canonical = {requirement.sr_id for requirement in parse_srs(root / 'docs/safety/SAFETY_REQUIREMENTS.md')} + cited = _cited_sr_tokens(ia) + invalid = sorted(cited - canonical) + results.append( + CheckResult( + 'E3', + 'fail' if invalid else 'pass', + f'invalid requirement IDs: {", ".join(invalid)}' if invalid else 'all cited requirement IDs exist', + ) + ) + + class_labels = sorted(label for label in labels if re.fullmatch(r'class-[abc]', label)) + paths = [item['filename'] for item in data.get('files', [])] + wire_changed = any(path.startswith('pstop_c/pstop/include/pstop/') for path in paths) + floor = minimum_class(paths, wire_changed) + if len(class_labels) != 1: + results.append(CheckResult('E4', 'fail', 'exactly one class-a, class-b, or class-c label is required')) + selected = None + else: + selected = class_labels[0][-1].upper() + under = CLASS_RANK[selected] < CLASS_RANK[floor] + results.append( + CheckResult( + 'E4', + 'fail' if under else 'pass', + f'Class {selected}; mechanical floor Class {floor}; checks existence/order, not classification adequacy', + ) + ) + + approvers = _approvers(data) + if selected == 'C': + status = 'pass' if len(approvers) >= 2 else 'fail' + results.append( + CheckResult('E5', status, f'current distinct non-author approving authorizers: {len(approvers)}/2') + ) + else: + results.append(CheckResult('E5', 'not-applicable', 'two-review requirement applies to Class C')) + + names = _named_tests(ia) + head = pr['head']['sha'] + evidence = { + item.get('name', '') + for item in data.get('check_runs', []) + if item.get('head_sha', head) == head and item.get('conclusion') == 'success' + } + evidence.update( + item.get('name', item.get('path', '')) + for item in data.get('workflow_runs', []) + if item.get('head_sha') == head and item.get('conclusion') == 'success' + ) + missing_tests = [name for name in names if not any(_evidence_matches(name, item) for item in evidence)] + explanation = 'check-run/workflow evidence cannot prove commands or tests inside a job executed' + if not names: + e6_status = 'fail' + e6_message = f'IA verification plan names no specific tests; {explanation}' + elif missing_tests: + e6_status = 'fail' + e6_message = f'missing head-SHA evidence: {", ".join(missing_tests)}; {explanation}' + else: + e6_status = 'pass' + e6_message = f'all named evidence matched; {explanation}' + results.append( + CheckResult( + 'E6', + e6_status, + e6_message, + ) + ) + + if emergency: + short_form = bool(ia and names and short_ia) + status = 'pass' if len(approvers) >= 2 and short_form else 'fail' + results.append( + CheckResult( + 'E7', + status, + 'emergency path requires two approvals and short-form IA; retrospective due within five working days of release', + ) + ) + else: + results.append(CheckResult('E7', 'not-applicable', 'PR is not labelled emergency')) + return results + + +def render_report(mode, results): + """Render one deterministic PR comment with the active mode visible.""" + lines = [f'mode: {mode}', '', '| Check | Result | Explanation |', '|---|---|---|'] + lines.extend(f'| {item.check_id} | {item.status} | {item.message.replace("|", "\\|")} |' for item in results) + lines.extend([ + '', + 'These checks verify artifact existence and ordering only, not truth, adequacy, or safety sufficiency.', + ]) + return '\n'.join(lines) + + +def upsert_comment(api, pr_number, report, comments): + """Create or update at most one marker-owned report comment.""" + body = f'{COMMENT_MARKER}\n{report}' + existing = next((comment for comment in comments if COMMENT_MARKER in comment.get('body', '')), None) + if existing: + api('PATCH', f'repos/{{repo}}/issues/comments/{existing["id"]}', {'body': body}) + else: + api('POST', f'repos/{{repo}}/issues/{pr_number}/comments', {'body': body}) diff --git a/tools/change_control/coverage_delta.py b/tools/change_control/coverage_delta.py new file mode 100644 index 00000000..d0b8f388 --- /dev/null +++ b/tools/change_control/coverage_delta.py @@ -0,0 +1,148 @@ +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Compare deterministic safety-linter output between two Git revisions.""" + +import argparse +import json +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +from .__main__ import GhApi + +MARKER = '' + + +def compare_reports(base, head): + """Render coverage and citation changes without assigning safety significance.""" + if head.get('unavailable'): + raise RuntimeError(f'head coverage unavailable: {head["unavailable"]}') + if base.get('unavailable'): + return ( + 'Coverage before: unavailable. The base predates the stacked dependency on ' + f'change-0001 (`tools/safety_lint`): {base["unavailable"]}\n' + f'Coverage after: {head.get("coverage", {}).get("cited_tests", "?")}/{head.get("coverage", {}).get("total", "?")} cited.' + ) + before = base.get('coverage', {}) + after = head.get('coverage', {}) + lines = [ + f'Coverage before: {before.get("cited_tests", "?")}/{before.get("total", "?")} cited.', + f'Coverage after: {after.get("cited_tests", "?")}/{after.get("total", "?")} cited.', + ] + base_citations = base.get('citations', {}) + head_citations = head.get('citations', {}) + for sr_id in sorted(set(base_citations) | set(head_citations)): + old = set(base_citations.get(sr_id, [])) + new = set(head_citations.get(sr_id, [])) + if old - new: + lines.append(f'- {sr_id} lost citation(s): {", ".join(sorted(old - new))}') + if new - old: + lines.append(f'- {sr_id} gained citation(s): {", ".join(sorted(new - old))}') + old_unresolved = { + (item.get('check_id'), item.get('subject'), item.get('message')) + for item in base.get('findings', []) + if item.get('check_id') in ('C3', 'C4') + } + new_unresolved = { + (item.get('check_id'), item.get('subject'), item.get('message')) + for item in head.get('findings', []) + if item.get('check_id') in ('C3', 'C4') + } + for _, subject, message in sorted(new_unresolved - old_unresolved): + lines.append(f'- Newly unresolvable citation for {subject}: {message}') + if len(lines) == 2: + lines.append('- No citation gains, losses, or newly unresolvable citations.') + lines.append( + 'Limitation: this is deterministic citation resolution, not evidence that a cited test executed or passed.' + ) + return '\n'.join(lines) + + +def run_linter_at_tree(worktree): + """Run the revision's own unchanged linter and add its parsed citation map.""" + if not (worktree / 'tools/safety_lint/__main__.py').is_file(): + return {'unavailable': 'tools/safety_lint is absent at this revision'} + result = subprocess.run( + [sys.executable, '-m', 'tools.safety_lint', '--json'], + cwd=worktree, + check=False, + capture_output=True, + text=True, + ) + if result.returncode == 2: + raise RuntimeError(result.stderr.strip() or 'safety linter could not run') + try: + report = json.loads(result.stdout) + except json.JSONDecodeError as error: + raise RuntimeError('safety linter emitted invalid JSON') from error + citation_code = ( + 'import json; from tools.safety_lint.runner import analyze; ' + 'print(json.dumps({r.sr_id: sorted(set(r.test_refs)) for r in analyze(".").trace}, sort_keys=True))' + ) + citations = subprocess.run( + [sys.executable, '-c', citation_code], cwd=worktree, check=False, capture_output=True, text=True + ) + if citations.returncode: + raise RuntimeError(citations.stderr.strip() or 'cannot extract linter citations') + report['citations'] = json.loads(citations.stdout) + return report + + +def report_at_revision(root, revision): + """Run the unchanged checked-in linter at one detached revision.""" + temporary = Path(tempfile.mkdtemp(prefix='pstop-coverage-delta-')) + try: + result = subprocess.run( + ['git', 'worktree', 'add', '--detach', str(temporary), revision], + cwd=root, + check=False, + capture_output=True, + text=True, + ) + if result.returncode: + raise RuntimeError(result.stderr.strip() or f'cannot materialize revision {revision}') + return run_linter_at_tree(temporary) + finally: + subprocess.run( + ['git', 'worktree', 'remove', '--force', str(temporary)], cwd=root, check=False, capture_output=True + ) + shutil.rmtree(temporary, ignore_errors=True) + + +def upsert_coverage_comment(api, repository, pr, report): + """Create or update the single marker-owned deterministic coverage comment.""" + comments = api('GET', f'repos/{repository}/issues/{pr}/comments') + existing = next((comment for comment in comments if MARKER in comment.get('body', '')), None) + body = f'{MARKER}\n{report}' + if existing: + api('PATCH', f'repos/{repository}/issues/comments/{existing["id"]}', {'body': body}) + else: + api('POST', f'repos/{repository}/issues/{pr}/comments', {'body': body}) + + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument('--root', default='.') + parser.add_argument('--base', required=True) + parser.add_argument('--head', default='HEAD') + parser.add_argument('--repository', required=True) + parser.add_argument('--pr', required=True, type=int) + parser.add_argument('--gh', default='gh') + parser.add_argument('--no-comment', action='store_true') + args = parser.parse_args(argv) + try: + root = Path(args.root).resolve() + report = compare_reports(report_at_revision(root, args.base), report_at_revision(root, args.head)) + print(report) + if not args.no_comment: + upsert_coverage_comment(GhApi(args.gh, args.repository), args.repository, args.pr, report) + return 0 + except (OSError, RuntimeError, ValueError) as error: + print(f'coverage-delta: cannot run: {error}', file=sys.stderr) + return 2 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/change_control/fixtures/fake_gh.py b/tools/change_control/fixtures/fake_gh.py new file mode 100755 index 00000000..990c599d --- /dev/null +++ b/tools/change_control/fixtures/fake_gh.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Deterministic gh replacement used only by change-control tests.""" + +import json +import os +import sys +from pathlib import Path + + +def main(): + """Serve API responses from the JSON file named by FAKE_GH_DATA.""" + data = json.loads(Path(os.environ['FAKE_GH_DATA']).read_text(encoding='utf-8')) + if data.get('exit_code'): + print(data.get('stderr', 'fake gh failure'), file=sys.stderr) + return int(data['exit_code']) + args = sys.argv[1:] + if not args or args[0] != 'api': + return 2 + endpoint = args[1] if len(args) > 1 else '' + method = 'GET' + if '--method' in args: + method = args[args.index('--method') + 1] + key = f'{method} {endpoint}' + calls = os.environ.get('FAKE_GH_CALLS') + if calls: + with Path(calls).open('a', encoding='utf-8') as stream: + stream.write(key + '\n') + response = data.get('responses', {}).get(key) + if response is None: + print(f'unconfigured fake gh request: {key}', file=sys.stderr) + return 1 + if isinstance(response, dict) and '__pages__' in response: + response = response['__pages__'] + elif '--slurp' in args: + response = [response] + if isinstance(response, str): + print(response) + else: + print(json.dumps(response)) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/change_control/issue_form.py b/tools/change_control/issue_form.py new file mode 100644 index 00000000..5313b86a --- /dev/null +++ b/tools/change_control/issue_form.py @@ -0,0 +1,120 @@ +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Parse and validate the constrained YAML used by the Change Request issue form.""" + +import re +from pathlib import Path + + +def _scalar(value): + value = value.strip() + if not value: + return '' + if value in ('true', 'false'): + return value == 'true' + if value.isdigit(): + return int(value) + if value.startswith('[') and value.endswith(']'): + return [part.strip().strip('\'"') for part in value[1:-1].split(',') if part.strip()] + return value.strip('\'"') + + +def parse_issue_form(path): + """Return issue-form controls from the repository's deliberately limited YAML subset.""" + lines = Path(path).read_text(encoding='utf-8').splitlines() + if any('\t' in line for line in lines): + raise ValueError('tabs are not valid indentation') + document = {'body': []} + item = None + section = None + options = False + in_body = False + for number, raw in enumerate(lines, 1): + if not raw.strip() or raw.lstrip().startswith('#') or raw.strip() == '---': + continue + indent = len(raw) - len(raw.lstrip(' ')) + text = raw.strip() + if indent == 0: + match = re.fullmatch(r'([a-z_]+):(?:\s*(.*))?', text) + if not match: + raise ValueError(f'{path}:{number}: unsupported top-level YAML') + key, value = match.groups() + if key == 'body': + if value: + raise ValueError(f'{path}:{number}: body must be a sequence') + in_body = True + else: + if in_body: + raise ValueError(f'{path}:{number}: top-level key after body') + document[key] = _scalar(value or '') + continue + if not in_body: + raise ValueError(f'{path}:{number}: unexpected indentation') + if indent == 2 and text.startswith('- type: '): + item = {'type': _scalar(text[8:]), 'required': False, 'options': []} + document['body'].append(item) + section = None + options = False + continue + if item is None: + raise ValueError(f'{path}:{number}: body entry must begin with type') + if indent == 4 and re.fullmatch(r'(id|attributes|validations):.*', text): + key, value = text.split(':', 1) + if key == 'id': + item['id'] = _scalar(value) + section = None + else: + if value.strip(): + raise ValueError(f'{path}:{number}: {key} must be a mapping') + section = key + options = False + continue + if indent == 6 and section in ('attributes', 'validations'): + if ':' not in text: + raise ValueError(f'{path}:{number}: expected mapping value') + key, value = text.split(':', 1) + if section == 'validations' and key == 'required': + item['required'] = _scalar(value) + elif section == 'attributes' and key == 'options': + if value.strip(): + raise ValueError(f'{path}:{number}: options must be a sequence') + options = True + elif section == 'attributes': + item[key] = _scalar(value) + options = False + else: + raise ValueError(f'{path}:{number}: unsupported validation') + continue + if indent == 8 and options and text.startswith('- '): + item['options'].append(_scalar(text[2:])) + continue + if indent >= 8 and section == 'attributes': + continue + raise ValueError(f'{path}:{number}: unsupported indentation or YAML construct') + validate_issue_form(document) + return document + + +def validate_issue_form(document): + """Reject forms that GitHub could silently replace with a blank issue page.""" + for key in ('name', 'description', 'title', 'labels'): + if not document.get(key): + raise ValueError(f'issue form missing top-level {key}') + body = document.get('body') + if not isinstance(body, list) or not body: + raise ValueError('issue form body must contain controls') + ids = [] + for field in body: + if field.get('type') not in ('input', 'textarea', 'dropdown'): + raise ValueError(f'unsupported issue form field type: {field.get("type")}') + if not field.get('id') or not field.get('label'): + raise ValueError('every issue form field needs id and label') + ids.append(field['id']) + if field['type'] == 'dropdown': + if not field.get('options') or not isinstance(field.get('default'), int): + raise ValueError('dropdown needs options and integer default') + if not 0 <= field['default'] < len(field['options']): + raise ValueError('dropdown default is outside options') + if len(ids) != len(set(ids)): + raise ValueError('duplicate issue form field id') + return document diff --git a/tools/change_control/labels.json b/tools/change_control/labels.json new file mode 100644 index 00000000..17ac2c33 --- /dev/null +++ b/tools/change_control/labels.json @@ -0,0 +1,18 @@ +[ + {"name": "change-request", "color": "0052cc", "description": "Modification-procedure change request"}, + {"name": "class-a", "color": "0e8a16", "description": "Safety class A: no safety impact"}, + {"name": "class-b", "color": "fbca04", "description": "Safety class B: indirect safety impact"}, + {"name": "class-c", "color": "b60205", "description": "Safety class C: direct safety impact"}, + {"name": "emergency", "color": "d93f0b", "description": "Compressed modification timeline"}, + {"name": "safety-defect", "color": "b60205", "description": "Released-baseline defect affecting safety"}, + {"name": "wire-break", "color": "5319e7", "description": "Coordinated remote and machine wire-format break"}, + {"name": "needs-change-request", "color": "c5def5", "description": "External contribution awaiting maintainer CR"}, + {"name": "status:proposed", "color": "ededed", "description": "CR proposed"}, + {"name": "status:under-analysis", "color": "d4c5f9", "description": "CR under analysis"}, + {"name": "status:authorized", "color": "0e8a16", "description": "CR authorized"}, + {"name": "status:rejected", "color": "b60205", "description": "CR rejected"}, + {"name": "status:in-implementation", "color": "1d76db", "description": "CR in implementation"}, + {"name": "status:in-verification", "color": "006b75", "description": "CR in verification"}, + {"name": "status:merged", "color": "5319e7", "description": "CR merged"}, + {"name": "status:released", "color": "0e8a16", "description": "CR released"} +] diff --git a/tools/change_control/self_test.py b/tools/change_control/self_test.py new file mode 100755 index 00000000..fa18df5e --- /dev/null +++ b/tools/change_control/self_test.py @@ -0,0 +1,831 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Spec-driven tests for repository change-control tooling.""" + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) + +from tools.change_control.__main__ import GhApi # noqa: E402 +from tools.change_control.checks import ( # noqa: E402 + AUTHORIZERS, + evaluate, + impact_analysis_complete, + minimum_class, + upsert_comment, +) +from tools.change_control.coverage_delta import ( # noqa: E402 + compare_reports, + run_linter_at_tree, + upsert_coverage_comment, +) +from tools.change_control.issue_form import parse_issue_form, validate_issue_form # noqa: E402 +from tools.change_control.wire_format import HEADER_NAMES, check_wire_format, compute_signature # noqa: E402 + +FORM = ROOT / '.github/ISSUE_TEMPLATE/change-request.yml' +WIRE_EXPECTED = ROOT / 'tools/change_control/wire_format.sha256' + + +def snapshot(**overrides): + """Return a complete synthetic GitHub state shaped like the real API snapshot.""" + ia = (ROOT / 'docs/process/templates/IMPACT_ANALYSIS.md').read_text(encoding='utf-8') + filled = ia.replace('Modules changed:', 'Modules changed: tools/change_control').replace( + '| | |', '| None | None |' + ) + data = { + 'pr': { + 'user': {'login': 'contributor'}, + 'body': 'Closes #17', + 'labels': [{'name': 'class-b'}], + 'head': {'sha': 'abc123'}, + }, + 'files': [{'filename': 'tools/change_control/checks.py'}], + 'issue': { + 'labels': [{'name': 'change-request'}, {'name': 'status:authorized'}], + 'body': '### Reason for the change\nNeeded\n### Hazards that may be affected\nNone identified, because tooling only\n### Description of the proposed change\nTooling\n### Baseline affected\nmain\n### Proposed class\nB', + }, + 'issue_comments': [ + {'user': {'login': AUTHORIZERS[0]}, 'body': 'Authorized: proceed.'}, + {'user': {'login': 'analyst'}, 'body': filled}, + ], + 'reviews': [], + 'check_runs': [{'name': 'change-control', 'conclusion': 'success', 'head_sha': 'abc123'}], + 'workflow_runs': [], + } + data.update(overrides) + return data + + +class IssueFormTests(unittest.TestCase): + def test_issue_form_is_valid_yaml_and_parses(self): + """The checked-in issue form must parse as the deliberately supported YAML subset.""" + parsed = parse_issue_form(FORM) + self.assertEqual(parsed['name'], 'Change Request') + self.assertGreater(len(parsed['body']), 5) + + def test_required_fields_present(self): + """All five creation-time fields mandated by the procedure must be required.""" + fields = {field['id']: field for field in parse_issue_form(FORM)['body']} + self.assertTrue( + all(fields[name]['required'] for name in ('reason', 'hazards', 'description', 'baseline', 'class')) + ) + + def test_class_dropdown_defaults_to_c(self): + """An unclassified request must conservatively default to Class C.""" + fields = {field['id']: field for field in parse_issue_form(FORM)['body']} + self.assertEqual(fields['class']['options'][fields['class']['default']], 'C') + + def test_every_source_field_is_represented(self): + """The issue form must carry every Change Request source section without a duplicate field list.""" + ids = {field['id'] for field in parse_issue_form(FORM)['body']} + expected = { + 'reason', + 'hazards', + 'description', + 'baseline', + 'requester', + 'impact-analysis', + 'class', + 'authorization', + 'implementation', + 'gate-0', + 'gate-1', + 'review', + 'deviations', + 'release', + 'status', + } + self.assertEqual(ids, expected) + + def test_source_field_details_survive_issue_form_conversion(self): + """YAML conversion must retain source details needed to complete implementation and Gate 1 records.""" + fields = {field['id']: field for field in parse_issue_form(FORM)['body']} + self.assertIn('Yes / No, with link', fields['implementation']['description']) + self.assertIn('Run by', fields['gate-1']['description']) + self.assertIn('Forward -', fields['gate-1']['description']) + self.assertIn('Backward -', fields['gate-1']['description']) + + def test_malformed_issue_form_is_rejected(self): + """Malformed indentation must fail instead of silently degrading to a blank issue.""" + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'form.yml' + path.write_text('name: Bad\nbody:\n - type: input\n id: broken\n', encoding='utf-8') + with self.assertRaises(ValueError): + parse_issue_form(path) + + def test_blank_issue_fallback_risk_is_rejected(self): + """A form lacking required top-level metadata or body controls must be invalid.""" + with self.assertRaises(ValueError): + validate_issue_form({'name': 'Change Request', 'body': []}) + + +class RepositoryPolicyTests(unittest.TestCase): + def test_issue_chooser_keeps_blank_issues_and_links_security_policy(self): + """Public reports must remain available while safety defects are directed to the private policy.""" + config = (ROOT / '.github/ISSUE_TEMPLATE/config.yml').read_text(encoding='utf-8') + self.assertIn('blank_issues_enabled: true', config) + self.assertIn('https://github.com/polymathrobotics/protective-stop/security/policy', config) + + def test_labels_json_has_no_duplicate_names(self): + """The reproducible label definition must contain unique names.""" + labels = json.loads((ROOT / 'tools/change_control/labels.json').read_text(encoding='utf-8')) + names = [label['name'] for label in labels] + self.assertEqual(len(names), len(set(names))) + + def test_labels_json_contains_the_complete_settled_label_set(self): + """The reproducible data must contain every label settled by the modification procedure plan.""" + labels = json.loads((ROOT / 'tools/change_control/labels.json').read_text(encoding='utf-8')) + names = {label['name'] for label in labels} + self.assertEqual( + names, + { + 'change-request', + 'class-a', + 'class-b', + 'class-c', + 'emergency', + 'safety-defect', + 'wire-break', + 'needs-change-request', + 'status:proposed', + 'status:under-analysis', + 'status:authorized', + 'status:rejected', + 'status:in-implementation', + 'status:in-verification', + 'status:merged', + 'status:released', + }, + ) + + def test_label_sync_dry_run_is_deterministic_and_network_free(self): + """Two dry runs must produce identical plans without invoking GitHub.""" + environment = os.environ.copy() + environment['PATH'] = '/usr/bin:/bin' + first = subprocess.run( + ['scripts/sync_labels.sh', '--dry-run'], + cwd=ROOT, + env=environment, + check=False, + capture_output=True, + text=True, + ) + second = subprocess.run( + ['scripts/sync_labels.sh', '--dry-run'], + cwd=ROOT, + env=environment, + check=False, + capture_output=True, + text=True, + ) + self.assertEqual((first.returncode, second.returncode), (0, 0)) + self.assertEqual(first.stdout, second.stdout) + self.assertEqual(first.stdout.count('would sync label:'), 16) + + def test_label_sync_targets_the_intended_repository_explicitly(self): + """Label writes must not depend on ambiguous git-remote repository inference.""" + script = (ROOT / 'scripts/sync_labels.sh').read_text(encoding='utf-8') + self.assertIn('--repo "$REPOSITORY"', script) + + def test_every_label_referenced_in_the_procedure_exists_in_labels_json(self): + """Every backticked process label must exist in the reproducible label set.""" + import re + + procedure = (ROOT / 'docs/process/MODIFICATION_PROCEDURE.md').read_text(encoding='utf-8') + referenced = set( + re.findall( + r'`((?:class-[abc]|change-request|emergency|safety-defect|wire-break|needs-change-request|status:[a-z-]+))`', + procedure, + ) + ) + labels = { + item['name'] for item in json.loads((ROOT / 'tools/change_control/labels.json').read_text(encoding='utf-8')) + } + self.assertTrue(referenced) + self.assertEqual(referenced - labels, set()) + + def test_codeowners_parses_and_covers_root(self): + """One CODEOWNERS rule must cover the repository root with all verified authorizers.""" + lines = [ + line.split() + for line in (ROOT / '.github/CODEOWNERS').read_text(encoding='utf-8').splitlines() + if line and not line.startswith('#') + ] + self.assertEqual(lines, [['*', *('@' + name for name in AUTHORIZERS)]]) + + def test_procedure_authorizers_match_enforcement_and_codeowners(self): + """The procedure, enforcement code, and CODEOWNERS must name one identical authorizer set.""" + import re + + procedure = (ROOT / 'docs/process/MODIFICATION_PROCEDURE.md').read_text(encoding='utf-8') + named = set(re.findall(r'@(iliabaranov|rajasimman-madhivanan|davidt315)', procedure)) + self.assertEqual(named, set(AUTHORIZERS)) + + @unittest.skipUnless( + os.environ.get('GH_TOKEN'), 'GH_TOKEN absent; CODEOWNERS handle resolution test visibly skipped' + ) + def test_codeowners_handles_resolve(self): + """Every CODEOWNERS account must resolve through authenticated GitHub API access.""" + for handle in AUTHORIZERS: + result = subprocess.run(['gh', 'api', f'users/{handle}'], check=False, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + + +class ImpactAndClassificationTests(unittest.TestCase): + def test_e2_rejects_heading_present_but_empty(self): + """An IA heading followed only by whitespace must fail completeness checking.""" + headings = ['# One', '# Two'] + complete, missing = impact_analysis_complete('# One\n \t\n# Two\nanswer\n', headings) + self.assertFalse(complete) + self.assertIn('# One', missing) + + def test_e2_accepts_na_content_without_judging_truth(self): + """The checker verifies nonblank content but does not judge whether N/A is adequate.""" + self.assertEqual(impact_analysis_complete('# One\nN/A\n# Two\nN/A\n', ['# One', '# Two']), (True, [])) + + def test_e3_rejects_invalid_sr_ids(self): + """A cited token shaped like an SR but outside the canonical grammar must be a finding.""" + data = snapshot() + data['issue_comments'][1]['body'] += '\nSR-BOGUS-99\n' + results = evaluate(ROOT, data) + self.assertEqual(next(item for item in results if item.check_id == 'E3').status, 'fail') + + def test_e4_under_classification_is_a_finding(self): + """A class label below the path-derived floor must fail E4.""" + data = snapshot(files=[{'filename': 'docs/safety/HARA.md'}]) + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E4').status, 'fail') + + def test_e4_over_classification_is_not_a_finding(self): + """A class label above the path-derived floor must be accepted.""" + data = snapshot(files=[{'filename': 'README.md'}]) + data['pr']['labels'] = [{'name': 'class-c'}] + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E4').status, 'pass') + + def test_missing_class_label_is_a_finding(self): + """A PR without a classification label must fail classification checking.""" + data = snapshot() + data['pr']['labels'] = [] + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E4').status, 'fail') + + def test_duplicate_class_labels_are_a_finding(self): + """Multiple classification labels are ambiguous and must fail E4.""" + data = snapshot() + data['pr']['labels'] = [{'name': 'class-a'}, {'name': 'class-b'}] + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E4').status, 'fail') + + def test_minimum_class_rules(self): + """Every settled path rule must produce its documented minimum classification floor.""" + self.assertEqual(minimum_class(['pstop_c/x.c']), 'C') + self.assertEqual(minimum_class(['firmware/main/main.c']), 'C') + self.assertEqual(minimum_class(['components/x.c']), 'B') + self.assertEqual(minimum_class(['firmware/sdkconfig.defaults']), 'B') + self.assertEqual(minimum_class(['.github/workflows/x.yml']), 'B') + + +class ApprovalAndLinkTests(unittest.TestCase): + def _class_c(self): + data = snapshot(files=[{'filename': 'docs/safety/HARA.md'}]) + data['pr']['labels'] = [{'name': 'class-c'}] + data['reviews'] = [ + {'user': {'login': AUTHORIZERS[0]}, 'state': 'APPROVED', 'commit_id': 'abc123'}, + {'user': {'login': AUTHORIZERS[1]}, 'state': 'APPROVED', 'commit_id': 'abc123'}, + ] + return data + + def test_class_c_two_distinct_approvals_pass(self): + """Class C requires two distinct current approving authorizers who are not the author.""" + self.assertEqual(next(item for item in evaluate(ROOT, self._class_c()) if item.check_id == 'E5').status, 'pass') + + def test_duplicate_reviews_count_once(self): + """Repeated approvals by one account must count as one approval.""" + data = self._class_c() + data['reviews'][1]['user']['login'] = AUTHORIZERS[0] + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E5').status, 'fail') + + def test_author_approval_is_excluded(self): + """The PR author's own approval must never satisfy Class C approval.""" + data = self._class_c() + data['pr']['user']['login'] = AUTHORIZERS[0] + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E5').status, 'fail') + + def test_stale_approval_is_excluded(self): + """An approval for a commit other than the PR head must not count.""" + data = self._class_c() + data['reviews'][1]['commit_id'] = 'oldsha' + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E5').status, 'fail') + + def test_approval_without_commit_identity_is_excluded(self): + """An approval with no commit identity cannot establish review of the current diff.""" + data = self._class_c() + data['reviews'][1].pop('commit_id') + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E5').status, 'fail') + + def test_authorization_after_pr_is_an_ordering_finding(self): + """An authorization timestamp after implementation began must fail E1 ordering.""" + data = snapshot() + data['pr']['created_at'] = '2026-09-10T10:00:00Z' + data['issue_comments'][0]['created_at'] = '2026-09-10T11:00:00Z' + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E1').status, 'fail') + + def test_authorization_before_impact_analysis_is_an_ordering_finding(self): + """Authorization must follow the completed Impact Analysis rather than merely precede implementation.""" + data = snapshot() + data['pr']['created_at'] = '2026-09-10T12:00:00Z' + data['issue_comments'][0]['created_at'] = '2026-09-10T10:00:00Z' + data['issue_comments'][1]['created_at'] = '2026-09-10T11:00:00Z' + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E1').status, 'fail') + + def test_pr_author_cannot_authorize_own_change(self): + """The implementer must not satisfy the Change Request authorization requirement.""" + data = snapshot() + data['pr']['user']['login'] = AUTHORIZERS[0] + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E1').status, 'fail') + + def test_class_c_requires_two_distinct_cr_authorizers_before_implementation(self): + """Class C implementation cannot start after only one Change Request authorization.""" + data = snapshot() + data['pr']['labels'] = [{'name': 'class-c'}] + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E1').status, 'fail') + data['issue_comments'].append({'user': {'login': AUTHORIZERS[1]}, 'body': 'Authorized: proceed.'}) + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E1').status, 'pass') + + def test_rejection_text_does_not_count_as_authorization(self): + """An authorizer saying a change is not authorized must not satisfy E1.""" + data = snapshot() + data['issue_comments'][0]['body'] = 'Rejected: this is not authorized.' + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E1').status, 'fail') + + def test_emergency_requires_two_approvals_and_short_form_ia(self): + """Emergency E7 must require two current approvals and all three short-form IA subjects.""" + data = snapshot() + data['pr']['labels'].append({'name': 'emergency'}) + data['issue_comments'][1]['body'] = ( + 'What changed: tooling\nWhat it could affect: process\nTests: change-control' + ) + data['reviews'] = [ + {'user': {'login': AUTHORIZERS[0]}, 'state': 'APPROVED', 'commit_id': 'abc123'}, + {'user': {'login': AUTHORIZERS[1]}, 'state': 'APPROVED', 'commit_id': 'abc123'}, + ] + data['check_runs'] = [{'name': 'change-control', 'conclusion': 'success', 'head_sha': 'abc123'}] + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E7').status, 'pass') + + def test_ambiguous_change_request_link_fails(self): + """A bare issue number without Closes or Refs syntax must not be guessed as the CR.""" + data = snapshot() + data['pr']['body'] = 'Issue #17' + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E1').status, 'fail') + + def test_multiple_change_request_links_fail(self): + """Multiple candidate CR links must fail rather than selecting one.""" + data = snapshot() + data['pr']['body'] = 'Closes #17\nRefs #18' + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E1').status, 'fail') + + def test_needs_change_request_label_exempts_e1(self): + """External PRs awaiting a maintainer CR must produce a pending E1 result.""" + data = snapshot() + data['pr']['labels'].append({'name': 'needs-change-request'}) + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E1').status, 'pending') + + +class EvidenceAndCommentTests(unittest.TestCase): + def test_e6_requires_at_least_one_named_test(self): + """An IA that names no specific test must fail the plan-versus-execution check.""" + result = next(item for item in evaluate(ROOT, snapshot()) if item.check_id == 'E6') + self.assertEqual(result.status, 'fail') + + def test_e6_missing_workflow_evidence_names_test(self): + """A test named by the IA without head-SHA check evidence must fail and be named.""" + data = snapshot() + data['issue_comments'][1]['body'] += '\n# 7. Verification plan for this change\n`missing-check`\n' + result = next(item for item in evaluate(ROOT, data) if item.check_id == 'E6') + self.assertEqual(result.status, 'fail') + self.assertIn('missing-check', result.message) + + def test_e6_ignores_evidence_for_other_sha(self): + """Evidence attached to an older commit must not satisfy the IA plan.""" + data = snapshot() + data['issue_comments'][1]['body'] += '\n# 7. Verification plan for this change\n`host-check`\n' + data['check_runs'] = [{'name': 'host-check', 'conclusion': 'success', 'head_sha': 'oldsha'}] + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E6').status, 'fail') + + def test_e6_does_not_accept_a_substring_check_name(self): + """A short IA test token must not match an unrelated longer check-run name.""" + data = snapshot() + data['issue_comments'][1]['body'] += '\n# 7. Verification plan for this change\n`host`\n' + data['check_runs'] = [{'name': 'host-check', 'conclusion': 'success', 'head_sha': 'abc123'}] + result = next(item for item in evaluate(ROOT, data) if item.check_id == 'E6') + self.assertEqual(result.status, 'fail') + self.assertIn('host', result.message) + + def test_e6_reads_plain_table_cells(self): + """Plain test names in the IA verification table must be checked, not only backticked names.""" + data = snapshot() + data['issue_comments'][1]['body'] = data['issue_comments'][1]['body'].replace( + '| Tests that validate the change itself | |', + '| Tests that validate the change itself | missing-table-check |', + ) + result = next(item for item in evaluate(ROOT, data) if item.check_id == 'E6') + self.assertEqual(result.status, 'fail') + self.assertIn('missing-table-check', result.message) + + def test_upsert_comment_updates_existing_comment(self): + """An existing bot report must be updated instead of appending another comment.""" + calls = [] + upsert_comment( + lambda method, path, body=None: calls.append((method, path, body)), + 9, + 'report', + [{'id': 44, 'body': 'old'}], + ) + self.assertEqual(calls[0][0:2], ('PATCH', 'repos/{repo}/issues/comments/44')) + + def test_upsert_comment_creates_when_absent(self): + """A bot report must be created exactly once when no marker exists.""" + calls = [] + upsert_comment(lambda method, path, body=None: calls.append((method, path, body)), 9, 'report', []) + self.assertEqual( + calls, [('POST', 'repos/{repo}/issues/9/comments', {'body': '\nreport'})] + ) + + +class WireFormatTests(unittest.TestCase): + def _copy_headers(self, directory): + include = Path(directory) / 'pstop_c/pstop/include/pstop' + include.mkdir(parents=True) + source = ROOT / 'pstop_c/pstop/include/pstop' + for name in HEADER_NAMES: + shutil.copy2(source / name, include / name) + return Path(directory) + + def test_signature_stable_across_comment_only_change(self): + """Adding a C comment to a watched header must not alter its signature.""" + with tempfile.TemporaryDirectory() as directory: + root = self._copy_headers(directory) + before = compute_signature(root) + path = root / 'pstop_c/pstop/include/pstop/protocol.h' + path.write_text(path.read_text(encoding='utf-8') + '\n/* comment only */\n', encoding='utf-8') + self.assertEqual(compute_signature(root).aggregate, before.aggregate) + + def test_comment_only_change_needs_no_wire_labels(self): + """A comment-only watched-header edit must pass the declaration check without wire-break labels.""" + with tempfile.TemporaryDirectory() as directory: + root = self._copy_headers(directory) + shutil.copy2(WIRE_EXPECTED, root / 'wire_format.sha256') + path = root / 'pstop_c/pstop/include/pstop/protocol.h' + path.write_text(path.read_text(encoding='utf-8') + '\n/* comment only */\n', encoding='utf-8') + code, _ = check_wire_format(root, root / 'wire_format.sha256', set(), [str(path)]) + self.assertEqual(code, 0) + + def test_signature_changes_on_field_addition(self): + """Adding a structure field must alter the aggregate wire signature.""" + with tempfile.TemporaryDirectory() as directory: + root = self._copy_headers(directory) + before = compute_signature(root) + path = root / 'pstop_c/pstop/include/pstop/pstop_msg.h' + path.write_text( + path.read_text(encoding='utf-8').replace('uint16_t checksum;', 'uint32_t added;\nuint16_t checksum;'), + encoding='utf-8', + ) + self.assertNotEqual(compute_signature(root).aggregate, before.aggregate) + + def test_signature_changes_on_constant_change(self): + """Changing a protocol constant must alter the aggregate wire signature.""" + with tempfile.TemporaryDirectory() as directory: + root = self._copy_headers(directory) + before = compute_signature(root) + path = root / 'pstop_c/pstop/include/pstop/pstop_msg.h' + path.write_text(path.read_text(encoding='utf-8').replace('0xADU', '0xAEU'), encoding='utf-8') + self.assertNotEqual(compute_signature(root).aggregate, before.aggregate) + + def test_signature_changes_on_message_size_change(self): + """Changing PSTOP_MESSAGE_SIZE must alter the signature and reported literal.""" + with tempfile.TemporaryDirectory() as directory: + root = self._copy_headers(directory) + before = compute_signature(root) + path = root / 'pstop_c/pstop/include/pstop/config.h' + path.write_text(path.read_text(encoding='utf-8').replace('48U', '49U'), encoding='utf-8') + after = compute_signature(root) + self.assertEqual(after.message_size, '49U') + self.assertNotEqual(after.aggregate, before.aggregate) + + def test_signature_changes_on_version_change(self): + """Changing PSTOP_VERSION must alter both the reported version and aggregate signature.""" + with tempfile.TemporaryDirectory() as directory: + root = self._copy_headers(directory) + before = compute_signature(root) + path = root / 'pstop_c/pstop/include/pstop/config.h' + path.write_text(path.read_text(encoding='utf-8').replace('0x02U', '0x03U'), encoding='utf-8') + after = compute_signature(root) + self.assertEqual(after.version, '0x03U') + self.assertNotEqual(after.aggregate, before.aggregate) + + def test_check_exits_one_on_mismatch_and_names_the_headers(self): + """A stale expected signature must fail and name each changed header.""" + with tempfile.TemporaryDirectory() as directory: + root = self._copy_headers(directory) + shutil.copy2(WIRE_EXPECTED, root / 'wire_format.sha256') + path = root / 'pstop_c/pstop/include/pstop/protocol.h' + path.write_text(path.read_text(encoding='utf-8') + '\nint changed;\n', encoding='utf-8') + code, message = check_wire_format(root, root / 'wire_format.sha256', {'wire-break', 'class-c'}, []) + self.assertEqual(code, 1) + self.assertIn('protocol.h', message) + + def test_expected_update_requires_both_labels(self): + """Changing the expected signature cannot pass without wire-break and class-c labels.""" + code, message = check_wire_format( + ROOT, WIRE_EXPECTED, {'wire-break'}, ['tools/change_control/wire_format.sha256'] + ) + self.assertEqual(code, 1) + self.assertIn('class-c', message) + + def test_initial_expected_signature_is_not_a_wire_break(self): + """Adding the first reviewed signature snapshot must not claim the existing wire format changed.""" + code, _ = check_wire_format( + ROOT, + WIRE_EXPECTED, + set(), + ['tools/change_control/wire_format.sha256'], + expectation_preexisted=False, + ) + self.assertEqual(code, 0) + + def test_initial_snapshot_cannot_hide_a_header_change(self): + """A wire-header edit accompanying the first snapshot must still require Class C declaration.""" + code, message = check_wire_format( + ROOT, + WIRE_EXPECTED, + {'wire-break'}, + ['pstop_c/pstop/include/pstop/protocol.h', 'tools/change_control/wire_format.sha256'], + expectation_preexisted=False, + ) + self.assertEqual(code, 1) + self.assertIn('class-c', message) + + def test_wire_cli_exposes_guard_exit_codes(self): + """The public wire CLI must return 0 for a match, 1 for policy mismatch, and 2 when it cannot run.""" + clean = subprocess.run( + [sys.executable, '-m', 'tools.change_control.wire_format', 'check', '--root', str(ROOT)], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + finding = subprocess.run( + [ + sys.executable, + '-m', + 'tools.change_control.wire_format', + 'check', + '--root', + str(ROOT), + '--changed-file', + 'tools/change_control/wire_format.sha256', + '--labels', + 'wire-break', + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + unable = subprocess.run( + [sys.executable, '-m', 'tools.change_control.wire_format', 'check', '--root', '/does/not/exist'], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + self.assertEqual((clean.returncode, finding.returncode, unable.returncode), (0, 1, 2)) + + def test_mismatch_explains_coordinated_rollout(self): + """A wire mismatch must explicitly require coordinated remote and machine rollout.""" + with tempfile.TemporaryDirectory() as directory: + root = self._copy_headers(directory) + shutil.copy2(WIRE_EXPECTED, root / 'wire_format.sha256') + path = root / 'pstop_c/pstop/include/pstop/constants.h' + path.write_text(path.read_text(encoding='utf-8').replace('10U', '11U'), encoding='utf-8') + _, message = check_wire_format(root, root / 'wire_format.sha256', set(), []) + self.assertIn('remote and machine', message.lower()) + + +class CoverageDeltaTests(unittest.TestCase): + def test_coverage_delta_detects_lost_citation(self): + """Deleting real cited evidence in a scratch repository must name the affected requirement.""" + with tempfile.TemporaryDirectory() as directory: + scratch = Path(directory) / 'repository' + shutil.copytree(ROOT / 'tools/safety_lint/fixtures/repository', scratch) + (scratch / 'tools').mkdir(exist_ok=True) + shutil.copytree( + ROOT / 'tools/safety_lint', + scratch / 'tools/safety_lint', + ignore=shutil.ignore_patterns('__pycache__'), + ) + base = run_linter_at_tree(scratch) + (scratch / 'tests/test_unique_probe.py').unlink() + head = run_linter_at_tree(scratch) + delta = compare_reports(base, head) + self.assertIn('SR-R-01', delta) + self.assertIn('unresolvable', delta.lower()) + + def test_base_without_linter_exposes_stacked_dependency(self): + """A base predating change-0001 must be reported as unavailable, never treated as zero coverage.""" + delta = compare_reports( + {'unavailable': 'tools/safety_lint absent'}, + {'coverage': {'total': 1, 'cited_tests': 1}, 'findings': [], 'citations': {}}, + ) + self.assertIn('stacked dependency', delta.lower()) + + def test_head_without_linter_is_an_execution_error(self): + """Missing head coverage must fail explicitly rather than render unknown values as a delta.""" + with self.assertRaisesRegex(RuntimeError, 'head coverage unavailable'): + compare_reports( + {'coverage': {'total': 1, 'cited_tests': 1}, 'findings': [], 'citations': {}}, + {'unavailable': 'tools/safety_lint absent'}, + ) + + def test_coverage_comment_updates_instead_of_appending(self): + """Coverage delta must update its marker-owned comment rather than append on every run.""" + writes = [] + + def api(method, path, body=None): + if method == 'GET': + return [{'id': 12, 'body': '\nold'}] + writes.append((method, path, body)) + return {} + + upsert_coverage_comment(api, 'acme/project', 7, 'new') + self.assertEqual(writes[0][0:2], ('PATCH', 'repos/acme/project/issues/comments/12')) + + def test_coverage_cli_no_comment_avoids_write_api(self): + """Fork-safe coverage reporting must support stdout-only operation when write tokens are unavailable.""" + result = subprocess.run( + [sys.executable, '-m', 'tools.change_control.coverage_delta', '--help'], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0) + self.assertIn('--no-comment', result.stdout) + + +class CliTests(unittest.TestCase): + def test_gh_api_flattens_all_paginated_list_pages(self): + """Policy evaluation must see every item returned across GitHub list pages.""" + with tempfile.TemporaryDirectory() as directory: + data = Path(directory) / 'gh.json' + data.write_text( + json.dumps({'responses': {'GET items': {'__pages__': [[{'id': 1}], [{'id': 2}]]}}}), + encoding='utf-8', + ) + with mock.patch.dict(os.environ, {'FAKE_GH_DATA': str(data)}): + api = GhApi(f'{sys.executable} {ROOT / "tools/change_control/fixtures/fake_gh.py"}', 'acme/project') + self.assertEqual(api('GET', 'items', paginate=True), [{'id': 1}, {'id': 2}]) + + def test_gh_api_merges_all_paginated_collection_pages(self): + """Check and workflow evidence must include every GitHub response page.""" + with tempfile.TemporaryDirectory() as directory: + data = Path(directory) / 'gh.json' + pages = [{'check_runs': [{'id': 1}]}, {'check_runs': [{'id': 2}]}] + data.write_text( + json.dumps({'responses': {'GET checks': {'__pages__': pages}}}), + encoding='utf-8', + ) + with mock.patch.dict(os.environ, {'FAKE_GH_DATA': str(data)}): + api = GhApi(f'{sys.executable} {ROOT / "tools/change_control/fixtures/fake_gh.py"}', 'acme/project') + self.assertEqual( + api('GET', 'checks', paginate=True, collection_key='check_runs'), + {'check_runs': [{'id': 1}, {'id': 2}]}, + ) + + def test_workflows_disable_writes_for_fork_pull_requests(self): + """Fork pull requests must still run checks without attempting unavailable comment or label writes.""" + change = (ROOT / '.github/workflows/change-control.yml').read_text(encoding='utf-8') + coverage = (ROOT / '.github/workflows/coverage-delta.yml').read_text(encoding='utf-8') + wire = (ROOT / '.github/workflows/wire-break.yml').read_text(encoding='utf-8') + self.assertIn('CAN_COMMENT', change) + self.assertIn('--no-comment', change) + self.assertIn('CAN_COMMENT', coverage) + self.assertIn('--no-comment', coverage) + self.assertIn('CAN_LABEL', wire) + + def test_warn_mode_exits_zero_with_findings(self): + """Warn mode must report findings while returning success to the caller.""" + result = self._run_cli('warn', {'responses': {}}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('mode: warn', result.stdout) + + def test_enforce_mode_exits_one_with_findings(self): + """Enforce mode must return one for the same findings that warn mode tolerates.""" + result = self._run_cli('enforce', {'responses': {}}) + self.assertEqual(result.returncode, 1, result.stderr) + + def test_mode_file_rejects_unknown_value(self): + """An invalid mode must return two rather than silently defaulting to warn.""" + result = self._run_cli('maybe', {'responses': {}}) + self.assertEqual(result.returncode, 2) + + def test_gh_api_failure_returns_two(self): + """A failed gh subprocess must make the checker unable to run, not create policy findings.""" + result = self._run_cli('warn', {'exit_code': 1, 'stderr': 'API unavailable'}) + self.assertEqual(result.returncode, 2) + self.assertIn('API unavailable', result.stderr) + + def test_partial_json_returns_two(self): + """A partial GitHub response must fail closed as an execution error.""" + responses = self._responses() + responses['GET repos/acme/project/pulls/7'] = {'body': 'Closes #17'} + result = self._run_cli('warn', {'responses': responses}) + self.assertEqual(result.returncode, 2) + + def test_no_comment_avoids_write_api(self): + """Fork-safe change-control checks must not call the comment API when writes are unavailable.""" + result, calls = self._run_cli('warn', {'responses': self._responses()}, no_comment=True, record_calls=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertNotIn('POST repos/acme/project/issues/7/comments', calls) + + def _responses(self): + data = snapshot() + data['pr']['labels'] = [] + return { + 'GET repos/acme/project/pulls/7': data['pr'], + 'GET repos/acme/project/pulls/7/files': data['files'], + 'GET repos/acme/project/issues/17': data['issue'], + 'GET repos/acme/project/issues/17/comments': data['issue_comments'], + 'GET repos/acme/project/pulls/7/reviews': data['reviews'], + 'GET repos/acme/project/commits/abc123/check-runs': {'check_runs': data['check_runs']}, + 'GET repos/acme/project/actions/runs?head_sha=abc123': {'workflow_runs': data['workflow_runs']}, + 'GET repos/acme/project/issues/7/comments': [], + 'POST repos/acme/project/issues/7/comments': {}, + } + + def _run_cli(self, mode, fake_data, no_comment=False, record_calls=False): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'docs/process').mkdir(parents=True) + (root / 'docs/process/enforcement-mode').write_text(mode + '\n', encoding='utf-8') + (root / 'docs/process/templates').mkdir() + shutil.copy2( + ROOT / 'docs/process/templates/IMPACT_ANALYSIS.md', root / 'docs/process/templates/IMPACT_ANALYSIS.md' + ) + (root / 'docs/safety').mkdir(parents=True) + shutil.copy2(ROOT / 'docs/safety/SAFETY_REQUIREMENTS.md', root / 'docs/safety/SAFETY_REQUIREMENTS.md') + (root / '.github/ISSUE_TEMPLATE').mkdir(parents=True) + shutil.copy2(FORM, root / '.github/ISSUE_TEMPLATE/change-request.yml') + data_path = root / 'gh.json' + data_path.write_text( + json.dumps( + fake_data + if fake_data.get('exit_code') + else {'responses': fake_data.get('responses') or self._responses()} + ), + encoding='utf-8', + ) + environment = os.environ.copy() + environment['FAKE_GH_DATA'] = str(data_path) + environment['PYTHONPATH'] = str(ROOT) + calls_path = root / 'gh-calls.txt' + if record_calls: + environment['FAKE_GH_CALLS'] = str(calls_path) + command = [ + sys.executable, + '-m', + 'tools.change_control', + '--root', + str(root), + '--repository', + 'acme/project', + '--pr', + '7', + '--gh', + f'{sys.executable} {ROOT / "tools/change_control/fixtures/fake_gh.py"}', + ] + if no_comment: + command.append('--no-comment') + result = subprocess.run( + command, + cwd=ROOT, + env=environment, + check=False, + capture_output=True, + text=True, + ) + if record_calls: + calls = calls_path.read_text(encoding='utf-8') if calls_path.exists() else '' + return result, calls + return result + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/tools/change_control/wire_format.py b/tools/change_control/wire_format.py new file mode 100644 index 00000000..5747a703 --- /dev/null +++ b/tools/change_control/wire_format.py @@ -0,0 +1,160 @@ +# SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Compute and enforce the normalized pstop_c public wire-header signature.""" + +import argparse +import hashlib +import re +import sys +from dataclasses import dataclass +from pathlib import Path + +HEADER_NAMES = ( + 'config.h', + 'constants.h', + 'protocol.h', + 'protocol_data.h', + 'pstop_msg.h', + 'checksum.h', + 'device_id.h', + 'endian.h', +) +HEADER_PREFIX = 'pstop_c/pstop/include/pstop/' + + +@dataclass(frozen=True) +class WireSignature: + """Per-header evidence and the aggregate normalized wire signature.""" + + version: str + message_size: str + headers: dict + aggregate: str + + +def _strip_comments(text): + text = re.sub(r'/\*.*?\*/', ' ', text, flags=re.DOTALL) + text = re.sub(r'//[^\n]*', ' ', text) + return text + + +def _literal(text, name): + match = re.search(rf'^\s*#\s*define\s+{name}\s+(\S+)', _strip_comments(text), re.MULTILINE) + if not match: + raise ValueError(f'{name} not found in config.h') + return match.group(1) + + +def compute_signature(root): + """Hash comment-free, whitespace-collapsed headers plus explicit protocol literals.""" + include = Path(root) / HEADER_PREFIX + normalized = {} + for name in HEADER_NAMES: + path = include / name + if not path.is_file(): + raise FileNotFoundError(path) + normalized[name] = ' '.join(_strip_comments(path.read_text(encoding='utf-8')).split()) + config = (include / 'config.h').read_text(encoding='utf-8') + version = _literal(config, 'PSTOP_VERSION') + message_size = _literal(config, 'PSTOP_MESSAGE_SIZE') + header_hashes = {name: hashlib.sha256(normalized[name].encode()).hexdigest() for name in HEADER_NAMES} + payload = ''.join(f'{name}\0{normalized[name]}\0' for name in HEADER_NAMES) + payload += f'PSTOP_VERSION\0{version}\0PSTOP_MESSAGE_SIZE\0{message_size}\0' + return WireSignature(version, message_size, header_hashes, hashlib.sha256(payload.encode()).hexdigest()) + + +def read_expected(path): + """Read the reviewable line-oriented wire signature record.""" + values = {} + headers = {} + for raw in Path(path).read_text(encoding='utf-8').splitlines(): + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) != 2: + raise ValueError(f'invalid expected signature line: {raw}') + key, value = parts + if key in HEADER_NAMES: + headers[key] = value + else: + values[key] = value + if set(headers) != set(HEADER_NAMES) or not {'PSTOP_VERSION', 'PSTOP_MESSAGE_SIZE', 'aggregate'} <= set(values): + raise ValueError('expected signature lacks version, size, aggregate, or per-header hashes') + return WireSignature(values['PSTOP_VERSION'], values['PSTOP_MESSAGE_SIZE'], headers, values['aggregate']) + + +def render_signature(signature): + """Render deterministic expected-signature content suitable for code review.""" + lines = [ + '# Normalized pstop_c wire headers; comments stripped and whitespace collapsed.', + f'# Corresponds to PSTOP_VERSION {signature.version}.', + f'PSTOP_VERSION {signature.version}', + f'PSTOP_MESSAGE_SIZE {signature.message_size}', + ] + lines.extend(f'{name} {signature.headers[name]}' for name in HEADER_NAMES) + lines.append(f'aggregate {signature.aggregate}') + return '\n'.join(lines) + '\n' + + +def check_wire_format(root, expected_path, labels, changed_files, expectation_preexisted=True): + """Return guard exit code and explanation for current headers, labels, and changed paths.""" + try: + current = compute_signature(root) + expected = read_expected(expected_path) + except (OSError, ValueError) as error: + return 2, f'wire-format: cannot run: {error}' + mismatched = [name for name in HEADER_NAMES if current.headers[name] != expected.headers[name]] + if current.version != expected.version or current.message_size != expected.message_size: + if 'config.h' not in mismatched: + mismatched.append('config.h') + expectation_change = expectation_preexisted and 'tools/change_control/wire_format.sha256' in changed_files + initial_header_change = not expectation_preexisted and any(path.startswith(HEADER_PREFIX) for path in changed_files) + signature_changed = bool(mismatched or current.aggregate != expected.aggregate) + required = {'wire-break', 'class-c'} if expectation_change or initial_header_change or signature_changed else set() + missing_labels = sorted(required - set(labels)) + rollout = 'Remote and machine must be released and deployed together for a coordinated rollout.' + if signature_changed: + names = ', '.join(sorted(set(mismatched))) or 'aggregate signature' + return 1, f'wire-format mismatch in: {names}. Update the reviewed expectation in this PR. {rollout}' + if missing_labels: + return ( + 1, + f'wire-format or expectation changed; required PR labels missing: {", ".join(missing_labels)}. {rollout}', + ) + return ( + 0, + f'wire-format signature matches PSTOP_VERSION {current.version}, PSTOP_MESSAGE_SIZE {current.message_size}', + ) + + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument('command', choices=('check', 'snapshot')) + parser.add_argument('--root', default='.') + parser.add_argument('--expected', default='tools/change_control/wire_format.sha256') + parser.add_argument('--labels', default='') + parser.add_argument('--changed-file', action='append', default=[]) + parser.add_argument('--initial-expectation', action='store_true') + args = parser.parse_args(argv) + if args.command == 'snapshot': + try: + print(render_signature(compute_signature(args.root)), end='') + return 0 + except (OSError, ValueError) as error: + print(f'wire-format: cannot run: {error}', file=sys.stderr) + return 2 + labels = {label.strip() for label in args.labels.split(',') if label.strip()} + code, message = check_wire_format( + args.root, + Path(args.root) / args.expected, + labels, + args.changed_file, + expectation_preexisted=not args.initial_expectation, + ) + print(message, file=sys.stderr if code == 2 else sys.stdout) + return code + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/change_control/wire_format.sha256 b/tools/change_control/wire_format.sha256 new file mode 100644 index 00000000..ac38da86 --- /dev/null +++ b/tools/change_control/wire_format.sha256 @@ -0,0 +1,13 @@ +# Normalized pstop_c wire headers; comments stripped and whitespace collapsed. +# Corresponds to PSTOP_VERSION 0x02U. +PSTOP_VERSION 0x02U +PSTOP_MESSAGE_SIZE 48U +config.h 3fb2ef9fef38451b283ef7e6b850cf06eb5f9b81e5dc07af64d43843c9ba0474 +constants.h 7f24ea90cac9a64008c69b16c4629682bfa150238e3f7825f4726b055b59ea54 +protocol.h a5250fac05f5c8ea6f95d4af821f07111323acef7c39827fd215d2822c56a7aa +protocol_data.h afe3ca0116d57832350defdfe50d22d9d0da3c6e8ee8690e0c084f24efd932bb +pstop_msg.h 6c5ab0d4847dc5fafd43890093473b202548e87cf0c4b8c382def5a36e301249 +checksum.h 47fddca6f7f7bee63cf0c8fda307ae438ce1e497b9d1c2b6639485df84adee4e +device_id.h 51047e992b03d156b2acc8acc2130e6cc0a246ec6035438a213f3e9e440bf422 +endian.h e22513590e1f28a54ec7d9e0bc2986d4adbb0ad38f6df069ec42da8dc82f9bce +aggregate a652abc231b4acb2d9014d73f459123e914fb4b1fcfe8c630f3f8d3853f012af From 01e1f74ae52d57fe0e37d133661b9df2d2932c46 Mon Sep 17 00:00:00 2001 From: Raj Madhivanan Date: Fri, 11 Sep 2026 17:15:59 -0700 Subject: [PATCH 03/12] fix: keep advisory checks non-blocking Co-authored-by: OpenCode --- .github/workflows/change-control.yml | 2 +- .github/workflows/coverage-delta.yml | 1 + tools/change_control/__main__.py | 12 +++++++++--- tools/change_control/checks.py | 4 +++- tools/change_control/coverage_delta.py | 5 ++++- tools/change_control/self_test.py | 8 ++++++++ 6 files changed, 26 insertions(+), 6 deletions(-) diff --git a/.github/workflows/change-control.yml b/.github/workflows/change-control.yml index 0ce2e9b8..bdbd5615 100644 --- a/.github/workflows/change-control.yml +++ b/.github/workflows/change-control.yml @@ -10,7 +10,7 @@ on: permissions: contents: read issues: write - pull-requests: read + pull-requests: write checks: read actions: read diff --git a/.github/workflows/coverage-delta.yml b/.github/workflows/coverage-delta.yml index 45cf700d..95d8cbb0 100644 --- a/.github/workflows/coverage-delta.yml +++ b/.github/workflows/coverage-delta.yml @@ -7,6 +7,7 @@ on: permissions: contents: read issues: write + pull-requests: write jobs: coverage-delta: diff --git a/tools/change_control/__main__.py b/tools/change_control/__main__.py index 5302a3a4..b5aea4bc 100644 --- a/tools/change_control/__main__.py +++ b/tools/change_control/__main__.py @@ -33,7 +33,8 @@ def __call__(self, method, path, body=None, paginate=False, collection_key=None) command.extend(['--field', f'{key}={value}']) result = subprocess.run(command, check=False, capture_output=True, text=True) if result.returncode: - raise RuntimeError(result.stderr.strip() or f'gh api failed for {endpoint}') + detail = result.stderr.strip() or 'no error detail' + raise RuntimeError(f'gh api failed for {endpoint}: {detail}') try: response = json.loads(result.stdout or '{}') except json.JSONDecodeError as error: @@ -126,9 +127,14 @@ def main(argv=None): results = evaluate(Path(args.root), data) report = render_report(mode, results) print(report) - if not args.no_comment: - upsert_comment(api, args.pr, report, data['pr_comments']) findings = any(item.status == 'fail' for item in results) + if not args.no_comment: + try: + upsert_comment(api, args.pr, report, data['pr_comments']) + except RuntimeError as error: + print(f'change-control: comment publication warning: {error}', file=sys.stderr) + if mode == 'enforce': + return 2 return 1 if findings and mode == 'enforce' else 0 except (OSError, RuntimeError, ValueError, KeyError, LintError) as error: print(f'change-control: cannot run: {error}', file=sys.stderr) diff --git a/tools/change_control/checks.py b/tools/change_control/checks.py index 6f360d7b..352a8c6e 100644 --- a/tools/change_control/checks.py +++ b/tools/change_control/checks.py @@ -347,7 +347,9 @@ def evaluate(root, data): def render_report(mode, results): """Render one deterministic PR comment with the active mode visible.""" lines = [f'mode: {mode}', '', '| Check | Result | Explanation |', '|---|---|---|'] - lines.extend(f'| {item.check_id} | {item.status} | {item.message.replace("|", "\\|")} |' for item in results) + for item in results: + message = item.message.replace('|', '\\|') + lines.append(f'| {item.check_id} | {item.status} | {message} |') lines.extend([ '', 'These checks verify artifact existence and ordering only, not truth, adequacy, or safety sufficiency.', diff --git a/tools/change_control/coverage_delta.py b/tools/change_control/coverage_delta.py index d0b8f388..01cb0c69 100644 --- a/tools/change_control/coverage_delta.py +++ b/tools/change_control/coverage_delta.py @@ -137,7 +137,10 @@ def main(argv=None): report = compare_reports(report_at_revision(root, args.base), report_at_revision(root, args.head)) print(report) if not args.no_comment: - upsert_coverage_comment(GhApi(args.gh, args.repository), args.repository, args.pr, report) + try: + upsert_coverage_comment(GhApi(args.gh, args.repository), args.repository, args.pr, report) + except RuntimeError as error: + print(f'coverage-delta: comment publication warning: {error}', file=sys.stderr) return 0 except (OSError, RuntimeError, ValueError) as error: print(f'coverage-delta: cannot run: {error}', file=sys.stderr) diff --git a/tools/change_control/self_test.py b/tools/change_control/self_test.py index fa18df5e..01968c30 100755 --- a/tools/change_control/self_test.py +++ b/tools/change_control/self_test.py @@ -755,6 +755,14 @@ def test_no_comment_avoids_write_api(self): self.assertEqual(result.returncode, 0, result.stderr) self.assertNotIn('POST repos/acme/project/issues/7/comments', calls) + def test_warn_mode_survives_comment_permission_failure(self): + """Warn-mode findings remain visible in logs when GitHub denies advisory comment writes.""" + responses = self._responses() + responses.pop('POST repos/acme/project/issues/7/comments') + result = self._run_cli('warn', {'responses': responses}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('comment publication warning', result.stderr) + def _responses(self): data = snapshot() data['pr']['labels'] = [] From bbb4bebbd2ae3e259854439bd495e56e05a88869 Mon Sep 17 00:00:00 2001 From: Raj Madhivanan Date: Sun, 13 Sep 2026 17:54:17 -0700 Subject: [PATCH 04/12] fix: keep generated coverage as the only numeric source The traceability summary currently contains coverage figures outside the generated regions. Those figures can drift independently and can claim that tests pass when the tooling has only resolved their citations. This adds a check that fails on competing figures and records the existing conflict until its separately authorized document correction lands. ## What changed - Added an error check for numeric requirements and function coverage claims outside generated section 3 markers. - Added an exact baseline entry owned by Raj for the current duplicate headline and footnote claims. - Made removed legacy claims valid while retaining mismatch detection for claims that remain. - Made stale --check output name python3 -m tools.safety_lint --write. - Documented manual regeneration and its citation-only limitation in CONTRIBUTING.md. - Added adversarial and drift tests for marker ownership, identifiers, later sections, and structural coverage exclusions. ## Safety lifecycle Verification and traceability. Bears on IEC 61508-3:2010 Annex A.8.7 and A.8.8, and IEC 61508-1:2010 section 7.18.2. Co-Authored-By: OpenCode --- CONTRIBUTING.md | 9 +++ docs/safety/lint-baseline.json | 7 +++ tools/safety_lint/__main__.py | 11 +++- tools/safety_lint/checks.py | 55 +++++++++++++++- tools/safety_lint/self_test.py | 111 +++++++++++++++++++++++++++++++-- 5 files changed, 185 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f584d605..4fb74e71 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -100,6 +100,15 @@ Note `pstop_c/` is intentionally excluded from the C/C++ hooks. - **No changes to `pstop_c/`** (contribute upstream instead). - **Clear commits.** Explain the design intent, not just the diff. Reference the relevant `docs/` design note where one applies. +- **Keep safety traceability lint clean.** The linter checks requirement and + function mappings, statuses, evidence citations, and ownership of numeric + coverage claims. Generated coverage becomes stale whenever document + citations or statuses change; refresh it with + `python3 -m tools.safety_lint --write`. This command recounts citations and + statuses from the documents; it does not execute tests or establish that + cited tests pass. Write mode refuses to modify the document while + unbaselined errors exist. Automatic pre-commit rewriting is deliberately not + configured because coverage drops require human review. Contributions are licensed according to where they land: software and firmware under Apache-2.0, hardware design files under CERN-OHL-P-2.0, and documentation diff --git a/docs/safety/lint-baseline.json b/docs/safety/lint-baseline.json index 2e29b797..638c3b8a 100644 --- a/docs/safety/lint-baseline.json +++ b/docs/safety/lint-baseline.json @@ -2,6 +2,13 @@ "generated": "2026-09-11", "note": "Pre-existing findings accepted at linter introduction. Each entry needs an exact finding message, owner, and reason. Removing a fixed entry is required; CI fails on a stale entry.", "findings": [ + { + "check_id": "C10", + "subject": "section 3 outside generated regions", + "finding": "numeric coverage claims outside generated regions: [32 / 40 = 80.0 %, 17 Verified, 14 Partially-verified, 1 Residual-with-test, 7 Unverified-gap SRs, 17 / 40 = 42.5 %, 14 Partials, 22 / 27 = 81.5 %, 22 / 25 = 88.0 %, 5 / 6 = 83.3 %]", + "reason": "TRACEABILITY.md section 3 retains superseded numeric headline and footnote claims outside generated regions. Owner-approved Class C reconciliation is pending; remove this baseline entry when that edit lands.", + "owner": "raj" + }, { "check_id": "C4", "subject": "SR-M-01", diff --git a/tools/safety_lint/__main__.py b/tools/safety_lint/__main__.py index a4a3f3e9..b0eec117 100644 --- a/tools/safety_lint/__main__.py +++ b/tools/safety_lint/__main__.py @@ -7,7 +7,7 @@ import sys from pathlib import Path -from .checks import apply_baseline, check_summary_prose, run_checks +from .checks import apply_baseline, check_numeric_coverage_claims, check_summary_prose, run_checks from .coverage import compute_coverage from .model import LintError from .render import render_traceability @@ -63,7 +63,10 @@ def main(argv=None): coverage = compute_coverage(analysis) trace_path = root / 'docs/safety/TRACEABILITY.md' original = trace_path.read_text(encoding='utf-8') - active, suppressed = apply_baseline(run_checks(analysis) + check_summary_prose(original, coverage), baseline) + active, suppressed = apply_baseline( + run_checks(analysis) + check_numeric_coverage_claims(original) + check_summary_prose(original, coverage), + baseline, + ) rendered = render_traceability(original, coverage) stale = rendered != original active_errors = [finding for finding in active if finding.severity == 'error'] @@ -100,7 +103,9 @@ def main(argv=None): for area, data in coverage.areas.items(): print(f' SR-{area}: {data["count"]} total, {data["cited"]} cited, {data["Verified"]} Verified') if args.check and stale: - print('docs/safety/TRACEABILITY.md: generated regions are stale') + print( + 'docs/safety/TRACEABILITY.md: generated regions are stale; run python3 -m tools.safety_lint --write' + ) failed = bool(active_errors) or (args.check and stale) return 1 if failed else 0 except (LintError, OSError) as error: diff --git a/tools/safety_lint/checks.py b/tools/safety_lint/checks.py index 876dc471..6c80c5a7 100644 --- a/tools/safety_lint/checks.py +++ b/tools/safety_lint/checks.py @@ -10,6 +10,21 @@ SRS_STATUSES = ('Partially satisfied', 'Residual-accepted', 'Satisfied', 'Gap') TRACE_STATUSES = ('Partially-verified', 'Residual-accepted', 'Unverified-gap', 'Verified') +_GENERATED_REGION = re.compile( + r'.*?' + r'', + re.DOTALL, +) +_COVERAGE_CLAIM = re.compile( + r'\d+\s*/\s*\d+\s*=\s*~?\d+(?:\.\d+)?\s*%' + r'|\d+\s*/\s*\d+' + r'|~?\d+(?:\.\d+)?\s*%' + r'|\d+\s+(?:Partially-verified|Residual-with-test|Residual-accepted|' + r'Unverified-gap(?:\s+SRs?)?|Verified|Partials?)\b', + re.IGNORECASE, +) +_COMPACT_SAFETY_ID = re.compile(r'\b(?:SR-[A-Z]+-\d+(?:/\d+)*|DU-\d+(?:/\d+)*)\b') + def _finding(check, severity, subject, message, file, line=1): return Finding(check, severity, subject, message, file, line) @@ -237,7 +252,9 @@ def check_summary_prose(text, coverage): findings = [] for pattern, wanted, subject in expected: match = re.search(pattern, text, re.IGNORECASE) - actual = tuple(map(int, match.groups())) if match else None + if match is None: + continue + actual = tuple(map(int, match.groups())) if actual != wanted: findings.append( _finding( @@ -251,6 +268,42 @@ def check_summary_prose(text, coverage): return tuple(findings) +def check_numeric_coverage_claims(text): + """Reject numeric section-3 coverage claims outside generated regions.""" + heading = re.search(r'^## 3\. Requirements coverage summary\s*$', text, re.MULTILINE) + if heading is None: + return () + remainder = text[heading.end() :] + boundaries = [ + match.start() + for pattern in (r'^\*\*Reading:\*\*', r'^## (?!3\.)') + if (match := re.search(pattern, remainder, re.MULTILINE)) is not None + ] + end = heading.end() + min(boundaries) if boundaries else len(text) + section = text[heading.start() : end] + section = _GENERATED_REGION.sub('', section) + section = _COMPACT_SAFETY_ID.sub(lambda match: ' ' * len(match.group()), section) + + claims = [] + for match in _COVERAGE_CLAIM.finditer(section): + claim = re.sub(r'\s*/\s*', ' / ', match.group()) + claim = re.sub(r'\s*=\s*', ' = ', claim) + claim = re.sub(r'\s*%', ' %', claim) + claims.append(' '.join(claim.split())) + if not claims: + return () + return ( + _finding( + 'C10', + 'error', + 'section 3 outside generated regions', + f'numeric coverage claims outside generated regions: [{", ".join(claims)}]', + 'docs/safety/TRACEABILITY.md', + text.count('\n', 0, heading.start()) + 1, + ), + ) + + def apply_baseline(findings, baseline): active = [] suppressed = [] diff --git a/tools/safety_lint/self_test.py b/tools/safety_lint/self_test.py index 2590de57..1533ad52 100755 --- a/tools/safety_lint/self_test.py +++ b/tools/safety_lint/self_test.py @@ -21,6 +21,7 @@ SRS_STATUSES, TRACE_STATUSES, apply_baseline, + check_numeric_coverage_claims, check_summary_prose, run_checks, ) @@ -40,6 +41,14 @@ def setUp(self): self.temp = tempfile.TemporaryDirectory() self.root = Path(self.temp.name) shutil.copytree(FIXTURE, self.root, dirs_exist_ok=True) + self.replace( + 'docs/safety/TRACEABILITY.md', + '- **(a) SRs with ≥1 passing verifying test: 2 / 2 = 100 %**\n' + '- **Strict, fully-verified only: 1 / 2 = 50.0 %.**\n' + '- **(b) Safety functions F-xx traced to ≥1 SR: 1 / 1 = 100 %.**\n' + ' Excluding the two declared-non-safety functions: 1 / 1 = 100 %.\n', + 'Coverage values are owned by the generated regions above.\n', + ) def tearDown(self): self.temp.cleanup() @@ -368,6 +377,89 @@ def test_undocumented_srs_status_is_informational(self): self.assertTrue([f for f in findings if f.check_id == 'C2' and f.severity == 'info']) +class NumericCoverageOwnershipTests(unittest.TestCase): + def test_current_section_emits_one_exact_c10_finding(self): + """Today's outside-marker claims aggregate into one exact baseline discriminator.""" + text = (REPO / 'docs/safety/TRACEABILITY.md').read_text(encoding='utf-8') + findings = check_numeric_coverage_claims(text) + self.assertEqual(len(findings), 1) + self.assertEqual( + (findings[0].check_id, findings[0].severity, findings[0].subject, findings[0].message), + ( + 'C10', + 'error', + 'section 3 outside generated regions', + 'numeric coverage claims outside generated regions: ' + '[32 / 40 = 80.0 %, 17 Verified, 14 Partially-verified, ' + '1 Residual-with-test, 7 Unverified-gap SRs, 17 / 40 = 42.5 %, ' + '14 Partials, 22 / 27 = 81.5 %, 22 / 25 = 88.0 %, 5 / 6 = 83.3 %]', + ), + ) + + def test_marker_bounded_numeric_claims_are_ignored(self): + """Generated coverage regions exclusively own every numeric claim they contain.""" + text = """## 3. Requirements coverage summary + +32 / 40 = 80.0 %; 17 Verified; 14 Partially-verified + + +22 / 27 = 81.5 % + +**Reading:** 56.8 % branch coverage +""" + self.assertEqual(check_numeric_coverage_claims(text), ()) + + def test_new_outside_claim_changes_exact_message(self): + """Any added outside-marker ratio or percentage breaks an exact C10 baseline.""" + base = """## 3. Requirements coverage summary +Legacy coverage: 2 / 3 = 66.7 %. +**Reading:** details +""" + changed = base.replace('**Reading:**', 'Another claim: 75 %.\n**Reading:**') + before = check_numeric_coverage_claims(base)[0] + after = check_numeric_coverage_claims(changed)[0] + self.assertEqual(before.subject, after.subject) + self.assertNotEqual(before.message, after.message) + self.assertIn('75 %', after.message) + + def test_identifiers_date_and_reconciliation_delta_do_not_trigger(self): + """Compact IDs, dates, and reconciliation deltas are not coverage values.""" + text = """## 3. Requirements coverage summary +Reconciled 2026-08-07: +4 for SR-M-01/03/05 and DU-1/2/3/4. +**Reading:** details +""" + self.assertEqual(check_numeric_coverage_claims(text), ()) + + def test_structural_percentages_after_reading_do_not_trigger(self): + """Structural coverage in Reading is outside C10's requirements-summary scope.""" + text = """## 3. Requirements coverage summary +Coverage details are generated above. +**Reading:** MC-DC 100 %, branch 56.8 %, line ~89 %. +""" + self.assertEqual(check_numeric_coverage_claims(text), ()) + + def test_later_sections_are_not_scanned_without_reading_paragraph(self): + """Coverage-like numbers in section 4 cannot become section-3 ownership findings.""" + text = """## 3. Requirements coverage summary +Coverage details are generated above. +## 4. Test-gap register +Historical result: 4 / 5 = 80 %. +""" + self.assertEqual(check_numeric_coverage_claims(text), ()) + + def test_clean_pointer_and_reconciliation_section_has_no_finding(self): + """Pointers and nonnumeric history may remain outside generated coverage regions.""" + text = """## 3. Requirements coverage summary + +32 / 40 = 80.0 % + +See the generated headline and area table. Reconciled 2026-08-07: +4 for +SR-M-01/03/05 and DU-1/2/3/4 after evidence review. +**Reading:** structural coverage is discussed here at 100 %. +""" + self.assertEqual(check_numeric_coverage_claims(text), ()) + + class CoverageRenderCliTests(FixtureRepo): def test_coverage_matches_committed_summary(self): """Real citation and status counts reproduce the ratified committed summary.""" @@ -393,6 +485,14 @@ def test_check_mode_detects_stale_block(self): proc = self.run_cli('--check') self.assertEqual(proc.returncode, 1) + def test_stale_check_names_exact_write_remedy(self): + """Stale check output gives the exact command that refreshes generated regions.""" + proc = self.run_cli('--check') + self.assertIn( + 'docs/safety/TRACEABILITY.md: generated regions are stale; run python3 -m tools.safety_lint --write', + proc.stdout, + ) + def test_render_does_not_touch_prose_outside_markers(self): """Rendering preserves hand-authored Reading prose byte-for-byte.""" path = self.root / 'docs/safety/TRACEABILITY.md' @@ -409,11 +509,14 @@ def test_mixed_numeric_prose_is_checked_not_generated(self): stale = text.replace('22 / 27 = 81.5 %', '21 / 27 = 77.8 %') self.assertTrue(check_summary_prose(stale, compute_coverage(result))) + def test_missing_legacy_summary_claims_do_not_trigger_summary(self): + """Removing superseded hand-authored headlines does not create SUMMARY errors.""" + coverage = compute_coverage(analyze(self.root)) + self.assertEqual(check_summary_prose('No hand-authored numeric coverage claims.\n', coverage), ()) + def test_cli_exit_code_zero_on_clean_tree(self): - """The real repository passes with its checked-in baseline and current generated regions.""" - proc = subprocess.run( - [sys.executable, '-m', 'tools.safety_lint'], cwd=REPO, capture_output=True, text=True, check=False - ) + """A repository with no active or stale findings returns the clean exit code.""" + proc = self.run_cli() self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) def test_cli_exit_code_one_on_injected_error(self): From 93dd1a522a9bef5580602ba81ac4d96e7857e8c3 Mon Sep 17 00:00:00 2001 From: Raj Madhivanan Date: Sun, 13 Sep 2026 18:00:11 -0700 Subject: [PATCH 05/12] fix: preserve the comparison symbol in generated coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The coverage table generator replaced the document's greater-than-or-equal symbol with two plain characters. Regenerating the table then created an avoidable formatting drift inside the generated region. This keeps the symbol consistent and pins that behavior with a regression test. ## What changed - Updated tools/safety_lint/render.py to emit ≥1 cited test %. - Ran python3 -m tools.safety_lint --write. - Regenerated only the marker-owned table header in docs/safety/TRACEABILITY.md. - Added a test that rejects the ASCII >= spelling in rendered output. - Confirmed the exact C10 baseline discriminator remains unchanged. ## Safety lifecycle Verification record generation and traceability presentation. Bears on IEC 61508-3:2010 Annex A.8.7 and A.8.8, and IEC 61508-1:2010 section 7.18.2. Co-Authored-By: OpenCode --- docs/safety/TRACEABILITY.md | 2 +- tools/safety_lint/render.py | 2 +- tools/safety_lint/self_test.py | 7 +++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/safety/TRACEABILITY.md b/docs/safety/TRACEABILITY.md index 5b9b5734..ee04fbea 100644 --- a/docs/safety/TRACEABILITY.md +++ b/docs/safety/TRACEABILITY.md @@ -156,7 +156,7 @@ Test-file shorthand: ### 3.2 Breakdown by area -| Area | Count | Verified | Partially-verified | Unverified-gap | Residual-accepted | >=1 cited test % | Fully-verified % | +| Area | Count | Verified | Partially-verified | Unverified-gap | Residual-accepted | ≥1 cited test % | Fully-verified % | |---|---|---|---|---|---|---|---| | SR-SYS | 9 | 2 | 6 | 1 | 0 | 88.9 % | 22.2 % | | SR-R | 15 | 6 | 3 | 6 | 0 | 60.0 % | 40.0 % | diff --git a/tools/safety_lint/render.py b/tools/safety_lint/render.py index 71858e28..ab88a11b 100644 --- a/tools/safety_lint/render.py +++ b/tools/safety_lint/render.py @@ -34,7 +34,7 @@ def render_traceability(text, coverage): '‡ Citation resolution, not test execution or passing state, is checked by the linter.' ) lines = [ - '| Area | Count | Verified | Partially-verified | Unverified-gap | Residual-accepted | >=1 cited test % | Fully-verified % |', + '| Area | Count | Verified | Partially-verified | Unverified-gap | Residual-accepted | ≥1 cited test % | Fully-verified % |', '|---|---|---|---|---|---|---|---|', ] for area in ('SYS', 'R', 'H', 'M', 'I'): diff --git a/tools/safety_lint/self_test.py b/tools/safety_lint/self_test.py index 1533ad52..08df98a4 100755 --- a/tools/safety_lint/self_test.py +++ b/tools/safety_lint/self_test.py @@ -502,6 +502,13 @@ def test_render_does_not_touch_prose_outside_markers(self): )[1] self.assertEqual(after, before) + def test_renderer_uses_unicode_greater_than_or_equal(self): + """The generated area heading preserves the document's Unicode comparison symbol.""" + text = (self.root / 'docs/safety/TRACEABILITY.md').read_text(encoding='utf-8') + rendered = render_traceability(text, compute_coverage(analyze(self.root))) + self.assertIn('≥1 cited test %', rendered) + self.assertNotIn('>=', rendered) + def test_mixed_numeric_prose_is_checked_not_generated(self): """A stale number outside generated markers remains a check failure.""" result = analyze(REPO) From dac1e14540a07e2dd1742cdd604562c979036fb2 Mon Sep 17 00:00:00 2001 From: Raj Madhivanan Date: Sun, 13 Sep 2026 18:08:16 -0700 Subject: [PATCH 06/12] docs: remove competing coverage claims from the summary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The traceability summary repeated coverage figures outside the generated regions and described resolvable citations as passing tests. Those statements could diverge while each still looked authoritative. This removes the competing figures, keeps the historical reconciliation note, and points readers to the generated source. ## What changed - Reduced section 3 bullets to pointers and retained the 2026-08-07 reconciliation and closure notes verbatim. - Reconciled the SR-M footnote with the generated ≥1 cited test % column. - Removed the now-stale C10 baseline entry. - Added Partially satisfied to the requirements conventions status list. - Updated tests to assert the reconciled real section is clean and injected outside-marker claims still fail. - Confirmed --write leaves both generated regions byte-identical. ## Safety lifecycle Safety requirements documentation, verification records, and forward and backward traceability. Bears on IEC 61508-3:2010 sections 7.2.2 and 7.9, Annex A.8.7 and A.8.8, and IEC 61508-1:2010 section 7.18.2. Co-Authored-By: OpenCode --- docs/safety/SAFETY_REQUIREMENTS.md | 1 + docs/safety/TRACEABILITY.md | 24 ++++++++---------------- docs/safety/lint-baseline.json | 7 ------- tools/safety_lint/self_test.py | 28 +++++++--------------------- 4 files changed, 16 insertions(+), 44 deletions(-) diff --git a/docs/safety/SAFETY_REQUIREMENTS.md b/docs/safety/SAFETY_REQUIREMENTS.md index 56501b6c..7fb43101 100644 --- a/docs/safety/SAFETY_REQUIREMENTS.md +++ b/docs/safety/SAFETY_REQUIREMENTS.md @@ -48,6 +48,7 @@ rationale) · **Verification** (Test / Fault-injection / Analysis / Inspection) - **Satisfied** — implemented; cited `file:line` (line numbers per the analyzed tree, ±a few lines as the tree evolves). +- **Partially satisfied** - **Gap** — no implementation yet. - **Residual-accepted** — deliberately not implemented; rationale given. diff --git a/docs/safety/TRACEABILITY.md b/docs/safety/TRACEABILITY.md index ee04fbea..835a967e 100644 --- a/docs/safety/TRACEABILITY.md +++ b/docs/safety/TRACEABILITY.md @@ -136,22 +136,13 @@ Test-file shorthand: ‡ Citation resolution, not test execution or passing state, is checked by the linter. -- **(a) SRs with ≥1 passing verifying test: 32 / 40 = 80.0 %** [Reconciled +- **(a) Requirements coverage:** See the generated citation and fully-verified figures above. [Reconciled 2026-08-07: +4 as DU-1/2/3/4 closures gained tests — SR-H-03/SR-H-04 now Verified, SR-R-03/SR-R-09 now Partially-verified]. - (17 Verified + 14 Partially-verified + 1 Residual-with-test [SR-M-06]. The - remaining Residual [SR-M-04] is inspection-only; the 7 Unverified-gap SRs - have no test.) - **Strict, fully-verified only: 17 / 40 = 42.5 %.** This is the honest number - for "requirement completely discharged by test" — the 14 Partials each leave a - named leg (end-to-end, quantification, golden-vector/replay, fault-injection - divergence, or the operator-list config plumbing) untested. - -- **(b) Safety functions F-xx traced to ≥1 SR: 22 / 27 = 81.5 %.** - Five functions carry **no** requirement (§5): F-R-08, F-R-10 (both declared - **non-safety**), and **F-H-04, F-M-01, F-M-06** (undeclared — genuine - requirements-coverage holes). Excluding the two declared-non-safety functions: - 22 / 25 = 88.0 %. + +- **(b) Function traceability:** See the generated figures above. Section 5 records + functions with no requirement and distinguishes declared non-safety functions + from genuine requirements-coverage holes. ### 3.2 Breakdown by area @@ -166,8 +157,9 @@ Test-file shorthand: | **Total** | **40** | **17** | **14** | **7** | **2** | **80.0 %** | **42.5 %** | -† SR-M ≥1-test counts SR-M-01/03/05 (Verified) + SR-M-02 (Partial) + SR-M-06 -(Residual-with-test) = 5/6 = 83.3 % (SR-M-01/03 verified 2026-08-02). +† The SR-M cited-test percentage includes SR-M-06 as Residual-with-test; see the +generated SR-M row above. Citation resolution does not establish test execution or +passing state. **Reading:** SR-I is fully *touched* by the pre-qualified `pstop_c` suite but never *completed* (golden-vector + replay integration missing). **SR-R is the diff --git a/docs/safety/lint-baseline.json b/docs/safety/lint-baseline.json index 638c3b8a..2e29b797 100644 --- a/docs/safety/lint-baseline.json +++ b/docs/safety/lint-baseline.json @@ -2,13 +2,6 @@ "generated": "2026-09-11", "note": "Pre-existing findings accepted at linter introduction. Each entry needs an exact finding message, owner, and reason. Removing a fixed entry is required; CI fails on a stale entry.", "findings": [ - { - "check_id": "C10", - "subject": "section 3 outside generated regions", - "finding": "numeric coverage claims outside generated regions: [32 / 40 = 80.0 %, 17 Verified, 14 Partially-verified, 1 Residual-with-test, 7 Unverified-gap SRs, 17 / 40 = 42.5 %, 14 Partials, 22 / 27 = 81.5 %, 22 / 25 = 88.0 %, 5 / 6 = 83.3 %]", - "reason": "TRACEABILITY.md section 3 retains superseded numeric headline and footnote claims outside generated regions. Owner-approved Class C reconciliation is pending; remove this baseline entry when that edit lands.", - "owner": "raj" - }, { "check_id": "C4", "subject": "SR-M-01", diff --git a/tools/safety_lint/self_test.py b/tools/safety_lint/self_test.py index 08df98a4..eb6e252d 100755 --- a/tools/safety_lint/self_test.py +++ b/tools/safety_lint/self_test.py @@ -378,23 +378,10 @@ def test_undocumented_srs_status_is_informational(self): class NumericCoverageOwnershipTests(unittest.TestCase): - def test_current_section_emits_one_exact_c10_finding(self): - """Today's outside-marker claims aggregate into one exact baseline discriminator.""" + def test_reconciled_real_section_has_no_c10_finding(self): + """The reconciled real summary keeps all numeric requirements coverage inside generated regions.""" text = (REPO / 'docs/safety/TRACEABILITY.md').read_text(encoding='utf-8') - findings = check_numeric_coverage_claims(text) - self.assertEqual(len(findings), 1) - self.assertEqual( - (findings[0].check_id, findings[0].severity, findings[0].subject, findings[0].message), - ( - 'C10', - 'error', - 'section 3 outside generated regions', - 'numeric coverage claims outside generated regions: ' - '[32 / 40 = 80.0 %, 17 Verified, 14 Partially-verified, ' - '1 Residual-with-test, 7 Unverified-gap SRs, 17 / 40 = 42.5 %, ' - '14 Partials, 22 / 27 = 81.5 %, 22 / 25 = 88.0 %, 5 / 6 = 83.3 %]', - ), - ) + self.assertEqual(check_numeric_coverage_claims(text), ()) def test_marker_bounded_numeric_claims_are_ignored(self): """Generated coverage regions exclusively own every numeric claim they contain.""" @@ -509,12 +496,11 @@ def test_renderer_uses_unicode_greater_than_or_equal(self): self.assertIn('≥1 cited test %', rendered) self.assertNotIn('>=', rendered) - def test_mixed_numeric_prose_is_checked_not_generated(self): - """A stale number outside generated markers remains a check failure.""" - result = analyze(REPO) + def test_new_numeric_prose_outside_markers_is_a_c10_failure(self): + """A newly introduced numeric claim outside generated markers remains a check failure.""" text = (REPO / 'docs/safety/TRACEABILITY.md').read_text(encoding='utf-8') - stale = text.replace('22 / 27 = 81.5 %', '21 / 27 = 77.8 %') - self.assertTrue(check_summary_prose(stale, compute_coverage(result))) + stale = text.replace('**Reading:**', 'Legacy claim: 21 / 27 = 77.8 %.\n\n**Reading:**') + self.assertTrue(check_numeric_coverage_claims(stale)) def test_missing_legacy_summary_claims_do_not_trigger_summary(self): """Removing superseded hand-authored headlines does not create SUMMARY errors.""" From 67234f592f9c505bb2d5b04ada28614a648c0978 Mon Sep 17 00:00:00 2001 From: Raj Madhivanan Date: Sun, 13 Sep 2026 18:14:30 -0700 Subject: [PATCH 07/12] docs: restore the meaning behind partial verification The numeric cleanup also removed an explanation of what incomplete verification means in practice. This restores that qualitative guidance without reintroducing counts or percentages, so the generated regions remain the sole owner of quantities. ## What changed - Restored the named untested legs beneath the requirements-coverage reconciliation note. - Clarified that the residual requirement is inspection-only and unverified gaps have no test. - Added no numeric coverage claim; C10 remains clean. ## Safety lifecycle Verification records and requirements traceability. Bears on IEC 61508-3:2010 sections 7.9 and 7.10, Annex A.8.7 and A.8.8, and IEC 61508-1:2010 section 7.18.2. Co-Authored-By: OpenCode --- docs/safety/TRACEABILITY.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/safety/TRACEABILITY.md b/docs/safety/TRACEABILITY.md index 835a967e..8a8915ba 100644 --- a/docs/safety/TRACEABILITY.md +++ b/docs/safety/TRACEABILITY.md @@ -139,6 +139,10 @@ Test-file shorthand: - **(a) Requirements coverage:** See the generated citation and fully-verified figures above. [Reconciled 2026-08-07: +4 as DU-1/2/3/4 closures gained tests — SR-H-03/SR-H-04 now Verified, SR-R-03/SR-R-09 now Partially-verified]. + The Partially-verified requirements each leave a named leg untested — end-to-end, + quantification, golden-vector/replay, fault-injection divergence, or the + operator-list config plumbing. SR-M-04 is Residual and inspection-only; the + Unverified-gap requirements have no test at all. - **(b) Function traceability:** See the generated figures above. Section 5 records functions with no requirement and distinguishes declared non-safety functions From d399b0d267f4f5270343a8be5f5f9f3c3b36feec Mon Sep 17 00:00:00 2001 From: Raj Madhivanan Date: Sun, 13 Sep 2026 19:35:06 -0700 Subject: [PATCH 08/12] fix: reject malformed traceability inputs instead of guessing Several malformed document shapes could be silently truncated, overwritten, or mistaken for evidence. That could make the reported coverage look complete while parts of the source data had disappeared. This makes those cases fail visibly and moves every published coverage figure into generated regions. ## What changed - Expanded arbitrary allocation slash chains and rejected malformed or descending forms. - Rejected duplicate system-function and reverse-map identifiers with both source lines. - Gave requirement-side and trace-side C6 findings distinct baseline subjects. - Restricted evidence to test artifacts, requirement-naming reports, and scripts/check_*.sh guards; rejected paths now produce C4 findings. - Validated every baseline container, entry, field type, and nonblank value before use. - Generated both function-coverage denominators and removed the obsolete hand-prose checker. - Added contents: read as the sole safety-lint workflow permission. - Added exact baselines for two rejected supplemental citations while preserving 32/40 citation coverage. - Added adversarial tests for every fail-closed path. ## Safety lifecycle Verification evidence integrity and forward and backward traceability. Bears on IEC 61508-3:2010 sections 7.9 and 7.10, Annex A.8.7 and A.8.8, and IEC 61508-1:2010 section 7.18.2. Co-Authored-By: OpenCode --- .github/workflows/safety-lint.yml | 3 + docs/safety/TRACEABILITY.md | 2 + docs/safety/lint-baseline.json | 14 ++ tools/safety_lint/__main__.py | 20 +- tools/safety_lint/checks.py | 71 +++--- tools/safety_lint/parse_srs.py | 21 +- tools/safety_lint/parse_system_definition.py | 5 + tools/safety_lint/parse_traceability.py | 56 ++--- tools/safety_lint/render.py | 4 + tools/safety_lint/self_test.py | 244 ++++++++++++++++++- 10 files changed, 342 insertions(+), 98 deletions(-) diff --git a/.github/workflows/safety-lint.yml b/.github/workflows/safety-lint.yml index 2c99c27e..64f7fa09 100644 --- a/.github/workflows/safety-lint.yml +++ b/.github/workflows/safety-lint.yml @@ -6,6 +6,9 @@ on: branches: [main] pull_request: +permissions: + contents: read + jobs: safety_lint: runs-on: ubuntu-latest diff --git a/docs/safety/TRACEABILITY.md b/docs/safety/TRACEABILITY.md index 8a8915ba..d7a635ec 100644 --- a/docs/safety/TRACEABILITY.md +++ b/docs/safety/TRACEABILITY.md @@ -132,6 +132,8 @@ Test-file shorthand: - **SRs with at least one cited verifying test: 32 / 40 = 80.0 %**‡ - **Strict, fully-verified only: 17 / 40 = 42.5 %** +- **Functions traced to at least one SR: 22 / 27 = 81.5 %** +- **Functions traced excluding declared non-safety functions: 22 / 25 = 88.0 %** ‡ Citation resolution, not test execution or passing state, is checked by the linter. diff --git a/docs/safety/lint-baseline.json b/docs/safety/lint-baseline.json index 2e29b797..11de7cba 100644 --- a/docs/safety/lint-baseline.json +++ b/docs/safety/lint-baseline.json @@ -9,6 +9,20 @@ "reason": "TRACEABILITY.md section 2.4 cites test_timing_floors, but test_timing_floors.cpp was added by 1f226be and deleted by 9a28d4a during ROS2 convention restoration. This records broken citation resolution only and does not decide SR-M-01 verification status.", "owner": "raj" }, + { + "check_id": "C4", + "subject": "SR-R-01", + "finding": "estop_verdict.c: firmware/main/estop_verdict.c: path is not an approved evidence class (test artifact, SR-naming docs report, or scripts/check_*.sh guard)", + "reason": "TRACEABILITY.md cites estop_verdict.c as structural-coverage context, but production source is not verification evidence. SR-R-01 retains independently accepted EV test evidence; the hand-authored citation needs human disposition.", + "owner": "raj" + }, + { + "check_id": "C4", + "subject": "SR-R-03", + "finding": "firmware-build.yml: .github/workflows/firmware-build.yml: path is not an approved evidence class (test artifact, SR-naming docs report, or scripts/check_*.sh guard)", + "reason": "TRACEABILITY.md cites firmware-build.yml as CI wiring context, but a workflow is not verification evidence. SR-R-03 retains accepted scripts/check_estop_diversity.sh evidence; the hand-authored workflow citation needs human disposition.", + "owner": "raj" + }, { "check_id": "C8", "subject": "F-H-04", diff --git a/tools/safety_lint/__main__.py b/tools/safety_lint/__main__.py index b0eec117..f6dd1d2a 100644 --- a/tools/safety_lint/__main__.py +++ b/tools/safety_lint/__main__.py @@ -7,7 +7,7 @@ import sys from pathlib import Path -from .checks import apply_baseline, check_numeric_coverage_claims, check_summary_prose, run_checks +from .checks import apply_baseline, check_numeric_coverage_claims, run_checks from .coverage import compute_coverage from .model import LintError from .render import render_traceability @@ -19,11 +19,21 @@ def _load_baseline(path): raise LintError(f'required document missing: {path}') try: document = json.loads(path.read_text(encoding='utf-8')) + if not isinstance(document, dict): + raise LintError(f'invalid baseline {path}: top level must be an object') + if 'findings' not in document or not isinstance(document['findings'], list): + raise LintError(f'invalid baseline {path}: findings must be a list') entries = document['findings'] baseline = {} - for entry in entries: - if not entry.get('owner') or not entry.get('reason') or not entry.get('finding'): - raise LintError(f'baseline entry needs owner, reason, and exact finding: {entry}') + required = ('check_id', 'subject', 'finding', 'reason', 'owner') + for position, entry in enumerate(entries): + if not isinstance(entry, dict): + raise LintError(f'invalid baseline {path}: finding entry {position} must be an object') + invalid = [field for field in required if not isinstance(entry.get(field), str) or not entry[field].strip()] + if invalid: + raise LintError( + f'invalid baseline {path}: finding entry {position} requires nonblank strings: {", ".join(invalid)}' + ) key = (entry['check_id'], entry['subject'], entry['finding']) if key in baseline: raise LintError(f'duplicate exact baseline entry: {key}') @@ -64,7 +74,7 @@ def main(argv=None): trace_path = root / 'docs/safety/TRACEABILITY.md' original = trace_path.read_text(encoding='utf-8') active, suppressed = apply_baseline( - run_checks(analysis) + check_numeric_coverage_claims(original) + check_summary_prose(original, coverage), + run_checks(analysis) + check_numeric_coverage_claims(original), baseline, ) rendered = render_traceability(original, coverage) diff --git a/tools/safety_lint/checks.py b/tools/safety_lint/checks.py index 6c80c5a7..36c1fd9b 100644 --- a/tools/safety_lint/checks.py +++ b/tools/safety_lint/checks.py @@ -4,6 +4,7 @@ import re from collections import Counter, defaultdict +from pathlib import Path from .model import Finding @@ -30,6 +31,33 @@ def _finding(check, severity, subject, message, file, line=1): return Finding(check, severity, subject, message, file, line) +def evidence_rejection(root, path, sr_id): + """Return why an existing path is not one of the three approved evidence classes.""" + candidate = Path(path) + parts = tuple(part.lower() for part in candidate.parts) + stem = candidate.stem.lower() + + if candidate.name.lower() == 'readme.md': + return 'README files are not approved evidence reports' + + # 1. Test sources are identified by a test/requirements directory or source naming. + if {'test', 'tests', 'requirements'} & set(parts) or stem.startswith('test_') or stem.endswith('_test'): + return None + + # 2. A docs Markdown report must not be a README and must name this requirement. + if parts and parts[0] == 'docs' and candidate.suffix.lower() == '.md': + content = (Path(root) / candidate).read_text(encoding='utf-8', errors='replace') + if sr_id not in content: + return 'evidence report does not name cited SR' + return None + + # 3. Repository guard scripts are scripts/check_*.sh only. + if len(parts) == 2 and parts[0] == 'scripts' and candidate.name.startswith('check_') and candidate.suffix == '.sh': + return None + + return 'path is not an approved evidence class (test artifact, SR-naming docs report, or scripts/check_*.sh guard)' + + def run_checks(analysis): findings = [] srs_counts = Counter(row.sr_id for row in analysis.srs) @@ -138,7 +166,7 @@ def run_checks(analysis): _finding( 'C6', 'error', - function_id, + f'SRS:{function_id}', f'allocated by {row.sr_id} but absent from system definition', 'docs/safety/SAFETY_REQUIREMENTS.md', row.source_line, @@ -151,7 +179,7 @@ def run_checks(analysis): _finding( 'C6', 'error', - function_id, + f'TRACE:{function_id}', f'allocated by {row.sr_id} but absent from system definition', 'docs/safety/TRACEABILITY.md', row.source_line, @@ -229,45 +257,6 @@ def run_checks(analysis): return tuple(findings) -def check_summary_prose(text, coverage): - """Check numeric claims embedded in hand-authored prose without rewriting it.""" - expected = ( - ( - r'SRs with ≥1 passing verifying test:\s*(\d+)\s*/\s*(\d+)', - (coverage.cited_tests, coverage.total), - 'passing-test headline', - ), - (r'Strict, fully-verified only:\s*(\d+)\s*/\s*(\d+)', (coverage.verified, coverage.total), 'strict headline'), - ( - r'Safety functions F-xx traced to ≥1 SR:\s*(\d+)\s*/\s*(\d+)', - (coverage.functions_traced, coverage.functions_total), - 'function headline', - ), - ( - r'Excluding the two declared-non-safety functions:\s*(\d+)\s*/\s*(\d+)', - (coverage.functions_traced, coverage.safety_functions_total), - 'safety-function headline', - ), - ) - findings = [] - for pattern, wanted, subject in expected: - match = re.search(pattern, text, re.IGNORECASE) - if match is None: - continue - actual = tuple(map(int, match.groups())) - if actual != wanted: - findings.append( - _finding( - 'SUMMARY', - 'error', - subject, - f'committed prose value {actual} != citation-derived {wanted}; agreement does not verify execution', - 'docs/safety/TRACEABILITY.md', - ) - ) - return tuple(findings) - - def check_numeric_coverage_claims(text): """Reject numeric section-3 coverage claims outside generated regions.""" heading = re.search(r'^## 3\. Requirements coverage summary\s*$', text, re.MULTILINE) diff --git a/tools/safety_lint/parse_srs.py b/tools/safety_lint/parse_srs.py index 507f9686..5d02d819 100644 --- a/tools/safety_lint/parse_srs.py +++ b/tools/safety_lint/parse_srs.py @@ -61,20 +61,23 @@ def normalize_status(cell, vocabulary, path, line): raise LintError(f'{path}:{line}: unknown status cell {cell!r}') -def expand_allocations(cell): - """Expand F-X-01..03 and F-X-01/02 notation into complete IDs.""" +def expand_allocations(cell, path='', line=1): + """Expand compact ranges and arbitrary slash chains, rejecting truncation.""" result = [] - occupied = [] - pattern = re.compile(r'F-([A-Z])-([0-9]{2})(?:\.\.([0-9]{2})|/([0-9]{2}))?') + pattern = re.compile(r'F-([A-Z])-([0-9]{2})(?:\.\.([0-9]{2})|((?:/[0-9]{2})+))?') for match in pattern.finditer(cell): - area, first, end, alternate = match.groups() - occupied.append(match.span()) + area, first, end, alternates = match.groups() + trailing = cell[match.end() :] + if trailing.startswith(('/', '.')) and not trailing.startswith('/F-'): + literal = re.match(r'[^\s,|)]+', cell[match.start() :]).group() + raise LintError(f'{path}:{line}: malformed allocation {literal!r}') if end: + if int(end) < int(first): + raise LintError(f'{path}:{line}: descending allocation range F-{area}-{first}..{end}') result.extend(f'F-{area}-{number:02d}' for number in range(int(first), int(end) + 1)) else: result.append(f'F-{area}-{first}') - if alternate: - result.append(f'F-{area}-{alternate}') + result.extend(f'F-{area}-{number}' for number in (alternates or '').lstrip('/').split('/') if number) return tuple(dict.fromkeys(result)) @@ -117,7 +120,7 @@ def parse_srs(path): int(match.group(2)), cells[columns['Requirement (shall)']], derived, - expand_allocations(cells[columns['Allocated to']]), + expand_allocations(cells[columns['Allocated to']], path, line_number), cells[columns['Integrity']], verify, status, diff --git a/tools/safety_lint/parse_system_definition.py b/tools/safety_lint/parse_system_definition.py index 4450ad05..09d298d4 100644 --- a/tools/safety_lint/parse_system_definition.py +++ b/tools/safety_lint/parse_system_definition.py @@ -26,6 +26,11 @@ def parse_system_definition(path): cells = split_row(line) if len(cells) >= 2 and re.fullmatch(r'F-[A-Z]-\d{2}', cells[0].strip('*')): function_id = cells[0].strip('*') + if function_id in functions: + raise LintError( + f'{path}:{index + 1}: duplicate function ID {function_id}; ' + f'lines {functions[function_id].source_line} and {index + 1}' + ) functions[function_id] = Function(function_id, cells[1], index + 1) if not functions: raise LintError(f'{path}: no functions found in section 4') diff --git a/tools/safety_lint/parse_traceability.py b/tools/safety_lint/parse_traceability.py index b883441e..e4fcbcbb 100644 --- a/tools/safety_lint/parse_traceability.py +++ b/tools/safety_lint/parse_traceability.py @@ -7,7 +7,7 @@ from collections import defaultdict from pathlib import Path -from .checks import TRACE_STATUSES +from .checks import TRACE_STATUSES, evidence_rejection from .model import LintError, ResolutionIssue, ReverseEntry, TraceRow from .parse_srs import SR_RE, expand_allocations, normalize_status, split_row @@ -60,13 +60,21 @@ def _test_refs(root, sr_id, cell, line, index): ) ) + def accept(path, literal): + reason = evidence_rejection(root, path, sr_id) + if reason is None: + refs.append(path) + return + kind = 'report-does-not-name-sr' if reason == 'evidence report does not name cited SR' else 'rejected-evidence' + issues.append(ResolutionIssue(kind, sr_id, literal, f'{path}: {reason}', line, 'test')) + for match in re.finditer(r'\bHIL(10|20|30)((?:/(?:10|20|30))*)', evidence_cell): numbers = (match.group(1), *match.group(2).lstrip('/').split('/')) for number in filter(None, numbers): token = f'HIL{number}' path = SHORTHAND[token] if path in all_paths: - refs.append(path) + accept(path, token) else: issues.append( ResolutionIssue( @@ -78,7 +86,7 @@ def _test_refs(root, sr_id, cell, line, index): continue if re.search(rf'\b{token}(?:\[[A-Z/]+\])?\b', evidence_cell): if path in all_paths: - refs.append(path) + accept(path, token) else: issues.append( ResolutionIssue( @@ -89,7 +97,7 @@ def _test_refs(root, sr_id, cell, line, index): for number in match.group(1).split('/'): path = f'pstop_c/pstop/test/src/pstop/requirements/req_{number}_test.c' if path in all_paths: - refs.append(path) + accept(path, f'REQ {number}') else: issues.append( ResolutionIssue( @@ -102,7 +110,7 @@ def _test_refs(root, sr_id, cell, line, index): r'(? 1: issues.append( ResolutionIssue( @@ -152,22 +143,12 @@ def _test_refs(root, sr_id, cell, line, index): return tuple(dict.fromkeys(refs)), tuple(issues) -def _is_test_artifact(path): - """Return whether a bare candidate is independently recognizable as test evidence.""" - candidate = Path(path) - lower_parts = {part.lower() for part in candidate.parts} - stem = candidate.stem.lower() - if candidate.suffix.lower() == '.md': - return True - return stem.startswith('test_') or stem.endswith('_test') or bool(lower_parts & {'test', 'tests', 'requirements'}) - - def _looks_like_test_citation(literal): without_line = re.sub(r':\d+(?:-\d+)?$', '', literal) return ( ('/' in without_line and not without_line.startswith('/') and '.' in Path(without_line).name) or without_line.startswith('test_') - or without_line.endswith(('.c', '.cc', '.cpp', '.py', '.md')) + or without_line.endswith(('.c', '.cc', '.cpp', '.py', '.md', '.yaml', '.yml', '.sh')) ) @@ -255,7 +236,7 @@ def parse_traceability(root): rows.append( TraceRow( sr_id, - expand_allocations(cells[1]), + expand_allocations(cells[1], path, line_number), code_refs, test_refs, tuple(part.strip() for part in cells[-2].split('+') if part.strip()), @@ -270,6 +251,11 @@ def parse_traceability(root): else: function_id = cells[0].strip('*') sr_cell = cells[2] + if function_id in reverse: + raise LintError( + f'{path}:{line_number}: duplicate reverse-map function ID {function_id}; ' + f'lines {reverse[function_id].source_line} and {line_number}' + ) reverse[function_id] = ReverseEntry( function_id, cells[1], diff --git a/tools/safety_lint/render.py b/tools/safety_lint/render.py index ab88a11b..9bfe9d2a 100644 --- a/tools/safety_lint/render.py +++ b/tools/safety_lint/render.py @@ -30,6 +30,10 @@ def render_traceability(text, coverage): f'{_percent(coverage.cited_tests, coverage.total)}**‡\n' f'- **Strict, fully-verified only: {coverage.verified} / {coverage.total} = ' f'{_percent(coverage.verified, coverage.total)}**\n' + f'- **Functions traced to at least one SR: {coverage.functions_traced} / {coverage.functions_total} = ' + f'{_percent(coverage.functions_traced, coverage.functions_total)}**\n' + f'- **Functions traced excluding declared non-safety functions: {coverage.functions_traced} / ' + f'{coverage.safety_functions_total} = {_percent(coverage.functions_traced, coverage.safety_functions_total)}**\n' '\n' '‡ Citation resolution, not test execution or passing state, is checked by the linter.' ) diff --git a/tools/safety_lint/self_test.py b/tools/safety_lint/self_test.py index eb6e252d..37688f93 100755 --- a/tools/safety_lint/self_test.py +++ b/tools/safety_lint/self_test.py @@ -22,7 +22,6 @@ TRACE_STATUSES, apply_baseline, check_numeric_coverage_claims, - check_summary_prose, run_checks, ) from tools.safety_lint.coverage import compute_coverage # noqa: E402 @@ -89,6 +88,29 @@ def test_allocated_to_slash_expansion(self): """Compact slash allocations expand to complete function IDs.""" self.assertEqual(expand_allocations('F-M-03/04'), ('F-M-03', 'F-M-04')) + def test_allocated_to_arbitrary_slash_chain_expansion(self): + """Every member of an arbitrary compact slash chain becomes a complete function ID.""" + self.assertEqual( + expand_allocations('F-R-01/02/03'), + ('F-R-01', 'F-R-02', 'F-R-03'), + ) + + def test_descending_allocation_range_fails_with_source_location(self): + """A descending allocation range fails at its document location instead of becoming empty.""" + with self.assertRaisesRegex(LintError, r'doc.md:17:.*descending.*F-R-03\.\.01'): + expand_allocations('F-R-03..01', 'doc.md', 17) + + def test_malformed_trailing_slash_allocation_fails_with_source_location(self): + """A malformed trailing slash member fails at its document location instead of being truncated.""" + with self.assertRaisesRegex(LintError, r'doc.md:19:.*F-R-01/02/XX'): + expand_allocations('F-R-01/02/XX', 'doc.md', 19) + + def test_trace_parser_reports_malformed_allocation_row_location(self): + """A truncated trace allocation fails at the exact matrix row rather than yielding partial data.""" + self.replace('docs/safety/TRACEABILITY.md', '| SR-R-01 | F-R-01 |', '| SR-R-01 | F-R-01/02/XX |') + with self.assertRaisesRegex(LintError, r'TRACEABILITY\.md:11: malformed allocation'): + parse_traceability(self.root) + def test_status_longest_match_wins(self): """A partial SRS status is never inflated to a fully satisfied status.""" rows = parse_srs(self.root / 'docs/safety/SAFETY_REQUIREMENTS.md') @@ -165,6 +187,26 @@ def test_system_definition_function_set_nonempty(self): """The authoritative function decomposition yields a nonempty function set.""" self.assertTrue(parse_system_definition(REPO / 'docs/safety/SYSTEM_DEFINITION.md')) + def test_duplicate_system_function_ids_fail_with_both_lines(self): + """A duplicate authoritative function ID fails and identifies both defining lines.""" + self.replace( + 'docs/safety/SYSTEM_DEFINITION.md', + '| F-R-01 | Sense | code.c |', + '| F-R-01 | Sense | code.c |\n| F-R-01 | Duplicate | code.c |', + ) + with self.assertRaisesRegex(LintError, r'F-R-01.*lines 10 and 11'): + parse_system_definition(self.root / 'docs/safety/SYSTEM_DEFINITION.md') + + def test_duplicate_reverse_map_ids_fail_with_both_lines(self): + """A duplicate reverse-map function ID fails and identifies both defining lines.""" + self.replace( + 'docs/safety/TRACEABILITY.md', + '| F-R-01 | Sense | SR-SYS-01, SR-R-01 |', + '| F-R-01 | Sense | SR-SYS-01, SR-R-01 |\n| F-R-01 | Duplicate | SR-R-01 |', + ) + with self.assertRaisesRegex(LintError, r'F-R-01.*lines 26 and 27'): + parse_traceability(self.root) + def test_unique_repository_wide_stem_resolves(self): """A repository-wide unique test stem resolves without directory guessing.""" decoy = self.root / 'tools/safety_lint/fixtures/repository/tests/test_unique_probe.py' @@ -197,6 +239,93 @@ def test_hil_report_must_name_sr(self): _, _, issues = parse_traceability(self.root) self.assertTrue([i for i in issues if i.kind == 'report-does-not-name-sr']) + def test_docs_markdown_report_naming_sr_is_evidence(self): + """A non-README Markdown report under docs counts when its content names the cited SR.""" + (self.root / 'docs/evidence.md').write_text('# Evidence for SR-R-01\n', encoding='utf-8') + self.replace('docs/safety/TRACEABILITY.md', 'test_unique_probe.py', 'docs/evidence.md') + rows, _, issues = parse_traceability(self.root) + self.assertEqual(rows[1].test_refs, ('docs/evidence.md',)) + self.assertFalse([issue for issue in issues if issue.literal == 'docs/evidence.md']) + + def test_test_artifact_directory_and_source_naming_are_evidence(self): + """Both test-directory membership and test-source naming independently identify test artifacts.""" + (self.root / 'tests/probe.c').write_text('/* test */\n', encoding='utf-8') + (self.root / 'tools/probe_test.c').write_text('/* test */\n', encoding='utf-8') + self.replace( + 'docs/safety/TRACEABILITY.md', + 'test_unique_probe.py', + 'tests/probe.c, tools/probe_test.c', + ) + rows, _, issues = parse_traceability(self.root) + self.assertEqual(rows[1].test_refs, ('tests/probe.c', 'tools/probe_test.c')) + self.assertFalse([issue for issue in issues if issue.literal in {'tests/probe.c', 'tools/probe_test.c'}]) + + def test_readme_never_counts_as_evidence_even_under_tests(self): + """A README is never evidence even when its path contains a test-artifact segment.""" + (self.root / 'tests/README.md').write_text('SR-R-01\n', encoding='utf-8') + self.replace('docs/safety/TRACEABILITY.md', 'test_unique_probe.py', 'tests/README.md') + rows, _, issues = parse_traceability(self.root) + self.assertFalse(rows[1].test_refs) + self.assertTrue([issue for issue in issues if issue.literal == 'tests/README.md']) + + def test_check_script_is_evidence(self): + """An existing scripts/check_*.sh guard counts as verifying evidence.""" + scripts = self.root / 'scripts' + scripts.mkdir() + (scripts / 'check_guard.sh').write_text('#!/bin/sh\n', encoding='utf-8') + self.replace('docs/safety/TRACEABILITY.md', 'test_unique_probe.py', 'scripts/check_guard.sh') + rows, _, issues = parse_traceability(self.root) + self.assertEqual(rows[1].test_refs, ('scripts/check_guard.sh',)) + self.assertFalse([issue for issue in issues if issue.literal == 'scripts/check_guard.sh']) + + def test_explicit_non_evidence_paths_are_rejected_with_reason(self): + """Existing source, workflow, and README paths each produce a reasoned evidence rejection.""" + candidates = { + 'code.c': 'src/code.c', + 'workflow.yml': '.github/workflows/example.yml', + 'README.md': 'docs/README.md', + } + (self.root / 'src').mkdir() + (self.root / 'src/code.c').write_text('/* production */\n', encoding='utf-8') + (self.root / '.github/workflows').mkdir(parents=True) + (self.root / '.github/workflows/example.yml').write_text('name: example\n', encoding='utf-8') + (self.root / 'docs/README.md').write_text('SR-R-01\n', encoding='utf-8') + for label, citation in candidates.items(): + with self.subTest(label=label): + copy = self.root / 'docs/safety/TRACEABILITY.md' + original = copy.read_text(encoding='utf-8') + try: + copy.write_text(original.replace('test_unique_probe.py', citation), encoding='utf-8') + rows, _, issues = parse_traceability(self.root) + rejected = [ + issue for issue in issues if issue.literal == citation and issue.kind == 'rejected-evidence' + ] + self.assertFalse(rows[1].test_refs) + self.assertEqual(len(rejected), 1) + self.assertIn(citation, f'{rejected[0].literal}: {rejected[0].message}') + self.assertRegex( + rejected[0].message, + r'not an approved evidence class|README files are not approved', + ) + finally: + copy.write_text(original, encoding='utf-8') + + def test_rejected_evidence_becomes_c4_error(self): + """A rejected existing path is exposed as a C4 error rather than silently discarded.""" + source = self.root / 'src' + source.mkdir() + (source / 'implementation.c').write_text('/* production */\n', encoding='utf-8') + self.replace('docs/safety/TRACEABILITY.md', 'test_unique_probe.py', 'src/implementation.c') + findings = [finding for finding in self.findings() if finding.check_id == 'C4'] + self.assertTrue([ + finding + for finding in findings + if finding.subject == 'SR-R-01' + and finding.severity == 'error' + and 'src/implementation.c' in finding.message + and 'not an approved evidence class' in finding.message + ]) + def test_prose_does_not_infer_evidence(self): """Words describing a successful test never become a test-file citation.""" self.replace('docs/safety/TRACEABILITY.md', 'test_unique_probe.py', 'bench test passed 8/8') @@ -299,8 +428,35 @@ def test_c5_flags_missing_code_file(self): def test_c6_flags_unknown_function(self): """C6 rejects allocations outside the authoritative function decomposition.""" - self.replace('docs/safety/SAFETY_REQUIREMENTS.md', 'F-R-01 |', 'F-X-99 |') - self.assert_check('C6', 'F-X-99') + self.replace( + 'docs/safety/SAFETY_REQUIREMENTS.md', + '| **SR-R-01** | Remain fresh with `a|b`. | SG-1 | F-R-01 |', + '| **SR-R-01** | Remain fresh with `a|b`. | SG-1 | F-X-99 |', + ) + self.assert_check('C6', 'SRS:F-X-99') + + def test_c6_document_findings_are_independently_baselineable(self): + """SRS and trace allocation violations have distinct exact keys suppressible one at a time.""" + self.replace( + 'docs/safety/SAFETY_REQUIREMENTS.md', + '| **SR-R-01** | Remain fresh with `a|b`. | SG-1 | F-R-01 |', + '| **SR-R-01** | Remain fresh with `a|b`. | SG-1 | F-X-99 |', + ) + self.replace('docs/safety/TRACEABILITY.md', '| SR-R-01 | F-R-01 |', '| SR-R-01 | F-X-99 |') + findings = [finding for finding in self.findings() if finding.check_id == 'C6'] + self.assertEqual({finding.subject for finding in findings}, {'SRS:F-X-99', 'TRACE:F-X-99'}) + suppressed_key = next(finding for finding in findings if finding.subject == 'SRS:F-X-99') + active, suppressed = apply_baseline( + findings, + { + ('C6', suppressed_key.subject, suppressed_key.message): { + 'reason': 'fixture', + 'owner': 'test', + } + }, + ) + self.assertEqual([finding.subject for finding in suppressed], ['SRS:F-X-99']) + self.assertEqual([finding.subject for finding in active], ['TRACE:F-X-99']) def test_c7_flags_unknown_upstream_reference(self): """C7 warns when a requirement cites an absent hazard, goal, or DU identifier.""" @@ -371,6 +527,48 @@ def test_duplicate_exact_baseline_entries_are_rejected(self): with self.assertRaises(LintError): _load_baseline(path) + def test_baseline_top_level_must_be_object(self): + """A baseline with a list at top level fails as malformed input.""" + self.assert_invalid_baseline([]) + + def test_baseline_findings_must_be_list(self): + """A baseline findings member must be a list rather than an iterable scalar or object.""" + self.assert_invalid_baseline({'findings': {}}) + + def test_baseline_entry_must_be_object(self): + """Every baseline findings entry must be an object.""" + self.assert_invalid_baseline({'findings': ['bad']}) + + def test_baseline_entry_requires_every_field(self): + """Every baseline entry requires all five exact-key and justification fields.""" + self.assert_invalid_baseline({'findings': [{'check_id': 'C4'}]}) + + def test_baseline_entry_fields_must_be_strings(self): + """Every required baseline field must be a string.""" + self.assert_invalid_baseline({'findings': [self.baseline_entry(owner=7)]}) + + def test_baseline_entry_fields_must_be_nonblank(self): + """Whitespace-only required baseline fields are rejected.""" + self.assert_invalid_baseline({'findings': [self.baseline_entry(reason=' ')]}) + + def assert_invalid_baseline(self, document): + path = self.root / 'invalid-baseline.json' + path.write_text(json.dumps(document), encoding='utf-8') + with self.assertRaises(LintError): + _load_baseline(path) + + @staticmethod + def baseline_entry(**changes): + entry = { + 'check_id': 'C4', + 'subject': 'SR-R-01', + 'finding': 'missing fixture', + 'reason': 'fixture reason', + 'owner': 'test', + } + entry.update(changes) + return entry + def test_undocumented_srs_status_is_informational(self): """A valid status used by requirements but omitted from conventions is informationally visible.""" findings = self.findings() @@ -396,6 +594,20 @@ def test_marker_bounded_numeric_claims_are_ignored(self): """ self.assertEqual(check_numeric_coverage_claims(text), ()) + def test_c10_catches_every_legacy_headline_numeric_pattern(self): + """C10 catches cited, strict, all-function, and safety-function legacy totals outside markers.""" + text = """## 3. Requirements coverage summary +SRs with ≥1 passing verifying test: 32 / 40 +Strict, fully-verified only: 17 / 40 +Safety functions F-xx traced to ≥1 SR: 22 / 27 +Excluding the two declared-non-safety functions: 22 / 25 +**Reading:** details +""" + message = check_numeric_coverage_claims(text)[0].message + for claim in ('32 / 40', '17 / 40', '22 / 27', '22 / 25'): + with self.subTest(claim=claim): + self.assertIn(claim, message) + def test_new_outside_claim_changes_exact_message(self): """Any added outside-marker ratio or percentage breaks an exact C10 baseline.""" base = """## 3. Requirements coverage summary @@ -496,17 +708,21 @@ def test_renderer_uses_unicode_greater_than_or_equal(self): self.assertIn('≥1 cited test %', rendered) self.assertNotIn('>=', rendered) + def test_generated_headline_reports_all_and_safety_function_totals(self): + """The generated headline labels both function denominators and the non-safety exclusion.""" + rendered = render_traceability( + (self.root / 'docs/safety/TRACEABILITY.md').read_text(encoding='utf-8'), + compute_coverage(analyze(self.root)), + ) + self.assertIn('Functions traced to at least one SR: 1 / 1', rendered) + self.assertIn('excluding declared non-safety functions: 1 / 1', rendered) + def test_new_numeric_prose_outside_markers_is_a_c10_failure(self): """A newly introduced numeric claim outside generated markers remains a check failure.""" text = (REPO / 'docs/safety/TRACEABILITY.md').read_text(encoding='utf-8') stale = text.replace('**Reading:**', 'Legacy claim: 21 / 27 = 77.8 %.\n\n**Reading:**') self.assertTrue(check_numeric_coverage_claims(stale)) - def test_missing_legacy_summary_claims_do_not_trigger_summary(self): - """Removing superseded hand-authored headlines does not create SUMMARY errors.""" - coverage = compute_coverage(analyze(self.root)) - self.assertEqual(check_summary_prose('No hand-authored numeric coverage claims.\n', coverage), ()) - def test_cli_exit_code_zero_on_clean_tree(self): """A repository with no active or stale findings returns the clean exit code.""" proc = self.run_cli() @@ -543,6 +759,13 @@ def test_cli_exit_code_two_on_missing_document(self): (self.root / 'docs/safety/SAFETY_REQUIREMENTS.md').unlink() self.assertEqual(self.run_cli().returncode, 2) + def test_cli_exit_code_two_on_malformed_baseline_without_traceback(self): + """Malformed baseline shapes return cannot-run without leaking an AttributeError traceback.""" + (self.root / 'docs/safety/lint-baseline.json').write_text('{"findings":[7]}\n', encoding='utf-8') + proc = self.run_cli() + self.assertEqual(proc.returncode, 2) + self.assertNotIn('AttributeError', proc.stderr) + def test_workflow_has_no_path_filters(self): """CI runs on every pull request and every main push without path filtering.""" text = (REPO / '.github/workflows/safety-lint.yml').read_text(encoding='utf-8') @@ -550,6 +773,11 @@ def test_workflow_has_no_path_filters(self): self.assertIn('pull_request:', text) self.assertIn('branches: [main]', text) + def test_workflow_declares_contents_read_as_sole_top_level_permission(self): + """Safety lint runs with only repository-content read permission at workflow scope.""" + text = (REPO / '.github/workflows/safety-lint.yml').read_text(encoding='utf-8') + self.assertIn('\npermissions:\n contents: read\n\njobs:', text) + def run_cli(self, *args): return subprocess.run( [sys.executable, '-m', 'tools.safety_lint', '--root', str(self.root), *args], From 64e4fde385db2d196327e4c2a6046620bef9eebc Mon Sep 17 00:00:00 2001 From: Raj Madhivanan Date: Sun, 13 Sep 2026 19:44:23 -0700 Subject: [PATCH 09/12] fix: watch the files that define bytes on the wire The wire guard watched declarations but not the source files that choose field order, checksum behavior, and byte order. Those files could therefore change compatibility without moving the expected signature. This expands the signature boundary and also stops the issue-form reader from silently truncating wrapped text. ## What changed - Added pstop_msg.c, checksum.c, and endian.c to the normalized signature and workflow watch set. - Stored repository-relative per-file hashes and updated the expected aggregate to d0819037896320c1f40b88573770809b42e4a11b7055d65c4991c598116eee4e. - Added one semantic mutation test per implementation file plus source-path diagnostics and comment-only stability tests. - Preserved initial signature bootstrap while preventing it from hiding a simultaneous source change. - Folded continued issue-form label, description, and placeholder text with one space. - Rejected orphan, option, and validation continuations instead of silently discarding them. - Updated the change record to describe the complete wire-behavior boundary. - Kept enforcement-mode at warn and wire-break enforcement independent. ## Safety lifecycle Configuration management, interface compatibility, and verification tooling. Bears on IEC 61508-3:2010 sections 7.4.4, 7.9, and 7.10, and IEC 61508-1:2010 sections 7.16 and 7.18.2. Co-Authored-By: OpenCode --- .github/workflows/wire-break.yml | 13 +- ...0002-modification-procedure-enforcement.md | 17 +- tools/change_control/issue_form.py | 22 ++- tools/change_control/self_test.py | 182 +++++++++++++++++- tools/change_control/wire_format.py | 57 +++--- tools/change_control/wire_format.sha256 | 23 ++- 6 files changed, 262 insertions(+), 52 deletions(-) diff --git a/.github/workflows/wire-break.yml b/.github/workflows/wire-break.yml index afb1fde6..ac7b18cd 100644 --- a/.github/workflows/wire-break.yml +++ b/.github/workflows/wire-break.yml @@ -28,10 +28,21 @@ jobs: if git cat-file -e "$BASE_SHA:tools/change_control/wire_format.sha256" 2>/dev/null; then expectation_preexisted=true fi + watched_paths='pstop_c/pstop/include/pstop/config.h + pstop_c/pstop/include/pstop/constants.h + pstop_c/pstop/include/pstop/protocol.h + pstop_c/pstop/include/pstop/protocol_data.h + pstop_c/pstop/include/pstop/pstop_msg.h + pstop_c/pstop/include/pstop/checksum.h + pstop_c/pstop/include/pstop/device_id.h + pstop_c/pstop/include/pstop/endian.h + pstop_c/pstop/src/pstop/pstop_msg.c + pstop_c/pstop/src/pstop/checksum.c + pstop_c/pstop/src/pstop/endian.c' if { [ "$expectation_preexisted" = true ] && \ git diff --name-only "$BASE_SHA"...HEAD | grep -Eq '^tools/change_control/wire_format\.sha256$'; } || \ { [ "$expectation_preexisted" = false ] && \ - git diff --name-only "$BASE_SHA"...HEAD | grep -Eq '^pstop_c/pstop/include/pstop/'; }; then + git diff --name-only "$BASE_SHA"...HEAD | grep -Fqx -f <(printf '%s\n' "$watched_paths"); }; then signature_changed=true elif ! python3 -m tools.change_control.wire_format check --root . >/dev/null; then signature_changed=true diff --git a/changes/change-0002-modification-procedure-enforcement.md b/changes/change-0002-modification-procedure-enforcement.md index 3327f287..c5a49150 100644 --- a/changes/change-0002-modification-procedure-enforcement.md +++ b/changes/change-0002-modification-procedure-enforcement.md @@ -107,7 +107,7 @@ in this change requires. | Checkout action version in use | `actions/checkout@v7` | | Authorizers | Ilia Baranov, Raj, David Tarazi — all three are code owners of everything and all three may authorize | | Protocol version constant | `pstop_c/pstop/include/pstop/config.h` — `PSTOP_VERSION 0x02U`, `PSTOP_MESSAGE_SIZE 48U` | -| Wire-format headers | `pstop_c/pstop/include/pstop/` — `config.h`, `constants.h`, `protocol.h`, `protocol_data.h`, `pstop_msg.h`, `checksum.h`, `device_id.h`, `endian.h` | +| Wire-behavior files | Eight public headers under `pstop_c/pstop/include/pstop/` — `config.h`, `constants.h`, `protocol.h`, `protocol_data.h`, `pstop_msg.h`, `checksum.h`, `device_id.h`, `endian.h` — plus `pstop_c/pstop/src/pstop/pstop_msg.c`, `pstop_c/pstop/src/pstop/checksum.c`, and `pstop_c/pstop/src/pstop/endian.c` | | `pstop_c` is vendored in-tree | A directory, NOT an ESP-IDF managed component. It does not appear in `firmware/dependencies.lock`. A "version bump" is a directory update, so the check compares header content, not a lockfile line | | Linter entry point (change-0001) | `python3 -m tools.safety_lint --json` | | Pre-commit exclusions | `pstop_c/`, `ros2/`, `archive/`, vendored wireguard and x25519, `hardware/` binaries. `tools/`, `docs/` and `.github/` are in scope | @@ -361,28 +361,31 @@ it fail-safes to STOP — safe, and permanently stopped until both ends are upda together. The build stays green throughout. This check makes that impossible to ship unannounced. -**5b. The signature.** A SHA-256 over the normalized content of the wire-format headers +**5b. The signature.** A SHA-256 over the normalized content of the wire-behavior files listed in §3, plus the literal values of `PSTOP_VERSION` and `PSTOP_MESSAGE_SIZE`. Normalize by stripping comments and collapsing whitespace so a comment edit does not trip it. Store the expected value in `wire_format.sha256` with a comment recording the `PSTOP_VERSION` it corresponds to. -**5c. Behaviour on mismatch.** Fail the PR with a message naming which headers changed +**5c. Behaviour on mismatch.** Fail the PR with a message naming which files changed and stating that remote and machine must be released and deployed together. Apply the `wire-break` label and require the `class-c` label. The fix path is to update `wire_format.sha256` in the same PR, which makes the change explicit in the diff and reviewable — the point is not to prevent wire changes, it is to prevent *silent* ones. **5d. This check ignores `docs/process/enforcement-mode` and always enforces.** It has -no judgement in it and no false positives — the headers either changed or they did not -— and the failure mode is a field outage rather than a process complaint. Hardcode +no judgement in it and no false positives — the watched files either changed or they +did not — and the failure mode is a field outage rather than a process complaint. Hardcode this; do not make it configurable. **Tests:** -- `test_signature_stable_across_comment_only_change` — add a comment to `protocol.h` - in a scratch copy, assert the signature is unchanged. +- `test_signature_stable_across_comment_only_change` — add comments to watched headers + and implementation files in a scratch copy, assert the signature is unchanged. - `test_signature_changes_on_field_addition` — add a field to a struct in a scratch copy, assert it changes. +- Mutate each watched implementation file in a scratch copy: reorder adjacent field + writes in `pstop_msg.c`, change the CRC polynomial in `checksum.c`, and change a + byte-order operation in `endian.c`; each file hash and the aggregate must change. - `test_signature_changes_on_message_size_change` - `test_check_exits_one_on_mismatch_and_names_the_headers` - Drift-verify: corrupt `wire_format.sha256`, confirm CI fails, restore. Paste the run. diff --git a/tools/change_control/issue_form.py b/tools/change_control/issue_form.py index 5313b86a..362a5261 100644 --- a/tools/change_control/issue_form.py +++ b/tools/change_control/issue_form.py @@ -28,6 +28,8 @@ def parse_issue_form(path): item = None section = None options = False + active_attribute_key = None + active_attribute_indent = None in_body = False for number, raw in enumerate(lines, 1): if not raw.strip() or raw.lstrip().startswith('#') or raw.strip() == '---': @@ -35,6 +37,8 @@ def parse_issue_form(path): indent = len(raw) - len(raw.lstrip(' ')) text = raw.strip() if indent == 0: + active_attribute_key = None + active_attribute_indent = None match = re.fullmatch(r'([a-z_]+):(?:\s*(.*))?', text) if not match: raise ValueError(f'{path}:{number}: unsupported top-level YAML') @@ -55,6 +59,8 @@ def parse_issue_form(path): document['body'].append(item) section = None options = False + active_attribute_key = None + active_attribute_indent = None continue if item is None: raise ValueError(f'{path}:{number}: body entry must begin with type') @@ -68,6 +74,8 @@ def parse_issue_form(path): raise ValueError(f'{path}:{number}: {key} must be a mapping') section = key options = False + active_attribute_key = None + active_attribute_indent = None continue if indent == 6 and section in ('attributes', 'validations'): if ':' not in text: @@ -75,20 +83,32 @@ def parse_issue_form(path): key, value = text.split(':', 1) if section == 'validations' and key == 'required': item['required'] = _scalar(value) + active_attribute_key = None + active_attribute_indent = None elif section == 'attributes' and key == 'options': if value.strip(): raise ValueError(f'{path}:{number}: options must be a sequence') options = True + active_attribute_key = None + active_attribute_indent = None elif section == 'attributes': item[key] = _scalar(value) options = False + active_attribute_key = key + active_attribute_indent = indent else: raise ValueError(f'{path}:{number}: unsupported validation') continue if indent == 8 and options and text.startswith('- '): item['options'].append(_scalar(text[2:])) continue - if indent >= 8 and section == 'attributes': + if ( + section == 'attributes' + and not options + and active_attribute_key is not None + and indent > active_attribute_indent + ): + item[active_attribute_key] = f'{item[active_attribute_key]} {_scalar(text)}'.strip() continue raise ValueError(f'{path}:{number}: unsupported indentation or YAML construct') validate_issue_form(document) diff --git a/tools/change_control/self_test.py b/tools/change_control/self_test.py index 01968c30..b730c90a 100755 --- a/tools/change_control/self_test.py +++ b/tools/change_control/self_test.py @@ -30,10 +30,23 @@ upsert_coverage_comment, ) from tools.change_control.issue_form import parse_issue_form, validate_issue_form # noqa: E402 -from tools.change_control.wire_format import HEADER_NAMES, check_wire_format, compute_signature # noqa: E402 +from tools.change_control.wire_format import WIRE_PATHS, check_wire_format, compute_signature # noqa: E402 FORM = ROOT / '.github/ISSUE_TEMPLATE/change-request.yml' WIRE_EXPECTED = ROOT / 'tools/change_control/wire_format.sha256' +EXPECTED_WIRE_PATHS = ( + 'pstop_c/pstop/include/pstop/config.h', + 'pstop_c/pstop/include/pstop/constants.h', + 'pstop_c/pstop/include/pstop/protocol.h', + 'pstop_c/pstop/include/pstop/protocol_data.h', + 'pstop_c/pstop/include/pstop/pstop_msg.h', + 'pstop_c/pstop/include/pstop/checksum.h', + 'pstop_c/pstop/include/pstop/device_id.h', + 'pstop_c/pstop/include/pstop/endian.h', + 'pstop_c/pstop/src/pstop/pstop_msg.c', + 'pstop_c/pstop/src/pstop/checksum.c', + 'pstop_c/pstop/src/pstop/endian.c', +) def snapshot(**overrides): @@ -67,6 +80,25 @@ def snapshot(**overrides): class IssueFormTests(unittest.TestCase): + def _parse_field(self, attributes): + form = ( + 'name: Test\n' + 'description: Test form\n' + "title: '[Test] '\n" + 'labels: [test]\n' + 'body:\n' + ' - type: textarea\n' + ' id: field\n' + ' attributes:\n' + f'{attributes}' + ' validations:\n' + ' required: true\n' + ) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'form.yml' + path.write_text(form, encoding='utf-8') + return parse_issue_form(path)['body'][0] + def test_issue_form_is_valid_yaml_and_parses(self): """The checked-in issue form must parse as the deliberately supported YAML subset.""" parsed = parse_issue_form(FORM) @@ -115,6 +147,49 @@ def test_source_field_details_survive_issue_form_conversion(self): self.assertIn('Forward -', fields['gate-1']['description']) self.assertIn('Backward -', fields['gate-1']['description']) + def test_plain_label_continuation_is_folded(self): + """A continued attributes label must be joined to its first line with one space.""" + field = self._parse_field(' label: First label line\n second label line\n') + self.assertEqual(field['label'], 'First label line second label line') + + def test_plain_description_continuation_is_folded(self): + """A continued attributes description must be joined to its first line with one space.""" + field = self._parse_field( + ' label: Field\n description: First description line\n second description line\n' + ) + self.assertEqual(field['description'], 'First description line second description line') + + def test_plain_placeholder_continuation_is_folded(self): + """A continued attributes placeholder must be joined to its first line with one space.""" + field = self._parse_field( + ' label: Field\n placeholder: First placeholder line\n second placeholder line\n' + ) + self.assertEqual(field['placeholder'], 'First placeholder line second placeholder line') + + def test_real_gate_one_description_is_complete(self): + """The checked-in folded Gate 1 description must retain its final continuation text.""" + fields = {field['id']: field for field in parse_issue_form(FORM)['body']} + self.assertTrue(fields['gate-1']['description'].endswith('requirements covered.')) + + def test_orphan_attribute_continuation_is_rejected(self): + """Indented text without an active scalar key must fail rather than disappear.""" + with self.assertRaises(ValueError): + self._parse_field(' orphan continuation\n label: Field\n') + + def test_option_continuation_is_rejected(self): + """Deeper text below an option must never be folded into an attributes scalar.""" + attributes = ' label: Field\n options:\n - First\n unsupported option continuation\n' + with self.assertRaises(ValueError): + self._parse_field(attributes) + + def test_validation_continuation_is_rejected(self): + """Deeper validation text must never be appended to the preceding attributes scalar.""" + attributes = ( + ' label: Field\n validations:\n required: true\n unsupported validation continuation\n' + ) + with self.assertRaises(ValueError): + self._parse_field(attributes) + def test_malformed_issue_form_is_rejected(self): """Malformed indentation must fail instead of silently degrading to a blank issue.""" with tempfile.TemporaryDirectory() as directory: @@ -460,14 +535,15 @@ def test_upsert_comment_creates_when_absent(self): class WireFormatTests(unittest.TestCase): - def _copy_headers(self, directory): - include = Path(directory) / 'pstop_c/pstop/include/pstop' - include.mkdir(parents=True) - source = ROOT / 'pstop_c/pstop/include/pstop' - for name in HEADER_NAMES: - shutil.copy2(source / name, include / name) + def _copy_wire_files(self, directory): + for relative in EXPECTED_WIRE_PATHS: + destination = Path(directory) / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(ROOT / relative, destination) return Path(directory) + _copy_headers = _copy_wire_files + def test_signature_stable_across_comment_only_change(self): """Adding a C comment to a watched header must not alter its signature.""" with tempfile.TemporaryDirectory() as directory: @@ -477,6 +553,66 @@ def test_signature_stable_across_comment_only_change(self): path.write_text(path.read_text(encoding='utf-8') + '\n/* comment only */\n', encoding='utf-8') self.assertEqual(compute_signature(root).aggregate, before.aggregate) + def test_source_signatures_stay_stable_across_comment_only_changes(self): + """Comments in each watched implementation file must not alter any wire signature.""" + with tempfile.TemporaryDirectory() as directory: + root = self._copy_wire_files(directory) + before = compute_signature(root) + for relative in EXPECTED_WIRE_PATHS[-3:]: + path = root / relative + path.write_text(path.read_text(encoding='utf-8') + '\n/* comment only */\n', encoding='utf-8') + after = compute_signature(root) + self.assertEqual(after, before) + + def test_reordered_field_writes_change_pstop_message_source_and_aggregate_hashes(self): + """Reordering adjacent encoded fields must change pstop_msg.c evidence and the aggregate.""" + relative = 'pstop_c/pstop/src/pstop/pstop_msg.c' + with tempfile.TemporaryDirectory() as directory: + root = self._copy_wire_files(directory) + before = compute_signature(root) + path = root / relative + original = ( + ' write_uint32(msg->counter, data, &pos);\n write_uint32(msg->received_counter, data, &pos);' + ) + replacement = ( + ' write_uint32(msg->received_counter, data, &pos);\n write_uint32(msg->counter, data, &pos);' + ) + changed = path.read_text(encoding='utf-8').replace(original, replacement) + self.assertNotEqual(changed, path.read_text(encoding='utf-8')) + path.write_text(changed, encoding='utf-8') + after = compute_signature(root) + self.assertNotEqual(after.files[relative], before.files[relative]) + self.assertNotEqual(after.aggregate, before.aggregate) + + def test_crc_polynomial_change_changes_checksum_source_and_aggregate_hashes(self): + """Changing the CRC polynomial must change checksum.c evidence and the aggregate.""" + relative = 'pstop_c/pstop/src/pstop/checksum.c' + with tempfile.TemporaryDirectory() as directory: + root = self._copy_wire_files(directory) + before = compute_signature(root) + path = root / relative + path.write_text(path.read_text(encoding='utf-8').replace('0x8D95U', '0x8D96U'), encoding='utf-8') + after = compute_signature(root) + self.assertNotEqual(after.files[relative], before.files[relative]) + self.assertNotEqual(after.aggregate, before.aggregate) + + def test_byte_order_change_changes_endian_source_and_aggregate_hashes(self): + """Changing one byte-order operation must change endian.c evidence and the aggregate.""" + relative = 'pstop_c/pstop/src/pstop/endian.c' + with tempfile.TemporaryDirectory() as directory: + root = self._copy_wire_files(directory) + before = compute_signature(root) + path = root / relative + path.write_text( + path.read_text(encoding='utf-8').replace( + 'bytes[3] = (uint8_t)(value & 0xFFU);', 'bytes[2] = (uint8_t)(value & 0xFFU);' + ), + encoding='utf-8', + ) + after = compute_signature(root) + self.assertNotEqual(after.files[relative], before.files[relative]) + self.assertNotEqual(after.aggregate, before.aggregate) + def test_comment_only_change_needs_no_wire_labels(self): """A comment-only watched-header edit must pass the declaration check without wire-break labels.""" with tempfile.TemporaryDirectory() as directory: @@ -541,6 +677,18 @@ def test_check_exits_one_on_mismatch_and_names_the_headers(self): self.assertEqual(code, 1) self.assertIn('protocol.h', message) + def test_source_mismatch_diagnostic_names_repository_path(self): + """A source mismatch diagnostic must identify the changed repository-relative path.""" + relative = 'pstop_c/pstop/src/pstop/checksum.c' + with tempfile.TemporaryDirectory() as directory: + root = self._copy_wire_files(directory) + shutil.copy2(WIRE_EXPECTED, root / 'wire_format.sha256') + path = root / relative + path.write_text(path.read_text(encoding='utf-8').replace('0x8D95U', '0x8D96U'), encoding='utf-8') + code, message = check_wire_format(root, root / 'wire_format.sha256', {'wire-break', 'class-c'}, []) + self.assertEqual(code, 1) + self.assertIn(relative, message) + def test_expected_update_requires_both_labels(self): """Changing the expected signature cannot pass without wire-break and class-c labels.""" code, message = check_wire_format( @@ -572,6 +720,26 @@ def test_initial_snapshot_cannot_hide_a_header_change(self): self.assertEqual(code, 1) self.assertIn('class-c', message) + def test_initial_snapshot_cannot_hide_a_source_change(self): + """A watched source edit accompanying the first snapshot must still require both declarations.""" + relative = 'pstop_c/pstop/src/pstop/endian.c' + code, message = check_wire_format( + ROOT, + WIRE_EXPECTED, + {'wire-break'}, + [relative, 'tools/change_control/wire_format.sha256'], + expectation_preexisted=False, + ) + self.assertEqual(code, 1) + self.assertIn('class-c', message) + + def test_workflow_initial_bootstrap_matcher_watches_all_wire_files(self): + """The reviewable workflow matcher must include every watched header and implementation path.""" + workflow = (ROOT / '.github/workflows/wire-break.yml').read_text(encoding='utf-8') + self.assertEqual(WIRE_PATHS, EXPECTED_WIRE_PATHS) + for relative in EXPECTED_WIRE_PATHS: + self.assertIn(relative, workflow) + def test_wire_cli_exposes_guard_exit_codes(self): """The public wire CLI must return 0 for a match, 1 for policy mismatch, and 2 when it cannot run.""" clean = subprocess.run( diff --git a/tools/change_control/wire_format.py b/tools/change_control/wire_format.py index 5747a703..961a82ab 100644 --- a/tools/change_control/wire_format.py +++ b/tools/change_control/wire_format.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: 2026 Polymath Robotics, Inc. # SPDX-License-Identifier: Apache-2.0 -"""Compute and enforce the normalized pstop_c public wire-header signature.""" +"""Compute and enforce the normalized pstop_c wire-behavior signature.""" import argparse import hashlib @@ -20,15 +20,20 @@ 'endian.h', ) HEADER_PREFIX = 'pstop_c/pstop/include/pstop/' +WIRE_PATHS = tuple(HEADER_PREFIX + name for name in HEADER_NAMES) + ( + 'pstop_c/pstop/src/pstop/pstop_msg.c', + 'pstop_c/pstop/src/pstop/checksum.c', + 'pstop_c/pstop/src/pstop/endian.c', +) @dataclass(frozen=True) class WireSignature: - """Per-header evidence and the aggregate normalized wire signature.""" + """Per-file evidence and the aggregate normalized wire signature.""" version: str message_size: str - headers: dict + files: dict aggregate: str @@ -46,27 +51,26 @@ def _literal(text, name): def compute_signature(root): - """Hash comment-free, whitespace-collapsed headers plus explicit protocol literals.""" - include = Path(root) / HEADER_PREFIX + """Hash comment-free, whitespace-collapsed wire files plus explicit protocol literals.""" normalized = {} - for name in HEADER_NAMES: - path = include / name + for relative in WIRE_PATHS: + path = Path(root) / relative if not path.is_file(): raise FileNotFoundError(path) - normalized[name] = ' '.join(_strip_comments(path.read_text(encoding='utf-8')).split()) - config = (include / 'config.h').read_text(encoding='utf-8') + normalized[relative] = ' '.join(_strip_comments(path.read_text(encoding='utf-8')).split()) + config = (Path(root) / HEADER_PREFIX / 'config.h').read_text(encoding='utf-8') version = _literal(config, 'PSTOP_VERSION') message_size = _literal(config, 'PSTOP_MESSAGE_SIZE') - header_hashes = {name: hashlib.sha256(normalized[name].encode()).hexdigest() for name in HEADER_NAMES} - payload = ''.join(f'{name}\0{normalized[name]}\0' for name in HEADER_NAMES) + file_hashes = {path: hashlib.sha256(normalized[path].encode()).hexdigest() for path in WIRE_PATHS} + payload = ''.join(f'{path}\0{normalized[path]}\0' for path in WIRE_PATHS) payload += f'PSTOP_VERSION\0{version}\0PSTOP_MESSAGE_SIZE\0{message_size}\0' - return WireSignature(version, message_size, header_hashes, hashlib.sha256(payload.encode()).hexdigest()) + return WireSignature(version, message_size, file_hashes, hashlib.sha256(payload.encode()).hexdigest()) def read_expected(path): """Read the reviewable line-oriented wire signature record.""" values = {} - headers = {} + files = {} for raw in Path(path).read_text(encoding='utf-8').splitlines(): line = raw.strip() if not line or line.startswith('#'): @@ -75,43 +79,44 @@ def read_expected(path): if len(parts) != 2: raise ValueError(f'invalid expected signature line: {raw}') key, value = parts - if key in HEADER_NAMES: - headers[key] = value + if key in WIRE_PATHS: + files[key] = value else: values[key] = value - if set(headers) != set(HEADER_NAMES) or not {'PSTOP_VERSION', 'PSTOP_MESSAGE_SIZE', 'aggregate'} <= set(values): - raise ValueError('expected signature lacks version, size, aggregate, or per-header hashes') - return WireSignature(values['PSTOP_VERSION'], values['PSTOP_MESSAGE_SIZE'], headers, values['aggregate']) + if set(files) != set(WIRE_PATHS) or not {'PSTOP_VERSION', 'PSTOP_MESSAGE_SIZE', 'aggregate'} <= set(values): + raise ValueError('expected signature lacks version, size, aggregate, or per-file hashes') + return WireSignature(values['PSTOP_VERSION'], values['PSTOP_MESSAGE_SIZE'], files, values['aggregate']) def render_signature(signature): """Render deterministic expected-signature content suitable for code review.""" lines = [ - '# Normalized pstop_c wire headers; comments stripped and whitespace collapsed.', + '# Normalized pstop_c wire files; comments stripped and whitespace collapsed.', f'# Corresponds to PSTOP_VERSION {signature.version}.', f'PSTOP_VERSION {signature.version}', f'PSTOP_MESSAGE_SIZE {signature.message_size}', ] - lines.extend(f'{name} {signature.headers[name]}' for name in HEADER_NAMES) + lines.extend(f'{path} {signature.files[path]}' for path in WIRE_PATHS) lines.append(f'aggregate {signature.aggregate}') return '\n'.join(lines) + '\n' def check_wire_format(root, expected_path, labels, changed_files, expectation_preexisted=True): - """Return guard exit code and explanation for current headers, labels, and changed paths.""" + """Return guard exit code and explanation for current files, labels, and changed paths.""" try: current = compute_signature(root) expected = read_expected(expected_path) except (OSError, ValueError) as error: return 2, f'wire-format: cannot run: {error}' - mismatched = [name for name in HEADER_NAMES if current.headers[name] != expected.headers[name]] + mismatched = [path for path in WIRE_PATHS if current.files[path] != expected.files[path]] if current.version != expected.version or current.message_size != expected.message_size: - if 'config.h' not in mismatched: - mismatched.append('config.h') + config_path = HEADER_PREFIX + 'config.h' + if config_path not in mismatched: + mismatched.append(config_path) expectation_change = expectation_preexisted and 'tools/change_control/wire_format.sha256' in changed_files - initial_header_change = not expectation_preexisted and any(path.startswith(HEADER_PREFIX) for path in changed_files) + initial_wire_change = not expectation_preexisted and bool(set(changed_files) & set(WIRE_PATHS)) signature_changed = bool(mismatched or current.aggregate != expected.aggregate) - required = {'wire-break', 'class-c'} if expectation_change or initial_header_change or signature_changed else set() + required = {'wire-break', 'class-c'} if expectation_change or initial_wire_change or signature_changed else set() missing_labels = sorted(required - set(labels)) rollout = 'Remote and machine must be released and deployed together for a coordinated rollout.' if signature_changed: diff --git a/tools/change_control/wire_format.sha256 b/tools/change_control/wire_format.sha256 index ac38da86..ee3bf72d 100644 --- a/tools/change_control/wire_format.sha256 +++ b/tools/change_control/wire_format.sha256 @@ -1,13 +1,16 @@ -# Normalized pstop_c wire headers; comments stripped and whitespace collapsed. +# Normalized pstop_c wire files; comments stripped and whitespace collapsed. # Corresponds to PSTOP_VERSION 0x02U. PSTOP_VERSION 0x02U PSTOP_MESSAGE_SIZE 48U -config.h 3fb2ef9fef38451b283ef7e6b850cf06eb5f9b81e5dc07af64d43843c9ba0474 -constants.h 7f24ea90cac9a64008c69b16c4629682bfa150238e3f7825f4726b055b59ea54 -protocol.h a5250fac05f5c8ea6f95d4af821f07111323acef7c39827fd215d2822c56a7aa -protocol_data.h afe3ca0116d57832350defdfe50d22d9d0da3c6e8ee8690e0c084f24efd932bb -pstop_msg.h 6c5ab0d4847dc5fafd43890093473b202548e87cf0c4b8c382def5a36e301249 -checksum.h 47fddca6f7f7bee63cf0c8fda307ae438ce1e497b9d1c2b6639485df84adee4e -device_id.h 51047e992b03d156b2acc8acc2130e6cc0a246ec6035438a213f3e9e440bf422 -endian.h e22513590e1f28a54ec7d9e0bc2986d4adbb0ad38f6df069ec42da8dc82f9bce -aggregate a652abc231b4acb2d9014d73f459123e914fb4b1fcfe8c630f3f8d3853f012af +pstop_c/pstop/include/pstop/config.h 3fb2ef9fef38451b283ef7e6b850cf06eb5f9b81e5dc07af64d43843c9ba0474 +pstop_c/pstop/include/pstop/constants.h 7f24ea90cac9a64008c69b16c4629682bfa150238e3f7825f4726b055b59ea54 +pstop_c/pstop/include/pstop/protocol.h a5250fac05f5c8ea6f95d4af821f07111323acef7c39827fd215d2822c56a7aa +pstop_c/pstop/include/pstop/protocol_data.h afe3ca0116d57832350defdfe50d22d9d0da3c6e8ee8690e0c084f24efd932bb +pstop_c/pstop/include/pstop/pstop_msg.h 6c5ab0d4847dc5fafd43890093473b202548e87cf0c4b8c382def5a36e301249 +pstop_c/pstop/include/pstop/checksum.h 47fddca6f7f7bee63cf0c8fda307ae438ce1e497b9d1c2b6639485df84adee4e +pstop_c/pstop/include/pstop/device_id.h 51047e992b03d156b2acc8acc2130e6cc0a246ec6035438a213f3e9e440bf422 +pstop_c/pstop/include/pstop/endian.h e22513590e1f28a54ec7d9e0bc2986d4adbb0ad38f6df069ec42da8dc82f9bce +pstop_c/pstop/src/pstop/pstop_msg.c 5e6b99165bb4757c8b1ef5b31d5f71bfbaa73228919ba78cd4dad33eebedecfa +pstop_c/pstop/src/pstop/checksum.c 3e65c06d7cb73fd6a86d500917b291ba062837efd50e1ae665a0b207cacae3b1 +pstop_c/pstop/src/pstop/endian.c 447b082a41b87949ccf6be3a9e7340ee65ffb8b69daae9d6b9399cb6b9b4f349 +aggregate d0819037896320c1f40b88573770809b42e4a11b7055d65c4991c598116eee4e From 74f1e7ee9f77f9976ee8ecfc040827be628233d6 Mon Sep 17 00:00:00 2001 From: Raj Madhivanan Date: Sun, 13 Sep 2026 20:53:39 -0700 Subject: [PATCH 10/12] fix: close the remaining traceability edge cases The previous parser hardening rejected ordinary punctuation and allowed some Markdown paths to bypass report validation. It also reused the all-function numerator for the safety-only fraction. This closes those interactions while preserving the current 32/40 citation result. ## What changed - Accepted normal punctuation after complete allocations while rejecting malformed continuations. - Required every Markdown report to live under docs and name the complete requirement token. - Allowed one lowercase decomposition suffix, such as SR-H-04b, to evidence its canonical parent. - Added a safety-only traced-function numerator to text, JSON, and generated rendering. - Added adversarial tests for rule ordering, identifier prefixes, punctuation, traversal, malformed IDs, duplicate baseline keys, and excluded-function arithmetic. - Confirmed disputed traversal, malformed-coverage, and duplicate-baseline findings are unreachable through public analysis paths. - Kept generated output byte-identical and citation coverage at 32/40. ## Safety lifecycle Verification evidence integrity and requirements traceability. Bears on IEC 61508-3:2010 sections 7.9 and 7.10, Annex A.8.7 and A.8.8, and IEC 61508-1:2010 section 7.18.2. Co-Authored-By: OpenCode --- tools/safety_lint/__main__.py | 3 +- tools/safety_lint/checks.py | 22 ++-- tools/safety_lint/coverage.py | 7 ++ tools/safety_lint/parse_srs.py | 7 +- tools/safety_lint/parse_traceability.py | 6 +- tools/safety_lint/render.py | 4 +- tools/safety_lint/self_test.py | 137 +++++++++++++++++++++++- 7 files changed, 169 insertions(+), 17 deletions(-) diff --git a/tools/safety_lint/__main__.py b/tools/safety_lint/__main__.py index f6dd1d2a..5c57dda3 100644 --- a/tools/safety_lint/__main__.py +++ b/tools/safety_lint/__main__.py @@ -52,6 +52,7 @@ def _coverage_dict(coverage): 'functions': { 'traced': coverage.functions_traced, 'total': coverage.functions_total, + 'safety_traced': coverage.safety_functions_traced, 'safety_total': coverage.safety_functions_total, }, } @@ -107,7 +108,7 @@ def main(argv=None): for finding in suppressed: print(f'{finding.file}:{finding.line}: [{finding.check_id}] {finding.subject} — {finding.message}') print( - f'Coverage: cited tests {coverage.cited_tests}/{coverage.total}; strict Verified {coverage.verified}/{coverage.total}; functions {coverage.functions_traced}/{coverage.functions_total} ({coverage.functions_traced}/{coverage.safety_functions_total} excluding declared non-safety)' + f'Coverage: cited tests {coverage.cited_tests}/{coverage.total}; strict Verified {coverage.verified}/{coverage.total}; functions {coverage.functions_traced}/{coverage.functions_total} ({coverage.safety_functions_traced}/{coverage.safety_functions_total} excluding declared non-safety)' ) print('Citation limitation: resolution does not verify test execution or passing state.') for area, data in coverage.areas.items(): diff --git a/tools/safety_lint/checks.py b/tools/safety_lint/checks.py index 36c1fd9b..b930d692 100644 --- a/tools/safety_lint/checks.py +++ b/tools/safety_lint/checks.py @@ -37,21 +37,23 @@ def evidence_rejection(root, path, sr_id): parts = tuple(part.lower() for part in candidate.parts) stem = candidate.stem.lower() - if candidate.name.lower() == 'readme.md': - return 'README files are not approved evidence reports' + # Markdown is always a report first; test-like paths and names cannot bypass report rules. + if candidate.suffix.lower() == '.md': + if candidate.name.lower() == 'readme.md': + return 'README files are not approved evidence reports' + if not parts or parts[0] != 'docs': + return 'Markdown evidence reports must be under docs/' + content = (Path(root) / candidate).read_text(encoding='utf-8', errors='replace') + token = re.compile(rf'(?', line=1): for match in pattern.finditer(cell): area, first, end, alternates = match.groups() trailing = cell[match.end() :] - if trailing.startswith(('/', '.')) and not trailing.startswith('/F-'): + malformed = ( + (trailing.startswith('/') and not trailing.startswith('/F-')) + or trailing.startswith('..') + or (trailing and trailing[0].isalnum()) + ) + if malformed: literal = re.match(r'[^\s,|)]+', cell[match.start() :]).group() raise LintError(f'{path}:{line}: malformed allocation {literal!r}') if end: diff --git a/tools/safety_lint/parse_traceability.py b/tools/safety_lint/parse_traceability.py index e4fcbcbb..9c83254c 100644 --- a/tools/safety_lint/parse_traceability.py +++ b/tools/safety_lint/parse_traceability.py @@ -65,7 +65,11 @@ def accept(path, literal): if reason is None: refs.append(path) return - kind = 'report-does-not-name-sr' if reason == 'evidence report does not name cited SR' else 'rejected-evidence' + kind = ( + 'report-does-not-name-sr' + if reason.startswith('evidence report does not name cited SR') + else 'rejected-evidence' + ) issues.append(ResolutionIssue(kind, sr_id, literal, f'{path}: {reason}', line, 'test')) for match in re.finditer(r'\bHIL(10|20|30)((?:/(?:10|20|30))*)', evidence_cell): diff --git a/tools/safety_lint/render.py b/tools/safety_lint/render.py index 9bfe9d2a..4fd2ccb3 100644 --- a/tools/safety_lint/render.py +++ b/tools/safety_lint/render.py @@ -32,8 +32,8 @@ def render_traceability(text, coverage): f'{_percent(coverage.verified, coverage.total)}**\n' f'- **Functions traced to at least one SR: {coverage.functions_traced} / {coverage.functions_total} = ' f'{_percent(coverage.functions_traced, coverage.functions_total)}**\n' - f'- **Functions traced excluding declared non-safety functions: {coverage.functions_traced} / ' - f'{coverage.safety_functions_total} = {_percent(coverage.functions_traced, coverage.safety_functions_total)}**\n' + f'- **Functions traced excluding declared non-safety functions: {coverage.safety_functions_traced} / ' + f'{coverage.safety_functions_total} = {_percent(coverage.safety_functions_traced, coverage.safety_functions_total)}**\n' '\n' '‡ Citation resolution, not test execution or passing state, is checked by the linter.' ) diff --git a/tools/safety_lint/self_test.py b/tools/safety_lint/self_test.py index 37688f93..2a4fca11 100755 --- a/tools/safety_lint/self_test.py +++ b/tools/safety_lint/self_test.py @@ -9,14 +9,16 @@ import sys import tempfile import unittest +from dataclasses import replace from pathlib import Path +from unittest import mock REPO = Path(__file__).resolve().parents[2] FIXTURE = Path(__file__).with_name('fixtures') / 'repository' sys.path.insert(0, str(REPO)) -from tools.safety_lint.__main__ import _load_baseline # noqa: E402 +from tools.safety_lint.__main__ import _coverage_dict, _load_baseline, main # noqa: E402 from tools.safety_lint.checks import ( # noqa: E402 SRS_STATUSES, TRACE_STATUSES, @@ -25,7 +27,7 @@ run_checks, ) from tools.safety_lint.coverage import compute_coverage # noqa: E402 -from tools.safety_lint.model import LintError # noqa: E402 +from tools.safety_lint.model import Function, LintError, ReverseEntry # noqa: E402 from tools.safety_lint.parse_srs import expand_allocations, parse_srs, split_row # noqa: E402 from tools.safety_lint.parse_system_definition import parse_system_definition # noqa: E402 from tools.safety_lint.parse_traceability import parse_traceability # noqa: E402 @@ -88,6 +90,10 @@ def test_allocated_to_slash_expansion(self): """Compact slash allocations expand to complete function IDs.""" self.assertEqual(expand_allocations('F-M-03/04'), ('F-M-03', 'F-M-04')) + def test_allocation_tokens_allow_ordinary_trailing_punctuation(self): + """Sentence punctuation after a complete allocation is not a malformed continuation.""" + self.assertEqual(expand_allocations('F-M-03/04. F-R-01)'), ('F-M-03', 'F-M-04', 'F-R-01')) + def test_allocated_to_arbitrary_slash_chain_expansion(self): """Every member of an arbitrary compact slash chain becomes a complete function ID.""" self.assertEqual( @@ -105,12 +111,38 @@ def test_malformed_trailing_slash_allocation_fails_with_source_location(self): with self.assertRaisesRegex(LintError, r'doc.md:19:.*F-R-01/02/XX'): expand_allocations('F-R-01/02/XX', 'doc.md', 19) + def test_every_malformed_allocation_continuation_fails_in_helper(self): + """Malformed slash, range, and numeric continuations can never leave a partial allocation.""" + for literal in ('F-R-01/XX', 'F-R-01..', 'F-R-01/2', 'F-R-01/003'): + with self.subTest(literal=literal), self.assertRaisesRegex(LintError, 'malformed allocation'): + expand_allocations(literal, 'doc.md', 21) + + def test_srs_and_trace_rows_allow_allocation_punctuation(self): + """Both authoritative allocation tables accept punctuation after complete function IDs.""" + self.replace('docs/safety/SAFETY_REQUIREMENTS.md', '| F-R-01 | SIL 3 |', '| F-R-01) | SIL 3 |') + self.replace('docs/safety/TRACEABILITY.md', '| SR-R-01 | F-R-01 |', '| SR-R-01 | F-R-01. |') + srs = parse_srs(self.root / 'docs/safety/SAFETY_REQUIREMENTS.md') + trace, _, _ = parse_traceability(self.root) + self.assertEqual((srs[1].allocated_to, trace[1].allocated_to), (('F-R-01',), ('F-R-01',))) + + def test_srs_parser_rejects_partial_range_in_real_row(self): + """A malformed continuation in an SRS table row fails instead of retaining its valid prefix.""" + self.replace('docs/safety/SAFETY_REQUIREMENTS.md', '| F-R-01 | SIL 3 |', '| F-R-01.. | SIL 3 |') + with self.assertRaisesRegex(LintError, r'SAFETY_REQUIREMENTS\.md:14: malformed allocation'): + parse_srs(self.root / 'docs/safety/SAFETY_REQUIREMENTS.md') + def test_trace_parser_reports_malformed_allocation_row_location(self): """A truncated trace allocation fails at the exact matrix row rather than yielding partial data.""" self.replace('docs/safety/TRACEABILITY.md', '| SR-R-01 | F-R-01 |', '| SR-R-01 | F-R-01/02/XX |') with self.assertRaisesRegex(LintError, r'TRACEABILITY\.md:11: malformed allocation'): parse_traceability(self.root) + def test_trace_parser_rejects_malformed_numeric_member_in_real_row(self): + """A malformed numeric member in a matrix row fails instead of retaining its valid prefix.""" + self.replace('docs/safety/TRACEABILITY.md', '| SR-R-01 | F-R-01 |', '| SR-R-01 | F-R-01/2 |') + with self.assertRaisesRegex(LintError, r'TRACEABILITY\.md:11: malformed allocation'): + parse_traceability(self.root) + def test_status_longest_match_wins(self): """A partial SRS status is never inflated to a fully satisfied status.""" rows = parse_srs(self.root / 'docs/safety/SAFETY_REQUIREMENTS.md') @@ -239,6 +271,22 @@ def test_hil_report_must_name_sr(self): _, _, issues = parse_traceability(self.root) self.assertTrue([i for i in issues if i.kind == 'report-does-not-name-sr']) + def test_test_named_markdown_without_sr_is_rejected_as_report(self): + """A Markdown file in tests must name the cited SR before test-like naming can matter.""" + (self.root / 'tests/test_report.md').write_text('# Report without requirement\n', encoding='utf-8') + self.replace('docs/safety/TRACEABILITY.md', 'test_unique_probe.py', 'tests/test_report.md') + rows, _, issues = parse_traceability(self.root) + self.assertFalse(rows[1].test_refs) + self.assertTrue([i for i in issues if i.literal == 'tests/test_report.md']) + + def test_test_named_markdown_with_sr_outside_docs_is_rejected(self): + """A Markdown report naming its SR is still ineligible when it is outside docs.""" + (self.root / 'tests/test_report.md').write_text('# Evidence for SR-R-01\n', encoding='utf-8') + self.replace('docs/safety/TRACEABILITY.md', 'test_unique_probe.py', 'tests/test_report.md') + rows, _, issues = parse_traceability(self.root) + self.assertFalse(rows[1].test_refs) + self.assertTrue([i for i in issues if i.literal == 'tests/test_report.md']) + def test_docs_markdown_report_naming_sr_is_evidence(self): """A non-README Markdown report under docs counts when its content names the cited SR.""" (self.root / 'docs/evidence.md').write_text('# Evidence for SR-R-01\n', encoding='utf-8') @@ -247,6 +295,50 @@ def test_docs_markdown_report_naming_sr_is_evidence(self): self.assertEqual(rows[1].test_refs, ('docs/evidence.md',)) self.assertFalse([issue for issue in issues if issue.literal == 'docs/evidence.md']) + def test_sr_report_match_requires_complete_identifier_token(self): + """A report must contain the exact cited SR token, not a prefixed or extended identifier.""" + path = self.root / 'docs/evidence.md' + self.replace('docs/safety/TRACEABILITY.md', 'test_unique_probe.py', 'docs/evidence.md') + for content in ('SR-R-010', 'XSR-R-01', 'xSR-R-01', 'SR-R-01bb', 'SR-R-01-extra', 'SR-R-01_extra'): + with self.subTest(content=content): + path.write_text(f'# Evidence for {content}\n', encoding='utf-8') + rows, _, issues = parse_traceability(self.root) + self.assertFalse(rows[1].test_refs) + self.assertTrue([i for i in issues if i.kind == 'report-does-not-name-sr']) + path.write_text('# Evidence for (SR-R-01).\n', encoding='utf-8') + rows, _, issues = parse_traceability(self.root) + self.assertEqual(rows[1].test_refs, ('docs/evidence.md',)) + self.assertFalse([i for i in issues if i.literal == 'docs/evidence.md']) + + def test_sr_report_match_allows_one_lowercase_decomposition_suffix(self): + """A report naming one lowercase requirement decomposition suffix evidences its canonical parent.""" + path = self.root / 'docs/evidence.md' + path.write_text('# Evidence for SR-R-01b\n', encoding='utf-8') + self.replace('docs/safety/TRACEABILITY.md', 'test_unique_probe.py', 'docs/evidence.md') + rows, _, issues = parse_traceability(self.root) + self.assertEqual(rows[1].test_refs, ('docs/evidence.md',)) + self.assertFalse([i for i in issues if i.literal == 'docs/evidence.md']) + + def test_explicit_parent_traversal_cannot_resolve_outside_root(self): + """An existing file reached through ../ is missing evidence because it is absent from the root index.""" + outside = self.root.parent / f'{self.root.name}-outside_test.py' + outside.write_text('# SR-R-01 outside repository\n', encoding='utf-8') + citation = f'../{outside.name}' + try: + self.replace('docs/safety/TRACEABILITY.md', 'test_unique_probe.py', f'`{citation}`') + rows, _, issues = parse_traceability(self.root) + self.assertFalse(rows[1].test_refs) + self.assertTrue([i for i in issues if i.kind == 'missing' and i.literal == citation]) + finally: + outside.unlink() + + def test_public_analysis_rejects_malformed_sr_before_coverage(self): + """The public CLI analysis seam cannot pass a malformed trace SR to coverage computation.""" + self.replace('docs/safety/TRACEABILITY.md', '| SR-R-01 |', '| SR-R-1 |') + with mock.patch('tools.safety_lint.__main__.compute_coverage') as compute: + self.assertEqual(main(['--root', str(self.root)]), 2) + compute.assert_not_called() + def test_test_artifact_directory_and_source_naming_are_evidence(self): """Both test-directory membership and test-source naming independently identify test artifacts.""" (self.root / 'tests/probe.c').write_text('/* test */\n', encoding='utf-8') @@ -375,6 +467,14 @@ def test_real_machine_test_line_citations_resolve_without_opening_symbols(self): class ConsistencyTests(FixtureRepo): + def test_reachable_findings_have_unique_baseline_discriminators(self): + """Real and independently injected findings never compete for one exact baseline key.""" + self.replace('docs/safety/TRACEABILITY.md', 'test_unique_probe.py', 'test_missing_one, test_missing_two') + self.replace('docs/safety/TRACEABILITY.md', '| SR-R-01 | F-R-01 |', '| SR-R-01 | F-X-99 |') + findings = self.findings() + keys = [(finding.check_id, finding.subject, finding.message) for finding in findings] + self.assertEqual(len(keys), len(set(keys))) + def test_c1_flags_sr_set_mismatch(self): """C1 reports an SRS requirement omitted from the matrix.""" self.replace( @@ -665,6 +765,39 @@ def test_coverage_matches_committed_summary(self): result = analyze(REPO) coverage = compute_coverage(result) self.assertEqual((coverage.total, coverage.cited_tests, coverage.verified), (40, 32, 17)) + self.assertEqual( + ( + coverage.functions_traced, + coverage.functions_total, + coverage.safety_functions_traced, + coverage.safety_functions_total, + ), + (22, 27, 22, 25), + ) + + def test_declared_non_safety_sr_does_not_inflate_safety_numerator(self): + """An SR on a declared non-safety function affects only the all-function traced numerator.""" + analysis = analyze(self.root) + functions = dict(analysis.functions) + reverse = dict(analysis.reverse) + functions['F-R-02'] = Function('F-R-02', 'Non-safety probe', 11) + reverse['F-R-02'] = ReverseEntry('F-R-02', 'Non-safety probe', ('SR-R-01',), True, 30) + coverage = compute_coverage(replace(analysis, functions=functions, reverse=reverse)) + self.assertEqual( + (coverage.functions_traced, coverage.safety_functions_traced, coverage.safety_functions_total), + (2, 1, 1), + ) + self.assertLessEqual(coverage.safety_functions_traced, coverage.safety_functions_total) + rendered = render_traceability( + (self.root / 'docs/safety/TRACEABILITY.md').read_text(encoding='utf-8'), + coverage, + ) + self.assertIn('excluding declared non-safety functions: 1 / 1 = 100 %', rendered) + + def test_json_exposes_safety_function_numerator(self): + """Machine-readable coverage distinguishes all-function and safety-only traced counts.""" + functions = _coverage_dict(compute_coverage(analyze(self.root)))['functions'] + self.assertEqual(functions['safety_traced'], 1) def test_generated_block_is_idempotent(self): """Rendering an already rendered traceability document is byte-idempotent.""" From 781c551529ec5cb164cf01980fba87b1e327a4d2 Mon Sep 17 00:00:00 2001 From: Raj Madhivanan Date: Sun, 13 Sep 2026 21:01:39 -0700 Subject: [PATCH 11/12] fix: validate every external value before using it The change-control workflows trusted several values that came from pull requests or automation metadata. Foreign issue links, missing commit identities, unchecked revisions, inherited credentials, and concurrent comment writers could produce misleading results or duplicate reports. This makes those boundaries explicit while keeping advisory comments and warn mode unchanged. ## What changed - Validated wire comparison revisions as full hexadecimal commit identifiers before invoking Git. - Rejected foreign and mixed-repository change-request links through one shared parser. - Required successful test evidence to name the exact pull-request head commit. - Removed GitHub and Actions credentials from revision-controlled coverage subprocesses while retaining the trusted parent token for comments. - Paginated coverage marker lookup so old comments are updated rather than duplicated. - Added per-PR cancellation groups to both marker-comment workflows. - Added adversarial tests for malformed revisions, foreign identity, absent commit identity, credential inheritance, pagination, and concurrency. - Kept enforcement-mode at warn and the wire signature at d0819037896320c1f40b88573770809b42e4a11b7055d65c4991c598116eee4e. ## Safety lifecycle Modification records, configuration management, and verification evidence integrity. Bears on IEC 61508-1:2010 sections 7.16 and 7.18.2, and IEC 61508-3:2010 sections 7.9 and 7.10. Co-Authored-By: OpenCode --- .github/workflows/change-control.yml | 4 + .github/workflows/coverage-delta.yml | 4 + scripts/check_wire_format.sh | 4 + tools/change_control/__main__.py | 17 +-- tools/change_control/checks.py | 38 +++-- tools/change_control/coverage_delta.py | 14 +- tools/change_control/self_test.py | 191 ++++++++++++++++++++++++- 7 files changed, 245 insertions(+), 27 deletions(-) diff --git a/.github/workflows/change-control.yml b/.github/workflows/change-control.yml index bdbd5615..f4c74998 100644 --- a/.github/workflows/change-control.yml +++ b/.github/workflows/change-control.yml @@ -7,6 +7,10 @@ on: pull_request_review: types: [submitted, dismissed] +concurrency: + group: change-control-${{ github.event.pull_request.number }} + cancel-in-progress: true + permissions: contents: read issues: write diff --git a/.github/workflows/coverage-delta.yml b/.github/workflows/coverage-delta.yml index 95d8cbb0..9132e5f3 100644 --- a/.github/workflows/coverage-delta.yml +++ b/.github/workflows/coverage-delta.yml @@ -4,6 +4,10 @@ name: Coverage delta on: pull_request: +concurrency: + group: coverage-delta-${{ github.event.pull_request.number }} + cancel-in-progress: true + permissions: contents: read issues: write diff --git a/scripts/check_wire_format.sh b/scripts/check_wire_format.sh index df9b94bc..c5cd97dd 100755 --- a/scripts/check_wire_format.sh +++ b/scripts/check_wire_format.sh @@ -11,6 +11,10 @@ cd "$ROOT" || exit 2 args=(check --root "$ROOT" --labels "${PSTOP_PR_LABELS:-}") if [ -n "${PSTOP_BASE_SHA:-}" ]; then + if [[ ! "$PSTOP_BASE_SHA" =~ ^[0-9A-Fa-f]{40}$ ]]; then + echo "wire-format: cannot run: PSTOP_BASE_SHA must be exactly 40 ASCII hexadecimal characters" >&2 + exit 2 + fi if ! git cat-file -e "$PSTOP_BASE_SHA:tools/change_control/wire_format.sha256" 2>/dev/null; then args+=(--initial-expectation) fi diff --git a/tools/change_control/__main__.py b/tools/change_control/__main__.py index b5aea4bc..e85f201c 100644 --- a/tools/change_control/__main__.py +++ b/tools/change_control/__main__.py @@ -11,7 +11,7 @@ from tools.safety_lint.model import LintError -from .checks import evaluate, load_mode, render_report, upsert_comment +from .checks import cr_number, evaluate, load_mode, render_report, upsert_comment class GhApi: @@ -64,18 +64,6 @@ def _require(mapping, path): return value -def _cr_number(body): - import re - - values = set( - re.findall( - r'(?im)^\s*(?:closes|refs)\s+(?:(?:https://github\.com/[^/]+/[^/]+/issues/)?#?)(\d+)\s*$', - body or '', - ) - ) - return int(next(iter(values))) if len(values) == 1 else None - - def collect(api, repository, pr_number): """Collect the complete GitHub snapshot used by pure policy evaluation.""" prefix = f'repos/{repository}' @@ -84,7 +72,7 @@ def collect(api, repository, pr_number): head = _require(pr, ('head', 'sha')) if 'body' not in pr or 'labels' not in pr: raise RuntimeError('partial GitHub response missing PR body or labels') - cr = _cr_number(pr['body']) + cr = cr_number(pr['body'], repository) issue = api('GET', f'{prefix}/issues/{cr}') if cr else {'labels': [], 'body': ''} comments = api('GET', f'{prefix}/issues/{cr}/comments', paginate=True) if cr else [] reviews = api('GET', f'{prefix}/pulls/{pr_number}/reviews', paginate=True) @@ -101,6 +89,7 @@ def collect(api, repository, pr_number): if not isinstance(value, list): raise RuntimeError(f'partial GitHub response: {name} is not a list') return { + 'repository': repository, 'pr': pr, 'issue': issue, 'issue_comments': comments, diff --git a/tools/change_control/checks.py b/tools/change_control/checks.py index 352a8c6e..82cc033e 100644 --- a/tools/change_control/checks.py +++ b/tools/change_control/checks.py @@ -90,12 +90,30 @@ def _labels(entity): return {label['name'] if isinstance(label, dict) else label for label in entity.get('labels', [])} -def _cr_number(body): - matches = re.findall( - r'(?im)^\s*(?:closes|refs)\s+(?:(?:https://github\.com/[^/]+/[^/]+/issues/)?#?)(\d+)\s*$', - body or '', - ) - return int(matches[0]) if len(set(matches)) == 1 else None +def cr_number(body, repository): + """Return one local CR number, rejecting foreign or ambiguous candidates.""" + if repository.count('/') != 1: + return None + current_owner, current_name = repository.casefold().split('/') + numbers = set() + foreign = False + for line in (body or '').splitlines(): + local = re.fullmatch(r'\s*(?:closes|refs)\s+#(\d+)\s*', line, re.IGNORECASE) + if local: + numbers.add(int(local.group(1))) + continue + url = re.fullmatch( + r'\s*(?:(?:closes|refs)\s+)?https://github\.com/([^/\s]+)/([^/\s]+)/issues/(\d+)/?\s*', + line, + re.IGNORECASE, + ) + if url: + owner, name, number = url.groups() + if (owner.casefold(), name.casefold()) == (current_owner, current_name): + numbers.add(int(number)) + else: + foreign = True + return next(iter(numbers)) if len(numbers) == 1 and not foreign else None def _issue_fields_complete(root, body): @@ -193,7 +211,7 @@ def evaluate(root, data): root = Path(root) pr = data['pr'] labels = _labels(pr) - cr_number = _cr_number(pr.get('body', '')) + linked_cr = cr_number(pr.get('body', ''), data['repository']) issue = data.get('issue', {}) issue_labels = _labels(issue) comments = data.get('issue_comments', []) @@ -230,9 +248,9 @@ def evaluate(root, data): len(authorization_times) >= required_authorizers and max(ia_times) <= min(authorization_times) ) ordered = before_implementation and after_analysis - missing_fields = _issue_fields_complete(root, issue.get('body', '')) if cr_number else ['change-request-link'] + missing_fields = _issue_fields_complete(root, issue.get('body', '')) if linked_cr else ['change-request-link'] okay = ( - cr_number is not None + linked_cr is not None and 'change-request' in issue_labels and 'status:authorized' in issue_labels and authorized_comment @@ -303,7 +321,7 @@ def evaluate(root, data): evidence = { item.get('name', '') for item in data.get('check_runs', []) - if item.get('head_sha', head) == head and item.get('conclusion') == 'success' + if item.get('head_sha') == head and item.get('conclusion') == 'success' } evidence.update( item.get('name', item.get('path', '')) diff --git a/tools/change_control/coverage_delta.py b/tools/change_control/coverage_delta.py index 01cb0c69..6b032f40 100644 --- a/tools/change_control/coverage_delta.py +++ b/tools/change_control/coverage_delta.py @@ -4,6 +4,7 @@ import argparse import json +import os import shutil import subprocess import sys @@ -64,9 +65,13 @@ def run_linter_at_tree(worktree): """Run the revision's own unchanged linter and add its parsed citation map.""" if not (worktree / 'tools/safety_lint/__main__.py').is_file(): return {'unavailable': 'tools/safety_lint is absent at this revision'} + child_environment = os.environ.copy() + for name in ('GH_TOKEN', 'GITHUB_TOKEN'): + child_environment.pop(name, None) result = subprocess.run( [sys.executable, '-m', 'tools.safety_lint', '--json'], cwd=worktree, + env=child_environment, check=False, capture_output=True, text=True, @@ -82,7 +87,12 @@ def run_linter_at_tree(worktree): 'print(json.dumps({r.sr_id: sorted(set(r.test_refs)) for r in analyze(".").trace}, sort_keys=True))' ) citations = subprocess.run( - [sys.executable, '-c', citation_code], cwd=worktree, check=False, capture_output=True, text=True + [sys.executable, '-c', citation_code], + cwd=worktree, + env=child_environment, + check=False, + capture_output=True, + text=True, ) if citations.returncode: raise RuntimeError(citations.stderr.strip() or 'cannot extract linter citations') @@ -113,7 +123,7 @@ def report_at_revision(root, revision): def upsert_coverage_comment(api, repository, pr, report): """Create or update the single marker-owned deterministic coverage comment.""" - comments = api('GET', f'repos/{repository}/issues/{pr}/comments') + comments = api('GET', f'repos/{repository}/issues/{pr}/comments', paginate=True) existing = next((comment for comment in comments if MARKER in comment.get('body', '')), None) body = f'{MARKER}\n{report}' if existing: diff --git a/tools/change_control/self_test.py b/tools/change_control/self_test.py index b730c90a..5cf47bb1 100755 --- a/tools/change_control/self_test.py +++ b/tools/change_control/self_test.py @@ -56,6 +56,7 @@ def snapshot(**overrides): '| | |', '| None | None |' ) data = { + 'repository': 'polymathrobotics/protective-stop', 'pr': { 'user': {'login': 'contributor'}, 'body': 'Closes #17', @@ -466,6 +467,42 @@ def test_multiple_change_request_links_fail(self): data['pr']['body'] = 'Closes #17\nRefs #18' self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E1').status, 'fail') + def test_same_repository_issue_url_is_accepted(self): + """A full issue URL for the current repository must identify its Change Request.""" + data = snapshot(repository='polymathrobotics/protective-stop') + data['pr']['body'] = 'https://github.com/polymathrobotics/protective-stop/issues/17' + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E1').status, 'pass') + + def test_foreign_repository_same_issue_number_is_rejected(self): + """A foreign issue URL must not map an equal issue number into the current repository.""" + data = snapshot(repository='polymathrobotics/protective-stop') + data['pr']['body'] = 'Refs https://github.com/other/protective-stop/issues/17' + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E1').status, 'fail') + + def test_foreign_repository_different_issue_number_is_rejected(self): + """A foreign issue URL must never select that number from the current repository.""" + data = snapshot(repository='polymathrobotics/protective-stop') + data['pr']['body'] = 'Closes https://github.com/other/project/issues/91' + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E1').status, 'fail') + + def test_mixed_local_and_foreign_issue_links_are_rejected(self): + """A local CR candidate mixed with any foreign candidate must be treated as ambiguous.""" + data = snapshot(repository='polymathrobotics/protective-stop') + data['pr']['body'] = 'Closes #17\nRefs https://github.com/other/project/issues/91' + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E1').status, 'fail') + + def test_duplicate_same_local_issue_link_is_accepted(self): + """Repeated equivalent local links must identify one unambiguous Change Request.""" + data = snapshot(repository='polymathrobotics/protective-stop') + data['pr']['body'] = 'Closes #17\nRefs #17' + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E1').status, 'pass') + + def test_repository_identity_comparison_is_case_insensitive(self): + """GitHub owner and repository casing must not make a same-repository URL foreign.""" + data = snapshot(repository='PolyMathRobotics/Protective-Stop') + data['pr']['body'] = 'Refs https://github.com/POLYMATHROBOTICS/protective-stop/issues/17' + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E1').status, 'pass') + def test_needs_change_request_label_exempts_e1(self): """External PRs awaiting a maintainer CR must produce a pending E1 result.""" data = snapshot() @@ -494,6 +531,36 @@ def test_e6_ignores_evidence_for_other_sha(self): data['check_runs'] = [{'name': 'host-check', 'conclusion': 'success', 'head_sha': 'oldsha'}] self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E6').status, 'fail') + def test_e6_ignores_check_evidence_without_head_sha(self): + """A successful check without explicit commit identity cannot satisfy the verification plan.""" + data = snapshot() + data['issue_comments'][1]['body'] += '\n# 7. Verification plan for this change\n`host-check`\n' + data['check_runs'] = [{'name': 'host-check', 'conclusion': 'success'}] + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E6').status, 'fail') + + def test_e6_ignores_workflow_evidence_with_null_head_sha(self): + """A successful workflow with null commit identity cannot satisfy the verification plan.""" + data = snapshot() + data['issue_comments'][1]['body'] += '\n# 7. Verification plan for this change\n`host-check`\n' + data['check_runs'] = [] + data['workflow_runs'] = [{'name': 'host-check', 'conclusion': 'success', 'head_sha': None}] + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E6').status, 'fail') + + def test_e6_ignores_check_evidence_with_empty_head_sha(self): + """A successful check with empty commit identity cannot satisfy the verification plan.""" + data = snapshot() + data['issue_comments'][1]['body'] += '\n# 7. Verification plan for this change\n`host-check`\n' + data['check_runs'] = [{'name': 'host-check', 'conclusion': 'success', 'head_sha': ''}] + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E6').status, 'fail') + + def test_e6_accepts_workflow_evidence_with_exact_head_sha(self): + """A successful exactly named workflow explicitly attached to PR head must satisfy the plan.""" + data = snapshot() + data['issue_comments'][1]['body'] += '\n# 7. Verification plan for this change\n`host-check`\n' + data['check_runs'] = [] + data['workflow_runs'] = [{'name': 'host-check', 'conclusion': 'success', 'head_sha': 'abc123'}] + self.assertEqual(next(item for item in evaluate(ROOT, data) if item.check_id == 'E6').status, 'pass') + def test_e6_does_not_accept_a_substring_check_name(self): """A short IA test token must not match an unrelated longer check-run name.""" data = snapshot() @@ -544,6 +611,71 @@ def _copy_wire_files(self, directory): _copy_headers = _copy_wire_files + def _run_wire_script(self, base_sha_marker, fake_git=False): + environment = os.environ.copy() + if base_sha_marker is None: + environment.pop('PSTOP_BASE_SHA', None) + else: + environment['PSTOP_BASE_SHA'] = base_sha_marker + marker = None + temporary = None + if fake_git: + temporary = tempfile.TemporaryDirectory() + directory = Path(temporary.name) + marker = directory / 'git-called' + fake = directory / 'git' + fake.write_text(f'#!/usr/bin/env bash\ntouch "{marker}"\nexit 99\n', encoding='utf-8') + fake.chmod(0o755) + environment['PATH'] = f'{directory}:{environment["PATH"]}' + result = subprocess.run( + ['scripts/check_wire_format.sh'], + cwd=ROOT, + env=environment, + check=False, + capture_output=True, + text=True, + ) + git_called = marker.exists() if marker else False + if temporary: + temporary.cleanup() + return result, git_called + + def test_wire_script_allows_unset_base_sha(self): + """A local invocation with PSTOP_BASE_SHA unset must run the current-tree check.""" + result, _ = self._run_wire_script(None) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_wire_script_allows_empty_base_sha(self): + """A local invocation with an empty PSTOP_BASE_SHA must run the current-tree check.""" + result, _ = self._run_wire_script('') + self.assertEqual(result.returncode, 0, result.stderr) + + def test_wire_script_accepts_valid_full_base_sha(self): + """An exact 40-character hexadecimal base SHA must reach the real git comparison seam.""" + head = subprocess.run( + ['git', 'rev-parse', 'HEAD'], cwd=ROOT, check=True, capture_output=True, text=True + ).stdout.strip() + result, _ = self._run_wire_script(head) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_wire_script_rejects_short_base_sha_before_git(self): + """A shortened base SHA must return cannot-run before invoking git.""" + result, git_called = self._run_wire_script('a' * 39, fake_git=True) + self.assertEqual((result.returncode, git_called), (2, False)) + self.assertIn('cannot run', result.stderr.lower()) + + def test_wire_script_rejects_nonhex_base_sha_before_git(self): + """A 40-character nonhex base SHA must return cannot-run before invoking git.""" + result, git_called = self._run_wire_script('g' * 40, fake_git=True) + self.assertEqual((result.returncode, git_called), (2, False)) + self.assertIn('PSTOP_BASE_SHA', result.stderr) + + def test_wire_script_rejects_option_like_base_sha_before_git(self): + """An option-like base value must never be passed to git as a revision argument.""" + result, git_called = self._run_wire_script('--help', fake_git=True) + self.assertEqual((result.returncode, git_called), (2, False)) + self.assertIn('PSTOP_BASE_SHA', result.stderr) + def test_signature_stable_across_comment_only_change(self): """Adding a C comment to a watched header must not alter its signature.""" with tempfile.TemporaryDirectory() as directory: @@ -806,6 +938,36 @@ def test_coverage_delta_detects_lost_citation(self): self.assertIn('SR-R-01', delta) self.assertIn('unresolvable', delta.lower()) + def test_revision_linter_subprocesses_cannot_observe_actions_credentials(self): + """Code from an untrusted revision must receive no Actions token while producing a normal report.""" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + package = root / 'tools/safety_lint' + package.mkdir(parents=True) + (root / 'tools/__init__.py').write_text('', encoding='utf-8') + (package / '__init__.py').write_text('', encoding='utf-8') + guard = ( + "if os.environ.get('GH_TOKEN') or os.environ.get('GITHUB_TOKEN'): raise RuntimeError('token leaked')" + ) + (package / '__main__.py').write_text( + 'import json, os\n' + f'{guard}\n' + "print(json.dumps({'coverage': {'cited_tests': 1, 'total': 1}, 'findings': []}))\n", + encoding='utf-8', + ) + (package / 'runner.py').write_text( + 'import os\nfrom types import SimpleNamespace\n' + f'{guard}\n' + "def analyze(root): return SimpleNamespace(trace=[SimpleNamespace(sr_id='SR-X-01', test_refs=['probe'])])\n", + encoding='utf-8', + ) + with mock.patch.dict(os.environ, {'GH_TOKEN': 'gh-sentinel', 'GITHUB_TOKEN': 'github-sentinel'}): + report = run_linter_at_tree(root) + self.assertEqual(os.environ['GH_TOKEN'], 'gh-sentinel') + self.assertEqual(os.environ['GITHUB_TOKEN'], 'github-sentinel') + self.assertEqual(report['coverage'], {'cited_tests': 1, 'total': 1}) + self.assertEqual(report['citations'], {'SR-X-01': ['probe']}) + def test_base_without_linter_exposes_stacked_dependency(self): """A base predating change-0001 must be reported as unavailable, never treated as zero coverage.""" delta = compare_reports( @@ -826,8 +988,9 @@ def test_coverage_comment_updates_instead_of_appending(self): """Coverage delta must update its marker-owned comment rather than append on every run.""" writes = [] - def api(method, path, body=None): + def api(method, path, body=None, paginate=False): if method == 'GET': + self.assertTrue(paginate) return [{'id': 12, 'body': '\nold'}] writes.append((method, path, body)) return {} @@ -835,6 +998,23 @@ def api(method, path, body=None): upsert_coverage_comment(api, 'acme/project', 7, 'new') self.assertEqual(writes[0][0:2], ('PATCH', 'repos/acme/project/issues/comments/12')) + def test_coverage_comment_marker_on_second_page_is_updated(self): + """Comment lookup must paginate so a marker beyond page one is updated rather than duplicated.""" + calls = [] + + def api(method, path, body=None, paginate=False): + calls.append((method, path, body, paginate)) + if method == 'GET': + self.assertTrue(paginate) + return [{'id': 1, 'body': 'first page'}, {'id': 12, 'body': '\nold'}] + return {} + + upsert_coverage_comment(api, 'acme/project', 7, 'new') + self.assertIn( + ('PATCH', 'repos/acme/project/issues/comments/12', {'body': '\nnew'}, False), calls + ) + self.assertFalse(any(call[0] == 'POST' for call in calls)) + def test_coverage_cli_no_comment_avoids_write_api(self): """Fork-safe coverage reporting must support stdout-only operation when write tokens are unavailable.""" result = subprocess.run( @@ -888,6 +1068,15 @@ def test_workflows_disable_writes_for_fork_pull_requests(self): self.assertIn('--no-comment', coverage) self.assertIn('CAN_LABEL', wire) + def test_marker_comment_workflows_cancel_superseded_pr_runs(self): + """Each marker-comment writer must cancel stale runs in its own per-PR concurrency group.""" + change = (ROOT / '.github/workflows/change-control.yml').read_text(encoding='utf-8') + coverage = (ROOT / '.github/workflows/coverage-delta.yml').read_text(encoding='utf-8') + self.assertIn('group: change-control-${{ github.event.pull_request.number }}', change) + self.assertIn('group: coverage-delta-${{ github.event.pull_request.number }}', coverage) + self.assertEqual(change.count('cancel-in-progress: true'), 1) + self.assertEqual(coverage.count('cancel-in-progress: true'), 1) + def test_warn_mode_exits_zero_with_findings(self): """Warn mode must report findings while returning success to the caller.""" result = self._run_cli('warn', {'responses': {}}) From 8189602a341fd67f22fa7946901ab14461da30bf Mon Sep 17 00:00:00 2001 From: Raj Madhivanan Date: Mon, 14 Sep 2026 08:26:40 -0700 Subject: [PATCH 12/12] fix: reject mistyped allocation continuations Allocation parsing still accepted several typo forms by keeping the valid prefix and dropping the malformed remainder. That could silently remove functions from traceability. This rejects those adjacent typo forms while preserving normal sentence punctuation and properly separated allocations. ## What changed - Rejected single dot, comma, and hyphen separators followed directly by digits. - Rejected malformed fully-prefixed slash members such as F-H-01/F-M02. - Added regression cases for every reproduced malformed form. - Disabled persisted checkout credentials for safety-lint CI. - Removed unused recursive submodule checkout from a repository with no submodules. - Confirmed citation coverage remains 32/40. ## Safety lifecycle Verification evidence integrity and requirements traceability. Bears on IEC 61508-3:2010 sections 7.9 and 7.10, Annex A.8.7 and A.8.8, and IEC 61508-1:2010 section 7.18.2. Co-Authored-By: OpenCode --- .github/workflows/safety-lint.yml | 2 +- tools/safety_lint/parse_srs.py | 2 ++ tools/safety_lint/self_test.py | 18 +++++++++++++++++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/safety-lint.yml b/.github/workflows/safety-lint.yml index 64f7fa09..36d74293 100644 --- a/.github/workflows/safety-lint.yml +++ b/.github/workflows/safety-lint.yml @@ -16,7 +16,7 @@ jobs: - name: Checkout uses: actions/checkout@v7 with: - submodules: recursive + persist-credentials: false - name: Safety linter self-tests run: python3 tools/safety_lint/self_test.py diff --git a/tools/safety_lint/parse_srs.py b/tools/safety_lint/parse_srs.py index ebaefb5a..8fb14f65 100644 --- a/tools/safety_lint/parse_srs.py +++ b/tools/safety_lint/parse_srs.py @@ -70,7 +70,9 @@ def expand_allocations(cell, path='', line=1): trailing = cell[match.end() :] malformed = ( (trailing.startswith('/') and not trailing.startswith('/F-')) + or (trailing.startswith('/F-') and re.match(r'^/F-[A-Z]-[0-9]{2}(?=$|[\s,.;)])', trailing) is None) or trailing.startswith('..') + or re.match(r'^[.,-][0-9]', trailing) is not None or (trailing and trailing[0].isalnum()) ) if malformed: diff --git a/tools/safety_lint/self_test.py b/tools/safety_lint/self_test.py index 2a4fca11..6bb39706 100755 --- a/tools/safety_lint/self_test.py +++ b/tools/safety_lint/self_test.py @@ -113,7 +113,17 @@ def test_malformed_trailing_slash_allocation_fails_with_source_location(self): def test_every_malformed_allocation_continuation_fails_in_helper(self): """Malformed slash, range, and numeric continuations can never leave a partial allocation.""" - for literal in ('F-R-01/XX', 'F-R-01..', 'F-R-01/2', 'F-R-01/003'): + for literal in ( + 'F-R-01/XX', + 'F-R-01..', + 'F-R-01/2', + 'F-R-01/003', + 'F-R-01.02', + 'F-R-01,02', + 'F-R-01-02', + 'F-H-01/F-M02', + 'F-H-01/F-M-1', + ): with self.subTest(literal=literal), self.assertRaisesRegex(LintError, 'malformed allocation'): expand_allocations(literal, 'doc.md', 21) @@ -911,6 +921,12 @@ def test_workflow_declares_contents_read_as_sole_top_level_permission(self): text = (REPO / '.github/workflows/safety-lint.yml').read_text(encoding='utf-8') self.assertIn('\npermissions:\n contents: read\n\njobs:', text) + def test_workflow_does_not_persist_credentials_or_fetch_submodules(self): + """Safety lint checkout keeps no Git credential and fetches no nonexistent submodules.""" + text = (REPO / '.github/workflows/safety-lint.yml').read_text(encoding='utf-8') + self.assertIn('persist-credentials: false', text) + self.assertNotIn('submodules:', text) + def run_cli(self, *args): return subprocess.run( [sys.executable, '-m', 'tools.safety_lint', '--root', str(self.root), *args],