From dac80e92ac53c4cebd73c4c357fe6d3800c55d59 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 28 Sep 2026 08:01:21 +0200 Subject: [PATCH 001/292] =?UTF-8?q?split(#4502=20PR-1):=20PVL=20rows=20fre?= =?UTF-8?q?sh=20from=20main=20=E2=80=94=20PMAT-4199=20EV-6b,=20PMAT-4200?= =?UTF-8?q?=20EV-7a,=20PMAT-4347,=20GH-4122=20EV-5a?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Takes the aprender-contracts / -cli / -staging trees and the additive aprender-common cli_roles module from #4502's r7 A head 0be7c00372, plus the cooperative-matrix delete (#4347) and the PVL ci/explicit-test-commands.d rows. Workspace versions stay at main's. Generated files are excluded (discharge-summary.json, Lean Axioms pin); 98 regenerates them as the single writer. Co-Authored-By: Claude Opus 5.5 --- ...nder-contracts-cli-pvl-discharge-check.cmd | 1 + ...r-contracts-cli-pvl-obligations-golden.cmd | 1 + ...ontracts-cli-pvl-discharge-leanchecker.cmd | 1 + ...-aprender-contracts-cli-pvl-comparator.cmd | 1 + ...2-aprender-contracts-cli-pvl-challenge.cmd | 1 + ...render-contracts-cli-pvl-ghost-binding.cmd | 3 + contracts/cooperative-matrix-gemm-v1.yaml | 79 - contracts/pvl-lint-ratchets-v1.yaml | 121 + crates/aprender-common/Cargo.toml | 7 + crates/aprender-common/src/cli_roles.rs | 732 +++ crates/aprender-common/src/lib.rs | 3 + .../src/backends/gpu/shaders/cooperative.rs | 55 - crates/aprender-contracts-cli/Cargo.toml | 8 + .../src/bin/pv-sat/liskov.rs | 200 + .../src/bin/pv-sat/main.rs | 243 + .../src/bin/pv-sat/plant.rs | 63 + .../src/bin/pv-sat/sat.rs | 75 + .../src/bin/pv-sat/tests.rs | 234 + crates/aprender-contracts-cli/src/cli.rs | 214 +- .../src/commands/census.rs | 20 + .../src/commands/challenge.rs | 89 + .../src/commands/discharge.rs | 1991 ++++++++ .../src/commands/extract_rdf.rs | 58 +- .../src/commands/lint.rs | 395 +- .../src/commands/lint_arming.rs | 81 +- .../src/commands/mod.rs | 4 + .../src/commands/obligations.rs | 444 ++ .../src/commands/ontology.rs | 81 + .../src/commands/proof_status.rs | 92 +- .../src/commands/verify_bindings.rs | 184 +- .../src/contract_walk.rs | 50 +- crates/aprender-contracts-cli/src/lib.rs | 30 +- .../tests/ev11_lint_ratchets.rs | 219 + .../tests/fixtures/pvl/obligations/README.md | 86 + .../tests/fixtures/pvl/obligations/broken.rc | 1 + .../fixtures/pvl/obligations/broken.stdout | 7 + .../broken/contracts/.hidden-v1.yaml | 1 + .../broken/contracts/a-invalid-v1.yaml | 8 + .../broken/contracts/b-hidden-v1.yaml | 29 + .../broken/contracts/c-unbound-v1.yaml | 36 + .../broken/contracts/d-proved-v1.yaml | 28 + .../obligations/broken/contracts/notes.yml | 1 + .../broken/contracts/sub/nested-v1.yaml | 1 + .../broken/contracts/z-binding.yaml | 1 + .../obligations/broken/src/deep/real.rs.txt | 3 + .../pvl/obligations/broken/src/gate.rs.txt | 4 + .../pvl/obligations/broken/src/ok.rs.txt | 3 + .../obligations/broken/src/prefixed.rs.txt | 4 + .../tests/fixtures/pvl/obligations/edge.rc | 1 + .../fixtures/pvl/obligations/edge.stdout | 4 + .../edge/contracts/e-merge-v1.yaml | 33 + .../edge/contracts/f-falsy-v1.yaml | 38 + .../edge/contracts/g-unused-type-v1.yaml | 25 + .../edge/contracts/h-false-doc-v1.yaml | 1 + .../pvl/obligations/edge/src/real.rs.txt | 1 + .../tests/fixtures/pvl/obligations/pmat.rc | 1 + .../fixtures/pvl/obligations/pmat.stdout | 1 + .../pmat/contracts/benchmarking-v1.yaml | 64 + .../obligations/pmat/contracts/binding.yaml | 308 ++ .../pmat/contracts/cli-usage-lines-v1.yaml | 71 + .../pmat/contracts/commit-enforcement-v1.yaml | 60 + .../pmat/contracts/comply-gate-effect-v1.yaml | 410 ++ .../contracts/comply-numeric-claims-v1.yaml | 283 ++ .../pmat/contracts/comply-ratchet-v1.yaml | 323 ++ .../comply-threshold-coherence-v1.yaml | 269 + .../pmat/contracts/dead-code-cache-v1.yaml | 79 + .../pmat/contracts/dogfood-published-v1.yaml | 50 + .../pmat/contracts/hook-debt-scope-v1.yaml | 136 + .../pmat/contracts/index-faithful-v1.yaml | 83 + .../pmat/contracts/macs-artifacts-v1.yaml | 96 + .../pmat/contracts/macs-cot-v1.yaml | 70 + .../pmat/contracts/macs-ladder-kernel-v1.yaml | 126 + .../pmat/contracts/macs-ladder-v1.yaml | 68 + .../pmat/contracts/macs-provenance-v1.yaml | 61 + .../pmat/contracts/macs-skill-effort-v1.yaml | 42 + .../pmat/contracts/macs-sweep-v1.yaml | 98 + .../pmat/contracts/pmat-book-build-v1.yaml | 71 + .../pmat/contracts/pmat-book-ch05-v1.yaml | 39 + .../pmat/contracts/pmat-book-ch07-v1.yaml | 37 + .../pmat/contracts/pmat-book-ch13-v1.yaml | 39 + .../pmat/contracts/pmat-book-ch35-v1.yaml | 42 + .../obligations/pmat/contracts/pmat-core.yaml | 287 ++ .../pmat/contracts/pmat-install-v1.yaml | 82 + .../contracts/pmat-no-fabrication-v1.yaml | 213 + .../contracts/pmat-quality-acceptance-v1.yaml | 83 + .../contracts/pmat-query-search-modes-v1.yaml | 97 + .../contracts/quality-check-content-v1.yaml | 73 + .../quality-gate-not-measured-v1.yaml | 78 + .../pmat/contracts/quorum-review-v1.yaml | 74 + .../pmat/contracts/tdg-grade-order-v1.yaml | 246 + .../pmat/contracts/verify-verdict-v1.yaml | 65 + .../pmat/contracts/work-ladder-claim-v1.yaml | 78 + .../pmat/contracts/workspace-scoring-v1.yaml | 74 + .../check_handlers/check_tdg_grade.rs.txt | 2596 ++++++++++ .../comply_handlers/muda_handlers.rs.txt | 160 + .../cli/handlers/work_contract_scoring.rs.txt | 468 ++ .../src/services/file_health_types.rs.txt | 240 + .../src/services/infra_score/models.rs.txt | 549 ++ .../pmat/src/services/normalized_score.rs.txt | 234 + .../services/repo_score/models_impls.rs.txt | 216 + .../rust_project_score/models_score.rs.txt | 155 + .../pmat/src/tdg/cuda_simd_scores.rs.txt | 293 ++ .../pvl/obligations/pmat/src/tdg/grade.rs.txt | 502 ++ .../tests/includes/dispatch_tests.rs | 29 +- .../tests/includes/version_identity_unit.rs | 25 + .../tests/ont2c_owl_tbox.rs | 196 + .../tests/ont4b_shapes_gate.rs | 24 +- .../tests/ont4c3_parity_receipts.rs | 20 +- .../tests/ont4c5_capability_cells.rs | 371 ++ .../tests/ont4c_doc_contracts.rs | 462 ++ .../tests/ont4d_subsumption.rs | 230 + .../tests/ont4e_refines_gate.rs | 240 + .../tests/ont4f_github_entities.rs | 167 + .../tests/ont5_consistency_gate.rs | 178 + .../tests/ont6_lint_verdict.rs | 53 +- .../tests/ont8_evidence_gate.rs | 213 + .../tests/ont9_self_contract.rs | 233 + .../tests/ont_release_readiness.rs | 1245 +++-- .../tests/pv_surface_gate.rs | 23 +- .../tests/pvl_challenge.rs | 199 + .../tests/pvl_comparator.rs | 295 ++ .../tests/pvl_discharge_check.rs | 440 ++ .../tests/pvl_discharge_leanchecker.rs | 167 + .../tests/pvl_ghost_binding.rs | 120 + .../tests/pvl_obligations_golden.rs | 291 ++ .../tests/pvl_zero_contracts.rs | 22 +- .../tests/shapes_vacuity.rs | 360 ++ .../contracts/cooperative-matrix-gemm-v1.md | 68 - .../book/src/verification-ladder.md | 44 +- .../contracts/cooperative-matrix-gemm-v1.yaml | 78 - .../specifications/sub/verification-ladder.md | 60 +- .../lean/Challenge/activation-kernel-v1.lean | 130 + .../lean/Challenge/alibi-kernel-v1.lean | 117 + .../lean/Challenge/apr-cli-sampling-v1.lean | 73 + .../lean/Challenge/attention-kernel-v1.lean | 205 + .../lean/Challenge/bpe-tokenization-v1.lean | 11 + .../lean/Challenge/conv1d-kernel-v1.lean | 85 + .../lean/Challenge/embedding-lookup-v1.lean | 159 + .../lean/Challenge/encoder-roundtrip-v1.lean | 32 + .../lean/Challenge/fp8-interchange-v1.lean | 60 + .../lean/Challenge/gated-delta-net-v1.lean | 10 + .../lean/Challenge/gqa-kernel-v1.lean | 62 + .../lean/Challenge/kernel-fusion-v1.lean | 40 + .../lean/Challenge/layernorm-kernel-v1.lean | 85 + .../lean/Challenge/matmul-kernel-v1.lean | 11 + .../lean/Challenge/q4k-q6k-superblock-v1.lean | 16 + .../lean/Challenge/silu-kernel-v1.lean | 93 + .../lean/Challenge/softmax-kernel-v1.lean | 57 + .../lean/Challenge/transpose-kernel-v1.lean | 18 + .../lean/ProvableContracts.lean | 24 +- .../lean/ProvableContracts/Defs/GPU.lean | 2 +- .../lean/ProvableContracts/Defs/Rand.lean | 6 +- .../Theorems/Alibi/Slopes.lean | 6 +- .../Theorems/AprCode/HarnessIrRoundtrip.lean | 6 +- .../ProvableContracts/Theorems/BLAS/Trmm.lean | 2 + .../ProvableContracts/Theorems/BLAS/Trsm.lean | 2 + .../CrossEntropy/CoreSoftmaxInvariants.lean | 2 +- .../lean/ProvableContracts/Theorems/DPO.lean | 91 +- .../ProvableContracts/Theorems/FFT/Fft2d.lean | 2 +- .../ProvableContracts/Theorems/FFT/Fft3d.lean | 2 +- .../Theorems/FFT/FftBatched.lean | 2 +- .../Theorems/GPU/DimensionIndependence.lean | 4 +- .../Theorems/Gqa/HeadMapping.lean | 1 - .../Theorems/Image/Canny.lean | 2 + .../Theorems/Image/Conv2d.lean | 2 +- .../Theorems/Image/Histogram.lean | 4 +- .../Theorems/Image/Morphology.lean | 2 +- .../Theorems/Image/Resize.lean | 2 + .../Theorems/Image/RgbToGray.lean | 2 + .../Theorems/Metrics/RegressionAnalytic.lean | 5 +- .../Theorems/Quantization/NF4Dequant.lean | 52 +- .../Theorems/Rand/Philox.lean | 2 +- .../Theorems/Rand/Threefry.lean | 2 +- .../Theorems/Sigmoid/SiluAsymptotic.lean | 2 +- .../Theorems/Sparse/Spgemm.lean | 2 + .../Theorems/Sparse/Spmm.lean | 2 + .../aprender-contracts-staging/lean/build.sh | 117 + .../lean/check-orphans.sh | 137 + .../lean/conleche.sh | 193 + .../lean/escape-allowlist.yaml | 5 + .../lean/fixtures/build/clean/lake.out | 2 + .../lean/fixtures/build/clean/want | 1 + .../lean/fixtures/build/clean/want-msg | 1 + .../cold-cache-failed-build-is-red/build-rc | 1 + .../cold-cache-failed-build-is-red/lake.out | 4 + .../build/cold-cache-failed-build-is-red/want | 1 + .../cold-cache-failed-build-is-red/want-msg | 3 + .../cold-cache-our-warning-is-red/lake.out | 4 + .../build/cold-cache-our-warning-is-red/want | 1 + .../cold-cache-our-warning-is-red/want-msg | 1 + .../dot-prefixed-warning-is-red/lake.out | 4 + .../build/dot-prefixed-warning-is-red/want | 1 + .../dot-prefixed-warning-is-red/want-msg | 1 + .../build/failed-build-is-red/build-rc | 1 + .../build/failed-build-is-red/lake.out | 3 + .../fixtures/build/failed-build-is-red/want | 1 + .../build/failed-build-is-red/want-msg | 1 + .../lake.out | 3 + .../mathlib-elaborated-is-a-cache-miss/want | 1 + .../want-msg | 1 + .../build/mathlib-warning-is-ignored/lake.out | 3 + .../build/mathlib-warning-is-ignored/want | 1 + .../build/mathlib-warning-is-ignored/want-msg | 1 + .../native-object-is-not-a-miss/lake.out | 3 + .../build/native-object-is-not-a-miss/want | 1 + .../native-object-is-not-a-miss/want-msg | 1 + .../build/our-warning-is-red/lake.out | 4 + .../fixtures/build/our-warning-is-red/want | 1 + .../build/our-warning-is-red/want-msg | 1 + .../build/root-module-warning-is-red/lake.out | 4 + .../build/root-module-warning-is-red/want | 1 + .../build/root-module-warning-is-red/want-msg | 1 + .../orphans/clean/ProvableContracts.lean | 1 + .../orphans/clean/ProvableContracts/A.lean | 1 + .../orphans/clean/ProvableContracts/B.lean | 1 + .../orphans/clean/ProvableContracts/C.lean | 1 + .../orphans/clean/orphan-allowlist.yaml | 3 + .../lean/fixtures/orphans/clean/want | 1 + .../lean/fixtures/orphans/clean/want-msg | 1 + .../ProvableContracts.lean | 1 + .../ProvableContracts/A.lean | 5 + .../ProvableContracts/C.lean | 1 + .../doc-comment-import-is-not-an-import/want | 1 + .../want-msg | 1 + .../ProvableContracts.lean | 1 + .../ProvableContracts/A.lean | 1 + .../ProvableContracts/B.lean | 1 + .../ProvableContracts/C.lean | 1 + .../orphan-allowlist.yaml | 2 + .../orphans/entry-without-reason-is-red/want | 1 + .../entry-without-reason-is-red/want-msg | 1 + .../ProvableContracts.lean | 1 + .../ProvableContracts/A.lean | 1 + .../ProvableContracts/B.lean | 1 + .../ProvableContracts/C.lean | 1 + .../orphan-allowlist.yaml | 2 + .../orphans/entry-without-ticket-is-red/want | 1 + .../entry-without-ticket-is-red/want-msg | 1 + .../ProvableContracts.lean | 2 + .../ProvableContracts/A.lean | 1 + .../want | 1 + .../want-msg | 1 + .../ProvableContracts.lean | 4 + .../ProvableContracts/A.lean | 1 + .../nested-header-comment-before-imports/want | 1 + .../want-msg | 1 + .../no-root-declines/ProvableContracts/A.lean | 1 + .../fixtures/orphans/no-root-declines/want | 1 + .../orphans/no-root-declines/want-msg | 1 + .../stale-entry-is-red/ProvableContracts.lean | 1 + .../ProvableContracts/A.lean | 1 + .../ProvableContracts/B.lean | 1 + .../ProvableContracts/C.lean | 1 + .../stale-entry-is-red/orphan-allowlist.yaml | 6 + .../fixtures/orphans/stale-entry-is-red/want | 1 + .../orphans/stale-entry-is-red/want-msg | 1 + .../ProvableContracts.lean | 1 + .../ProvableContracts/A.lean | 1 + .../ProvableContracts/B.lean | 1 + .../orphans/transitive-import-is-in-cone/want | 1 + .../transitive-import-is-in-cone/want-msg | 1 + .../ProvableContracts.lean | 1 + .../ProvableContracts/A.lean | 1 + .../ProvableContracts/B.lean | 1 + .../ProvableContracts/C.lean | 1 + .../orphans/unlisted-orphan-is-red/want | 1 + .../orphans/unlisted-orphan-is-red/want-msg | 1 + .../lean/formalization.yaml | 50 + .../lean/lake-manifest.json | 2 +- .../lean/lakefile.lean | 2 +- .../lean/orphan-allowlist.yaml | 172 + .../lean/scripts/Comparator.lean | 219 + .../lean/unresolved-labels.json | 4517 +++++++++++++++++ crates/aprender-contracts/Cargo.toml | 9 + crates/aprender-contracts/build.rs | 17 +- .../fixtures/liskov-corrupt.json | 35 + .../fixtures/unsat-core-corrupt.json | 14 + crates/aprender-contracts/src/binding.rs | 22 +- crates/aprender-contracts/src/build_helper.rs | 183 +- .../src/discharge/challenge.rs | 448 ++ .../src/discharge/challenge_tests.rs | 262 + .../src/discharge/comparator.rs | 237 + .../src/discharge/comparator_tests.rs | 403 ++ .../src/discharge/formalization.rs | 137 + .../src/discharge/formalization_tests.rs | 135 + .../aprender-contracts/src/discharge/lex.rs | 368 ++ .../aprender-contracts/src/discharge/mod.rs | 828 +++ .../src/discharge/refinement.rs | 216 + .../src/discharge/refinement_tests.rs | 170 + .../src/discharge/summary.rs | 311 ++ .../src/discharge/summary_tests.rs | 264 + .../aprender-contracts/src/discharge/tests.rs | 719 +++ crates/aprender-contracts/src/env_key.rs | 16 + crates/aprender-contracts/src/explain.rs | 3 + .../aprender-contracts/src/explain_render.rs | 16 +- .../aprender-contracts/src/explain_tests.rs | 20 +- crates/aprender-contracts/src/kani_assume.rs | 273 + .../src/kani_assume_tests.rs | 148 + crates/aprender-contracts/src/kani_gen/mod.rs | 44 +- .../src/kani_gen/strategies.rs | 35 + .../src/kernels/kani_proofs.rs | 41 +- .../src/kernels/kani_proofs_ab.rs | 38 +- .../src/kernels/kani_proofs_cd.rs | 62 +- crates/aprender-contracts/src/lean_gen/mod.rs | 98 +- crates/aprender-contracts/src/levels.rs | 256 + crates/aprender-contracts/src/lib.rs | 4 + .../src/lint/bindings_gate.rs | 345 ++ .../src/lint/bindings_gate_tests.rs | 364 ++ .../src/lint/capability_cells_gate.rs | 297 ++ .../src/lint/capability_cells_plant.yaml | 14 + .../src/lint/consistency_gate.rs | 267 + .../src/lint/consistency_gate_tests.rs | 263 + .../src/lint/evidence_gate.rs | 500 ++ .../src/lint/evidence_gate_tests.rs | 364 ++ crates/aprender-contracts/src/lint/mod.rs | 729 ++- .../aprender-contracts/src/lint/mod_tests.rs | 256 +- .../src/lint/ratchet_gates.rs | 397 ++ .../src/lint/ratchet_gates_tests.rs | 546 ++ .../src/lint/refinement_gate.rs | 492 ++ .../src/lint/refinement_gate_tests.rs | 370 ++ .../src/lint/refines_gate.rs | 336 ++ .../src/lint/refines_gate_tests.rs | 188 + .../src/lint/relations_gate.rs | 104 +- crates/aprender-contracts/src/lint/rules.rs | 2 +- .../src/lint/shapes_gate.rs | 758 ++- .../src/lint/subsumption.rs | 298 ++ .../aprender-contracts/src/lint/tbox_gate.rs | 149 + .../src/ontology/capability_cells.rs | 277 + .../src/ontology/capability_cells_tests.rs | 400 ++ .../src/ontology/extract/apr_model.rs | 2 + .../src/ontology/extract/binary.rs | 634 +++ .../src/ontology/extract/binary_tests.rs | 279 + .../src/ontology/extract/claims.rs | 770 +++ .../src/ontology/extract/cli_surface.rs | 464 ++ .../src/ontology/extract/code.rs | 855 +++- .../src/ontology/extract/covering.rs | 201 + .../src/ontology/extract/csv.rs | 249 + .../src/ontology/extract/example.rs | 335 ++ .../src/ontology/extract/example_tests.rs | 233 + .../src/ontology/extract/gguf.rs | 2 + .../src/ontology/extract/json.rs | 3 + .../src/ontology/extract/json/github.rs | 972 ++++ .../src/ontology/extract/kernel.rs | 598 +++ .../src/ontology/extract/kernel_tests.rs | 300 ++ .../src/ontology/extract/llm_context.rs | 322 ++ .../src/ontology/extract/mod.rs | 86 +- .../src/ontology/extract/pv_contract.rs | 11 +- .../src/ontology/extract/readme.rs | 268 + .../src/ontology/extract/release_cells.rs | 546 ++ .../src/ontology/extract/release_crux.rs | 428 ++ .../src/ontology/extract/release_evidence.rs | 636 ++- .../extract/release_evidence_tests.rs | 246 +- .../src/ontology/extract/release_inputs.rs | 43 + .../aprender-contracts/src/ontology/liskov.rs | 552 ++ .../src/ontology/liskov_tests.rs | 389 ++ .../src/ontology/measured_sets.rs | 176 + crates/aprender-contracts/src/ontology/mod.rs | 14 + crates/aprender-contracts/src/ontology/owl.rs | 558 ++ .../src/ontology/receipts.rs | 58 +- .../aprender-contracts/src/ontology/shapes.rs | 822 ++- .../aprender-contracts/src/ontology/sigma.rs | 444 +- crates/aprender-contracts/src/ontology/w3c.rs | 20 +- .../src/ontology/witness.rs | 624 +++ .../src/ontology/witness_planted.rs | 287 ++ .../src/ontology/witness_small.rs | 458 ++ .../src/ontology/witness_tests.rs | 233 + crates/aprender-contracts/src/proof_status.rs | 303 +- .../src/proof_status_tests.rs | 309 +- crates/aprender-contracts/src/readme_gen.rs | 49 +- .../src/schema/artifact_tests.rs | 37 + .../aprender-contracts/src/schema/clause.rs | 58 + crates/aprender-contracts/src/schema/mod.rs | 24 +- crates/aprender-contracts/src/schema/types.rs | 17 +- .../src/schema/types_tests.rs | 1 + .../src/schema/validator.rs | 36 + .../src/schema/validator_tests.rs | 39 + .../src/schema/validator_tests_top_level.rs | 14 +- .../src/scoring/codebase.rs | 8 +- .../src/scoring/codebase_tests.rs | 2 + crates/aprender-contracts/src/scoring/mod.rs | 3 +- .../src/scoring/scoring_probe_tests.rs | 2 + .../src/scoring/scoring_tests.rs | 2 + crates/aprender-contracts/src/tree.rs | 254 + .../tests/fixtures/ont4g/snapshot.jsonl | 3 + .../tests/fixtures/ont4g/surface_audit.csv | 9 + .../w3c/property-lessThan-001.yaml | 35 + .../w3c/property-lessThan-002.yaml | 23 + .../w3c/property-lessThanOrEquals-001.yaml | 34 + .../quorum-GH-4122-pvl-5a-degraded.json | 38 + scripts/check_lean_modules_reachable.sh | 112 + scripts/check_leanchecker_scoped.sh | 102 + tests/fixtures/pvl/ghost-binding.yaml | 12 + tests/fixtures/pvl/resolved-binding.yaml | 12 + 393 files changed, 55108 insertions(+), 1876 deletions(-) create mode 100644 ci/explicit-test-commands.d/461-aprender-contracts-cli-pvl-discharge-check.cmd create mode 100644 ci/explicit-test-commands.d/462-aprender-contracts-cli-pvl-obligations-golden.cmd create mode 100644 ci/explicit-test-commands.d/471-aprender-contracts-cli-pvl-discharge-leanchecker.cmd create mode 100644 ci/explicit-test-commands.d/481-aprender-contracts-cli-pvl-comparator.cmd create mode 100644 ci/explicit-test-commands.d/482-aprender-contracts-cli-pvl-challenge.cmd create mode 100644 ci/explicit-test-commands.d/491-aprender-contracts-cli-pvl-ghost-binding.cmd delete mode 100644 contracts/cooperative-matrix-gemm-v1.yaml create mode 100644 contracts/pvl-lint-ratchets-v1.yaml create mode 100644 crates/aprender-common/src/cli_roles.rs delete mode 100644 crates/aprender-compute/src/backends/gpu/shaders/cooperative.rs create mode 100644 crates/aprender-contracts-cli/src/bin/pv-sat/liskov.rs create mode 100644 crates/aprender-contracts-cli/src/bin/pv-sat/main.rs create mode 100644 crates/aprender-contracts-cli/src/bin/pv-sat/plant.rs create mode 100644 crates/aprender-contracts-cli/src/bin/pv-sat/sat.rs create mode 100644 crates/aprender-contracts-cli/src/bin/pv-sat/tests.rs create mode 100644 crates/aprender-contracts-cli/src/commands/challenge.rs create mode 100644 crates/aprender-contracts-cli/src/commands/discharge.rs create mode 100644 crates/aprender-contracts-cli/src/commands/obligations.rs create mode 100644 crates/aprender-contracts-cli/src/commands/ontology.rs create mode 100644 crates/aprender-contracts-cli/tests/ev11_lint_ratchets.rs create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/README.md create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken.rc create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken.stdout create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/.hidden-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/a-invalid-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/b-hidden-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/c-unbound-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/d-proved-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/notes.yml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/sub/nested-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/z-binding.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/src/deep/real.rs.txt create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/src/gate.rs.txt create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/src/ok.rs.txt create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/src/prefixed.rs.txt create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge.rc create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge.stdout create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge/contracts/e-merge-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge/contracts/f-falsy-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge/contracts/g-unused-type-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge/contracts/h-false-doc-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge/src/real.rs.txt create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat.rc create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat.stdout create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/benchmarking-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/binding.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/cli-usage-lines-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/commit-enforcement-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/comply-gate-effect-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/comply-numeric-claims-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/comply-ratchet-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/comply-threshold-coherence-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/dead-code-cache-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/dogfood-published-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/hook-debt-scope-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/index-faithful-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-artifacts-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-cot-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-ladder-kernel-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-ladder-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-provenance-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-skill-effort-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-sweep-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-book-build-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-book-ch05-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-book-ch07-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-book-ch13-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-book-ch35-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-core.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-install-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-no-fabrication-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-quality-acceptance-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-query-search-modes-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/quality-check-content-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/quality-gate-not-measured-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/quorum-review-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/tdg-grade-order-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/verify-verdict-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/work-ladder-claim-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/workspace-scoring-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/cli/handlers/comply_handlers/check_handlers/check_tdg_grade.rs.txt create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/cli/handlers/comply_handlers/muda_handlers.rs.txt create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/cli/handlers/work_contract_scoring.rs.txt create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/services/file_health_types.rs.txt create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/services/infra_score/models.rs.txt create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/services/normalized_score.rs.txt create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/services/repo_score/models_impls.rs.txt create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/services/rust_project_score/models_score.rs.txt create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/tdg/cuda_simd_scores.rs.txt create mode 100644 crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/tdg/grade.rs.txt create mode 100644 crates/aprender-contracts-cli/tests/ont2c_owl_tbox.rs create mode 100644 crates/aprender-contracts-cli/tests/ont4c5_capability_cells.rs create mode 100644 crates/aprender-contracts-cli/tests/ont4c_doc_contracts.rs create mode 100644 crates/aprender-contracts-cli/tests/ont4d_subsumption.rs create mode 100644 crates/aprender-contracts-cli/tests/ont4e_refines_gate.rs create mode 100644 crates/aprender-contracts-cli/tests/ont4f_github_entities.rs create mode 100644 crates/aprender-contracts-cli/tests/ont5_consistency_gate.rs create mode 100644 crates/aprender-contracts-cli/tests/ont8_evidence_gate.rs create mode 100644 crates/aprender-contracts-cli/tests/ont9_self_contract.rs create mode 100644 crates/aprender-contracts-cli/tests/pvl_challenge.rs create mode 100644 crates/aprender-contracts-cli/tests/pvl_comparator.rs create mode 100644 crates/aprender-contracts-cli/tests/pvl_discharge_check.rs create mode 100644 crates/aprender-contracts-cli/tests/pvl_discharge_leanchecker.rs create mode 100644 crates/aprender-contracts-cli/tests/pvl_ghost_binding.rs create mode 100644 crates/aprender-contracts-cli/tests/pvl_obligations_golden.rs create mode 100644 crates/aprender-contracts-cli/tests/shapes_vacuity.rs delete mode 100644 crates/aprender-contracts-staging/book/src/contracts/cooperative-matrix-gemm-v1.md delete mode 100644 crates/aprender-contracts-staging/contracts/cooperative-matrix-gemm-v1.yaml create mode 100644 crates/aprender-contracts-staging/lean/Challenge/activation-kernel-v1.lean create mode 100644 crates/aprender-contracts-staging/lean/Challenge/alibi-kernel-v1.lean create mode 100644 crates/aprender-contracts-staging/lean/Challenge/apr-cli-sampling-v1.lean create mode 100644 crates/aprender-contracts-staging/lean/Challenge/attention-kernel-v1.lean create mode 100644 crates/aprender-contracts-staging/lean/Challenge/bpe-tokenization-v1.lean create mode 100644 crates/aprender-contracts-staging/lean/Challenge/conv1d-kernel-v1.lean create mode 100644 crates/aprender-contracts-staging/lean/Challenge/embedding-lookup-v1.lean create mode 100644 crates/aprender-contracts-staging/lean/Challenge/encoder-roundtrip-v1.lean create mode 100644 crates/aprender-contracts-staging/lean/Challenge/fp8-interchange-v1.lean create mode 100644 crates/aprender-contracts-staging/lean/Challenge/gated-delta-net-v1.lean create mode 100644 crates/aprender-contracts-staging/lean/Challenge/gqa-kernel-v1.lean create mode 100644 crates/aprender-contracts-staging/lean/Challenge/kernel-fusion-v1.lean create mode 100644 crates/aprender-contracts-staging/lean/Challenge/layernorm-kernel-v1.lean create mode 100644 crates/aprender-contracts-staging/lean/Challenge/matmul-kernel-v1.lean create mode 100644 crates/aprender-contracts-staging/lean/Challenge/q4k-q6k-superblock-v1.lean create mode 100644 crates/aprender-contracts-staging/lean/Challenge/silu-kernel-v1.lean create mode 100644 crates/aprender-contracts-staging/lean/Challenge/softmax-kernel-v1.lean create mode 100644 crates/aprender-contracts-staging/lean/Challenge/transpose-kernel-v1.lean create mode 100755 crates/aprender-contracts-staging/lean/build.sh create mode 100755 crates/aprender-contracts-staging/lean/check-orphans.sh create mode 100755 crates/aprender-contracts-staging/lean/conleche.sh create mode 100644 crates/aprender-contracts-staging/lean/escape-allowlist.yaml create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/clean/lake.out create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/clean/want create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/clean/want-msg create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/cold-cache-failed-build-is-red/build-rc create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/cold-cache-failed-build-is-red/lake.out create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/cold-cache-failed-build-is-red/want create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/cold-cache-failed-build-is-red/want-msg create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/cold-cache-our-warning-is-red/lake.out create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/cold-cache-our-warning-is-red/want create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/cold-cache-our-warning-is-red/want-msg create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/dot-prefixed-warning-is-red/lake.out create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/dot-prefixed-warning-is-red/want create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/dot-prefixed-warning-is-red/want-msg create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/failed-build-is-red/build-rc create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/failed-build-is-red/lake.out create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/failed-build-is-red/want create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/failed-build-is-red/want-msg create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/mathlib-elaborated-is-a-cache-miss/lake.out create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/mathlib-elaborated-is-a-cache-miss/want create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/mathlib-elaborated-is-a-cache-miss/want-msg create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/mathlib-warning-is-ignored/lake.out create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/mathlib-warning-is-ignored/want create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/mathlib-warning-is-ignored/want-msg create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/native-object-is-not-a-miss/lake.out create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/native-object-is-not-a-miss/want create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/native-object-is-not-a-miss/want-msg create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/our-warning-is-red/lake.out create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/our-warning-is-red/want create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/our-warning-is-red/want-msg create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/root-module-warning-is-red/lake.out create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/root-module-warning-is-red/want create mode 100644 crates/aprender-contracts-staging/lean/fixtures/build/root-module-warning-is-red/want-msg create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/clean/ProvableContracts.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/clean/ProvableContracts/A.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/clean/ProvableContracts/B.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/clean/ProvableContracts/C.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/clean/orphan-allowlist.yaml create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/clean/want create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/clean/want-msg create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/doc-comment-import-is-not-an-import/ProvableContracts.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/doc-comment-import-is-not-an-import/ProvableContracts/A.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/doc-comment-import-is-not-an-import/ProvableContracts/C.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/doc-comment-import-is-not-an-import/want create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/doc-comment-import-is-not-an-import/want-msg create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/entry-without-reason-is-red/ProvableContracts.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/entry-without-reason-is-red/ProvableContracts/A.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/entry-without-reason-is-red/ProvableContracts/B.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/entry-without-reason-is-red/ProvableContracts/C.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/entry-without-reason-is-red/orphan-allowlist.yaml create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/entry-without-reason-is-red/want create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/entry-without-reason-is-red/want-msg create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/entry-without-ticket-is-red/ProvableContracts.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/entry-without-ticket-is-red/ProvableContracts/A.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/entry-without-ticket-is-red/ProvableContracts/B.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/entry-without-ticket-is-red/ProvableContracts/C.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/entry-without-ticket-is-red/orphan-allowlist.yaml create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/entry-without-ticket-is-red/want create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/entry-without-ticket-is-red/want-msg create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/line-comment-with-slash-dash-is-not-a-block/ProvableContracts.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/line-comment-with-slash-dash-is-not-a-block/ProvableContracts/A.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/line-comment-with-slash-dash-is-not-a-block/want create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/line-comment-with-slash-dash-is-not-a-block/want-msg create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/nested-header-comment-before-imports/ProvableContracts.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/nested-header-comment-before-imports/ProvableContracts/A.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/nested-header-comment-before-imports/want create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/nested-header-comment-before-imports/want-msg create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/no-root-declines/ProvableContracts/A.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/no-root-declines/want create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/no-root-declines/want-msg create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/stale-entry-is-red/ProvableContracts.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/stale-entry-is-red/ProvableContracts/A.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/stale-entry-is-red/ProvableContracts/B.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/stale-entry-is-red/ProvableContracts/C.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/stale-entry-is-red/orphan-allowlist.yaml create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/stale-entry-is-red/want create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/stale-entry-is-red/want-msg create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/transitive-import-is-in-cone/ProvableContracts.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/transitive-import-is-in-cone/ProvableContracts/A.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/transitive-import-is-in-cone/ProvableContracts/B.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/transitive-import-is-in-cone/want create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/transitive-import-is-in-cone/want-msg create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/unlisted-orphan-is-red/ProvableContracts.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/unlisted-orphan-is-red/ProvableContracts/A.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/unlisted-orphan-is-red/ProvableContracts/B.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/unlisted-orphan-is-red/ProvableContracts/C.lean create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/unlisted-orphan-is-red/want create mode 100644 crates/aprender-contracts-staging/lean/fixtures/orphans/unlisted-orphan-is-red/want-msg create mode 100644 crates/aprender-contracts-staging/lean/formalization.yaml create mode 100644 crates/aprender-contracts-staging/lean/orphan-allowlist.yaml create mode 100644 crates/aprender-contracts-staging/lean/scripts/Comparator.lean create mode 100644 crates/aprender-contracts-staging/lean/unresolved-labels.json create mode 100644 crates/aprender-contracts/fixtures/liskov-corrupt.json create mode 100644 crates/aprender-contracts/fixtures/unsat-core-corrupt.json create mode 100644 crates/aprender-contracts/src/discharge/challenge.rs create mode 100644 crates/aprender-contracts/src/discharge/challenge_tests.rs create mode 100644 crates/aprender-contracts/src/discharge/comparator.rs create mode 100644 crates/aprender-contracts/src/discharge/comparator_tests.rs create mode 100644 crates/aprender-contracts/src/discharge/formalization.rs create mode 100644 crates/aprender-contracts/src/discharge/formalization_tests.rs create mode 100644 crates/aprender-contracts/src/discharge/lex.rs create mode 100644 crates/aprender-contracts/src/discharge/mod.rs create mode 100644 crates/aprender-contracts/src/discharge/refinement.rs create mode 100644 crates/aprender-contracts/src/discharge/refinement_tests.rs create mode 100644 crates/aprender-contracts/src/discharge/summary.rs create mode 100644 crates/aprender-contracts/src/discharge/summary_tests.rs create mode 100644 crates/aprender-contracts/src/discharge/tests.rs create mode 100644 crates/aprender-contracts/src/env_key.rs create mode 100644 crates/aprender-contracts/src/kani_assume.rs create mode 100644 crates/aprender-contracts/src/kani_assume_tests.rs create mode 100644 crates/aprender-contracts/src/levels.rs create mode 100644 crates/aprender-contracts/src/lint/bindings_gate.rs create mode 100644 crates/aprender-contracts/src/lint/bindings_gate_tests.rs create mode 100644 crates/aprender-contracts/src/lint/capability_cells_gate.rs create mode 100644 crates/aprender-contracts/src/lint/capability_cells_plant.yaml create mode 100644 crates/aprender-contracts/src/lint/consistency_gate.rs create mode 100644 crates/aprender-contracts/src/lint/consistency_gate_tests.rs create mode 100644 crates/aprender-contracts/src/lint/evidence_gate.rs create mode 100644 crates/aprender-contracts/src/lint/evidence_gate_tests.rs create mode 100644 crates/aprender-contracts/src/lint/ratchet_gates.rs create mode 100644 crates/aprender-contracts/src/lint/ratchet_gates_tests.rs create mode 100644 crates/aprender-contracts/src/lint/refinement_gate.rs create mode 100644 crates/aprender-contracts/src/lint/refinement_gate_tests.rs create mode 100644 crates/aprender-contracts/src/lint/refines_gate.rs create mode 100644 crates/aprender-contracts/src/lint/refines_gate_tests.rs create mode 100644 crates/aprender-contracts/src/lint/subsumption.rs create mode 100644 crates/aprender-contracts/src/lint/tbox_gate.rs create mode 100644 crates/aprender-contracts/src/ontology/capability_cells.rs create mode 100644 crates/aprender-contracts/src/ontology/capability_cells_tests.rs create mode 100644 crates/aprender-contracts/src/ontology/extract/binary.rs create mode 100644 crates/aprender-contracts/src/ontology/extract/binary_tests.rs create mode 100644 crates/aprender-contracts/src/ontology/extract/claims.rs create mode 100644 crates/aprender-contracts/src/ontology/extract/cli_surface.rs create mode 100644 crates/aprender-contracts/src/ontology/extract/covering.rs create mode 100644 crates/aprender-contracts/src/ontology/extract/csv.rs create mode 100644 crates/aprender-contracts/src/ontology/extract/example.rs create mode 100644 crates/aprender-contracts/src/ontology/extract/example_tests.rs create mode 100644 crates/aprender-contracts/src/ontology/extract/json/github.rs create mode 100644 crates/aprender-contracts/src/ontology/extract/kernel.rs create mode 100644 crates/aprender-contracts/src/ontology/extract/kernel_tests.rs create mode 100644 crates/aprender-contracts/src/ontology/extract/llm_context.rs create mode 100644 crates/aprender-contracts/src/ontology/extract/readme.rs create mode 100644 crates/aprender-contracts/src/ontology/extract/release_cells.rs create mode 100644 crates/aprender-contracts/src/ontology/extract/release_crux.rs create mode 100644 crates/aprender-contracts/src/ontology/liskov.rs create mode 100644 crates/aprender-contracts/src/ontology/liskov_tests.rs create mode 100644 crates/aprender-contracts/src/ontology/measured_sets.rs create mode 100644 crates/aprender-contracts/src/ontology/owl.rs create mode 100644 crates/aprender-contracts/src/ontology/witness.rs create mode 100644 crates/aprender-contracts/src/ontology/witness_planted.rs create mode 100644 crates/aprender-contracts/src/ontology/witness_small.rs create mode 100644 crates/aprender-contracts/src/ontology/witness_tests.rs create mode 100644 crates/aprender-contracts/src/schema/clause.rs create mode 100644 crates/aprender-contracts/src/tree.rs create mode 100644 crates/aprender-contracts/tests/fixtures/ont4g/snapshot.jsonl create mode 100644 crates/aprender-contracts/tests/fixtures/ont4g/surface_audit.csv create mode 100644 crates/aprender-contracts/w3c/property-lessThan-001.yaml create mode 100644 crates/aprender-contracts/w3c/property-lessThan-002.yaml create mode 100644 crates/aprender-contracts/w3c/property-lessThanOrEquals-001.yaml create mode 100644 docs/audits/quorum-GH-4122-pvl-5a-degraded.json create mode 100644 scripts/check_lean_modules_reachable.sh create mode 100755 scripts/check_leanchecker_scoped.sh create mode 100644 tests/fixtures/pvl/ghost-binding.yaml create mode 100644 tests/fixtures/pvl/resolved-binding.yaml diff --git a/ci/explicit-test-commands.d/461-aprender-contracts-cli-pvl-discharge-check.cmd b/ci/explicit-test-commands.d/461-aprender-contracts-cli-pvl-discharge-check.cmd new file mode 100644 index 0000000000..f10fbfea13 --- /dev/null +++ b/ci/explicit-test-commands.d/461-aprender-contracts-cli-pvl-discharge-check.cmd @@ -0,0 +1 @@ +cargo test -p aprender-contracts-cli --test pvl_discharge_check diff --git a/ci/explicit-test-commands.d/462-aprender-contracts-cli-pvl-obligations-golden.cmd b/ci/explicit-test-commands.d/462-aprender-contracts-cli-pvl-obligations-golden.cmd new file mode 100644 index 0000000000..7f4042f19d --- /dev/null +++ b/ci/explicit-test-commands.d/462-aprender-contracts-cli-pvl-obligations-golden.cmd @@ -0,0 +1 @@ +cargo test -p aprender-contracts-cli --test pvl_obligations_golden diff --git a/ci/explicit-test-commands.d/471-aprender-contracts-cli-pvl-discharge-leanchecker.cmd b/ci/explicit-test-commands.d/471-aprender-contracts-cli-pvl-discharge-leanchecker.cmd new file mode 100644 index 0000000000..663ef8f32a --- /dev/null +++ b/ci/explicit-test-commands.d/471-aprender-contracts-cli-pvl-discharge-leanchecker.cmd @@ -0,0 +1 @@ +cargo test -p aprender-contracts-cli --test pvl_discharge_leanchecker diff --git a/ci/explicit-test-commands.d/481-aprender-contracts-cli-pvl-comparator.cmd b/ci/explicit-test-commands.d/481-aprender-contracts-cli-pvl-comparator.cmd new file mode 100644 index 0000000000..02c3c9635d --- /dev/null +++ b/ci/explicit-test-commands.d/481-aprender-contracts-cli-pvl-comparator.cmd @@ -0,0 +1 @@ +cargo test -p aprender-contracts-cli --test pvl_comparator diff --git a/ci/explicit-test-commands.d/482-aprender-contracts-cli-pvl-challenge.cmd b/ci/explicit-test-commands.d/482-aprender-contracts-cli-pvl-challenge.cmd new file mode 100644 index 0000000000..1f5f60d80e --- /dev/null +++ b/ci/explicit-test-commands.d/482-aprender-contracts-cli-pvl-challenge.cmd @@ -0,0 +1 @@ +cargo test -p aprender-contracts-cli --test pvl_challenge diff --git a/ci/explicit-test-commands.d/491-aprender-contracts-cli-pvl-ghost-binding.cmd b/ci/explicit-test-commands.d/491-aprender-contracts-cli-pvl-ghost-binding.cmd new file mode 100644 index 0000000000..ddafe0fb95 --- /dev/null +++ b/ci/explicit-test-commands.d/491-aprender-contracts-cli-pvl-ghost-binding.cmd @@ -0,0 +1,3 @@ +# PVL-001 EV-2 (#4080): pv proof-status --binding resolves bindings; a ghost binding is a reject. +# Reads tests/fixtures/pvl/ and contracts/ from the workspace root (the resolver is CWD-sensitive). +cargo test -p aprender-contracts-cli --test pvl_ghost_binding diff --git a/contracts/cooperative-matrix-gemm-v1.yaml b/contracts/cooperative-matrix-gemm-v1.yaml deleted file mode 100644 index 3ad3cf8b68..0000000000 --- a/contracts/cooperative-matrix-gemm-v1.yaml +++ /dev/null @@ -1,79 +0,0 @@ -metadata: - version: 1.0.0 - created: '2026-04-02' - author: PAIML Engineering - description: > - Cooperative matrix GEMM — hardware tensor core acceleration via - VK_KHR_cooperative_matrix (wgpu 29.0+). Replaces software tiled GEMM - (375 GFLOPS) with hardware WMMA (expected 1000+ GFLOPS on GB10). - references: - - "VK_KHR_cooperative_matrix Vulkan extension" - - "wgpu v29.0.0 cooperative matrix support (2026-03-19)" - - "NVIDIA Blackwell GB10: BF16+FP8 cooperative matrix, revision 2" - -equations: - cooperative_gemm: - formula: "C[m,n] = α * Σ_k A[m,k] * B[k,n] + β * C[m,n]" - domain: "A ∈ ℝ^{M×K}, B ∈ ℝ^{K×N}, α,β ∈ ℝ" - codomain: "C ∈ ℝ^{M×N}" - invariants: - - "Result matches software tiled GEMM within ε < 1e-3 (f32)" - - "F16 input, F32 accumulation (GB10 config 3: M=16 K=16 N=16)" - lean_theorem: ProvableContracts.CooperativeMatrix.matmul_block_sum - f16_error_bound: - formula: "|C_f32_accum - C_exact| ≤ K * ε_f16 * max|A| * max|B|" - domain: "K reduction dimension, ε_f16 = 2^{-10}" - codomain: "error bound ∈ ℝ≥0" - lean_theorem: ProvableContracts.CooperativeMatrix.f16_accumulation_error_bound - notes: - - "Uses wgpu::Features::SUBGROUP_VERTEX_STAGE or cooperative_matrix feature" - - "Dispatch: ceil(M/coop_M) × ceil(N/coop_N) workgroups" - -invariants: - parity_with_tiled: - statement: "|coop_result - tiled_result| < 1e-3 for all elements" - rationale: "Hardware GEMM must match software reference" - - throughput_improvement: - statement: "coop_gflops >= 2 * tiled_gflops" - rationale: "Tensor cores should at least double throughput" - - fallback: - statement: "If cooperative matrix not available, fall back to tiled GEMM" - rationale: "Not all GPUs support VK_KHR_cooperative_matrix" - -falsification_tests: - - id: FALSIFY-COOP-001 - rule: parity_with_tiled - prediction: "max |coop - tiled| < 1e-3" - if_fails: "Cooperative matrix GEMM diverges from tiled reference" - - id: FALSIFY-COOP-002 - rule: throughput_improvement - prediction: "coop GFLOPS > 2 * tiled GFLOPS" - if_fails: "Cooperative matrix provides no performance benefit" - - id: FALSIFY-COOP-003 - rule: fallback - prediction: "Falls back to tiled GEMM, no crash" - if_fails: "Missing fallback causes crash on unsupported hardware" - -proof_obligations: - - id: CMG-POST-001 - type: postcondition - property: Parity with tiled reference - formal: "|coop - tiled| < 1e-3" - -kani_harnesses: - - id: KANI-COOP-001 - obligation: parity_with_tiled - property: Output parity - bound: 8 - strategy: bounded_int - -qa_gate: - id: F-COOP-GEMM-001 - name: Cooperative Matrix GEMM Contract - description: Quality gate for cooperative matrix GEMM hardware tensor core acceleration - checks: - - validation - - falsification - diff --git a/contracts/pvl-lint-ratchets-v1.yaml b/contracts/pvl-lint-ratchets-v1.yaml new file mode 100644 index 0000000000..e8160db4af --- /dev/null +++ b/contracts/pvl-lint-ratchets-v1.yaml @@ -0,0 +1,121 @@ +# ───────────────────────────────────────────────────────────────────────────── +# pvl-lint-ratchets-v1 — theorem pairing + depends_on, shrink-only (PVL-001 EV-11) +# (paiml/infra PVL-001 @00553b0b §EV-11; issue #4166; ticket PMAT-4166) +# +# WHAT IT GOVERNS +# provable_contracts::lint::ratchet_gates — PV-RAT-001 (theorem-pairing) and +# PV-RAT-002 (depends-on-present); `pv lint --gate theorem-pairing --gate +# depends-on-present` and gates 14/15 of the full run (R-8: computed +# everywhere, armed per repo); `make lint-ratchet` (scripts/lint_ratchet.sh), +# the only writer of the two baselines, which only turns them down. +# +# THE PAIRING RULE IS THIS ROW'S DESIGN. The row says "module name appears in +# book/"; the full dotted module name on identifier boundaries is the reading +# that cannot be satisfied by accident (the book names 80 of 165 file STEMS, +# mostly as ordinary words; measured 2026-09-24). +# ───────────────────────────────────────────────────────────────────────────── +name: pvl-lint-ratchets +version: "1.0.0" +scope: > + Two debts that may fall and never rise: Lean theorem modules no book page names, and kernel-kind contracts + with an empty `metadata.depends_on`. Out of scope: whether a book page's mention is a real explanation, and + whether a declared dependency is the right one. +status: active + +metadata: + version: "1.0.0" + kind: pattern + created: '2026-09-24' + last_modified: '2026-09-24' + author: PAIML Engineering + description: > + `pv lint --gate theorem-pairing` counts Lean theorem modules whose full module name no book page mentions; + `--gate depends-on-present` counts kernel-kind contracts with no depends_on. Each rejects a count above its + top-level baseline in contracts/lint-baseline.json, reports (never passes) with no baseline, declines when + nothing was measured, and never writes the file. + references: + - 'paiml/infra docs/specifications/pvl-001 @00553b0b §EV-11 (probe + mutation)' + - 'crates/aprender-contracts/src/lint/ratchet_gates.rs — PV-RAT-001, PV-RAT-002' + - 'scripts/lint_ratchet.sh — `make lint-ratchet`, the downward-only writer' + +relations: + depends_on: [ont-verdict-lattice-v1] + +equations: + shrink_only_ratchet: + formula: "admits(corpus) ⟺ baseline ≠ ⊥ ∧ debt(corpus) ≤ baseline" + domain: "a contract corpus, its repo root (lean/, book/) and contracts/lint-baseline.json" + codomain: "Pass | Fail | Unknown" + invariants: + - "no baseline is Unknown(Report) with the count printed, never Pass" + - "nothing measured (no Lean base, no module, no book page, no kernel contract) is a decline" + preconditions: + - "the baseline, when present, is a non-negative integer at the top level" + postconditions: + - "verdict ∈ {Pass, Fail, Unknown}" + - "contracts/lint-baseline.json is byte-identical after the gate" + lean_theorem: none — L4 not declared + +invariants: + - id: RAT-INV-001 + property: the unpaired theorem module debt is shrink-only + formal: 'unpaired(corpus) ≤ baseline(unpaired_theorem_modules)' + prose: false + - id: RAT-INV-002 + property: the kernel contracts without depends_on debt is shrink-only + formal: 'without_depends_on(corpus) ≤ baseline(contracts_without_depends_on)' + prose: false + - id: RAT-INV-003 + property: a module is paired only by its full dotted name on identifier boundaries + formal: 'paired(m) ⟺ ∃ p ∈ book: mentions(p, name(m))' + prose: false + - id: RAT-INV-004 + property: a gate never writes its baseline + formal: '¬(after_gate(baseline) ≠ before_gate(baseline))' + prose: false + +falsification_tests: + - id: FALSIFY-RAT-001 + rule: the spec's mutation + prediction: adding an unpaired Theorem module turns `pv lint --gate theorem-pairing --gate depends-on-present` RED with PV-RAT-001 + test: cargo test -p aprender-contracts-cli --test ev11_lint_ratchets + if_fails: a theorem module can be added that no book page names, unrecorded + - id: FALSIFY-RAT-002 + rule: the depends_on ratchet + prediction: a kernel-kind contract with no depends_on above the baseline is PV-RAT-002; equal or lower passes + test: cargo test -p aprender-contracts --lib lint::ratchet_gates::tests::adding_a_kernel_contract_with_no_depends_on_is_red + if_fails: a kernel contract that names no composition can be added unrecorded + - id: FALSIFY-RAT-003 + rule: the pairing boundary + prediction: a file stem, or a longer module name that starts with it, does not pair a module + test: cargo test -p aprender-contracts --lib lint::ratchet_gates::tests::the_file_stem_or_a_longer_name_does_not_pair_a_module + if_fails: the book's ordinary words pair modules by accident + - id: FALSIFY-RAT-004 + rule: no baseline is reported, never judged + prediction: with no baseline key both gates are Unknown(Report), exit 2, the count printed + test: cargo test -p aprender-contracts --lib lint::ratchet_gates::tests::no_baseline_is_reported_and_never_a_pass + if_fails: deleting the baseline turns the ratchet into a pass + - id: FALSIFY-RAT-005 + rule: the gates never write + prediction: the baseline is byte-identical after both gates, on fixtures and on the real corpus + test: cargo test -p aprender-contracts --lib lint::ratchet_gates::tests::neither_gate_writes_the_baseline + if_fails: a gate can raise its own baseline + - id: FALSIFY-RAT-006 + rule: R-8 + prediction: the full `pv lint` run computes theorem-pairing and depends-on-present, reported and not armed + test: cargo test -p aprender-contracts --lib lint::tests + if_fails: armed_gates could name a gate no run computes + +qa_gate: + id: F-RAT-001 + name: lint ratchets + description: Theorem pairing and depends_on debts may fall, never rise (PVL-001 EV-11) + checks: + - unpaired_theorem_modules_shrink_only + - contracts_without_depends_on_shrink_only + - strict_pairing_boundary + - no_baseline_is_report + - gate_never_writes + - computed_in_every_run + pass_criteria: All 6 falsification tests pass; the EV-11 probe exits 0 on the real corpus and `git diff --exit-code contracts/lint-baseline.json` is clean after it + falsification: 'add lean/ProvableContracts/Theorems/S/B.lean named by no book page → exit 1, PV-RAT-001 (measured on a fixture 2026-09-24)' diff --git a/crates/aprender-common/Cargo.toml b/crates/aprender-common/Cargo.toml index 1861e9cd0b..6bd5a61a52 100644 --- a/crates/aprender-common/Cargo.toml +++ b/crates/aprender-common/Cargo.toml @@ -12,7 +12,14 @@ categories = ["command-line-utilities", "text-processing"] [lib] name = "batuta_common" +[features] +default = [] +# #3745 S1: typed CLI argument roles (`batuta_common::cli_roles`). Optional so the +# default build of this crate stays clap-free. +cli-roles = ["dep:clap"] + [dependencies] +clap = { workspace = true, optional = true } lz4_flex = { version = "0.11", default-features = false, features = ["safe-encode", "safe-decode"] } zstd = { version = "0.13", default-features = false } diff --git a/crates/aprender-common/src/cli_roles.rs b/crates/aprender-common/src/cli_roles.rs new file mode 100644 index 0000000000..cd0da3c474 --- /dev/null +++ b/crates/aprender-common/src/cli_roles.rs @@ -0,0 +1,732 @@ +//! Typed CLI argument roles (#3745 S1, PMAT-3749). +//! +//! A release cell can only be derived for an argument whose ROLE the gate can +//! see. Before these types, the role of an `apr` argument was knowable only +//! from its name: `apr run SOURCE`, `--prompt` and `-i FILE` were a `String`, a +//! `String` and a `PathBuf`, the same types as `--name`, `--tag` and +//! `--output`. Every release list was therefore typed by hand, and a verb, flag +//! or input shape missing from that list was invisible to the gate. That is how +//! `apr run --prompt P --chat` shipped double-templated (#3743) while +//! `-i file --chat` was covered. +//! +//! Each type here is a transparent newtype over `PathBuf` or `String` whose clap +//! value parser is the SAME parser clap uses for the raw type +//! (`PathBufValueParser` / `StringValueParser`), mapped into the newtype. So a +//! migrated argument accepts exactly the values it accepted before. The only +//! thing that changes is the parser's output TYPE, and that is what `apr surface` +//! reads, through `ValueParser::type_id()`, to decide the role. The role is +//! declared by how the argument is BUILT; its name never enters into it. +//! +//! The types live here rather than in `apr-cli` because a marker compared by +//! `TypeId` needs ONE defining crate, and some of `apr`'s surface is declared by +//! crates that do not depend on `apr-cli` (`apr pv …` from +//! `aprender-contracts-cli`, `apr rag …` from `aprender-rag-cli`). + +use std::any::TypeId; +use std::ffi::OsStr; +use std::fmt; +use std::ops::Deref; +use std::path::{Path, PathBuf}; + +use clap::builder::{MapValueParser, PathBufValueParser, StringValueParser, TypedValueParser}; + +/// The role an argument plays in a release cell (#3745). +/// +/// `Mode` and `Backend` are not listed in [`MARKERS`]. Mode is structural (a +/// flag action, or a finite value set), and backend is membership in the shared +/// `BackendArg` group; `apr surface` decides both. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum Role { + /// A model: a local file, or a reference `apr` resolves to one. + Model, + /// Text the model is prompted with. + Prompt, + /// A file the command consumes as input data (audio, text, a dataset). + InputFile, + /// The compute backend override. + Backend, + /// A generation sampling control: seed, temperature, top-k, top-p, + /// repeat penalty (S1 v1.1, for S2.5's flag-effect sampling controls). + Sampling, + /// A flag, or a value drawn from a finite set. + Mode, + /// Declared, and none of the above: outputs, directories, configs, names. + Other, + /// A free-form value no role type declares. Only a foreign subtree may + /// carry one; the marker guard is RED on it anywhere else. + Unknown, +} + +impl Role { + /// The spelling used in the `apr-cli-surface` JSON. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Model => "model", + Self::Prompt => "prompt", + Self::InputFile => "input-file", + Self::Backend => "backend", + Self::Sampling => "sampling", + Self::Mode => "mode", + Self::Other => "other", + Self::Unknown => "unknown", + } + } +} + +/// What a marker's underlying value is: a filesystem path, or free text. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Carrier { + /// Backed by `PathBuf`. + Path, + /// Backed by `String`. + Text, +} + +/// One role type: the `TypeId` its value parser yields, and what it declares. +#[derive(Debug, Clone, Copy)] +pub struct Marker { + /// The type's name as written in code, e.g. `ModelPath`. + pub name: &'static str, + /// The role this type declares. + pub role: Role, + /// Path- or text-backed. + pub carrier: Carrier, + type_id: fn() -> TypeId, +} + +impl Marker { + /// The `TypeId` of the marker type, which is the `TypeId` its clap value + /// parser reports. + #[must_use] + pub fn type_id(&self) -> TypeId { + (self.type_id)() + } +} + +macro_rules! path_role { + ($(#[$doc:meta])* $name:ident) => { + $(#[$doc])* + #[derive(Clone, Default, PartialEq, Eq, Hash, PartialOrd, Ord)] + pub struct $name(PathBuf); + + // `Debug` is the inner value's, so `{:?}` of a migrated field prints + // `"m.gguf"` exactly as the raw `PathBuf` did, never `ModelPath("m.gguf")`. + impl fmt::Debug for $name { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&self.0, f) + } + } + + impl $name { + /// Wrap a path. + #[must_use] + pub fn new(path: impl Into) -> Self { + Self(path.into()) + } + /// The path, borrowed. + #[must_use] + pub fn as_path(&self) -> &Path { + &self.0 + } + /// The path, owned. + #[must_use] + pub fn into_path_buf(self) -> PathBuf { + self.0 + } + } + + // `Path`, not `PathBuf`: the same target `PathBuf` derefs to, so + // `Option::as_deref()` is `Option<&Path>` exactly as + // `Option::as_deref()` was, and a migrated field reads the same. + impl Deref for $name { + type Target = Path; + fn deref(&self) -> &Path { + &self.0 + } + } + impl AsRef for $name { + fn as_ref(&self) -> &Path { + &self.0 + } + } + impl AsRef for $name { + fn as_ref(&self) -> &OsStr { + self.0.as_os_str() + } + } + impl From for $name { + fn from(p: PathBuf) -> Self { + Self(p) + } + } + impl From<&Path> for $name { + fn from(p: &Path) -> Self { + Self(p.to_path_buf()) + } + } + impl From<&str> for $name { + fn from(p: &str) -> Self { + Self(PathBuf::from(p)) + } + } + impl From for $name { + fn from(p: String) -> Self { + Self(PathBuf::from(p)) + } + } + impl From<$name> for PathBuf { + fn from(p: $name) -> PathBuf { + p.0 + } + } + impl PartialEq for $name { + fn eq(&self, other: &PathBuf) -> bool { + &self.0 == other + } + } + impl PartialEq for $name { + fn eq(&self, other: &Path) -> bool { + self.0 == other + } + } + impl clap::builder::ValueParserFactory for $name { + type Parser = MapValueParser $name>; + fn value_parser() -> Self::Parser { + PathBufValueParser::new().map($name as fn(PathBuf) -> $name) + } + } + }; +} + +macro_rules! text_role { + ($(#[$doc:meta])* $name:ident) => { + $(#[$doc])* + #[derive(Clone, Default, PartialEq, Eq, Hash, PartialOrd, Ord)] + pub struct $name(String); + + // `Debug` is the inner value's: `{:?}` prints `"text"`, as `String` did. + impl fmt::Debug for $name { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&self.0, f) + } + } + + impl $name { + /// Wrap a string. + #[must_use] + pub fn new(s: impl Into) -> Self { + Self(s.into()) + } + /// The text, borrowed. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } + /// The text as a `&String`, for callees that take one. + #[must_use] + pub fn as_string(&self) -> &String { + &self.0 + } + /// The text, owned. + #[must_use] + pub fn into_string(self) -> String { + self.0 + } + } + + // `str`, not `String`: the same target `String` derefs to, so + // `Option::as_deref()` is `Option<&str>` exactly as + // `Option::as_deref()` was. + impl Deref for $name { + type Target = str; + fn deref(&self) -> &str { + &self.0 + } + } + impl AsRef for $name { + fn as_ref(&self) -> &str { + &self.0 + } + } + impl AsRef for $name { + fn as_ref(&self) -> &Path { + Path::new(&self.0) + } + } + impl AsRef for $name { + fn as_ref(&self) -> &OsStr { + OsStr::new(&self.0) + } + } + impl fmt::Display for $name { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } + } + impl From for $name { + fn from(s: String) -> Self { + Self(s) + } + } + impl From<&str> for $name { + fn from(s: &str) -> Self { + Self(s.to_string()) + } + } + impl From<$name> for String { + fn from(s: $name) -> String { + s.0 + } + } + impl PartialEq for $name { + fn eq(&self, other: &str) -> bool { + self.0 == other + } + } + impl PartialEq<&str> for $name { + fn eq(&self, other: &&str) -> bool { + self.0 == *other + } + } + impl PartialEq for $name { + fn eq(&self, other: &String) -> bool { + &self.0 == other + } + } + impl clap::builder::ValueParserFactory for $name { + type Parser = MapValueParser $name>; + fn value_parser() -> Self::Parser { + StringValueParser::new().map($name as fn(String) -> $name) + } + } + }; +} + +path_role!( + /// A local model file (`.apr`, `.gguf`, `.safetensors`, …). + ModelPath +); +path_role!( + /// A file the command reads as input data, not as a model. + InputFile +); +path_role!( + /// A file the command writes. + OutputPath +); +path_role!( + /// A directory: a workspace, a cache, a corpus root, an output tree. + DirPath +); +path_role!( + /// A configuration, manifest, contract, schema or receipt the command + /// reads to decide what to do, as distinct from the data it operates on. + ConfigPath +); +text_role!( + /// A model reference: a local path, `hf://org/repo`, a URL, or a cache + /// name, which `apr` resolves to a model file. + ModelRef +); +text_role!( + /// Text the model is prompted with. + PromptText +); +text_role!( + /// Any other free text: a name, id, tag, URL, pattern or list. + FreeText +); +text_role!( + /// Text a model ENCODES or SCORES rather than generates from: `apr embed + /// --text`, `apr rerank --query/--passage`, `apr eval --text` (perplexity). + /// Role `other`, so the command does not read as a generator: `generates` + /// drives the thinking × context-rung cells, which mean nothing for an + /// encoder or a scorer (aprender-97 on #3745, v1.1). + EncodeText +); + +/// Command-level marker: this command GENERATES text from prompts that do not +/// arrive through a `PromptText` argument (#3745 S1, schema v1.1). +/// +/// `apr serve run` takes its prompts over HTTP, `apr chat` reads them from +/// stdin, and `apr mcp` receives them as JSON-RPC tool calls. None of them has a +/// prompt ARGUMENT, so a rule that reads "generates" off the prompt role alone +/// would derive no prompt-shape cells for the very verb whose 0.69.0 leak +/// (#3571) started this. The marker is attached to the subcommand itself, with +/// no argument and no field: +/// +/// ```ignore +/// #[command(group(batuta_common::cli_roles::ServesGeneration::group()))] +/// Run { /* … */ }, +/// ``` +/// +/// `apr surface` reports `generates: true` for a command that has a +/// `PromptText` argument OR carries this group, and it matches the group by +/// [`ServesGeneration::ID`], never by the command's name. +#[derive(Debug, Clone, Copy)] +pub struct ServesGeneration; + +impl ServesGeneration { + /// The group id the marker is recognised by. + pub const ID: &'static str = "batuta_common::cli_roles::ServesGeneration"; + + /// The argument-less group that marks a command as a generator. It names + /// no argument, so it changes neither parsing nor `--help`. + #[must_use] + pub fn group() -> clap::ArgGroup { + clap::ArgGroup::new(Self::ID).multiple(true) + } + + /// True iff `cmd` carries the marker. + #[must_use] + pub fn is_on(cmd: &clap::Command) -> bool { + cmd.get_groups().any(|g| g.get_id() == Self::ID) + } +} + +/// Which generation sampling control an argument is (#3745 S1, v1.1). S2 +/// derives its sampling cells per kind (sampled ≠ greedy, same seed twice is +/// byte-identical, seed A ≠ seed B, top-k 1 / T 0 is byte-greedy), so the kind +/// is declared, not read off the argument's name. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum SamplingKind { + /// RNG seed of the sampler. + Seed, + /// Softmax temperature. + Temperature, + /// Top-k cut-off. + TopK, + /// Nucleus (top-p) threshold. + TopP, + /// Min-p threshold. + MinP, + /// Repetition penalty. + RepeatPenalty, + /// Window the repetition penalty looks back over. + RepeatLastN, +} + +impl SamplingKind { + /// Every kind. + pub const ALL: [SamplingKind; 7] = [ + Self::Seed, + Self::Temperature, + Self::TopK, + Self::TopP, + Self::MinP, + Self::RepeatPenalty, + Self::RepeatLastN, + ]; + + /// The spelling used in the `apr-cli-surface` JSON (`sampling_kind`). + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Seed => "seed", + Self::Temperature => "temperature", + Self::TopK => "top_k", + Self::TopP => "top_p", + Self::MinP => "min_p", + Self::RepeatPenalty => "repeat_penalty", + Self::RepeatLastN => "repeat_last_n", + } + } + + /// The group id an argument of this kind joins: `#[arg(group = SamplingKind::Temperature.id())]`. + #[must_use] + pub const fn id(self) -> &'static str { + match self { + Self::Seed => "batuta_common::cli_roles::SamplingArg::seed", + Self::Temperature => "batuta_common::cli_roles::SamplingArg::temperature", + Self::TopK => "batuta_common::cli_roles::SamplingArg::top_k", + Self::TopP => "batuta_common::cli_roles::SamplingArg::top_p", + Self::MinP => "batuta_common::cli_roles::SamplingArg::min_p", + Self::RepeatPenalty => "batuta_common::cli_roles::SamplingArg::repeat_penalty", + Self::RepeatLastN => "batuta_common::cli_roles::SamplingArg::repeat_last_n", + } + } +} + +/// Argument-level marker for a generation SAMPLING control (#3745 S1, v1.1). +/// +/// These are numbers, so a newtype would change every field's type and every +/// call site that reads it. Instead the argument joins the argument-less group +/// of its [`SamplingKind`], and the command declares those groups once, each +/// `multiple(true)`, so nothing about parsing changes: +/// +/// ```ignore +/// #[command(groups(batuta_common::cli_roles::SamplingArg::groups()))] +/// Run { +/// #[arg(long, default_value = "0.0", +/// group = batuta_common::cli_roles::SamplingKind::Temperature.id())] +/// temperature: f32, +/// } +/// ``` +/// +/// `apr surface` reports role `sampling` and `sampling_kind` for a member of +/// one of these groups, matched by id, never by the argument's name. +#[derive(Debug, Clone, Copy)] +pub struct SamplingArg; + +impl SamplingArg { + /// One argument-less `multiple(true)` group per [`SamplingKind`], for + /// `#[command(groups(SamplingArg::groups()))]`. + #[must_use] + pub fn groups() -> Vec { + SamplingKind::ALL + .iter() + .map(|k| clap::ArgGroup::new(k.id()).multiple(true)) + .collect() + } + + /// The sampling kind of `arg` on `built`, if it has one. `built` must be a + /// BUILT command: clap merges each argument's `group = …` into the + /// command's groups only when the command is built. + #[must_use] + pub fn kind_of(built: &clap::Command, arg: &clap::Arg) -> Option { + SamplingKind::ALL.into_iter().find(|k| { + built + .get_groups() + .any(|g| g.get_id() == k.id() && g.get_args().any(|a| a == arg.get_id())) + }) + } +} + +/// Owned `String`s from a slice of text role values, for callees that take +/// `&[String]`. +#[must_use] +pub fn strings>(values: &[T]) -> Vec { + values.iter().map(|v| v.as_ref().to_string()).collect() +} + +/// Owned `PathBuf`s from a slice of path role values, for callees that take +/// `&[PathBuf]`. +#[must_use] +pub fn path_bufs>(values: &[T]) -> Vec { + values.iter().map(|v| v.as_ref().to_path_buf()).collect() +} + +/// Every role type, in one table. `apr surface` reads roles through it, and the +/// marker guard refuses a free-form argument whose parser yields none of these. +pub const MARKERS: &[Marker] = &[ + Marker { + name: "ModelPath", + role: Role::Model, + carrier: Carrier::Path, + type_id: TypeId::of::, + }, + Marker { + name: "ModelRef", + role: Role::Model, + carrier: Carrier::Text, + type_id: TypeId::of::, + }, + Marker { + name: "PromptText", + role: Role::Prompt, + carrier: Carrier::Text, + type_id: TypeId::of::, + }, + Marker { + name: "InputFile", + role: Role::InputFile, + carrier: Carrier::Path, + type_id: TypeId::of::, + }, + Marker { + name: "OutputPath", + role: Role::Other, + carrier: Carrier::Path, + type_id: TypeId::of::, + }, + Marker { + name: "DirPath", + role: Role::Other, + carrier: Carrier::Path, + type_id: TypeId::of::, + }, + Marker { + name: "ConfigPath", + role: Role::Other, + carrier: Carrier::Path, + type_id: TypeId::of::, + }, + Marker { + name: "FreeText", + role: Role::Other, + carrier: Carrier::Text, + type_id: TypeId::of::, + }, + Marker { + name: "EncodeText", + role: Role::Other, + carrier: Carrier::Text, + type_id: TypeId::of::, + }, +]; + +/// The marker whose type a value parser yields, if any. +#[must_use] +pub fn marker_for(type_id: TypeId) -> Option<&'static Marker> { + MARKERS.iter().find(|m| m.type_id() == type_id) +} + +#[cfg(test)] +mod tests { + use super::*; + use clap::{Arg, Command}; + + fn parser_type() -> TypeId + where + T::Parser: TypedValueParser + Send + Sync + 'static, + ::Value: Send + Sync + Clone + 'static, + { + let vp: clap::builder::ValueParser = T::value_parser().into(); + let arg = Arg::new("x").value_parser(vp); + // `AnyValueId` is clap's; it compares equal to a std `TypeId`. + let id = arg.get_value_parser().type_id(); + MARKERS + .iter() + .map(Marker::type_id) + .chain([TypeId::of::(), TypeId::of::()]) + .find(|t| id == *t) + .expect("value parser yields a known type") + } + + #[test] + fn every_marker_parser_reports_its_own_type_id() { + assert_eq!(parser_type::(), TypeId::of::()); + assert_eq!(parser_type::(), TypeId::of::()); + assert_eq!(parser_type::(), TypeId::of::()); + assert_eq!(parser_type::(), TypeId::of::()); + assert_eq!(parser_type::(), TypeId::of::()); + assert_eq!(parser_type::(), TypeId::of::()); + assert_eq!(parser_type::(), TypeId::of::()); + assert_eq!(parser_type::(), TypeId::of::()); + assert_eq!(parser_type::(), TypeId::of::()); + } + + #[test] + fn marker_table_is_one_to_one_with_types() { + let mut ids: Vec = MARKERS.iter().map(Marker::type_id).collect(); + ids.sort(); + ids.dedup(); + assert_eq!(ids.len(), MARKERS.len(), "two markers share a TypeId"); + assert!( + marker_for(TypeId::of::()).is_none(), + "raw PathBuf must not be a marker" + ); + assert!( + marker_for(TypeId::of::()).is_none(), + "raw String must not be a marker" + ); + assert_eq!( + marker_for(TypeId::of::()).map(|m| m.role), + Some(Role::Model) + ); + assert_eq!( + marker_for(TypeId::of::()).map(|m| m.role), + Some(Role::Prompt) + ); + } + + /// A migrated argument must accept exactly what the raw type accepted: the + /// marker parser is the raw parser, mapped. An empty path is refused by + /// both, a normal value is accepted by both. + #[test] + fn marker_parsers_accept_what_the_raw_parsers_accept() { + let raw = Command::new("t").arg(Arg::new("p").value_parser(clap::value_parser!(PathBuf))); + let typed = + Command::new("t").arg(Arg::new("p").value_parser(clap::value_parser!(ModelPath))); + for argv in [vec!["t", "m.gguf"], vec!["t", ""], vec!["t", "-"]] { + let a = raw.clone().try_get_matches_from(&argv).is_ok(); + let b = typed.clone().try_get_matches_from(&argv).is_ok(); + assert_eq!(a, b, "PathBuf vs ModelPath disagree on {argv:?}"); + } + let m = typed.try_get_matches_from(["t", "m.gguf"]).expect("parses"); + assert_eq!( + m.get_one::("p").map(ModelPath::as_path), + Some(Path::new("m.gguf")) + ); + + let raw = Command::new("t").arg(Arg::new("s").value_parser(clap::value_parser!(String))); + let typed = + Command::new("t").arg(Arg::new("s").value_parser(clap::value_parser!(PromptText))); + for argv in [vec!["t", "What is 2+2?"], vec!["t", ""]] { + let a = raw.clone().try_get_matches_from(&argv).is_ok(); + let b = typed.clone().try_get_matches_from(&argv).is_ok(); + assert_eq!(a, b, "String vs PromptText disagree on {argv:?}"); + } + } + + /// The generation marker names no argument, so it changes neither what + /// parses nor what `--help` prints, and it is found by id alone. + #[test] + fn serves_generation_marks_a_command_without_changing_it() { + let plain = Command::new("t").arg(Arg::new("port").long("port")); + let marked = plain.clone().group(ServesGeneration::group()); + assert!(!ServesGeneration::is_on(&plain)); + assert!(ServesGeneration::is_on(&marked)); + for argv in [vec!["t"], vec!["t", "--port", "8080"], vec!["t", "--nope"]] { + let a = plain.clone().try_get_matches_from(&argv).is_ok(); + let b = marked.clone().try_get_matches_from(&argv).is_ok(); + assert_eq!(a, b, "the marker changed parsing of {argv:?}"); + } + let help = |c: &Command| c.clone().render_long_help().to_string(); + assert_eq!(help(&plain), help(&marked), "the marker changed --help"); + // A look-alike group spelled with the type's short name is not the marker. + let lookalike = plain.group(clap::ArgGroup::new("ServesGeneration").multiple(true)); + assert!(!ServesGeneration::is_on(&lookalike)); + } + + /// A migrated field must print the same under `{:?}` as the raw type did: + /// error messages and tests format arguments that way. + #[test] + fn debug_output_is_the_raw_types() { + assert_eq!( + format!("{:?}", ModelPath::from("m.gguf")), + format!("{:?}", PathBuf::from("m.gguf")) + ); + assert_eq!( + format!("{:?}", Some(OutputPath::from("o"))), + format!("{:?}", Some(PathBuf::from("o"))) + ); + assert_eq!( + format!("{:?}", PromptText::from("hi")), + format!("{:?}", String::from("hi")) + ); + assert_eq!( + format!("{:?}", vec![FreeText::from("a")]), + format!("{:?}", vec![String::from("a")]) + ); + } + + #[test] + fn derive_picks_the_marker_parser_for_a_marker_field() { + #[derive(clap::Parser)] + struct T { + model: ModelPath, + #[arg(long)] + prompt: Option, + #[arg(long)] + inputs: Vec, + } + use clap::{CommandFactory, Parser}; + let cmd = T::command(); + let id = |name: &str| { + cmd.get_arguments() + .find(|a| a.get_id() == name) + .map(|a| a.get_value_parser().type_id()) + .expect("arg exists") + }; + assert!(id("model") == TypeId::of::()); + assert!(id("prompt") == TypeId::of::()); + assert!(id("inputs") == TypeId::of::()); + let t = T::try_parse_from(["t", "a.apr", "--prompt", "hi", "--inputs", "x.wav"]) + .expect("parses"); + assert_eq!(t.model.as_path(), Path::new("a.apr")); + assert_eq!(t.prompt.as_deref(), Some("hi")); + assert_eq!(t.inputs.len(), 1); + } +} diff --git a/crates/aprender-common/src/lib.rs b/crates/aprender-common/src/lib.rs index 7ee0cb44c6..4128ca843a 100644 --- a/crates/aprender-common/src/lib.rs +++ b/crates/aprender-common/src/lib.rs @@ -8,11 +8,14 @@ //! - [`fmt`] — Byte, percentage, duration, and number formatting //! - [`sys`] — System information (cgroup detection, CPU info) //! - [`display`] — Column alignment, truncation, and builder traits +//! - `cli_roles` — typed CLI argument roles (feature `cli-roles`, #3745) // `.unwrap()` is banned in production via workspace `.clippy.toml`, but is // idiomatic in `#[cfg(test)]` assertions. Allow it only under `cfg(test)`. #![cfg_attr(test, allow(clippy::disallowed_methods))] +#[cfg(feature = "cli-roles")] +pub mod cli_roles; pub mod compression; pub mod display; pub mod fmt; diff --git a/crates/aprender-compute/src/backends/gpu/shaders/cooperative.rs b/crates/aprender-compute/src/backends/gpu/shaders/cooperative.rs deleted file mode 100644 index 07a57cc920..0000000000 --- a/crates/aprender-compute/src/backends/gpu/shaders/cooperative.rs +++ /dev/null @@ -1,55 +0,0 @@ -//! Cooperative matrix GEMM shader — hardware tensor core acceleration -//! -//! Uses VK_KHR_cooperative_matrix via wgpu 29's WGSL extension. -//! GB10 Blackwell: M=16, K=16, N=16, F16 input, F32 accumulation. -//! -//! Contract: cooperative-matrix-gemm-v1 -//! Lean theorem: ProvableContracts.CooperativeMatrix.matmul_block_sum - -/// Cooperative matrix GEMM: C = α * A[M,K] @ B[K,N] -/// -/// Dispatch: ceil(N/16) × ceil(M/16) workgroups, @workgroup_size(32) -/// Fallback: if cooperative matrix not available, use TILED_GEMM_SHADER. -pub const COOPERATIVE_GEMM_SHADER: &str = r#" -enable f16; -enable wgpu_cooperative_matrix; - -@group(0) @binding(0) var a: array; -@group(0) @binding(1) var b: array; -@group(0) @binding(2) var c: array; - -struct Dimensions { - M: u32, - K: u32, - N: u32, - alpha: f32, -} - -@group(0) @binding(3) var dims: Dimensions; - -@compute @workgroup_size(32) -fn main( - @builtin(workgroup_id) wg_id: vec3, -) { - // No early return — cooperative ops require uniform control flow - let tile_row = wg_id.y * 16u; - let tile_col = wg_id.x * 16u; - - var acc = coop_mat16x16(); - let num_k_tiles = (dims.K + 15u) / 16u; - for (var kt = 0u; kt < num_k_tiles; kt++) { - let k_offset = kt * 16u; - let a_tile = coopLoad>( - &a[tile_row * dims.K + k_offset], dims.K - ); - let b_tile = coopLoad>( - &b[k_offset * dims.N + tile_col], dims.N - ); - acc = coopMultiplyAdd(a_tile, b_tile, acc); - } - // Only store if within bounds (uniform control flow maintained above) - if (tile_row < dims.M && tile_col < dims.N) { - coopStore(acc, &c[tile_row * dims.N + tile_col], dims.N); - } -} -"#; diff --git a/crates/aprender-contracts-cli/Cargo.toml b/crates/aprender-contracts-cli/Cargo.toml index 5a43b29a00..2f97d3998d 100644 --- a/crates/aprender-contracts-cli/Cargo.toml +++ b/crates/aprender-contracts-cli/Cargo.toml @@ -12,8 +12,16 @@ repository.workspace = true name = "pv" path = "src/main.rs" +# ONT-5: the reasoner. Private to this target (F-7); the library holds only the graph and the checker. +[[bin]] +name = "pv-sat" +path = "src/bin/pv-sat/main.rs" + [dependencies] aprender-contracts = { path = "../aprender-contracts", version = "0.69.3" } +# #3745 S1: `--model` is declared through the shared role type `ModelPath`, so +# `apr surface` reads `apr pv verify-structure --model` as a model argument. +batuta-common = { workspace = true, features = ["cli-roles"] } clap = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/aprender-contracts-cli/src/bin/pv-sat/liskov.rs b/crates/aprender-contracts-cli/src/bin/pv-sat/liskov.rs new file mode 100644 index 0000000000..bd4b222e94 --- /dev/null +++ b/crates/aprender-contracts-cli/src/bin/pv-sat/liskov.rs @@ -0,0 +1,200 @@ +//! ONT-4e (R-20): the Liskov half of pv-sat. For every checkable `A refines B` it decides the three obligations and +//! writes the certificate the `refines` gate re-checks: `contracts/witness/liskov/.json`. +//! +//! | obligation | premise | conclusion | broken when | +//! |------------|----------------|----------------|--------------------------------------------| +//! | pre | B.requires | A.requires | A demands something B did not (strengthened) | +//! | post | A.ensures | B.ensures | A drops a promise B made (weakened) | +//! | inv | A.invariants | B.invariants | A drops an invariant B kept (dropped) | +//! +//! Atoms are opaque: a conclusion clause is implied only by a premise clause with the same `formal` (whitespace +//! collapsed). The direction is spelled out HERE, not borrowed from the library's `Kind::sides`, so a reasoner and a +//! checker that disagree about which way refinement runs disagree on every pair, and the checker refuses the +//! witness. Before it writes, pv-sat runs `pc_liskov_reasoner`: a planted strengthened precondition must come back as +//! a counter-model naming it, and pass the library checker. + +use std::path::Path; + +use provable_contracts::ontology::liskov::{ + check, liskov_corpus, liskov_sha256, liskov_witness_path, Clauses, CounterModel, Kind, + LiskovWitness, Obligation, Pair, PairClass, PairWitness, Step, +}; +use provable_contracts::ontology::witness::FIRED; +use provable_contracts::schema::{Clause, FormalStatus}; + +/// Decide one obligation: a chain when every conclusion clause has a same-atom premise, else the counter-model +/// making exactly the premises true. +fn decide(kind: Kind, premise: &[Clause], conclusion: &[Clause]) -> Obligation { + let mut chain = Vec::new(); + let mut violated = Vec::new(); + for c in conclusion { + match premise + .iter() + .find(|p| p.atom().is_some() && p.atom() == c.atom()) + { + Some(p) => chain.push(Step { + clause: c.id.clone(), + from: p.id.clone(), + }), + None => violated.push(c.id.clone()), + } + } + if violated.is_empty() { + Obligation { + kind, + chain: Some(chain), + counter_model: None, + } + } else { + Obligation { + kind, + chain: None, + counter_model: Some(CounterModel { + violated, + holds: premise.iter().map(|p| p.id.clone()).collect(), + }), + } + } +} + +/// The three obligations of `a refines b`, each in its own direction. +pub fn reason(p: &Pair) -> PairWitness { + let (a, b) = (&p.a_clauses, &p.b_clauses); + PairWitness { + a: p.a.clone(), + b: p.b.clone(), + obligations: vec![ + decide(Kind::Pre, &b.requires, &a.requires), + decide(Kind::Post, &a.ensures, &b.ensures), + decide(Kind::Inv, &a.invariants, &b.invariants), + ], + } +} + +/// The witness for `pairs`, before `pc_reasoner` and the git sha are stamped on it. +pub fn witness( + pairs: &[Pair], + pc_reasoner: &str, + reasoner_git_sha: Option, +) -> LiskovWitness { + LiskovWitness { + liskov_sha256: liskov_sha256(pairs), + pairs_checked: pairs.len(), + reasoner_git_sha, + pc_reasoner: pc_reasoner.into(), + pairs: pairs.iter().map(reason).collect(), + } +} + +fn parsed(id: &str, formal: &str) -> Clause { + Clause { + id: id.into(), + statement: id.into(), + formal: Some(formal.into()), + formal_status: FormalStatus::Parsed, + } +} + +/// The plant: `a` keeps `b`'s precondition and adds `PRE-2`; its postcondition and invariant are `b`'s. +pub fn plant() -> Pair { + Pair { + a: "a".into(), + b: "b".into(), + a_clauses: Clauses { + requires: vec![parsed("PRE-1", "len(x) > 0"), parsed("PRE-2", "len(x) > 1")], + ensures: vec![parsed("POST-1", "len(y) = len(x)")], + invariants: vec![parsed("INV-1", "y ≥ 0")], + }, + b_clauses: Clauses { + requires: vec![parsed("PRE-1", "len(x) > 0")], + ensures: vec![parsed("POST-1", "len(y) = len(x)")], + invariants: vec![parsed("INV-1", "y ≥ 0")], + }, + } +} + +/// `pc_liskov_reasoner`: `Ok(FIRED)` when the plant's ONLY violation is `a refines b: precondition strengthened +/// (PRE-2)`, and the library checker proves exactly that from the reasoner's witness. +pub fn pc_reasoner() -> Result<&'static str, String> { + let pairs = [plant()]; + let w = witness(&pairs, FIRED, None); + let found = check(&pairs, &w) + .map_err(|e| format!("the checker refused the reasoner's witness for the plant: {e}"))?; + let found: Vec = found.iter().map(ToString::to_string).collect(); + if found == ["a refines b: precondition strengthened (PRE-2)"] { + Ok(FIRED) + } else { + Err(format!( + "the plant strengthens exactly PRE-2, and the reasoner proved {found:?}" + )) + } +} + +/// The checkable pairs of `dir`, read the way the gate reads them. +pub fn checkable_pairs( + dir: &Path, + edges: &std::collections::BTreeSet, +) -> Vec { + let docs = provable_contracts::lint::relations_gate::corpus_documents(dir); + liskov_corpus(&docs, edges) + .pairs + .into_iter() + .filter(|p| p.class() == PairClass::Checkable) + .collect() +} + +/// Write (or confirm) the Liskov witness for `pairs` under `dir`. Nothing is written when there is no checkable +/// pair. Exit-code semantics are the caller's: `Err(1)` a control or the self-check failed, `Err(3)` an I/O error. +pub fn write(dir: &Path, pairs: &[Pair], git_sha: Option) -> Result<(), u8> { + let pc = pc_reasoner().map_err(|e| { + eprintln!("pv-sat: pc_liskov_reasoner did not fire, no Liskov witness written: {e}"); + 1 + })?; + let sha = liskov_sha256(pairs); + let path = liskov_witness_path(dir, &sha); + if pairs.is_empty() { + // Nothing to certify: a witness left from a corpus that had pairs names nothing, so it goes. + let dir_exists = path.parent().is_some_and(Path::is_dir); + return if dir_exists { + super::prune(&path).map_err(|()| 3) + } else { + Ok(()) + }; + } + let fresh = std::fs::read_to_string(&path) + .ok() + .and_then(|t| serde_json::from_str::(&t).ok()) + .filter(|w| w.liskov_sha256 == sha && w.pc_reasoner == FIRED && check(pairs, w).is_ok()); + if fresh.is_some() { + println!( + "pv-sat: {} is fresh ({} pair(s))", + path.display(), + pairs.len() + ); + return super::prune(&path).map_err(|()| 3); + } + let w = witness(pairs, pc, git_sha); + let violations = check(pairs, &w).map_err(|e| { + eprintln!("pv-sat: the reasoner's own Liskov witness does not check, none written: {e}"); + 1 + })?; + let written = path + .parent() + .map_or(Ok(()), std::fs::create_dir_all) + .and_then(|()| { + let mut text = serde_json::to_string_pretty(&w).map_err(std::io::Error::other)?; + text.push('\n'); + std::fs::write(&path, text) + }); + if let Err(e) = written { + eprintln!("pv-sat: cannot write {}: {e}", path.display()); + return Err(3); + } + println!( + "pv-sat: wrote {} ({} pair(s); {} Liskov violation(s))", + path.display(), + pairs.len(), + violations.len() + ); + super::prune(&path).map_err(|()| 3) +} diff --git a/crates/aprender-contracts-cli/src/bin/pv-sat/main.rs b/crates/aprender-contracts-cli/src/bin/pv-sat/main.rs new file mode 100644 index 0000000000..d68cf0762a --- /dev/null +++ b/crates/aprender-contracts-cli/src/bin/pv-sat/main.rs @@ -0,0 +1,243 @@ +//! `pv-sat` — ONT-5's reasoner. Decides whether a corpus's typed relations are jointly satisfiable and writes the +//! certificate the `ont-consistency` gate re-checks: `contracts/witness/.json`. +//! +//! ```text +//! pv-sat [CONTRACT_DIR] write (or confirm) the witness; default `contracts` +//! pv-sat --self-test the reasoner's plant, its satisfiable twin, and the checker's corrupt core +//! (and ONT-4e's: `pc_liskov_reasoner`, `pc_liskov_checker`) +//! ``` +//! +//! Private to this bin target (F-7): the library exports the graph and the checker, and nothing in it can reach +//! the reasoner. Before it writes, pv-sat runs its plant (`pc_reasoner`); a reasoner that cannot find the planted +//! core never writes a witness. A witness already on disk that names the same graph and still checks is left +//! untouched, so `make contracts` leaves a clean tree clean. Other `.json` files in `witness/` are pruned — +//! one graph, one witness. +//! +//! ONT-4e: once the consistency witness stands, pv-sat also certifies every checkable `A refines B` (R-20) in +//! `contracts/witness/liskov/.json` — see `liskov.rs`. +//! +//! Exit: 0 written or confirmed · 1 a control failed · 2 nothing to reason over (no Σ, no typed relation) · 3 Σ +//! malformed or the witness could not be written. + +mod liskov; +mod plant; +mod sat; + +use std::path::{Path, PathBuf}; +use std::process::ExitCode; +use std::time::Instant; + +use provable_contracts::lint::relations_gate::{typed_graph, TypedGraph}; +use provable_contracts::ontology::witness::{ + census_id_set_sha256, check, pc_checker, relations_sha256, witness_path, ClauseSet, Witness, + FIRED, +}; + +fn main() -> ExitCode { + let args: Vec = std::env::args().skip(1).collect(); + match args.as_slice() { + [flag] if flag == "--self-test" => self_test(), + [flag] if flag == "-h" || flag == "--help" => { + println!("usage: pv-sat [CONTRACT_DIR] | pv-sat --self-test | pv-sat --version"); + ExitCode::SUCCESS + } + // G0.1 (#4476): the same ` ()` shape as `pv --version`. + [flag] if flag == "-V" || flag == "--version" => { + println!("{}", version_line()); + ExitCode::SUCCESS + } + [] => write_witness(Path::new("contracts")), + [dir] => write_witness(Path::new(dir)), + _ => { + eprintln!("usage: pv-sat [CONTRACT_DIR] | pv-sat --self-test | pv-sat --version"); + ExitCode::from(3) + } + } +} + +fn version_line() -> String { + format!( + "pv-sat {} ({}) (aprender ONT-5 reasoner)", + env!("CARGO_PKG_VERSION"), + env!("APR_GIT_SHA") + ) +} + +fn self_test() -> ExitCode { + let checks: [(&str, Result<(), String>); 5] = [ + ("pc_reasoner", plant::pc_reasoner().map(|_| ())), + ("pc_model", plant::pc_model()), + ("pc_checker", pc_checker().map(|_| ())), + ("pc_liskov_reasoner", liskov::pc_reasoner().map(|_| ())), + ( + "pc_liskov_checker", + provable_contracts::ontology::liskov::pc_checker().map(|_| ()), + ), + ]; + let mut ok = true; + for (name, r) in &checks { + match r { + Ok(()) => println!("pv-sat self-test: {name} {FIRED}"), + Err(e) => { + ok = false; + eprintln!("pv-sat self-test: {name} FAILED: {e}"); + } + } + } + if ok { + ExitCode::SUCCESS + } else { + ExitCode::from(1) + } +} + +/// The consistency witness, then (ONT-4e) the Liskov witness for every checkable `refines` pair. +fn write_witness(dir: &Path) -> ExitCode { + let code = write_consistency(dir); + if code != ExitCode::SUCCESS { + return code; + } + let TypedGraph::Read { edges, .. } = typed_graph(dir) else { + return code; + }; + let pairs = liskov::checkable_pairs(dir, &edges); + liskov::write(dir, &pairs, git_head(dir)).map_or_else(ExitCode::from, |()| ExitCode::SUCCESS) +} + +fn write_consistency(dir: &Path) -> ExitCode { + let pc = match plant::pc_reasoner() { + Ok(fired) => fired, + Err(e) => { + eprintln!("pv-sat: pc_reasoner did not fire, no witness written: {e}"); + return ExitCode::from(1); + } + }; + let (ids, edges) = match typed_graph(dir) { + TypedGraph::NoSigma => { + eprintln!( + "pv-sat: no {}/ontology.yaml — nothing to reason over", + dir.display() + ); + return ExitCode::from(2); + } + TypedGraph::Malformed(e) => { + eprintln!("pv-sat: Σ is malformed: {e}"); + return ExitCode::from(3); + } + TypedGraph::Read { ids, edges } => (ids, edges), + }; + let cs = ClauseSet::from_graph(&ids, &edges); + if cs.checkable_n() == 0 { + eprintln!( + "pv-sat: no typed relation clause in {} contracts — nothing to reason over", + ids.len() + ); + return ExitCode::from(2); + } + let census_sha = census_id_set_sha256(&ids); + let relations_sha = relations_sha256(&edges); + let path = witness_path(dir, &relations_sha); + + if let Some(w) = fresh_witness(&path, &cs, &census_sha, &relations_sha) { + println!( + "pv-sat: {} is fresh ({}; checkable_n {})", + path.display(), + kind(&w), + w.checkable_n + ); + return prune(&path).map_or(ExitCode::from(3), |()| ExitCode::SUCCESS); + } + + let start = Instant::now(); + let result = sat::solve(&cs); + let cpu_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX); + if let Err(e) = check(&cs, &result) { + eprintln!("pv-sat: the reasoner's own answer does not check, no witness written: {e}"); + return ExitCode::from(1); + } + let witness = Witness { + census_id_set_sha256: census_sha, + relations_sha256: relations_sha, + reasoner_git_sha: git_head(dir), + checkable_n: cs.checkable_n(), + result, + pc_reasoner: pc.to_string(), + cpu_ms, + }; + let written = path + .parent() + .map_or(Ok(()), std::fs::create_dir_all) + .and_then(|()| { + let mut text = serde_json::to_string_pretty(&witness).map_err(std::io::Error::other)?; + text.push('\n'); + std::fs::write(&path, text) + }); + if let Err(e) = written { + eprintln!("pv-sat: cannot write {}: {e}", path.display()); + return ExitCode::from(3); + } + println!( + "pv-sat: wrote {} ({}; checkable_n {}; {} ms)", + path.display(), + kind(&witness), + witness.checkable_n, + cpu_ms + ); + prune(&path).map_or(ExitCode::from(3), |()| ExitCode::SUCCESS) +} + +fn kind(w: &Witness) -> &'static str { + match w.result { + provable_contracts::ontology::witness::WitnessResult::UnsatCore(_) => "unsat_core", + provable_contracts::ontology::witness::WitnessResult::Model(_) => "model", + } +} + +/// The witness on disk, when it names this graph, its reasoner's plant fired, and it still checks. +fn fresh_witness(path: &Path, cs: &ClauseSet, census: &str, relations: &str) -> Option { + let w: Witness = serde_json::from_str(&std::fs::read_to_string(path).ok()?).ok()?; + let fresh = w.census_id_set_sha256 == census + && w.relations_sha256 == relations + && w.pc_reasoner == FIRED + && w.checkable_n == cs.checkable_n() + && check(cs, &w.result).is_ok(); + fresh.then_some(w) +} + +/// Remove every other `<64 hex>.json` beside `keep`. +fn prune(keep: &Path) -> Result<(), ()> { + let Some(dir) = keep.parent() else { + return Ok(()); + }; + let entries = std::fs::read_dir(dir) + .map_err(|e| eprintln!("pv-sat: cannot list {}: {e}", dir.display()))?; + for entry in entries.flatten() { + let p: PathBuf = entry.path(); + let is_witness = p.extension().is_some_and(|e| e == "json") + && p.file_stem().and_then(|s| s.to_str()).is_some_and(|s| { + s.len() == 64 + && s.bytes() + .all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()) + }); + if is_witness && p != keep { + std::fs::remove_file(&p) + .map_err(|e| eprintln!("pv-sat: cannot prune {}: {e}", p.display()))?; + println!("pv-sat: pruned {}", p.display()); + } + } + Ok(()) +} + +fn git_head(dir: &Path) -> Option { + let out = std::process::Command::new("git") + .arg("-C") + .arg(dir) + .args(["rev-parse", "HEAD"]) + .output() + .ok()?; + let sha = String::from_utf8(out.stdout).ok()?.trim().to_string(); + (out.status.success() && sha.len() == 40).then_some(sha) +} + +#[cfg(test)] +mod tests; diff --git a/crates/aprender-contracts-cli/src/bin/pv-sat/plant.rs b/crates/aprender-contracts-cli/src/bin/pv-sat/plant.rs new file mode 100644 index 0000000000..7928d14625 --- /dev/null +++ b/crates/aprender-contracts-cli/src/bin/pv-sat/plant.rs @@ -0,0 +1,63 @@ +//! `pc_reasoner`: the plant pv-sat must see through before it may write a witness. +//! +//! `A⇒B, B⇒C, contradicts(A,C)` with only `A` asserted is unsatisfiable, and its only core is `[A, B, C]` — +//! a reasoner that stops one implication early, or that "finds" the conflict without deriving `C`, draws +//! something else. The core must also pass the library's checker: a reasoner and a checker that disagree about +//! the plant cannot both be trusted with the corpus. + +use std::collections::BTreeSet; + +use provable_contracts::ontology::witness::{check, Checked, ClauseSet, WitnessResult, FIRED}; + +use crate::sat::solve; + +/// The planted clause set. +pub fn plant() -> ClauseSet { + let s = |x: &str| x.to_string(); + ClauseSet { + units: [s("A")].into(), + implies: [(s("A"), s("B")), (s("B"), s("C"))].into(), + conflicts: [(s("A"), s("C"))].into(), + } +} + +/// `Ok(FIRED)` when the reasoner draws the core `[A, B, C]` and the checker accepts it. +pub fn pc_reasoner() -> Result<&'static str, String> { + let cs = plant(); + let result = solve(&cs); + let WitnessResult::UnsatCore(_) = &result else { + return Err(format!( + "the plant is unsatisfiable and the reasoner answered {result:?}" + )); + }; + match check(&cs, &result) { + Ok(Checked::Unsat { core }) => { + let want: BTreeSet = ["A", "B", "C"].map(String::from).into(); + if core == want { + Ok(FIRED) + } else { + Err(format!( + "the plant's core is [A, B, C] and the reasoner drew {core:?}" + )) + } + } + Ok(Checked::Sat) => Err("the checker read the plant's core as a model".into()), + Err(e) => Err(format!( + "the checker refused the reasoner's core for the plant: {e}" + )), + } +} + +/// The satisfiable twin of the plant — the conflict removed — must yield a model the checker accepts, so a +/// reasoner that answers "unsat" to everything cannot pass `--self-test`. +pub fn pc_model() -> Result<(), String> { + let mut cs = plant(); + cs.conflicts.clear(); + let result = solve(&cs); + match check(&cs, &result) { + Ok(Checked::Sat) => Ok(()), + other => Err(format!( + "the plant without its conflict is satisfiable, and got {other:?}" + )), + } +} diff --git a/crates/aprender-contracts-cli/src/bin/pv-sat/sat.rs b/crates/aprender-contracts-cli/src/bin/pv-sat/sat.rs new file mode 100644 index 0000000000..279890be4c --- /dev/null +++ b/crates/aprender-contracts-cli/src/bin/pv-sat/sat.rs @@ -0,0 +1,75 @@ +//! The reasoner: Horn-SAT by unit propagation, with a certificate for either answer. +//! +//! Propagation is breadth-first from the units in sorted order, so the derivation — and the witness — is the same +//! bytes on every run for the same clause set. On a conflict the core is the derivation of its two sides and +//! nothing else, in the order propagation reached each variable, which is an order the checker accepts: every +//! premise is derived before the implication that uses it. + +use std::collections::{BTreeMap, VecDeque}; + +use provable_contracts::ontology::witness::{ + implication_adjacency, variables, ClauseSet, Core, Model, Step, WitnessResult, +}; + +/// Each derived variable: (derivation index, the premise that derived it; `None` for a unit). +type Derived<'a> = BTreeMap<&'a str, (usize, Option<&'a str>)>; + +/// Decide `cs`: an `unsat_core` for the first violated conflict (in sorted order), else the least model. +pub fn solve(cs: &ClauseSet) -> WitnessResult { + let derived = propagate(cs); + let violated = cs + .conflicts + .iter() + .find(|(a, b)| derived.contains_key(a.as_str()) && derived.contains_key(b.as_str())); + if let Some((a, b)) = violated { + let mut needed: BTreeMap = BTreeMap::new(); + trace(&derived, a, &mut needed); + trace(&derived, b, &mut needed); + return WitnessResult::UnsatCore(Core { + steps: needed.into_values().collect(), + conflict: (a.clone(), b.clone()), + }); + } + WitnessResult::Model(Model { + false_vars: variables(cs) + .into_iter() + .filter(|v| !derived.contains_key(v.as_str())) + .collect(), + }) +} + +/// Breadth-first unit propagation from the sorted units. +fn propagate(cs: &ClauseSet) -> Derived<'_> { + let adj = implication_adjacency(cs); + let mut derived: Derived<'_> = BTreeMap::new(); + let mut queue: VecDeque<&str> = VecDeque::new(); + for u in &cs.units { + derived.insert(u.as_str(), (derived.len(), None)); + queue.push_back(u.as_str()); + } + while let Some(v) = queue.pop_front() { + for &w in adj.get(v).map_or(&[][..], Vec::as_slice) { + if !derived.contains_key(w) { + derived.insert(w, (derived.len(), Some(v))); + queue.push_back(w); + } + } + } + derived +} + +/// Add the derivation of `side` to `needed`, back to its unit or to a step already there. +fn trace(derived: &Derived<'_>, side: &str, needed: &mut BTreeMap) { + let mut v = side; + while let Some(&(idx, premise)) = derived.get(v) { + if needed.contains_key(&idx) { + return; + } + let Some(p) = premise else { + needed.insert(idx, Step::Unit(v.to_string())); + return; + }; + needed.insert(idx, Step::Implies(p.to_string(), v.to_string())); + v = p; + } +} diff --git a/crates/aprender-contracts-cli/src/bin/pv-sat/tests.rs b/crates/aprender-contracts-cli/src/bin/pv-sat/tests.rs new file mode 100644 index 0000000000..c632147f69 --- /dev/null +++ b/crates/aprender-contracts-cli/src/bin/pv-sat/tests.rs @@ -0,0 +1,234 @@ +//! The reasoner is only trusted through the checker, so the property here is "whatever `solve` says, `check` +//! accepts" — swept over seeded random clause sets (stdlib LCG, no proptest) — plus the controls and the writer's +//! file discipline over a copy of `tests/fixtures/ont/relations-ok`. + +use std::collections::BTreeSet; +use std::path::Path; + +use provable_contracts::lint::consistency_gate::{run_consistency_gate, ConsistencyOutcome}; +use provable_contracts::ontology::witness::{Checked, Core, Step, WitnessResult}; + +use super::*; + +fn s(x: &str) -> String { + x.to_string() +} + +#[test] +fn the_plant_draws_its_core_in_derivation_order() { + assert_eq!( + sat::solve(&plant::plant()), + WitnessResult::UnsatCore(Core { + steps: vec![ + Step::Unit(s("A")), + Step::Implies(s("A"), s("B")), + Step::Implies(s("B"), s("C")), + ], + conflict: (s("A"), s("C")), + }) + ); + assert_eq!(plant::pc_reasoner(), Ok(FIRED)); + assert_eq!(plant::pc_model(), Ok(())); + assert_eq!(pc_checker(), Ok(FIRED)); +} + +/// Deterministic LCG (Knuth MMIX constants). +struct Lcg(u64); +impl Lcg { + fn next(&mut self, n: u64) -> usize { + self.0 = self + .0 + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + usize::try_from((self.0 >> 33) % n).expect("small") + } +} + +#[test] +fn every_answer_the_reasoner_gives_checks() { + let names: Vec = (0..8).map(|i| format!("v{i}")).collect(); + let (mut sat_n, mut unsat_n) = (0, 0); + for seed in 0..500_u64 { + let mut r = Lcg(seed); + let mut cs = ClauseSet { + units: BTreeSet::new(), + implies: BTreeSet::new(), + conflicts: BTreeSet::new(), + }; + for _ in 0..=r.next(3) { + cs.units.insert(names[r.next(8)].clone()); + } + for _ in 0..r.next(10) { + let (a, b) = (r.next(8), r.next(8)); + if a != b { + cs.implies.insert((names[a].clone(), names[b].clone())); + } + } + for _ in 0..r.next(5) { + let (a, b) = (r.next(8), r.next(8)); + if a != b { + cs.conflicts + .insert((names[a.min(b)].clone(), names[a.max(b)].clone())); + } + } + let first = sat::solve(&cs); + assert_eq!(first, sat::solve(&cs), "seed {seed}: not deterministic"); + match check(&cs, &first) { + Ok(Checked::Sat) => sat_n += 1, + Ok(Checked::Unsat { .. }) => unsat_n += 1, + Err(e) => panic!( + "seed {seed}: the reasoner's own answer does not check: {e}\n{cs:?}\n{first:?}" + ), + } + } + // The sweep must visit both answers, or it proves half the claim. + assert!(sat_n > 50 && unsat_n > 50, "sat {sat_n} / unsat {unsat_n}"); +} + +fn corpus() -> tempfile::TempDir { + let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../tests/fixtures/ont/relations-ok"); + let dir = tempfile::tempdir().expect("tempdir"); + for entry in std::fs::read_dir(src).expect("fixture") { + let entry = entry.expect("entry"); + std::fs::copy(entry.path(), dir.path().join(entry.file_name())).expect("copy"); + } + dir +} + +fn exit(code: ExitCode) -> String { + format!("{code:?}") +} + +#[test] +fn the_written_witness_is_the_one_the_gate_checks_and_a_rerun_leaves_it() { + let dir = corpus(); + let stray = dir + .path() + .join("witness") + .join(format!("{}.json", "a".repeat(64))); + std::fs::create_dir_all(stray.parent().expect("dir")).expect("mkdir"); + std::fs::write(&stray, "{}").expect("stray"); + let notes = dir.path().join("witness").join("README.json"); + std::fs::write(¬es, "{}").expect("notes"); + + assert_eq!(exit(write_witness(dir.path())), exit(ExitCode::SUCCESS)); + assert!(!stray.exists(), "another graph's witness was not pruned"); + assert!(notes.exists(), "a file that is not a witness was pruned"); + + // relations-ok is inconsistent (`a contradicts d`, both live): the gate must reach a verdict, and it is a Fail. + match run_consistency_gate(dir.path()) { + ConsistencyOutcome::Ran { findings, .. } => { + let ids: Vec<&str> = findings.iter().map(|f| f.rule_id.as_str()).collect(); + assert_eq!(ids, ["PV-ONT-022"]); + } + other => panic!("expected a verdict, got {other:?}"), + } + + let written: Vec<_> = std::fs::read_dir(dir.path().join("witness")) + .expect("witness dir") + .flatten() + .map(|e| e.path()) + .filter(|p| p.file_name() != notes.file_name()) + .collect(); + assert_eq!(written.len(), 1); + let before = std::fs::read(&written[0]).expect("witness"); + let mtime = std::fs::metadata(&written[0]) + .and_then(|m| m.modified()) + .expect("mtime"); + assert_eq!(exit(write_witness(dir.path())), exit(ExitCode::SUCCESS)); + assert_eq!(std::fs::read(&written[0]).expect("witness"), before); + assert_eq!( + std::fs::metadata(&written[0]) + .and_then(|m| m.modified()) + .expect("mtime"), + mtime + ); +} + +#[test] +fn nothing_to_reason_over_is_exit_2_and_a_broken_sigma_exit_3() { + let dir = corpus(); + std::fs::remove_file(dir.path().join("ontology.yaml")).expect("rm"); + assert_eq!(exit(write_witness(dir.path())), exit(ExitCode::from(2))); + assert!(!dir.path().join("witness").exists()); + + let dir = corpus(); + std::fs::write(dir.path().join("ontology.yaml"), "roles: [unclosed\n").expect("write"); + assert_eq!(exit(write_witness(dir.path())), exit(ExitCode::from(3))); +} + +/// ONT-4e: the Liskov plant is found, and the reasoner's directions are the ones the checker proves — a strengthened +/// precondition, a weakened postcondition and a dropped invariant each name their clause, and a weakened +/// precondition / strengthened postcondition are Liskov. +#[test] +fn the_liskov_reasoner_decides_each_direction_the_checker_proves() { + use provable_contracts::ontology::liskov::{check, Clauses, Pair}; + use provable_contracts::schema::{Clause, FormalStatus}; + + assert_eq!(liskov::pc_reasoner(), Ok(FIRED)); + assert_eq!( + provable_contracts::ontology::liskov::pc_checker(), + Ok(FIRED) + ); + + let c = |id: &str, f: &str| Clause { + id: s(id), + statement: s(id), + formal: Some(s(f)), + formal_status: FormalStatus::Parsed, + }; + let verdict = |a: Clauses, b: Clauses| -> Vec { + let pairs = [Pair { + a: s("a"), + b: s("b"), + a_clauses: a, + b_clauses: b, + }]; + let w = liskov::witness(&pairs, FIRED, None); + check(&pairs, &w) + .expect("the reasoner's witness checks") + .iter() + .map(ToString::to_string) + .collect() + }; + let req = |v: Vec| Clauses { + requires: v, + ..Clauses::default() + }; + let ens = |v: Vec| Clauses { + ensures: v, + ..Clauses::default() + }; + let inv = |v: Vec| Clauses { + invariants: v, + ..Clauses::default() + }; + assert_eq!( + verdict(req(vec![c("PRE-1", "p")]), Clauses::default()), + ["a refines b: precondition strengthened (PRE-1)"] + ); + assert!(verdict(Clauses::default(), req(vec![c("PRE-1", "p")])).is_empty()); + assert_eq!( + verdict(Clauses::default(), ens(vec![c("POST-1", "q")])), + ["a refines b: postcondition weakened (POST-1)"] + ); + assert!(verdict(ens(vec![c("POST-1", "q")]), Clauses::default()).is_empty()); + assert_eq!( + verdict(Clauses::default(), inv(vec![c("INV-1", "r")])), + ["a refines b: invariant dropped (INV-1)"] + ); +} + +/// G0.1 (#4476): `pv-sat --version` exists and names the semver and the commit. +#[test] +fn g0_1_version_line_names_semver_and_commit() { + let v = version_line(); + let f: Vec<&str> = v.split_whitespace().collect(); + assert_eq!(f.first(), Some(&"pv-sat"), "{v}"); + assert_eq!(f.get(1), Some(&env!("CARGO_PKG_VERSION")), "{v}"); + assert_eq!( + f.get(2).copied(), + Some(format!("({})", env!("APR_GIT_SHA")).as_str()), + "{v}" + ); +} diff --git a/crates/aprender-contracts-cli/src/cli.rs b/crates/aprender-contracts-cli/src/cli.rs index c790ff16f8..487a1247da 100644 --- a/crates/aprender-contracts-cli/src/cli.rs +++ b/crates/aprender-contracts-cli/src/cli.rs @@ -84,6 +84,17 @@ pub enum Commands { /// Path to the new contract YAML file new: PathBuf, }, + /// What the Lean proofs rest on: axiom subset pins and the compiler-escape allowlist (PVL-001 EV-6a, #4139) + Discharge { + #[command(subcommand)] + action: DischargeAction, + }, + /// Pin each contract-bound theorem's STATEMENT apart from its proof: `/Challenge/.lean` + /// (PVL-001 EV-7a, #4200) + Challenge { + #[command(subcommand)] + action: ChallengeAction, + }, /// Census the contract corpus: one cardinality, by_anchoring, by_entity_type (ONT-001 ONT-1) Census { /// Directory containing contract YAML files @@ -109,9 +120,18 @@ pub enum Commands { /// the tracked contracts.nt / shapes.ttl untouched (aprender#3715) #[arg(long)] out: Option, + /// With `--release-*` and `--surface`: write every DERIVED release cell (the producer's work list, keyed + /// by `cell_id`) to this JSON file (aprender#3745 S2) + #[arg(long)] + cells_out: Option, #[command(flatten)] release: Box, }, + /// Σ as OWL and its advisory TBox (ONT-001 §3.8, ONT-2c) + Ontology { + #[command(subcommand)] + command: OntologyCommand, + }, /// Show cross-contract obligation coverage report Coverage { /// Directory containing contract YAML files @@ -182,10 +202,10 @@ pub enum Commands { /// Path to binding registry YAML (adds binding coverage) #[arg(long)] binding: Option, - /// L5 gate: before counting a binding as implemented, verify its - /// `function` actually exists in source (scanned from the given root, - /// default `.`). Phantom "implemented" bindings are downgraded, so L5 - /// means "verified as implemented", not self-declared. + /// No-op alias (PVL-001 EV-2): `--binding` now ALWAYS resolves every + /// `implemented` binding against source with the `pv verify-bindings` + /// resolver, lists ghosts under `GHOST BINDINGS (n)` and exits 1. Kept so + /// existing invocations still parse; its root argument is ignored. #[arg(long, num_args = 0..=1, default_missing_value = ".")] verify_bindings: Option, /// Output format: text (default) or json @@ -209,8 +229,9 @@ pub enum Commands { /// Path to binding registry YAML #[arg(long)] binding: Option, - /// Output format: text (default), json, sarif, github - #[arg(short, long)] + /// Output format: text (default), json, sarif, github, html. + /// G1.3 (#4476): parsed by clap, so an unknown value exits 2 and names these. + #[arg(short, long, value_parser = ["text", "json", "sarif", "github", "html"])] format: Option, /// Minimum severity to report: error, warning, info #[arg(long)] @@ -287,9 +308,11 @@ pub enum Commands { /// merge-base(HEAD, origin/main), else the origin/main tip; with neither, NOT CHECKED is printed. #[arg(long)] armed_baseline_ref: Option, - /// Run ONE named gate and report only it (ONT-001 section 5 ONT-2b): `--gate sigma`. + /// Run ONE named gate and report only it (ONT-001 section 5 ONT-2b): `--gate sigma`. Repeatable + /// (PVL-001 EV-11): every named gate runs and reports, and the exit is their meet — a refusal over a + /// reject over a decline over a pass. #[arg(long)] - gate: Option, + gate: Vec, /// With `--gate shapes`: grade only this shape family (the shape and every `.*` shape), armed /// whatever `armed_shapes` says (aprender#3715: `--shape release-readiness-v1`). #[arg(long)] @@ -374,6 +397,17 @@ pub enum Commands { #[arg(long, default_value = "20")] top: usize, }, + /// Obligation gate (PVL-001 EV-10): every contract under ROOT/contracts validates, hides no + /// test under `falsification:`, and binds each `applies_to` to a `fn` under ROOT/src that + /// mentions the contract's `proved_type`. Replaces pmat's `scripts/pv-obligation-gate.py`. + Obligations { + /// Repository root holding `contracts/` and `src/` + #[arg(default_value = ".")] + root: PathBuf, + /// Exit 1 (`reject:`) when any problem is found; without it the report exits 0 + #[arg(long)] + gate: bool, + }, /// Remove enforcement level lock from a contract (requires --reason) Unlock { /// Path to the contract YAML file @@ -442,8 +476,9 @@ pub enum Commands { contract_dir: PathBuf, #[arg(long)] config: Option, + /// #3745 S1: typed `ModelPath`, which is how `apr surface` knows this is a model. #[arg(long)] - model: Option, + model: Option, }, /// Verify compositional shape flow across contract dependency graph #[command(name = "verify-pipeline")] @@ -473,6 +508,131 @@ pub enum Commands { }, } +/// `pv discharge` actions (PVL-001 EV-6a, #4139). +#[derive(Subcommand, Clone, Debug)] +pub enum DischargeAction { + /// Generate `/Axioms.lean`: a subset axiom pin per contract-bound theorem in the root's import cone + GenAxioms { + /// The Lean dir (holds ProvableContracts.lean) + lean_dir: PathBuf, + /// Directory of the contracts whose `lean_theorem:` references bind the roots + #[arg(long, default_value = "contracts")] + contracts: PathBuf, + /// Do not write: rc 1 when the tracked Axioms.lean differs from its regeneration + #[arg(long)] + check: bool, + }, + /// Judge the tree: escapes vs escape-allowlist.yaml, exact-name roots, the label ratchet, Axioms.lean + /// freshness, then `lake env lean Axioms.lean` (after `build.sh`) unless `--no-lake` + Check { + lean_dir: PathBuf, + #[arg(long, default_value = "contracts")] + contracts: PathBuf, + /// Skip the Lean elaboration of Axioms.lean + #[arg(long)] + no_lake: bool, + /// Allowlist entries still `confirmed_by: pending` are RED + #[arg(long)] + strict: bool, + /// Also judge `/formalization.yaml`: `main_results` listed by discharge-summary.json, + /// `status.axioms` the pinned kernel set, `sorry_count` the measured count; a missing file is RED + /// (PVL-001 EV-8b, #4082) + #[arg(long)] + validate_formalization: bool, + /// Also re-check the BUILT tree's .olean files: `timeout lake env leanchecker ProvableContracts` + /// (non-fresh; `--fresh`, which replays Mathlib, is the nightly's, PVL-F7). rc != 0 rejects; no + /// `leanchecker` in the toolchain declines (PVL-001 EV-6b, #4199) + #[arg(long, conflicts_with = "no_lake")] + leanchecker: bool, + /// `--leanchecker`'s wall-clock limit, seconds + #[arg(long, default_value_t = 3600, requires = "leanchecker")] + leanchecker_timeout: u64, + /// `--leanchecker` under `ulimit -v ` (virtual memory, KiB); unset = no limit + #[arg(long, requires = "leanchecker")] + leanchecker_ulimit_v: Option, + /// `--leanchecker` with at most N Lean worker threads (`LEAN_NUM_THREADS`), 1..=8: the default is also the ceiling. leanchecker replays one full + /// environment per concurrent module task, so memory scales with this: 51 threads took 58-67 GB (#4348) + #[arg(long, default_value_t = crate::commands::discharge::LEANCHECKER_THREADS, value_parser = clap::value_parser!(u32).range(1..=i64::from(crate::commands::discharge::LEANCHECKER_THREADS)), requires = "leanchecker")] + leanchecker_threads: u32, + /// `--leanchecker` inside `systemd-run --user --scope -p MemoryMax=G -p CPUQuota=%`: this memory cap, + /// GiB, 1..=24: a caller may lower it, never raise it (#4348: the host must stay usable) + #[arg(long, default_value_t = crate::commands::discharge::LEANCHECKER_MEMORY_MAX_GIB, value_parser = clap::value_parser!(u32).range(1..=i64::from(crate::commands::discharge::LEANCHECKER_MEMORY_MAX_GIB)), requires = "leanchecker")] + leanchecker_memory_max_gib: u32, + /// `--leanchecker`'s scope CPU quota, percent of one core, 1..=800 + #[arg(long, default_value_t = crate::commands::discharge::LEANCHECKER_CPU_QUOTA_PCT, value_parser = clap::value_parser!(u32).range(1..=i64::from(crate::commands::discharge::LEANCHECKER_CPU_QUOTA_PCT)), requires = "leanchecker")] + leanchecker_cpu_quota_pct: u32, + /// Run `--leanchecker` WITHOUT the systemd scope (a host with no user systemd). The thread cap still applies + #[arg(long, requires = "leanchecker")] + leanchecker_unscoped: bool, + /// Also run the comparator: `lake env lean --run scripts/Comparator.lean Challenge/*.lean` on the BUILT + /// tree. Each EV-7a challenge must be closed by a sorry-free solution of the SAME statement (sha256 of + /// the canonical type). No Challenge file, or zero rows, declines (PVL-001 EV-7b, #4201) + #[arg(long, conflicts_with = "no_lake")] + comparator: bool, + /// Wall-clock limit on each `lake env lean` call (Axioms.lean, the comparator), seconds. A call that + /// exceeds it is killed with its process group and rejects: a hang is RED, not a wait (#4239) + #[arg(long, default_value_t = crate::commands::discharge::LAKE_TIMEOUT_S)] + lake_timeout: u64, + }, + /// `build.sh`, then `check` with every arm (`--strict`, the comparator, `--leanchecker`), then write the + /// untracked full log `/discharge.json` and the TRACKED `/../discharge-summary.json` -- + /// on failure too. The Lean steps run only after `build.sh` exits 0 (PVL-001 EV-8a, #4202) + Run { + lean_dir: PathBuf, + #[arg(long, default_value = "contracts")] + contracts: PathBuf, + /// The leanchecker arm's wall-clock limit, seconds + #[arg(long, default_value_t = 3600)] + leanchecker_timeout: u64, + /// The leanchecker arm under `ulimit -v ` (virtual memory, KiB); unset = no limit + #[arg(long)] + leanchecker_ulimit_v: Option, + /// the leanchecker arm with at most N Lean worker threads (`LEAN_NUM_THREADS`), 1..=8: the default is also the ceiling. leanchecker replays one full + /// environment per concurrent module task, so memory scales with this: 51 threads took 58-67 GB (#4348) + #[arg(long, default_value_t = crate::commands::discharge::LEANCHECKER_THREADS, value_parser = clap::value_parser!(u32).range(1..=i64::from(crate::commands::discharge::LEANCHECKER_THREADS)))] + leanchecker_threads: u32, + /// the leanchecker arm inside `systemd-run --user --scope -p MemoryMax=G -p CPUQuota=%`: this memory cap, + /// GiB, 1..=24: a caller may lower it, never raise it (#4348: the host must stay usable) + #[arg(long, default_value_t = crate::commands::discharge::LEANCHECKER_MEMORY_MAX_GIB, value_parser = clap::value_parser!(u32).range(1..=i64::from(crate::commands::discharge::LEANCHECKER_MEMORY_MAX_GIB)))] + leanchecker_memory_max_gib: u32, + /// the leanchecker arm's scope CPU quota, percent of one core, 1..=800 + #[arg(long, default_value_t = crate::commands::discharge::LEANCHECKER_CPU_QUOTA_PCT, value_parser = clap::value_parser!(u32).range(1..=i64::from(crate::commands::discharge::LEANCHECKER_CPU_QUOTA_PCT)))] + leanchecker_cpu_quota_pct: u32, + /// Run the leanchecker arm WITHOUT the systemd scope (a host with no user systemd). The thread cap still applies + #[arg(long)] + leanchecker_unscoped: bool, + /// Wall-clock limit on each `lake env` call outside the leanchecker arm, seconds; a timeout rejects (#4239) + #[arg(long, default_value_t = crate::commands::discharge::LAKE_TIMEOUT_S)] + lake_timeout: u64, + }, + /// `make label-ratchet`: rewrite /unresolved-labels.json DOWNWARD (it never gains a label; a missing + /// file is seeded). `check` never writes it. + LabelRatchet { + lean_dir: PathBuf, + #[arg(long, default_value = "contracts")] + contracts: PathBuf, + }, +} + +/// `pv challenge` actions (PVL-001 EV-7a, #4200). +#[derive(Subcommand, Clone, Debug)] +pub enum ChallengeAction { + /// Write `/Challenge/.lean`: every bound theorem restated as `PvlChallenge.` with + /// its proof replaced by `sorry`. Stale files are removed. + Gen { + /// Directory of the contracts whose `lean_theorem:` references bind the roots + contracts: PathBuf, + /// The Lean dir (holds ProvableContracts.lean) + lean_dir: PathBuf, + }, + /// Regenerate in memory and compare with `/Challenge/`: rc 1 on any difference, rc 2 on zero + /// challenges + Check { + contracts: PathBuf, + lean_dir: PathBuf, + }, +} + /// `pv census` output format (ONT-001 ONT-1). #[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)] pub enum CensusFormat { @@ -508,6 +668,12 @@ pub struct ReleaseArgs { /// The tokenizer-parity receipts, apr vs the pinned llama.cpp (default: evidence/dogfood/tokenizer//) #[arg(long)] pub tokenizer_receipts: Option, + /// The release candidate's `apr surface --json` (#3745): the release cells are DERIVED from it + #[arg(long)] + pub surface: Option, + /// The CRUX receipts, apr vs its comparators (default: evidence/crux//) + #[arg(long)] + pub crux_receipts: Option, } impl ReleaseArgs { @@ -521,6 +687,8 @@ impl ReleaseArgs { || self.kernel_receipts.is_some() || self.dogfood_receipt.is_some() || self.tokenizer_receipts.is_some() + || self.surface.is_some() + || self.crux_receipts.is_some() } /// The subject, or `None` when no flag was passed. A partial set is refused, never completed by a default. @@ -548,6 +716,34 @@ impl ReleaseArgs { s.dogfood_receipt.clone_from(&self.dogfood_receipt); s.tokenizer_receipts_dir .clone_from(&self.tokenizer_receipts); + s.surface.clone_from(&self.surface); + s.crux_receipts_dir.clone_from(&self.crux_receipts); Ok(Some(s)) } } + +/// `pv ontology …` (ONT-001 §3.8, row ONT-2c). +#[derive(Subcommand, Clone, Debug)] +pub enum OntologyCommand { + /// Write Σ as OWL 2 EL functional syntax (the in-house writer; byte-deterministic) + Export { + /// Σ, the ontology declaration + #[arg(default_value = "contracts/ontology.yaml")] + sigma: PathBuf, + /// OWL 2 functional syntax. The only format this command writes; required so the output is named + #[arg(long)] + owl: bool, + /// Write `ontology.ofn` next to Σ instead of printing it + #[arg(long)] + write: bool, + }, + /// The told-closure TBox report (advisory; `tbox-report.json`). Exit 3 if its precondition fails + Tbox { + /// Σ, the ontology declaration + #[arg(default_value = "contracts/ontology.yaml")] + sigma: PathBuf, + /// Write `tbox-report.json` next to Σ instead of printing it + #[arg(long)] + write: bool, + }, +} diff --git a/crates/aprender-contracts-cli/src/commands/census.rs b/crates/aprender-contracts-cli/src/commands/census.rs index ab379a1d48..56e8c29f1f 100644 --- a/crates/aprender-contracts-cli/src/commands/census.rs +++ b/crates/aprender-contracts-cli/src/commands/census.rs @@ -120,6 +120,11 @@ pub struct Census { pub quarantined_n: usize, pub by_kind: BTreeMap, pub by_entity_type: BTreeMap, + /// ONT-4d: instances per Σ concept in the extracted graph AFTER the subsumption closure, so a concept + /// counts its sub-concepts' instances too. Empty when there is no well-formed Σ or the extraction refused + /// (the `sigma` / `shapes` gates report why); never a guess. + #[serde(default)] + pub by_concept: BTreeMap, pub by_anchoring: AnchoringCounts, /// sha256 over the sorted, unique contract ids (file stems), newline /// separated. Two corpora with the same ids hash the same; adding, removing @@ -271,9 +276,23 @@ pub fn census_of(dir: &Path) -> Result> { } census.id_set_sha256 = id_set_sha256(&ids); census.declared_external = declared_external(dir)?; + census.by_concept = by_concept(dir); Ok(census) } +/// ONT-4d: `concept → instances` over the closed graph (see [`Census::by_concept`]). +fn by_concept(dir: &Path) -> BTreeMap { + use provable_contracts::ontology::{extract, rdf::ont}; + let (Some(sigma), Ok(x)) = (extract::sigma_of(dir), extract::all(dir)) else { + return BTreeMap::new(); + }; + sigma + .concepts + .keys() + .map(|c| (c.clone(), x.graph.instances_of(&ont(c)).len())) + .collect() +} + /// Contracts held OUT of the corpus, counted but never parsed. The shared walker /// skips `quarantine/`, so this is its own scan: a number the census reports is a /// number the census measured. @@ -311,6 +330,7 @@ fn empty_census(n_files: usize, quarantined_n: usize) -> Census { quarantined_n, by_kind: BTreeMap::new(), by_entity_type: BTreeMap::new(), + by_concept: BTreeMap::new(), by_anchoring: AnchoringCounts::default(), id_set_sha256: String::new(), declared_external: Vec::new(), diff --git a/crates/aprender-contracts-cli/src/commands/challenge.rs b/crates/aprender-contracts-cli/src/commands/challenge.rs new file mode 100644 index 0000000000..8673cb8ac8 --- /dev/null +++ b/crates/aprender-contracts-cli/src/commands/challenge.rs @@ -0,0 +1,89 @@ +//! `pv challenge gen | check` (PVL-001 EV-7a, #4200). The rendering lives in +//! [`provable_contracts::discharge::challenge`]; this module writes or compares, and prints. +//! +//! Exit: 0 fresh · 1 reject (a statement could not be lifted, or `Challenge/` differs from its regeneration) +//! · 2 decline (the tree does not load, or zero challenges: nothing is pinned, which must not read as fresh). + +use std::path::Path; + +use provable_contracts::discharge::challenge::{self, Rendered, CHALLENGE_DIR}; + +use super::discharge::{DischargeDeclined, DischargeRejected}; +use crate::cli::ChallengeAction; + +type Res = Result<(), Box>; + +pub fn run(action: ChallengeAction) -> Res { + match action { + ChallengeAction::Gen { + contracts, + lean_dir, + } => gen(&contracts, &lean_dir), + ChallengeAction::Check { + contracts, + lean_dir, + } => check(&contracts, &lean_dir), + } +} + +fn load(contracts: &Path, lean_dir: &Path) -> Result { + let r = challenge::render(lean_dir, contracts).map_err(DischargeDeclined)?; + if r.files.is_empty() && r.unrestated.is_empty() { + return Err(DischargeDeclined(format!( + "zero challenges: no contract under {} binds a theorem of {}", + contracts.display(), + lean_dir.display() + ))); + } + Ok(r) +} + +/// One FAIL line per root whose statement could not be lifted; the count. +fn report_unrestated(r: &Rendered) -> usize { + for (stem, fqn, why) in &r.unrestated { + println!("FAIL UNRESTATED {stem}: {fqn}: {why}"); + } + r.unrestated.len() +} + +fn gen(contracts: &Path, lean_dir: &Path) -> Res { + let r = load(contracts, lean_dir)?; + let written = challenge::write(lean_dir, &r) + .map_err(|e| DischargeDeclined(format!("{}/{CHALLENGE_DIR}: {e}", lean_dir.display())))?; + for rel in &written { + println!("wrote {rel}"); + } + println!( + "{} challenge file(s) under {}/{CHALLENGE_DIR} ({} rewritten)", + r.files.len(), + lean_dir.display(), + written.len() + ); + match report_unrestated(&r) { + 0 => Ok(()), + n => Err(DischargeRejected(format!("{n} bound theorem(s) have no challenge")).into()), + } +} + +fn check(contracts: &Path, lean_dir: &Path) -> Res { + let r = load(contracts, lean_dir)?; + let diffs = challenge::diff(lean_dir, &r); + for d in &diffs { + println!("FAIL STALE {d}"); + } + let n = diffs.len() + report_unrestated(&r); + if n > 0 { + return Err(DischargeRejected(format!( + "{n} failure(s); regenerate with `pv challenge gen {} {}`", + contracts.display(), + lean_dir.display() + )) + .into()); + } + println!( + "challenge-fresh: {} file(s) under {}/{CHALLENGE_DIR} match their regeneration", + r.files.len(), + lean_dir.display() + ); + Ok(()) +} diff --git a/crates/aprender-contracts-cli/src/commands/discharge.rs b/crates/aprender-contracts-cli/src/commands/discharge.rs new file mode 100644 index 0000000000..c1b8b9201a --- /dev/null +++ b/crates/aprender-contracts-cli/src/commands/discharge.rs @@ -0,0 +1,1991 @@ +//! `pv discharge gen-axioms | check | run` (PVL-001 EV-6a, #4139; `--leanchecker` EV-6b, #4199; `--comparator` +//! EV-7b, #4201; `run` EV-8a, #4202). The judging lives in +//! [`provable_contracts::discharge`]; this module prints the report and runs Lean. +//! +//! Exit: 0 accept · 1 reject (`reject:`) · 2 decline (`decline:` — no root file, zero roots, no `lake`, no +//! `leanchecker` in the toolchain, no `Challenge/*.lean` or zero comparator rows). +//! +//! `--leanchecker` is NON-fresh: it re-checks the tree's own .olean files and trusts the Mathlib .oleans they +//! import. `--fresh` replays Mathlib and is the nightly's (PVL-F7). `formalization.yaml` `scope` says so. + +use std::ffi::OsStr; +use std::fmt; +use std::path::Path; +use std::process::Command; +use std::time::{Duration, Instant}; + +use provable_contracts::discharge::comparator::{self, CHALLENGE_DIR, COMPARATOR}; +use provable_contracts::discharge::summary::{self, LOG_FILE}; +use provable_contracts::discharge::{self, CheckOpts, Report, Tree, AXIOMS_FILE}; + +use crate::cli::DischargeAction; + +/// Nothing could be judged. Exit 2. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DischargeDeclined(pub String); + +impl fmt::Display for DischargeDeclined { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl std::error::Error for DischargeDeclined {} + +/// Judged, and failed. Exit 1. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DischargeRejected(pub String); + +impl fmt::Display for DischargeRejected { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl std::error::Error for DischargeRejected {} + +type Res = Result<(), Box>; + +pub fn run(action: DischargeAction) -> Res { + match action { + DischargeAction::GenAxioms { + lean_dir, + contracts, + check, + } => gen_axioms(&lean_dir, &contracts, check), + DischargeAction::Check { + lean_dir, + contracts, + no_lake, + strict, + validate_formalization, + leanchecker, + leanchecker_timeout, + leanchecker_ulimit_v, + leanchecker_threads, + leanchecker_memory_max_gib, + leanchecker_cpu_quota_pct, + leanchecker_unscoped, + comparator, + lake_timeout, + } => { + let opts = CheckOpts { + strict, + validate_formalization, + }; + let r = discharge::check(&lean_dir, &contracts, opts); + let lc = leanchecker.then_some(Leanchecker::from_flags( + leanchecker_timeout, + leanchecker_ulimit_v, + leanchecker_threads, + leanchecker_memory_max_gib, + leanchecker_cpu_quota_pct, + leanchecker_unscoped, + )); + finish_with( + Lake::new(lake_timeout), + r, + &lean_dir, + no_lake, + comparator, + lc, + ) + } + DischargeAction::Run { + lean_dir, + contracts, + leanchecker_timeout, + leanchecker_ulimit_v, + leanchecker_threads, + leanchecker_memory_max_gib, + leanchecker_cpu_quota_pct, + leanchecker_unscoped, + lake_timeout, + } => run_all( + Lake::new(lake_timeout), + "build.sh", + &lean_dir, + &contracts, + Leanchecker::from_flags( + leanchecker_timeout, + leanchecker_ulimit_v, + leanchecker_threads, + leanchecker_memory_max_gib, + leanchecker_cpu_quota_pct, + leanchecker_unscoped, + ), + ), + DischargeAction::LabelRatchet { + lean_dir, + contracts, + } => finish_with( + Lake::new(LAKE_TIMEOUT_S), + discharge::ratchet_labels(&lean_dir, &contracts), + &lean_dir, + true, + false, + None, + ), + } +} + +fn gen_axioms(lean_dir: &Path, contracts: &Path, check: bool) -> Res { + let g = discharge::generate(lean_dir, contracts).map_err(DischargeDeclined)?; + let (text, b) = (g.text, g.binding); + let path = lean_dir.join(AXIOMS_FILE); + if check { + return match std::fs::read_to_string(&path) { + Ok(on_disk) if on_disk == text => { + println!( + "ok {} is its regeneration ({} root(s) bound)", + path.display(), + b.roots.len() + ); + Ok(()) + } + Ok(_) => Err(DischargeRejected(format!( + "{} differs from its regeneration", + path.display() + )) + .into()), + Err(e) => Err(DischargeRejected(format!("{}: {e}", path.display())).into()), + }; + } + std::fs::write(&path, &text)?; + println!("wrote {} ({} root(s) bound)", path.display(), b.roots.len()); + Ok(()) +} + +/// How `--leanchecker` runs (PVL-001 EV-6b, #4199; the caps #4348). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Leanchecker { + pub timeout_s: u64, + pub ulimit_v_kib: Option, + /// `LEAN_NUM_THREADS`: leanchecker replays one full environment (Mathlib included) per concurrent module + /// task, so its memory is threads x environment. Uncapped it took 51 threads and 58-67 GB on lambda (#4348). + pub threads: u32, + /// `None` = no systemd scope (`--leanchecker-unscoped`). + pub scope: Option, +} + +impl Leanchecker { + /// The `--leanchecker-*` flags, shared by `check` and `run`. + fn from_flags( + timeout_s: u64, + ulimit_v_kib: Option, + threads: u32, + memory_max_gib: u32, + cpu_quota_pct: u32, + unscoped: bool, + ) -> Self { + Self { + timeout_s, + ulimit_v_kib, + threads, + scope: (!unscoped).then_some(Scope { + memory_max_gib, + cpu_quota_pct, + }), + } + } +} + +/// `systemd-run --user --scope -p MemoryMax=G -p CPUQuota=%` around leanchecker (#4348, operator: "this +/// host must be able to do other work, so never let it get overloaded"). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Scope { + pub memory_max_gib: u32, + pub cpu_quota_pct: u32, +} + +/// The defaults the cop ruled for lambda (#4348): <= 8 Lean threads, 24G, 800%. +pub const LEANCHECKER_THREADS: u32 = 8; +pub const LEANCHECKER_MEMORY_MAX_GIB: u32 = 24; +pub const LEANCHECKER_CPU_QUOTA_PCT: u32 = 800; + +/// The slice the scope is created under. lambda's `agent-slice-sweep` adopts any lean/lake process OUTSIDE it into +/// it -- measured 2026-09-25: a leanchecker in a 24G `run-*.scope` in app.slice was moved to agent.slice (96G) and +/// grew to 71G. A scope nested in agent.slice is left alone and its own, tighter MemoryMax still binds. On a host +/// without that slice systemd creates it as a plain transient slice. +const AGENT_SLICE: &str = "agent.slice"; + +/// The script `sh -c` runs; `$1` ulimit, `$2` timeout, `$3` lake. +const RECHECK_SH: &str = r#"if [ -n "$1" ]; then ulimit -v "$1" || exit 125; fi; exec timeout -k 30 "$2" "$3" env leanchecker ProvableContracts"#; + +/// The argv of the leanchecker arm: `[systemd-run --user --scope -q -p MemoryMax=.. -p CPUQuota=.. --] sh -c ..`. +/// `LEAN_NUM_THREADS` is set on the command, which a `--scope` unit inherits (it runs in the caller's process). +fn recheck_argv(lake_bin: &str, lc: Leanchecker) -> Vec { + let mut v: Vec = Vec::new(); + if let Some(s) = lc.scope { + v.extend( + [ + "systemd-run".to_string(), + "--user".into(), + "--scope".into(), + format!("--slice={AGENT_SLICE}"), + "-q".into(), + "-p".into(), + format!("MemoryMax={}G", s.memory_max_gib), + "-p".into(), + format!("CPUQuota={}%", s.cpu_quota_pct), + "--".into(), + ] + .into_iter(), + ); + } + let limit = lc.ulimit_v_kib.map(|k| k.to_string()).unwrap_or_default(); + v.extend([ + "sh".to_string(), + "-c".into(), + RECHECK_SH.into(), + "pv-leanchecker".into(), + limit, + lc.timeout_s.to_string(), + lake_bin.into(), + ]); + v +} + +/// The Lean steps run only on a tree nothing else has already failed or declined: a 60-minute re-check of a tree +/// that is already RED would only delay the verdict. They record their raw exits in `r.lake_exit` and +/// `r.leanchecker_exit`, and the comparator's closure in `r.challenges` (`None` = never ran), for +/// `discharge-summary.json` (EV-8a). The comparator runs before the (much slower) leanchecker. +pub(crate) fn lean_steps( + lake: Lake<'_>, + r: &mut Report, + lean_dir: &Path, + no_lake: bool, + cmp: bool, + lc: Option, +) { + let open = |r: &Report| !r.reject && r.decline.is_none(); + if open(r) && !no_lake { + elaborate(lake, lean_dir, r); + } + if cmp && open(r) { + compare(lake, lean_dir, r); + } + if let Some(lc) = lc { + if open(r) { + recheck(lake, lean_dir, lc, r); + } + } +} + +fn finish_with( + lake: Lake<'_>, + mut r: Report, + lean_dir: &Path, + no_lake: bool, + cmp: bool, + lc: Option, +) -> Res { + lean_steps(lake, &mut r, lean_dir, no_lake, cmp, lc); + verdict(r, lean_dir) +} + +/// Print the report and turn it into the exit: reject (1) before decline (2) before accept (0). +fn verdict(r: Report, lean_dir: &Path) -> Res { + for l in &r.lines { + println!("{l}"); + } + if r.reject { + let n = r.lines.iter().filter(|l| l.starts_with("FAIL")).count(); + return Err( + DischargeRejected(format!("{n} failure(s) under {}", lean_dir.display())).into(), + ); + } + if let Some(why) = r.decline { + return Err(DischargeDeclined(why).into()); + } + println!("ok discharge {}", lean_dir.display()); + Ok(()) +} + +/// `pv discharge run` (PVL-001 EV-8a, #4202): `build` (run by bash in ``), then `check --strict` and +/// every Lean arm, then the two files -- written whatever the verdict, so a RED run leaves a RED summary, never +/// none. `build.sh` rc 2 is its cache-miss decline and stays a decline; any other non-zero rejects; either way the +/// Lean steps do not run on a tree that did not build. A summary that cannot be written rejects: it is the output. +pub(crate) fn run_all( + lake: Lake<'_>, + build: &str, + lean_dir: &Path, + contracts: &Path, + lc: Leanchecker, +) -> Res { + let (build_exit, build_out) = match Command::new("bash") + .arg(build) + .current_dir(lean_dir) + .output() + { + Ok(o) => ( + Some(raw_exit(o.status)), + format!( + "{}{}", + String::from_utf8_lossy(&o.stdout), + String::from_utf8_lossy(&o.stderr) + ), + ), + Err(e) => (None, e.to_string()), + }; + let mut r = discharge::check( + lean_dir, + contracts, + CheckOpts { + strict: true, + validate_formalization: false, + }, + ); + match build_exit { + Some(0) => { + r.lines.insert(0, format!("ok {build}")); + lean_steps(lake, &mut r, lean_dir, false, true, Some(lc)); + } + Some(2) => { + r.lines.push(format!("{build} declined (rc 2):")); + r.lines.extend(tail(&build_out)); + r.decline.get_or_insert_with(|| { + format!("{build} declined: the Lean steps did not run -- not a verdict") + }); + } + Some(rc) => { + r.lines.push(format!("FAIL {build} exited {rc}")); + r.lines.extend(tail(&build_out)); + r.reject = true; + } + None => { + r.decline + .get_or_insert_with(|| format!("{build} could not be run ({build_out})")); + } + } + let tree = Tree::load(lean_dir).ok(); + let s = summary::summarize( + &r, + tree.as_ref(), + lean_dir, + summary::current_tree_sha(lean_dir), + build_exit, + ); + let spath = summary::summary_path(lean_dir); + // The summary first: the log is built after it, so a summary that cannot be written is in the log's verdict. + write_or_reject(&mut r, &spath, Ok(s.render())); + let log = RunLog { + verdict: if r.reject { + "reject" + } else if r.decline.is_some() { + "decline" + } else { + "accept" + }, + decline: r.decline.as_deref(), + lines: &r.lines, + summary: &s, + }; + let log_text = serde_json::to_string_pretty(&log) + .map(|t| t + "\n") + .map_err(|e| e.to_string()); + write_or_reject(&mut r, &lean_dir.join(LOG_FILE), log_text); + verdict(r, lean_dir) +} + +/// Write `text` to `p`, or record the failure and reject the run. +fn write_or_reject(r: &mut Report, p: &Path, text: Result) { + match text.and_then(|t| std::fs::write(p, t).map_err(|e| e.to_string())) { + Ok(()) => r.lines.push(format!("wrote {}", p.display())), + Err(e) => { + r.lines + .push(format!("FAIL cannot write {}: {e}", p.display())); + r.reject = true; + } + } +} + +/// `/discharge.json`, the untracked full log of one `run`: the verdict, every line, and the summary. +#[derive(serde::Serialize)] +struct RunLog<'a> { + verdict: &'a str, + decline: Option<&'a str>, + lines: &'a [String], + summary: &'a summary::Summary, +} + +/// `lake env lean Axioms.lean`: the subset and capstone pins, elaborated against the BUILT tree (run `build.sh` +/// first; `lake env` builds nothing). +fn elaborate(lake: Lake<'_>, lean_dir: &Path, r: &mut Report) { + let what = format!("lake env lean {AXIOMS_FILE}"); + match lake.run(&["env", "lean", AXIOMS_FILE], lean_dir) { + Ok(Bounded::TimedOut(pgid)) => { + r.lake_exit = Some(TIMED_OUT); + r.lines.push(lake.timed_out(&what, pgid)); + r.reject = true; + } + Err(e) => { + r.decline = Some(format!( + "lake could not be run ({e}): Axioms.lean was not elaborated" + )) + } + Ok(Bounded::Done(o)) if o.status.success() => { + r.lake_exit = Some(raw_exit(o.status)); + r.lines.push(format!("ok {what}")); + } + Ok(Bounded::Done(o)) => { + r.lake_exit = Some(raw_exit(o.status)); + let text = format!( + "{}{}", + String::from_utf8_lossy(&o.stdout), + String::from_utf8_lossy(&o.stderr) + ); + r.lines.push(format!( + "FAIL lake env lean {AXIOMS_FILE} exited {:?}", + o.status.code() + )); + r.lines.extend( + text.lines() + .filter(|l| l.contains("error")) + .take(20) + .map(|l| format!(" {l}")), + ); + r.reject = true; + } + } +} + +/// `lake env lean --run scripts/Comparator.lean Challenge/*.lean` (PVL-001 EV-7b): the script MEASURES, one NDJSON +/// row per challenge, and [`comparator::judge_rows`] judges. No Challenge file, no script or no `lake` declines; +/// a Challenge file that does not elaborate, or output that is not rows, rejects: its rows were never judged. +fn compare(lake: Lake<'_>, lean_dir: &Path, r: &mut Report) { + let files = comparator::challenge_files(lean_dir); + if files.is_empty() { + r.decline = Some(format!( + "comparator: no {CHALLENGE_DIR}/*.lean under {} -- EV-7a writes them; nothing to compare", + lean_dir.display() + )); + return; + } + if !lean_dir.join(COMPARATOR).is_file() { + r.decline = Some(format!( + "comparator: {} does not exist -- not a verdict", + lean_dir.join(COMPARATOR).display() + )); + return; + } + if !self_test(lake, lean_dir, r) { + return; + } + let what = format!("lake env lean --run {COMPARATOR} ({} file(s))", files.len()); + let mut args: Vec<&OsStr> = ["env", "lean", "--run", COMPARATOR] + .map(OsStr::new) + .to_vec(); + args.extend(files.iter().map(|f| f.as_os_str())); + let out = match lake.run(&args, lean_dir) { + Ok(Bounded::Done(o)) => o, + Ok(Bounded::TimedOut(pgid)) => { + r.lines.push(lake.timed_out(&what, pgid)); + r.reject = true; + return; + } + Err(e) => { + r.decline = Some(format!( + "lake could not be run ({e}): the comparator did not run" + )); + return; + } + }; + let stdout = String::from_utf8_lossy(&out.stdout); + if !out.status.success() { + r.lines.push(format!( + "FAIL {what} exited {} -- a Challenge file did not elaborate; its rows were withheld", + raw_exit(out.status) + )); + r.lines.extend(tail(&String::from_utf8_lossy(&out.stderr))); + r.reject = true; + return; + } + match comparator::parse_rows(&stdout) { + Ok(rows) => { + r.lines.push(format!("ok {what}")); + let mut c = comparator::judge_rows(&rows, r); + match comparator::expected_roots(lean_dir, &files) { + Ok(roots) => comparator::cross_check(&rows, &roots, &mut c, r), + Err(e) => { + r.lines.push(format!( + "FAIL comparator: a Challenge file could not be read, its roots were never counted: {e}" + )); + r.reject = true; + } + } + r.challenges = Some(c); + } + Err(e) => { + r.lines.push(format!("FAIL {what}: {e}")); + r.reject = true; + } + } +} + +/// The FIPS 180-4 vectors `Comparator.lean --self-test` checks: "", "abc" and the two-block 448-bit message. +const FIPS_VECTORS: usize = 3; + +/// `lake env lean --run scripts/Comparator.lean --self-test` (#4238), before any row is trusted: the pure-Lean +/// sha256 against the FIPS vectors, then the defeq controls. A regressed sha256 still hashes consistently, so the +/// rows alone cannot catch it. rc != 0, a timeout, or fewer than [`FIPS_VECTORS`] `ok … sha256` lines rejects -- +/// a self-test that checked nothing is not a pass. +fn self_test(lake: Lake<'_>, lean_dir: &Path, r: &mut Report) -> bool { + let what = format!("lake env lean --run {COMPARATOR} --self-test"); + let o = match lake.run( + &["env", "lean", "--run", COMPARATOR, "--self-test"], + lean_dir, + ) { + Ok(Bounded::Done(o)) => o, + Ok(Bounded::TimedOut(pgid)) => { + r.lines.push(lake.timed_out(&what, pgid)); + r.reject = true; + return false; + } + Err(e) => { + r.decline = Some(format!( + "lake could not be run ({e}): the comparator did not run (nor its self-test)" + )); + return false; + } + }; + let stdout = String::from_utf8_lossy(&o.stdout); + let vectors = stdout + .lines() + .filter(|l| l.starts_with("ok") && l.contains(" sha256 ")) + .count(); + if o.status.success() && vectors >= FIPS_VECTORS { + r.lines + .push(format!("ok {what} ({vectors} FIPS vectors)")); + return true; + } + r.lines.push(format!( + "FAIL {what} exited {} with {vectors}/{FIPS_VECTORS} FIPS vectors ok -- the statement hash is not trusted", + raw_exit(o.status) + )); + r.lines.extend(tail(&format!( + "{stdout}{}", + String::from_utf8_lossy(&o.stderr) + ))); + r.reject = true; + false +} + +/// The last 20 lines of `text`, indented. +fn tail(text: &str) -> Vec { + let mut v: Vec = text + .lines() + .rev() + .take(20) + .map(|l| format!(" {l}")) + .collect(); + v.reverse(); + v +} + +/// The process's exit code, or 128+signal when a signal ended it (the shell's convention). +fn raw_exit(s: std::process::ExitStatus) -> i32 { + use std::os::unix::process::ExitStatusExt; + s.code().unwrap_or_else(|| 128 + s.signal().unwrap_or(0)) +} + +/// The default bound on one `lake` call, seconds. `check`/`run` take `--lake-timeout` (#4239). +pub(crate) const LAKE_TIMEOUT_S: u64 = 1800; + +/// The exit recorded for a step the bound killed: `timeout(1)`'s, as the leanchecker arm records. +const TIMED_OUT: i32 = 124; + +/// A `lake` binary and the wall-clock bound on each call to it (#4239: a hang is RED, not a wait). +#[derive(Clone, Copy, Debug)] +pub(crate) struct Lake<'a> { + pub bin: &'a str, + pub timeout_s: u64, +} + +/// How a bounded call ended. `TimedOut` carries the killed process group (the child's own PID). +enum Bounded { + Done(std::process::Output), + TimedOut(u32), +} + +impl Lake<'static> { + fn new(timeout_s: u64) -> Self { + Lake { + bin: "lake", + timeout_s, + } + } +} + +impl Lake<'_> { + /// `lake ` in `dir`, in its own process group, waited on for at most `timeout_s`. At the deadline the + /// group is SIGKILLed by the child's own PID (pgid == pid) -- never by pattern -- so a `lean` under `lake env` + /// dies with it. The deadline also bounds the output drain: a grandchild holding a pipe cannot stall it. + fn run>(self, args: &[S], dir: &Path) -> std::io::Result { + let mut child = self.spawn(args, dir)?; + let pgid = child.id(); + let rx = drain_pipes(&mut child); + let deadline = Instant::now() + Duration::from_secs(self.timeout_s); + let Some(status) = wait_until(&mut child, deadline)? else { + return Ok(kill_group(&mut child, pgid)); + }; + let Some([stdout, stderr]) = collect(&rx, deadline) else { + return Ok(kill_group(&mut child, pgid)); + }; + Ok(Bounded::Done(std::process::Output { + status, + stdout, + stderr, + })) + } + + /// `lake ` spawned in its own process group, piped. ETXTBSY (26): a just-written `lake` whose write fd a + /// concurrent fork still holds until its exec. Transient, so it is retried. + fn spawn>( + self, + args: &[S], + dir: &Path, + ) -> std::io::Result { + use std::os::unix::process::CommandExt; + use std::process::Stdio; + let spawn = || { + Command::new(self.bin) + .args(args) + .current_dir(dir) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .process_group(0) + .spawn() + }; + let mut child = spawn(); + for _ in 0..20 { + match &child { + Err(e) if e.raw_os_error() == Some(26) => { + std::thread::sleep(Duration::from_millis(25)); + child = spawn(); + } + _ => break, + } + } + child + } + + fn timed_out(self, what: &str, pgid: u32) -> String { + format!( + "FAIL {what} timed out after {}s -- killed its process group {pgid}; a hang is RED, not a wait", + self.timeout_s + ) + } +} + +type Drained = std::sync::mpsc::Receiver<(usize, Vec)>; + +/// Read the child's stdout (0) and stderr (1) on their own threads, so a full pipe cannot block its exit. +fn drain_pipes(child: &mut std::process::Child) -> Drained { + use std::io::Read; + let (tx, rx) = std::sync::mpsc::channel(); + let drain = |mut pipe: Box, which: usize| { + let tx = tx.clone(); + std::thread::spawn(move || { + let mut buf = Vec::new(); + let _ = pipe.read_to_end(&mut buf); + let _ = tx.send((which, buf)); + }); + }; + if let Some(p) = child.stdout.take() { + drain(Box::new(p), 0); + } + if let Some(p) = child.stderr.take() { + drain(Box::new(p), 1); + } + rx +} + +/// The child's exit status, or `None` once `deadline` passes with it still running. +fn wait_until( + child: &mut std::process::Child, + deadline: Instant, +) -> std::io::Result> { + loop { + if let Some(status) = child.try_wait()? { + return Ok(Some(status)); + } + if Instant::now() >= deadline { + return Ok(None); + } + std::thread::sleep(Duration::from_millis(20)); + } +} + +/// Both drained pipes, or `None` if `deadline` passes first: a grandchild holding a pipe cannot stall the drain. +fn collect(rx: &Drained, deadline: Instant) -> Option<[Vec; 2]> { + use std::sync::mpsc::RecvTimeoutError; + let mut out = [Vec::new(), Vec::new()]; + for _ in 0..2 { + let left = deadline.saturating_duration_since(Instant::now()); + match rx.recv_timeout(left) { + Ok((which, buf)) => out[which] = buf, + Err(RecvTimeoutError::Disconnected) => break, + Err(RecvTimeoutError::Timeout) => return None, + } + } + Some(out) +} + +/// SIGKILL the whole process group by the child's own PID (pgid == pid) -- never by pattern -- then reap the child. +fn kill_group(child: &mut std::process::Child, pgid: u32) -> Bounded { + let _ = Command::new("kill") + .args(["-s", "KILL", "--", &format!("-{pgid}")]) + .status(); + let _ = child.kill(); + let _ = child.wait(); + Bounded::TimedOut(pgid) +} + +/// The toolchain `lake env` puts on PATH for this tree: `lake env printenv LEAN_SYSROOT`. `Err` is the decline; +/// a timeout rejects in place and yields `Err(None)` (#4239). +fn sysroot( + lake: Lake<'_>, + lean_dir: &Path, + r: &mut Report, +) -> Result> { + let o = match lake.run(&["env", "printenv", "LEAN_SYSROOT"], lean_dir) { + Ok(Bounded::Done(o)) => o, + Ok(Bounded::TimedOut(pgid)) => { + r.lines + .push(lake.timed_out("lake env printenv LEAN_SYSROOT", pgid)); + r.reject = true; + return Err(None); + } + Err(e) => { + return Err(Some(format!( + "lake could not be run ({e}): leanchecker did not run" + ))) + } + }; + let root = String::from_utf8_lossy(&o.stdout).trim().to_string(); + if !o.status.success() || root.is_empty() { + return Err(Some(format!( + "`lake env printenv LEAN_SYSROOT` exited {:?} and named no toolchain: leanchecker did not run", + o.status.code() + ))); + } + Ok(root.into()) +} + +/// `timeout lake env leanchecker ProvableContracts` (PVL-001 EV-6b): rc != 0 rejects, a timeout rejects, and a +/// toolchain without `leanchecker` declines -- measured: elan's `lake env leanchecker` on v4.15.0 exits 1 "does not +/// have the binary", which would otherwise read as a failed check. `timeout` signals the whole process group, so +/// the `leanchecker` under `lake` does not outlive it. +fn recheck(lake: Lake<'_>, lean_dir: &Path, lc: Leanchecker, r: &mut Report) { + let root = match sysroot(lake, lean_dir, r) { + Ok(root) => root, + Err(why) => { + r.decline = why; + return; + } + }; + let bin = root.join("bin").join("leanchecker"); + if !bin.is_file() { + r.decline = Some(format!( + "leanchecker not in toolchain: {} does not exist", + bin.display() + )); + return; + } + // A scope that cannot be created would otherwise read as leanchecker's own non-zero exit: probe it first. + if lc.scope.is_some() { + let probe = Command::new("systemd-run") + .args([ + "--user", + "--scope", + &format!("--slice={AGENT_SLICE}"), + "-q", + "--", + "true", + ]) + .output(); + if !matches!(&probe, Ok(o) if o.status.success()) { + r.decline = Some( + "`systemd-run --user --scope` is unavailable here: leanchecker did not run (pass \ + --leanchecker-unscoped only on a host where an uncapped run cannot starve other work, #4348)" + .to_string(), + ); + return; + } + } + let argv = recheck_argv(lake.bin, lc); + let out = Command::new(&argv[0]) + .args(&argv[1..]) + .env("LEAN_NUM_THREADS", lc.threads.to_string()) + .current_dir(lean_dir) + .output(); + let what = format!( + "lake env leanchecker ProvableContracts (timeout {}s, {} threads, {})", + lc.timeout_s, + lc.threads, + lc.scope.map_or("unscoped".to_string(), |s| format!( + "scope MemoryMax={}G CPUQuota={}%", + s.memory_max_gib, s.cpu_quota_pct + )) + ); + match out { + Err(e) => { + r.decline = Some(format!( + "sh could not be run ({e}): leanchecker did not run" + )) + } + Ok(o) if o.status.success() => { + r.leanchecker_exit = Some(0); + r.lines.push(format!("ok {what}")); + } + Ok(o) => match o.status.code() { + // timeout/ulimit/lake could not start leanchecker: it never ran, so no exit is recorded. + Some(c @ 125..=127) => { + r.decline = Some(format!( + "{what} could not be started (rc {c}: timeout/ulimit/lake): not a verdict" + )); + } + code => { + r.leanchecker_exit = Some(raw_exit(o.status)); + let why = if matches!(code, Some(124 | 137)) { + "timed out".to_string() + } else { + format!("exited {code:?}") + }; + r.lines.push(format!("FAIL {what} {why}")); + let text = format!( + "{}{}", + String::from_utf8_lossy(&o.stdout), + String::from_utf8_lossy(&o.stderr) + ); + r.lines.extend( + text.lines() + .rev() + .take(20) + .collect::>() + .into_iter() + .rev() + .map(|l| format!(" {l}")), + ); + r.reject = true; + } + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::contract_walk::{exit_code_for, verdict_for}; + use std::path::PathBuf; + + /// A one-theorem tree bound by the label `Theorems.Gelu` (the integration fixture, in-process). + fn tree() -> (tempfile::TempDir, PathBuf, PathBuf) { + let d = tempfile::tempdir().expect("tempdir"); + let lean = d.path().join("lean"); + let contracts = d.path().join("contracts"); + std::fs::create_dir_all(lean.join("ProvableContracts/Theorems/Gelu")).expect("mkdir"); + std::fs::create_dir_all(&contracts).expect("mkdir"); + std::fs::write( + lean.join("ProvableContracts.lean"), + "import ProvableContracts.Theorems.Gelu.Bound\n", + ) + .expect("w"); + std::fs::write( + lean.join("ProvableContracts/Theorems/Gelu/Bound.lean"), + "namespace ProvableContracts.Gelu\ntheorem gelu_bound : True := trivial\nend ProvableContracts.Gelu\n", + ) + .expect("w"); + std::fs::write( + contracts.join("gelu-v1.yaml"), + "equations:\n e:\n lean_theorem: Theorems.Gelu\n", + ) + .expect("w"); + (d, lean, contracts) + } + + fn gen(lean: &Path, contracts: &Path, check: bool) -> Res { + run(DischargeAction::GenAxioms { + lean_dir: lean.into(), + contracts: contracts.into(), + check, + }) + } + + fn check(lean: &Path, contracts: &Path, strict: bool) -> Res { + run(DischargeAction::Check { + lean_dir: lean.into(), + contracts: contracts.into(), + no_lake: true, + strict, + validate_formalization: false, + leanchecker: false, + leanchecker_timeout: 3600, + leanchecker_ulimit_v: None, + leanchecker_threads: LEANCHECKER_THREADS, + leanchecker_memory_max_gib: LEANCHECKER_MEMORY_MAX_GIB, + leanchecker_cpu_quota_pct: LEANCHECKER_CPU_QUOTA_PCT, + leanchecker_unscoped: false, + comparator: false, + lake_timeout: LAKE_TIMEOUT_S, + }) + } + + fn is_reject(r: &Res) -> bool { + r.as_ref() + .err() + .is_some_and(|e| e.downcast_ref::().is_some()) + } + + fn is_decline(r: &Res) -> bool { + r.as_ref() + .err() + .is_some_and(|e| e.downcast_ref::().is_some()) + } + + #[test] + fn the_verdict_words_and_exit_codes_are_pvl_1s() { + let d: Box = Box::new(DischargeDeclined("x".into())); + let r: Box = Box::new(DischargeRejected("y".into())); + assert_eq!( + (exit_code_for(d.as_ref()), verdict_for(d.as_ref())), + (2, "decline") + ); + assert_eq!( + (exit_code_for(r.as_ref()), verdict_for(r.as_ref())), + (1, "reject") + ); + assert_eq!( + (d.to_string(), r.to_string()), + ("x".to_string(), "y".to_string()) + ); + } + + #[test] + fn gen_axioms_writes_and_check_mode_judges_freshness() { + let (_d, lean, contracts) = tree(); + assert!( + is_reject(&gen(&lean, &contracts, true)), + "no Axioms.lean yet must reject" + ); + gen(&lean, &contracts, false).expect("write"); + let text = std::fs::read_to_string(lean.join(AXIOMS_FILE)).expect("written"); + assert!( + text.contains("`ProvableContracts.Gelu.gelu_bound"), + "{text}" + ); + gen(&lean, &contracts, true).expect("fresh"); + std::fs::write(lean.join(AXIOMS_FILE), format!("{text}-- edit\n")).expect("w"); + assert!(is_reject(&gen(&lean, &contracts, true))); + std::fs::remove_file(lean.join("ProvableContracts.lean")).expect("rm"); + assert!(is_decline(&gen(&lean, &contracts, false))); + } + + #[test] + fn check_accepts_rejects_and_declines() { + let (_d, lean, contracts) = tree(); + gen(&lean, &contracts, false).expect("write"); + check(&lean, &contracts, false).expect("clean tree"); + let f = lean.join("ProvableContracts/Theorems/Gelu/Bound.lean"); + let src = std::fs::read_to_string(&f).expect("r"); + std::fs::write( + &f, + src.replace( + "end ProvableContracts.Gelu", + "axiom m : False\nend ProvableContracts.Gelu", + ), + ) + .expect("w"); + assert!(is_reject(&check(&lean, &contracts, false))); + std::fs::write( + lean.join("escape-allowlist.yaml"), + "- file: ProvableContracts/Theorems/Gelu/Bound.lean\n decl: ProvableContracts.Gelu.m\n kind: axiom\n reason: r\n ticket: t\n confirmed_by: pending\n", + ) + .expect("w"); + gen(&lean, &contracts, false).expect("regen"); + check(&lean, &contracts, false).expect("pending is accepted"); + assert!( + is_reject(&check(&lean, &contracts, true)), + "--strict rejects pending" + ); + std::fs::write( + contracts.join("gelu-v1.yaml"), + "equations:\n e:\n lean_theorem: none\n", + ) + .expect("w"); + gen(&lean, &contracts, false).expect("regen"); + assert!( + is_decline(&check(&lean, &contracts, false)), + "zero roots declines" + ); + } + + #[test] + fn label_ratchet_writes_the_set() { + let (_d, lean, contracts) = tree(); + run(DischargeAction::LabelRatchet { + lean_dir: lean.clone(), + contracts, + }) + .expect("seed"); + assert!(lean.join(discharge::LABELS).is_file()); + } + + /// A `lake` under a 60 s bound: long enough for any fake, short enough that a hang fails the test. + fn lk(bin: &str) -> Lake<'_> { + Lake { bin, timeout_s: 60 } + } + + /// A fake `lake`: exits `rc` after printing `out`. + fn fake_lake(dir: &Path, rc: i32, out: &str) -> String { + let p = dir.join(format!("lake-{rc}")); + std::fs::write(&p, format!("#!/bin/sh\necho '{out}'\nexit {rc}\n")).expect("w"); + let mut perm = std::fs::metadata(&p).expect("meta").permissions(); + std::os::unix::fs::PermissionsExt::set_mode(&mut perm, 0o755); + std::fs::set_permissions(&p, perm).expect("chmod"); + p.to_string_lossy().into_owned() + } + + #[test] + fn the_lean_elaboration_decides_the_verdict() { + let (d, lean, _) = tree(); + let ok = fake_lake(d.path(), 0, "fine"); + let bad = fake_lake(d.path(), 1, "Axioms.lean:3:0: error: AXIOMS x"); + finish_with(lk(&ok), Report::default(), &lean, false, false, None).expect("lake ok"); + assert!( + is_reject(&finish_with( + lk(&bad), + Report::default(), + &lean, + false, + false, + None + )), + "a failing elaboration rejects" + ); + assert!( + is_decline(&finish_with( + lk("/nonexistent/lake"), + Report::default(), + &lean, + false, + false, + None + )), + "no lake declines" + ); + finish_with(lk(&bad), Report::default(), &lean, true, false, None) + .expect("--no-lake skips it"); + let rejected = Report { + reject: true, + ..Report::default() + }; + assert!( + is_reject(&finish_with(lk(&ok), rejected, &lean, false, false, None)), + "a prior failure is not cleared by lake" + ); + } + + /// A `lake` for `--leanchecker`: `lake env printenv LEAN_SYSROOT` names `/sysroot` (with `bin/leanchecker` + /// when `has_checker`), `lake env lean …` passes, and `lake env leanchecker …` prints `out` and exits `rc`. + fn checker_lake(dir: &Path, has_checker: bool, rc: i32, out: &str) -> String { + let root = dir.join(format!("sysroot-{has_checker}")); + std::fs::create_dir_all(root.join("bin")).expect("mkdir"); + if has_checker { + std::fs::write(root.join("bin").join("leanchecker"), "").expect("w"); + } + let p = dir.join(format!("checker-lake-{has_checker}-{rc}")); + let script = format!( + "#!/bin/sh\ncase \"$2\" in\n printenv) echo '{}' ;;\n lean) exit 0 ;;\n leanchecker) echo '{out}'; exit {rc} ;;\n *) exit 99 ;;\nesac\n", + root.display() + ); + std::fs::write(&p, script).expect("w"); + let mut perm = std::fs::metadata(&p).expect("meta").permissions(); + std::os::unix::fs::PermissionsExt::set_mode(&mut perm, 0o755); + std::fs::set_permissions(&p, perm).expect("chmod"); + p.to_string_lossy().into_owned() + } + + const LC: Option = Some(Leanchecker { + timeout_s: 60, + ulimit_v_kib: None, + threads: 8, + scope: None, + }); + + /// PVL-001 EV-6b: leanchecker's rc decides; an ABSENT leanchecker declines and is never read as a failure. + #[test] + fn leanchecker_rc_rejects_and_an_absent_checker_declines() { + let (d, lean, _) = tree(); + let pass = checker_lake(d.path(), true, 0, "ok"); + let fail = checker_lake( + d.path(), + true, + 1, + "error: kernel rejected Theorems.Gelu.bound", + ); + let absent = checker_lake(d.path(), false, 0, "never reached"); + finish_with(lk(&pass), Report::default(), &lean, false, false, LC) + .expect("leanchecker rc 0 accepts"); + assert!( + is_reject(&finish_with( + lk(&fail), + Report::default(), + &lean, + false, + false, + LC + )), + "leanchecker rc 1 rejects" + ); + assert!( + is_decline(&finish_with( + lk(&absent), + Report::default(), + &lean, + false, + false, + LC + )), + "no leanchecker in the toolchain declines" + ); + assert!( + is_decline(&finish_with( + lk("/nonexistent/lake"), + Report::default(), + &lean, + true, + false, + LC + )), + "no lake declines under --leanchecker" + ); + finish_with(lk(&fail), Report::default(), &lean, false, false, None) + .expect("without --leanchecker it never runs"); + } + + #[test] + fn a_timed_out_leanchecker_rejects_and_the_ulimit_reaches_the_checker() { + let (d, lean, _) = tree(); + let slow = checker_lake(d.path(), true, 0, "x"); + let body = std::fs::read_to_string(&slow) + .expect("r") + .replace("echo 'x'; exit 0", "sleep 30"); + std::fs::write(&slow, body).expect("w"); + let t1 = Some(Leanchecker { + timeout_s: 1, + ulimit_v_kib: None, + threads: 8, + scope: None, + }); + assert!( + is_reject(&finish_with( + lk(&slow), + Report::default(), + &lean, + true, + false, + t1 + )), + "a timeout rejects" + ); + // The limit reaches the checker: a stub that prints its own `ulimit -v` and fails shows it in the reject. + let shows = checker_lake(d.path(), true, 1, "x"); + let body = std::fs::read_to_string(&shows) + .expect("r") + .replace("echo 'x'", "echo \"vlimit=$(ulimit -v)\""); + std::fs::write(&shows, body).expect("w"); + let mut r = Report::default(); + recheck( + lk(&shows), + &lean, + Leanchecker { + timeout_s: 60, + ulimit_v_kib: Some(4_194_304), + threads: 8, + scope: None, + }, + &mut r, + ); + assert!(r.reject, "{:?}", r.lines); + assert!( + r.lines.iter().any(|l| l.contains("vlimit=4194304")), + "{:?}", + r.lines + ); + } + + /// #4348: the thread cap reaches the checker -- a stub that prints its own `LEAN_NUM_THREADS` and fails shows it. + #[test] + fn the_thread_cap_reaches_the_checker() { + let (d, lean, _) = tree(); + let shows = checker_lake(d.path(), true, 1, "x"); + let body = std::fs::read_to_string(&shows) + .expect("r") + .replace("echo 'x'", "echo \"threads=$LEAN_NUM_THREADS\""); + std::fs::write(&shows, body).expect("w"); + let mut r = Report::default(); + let lc = Leanchecker { + threads: 3, + ..LC.expect("LC") + }; + recheck(lk(&shows), &lean, lc, &mut r); + assert!(r.reject, "{:?}", r.lines); + assert!( + r.lines.iter().any(|l| l.contains("threads=3")), + "{:?}", + r.lines + ); + } + + /// #4348: scoped, the arm is `systemd-run --user --scope` with the memory and CPU caps around the same `sh -c`; + /// unscoped it is the bare `sh -c`. + #[test] + fn the_scope_wraps_the_checker_with_both_caps() { + let scoped = recheck_argv( + "lake", + Leanchecker { + scope: Some(Scope { + memory_max_gib: 24, + cpu_quota_pct: 800, + }), + ..LC.expect("LC") + }, + ); + assert_eq!( + scoped[..10], + [ + "systemd-run", + "--user", + "--scope", + "--slice=agent.slice", + "-q", + "-p", + "MemoryMax=24G", + "-p", + "CPUQuota=800%", + "--" + ] + ); + let bare = recheck_argv("lake", LC.expect("LC")); + assert_eq!(bare[..2], ["sh", "-c"]); + assert_eq!(scoped[10..], bare[..]); + } + + /// #4348: the DEFAULT `--leanchecker` is capped -- 8 threads inside a 24G/800% scope -- on `check` and `run` + /// alike; only an explicit `--leanchecker-unscoped` drops the scope. + #[test] + fn the_default_leanchecker_is_capped() { + #[derive(clap::Parser)] + struct T { + #[command(subcommand)] + a: crate::cli::DischargeAction, + } + let parse = |args: &[&str]| { + ::try_parse_from(std::iter::once("t").chain(args.iter().copied())) + .expect("parses") + .a + }; + for a in [ + parse(&["check", "L", "--leanchecker"]), + parse(&["run", "L"]), + ] { + let (t, g, q, u) = match a { + DischargeAction::Check { + leanchecker_threads: t, + leanchecker_memory_max_gib: g, + leanchecker_cpu_quota_pct: q, + leanchecker_unscoped: u, + .. + } + | DischargeAction::Run { + leanchecker_threads: t, + leanchecker_memory_max_gib: g, + leanchecker_cpu_quota_pct: q, + leanchecker_unscoped: u, + .. + } => (t, g, q, u), + _ => unreachable!(), + }; + let lc = Leanchecker::from_flags(3600, None, t, g, q, u); + assert_eq!( + (lc.threads, lc.scope), + ( + 8, + Some(Scope { + memory_max_gib: 24, + cpu_quota_pct: 800 + }) + ) + ); + } + match parse(&["check", "L", "--leanchecker", "--leanchecker-unscoped"]) { + DischargeAction::Check { + leanchecker_unscoped, + .. + } => assert!(leanchecker_unscoped), + _ => unreachable!(), + } + } + + /// The defaults are also the ceilings: a caller may lower a cap, never raise it (51 threads took 58-67 GB). + #[test] + fn a_leanchecker_cap_can_be_lowered_never_raised() { + #[derive(clap::Parser)] + struct T { + #[command(subcommand)] + a: crate::cli::DischargeAction, + } + let parses = |args: &[&str]| { + ::try_parse_from(std::iter::once("t").chain(args.iter().copied())) + .is_ok() + }; + for (flag, max) in [ + ("--leanchecker-threads", LEANCHECKER_THREADS), + ("--leanchecker-memory-max-gib", LEANCHECKER_MEMORY_MAX_GIB), + ("--leanchecker-cpu-quota-pct", LEANCHECKER_CPU_QUOTA_PCT), + ] { + for base in [&["check", "L", "--leanchecker"][..], &["run", "L"][..]] { + let with = |v: u32| { + let v = v.to_string(); + let mut a = base.to_vec(); + a.extend([flag, v.as_str()]); + parses(&a) + }; + assert!( + with(1) && with(max), + "{flag} {base:?}: 1 and {max} must parse" + ); + assert!( + !with(max + 1), + "{flag} {base:?}: {} must be refused", + max + 1 + ); + assert!(!with(0), "{flag} {base:?}: 0 must be refused"); + } + } + } + + /// EV-8a reads the raw exits: `Some(n)` for a step that ran (124 on a timeout), `None` for one that never ran. + #[test] + fn the_lean_steps_record_their_raw_exits_and_none_when_they_did_not_run() { + let (d, lean, _) = tree(); + let exits = |lake: &str, no_lake: bool, lc: Option| { + let mut r = Report::default(); + lean_steps(lk(lake), &mut r, &lean, no_lake, false, lc); + (r.lake_exit, r.leanchecker_exit) + }; + let pass = checker_lake(d.path(), true, 0, "ok"); + assert_eq!(exits(&pass, false, LC), (Some(0), Some(0))); + assert_eq!( + exits(&pass, false, None), + (Some(0), None), + "no --leanchecker" + ); + assert_eq!(exits(&pass, true, None), (None, None), "--no-lake"); + let fail = checker_lake(d.path(), true, 3, "kernel error"); + assert_eq!(exits(&fail, true, LC), (None, Some(3))); + let absent = checker_lake(d.path(), false, 0, "never reached"); + assert_eq!( + exits(&absent, true, LC), + (None, None), + "absent checker never ran" + ); + let slow = checker_lake(d.path(), true, 0, "x"); + let body = std::fs::read_to_string(&slow) + .expect("r") + .replace("echo 'x'; exit 0", "sleep 30"); + std::fs::write(&slow, body).expect("w"); + let t1 = Some(Leanchecker { + timeout_s: 1, + ulimit_v_kib: None, + threads: 8, + scope: None, + }); + assert_eq!(exits(&slow, true, t1), (None, Some(124)), "timeout"); + } + + /// A passing `Comparator.lean --self-test`: the three FIPS vector lines, rc 0 (#4238). + const SELF_TEST_OK: &str = "if [ \"$5\" = --self-test ]; then printf 'ok sha256 \"\" = e3\\nok sha256 \"abc\" = ba\\nok sha256 \"abcdbcde\" = 24\\n'; exit 0; fi"; + + /// The root EV-7a's `render` declares for the fixture's one solution. The cross-check (#4240) counts + /// these lines, so an empty Challenge file would make every fixture row an UNEXPECTED-ROW. + const GELU_CHALLENGE: &str = + "theorem _root_.PvlChallenge.ProvableContracts.Gelu.gelu_bound : True := by\n sorry\n"; + + /// A `lake` for `--comparator`: `env lean --run …` writes `rows` to stdout, `stderr` to stderr, and exits + /// `rc`; every other `env lean` passes. The tree gets `Challenge/gelu-v1.lean` and the comparator script. + fn comparator_lake(dir: &Path, lean: &Path, rc: i32, rows: &str, stderr: &str) -> String { + std::fs::create_dir_all(lean.join(CHALLENGE_DIR)).expect("mkdir"); + // The Challenge file declares exactly the roots the canned rows name (#4240 cross-checks the two); + // output that is not rows declares nothing, so those cases still judge the output alone. + let decls: String = comparator::parse_rows(rows) + .map(|rs| { + rs.iter() + .map(|r| { + format!( + "theorem _root_.PvlChallenge.{} : True := by\n sorry\n", + r.name + ) + }) + .collect() + }) + .unwrap_or_default(); + std::fs::write(lean.join(CHALLENGE_DIR).join("gelu-v1.lean"), decls).expect("w"); + std::fs::create_dir_all(lean.join("scripts")).expect("mkdir"); + std::fs::write(lean.join(COMPARATOR), "").expect("w"); + std::fs::write(dir.join(format!("rows-{rc}")), rows).expect("w"); + let p = dir.join(format!("cmp-lake-{rc}")); + let script = format!( + "#!/bin/sh\n{SELF_TEST_OK}\nif [ \"$3\" = --run ]; then\n [ \"$4 $5\" = \"{COMPARATOR} {CHALLENGE_DIR}/gelu-v1.lean\" ] || {{ echo \"bad args: $*\" >&2; exit 98; }}\n cat '{}'; echo '{stderr}' >&2; exit {rc}\nfi\nexit 0\n", + dir.join(format!("rows-{rc}")).display() + ); + std::fs::write(&p, script).expect("w"); + let mut perm = std::fs::metadata(&p).expect("meta").permissions(); + std::os::unix::fs::PermissionsExt::set_mode(&mut perm, 0o755); + std::fs::set_permissions(&p, perm).expect("chmod"); + p.to_string_lossy().into_owned() + } + + fn cmp_row(name: &str, ch: &str, sol: &str, axioms: &str) -> String { + format!( + "{{\"name\": \"{name}\", \"challenge_type_hash\": \"{ch}\", \"solution_type_hash\": {sol}, \"axioms\": {axioms}}}\n" + ) + } + + fn compared(lake: &str, lean: &Path) -> Report { + let mut r = Report::default(); + lean_steps(lk(lake), &mut r, lean, false, true, None); + r + } + + /// PVL-001 EV-7b: a matching, sorry-free solution closes its challenge; the closure is recorded for EV-8a. + #[test] + fn the_comparator_closes_a_matching_challenge() { + let (d, lean, _) = tree(); + let h = "ab".repeat(32); + let lake = comparator_lake( + d.path(), + &lean, + 0, + &cmp_row( + "ProvableContracts.Gelu.gelu_bound", + &h, + &format!("\"{h}\""), + "[]", + ), + "", + ); + let r = compared(&lake, &lean); + assert!(!r.reject && r.decline.is_none(), "{:?}", r.lines); + assert_eq!( + r.challenges, + Some(comparator::Closure { + closed: 1, + total: 1 + }) + ); + assert!( + r.lines + .iter() + .any(|l| l == "COMPARATOR 1/1 challenge(s) closed"), + "{:?}", + r.lines + ); + finish_with(lk(&lake), Report::default(), &lean, false, true, None).expect("accepts"); + } + + /// The spec's RED: a solution of a WEAKER statement is a mismatch, and a sorry'd one closes nothing. + #[test] + fn a_mismatched_or_sorry_solution_rejects() { + let (d, lean, _) = tree(); + let (c, s) = ("ab".repeat(32), "cd".repeat(32)); + let rows = format!( + "{}{}", + cmp_row("A.weak", &c, &format!("\"{s}\""), "[]"), + cmp_row("A.sorried", &c, &format!("\"{c}\""), "[\"sorryAx\"]") + ); + let lake = comparator_lake(d.path(), &lean, 0, &rows, ""); + let r = compared(&lake, &lean); + assert!(r.reject, "{:?}", r.lines); + assert!(r + .lines + .iter() + .any(|l| l.starts_with("FAIL MISMATCH A.weak"))); + assert!(r + .lines + .iter() + .any(|l| l.starts_with("FAIL SORRY A.sorried"))); + assert_eq!(r.challenges.map(|c| (c.closed, c.total)), Some((0, 2))); + } + + #[test] + fn a_challenge_that_does_not_elaborate_rejects_with_its_errors() { + let (d, lean, _) = tree(); + let lake = comparator_lake( + d.path(), + &lean, + 1, + "", + "Challenge/gelu-v1.lean:5:0: error: unknown g", + ); + let r = compared(&lake, &lean); + assert!(r.reject, "{:?}", r.lines); + assert!( + r.lines.iter().any(|l| l.contains("exited 1")), + "{:?}", + r.lines + ); + assert!( + r.lines.iter().any(|l| l.contains("unknown g")), + "{:?}", + r.lines + ); + assert_eq!(r.challenges, None, "rows withheld: nothing was judged"); + } + + #[test] + fn unreadable_comparator_output_rejects() { + let (d, lean, _) = tree(); + let lake = comparator_lake(d.path(), &lean, 0, "not a row\n", ""); + let r = compared(&lake, &lean); + assert!(r.reject, "{:?}", r.lines); + assert!( + r.lines.iter().any(|l| l.contains("line 1")), + "{:?}", + r.lines + ); + } + + /// Zero measured is a decline, never a pass: no Challenge file, no script, zero rows, no lake. + #[test] + fn the_comparator_declines_when_nothing_was_compared() { + let (d, lean, _) = tree(); + let lake = comparator_lake(d.path(), &lean, 0, "", ""); + let r = compared(&lake, &lean); + assert!(!r.reject, "{:?}", r.lines); + assert!( + r.decline + .as_deref() + .is_some_and(|w| w.contains("0 challenge rows")), + "{r:?}" + ); + assert_eq!(r.challenges.map(|c| c.total), Some(0)); + + std::fs::remove_file(lean.join(COMPARATOR)).expect("rm"); + let r = compared(&lake, &lean); + assert!( + r.decline + .as_deref() + .is_some_and(|w| w.contains("does not exist")), + "{r:?}" + ); + + std::fs::remove_dir_all(lean.join(CHALLENGE_DIR)).expect("rm"); + let r = compared(&lake, &lean); + assert!( + r.decline + .as_deref() + .is_some_and(|w| w.contains("no Challenge/*.lean")), + "{r:?}" + ); + assert!(is_decline(&finish_with( + lk(&lake), + Report::default(), + &lean, + false, + true, + None + ))); + + let (d2, lean2, _) = tree(); + comparator_lake(d2.path(), &lean2, 0, "", ""); + let mut r = Report::default(); + lean_steps(lk("/nonexistent/lake"), &mut r, &lean2, true, true, None); + assert!( + r.decline + .as_deref() + .is_some_and(|w| w.contains("comparator did not run")), + "{r:?}" + ); + } + + #[test] + fn without_the_flag_the_comparator_never_runs() { + let (d, lean, _) = tree(); + let lake = comparator_lake(d.path(), &lean, 1, "", "would reject"); + let mut r = Report::default(); + lean_steps(lk(&lake), &mut r, &lean, false, false, None); + assert!(!r.reject && r.challenges.is_none(), "{:?}", r.lines); + } + + /// PVL-001 EV-8a: a `lake` for `run` — every `env lean` passes, `--run` prints one closing comparator row, + /// `printenv` names a sysroot with a leanchecker, and `env leanchecker` exits `lc_rc`. + fn run_lake(dir: &Path, lean: &Path, lc_rc: i32) -> String { + std::fs::create_dir_all(lean.join(CHALLENGE_DIR)).expect("mkdir"); + std::fs::write( + lean.join(CHALLENGE_DIR).join("gelu-v1.lean"), + GELU_CHALLENGE, + ) + .expect("w"); + std::fs::create_dir_all(lean.join("scripts")).expect("mkdir"); + std::fs::write(lean.join(COMPARATOR), "").expect("w"); + let root = dir.join("run-sysroot"); + std::fs::create_dir_all(root.join("bin")).expect("mkdir"); + std::fs::write(root.join("bin").join("leanchecker"), "").expect("w"); + let h = "ab".repeat(32); + let rows = dir.join("run-rows"); + std::fs::write( + &rows, + cmp_row( + "ProvableContracts.Gelu.gelu_bound", + &h, + &format!("\"{h}\""), + "[]", + ), + ) + .expect("w"); + let p = dir.join(format!("run-lake-{lc_rc}")); + let script = format!( + "#!/bin/sh\n{SELF_TEST_OK}\ncase \"$2 $3\" in\n 'lean --run') cat '{}' ;;\n printenv*) echo '{}' ;;\n leanchecker*) exit {lc_rc} ;;\nesac\nexit 0\n", + rows.display(), + root.display() + ); + std::fs::write(&p, script).expect("w"); + let mut perm = std::fs::metadata(&p).expect("meta").permissions(); + std::os::unix::fs::PermissionsExt::set_mode(&mut perm, 0o755); + std::fs::set_permissions(&p, perm).expect("chmod"); + p.to_string_lossy().into_owned() + } + + fn build_sh(dir: &Path, rc: i32) -> String { + let p = dir.join(format!("build-{rc}.sh")); + std::fs::write(&p, format!("echo building\nexit {rc}\n")).expect("w"); + p.to_string_lossy().into_owned() + } + + fn summary_of(lean: &Path) -> summary::Summary { + summary::load(&summary::summary_path(lean)).expect("a summary was written") + } + + fn lc() -> Leanchecker { + LC.expect("LC") + } + + /// The accept: every step green → a green summary deriving the tree's theorem, and the untracked log beside it. + #[test] + fn a_green_run_writes_a_green_summary_and_the_log() { + let (d, lean, contracts) = tree(); + gen(&lean, &contracts, false).expect("gen-axioms"); + let lake = run_lake(d.path(), &lean, 0); + let r = run_all(lk(&lake), &build_sh(d.path(), 0), &lean, &contracts, lc()); + let s = summary_of(&lean); + assert!(r.is_ok(), "{r:?} {s:?}"); + assert!(s.is_green(), "{s:?}"); + assert_eq!(s.challenges_closed.as_deref(), Some("1/1")); + assert!(s.derived().contains("ProvableContracts.Gelu.gelu_bound")); + assert!(s.tree_sha.is_none(), "a tempdir is no git checkout"); + let log: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(lean.join(LOG_FILE)).expect("log")) + .expect("json"); + assert_eq!(log["verdict"], "accept"); + assert_eq!(log["summary"]["leanchecker_exit"], 0); + } + + /// The spec's mutation: a summary hand-edited toward pass differs from its regeneration, byte for byte. + #[test] + fn a_regeneration_reproduces_the_summary_and_catches_a_hand_edit() { + let (d, lean, contracts) = tree(); + gen(&lean, &contracts, false).expect("gen-axioms"); + let lake = run_lake(d.path(), &lean, 1); + let build = build_sh(d.path(), 0); + assert!(is_reject(&run_all( + lk(&lake), + &build, + &lean, + &contracts, + lc() + ))); + let spath = summary::summary_path(&lean); + let first = std::fs::read_to_string(&spath).expect("r"); + assert!(first.contains("\"leanchecker_exit\": 1"), "{first}"); + let edited = first.replace("\"leanchecker_exit\": 1", "\"leanchecker_exit\": 0"); + std::fs::write(&spath, &edited).expect("w"); + assert!(summary_of(&lean).is_green(), "the edit alone would derive"); + assert!(is_reject(&run_all( + lk(&lake), + &build, + &lean, + &contracts, + lc() + ))); + let again = std::fs::read_to_string(&spath).expect("r"); + assert_eq!( + again, first, + "the regeneration is byte-identical to the honest run" + ); + assert_ne!( + again, edited, + "and so differs from the edit: git diff --exit-code fails" + ); + } + + /// A tree that did not build runs no Lean step, and still leaves a summary saying so. + #[test] + fn a_failed_build_rejects_a_cache_miss_declines_and_both_leave_a_red_summary() { + let (d, lean, contracts) = tree(); + gen(&lean, &contracts, false).expect("gen-axioms"); + let lake = run_lake(d.path(), &lean, 0); + for (rc, want_reject) in [(1, true), (2, false)] { + let r = run_all(lk(&lake), &build_sh(d.path(), rc), &lean, &contracts, lc()); + assert_eq!(is_reject(&r), want_reject, "build rc {rc}: {r:?}"); + assert_eq!(is_decline(&r), !want_reject, "build rc {rc}: {r:?}"); + let s = summary_of(&lean); + assert_eq!(s.build_exit, Some(rc)); + assert_eq!( + (s.lake_exit, s.leanchecker_exit), + (None, None), + "no Lean step ran" + ); + assert!(!s.is_green() && s.derived().is_empty()); + assert!(lean.join(LOG_FILE).is_file()); + } + } + + /// A summary that cannot be written is the run's output missing: reject. + #[test] + fn an_unwritable_summary_rejects() { + let (d, lean, contracts) = tree(); + gen(&lean, &contracts, false).expect("gen-axioms"); + std::fs::create_dir_all(summary::summary_path(&lean)).expect("a dir where the file goes"); + let lake = run_lake(d.path(), &lean, 0); + assert!(is_reject(&run_all( + lk(&lake), + &build_sh(d.path(), 0), + &lean, + &contracts, + lc() + ))); + // The log is written after the summary, so it carries the failed write and the reject (EV-8a quorum). + let log = std::fs::read_to_string(lean.join(LOG_FILE)).expect("the log is still written"); + assert!(log.contains("\"verdict\": \"reject\""), "{log}"); + assert!(log.contains("cannot write"), "{log}"); + } + + /// A `lake` whose `env ` call runs `body` (every other call passes); `$PIDF` is a file for a pid. + fn hang_lake(dir: &Path, step: &str, body: &str) -> (String, PathBuf) { + let pidf = dir.join(format!("grandchild-{step}")); + let p = dir.join(format!("hang-lake-{step}")); + let script = format!( + "#!/bin/sh\nPIDF='{}'\nif [ \"$2\" = {step} ]; then\n {body}\nfi\nexit 0\n", + pidf.display() + ); + std::fs::write(&p, script).expect("w"); + let mut perm = std::fs::metadata(&p).expect("meta").permissions(); + std::os::unix::fs::PermissionsExt::set_mode(&mut perm, 0o755); + std::fs::set_permissions(&p, perm).expect("chmod"); + (p.to_string_lossy().into_owned(), pidf) + } + + /// Whether `pid` is gone (or a zombie awaiting its reaper), polled for up to 5 s. + fn gone(pid: &str) -> bool { + (0..250).any(|_| { + let alive = std::fs::read_to_string(format!("/proc/{pid}/stat")).is_ok_and(|st| { + st.rsplit(')') + .next() + .is_some_and(|t| !t.trim_start().starts_with('Z')) + }); + if alive { + std::thread::sleep(Duration::from_millis(20)); + } + !alive + }) + } + + /// #4239: a hung `lake env lean Axioms.lean` rejects with a named reason inside its bound, records 124, and + /// takes the `lean` under it (a grandchild in its process group) down with it. + #[test] + fn a_hung_elaboration_rejects_within_its_bound_and_kills_its_process_group() { + let (d, lean, _) = tree(); + let (lake, pidf) = hang_lake(d.path(), "lean", "sleep 60 & echo $! > \"$PIDF\"; wait"); + let mut r = Report::default(); + let t0 = Instant::now(); + lean_steps( + Lake { + bin: &lake, + timeout_s: 1, + }, + &mut r, + &lean, + false, + false, + None, + ); + assert!( + t0.elapsed() < Duration::from_secs(20), + "the bound held: {:?}", + t0.elapsed() + ); + assert!( + r.reject && r.decline.is_none(), + "a hang is RED, not a decline: {:?}", + r.lines + ); + assert_eq!(r.lake_exit, Some(TIMED_OUT)); + assert!( + r.lines + .iter() + .any(|l| l.contains("lake env lean Axioms.lean timed out after 1s")), + "{:?}", + r.lines + ); + let pid = std::fs::read_to_string(&pidf).expect("the fake recorded its grandchild"); + assert!( + gone(pid.trim()), + "grandchild {} outlived the kill", + pid.trim() + ); + } + + /// #4239: the comparator and the leanchecker arm's `printenv LEAN_SYSROOT` are bounded too, and a child that + /// exits while a grandchild holds its stdout cannot stall the drain past the bound. + #[test] + fn every_lake_call_is_bounded_and_a_held_pipe_cannot_stall_it() { + let (d, lean, _) = tree(); + std::fs::create_dir_all(lean.join(CHALLENGE_DIR)).expect("mkdir"); + std::fs::write( + lean.join(CHALLENGE_DIR).join("gelu-v1.lean"), + GELU_CHALLENGE, + ) + .expect("w"); + std::fs::create_dir_all(lean.join("scripts")).expect("mkdir"); + std::fs::write(lean.join(COMPARATOR), "").expect("w"); + let bounded = |lake: &str, cmp: bool, lc: Option| { + let mut r = Report::default(); + let t0 = Instant::now(); + lean_steps( + Lake { + bin: lake, + timeout_s: 1, + }, + &mut r, + &lean, + lc.is_some(), + cmp, + lc, + ); + assert!(t0.elapsed() < Duration::from_secs(20), "{:?}", t0.elapsed()); + r + }; + // `env lean --run …`: $2 is `lean` for elaboration too, so hang only on `--run`. + let (cmp, _) = hang_lake( + d.path(), + "lean", + &format!("{SELF_TEST_OK}; [ \"$3\" = --run ] && sleep 60"), + ); + let r = bounded(&cmp, true, None); + assert!(r.reject && r.decline.is_none(), "{:?}", r.lines); + assert!( + r.lines + .iter() + .any(|l| l.contains("file(s)) timed out after 1s")), + "{:?}", + r.lines + ); + let (sys, _) = hang_lake(d.path(), "printenv", "sleep 60"); + let r = bounded(&sys, false, LC); + assert!(r.reject && r.decline.is_none(), "{:?}", r.lines); + assert!( + r.lines + .iter() + .any(|l| l.contains("printenv LEAN_SYSROOT timed out")), + "{:?}", + r.lines + ); + assert_eq!(r.leanchecker_exit, None, "leanchecker never ran"); + // `lake` exits 0 at once, but a background child keeps stdout open: still bounded, still RED. + let (held, pidf) = hang_lake(d.path(), "lean", "sleep 60 & echo $! > \"$PIDF\"; exit 0"); + let r = bounded(&held, false, None); + assert!(r.reject, "{:?}", r.lines); + let pid = std::fs::read_to_string(&pidf).expect("pid"); + assert!( + gone(pid.trim()), + "pipe holder {} outlived the kill", + pid.trim() + ); + } + + /// #4238: the comparator's sha256 is checked against the FIPS vectors before any row is judged. A failing + /// self-test, a hung one, or one that exits 0 having checked fewer than three vectors rejects, and the rows + /// never run; a passing one is recorded first. + #[test] + fn the_comparator_self_test_gates_the_rows() { + let (d, lean, _) = tree(); + let h = "ab".repeat(32); + let lake = comparator_lake( + d.path(), + &lean, + 0, + &cmp_row( + "ProvableContracts.Gelu.gelu_bound", + &h, + &format!("\"{h}\""), + "[]", + ), + "", + ); + let r = compared(&lake, &lean); + let st = r + .lines + .iter() + .position(|l| l.contains("--self-test (3 FIPS vectors)")); + let rows = r.lines.iter().position(|l| l.contains("file(s))")); + assert!(st.is_some() && st < rows, "self-test first: {:?}", r.lines); + let body = std::fs::read_to_string(&lake).expect("r"); + for (name, stub) in [ + ("fails", "printf 'FAIL sha256 \\\"abc\\\" = 00\\n'; exit 1"), + ( + "two-vectors", + "printf 'ok sha256 a\\nok sha256 b\\n'; exit 0", + ), + ("silent", "exit 0"), + ("hangs", "sleep 60"), + ] { + let p = d.path().join(format!("self-test-{name}")); + std::fs::write( + &p, + body.replace( + SELF_TEST_OK, + &format!("if [ \"$5\" = --self-test ]; then {stub}; fi"), + ), + ) + .expect("w"); + let mut perm = std::fs::metadata(&p).expect("meta").permissions(); + std::os::unix::fs::PermissionsExt::set_mode(&mut perm, 0o755); + std::fs::set_permissions(&p, perm).expect("chmod"); + let mut r = Report::default(); + lean_steps( + Lake { + bin: &p.to_string_lossy(), + timeout_s: 2, + }, + &mut r, + &lean, + false, + true, + None, + ); + assert!(r.reject && r.decline.is_none(), "{name}: {:?}", r.lines); + assert!(r.challenges.is_none(), "{name}: rows judged: {:?}", r.lines); + assert!( + r.lines + .iter() + .any(|l| l.starts_with("FAIL") && l.contains("--self-test")), + "{name}: {:?}", + r.lines + ); + } + } +} diff --git a/crates/aprender-contracts-cli/src/commands/extract_rdf.rs b/crates/aprender-contracts-cli/src/commands/extract_rdf.rs index a962e180df..c7a745d012 100644 --- a/crates/aprender-contracts-cli/src/commands/extract_rdf.rs +++ b/crates/aprender-contracts-cli/src/commands/extract_rdf.rs @@ -25,6 +25,27 @@ struct ExtractReport<'a> { check: Option>, } +/// `--out` and `--cells-out` describe a release; without a release subject they are refused. +fn refuse_release_only_args( + out: Option<&Path>, + cells_out: Option<&Path>, +) -> Result<(), Box> { + if cells_out.is_some() { + return Err(ReleaseArgsRefused( + "--cells-out lists a release's derived cells; it needs --release-version, --release-commit and --surface" + .into(), + ) + .into()); + } + if out.is_some() { + return Err(ReleaseArgsRefused( + "--out writes the release evidence graph; it needs --release-version and --release-commit".into(), + ) + .into()); + } + Ok(()) +} + /// `check == true` → compare and report drift, write nothing. With a release `subject` (aprender#3715) the /// release evidence joins the graph, and the result goes ONLY to `out`: the tracked `contracts.nt` is the corpus, /// and a release's receipts written into it would be a release baked into every later PR's baseline. @@ -33,16 +54,12 @@ pub fn run( check: bool, subject: Option<&Subject>, out: Option<&Path>, + cells_out: Option<&Path>, ) -> Result<(), Box> { if let Some(subject) = subject { - return run_release(contract_dir, check, subject, out); - } - if out.is_some() { - return Err(ReleaseArgsRefused( - "--out writes the release evidence graph; it needs --release-version and --release-commit".into(), - ) - .into()); + return run_release(contract_dir, check, subject, out, cells_out); } + refuse_release_only_args(out, cells_out)?; let extraction = match extract::all(contract_dir) { Ok(x) => x, Err(e) => { @@ -125,6 +142,7 @@ fn run_release( check: bool, subject: &Subject, out: Option<&Path>, + cells_out: Option<&Path>, ) -> Result<(), Box> { if check { return Err(ReleaseArgsRefused( @@ -151,6 +169,9 @@ fn run_release( let mut hasher = Sha256::new(); hasher.update(nt.as_bytes()); std::fs::write(out, &nt)?; + if let Some(path) = cells_out { + write_cells(path, subject, extraction.release.as_ref())?; + } let report = ReleaseExtractReport { triples: extraction.graph.len(), sha256: format!("{:x}", hasher.finalize()), @@ -160,3 +181,26 @@ fn run_release( println!("{}", serde_json::to_string_pretty(&report)?); Ok(()) } + +/// `--cells-out`: every derived cell — the producer's work list, keyed by `cell_id` (aprender#3745 S2). +fn write_cells( + path: &Path, + subject: &Subject, + release: Option<&provable_contracts::ontology::extract::release_evidence::ReleaseStats>, +) -> Result<(), Box> { + let derived = release.map(|r| r.derived.as_slice()).unwrap_or_default(); + if derived.is_empty() { + return Err(ReleaseArgsRefused( + "--cells-out: no cell was derived (no --surface, or a surface with no leaf command)" + .into(), + ) + .into()); + } + let mut doc = serde_json::Map::new(); + doc.insert("schema".into(), "apr-release-cells/v1".into()); + doc.insert("version".into(), subject.version.clone().into()); + doc.insert("release_commit".into(), subject.commit.clone().into()); + doc.insert("cells".into(), serde_json::to_value(derived)?); + std::fs::write(path, serde_json::to_string_pretty(&doc)?)?; + Ok(()) +} diff --git a/crates/aprender-contracts-cli/src/commands/lint.rs b/crates/aprender-contracts-cli/src/commands/lint.rs index 0f590b07a1..7ed791536f 100644 --- a/crates/aprender-contracts-cli/src/commands/lint.rs +++ b/crates/aprender-contracts-cli/src/commands/lint.rs @@ -48,7 +48,7 @@ pub fn run( watch: bool, strict_test_binding: bool, armed_baseline_ref: Option<&str>, - gate: Option<&str>, + gate: &[String], shapes_opts: ShapesOptions, ) -> Result<(), Box> { refuse_missing_corpus(contract_dir)?; @@ -57,8 +57,10 @@ pub fn run( // is asked for, because run_single_gate would otherwise report every ref // missing on exactly the input the refusal exists for. refuse_single_file_strict_binding(contract_dir, strict_test_binding)?; - if let Some(name) = gate { - return run_single_gate(contract_dir, name, &shapes_opts); + match gate { + [] => {} + [name] => return run_single_gate(contract_dir, name, &shapes_opts, armed_baseline_ref), + names => return run_gates(contract_dir, names, &shapes_opts, armed_baseline_ref), } if watch { return run_watch( @@ -215,8 +217,10 @@ fn run_single_gate( contract_dir: &Path, name: &str, shapes_opts: &ShapesOptions, + armed_baseline_ref: Option<&str>, ) -> Result<(), Box> { - let (result, findings) = decide_named_gate(contract_dir, name, shapes_opts)?; + let (mut result, mut findings) = decide_named_gate(contract_dir, name, shapes_opts)?; + apply_measured_ratchet(contract_dir, armed_baseline_ref, &mut result, &mut findings)?; let report = SingleGateReport { gate: &result.name, @@ -237,6 +241,19 @@ fn run_single_gate( }; println!("{}", serde_json::to_string_pretty(&report)?); + // ONT-4e: `refines` says WHICH clause broke Liskov on the reject line, and which prose clause left it Unknown. + if result.name == provable_contracts::lint::refines_gate::GATE { + let lines = provable_contracts::lint::refines_gate::explain(&result, &findings); + if result.verdict == provable_contracts::ontology::verdict::Verdict::Fail + && !lines.is_empty() + { + return Err(crate::contract_walk::GateRejected(lines.join("; ")).into()); + } + for line in &lines { + eprintln!("{line}"); + } + } + // The exit is the gate's VERDICT, not its `passed` bit: a gate that ran and answered `Unknown{Warn}` (ONT-4b, warnings // and no violation) is a decline, exit 2 — `passed` alone would print 0 for a corpus nobody judged clean. match result.verdict { @@ -252,6 +269,100 @@ fn run_single_gate( } } +/// ONT-4c (v4.14) F-34 on the shapes gate: the measured-set withdrawal ratchet needs git, so it is applied here +/// rather than in the library gate. It upgrades `ratchets.measured_sets` from the library's `not-checked` to +/// `checked` when a comparand was asked, and each violation is an Error finding (PV-ONT-014) that turns the +/// verdict to `Fail` — a ratchet that prints and exits 0 is the shape this corpus keeps refusing. +fn apply_measured_ratchet( + contract_dir: &Path, + armed_baseline_ref: Option<&str>, + result: &mut provable_contracts::lint::GateResult, + findings: &mut Vec, +) -> Result<(), Box> { + use provable_contracts::lint::finding::LintFinding; + use provable_contracts::lint::rules::RuleSeverity; + use provable_contracts::lint::GateExtra; + use std::collections::BTreeSet; + + let Some(GateExtra::Shapes { + readme, + claude_md, + ratchets, + .. + }) = result.extra.as_mut() + else { + return Ok(()); + }; + let live_readme: BTreeSet = readme.verified_commands.iter().cloned().collect(); + let live_claude: BTreeSet = claude_md.verified_commands.iter().cloned().collect(); + let answer = lint_arming::measured_ratchet( + contract_dir, + armed_baseline_ref, + [&live_readme, &live_claude], + )?; + ratchets.measured_sets = answer.status.to_string(); + if answer.violations.is_empty() { + return Ok(()); + } + let file = contract_dir + .join("lint-baseline.json") + .display() + .to_string(); + for m in answer.violations { + findings.push(LintFinding::new( + "PV-ONT-014", + RuleSeverity::Error, + m, + file.clone(), + )); + } + result.passed = false; + result.verdict = provable_contracts::ontology::verdict::Verdict::Fail; + Ok(()) +} + +/// PVL-001 EV-11: `--gate a --gate b` runs every named gate, each printing its own report as `--gate a` alone +/// would, and exits with their MEET: any refusal (exit 3) over any reject (1) over any decline (2) over pass (0). +/// A name this build does not compute is refused before any gate runs, so a typo never reports a partial pass. +fn run_gates( + contract_dir: &Path, + names: &[String], + shapes_opts: &ShapesOptions, + armed_baseline_ref: Option<&str>, +) -> Result<(), Box> { + use provable_contracts::lint::NAMED_GATES; + if let Some(bad) = names.iter().find(|n| !NAMED_GATES.contains(&n.as_str())) { + return Err(crate::contract_walk::UnknownGate { + asked: bad.clone(), + known: NAMED_GATES.iter().map(|g| (*g).to_string()).collect(), + } + .into()); + } + let outcomes: Vec<_> = names + .iter() + .map(|n| run_single_gate(contract_dir, n, shapes_opts, armed_baseline_ref)) + .collect(); + let rank = |r: &Result<(), Box>| match r { + Ok(()) => 0, + Err(e) if e.is::() => 1, + Err(e) if e.is::() => 2, + Err(_) => 3, + }; + let passed = outcomes.iter().filter(|r| rank(r) == 0).count(); + let worst = outcomes + .into_iter() + .max_by_key(|r| rank(r)) + .unwrap_or(Ok(())); + match worst { + Err(e) if e.is::() => Err(LintRejected { + passed, + armed: names.len(), + } + .into()), + other => other, + } +} + /// One gate's run, mapped to a report or to the refusal/decline that stands in its place. Every non-verdict /// answer says WHY on stderr before it returns, because an exit code without a reason is the thing this gate /// exists to refuse. @@ -265,10 +376,7 @@ fn decide_named_gate( name: &str, shapes_opts: &ShapesOptions, ) -> Result> { - use provable_contracts::lint::{ - relations_gate::RelationsOutcome, sigma_gate::SigmaOutcome, - valid_under_gate::ValidUnderOutcome, NamedGateOutcome, NAMED_GATES, - }; + use provable_contracts::lint::{NamedGateOutcome, NAMED_GATES}; match provable_contracts::lint::run_named_gate_with(contract_dir, name, shapes_opts) { NamedGateOutcome::UnknownGate => Err(crate::contract_walk::UnknownGate { @@ -276,43 +384,209 @@ fn decide_named_gate( known: NAMED_GATES.iter().map(|g| (*g).to_string()).collect(), } .into()), - NamedGateOutcome::Sigma(SigmaOutcome::NoSigma) => Err(LintDeclined { - reason: provable_contracts::ontology::verdict::Reason::NoCheckable, + NamedGateOutcome::Shapes(outcome) => decide_shapes_gate(outcome), + NamedGateOutcome::Consistency(outcome) => decide_consistency_gate(outcome), + NamedGateOutcome::Refines(outcome) => decide_refines_gate(outcome), + NamedGateOutcome::Tbox(outcome) => decide_tbox_gate(outcome), + other => decide_scored_gate(name, other), + } +} + +/// A gate whose decline is a bare "nothing checkable" reason, with no message of its own. +fn no_checkable() -> Box { + LintDeclined { + reason: provable_contracts::ontology::verdict::Reason::NoCheckable, + } + .into() +} + +/// The remaining named-gate outcomes that share the `Sigma`-derived answer shape (no Σ, malformed Σ, a +/// verdict): `sigma`, `relations`, `evidence`, `valid-under` and any PVL-001 ratchet. +fn decide_scored_gate( + name: &str, + outcome: provable_contracts::lint::NamedGateOutcome, +) -> Result> { + use provable_contracts::lint::NamedGateOutcome; + + match outcome { + NamedGateOutcome::Sigma(o) => decide_sigma_gate(o), + NamedGateOutcome::Relations(o) => decide_relations_gate(o), + NamedGateOutcome::Evidence(o) => decide_evidence_gate(o), + NamedGateOutcome::ValidUnder(o) => decide_valid_under_gate(o), + NamedGateOutcome::Ratchet(o) => decide_ratchet_gate(name, o), + NamedGateOutcome::Ran { result, findings } => Ok((result, findings)), + _ => unreachable!( + "decide_named_gate routes UnknownGate/Shapes/Consistency/Refines/Tbox directly" + ), + } +} + +/// The `sigma` gate's answers: no Σ, malformed Σ, or a verdict. +fn decide_sigma_gate( + outcome: provable_contracts::lint::sigma_gate::SigmaOutcome, +) -> Result> { + use provable_contracts::lint::sigma_gate::SigmaOutcome; + + match outcome { + SigmaOutcome::NoSigma => Err(no_checkable()), + SigmaOutcome::Malformed(e) => { + Err(crate::contract_walk::SigmaMalformed(e.to_string()).into()) } - .into()), - NamedGateOutcome::Sigma(SigmaOutcome::Malformed(e)) => { + SigmaOutcome::Ran { result, findings } => Ok((result, findings)), + } +} + +/// The `relations` gate's answers (ONT-4): no Σ, no typed relations, malformed Σ, or a verdict. +fn decide_relations_gate( + outcome: provable_contracts::lint::relations_gate::RelationsOutcome, +) -> Result> { + use provable_contracts::lint::relations_gate::RelationsOutcome; + + match outcome { + RelationsOutcome::NoSigma | RelationsOutcome::NoRelations { .. } => Err(no_checkable()), + RelationsOutcome::Malformed(e) => { Err(crate::contract_walk::SigmaMalformed(e.to_string()).into()) } - NamedGateOutcome::Relations( - RelationsOutcome::NoSigma | RelationsOutcome::NoRelations { .. }, - ) => Err(LintDeclined { - reason: provable_contracts::ontology::verdict::Reason::NoCheckable, + RelationsOutcome::Ran { result, findings } => Ok((result, findings)), + } +} + +/// The `evidence` gate's answers (ONT-8): no Σ, no evidence block (prints why first), malformed Σ, or a verdict. +fn decide_evidence_gate( + outcome: provable_contracts::lint::evidence_gate::EvidenceOutcome, +) -> Result> { + use provable_contracts::lint::evidence_gate::EvidenceOutcome; + + match outcome { + EvidenceOutcome::NoSigma => Err(no_checkable()), + EvidenceOutcome::NoEvidence { contracts_checked } => { + eprintln!( + "evidence: no evidence block in {contracts_checked} contract(s) — nothing was measured" + ); + Err(no_checkable()) } - .into()), - NamedGateOutcome::Relations(RelationsOutcome::Malformed(e)) => { + EvidenceOutcome::Malformed(e) => { Err(crate::contract_walk::SigmaMalformed(e.to_string()).into()) } - NamedGateOutcome::Shapes(outcome) => decide_shapes_gate(outcome), - NamedGateOutcome::ValidUnder(ValidUnderOutcome::NoSigma) => Err(LintDeclined { - reason: provable_contracts::ontology::verdict::Reason::NoCheckable, + EvidenceOutcome::Ran { result, findings } => Ok((result, findings)), + } +} + +/// The `valid-under` gate's answers (ONT-7): no Σ, no kernels (prints why first), malformed Σ, or a verdict. +fn decide_valid_under_gate( + outcome: provable_contracts::lint::valid_under_gate::ValidUnderOutcome, +) -> Result> { + use provable_contracts::lint::valid_under_gate::ValidUnderOutcome; + + match outcome { + ValidUnderOutcome::NoSigma => Err(no_checkable()), + ValidUnderOutcome::NoKernels { contracts_checked } => { + eprintln!( + "valid-under: no kernel-kind contract and no valid_under in {contracts_checked} contract(s) — nothing was measured" + ); + Err(no_checkable()) + } + ValidUnderOutcome::Malformed(e) => { + Err(crate::contract_walk::SigmaMalformed(e.to_string()).into()) + } + ValidUnderOutcome::Ran { result, findings } => Ok((result, findings)), + } +} + +/// A PVL-001 EV-11 ratchet's answers: a decline (prints why first) or a verdict. +fn decide_ratchet_gate( + name: &str, + outcome: provable_contracts::lint::ratchet_gates::RatchetOutcome, +) -> Result> { + use provable_contracts::lint::ratchet_gates::RatchetOutcome; + + match outcome { + RatchetOutcome::Declined(why) => { + eprintln!("{name}: {why}"); + Err(no_checkable()) + } + RatchetOutcome::Ran { result, findings } => Ok((result, findings)), + } +} + +/// The `ont-consistency` gate's answers (ONT-5). Only `Ran` is a verdict; every decline prints WHY first — a stale +/// witness names the command that regenerates it. +fn decide_consistency_gate( + outcome: provable_contracts::lint::consistency_gate::ConsistencyOutcome, +) -> Result> { + use provable_contracts::lint::consistency_gate::{decline_reason, why, ConsistencyOutcome}; + + match outcome { + ConsistencyOutcome::Ran { result, findings } => Ok((result, findings)), + ConsistencyOutcome::Malformed(e) => { + Err(crate::contract_walk::SigmaMalformed(e.to_string()).into()) + } + other => { + eprintln!("ont-consistency: {}", why(&other)); + let reason = decline_reason(&other) + .unwrap_or(provable_contracts::ontology::verdict::Reason::NoCheckable); + Err(LintDeclined { reason }.into()) + } + } +} + +/// The `refines` gate's answers (ONT-4e). Only `Ran` is a verdict; every decline prints WHY first — a stale Liskov +/// witness names `make contracts`. +fn decide_refines_gate( + outcome: provable_contracts::lint::refines_gate::RefinesOutcome, +) -> Result> { + use provable_contracts::lint::refines_gate::{decline_reason, why, RefinesOutcome, GATE}; + + match outcome { + RefinesOutcome::Ran { result, findings } => Ok((result, findings)), + RefinesOutcome::Malformed(e) => { + Err(crate::contract_walk::SigmaMalformed(e.to_string()).into()) + } + other => { + eprintln!("{GATE}: {}", why(&other)); + let reason = decline_reason(&other) + .unwrap_or(provable_contracts::ontology::verdict::Reason::NoCheckable); + Err(LintDeclined { reason }.into()) + } + } +} + +/// The `tbox` gate's answers (ONT-2c). There is no verdict arm: a clean classification is `Unknown{Advisory}` +/// (R-7, no inferred fact arms a merge); everything else is the declaration's fault (exit 3). +fn decide_tbox_gate( + outcome: provable_contracts::lint::tbox_gate::TboxOutcome, +) -> Result> { + use provable_contracts::lint::tbox_gate::TboxOutcome; + use provable_contracts::ontology::verdict::Reason; + + match outcome { + TboxOutcome::NoSigma => Err(LintDeclined { + reason: Reason::NoCheckable, } .into()), - NamedGateOutcome::ValidUnder(ValidUnderOutcome::NoKernels { contracts_checked }) => { + TboxOutcome::Malformed(e) | TboxOutcome::Stale(e) => { + Err(crate::contract_walk::SigmaMalformed(e).into()) + } + TboxOutcome::PreconditionFailed(refused) => { + Err(crate::contract_walk::SigmaMalformed(format!( + "told-closure precondition fails, so no classification is claimed: {}", + refused.join("; ") + )) + .into()) + } + TboxOutcome::Advisory(report) => { eprintln!( - "valid-under: no kernel-kind contract and no valid_under in {contracts_checked} contract(s) — nothing was measured" + "tbox: {} classes, consistent={}, unintended_subsumptions={} (method {}, advisory; never arms)", + report.classes, + report.consistent, + report.unintended_subsumptions.len(), + report.method ); Err(LintDeclined { - reason: provable_contracts::ontology::verdict::Reason::NoCheckable, + reason: Reason::Advisory, } .into()) } - NamedGateOutcome::ValidUnder(ValidUnderOutcome::Malformed(e)) => { - Err(crate::contract_walk::SigmaMalformed(e.to_string()).into()) - } - NamedGateOutcome::Sigma(SigmaOutcome::Ran { result, findings }) - | NamedGateOutcome::Relations(RelationsOutcome::Ran { result, findings }) - | NamedGateOutcome::ValidUnder(ValidUnderOutcome::Ran { result, findings }) - | NamedGateOutcome::Ran { result, findings } => Ok((result, findings)), } } @@ -322,7 +596,6 @@ fn decide_shapes_gate( outcome: provable_contracts::lint::shapes_gate::ShapesOutcome, ) -> Result> { use provable_contracts::lint::shapes_gate::ShapesOutcome; - use provable_contracts::ontology::verdict::Reason; match outcome { ShapesOutcome::Unsupported(e) => { @@ -331,14 +604,24 @@ fn decide_shapes_gate( ShapesOutcome::ExtractFailed(e) => { Err(crate::contract_walk::SigmaMalformed(e.to_string()).into()) } - ShapesOutcome::NoShapes { .. } => Err(LintDeclined { - reason: Reason::NoShapes, - } - .into()), - ShapesOutcome::NoFocus { .. } => Err(LintDeclined { - reason: Reason::NoFocus, + ShapesOutcome::Ran { result, findings } => Ok((result, findings)), + declined => Err(LintDeclined { + reason: shapes_decline_reason(declined), } .into()), + } +} + +/// A `shapes` outcome that is a decline: print what could not be checked, return the decline's reason. +fn shapes_decline_reason( + outcome: provable_contracts::lint::shapes_gate::ShapesOutcome, +) -> provable_contracts::ontology::verdict::Reason { + use provable_contracts::lint::shapes_gate::ShapesOutcome; + use provable_contracts::ontology::verdict::Reason; + + match outcome { + ShapesOutcome::NoShapes { .. } => Reason::NoShapes, + ShapesOutcome::NoFocus { .. } => Reason::NoFocus, ShapesOutcome::WrongCorpus { shapes_n, expected, @@ -351,10 +634,7 @@ fn decide_shapes_gate( for r in &refused { eprintln!("shapes: refused {r}"); } - Err(LintDeclined { - reason: Reason::WrongCorpus, - } - .into()) + Reason::WrongCorpus } ShapesOutcome::NoReceipts { shapes_n, dir } => { // ONT-4c1: the WHY travels with the decline — the lattice has no ReceiptUnmeasured element (ONT-6's @@ -362,17 +642,24 @@ fn decide_shapes_gate( eprintln!( "shapes: {shapes_n} shape(s) resolve receipts and the tree holds none under {dir}/" ); - Err(LintDeclined { - reason: Reason::NoCheckable, + Reason::NoCheckable + } + ShapesOutcome::HarnessBroken { causes } => { + for c in &causes { + eprintln!("shapes: CRUX harness broken — {c}"); } - .into()) + Reason::NoCheckable + } + ShapesOutcome::EmptyDomain { shapes_n } => { + // ONT-4c5 / R-2: |D| = 0 answers nothing about a required cell — a decline, never Pass, never RED + eprintln!( + "shapes: capability-cells domain D is empty ({shapes_n} shape(s)) — no required rung has a host to be measured on" + ); + Reason::NoCheckable } ShapesOutcome::PositiveControlFailed { which, .. } => { eprintln!("shapes: positive control {which} did not fire"); - Err(LintDeclined { - reason: Reason::PositiveControlFailed, - } - .into()) + Reason::PositiveControlFailed } ShapesOutcome::Differential { passed, n, failed, .. @@ -384,12 +671,12 @@ fn decide_shapes_gate( for f in &failed { eprintln!(" {f}"); } - Err(LintDeclined { - reason: Reason::Differential, - } - .into()) + Reason::Differential } - ShapesOutcome::Ran { result, findings } => Ok((result, findings)), + // answered by decide_shapes_gate before it asks for a decline reason + ShapesOutcome::Unsupported(_) + | ShapesOutcome::ExtractFailed(_) + | ShapesOutcome::Ran { .. } => Reason::NoCheckable, } } diff --git a/crates/aprender-contracts-cli/src/commands/lint_arming.rs b/crates/aprender-contracts-cli/src/commands/lint_arming.rs index 0c27c32d97..2ab9bd7af4 100644 --- a/crates/aprender-contracts-cli/src/commands/lint_arming.rs +++ b/crates/aprender-contracts-cli/src/commands/lint_arming.rs @@ -14,6 +14,7 @@ //! A committed `lint-baseline.json` that does not parse is an error on both paths: the committed value //! exists, it just cannot be read, and skipping it would disarm the check for exactly that commit. +use std::collections::BTreeSet; use std::error::Error; use std::path::{Path, PathBuf}; use std::process::Command; @@ -21,6 +22,7 @@ use std::process::Command; use provable_contracts::ontology::arming::{ check_monotone, check_shapes_monotone, ArmedGates, ArmedShapes, }; +use provable_contracts::ontology::measured_sets; const BASELINE: &str = "lint-baseline.json"; const NO_COMPARAND: &str = "NOT CHECKED (no comparand)"; @@ -41,7 +43,7 @@ pub fn resolve(contract_dir: &Path, explicit_ref: Option<&str>) -> Result ( + Comparand::Absent(_, why) => ( format!("{NO_COMPARAND} — {why}"), format!("{NO_COMPARAND} — {why}"), ), @@ -127,6 +129,62 @@ pub fn declared(contract_dir: &Path) -> Result> { )?) } +/// ONT-4c (v4.14) F-34: the measured-set withdrawal ratchet's answer for one run. +pub struct MeasuredRatchet { + /// `checked` when F-34 was computed against a comparand; `not-checked` outside a git work tree. + pub status: &'static str, + /// One RED message per violation (a drop with no `withdrawn[]` entry, an entry with no drop, no comparand + /// ref in a work tree, a set that cannot be read). + pub violations: Vec, +} + +/// F-34 against the comparand `resolve` uses: `withdrawn(HEAD) \ withdrawn(comparand) == set(comparand) \ +/// set(live)` for each key of [`measured_sets::KEYS`], `live` in the same order. The three no-comparand cases +/// are NOT the arming monotone's single `NOT CHECKED`: outside a work tree there is nothing to ask +/// (`not-checked`, which a probe reads as not green); a work tree whose comparand ref does not resolve is RED — +/// on a CI checkout that is a fetch misconfiguration, and answering "not checked" there would disarm the +/// ratchet on exactly the runs that gate; a corpus untracked at the comparand has every set ∅ there. +pub fn measured_ratchet( + contract_dir: &Path, + explicit_ref: Option<&str>, + live: [&BTreeSet; 2], +) -> Result> { + let comparand_text = match comparand(contract_dir, explicit_ref)? { + Comparand::Absent(Missing::WorkTree, _) => { + return Ok(MeasuredRatchet { + status: "not-checked", + violations: Vec::new(), + }) + } + Comparand::Absent(Missing::Ref, why) => { + return Ok(MeasuredRatchet { + status: "checked", + violations: vec![format!( + "F-34: no comparand in a git work tree ({why}) — fetch origin/main; {}", + measured_sets::REMEDY + )], + }) + } + Comparand::Absent(Missing::Untracked, _) => None, + Comparand::At { text, .. } => text, + }; + let head_text = read_baseline(contract_dir)?; + let mut violations = Vec::new(); + for (key, live) in measured_sets::KEYS.iter().zip(live) { + let head = measured_sets::read(head_text.as_deref(), key); + let cmp = measured_sets::read(comparand_text.as_deref(), key); + match (head, cmp) { + (Ok(h), Ok(c)) => violations.extend(measured_sets::withdrawal(key, &h, &c, live)), + (Err(e), _) => violations.push(format!("F-34: {e}")), + (_, Err(e)) => violations.push(format!("F-34: comparand {e}")), + } + } + Ok(MeasuredRatchet { + status: "checked", + violations, + }) +} + /// `/lint-baseline.json`, or `None` when the corpus has none (a single-file corpus never does). fn read_baseline(contract_dir: &Path) -> Result, Box> { if !contract_dir.is_dir() { @@ -140,9 +198,21 @@ fn read_baseline(contract_dir: &Path) -> Result, Box> } } +/// Why the default path has no comparand. ONT-4c (v4.14) splits the three: the measured-set ratchet (F-34) +/// answers each differently, where the arming monotone prints `NOT CHECKED` for all of them. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Missing { + /// The corpus is not in a git work tree: nothing to ask. + WorkTree, + /// A work tree with neither merge-base(HEAD, origin/main) nor origin/main. + Ref, + /// A comparand commit at which the corpus is not tracked: every committed set there is ∅. + Untracked, +} + enum Comparand { - /// Default path only: nothing committed to compare against, and why. - Absent(String), + /// Default path only: nothing committed to compare against, which case, and why. + Absent(Missing, String), /// The committed baseline text at `label` (`None`: the corpus is tracked there without a baseline, /// which arms the default set). At { @@ -164,7 +234,7 @@ fn comparand(contract_dir: &Path, explicit_ref: Option<&str>) -> Result Err(format!("--armed-baseline-ref {r}: {why}").into()), - None => Ok(Comparand::Absent(why)), + None => Ok(Comparand::Absent(Missing::WorkTree, why)), }; }; let (commit, label) = match explicit_ref { @@ -184,6 +254,7 @@ fn comparand(contract_dir: &Path, explicit_ref: Option<&str>) -> Result found, None => { return Ok(Comparand::Absent( + Missing::Ref, "neither merge-base(HEAD, origin/main) nor origin/main resolves".into(), )) } @@ -193,7 +264,7 @@ fn comparand(contract_dir: &Path, explicit_ref: Option<&str>) -> Result Err(format!("--armed-baseline-ref {r}: {why}").into()), - None => Ok(Comparand::Absent(why)), + None => Ok(Comparand::Absent(Missing::Untracked, why)), }; } let path = if rel.is_empty() { diff --git a/crates/aprender-contracts-cli/src/commands/mod.rs b/crates/aprender-contracts-cli/src/commands/mod.rs index c9734eccc5..0d2d4a9e82 100644 --- a/crates/aprender-contracts-cli/src/commands/mod.rs +++ b/crates/aprender-contracts-cli/src/commands/mod.rs @@ -2,11 +2,13 @@ pub mod audit; pub mod book; pub mod census; pub mod certify; +pub mod challenge; pub mod check_parity; pub mod codegen; pub mod coq; pub mod coverage; pub mod diff; +pub mod discharge; pub mod equations; pub mod explain; pub mod extract; @@ -24,6 +26,8 @@ pub mod lean_status; pub mod lint; pub mod migrate; pub mod mirai; +pub mod obligations; +pub mod ontology; pub mod pipeline; pub mod probar; pub mod proof_status; diff --git a/crates/aprender-contracts-cli/src/commands/obligations.rs b/crates/aprender-contracts-cli/src/commands/obligations.rs new file mode 100644 index 0000000000..afcbbb912e --- /dev/null +++ b/crates/aprender-contracts-cli/src/commands/obligations.rs @@ -0,0 +1,444 @@ +//! `pv obligations [ROOT] [--gate]` — PVL-001 EV-10 (paiml/aprender#4198). +//! +//! A native replacement for pmat's `scripts/pv-obligation-gate.py` (sha256 `e4fa9719…`, pmat +//! `7c93535e`), which pmat's `Makefile:227` and `quality-gate.yml:246` run as a blocking gate. +//! Every contract must be readable by pv, and its obligations must bind to code. Three checks, +//! run per contract in the script's order, over `ROOT/contracts/*.yaml` (not recursive; a name +//! ending in `binding.yaml` is skipped) with functions searched under `ROOT/src/`: +//! +//! 1. `pv validate` passes — here in-process ([`validate_artifact`], the same decision +//! `pv validate ` makes: any error-severity violation, or no verdict at all, fails). +//! 2. No test-bearing entry (a mapping with a `test` key) hides under `falsification:`, the key +//! pv does not read. Entries without `test` are alert thresholds and are left alone. +//! 3. Every `applies_to` other than `all`, a falsy value, or an equation of the contract names a +//! `fn` under `src/`; where the contract declares `metadata.proved_type`, at least one file +//! defining that `fn` mentions the type as a word. +//! +//! Output is the script's, byte for byte: one `::error::` line per problem, then +//! `pv obligation gate: N problem(s) over M contracts`. Without `--gate` the exit is 0; with it, +//! any problem is `reject:` at exit 1. `tests/pvl_obligations_golden.rs` holds the two against +//! outputs recorded from the script. +//! +//! **Where it deliberately differs from the script** (each is an input the script does not +//! survive, or PVL-1): +//! - zero contracts is `decline:` at exit 2 (PVL-1), where the script reports 0 over 0 at exit 0; +//! - an input the script dies on with a Python traceback is a named problem here: a file that +//! cannot be read or is not YAML; a truthy document that is not a mapping; a truthy +//! `equations`/`metadata` that is not a mapping, or `proof_obligations`/`falsification` that +//! is not a list (a `falsification` mapping or string counts 0, as the script's loop does); +//! a `proof_obligations` entry that is not a mapping; a truthy `applies_to` that is not a +//! string; a truthy `proved_type` that is not a string, where a bound `fn` would be searched +//! for it; +//! - YAML is read by `serde_yaml` (YAML 1.2), the script's by PyYAML (YAML 1.1): a plain +//! `yes`/`no`/`on`/`off` is a string here and a boolean there; +//! - a document `serde_yaml` cannot read that PyYAML can (a duplicate key, an integer beyond +//! 64 bits, …) gets no checks 2 and 3 here. `pv validate` reads with the same parser, so +//! check 1 fails on it on both sides, and pv adds no parse line of its own: the report is +//! the script's unless PyYAML also finds a check-2 or check-3 problem in it, which pv then +//! does not name (the verdict still agrees); +//! - `src/` is walked on disk (symlinks not followed), where the script's `git grep` searches +//! tracked files only: an untracked file under `src/` counts here; +//! - a word character is `char::is_alphanumeric` or `_`; next to non-ASCII text this can differ +//! from Python's `re` and from `git grep`'s locale. +//! +//! **Where it deliberately agrees** (each held by `tests/fixtures/pvl/obligations/edge/`): merge +//! keys (`<<`) are resolved as PyYAML resolves them, and falsy values follow Python truthiness — +//! a falsy document is `{}`, a falsy `applies_to` is skipped, a falsy `proved_type` is none. +//! +//! Line-by-line mapping (script → here): `CONTRACTS` → [`contract_files`]; `_fn_files` → +//! [`SrcTree::fn_files`]; `check_validate` → [`check_validate`]; `check_visible` → +//! [`check_visible`]; `check_bindings` → [`check_bindings`]; `main` → [`run`]; `{x!r}` → +//! [`py_repr`]; `{files}` → [`py_list`]; `re` `\b` → [`boundary`]. + +use std::path::{Path, PathBuf}; + +use provable_contracts::error::Severity; +use provable_contracts::schema::validate_artifact; +use serde_yaml::{Mapping, Value}; + +use crate::contract_walk::{ObligationsRejected, ZeroContracts}; + +/// Run the three checks over `root`. With `gate`, any problem is an error (exit 1). +/// +/// # Errors +/// [`ZeroContracts`] when `root/contracts` holds no contract; [`ObligationsRejected`] under +/// `gate` when any problem was found. +pub fn run(root: &Path, gate: bool) -> Result<(), Box> { + let contracts = contract_files(root); + if contracts.is_empty() { + return Err(ZeroContracts { + path: root.join("contracts"), + filter: None, + } + .into()); + } + let mut src = SrcTree::new(root.join("src")); + let mut problems = Vec::new(); + for rel in &contracts { + problems.extend(check_contract(root, rel, &mut src)); + } + for p in &problems { + println!("::error::{p}"); + } + println!( + "pv obligation gate: {} problem(s) over {} contracts", + problems.len(), + contracts.len() + ); + if gate && !problems.is_empty() { + return Err(ObligationsRejected { + problems: problems.len(), + contracts: contracts.len(), + } + .into()); + } + Ok(()) +} + +/// `contracts/*.yaml` under `root`, relative and `/`-joined, sorted, `*binding.yaml` excluded. +/// Hidden names are excluded as `glob`'s `*` excludes them. Like `glob`, the name alone decides: +/// a symlink is followed when read, and an entry that cannot be read is a named problem. +fn contract_files(root: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(root.join("contracts")) else { + return Vec::new(); + }; + let mut out: Vec = entries + .filter_map(Result::ok) + .map(|e| e.file_name().to_string_lossy().into_owned()) + .filter(|n| !n.starts_with('.') && n.ends_with(".yaml") && !n.ends_with("binding.yaml")) + .map(|n| format!("contracts/{n}")) + .collect(); + out.sort(); + out +} + +fn check_contract(root: &Path, rel: &str, src: &mut SrcTree) -> Vec { + let mut problems = check_validate(root, rel); + let doc = match load(&root.join(rel)) { + Ok(doc) => doc, + // A document `serde_yaml` refuses also fails `pv validate`, the script's only + // report on it when PyYAML finds nothing else; a second line would miscount it. + Err(Load::Parse(_)) if !problems.is_empty() => return problems, + Err(Load::Parse(why) | Load::Other(why)) => { + problems.push(format!("{rel}: {why}")); + return problems; + } + }; + problems.extend(check_visible(rel, &doc)); + problems.extend(check_bindings(rel, &doc, src)); + problems +} + +/// Why a contract could not be loaded. `Parse` is `serde_yaml` refusing the text. +enum Load { + Parse(String), + Other(String), +} + +/// `yaml.safe_load(...) or {}`: merge keys (`<<`) resolved as PyYAML resolves them, and a +/// falsy document (empty, `false`, `0`, `[]`) is an empty mapping. +fn load(path: &Path) -> Result { + let text = + std::fs::read_to_string(path).map_err(|e| Load::Other(format!("cannot be read: {e}")))?; + let mut doc: Value = serde_yaml::from_str(&text) + .map_err(|e| Load::Parse(format!("does not parse as YAML: {e}")))?; + // One pass resolves one level; a merged mapping that itself merges needs another. + loop { + let before = doc.clone(); + doc.apply_merge() + .map_err(|e| Load::Other(format!("has a merge key PyYAML refuses: {e}")))?; + if doc == before { + break; + } + } + match doc { + Value::Mapping(m) => Ok(m), + v if !truthy(&v) => Ok(Mapping::new()), + _ => Err(Load::Other("is not a YAML mapping".into())), + } +} + +/// Python's `bool(x)` over a YAML value — what the script's `x or {}`, `if not target` and +/// `if proved` decide on. +fn truthy(v: &Value) -> bool { + match v { + Value::Null => false, + Value::Bool(b) => *b, + Value::Number(n) => n.as_f64().is_none_or(|f| f != 0.0), + Value::String(s) => !s.is_empty(), + Value::Sequence(s) => !s.is_empty(), + Value::Mapping(m) => !m.is_empty(), + Value::Tagged(_) => true, + } +} + +/// `m.get(key) or `: `None` when the key is absent or its value is falsy. +fn get_truthy<'a>(m: &'a Mapping, key: &str) -> Option<&'a Value> { + m.get(key).filter(|v| truthy(v)) +} + +fn check_validate(root: &Path, rel: &str) -> Vec { + let passes = validate_artifact(&root.join(rel)) + .is_ok_and(|(_, v)| v.iter().all(|v| v.severity != Severity::Error)); + if passes { + Vec::new() + } else { + vec![format!("{rel}: pv validate failed")] + } +} + +fn check_visible(rel: &str, doc: &Mapping) -> Vec { + let hidden = match get_truthy(doc, "falsification") { + None => 0, + Some(Value::Sequence(entries)) => entries + .iter() + .filter(|e| e.as_mapping().is_some_and(|m| m.contains_key("test"))) + .count(), + // Iterating a dict or a str yields keys or characters, never a dict. + Some(Value::Mapping(_) | Value::String(_)) => 0, + Some(_) => return vec![format!("{rel}: falsification is not a list")], + }; + if hidden == 0 { + return Vec::new(); + } + vec![format!( + "{rel}: {hidden} test-bearing obligation(s) under `falsification:`, \ + which pv cannot read — use `falsification_tests:`" + )] +} + +/// `metadata.proved_type` as the script reads it: `(doc.get("metadata") or {}).get(...)`, then +/// `if proved`. A truthy non-string is only an error where the script would `re.escape` it. +#[derive(Clone, Copy)] +enum Proved<'a> { + Absent, + Type(&'a str), + NotAString, +} + +fn check_bindings(rel: &str, doc: &Mapping, src: &mut SrcTree) -> Vec { + let (equations, proved, obligations) = match bindings_of(doc) { + Ok(parts) => parts, + Err(why) => return vec![format!("{rel}: {why}")], + }; + let mut problems = Vec::new(); + for ob in obligations { + let Some(ob) = ob.as_mapping() else { + problems.push(format!("{rel}: a proof_obligations entry is not a mapping")); + continue; + }; + // `if not target or target == "all" or target in equations: continue` + let Some(target) = get_truthy(ob, "applies_to") else { + continue; + }; + if equations.contains(&target) { + continue; + } + let Some(target) = target.as_str() else { + problems.push(format!("{rel}: an applies_to is not a string")); + continue; + }; + if target != "all" { + problems.extend(check_target(rel, target, proved, src)); + } + } + problems +} + +/// The equation names, the proved type and the obligations, each read as the script reads +/// them; a shape the script would die on is an error naming it. +fn bindings_of(doc: &Mapping) -> Result<(Vec<&Value>, Proved<'_>, &[Value]), String> { + let equations = match get_truthy(doc, "equations") { + None => Vec::new(), + Some(Value::Mapping(m)) => m.keys().collect(), + Some(_) => return Err("equations is not a mapping".into()), + }; + let proved = match get_truthy(doc, "metadata") { + None => Proved::Absent, + Some(Value::Mapping(m)) => match get_truthy(m, "proved_type") { + None => Proved::Absent, + Some(Value::String(s)) => Proved::Type(s), + Some(_) => Proved::NotAString, + }, + Some(_) => return Err("metadata is not a mapping".into()), + }; + let obligations = match get_truthy(doc, "proof_obligations") { + None => &[][..], + Some(Value::Sequence(obs)) => obs.as_slice(), + Some(_) => return Err("proof_obligations is not a list".into()), + }; + Ok((equations, proved, obligations)) +} + +fn check_target(rel: &str, target: &str, proved: Proved<'_>, src: &mut SrcTree) -> Option { + let files = src.fn_files(target); + if files.is_empty() { + return Some(format!( + "{rel}: applies_to {} names neither an equation of this contract nor any `fn` under src/", + py_repr(target) + )); + } + let proved = match proved { + Proved::Absent => return None, + Proved::Type(t) => t, + Proved::NotAString => { + return Some(format!( + "{rel}: applies_to {} is proved against a metadata.proved_type that is not a string", + py_repr(target) + )) + } + }; + if files.iter().any(|f| src.mentions(f, proved)) { + return None; + } + Some(format!( + "{rel}: applies_to {} is proved against {}, but none of {} mentions it — the proof does \ + not reach the code it names", + py_repr(target), + py_repr(proved), + py_list(&files) + )) +} + +/// Every file under `src/`, read once, lossily, on first use. Keys are `src/…`, `/`-joined, +/// in byte order (`git grep -l`'s order). +struct SrcTree { + dir: PathBuf, + files: Option>, +} + +impl SrcTree { + fn new(dir: PathBuf) -> Self { + Self { dir, files: None } + } + + fn loaded(&mut self) -> &[(String, String)] { + let dir = &self.dir; + self.files.get_or_insert_with(|| { + let mut paths = Vec::new(); + walk(dir, &mut paths); + let mut files: Vec<(String, String)> = paths + .into_iter() + .filter_map(|p| { + let bytes = std::fs::read(&p).ok()?; + let rel = p.strip_prefix(dir).ok()?; + let key = std::iter::once("src".to_owned()) + .chain( + rel.components() + .map(|c| c.as_os_str().to_string_lossy().into_owned()), + ) + .collect::>() + .join("/"); + Some((key, String::from_utf8_lossy(&bytes).into_owned())) + }) + .collect(); + files.sort(); + files + }) + } + + /// `git grep -l -E "fn \b" -- src/`. + fn fn_files(&mut self, name: &str) -> Vec { + let needle = format!("fn {name}"); + self.loaded() + .iter() + .filter(|(_, text)| { + text.match_indices(&needle) + .any(|(i, m)| boundary(&text[..i + m.len()], &text[i + m.len()..])) + }) + .map(|(k, _)| k.clone()) + .collect() + } + + /// `re.search(rf"\b{re.escape(word)}\b", text)`. + fn mentions(&mut self, key: &str, word: &str) -> bool { + self.loaded().iter().any(|(k, text)| { + k == key + && text.match_indices(word).any(|(i, m)| { + boundary(&text[..i], &text[i..]) + && boundary(&text[..i + m.len()], &text[i + m.len()..]) + }) + }) + } +} + +fn walk(dir: &Path, out: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.filter_map(Result::ok) { + let Ok(kind) = entry.file_type() else { + continue; + }; + if kind.is_dir() { + walk(&entry.path(), out); + } else if kind.is_file() { + out.push(entry.path()); + } + } +} + +/// A regex `\b` between `before` and `after`: exactly one side is a word character. +fn boundary(before: &str, after: &str) -> bool { + let word = |c: Option| c.is_some_and(|c| c.is_alphanumeric() || c == '_'); + word(before.chars().next_back()) != word(after.chars().next()) +} + +/// Python's `repr` of a `str`: single quotes unless the text holds `'` and no `"`. +fn py_repr(s: &str) -> String { + let quote = if s.contains('\'') && !s.contains('"') { + '"' + } else { + '\'' + }; + let mut out = String::from(quote); + for c in s.chars() { + match c { + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + c if c == quote => { + out.push('\\'); + out.push(c); + } + c => out.push(c), + } + } + out.push(quote); + out +} + +/// Python's `repr` of a `list[str]`. +fn py_list(items: &[String]) -> String { + let inner: Vec = items.iter().map(|s| py_repr(s)).collect(); + format!("[{}]", inner.join(", ")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn repr_matches_python() { + assert_eq!(py_repr("from_score"), "'from_score'"); + assert_eq!(py_repr("it's"), "\"it's\""); + assert_eq!(py_repr("a'b\"c"), "'a\\'b\"c'"); + assert_eq!(py_repr("a\\b"), "'a\\\\b'"); + assert_eq!( + py_list(&["src/a.rs".into(), "src/b.rs".into()]), + "['src/a.rs', 'src/b.rs']" + ); + assert_eq!(py_list(&[]), "[]"); + } + + #[test] + fn boundary_is_regex_backslash_b() { + assert!(boundary("fn score", "(x)")); + assert!(boundary("fn score", "")); + assert!(!boundary("fn score", "_v2")); + assert!(!boundary("fn score", "2")); + assert!(boundary(" ", "Grade")); + assert!(!boundary("Tdg", "Grade")); + } +} diff --git a/crates/aprender-contracts-cli/src/commands/ontology.rs b/crates/aprender-contracts-cli/src/commands/ontology.rs new file mode 100644 index 0000000000..471556b112 --- /dev/null +++ b/crates/aprender-contracts-cli/src/commands/ontology.rs @@ -0,0 +1,81 @@ +//! `pv ontology export --owl` / `pv ontology tbox` — ONT-001 §3.8, row ONT-2c. +//! +//! `export --owl` writes Σ as OWL 2 functional syntax with the in-house writer, byte-deterministic, so CI can +//! `cmp` a fresh export against the tracked `contracts/ontology.ofn` (R-18). `tbox` writes the told-closure +//! classification, `contracts/tbox-report.json`, which is ADVISORY: the `tbox` lint gate maps it to +//! `Unknown{Advisory}` and it never arms (R-7: no inferred fact arms a merge). +//! +//! Exit codes follow `pv lint`: 3 `error:` when Σ is malformed, cannot be written as OWL, or the TBox +//! precondition fails. In that last case the report is still printed, so the refusal names its axiom. + +use std::path::Path; + +use provable_contracts::ontology::owl; +use provable_contracts::ontology::sigma::Sigma; + +use crate::cli::OntologyCommand; + +pub fn run(command: &OntologyCommand) -> Result<(), Box> { + match command { + OntologyCommand::Export { + sigma, + owl: as_owl, + write, + } => { + if !as_owl { + eprintln!("error: `pv ontology export` writes one format today; pass --owl"); + std::process::exit(2); + } + let export = load_export(sigma); + emit(sigma, "ontology.ofn", &owl::to_ofn(&export), *write) + } + OntologyCommand::Tbox { sigma, write } => { + let export = load_export(sigma); + let report = owl::tbox(&export); + emit( + sigma, + "tbox-report.json", + &owl::report_json(&report), + *write, + )?; + if !report.precondition.holds { + for r in &report.precondition.refused { + eprintln!("error: told-closure precondition fails: {r}"); + } + std::process::exit(3); + } + Ok(()) + } + } +} + +fn load_export(sigma_path: &Path) -> owl::OwlExport { + let fail = |msg: String| -> ! { + eprintln!("error: {msg}"); + std::process::exit(3); + }; + let text = std::fs::read_to_string(sigma_path) + .unwrap_or_else(|e| fail(format!("cannot read Σ {}: {e}", sigma_path.display()))); + let sigma = Sigma::from_yaml(&text).unwrap_or_else(|e| fail(format!("Σ: {e}"))); + if let Err(e) = sigma.check_integrity() { + fail(format!("Σ: {e}")); + } + owl::export(&sigma).unwrap_or_else(|e| fail(e.to_string())) +} + +fn emit( + sigma_path: &Path, + name: &str, + body: &str, + write: bool, +) -> Result<(), Box> { + if write { + let dir = sigma_path.parent().unwrap_or_else(|| Path::new(".")); + let out = dir.join(name); + std::fs::write(&out, body)?; + eprintln!("wrote {}", out.display()); + } else { + print!("{body}"); + } + Ok(()) +} diff --git a/crates/aprender-contracts-cli/src/commands/proof_status.rs b/crates/aprender-contracts-cli/src/commands/proof_status.rs index f3fa582ffc..edf8671823 100644 --- a/crates/aprender-contracts-cli/src/commands/proof_status.rs +++ b/crates/aprender-contracts-cli/src/commands/proof_status.rs @@ -1,6 +1,6 @@ use std::path::Path; -use provable_contracts::binding::parse_binding; +use provable_contracts::binding::{parse_binding, BindingRegistry, ImplStatus}; use provable_contracts::obligation_matrix::{format_obligation_table, obligation_matrix}; use provable_contracts::proof_status::{format_text, proof_status_report}; use provable_contracts::schema::ContractKind; @@ -15,15 +15,20 @@ pub fn run( table: bool, kind_filter: Option<&str>, ) -> Result<(), Box> { - let binding = match binding_path { - // L5 gate: when --verify-bindings is set, downgrade any `implemented` - // binding whose function is absent from source, so L5 requires bindings - // that are VERIFIED as implemented rather than merely self-declared. - Some(bp) => Some(match verify_root { - Some(root) => parse_binding(bp)?.verified(root), - None => parse_binding(bp)?, - }), - None => None, + // PVL-001 EV-2 (PVL-2): a binding is RESOLVED, always. `--binding` used to count + // entries without resolving them (a binding naming a function that exists + // nowhere printed its report and exited 0), and `--verify-bindings` downgraded + // silently. Now every `implemented` binding is looked up with the + // `pv verify-bindings` resolver; a ghost is downgraded, listed under + // `GHOST BINDINGS (n)`, and the command exits 1. `--verify-bindings` is a no-op + // alias, kept so existing invocations still parse. + let _ = verify_root; + let (binding, ghosts) = match binding_path { + Some(bp) => { + let (reg, ghosts) = resolve_bindings(bp, parse_binding(bp)?); + (Some(reg), ghosts) + } + None => (None, Vec::new()), }; let kind = kind_filter.map(parse_kind).transpose()?; @@ -61,7 +66,72 @@ pub fn run( print!("{}", format_obligation_table(&matrices)); } - Ok(()) + if ghosts.is_empty() { + return Ok(()); + } + // Text mode prints the block on stdout with the report; JSON mode keeps stdout a + // single JSON document and prints the block on stderr. + let block = ghost_block(&ghosts); + if format == "json" { + eprint!("{block}"); + } else { + print!("{block}"); + } + Err(format!( + "{} ghost binding(s): claimed implemented, not found in source", + ghosts.len() + ) + .into()) +} + +/// One binding that claims `implemented` for a function the resolver cannot find. +struct Ghost { + contract: String, + equation: String, + function: String, +} + +/// Resolve every `implemented` binding against source with the `pv verify-bindings` +/// resolver (`scan_all_sources`: the binding's derived source root, its `crates/`, +/// and the local `src/`). A ghost is downgraded to `not_implemented` so the report's +/// levels are honest, and returned so the caller can name it and reject. +fn resolve_bindings(binding_path: &Path, reg: BindingRegistry) -> (BindingRegistry, Vec) { + use crate::commands::verify_bindings::{scan_all_sources, short_name}; + let found = scan_all_sources(binding_path, ®.target_crate); + let mut ghosts = Vec::new(); + let bindings = reg + .bindings + .into_iter() + .map(|mut b| { + let unresolved = b.status == ImplStatus::Implemented + && b.function + .as_deref() + .and_then(short_name) + .is_some_and(|s| !found.contains(&s)); + if unresolved { + ghosts.push(Ghost { + contract: b.contract.clone(), + equation: b.equation.clone(), + function: b.function.clone().unwrap_or_default(), + }); + b.status = ImplStatus::NotImplemented; + } + b + }) + .collect(); + (BindingRegistry { bindings, ..reg }, ghosts) +} + +/// `GHOST BINDINGS (n)`, then one line per ghost: the line PVL-001 EV-2's probe reads. +fn ghost_block(ghosts: &[Ghost]) -> String { + let mut out = format!("\nGHOST BINDINGS ({})\n", ghosts.len()); + for g in ghosts { + out.push_str(&format!( + " {} {}: {}\n", + g.contract, g.equation, g.function + )); + } + out } fn print_kind_breakdown(contracts: &[(String, provable_contracts::schema::Contract)]) { diff --git a/crates/aprender-contracts-cli/src/commands/verify_bindings.rs b/crates/aprender-contracts-cli/src/commands/verify_bindings.rs index 60b76cb115..d55d12b4c5 100644 --- a/crates/aprender-contracts-cli/src/commands/verify_bindings.rs +++ b/crates/aprender-contracts-cli/src/commands/verify_bindings.rs @@ -50,21 +50,28 @@ fn parse_expected_functions(content: &str) -> HashSet { let Some(rest) = line.trim().strip_prefix("function:") else { continue; }; - let func = rest.trim().trim_matches('"').trim_matches('\'').trim(); - if func.is_empty() || func == "N/A" { - continue; - } - let short = func.rsplit("::").next().unwrap_or(func).to_lowercase(); - if !short.is_empty() { + if let Some(short) = short_name(rest) { expected.insert(short); } } expected } +/// The name a `function:` value is resolved by: its last `::` segment, lowercased. +/// `None` for an empty value or `N/A`. ONE normalization, shared with +/// `pv proof-status --binding` (PVL-001 EV-2), so the two commands cannot disagree. +pub(crate) fn short_name(function: &str) -> Option { + let func = function.trim().trim_matches('"').trim_matches('\'').trim(); + if func.is_empty() || func == "N/A" { + return None; + } + let short = func.rsplit("::").next().unwrap_or(func).to_lowercase(); + (!short.is_empty()).then_some(short) +} + /// Scan the crate's `src/`, `crates/`, and the current-dir `src/` (if different) /// for `fn` declarations. -fn scan_all_sources(binding_path: &Path, label: &str) -> HashSet { +pub(crate) fn scan_all_sources(binding_path: &Path, label: &str) -> HashSet { let src_dir = derive_src_root(binding_path, label); let mut found: HashSet = HashSet::new(); let src = src_dir.join("src"); @@ -82,16 +89,30 @@ fn scan_all_sources(binding_path: &Path, label: &str) -> HashSet { found } -/// binding.yaml lives in `contracts//` — source is `../..//`. -/// Falls back to `.` when the path has no usable parent chain. +/// Where a binding's source lives. +/// +/// The multi-repo layout (`contracts//binding.yaml`, source at `../..//`) +/// is used when that directory exists. In this monorepo it does not +/// (`contracts/aprender/binding.yaml` -> `./aprender/`, absent), and scanning an +/// absent root made nearly every binding a ghost: under PVL-001 EV-2's reject that +/// is a false reject, and one that depended on the caller's cwd. So otherwise the +/// root is the nearest ancestor of the binding file holding a `crates/` or `src/` +/// tree (the workspace root), and `.` only when there is none. fn derive_src_root(binding_path: &Path, label: &str) -> std::path::PathBuf { - let Some(parent) = binding_path.parent() else { - return Path::new(".").to_path_buf(); - }; - parent + let has_tree = |d: &Path| d.join("src").is_dir() || d.join("crates").is_dir(); + let legacy = binding_path .parent() - .and_then(|p| p.parent()) - .map_or_else(|| Path::new(".").to_path_buf(), |p| p.join(label)) + .and_then(Path::parent) + .and_then(Path::parent) + .map(|p| p.join(label)); + if let Some(l) = legacy.filter(|l| has_tree(l)) { + return l; + } + let abs = std::fs::canonicalize(binding_path).unwrap_or_else(|_| binding_path.to_path_buf()); + abs.ancestors() + .skip(1) + .find(|d| has_tree(d)) + .map_or_else(|| Path::new(".").to_path_buf(), Path::to_path_buf) } /// Sort the expected names missing from `found` for stable reporting. @@ -159,33 +180,118 @@ fn scan_fns(dir: &Path, found: &mut HashSet) { } } -/// Extract lowercased `fn`/`pub fn`/`pub async fn`/`pub(crate) fn` names from source. +/// Extract lowercased function names from source: every `fn` item, whatever its +/// visibility (`pub`, `pub(crate)`, `pub(super)`, `pub(in path)`) and qualifiers +/// (`const`, `async`, `unsafe`, `extern "ABI"`). PVL-001 EV-2: `pv proof-status` +/// now REJECTS on a ghost, so a real function the scanner cannot see is a false +/// reject. Measured on aprender's contracts/binding.yaml: `compute_mse` is +/// `pub(super) fn` (crates/aprender-core/src/tree/regression_helpers.rs:27) and was +/// reported a ghost by the old four-prefix scanner (scripts/dogfood.sh records the +/// same defect for rmedia's `apply_loudnorm`). fn extract_fn_names(content: &str, found: &mut HashSet) { for line in content.lines() { - let t = line.trim(); - if !(t.starts_with("pub fn ") - || t.starts_with("pub async fn ") - || t.starts_with("pub(crate) fn ") - || t.starts_with("fn ")) - { - continue; - } - let part = t - .trim_start_matches("pub async fn ") - .trim_start_matches("pub(crate) fn ") - .trim_start_matches("pub fn ") - .trim_start_matches("fn "); - let name = part - .split('(') - .next() - .unwrap_or("") - .split('<') - .next() - .unwrap_or("") - .trim() - .to_lowercase(); - if !name.is_empty() { + if let Some(name) = fn_item_name(line) { found.insert(name); } } } + +/// The lowercased name of the item (`fn`, `struct`, `enum`, `type`, `trait`) a source +/// line declares, if it declares one. +fn fn_item_name(line: &str) -> Option { + let mut t = line.trim_start(); + // visibility: `pub` or `pub(...)` + if let Some(rest) = t.strip_prefix("pub") { + let rest_trim = rest.trim_start(); + if let Some(inner) = rest_trim.strip_prefix('(') { + t = inner.split_once(')')?.1.trim_start(); + } else if rest.starts_with(char::is_whitespace) { + t = rest_trim; + } else { + return None; // `pubfoo`, `pub_x`: an identifier, not a visibility + } + } + // qualifiers, in any order the grammar allows them to appear + loop { + let before = t; + for q in ["const ", "async ", "unsafe ", "default "] { + if let Some(rest) = t.strip_prefix(q) { + t = rest.trim_start(); + } + } + if let Some(rest) = t.strip_prefix("extern ") { + let rest = rest.trim_start(); + t = match rest.strip_prefix('"') { + Some(abi) => abi.split_once('"')?.1.trim_start(), + None => rest, + }; + } + if t == before { + break; + } + } + // A binding may name a function or a type (setfit-apr-v1 binds + // `SetFitArtifactDoc` and `ClassifyResponse`, both `pub struct`): the resolver + // sees every item kind a binding can name. + let part = ["fn ", "struct ", "enum ", "type ", "trait "] + .iter() + .find_map(|kw| t.strip_prefix(kw))?; + let name = part + .split(|c: char| matches!(c, '(' | '<' | ';' | '{' | ':' | '=') || c.is_whitespace()) + .next() + .unwrap_or("") + .trim() + .to_lowercase(); + (!name.is_empty()).then_some(name) +} + +#[cfg(test)] +mod tests { + use super::fn_item_name; + + /// The resolver's case table: every declaration form a binding can name must be + /// seen (a miss is a false GHOST reject), and non-declarations must not be. + #[test] + fn fn_item_name_case_table() { + let must_match = [ + ("fn plain() {}", "plain"), + ("pub fn public(x: u8) -> u8 {", "public"), + ("pub(crate) fn in_crate() {", "in_crate"), + ( + "pub(super) fn compute_mse(y_left: &[f32], y_right: &[f32]) -> f32 {", + "compute_mse", + ), + ("pub(in crate::tree) fn scoped() {", "scoped"), + ("pub async fn serve() {", "serve"), + ("pub(crate) async fn load() {", "load"), + ("pub const fn size() -> usize {", "size"), + ("pub unsafe fn raw() {", "raw"), + ("pub const unsafe fn both() {", "both"), + ("pub extern \"C\" fn ffi() {", "ffi"), + (" fn indented(t: T) {", "indented"), + ("fn Mixed_Case() {", "mixed_case"), + ("pub struct SetFitArtifactDoc {", "setfitartifactdoc"), + ("pub struct ClassifyResponse {", "classifyresponse"), + ("pub(crate) enum Mode {", "mode"), + ("pub type Alias = Vec;", "alias"), + ("pub trait Estimator {", "estimator"), + ("pub unsafe trait Marker {}", "marker"), + ("pub struct Fn;", "fn"), + ]; + for (line, want) in must_match { + assert_eq!(fn_item_name(line).as_deref(), Some(want), "{line}"); + } + let must_not_match = [ + "// fn commented_out() {}", + "let f = fn_pointer;", + "pubfn not_a_decl() {}", + "impl Fn for X {}", + "call(fn_like);", + "let structure = 3;", + "// struct Commented {}", + ]; + for line in must_not_match { + assert_eq!(fn_item_name(line), None, "{line}"); + } + } +} diff --git a/crates/aprender-contracts-cli/src/contract_walk.rs b/crates/aprender-contracts-cli/src/contract_walk.rs index 45dff87517..e58b85dbb2 100644 --- a/crates/aprender-contracts-cli/src/contract_walk.rs +++ b/crates/aprender-contracts-cli/src/contract_walk.rs @@ -193,6 +193,39 @@ impl fmt::Display for LintRejected { impl std::error::Error for LintRejected {} +/// `pv obligations --gate` found problems (PVL-001 EV-10): measured, and failed. Exit 1. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ObligationsRejected { + /// Problems reported. + pub problems: usize, + /// Contracts checked. + pub contracts: usize, +} + +impl fmt::Display for ObligationsRejected { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "obligation gate failed ({} problem(s) over {} contracts)", + self.problems, self.contracts + ) + } +} + +impl std::error::Error for ObligationsRejected {} +/// One `--gate` run measured and failed, and says WHAT failed: the line after `reject:` is the gate's own findings +/// (ONT-4e: `reject: A refines B: precondition strengthened (PRE-1)`), not a gate count. Exit 1. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GateRejected(pub String); + +impl fmt::Display for GateRejected { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for GateRejected {} + /// `pv lint --gate sigma` found Σ itself malformed (ONT-001 §5 ONT-2b): the DECLARATION is wrong, not the corpus, /// so it is `error:` at exit 3 and never `reject:`. #[derive(Debug, Clone, PartialEq, Eq)] @@ -246,7 +279,11 @@ pub const ARMED_GATES_SHRANK_EXIT: i32 = 3; /// empty corpus or a declined lint meet, [`ARMED_GATES_SHRANK_EXIT`] for a shrunk /// armed set, 1 for everything else (a parse failure and a rejected meet included). pub fn exit_code_for(err: &(dyn std::error::Error + 'static)) -> i32 { - if err.downcast_ref::().is_some() || err.downcast_ref::().is_some() + if err.downcast_ref::().is_some() + || err.downcast_ref::().is_some() + || err + .downcast_ref::() + .is_some() { ZERO_CONTRACTS_EXIT } else if err.downcast_ref::().is_some() @@ -267,11 +304,20 @@ pub fn exit_code_for(err: &(dyn std::error::Error + 'static)) -> i32 { /// ONT-1 asserts both halves of the line. #[must_use] pub fn verdict_for(err: &(dyn std::error::Error + 'static)) -> &'static str { - if err.downcast_ref::().is_some() || err.downcast_ref::().is_some() + if err.downcast_ref::().is_some() + || err.downcast_ref::().is_some() + || err + .downcast_ref::() + .is_some() { "decline" } else if err.downcast_ref::().is_some() || err.downcast_ref::().is_some() + || err.downcast_ref::().is_some() + || err.downcast_ref::().is_some() + || err + .downcast_ref::() + .is_some() { "reject" } else { diff --git a/crates/aprender-contracts-cli/src/lib.rs b/crates/aprender-contracts-cli/src/lib.rs index e4c514eb86..3a942e693d 100644 --- a/crates/aprender-contracts-cli/src/lib.rs +++ b/crates/aprender-contracts-cli/src/lib.rs @@ -22,11 +22,10 @@ use cli::Commands; /// Glance form, printed by `pv -V`. One line, and it names the tool. /// /// clap renders `{name} {version}`, so this yields -/// `pv 0.69.0 (aa7c6ef03) (aprender provable-contracts verifier)` — the build -/// SHA from `aprender-build-sha` (#4219), as every workspace binary prints. -/// The bare semver stays the SECOND whitespace field because -/// `scripts/pv_bin.sh` reads it positionally to prove a resolved binary was -/// built from HEAD. +/// `pv 0.70.0 (817d63361) (aprender provable-contracts verifier)`. The bare +/// semver stays the SECOND whitespace field because `scripts/pv_bin.sh` reads it +/// positionally; the sha (G0.1, #4476, from `build.rs`) is what tells two trees +/// at the same workspace version apart. const SHORT_VERSION: &str = concat!( env!("CARGO_PKG_VERSION"), " (", @@ -120,6 +119,8 @@ pub fn dispatch(command: Commands) -> Result<(), Box> { contract, binding, .. } => commands::audit::run(&contract, binding.as_deref()), Commands::Diff { old, new } => commands::diff::run(&old, &new), + Commands::Discharge { action } => commands::discharge::run(action), + Commands::Challenge { action } => commands::challenge::run(action), Commands::Census { contract_dir, format, @@ -132,13 +133,21 @@ pub fn dispatch(command: Commands) -> Result<(), Box> { contract_dir, check, out, + cells_out, release, } => { let subject = release .subject() .map_err(crate::contract_walk::ReleaseArgsRefused)?; - commands::extract_rdf::run(&contract_dir, check, subject.as_ref(), out.as_deref()) + commands::extract_rdf::run( + &contract_dir, + check, + subject.as_ref(), + out.as_deref(), + cells_out.as_deref(), + ) } + Commands::Ontology { command } => commands::ontology::run(&command), Commands::Coverage { contract_dir, binding, @@ -250,8 +259,12 @@ pub fn dispatch(command: Commands) -> Result<(), Box> { watch, strict_test_binding, armed_baseline_ref.as_deref(), - gate.as_deref(), - commands::lint::shapes_options(gate.as_deref(), shape, &release)?, + &gate, + commands::lint::shapes_options( + gate.iter().any(|g| g == "shapes").then_some("shapes"), + shape, + &release, + )?, ) } Commands::Score { @@ -332,6 +345,7 @@ pub fn dispatch(command: Commands) -> Result<(), Box> { contract_dir, top, } => commands::infer::run(&crate_dir, &binding, &contract_dir, top), + Commands::Obligations { root, gate } => commands::obligations::run(&root, gate), Commands::Unlock { contract, reason } => commands::unlock::run(&contract, &reason), Commands::Roofline { contract_dir, diff --git a/crates/aprender-contracts-cli/tests/ev11_lint_ratchets.rs b/crates/aprender-contracts-cli/tests/ev11_lint_ratchets.rs new file mode 100644 index 0000000000..6d76110cf4 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/ev11_lint_ratchets.rs @@ -0,0 +1,219 @@ +//! PVL-001 EV-11 (PMAT-4166) — `pv lint --gate theorem-pairing --gate depends-on-present`: two shrink-only +//! ratchets read from `contracts/lint-baseline.json`, which the gates never write. +//! +//! The row's probe, verbatim (paiml/infra PVL-001 @00553b0b): `json_object contracts/lint-baseline.json && jq -e +//! '(.unpaired_theorem_modules|numbers) and (.contracts_without_depends_on|numbers) and (.command|strings)' +//! contracts/lint-baseline.json && "$PV" lint contracts/ --gate theorem-pairing --gate depends-on-present`, and the +//! accept adds `git diff --exit-code contracts/lint-baseline.json` after it. The first test is that probe on the +//! real corpus; the rest run on a throwaway repo built in a tempdir, because the gates read the repo ROOT +//! (`lean/`, `book/`) as well as the contract dir. +//! +//! | case | expected | +//! |---|---| +//! | real corpus, both gates | exit 0, both Pass, the baseline byte-identical after | +//! | tempdir at the baseline | exit 0 | +//! | the spec's mutation: add an unpaired Theorem module | exit 1, PV-RAT-001 — the meet rejects | +//! | a kernel contract with no `depends_on` above the baseline | exit 1, PV-RAT-002 | +//! | no baseline key | exit 2, `Unknown(Report)` with the count printed — reported, never a pass | +//! | no Lean base | exit 2, decline naming what is missing | +//! | an unknown name among several | exit 1 before anything runs | + +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn pv_bin() -> PathBuf { + PathBuf::from(env!("CARGO_BIN_EXE_pv")) +} + +struct Run { + code: i32, + stdout: String, + stderr: String, +} + +fn pv(args: &[&str]) -> Run { + let scratch = tempfile::tempdir().expect("scratch cwd is creatable"); + let out = Command::new(pv_bin()) + .current_dir(scratch.path()) + .args(args) + .output() + .expect("failed to spawn pv"); + Run { + code: out.status.code().unwrap_or(-1), + stdout: String::from_utf8_lossy(&out.stdout).into_owned(), + stderr: String::from_utf8_lossy(&out.stderr).into_owned(), + } +} + +fn show(r: &Run) -> String { + format!( + "exit {}\n--- stdout\n{}\n--- stderr\n{}", + r.code, r.stdout, r.stderr + ) +} + +fn s(p: &Path) -> &str { + p.to_str().expect("utf-8 path") +} + +fn repo_contracts() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../contracts") +} + +fn write(root: &Path, rel: &str, text: &str) { + let p = root.join(rel); + std::fs::create_dir_all(p.parent().expect("has a parent")).expect("mkdir"); + std::fs::write(p, text).expect("write"); +} + +/// `pv lint --gate X` prints one JSON object per gate, pretty-printed and back to back. +fn reports(stdout: &str) -> Vec { + serde_json::Deserializer::from_str(stdout) + .into_iter::() + .map(|v| v.expect("each gate report is JSON")) + .collect() +} + +const BOTH: [&str; 4] = ["--gate", "theorem-pairing", "--gate", "depends-on-present"]; + +/// A repo with one paired theorem module and one kernel contract (the PVL-1 control, which has no +/// `depends_on`), at baselines 0 and 1. +fn repo() -> tempfile::TempDir { + let t = tempfile::tempdir().expect("tempdir"); + write( + t.path(), + "lean/ProvableContracts/Theorems/S/A.lean", + "theorem a : True := trivial\n", + ); + write( + t.path(), + "book/a.md", + "see ProvableContracts.Theorems.S.A\n", + ); + std::fs::create_dir_all(t.path().join("contracts")).expect("mkdir"); + std::fs::copy( + repo_contracts().join("softmax-kernel-v1.yaml"), + t.path().join("contracts/softmax-kernel-v1.yaml"), + ) + .expect("control contract copies"); + write( + t.path(), + "contracts/lint-baseline.json", + "{\n \"unpaired_theorem_modules\": 0,\n \"contracts_without_depends_on\": 1\n}\n", + ); + t +} + +fn lint(root: &Path, gates: &[&str]) -> Run { + let dir = root.join("contracts"); + let mut args = vec!["lint", s(&dir)]; + args.extend_from_slice(gates); + pv(&args) +} + +#[test] +fn the_probe_passes_on_the_real_corpus_and_writes_nothing() { + let baseline = repo_contracts().join("lint-baseline.json"); + let before = std::fs::read(&baseline).expect("baseline readable"); + let doc: serde_json::Value = serde_json::from_slice(&before).expect("baseline is JSON"); + assert!(doc.is_object()); + assert!(doc["unpaired_theorem_modules"].is_u64(), "{doc}"); + assert!(doc["contracts_without_depends_on"].is_u64(), "{doc}"); + assert_eq!(doc["command"], "make lint-ratchet"); + + let dir = repo_contracts(); + let mut args = vec!["lint", s(&dir)]; + args.extend_from_slice(&BOTH); + let r = pv(&args); + assert_eq!(r.code, 0, "{}", show(&r)); + let got = reports(&r.stdout); + assert_eq!(got.len(), 2, "{}", show(&r)); + for g in &got { + assert_eq!(g["verdict"], "Pass", "{g}"); + } + assert_eq!( + got[0]["unpaired_theorem_modules"], doc["unpaired_theorem_modules"], + "the recorded baseline is the measured count (`make lint-ratchet` wrote it)" + ); + assert_eq!( + got[1]["contracts_without_depends_on"], + doc["contracts_without_depends_on"] + ); + assert_eq!( + std::fs::read(&baseline).expect("baseline readable"), + before, + "a gate wrote the baseline" + ); +} + +#[test] +fn at_the_baseline_both_gates_pass() { + let t = repo(); + let r = lint(t.path(), &BOTH); + assert_eq!(r.code, 0, "{}", show(&r)); +} + +/// PVL-001 EV-11's mutation, verbatim: "add an unpaired Theorem module → RED". +#[test] +fn adding_an_unpaired_theorem_module_is_red() { + let t = repo(); + write( + t.path(), + "lean/ProvableContracts/Theorems/S/B.lean", + "theorem b : True := trivial\n", + ); + let r = lint(t.path(), &BOTH); + assert_eq!(r.code, 1, "{}", show(&r)); + assert!(r.stdout.contains("PV-RAT-001"), "{}", show(&r)); + assert!(!r.stdout.contains("PV-RAT-002"), "{}", show(&r)); + assert!( + r.stderr.contains("1/2 armed gates passed"), + "the meet names the one that held: {}", + show(&r) + ); +} + +#[test] +fn a_kernel_contract_without_depends_on_above_the_baseline_is_red() { + let t = repo(); + let text = std::fs::read_to_string(repo_contracts().join("softmax-kernel-v1.yaml")) + .expect("control contract readable"); + write(t.path(), "contracts/softmax-kernel-copy-v1.yaml", &text); + let r = lint(t.path(), &["--gate", "depends-on-present"]); + assert_eq!(r.code, 1, "{}", show(&r)); + assert!(r.stdout.contains("PV-RAT-002"), "{}", show(&r)); +} + +#[test] +fn no_baseline_reports_the_count_and_is_never_a_pass() { + let t = repo(); + write(t.path(), "contracts/lint-baseline.json", "{}\n"); + let r = lint(t.path(), &BOTH); + assert_eq!(r.code, 2, "{}", show(&r)); + let got = reports(&r.stdout); + assert_eq!(got.len(), 2, "{}", show(&r)); + for g in &got { + assert_eq!(g["verdict"], "Unknown(Report)", "{g}"); + } + // the count `make lint-ratchet` records as the first baseline + assert_eq!(got[0]["unpaired_theorem_modules"], 0); + assert_eq!(got[1]["contracts_without_depends_on"], 1); +} + +#[test] +fn no_lean_base_is_a_decline_naming_what_is_missing() { + let t = repo(); + std::fs::remove_dir_all(t.path().join("lean")).expect("rm lean"); + let r = lint(t.path(), &["--gate", "theorem-pairing"]); + assert_eq!(r.code, 2, "{}", show(&r)); + assert!(r.stderr.contains("no Lean theorem base"), "{}", show(&r)); +} + +#[test] +fn an_unknown_name_among_several_is_refused_before_anything_runs() { + let t = repo(); + let r = lint(t.path(), &["--gate", "theorem-pairing", "--gate", "bogus"]); + assert_eq!(r.code, 1, "{}", show(&r)); + assert!(r.stderr.contains("--gate bogus"), "{}", show(&r)); + assert!(r.stdout.is_empty(), "nothing ran: {}", show(&r)); +} diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/README.md b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/README.md new file mode 100644 index 0000000000..d12f1c1558 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/README.md @@ -0,0 +1,86 @@ +# PVL-001 EV-10 golden fixtures — `pv obligations --gate` vs `pv-obligation-gate.py` + +`tests/pvl_obligations_golden.rs` holds `pv obligations --gate` to `.stdout` +byte for byte and to `.rc`. Both files were written by the Python script it replaces, +never by pv. + +| fixture | what | script verdict | +|---|---|---| +| `pmat/` | pmat's 35 `contracts/*.yaml` + `binding.yaml`, verbatim, and the 10 `src/` files that `git grep -l -E "fn \b" -- src/` returns for the 5 function-valued `applies_to` (all in `tdg-grade-order-v1.yaml`, `proved_type: Grade`) | 0 problems over 35, exit 0 | +| `broken/` | one planted defect per check, plus a control for every rule that must stay silent | 6 problems over 4, exit 1 | +| `edge/` | YAML the script reads through PyYAML and Python truthiness: merge keys and falsy values | 3 problems over 4, exit 1 | + +## Provenance + +- pmat (`paiml/paiml-mcp-agent-toolkit`) at `d70a78f67` (2026-09-15). +- script `scripts/pv-obligation-gate.py`, sha256 + `e4fa971964f7d902d2491ef81eb667f8f1eee68a4ec411590b138c70dfad4e12` (last changed in `7c93535e`). +- PyYAML 6.0.3. The script's `pv validate` resolved to the pv built from this tree (first on `PATH`). + +## Why the `src/` files end in `.rs.txt` + +They are pmat's sources, and ten aprender gates scan every tracked `*.rs`. They were renamed +after copying and not otherwise changed. Neither side cares about the extension: the script's `git grep … -- src/` searches +every tracked file under `src/`, and so does pv. A name only reaches the output inside a check-3b +problem (`['src/gate.rs.txt']` in `broken.stdout`). + +## `broken/`, line by line + +- `a-invalid-v1.yaml`: no `metadata:`, so `pv validate` fails (check 1). +- `b-hidden-v1.yaml`: two `test:` entries under `falsification:` (check 2). The `action:`-only + entry is an alert threshold and is not counted. +- `c-unbound-v1.yaml`: `ghost_fn_xyz` names no `fn` (check 3a). `bounded` is only found as a + prefix of `fn bounded_v2`, and `\b` refuses that match. The controls are `all`, the equation + `relu`, `real_fn` in a nested file, and an obligation with no `applies_to`. +- `d-proved-v1.yaml`: `grade_gate`'s file only says `GradeTable`, and `grade_prefixed`'s file + only says `TdgGrade`. Neither says the word `Grade` (check 3b, one case for each side of + `\b`). The control is `grade_ok`, whose file does say `Grade`. +- Excluded, and never reported: + - `z-binding.yaml` (its name ends in `binding.yaml`); + - `.hidden-v1.yaml` (`glob`'s `*` skips dotfiles); + - `sub/nested-v1.yaml` (the walk is not recursive); + - `notes.yml` (not `.yaml`). + +## `edge/`, line by line + +- `e-merge-v1.yaml`: two entries under `falsification:` have no `test:` of their own. `F-1` + merges one in with `<<: *hidden`, and `F-2` merges it through `chained`, a mapping that + merges in turn. PyYAML resolves both, so the script counts 2 (check 2). `F-0` holds the + templates one level down and is not counted. +- `f-falsy-v1.yaml`: `proved_type: 0`, and `applies_to` set to `0`, `[]`, `{}`, `false` and `""`. + Python treats every one of them as absent, so the only problem is `pv validate` (check 1). + `real_fn` is bound and never checked against the falsy type. +- `g-unused-type-v1.yaml`: `proved_type: 7`, which no function target reaches, so the script + never evaluates it and reports nothing. +- `h-false-doc-v1.yaml`: the document is `false`. `yaml.safe_load(...) or {}` makes it `{}`, so + only `pv validate` fails. + +## Re-recording + +The fixture must be tracked (`git add`) first, because the script's `git grep` only sees +tracked files. + +```bash +cargo build -p aprender-contracts-cli --bin pv +S=~/src/paiml-mcp-agent-toolkit/scripts/pv-obligation-gate.py # sha256 as above +for fx in pmat broken edge; do + (cd crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/$fx && + PATH="$PWD/../../../../../../../target/debug:$PATH" python3 "$S" > ../$fx.stdout; echo $? > ../$fx.rc) +done +``` + +## Where pv deliberately differs + +These are documented in `src/commands/obligations.rs`. No fixture here exercises them, because the +script has no verdict to record: + +- zero contracts is `decline:` at exit 2 (PVL-1), where the script reports 0 over 0 at exit 0; +- an input the script dies on with a traceback is a named problem. The module doc lists each + one, and `a_non_string_proved_type_is_named_where_a_bound_fn_reaches_it` tests one; +- YAML 1.2 (`serde_yaml`) against PyYAML's YAML 1.1: a plain `no` is a string here; +- a document `serde_yaml` refuses and PyYAML reads (a duplicate key, an integer beyond 64 bits) + fails `pv validate` on both sides. pv reports only that, as the script does, but cannot run + checks 2 and 3 on it, so a problem PyYAML would also find there goes unnamed (the verdict + agrees). `a_document_serde_yaml_refuses_fails_validate_so_the_verdict_agrees` holds both + shapes to the script's measured output; +- `src/` is walked on disk, so untracked files count. diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken.rc b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken.rc new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken.rc @@ -0,0 +1 @@ +1 diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken.stdout b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken.stdout new file mode 100644 index 0000000000..7c03f01043 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken.stdout @@ -0,0 +1,7 @@ +::error::contracts/a-invalid-v1.yaml: pv validate failed +::error::contracts/b-hidden-v1.yaml: 2 test-bearing obligation(s) under `falsification:`, which pv cannot read — use `falsification_tests:` +::error::contracts/c-unbound-v1.yaml: applies_to 'ghost_fn_xyz' names neither an equation of this contract nor any `fn` under src/ +::error::contracts/c-unbound-v1.yaml: applies_to 'bounded' names neither an equation of this contract nor any `fn` under src/ +::error::contracts/d-proved-v1.yaml: applies_to 'grade_gate' is proved against 'Grade', but none of ['src/gate.rs.txt'] mentions it — the proof does not reach the code it names +::error::contracts/d-proved-v1.yaml: applies_to 'grade_prefixed' is proved against 'Grade', but none of ['src/prefixed.rs.txt'] mentions it — the proof does not reach the code it names +pv obligation gate: 6 problem(s) over 4 contracts diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/.hidden-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/.hidden-v1.yaml new file mode 100644 index 0000000000..1363bfc814 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/.hidden-v1.yaml @@ -0,0 +1 @@ +not a contract diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/a-invalid-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/a-invalid-v1.yaml new file mode 100644 index 0000000000..266414ebf4 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/a-invalid-v1.yaml @@ -0,0 +1,8 @@ +# CHECK 1: no `metadata:` — `pv validate` fails. The script and pv both report it. +equations: + id: + formula: "y = x" +proof_obligations: + - type: invariant + property: "identity" + applies_to: all diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/b-hidden-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/b-hidden-v1.yaml new file mode 100644 index 0000000000..26ea07fe41 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/b-hidden-v1.yaml @@ -0,0 +1,29 @@ +metadata: + version: "1.0.0" + kind: pattern + references: + - "PVL-001 EV-10 golden fixture" + description: "CHECK 2: two test-bearing entries hide under falsification:; the action-only entry is an alert threshold and is not counted" + +equations: + relu: + formula: "y_i = max(0, x_i)" + invariants: + - "y_i >= 0" + +proof_obligations: + - type: invariant + property: "Non-negativity" + applies_to: all +falsification: + - id: HIDDEN-1 + test: "relu_nonneg" + - id: HIDDEN-2 + test: "relu_idempotent" + - id: ALERT-1 + action: "page on-call" +falsification_tests: + - id: FALSIFY-OBL-001 + rule: "Non-negativity" + prediction: "relu(x)_i >= 0" + if_fails: "Comparison inverted" diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/c-unbound-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/c-unbound-v1.yaml new file mode 100644 index 0000000000..9489752389 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/c-unbound-v1.yaml @@ -0,0 +1,36 @@ +metadata: + version: "1.0.0" + kind: pattern + references: + - "PVL-001 EV-10 golden fixture" + description: "CHECK 3a: an applies_to that names no fn; controls: all, an equation, a real fn, a missing/empty target" + +equations: + relu: + formula: "y_i = max(0, x_i)" + invariants: + - "y_i >= 0" + +proof_obligations: + - type: invariant + property: "ghost" + applies_to: ghost_fn_xyz + - type: invariant + property: "prefix only: src has fn bounded_v2, not fn bounded" + applies_to: bounded + - type: invariant + property: "control: all" + applies_to: all + - type: invariant + property: "control: an equation of this contract" + applies_to: relu + - type: invariant + property: "control: a real fn, found in a nested file" + applies_to: real_fn + - type: invariant + property: "control: no applies_to" +falsification_tests: + - id: FALSIFY-OBL-001 + rule: "Non-negativity" + prediction: "relu(x)_i >= 0" + if_fails: "Comparison inverted" diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/d-proved-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/d-proved-v1.yaml new file mode 100644 index 0000000000..e2562672a5 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/d-proved-v1.yaml @@ -0,0 +1,28 @@ +metadata: + version: "1.0.0" + kind: pattern + references: + - "PVL-001 EV-10 golden fixture" + description: "CHECK 3b: proved_type Grade; grade_gate s file only says GradeTable and grade_prefixed s only TdgGrade; grade_ok s says Grade" + proved_type: Grade +equations: + relu: + formula: "y_i = max(0, x_i)" + invariants: + - "y_i >= 0" + +proof_obligations: + - type: invariant + property: "gate is proved against Grade" + applies_to: grade_gate + - type: invariant + property: "prefixed is proved against Grade, but its file only says TdgGrade" + applies_to: grade_prefixed + - type: invariant + property: "control: reaches Grade" + applies_to: grade_ok +falsification_tests: + - id: FALSIFY-OBL-001 + rule: "Non-negativity" + prediction: "relu(x)_i >= 0" + if_fails: "Comparison inverted" diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/notes.yml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/notes.yml new file mode 100644 index 0000000000..1363bfc814 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/notes.yml @@ -0,0 +1 @@ +not a contract diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/sub/nested-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/sub/nested-v1.yaml new file mode 100644 index 0000000000..1363bfc814 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/sub/nested-v1.yaml @@ -0,0 +1 @@ +not a contract diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/z-binding.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/z-binding.yaml new file mode 100644 index 0000000000..49a6a7df4a --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/contracts/z-binding.yaml @@ -0,0 +1 @@ +not: [a, contract diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/src/deep/real.rs.txt b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/src/deep/real.rs.txt new file mode 100644 index 0000000000..2e03ada5a4 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/src/deep/real.rs.txt @@ -0,0 +1,3 @@ +pub fn real_fn() {} + +fn bounded_v2() {} diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/src/gate.rs.txt b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/src/gate.rs.txt new file mode 100644 index 0000000000..755c25730f --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/src/gate.rs.txt @@ -0,0 +1,4 @@ +// The type is only ever named as part of a longer word here. +pub fn grade_gate(t: &GradeTable) -> bool { + t.is_empty() +} diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/src/ok.rs.txt b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/src/ok.rs.txt new file mode 100644 index 0000000000..05b44998dd --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/src/ok.rs.txt @@ -0,0 +1,3 @@ +pub fn grade_ok(g: Grade) -> bool { + matches!(g, Grade::A) +} diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/src/prefixed.rs.txt b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/src/prefixed.rs.txt new file mode 100644 index 0000000000..c982ac7b15 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/broken/src/prefixed.rs.txt @@ -0,0 +1,4 @@ +// The type is only ever named as the tail of a longer word here. +pub fn grade_prefixed(g: &TdgGrade) -> bool { + g.is_passing() +} diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge.rc b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge.rc new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge.rc @@ -0,0 +1 @@ +1 diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge.stdout b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge.stdout new file mode 100644 index 0000000000..41ec5d3647 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge.stdout @@ -0,0 +1,4 @@ +::error::contracts/e-merge-v1.yaml: 2 test-bearing obligation(s) under `falsification:`, which pv cannot read — use `falsification_tests:` +::error::contracts/f-falsy-v1.yaml: pv validate failed +::error::contracts/h-false-doc-v1.yaml: pv validate failed +pv obligation gate: 3 problem(s) over 4 contracts diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge/contracts/e-merge-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge/contracts/e-merge-v1.yaml new file mode 100644 index 0000000000..d306a8de79 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge/contracts/e-merge-v1.yaml @@ -0,0 +1,33 @@ +metadata: + version: "1.0.0" + kind: pattern + references: + - "PVL-001 EV-10 golden fixture" + description: "CHECK 2 through a YAML merge key: F-1 merges a test from F-0's template, F-2 merges it through a merge that itself merges; the script (PyYAML) counts both, F-0 is not counted" +equations: + relu: + formula: "y_i = max(0, x_i)" + invariants: + - "y_i >= 0" + +proof_obligations: + - type: invariant + property: "Non-negativity" + applies_to: all +falsification: + - id: F-0 + action: "page on-call" + template: &hidden + test: "relu_nonneg" + chained: &chained + <<: *hidden + note: "merges the template in turn" + - <<: *hidden + id: F-1 + - <<: *chained + id: F-2 +falsification_tests: + - id: FALSIFY-OBL-001 + rule: "Non-negativity" + prediction: "relu(x)_i >= 0" + if_fails: "Comparison inverted" diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge/contracts/f-falsy-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge/contracts/f-falsy-v1.yaml new file mode 100644 index 0000000000..f5ebee3833 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge/contracts/f-falsy-v1.yaml @@ -0,0 +1,38 @@ +metadata: + version: "1.0.0" + kind: pattern + references: + - "PVL-001 EV-10 golden fixture" + description: "Python falsiness: a falsy proved_type is no proved_type, and falsy applies_to values are skipped like an absent one; real_fn is bound and never checked against a type" + proved_type: 0 +equations: + relu: + formula: "y_i = max(0, x_i)" + invariants: + - "y_i >= 0" + +proof_obligations: + - type: invariant + property: "zero" + applies_to: 0 + - type: invariant + property: "empty list" + applies_to: [] + - type: invariant + property: "empty map" + applies_to: {} + - type: invariant + property: "false" + applies_to: false + - type: invariant + property: "empty string" + applies_to: "" + - type: invariant + property: "bound, and not checked against the falsy type" + applies_to: real_fn +falsification: [] +falsification_tests: + - id: FALSIFY-OBL-001 + rule: "Non-negativity" + prediction: "relu(x)_i >= 0" + if_fails: "Comparison inverted" diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge/contracts/g-unused-type-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge/contracts/g-unused-type-v1.yaml new file mode 100644 index 0000000000..95eee374b9 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge/contracts/g-unused-type-v1.yaml @@ -0,0 +1,25 @@ +metadata: + version: "1.0.0" + kind: pattern + references: + - "PVL-001 EV-10 golden fixture" + description: "A truthy non-string proved_type that no function target ever reaches: the script never evaluates it" + proved_type: 7 +equations: + relu: + formula: "y_i = max(0, x_i)" + invariants: + - "y_i >= 0" + +proof_obligations: + - type: invariant + property: "equation, not a function" + applies_to: relu + - type: invariant + property: "everything" + applies_to: all +falsification_tests: + - id: FALSIFY-OBL-001 + rule: "Non-negativity" + prediction: "relu(x)_i >= 0" + if_fails: "Comparison inverted" diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge/contracts/h-false-doc-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge/contracts/h-false-doc-v1.yaml new file mode 100644 index 0000000000..c508d5366f --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge/contracts/h-false-doc-v1.yaml @@ -0,0 +1 @@ +false diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge/src/real.rs.txt b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge/src/real.rs.txt new file mode 100644 index 0000000000..3ea2e60028 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/edge/src/real.rs.txt @@ -0,0 +1 @@ +pub fn real_fn() {} diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat.rc b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat.rc new file mode 100644 index 0000000000..573541ac97 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat.rc @@ -0,0 +1 @@ +0 diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat.stdout b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat.stdout new file mode 100644 index 0000000000..a5e76d2a4c --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat.stdout @@ -0,0 +1 @@ +pv obligation gate: 0 problem(s) over 35 contracts diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/benchmarking-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/benchmarking-v1.yaml new file mode 100644 index 0000000000..1a525015a0 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/benchmarking-v1.yaml @@ -0,0 +1,64 @@ +metadata: + version: "1.0.0" + created: "2026-04-08" + author: PAIML Engineering + references: + - "benches/coverage_gaps_bench.rs (criterion harness these budgets gate)" + - ".github/workflows/nightly-bench.yml (where the budgets are enforced)" + - "docs/specifications/components/repo-health.md (Build Performance category, 15pts)" + registry: true + description: > + Benchmarking and profiling contract — performance budgets for all + critical pmat operations. Regression detection via recorded baselines. + contract: benchmarking + status: enforced + +equations: + query_latency: + formula: latency = time(pmat query "q" --limit N) + domain: q in String, N in {1..100} + codomain: latency in Duration + invariants: + - warm_latency < 500ms for semantic/regex/literal queries + - cold_latency < 5s for local-only index load + preconditions: + - .pmat/context.db exists (index pre-built) + lean_theorem: Theorems.Query_Latency_Bounded + + index_build: + formula: duration = time(build_index(project_path)) + domain: project_path in Path + codomain: duration in Duration + invariants: + - duration < 60s for projects with <5000 .rs files + - incremental_duration < 5s when <10 files changed + preconditions: + - project_path.join("Cargo.toml").exists() + + scoring_latency: + formula: latency = time(pmat rust-project-score) + domain: project_path in Path + codomain: latency in Duration + invariants: + - fast_mode < 5s + - full_mode < 300s + preconditions: + - project_path.join("Cargo.toml").exists() + +preconditions: + - "pmat binary installed" + - ".pmat/context.db exists" + +falsification: + - condition: "pmat query 'test' --limit 1 takes >500ms warm" + severity: P1 + action: investigate_regression + - condition: "pmat rust-project-score takes >5s in fast mode" + severity: P1 + action: investigate_regression + - condition: "pmat comply check takes >30s" + severity: P2 + action: investigate_regression + - condition: "Index build >60s for <5000 files" + severity: P2 + action: optimize_indexer diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/binding.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/binding.yaml new file mode 100644 index 0000000000..c7a90ad95b --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/binding.yaml @@ -0,0 +1,308 @@ +version: 1.0.0 +target_crate: pmat +bindings: +- contract: pmat-core.yaml + equation: refresh_bindings + source_file: src/cli/handlers/comply_handlers/check_handlers/check_commit_enforcement.rs + function: handle_refresh_bindings + status: implemented +- contract: pmat-core.yaml + equation: check_compliance + source_file: src/cli/handlers/comply_handlers/check_handlers/check.rs + function: handle_check + status: implemented +- contract: pmat-core.yaml + equation: check_compliance + source_file: src/services/contract_index.rs + function: ContractIndex::load + status: implemented +- contract: pmat-core.yaml + equation: check_compliance + source_file: src/services/asset_validator.rs + function: validate_all_assets + status: implemented +- contract: pmat-core.yaml + equation: score_range + source_file: src/cli/handlers/score_handler.rs + function: handle_score + status: implemented +- contract: pmat-core.yaml + equation: score_range + source_file: src/cli/handlers/score_handler.rs + function: compute_composite + status: implemented +- contract: pmat-core.yaml + equation: score_range + source_file: src/services/rust_project_score/orchestrator.rs + function: score_with_mode + status: implemented +- contract: pmat-core.yaml + equation: score_range + source_file: src/cli/handlers/rust_project_score_handlers.rs + function: handle_rust_project_score + status: implemented +- contract: pmat-core.yaml + equation: score_range + source_file: src/services/popper_score/orchestrator.rs + function: score + status: implemented +- contract: pmat-core.yaml + equation: score_range + source_file: src/cli/handlers/popper_score_handlers.rs + function: handle_popper_score + status: implemented +- contract: pmat-core.yaml + equation: score_range + source_file: src/cli/handlers/work_contract_scoring.rs + function: score_contract + status: implemented +- contract: pmat-core.yaml + equation: path_exists + source_file: src/cli/handlers/query_handler/query_execution.rs + function: handle_query + status: implemented +- contract: pmat-core.yaml + equation: path_exists + source_file: src/cli/handlers/advanced_analysis_handlers.rs + function: handle_analyze_tdg + status: implemented +- contract: pmat-core.yaml + equation: path_exists + source_file: src/cli/handlers/dead_code_handlers.rs + function: handle_analyze_dead_code + status: implemented +- contract: pmat-core.yaml + equation: path_exists + source_file: src/cli/handlers/comprehensive_handler.rs + function: handle_analyze_comprehensive + status: implemented +- contract: pmat-core.yaml + equation: path_exists + source_file: src/cli/handlers/five_whys_handlers.rs + function: handle_debug + status: implemented +- contract: pmat-core.yaml + equation: path_exists + source_file: src/cli/handlers/kaizen_handler/mod.rs + function: handle_kaizen + status: implemented +- contract: pmat-core.yaml + equation: path_exists + source_file: src/cli/handlers/demo_handlers.rs + function: handle_quality_gate + status: implemented +- contract: pmat-core.yaml + equation: path_exists + source_file: src/cli/handlers/bottleneck_handler.rs + function: handle_bottleneck + status: implemented +- contract: pmat-core.yaml + equation: path_exists + source_file: src/cli/handlers/infra_score_handlers.rs + function: handle_infra_score + status: implemented +- contract: pmat-core.yaml + equation: path_exists + source_file: src/cli/handlers/repo_score_handlers.rs + function: handle_repo_score + status: implemented +- contract: pmat-core.yaml + equation: path_exists + source_file: src/cli/handlers/brick_score_handlers.rs + function: handle_brick_score + status: implemented +- contract: pmat-core.yaml + equation: path_exists + source_file: src/cli/handlers/demo_score_handlers.rs + function: handle_demo_score + status: implemented +- contract: pmat-core.yaml + equation: lint_valid + source_file: src/cli/handlers/work_contract_lint.rs + function: lint_contract + status: implemented +- contract: pmat-core.yaml + equation: non_empty_index + source_file: src/services/agent_context/function_index/build_persistence.rs + function: save + status: implemented +- contract: macs-provenance-v1.yaml + equation: provenance_roundtrip + source_file: src/cli/handlers/work_ledger_types.rs + function: AgentProvenance + status: pending +- contract: macs-provenance-v1.yaml + equation: hash_stability + source_file: src/cli/handlers/work_ledger_receipt.rs + function: FalsificationReceipt::compute_content_hash + status: pending +- contract: macs-provenance-v1.yaml + equation: refusal_gates_completion + source_file: src/cli/handlers/work_handlers/core_handlers/handlers.rs + function: check_unacked_refusals + status: pending +- contract: macs-ladder-v1.yaml + equation: parse_total_strict + source_file: src/cli/handlers/work_verification_level.rs + function: VerificationLevel::parse_strict + status: pending +- contract: macs-ladder-v1.yaml + equation: gate_monotone + source_file: src/quality/ladder_evidence.rs + function: achieved_level + status: pending +- contract: macs-cot-v1.yaml + equation: chain_integrity + source_file: src/models/work_cot.rs + function: check_chain + status: pending +- contract: macs-cot-v1.yaml + equation: derivation_complete + source_file: src/models/work_cot.rs + function: derive + status: pending +- contract: macs-sweep-v1.yaml + equation: sweep_deterministic + source_file: src/cli/handlers/qa_mcp_sweep.rs + function: handle_mcp_sweep + status: pending +- contract: macs-artifacts-v1.yaml + equation: manifest_faithful + source_file: src/mcp_pmcp/tool_manifest.rs + function: render_manifest + status: pending +- contract: macs-artifacts-v1.yaml + equation: roadmap_canonical + source_file: src/roadmap/sync.rs + function: render_roadmap + status: pending +- contract: macs-skill-effort-v1.yaml + equation: skill_effort_pinned + source_file: src/cli/handlers/comply_handlers/check_handlers/check_macs_skill_effort.rs + function: check_skill_effort_pinned + status: pending +- contract: macs-ladder-kernel-v1.yaml + equation: parse_strict + source_file: src/cli/handlers/work_verification_level.rs + function: VerificationLevel::parse_strict + status: implemented +- contract: macs-ladder-kernel-v1.yaml + equation: ord_monotone + source_file: src/cli/handlers/work_verification_level.rs + function: VerificationLevel::as_str + status: implemented +- contract: pmat-quality-acceptance-v1.yaml + equation: quality_acceptance + source_file: src/cli/handlers/work_quality_handlers.rs + function: run_popper_falsification + status: implemented +# --- pmat-no-fabrication-v1: every function fixed in the anti-fabrication sweep +# binds to the equation it now satisfies. A binding here is the claim that this +# function no longer reports a value it did not measure; the equation's +# falsification_tests are what make that claim refutable. +- contract: pmat-no-fabrication-v1.yaml + equation: measured_or_absent + source_file: src/services/deep_context/analyzer_core/quality.rs + function: DeepContextAnalyzer::calculate_quality_scorecard + status: implemented +- contract: pmat-no-fabrication-v1.yaml + equation: measured_or_absent + source_file: src/services/deep_context/deep_context_quality_types.rs + function: QualityScorecard::render + status: implemented +- contract: pmat-no-fabrication-v1.yaml + equation: output_derived_from_input + source_file: src/services/deep_context/analyzer_core/quality.rs + function: DeepContextAnalyzer::measured_line_coverage + status: implemented +- contract: pmat-no-fabrication-v1.yaml + equation: detection_mode_superset + source_file: src/cli/analysis/duplicates_extraction.rs + function: extract_blocks + status: implemented +- contract: pmat-no-fabrication-v1.yaml + equation: measured_or_absent + source_file: src/cli/analysis/duplicates_output.rs + function: format_json_output + status: implemented +- contract: pmat-no-fabrication-v1.yaml + equation: bounded_time_arithmetic + source_file: src/services/git_analysis.rs + function: GitAnalysisService::analyze_code_churn + status: implemented +- contract: pmat-no-fabrication-v1.yaml + equation: session_survives_recoverable_frame + source_file: src/mcp_pmcp/simple_unified_server.rs + function: EofSignalingTransport::is_session_over + status: implemented +- contract: pmat-no-fabrication-v1.yaml + equation: session_survives_recoverable_frame + source_file: src/mcp_pmcp/simple_unified_server.rs + function: EofSignalingTransport::receive + status: implemented +- contract: comply-gate-effect-v1.yaml + equation: gate_effect + source_file: src/cli/handlers/comply_handlers/check_handlers/check_gate_effect.rs + function: check_comply_gate_effect + status: implemented +- contract: comply-gate-effect-v1.yaml + equation: reachable + source_file: src/services/gate_effect/kernel.rs + function: reachable + status: implemented +- contract: comply-gate-effect-v1.yaml + equation: select_by_context + source_file: src/services/gate_effect/kernel.rs + function: select_by_context + status: implemented +- contract: comply-gate-effect-v1.yaml + equation: gates + source_file: src/services/gate_effect/kernel.rs + function: gates + status: implemented +- contract: comply-gate-effect-v1.yaml + equation: context_string_resolution + source_file: src/services/gate_effect/resolve.rs + function: resolve_context + status: implemented +- contract: comply-gate-effect-v1.yaml + equation: failure_propagation + source_file: src/services/gate_effect/effect.rs + function: assess + status: implemented +- contract: comply-gate-effect-v1.yaml + equation: enforcement_ledger + source_file: src/services/gate_effect/ledger.rs + function: render + status: implemented +# CB-2102 (ratchet) — the module was annotated with #[contract(...)] since it was +# written and never registered here, so `AllImplemented` had nothing to check and +# the `#[contract]` macro emitted no `=bound` env var for it. Registered now +# alongside CB-2101 rather than left as a second silent gap in the pair of rules +# that exist to find silent gaps. +- contract: comply-ratchet-v1.yaml + equation: verdict + source_file: src/services/metrics_ratchet/kernel.rs + function: ratchet_verdict + status: implemented +- contract: comply-ratchet-v1.yaml + equation: next + source_file: src/services/metrics_ratchet/kernel.rs + function: next_baseline + status: implemented +- contract: comply-ratchet-v1.yaml + equation: ratchet + source_file: src/cli/handlers/comply_handlers/check_handlers/check_metrics_ratchet.rs + function: check_metrics_ratchet + status: implemented +# CB-2101 (threshold coherence). +- contract: comply-threshold-coherence-v1.yaml + equation: classify + source_file: src/services/metrics_ratchet/kernel.rs + function: classify + status: implemented +- contract: comply-threshold-coherence-v1.yaml + equation: audit + source_file: src/cli/handlers/comply_handlers/check_handlers/check_threshold_coherence.rs + function: check_threshold_coherence + status: implemented diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/cli-usage-lines-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/cli-usage-lines-v1.yaml new file mode 100644 index 0000000000..cdf681aa2e --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/cli-usage-lines-v1.yaml @@ -0,0 +1,71 @@ +metadata: + version: "1.0.0" + created: "2026-09-02" + author: PAIML Engineering + references: + - "docs/specifications/pmat-architecture-crux-audit.md section 8.5 (CRUX-05, issue 1148, ticket PMAT-634)" + - "Popper (1959) The Logic of Scientific Discovery" + registry: true + description: > + Every pmat subcommand prints a usable Usage line, every CLI error names the + offending token, and no guard in the tree may certify a usage section by the + bare substring "Usage:". Cargo.toml built clap with default-features = false + and without usage/error-context/suggestions; 70 of 71 subcommands printed an + empty "Usage: " line and five separate guards passed on it. + contract: cli-usage-lines + status: draft + +equations: + usage_line_present: + formula: "for every subcommand c, help(c) contains a line matching ^Usage:[ \\t]+\\S" + domain: "pmat --help and pmat --help for every advertised c" + codomain: bool + invariants: + - "the line names the binary and the full command path, not a placeholder alone" + - "generated by clap's usage feature, never by Command::override_usage or a hand-rolled printer" + preconditions: [] + + error_names_token: + formula: "for an unknown flag f, stderr(pmat ... f) contains f" + domain: "any subcommand invoked with an unrecognised flag or a misspelt subcommand" + codomain: bool + invariants: + - "a near-miss subcommand is suggested (clap suggestions)" + - "the error carries clap's 'For more information, try' footer (error-context)" + preconditions: [] + + no_blind_usage_guard: + formula: "count of contains(\"Usage:\") predicates in src/ and tests/ = 0" + domain: "src/**, tests/**" + codomain: bool + invariants: + - "a guard asserts CONTENT after the heading, so an empty heading cannot satisfy it" + preconditions: [] + +falsification: + - condition: "A subcommand's --help prints an empty Usage: heading with green CI (CRUX-05, issue 1148)" + severity: P0 + action: reject_push + - condition: "A CLI error does not name the token it rejected" + severity: P1 + action: reject_push + - condition: "A bare contains(\"Usage:\") predicate returns to src/ or tests/" + severity: P1 + action: reject_push + +falsification_tests: + - id: usage_line_present + rule: "an empty Usage: heading is not a usage section" + prediction: "help text 'Usage: \\n\\nOptions:' fails has_usage_section; 'Usage: pmat analyze complexity [OPTIONS]' passes" + test: "cargo test --lib an_empty_usage_heading_is_not_a_usage_section" + if_fails: "pmat's own docs checker certifies the defect it exists to catch" + - id: error_names_token + rule: "the error for --zzz-nope contains --zzz-nope, and a near-miss subcommand is suggested" + prediction: "legs 3 and 4 of the spec section 8.5 script fail on the pre-fix binary and pass after" + test: "bash scripts/cli-usage-audit.sh (the spec section 8.5 acceptance script) against the release binary" + if_fails: "an agent cannot tell which of its arguments was wrong" + - id: no_blind_usage_guard + rule: "no bare contains(\"Usage:\") predicate in src/ or tests/" + prediction: "grep -rn 'contains(\"Usage:\")' src/ tests/ | wc -l == 0" + test: "grep -rn 'contains(\"Usage:\")' src/ tests/" + if_fails: "the regression returns the way it arrived, past five green guards" diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/commit-enforcement-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/commit-enforcement-v1.yaml new file mode 100644 index 0000000000..88236c2694 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/commit-enforcement-v1.yaml @@ -0,0 +1,60 @@ +metadata: + version: "1.0.0" + created: "2026-09-04" + author: PAIML Engineering + references: + - "docs/specifications/agentic-delivery-pmat.md section 4.7 / 9.3 (AD-03)" + - "issue 1126 — the generated hook's SATD and task-ID checks warn and exit 0" + - "Popper (1959) The Logic of Scientific Discovery" + registry: true + description: > + The hooks pmat installs must be able to refuse a commit. Under [hooks] + strict = true (or pmat hooks install --strict) the pre-commit SATD check + exits 1 over the threshold, and a generated commit-msg hook refuses a + message that carries neither a Pmat-Ticket trailer nor a ticket or issue + reference matching [hooks] ticket_pattern. Non-strict keeps warning so an + upgrade cannot lock a repository out. The trailer is the machine-readable + link between a commit and its work: git log --format=%(trailers:key=Pmat-Ticket,valueonly). + contract: commit-enforcement + status: draft + +equations: + strict_refuses_unlinked_commits: + formula: "strict and not (has_trailer(msg) or matches(msg, ticket_pattern)) => commit-msg exits 1 naming Pmat-Ticket and no commit is created" + domain: "any git commit through a repository where pmat hooks install --strict ran" + codomain: bool + invariants: + - "a message with the trailer is accepted and git reads the trailer back" + - "a message with an issue reference (#NNN) is accepted" + - "non-strict warns (stderr names Pmat-Ticket) and accepts" + - "a pre-commit hook receives no message, so the ticket check lives in commit-msg only" + - "hooks verify --fix, hooks update and the comply auto-install re-generate the hook with the strictness it was installed with (installed_strict reads PMAT_HOOKS_STRICT=1 back)" + preconditions: [] + + strict_satd_is_a_refusal: + formula: "strict and satd_count > PMAT_MAX_SATD_COMMENTS => pre-commit exits 1 before the success banner" + domain: "the generated pre-commit hook" + codomain: bool + invariants: + - "PMAT_HOOKS_STRICT is exported into the pre-commit hook from --strict OR [hooks] strict, resolved once for both hooks; the ticket pattern lives in the commit-msg hook only (the pre-commit export was dead — nothing read it)" + preconditions: [] + +falsification: + - condition: "under strict, a commit without a trailer or ticket reference is created (AD-03)" + severity: P0 + action: reject_push + - condition: "non-strict refuses a commit, or strict refuses a commit that carries the trailer" + severity: P0 + action: reject_push + +falsification_tests: + - id: strict_refuses_unlinked_commits + rule: "legs 1-4 of scripts/commit-enforcement-audit.sh; of the commit_enforcement_tests, four drive git commit in a temp repository and the rest assert the generated hook text, the strict plumbing and the --strict/--stack refusal" + prediction: "pre-fix binary: FAIL at leg 1 (no --strict, no commit-msg hook); fixed binary: PASS" + test: "PMAT= bash scripts/commit-enforcement-audit.sh" + if_fails: "ticket linking stays a habit and #1126's warning-that-passes returns" + - id: strict_satd_is_a_refusal + rule: "the generated hook text carries the strict SATD refusal branch" + prediction: "the_pre_commit_satd_block_exits_1_under_strict passes" + test: "cargo test --lib the_pre_commit_satd_block_exits_1_under_strict" + if_fails: "SATD over the threshold commits under a green banner" diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/comply-gate-effect-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/comply-gate-effect-v1.yaml new file mode 100644 index 0000000000..60b6be77fd --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/comply-gate-effect-v1.yaml @@ -0,0 +1,410 @@ +metadata: + version: "2.0.0" + kind: kernel + created: "2026-08-18" + author: PAIML Engineering + registry: true + description: > + Gate-effect verification (CB-2100). pmat enforces "gates or theatre" across + the fleet and, until this rule, had nothing that checked whether its own + rules block anything. Every "enforced in CI" claim rests on one chain: a + REQUIRED STATUS CHECK CONTEXT STRING -> the job that reports that context -> + a step (or a Makefile/script one hop away) that invokes the rule -> whose + non-zero exit can still fail the job. Break any link and the rule is + decoration. + + The roots are the UNION of the repository's required contexts, taken from + branch protection. Nothing in the rule may name a gate: a rule that + hardcodes `gate` stops checking the day a repository renames a job, which + is the failure INV-2100-3 already describes, self-inflicted. + + Rule id: CB-2100. Every id the source backlog proposed collides with a live + rule (CB-1400 "Agent Contract Existence", CB-1403 "Assume-Guarantee Chain", + CB-1404 "Agent Comply Usage", CB-1211 "Codegen Fidelity", CB-1300 "CLI Arg + Contracts", CB-1302 "MCP Schema Contracts", CB-1305 "Contract Surface + Classification"), and reusing one would have deleted it. CB-21xx was + audited free across src/, contracts/, docs/, .github/, the work list and + the whole git history. v1 of this contract shipped the same engine as + CB-1411 with three invariants; v2 renames it to CB-2100 and adds four. + contract: comply-gate-effect + status: enforced + verification: + kani_status: not_discharged + measured_at: "pmat 3.32.0, kani 0.67.0, 2026-08-18" + detail: > + The three `kani_harness` names below are REAL harnesses — `#[kani::proof]` + functions at the bottom of src/services/gate_effect/kernel.rs, held to + their names by + tests_cb2100.rs::every_kani_harness_the_contract_names_exists. They are + NOT discharged: nothing in this repository has ever run them, and with + kani installed they cannot be run at all. Measured, not assumed: + + (1) `cargo kani --harness verify_gates_only_on_exit_code` fails with + "error: target `pmat-agent` in package `pmat` requires the features: + `mcp-integration`" — kani enumerates every target in the package, and one + of them does not build under default features. + + (2) `cargo kani --harness verify_gates_only_on_exit_code --features + mcp-integration` fails with "error: rustc 1.93.0-nightly is not supported + by the following package: sysinfo@0.39.6 requires rustc 1.95" — kani + 0.67.0 pins a nightly older than a transitive dependency's MSRV. + + (3) No .github/workflows job, Makefile target or script invokes kani, so + not one of the repository's `#[kani::proof]` harnesses is compiled by any + gate (count them with `grep -rn '#\[kani::proof\]' src/ | wc -l`; the + number moves, the fact that nothing runs them does not). `#[cfg(kani)]` code is not built by `cargo test` either. + + These harnesses are therefore ASPIRATIONAL: they state the intended + theorems and are the specification a future proof will discharge, but + `kind: kernel` here must not be read as "proved". Naming that is the + honest alternative to deleting the harnesses or letting the reader infer + a proof that never ran — a contract that names a proof which does not + discharge is exactly the defect class this backlog exists to rule on. + + To make them real: kernel.rs has no dependencies beyond the contract + attribute macro, so lifting it into its own workspace crate would give + kani a dependency graph it can build. That is a separate work item and + has not been done. + references: + - src/services/gate_effect/kernel.rs + - src/services/gate_effect/mod.rs + - src/services/gate_effect/graph.rs + - src/services/gate_effect/resolve.rs + - src/services/gate_effect/effect.rs + - src/services/gate_effect/reach.rs + - src/services/gate_effect/required.rs + - src/services/gate_effect/roster.rs + - src/services/gate_effect/ledger.rs + - src/cli/handlers/comply_handlers/check_handlers/check_gate_effect.rs + - docs/status/comply-enforcement-ledger.md + +equations: + reachable: + formula: > + reachable(G, R, n) = exists r in R, exists a path r -> ... -> n in G whose + every edge is live + domain: > + G a finite directed graph over node indices 0..node_count with edges + carrying a `live` flag; R a set of root indices; n a node index + codomain: bool + invariants: + - "INV-2100-1: every severity=error comply rule is reachable from SOME required context — the roots are unioned, never taken one at a time" + - "INV-2100-2: a reachable invocation whose failure cannot propagate is NOT reachable (not continue_on_error and not suppressed and exit_code_compared)" + - "reachable agrees with an independent transitive closure over any bounded graph" + - "neutering every edge leaves exactly the roots reachable" + - "an empty root set reaches nothing: fail closed, never a vacuous pass" + - "an out-of-range target is unreachable, never an error and never a pass" + preconditions: + - "node indices are less than node_count; out-of-range indices are ignored rather than trusted" + kani_harness: verify_reachable_matches_transitive_closure + + select_by_context: + formula: > + select_by_context(C, q) = min { i : C[i].context = q }, where C[i] = + (context, display_name) + domain: a list of (context, display) candidates and one required context string + codomain: Option + invariants: + - "INV-2100-3: reachability is computed against the required-check CONTEXT STRING, never the job display name" + - "two candidate lists differing only in display names select the same candidate" + - "a selected candidate's context equals the required string" + - "a display name equal to a required context never, on its own, satisfies reachability" + - "`gate` and `ci / gate` are different contexts and may resolve to different jobs" + - "INV-2100-7: no gate name is hardcoded anywhere in the rule; the roots are branch protection UNIONED WITH repository rulesets (PMAT-717) — two independent mechanisms, either of which may be the only one a repository configures" + preconditions: + - "the candidate list enumerates every context this repository's workflows can produce; an external reusable callee is Opaque, never a Job" + kani_harness: verify_select_ignores_display_names + + gates: + formula: "gates(printed_failure_verdict, exit_code) = not printed_failure_verdict or exit_code != 0" + domain: "one command's observed output classification and its exit status" + codomain: bool + invariants: + - "INV-2100-4: a command that PRINTS a failure verdict and exits 0 does not gate" + - "a command that printed no failure verdict gates: there was no verdict to contradict" + - "any non-zero exit gates" + preconditions: [] + kani_harness: verify_gates_only_on_exit_code + + gate_effect: + formula: > + enforced(rule) = exists ctx in required_contexts, exists job in + resolve(ctx) union needs_closure(resolve(ctx)), exists step in job.steps, + invokes(step, rule) and propagates(step) + domain: > + required_contexts is a non-empty list of GitHub branch-protection context + strings; workflows is the parsed set of .github/workflows/*.y[a]ml + codomain: > + GateEffectReport with passed() true only when every severity=error rule is + enforced and no hole was recorded + invariants: + - "INV-2100-1: every severity=error comply rule is reachable from a required check context" + - "INV-2100-2: a reachable invocation whose failure cannot propagate is NOT reachable" + - "INV-2100-3: reachability is computed against the required-check CONTEXT STRING, not the job display name" + - "INV-2100-4: a command that prints a failure verdict and exits 0 does not gate" + - "INV-2100-5: an invocation that can never succeed does not gate" + - "INV-2100-6: a job that COMPILES tests without executing them does not establish reachability for those tests" + - "INV-2100-7: no hardcoded gate name anywhere in the rule" + - "an empty severity=error roster is a failure, not a vacuous pass" + - "an empty or unresolvable required-context list is a failure, not a pass" + - "a required context produced by no job (phantom gate) is a failure" + - "a required context resolving into an unreadable external reusable workflow is a failure, because no rule can be SHOWN to run through it" + - "a workflow that does not parse is a hole in the reachability graph, and a hole fails" + - "zero jobs across all workflows is a failure, not a pass" + - "a required context that reaches no invocation is named in the verdict, not silently counted as coverage" + - "a required context that was READ IN FULL and reaches nothing is a measured zero; one that is opaque, phantom or unresolved is a HOLE, and the two must never render identically" + preconditions: + - ".github/workflows exists (otherwise the rule Skips: there is no Actions gate to verify)" + + context_string_resolution: + formula: > + context(job) = job.name if present else job.id; + context(caller.uses(callee)) = context(caller) + " / " + context(callee_job) + domain: a parsed workflow set and one required context string + codomain: Resolution in {Job, Opaque, Phantom} + invariants: + - "a top-level job reports as its display name when it has one, else its job id" + - "a reusable-workflow call reports one context per callee job, namespaced ` / `" + - "an external `owner/repo/...@ref` callee resolves to Opaque, never to a Job" + - "Opaque is never evidence of enforcement" + preconditions: + - "workflow YAML parses" + + failure_propagation: + formula: > + propagates(inv) = not job.continue_on_error and not step.continue_on_error + and not suppressed(line) and exit_code_compared(script, line) + and not compiles_only(line) and not dead(line) and not verdict_without_exit(script, line) + domain: one shell line invoking the rule, within its script and its job + codomain: boolean, with the reasons it is false + invariants: + - "`continue-on-error: true` at job or step level suppresses" + - "a `continue-on-error` EXPRESSION suppresses: `might be false` is not `provably propagates`" + - "`|| true`, `|| :`, `|| echo ...`, `|| exit 0` suppress" + - "a Makefile recipe line prefixed with `-` suppresses" + - "a pipeline suppresses unless the script sets pipefail (GitHub runs `run:` under `bash -e`, pipefail off)" + - "an exit code captured into a variable, consumed by `if`, or disarmed by `set +e` propagates only when a later line exits non-zero" + - "a `needs` edge under `if: always()` propagates only when the job inspects needs..result" + - "INV-2100-4: a wrapper that prints a failure verdict and then exits 0 suppresses" + - "INV-2100-5: a line after an unconditional failure under errexit never runs, and a job containing one can never succeed" + - "INV-2100-5 control: `exit 1` inside an `if` is a working gate, not an impossibility" + - "INV-2100-6: `--no-run`, `--dry-run` and `--list` build the invocation without executing it" + preconditions: + - "the invocation line was found in a `run:` block, or one hop away in a Makefile recipe or shell script" + + enforcement_ledger: + formula: > + ledger = { (rule, severity, status, carrier, file:line) : rule in roster }, + status in {ENFORCED, NEUTERED, UNREACHABLE} + domain: > + the comply rule REGISTRY — the clause ids the check builders register with + filter_check_by_config, as enumerated by enumerate_comply_rule_ids, the + same authority CB-1703 holds the documentation to + codomain: a deterministic markdown document + invariants: + - "the ledger covers exactly the registered rule set: no invented rows, no omissions" + - "one row per CB rule, each with a file:line citation" + - "a rule whose status cannot be established is UNREACHABLE, never blank" + - "a registered rule with no findable definition site is UNREACHABLE and says so" + - "an unenumerable or empty registry is an error, not a clean sheet" + - "the rendering is deterministic: no timestamps, no version stamps, rows sorted by rule id" + - "a required context that carries no rule is listed as carrying none" + - "the population is read from the existing registry, never recounted: one rule, one answer" + - "drift is keyed on the ledger's DATA — rule id, title, severity, status, carrier, and the FILE a rule is declared in — never on its presentation" + - "a moved line number is NOT drift: an edit above a rule declaration changes nothing about what the repository enforces" + - "the provenance label of an IDENTICAL root list is NOT drift: the same four contexts read from PMAT_REQUIRED_STATUS_CHECKS instead of the manifest are the same four contexts" + - "loosening drift must not blind it: a changed status, carrier, declaring file, severity, title or id, and a missing or added row, are each still drift" + preconditions: + - "the repository declares CB rules in-tree (src/cli/handlers/comply_handlers exists); a repo that merely runs the rules owes no ledger" + +preconditions: + - "pmat >= 3.32.0 with `pmat comply check` running CB-2100" + - "required contexts resolvable from PMAT_REQUIRED_STATUS_CHECKS, .github/required-status-checks.txt, or the GitHub branch-protection API" + +postconditions: + - "CB-2100 reports Skip only when .github/workflows/ is absent" + - "CB-2100 reports Fail with severity=error whenever any invariant above is violated" + - "CB-2100's Fail message names the job and the key responsible" + - "CB-2100 fails when docs/status/comply-enforcement-ledger.md is missing or has drifted" + +falsification_tests: + - id: f_1 + rule: "F-1: the job invoking `pmat comply check` carries `continue-on-error: true` and CB-2100 still passes" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::f1_job_level_continue_on_error_fails_citing_job_and_key + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_2 + rule: "F-2: the step is `pmat comply check || true` and CB-2100 still passes" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::f2_or_true_fails + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_3 + rule: "F-3: a job whose DISPLAY NAME equals the required context string is credited as the job that reports it" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::f3_a_display_name_equal_to_the_required_context_is_not_a_match + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_4 + rule: "F-4: a workflow with zero jobs yields a pass instead of a fail" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::f4_zero_jobs_fails_closed + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_5 + rule: "F-5: an unfetchable branch-protection API yields a pass instead of a fail" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::f5_unresolvable_required_contexts_fail_closed + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_6_control + rule: "F-6 (the control): the live shape — required context `ci / gate` beside an unrequired job literally named `gate` — fails, so every other fixture proves nothing" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::f6_required_context_beside_an_unrequired_job_named_the_same_thing_passes + if_fails: "P0 — this contract's guarantee does not hold" + - id: f + rule: "INV-2100-4: a wrapper that prints a failure verdict and exits 0 is credited as a gate" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::inv_2100_4_a_wrapper_that_prints_failed_and_exits_zero_does_not_gate + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_6 + rule: "INV-2100-5: an invocation inside a job that can never succeed is credited as a gate" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::inv_2100_5_a_job_that_can_never_succeed_does_not_gate + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_control + rule: "INV-2100-5 control: `exit 1` inside an `if` is misread as an impossibility, failing every correctly-written workflow" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::inv_2100_5_a_guarded_exit_is_a_gate_not_an_impossibility + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_7 + rule: "INV-2100-6: a job that compiles a test suite without running it establishes reachability for those tests" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::inv_2100_6_compiling_a_test_is_not_running_it + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_8 + rule: "INV-2100-7: the rule hardcodes a gate name and so stops checking the day a job is renamed" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::inv_2100_7_no_gate_name_is_hardcoded_in_the_rule + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_9 + rule: "INV-2100-1: the roots are taken one at a time, so one healthy required check answers for four" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::inv_2100_1_any_root_may_carry_the_roster + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_10 + rule: "an empty severity=error roster yields a vacuous pass" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests.rs::control_roster_is_not_empty + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_11 + rule: "an empty CB rule roster yields an empty ledger instead of an error" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::ledger_over_an_empty_roster_is_an_error_not_a_clean_sheet + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_12 + rule: "the ledger invents or omits rows because it recounts the rules instead of reading the registry" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::the_roster_is_exactly_the_registered_rule_set + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_13 + rule: "a configuration that declares rules and grades none of them error is reported as a clean roster" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::a_config_whose_checks_are_all_sub_error_is_a_hole_with_a_diagnosis + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_14 + rule: "the contract names kani harnesses that do not exist, or that were renamed out from under it — `#[cfg(kani)]` code is never compiled by `cargo test`, so nothing would notice" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::every_kani_harness_the_contract_names_exists + if_fails: "P1 — this contract's guarantee does not hold" + - id: f_15 + rule: "`kind: kernel` plus a `kani_harness` per equation is read as a discharged proof, though nothing in this repository can run one" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::the_contract_states_whether_its_kani_harnesses_discharge + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_16 + rule: "the not_discharged note rots into a false claim: the contract says the harnesses are discharged while nothing runs kani, or a runner is added and the note is left stale" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::a_discharged_claim_requires_something_that_actually_runs_kani + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_17 + rule: "an opaque required context — one resolving into a workflow this repository cannot read — renders identically to a context that was read in full and reaches nothing, so a hole is reported as a measured zero" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::an_opaque_root_is_not_rendered_as_a_measured_zero + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_18 + rule: "a phantom required context renders as a measured zero" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::a_phantom_root_is_not_rendered_as_a_measured_zero + if_fails: "P1 — this contract's guarantee does not hold" + - id: f_control_2 + rule: "the control: every root is rendered `unknown`, which satisfies the two tests above and destroys the table" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::a_root_that_carries_the_roster_still_reads_as_yes + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_19 + rule: "a ledger row renders a blank Title, which is indistinguishable from `has no title`, `never names itself` and `the scanner lost it`" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::the_ledger_never_renders_a_blank_title + if_fails: "P1 — this contract's guarantee does not hold" + - id: f_20 + rule: "the roster's `#[cfg(test)]` guard is a substring match, so a file that MENTIONS the attribute in a comment loses every rule declared below the mention" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::a_cfg_test_mention_in_a_comment_does_not_truncate_the_roster_scan + if_fails: "P1 — this contract's guarantee does not hold" + - id: f_control_3 + rule: "the control: loosening that guard makes the roster cite lines that do not declare the rule they are attributed to" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::every_citation_points_at_a_line_that_names_its_rule + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_21 + rule: "a check owns a CB id and never reports it, so no finding it emits can be tied back to its ledger row" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::every_rule_that_owns_a_cb_id_reports_it_at_runtime + if_fails: "P1 — this contract's guarantee does not hold" + - id: f_22 + rule: "an edit two lines above a rule declaration moves its citation and is reported as ledger drift, so the gate reddens on changes it does not measure" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::an_edit_above_a_rule_declaration_is_not_ledger_drift + if_fails: "P1 — this contract's guarantee does not hold" + - id: f_23 + rule: "supplying the IDENTICAL required contexts through PMAT_REQUIRED_STATUS_CHECKS instead of the manifest is reported as ledger drift" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::the_provenance_of_the_root_list_is_not_ledger_drift + if_fails: "P1 — this contract's guarantee does not hold" + - id: f_control_4 + rule: "the control: drift is loosened until a changed status, carrier, declaring file, severity, title or id no longer registers" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::a_changed_status_carrier_or_file_is_still_ledger_drift + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_control_5 + rule: "the control: drift is loosened until a ledger missing a whole rule still reads as up to date" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::a_missing_row_is_still_ledger_drift + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_24 + rule: "the ledger is non-deterministic, so nobody can diff it and drift goes unnoticed" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests_cb2100.rs::the_rendered_ledger_is_deterministic + if_fails: "P1 — this contract's guarantee does not hold" + - id: f_25 + rule: "a required context that no job produces is treated as absence of evidence rather than a phantom gate" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests.rs::a_phantom_required_context_fails + if_fails: "P1 — this contract's guarantee does not hold" + - id: f_26 + rule: "an unreadable external reusable workflow is credited as enforcement" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests.rs::falsify_2100_3b_external_reusable_workflow_is_opaque_not_compliant + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_27 + rule: "a `run:` block that pipes the invocation is credited with its exit code although bash -e leaves pipefail off" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests.rs::falsify_2100_2c_piped_invocation_loses_its_status + if_fails: "P1 — this contract's guarantee does not hold" + - id: f_28 + rule: "an `if: always()` summary job is credited with its needs' failures although it never inspects needs..result" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests.rs::if_always_without_a_result_check_breaks_the_edge + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_29 + rule: "every fixture fails, including the healthy one, so the rule proves nothing" + prediction: "the named test fails when this condition holds" + test: src/services/gate_effect/tests.rs::control_a_plain_enforcing_job_passes + if_fails: "P0 — this contract's guarantee does not hold" diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/comply-numeric-claims-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/comply-numeric-claims-v1.yaml new file mode 100644 index 0000000000..6a88cfc4f8 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/comply-numeric-claims-v1.yaml @@ -0,0 +1,283 @@ +metadata: + version: "1.0.0" + kind: kernel + created: "2026-08-24" + author: PAIML Engineering + registry: true + description: > + CB-2104 `pmat comply numeric-claims`. Numbers a repository writes down about + itself, judged against each other. + + Two rules over one corpus of git-tracked files. They cover disjoint surfaces + and neither subsumes the other — on the two reference repositories R2 found + none of R1's findings and R1 found none of R2's: + + R1 REPLICATED DIVERGENT CLAIM reads UNNAMED prose numerals. N files carry + the same sentence with a number in it and the numbers disagree, so at least + N - m of them are wrong. Its specimen: 45 per-crate READMEs in aprender + saying `70 workspace crates` where 5 say `75` and `cargo metadata` says 78. + + R2 CONTRADICTION reads NAMED quantities — config keys, Rust consts — and + their same-line annotations. Its specimen: this repository's + src/tests/binary_size.rs declaring `50 * 1024 * 1024` (52,428,800) with the + comment "aligned with .pmat-metrics.toml binary_max_bytes", which is + 50,000,000. + + SEVERITY IS WARN AND THE CHECK NEVER BLOCKS. Findings exit 0, whatever their + count. Exit 2 is reserved for UNMEASURABLE, and that distinction is the + point of the rule's design: "I analysed 12,693 numbers and found nothing" + must never be byte-identical to "`git ls-files` returned nothing and I + analysed nothing", which is exactly what the researched design printed for + both. + + WHY IT IS NOT A DUPLICATE OF CB-2101 OR CB-2102. The unit of analysis is + different. CB-2101 binds ONE DECLARED key to one hand-written binding; + CB-2102 binds ONE DECLARED metric to its reproducing command; both are total + over a curated set and both cost roughly fifteen lines of hand-written TOML + per number. CB-2104 compares N UNDECLARED copies against each other and + requires no prior human declaration at all — which is the whole point, + because nobody declares the copy. Measured: CB-2101 covers 17 of this + repository's 539 config keys and 0 of aprender's 14,385, because it opens + exactly one path and aprender's are in crates/*/. CB-2104 reads 3,883 + mentions here and 25,522 there. CB-2101 also DOCUMENTED the binary-size + contradiction inside a justification string and then passed it, because + binary_max_bytes is kind = "budget". + + The relationship runs the other way: every R1 finding's remediation is + "declare this quantity once, with the command that reproduces it", i.e. + enroll it in .pmat-ratchet.toml or a CB-2101 binding. CB-2104 is the + discovery front-end that populates the two curated checks. It can never do + their job — it cannot tell you WHICH of 45 values is correct. Only an anchor + can, so every finding reports `anchored: false` and a disagreement FLOOR, + never a wrong-count. + + WHAT WAS RESEARCHED AND DELIBERATELY NOT BUILT, each rejected on a + measurement rather than on taste: + + (a) Benford's Law. Rust source scores MAD 0.0382 against Nigrini's 0.015 + nonconformity band BEFORE any contamination, and injecting fabricated + numbers moves the corpus TOWARD conformance. A check that gets happier when + you plant fraud is worse than no check. + + (b) Anchored-claim execution (run the documented predicate, compare). 0.1% + coverage — 12 anchored numerals in 11,282 claim-shaped ones — and 0/4 + precision on this repository. All four findings were well-written documents, + including one correctly pinned to a SHA. Acting on the flagship finding + would have replaced a correctly pinned number with one that rots next week. + + (c) Automatic anchor resolution for R1. Built and measured: 8 labels on + aprender, all of them spec-doc examples, 0 on pmat, and it misses the one + anchor that matters because aprender's README cites `cargo metadata + --no-deps`, which names a method without printing a number. + + (d) The cohort rule AS DESIGNED, at 1/3 precision. Two of three flagship + findings were machine-generated per-crate headers whose counts co-vary + correctly. Two guards, each derived from one of those audited false + positives, restore it to 1/1 and both ship on. + contract: comply-numeric-claims + # ADVISORY, not enforced, and the word is chosen. CB-2104 is a standalone + # subcommand; no `pmat comply check` clause runs it, no CI job gates on it, + # and `default_checks` registers it at severity Warning on purpose. Writing + # `enforced` here would be a claim about this repository that nothing in it + # backs — the exact shape CB-2100 exists to find. + status: advisory + verification: + kani_status: not_applicable + measured_at: "pmat 3.32.0, 2026-08-24; pmat 583ea9ac2, aprender d6c6c6f8f" + detail: > + No kani harness is declared, and none is claimed. Every rule here is a + comparison over bytes that are already checked in: there is no arithmetic + kernel with an invariant a bounded proof would add confidence to, and a + `kind: kernel` with aspirational harnesses would be the decorative + verification claim CB-2100 exists to find. + + What actually runs is the module's own test suite — + src/services/numeric_claims/{census,cohort,corpus,extract,frame,annotate,render,rules}_tests.rs + — reachable from `cargo test --lib numeric_claims`, plus the committed + self-test fixture, which runs on EVERY invocation of the shipped binary + and not only under `cargo test`. + references: + - src/services/numeric_claims/mod.rs + - src/services/numeric_claims/census.rs + - src/services/numeric_claims/render.rs + - src/services/numeric_claims/corpus.rs + - src/services/numeric_claims/frame.rs + - src/services/numeric_claims/cohort.rs + - src/services/numeric_claims/extract.rs + - src/services/numeric_claims/annotate.rs + - src/services/numeric_claims/rules.rs + - src/cli/handlers/comply_handlers/numeric_claims_handler.rs + - tests/fixtures/numeric_claims/README.md + +equations: + exit_policy: + formula: "exit(report) = 2 <=> status(report) = UNMEASURABLE ; else 0" + domain: "one NumericClaimsReport over one repository" + codomain: "{0, 2}" + invariants: + - "INV-2104-1: findings never change the exit code. A repository with 45 divergent claims exits 0, because this check is advisory and the user's constraint was explicit" + - "INV-2104-2: UNMEASURABLE is the ONLY non-zero exit, and it never carries findings — when the check cannot vouch for its own rules, a list of things it thinks it found is worse than silence" + - "an empty corpus is UNMEASURABLE, not clean: 'no contradictions' over nothing is a claim the run did not earn" + preconditions: [] + + self_test: + formula: > + measurable(run) => self_test(committed fixture) = (4/4 planted recovered, + 0 innocent flagged, 0 unexpected) + domain: "tests/fixtures/numeric_claims, include_str!-ed into the binary" + codomain: "SelfTest { passed, recovered, missed, false_positives, unexpected }" + invariants: + - "INV-2104-3: the self-test runs BEFORE the real corpus on every invocation, and a failure suppresses the real result entirely. A rule that has silently stopped firing and a repository that is genuinely clean produce the same empty output; only a corpus that MUST fire separates them" + - "INV-2104-4: one planted defect per rule family. A fixture that exercises three of four rules leaves the fourth's silence indistinguishable from a rule that no longer runs" + - "INV-2104-5: the self-test always runs at CohortConfig::default(), never at the user's settings. It is a control on the RULES, not on the configuration, and a --min-sites 20 that made it fail would be reporting the wrong thing" + - "the innocent half is REPLICATED — each of the 26 classes across eight files with a 6/2 value split, the exact shape R1 hunts — because an innocent control that could not fire even if the rules rotted measures nothing" + - "the fixture lives under tests/fixtures/, which the corpus rules exclude, so the four defects this check plants in its own repository can never reach a user's report" + preconditions: [] + + vacuity: + formula: > + UNMEASURABLE <= files_scanned = 0, or (files_scanned > 20 and + framed_numerals = 0), or (files_scanned > 20 and mentions = 0), or not a + git work tree, or the self-test did not round-trip + domain: "the census of one run" + codomain: "Option" + invariants: + - "INV-2104-6: stated in .pmat-ratchet.toml's idiom — a metric that measures 0 against a baseline above 0 is UNMEASURABLE, not passed" + - "INV-2104-7: the census is UNCONDITIONAL and machine-readable. A clean result that does not carry files_scanned, r1_framed_numerals, r2_mentions and every suppression counter is a bug, not a pass" + - "INV-2104-8: every guard that can hide a finding reports how often it did — G1 generated, G2 multi-slot, derivation, unit-ambiguity, unresolved-xref" + - "the guard counters are ORDERED: G1 runs before G2, so a 0 in the G2 column records that G1 got there first and never that G2 is idle. The renderer says so rather than letting a reader infer that G2 is dead weight" + - "a corpus that is merely SMALL is not a rotted one: the plausibility rules only accuse the extractor once more than 20 files have been scanned" + preconditions: + - "git is available and the path is inside a work tree" + + precision: + formula: "precision(min_sites, guards) is monotone in min_sites and in guards-on" + domain: "the reference corpus: pmat, aprender, bashrs, batuta, renacer, trueno, actix-web" + codomain: "a measured hit rate" + invariants: + - "INV-2104-9: --min-sites below the default 7 MUST print a precision warning. Measured at --min-sites 3: 1/10 on the reference corpus, the rejected nine being step numbers in per-language example workflows, sprint records written weeks apart, and three tickets about three different modules" + - "INV-2104-10: --include-generated (G1 off) MUST print a precision warning. With either guard off the reference corpus measured 1/3" + - "the rule reports DISAGREEMENT, never a verdict on which value is right. Nothing in the output says a value is wrong, or fabricated: a number that is wrong and a number that was invented are indistinguishable to every rule here" + - > + MEASURED BLIND SPOT, found by running the shipped check over the reference + corpus rather than by testing it. On bashrs 90e68cf0af R1 reports one + finding: `- All N rules already verified as implemented` across eight + docs/RULE-CLASSIFICATION-BATCH*.md files, saying 20 six times, then 10, + then 6. It is a false positive of exactly the class G2 exists to kill — + per-entity parameterisation — and G2 cannot see it, because the entity + (the batch number) is in the FILENAME rather than in the line, so the line + has exactly one varying slot and looks like a replicated singleton. + + The spec's negative-control table records bashrs at 0 findings. That row + was an R2-only measurement: R1 had never been run over bashrs, or over any + of the five control repositories. This is therefore the first R1 + measurement on them, and it moves the honest claim from "5 healthy repos, + 0 false positives" to "4 of 5 silent; bashrs yields 1 R1 false positive". + batuta, renacer and trueno stay at 0/0; actix-web was not re-measured here. + + Reported rather than patched. A third guard keyed on path tokens would + change the rule's measured ablation table, and the spec's own instruction + is that any change to the check's loudness must be assumed to break + precision until re-measured on the reference corpus. + preconditions: [] + +preconditions: + - "a pmat whose `pmat comply` carries the `numeric-claims` subcommand (added after 3.32.0; no released version is claimed here, because none has shipped it)" + - "the project root is inside a git work tree" + +postconditions: + - "the census is printed on every run, findings or not, measurable or not" + - "findings exit 0; only UNMEASURABLE exits 2" + - "every reported site carries file:line, so a developer can act without re-running the tool" + +falsification_tests: + - id: falsify_nc_001 + rule: "FALSIFY-NC-001: the self-test cannot fail — '4/4 recovered' is a constant rather than a measurement. Ablate each planted defect in turn and the runner must go red for each" + prediction: "the named test fails when this condition holds" + test: src/services/numeric_claims/census_tests.rs::removing_a_planted_defect_fails_the_self_test + if_fails: "P0 — the strongest guard in the design is decoration" + - id: falsify_nc_002 + rule: "FALSIFY-NC-002: a planted defect is not recovered, or a rule family has no planted defect at all so its silence proves nothing" + prediction: "the named test fails when this condition holds" + test: src/services/numeric_claims/census_tests.rs::the_self_test_recovers_all_four_planted_defects + if_fails: "P0 — a rule that has stopped firing would report a clean tree" + - id: falsify_nc_003 + rule: "FALSIFY-NC-003: an innocent number is flagged, or the innocent half is too small to be a control" + prediction: "the named test fails when this condition holds" + test: src/services/numeric_claims/census_tests.rs::the_self_test_flags_none_of_the_innocent_numbers + if_fails: "P0 — precision is above 70% only because the check is quiet" + - id: falsify_nc_004 + rule: "FALSIFY-NC-004: an empty or non-git corpus reports as clean rather than UNMEASURABLE" + prediction: "the named test fails when this condition holds" + test: src/services/numeric_claims/census_tests.rs::r13_unmeasurable_corpus_exits_two + if_fails: "P0 — 'we could not measure it' would read as 'it did not regress'" + - id: falsify_nc_005 + rule: "FALSIFY-NC-005: a rotted extractor over a real corpus passes as clean, or the guard fires on a corpus that is merely small" + prediction: "the named test fails when this condition holds" + test: src/services/numeric_claims/census_tests.rs::plausibility_separates_nothing_to_say_from_nothing_measured + if_fails: "P0 — this contract's guarantee does not hold" + - id: falsify_nc_006 + rule: "FALSIFY-NC-006: findings change the exit code. The user's hard constraint: this check warns, it never blocks" + prediction: "the named test fails when this condition holds" + test: src/services/numeric_claims/census_tests.rs::r12_exit_code_is_zero_with_findings + if_fails: "P0 — the check would start blocking builds it was promised never to block" + - id: falsify_nc_007 + rule: "FALSIFY-NC-007: a run reports without a populated census, so a silent pass carries no proof it ran" + prediction: "the named test fails when this condition holds" + test: src/services/numeric_claims/census_tests.rs::r14_census_is_always_emitted + if_fails: "P0 — the defect the vacuity guard exists to prevent" + - id: falsify_nc_008 + rule: "FALSIFY-NC-008: a guard hides a finding without counting it, so a suppression leaves no trace and is indistinguishable from a rule that never ran" + prediction: "the named test fails when this condition holds" + test: src/services/numeric_claims/census_tests.rs::r15_suppression_counters_are_reported + if_fails: "P0 — this contract's guarantee does not hold" + - id: falsify_nc_009 + rule: "FALSIFY-NC-009: --min-sites is lowered below the default without the precision warning, which is the one knob that destroys the check (1/10 measured at 3)" + prediction: "the named test fails when this condition holds" + test: src/services/numeric_claims/census_tests.rs::r16_min_sites_below_default_warns + if_fails: "P0 — precision would silently fall by an order of magnitude" + - id: falsify_nc_010 + rule: "FALSIFY-NC-010: the fixture's planted defects reach the real report, because the fixture-tree exclusion rotted and pmat started reporting its own test data" + prediction: "the named test fails when this condition holds" + test: src/services/numeric_claims/census_tests.rs::the_fixture_is_excluded_from_the_real_corpus + if_fails: "P0 — every run on this repository would carry four false findings" + - id: falsify_nc_011 + rule: "FALSIFY-NC-011: R1 reads JSON, pulling 3.3M numerals out of machine-written contract.json files, or R2 stops reading it" + prediction: "the named test fails when this condition holds" + test: src/services/numeric_claims/census_tests.rs::r2_reads_json_and_r1_does_not + if_fails: "P1 — 93% of the corpus would be machine-written interchange" + - id: falsify_nc_012 + rule: "FALSIFY-NC-012: the census stops leading the output, or a clean measured run prints the same bytes as an empty corpus" + prediction: "the named test fails when this condition holds" + test: src/services/numeric_claims/render_tests.rs::a_clean_run_still_shows_its_working + if_fails: "P0 — the exact failure of the researched design" + - id: falsify_nc_013 + rule: "FALSIFY-NC-013: a finding is reported without naming file:line for every site, so a developer has to re-run the tool to find out where to look" + prediction: "the named test fails when this condition holds" + test: src/services/numeric_claims/render_tests.rs::every_site_is_named_with_file_and_line + if_fails: "P1 — this contract's guarantee does not hold" + - id: falsify_nc_014 + rule: "FALSIFY-NC-014: the output claims to know which of two disagreeing values is wrong, or calls a number fabricated. The check reports contradiction; only an anchor can report truth" + prediction: "the named test fails when this condition holds" + test: src/services/numeric_claims/render_tests.rs::the_output_never_claims_to_know_which_value_is_wrong + if_fails: "P0 — the check would assert something no rule in it can establish" + - id: falsify_nc_015 + rule: "FALSIFY-NC-015: an UNMEASURABLE run prints findings it cannot vouch for, or does not say it was unmeasurable" + prediction: "the named test fails when this condition holds" + test: src/services/numeric_claims/render_tests.rs::unmeasurable_says_so_and_prints_no_findings + if_fails: "P0 — this contract's guarantee does not hold" + - id: falsify_nc_016 + rule: "FALSIFY-NC-016: the JSON drops the exit code, the status or the self-test result, so a CI job cannot tell a clean run from an unmeasurable one" + prediction: "the named test fails when this condition holds" + test: src/services/numeric_claims/render_tests.rs::json_carries_the_exit_code_the_status_and_the_self_test + if_fails: "P1 — this contract's guarantee does not hold" + - id: falsify_nc_017 + rule: "FALSIFY-NC-017: the G2 suppression counter's zero is presented as evidence G2 is idle, when G1 simply ran first" + prediction: "the named test fails when this condition holds" + test: src/services/numeric_claims/render_tests.rs::the_g2_zero_is_not_presented_as_evidence_g2_is_idle + if_fails: "P1 — a working guard would be read as dead weight and removed" + - id: falsify_nc_018 + rule: "FALSIFY-NC-018: CB-2104 is registered under an id .pmat.yaml cannot address, or registered at severity=error, which would put an advisory check into the roster CB-2100 checks reachability for and make it block" + prediction: "the named test fails when this condition holds" + test: src/services/numeric_claims/census_tests.rs::cb_2104_is_registered_as_a_warning_and_can_be_disabled + if_fails: "P0 — the check would either be unaddressable or would block" diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/comply-ratchet-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/comply-ratchet-v1.yaml new file mode 100644 index 0000000000..35e5e64256 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/comply-ratchet-v1.yaml @@ -0,0 +1,323 @@ +metadata: + version: "1.0.0" + kind: kernel + created: "2026-08-19" + author: PAIML Engineering + registry: true + description: > + Ratchet baselines (CB-2102). A threshold is a number somebody guessed once; + a ratchet is the number the repository actually had. `.pmat-metrics.toml:45` + is this project's own worked example of both failure modes at once: it + declares `max_unwrap_calls = 100`, annotates it `Current: 570`, and sits in + a tree that measures 20,390 by the predicate `.pmat-ratchet.toml` pins. + Three numbers, no two of which agree, none of which anything in the tree + reads, and a green build throughout. + + CB-2102 replaces the guess with a captured value and a direction. + `.pmat-ratchet.toml` records what each metric WAS; the gate asserts only + `observed <= baseline`; a scheduled pass LOWERS a baseline whenever the + measurement drops; and RAISING one requires a written `justification` on + that entry, checked against the previous committed version of the file. + + The load-bearing decision is the SCOPE PREDICATE, so it is not in prose and + not in source: every entry carries the exact command that reproduces its + baseline plus an explicit `includes_test_files`, and the gate RUNS that + command rather than reading the number. The "unwrap count" of this + repository has been quoted as 570, 11,002, 20,326 and 20,378 inside a + single programme of work; two of those differ by 9,324 because they meant + different file sets, and one moved by 52 within a single session. + + PROVENANCE, and the defect this contract itself closes. The pure half of + this rule — `src/services/metrics_ratchet/{kernel,config}.rs`, 1,836 lines + with 34 falsification tests — was already committed on release/3.32.0 and + had NO caller anywhere in the tree: no comply check, no `.pmat-ratchet.toml` + to read, a doc comment naming a verifier (`command_reproduces_measurement`) + that did not exist, and two `contracts/` files named in its module headers + that were never written, this one included. It was the exact defect class + CB-2100 exists to find, sitting inside the module written to find it. This + version adds the drive (measurement, git history, write-back), the comply + check, the baseline file, and this contract — and fixes one live bug the + orphaning had hidden: `evaluate_ratchet` folded `Outcome::Ok` over an empty + metric map, so a ratchet declaring nothing passed every run forever. + + Rule ids: CB-2102 (ratchet) and CB-2101 (threshold coherence). The module + previously used CB-1421/CB-1420 in prose only — neither was ever registered, + so neither could be addressed from `.pmat.yaml` or appear in CB-2100's + enforcement ledger. They were renamed rather than left as a second name for + the same rule. CB-1403/CB-1404 were NOT reused: both are live rules + ("Assume-Guarantee Chain", "Agent Comply Usage"), and taking either id would + have deleted it. + contract: comply-ratchet + status: enforced + verification: + kani_status: not_discharged + measured_at: "pmat 3.32.0, kani 0.67.0, 2026-08-19" + detail: > + The two `kani_harness` names below are REAL harnesses — `#[kani::proof]` + functions in `mod kani_proofs` at the bottom of + src/services/metrics_ratchet/kernel.rs, held to their names by + drive_tests.rs::every_kani_harness_the_contract_names_exists. They are NOT + discharged: nothing in this repository has ever run them, and with kani + installed they cannot be run at all. Measured, not assumed, and identical + to the blockers recorded for CB-2100: + + (1) `cargo kani --harness verify_next_baseline_monotone_idempotent` fails + with "error: target `pmat-agent` in package `pmat` requires the features: + `mcp-integration`" — kani enumerates every target in the package and one + of them does not build under default features. + + (2) Adding `--features mcp-integration` fails with "error: rustc + 1.93.0-nightly is not supported by the following package: sysinfo@0.39.6 + requires rustc 1.95" — kani 0.67.0 pins a nightly older than a transitive + dependency's MSRV. + + (3) No .github/workflows job, Makefile target or script invokes kani, so + none of this repository's `#[kani::proof]` harnesses is compiled by any + gate. `#[cfg(kani)]` code is not built by `cargo test` either, which is + why a test pins the names rather than the compiler. + + These harnesses are therefore ASPIRATIONAL: they state the intended + theorems and are the specification a future proof will discharge, but + `kind: kernel` here must NOT be read as "proved". What actually runs is + kernel_tests.rs, which checks the same theorems over a bounded box and at + the i64 extremes. That is a test, not a proof, and this contract says so + rather than letting a reader infer a proof that never ran — a contract + naming a harness that silently does not run is precisely the defect class + this backlog exists to rule on. + references: + - src/services/metrics_ratchet/kernel.rs + - src/services/metrics_ratchet/config.rs + - src/services/metrics_ratchet/measure.rs + - src/services/metrics_ratchet/history.rs + - src/services/metrics_ratchet/rewrite.rs + - src/services/metrics_ratchet/mod.rs + - src/cli/handlers/comply_handlers/check_handlers/check_metrics_ratchet.rs + - .pmat-ratchet.toml + +equations: + verdict: + formula: "ratchet_verdict(b, o) = Fail <=> o > b" + domain: "an i64 captured baseline and an i64 observation, both in the metric's declared unit" + codomain: "Pass | Fail" + invariants: + - "INV-2102-1: verdict(b, o) = Fail <=> o > b" + - "equal to the baseline is not 'exceeds': verdict(b, b) = Pass, or every metric is red the day it is captured" + - "no direction parameter exists: every ratcheted metric is normalised so that bigger is worse, because a direction on a comparator is one more thing that can be set the wrong way round and silently invert a gate" + preconditions: + - "the metric declares the debt, never the virtue (uncovered lines, not covered ones)" + kani_harness: verify_next_baseline_monotone_idempotent + + next: + formula: "next_baseline(b, o) = o when o <= b ; = b when o > b" + domain: "an i64 captured baseline and an i64 observation" + codomain: i64 + invariants: + - "INV-2102-2: monotone non-increasing — next(b, o) <= b for every o, so no automated pass can ever loosen a baseline whatever it measures" + - "INV-2102-3: idempotent — next(next(b, o), o) = next(b, o)" + - "a Fail leaves the baseline untouched" + preconditions: [] + kani_harness: verify_next_baseline_monotone_idempotent + + ratchet: + formula: > + outcome(report) = Fail when any metric verdict is Fail, or any baseline + rose against the previous committed file without a justification, or any + hole was recorded + domain: > + .pmat-ratchet.toml, the output of running each metric's own `command` + from the repository root under `bash -o pipefail -c`, and the newest + COMMITTED version of the same file that differs from the one on disk + codomain: "RatchetReport { metrics, unjustified_raises, holes, outcome }" + invariants: + - "INV-2102-4: a metric absent from the measurement run, or measured as Unavailable, is FAIL — 'we could not measure it' must never read as 'it did not regress'" + - "INV-2102-5: RAISING a baseline requires an explicit justification on that entry" + - "an empty [metric.*] set is a FAILURE, not a vacuous pass: a ratchet with no metrics cannot fail and is not a gate" + - "an unreadable or unparsable .pmat-ratchet.toml is a FAILURE" + - "an unknown schema version is a FAILURE: a future file must not be read with today's meaning" + - "a metric whose command did not run, exited outside {0,1}, or printed something that is not a count, is FAIL" + - "exit 1 is a legitimate count of zero: the grep family uses it for 'no matches'" + - "INV-2102-6: a measurement of 0 against a baseline above 0 is UNAVAILABLE unless the metric declares `zero_is_reachable = true`. A rotted `git grep` pathspec and a genuine zero are byte-identical at the shell — both print 0, both exit 1, neither writes to stderr — so the exit-code guard cannot separate them and the baseline is the only place they differ. Measured on the shipped binary before the guard existed: repointing one metric's pathspec at a directory that does not exist made the coherence audit report `FIRING measured 0 against limit 100` and exit 0, while the ratchet read `0 <= 20390` as a Pass" + - "the measurement shell is `bash -o pipefail`: without pipefail the canonical ` | wc -l` reports wc's status, so a producer that failed outright reads as a clean zero — and a ratchet, which only ever looks upward, greets that as the largest improvement in the project's history" + - "an unreadable previous version of the file is a HOLE, never `None`: `None` legitimately means 'initial capture, nothing was raised', and letting an unreadable history borrow that meaning hides an unjustified raise on exactly the machines where history is hardest to read" + - "deleting .pmat-ratchet.toml from a repository that once committed one is a FAILURE, not a Skip" + - "the lowering pass writes back min(baseline, observed) and never anything larger, never for an unmeasured metric, and preserves the file's comments byte for byte — a job that strips a config's documentation each time it runs is worse than no job" + - "the lowering pass re-parses what it wrote and verifies it says exactly what was asked, because a plausible-looking wrong number silently becomes the new truth" + - "INV-2102-7: every measurement is bounded by the budget the metric declares in `timeout_secs`, defaulting to 300s when it declares none, and exceeding that budget is UNAVAILABLE — which is FAIL by INV-2102-4 — never a pass. The budget is per-metric because one constant has to bound a runaway AND let a legitimately expensive measurement finish, and no single number does both: `unwrap_calls_shipped_code` is a cold full-crate clippy that measured 203s and passed on intel-clean-room-8 (run 34676994867) and was killed at the 300s default on the more loaded intel-clean-room-6 (run 34680577045) — the same commit and the same command, reported as a broken measurement on one host and a clean one on the other. Declaring a wider budget therefore buys time, never a verdict; and the lowering pass must preserve the declaration, since a budget a nightly job silently dropped would re-arm this on the next slow runner with nothing in the diff but a baseline. Held by src/services/metrics_ratchet/drive_tests.rs::a_metric_declares_its_own_budget" + preconditions: + - ".pmat-ratchet.toml exists (otherwise CB-2102 Skips: the project declares no baselines), and bash and git are available" + +preconditions: + - "pmat >= 3.32.0 with `pmat comply check` running CB-2102" + - "the project root is inside a git work tree" + +postconditions: + - "CB-2102 reports Skip only when .pmat-ratchet.toml is absent AND was never committed" + - "CB-2102 reports Fail with severity=error whenever any invariant above is violated" + - "CB-2102's Fail message names the metric and both numbers" + +falsification_tests: + - id: f_1 + rule: "F-1: a change adds one to a metric (B+1) and the ratchet still passes" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/kernel_tests.rs::falsify_2102_1_one_added_unwrap_fails_against_the_measured_baseline + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_2 + rule: "F-2: an improvement (B-3) fails, or the lowering pass does not rewrite the baseline to B-3" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/kernel_tests.rs::falsify_2102_2_removing_three_passes_and_lowers_the_baseline + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_3 + rule: "F-3: the file is edited upward with no justification and the ratchet still passes — the ratchet must not be silently loosenable, which is the whole point" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/kernel_tests.rs::falsify_2102_3_raising_a_baseline_without_justification_fails + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_3_control + rule: "F-3 control: raise detection reads the direction of the edit rather than the sign of the metric" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/kernel_tests.rs::falsify_2102_3_raise_detection_is_direction_free + if_fails: "P1 — this contract's guarantee does not hold" + - id: f_4 + rule: "F-4: a metric absent from the measurement run passes" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/kernel_tests.rs::falsify_2102_4_missing_measurement_fails_it_does_not_pass + if_fails: "P0 — this contract's guarantee does not hold" + - id: f + rule: "INV-2102-1: the verdict does not fail exactly when the observation exceeds the baseline" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/kernel_tests.rs::inv_2102_1_verdict_fails_exactly_when_observed_exceeds_baseline + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_5 + rule: "INV-2102-2: the lowering arithmetic raises a baseline" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/kernel_tests.rs::inv_2102_2_next_baseline_is_monotone_non_increasing + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_6 + rule: "INV-2102-3: running the lowering pass twice differs from running it once" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/kernel_tests.rs::inv_2102_3_next_baseline_is_idempotent + if_fails: "P1 — this contract's guarantee does not hold" + - id: f_7 + rule: "a ratchet declaring no metrics folds to Ok and passes every run forever while reading as a gate — the live bug this rule's own module carried while it was orphaned" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/config_tests.rs::an_empty_metric_set_is_a_failure_not_a_clean_sheet + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_8 + rule: "` | wc -l` with a failing producer is read as a count of zero — the single most dangerous shape in the design, and invisible to a gate that only looks upward" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/drive_tests.rs::a_failing_producer_in_a_pipeline_is_unavailable_not_a_zero + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_control + rule: "the control: an honest pipeline of exactly that shape stops measuring, so the guard above has broken every metric" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/drive_tests.rs::an_honest_pipeline_measures + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_9 + rule: "grep's exit 1 for 'no matches' is treated as a broken measurement, so a metric can never legitimately reach zero" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/drive_tests.rs::no_matches_is_a_zero_not_a_failure + if_fails: "P1 — this contract's guarantee does not hold" + - id: f_10 + rule: "prose, or nothing, is parsed as a count" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/drive_tests.rs::non_numeric_output_is_not_a_measurement + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_11 + rule: "an unreadable git history is passed through as `None`, which means 'initial capture' — so an unjustified raise becomes invisible" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/drive_tests.rs::an_unreadable_prior_is_a_hole_not_an_initial_capture + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_12 + rule: "deleting .pmat-ratchet.toml is a way of passing CB-2102" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/drive_tests.rs::deleting_the_ratchet_file_is_not_a_way_of_passing + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_13 + rule: "the lowering pass destroys the file's comments, or writes a number it did not verify" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/drive_tests.rs::lowering_preserves_comments_and_verifies_what_it_wrote + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_14 + rule: "the lowering pass raises a baseline when the measurement grew" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/drive_tests.rs::lowering_never_raises_a_baseline + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_15 + rule: "a metric's pathspec rots so that it matches no files, prints a clean 0, and both gates greet the largest possible regression in measurement quality as the largest improvement in the project's history" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/drive_tests.rs::a_zero_against_a_nonzero_baseline_is_a_hole_not_the_best_day_in_project_history + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_control_2 + rule: "the control: the zero guard rejects every zero, so a metric can never reach the one number every ratchet is trying to get to" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/drive_tests.rs::zero_is_still_reachable_when_it_is_declared_or_already_the_baseline + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_16 + rule: "this repository's own committed ratchet is red, or was captured from a number rather than from its command" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/drive_tests.rs::the_committed_ratchet_holds_at_head + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_17 + rule: "a committed metric's command no longer runs, so its baseline has rotted while the number still looks plausible" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/drive_tests.rs::every_committed_metric_command_still_measures + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_18 + rule: "the committed baselines carry slack the tree has already beaten, so a regression can hide inside the gap" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/drive_tests.rs::the_committed_baselines_have_no_slack_left_in_them + if_fails: "P1 — this contract's guarantee does not hold" + - id: f_19 + rule: "two metrics share a command, so a scope that was supposed to be pinned twice is pinned once — the 9,324-wide question of which files count is answered once and copied" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/drive_tests.rs::no_two_metrics_share_a_command + if_fails: "P1 — this contract's guarantee does not hold" + - id: f_20 + rule: "the contract names kani harnesses that do not exist, or that were renamed out from under it — `#[cfg(kani)]` code is never compiled by `cargo test`, so nothing would notice" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/drive_tests.rs::every_kani_harness_the_contract_names_exists + if_fails: "P1 — this contract's guarantee does not hold" + - id: f_21 + rule: "`kind: kernel` plus a `kani_harness` per equation is read as a discharged proof, though nothing in this repository can run one" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/drive_tests.rs::the_contract_states_whether_its_kani_harnesses_discharge + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_22 + rule: "the not_discharged note rots into a false claim: a kani runner is added and the note is left stale" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/drive_tests.rs::a_discharged_claim_requires_something_that_actually_runs_kani + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_23 + rule: "CB-2102 is registered with no declared severity, so it defaults to Warning, does not fail a non-strict comply run, and falls outside the severity=error roster CB-2100 checks reachability for" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/drive_tests.rs::cb_2102_is_declared_error_not_left_at_the_warning_default + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_24 + rule: "CB-2102 is registered under an id .pmat.yaml cannot address, so it can be neither configured nor found in CB-2100's enforcement ledger" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/drive_tests.rs::cb_2102_is_in_the_comply_rule_registry + if_fails: "P1 — this contract's guarantee does not hold" + - id: f_25 + rule: "the rule is orphaned again: the engine exists and no comply check drives it, which is the state this contract was written to end" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/drive_tests.rs::cb_2102_is_in_the_comply_rule_registry + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_26 + rule: "INV-2102-7: a metric's declared `timeout_secs` is ignored and every measurement is bounded by the flat default, so the cap that exists to stop a runaway also kills a legitimately slow measurement — and the failure reads as UNAVAILABLE, which is indistinguishable from a rotted command" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/drive_tests.rs::a_metric_declares_its_own_budget + if_fails: "P1 — this contract's guarantee does not hold" + - id: f_26_control + rule: "the control: a metric that declares no budget stops getting the 300s default — a per-metric budget implemented by shortening every measurement would kill the cold clippy metric on every runner instead of only the loaded ones" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/drive_tests.rs::a_metric_without_a_budget_keeps_the_default + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_27 + rule: "the lowering pass drops a metric's `timeout_secs`, re-arming the flat-budget defect on the next slow runner with nothing in the nightly job's diff to review but a baseline" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/drive_tests.rs::lowering_preserves_a_timeout_secs_line + if_fails: "P1 — this contract's guarantee does not hold" + - id: f_28 + rule: "this repository's own cold full-crate clippy metric stops declaring a budget and goes back to the 300s default, which does not make the gate stricter — it makes it flaky, and a flaky UNAVAILABLE reads as a failed measurement rather than a regression anyone can act on" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/drive_tests.rs::the_committed_unwrap_metric_declares_a_budget + if_fails: "P1 — this contract's guarantee does not hold" diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/comply-threshold-coherence-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/comply-threshold-coherence-v1.yaml new file mode 100644 index 0000000000..37da6c065b --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/comply-threshold-coherence-v1.yaml @@ -0,0 +1,269 @@ +metadata: + version: "1.0.0" + kind: kernel + created: "2026-08-19" + author: PAIML Engineering + registry: true + description: > + Threshold coherence (CB-2101). CB-2102 asks whether a metric got worse. + CB-2101 asks the prior question nobody was asking: does the number written + down mean anything at all? Every scalar in a declared threshold section of + `.pmat-metrics.toml` is classified against a live measurement as exactly one + of FIRING, VIOLATED or VACUOUS, and the classification decides the verdict. + + VIOLATED — the bound is breached at HEAD and the build is green. This is + strictly worse than having no threshold: it reads as enforcement, it + enforces nothing, and every reader who greps the config comes away with a + false belief about the tree. `.pmat-metrics.toml:45` is this repository's + own case, `max_unwrap_calls = 100` annotated `Current: 570` against a + measured 20,390 — three numbers, no two of which agree. + + VACUOUS — the limit is further from the measurement than the band, so no + movement the ratchet tolerates can reach it. A vacuous threshold is green on + the day the project is abandoned. It is permitted, because a recorded budget + is a legitimate thing to write down, but only with an explicit + `justification`; without one it fails, so decoration cannot pass itself off + as enforcement by silence. + + The classification is TOTAL. There is no fourth "unknown" class: a threshold + whose metric could not be measured FAILS, because "we could not measure it" + must never render as "it is fine". Making unmeasurability a class is exactly + how an unenforceable number gets to look like a gate in a report. + + PROVENANCE, and the defect this contract itself closes. The pure evaluator — + `classify` in kernel.rs and `evaluate_coherence` in config.rs, with 14 + falsification tests — shipped on release/3.32.0 with NO caller, NO rule id + that `.pmat.yaml` could address, and THIS FILE named in three module headers + while not existing. A classifier nobody calls classifies nothing. This + version adds the driver, the comply check, the rule registration, the + seventeen bindings, and the contract. + + Scope, stated once so it cannot drift: CB-2101 audits the CONFIG, not the + tree. It never asserts that a metric is low enough — that is CB-2102's job, + and the two must not be able to disagree, which is why both read the same + `.pmat-ratchet.toml` and both judge against the number a metric's own + `command` prints rather than the number written beside it. + contract: comply-threshold-coherence + status: enforced + verification: + kani_status: not_discharged + measured_at: "pmat 3.32.0, kani 0.67.0, 2026-08-19" + detail: > + The two `kani_harness` names below are REAL harnesses — `#[kani::proof]` + functions in `mod kani_proofs` at the bottom of + src/services/metrics_ratchet/kernel.rs, held to their names by + coherence_drive_tests.rs::every_kani_harness_the_contract_names_exists. + They are NOT discharged: nothing in this repository has ever run them, and + with kani 0.67.0 installed they cannot be run at all. Re-measured + first-hand for this contract, not copied from CB-2102's: + + (1) `cargo kani --harness verify_classify_total_and_sound` fails with + "error: target `pmat-agent` in package `pmat` requires the features: + `mcp-integration`" — kani enumerates every target in the package and one + of them does not build under default features. + + (2) Adding `--features mcp-integration` fails with "error: rustc + 1.93.0-nightly is not supported by the following package: sysinfo@0.39.6 + requires rustc 1.95" — kani 0.67.0 pins a nightly older than a transitive + dependency's MSRV. + + (3) No .github/workflows job, Makefile target or script invokes kani, so + none of this repository's `#[kani::proof]` harnesses is compiled by any + gate. `#[cfg(kani)]` code is not built by `cargo test` either, which is + why a test pins the names rather than the compiler. + + These harnesses are therefore ASPIRATIONAL: they state the intended + theorems and are the specification a future proof will discharge, but + `kind: kernel` here must NOT be read as "proved". What actually runs is + kernel_tests.rs, which checks the same theorems over a bounded box and at + the i64 extremes. That is a test, not a proof, and this contract says so + rather than letting a reader infer a proof that never ran — a contract + naming a harness that silently does not run is precisely the defect class + this backlog exists to rule on. + references: + - src/services/metrics_ratchet/kernel.rs + - src/services/metrics_ratchet/config.rs + - src/services/metrics_ratchet/mod.rs + - src/cli/handlers/comply_handlers/check_handlers/check_threshold_coherence.rs + - src/cli/handlers/comply_handlers/coherence_handler.rs + - .pmat-ratchet.toml + - .pmat-metrics.toml + +equations: + classify: + formula: > + slack(l, m, Max) = l - m ; slack(l, m, Min) = m - l. + classify = Violated <=> slack < 0 ; = Vacuous <=> slack > band ; + = Firing <=> 0 <= slack <= band + domain: "an i64 limit, an i64 measurement in the same unit, a u64 band, and a direction" + codomain: "Firing | Violated | Vacuous" + invariants: + - "INV-2101-1: dir = Max and measured > limit => the threshold is breached, and the verdict of the audit is Fail" + - "INV-2101-2: dir = Max and limit > measured + band => the class is Vacuous; symmetrically for Min" + - "INV-2101-3: classification is TOTAL and DISJOINT — every (limit, measured, band, dir) lands in exactly one of the three classes" + - "KANI-2101-2: monotone in the limit — loosening a Firing threshold never yields Violated, so a class can never be made to look worse by relaxing the number" + - "slack is computed in i128 so that i64::MIN / i64::MAX operands cannot overflow and silently invert a verdict" + - "the direction is declared per binding and never inferred from the key's name: a `max_*` key that is really a floor is exactly the mislabelling this rule exists to catch" + preconditions: + - "limit and measurement are in the same declared unit; a gate threshold must therefore be an integer, and a string-valued threshold such as `min_tdg_grade = \"A-\"` cannot be a gate" + kani_harness: verify_classify_total_and_sound + + audit: + formula: > + outcome(report) = Fail when any threshold verdict is Fail, or any section + of .pmat-metrics.toml appears in neither threshold_sections nor + non_threshold_sections; = Warn when any verdict is Warn; = Ok otherwise + domain: > + .pmat-metrics.toml, the [coherence] declarations and [metric.*] baselines + of .pmat-ratchet.toml, and the output of running each metric's own + `command` from the repository root + codomain: "CoherenceReport { thresholds, undeclared_sections, outcome }" + invariants: + - "INV-2101-3 at file level: every scalar in a declared threshold section produces exactly one verdict carrying exactly one classification — nothing is skipped" + - "INV-2101-4: a gate whose metric produced no measurement, or measured Unavailable, is FAIL — unmeasurable is not compliant, and a justification cannot buy it a pass" + - "a threshold with no [coherence.binding.\"
.\"] entry is UNDECLARED and FAILS: an unbound number is enforced by nothing and can never fire" + - "a NEW section of .pmat-metrics.toml is in neither list and therefore fails closed, rather than being silently exempt" + - "a Vacuous threshold WARNS with a justification and FAILS without one" + - "kind = budget requires a justification; kind = external requires an enforced_by path, and that path must exist" + - "kind = external is a WARN, never a pass: this gate can check that the named reader exists, and cannot check that the bound it enforces is the same bound — `thresholds.binary_max_bytes` is the measured worked example, 50,000,000 declared against a hardcoded 52,428,800 under a comment claiming the two are aligned" + - "an audit that classified nothing is a FAILURE, not a clean sheet: an empty audit passes forever" + - "INV-2101-4 has an upstream dependency, and F-3 found it broken: a rotted pathspec is not an Unavailable measurement, it is a clean 0, so the zero guard in measure::guard_zero (INV-2102-6) is what makes 'unmeasurable is not compliant' reachable at all. Without it a gate whose metric had stopped measuring anything classified FIRING and the audit exited 0" + - "the limit is judged against the number the metric's command PRINTS, never against the baseline written beside it — a limit judged against a remembered number is this rule's own defect, one indirection out" + preconditions: + - ".pmat-ratchet.toml exists (otherwise CB-2101 Skips: the project declares no bindings), and .pmat-metrics.toml parses" + +preconditions: + - "pmat >= 3.32.0 with `pmat comply check` running CB-2101" + - "the project root is inside a git work tree" + +postconditions: + - "CB-2101 reports Skip only when .pmat-ratchet.toml is absent AND was never committed" + - "CB-2101 reports Fail with severity=error whenever any invariant above is violated" + - "every classified threshold appears as `=` in the check's message, uncapped, so `pmat comply check --format json` carries the whole classification" + +falsification_tests: + - id: f_1 + rule: "F-1: max_unwrap_calls = 100 against ~11,002 measured does not classify VIOLATED, or classifies VIOLATED and does not Fail" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/config_tests.rs::arm_a_violated_threshold_on_a_green_build_fails + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_1_2 + rule: "F-1 on real data: this repository's own committed max_unwrap_calls is judged against a remembered number rather than a live measurement, or its class disagrees with the bound" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/coherence_drive_tests.rs::arm_a_max_unwrap_calls_is_judged_against_a_live_measurement + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_2 + rule: "F-2: max_unwrap_calls = 100000 against ~11,002 measured does not classify VACUOUS" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/config_tests.rs::mutation_max_unwrap_calls_100000_classifies_vacuous + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_2_2 + rule: "F-2 second half: a vacuous threshold with no justification passes" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/config_tests.rs::arm_b_vacuous_threshold_warns_with_justification_and_fails_without + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_3 + rule: "F-3: a metric with no measurement available passes, or warns instead of failing, or is bought a pass by a justification" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/config_tests.rs::falsify_2101_3_unmeasurable_metric_fails + if_fails: "P0 — this contract's guarantee does not hold" + - id: f + rule: "INV-2101-3: some (limit, measured, band, dir) lands in zero classes or in two" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/kernel_tests.rs::inv_2101_3_classification_is_total_and_disjoint + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_4 + rule: "INV-2101-2: VIOLATED is reported for something other than a breached bound" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/kernel_tests.rs::inv_2101_2_violated_iff_bound_breached + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_5 + rule: "INV-2101-1: the FIRING band is not exactly the reachable one — an off-by-one at slack = 0 or slack = band" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/kernel_tests.rs::inv_2101_1_firing_is_exactly_the_reachable_band + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_6 + rule: "KANI-2101-2 in test form: loosening a FIRING threshold yields VIOLATED" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/kernel_tests.rs::classify_is_monotone_in_limit + if_fails: "P1 — this contract's guarantee does not hold" + - id: f_7 + rule: "a threshold in a declared threshold section is skipped rather than classified, so totality is claimed and not held" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/config_tests.rs::inv_2101_3_every_threshold_gets_exactly_one_classification + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_8 + rule: "the fixed state is unreachable: a threshold sitting exactly at the measurement cannot be FIRING, so there is no value the config could take that this rule would accept" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/config_tests.rs::firing_threshold_absorbs_one_unwrap_so_only_the_ratchet_moves + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_9 + rule: "kind = external names an enforcing file that does not exist, and the audit reports it as enforced — the doc comment claimed the gate verified this for a whole release while nothing did" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/config_tests.rs::external_enforcement_must_name_a_file_that_exists + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_control + rule: "the control for the above: a real enforcing path is rejected, so the existence check has broken every external binding" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/config_tests.rs::external_enforcement_accepts_a_path_that_exists + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_3_2 + rule: "F-3's upstream hole: a gate's metric stops matching anything, measures a clean 0, and classifies FIRING instead of failing — found by running this rule's own F-3 against the shipped binary" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/drive_tests.rs::a_zero_against_a_nonzero_baseline_is_a_hole_not_the_best_day_in_project_history + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_10 + rule: "this repository's own .pmat-metrics.toml has a threshold that no binding declares, or a section in neither list" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/coherence_drive_tests.rs::every_committed_threshold_is_classified + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_11 + rule: "a binding names an enforced_by path or a metric id that does not exist in this repository" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/coherence_drive_tests.rs::every_committed_binding_names_something_real + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_12 + rule: "an audit that classified nothing folds to Ok and passes every run forever — the exact live bug CB-2102's evaluator carried while it was orphaned" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/coherence_drive_tests.rs::an_empty_audit_is_a_failure_not_a_clean_sheet + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_13 + rule: "the check's message drops thresholds when there are many, so `pmat comply check --format json` no longer classifies all of them" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/coherence_drive_tests.rs::the_json_output_classifies_every_threshold + if_fails: "P1 — this contract's guarantee does not hold" + - id: f_14 + rule: "the contract names kani harnesses that do not exist, or that were renamed out from under it — `#[cfg(kani)]` code is never compiled by `cargo test`, so nothing would notice" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/coherence_drive_tests.rs::every_kani_harness_the_contract_names_exists + if_fails: "P1 — this contract's guarantee does not hold" + - id: f_15 + rule: "`kind: kernel` plus a `kani_harness` per equation is read as a discharged proof, though nothing in this repository can run one" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/coherence_drive_tests.rs::the_contract_states_whether_its_kani_harnesses_discharge + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_16 + rule: "the not_discharged note rots into a false claim: a kani runner is added and the note is left stale" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/coherence_drive_tests.rs::a_discharged_claim_requires_something_that_actually_runs_kani + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_17 + rule: "CB-2101 is registered with no declared severity, so it defaults to Warning and cannot fail a non-strict comply run" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/coherence_drive_tests.rs::cb_2101_is_declared_error_not_left_at_the_warning_default + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_18 + rule: "CB-2101 is registered under an id .pmat.yaml cannot address, so it can be neither configured nor found in CB-2100's enforcement ledger" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/coherence_drive_tests.rs::cb_2101_is_in_the_comply_rule_registry + if_fails: "P1 — this contract's guarantee does not hold" + - id: f_19 + rule: "the rule is orphaned again: the evaluator exists and no comply check drives it, which is the state this contract was written to end" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/coherence_drive_tests.rs::cb_2101_is_in_the_comply_rule_registry + if_fails: "P0 — this contract's guarantee does not hold" + - id: f_20 + rule: "this contract is named by the module headers and does not exist on disk, as it did not for an entire release" + prediction: "the named test fails when this condition holds" + test: src/services/metrics_ratchet/coherence_drive_tests.rs::the_coherence_contract_is_committed + if_fails: "P0 — this contract's guarantee does not hold" diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/dead-code-cache-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/dead-code-cache-v1.yaml new file mode 100644 index 0000000000..cb17671edc --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/dead-code-cache-v1.yaml @@ -0,0 +1,79 @@ +metadata: + version: "1.0.0" + created: "2026-09-03" + author: PAIML Engineering + references: + - "docs/specifications/pmat-architecture-crux-audit.md section 8.4 (CRUX-04, epic 1153)" + - "issue 748 (CLOSED) — the hooks cache keyed on HEAD^{tree} was invariant to the change it gated; moved to git write-tree" + - "Popper (1959) The Logic of Scientific Discovery" + registry: true + description: > + pmat analyze dead-code memoised cargo check on git rev-parse HEAD: — the + committed tree, byte-identical before and after any uncommitted edit — so + a warm run replayed 0 dead functions over a function appended a second + ago, and 1 dead function on line 9 of an 8-line file once the edit was + reverted; and a replay that opened no file and exec'd no compiler still + said compiler-lint-ran in the present tense with no cache marker. The key + is now the working tree (a scratch-index git write-tree; the user's index + is untouched), the report carries cache {hit, tree_hash, written_at, + pmat_version}, a replay says compiler-lint-cached, a schema-4 entry is a + miss, and --no-cache is the escape hatch. + contract: dead-code-cache + status: draft + +equations: + key_is_the_working_tree: + formula: "key(tree) = write-tree(scratch index filled by git add -A) ; fallback HEAD^{tree} ; None outside git => nothing cached" + domain: "any analyze dead-code run on a Rust crate inside a git checkout" + codomain: "tree hash" + invariants: + - "an unstaged edit changes the key; reverting it restores the key (states A, C, E)" + - "the user's index is never read or written; the scratch index is removed after use" + - "an entry written under the previous schema is a miss (state G)" + preconditions: [] + + replay_and_exec_count_move_together: + formula: "cache.hit == true <=> 0 cargo check execs in this invocation and compiler_scan.reason == compiler-lint-cached ; cache.hit == false <=> exactly 1 cargo check exec and reason == compiler-lint-ran" + domain: "states A-E of scripts/dead-code-cache-audit.sh, with a PATH shim counting cargo execs" + codomain: bool + invariants: + - "state D (rerun after an uncommitted edit) is a HIT: a fix that bypasses the cache on any dirty tree fails here" + - "a reduced compiler scan replayed keeps its own reason; only the cache object says it was replayed" + - "--no-cache forces exactly one cargo check and hit == false" + preconditions: [] + + the_gate_sees_the_working_tree: + formula: "quality-gate --checks dead-code on state C reports the uncommitted dead function (same analyzer, use_cache on)" + domain: "state F of the acceptance script" + codomain: bool + invariants: + - "with the cache deleted, states A/C/E answer 0/1/0 (control: a fix that changed the answers rather than the key is caught)" + preconditions: [] + +falsification: + - condition: "a warm run over an uncommitted dead function reports 0, or a run after reverting reports the reverted function (CRUX-04)" + severity: P0 + action: reject_push + - condition: "a run with zero cargo execs reports compiler-lint-ran, or omits cache.hit" + severity: P0 + action: reject_push + - condition: "a schema-4 cache entry is served after upgrade" + severity: P1 + action: reject_push + +falsification_tests: + - id: key_is_the_working_tree + rule: "unstaged edit => new key; revert => old key; index untouched; old schema => miss" + prediction: "working_tree_key_tests (5) pass; states A, C, E, G of scripts/dead-code-cache-audit.sh" + test: "cargo test --lib working_tree_key_tests" + if_fails: "the cache answers for the last commit, not the checkout being analysed" + - id: replay_and_exec_count_move_together + rule: "hit pairs with +0 execs and compiler-lint-cached; miss with +1 and compiler-lint-ran" + prediction: "states B and D (hit, +0), A/C/E (miss, +1), and the --no-cache leg of the script" + test: "bash scripts/dead-code-cache-audit.sh against the release binary" + if_fails: "a 0.2 s replay is byte-identical to a 50 s compiler pass" + - id: the_gate_sees_the_working_tree + rule: "quality-gate --checks dead-code counts an uncommitted dead function; cache-deleted control answers 0/1/0" + prediction: "state F and the control legs of the script" + test: "bash scripts/dead-code-cache-audit.sh" + if_fails: "the pre-commit gate is stale for exactly the change it gates" diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/dogfood-published-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/dogfood-published-v1.yaml new file mode 100644 index 0000000000..9b923beeb0 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/dogfood-published-v1.yaml @@ -0,0 +1,50 @@ +metadata: + version: "1.0.0" + created: "2026-09-03" + author: PAIML Engineering + references: + - "docs/specifications/agentic-delivery-pmat.md section 3.6 / 9.2 (AD-02)" + - "docs/audits/release-3.36.0-dogfood.md — the release-chain receipt, written by hand; it links to the script output release-3.36.0-dogfood-published.md" + - "Popper (1959) The Logic of Scientific Discovery" + registry: true + description: > + Nothing ships without dog-fooding the bytes crates.io serves. The + pre-publish probe reads the packaged tarball and the release gate reads a + local build; neither touches what a consumer obtains with cargo install. + scripts/dogfood-published.sh installs the published version from the + registry into a throwaway root, requires the binary to report that version + and answer --help, runs scripts/dogfood-use.sh against it, and writes + docs/audits/release--dogfood.md pinned by the registry's size and stamp. + contract: dogfood-published + status: draft + +equations: + the_installed_bytes_are_the_measured_bytes: + formula: "for version v: registry(v) present and not yanked => cargo install --version v --locked succeeds => installed --version starts with ' v' => dogfood-use.sh(BIN=installed) exits 0 => receipt names registry size and created stamp" + domain: "any version string matching ^[0-9]+\\.[0-9]+\\.[0-9]+$" + codomain: bool + invariants: + - "a version the registry does not carry fails at the registry leg with exit 1 (the control)" + - "a non-version argument is refused before any network call" + - "the receipt records the installed binary's own --version line, not the local tree's" + preconditions: [] + +falsification: + - condition: "the script exits 0 for a version crates.io does not serve, or for an installed binary that reports another version (AD-02)" + severity: P0 + action: reject_push + - condition: "the receipt is written when the release gate failed against the installed binary" + severity: P0 + action: reject_push + +falsification_tests: + - id: control_fabricated_version + rule: "9.9.9 fails at the registry leg, exit 1, no receipt" + prediction: "FAIL: dogfood-published: pmat 9.9.9 is not on crates.io" + test: "bash scripts/dogfood-published.sh 9.9.9; test $? = 1" + if_fails: "the script can pass on a version that does not exist" + - id: real_run_published_version + rule: "the published 3.36.0 installs, reports itself, passes the release gate, and the receipt names the registry size and stamp" + prediction: "GO: pmat 3.36.0 — 13 checks, 0 failure(s); receipt docs/audits/release-3.36.0-dogfood-published.md" + test: "bash scripts/dogfood-published.sh 3.36.0" + if_fails: "the release announcement is not backed by the bytes consumers receive" diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/hook-debt-scope-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/hook-debt-scope-v1.yaml new file mode 100644 index 0000000000..45cff07d8c --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/hook-debt-scope-v1.yaml @@ -0,0 +1,136 @@ +metadata: + version: "1.0.0" + created: "2026-09-07" + author: PAIML Engineering + references: + - "paiml/infra docs/specifications/build-system-enhancement.md BSE-12 (report O11)" + - "PMAT-707 — pre-commit debt rule scoped to the diff's touched functions" + - "src/cli/handlers/hooks_command_handlers/hook_debt_scope.rs" + - "Popper (1959) The Logic of Scientific Discovery" + registry: true + description: > + The pre-commit hook's complexity verdict is a function of the diff's + TOUCHED functions only. A function is touched when its measured + [line_start, line_end] span in the staged file overlaps a line added or + modified by a staged hunk (the + side of git diff --cached -U0). A touched + function is refused only when a metric exceeds the threshold AND exceeds + the same function's value in the HEAD version of the file, measured the + same way; a function with no HEAD counterpart is judged against the + threshold alone, never as growth. Untouched functions never affect the + verdict, over threshold or not. The hook is FEEDBACK, not the gate — ci / + gate enforces the thresholds on the merge path — so scoping the hook to the + diff removes no enforcement, it only stops refusing a one-line fix in an + undebted function of a file that carries pre-existing debt. + contract: hook-debt-scope + status: draft + +equations: + verdict_is_a_function_of_touched_functions_only: + formula: "verdict(old, new, diff, limits) = REFUSE iff exists f in touched(new, ranges(diff)) with (m(f) > limit and (old(f) is none or m(f) > m(old(f))))" + domain: "one staged source file, its HEAD version, and its git diff --cached -U0 hunk headers" + codomain: bool + invariants: + - "an untouched function over the threshold does not change the verdict" + - "a touched function over the threshold but no worse than at HEAD does not change the verdict" + - "a touched function with no HEAD counterpart is judged against the threshold alone" + - "spans and metrics come from collect_functions + measure_block + FunctionSpans, the one existing measurement path; no second parser" + - "a refusal names the function, the measured value, the limit, and the previous value (or says there is none)" + preconditions: [] + +falsification: + - condition: "a commit whose only change is a one-line edit in an undebted function is refused because another function in the same file carries pre-existing debt" + severity: P1 + action: reject_push + - condition: "complexity growth inside a touched function is allowed, or the refusal does not name the function and both numbers" + severity: P0 + action: reject_push + +falsification_tests: + - id: verdict_is_a_function_of_touched_functions_only + rule: "the hook_debt_scope_* tests in src/cli/handlers/hooks_command_handlers/hook_debt_scope_tests.rs: one-line fix beside pre-existing debt is allowed; growth in a touched function is refused with both numbers; a new over-threshold function is refused; an untouched over-threshold function is ignored; the pre-BSE-12 whole-file rule refuses the first case" + prediction: "diff-scoped rule: 13 passed (7 in-memory fixtures + 6 over a fixture repo). Same suite under the pre-BSE-12 whole-file rule (touched set = every function, previous = none): 8 of 13 FAIL, including the one-line-fix case." + test: "cargo test -p pmat hook_debt_scope" + if_fails: "the hook refuses a one-line fix for debt the commit did not write, and the developer decomposes functions the change never touched" + - id: the_verdict_is_read_out_of_git_not_out_of_the_working_tree + rule: "the staged_repo::* tests in src/cli/handlers/hooks_command_handlers/hook_debt_scope_tests.rs run the rule over a real fixture repo through staged_verdict / staged_verdict_for_file: the NEW source is `git show :` (the INDEX), the OLD source is `git show HEAD:`, and the ranges come from `git diff --cached -U0 -- `" + prediction: "6 passed. One of them stages the allowed content and then writes over-threshold growth into the WORKING TREE: a verdict read from disk instead of the index fails it. An unstaged path and an unreadable diff are errors, so neither can render as 'no violations'." + test: "cargo test -p pmat hook_debt_scope" + if_fails: "the hook grades content that is not being committed, in either direction: unstaged work refuses a clean commit, or staged debt slips past because the disk copy is clean" + - id: the_cli_entry_point_prints_the_offender_and_exits_non_zero + rule: "the diff_scope_entry_point::* tests in src/cli/handlers/complexity_handlers/complexity_handlers_tests.rs call handle_analyze_complexity_diff_scoped over a fixture repo at the DEFAULT thresholds (10/15, the same ComplexityConfig::from_args uses)" + prediction: "4 passed. Report O11 — a one-line fix in `innocent` beside a committed `debted` of cyclomatic 11 — returns Ok; `innocent` grown to 12 returns Err naming 'innocent - Cyclomatic 12 > 10 (was 1)'; a path with nothing staged is an Err, not a pass." + test: "cargo test -p pmat diff_scope" + if_fails: "the developer is told a file is at fault without being told which function, which is the report the whole-file mode already gives" + +mutation_matrix: + - mutation: "diff_scoped_verdict judges every function instead of the touched ones (`for func in new_functions.iter()`), leaving the growth comparison in place" + observed: "hook_debt_scope: 13 passed, 0 failed. diff_scope: 4 passed, 0 failed." + reading: > + Not a weak suite — a fact about the rule. An UNTOUCHED function measures + identically in the HEAD blob and the staged blob, so the growth + comparison already spares it and the touched filter is redundant for a + file that HAS a HEAD counterpart. The filter is load-bearing exactly + where growth cannot be computed: a file absent from HEAD has no previous + value for any function, and there the scope is the only thing between the + developer and the whole file. That leg is + hook_debt_scope_a_file_absent_from_head_is_judged_on_the_threshold. + - mutation: "the above PLUS `previous = None` — i.e. the complete pre-BSE-12 whole-file rule the hook still runs today" + observed: "hook_debt_scope: 5 passed, 8 FAILED. diff_scope: 2 passed, 2 FAILED (the O11 case and the offender-naming case)." + reading: "the suite discriminates the shipped rule from the rule it replaces" + + - id: the_generated_hook_puts_the_flag_on_a_command_line_for_rust_only + rule: "complexity_gate_tests::complexity_gate_scopes_rust_to_the_diff_only and ::dropping_the_flag_from_the_generated_hook_goes_red in src/cli/handlers/hooks_command_handlers/hook_generation.rs READ the SCOPE_FLAG case arms out of the generated text (a bare hook.contains(\"--diff-scope\") would now pass on the doctrine COMMENT alone) and assert the *.rs arm asks for the scoped verdict, the fallback arm does not, and the invocation passes $SCOPE_FLAG without hardcoding the flag" + prediction: "20 passed. Two mutants are built inside the test and run through the SAME predicate: blanking the *.rs arm, and deleting $SCOPE_FLAG from the invocation. Both are asserted to be caught, and each mutant is first asserted to differ from the real hook so the legs cannot pass vacuously." + test: "cargo test -p pmat --lib complexity_gate" + if_fails: "the flag is declared and routed but never reaches a command line -- the exact state this ticket found, where a tested rule was not the rule the hook ran" + - id: end_to_end_the_hook_allows_the_o11_case_and_refuses_growth + rule: "a minimal git repo (no Cargo.toml, so the hook's fmt and clippy gates are inert), fixture.rs with `debted` at cyclomatic 36 and `innocent` at 1, committed BEFORE `pmat hooks install` so the debt is pre-existing; thresholds read back out of the installed hook (30/25); no --no-verify" + prediction: "a one-line edit inside `innocent` commits cleanly (rc 0, HEAD moves) although the whole-file measurement of the same file reports `Errors: 2` / `Max Cyclomatic: 36`; growing `innocent` itself to 36 is refused (rc 1, HEAD unchanged) printing `innocent - Cyclomatic 36 > 30 (was 1)` and `innocent - Cognitive 35 > 25 (was 0)`, with the untouched `debted` absent from the refusal" + test: "cargo test -p pmat --test hook_debt_scope_e2e" + if_fails: "report O11 is still reproducible in the field: a developer is charged for debt the commit did not write" +wiring_status: + measured_on: "2026-09-08, PMAT-707 phase 3" + wired: > + YES, for Rust. `diff_scope: bool` is declared on AnalyzeCommands::Complexity + (clap `--diff-scope`, `requires = "file"`), route_complexity_analysis in + src/cli/handlers/analysis_handlers/core_routes.rs dispatches it to + handle_analyze_complexity_diff_scoped, and the generated pre-commit hook in + hook_generation.rs selects it PER STAGED FILE: `*.rs` gets + `SCOPE_FLAG="--diff-scope"`, every other language gets `SCOPE_FLAG=""` and + keeps the whole-file measurement, because the scoped measurement is + syn-based and there is no honest touched-set for a .py/.ts/.go file. + end_to_end: + procedure: > + A minimal `git init` repo (no Cargo.toml, so the hook's fmt and clippy + gates are inert and the complexity gate is the only variable), fixture.rs + holding `debted` (35 ifs, cyclomatic 36) and `innocent` (cyclomatic 1), + COMMITTED BEFORE `pmat hooks install` so the debt is genuinely + pre-existing. The hook exported PMAT_MAX_CYCLOMATIC_COMPLEXITY=30 / + PMAT_MAX_COGNITIVE_COMPLEXITY=25, read back out of the installed file + rather than taken from what the installer printed. No --no-verify. + baseline: > + The whole-file measurement over the same file reports `Errors: 2`, + `Max Cyclomatic: 36` — i.e. under the pre-BSE-12 rule EVERY commit + touching this file is refused. That is report O11. + allowed: > + A one-line change inside `innocent` (`acc = 0i64` -> `acc = 7i64`, + a single -/+ hunk at line 42) committed cleanly: + "Complexity check... OK / All quality gates passed!", rc 0, HEAD moved. + `debted` is over the threshold in the same file and did not affect it. + refused: > + Growing `innocent` itself to 35 ifs in the next commit was refused, rc 1, + HEAD unchanged, naming the function and both numbers: + "innocent - Cyclomatic 36 > 30 (was 1)" and + "innocent - Cognitive 35 > 25 (was 0)". `debted`, untouched and also at + cyclomatic 36, is ABSENT from the refusal — which is what distinguishes + the scoped verdict from the whole-file one. + remaining: > + Every AnalyzeCommands::Complexity struct literal in the crate (24 sites, the last + at tests/modules/analysis_timeout_test.rs, outside PMAT-707's scope and added by + the orchestrator at commit 3842c57e4) carries `diff_scope: false`; the pre-commit + clippy stage (--all-targets) is the check that refuses a missing site. + + LIMITATION: Rename-to-a-deleted-name. A touched function renamed to the name of a + DELETED function inherits the deleted one's baseline. This is an accepted + limitation of name-keyed pairing and is out of scope for this PR. It must be + fixed in a follow-up PR when AST-based identity tracking is introduced. diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/index-faithful-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/index-faithful-v1.yaml new file mode 100644 index 0000000000..ef29d985bf --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/index-faithful-v1.yaml @@ -0,0 +1,83 @@ +metadata: + version: "1.0.0" + created: "2026-09-04" + author: PAIML Engineering + references: + - "docs/specifications/pmat-architecture-crux-audit.md section 8.7 (CRUX-07)" + - "scripts/index-faithful-audit.sh — the acceptance, five legs with controls" + - "Popper (1959) The Logic of Scientific Discovery" + registry: true + description: > + The index CLAUDE.md mandates over grep is a faithful, reproducible view + of the tree. (a) The incremental fast path may skip re-hashing a file + only when its mtime, its length and (on unix) its ctime all predate + built_at, so a rewrite with a backdated mtime is re-read and the stale + checksum is not carried forward — while a quiescent tree is still skipped + by mtime. (b) The two walks that define persisted order are sorted and + the ranker breaks ties on (file_path, start_line), so tied results come + back the same on every filesystem. (c) analyze churn serialises + author_contributions as a BTreeMap and ranks ties by author, so its raw + JSON is byte-stable. (d) manifest.json is written atomically and a torn + or unparsable manifest is named and rebuilt, never served. (e) A failed + index save is reported as a failure; success is announced only after it. + contract: index-faithful + status: draft + +equations: + backdated_rewrite_is_seen: + formula: "content(f) changed and mtime(f) < built_at => (len(f) != recorded_len or ctime(f) >= built_at) => f is re-hashed; query returns the new symbols and not the deleted ones" + domain: "the incremental index update (function_index build_helpers)" + codomain: bool + invariants: + - "a file whose mtime, length and ctime all predate built_at is skipped: `Incremental update: N mtime-skipped, ..., 0 re-parsed` on a quiescent tree (the fast-path-survival control)" + - "old manifests without the len/ctime fields load and re-hash once" + preconditions: [] + + order_is_the_sorted_sequence: + formula: "query --limit N --format json over N tied one-function files returns all N, and the emitted (file_path, start_line) sequence equals its sorted form" + domain: "the persisted walk order and the ranker tie-break" + codomain: bool + invariants: + - "no result is truncated at the limit" + preconditions: [] + + churn_json_is_byte_stable: + formula: "over 12 runs, sha256(raw JSON minus generated_at, key order preserved) has one value (c1) and sha256(key-sorted JSON) has one value (c2)" + domain: "analyze churn --format json" + codomain: bool + invariants: + - "author_contributions names exactly the git shortlog -sn authors; the counts differ today and are reported, not judged, by the acceptance (a separate finding)" + - "c2 alone is not a gate (it normalised the defect); c1 alone misses an added timestamp field" + preconditions: [] + + manifest_is_atomic_and_torn_is_named: + formula: "a manifest truncated mid-object after a good db write is reported with one of torn|corrupt|invalid|unreadable|rebuild and the index rebuilds; a clean pair prints none of those words" + domain: "index load" + codomain: bool + invariants: [] + preconditions: [] + + failed_save_is_reported: + formula: "index directory read-only => the incremental save prints a failure line naming the path; index directory writable => no failure line" + domain: "query_handler indexing" + codomain: bool + invariants: + - "persist_score returns Result and its caller warns; the FTS DELETE in sqlite_docs propagates with ?" + preconditions: [] + +falsification_tests: + - id: F1 + rule: "the released binary fails five legs and passes every control" + prediction: "on pmat 3.36.0: a (both assertions), b, c1, d, e are red; the fast-path, c2, author-set, clean-pair and writable controls are green" + test: "PMAT= bash scripts/index-faithful-audit.sh" + if_fails: "a leg is already green on the old binary: its defect was never reproduced and the leg is vacuous — re-establish RED before trusting GREEN" + - id: F2 + rule: "this tree passes every leg and every control" + prediction: "eleven ✓, GREEN, exit 0" + test: "PMAT=$(cargo build --bin pmat --message-format json | ...executable) bash scripts/index-faithful-audit.sh" + if_fails: "the red leg names which rule regressed; a red control means the fix deleted the optimisation or the disclosure it was meant to keep" + - id: F3 + rule: "the fast path was kept, not deleted (named mutation)" + prediction: "making check_mtime_reuse always re-hash turns the fast-path control RED (0 mtime-skipped) while leg a stays green" + test: "mutate build_helpers.rs to always return None from the fast path, rebuild, run --only a, revert and REBUILD" + if_fails: "the control does not discriminate deletion of the optimisation; also confirm the binary was rebuilt after the revert" diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-artifacts-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-artifacts-v1.yaml new file mode 100644 index 0000000000..199cd60675 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-artifacts-v1.yaml @@ -0,0 +1,96 @@ +metadata: + version: "1.1.0" + created: "2026-07-02" + author: PAIML Engineering + references: + - "docs/specifications/components/modern-agentic-coding-support.md (MACS v1.0.0)" + - "arXiv:2602.22302 Bruni et al. (2026) Agent Behavioral Contracts" + - "Popper (1959) The Logic of Scientific Discovery" + - "Meyer (1997) Object-Oriented Software Construction 2e (Design by Contract)" + registry: true + description: > + Canonical planning artifacts: ROADMAP.yaml rendered from the work store, + mcp.json regenerated from tool defs, and doc model-id drift linted. + MACS F6 (docs/specifications/components/modern-agentic-coding-support.md). + contract: macs-artifacts + status: draft + +equations: + roadmap_canonical: + formula: render(state) is byte-stable and hash covers sources not wall-clock + domain: state in .pmat-work store + optional pinned GitHub snapshot + codomain: ROADMAP.yaml bytes + invariants: + - BTreeMap ordering, ids sorted + - generation timestamp excluded from content hash + - source snapshot ids included in content hash + preconditions: + - work store readable + + manifest_faithful: + formula: tools(mcp.json) = tools(mcp_server_tool_defs) + domain: mcp.json at repo root + codomain: bool + invariants: + - manifest generation is pure (defs -> bytes) + - hand-edits are drift (CB-1656 red) + preconditions: [] + + manifest_schema_faithful: + formula: inputSchema(mcp.json, t) = inputSchema(tools/list, t) for every live tool t + domain: mcp.json at repo root, and the handlers' metadata() the server serves + codomain: bool + invariants: + - the manifest renders each inputSchema FROM handler metadata, never from a shape chosen by tool name + - the committed file equals render_manifest() byte-for-byte + - every live tool declares metadata (no open-object fallback is reachable) + preconditions: [] + + docs_current: + formula: denylist_hits(docs) = 0 outside allow-list + domain: docs/** with deny-list {claude-3-*, claude-2*, gpt-4-turbo} + codomain: bool + invariants: + - allow-listed only inside docs/agent-models.md history table + preconditions: [] + +falsification: + - condition: "ROADMAP.yaml drifts from the work store with green comply" + severity: P0 + action: reject_push + - condition: "mcp.json drifts from tool defs with green comply" + severity: P0 + action: reject_push + - condition: "mcp.json advertises an inputSchema a tool's handler does not accept (CRUX-09, #1150)" + severity: P0 + action: reject_push + - condition: "A deny-listed model id appears outside docs/agent-models.md" + severity: P1 + action: block_release + +falsification_tests: + - id: manifest_faithful + rule: mcp.json advertises exactly the live tool set + prediction: legacy 2-tool manifest fails; generated 20-tool manifest passes + test: cargo test --lib generated_equals_tool_defs + if_fails: agents read a manifest that lies about the tool surface + - id: roadmap_canonical + rule: render is byte-stable and hash covers sources not wall-clock + prediction: two renders at different times share one content_hash + test: cargo test --lib hash_covers_sources_not_wallclock + if_fails: ROADMAP.yaml churns on every render, defeating drift detection + - id: manifest_schema_faithful + rule: the shipped inputSchema IS the served inputSchema, per tool + prediction: a canned paths-array schema for pmat_index_stats fails; the handler's own schema passes + test: cargo test --lib manifest_schemas_match_handler_metadata + if_fails: a validating client rejects its own valid call, or calls a tool as the manifest says and gets -32602 + - id: manifest_pinned + rule: the committed mcp.json equals the renderer's output byte-for-byte + prediction: editing one property in mcp.json without regenerating fails; regenerating passes + test: cargo test --lib committed_mcp_json_is_pinned_to_the_renderer + if_fails: a renderer fix ships with the old file still in the tarball, and nothing notices + - id: docs_current + rule: no superseded model ids in docs outside the registry + prediction: reintroducing claude-3-opus outside agent-models.md fails CB-1657 + test: cargo test --lib cb1657_red_on_denylist_hit + if_fails: stale model ids in agent docs become executable misinformation diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-cot-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-cot-v1.yaml new file mode 100644 index 0000000000..176b065c19 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-cot-v1.yaml @@ -0,0 +1,70 @@ +metadata: + version: "1.0.0" + created: "2026-07-02" + author: PAIML Engineering + references: + - "docs/specifications/components/modern-agentic-coding-support.md (MACS v1.0.0)" + - "arXiv:2602.22302 Bruni et al. (2026) Agent Behavioral Contracts" + - "Popper (1959) The Logic of Scientific Discovery" + - "Meyer (1997) Object-Oriented Software Construction 2e (Design by Contract)" + registry: true + description: > + Structured chain-of-thought with mandatory discharge + auto-derivation (C31). + MACS F3 (docs/specifications/components/modern-agentic-coding-support.md). + contract: macs-cot + status: draft + +equations: + chain_integrity: + formula: forall s in steps -> discharged(s.assumption) + domain: steps in Vec + codomain: bool + invariants: + - discharge graph is acyclic + - roots are equations or E-facts only + preconditions: + - ticket has >=1 CoT step when kind in {feature,bugfix,refactor} + lean_theorem: Theorems.Macs.CoT_Discharge_DAG + + derivation_complete: + formula: count(claims_from(steps)) = count(steps) and count(obligations_from(steps)) = count(steps) + domain: steps in Vec + codomain: bool + invariants: + - claim.hypothesis = step.implication verbatim + - claim.method = step.evidence_method verbatim + preconditions: + - MACS-007 schema active + +falsification: + - condition: "A step's assumption has no discharge and CB-1640 stays green" + severity: P0 + action: reject_push + - condition: "A ticket closes with fewer FalsifiableClaims than CoT steps" + severity: P0 + action: reject_push + - condition: "Legacy prose steps crash the parser instead of migrating to v2-with-L0 annotation" + severity: P1 + action: block_release + +falsification_tests: + - id: cot_chain_cycle_rejected + rule: Discharge graph must be a DAG + prediction: a two-step mutual discharge is rejected by check_chain + test: cargo test --lib cycle_detected_violates_1640 + if_fails: circular reasoning passes the integrity check (C31) + - id: cot_undischarged_rejected + rule: Every assumption must be discharged + prediction: a step with no discharge source and no evidence_method is rejected + test: cargo test --lib undischarged_assumption_violates_1640 + if_fails: assumptions inflate without discharge (a hallucination magnet) + - id: cot_spec_self_check + rule: The spec is subject to the check it specifies + prediction: MACS spec §3.1's own chain parses and passes check_chain + test: cargo test --lib spec_section_3_1_passes + if_fails: the schema cannot express its own specification's reasoning + - id: cot_derivation_verbatim + rule: claim.hypothesis/method copied verbatim from step fields + prediction: any paraphrase drift fails CB-1658 + test: cargo test --lib one_claim_per_step_verbatim_fields + if_fails: derived claims drift from the reasoning they discharge diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-ladder-kernel-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-ladder-kernel-v1.yaml new file mode 100644 index 0000000000..7b49ad1105 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-ladder-kernel-v1.yaml @@ -0,0 +1,126 @@ +metadata: + version: 1.0.0 + created: '2026-07-04' + author: PAIML Engineering + description: > + Verification-ladder parser kernel — the lighthouse dogfooding contract for + docs/specifications/audit-pmat-support-l1-l5-aprender-provable-contracts.md. + "The ladder proves itself": pmat's own VerificationLevel parser is climbed + L1→L5 (types → falsification #[test] → proptest → Kani → Lean). + references: + - docs/specifications/components/pmat-work-verification-ladder.md + - docs/specifications/audit-pmat-support-l1-l5-aprender-provable-contracts.md + - Popper (1959) The Logic of Scientific Discovery +equations: + parse_strict: + formula: parse_strict(s) = Some(l) ⇔ s ∈ {"L0".."L5"} + domain: s ∈ String + codomain: Option + invariants: + - parse_strict is total (never panics) on any &str + - parse_strict(l.as_str()) = Some(l) for every level l (left inverse) + - parse_strict(s) = Some(l) ⇒ l.as_str() = s (strict soundness) + preconditions: [] + lean_theorem: Theorems.Macs.Ladder_Parse_Total + ord_monotone: + formula: (a < b) ⇔ (a as u8) < (b as u8) + domain: a, b ∈ VerificationLevel + codomain: bool + invariants: + - the derived Ord agrees with the numeric rung (as u8) + - to_nat is injective, so rung comparisons never conflate two levels + preconditions: [] + lean_theorem: Theorems.Macs.Ladder_Ord_Injective +proof_obligations: +- type: postcondition + property: parse round-trips every canonical level string + formal: '∀l: parse_strict(as_str(l)) = Some(l)' + applies_to: parse_strict + lean: + theorem: Theorems.Macs.Ladder_Parse_Total + module: Theorems.Macs.Ladder + status: proved + notes: 'by cases l <;> decide — depends on no axioms' +- type: invariant + property: strict parse rejects case/whitespace/out-of-set corruptions + formal: 'parse_strict("l3") = None ∧ parse_strict("L3 ") = None ∧ parse_strict("strong") = None' + applies_to: parse_strict + lean: + theorem: Theorems.Macs.Ladder_Parse_Strict + module: Theorems.Macs.Ladder + status: proved + notes: 'by decide — mirrors Rust falsification test parse_strict_rejects_typos' +- type: invariant + property: Ord agrees with the numeric rung (gate comparisons never invert) + formal: '∀a,b: (a < b) = ((a as u8) < (b as u8))' + applies_to: ord_monotone + lean: + theorem: Theorems.Macs.Ladder_Ord_Injective + module: Theorems.Macs.Ladder + status: proved + notes: 'by cases a <;> cases b <;> decide' +- type: bound + property: distinct levels have distinct wire strings (Display never collides) + formal: '∀a,b: as_str(a) = as_str(b) ⇒ a = b' + applies_to: parse_strict + lean: + theorem: Theorems.Macs.Ladder_toStr_Injective + module: Theorems.Macs.Ladder + status: proved + notes: 'by cases a <;> cases b <;> decide' +falsification_tests: +- id: parse_total_strict_prop + rule: parse is total and strict — accepts exactly L0..L5 + prediction: for 8-char random strings, parse_strict is Some iff s ∈ {L0..L5} + test: cargo test --lib -- work_verification_level::tests::parse_total_strict_prop + if_fails: misspelled levels silently downgrade tickets (C28) +- id: parse_strict_rejects_typos + rule: strict parse rejects case/whitespace corruptions + prediction: parse_strict("l3"), ("L3 "), (" L3"), ("strong"), ("L6"), ("") all None + test: cargo test --lib -- work_verification_level::tests::parse_strict_rejects_typos + if_fails: over-claims like "l4" pass the completion gate +- id: ord_matches_numeric + rule: Ord matches numeric ladder order + prediction: L0 < L1 < L2 < L3 < L4 < L5 and (v as u8) is monotone + test: cargo test --lib -- work_verification_level::tests::ord_matches_numeric + if_fails: achieved >= claimed comparison inverts; over-claims pass +- id: display_parse_id + rule: Display then parse is the identity on valid levels + prediction: as_str(l).parse() = Some(l) for every l + test: cargo test --lib -- work_verification_level::tests::display_parse_id + if_fails: read/write round-trip breaks; stored levels corrupt on reload +- id: serde_string_repr + rule: wire format is the display string (no wire break vs legacy) + prediction: serde serializes L4 as "L4"; legacy contract.json still parses + test: cargo test --lib -- work_verification_level::tests::serde_string_repr + if_fails: every existing .pmat-work contract breaks on read +kani_harnesses: +- id: KANI-LADDER-001 + obligation: parse round-trips every canonical level string + property: parse_strict(l.as_str()) == Some(l), exhaustive over the six variants + bound: 6 + strategy: exhaustive + solver: cadical + harness: verify_ladder_parse_roundtrip +- id: KANI-LADDER-002 + obligation: Ord agrees with the numeric rung + property: (a < b) == ((a as u8) < (b as u8)) for all level pairs + bound: 6 + strategy: exhaustive + solver: cadical + harness: verify_ladder_ord_matches_numeric +- id: KANI-LADDER-003 + obligation: strict parse is total and accepts exactly L0..L5 + property: parse_strict never panics on 2-ASCII input; Some iff L{0..5} + bound: 128 + strategy: exhaustive + solver: cadical + harness: verify_ladder_parse_strict_total_two_ascii +qa_gate: + id: LADDER-KERNEL-GATE + name: Verification-ladder parser kernel gate + description: L1→L5 provability gate for pmat's own VerificationLevel parser + checks: + - validation + - falsification + pass_criteria: All 5 falsification tests pass and all 4 Lean obligations are proved (0 sorry) diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-ladder-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-ladder-v1.yaml new file mode 100644 index 0000000000..0b5f011919 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-ladder-v1.yaml @@ -0,0 +1,68 @@ +metadata: + version: "1.0.0" + created: "2026-07-02" + author: PAIML Engineering + references: + - "docs/specifications/components/modern-agentic-coding-support.md (MACS v1.0.0)" + - "arXiv:2602.22302 Bruni et al. (2026) Agent Behavioral Contracts" + - "Popper (1959) The Logic of Scientific Discovery" + - "Meyer (1997) Object-Oriented Software Construction 2e (Design by Contract)" + registry: true + description: > + Typed L0-L5 verification ladder with evidence-gated completion (C28). + MACS F2 (docs/specifications/components/modern-agentic-coding-support.md). + contract: macs-ladder + status: draft + +equations: + parse_total_strict: + formula: parse(s) = Ok(l) <-> s in {"L0".."L5"} + domain: s in String (proptest-generated incl. corruptions) + codomain: Result + invariants: + - Ord matches numeric order + - Display then parse = id on valid set + preconditions: [] + lean_theorem: Theorems.Macs.Ladder_Parse_Total + + gate_monotone: + formula: complete(t)=Ok -> achieved_level(t) >= t.claimed_level + domain: t in .pmat-work/* + codomain: bool + invariants: + - achieved_level is computed from evidence, never stored + preconditions: + - binding.yaml well-formed (CB-1250 family) + +falsification: + - condition: "\"l4\" or \"L3 \" parses successfully" + severity: P0 + action: reject_push + - condition: "A ticket claiming L4 closes while a kani harness fails" + severity: P0 + action: reject_push + - condition: "A ticket claiming L5 closes with any `sorry` in its Lean proof" + severity: P0 + action: reject_push + +falsification_tests: + - id: ladder_parse_strict_rejects_typos + rule: Strict parse rejects case/whitespace corruptions + prediction: parse("l4"), parse("L3 "), parse("strong") all fail strict parse + test: cargo test --lib parse_total_strict_prop + if_fails: misspelled levels silently downgrade tickets (C28) + - id: ladder_ord_matches_numeric + rule: Ord matches numeric order + prediction: L0 < L1 < L2 < L3 < L4 < L5 under derived Ord + test: cargo test --lib ord_matches_numeric + if_fails: gate comparisons invert and over-claims pass the completion gate + - id: ladder_serde_string_repr + rule: Wire format stays the display string (no wire break) + prediction: serde serializes L4 as "L4"; legacy contract.json still parses + test: cargo test --lib serde_string_repr + if_fails: every existing .pmat-work contract breaks on read + - id: ladder_gate_monotone + rule: complete implies achieved_level >= claimed level + prediction: a ticket claiming L4 without kani evidence blocks with LadderShortfall + test: cargo test --lib claim_l4_without_kani_blocks + if_fails: tickets close above their evidenced level (C28 completion gates are hard) diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-provenance-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-provenance-v1.yaml new file mode 100644 index 0000000000..d4f45f1c6e --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-provenance-v1.yaml @@ -0,0 +1,61 @@ +metadata: + version: "1.0.0" + created: "2026-07-02" + author: PAIML Engineering + references: + - "docs/specifications/components/modern-agentic-coding-support.md (MACS v1.0.0)" + - "arXiv:2602.22302 Bruni et al. (2026) Agent Behavioral Contracts" + - "Popper (1959) The Logic of Scientific Discovery" + - "Meyer (1997) Object-Oriented Software Construction 2e (Design by Contract)" + registry: true + description: > + Agent provenance and interruption events on falsification receipts. + Declared-first; detection advisory; hash-versioned for old receipts. + MACS F1 (docs/specifications/components/modern-agentic-coding-support.md). + contract: macs-provenance + status: draft + +equations: + provenance_roundtrip: + formula: deserialize(serialize(r)) = r + domain: r in FalsificationReceipt{schema_version=2} + codomain: bool + invariants: + - serialization uses canonical JSON (sorted keys, no NaN/Inf) + - unknown future fields are rejected (deny_unknown_fields) + preconditions: + - proptest deterministic mode enabled (MACS-000) + lean_theorem: Theorems.Macs.Provenance_Roundtrip + + hash_stability: + formula: content_hash(r) = golden(r) + domain: r in fixtures/receipts/v2/*.json + codomain: hex64 + invariants: + - byte-identical across OS/arch (CI matrix) + - v1 receipts verify under v1 rules keyed by schema_version + preconditions: + - fixtures committed at MACS-001 + + refusal_gates_completion: + formula: has_unacked(Refusal) -> complete(ticket) = Err(Blocked) + domain: ticket in .pmat-work/* + codomain: Result + invariants: + - Refusal never auto-acks; ack requires --ack-event with reason + preconditions: + - MACS-003 landed + +falsification: + - condition: "A receipt with schema_version=2 lacks `agent` yet CB-1651 passes" + severity: P0 + action: reject_push + - condition: "Two serializations of the same receipt differ in bytes" + severity: P0 + action: reject_push + - condition: "`pmat work complete` succeeds while an unacked Refusal event exists" + severity: P0 + action: reject_push + - condition: "A v1 (pre-MACS) receipt fails ledger verify after upgrade" + severity: P1 + action: block_release diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-skill-effort-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-skill-effort-v1.yaml new file mode 100644 index 0000000000..6c87758c03 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-skill-effort-v1.yaml @@ -0,0 +1,42 @@ +metadata: + version: "1.0.0" + created: "2026-07-02" + author: PAIML Engineering + references: + - "docs/specifications/components/modern-agentic-coding-support.md (MACS v1.0.0)" + - "arXiv:2602.22302 Bruni et al. (2026) Agent Behavioral Contracts" + - "Popper (1959) The Logic of Scientific Discovery" + - "Meyer (1997) Object-Oriented Software Construction 2e (Design by Contract)" + registry: true + description: > + Per-skill effort pinning: every repo skill pins a model-level effort in + frontmatter so behavior and cost are reproducible across sessions. + MACS F4 (docs/specifications/components/modern-agentic-coding-support.md). + contract: macs-skill-effort + status: draft + +equations: + skill_effort_pinned: + formula: forall f in skills -> effort(f) in {low, medium, high, xhigh} + domain: f in .claude/skills/**/{SKILL,skill}.md + codomain: bool + invariants: + - session-only values (max, ultracode) are rejected + - mechanical skills pin low/medium; adversarial skills pin high/xhigh + preconditions: + - frontmatter effort is honored by the harness (spec E4) + +falsification: + - condition: "A skill file lacks `effort:` frontmatter yet CB-1650 passes" + severity: P0 + action: reject_push + - condition: "A skill pins `max` or `ultracode` and CB-1650 stays green" + severity: P0 + action: reject_push + +falsification_tests: + - id: skill_effort_pinned + rule: every skill pins effort in {low,medium,high,xhigh} + prediction: a skill missing effort or pinning max/ultracode fails CB-1650 + test: cargo test --lib cb1650_red_on_session_only_values + if_fails: per-skill behavior varies run-to-run with session effort (E6) diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-sweep-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-sweep-v1.yaml new file mode 100644 index 0000000000..f2231f12fd --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/macs-sweep-v1.yaml @@ -0,0 +1,98 @@ +metadata: + version: "1.0.0" + created: "2026-07-02" + author: PAIML Engineering + references: + - "docs/specifications/components/modern-agentic-coding-support.md (MACS v1.0.0)" + - "arXiv:2602.22302 Bruni et al. (2026) Agent Behavioral Contracts" + - "Popper (1959) The Logic of Scientific Discovery" + - "Meyer (1997) Object-Oriented Software Construction 2e (Design by Contract)" + registry: true + description: > + LLM-free deterministic MCP sweep harness (pmat qa mcp-sweep) plus a + committed judgment workflow for anomalies only. + MACS F5 (docs/specifications/components/modern-agentic-coding-support.md). + contract: macs-sweep + status: draft + +equations: + sweep_deterministic: + formula: run1 = run2 modulo {timestamp, duration} fields + domain: runs of pmat qa mcp-sweep --format json + codomain: bool + invariants: + - anomaly ordering is stable (sorted by tool name) + preconditions: + - pmat binary built from working tree + + framing_pure: + formula: stdout_bytes subset_of JSONRPC_frames + domain: bytes on the MCP server stdout during a sweep + codomain: bool + invariants: + - no stray print/log bytes interleave JSON-RPC frames + preconditions: [] + + no_raw_resume: + formula: raw_resume_calls(release-sweep.ultracode.mjs) = 0 + domain: the committed ultracode judgment workflow, comments stripped + codomain: bool + invariants: + - durable state is .pmat-work receipts, never a session-bound continuation (MACS E7) + - exactly one spawnSubagent site, so PMAT_AGENT_* stamping is structural not observed + preconditions: + - contracts/workflows/release-sweep.ultracode.mjs is committed + + concurrency_safe: + formula: lock_errors = 0 and scratch_leftovers = 0 for N <= 8 + domain: N in {1..8} concurrent tool calls against one working tree + codomain: bool + invariants: + - advisory-lock discipline per src/services/metric_trends_core.rs + preconditions: [] + +falsification: + - condition: "Any LLM/model API call in the mcp-sweep code path" + severity: P0 + action: reject_push + - condition: "Two sweep runs differ beyond timestamp/duration fields" + severity: P0 + action: reject_push + - condition: "Orchestration layer calls a session-bound resume instead of .pmat-work receipts" + severity: P1 + action: reject_push + - condition: "Concurrency N<=8 produces lock errors or scratch leftovers" + severity: P0 + action: reject_push + +falsification_tests: + - id: sweep_args_derived_from_schema + rule: Minimal valid args derived from each tool inputSchema + prediction: every required property gets a type-appropriate placeholder; path fields get the target + test: cargo test --lib args_derived_from_every_tool_schema + if_fails: schema-derivable args need an LLM, defeating the LLM-free premise + - id: sweep_framing_pure + rule: stdout carries only JSON-RPC frames + prediction: framing_stray_lines flags any non-JSON stdout line + test: cargo test --lib framing_stdout_pure_jsonrpc_golden + if_fails: stray bytes corrupt the JSON-RPC stream (D82 class) + - id: sweep_deterministic + rule: two runs byte-identical modulo timestamp/duration + prediction: identical tool data serializes identically (report carries no wall-clock) + test: cargo test --lib two_runs_byte_identical_modulo_time + if_fails: the sweep is not replayable, breaking green-here-green-in-CI + - id: sweep_no_llm + rule: the sweep path links no model/API client + prediction: qa_mcp_sweep.rs references no LLM client symbols + test: cargo test --lib no_llm_symbols_linked + if_fails: stochastic tokens leak into a deterministic gate (E6) + - id: no_raw_resume + rule: the committed judgment workflow never relies on session-bound resume + prediction: workflow source (comments stripped) contains no `resume`; exactly one spawn site + test: cargo test --lib qa_mcp_sweep::tests::workflow_ + if_fails: durable state silently becomes session-bound, so a run cannot be reproduced by the team (E7) + - id: macs012_workflow_properties + rule: the four MACS-012 RED tests run where CI looks + prediction: workflow file is embedded at compile time; reads only the sweep artifact; stamps PMAT_AGENT_*; records refusals + test: cargo test --lib qa_mcp_sweep::tests::workflow_ + if_fails: the workflow's properties hold only until someone edits it, which is not enforcement diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-book-build-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-book-build-v1.yaml new file mode 100644 index 0000000000..3912c03c4c --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-book-build-v1.yaml @@ -0,0 +1,71 @@ +metadata: + version: "1.0.0" + created: "2026-04-08" + author: PAIML Engineering + registry: true + references: + - "../pmat-book/book.toml" + description: > + Provable contract for the PMAT mdBook build and GitHub Pages deployment. + 62 chapters covering CLI, MCP, TDG, quality gates, compliance, and + provable contracts. Falsification-first: every claim must be testable. + contract: pmat-book-build + status: enforced + title: "PMAT Book Build Contract" + tool: mdbook + source: "../pmat-book/" + config: "../pmat-book/book.toml" + summary: "../pmat-book/src/SUMMARY.md" + deploy: GitHub Pages via actions/deploy-pages@v4 + +invariants: + structure: + - "book.toml exists and is valid TOML" + - "src/SUMMARY.md exists and lists all chapters" + - "Every .md path in SUMMARY.md exists on disk" + - "No include directive references a nonexistent file" + + build: + - "mdbook build exits 0 with zero ERROR lines" + - "book/index.html exists after build" + - "No missing font errors in build output" + + content: + - "SUMMARY.md spine covers chapters 1-62" + - "No stub-only chapters (every chapter has substantive content)" + - "Critical chapters (5, 7, 13, 14) pass validate-book" + - "All pmat CLI examples in chapters are valid commands" + + ci: + - "Workflow triggers on push to main for pmat-book/** changes" + - "mdBook version pinned to specific release" + +preconditions: + - "mdbook binary available" + - "All chapter .md files committed to git" + - "pmat binary installed (for CLI example validation)" + +postconditions: + - "book/ contains valid HTML site" + - "GitHub Pages deployment succeeds on push to main" + +falsification: + - condition: "mdbook build exits non-zero" + severity: P0 + action: reject_push + - condition: "SUMMARY.md references nonexistent .md file" + severity: P0 + action: reject_push + - condition: "Critical chapter (5, 7, 13, 14) fails validate-book" + severity: P0 + action: reject_push + - condition: "Chapter contains pmat command that exits non-zero" + severity: P1 + action: fix_example + - condition: "SUMMARY.md contains stub-only chapter (< 50 words)" + severity: P0 + action: delete_or_expand + +verification: + local: "make validate-book" + ci: ".github/workflows/book.yml" diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-book-ch05-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-book-ch05-v1.yaml new file mode 100644 index 0000000000..2d4953dd20 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-book-ch05-v1.yaml @@ -0,0 +1,39 @@ +metadata: + version: "1.0.0" + created: "2026-04-08" + author: PAIML Engineering + references: + - "../pmat-book/src/ch05-00-analyze-suite.md (the chapter this contract gates)" + - "Makefile target validate-book (runs the chapter's test scripts)" + registry: true + description: > + PMAT-BOOK Chapter 5: The Analyze Command Suite + contract: pmat-book-ch05 + status: enforced + title: "The Analyze Command Suite" + part: III + chapter_file: ch05-00-analyze-suite.md + +preconditions: + - "pmat binary installed and in PATH" + - "pmat analyze complexity --help exits 0" + - "pmat analyze dead-code --help exits 0" + - "pmat analyze satd --help exits 0" + +postconditions: + - "All pmat analyze subcommands documented" + - "Example output matches current pmat version" + +falsification: + - condition: "pmat analyze complexity --path . exits non-zero on test project" + severity: P0 + action: reject_chapter + - condition: "pmat analyze dead-code --path . exits non-zero on test project" + severity: P0 + action: reject_chapter + - condition: "Chapter claims feature that doesn't exist in current pmat --help" + severity: P0 + action: reject_chapter + - condition: "Example output doesn't match pmat version in Cargo.toml" + severity: P1 + action: update_examples diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-book-ch07-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-book-ch07-v1.yaml new file mode 100644 index 0000000000..d6cc630301 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-book-ch07-v1.yaml @@ -0,0 +1,37 @@ +metadata: + version: "1.0.0" + created: "2026-04-08" + author: PAIML Engineering + references: + - "../pmat-book/src/ch07-00-quality-gates.md (the chapter this contract gates)" + - "src/cli/handlers/quality_gate_handlers.rs (the command the chapter documents)" + registry: true + description: > + PMAT-BOOK Chapter 7: Quality Gates + contract: pmat-book-ch07 + status: enforced + title: "Quality Gates" + part: III + chapter_file: ch07-00-quality-gate.md + +preconditions: + - "pmat binary installed and in PATH" + - ".pmat-metrics/ directory exists (from make lint/test-fast)" + +postconditions: + - "O(1) quality gate concept explained with timing evidence" + - "Pre-commit hook configuration documented" + +falsification: + - condition: "Pre-commit hook takes >45ms on cached metrics" + severity: P0 + action: reject_chapter + - condition: "Chapter references .pmat-metrics.toml thresholds that don't match actual file" + severity: P0 + action: reject_chapter + - condition: "make lint exits non-zero" + severity: P0 + action: reject_chapter + - condition: "Chapter claims metric name not in .pmat-metrics.toml" + severity: P1 + action: fix_claim diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-book-ch13-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-book-ch13-v1.yaml new file mode 100644 index 0000000000..9733b9ff0e --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-book-ch13-v1.yaml @@ -0,0 +1,39 @@ +metadata: + version: "1.0.0" + created: "2026-04-08" + author: PAIML Engineering + references: + - "../pmat-book/src/ch13-00-multi-language.md (the chapter this contract gates)" + - "CLAUDE.md (Chapter 13 is CRITICAL - must always pass)" + registry: true + description: > + PMAT-BOOK file:ch13 — Multi-Language Project Examples (CRITICAL) + contract: pmat-book-ch13 + status: enforced + title: "Multi-Language Project Examples" + part: IV + chapter_file: ch13-00-language-examples.md + critical: true + +preconditions: + - "pmat binary installed with core-languages feature" + - "pmat context --help exits 0" + - "Test fixtures for Rust, Python, TypeScript, Go, C exist" + +postconditions: + - "All documented languages produce valid AST output" + - "pmat context on multi-language project succeeds" + +falsification: + - condition: "pmat context fails on multi-language test fixture" + severity: P0 + action: reject_chapter + - condition: "Chapter claims language support that is not in --features list" + severity: P0 + action: reject_chapter + - condition: "Language listed in chapter but tree-sitter parser not in Cargo.toml" + severity: P0 + action: reject_chapter + - condition: "Example AST output for any language doesn't match current parser" + severity: P1 + action: update_examples diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-book-ch35-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-book-ch35-v1.yaml new file mode 100644 index 0000000000..40679525b7 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-book-ch35-v1.yaml @@ -0,0 +1,42 @@ +metadata: + version: "1.0.0" + created: "2026-04-08" + author: PAIML Engineering + references: + - "../pmat-book/src/ch35-00-semantic-search.md (the chapter this contract gates)" + - "docs/specifications/semantic-search-feature.md" + registry: true + description: > + PMAT-BOOK file:ch35 — Semantic Search and Code Clustering (pmat query) + contract: pmat-book-ch35 + status: enforced + title: "Semantic Search and Code Clustering" + part: VI + chapter_file: ch35-00-semantic-search.md + critical: true + +preconditions: + - "pmat binary installed and in PATH" + - ".pmat/context.db exists (index built)" + +postconditions: + - "All pmat query flags documented" + - "Contract metadata enrichment (PV:L2) shown in examples" + - "Coverage gaps mode documented with examples" + +falsification: + - condition: "pmat query 'test' --limit 1 exits non-zero" + severity: P0 + action: reject_chapter + - condition: "Chapter documents --flag that doesn't exist in pmat query --help" + severity: P0 + action: reject_chapter + - condition: "Chapter omits PV: contract enrichment (new in v3.12)" + severity: P1 + action: add_section + - condition: "Example query output doesn't show TDG grade" + severity: P1 + action: update_examples + - condition: "Chapter claims O(1) but query latency >500ms on cached index" + severity: P2 + action: investigate_perf diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-core.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-core.yaml new file mode 100644 index 0000000000..5ab0b72e6c --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-core.yaml @@ -0,0 +1,287 @@ +metadata: + version: 1.0.0 + created: '2026-04-05' + author: PAIML Engineering + description: PMAT core contract — commit-level enforcement invariants + references: + - docs/specifications/components/commit-level-contract-enforcement.md + - src/models/comply_config_types.rs +equations: + refresh_bindings: + formula: binding_index = scan(contracts/, binding.yaml) + domain: project_path is Path, project_path.exists() + codomain: binding_index in HashMap> + invariants: + - binding_index.is_object() (valid JSON) + - file_count <= binding_count (each file has at least 1 binding) + preconditions: + - binding_count.len() > 0 + lean_theorem: Theorems.Refresh_Bindings_Monotonic + # ── Declared because 1,187 annotation sites already assert them. + # + # `path_exists` (1,174 sites), `non_empty_index` (10) and `lint_valid` (3) + # were cited by `#[provable_contracts_macros::contract("pmat-core.yaml", + # equation = "...")]` throughout src/ and were NEVER DECLARED HERE. The macro + # accepts any string, build.rs emits CONTRACT_*=bound without opening the + # contract, and nothing checked that the name resolved — so 1,187 functions + # carried what reads as a proof obligation and bound to nothing. + # + # They are declared rather than deleted because the annotations are not + # decorative: the functions carrying `path_exists` really do require the path + # to exist, and several assert it at runtime (`debug_assert!(path.exists())`, + # e.g. check_individual_basic.rs). Writing the equation down turns 1,187 dead + # citations into live ones; deleting them would discard a precondition the + # code actually holds. + # + # `contracts/pv-registry-integrity-v1.yaml` is what stops the next phantom. + path_exists: + formula: "analysed(path) implies exists(path) and readable(path)" + domain: "path in Path" + codomain: "Result where E names the path" + invariants: + - "a path that does not exist is REFUSED, never reported as an empty result" + - "the refusal names the path, so a typo is distinguishable from a clean tree" + preconditions: + - "the caller has resolved the path (no further canonicalisation is implied)" + + non_empty_index: + formula: "query(index) requires count(index) > 0" + domain: "index in AgentContextIndex" + codomain: "Result, E>" + invariants: + - "a query against an empty or absent index is REFUSED, not answered with zero hits" + - "zero hits from a populated index is a measurement; zero hits from no index is not" + preconditions: + - "the index path is known to the caller" + + lint_valid: + formula: "lint(source) yields findings union refusal, never silence" + domain: "source in Path" + codomain: "Result" + invariants: + - "a linter that could not run is REFUSED, never reported as zero findings" + - "findings are counted BEFORE any confidence or severity filter is applied" + + check_compliance: + formula: check_result = run_cb_checks(project_path) + domain: project_path in Path + codomain: check_result in {Pass, Warn, Fail, Skip} + invariants: + - check_result != Panic (checks never panic) + - cached_data.is_fresh() implies latency < 45ms + preconditions: + - checks.len() > 0 + lean_theorem: Theorems.Check_Determinism + severity_resolution: + formula: > + severity(id) = (patch(id) applied over declared(id)).severity, where + declared = default_checks() and patch = the `comply.checks:` map read from + .pmat.yaml + domain: > + a check id, the declared check roster, and a possibly-partial + `comply.checks:` map + codomain: severity in {info, warning, error, critical} + invariants: + - "PMAT-630: a partial `checks:` map is a PATCH over the declared roster, never a replacement" + - "an id ABSENT from the map keeps the severity and threshold the code declared for it" + - "an entry that omits `severity:` keeps the declared severity; omitting `threshold:` keeps the declared threshold" + - "an explicit `severity:` still overrides the declared one: merging must not make the config unwritable" + - "an id the roster does not declare is added, not rejected: merging adds, it does not restrict" + - "an unknown key inside a check entry is a parse ERROR, never a silently discarded override" + - "a `comply:` section with no `checks:` key keeps the whole declared roster, not an empty map" + - "the resolved roster is never smaller than the declared one" + preconditions: + - ".pmat.yaml parses as YAML" + score_range: + formula: score = hit_rate * throughput + domain: hit_rate in [0.0, 1.0], throughput in [0.0, +inf) + codomain: score in [0.0, +inf) + invariants: + - score >= 0.0 (non-negative product of non-negative factors) + preconditions: + - hit_rate >= 0.0 + - throughput >= 0.0 + lean_theorem: Theorems.Score_Range_NonNeg + composite_bounded: + formula: composite = geometric_mean(sub_scores) + domain: sub_scores in Vec, all >= 0.0 + codomain: composite in [0.0, 100.0] + invariants: + - composite >= 0.0 (geometric mean of non-negatives) + - composite <= max(sub_scores) (geometric mean never exceeds max) + preconditions: + - sub_scores.len() > 0 + lean_theorem: Theorems.Geometric_Mean_Bounded + rps_category_sum: + formula: total = sum(category.earned for category in categories) + domain: categories in Vec + codomain: total in [0.0, 289.0] + invariants: + - total >= 0.0 + - total <= sum(category.max for category in categories) + preconditions: + - categories.len() == 11 + lean_theorem: Theorems.Category_Sum_Bounded + tdg_grade_monotonic: + formula: grade = score_to_grade(tdg_score) + domain: tdg_score in [0.0, 100.0] + codomain: grade in {A, B, C, D, F} + invariants: + - tdg_score >= threshold implies grade >= corresponding_grade + - higher score always yields same or better grade + preconditions: + - tdg_score >= 0.0 + lean_theorem: Theorems.TDG_Grade_Monotonic + index_load_idempotent: + formula: index = load(save(index)) + domain: index in AgentContextIndex + codomain: loaded_index in AgentContextIndex + invariants: + - loaded_index.functions.len() == index.functions.len() + - loaded_index.file_index.keys() == index.file_index.keys() + preconditions: + - index.functions.len() > 0 + lean_theorem: Theorems.Index_Roundtrip_Identity + bm25_relevance_positive: + formula: score = bm25(query, document, corpus) + domain: query in String, document in String, corpus in Vec + codomain: score in [0.0, +inf) + invariants: + - score >= 0.0 (BM25 produces non-negative scores) + - empty query implies score == 0.0 + preconditions: + - corpus.len() > 0 + lean_theorem: Theorems.BM25_Non_Negative + pagerank_convergent: + formula: ranks = pagerank(graph, damping, iterations) + domain: graph in CSRGraph, damping in (0.0, 1.0), iterations in N+ + codomain: ranks in Vec, sum(ranks) approx 1.0 + invariants: + - all(r >= 0.0 for r in ranks) + - sum(ranks) in [0.99, 1.01] (convergence within epsilon) + preconditions: + - graph.num_nodes() > 0 + - damping > 0.0 && damping < 1.0 + lean_theorem: Theorems.PageRank_Convergent + muda_waste_bounded: + formula: muda = compute_muda(project) + domain: project in ProjectAnalysis + codomain: muda in [0.0, 100.0] + invariants: + - muda >= 0.0 (waste score non-negative) + - muda <= 100.0 (bounded) + preconditions: + - project.files.len() > 0 + lean_theorem: Theorems.Muda_Score_Bounded +proof_obligations: +- type: invariant + property: TDG baseline valid JSON after every commit + formal: is_json(.pmat/baseline.json) + applies_to: all + check: contract_tests::FALSIFY-PMAT-001 + severity: ERROR +- type: invariant + property: Pre-commit hook < 30s + formal: hook_duration < 30.seconds + applies_to: all + check: contract_tests::FALSIFY-PMAT-002 + severity: ERROR +- type: bound + property: CB checks handle missing files gracefully + formal: file.missing() implies check_result == Skip + applies_to: all + check: contract_tests::FALSIFY-PMAT-003 + severity: WARNING +falsification_tests: +- id: FALSIFY-PMAT-001 + rule: TDG baseline valid JSON + prediction: .pmat/baseline.json is parseable as JSON after every commit + test: Post-commit hook verifies JSON parses, fails commit if corrupt + if_fails: TDG baseline corrupted, subsequent commits cannot compute drift +- id: FALSIFY-PMAT-002 + rule: Pre-commit hook performance budget + prediction: hook completes in under 30 seconds + test: .pmat-metrics/hook-timing.jsonl shows p95 < 30000ms over 10 runs + if_fails: Hook exceeds budget, developers bypass with --no-verify +- id: FALSIFY-PMAT-003 + rule: CB check graceful degradation + prediction: missing files produce Skip status, never panic + test: cargo test --lib -- check_handlers -- with tempdir (no files) + if_fails: Check panics on missing input, breaks CI for clean projects +kani_harnesses: +- id: KANI-PMAT-001 + obligation: geometric mean of non-negatives stays within [0, max] + property: composite in [0.0, 100.0] for all sub_scores >= 0.0 + bound: 8 + strategy: bounded_int + solver: cadical + harness: verify_geometric_mean_bounded +- id: KANI-PMAT-002 + obligation: geometric mean of an empty set is undefined, not zero + property: sub_scores.len() == 0 implies no composite is produced + bound: 8 + strategy: bounded_int + solver: cadical + harness: verify_geometric_mean_empty +- id: KANI-PMAT-003 + obligation: geometric mean of identical values is that value + property: all(s == k) implies composite == k + bound: 8 + strategy: bounded_int + solver: cadical + harness: verify_geometric_mean_identity +- id: KANI-PMAT-004 + obligation: a zero sub-score absorbs the geometric mean + property: any(s == 0.0) implies composite == 0.0 + bound: 8 + strategy: bounded_int + solver: cadical + harness: verify_geometric_mean_zero_absorbing +- id: KANI-PMAT-005 + obligation: impact score is never negative + property: score >= 0.0 for all non-negative factors + bound: 8 + strategy: bounded_int + solver: cadical + harness: verify_impact_score_non_negative +- id: KANI-PMAT-006 + obligation: zero missed lines yields zero impact + property: missed == 0 implies score == 0.0 + bound: 8 + strategy: bounded_int + solver: cadical + harness: verify_impact_score_zero_missed +- id: KANI-PMAT-007 + obligation: impact is monotonic in missed lines + property: missed_a <= missed_b implies score_a <= score_b + bound: 8 + strategy: bounded_int + solver: cadical + harness: verify_impact_score_monotonic_missed +- id: KANI-PMAT-008 + obligation: RRF score stays within its bound + property: rrf in (0.0, 1.0] for rank >= 0 + bound: 8 + strategy: bounded_int + solver: cadical + harness: verify_rrf_score_bounded +- id: KANI-PMAT-009 + obligation: RRF decreases monotonically with rank + property: rank_a < rank_b implies rrf_a > rrf_b + bound: 8 + strategy: bounded_int + solver: cadical + harness: verify_rrf_score_monotonic_decreasing +- id: KANI-PMAT-010 + obligation: rank zero maximises RRF + property: argmax(rrf) == 0 + bound: 8 + strategy: bounded_int + solver: cadical + harness: verify_rrf_rank_zero_maximum +verification_summary: + total_obligations: 3 + l2_property_tested: 3 + l3_kani_proved: 10 + l4_lean_proved: 0 + l4_sorry_count: 0 diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-install-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-install-v1.yaml new file mode 100644 index 0000000000..f72a67affb --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-install-v1.yaml @@ -0,0 +1,82 @@ +metadata: + version: "1.0.0" + created: "2026-05-05" + author: PAIML Engineering + registry: true + description: > + Provable contract for scripts/install.sh — the documented one-liner + installer (curl ... | sh). Captures the URL pattern, tarball layout, + and Linux platform default that must hold for `curl ... | sh` to + succeed on every supported base image. + contract: pmat-install + status: enforced + references: + - scripts/install.sh + - https://github.com/paiml/paiml-mcp-agent-toolkit/issues/561 + +equations: + url_pattern_matches_release_assets: + formula: download_url == "https://github.com/${REPO}/releases/download/v${VERSION}/pmat-v${VERSION}-${PLATFORM}.tar.gz" + domain: VERSION in semver, PLATFORM in supported_targets + codomain: download_url returns HTTP 302 (redirect to actual asset) + invariants: + - download_url contains "pmat-v${VERSION}-${PLATFORM}.tar.gz" + - download_url does NOT contain the legacy "paiml-mcp-agent-toolkit-" prefix + preconditions: + - Release v${VERSION} exists with the matching asset + lean_theorem: Theorems.Install_URL_Matches_Asset + + tarball_extraction_handles_subdirectory: + formula: tar -xzf archive.tar.gz -C $TMP_DIR --strip-components=1 + domain: archive.tar.gz contains "pmat-v${VERSION}-${PLATFORM}/pmat" + codomain: $TMP_DIR/pmat exists and is executable + invariants: + - Binary lands at $INSTALL_DIR/pmat (NOT nested under a subdirectory) + - $TMP_DIR/pmat is regular file after extraction + preconditions: + - download succeeded + - tar binary available + lean_theorem: Theorems.Install_Tarball_Flat + + default_linux_platform_is_musl_for_glibc_independence: + formula: PLATFORM == "${arch}-unknown-linux-musl" when uname -s == "Linux" + domain: arch in {x86_64, aarch64}, os == Linux + codomain: PLATFORM ends with "-musl" + invariants: + - musl variant is static-pie (file output contains "static-pie linked") + - musl variant runs on Ubuntu 22.04 (GLIBC 2.35) without GLIBC_2.39 errors + preconditions: + - musl-target asset exists in the release + lean_theorem: Theorems.Install_Musl_Default + +preconditions: + - "GitHub release v${VERSION} exists" + - "Release contains pmat-v${VERSION}-x86_64-unknown-linux-musl.tar.gz asset" + - "Release contains pmat-v${VERSION}-aarch64-unknown-linux-musl.tar.gz asset" + - "curl, tar, sh available on the target machine" + +postconditions: + - "$INSTALL_DIR/pmat is an executable producing 'pmat ${VERSION}' on --version" + - "Binary is static-pie (no GLIBC dependency)" + - "INSTALL_DIR env var is honoured if set, falls back to ~/.local/bin" + +falsification: + - condition: "DOWNLOAD_URL still references 'paiml-mcp-agent-toolkit-' prefix" + severity: P0 + action: reject_push + - condition: "Linux platform detection returns a -gnu target instead of -musl" + severity: P0 + action: reject_push + - condition: "tar extraction fails to flatten subdirectory (binary not at $TMP_DIR/pmat)" + severity: P0 + action: reject_push + - condition: "INSTALL_DIR env var is overwritten by unconditional assignment" + severity: P1 + action: reject_push + - condition: "Smoke test `INSTALL_DIR=/tmp/x bash install.sh v${VERSION}` exits non-zero" + severity: P0 + action: reject_push + +verification: + local: "INSTALL_DIR=/tmp/pmat-smoke bash scripts/install.sh v${VERSION} && /tmp/pmat-smoke/pmat --version" + ci: ".github/workflows/binary-release.yml" diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-no-fabrication-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-no-fabrication-v1.yaml new file mode 100644 index 0000000000..901dfd58a9 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-no-fabrication-v1.yaml @@ -0,0 +1,213 @@ +metadata: + version: "1.0.0" + created: "2026-07-31" + author: PAIML Engineering + references: + - "docs/specifications/documentation-accuracy-enforcement.md (zero-hallucination policy)" + - "CHANGELOG.md v3.28.0 (analyze comprehensive no longer fabricates its results)" + - "CHANGELOG.md v3.27.0 (every falsification claim now measures something)" + - "CHANGELOG.md v3.26.0 (falsification ladder audit: 14 of 22 claims verified nothing)" + - "scripts/dogfood-release.py (interface-coverage harness that detects fabricated output)" + - "Popper (1959) The Logic of Scientific Discovery" + registry: true + description: > + The anti-fabrication contract for pmat's analysis surface. pmat is a code + QUALITY tool: its output is consumed as evidence, and increasingly by LLMs + that cannot tell a measurement from a placeholder. A number pmat prints is + therefore a claim about the input, and a claim that was never computed from + the input is a fabrication regardless of how plausible it looks. + + This contract exists because the 3.29.0 pre-release dogfood found nine + blocker-level fabrications shipped simultaneously: a Quality Scorecard whose + health/maintainability/coverage were compile-time constants identical for an + empty directory and the whole pmat repo; a provability analyzer that emitted + the same two phantom functions (main@1, test@10) for every file; a + proof-annotation command that emitted ten annotations for files that exist + nowhere, each with a freshly generated UUID and a current-time dateVerified; + a duplicate detector whose DEFAULT mode returned zero on byte-identical + files; and analysis commands that exited 0 with a score for paths that do + not exist. + + The unifying failure is that nothing in the build, the test suite, or CI + distinguishes "computed from the input" from "constant that type-checks". + These equations make that distinction falsifiable. + contract: pmat-no-fabrication + status: active + +equations: + measured_or_absent: + formula: > + forall m in reported_metrics(cmd, input): + derived_from(m, input) or marked_unmeasured(m) + domain: cmd in AnalysisCommand, input in Path + codomain: Report + invariants: + - a metric that is a compile-time constant is NEVER rendered as a measurement + - a metric that cannot be measured in the command's scope is rendered as an + explicit unmeasured marker (None / "not measured" / omitted field), never + as a plausible default + - varying the input must be able to vary every reported metric; a metric + that is invariant across all inputs is by definition not measuring the input + - "test coverage is only reported when a coverage run actually produced it" + preconditions: + - the command declares which metrics it computes + + output_derived_from_input: + formula: > + analyze(empty_input) = empty_result and analyze(i) = f(files_read(i)) + domain: i in Path + codomain: AnalysisResult + invariants: + - an input containing zero analysable files yields zero findings + - every finding references a file that was actually read from the given path + - no finding names a path that does not exist on disk + - result size is a function of input size, not a constant + preconditions: + - the path exists and is readable + + missing_path_fails: + formula: "not exists(p) -> analyze(p) = Err(PathNotFound) and exit_code != 0" + domain: p in Path + codomain: Result + invariants: + - a nonexistent path NEVER yields exit 0 + - a nonexistent path NEVER yields a score, grade, percentage or "PASSED" + - the failure names the offending path + preconditions: [] + + detection_mode_superset: + formula: "results(All) >= union(results(m) for m in sub_modes)" + domain: mode in DetectionType + codomain: Vec + invariants: + - "|results(All)| >= |results(m)| for every single mode m" + - the documented default mode can never report fewer findings than an + explicitly selected sub-mode on the same input + - byte-identical files are reported as duplicates under the default mode + preconditions: + - at least one sub-mode is implemented + + source_location_fidelity: + formula: "reported_line(item) = true_source_line(item)" + domain: item in AstItem + codomain: LineNumber + invariants: + - a reported line is the item's line in its source file + - a reported line is NEVER an enumeration index or array position + - line numbers agree across commands analysing the same file + - reported_line <= file_line_count + preconditions: + - the AST layer carries the true line + + bounded_time_arithmetic: + formula: "clamp(now - days(n)) is defined for all n in u32" + domain: n in u32 (user-supplied lookback) + codomain: DateTime + invariants: + - no user-supplied integer reaches unchecked DateTime arithmetic + - an out-of-range lookback is clamped or rejected, never panicked on + - the process never terminates by SIGABRT on user input + preconditions: [] + + session_survives_recoverable_frame: + formula: > + recoverable(err) -> (emit_jsonrpc_error(err) and session_alive) + and connection_closed(err) -> session_ends + domain: err in TransportError + codomain: SessionState + invariants: + - a malformed frame, unknown method or invalid params yields a JSON-RPC + error response and the session continues to serve later requests + - only ConnectionClosed or a genuine IO failure ends the session + - the server NEVER exits 0 silently mid-conversation + - requests already accepted are still answered before the session ends + preconditions: + - the transport distinguishes recoverable errors from connection loss + +falsification: + - condition: "A reported metric is identical for an empty directory and a large real repository" + severity: P0 + action: reject_push + - condition: "An analysis result names a file path that does not exist on disk" + severity: P0 + action: reject_push + - condition: "A command exits 0 and prints a score for a path that does not exist" + severity: P0 + action: reject_push + - condition: "A proof/verification artifact is emitted with a current-time dateVerified without a verification having run" + severity: P0 + action: reject_push + - condition: "The default detection mode returns fewer findings than a sub-mode on identical input" + severity: P0 + action: reject_push + - condition: "A reported source line is an enumeration index rather than the true line" + severity: P1 + action: reject_push + - condition: "User-supplied integer input causes a panic or SIGABRT" + severity: P0 + action: reject_push + - condition: "An MCP session terminates silently with exit 0 on a recoverable protocol error" + severity: P0 + action: reject_push + +falsification_tests: + - id: FALSIFY-NOFAB-001 + rule: A reported metric must vary with the input, or be marked unmeasured + prediction: > + Running a metric-reporting command against an empty directory and against a + populated project yields different values, OR the metric is rendered as an + explicit unmeasured marker. Before the fix, `pmat context` reported + Overall Health 85.0 / Maintainability Index 70.0 / Test Coverage 65.0 for + BOTH an empty directory and the 3252-file pmat repo. + test: cargo test --lib no_fabrication::metric_varies_with_input + if_fails: pmat reports compile-time constants as measurements + - id: FALSIFY-NOFAB-002 + rule: An empty input yields an empty result + prediction: > + `analyze proof-annotations` on an empty directory reports zero annotations. + Before the fix it emitted ten annotations naming borrow_checker_0.rs and + other files that exist nowhere on the filesystem. + test: cargo test --lib no_fabrication::empty_input_empty_result + if_fails: pmat invents findings for input it never read + - id: FALSIFY-NOFAB-003 + rule: A nonexistent path fails loudly + prediction: > + Every analysis command given a path that does not exist exits non-zero and + names the path. Before the fix, `analyze proof-annotations`, `cuda-tdg`, + `analyze comprehensive`, `analyze dag`, `analyze provability`, + `analyze defect-prediction` and `analyze defects` all exited 0, some + printing "Quality Score: 100.0%" or "Gateway: PASSED". + test: cargo test --lib no_fabrication::missing_path_is_an_error + if_fails: a typo in a path silently produces a passing quality report + - id: FALSIFY-NOFAB-004 + rule: The default duplicate-detection mode is a superset of every sub-mode + prediction: > + On two byte-identical files, the default (`--detection-type all`) reports + at least as many duplicates as `--detection-type exact`. Before the fix, + exact found 13 and all found 0 on the same input (124 vs 0 on real source). + test: cargo test --lib no_fabrication::all_mode_is_superset + if_fails: the default invocation reports clean code as clean when it is duplicated + - id: FALSIFY-NOFAB-005 + rule: Reported source lines are real source lines + prediction: > + `context --format json` reports the same line for a function as + `quality-gate` does for the same file. Before the fix, context emitted + 1,2,3,4,5 (enumeration indices) where the true lines were 6,651,776,777,778. + test: cargo test --lib no_fabrication::line_numbers_are_source_lines + if_fails: any tool consuming context JSON to jump to code lands in the wrong place + - id: FALSIFY-NOFAB-006 + rule: User-supplied lookback never panics + prediction: > + `analyze churn -d N` terminates normally for every u32 N, including + 2147483647 and 4294967295. Before the fix, N >= ~100000000 aborted with + SIGABRT at src/services/git_analysis.rs:65 ("`DateTime - TimeDelta` overflowed"). + test: cargo test --lib no_fabrication::churn_lookback_is_clamped + if_fails: a user-supplied integer crashes the process with a core dump + - id: FALSIFY-NOFAB-007 + rule: A recoverable MCP frame does not end the session + prediction: > + An unknown method or malformed JSON line produces a JSON-RPC error and the + NEXT valid request is still answered. Before the fix, the session died + silently with exit 0 and every subsequent request was lost. + test: cargo test --lib no_fabrication::session_survives_bad_frame + if_fails: an MCP host sees the server "succeed" and vanish mid-conversation diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-quality-acceptance-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-quality-acceptance-v1.yaml new file mode 100644 index 0000000000..b8b98f6fc0 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-quality-acceptance-v1.yaml @@ -0,0 +1,83 @@ +metadata: + version: "1.0.0" + created: "2026-07-05" + author: PAIML Engineering + references: + - "docs/specifications/pmat-work-contract.md (Popperian falsification, PMAT-458)" + - "Popper (1959) The Logic of Scientific Discovery" + - "src/cli/handlers/work_contract_falsification.rs (FalsificationMethod: 23 quality guards)" + - "src/cli/handlers/work_quality_handlers.rs (run_popper_falsification: suite dispatch)" + registry: true + description: > + The quality-acceptance contract every pmat work ticket is completed under + (PMAT-458, Popperian falsification). A ticket is accepted iff NO quality + guard in FalsificationMethod can falsify it against the ticket's baseline + snapshot. This is exactly the invariant `pmat work complete` enforces via + run_popper_falsification. Binding a completed ticket to this contract + records the falsifiable suite that governed its acceptance (evidence + level L3: a bound equation whose falsification_tests execute and pass). + contract: pmat-quality-acceptance + status: active + +equations: + quality_acceptance: + formula: > + complete(t) = Ok(()) <-> forall g in guards: attempt_falsify(g, t) = NotFalsified + domain: t in .pmat-work/* (a work ticket carrying a baseline snapshot) + codomain: Result<()> + invariants: + - guards = FalsificationMethod::{ManifestIntegrity, DifferentialCoverage, + AbsoluteCoverage, TdgRegression, ComplexityRegression, FileSizeRegression, + SpecQuality, RoadmapUpdate, GitHubSync, CoverageGaming, SupplyChainIntegrity, + MetaFalsification, ExamplesCompile, BookValidation, SatdDetection, + DeadCodeDetection, PerFileCoverage, LintPass, VariantCoverage, FixChainLimit, + CrossCrateParity, RegressionGate, FormalProofVerification} — 23 independent guards + - acceptance requires EVERY guard to return NotFalsified; a single + falsified guard blocks completion (conjunction, not sampling) + - each guard is independently executable and independently falsifiable + against the baseline (evidence that the suite is not a rubber stamp) + preconditions: + - a baseline snapshot is captured at work start (contract.json baseline_commit, + baseline_file_manifest, baseline_coverage, baseline_tdg, baseline_rust_score) + +falsification: + - condition: "A ticket closes while any FalsificationMethod guard reports FAILED" + severity: P0 + action: reject_completion + - condition: "A guard reports PASSED without executing its check (rubber-stamp acceptance)" + severity: P0 + action: reject_completion + - condition: "The guard set silently drops a variant (acceptance narrows unnoticed)" + severity: P1 + action: reject_push + +falsification_tests: + - id: acceptance_requires_all_guards + rule: A ticket is accepted only if every quality guard returns NotFalsified + prediction: > + `pmat work falsify ` dispatches all 23 FalsificationMethod guards via + run_popper_falsification; if any guard finds a violation it reports FAILED + and completion is blocked. (Observed live: baseline-v1 fails ManifestIntegrity + and SupplyChainIntegrity — the suite genuinely falsifies.) + test: cargo run --bin pmat -- work falsify + if_fails: a ticket could complete with an unchecked or violated quality guard + - id: falsification_result_passed_semantics + rule: A guard that finds no counterexample yields PASSED (NotFalsified) + prediction: FalsificationResult::passed() reports success carrying no evidence + test: cargo test --lib test_falsification_result_passed + if_fails: passing guards misreport, allowing false acceptances or false rejections + - id: falsification_result_failed_semantics + rule: A guard that finds a counterexample yields FAILED carrying the evidence + prediction: FalsificationResult::failed() reports failure with the falsifying evidence + test: cargo test --lib test_falsification_result_failed + if_fails: a violated guard could be recorded as passing (rubber-stamp) + - id: falsification_coverage_accounting + rule: Falsification coverage is computed from evaluated-vs-total claims, never assumed + prediction: an empty claim set yields 0 coverage; a partial set yields proportional coverage + test: cargo test --lib test_compute_falsification_coverage_empty test_compute_falsification_coverage_partial + if_fails: acceptance could be declared while claims were never evaluated + - id: ladder_blocks_hollow_upgrade + rule: kani/lean declared without falsification_tests caps at L2 (no hollow L3+) + prediction: a YAML with kani_harnesses but no falsification_tests evidences L2, never L3 + test: cargo test --lib kani_without_falsification_tests_caps_at_l2 + if_fails: tickets earn L3+ without a real falsifiable test actually present diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-query-search-modes-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-query-search-modes-v1.yaml new file mode 100644 index 0000000000..a77ff0b5e3 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/pmat-query-search-modes-v1.yaml @@ -0,0 +1,97 @@ +metadata: + version: "1.0.0" + created: "2026-05-06" + author: PAIML Engineering + registry: true + description: > + Provable contract for the `pmat query --search-mode {semantic,lexical,hybrid}` + flag (issue #562). Captures the three teaching invariants that made the + flag necessary: (a) all three modes return results against a fresh + fixture corpus, (b) the lexical path requires NO embedding index (no + `pmat embed sync` gate), (c) the default `pmat query` (no flag) is + behaviour-identical to the v3.16.0 baseline. + contract: pmat-query-search-modes + status: enforced + references: + - https://github.com/paiml/paiml-mcp-agent-toolkit/issues/562 + - src/cli/handlers/query_handler/query_execution.rs + - src/cli/commands/commands_enum/definition.rs + +equations: + all_three_modes_return_results: + formula: results = pmat_query(query, search_mode in {semantic, lexical, hybrid}, corpus) + domain: query is non-empty String, corpus has >= 1 indexed function matching query + codomain: results in Vec, len >= 1 + invariants: + - lexical mode returns >= 1 result for canonical "literal" query against fixture corpus + - semantic mode returns >= 1 result for canonical "literal" query against fixture corpus + - hybrid mode returns >= 1 result for canonical "literal" query against fixture corpus + preconditions: + - corpus has been indexed via `pmat query --rebuild-index` + lean_theorem: Theorems.Search_Modes_NonEmpty + + lexical_does_not_require_embeddings: + formula: lexical_results = pmat_query(query, lexical, corpus_without_embeddings) + domain: corpus_without_embeddings has NO `.pmat/embeddings.idx` file + codomain: lexical_results in Vec, no error + invariants: + - "`--search-mode lexical` works on a fresh repo with no embeddings index" + - "lexical engine path uses calculate_literal_scores against name+signature+source+path" + - "lexical path NEVER calls into the semantic/embedding subsystem" + preconditions: + - function index exists (built by pmat query --rebuild-index) + lean_theorem: Theorems.Lexical_No_Embedding_Dep + + hybrid_top3_subset_of_lexical_or_semantic_top10: + formula: "hybrid_top3 is a subset of (lexical_top10 union semantic_top10)" + domain: same query against same corpus + codomain: "every key (file_path, function_name) in hybrid_top3 appears in lex top-10 or sem top-10" + invariants: + - RRF fusion never invents results + - "RRF score formula = 1 / (k + rank), k = 60, rank 1-indexed" + - "fused score = sum(rrf scores across both lists)" + preconditions: + - both lists non-empty + lean_theorem: Theorems.RRF_Closure_Subset + + default_unchanged: + formula: "pmat_query(query, no_flag) == pmat_query(query, semantic)" + domain: any non-empty query against any corpus + codomain: same result set, same ranking + invariants: + - "No --search-mode flag preserves v3.16.0 behaviour exactly (zero breaking change)" + - "`--search-mode semantic` is a no-op alias of the historical default" + preconditions: + - none + lean_theorem: Theorems.Default_Behaviour_Stable + +preconditions: + - "pmat binary >= 3.17.0 with --search-mode flag wired through" + - "Workspace contains at least one .rs/.py/.ts file with parseable functions" + +postconditions: + - "`pmat query --search-mode lexical` exits 0 against a fresh tempdir corpus" + - "`pmat query --search-mode semantic` exits 0 (default behaviour)" + - "`pmat query --search-mode hybrid` exits 0 with RRF-fused top-N results" + - "`pmat query ` without --search-mode is behaviour-identical to v3.16.0 output" + +falsification: + - condition: "--search-mode lexical errors when no embeddings index exists" + severity: P0 + action: reject_push + - condition: "--search-mode hybrid returns a result not present in either lexical or semantic top-10" + severity: P0 + action: reject_push + - condition: "Default `pmat query` output differs from `pmat query --search-mode semantic`" + severity: P0 + action: reject_push + - condition: "Adding --search-mode breaks the existing --regex / --literal flags" + severity: P0 + action: reject_push + - condition: "RRF k constant drifts from 60 (matches `pmat semantic search --search-mode hybrid`)" + severity: P1 + action: review_change + +verification: + local: "cargo test -p pmat --lib query_handler::tests::test_search_mode" + ci: ".github/workflows/ci.yml (test job)" diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/quality-check-content-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/quality-check-content-v1.yaml new file mode 100644 index 0000000000..02df699285 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/quality-check-content-v1.yaml @@ -0,0 +1,73 @@ +metadata: + version: "1.0.0" + created: "2026-09-02" + author: PAIML Engineering + references: + - "docs/specifications/pmat-architecture-crux-audit.md section 8.10 (CRUX-10, issue 1151)" + - "CHANGELOG.md 3.33.0 'Not done, deliberately' — no pmat_write_file; the harness PreToolUse hook is the write gate" + - "Popper (1959) The Logic of Scientific Discovery" + registry: true + description: > + The MCP tool formerly named quality_proxy advertised a write it never + performed: nine live calls across every mode and operation returned + accepted and created no file, and no response field said so. It is now + quality_check_content (alias quality_proxy for one release): it grades + content, discloses that it wrote nothing, never launders a failing verdict + as accepted, lets a client only tighten the project's gate, and keeps its + counts consistent with its lists. + contract: quality-check-content + status: draft + +equations: + never_writes_and_says_so: + formula: "for every call, response.written == false and the target path's bytes are unchanged" + domain: "any call to quality_check_content or its alias, any mode" + codomain: bool + invariants: + - "a request carrying operation is refused with -32602 (deny_unknown_fields), never silently accepted" + - "the alias returns a payload identical to the new name for the same request" + preconditions: [] + + advisory_does_not_launder: + formula: "mode == advisory and quality_report.passed == false implies status != accepted" + domain: "advisory-mode calls" + codomain: bool + invariants: + - "advisory still returns final_content, so the caller can proceed knowingly" + preconditions: [] + + client_config_only_tightens: + formula: "effective = tighten(project [quality] floor, client quality_config): min complexity, allow_satd only if both, require_docs if either" + domain: "a target path under a project with pmat.toml; no floor when there is none" + codomain: QualityConfig + invariants: + - "metrics.satd_count == count(violations[] where type == satd) in the same response, whatever allow_satd says" + preconditions: [] + +falsification: + - condition: "a response reports accepted for content that created or changed a file, or omits written (CRUX-10, issue 1151)" + severity: P0 + action: reject_push + - condition: "advisory returns accepted with passed == false" + severity: P0 + action: reject_push + - condition: "a client quality_config raises max_complexity or enables allow_satd above the project's pmat.toml" + severity: P1 + action: reject_push + +falsification_tests: + - id: never_writes_and_says_so + rule: "written is a boolean, false, and agrees with the filesystem; operation is refused" + prediction: "legs S, B0, R1, R2, R3, R5 of scripts/quality-check-content-audit.sh" + test: "bash scripts/quality-check-content-audit.sh against the release binary" + if_fails: "agents keep writing with their own harness believing the gate did" + - id: advisory_does_not_launder + rule: "advisory with a failing report is not accepted" + prediction: "leg B1; test_quality_proxy_advisory_mode asserts Rejected with content still returned" + test: "cargo test --test all test_quality_proxy_advisory_mode" + if_fails: "a status-only client treats rejected content as approved" + - id: client_config_only_tightens + rule: "the merge takes the stricter of project and client on every axis; satd_count equals the satd list" + prediction: "client_quality_config_can_only_tighten_the_project_floor and without_a_pmat_toml_the_default_config_is_the_floor pass; leg B2" + test: "cargo test --lib client_quality_config_can_only_tighten_the_project_floor" + if_fails: "any client can switch the gate off with two JSON keys" diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/quality-gate-not-measured-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/quality-gate-not-measured-v1.yaml new file mode 100644 index 0000000000..7ebc2e7c73 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/quality-gate-not-measured-v1.yaml @@ -0,0 +1,78 @@ +metadata: + version: "1.0.0" + created: "2026-09-03" + author: PAIML Engineering + references: + - "docs/specifications/pmat-architecture-crux-audit.md section 8.2 (CRUX-02, epic 1153)" + - "issue 1035 — quality-gate reports satd_violations=0 while analyze satd reports 55 (same shape)" + - "Popper (1959) The Logic of Scientific Discovery" + registry: true + description: > + pmat quality-gate rendered three unmeasured dimensions as clean: dead code + on a crate cargo check could not compile printed 0 violations found, the + coverage check trusted any cache file it found, and duplicate_violations + reported 0 beside a 21.67 percent block-level duplication it never looked + for. The gate now discloses each: not_measured names the dead-code + compile failure and the block-level half of duplicates; not_applicable + names a path with no Cargo.toml; a coverage report is accepted only if + its commit is HEAD-or-ancestor, it is newer than the newest tracked + source, and it covers at least 25 percent of the tree's Rust sources — a + rejection names the guard; the whole-file count is identical_files. + contract: quality-gate-not-measured + status: draft + +equations: + unmeasured_is_disclosed_not_zeroed: + formula: "dead_code on a crate cargo check cannot compile => results.not_measured contains {check: dead_code, reason ~ 'could not compile'}; no Cargo.toml at or above the path => results.not_applicable contains {check: dead_code}" + domain: "any pmat quality-gate run that selects dead-code" + codomain: bool + invariants: + - "a compiling crate has no dead_code entry in either list (control A)" + - "not_applicable is never reported as not_measured (control B): a non-Rust repository is not permanently amber" + - "both lists always serialize, empty included" + preconditions: [] + + coverage_report_must_come_from_this_tree: + formula: "accept(cache) iff git_hash is HEAD-or-ancestor and mtime(cache) >= mtime(newest tracked source) and covered_sources / rust_sources >= 0.25; else a coverage finding whose message says REJECTED and names the guard (git_hash: | mtime: | breadth:)" + domain: ".pmat/coverage-cache.json under the gated path" + codomain: bool + invariants: + - "a fresh report from HEAD covering the tree is accepted (control)" + - "an accepted detail cache is preferred over .pmat-metrics/coverage.json" + preconditions: [] + + duplicates_named_for_what_is_measured: + formula: "results.identical_files == count(whole files with byte-identical content); results has no duplicate_violations key; results.not_measured contains {check: duplicates, reason ~ 'block-level'}" + domain: "any run that selects duplicates" + codomain: bool + invariants: + - "the block-level detector is a separate item; the gate does not claim it" + preconditions: [] + +falsification: + - condition: "quality-gate reports dead_code_violations 0 with no not_measured entry on a crate cargo check fails to compile (CRUX-02)" + severity: P0 + action: reject_push + - condition: "a coverage cache whose git_hash is not HEAD-or-ancestor, or listing only nonexistent files, yields coverage_violations 0" + severity: P0 + action: reject_push + - condition: "the payload carries duplicate_violations, or identical_files without the block-level disclosure" + severity: P1 + action: reject_push + +falsification_tests: + - id: unmeasured_is_disclosed_not_zeroed + rule: "uncompilable crate => not_measured (could not compile); no manifest => not_applicable; compiling crate => neither" + prediction: "leg 1 with controls A and B of scripts/quality-gate-not-measured-audit.sh; dead_code_outcome_tests (4) pass" + test: "cargo test --lib dead_code_outcome_tests" + if_fails: "a pre-commit gate on a broken crate reports clean" + - id: coverage_report_must_come_from_this_tree + rule: "three named guards; the first to trip is the reason" + prediction: "leg 2 and its control; the a_report_* and a_rejected_report_* tests in coverage_sections_tests pass" + test: "cargo test --lib coverage_sections_tests" + if_fails: "a 114-commit-old or fabricated report passes the coverage floor" + - id: duplicates_named_for_what_is_measured + rule: "identical_files present, duplicate_violations absent, duplicates disclosed as block-level not measured" + prediction: "leg 3; the_results_payload_carries_the_honest_name_and_both_disclosure_lists passes" + test: "cargo test --lib the_results_payload_carries_the_honest_name_and_both_disclosure_lists" + if_fails: "0 duplicates is read as 0 clones on a 21.67 percent tree" diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/quorum-review-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/quorum-review-v1.yaml new file mode 100644 index 0000000000..a32a89f687 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/quorum-review-v1.yaml @@ -0,0 +1,74 @@ +metadata: + version: "1.0.0" + created: "2026-09-04" + author: PAIML Engineering + references: + - "docs/specifications/agentic-delivery-pmat.md section 3.1 / 5.2 / 9.4 (AD-04)" + - "paiml-implement bundle: skills/quorum-review/{SKILL.md,quorum-review.sh,pmat-merge}, agy/quorum-schema.json" + - "Popper (1959) The Logic of Scientific Discovery" + registry: true + description: > + Nothing merges without a quorum verdict. quorum-review.sh sends the same + prompt (git diff base...HEAD, the ticket, the receipt, the refutation + doctrine) to N independent agy lanes under agy/quorum-schema.json and + writes docs/audits/quorum-.json with agreed = every lane PASS and + the HEAD it judged. pmat-merge, the gh pr merge wrapper, refuses --auto + unless an artifact agrees for the PR's current head; a non-auto merge is + passed through. A concurrency cap is not a vote: this is the vote. + contract: quorum-review + status: draft + +equations: + auto_merge_requires_agreement: + formula: "auto and not exists(a in docs/audits/quorum-*.json : a.agreed and (a.head == pr.head or (a.head == parent(pr.head) and files(pr.head) subset docs/audits/quorum-*.json))) => pmat-merge exits 1 naming docs/audits/quorum-.json and never calls gh pr merge" + domain: "any invocation of pmat-merge --auto in a repository checkout" + codomain: bool + invariants: + - "an agreeing artifact for another head does not count (a rebase invalidates the verdict)" + - "the one exception is the verdict commit itself: the artifact is committed on the branch it judged, so an agreeing artifact for the head's first parent counts when the head commit touches nothing but docs/audits/quorum-*.json (paiml-implement#6); a verdict commit that also changes code is refused" + - "an artifact for this head with agreed = false does not count" + - "an agreeing artifact for this head hands the call to gh pr merge unchanged, --auto included" + - "pmat-merge without --auto is passed through whatever docs/audits holds" + preconditions: [] + + agreement_is_unanimous: + formula: "agreed == (len(lanes) == width and all(lane.verdict == PASS))" + domain: "the artifact quorum-review.sh writes" + codomain: bool + invariants: + - "a lane whose output carries no verdict in the schema enum is recorded as NO-VERDICT and blocks" + - "the schema object echoed back by the executor is never mistaken for a verdict (verdict must be an enum string and findings a list)" + - "raw lane outputs are kept beside the artifact under .lanes/ for audit" + preconditions: + - "width in 1..10" + - "ticket matches ^[A-Z]+-[0-9]+$" + + lanes_refute: + formula: "a diff containing a test that asserts the opposite of the ticket draws >= 1 FAIL whose finding names that file; a diff that does what the ticket says draws width PASS" + domain: "the two live controls in docs/audits/impl-PMAT-656-receipt.md" + codomain: bool + invariants: + - "findings carry file, claim and grounding in {cited, measured, asserted}; asserted findings are reported, not counted as refutation on their own" + preconditions: [] + +falsification_tests: + - id: F1 + rule: "the merge helper exists in the installed skill" + prediction: "with QUORUM_SKILL_DIR pointing at a directory without pmat-merge the audit exits 1 and its first leg reads helper present (missing ...) — the 3.36.0 state" + test: "QUORUM_SKILL_DIR=/nonexistent bash scripts/quorum-review-audit.sh" + if_fails: "the audit passed without a helper: its first leg no longer checks presence, or a helper leaked onto the path" + - id: F2 + rule: "the seven offline legs through a stub gh" + prediction: "GREEN, exit 0: refused without an artifact naming the file; refused for another head and for agreed=false; gh pr merge --auto called on agreement; non-auto passthrough; the docs-only verdict commit accepted for its parent; a verdict commit that also changes code refused" + test: "bash scripts/quorum-review-audit.sh" + if_fails: "pmat-merge changed its refusal or acceptance rule; read which leg is red and compare with auto_merge_requires_agreement" + - id: F3 + rule: "the helper can be broken (named mutations M1, M2, M3)" + prediction: "a helper that ignores agreed, one that ignores head, and one that accepts the parent unconditionally each turn the corresponding leg RED" + test: "QUORUM_SKILL_DIR= bash scripts/quorum-review-audit.sh" + if_fails: "a mutant passed: the audit no longer discriminates that rule and is vacuous for it" + - id: F4 + rule: "the live lanes refute and agree" + prediction: "the planted-contradiction artifact carries at least one FAIL naming commit_enforcement_tests and agreed=false; the after-fix artifact carries width PASS and agreed=true" + test: "bash scripts/quorum-review-audit.sh --clean docs/audits/quorum-PMAT-655-after-fix.json --planted docs/audits/quorum-PMAT-655-planted-control.json" + if_fails: "the lanes no longer refute a planted contradiction, or the parser mistook something other than an enum-string verdict for a verdict" diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/tdg-grade-order-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/tdg-grade-order-v1.yaml new file mode 100644 index 0000000000..42b0e3af57 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/tdg-grade-order-v1.yaml @@ -0,0 +1,246 @@ +metadata: + version: "1.0.0" + kind: kernel + created: "2026-08-20" + author: PAIML Engineering + registry: true + # The type these obligations are proved ABOUT. `scripts/pv-obligation-gate.py` + # requires that every function named by an `applies_to` actually mentions it, + # so a contract cannot be green while the code it names ignores the proof. + # This is not hypothetical: thirteen theorems below proved the grade order + # while `check_tdg_grade_gate` still carried its own ["A","B","C","D","F"] + # table and never referenced `Grade`, and every gate in the repository was + # green throughout. + proved_type: Grade + contract: tdg-grade-order-v1 + status: active + description: > + The TDG grade scale as an ORDER, and the CB-200 threshold as a predicate + over it. Written because CB-200 shipped neither. + + The gate carried a private five-letter table, ["A","B","C","D","F"], and + compared it to the stored spelling with SQL `IN`. `IN` answers "no row" for + a value it does not list, never an error, so an eleven-letter writer and a + five-letter reader coexisted for a release: at a floor of "A" the gate saw + 247 violations and could not see 1,719 more. It was also not monotone in + quality — a C- function passed while a C function failed. + + Its ordinal function mapped every modified grade to a catch-all ranked + WORSE THAN F. So a floor of "A-" produced an empty failing set, the caller + took its is_empty() branch, and the rule returned Pass with the message + "Minimum grade A- - no grades below threshold". `.pmat-metrics.toml:44` + declares exactly that spelling. A stricter-looking threshold turned the gate + off while printing a sentence that reads like enforcement. + + WHY THE OBVIOUS INVARIANTS ARE NOT ENOUGH, which is the reason this contract + reaches L5 rather than stopping at a const assertion. An adversarial review + of the first fix showed that reversing the enum declaration together with + `Grade::ALL` leaves EVERY self-referential invariant intact: the const block + passes, Display stays injective, parse stays a left inverse, all 121 pairs + and 1,331 triples pass, and passing/below still partition — while + `meets_threshold` reports that A+ fails an F floor. All of those invariants + are internal to the order; none anchors rank to anything outside itself. + + `Grade_Rank_Anchored_To_Score` is that anchor. `bandFloor` is attached to + variant NAMES (mirroring GRADE_BANDS, grade.rs:172-182), so a better rank + must carry a strictly higher score floor. Under the reversal above, Lean + rejects it: "decide proved that the proposition is false". Verified by + mutation, not asserted. + + verification: + lean_status: discharged + lean_toolchain: "leanprover/lean4:v4.15.0" + measured_at: "pmat 3.32.0, lean 4.15.0, 2026-08-20" + detail: > + All 13 theorems in contracts/lean/Theorems/Tdg/Grade.lean build under + `lake build`, contain no `sorry` or `admit`, and `#print axioms` reports + that each depends on NO axioms — every proof is closed by `decide` or + `cases ... <;> decide` over a finite domain. Unlike a kani harness, these + are EXECUTED in CI: .github/workflows/quality-gate.yml:163 runs + `lake build` and :170 greps for proof holes, both inside the + `provable ladder` job. + + Kani is deliberately NOT used for the order algebra. The domain is 11, + 121 and 1,331 closed enum-to-enum cases, so a loop over Grade::ALL is a + complete proof over it AND runs under `cargo test --lib`. Declaring a + kani harness there would name a proof technique without carrying it out, + which is the defect commit a44ec5d3e already ruled on. + references: + - src/tdg/grade.rs + - src/cli/handlers/comply_handlers/check_handlers/check_tdg_grade.rs + - src/services/agent_context/query/grades.rs + - contracts/lean/Theorems/Tdg/Grade.lean + - contracts/macs-ladder-kernel-v1.yaml + +equations: + grade_rank_anchored: + formula: "rank(a) < rank(b) => band_floor(b) < band_floor(a) for a,b != F" + domain: "two of the eleven Grade variants, neither being F" + codomain: "bool" + invariants: + - "INV-2200-1: a better rank carries a strictly higher score floor" + - "the ONLY invariant that is not internal to the order: it fails under a reversal of the enum declaration that every other invariant here waves through" + - "F is excluded because it is the fallthrough of GRADE_BANDS and has no floor of its own" + lean_theorem: Theorems.Tdg.Grade_Rank_Anchored_To_Score + + grade_from_score: + formula: "s1 + 1 >= s1 => rank(from_score(s1 + 1)) <= rank(from_score(s1))" + domain: "integer score positions 0..100" + codomain: "Grade" + invariants: + - "INV-2200-2: one more point of score never yields a worse grade" + - "INV-2200-3: from_score(band_floor(g)) = g for every g != F, so the two tables cannot drift apart" + - "stepwise, not pairwise: on a discrete domain the two are equivalent by transitivity, and the pairwise form exhausts Lean's kernel recursion depth at 10,201 cases" + - "checks every integer position; the Rust sweep at grade.rs:380-407 samples 1,001 points and can miss a band edge falling between two samples" + lean_theorem: Theorems.Tdg.Grade_FromScore_Antitone_Step + + grade_floor_partition: + formula: "|passing(t)| + |below(t)| = |all| for every floor t" + domain: "one of the eleven Grade variants, as a floor" + codomain: "two disjoint grade sets covering the scale" + invariants: + - "INV-2200-4: passing and below partition the whole scale at every floor" + - "this is Defect 1 stated as a theorem: the shipped five-letter table partitioned nothing, so 1,719 of 1,966 violations were invisible" + - "INV-2200-5: below(t) is empty for exactly one floor, F" + - "this is Defect 2 stated as a theorem: the catch-all ranked modified grades worse than F, so a floor of A- produced an empty failing set and the rule returned Pass" + - "INV-2200-6: below is antitone in the floor - a stricter floor never fails fewer grades" + lean_theorem: Theorems.Tdg.Grade_Partition + + grade_order_algebra: + formula: "meets_threshold(g, t) = (rank(g) <= rank(t)); total, transitive, antisymmetric" + domain: "two of the eleven Grade variants" + codomain: "bool" + invariants: + - "INV-2200-7: totality - any two grades are comparable" + - "INV-2200-8: transitivity over all 1,331 triples" + - "INV-2200-9: antisymmetry" + - "INV-2200-10: rank is injective, so Ord never conflates two grades" + - "INV-2200-11: to_str is injective, so the stored TEXT column is an unambiguous encoding of the order" + lean_theorem: Theorems.Tdg.Grade_Order_Transitive + +preconditions: + - "pmat >= 3.32.0 with CB-200 reading .pmat/context.db" + - "the stored tdg_grade spelling is one of the eleven Grade::ALL spellings" + +postconditions: + - "a grade absent from the gate's vocabulary is REPORTED, never silently unmatched" + - "a threshold that does not parse FAILS the rule; it never yields an empty failing set" + - "the failing set is computed as the complement of the passing set, never listed by hand" + +proof_obligations: + - type: postcondition + property: "a better rank carries a strictly higher score floor" + formal: "∀a,b ≠ F: rank(a) < rank(b) ⇒ band_floor(b) < band_floor(a)" + applies_to: meets_threshold + lean: + theorem: Theorems.Tdg.Grade_Rank_Anchored_To_Score + module: Theorems.Tdg.Grade + status: proved + notes: > + cases a <;> cases b <;> decide — depends on no axioms. THE anchoring + obligation: it is the only one here that is not internal to the order, + and it is the only one that fails when the enum declaration and + Grade::ALL are reversed together. Verified by that mutation, which Lean + rejects with "decide proved that the proposition is false". + - type: postcondition + property: "passing and below partition the scale at every floor" + formal: "∀t: |passing(t)| + |below(t)| = |all|" + applies_to: check_tdg_grade_gate + lean: + theorem: Theorems.Tdg.Grade_Partition + module: Theorems.Tdg.Grade + status: proved + notes: > + Defect 1 as an obligation. The shipped five-letter table partitioned + nothing, so at a floor of A the gate saw 247 violations and 1,719 were + invisible. + - type: postcondition + property: "the failing set is empty for exactly one floor, F" + formal: "∀t: below(t) = ∅ ⇔ t = F" + applies_to: check_tdg_grade_gate + lean: + theorem: Theorems.Tdg.Grade_Below_Empty_Only_At_F + module: Theorems.Tdg.Grade + status: proved + notes: > + Defect 2 as an obligation. grade_ordinal ranked every modified grade + worse than F, so a floor of "A-" produced an empty failing set and the + rule returned Pass. .pmat-metrics.toml:44 declares that spelling. + - type: postcondition + property: "one more point of score never yields a worse grade" + formal: "∀s ∈ 0..100: rank(from_score(s+1)) ≤ rank(from_score(s))" + applies_to: from_score + lean: + theorem: Theorems.Tdg.Grade_FromScore_Antitone_Step + module: Theorems.Tdg.Grade + status: proved + notes: > + Checks every integer position. The Rust sweep at grade.rs:380-407 + samples 1,001 points and can miss a band edge falling between two + samples. Stepwise rather than pairwise: equivalent on a discrete domain + by transitivity, and the 10,201-case pairwise form exhausts Lean's + kernel recursion depth. + - type: postcondition + property: "the order is total, transitive and antisymmetric; rank and spelling are injective" + formal: "meets_threshold(g,t) = rank(g) ≤ rank(t), a total order" + applies_to: meets_threshold + lean: + theorem: Theorems.Tdg.Grade_Order_Transitive + module: Theorems.Tdg.Grade + status: proved + notes: > + 1,331 closed triples checked by the kernel, plus totality, + antisymmetry, rank injectivity and to_str injectivity as separate + theorems in the same module. + +falsification_tests: + - id: grade_order_reversal + rule: "the grade order is anchored to the score scale, not merely self-consistent" + prediction: "reversing the enum declaration and Grade::ALL together makes lake build FAIL" + test: "cd contracts/lean && lake build # Theorems.Tdg.Grade_Rank_Anchored_To_Score" + if_fails: "A+ can rank worse than F while every internal invariant still holds, and CB-200 inverts silently" + - id: grade_modified_invisible + rule: "every grade below the floor is reported" + prediction: "a five-letter failing list makes |passing| + |below| < |all| for some floor" + test: "cd contracts/lean && lake build # Theorems.Tdg.Grade_Partition" + if_fails: "1,719 of 1,966 violations are invisible and CB-200 reports 247 (measured at HEAD)" + - id: grade_floor_disables_gate + rule: "only a floor of F admits every grade" + prediction: "a catch-all ordinal makes below(A-) empty, so the rule returns Pass" + test: "cd contracts/lean && lake build # Theorems.Tdg.Grade_Below_Empty_Only_At_F" + if_fails: "min_tdg_grade: \"A-\" silently disables CB-200 while printing a sentence that reads like enforcement" + - id: grade_band_edge_inverts + rule: "score and grade move in opposite directions, monotonically" + prediction: "an inverted band edge makes one step of score yield a worse grade" + test: "cd contracts/lean && lake build # Theorems.Tdg.Grade_FromScore_Antitone_Step" + if_fails: "a function scoring higher is graded worse, and every downstream ranking inverts at that edge" + - id: grade_spelling_collision + rule: "rank and wire spelling are both injective" + prediction: "two grades sharing a rank or a spelling makes the stored TEXT column ambiguous" + test: "cd contracts/lean && lake build # Theorems.Tdg.Grade_Rank_Injective, Grade_ToStr_Injective" + if_fails: "the SQLite tdg_grade column stops being a faithful encoding of the order" + - id: grade_parse_corruption + rule: "corrupted spellings are rejected, never ranked" + prediction: "empty, padded, out-of-set or unicode-minus spellings parse to None" + test: "cd contracts/lean && lake build # Theorems.Tdg.Grade_Parse_Strict" + if_fails: "a copy-pasted \"A−\" (U+2212) is ranked by a catch-all instead of rejected" + +kani_harnesses: [] + +qa_gate: + id: TDG-GRADE-ORDER-GATE + name: TDG grade order and threshold gate + description: > + L5 provability gate for pmat's own grade scale. Kani is deliberately empty: + the order algebra is 11, 121 and 1,331 closed enum-to-enum cases, so a loop + over Grade::ALL is a complete proof over that domain and it RUNS under + `cargo test --lib`. Declaring a harness there would record the name of a + proof technique without carrying it out — the defect commit a44ec5d3e + already ruled on. The Lean proofs, by contrast, are executed in CI: + quality-gate.yml:163 runs `lake build` and :170 fails on any sorry/admit. + checks: + - validation + - falsification + pass_criteria: > + All 13 theorems in Theorems.Tdg.Grade build with 0 sorry / 0 admit, and + `#print axioms` reports no axiom dependency for any of the 5 obligations. diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/verify-verdict-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/verify-verdict-v1.yaml new file mode 100644 index 0000000000..71700ddfc4 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/verify-verdict-v1.yaml @@ -0,0 +1,65 @@ +metadata: + version: "1.0.0" + created: "2026-09-02" + author: PAIML Engineering + references: + - "docs/specifications/pmat-architecture-crux-audit.md section 8.1 (CRUX-01, issue 1146)" + - "docs/specifications/pmat-verify-autonomous-preflight.md" + - "Popper (1959) The Logic of Scientific Discovery" + registry: true + description: > + pmat verify is the one gate agents are told to trust before every commit. + Its composite verdict must never assert safety over a stage it did not + measure, and its strict SATD stage must see every marker quality-gate + blocks on. On the repository's own clean tree it returned ok:true with the + complexity stage declined and strict SATD reading 0 where the default read 3. + contract: verify-verdict + status: draft + +equations: + verdict_tri_state: + formula: "ok = Some(false) if failed or measured == 0; None if measured > 0 and declined > 0; Some(true) otherwise" + domain: "the selected stages of one pmat verify run" + codomain: "Option plus exit code (1 iff Some(false))" + invariants: + - "not_measured[] is derived from the stages that returned not_applicable, never from a constant" + - "a --skip'ped stage is not declined and never appears in not_measured[]" + - "a measured failure is Some(false) whatever else declined" + preconditions: [] + + strict_sees_what_the_gate_blocks: + formula: "for every SATD finding quality-gate reports at severity error, analyze satd --strict reports the same line" + domain: "any tree; the repository's own tdg_calculator_core.rs:110 and quality_checks_part4.rs:117 are the motivating sites" + codomain: bool + invariants: + - "strict accepts the canonical markers upper case or capitalised, any standard separator, and a non-empty work item" + - "strict remains a subset of default: todo:, TODO x, and prose are not debt in strict" + preconditions: [] + +falsification: + - condition: "verify prints ok:true on a run in which a selected stage declined to measure (CRUX-01, issue 1146)" + severity: P0 + action: reject_push + - condition: "a stage that measured a failure is reported as anything but ok:false with exit 1" + severity: P0 + action: reject_push + - condition: "quality-gate blocks on a SATD line that analyze satd --strict does not report" + severity: P1 + action: reject_push + +falsification_tests: + - id: verdict_tri_state + rule: "the composite table, every row" + prediction: "composite_verdict(false, 2, 1) is None; (true, 1, 1) and (false, 0, 3) are Some(false); (false, 3, 0) is Some(true)" + test: "cargo test --lib composite_verdict_withdraws_rather_than_asserts_over_a_declined_stage" + if_fails: "verify asserts safe-to-commit over a stage it never looked at" + - id: verdict_end_to_end + rule: "on a clean crate with no Rust change vs HEAD, verify reports ok:null, not_measured [complexity], exit 0; with a TODO it reports ok:false, exit 1" + prediction: "legs 1, 1-RED, 1-EMPTY and 1-SKIP of scripts/verify-verdict-audit.sh" + test: "bash scripts/verify-verdict-audit.sh against the release binary" + if_fails: "the JSON contract agents read is not the one the table promises" + - id: strict_sees_what_the_gate_blocks + rule: "TODO(x): and TODO[x]: and Bug: are debt in strict; todo:, TODO x and TODO: with no work item are not" + prediction: "strict_accepts_every_standard_separator_and_the_capitalised_marker passes; strict_is_a_subset_of_default still passes" + test: "cargo test --lib strict_accepts_every_standard_separator_and_the_capitalised_marker strict_is_a_subset_of_default" + if_fails: "verify's SATD stage is green on a tree quality-gate blocks" diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/work-ladder-claim-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/work-ladder-claim-v1.yaml new file mode 100644 index 0000000000..282065fda6 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/work-ladder-claim-v1.yaml @@ -0,0 +1,78 @@ +metadata: + version: "1.0.0" + created: "2026-09-04" + author: PAIML Engineering + references: + - "issue 1186 — pmat work claims L3 by default; --implements only on start; ladder check ran after the quality gate" + - "docs/specifications/agentic-delivery-pmat.md section 5 (sub-agent level compliance)" + - "Popper (1959) The Logic of Scientific Discovery" + registry: true + description: > + A ticket's verification claim is what the work can honestly show. A ticket + started without a contract binding claims L1; one started with --implements + claims L2 and carries the binding; --level states the claim explicitly on + add, start and edit. --implements on edit binds a ticket that is already in + progress, so the normal flow can reach L2 without restarting. A binding must + name an equation the contract declares. On complete, the ladder is judged + before the quality gate, so an over-claim is refused as LadderShortfall + rather than surfacing behind unrelated gate output. + contract: work-ladder-claim + status: draft +equations: + initial_claim: + formula: "claim(start) = explicit_level if given else (L2 if bound(implements) else L1)" + domain: "pmat work start, pmat work add --level" + codomain: VerificationLevel + invariants: + - "an unbound start claims L1, never L3" + - "a start with --implements claims L2 and the contract carries the binding" + - "--level on add is stored on the ticket and becomes the claim at start" + - "a --level that is not a ladder level (L0..L5) is refused before anything is written" + preconditions: [] + edit_rebinds: + formula: "edit --level L | edit --implements c/e => contract.level = L | contract.implements += (c, e, sha)" + domain: "pmat work edit on a ticket whose contract exists (started)" + codomain: bool + invariants: + - "edit --implements on an InProgress ticket binds and lifts an L1 claim to L2" + - "a binding naming an equation the contract does not declare is refused, naming the declared ones" + - "edit --level alone is a change: it must not exit 0 as 'No changes specified'" + - "edit on a ticket with no contract yet names `pmat work start` as the prerequisite" + preconditions: [] + ladder_before_gate: + formula: "complete: ladder_shortfall(claim, evidence) is judged before run_quality_check" + domain: "pmat work complete" + codomain: bool + invariants: + - "an over-claim is refused with LadderShortfall and no quality-gate output precedes it" + - "an honestly-claimed L1 ticket is not refused by the ladder" + preconditions: [] +falsification: + - condition: "an unbound ticket claims above L1 at start, or a bound one claims below L2" + severity: P0 + action: reject_push + - condition: "edit --level or edit --implements exits 0 without changing the contract" + severity: P0 + action: reject_push + - condition: "a binding to an undeclared equation is accepted" + severity: P1 + action: reject_push + - condition: "quality-gate output precedes a LadderShortfall refusal on complete" + severity: P1 + action: reject_push +falsification_tests: + - id: initial_claim + rule: "legs 1, 2, 3a and control 3c of scripts/work-ladder-claim-audit.sh" + prediction: "3.36.0 binary: legs 1, 2, 3a red (L3 default; add has no --level); fixed binary: green" + if_fails: "the claim is not derived from the binding; the ladder gate judges a number nobody chose" + test: "PMAT= bash scripts/work-ladder-claim-audit.sh" + - id: edit_rebinds + rule: "legs 3b, 4 and controls 3c, 4b of scripts/work-ladder-claim-audit.sh; unit test resolve_refuses_an_equation_the_contract_does_not_declare" + prediction: "3.36.0 binary: edit has neither flag (clap refuses); fixed binary: both flags act on the contract and 4b refuses" + if_fails: "an in-progress ticket can never reach L2 legitimately, or binds to nothing" + test: "PMAT= bash scripts/work-ladder-claim-audit.sh" + - id: ladder_before_gate + rule: "legs 5a, 5b and control 6 of scripts/work-ladder-claim-audit.sh" + prediction: "3.36.0 binary: 5b red (quality output first); fixed binary: LadderShortfall is the first refusal" + if_fails: "an over-claim hides behind gate noise and the operator reaches for --override-claims" + test: "PMAT= bash scripts/work-ladder-claim-audit.sh" diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/workspace-scoring-v1.yaml b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/workspace-scoring-v1.yaml new file mode 100644 index 0000000000..addbbab1d1 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/contracts/workspace-scoring-v1.yaml @@ -0,0 +1,74 @@ +metadata: + version: "1.0.0" + created: "2026-04-08" + author: PAIML Engineering + registry: true + description: > + Workspace-aware rust-project-score: detect and score subcrates in + monorepo workspaces (e.g., aprender with 60+ crates). Each subcrate + gets its own RPS score. Aggregate scores into workspace-level grade. + contract: workspace-scoring + status: planned + references: + - docs/specifications/components/repo-health.md + +equations: + subcrate_discovery: + formula: subcrates = discover_workspace_members(project_path) + domain: project_path in Path, Cargo.toml exists + codomain: subcrates in Vec<(String, PathBuf)> + invariants: + - subcrates.len() >= 1 (workspace always has at least the root) + - all paths in subcrates are valid directories with Cargo.toml + preconditions: + - project_path.join("Cargo.toml").exists() + lean_theorem: Theorems.Workspace_Discovery_Nonempty + + workspace_aggregate: + formula: workspace_score = geometric_mean(subcrate_scores) + domain: subcrate_scores in Vec, all >= 0.0 + codomain: workspace_score in [0.0, 100.0] + invariants: + - workspace_score >= min(subcrate_scores) (geometric mean property) + - workspace_score <= max(subcrate_scores) + preconditions: + - subcrate_scores.len() > 0 + lean_theorem: Theorems.Geometric_Mean_Bounded + + per_crate_scoring: + formula: crate_score = rps_score(subcrate_path) + domain: subcrate_path in Path, subcrate_path.join("Cargo.toml").exists() + codomain: crate_score in CategoryScore + invariants: + - crate_score.earned >= 0.0 + - crate_score.earned <= crate_score.max + preconditions: + - subcrate_path.join("src").exists() || subcrate_path.join("lib.rs").exists() + lean_theorem: Theorems.Score_Range_NonNeg + +preconditions: + - "pmat rust-project-score --help exits 0" + - "Target workspace (e.g., ../aprender) has Cargo.toml with [workspace]" + - "At least 1 workspace member has src/ directory" + +postconditions: + - "pmat rust-project-score --path ../aprender produces per-crate table" + - "Workspace aggregate score is geometric mean of subcrate scores" + - "Subcrates without src/ are skipped (not errored)" + +falsification: + - condition: "pmat rust-project-score --path ../aprender panics" + severity: P0 + action: fix_panic + - condition: "Workspace with 0 members returns non-zero score" + severity: P0 + action: fix_logic + - condition: "Per-crate score exceeds 289 (max RPS)" + severity: P0 + action: fix_bounds + - condition: "Subcrate without Cargo.toml is included in scoring" + severity: P1 + action: skip_invalid + - condition: "Aggregate geometric mean exceeds max individual score" + severity: P0 + action: fix_math diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/cli/handlers/comply_handlers/check_handlers/check_tdg_grade.rs.txt b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/cli/handlers/comply_handlers/check_handlers/check_tdg_grade.rs.txt new file mode 100644 index 0000000000..39bd4c0d67 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/cli/handlers/comply_handlers/check_handlers/check_tdg_grade.rs.txt @@ -0,0 +1,2596 @@ +// CB-200: TDG Grade Gate (#214) +// +// Reads the SQLite agent-context index and fails if definitions fall below a +// configurable minimum TDG grade (default A). +// +// The index is the project's own `.pmat/context.db` when it has one, and +// otherwise the copy pmat keeps for that project OUTSIDE it, under the user's +// cache directory, built here on demand. See `TdgIndex` for why the second +// exists: without it this gate could only ever fail on a machine that happened +// to have an index lying around, which is never a CI checkout (#1008). + +use crate::models::comply_config::ComplyConfig; +use crate::tdg::grade::GRADE_VARIANTS; +use crate::tdg::Grade; +use std::path::{Path, PathBuf}; + +use super::types::*; + +/// Convert a TDG grade letter to a numeric ordinal for comparison. +/// The spellings a floor ADMITS, computed from the proved order. +/// +/// Returns `None` when the threshold is not a grade this codebase produces, so +/// an unreadable threshold fails the rule instead of yielding an empty set that +/// reads as "nothing violates". +/// +/// This replaces a private `grade_ordinal` over `["A","B","C","D","F"]` with a +/// `_ => 5` catch-all. Both halves were wrong. The five-letter list could never +/// match a MODIFIED grade against `WHERE tdg_grade IN (...)`, so at a floor of +/// "A" the gate saw 247 violations and could not see 1,719. And the catch-all +/// ranked every modified grade WORSE THAN F, so `grades_below("A-")` was empty +/// and the caller returned Pass — `.pmat-metrics.toml:44` declares exactly that +/// spelling. +/// +/// The order is `Grade`'s, anchored to the score bands in +/// `contracts/lean/Theorems/Tdg/Grade.lean::Grade_Rank_Anchored_To_Score`. +fn passing_spellings(min_grade: &str) -> Option> { + let floor = Grade::from_variant_name(min_grade.trim())?; + Some( + GRADE_VARIANTS + .iter() + .copied() + .filter(|g| Grade::from_variant_name(g).is_some_and(|x| x.meets_threshold(floor))) + .collect(), + ) +} + +struct TdgViolation { + file_path: String, + function_name: String, + tdg_grade: String, + complexity: u32, + start_line: usize, +} + +/// How many below-floor definitions this project has already agreed to carry, +/// read from `.pmat-gates.toml` `[tdg] baseline`. +/// +/// CB-200 is a RATCHET, not a threshold, and the distinction is the whole +/// design. `.pmat-ratchet.toml`'s own header states it: "A number in a config +/// file that nobody re-runs is not a gate; it is a wish with a colon after it." +/// `.pmat-metrics.toml:45` is this repository's worked example — +/// `max_unwrap_calls = 100`, annotated `Current: 570`, in a tree measuring +/// 20,390: three numbers, no two agreeing, nothing reading the key, green +/// throughout. +/// +/// This number is different in exactly one way, and it is the only way that +/// matters: it is RE-DERIVED on every run by the same query that produced it, +/// and compared. It can never quietly become a transcription. It is a record of +/// debt, not a permission to add more — a definition below the floor that +/// pushes the count past it fails the gate, closed, and the absolute count is +/// printed on every outcome so "passing" can never be read as "clean". +/// +/// It does NOT touch `min_tdg_grade` and adds no exclude glob. Both of those +/// are threshold-lowering in disguise: they make debt invisible, where this +/// keeps every unit of it counted and reported. +#[derive(Debug, PartialEq, Eq)] +enum TdgBaseline { + /// No `baseline` key. Zero tolerance — any violation fails, exactly as + /// before. This is the default precisely so that ratcheting THIS repo + /// cannot silently relax any other repo that runs pmat. + Absent, + /// The count this project last agreed to hold flat. + Held(usize), + /// The key is present and is not a count. Fails, for the same reason an + /// unparseable `min_grade` fails: a bound nobody can read must not be read + /// as a bound nothing exceeds. + Unreadable(String), +} + +struct TdgGateOverrides { + min_grade: Option, + exclude: Vec, + baseline: TdgBaseline, +} + +impl Default for TdgGateOverrides { + /// No overrides: the floor comes from `.pmat.yaml`, nothing extra is + /// excluded, and the gate has zero tolerance. An unreadable or unparsable + /// `.pmat-gates.toml` lands here, which fails CLOSED — the excludes vanish + /// and the baseline vanishes with them, so a typo cannot buy headroom. + fn default() -> Self { + Self { + min_grade: None, + exclude: Vec::new(), + baseline: TdgBaseline::Absent, + } + } +} + +/// A `baseline` value is a count or it is nothing. A string, a float, a +/// negative — anything that is not a non-negative integer — is reported rather +/// than rounded down to "no baseline", because "the key you wrote does nothing" +/// and "you configured zero tolerance" are opposite claims and look identical +/// from the outside. +fn parse_tdg_baseline(value: Option<&toml::Value>) -> TdgBaseline { + let Some(value) = value else { + return TdgBaseline::Absent; + }; + match value.as_integer() { + Some(n) if n >= 0 => TdgBaseline::Held(n as usize), + _ => TdgBaseline::Unreadable(value.to_string()), + } +} + +fn load_tdg_gate_overrides(project_path: &Path) -> TdgGateOverrides { + let path = project_path.join(".pmat-gates.toml"); + let Ok(content) = std::fs::read_to_string(&path) else { + return TdgGateOverrides::default(); + }; + let Ok(table) = content.parse::() else { + return TdgGateOverrides::default(); + }; + let tdg = table.get("tdg"); + let min_grade = tdg + .and_then(|t| t.get("min_grade")) + .and_then(|v| v.as_str()) + .map(String::from); + let exclude = tdg + .and_then(|t| t.get("exclude")) + .and_then(|v| v.as_array()) + .map(|a| { + a.iter() + .filter_map(|v| v.as_str().map(String::from)) + .collect() + }) + .unwrap_or_default(); + let baseline = parse_tdg_baseline(tdg.and_then(|t| t.get("baseline"))); + TdgGateOverrides { + min_grade, + exclude, + baseline, + } +} + +/// Is `.pmat/context.db` older than the sources it claims to describe? +/// +/// `pub(crate)` because `services::tdg_baseline` asks the same question before +/// it treats CB-200's count as the count at HEAD. It CALLS this rather than +/// re-deriving it: two implementations of one predicate is how CB-200 went +/// blind for a release (see [`passing_spellings`]). +pub(crate) fn is_index_stale(project_path: &Path, db_path: &Path) -> bool { + let db_mtime = match std::fs::metadata(db_path).and_then(|m| m.modified()) { + Ok(t) => t, + Err(_) => return true, + }; + for dir_name in ["src", "lib"] { + let dir = project_path.join(dir_name); + if !dir.exists() { + continue; + } + if has_newer_source_file(&dir, db_mtime) { + return true; + } + } + false +} + +fn has_newer_source_file(dir: &Path, threshold: std::time::SystemTime) -> bool { + let Ok(entries) = std::fs::read_dir(dir) else { + return false; + }; + entries.flatten().any(|entry| { + let path = entry.path(); + if path.is_dir() { + has_newer_source_file(&path, threshold) + } else { + is_source_file(&path) && entry_is_newer(&entry, threshold) + } + }) +} + +/// Is a directory entry's mtime newer than `threshold`? (false on any I/O error) +fn entry_is_newer(entry: &std::fs::DirEntry, threshold: std::time::SystemTime) -> bool { + entry + .metadata() + .and_then(|m| m.modified()) + .map(|mtime| mtime > threshold) + .unwrap_or(false) +} + +fn is_source_file(path: &Path) -> bool { + path.extension() + .and_then(|e| e.to_str()) + .is_some_and(|ext| { + matches!( + ext, + "rs" | "ts" + | "tsx" + | "js" + | "jsx" + | "py" + | "go" + | "java" + | "kt" + | "swift" + | "c" + | "cpp" + | "cs" + ) + }) +} + +// `rebuild_index` used to live here: CB-200 called +// `AgentContextIndex::build[_incremental]` and SAVED the result whenever +// `.pmat/context.db` was missing or stale, so merely ASKING for a compliance +// verdict wrote 160KB+ of index into the project under audit and, on a large +// repo, spent minutes doing it. Building the index is `pmat query`'s job; the +// gate now reads what exists and refuses honestly when nothing does (#939). + +/// Check TDG grade gate against the SQLite index. +/// Query violations from the context database for grades below threshold +fn query_tdg_violations( + db_path: &Path, + passing_grades: &[&str], +) -> Result, ComplianceCheck> { + let conn = rusqlite::Connection::open_with_flags( + db_path, + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX, + ) + .map_err(|e| ComplianceCheck { + name: "CB-200: TDG Grade Gate".into(), + status: CheckStatus::Skip, + message: format!("Failed to open context.db: {e}"), + severity: Severity::Info, + })?; + let placeholders: Vec = passing_grades + .iter() + .enumerate() + .map(|(i, _)| format!("?{}", i + 1)) + .collect(); + // NOT IN the PASSING set, never IN a failing list. `IN` answers "no row" + // for a value it does not list and never an error, so an eleven-letter + // writer and a five-letter reader coexisted silently for a release. + // Enumerating what passes makes any future alphabet drift return a row that + // must be classified, instead of a smaller number nobody can see. + let sql = format!("SELECT file_path, function_name, tdg_grade, complexity, start_line FROM functions WHERE tdg_grade NOT IN ({})", placeholders.join(", ")); + let mut stmt = conn.prepare(&sql).map_err(|e| ComplianceCheck { + name: "CB-200: TDG Grade Gate".into(), + status: CheckStatus::Skip, + message: format!("Failed to query context.db: {e}"), + severity: Severity::Info, + })?; + let params: Vec<&dyn rusqlite::types::ToSql> = passing_grades + .iter() + .map(|g| g as &dyn rusqlite::types::ToSql) + .collect(); + stmt.query_map(params.as_slice(), |row| { + Ok(TdgViolation { + file_path: row.get(0)?, + function_name: row.get(1)?, + tdg_grade: row.get(2)?, + complexity: row.get::<_, i64>(3)? as u32, + start_line: row.get::<_, i64>(4)? as usize, + }) + }) + .map(|iter| iter.filter_map(|r| r.ok()).collect()) + .map_err(|e| ComplianceCheck { + name: "CB-200: TDG Grade Gate".into(), + status: CheckStatus::Skip, + message: format!("Query failed: {e}"), + severity: Severity::Info, + }) +} + +/// Check if a violation should be excluded (test files or glob patterns) +fn is_tdg_violation_excluded(v: &TdgViolation, exclude_patterns: &[glob::Pattern]) -> bool { + if v.file_path.contains("/tests/") + || v.file_path.contains("/test/") + || v.file_path.ends_with("_test.rs") + || v.file_path.ends_with("_tests.rs") + { + return true; + } + let opts = glob::MatchOptions { + case_sensitive: true, + require_literal_separator: false, + require_literal_leading_dot: false, + }; + exclude_patterns + .iter() + .any(|pat| pat.matches_with(&v.file_path, opts)) +} + +/// Which index CB-200 measured, and where it came from. +/// +/// Two places, tried in this order, and the order is the whole of #1008: +/// +/// 1. `/.pmat/context.db` — the project has an index of its own, +/// built by `pmat query`. Read it, exactly as before. A developer's index +/// and CB-200's verdict must not be two different readings of one tree. +/// 2. pmat's own index for this project, under the user's cache directory +/// ([`comply_index_path`](crate::utils::pmat_cache_dir::comply_index_path)), +/// built here if it is missing or stale. +/// +/// (2) is the fix. Before it, an audit refused to build an index at all — +/// correctly, because building it in the project writes `.pmat/context.db` and +/// `.pmat/context.idx` into a tree an auditor has no business writing to (#939) +/// — and the consequence was a gate that could only fail on a machine that +/// happened to have an index lying around. A fresh CI checkout never does, so +/// on the one machine whose verdict decides a merge, CB-200 measured nothing +/// and `is_compliant` (which tallies `Fail` only) read that silence as consent. +/// Same tree, same commit, index the only variable: 2 failing checks against 3. +/// +/// The audited tree is still never written to. It is the CACHE that gets the +/// index, keyed by project path, and `git status --porcelain` in the audited +/// repository is empty afterwards — which is the property the refusal was +/// protecting and the only one worth keeping. +enum TdgIndex { + /// `/.pmat/context.db`, built by whoever ran `pmat query`. + InProject(PathBuf), + /// pmat's own index for this project, outside it. + OutOfTree(PathBuf), +} + +impl TdgIndex { + fn db_path(&self) -> &Path { + match self { + Self::InProject(path) | Self::OutOfTree(path) => path, + } + } +} + +/// Say where a measurement came from when it did NOT come from the project. +/// +/// Appended to the verdict rather than folded into it: the in-project sentence +/// stays byte-identical, so this cannot move the output of any repository that +/// already has an index — and a reader who is surprised by a CB-200 result on a +/// machine with no `.pmat/` is told, in the verdict itself, which file was read. +fn note_index_provenance(check: ComplianceCheck, index: &TdgIndex) -> ComplianceCheck { + let TdgIndex::OutOfTree(db) = index else { + return check; + }; + let message = format!( + "{} [measured against {}, the index pmat keeps for this project outside it \u{2014} \ + the audited tree has no .pmat/context.db and was not written to (#1008)]", + check.message, + db.display() + ); + ComplianceCheck { message, ..check } +} + +/// Pick the index to read, building pmat's own copy when the project has none. +/// +/// The project's own index wins whenever it exists, fresh or stale — a stale +/// one is reported as stale rather than replaced, because that is a decision +/// about the project's file and #1045 already settled it. Only the copy pmat +/// owns is rebuilt here, which is why this can rebuild at all. +fn resolve_tdg_index(project_path: &Path, cache_index: &Path) -> Result { + let in_project = project_path.join(".pmat").join("context.db"); + if in_project.exists() { + return Ok(TdgIndex::InProject(in_project)); + } + let cached_db = cache_index.with_extension("db"); + if cached_db.exists() && !is_index_stale(project_path, &cached_db) { + return Ok(TdgIndex::OutOfTree(cached_db)); + } + match build_out_of_tree_index(project_path, cache_index) { + Ok(()) => Ok(TdgIndex::OutOfTree(cached_db)), + // A failed rebuild does not throw away a measurement already in hand: a + // stale index describes an OLDER tree, `demote_pass_when_stale` refuses + // to let that read as a pass, and the note names it. Reporting nothing + // would be strictly less information. + Err(reason) if cached_db.exists() => { + crate::status_eprintln!("CB-200: keeping the previous index ({reason})"); + Ok(TdgIndex::OutOfTree(cached_db)) + } + Err(reason) => Err(reason), + } +} + +/// Build the agent-context index for `project_path` at `index_path`, which is +/// outside `project_path`. +/// +/// `AgentContextIndex::build` only walks and reads; `save` writes +/// `/manifest.json` and `.db`, and `index_path` is in +/// the user's cache. Nothing here touches the audited tree. +/// +/// No staging layer is added on top of that, deliberately: `save_to_sqlite` +/// already builds into a process-unique scratch file and `rename`s it into +/// place, so two audits of one project racing here leave a whole index or the +/// previous one, never a half-populated `functions` table — which would +/// under-report violations and read as a pass. +/// +/// An index with no definitions in it is an ERROR, not an empty pass. A +/// directory holding no code parses to zero functions, zero functions violate +/// no floor, and "nothing below grade A" is a sentence that would then be +/// printed over a project pmat never read a line of. Absence rendered as +/// success is this codebase's signature defect; the caller turns this Err into +/// the same "not measured" verdict an unbuildable index gets. +fn build_out_of_tree_index(project_path: &Path, index_path: &Path) -> Result<(), String> { + crate::status_eprintln!( + "CB-200: no index in {} \u{2014} building pmat's own at {} (the audited tree is not written to)", + project_path.display(), + index_path.display() + ); + let index = crate::services::agent_context::AgentContextIndex::build(project_path) + .map_err(|e| format!("no index could be built for this project ({e})"))?; + if index.all_functions().is_empty() { + return Err( + "an index built from this project holds no definitions, so there is nothing to grade" + .to_string(), + ); + } + crate::utils::pmat_cache_dir::ensure_parent_dir(index_path) + .map_err(|e| format!("pmat's cache directory is not writable ({e})"))?; + // Reclaim entries for projects that no longer exist before adding another + // 79 MB one. Here rather than on the read path: this is the rare branch, + // and a cache that is only ever added to is a disk leak with a nice name. + crate::utils::pmat_cache_dir::sweep_idle_state( + "index", + crate::utils::pmat_cache_dir::STATE_MAX_IDLE, + ); + index + .save(index_path) + .map_err(|e| format!("the index could not be saved to pmat's cache ({e})"))?; + let db_path = index_path.with_extension("db"); + if !db_path.exists() { + return Err(format!( + "the index was built but {} was not written", + db_path.display() + )); + } + Ok(()) +} + +/// A threshold that cannot be parsed must not be read as a threshold nothing +/// violates. Decided from the CONFIG alone, so it is answered before any index +/// is looked for: "no index" must not launder a broken floor into a skip. +fn unparseable_floor_verdict(min_grade: &str) -> ComplianceCheck { + tdg_check( + CheckStatus::Fail, + Severity::Error, + format!( + "minimum grade {min_grade:?} is not a grade this codebase produces (known: {}). \ + A threshold that cannot be parsed must not be read as a threshold nothing violates.", + GRADE_VARIANTS.join(", ") + ), + ) +} + +/// The floor admits every grade the codebase produces, so this verdict is +/// derived from the CONFIG and not from the index: no row, fresh or stale, can +/// violate it. It therefore carries no staleness note and is not demoted — the +/// invariant being kept is that a `Pass` never rests on a stale reading, and +/// this one rests on no reading at all. It is also answered before the index is +/// resolved, so a floor of "F" never pays for an index build it cannot use. +fn floor_admits_everything_verdict(min_grade: &str) -> ComplianceCheck { + tdg_check( + CheckStatus::Pass, + Severity::Info, + format!("Minimum grade {min_grade} \u{2014} no grades below threshold"), + ) +} + +/// Every glob a violation may be excluded by: `.pmat.yaml`'s, then +/// `.pmat-gates.toml`'s. A pattern that does not compile is dropped, which +/// keeps a typo from excluding everything. +fn tdg_exclude_patterns( + comply_config: &ComplyConfig, + overrides: &TdgGateOverrides, +) -> Vec { + comply_config + .thresholds + .tdg_exclude_paths + .iter() + .map(String::as_str) + .chain(overrides.exclude.iter().map(String::as_str)) + .filter_map(|p| glob::Pattern::new(p).ok()) + .collect() +} + +#[provable_contracts_macros::contract("pmat-core.yaml", equation = "path_exists")] +pub(crate) fn check_tdg_grade_gate( + project_path: &Path, + comply_config: &ComplyConfig, +) -> ComplianceCheck { + check_tdg_grade_gate_with_index( + project_path, + comply_config, + &crate::utils::pmat_cache_dir::comply_index_path(project_path), + ) +} + +/// CB-200 with the out-of-tree index location passed in. +/// +/// A parameter and not an environment variable for the tests' sake: a test that +/// has to `set_var` to stay out of the developer's real cache is a test that +/// cannot run in parallel with any other, and this file has forty that do. +/// `$PMAT_CACHE_DIR` still moves the default — see +/// [`user_cache_root`](crate::utils::pmat_cache_dir::user_cache_root) — it is +/// just not how a test gets a private one. +fn check_tdg_grade_gate_with_index( + project_path: &Path, + comply_config: &ComplyConfig, + cache_index: &Path, +) -> ComplianceCheck { + let overrides = load_tdg_gate_overrides(project_path); + let min_grade = overrides + .min_grade + .as_deref() + .unwrap_or(&comply_config.thresholds.min_tdg_grade); + let Some(passing_grades) = passing_spellings(min_grade) else { + return unparseable_floor_verdict(min_grade); + }; + if passing_grades.len() == GRADE_VARIANTS.len() { + return floor_admits_everything_verdict(min_grade); + } + let index = match resolve_tdg_index(project_path, cache_index) { + Ok(index) => index, + Err(reason) => return absent_index_verdict(&overrides.baseline, &reason), + }; + // A stale index still measures something REAL — an OLDER tree — and it is a + // rebuild that would measure nothing. For the project's own index the + // staleness is reported instead of silently repaired (#939, #1045); the + // copy pmat owns was rebuilt above unless the rebuild failed, so reaching + // here with a stale one means the note is carrying real news. + let stale = is_index_stale(project_path, index.db_path()); + let violations = match query_tdg_violations(index.db_path(), &passing_grades) { + Ok(v) => v, + Err(check) => return check, + }; + let exclude_patterns = tdg_exclude_patterns(comply_config, &overrides); + let filtered: Vec<&TdgViolation> = violations + .iter() + .filter(|v| !is_tdg_violation_excluded(v, &exclude_patterns)) + .collect(); + let verdict = tdg_grade_verdict( + &filtered, + violations.len(), + min_grade, + &overrides.baseline, + if stale { STALE_INDEX_NOTE } else { "" }, + ); + note_index_provenance(demote_pass_when_stale(verdict, stale), &index) +} + +/// What a stale index costs the reader, and the command that actually fixes it. +/// +/// The advice used to be `pmat query "x"`, which does not refresh an index that +/// already exists (#1045). `load_or_build_index` +/// (`cli/handlers/query_handler/indexing.rs:251`) BUILDS only when both +/// `.pmat/context.idx` and `.pmat/context.db` are missing; otherwise it loads +/// and updates in memory, and `maybe_save_incremental` writes that back only +/// when more than 50 files or 5% of the index changed (#212). Edit a handful of +/// files and the db is never rewritten — so a reader following the advice saw +/// the identical stale verdict, with the identical advice, indefinitely. +/// `--rebuild-index` forces the rebuild and the save. +/// +/// This is not academic: `.pmat-gates.toml`'s CB-200 baseline was first +/// committed 216 too high because it was derived against an index that was +/// stale by 265 definitions, and nothing in the loop said so loudly enough. +const STALE_INDEX_NOTE: &str = " (index is stale: source files are newer than .pmat/context.db, \ + so this count describes an OLDER tree - run `pmat query \"x\" --rebuild-index` to refresh; \ + a plain `pmat query` will NOT rewrite an index that already exists unless more than 50 \ + files or 5% of it changed)"; + +/// A stale index may report debt, but it may never report a clean bill. +/// +/// CB-200 reads `.pmat/context.db`. When sources are newer than the db, the +/// count is a measurement of an older tree: the definitions it names may +/// already be fixed, and — the direction that matters — the ones added since +/// are not in it at all. Passing on that is this project's signature defect, +/// absence rendered as success, and it is how the ratchet baseline was first +/// banked 216 units too high. +/// +/// `Warn`, not `Fail`, and the choice is deliberate on both sides: +/// +/// * not `Pass`, because `retain_blocking_checks` (check.rs:270) switches on +/// `CheckStatus` alone and drops every `Pass`, so under +/// `comply check --failures-only` — the exact invocation `quality-gate.yml` +/// runs — the sentence saying the measurement is stale would be discarded. +/// `Warn` survives into `report.summary.warn`, which is tallied before the +/// list is narrowed. +/// * not `Fail`, because a stale index is not evidence of a regression, and a +/// gate that goes red the moment anyone edits a file after indexing would be +/// turned off within a day. `--strict` still escalates warnings. +/// +/// A stale `Fail` or `Warn` is left exactly as it is: staleness never makes a +/// verdict more lenient, only less conclusive. +fn demote_pass_when_stale(check: ComplianceCheck, stale: bool) -> ComplianceCheck { + if !stale || check.status != CheckStatus::Pass { + return check; + } + ComplianceCheck { + status: CheckStatus::Warn, + severity: Severity::Warning, + message: format!( + "NOT A VERDICT ON THE CURRENT TREE - the index is stale, so this is not a pass: {}", + check.message + ), + ..check + } +} + +/// CB-200 measured nothing at all: neither the project's index nor pmat's own +/// could be read, and `reason` says which failed and how. +/// +/// `Skip` for a project that never opted into the ratchet. It has recorded no +/// baseline to hold, and pmat must not fail every fresh clone of every repo +/// that runs `comply check` — around 40 of 155 checks only run where some +/// state exists, and a blanket "unmeasured is a failure" rule turns a fresh +/// clone into a wall of red and makes failure counts incomparable between +/// machines. That is a different defect, not a fix for this one. +/// +/// `Fail` for a project that RECORDED a `[tdg] baseline`. `Skip` is not counted +/// by `ComplianceReport::is_compliant`, which tallies `Fail` only, so a +/// declared ratchet that could not be measured used to report success. A +/// ratchet that did not run has not held; it has not been asked. "We could not +/// measure it" must never read as "it did not regress" — the rule +/// `.pmat-ratchet.toml` already applies to its own baselines, where an +/// UNMEASURABLE metric fails rather than passes. +/// +/// Since #1008 this is a genuinely exceptional path rather than the normal one: +/// an absent index is now built out-of-tree (see [`TdgIndex`]), so getting here +/// means the project holds no code to grade, or pmat has nowhere writable to +/// keep an index. +fn absent_index_verdict(baseline: &TdgBaseline, reason: &str) -> ComplianceCheck { + const HOW: &str = "CB-200 builds its own index OUTSIDE the audited project \ + (under $PMAT_CACHE_DIR, else the platform cache directory) so that a \ + fresh checkout can be measured without being written to (#1008) \ + \u{2014} this run could not. Point $PMAT_CACHE_DIR at a writable \ + directory, or build an index in the project with `pmat query \"x\" \ + --rebuild-index`, then re-run."; + match baseline { + TdgBaseline::Absent => tdg_check( + CheckStatus::Skip, + Severity::Info, + format!( + "Not measured: {reason}, and this project records no \ + `[tdg] baseline` for CB-200 to hold. {HOW}" + ), + ), + TdgBaseline::Held(b) => tdg_check( + CheckStatus::Fail, + Severity::Error, + format!( + "Not measured: {reason}, so the recorded `[tdg] baseline` of {b} \ + was never checked. A ratchet that did not run has not held - an unmeasured \ + gate must not report success. {HOW}" + ), + ), + TdgBaseline::Unreadable(raw) => unreadable_baseline_verdict(raw), + } +} + +/// The most anyone reads before scrolling. Both listings stop here. +const TDG_MAX_LISTED: usize = 10; + +/// Worst first, and deterministically so. +/// +/// The listing used to be `filtered.iter().take(10)` over an unordered SQLite +/// scan with no `ORDER BY`. Two runs of the same gate over the same database +/// printing a different ten is indistinguishable, to the reader, from the tree +/// having changed — and on a flat distribution of 1,905 across 1,052 files +/// there is nothing else in the message to tell them apart. +/// +/// `Grade`'s derived `Ord` runs best-to-worst (`APlus` first), so the worst +/// grade is the LARGEST; a spelling the scale does not know sorts above even +/// `F`, because a violation nobody can rank is the one most worth looking at. +fn rank_tdg_violations<'a>(filtered: &[&'a TdgViolation]) -> Vec<&'a TdgViolation> { + fn severity_rank(v: &TdgViolation) -> usize { + Grade::from_variant_name(&v.tdg_grade).map_or(usize::MAX, |g| g as usize) + } + let mut ranked: Vec<&TdgViolation> = filtered.to_vec(); + ranked.sort_by(|a, b| { + severity_rank(b) + .cmp(&severity_rank(a)) + .then_with(|| b.complexity.cmp(&a.complexity)) + .then_with(|| a.file_path.cmp(&b.file_path)) + .then_with(|| a.start_line.cmp(&b.start_line)) + }); + ranked +} + +/// Up to `limit` offenders, worst first, with a truncation line that names how +/// many were not shown. `limit` of 0 still lists one: a Fail that names nothing +/// is a number the reader cannot act on. +fn tdg_offender_listing(filtered: &[&TdgViolation], limit: usize) -> String { + let limit = limit.clamp(1, TDG_MAX_LISTED); + let ranked = rank_tdg_violations(filtered); + let mut details: Vec = ranked + .iter() + .take(limit) + .map(|v| { + format!( + " {}:{} {} [{}] (complexity: {})", + v.file_path, v.start_line, v.function_name, v.tdg_grade, v.complexity + ) + }) + .collect(); + if ranked.len() > limit { + details.push(format!(" ... and {} more", ranked.len() - limit)); + } + details.join("\n") +} + +/// How many distinct files the surviving violations span — the shape of the +/// debt, not just its size. 1,905 in one file is a refactor; 1,905 across 1,052 +/// files is a policy, and the reader cannot tell which from a count alone. +fn tdg_violation_file_count(filtered: &[&TdgViolation]) -> usize { + filtered + .iter() + .map(|v| v.file_path.as_str()) + .collect::>() + .len() +} + +fn tdg_check(status: CheckStatus, severity: Severity, message: String) -> ComplianceCheck { + ComplianceCheck { + name: "CB-200: TDG Grade Gate".into(), + status, + message, + severity, + } +} + +/// CB-200's verdict over a completed measurement. +/// +/// Four outcomes, and which one you get depends only on the baseline the +/// project recorded: +/// +/// ```text +/// baseline unreadable -> Fail (a bound nobody can read is not a bound) +/// no baseline -> Fail on any violation (unchanged) +/// count <= baseline -> Pass, carrying count AND baseline +/// count > baseline -> Fail, naming the excess and the offenders +/// ``` +fn tdg_grade_verdict( + filtered: &[&TdgViolation], + queried_rows: usize, + min_grade: &str, + baseline: &TdgBaseline, + staleness: &str, +) -> ComplianceCheck { + match baseline { + TdgBaseline::Unreadable(raw) => unreadable_baseline_verdict(raw), + TdgBaseline::Absent => zero_tolerance_verdict(filtered, queried_rows, min_grade, staleness), + TdgBaseline::Held(b) if filtered.len() > *b => { + over_baseline_verdict(filtered, *b, min_grade, staleness) + } + TdgBaseline::Held(b) => within_baseline_verdict(filtered, *b, min_grade, staleness), + } +} + +fn unreadable_baseline_verdict(raw: &str) -> ComplianceCheck { + tdg_check( + CheckStatus::Fail, + Severity::Error, + format!( + "`[tdg] baseline` in .pmat-gates.toml is {raw}, which is not a count. A ratchet \ + baseline that cannot be read must not be read as a baseline nothing exceeds \ + \u{2014} write a non-negative integer, or delete the key to restore zero tolerance." + ), + ) +} + +/// A project that records no baseline gets exactly what it got before: any +/// surviving violation fails. The message is unchanged, byte for byte, so +/// adding a ratchet here cannot move another repository's output. +fn zero_tolerance_verdict( + filtered: &[&TdgViolation], + queried_rows: usize, + min_grade: &str, + staleness: &str, +) -> ComplianceCheck { + let count = filtered.len(); + if count == 0 { + return tdg_check( + CheckStatus::Pass, + Severity::Info, + format!( + "All non-test functions meet minimum grade {min_grade}{}{staleness}", + if queried_rows == 0 { + String::new() + } else { + format!(" ({queried_rows} test/excluded functions skipped)") + } + ), + ); + } + tdg_check( + CheckStatus::Fail, + Severity::Error, + format!( + "{count} function(s) below minimum grade {min_grade}{staleness}\n{}", + tdg_offender_listing(filtered, TDG_MAX_LISTED) + ), + ) +} + +/// At or under the recorded baseline: the gate passes, and says why it is not +/// clean while doing so. +/// +/// `Warn`, not `Pass`, and the reason is mechanical rather than aesthetic. +/// +/// `Pass` was the first choice — held-flat debt must not block a release, which +/// is the whole point of a ratchet — with `Severity::Warning` carrying the "not +/// clean" signal. That does not work: `retain_blocking_checks` (check.rs:270) +/// switches on `CheckStatus` ALONE and drops `Pass` unconditionally, ignoring +/// `Severity` entirely. `quality-gate.yml` runs +/// `pmat comply check --failures-only`, so the one line saying "1,904 +/// definitions are below the floor" was discarded by the exact invocation CI +/// uses. A gate that hides its own debt from the only place anyone reads it is +/// how 1,904 accumulated unseen in the first place. +/// +/// `Warn` does not block either — `exit_policy` (check.rs:241) only turns +/// warnings into a non-zero code under `--strict` — but it IS counted in +/// `report.summary.warn`, and the summary is deliberately tallied before the +/// list is narrowed, so the count survives `--failures-only`. The debt is +/// therefore always reachable, which was the requirement. +/// +/// A clean tree at baseline 0 still reports `Pass`/`Info`: nothing is being +/// held, so there is nothing to warn about. +fn within_baseline_verdict( + filtered: &[&TdgViolation], + baseline: usize, + min_grade: &str, + staleness: &str, +) -> ComplianceCheck { + let count = filtered.len(); + let slack = baseline - count; + let slack_note = if slack == 0 { + String::new() + } else { + format!( + " The tree is {slack} under the recorded baseline: lower `[tdg] baseline` to \ + {count} in .pmat-gates.toml to bank it \u{2014} a baseline the tree has already \ + beaten is headroom for new debt." + ) + }; + if count == 0 { + return tdg_check( + CheckStatus::Pass, + if slack == 0 { + Severity::Info + } else { + Severity::Warning + }, + format!( + "0 definitions below minimum grade {min_grade}, against a recorded baseline of \ + {baseline}.{slack_note}{staleness}" + ), + ); + } + tdg_check( + CheckStatus::Warn, + Severity::Warning, + format!( + "{count} definition(s) below minimum grade {min_grade} across {} file(s), at the \ + recorded baseline of {baseline} \u{2014} this is debt held flat, not a clean tree. \ + Any new definition below {min_grade} fails this gate.{slack_note}{staleness}", + tdg_violation_file_count(filtered) + ), + ) +} + +/// Over the recorded baseline: closed, naming how many and by how much. +/// +/// The caveat about the listing is not hedging. A baseline is a COUNT, so it +/// cannot identify WHICH definitions are new — only that there are more than +/// there were. Presenting the worst-graded survivors as "the ones you just +/// added" would be a claim the measurement does not support, and the reader +/// would chase the wrong functions. +fn over_baseline_verdict( + filtered: &[&TdgViolation], + baseline: usize, + min_grade: &str, + staleness: &str, +) -> ComplianceCheck { + let count = filtered.len(); + let over = count - baseline; + tdg_check( + CheckStatus::Fail, + Severity::Error, + format!( + "{count} definition(s) below minimum grade {min_grade} \u{2014} {over} OVER the \ + recorded baseline of {baseline}. A ratchet holds only if new debt is refused: fix \ + {over}, or revert what added them. Raising `[tdg] baseline` is not the fix \u{2014} \ + a baseline may only go down.{staleness}\n (the baseline is a count, not a \ + roster, so these are the worst-graded survivors, not necessarily the ones just \ + added)\n{}", + tdg_offender_listing(filtered, over) + ), + ) +} + +/// Evaluate a single custom score definition and return the compliance check +fn evaluate_custom_score( + project_path: &Path, + score_def: &crate::models::comply_config::CustomScoreDefinition, +) -> ComplianceCheck { + let check_name = format!("CB-1100: Custom Score [{}]", score_def.id); + let output = match std::process::Command::new("sh") + .args(["-c", &score_def.command]) + .current_dir(project_path) + .output() + { + Ok(o) => o, + Err(e) => { + return ComplianceCheck { + name: check_name, + status: CheckStatus::Skip, + message: format!("Failed to run command: {e}"), + severity: Severity::Info, + } + } + }; + if !output.status.success() { + return ComplianceCheck { + name: check_name, + status: CheckStatus::Fail, + message: format!( + "{}: command failed (exit {})", + score_def.name, + output.status.code().unwrap_or(-1) + ), + severity: Severity::from(score_def.severity), + }; + } + let stdout = String::from_utf8_lossy(&output.stdout); + let actual_score = match extract_score_from_output(&stdout) { + Some(s) => s, + None => { + return ComplianceCheck { + name: check_name, + status: CheckStatus::Skip, + message: format!( + "{}: could not parse score from command output", + score_def.name + ), + severity: Severity::Info, + } + } + }; + match score_def.min_score { + Some(min) if actual_score < min => ComplianceCheck { + name: check_name, + status: CheckStatus::Fail, + message: format!( + "{}: score {:.1} below minimum {:.1}", + score_def.name, actual_score, min + ), + severity: Severity::from(score_def.severity), + }, + Some(min) => ComplianceCheck { + name: check_name, + status: CheckStatus::Pass, + message: format!( + "{}: score {:.1} (min: {:.1})", + score_def.name, actual_score, min + ), + severity: Severity::Info, + }, + None => ComplianceCheck { + name: check_name, + status: CheckStatus::Pass, + message: format!("{}: score {:.1}", score_def.name, actual_score), + severity: Severity::Info, + }, + } +} + +/// CB-1100: Custom Project Scores +#[provable_contracts_macros::contract("pmat-core.yaml", equation = "path_exists")] +pub(crate) fn check_custom_scores(project_path: &Path) -> Vec { + let config = match crate::models::comply_config::PmatYamlConfig::load(project_path) { + Ok(c) => c, + Err(_) => return vec![], + }; + if config.scoring.custom_scores.is_empty() { + return vec![]; + } + config + .scoring + .custom_scores + .iter() + .map(|s| evaluate_custom_score(project_path, s)) + .collect() +} + +fn extract_score_from_output(output: &str) -> Option { + for line in output.lines() { + let line = line.trim(); + if let Ok(json) = serde_json::from_str::(line) { + if let Some(score) = json.get("score").and_then(|s| s.as_f64()) { + return Some(score); + } + } + } + for line in output.lines() { + let line = line.trim(); + if let Some(rest) = line.strip_prefix("SCORE:") { + if let Ok(score) = rest.trim().parse::() { + return Some(score); + } + } + } + None +} + +#[cfg_attr(coverage_nightly, coverage(off))] +#[cfg(test)] +mod tests_tdg_grade { + use super::*; + use crate::models::comply_config::ComplyConfig; + use std::path::PathBuf; + + /// The floor admits exactly the grades at least as good as it. + /// + /// Replaces `test_grade_ordinal` and `test_grades_below`, which pinned the + /// defect rather than the rule: they asserted `grade_ordinal("X") == 5` + /// (the catch-all that ranked every unknown, and every MODIFIED, grade + /// worse than F) and `grades_below("A") == ["B","C","D","F"]` (the + /// five-letter blindness). Both were true of the old code and both were + /// the bug. + #[test] + fn passing_set_is_the_up_set_of_the_floor() { + assert_eq!( + passing_spellings("A+").expect("A+ is a canonical grade spelling"), + vec!["A+"] + ); + assert_eq!( + passing_spellings("A").expect("A is a canonical grade spelling"), + vec!["A+", "A"] + ); + // The spelling that used to produce an EMPTY failing set and a Pass. + assert_eq!( + passing_spellings("A-").expect("A- is a canonical grade spelling"), + vec!["A+", "A", "A-"] + ); + assert_eq!( + passing_spellings("B").expect("B is a canonical grade spelling"), + vec!["A+", "A", "A-", "B+", "B"] + ); + // Only F admits everything, and that is the one honest vacuous floor. + assert_eq!( + passing_spellings("F") + .expect("F is a canonical grade spelling") + .len(), + GRADE_VARIANTS.len() + ); + } + + /// Passing and failing partition the scale at every floor — the property + /// the five-letter table did not have, proved for all eleven floors in + /// `contracts/lean/Theorems/Tdg/Grade.lean::Grade_Partition`. + #[test] + fn passing_and_failing_partition_the_scale() { + for floor in GRADE_VARIANTS { + let passing = passing_spellings(floor).expect("canonical spelling parses"); + let failing: Vec<_> = GRADE_VARIANTS + .iter() + .filter(|g| !passing.contains(g)) + .collect(); + assert_eq!( + passing.len() + failing.len(), + GRADE_VARIANTS.len(), + "floor {floor} does not partition the scale" + ); + } + } + + /// An unreadable threshold yields `None`, never an empty set. An empty set + /// is what the caller reads as "nothing violates". + #[test] + fn an_unparseable_floor_is_none_not_empty() { + for bad in ["X", "", " ", "A--", "Q", "E"] { + assert!( + passing_spellings(bad).is_none(), + "{bad:?} must not parse as a grade floor" + ); + } + // Counter-test: every canonical spelling still parses, so the guard did + // not become a spelling police. + for good in GRADE_VARIANTS { + assert!(passing_spellings(good).is_some(), "{good} must parse"); + } + } + + #[test] + fn test_missing_db_returns_skip() { + let tmp = PathBuf::from("/tmp/pmat-test-tdg-missing-db"); + let config = ComplyConfig::default(); + let result = check_tdg_grade_gate(&tmp, &config); + assert_eq!(result.status, CheckStatus::Skip); + assert!( + result.message.contains("Not measured"), + "{}", + result.message + ); + } + + /// #939: CB-200 used to BUILD and SAVE the agent-context index whenever + /// `.pmat/context.db` was missing or stale — so asking for a compliance + /// verdict wrote `.pmat/context.db` and `.pmat/context.idx` into the + /// project being audited (and spent minutes doing it on a large repo). + /// + /// It still never writes there, and that is what this test is for. What + /// #1008 changed is where the index comes from when the project has none: + /// pmat's own copy, outside the tree. The `Skip` this used to assert WAS + /// the defect — a gate that declines to measure a fresh checkout cannot + /// run in the one place a verdict decides a merge. + #[test] + fn cb200_never_builds_an_index_inside_the_audited_project() { + let cache = tempfile::tempdir().expect("create cache dir"); + let tmp = tempfile::tempdir().expect("create tempdir"); + std::fs::create_dir_all(tmp.path().join("src")).expect("create src"); + std::fs::write(tmp.path().join("src/lib.rs"), "pub fn f() {}\n").expect("write source"); + + let result = check_tdg_grade_gate_with_index( + tmp.path(), + &ComplyConfig::default(), + &cache.path().join("context.idx"), + ); + + assert!( + !tmp.path().join(".pmat").exists(), + "comply check must not write .pmat/ into the project it audits" + ); + assert_ne!( + result.status, + CheckStatus::Skip, + "#1008: a tree pmat could read must be measured, not skipped: {}", + result.message + ); + assert!( + cache.path().join("context.db").exists(), + "the index it read must be the one outside the audited tree" + ); + } + + /// A stale index is still a real measurement — it is reported, with the + /// staleness named, rather than silently repaired by a write. + #[test] + fn cb200_reports_staleness_instead_of_rebuilding() { + let tmp = tempfile::tempdir().expect("create tempdir"); + let pmat_dir = tmp.path().join(".pmat"); + std::fs::create_dir_all(&pmat_dir).expect("create .pmat"); + let db_path = pmat_dir.join("context.db"); + let conn = rusqlite::Connection::open(&db_path).expect("open db"); + conn.execute_batch( + "CREATE TABLE functions (id INTEGER PRIMARY KEY, file_path TEXT NOT NULL, \ + function_name TEXT NOT NULL, tdg_grade TEXT NOT NULL DEFAULT 'A', \ + complexity INTEGER NOT NULL DEFAULT 1, start_line INTEGER NOT NULL DEFAULT 0)", + ) + .expect("schema"); + drop(conn); + + // A source file newer than the db makes the index stale. + // + // The mtime is SET, not raced for. Writing the source after the db and + // trusting wall-clock ordering failed on CI with "fixture must be + // stale": both files landed in the same filesystem timestamp tick, so + // `is_index_stale` correctly saw nothing newer and the fixture never + // reached the behaviour under test. Filesystem timestamp granularity is + // coarser than the microseconds between two writes, and varies by + // filesystem — which is exactly the kind of thing that passes on the + // author's machine and fails in CI. + std::fs::create_dir_all(tmp.path().join("src")).expect("create src"); + let src = tmp.path().join("src/lib.rs"); + std::fs::write(&src, "pub fn f() {}\n").expect("write source"); + let db_mtime = std::fs::metadata(&db_path) + .and_then(|m| m.modified()) + .expect("db mtime"); + std::fs::File::options() + .write(true) + .open(&src) + .and_then(|f| f.set_modified(db_mtime + std::time::Duration::from_secs(60))) + .expect("set source mtime a clear minute past the db"); + assert!( + is_index_stale(tmp.path(), &db_path), + "fixture must be stale" + ); + + let before = std::fs::metadata(&db_path).expect("meta").len(); + let result = check_tdg_grade_gate(tmp.path(), &ComplyConfig::default()); + let after = std::fs::metadata(&db_path).expect("meta").len(); + + assert_eq!(before, after, "the audit rewrote the index it was reading"); + assert!( + !pmat_dir.join("context.idx").exists(), + "comply check must not build .pmat/context.idx" + ); + assert!( + result.message.contains("index is stale"), + "staleness must be reported, not silently repaired: {}", + result.message + ); + } + + #[test] + fn test_grade_f_no_violations() { + let tmp = PathBuf::from("/tmp/pmat-test-tdg-grade-f"); + let mut config = ComplyConfig::default(); + config.thresholds.min_tdg_grade = "F".to_string(); + let result = check_tdg_grade_gate(&tmp, &config); + assert!(result.status == CheckStatus::Skip || result.status == CheckStatus::Pass); + } + + #[test] + fn test_in_memory_db_with_violations() { + let tmp = tempfile::tempdir().expect("create tempdir"); + let pmat_dir = tmp.path().join(".pmat"); + std::fs::create_dir_all(&pmat_dir).expect("create .pmat"); + let db_path = pmat_dir.join("context.db"); + let conn = rusqlite::Connection::open(&db_path).expect("open db"); + conn.execute_batch("CREATE TABLE IF NOT EXISTS functions (id INTEGER PRIMARY KEY, file_path TEXT NOT NULL, function_name TEXT NOT NULL, signature TEXT NOT NULL DEFAULT '', definition_type TEXT NOT NULL DEFAULT 'function', doc_comment TEXT NOT NULL DEFAULT '', source TEXT NOT NULL DEFAULT '', start_line INTEGER NOT NULL DEFAULT 0, end_line INTEGER NOT NULL DEFAULT 0, language TEXT NOT NULL DEFAULT 'Rust', checksum TEXT NOT NULL DEFAULT '', tdg_score REAL NOT NULL DEFAULT 0.0, tdg_grade TEXT NOT NULL DEFAULT 'A', complexity INTEGER NOT NULL DEFAULT 1, cognitive_complexity INTEGER NOT NULL DEFAULT 1, big_o TEXT NOT NULL DEFAULT 'O(1)', satd_count INTEGER NOT NULL DEFAULT 0, loc INTEGER NOT NULL DEFAULT 0, commit_count INTEGER NOT NULL DEFAULT 0, churn_score REAL NOT NULL DEFAULT 0.0, clone_count INTEGER NOT NULL DEFAULT 0, pattern_diversity REAL NOT NULL DEFAULT 0.0, fault_annotations TEXT NOT NULL DEFAULT '[]')").expect("create schema"); + conn.execute("INSERT INTO functions (file_path, function_name, tdg_grade, complexity, start_line) VALUES ('src/core.rs', 'good_fn', 'A', 5, 10)", []).expect("insert A"); + conn.execute("INSERT INTO functions (file_path, function_name, tdg_grade, complexity, start_line) VALUES ('src/legacy.rs', 'bad_fn', 'D', 42, 20)", []).expect("insert D"); + conn.execute("INSERT INTO functions (file_path, function_name, tdg_grade, complexity, start_line) VALUES ('src/awful.rs', 'terrible_fn', 'F', 60, 30)", []).expect("insert F"); + conn.execute("INSERT INTO functions (file_path, function_name, tdg_grade, complexity, start_line) VALUES ('src/tests/helpers.rs', 'test_helper', 'D', 35, 40)", []).expect("insert test D"); + drop(conn); + let config = ComplyConfig::default(); + let result = check_tdg_grade_gate(tmp.path(), &config); + assert_eq!(result.status, CheckStatus::Fail); + assert!(result + .message + .contains("2 function(s) below minimum grade A")); + assert!(result.message.contains("src/legacy.rs:20 bad_fn [D]")); + assert!(result.message.contains("src/awful.rs:30 terrible_fn [F]")); + assert!(!result.message.contains("test_helper")); + } + + #[test] + fn test_tdg_exclude_paths() { + let tmp = tempfile::tempdir().expect("create tempdir"); + let pmat_dir = tmp.path().join(".pmat"); + std::fs::create_dir_all(&pmat_dir).expect("create .pmat"); + let db_path = pmat_dir.join("context.db"); + let conn = rusqlite::Connection::open(&db_path).expect("open db"); + conn.execute_batch("CREATE TABLE IF NOT EXISTS functions (id INTEGER PRIMARY KEY, file_path TEXT NOT NULL, function_name TEXT NOT NULL, signature TEXT NOT NULL DEFAULT '', definition_type TEXT NOT NULL DEFAULT 'function', doc_comment TEXT NOT NULL DEFAULT '', source TEXT NOT NULL DEFAULT '', start_line INTEGER NOT NULL DEFAULT 0, end_line INTEGER NOT NULL DEFAULT 0, language TEXT NOT NULL DEFAULT 'Rust', checksum TEXT NOT NULL DEFAULT '', tdg_score REAL NOT NULL DEFAULT 0.0, tdg_grade TEXT NOT NULL DEFAULT 'A', complexity INTEGER NOT NULL DEFAULT 1, cognitive_complexity INTEGER NOT NULL DEFAULT 1, big_o TEXT NOT NULL DEFAULT 'O(1)', satd_count INTEGER NOT NULL DEFAULT 0, loc INTEGER NOT NULL DEFAULT 0, commit_count INTEGER NOT NULL DEFAULT 0, churn_score REAL NOT NULL DEFAULT 0.0, clone_count INTEGER NOT NULL DEFAULT 0, pattern_diversity REAL NOT NULL DEFAULT 0.0, fault_annotations TEXT NOT NULL DEFAULT '[]')").expect("create schema"); + conn.execute("INSERT INTO functions (file_path, function_name, tdg_grade, complexity, start_line) VALUES ('vendor/lib.rs', 'vendor_fn', 'D', 40, 10)", []).expect("insert vendor D"); + conn.execute("INSERT INTO functions (file_path, function_name, tdg_grade, complexity, start_line) VALUES ('src/main.rs', 'main_fn', 'D', 30, 5)", []).expect("insert src D"); + drop(conn); + let mut config = ComplyConfig::default(); + config.thresholds.tdg_exclude_paths = vec!["vendor/*".to_string()]; + let result = check_tdg_grade_gate(tmp.path(), &config); + assert_eq!(result.status, CheckStatus::Fail); + assert!(result.message.contains("1 function(s)")); + assert!(result.message.contains("main_fn")); + assert!(!result.message.contains("vendor_fn")); + } + + #[test] + fn test_all_pass_with_good_grades() { + let tmp = tempfile::tempdir().expect("create tempdir"); + let pmat_dir = tmp.path().join(".pmat"); + std::fs::create_dir_all(&pmat_dir).expect("create .pmat"); + let db_path = pmat_dir.join("context.db"); + let conn = rusqlite::Connection::open(&db_path).expect("open db"); + conn.execute_batch("CREATE TABLE IF NOT EXISTS functions (id INTEGER PRIMARY KEY, file_path TEXT NOT NULL, function_name TEXT NOT NULL, signature TEXT NOT NULL DEFAULT '', definition_type TEXT NOT NULL DEFAULT 'function', doc_comment TEXT NOT NULL DEFAULT '', source TEXT NOT NULL DEFAULT '', start_line INTEGER NOT NULL DEFAULT 0, end_line INTEGER NOT NULL DEFAULT 0, language TEXT NOT NULL DEFAULT 'Rust', checksum TEXT NOT NULL DEFAULT '', tdg_score REAL NOT NULL DEFAULT 0.0, tdg_grade TEXT NOT NULL DEFAULT 'A', complexity INTEGER NOT NULL DEFAULT 1, cognitive_complexity INTEGER NOT NULL DEFAULT 1, big_o TEXT NOT NULL DEFAULT 'O(1)', satd_count INTEGER NOT NULL DEFAULT 0, loc INTEGER NOT NULL DEFAULT 0, commit_count INTEGER NOT NULL DEFAULT 0, churn_score REAL NOT NULL DEFAULT 0.0, clone_count INTEGER NOT NULL DEFAULT 0, pattern_diversity REAL NOT NULL DEFAULT 0.0, fault_annotations TEXT NOT NULL DEFAULT '[]')").expect("create schema"); + conn.execute("INSERT INTO functions (file_path, function_name, tdg_grade, complexity, start_line) VALUES ('src/lib.rs', 'good_fn', 'A', 3, 1)", []).expect("insert A"); + conn.execute("INSERT INTO functions (file_path, function_name, tdg_grade, complexity, start_line) VALUES ('src/util.rs', 'ok_fn', 'B', 8, 10)", []).expect("insert B"); + drop(conn); + let mut config = ComplyConfig::default(); + config.thresholds.min_tdg_grade = "B".to_string(); + let result = check_tdg_grade_gate(tmp.path(), &config); + assert_eq!(result.status, CheckStatus::Pass); + assert!(result.message.contains("meet minimum grade B")); + } + + #[test] + fn test_pmat_gates_toml_min_grade_override() { + let tmp = tempfile::tempdir().expect("create tempdir"); + let pmat_dir = tmp.path().join(".pmat"); + std::fs::create_dir_all(&pmat_dir).expect("create .pmat"); + let db_path = pmat_dir.join("context.db"); + let conn = rusqlite::Connection::open(&db_path).expect("open db"); + conn.execute_batch("CREATE TABLE IF NOT EXISTS functions (id INTEGER PRIMARY KEY, file_path TEXT NOT NULL, function_name TEXT NOT NULL, signature TEXT NOT NULL DEFAULT '', definition_type TEXT NOT NULL DEFAULT 'function', doc_comment TEXT NOT NULL DEFAULT '', source TEXT NOT NULL DEFAULT '', start_line INTEGER NOT NULL DEFAULT 0, end_line INTEGER NOT NULL DEFAULT 0, language TEXT NOT NULL DEFAULT 'Rust', checksum TEXT NOT NULL DEFAULT '', tdg_score REAL NOT NULL DEFAULT 0.0, tdg_grade TEXT NOT NULL DEFAULT 'A', complexity INTEGER NOT NULL DEFAULT 1, cognitive_complexity INTEGER NOT NULL DEFAULT 1, big_o TEXT NOT NULL DEFAULT 'O(1)', satd_count INTEGER NOT NULL DEFAULT 0, loc INTEGER NOT NULL DEFAULT 0, commit_count INTEGER NOT NULL DEFAULT 0, churn_score REAL NOT NULL DEFAULT 0.0, clone_count INTEGER NOT NULL DEFAULT 0, pattern_diversity REAL NOT NULL DEFAULT 0.0, fault_annotations TEXT NOT NULL DEFAULT '[]')").expect("create schema"); + conn.execute("INSERT INTO functions (file_path, function_name, tdg_grade, complexity, start_line) VALUES ('src/lib.rs', 'good_fn', 'A', 3, 1)", []).expect("insert A"); + conn.execute("INSERT INTO functions (file_path, function_name, tdg_grade, complexity, start_line) VALUES ('src/util.rs', 'ok_fn', 'B', 8, 10)", []).expect("insert B"); + drop(conn); + std::fs::write( + tmp.path().join(".pmat-gates.toml"), + "[tdg]\nmin_grade = \"B\"\n", + ) + .expect("write gates toml"); + let config = ComplyConfig::default(); + let result = check_tdg_grade_gate(tmp.path(), &config); + assert_eq!(result.status, CheckStatus::Pass); + } + + #[test] + fn test_pmat_gates_toml_exclude_override() { + let tmp = tempfile::tempdir().expect("create tempdir"); + let pmat_dir = tmp.path().join(".pmat"); + std::fs::create_dir_all(&pmat_dir).expect("create .pmat"); + let db_path = pmat_dir.join("context.db"); + let conn = rusqlite::Connection::open(&db_path).expect("open db"); + conn.execute_batch("CREATE TABLE IF NOT EXISTS functions (id INTEGER PRIMARY KEY, file_path TEXT NOT NULL, function_name TEXT NOT NULL, signature TEXT NOT NULL DEFAULT '', definition_type TEXT NOT NULL DEFAULT 'function', doc_comment TEXT NOT NULL DEFAULT '', source TEXT NOT NULL DEFAULT '', start_line INTEGER NOT NULL DEFAULT 0, end_line INTEGER NOT NULL DEFAULT 0, language TEXT NOT NULL DEFAULT 'Rust', checksum TEXT NOT NULL DEFAULT '', tdg_score REAL NOT NULL DEFAULT 0.0, tdg_grade TEXT NOT NULL DEFAULT 'A', complexity INTEGER NOT NULL DEFAULT 1, cognitive_complexity INTEGER NOT NULL DEFAULT 1, big_o TEXT NOT NULL DEFAULT 'O(1)', satd_count INTEGER NOT NULL DEFAULT 0, loc INTEGER NOT NULL DEFAULT 0, commit_count INTEGER NOT NULL DEFAULT 0, churn_score REAL NOT NULL DEFAULT 0.0, clone_count INTEGER NOT NULL DEFAULT 0, pattern_diversity REAL NOT NULL DEFAULT 0.0, fault_annotations TEXT NOT NULL DEFAULT '[]')").expect("create schema"); + conn.execute("INSERT INTO functions (file_path, function_name, tdg_grade, complexity, start_line) VALUES ('src/core_generated.rs', 'gen_fn', 'D', 40, 10)", []).expect("insert generated D"); + conn.execute("INSERT INTO functions (file_path, function_name, tdg_grade, complexity, start_line) VALUES ('src/real.rs', 'real_fn', 'D', 30, 5)", []).expect("insert real D"); + drop(conn); + std::fs::write( + tmp.path().join(".pmat-gates.toml"), + "[tdg]\nexclude = [\"**/*_generated.rs\"]\n", + ) + .expect("write gates toml"); + let config = ComplyConfig::default(); + let result = check_tdg_grade_gate(tmp.path(), &config); + assert_eq!(result.status, CheckStatus::Fail); + assert!(result.message.contains("1 function(s)")); + assert!(result.message.contains("real_fn")); + assert!(!result.message.contains("gen_fn")); + } + + #[test] + fn test_is_index_stale_no_db() { + let tmp = tempfile::tempdir().expect("create tempdir"); + assert!(is_index_stale( + tmp.path(), + &tmp.path().join("nonexistent.db") + )); + } + + #[test] + fn test_is_index_stale_fresh_db() { + let tmp = tempfile::tempdir().expect("create tempdir"); + let src_dir = tmp.path().join("src"); + std::fs::create_dir_all(&src_dir).expect("create src"); + std::fs::write(src_dir.join("lib.rs"), "fn main() {}").expect("write src"); + std::thread::sleep(std::time::Duration::from_millis(50)); + let db_path = tmp.path().join("context.db"); + std::fs::write(&db_path, "").expect("write db"); + assert!(!is_index_stale(tmp.path(), &db_path)); + } + + #[test] + fn test_is_index_stale_outdated_db() { + let tmp = tempfile::tempdir().expect("create tempdir"); + let db_path = tmp.path().join("context.db"); + std::fs::write(&db_path, "").expect("write db"); + std::thread::sleep(std::time::Duration::from_millis(50)); + let src_dir = tmp.path().join("src"); + std::fs::create_dir_all(&src_dir).expect("create src"); + std::fs::write(src_dir.join("lib.rs"), "fn main() {}").expect("write src"); + assert!(is_index_stale(tmp.path(), &db_path)); + } + + #[test] + fn test_is_source_file() { + assert!(is_source_file(Path::new("foo.rs"))); + assert!(is_source_file(Path::new("foo.py"))); + assert!(is_source_file(Path::new("foo.ts"))); + assert!(!is_source_file(Path::new("foo.txt"))); + assert!(!is_source_file(Path::new("foo.toml"))); + assert!(!is_source_file(Path::new("Makefile"))); + } + + // ── CB-200 as a RATCHET ────────────────────────────────────────────── + // + // The gate was BLIND until 2026-08-20: a five-letter reader against an + // eleven-letter writer, so it saw 247 violations and could not see 1,719. + // Every historical "CB-200 passed" came from that version. Once it could + // see, it measured 1,905 below-A definitions across 1,052 files, max 12 per + // file — a flat distribution with no hotspot and no bounded refactor. + // + // The two ways to make that green are both threshold-lowering in disguise: + // drop `min_tdg_grade`, or add an exclude glob. One of them (187f506885) is + // how a past "pass" was manufactured. Neither is done here. The gate holds + // the count flat instead, refuses any increase closed, and prints the + // absolute number on every outcome so that passing can never be mistaken + // for clean. + + /// A project with `.pmat/context.db` holding exactly these rows. + pub(super) fn tdg_fixture(rows: &[(&str, &str, &str, u32, usize)]) -> tempfile::TempDir { + let tmp = tempfile::tempdir().expect("create tempdir"); + let pmat_dir = tmp.path().join(".pmat"); + std::fs::create_dir_all(&pmat_dir).expect("create .pmat"); + let conn = rusqlite::Connection::open(pmat_dir.join("context.db")).expect("open db"); + conn.execute_batch( + "CREATE TABLE functions (id INTEGER PRIMARY KEY, file_path TEXT NOT NULL, \ + function_name TEXT NOT NULL, tdg_grade TEXT NOT NULL DEFAULT 'A', \ + complexity INTEGER NOT NULL DEFAULT 1, start_line INTEGER NOT NULL DEFAULT 0)", + ) + .expect("schema"); + for (file, name, grade, complexity, line) in rows { + conn.execute( + "INSERT INTO functions (file_path, function_name, tdg_grade, complexity, \ + start_line) VALUES (?1, ?2, ?3, ?4, ?5)", + rusqlite::params![file, name, grade, complexity, line], + ) + .expect("insert row"); + } + tmp + } + + pub(super) fn write_gates(tmp: &tempfile::TempDir, body: &str) { + std::fs::write(tmp.path().join(".pmat-gates.toml"), body).expect("write gates toml"); + } + + pub(super) fn judge(tmp: &tempfile::TempDir) -> ComplianceCheck { + check_tdg_grade_gate(tmp.path(), &ComplyConfig::default()) + } + + /// The key is OPTIONAL, and its absence is not a softer gate. + /// + /// This is the counter-test that bounds the whole change: pmat runs against + /// other repositories, and none of them has agreed to anything. A project + /// that records no baseline must get zero tolerance, byte for byte, whether + /// it has a `.pmat-gates.toml` with no `baseline` key or no file at all. + #[test] + fn without_a_baseline_key_any_violation_still_fails() { + let no_file = tdg_fixture(&[("src/a.rs", "bad", "D", 30, 5)]); + let verdict = judge(&no_file); + assert_eq!(verdict.status, CheckStatus::Fail, "{}", verdict.message); + assert!( + verdict + .message + .contains("1 function(s) below minimum grade A"), + "the no-baseline message must be unchanged: {}", + verdict.message + ); + + let other_keys = tdg_fixture(&[("src/a.rs", "bad", "D", 30, 5)]); + write_gates(&other_keys, "[tdg]\nexclude = [\"nothing/**\"]\n"); + let verdict = judge(&other_keys); + assert_eq!(verdict.status, CheckStatus::Fail, "{}", verdict.message); + assert!( + verdict + .message + .contains("1 function(s) below minimum grade A"), + "a [tdg] table without a baseline is still zero tolerance: {}", + verdict.message + ); + } + + /// At the baseline the gate PASSES — and must not read as clean. + /// + /// "1905 below A" printed at `Info` next to a genuinely empty tree is how + /// 1,905 accumulated unseen. The count, the baseline and the word "debt" + /// are all load-bearing, and so is `Severity::Warning` on a `Pass`. + /// Held debt must survive `--failures-only`, which is the ONLY invocation + /// CI runs. + /// + /// This is the counter-test for the status choice, and it exists because the + /// obvious simplification — "it passed, so return `Pass`" — is wrong in a way + /// nothing else here would catch. `retain_blocking_checks` (check.rs) matches + /// on `CheckStatus` ALONE and drops `Pass` unconditionally; `Severity` is not + /// consulted. `quality-gate.yml` runs `pmat comply check --failures-only`, so + /// a `Pass` verdict deletes the one line reporting that 1,904 definitions sit + /// below the floor, from the only report anyone reads. + /// + /// `Warn` does not block — `exit_policy` only escalates warnings under + /// `--strict` — but it is counted into `summary.warn`, and the summary is + /// tallied BEFORE the list is narrowed. That is what keeps the number + /// reachable. + /// + /// RED: change `within_baseline_verdict`'s non-empty branch back to + /// `CheckStatus::Pass` and this fails with `left: Pass, right: Warn`. + #[test] + fn held_debt_is_warn_so_failures_only_cannot_hide_it() { + let tmp = tdg_fixture(&[("src/a.rs", "one", "D", 30, 5)]); + write_gates(&tmp, "[tdg]\nbaseline = 5\n"); + let verdict = judge(&tmp); + + assert_eq!( + verdict.status, + CheckStatus::Warn, + "held debt reported as Pass is dropped by `--failures-only`; the \ + count must stay reachable. Message was: {}", + verdict.message + ); + + // ...and the counter-test: a genuinely clean tree is still a Pass, so + // "always Warn" cannot satisfy the assertion above. + let clean = tdg_fixture(&[]); + write_gates(&clean, "[tdg]\nbaseline = 5\n"); + let clean_verdict = judge(&clean); + assert_eq!( + clean_verdict.status, + CheckStatus::Pass, + "nothing is being held, so there is nothing to warn about: {}", + clean_verdict.message + ); + } + + #[test] + fn at_the_baseline_it_passes_carrying_the_count_and_the_baseline() { + let tmp = tdg_fixture(&[ + ("src/a.rs", "one", "D", 30, 5), + ("src/b.rs", "two", "C-", 20, 9), + ]); + write_gates(&tmp, "[tdg]\nbaseline = 2\n"); + let verdict = judge(&tmp); + + assert_eq!(verdict.status, CheckStatus::Warn, "{}", verdict.message); + assert_eq!( + verdict.severity, + Severity::Warning, + "held debt at Info reads as clean: {}", + verdict.message + ); + assert!( + verdict + .message + .contains("2 definition(s) below minimum grade A"), + "the absolute count must lead: {}", + verdict.message + ); + assert!( + verdict.message.contains("recorded baseline of 2"), + "the baseline must be named: {}", + verdict.message + ); + assert!( + verdict.message.contains("not a clean tree"), + "passing at baseline must not read as passing clean: {}", + verdict.message + ); + assert!( + verdict.message.contains("2 file(s)"), + "the shape of the debt is part of the report: {}", + verdict.message + ); + } + + /// One over is a failure, named as one over. + #[test] + fn one_definition_over_the_baseline_fails_and_says_by_how_much() { + let tmp = tdg_fixture(&[ + ("src/a.rs", "one", "D", 30, 5), + ("src/b.rs", "two", "C-", 20, 9), + ("src/c.rs", "three", "F", 44, 2), + ]); + write_gates(&tmp, "[tdg]\nbaseline = 2\n"); + let verdict = judge(&tmp); + + assert_eq!(verdict.status, CheckStatus::Fail, "{}", verdict.message); + assert_eq!(verdict.severity, Severity::Error); + assert!( + verdict + .message + .contains("3 definition(s) below minimum grade A"), + "the absolute count must lead even on a failure: {}", + verdict.message + ); + assert!( + verdict + .message + .contains("1 OVER the recorded baseline of 2"), + "the excess must be named: {}", + verdict.message + ); + // Exactly `over` offenders are listed, worst first, and the rest are + // counted rather than dropped. + assert!( + verdict.message.contains("src/c.rs:2 three [F]"), + "the worst survivor must be named: {}", + verdict.message + ); + assert!( + verdict.message.contains("... and 2 more"), + "the unlisted remainder must be counted: {}", + verdict.message + ); + assert!( + verdict.message.contains("may only go down"), + "the fix must not read as 'raise the baseline': {}", + verdict.message + ); + } + + /// Under the baseline is a pass that asks to be banked. + /// + /// A baseline the tree has already beaten is headroom for new debt, which + /// is the failure mode `.pmat-ratchet.toml`'s `--lower` job exists to + /// prevent. CB-200 has no such job, so it asks in the message instead. + #[test] + fn under_the_baseline_passes_and_asks_for_the_baseline_to_be_lowered() { + let tmp = tdg_fixture(&[("src/a.rs", "one", "D", 30, 5)]); + write_gates(&tmp, "[tdg]\nbaseline = 5\n"); + let verdict = judge(&tmp); + + assert_eq!(verdict.status, CheckStatus::Warn, "{}", verdict.message); + assert!( + verdict.message.contains("4 under the recorded baseline"), + "slack must be reported: {}", + verdict.message + ); + assert!( + verdict.message.contains("lower `[tdg] baseline` to 1"), + "the message must name the number to bank: {}", + verdict.message + ); + } + + /// A clean tree under a stale baseline reports the slack, not silence. + #[test] + fn a_clean_tree_still_reports_a_baseline_it_has_outgrown() { + let tmp = tdg_fixture(&[("src/a.rs", "fine", "A", 3, 1)]); + write_gates(&tmp, "[tdg]\nbaseline = 5\n"); + let verdict = judge(&tmp); + + assert_eq!(verdict.status, CheckStatus::Pass, "{}", verdict.message); + assert!( + verdict + .message + .contains("0 definitions below minimum grade A"), + "{}", + verdict.message + ); + assert!( + verdict.message.contains("lower `[tdg] baseline` to 0"), + "a baseline of 5 over an empty tree is pure headroom: {}", + verdict.message + ); + } + + /// A baseline nobody can read is not a baseline nothing exceeds. + /// + /// The same rule `passing_spellings` already enforces for `min_grade`: the + /// unreadable value is REPORTED, never quietly rounded to "no baseline", + /// because "your key does nothing" and "you chose zero tolerance" are + /// opposite claims that look identical from outside. + #[test] + fn an_unreadable_baseline_fails_closed_and_names_itself() { + for bad in ["\"many\"", "-1", "1.5", "true", "[1905]"] { + let tmp = tdg_fixture(&[("src/a.rs", "one", "D", 30, 5)]); + write_gates(&tmp, &format!("[tdg]\nbaseline = {bad}\n")); + let verdict = judge(&tmp); + assert_eq!( + verdict.status, + CheckStatus::Fail, + "baseline = {bad} must fail: {}", + verdict.message + ); + assert!( + verdict.message.contains("is not a count"), + "baseline = {bad} must name itself: {}", + verdict.message + ); + } + // Counter-test: the guard did not become a "no baseline may be small" + // rule. Zero is a real baseline — it is zero tolerance, said out loud. + let clean = tdg_fixture(&[("src/a.rs", "fine", "A", 3, 1)]); + write_gates(&clean, "[tdg]\nbaseline = 0\n"); + assert_eq!(judge(&clean).status, CheckStatus::Pass); + let dirty = tdg_fixture(&[("src/a.rs", "one", "D", 30, 5)]); + write_gates(&dirty, "[tdg]\nbaseline = 0\n"); + let verdict = judge(&dirty); + assert_eq!(verdict.status, CheckStatus::Fail, "{}", verdict.message); + assert!( + verdict + .message + .contains("1 OVER the recorded baseline of 0"), + "{}", + verdict.message + ); + } + + /// The over-correction this must NOT become: a baseline that hides debt. + /// + /// A generous baseline buys silence in exactly one place — the pass/fail + /// verdict. It must not raise the floor, must not exclude a path, and must + /// not remove a single definition from the count. Nothing here may read as + /// the sentence a genuinely clean tree gets. + #[test] + fn a_baseline_holds_debt_flat_without_hiding_any_of_it() { + let tmp = tdg_fixture(&[ + ("src/a.rs", "one", "D", 30, 5), + ("src/b.rs", "two", "C-", 20, 9), + ("src/c.rs", "three", "B+", 12, 3), + ]); + write_gates(&tmp, "[tdg]\nbaseline = 100\n"); + let verdict = judge(&tmp); + + assert_eq!(verdict.status, CheckStatus::Warn, "{}", verdict.message); + assert!( + verdict.message.contains("3 definition(s)"), + "every unit of debt stays counted: {}", + verdict.message + ); + assert!( + verdict.message.contains("minimum grade A"), + "the floor is still A — a baseline is not a lowered threshold: {}", + verdict.message + ); + // B+ is below A and is still counted: the baseline did not quietly + // narrow the alphabet the way the five-letter reader did. + assert!( + !verdict + .message + .contains("All non-test functions meet minimum grade"), + "held debt must never borrow the clean tree's sentence: {}", + verdict.message + ); + } + + /// The listing is deterministic, worst first. + /// + /// It used to be `take(10)` off a `SELECT` with no `ORDER BY`. On a flat + /// 1,905-across-1,052-files distribution there is nothing else in the + /// message to distinguish "the tree changed" from "SQLite scanned in a + /// different order". + #[test] + fn the_offender_listing_is_worst_first_and_stable() { + let tmp = tdg_fixture(&[ + ("src/mild.rs", "mild", "B+", 11, 1), + ("src/worst.rs", "worst", "F", 9, 1), + ("src/bad_simple.rs", "bad_simple", "D", 4, 1), + ("src/bad_complex.rs", "bad_complex", "D", 90, 1), + ]); + let verdict = judge(&tmp); + assert_eq!(verdict.status, CheckStatus::Fail, "{}", verdict.message); + + for name in ["worst", "bad_complex", "bad_simple", "mild"] { + assert!( + verdict.message.contains(name), + "{name} must be listed: {}", + verdict.message + ); + } + let order: Vec = ["worst", "bad_complex", "bad_simple", "mild"] + .iter() + .map(|name| verdict.message.find(name).unwrap_or(usize::MAX)) + .collect(); + let mut sorted = order.clone(); + sorted.sort_unstable(); + assert_eq!( + order, sorted, + "worst grade first, then highest complexity: {}", + verdict.message + ); + } +} + +#[cfg_attr(coverage_nightly, coverage(off))] +#[cfg(test)] +mod tests_stale_and_absent_index { + use super::tests_tdg_grade::{judge, tdg_fixture, write_gates}; + use super::*; + use crate::models::comply_config::ComplyConfig; + + /// Make `tmp`'s index stale by planting a source file a clear minute newer + /// than `.pmat/context.db`. + /// + /// The mtime is SET, not raced for. Writing the source after the db and + /// trusting wall-clock ordering fails on filesystems whose timestamp + /// granularity is coarser than the microseconds between two writes — both + /// files land in the same tick, `is_index_stale` correctly sees nothing + /// newer, and the fixture silently never reaches the behaviour under test. + fn make_stale(tmp: &tempfile::TempDir) { + let db_path = tmp.path().join(".pmat").join("context.db"); + std::fs::create_dir_all(tmp.path().join("src")).expect("create src"); + let src = tmp.path().join("src/lib.rs"); + std::fs::write(&src, "pub fn f() {}\n").expect("write source"); + let db_mtime = std::fs::metadata(&db_path) + .and_then(|m| m.modified()) + .expect("db mtime"); + std::fs::File::options() + .write(true) + .open(&src) + .and_then(|f| f.set_modified(db_mtime + std::time::Duration::from_secs(60))) + .expect("set source mtime a clear minute past the db"); + assert!( + is_index_stale(tmp.path(), &db_path), + "fixture must be stale before the behaviour under test can be reached" + ); + } + + /// Plant a source file OLDER than the db, so the fixture is provably fresh + /// while still containing sources — the counter-fixture to `make_stale`. + fn make_fresh(tmp: &tempfile::TempDir) { + let db_path = tmp.path().join(".pmat").join("context.db"); + std::fs::create_dir_all(tmp.path().join("src")).expect("create src"); + let src = tmp.path().join("src/lib.rs"); + std::fs::write(&src, "pub fn f() {}\n").expect("write source"); + let db_mtime = std::fs::metadata(&db_path) + .and_then(|m| m.modified()) + .expect("db mtime"); + std::fs::File::options() + .write(true) + .open(&src) + .and_then(|f| f.set_modified(db_mtime - std::time::Duration::from_secs(60))) + .expect("set source mtime a clear minute before the db"); + assert!( + !is_index_stale(tmp.path(), &db_path), + "counter-fixture must be fresh" + ); + } + + /// #1045. A clean count taken from a stale index is not a clean tree: the + /// definitions added since the index was built are not in it AT ALL, so + /// "nothing below the floor" is a statement about yesterday. + /// + /// RED, with `demote_pass_when_stale` reduced to `check` (the identity it + /// replaced): + /// ```text + /// a clean count over a STALE index must not be a Pass, got Pass: + /// All non-test functions meet minimum grade A (index is stale: ...) + /// ``` + #[test] + fn a_stale_index_may_not_report_a_pass() { + let tmp = tdg_fixture(&[("src/lib.rs", "fine", "A", 3, 1)]); + make_stale(&tmp); + let verdict = judge(&tmp); + assert_ne!( + verdict.status, + CheckStatus::Pass, + "a clean count over a STALE index must not be a Pass, got Pass:\n {}", + verdict.message + ); + assert_eq!(verdict.status, CheckStatus::Warn, "{}", verdict.message); + assert!( + verdict + .message + .contains("NOT A VERDICT ON THE CURRENT TREE"), + "the reader must be told the verdict is not about their tree: {}", + verdict.message + ); + assert!( + verdict.message.contains("index is stale"), + "the reason must survive the demotion: {}", + verdict.message + ); + } + + /// The counter-test that bounds the demotion. Staleness is the trigger, not + /// the presence of sources or the mere act of reading an index: an index + /// NEWER than every source still reports a clean tree as clean. + /// + /// Without this, `demote_pass_when_stale` returning `Warn` unconditionally + /// would pass every other test in this module. + #[test] + fn a_fresh_index_still_reports_a_pass() { + let tmp = tdg_fixture(&[("src/lib.rs", "fine", "A", 3, 1)]); + make_fresh(&tmp); + let verdict = judge(&tmp); + assert_eq!( + verdict.status, + CheckStatus::Pass, + "a fresh index over a clean tree must still Pass: {}", + verdict.message + ); + assert!( + !verdict.message.contains("index is stale"), + "a fresh index must not be described as stale: {}", + verdict.message + ); + } + + /// Staleness makes a verdict LESS conclusive, never more lenient. A count + /// over the floor is still a Fail — those definitions exist somewhere in + /// history and the fix is to look, not to shrug. + #[test] + fn a_stale_index_does_not_soften_a_failure() { + let tmp = tdg_fixture(&[("src/lib.rs", "bad", "D", 30, 5)]); + make_stale(&tmp); + let verdict = judge(&tmp); + assert_eq!( + verdict.status, + CheckStatus::Fail, + "staleness must not downgrade a Fail: {}", + verdict.message + ); + } + + /// #1045's second half: the advice printed beside a stale index must name a + /// command that actually refreshes one. + /// + /// `pmat query "x"` does not. `load_or_build_index` + /// (`cli/handlers/query_handler/indexing.rs:251`) rebuilds only when BOTH + /// `.pmat/context.idx` and `.pmat/context.db` are absent; with an index + /// present it updates in memory and `maybe_save_incremental` persists that + /// only past 50 changed files or 5% of the index. Following the old advice + /// after editing a handful of files left the db byte-identical and the next + /// run printed the same "index is stale" with the same useless remedy. + /// + /// RED, restoring the old text: `the stale-index advice must name + /// --rebuild-index`. + #[test] + fn the_stale_advice_names_a_command_that_rebuilds() { + let tmp = tdg_fixture(&[("src/lib.rs", "fine", "A", 3, 1)]); + make_stale(&tmp); + let verdict = judge(&tmp); + assert!( + verdict.message.contains("--rebuild-index"), + "the stale-index advice must name --rebuild-index, because a plain \ + `pmat query` does not rewrite an index that already exists: {}", + verdict.message + ); + } + + /// #1008. `.pmat/` is gitignored and no CI leg builds an index, so an + /// absent index answered `Skip` — and `is_compliant` counts `Fail` only. + /// A project that RECORDED a ratchet baseline therefore had a gate that + /// could only fail on a machine which happened to have an index lying + /// around: unenforceable exactly where it decides a merge. + /// + /// RED, with `absent_index_verdict` returning the old unconditional `Skip`: + /// ```text + /// a recorded baseline that was never checked must not report success, + /// got Skip + /// ``` + #[test] + fn an_absent_index_fails_a_project_that_recorded_a_baseline() { + let tmp = tempfile::tempdir().expect("create tempdir"); + std::fs::write( + tmp.path().join(".pmat-gates.toml"), + "[tdg]\nbaseline = 1688\n", + ) + .expect("write gates toml"); + let verdict = check_tdg_grade_gate(tmp.path(), &ComplyConfig::default()); + assert_eq!( + verdict.status, + CheckStatus::Fail, + "a recorded baseline that was never checked must not report success, got {:?}: {}", + verdict.status, + verdict.message + ); + assert!( + verdict.message.contains("1688"), + "the unchecked baseline must be named: {}", + verdict.message + ); + assert!( + verdict.message.contains("Not measured"), + "the failure is 'unmeasured', not 'violated': {}", + verdict.message + ); + assert!( + !tmp.path().join(".pmat").exists(), + "#939: an audit must not build an index inside the tree it audits" + ); + } + + /// The counter-test that bounds #1008's fix, and it is the one that matters + /// most: pmat runs `comply check` against repositories that never opted + /// into this ratchet. A project with no `[tdg] baseline` — no + /// `.pmat-gates.toml` at all, or one without the key — must still be told + /// "not measured" and must NOT be failed for it. + /// + /// Without this, "make the absent index fail" is a one-line change that + /// reddens every fresh clone of every project pmat has ever been pointed at. + #[test] + fn an_absent_index_still_skips_a_project_with_no_baseline() { + let nothing = tempfile::tempdir().expect("create tempdir"); + let verdict = check_tdg_grade_gate(nothing.path(), &ComplyConfig::default()); + assert_eq!( + verdict.status, + CheckStatus::Skip, + "a project holding no baseline has nothing to fail: {}", + verdict.message + ); + + let no_key = tempfile::tempdir().expect("create tempdir"); + std::fs::write( + no_key.path().join(".pmat-gates.toml"), + "[tdg]\nexclude = [\"nothing/**\"]\n", + ) + .expect("write gates toml"); + let verdict = check_tdg_grade_gate(no_key.path(), &ComplyConfig::default()); + assert_eq!( + verdict.status, + CheckStatus::Skip, + "a [tdg] table without a baseline records no ratchet: {}", + verdict.message + ); + } + + /// An unreadable baseline is a Fail with or without an index. It was + /// already a Fail once the index was read; routing the absent-index case + /// through the same verdict keeps the two answers identical rather than + /// letting "no index" launder a broken config into a Skip. + #[test] + fn an_unreadable_baseline_fails_even_with_no_index() { + let tmp = tempfile::tempdir().expect("create tempdir"); + std::fs::write( + tmp.path().join(".pmat-gates.toml"), + "[tdg]\nbaseline = \"lots\"\n", + ) + .expect("write gates toml"); + let verdict = check_tdg_grade_gate(tmp.path(), &ComplyConfig::default()); + assert_eq!(verdict.status, CheckStatus::Fail, "{}", verdict.message); + assert!( + verdict.message.contains("not a count"), + "{}", + verdict.message + ); + } + + /// The demotion is a funnel, not a special case bolted onto one verdict: + /// the ratchet's own clean-at-baseline `Pass` goes through it too. + #[test] + fn the_ratchet_clean_pass_is_demoted_when_stale() { + let tmp = tdg_fixture(&[("src/lib.rs", "fine", "A", 3, 1)]); + write_gates(&tmp, "[tdg]\nbaseline = 0\n"); + make_stale(&tmp); + let verdict = judge(&tmp); + assert_eq!( + verdict.status, + CheckStatus::Warn, + "0-against-baseline-0 over a stale index is not a Pass: {}", + verdict.message + ); + } +} + +#[cfg_attr(coverage_nightly, coverage(off))] +#[cfg(test)] +mod tests_index_outside_the_audited_tree { + use super::*; + use std::process::Command; + + /// A tree with one definition no floor of A can admit, and one that passes, + /// so a verdict of "everything is fine" and a verdict of "nothing was read" + /// are distinguishable from each other. + /// + /// The grade is driven by complexity alone. It used to carry two + /// self-admitted-debt comments as well, until `.pmat-ratchet.toml`'s + /// `satd_markers_src_comments` went red at HEAD: that metric greps comment + /// lines across `src/*.rs`, and a raw string literal inside a test is still + /// a line in a file. A fixture is not a licence to move a baseline — nor is + /// a comment about one, which is why this paragraph does not spell the four + /// words out either. + const AWFUL: &str = r#" +pub fn fine(a: u32) -> u32 { a + 1 } + +pub fn awful(a: i32, b: i32, c: i32, d: i32) -> i32 { + let mut t = 0; + for i in 0..a { + if i % 2 == 0 { + for j in 0..b { + if j % 3 == 0 { + while t < c { + if t % 5 == 0 { t += 1; } else if t % 7 == 0 { t += 2; } else { t += 3; } + match t % 4 { + 0 => t += 1, + 1 => t += 2, + 2 => { if d > 0 { t += 3 } else { t -= 1 } } + _ => t += 4, + } + } + } else if j % 5 == 0 { t -= 1; } else if j % 7 == 0 { t += j; } else { t += 2; } + } + } else if i % 3 == 0 { + t += 2; + } else { + for k in 0..d { if k > 3 { t += k } else if k > 1 { t -= k } else { t += 1 } } + } + } + t +} +"#; + + pub(super) fn committed_cargo_project() -> tempfile::TempDir { + let d = git_project_with_code(); + std::fs::write( + d.path().join("Cargo.toml"), + "[package]\nname = \"fixture\"\nversion = \"0.1.0\"\nedition = \"2021\"\n", + ) + .expect("write manifest"); + git(d.path(), &["add", "-A"]); + git( + d.path(), + &[ + "-c", + "user.email=t@t", + "-c", + "user.name=t", + "commit", + "--no-verify", + "-qm", + "manifest", + ], + ); + d + } + + pub(super) fn porcelain_of(p: &Path) -> String { + porcelain(p) + } + + pub(super) fn build_index_in(p: &Path) { + build_in_project_index(p) + } + + fn project_with_code() -> tempfile::TempDir { + let dir = tempfile::tempdir().expect("create tempdir"); + std::fs::create_dir_all(dir.path().join("src")).expect("create src"); + std::fs::write(dir.path().join("src/lib.rs"), AWFUL).expect("write source"); + dir + } + + /// The index `pmat query` would leave in the project — built the same way, + /// so side B of the A/B is the real thing and not a hand-written fixture. + fn build_in_project_index(project: &Path) { + let index_path = project.join(".pmat").join("context.idx"); + std::fs::create_dir_all(project.join(".pmat")).expect("create .pmat"); + crate::services::agent_context::AgentContextIndex::build(project) + .expect("build index") + .save(&index_path) + .expect("save index"); + } + + /// The verdict without the provenance note, which is the only part of the + /// message that is *supposed* to differ between the two sides. + fn verdict_core(check: &ComplianceCheck) -> String { + check + .message + .split(" [measured against") + .next() + .unwrap_or_default() + .to_string() + } + + /// `$PMAT_CACHE_DIR`, restored on drop so a failing assertion cannot leak + /// it into the rest of the suite. + pub(super) struct CacheDirGuard(Option); + + /// `$PMAT_CACHE_DIR` for the duration of one test, for the sibling module + /// that runs the whole compliance report rather than one check. + pub(super) fn cache_dir_guard(dir: &Path) -> CacheDirGuard { + CacheDirGuard::pointing_at(dir) + } + + impl CacheDirGuard { + fn pointing_at(dir: &Path) -> Self { + let previous = std::env::var_os(crate::utils::pmat_cache_dir::CACHE_DIR_ENV); + std::env::set_var(crate::utils::pmat_cache_dir::CACHE_DIR_ENV, dir); + Self(previous) + } + } + + impl Drop for CacheDirGuard { + fn drop(&mut self) { + match self.0.take() { + Some(v) => std::env::set_var(crate::utils::pmat_cache_dir::CACHE_DIR_ENV, v), + None => std::env::remove_var(crate::utils::pmat_cache_dir::CACHE_DIR_ENV), + } + } + } + + fn git(dir: &Path, args: &[&str]) -> std::process::Output { + Command::new("git") + .current_dir(dir) + .args(args) + .output() + .expect("git must be runnable") + } + + fn porcelain(dir: &Path) -> String { + String::from_utf8_lossy(&git(dir, &["status", "--porcelain"]).stdout).into_owned() + } + + /// `--template=` keeps a developer's global hook template out of the + /// fixture; `--no-verify` on the commit keeps a global `core.hooksPath` + /// from running this repository's own gates inside a two-file tempdir. + fn git_project_with_code() -> tempfile::TempDir { + let dir = project_with_code(); + assert!( + git(dir.path(), &["init", "-q", "--template=", "."]) + .status + .success(), + "git init failed" + ); + git(dir.path(), &["add", "-A"]); + assert!( + git( + dir.path(), + &[ + "-c", + "user.email=t@t", + "-c", + "user.name=t", + "commit", + "--no-verify", + "-qm", + "fixture", + ], + ) + .status + .success(), + "commit failed" + ); + dir + } + + /// #1008's own A/B, at the check that showed it: one tree, one commit, the + /// index the only variable. The two sides must return the SAME verdict. + /// + /// Reported on `paiml/rmedia` at `ffe4f68` as 2 failing checks without an + /// index against 3 with one — a gate that structurally could not fail on a + /// fresh CI checkout, which is the only checkout whose verdict decides a + /// merge. + /// + /// RED, against the code this replaces: + /// ```text + /// thread 'a_tree_gets_the_same_verdict_with_or_without_an_index_of_its_own' + /// panicked at check_tdg_grade.rs: + /// a project with no index of its own was not measured at all (Skip) - #1008: + /// a gate that can only run where an index happens to exist cannot run in CI + /// ``` + #[test] + #[serial_test::serial] + fn a_tree_gets_the_same_verdict_with_or_without_an_index_of_its_own() { + let cache = tempfile::tempdir().expect("create cache dir"); + let _env = CacheDirGuard::pointing_at(cache.path()); + let project = project_with_code(); + let config = ComplyConfig::default(); + + // A: no `.pmat/` anywhere in the tree, exactly as a fresh checkout. + let without = check_tdg_grade_gate(project.path(), &config); + assert!( + !project.path().join(".pmat").exists(), + "#939: the audited tree must not be written to, index or no index" + ); + assert_ne!( + without.status, + CheckStatus::Skip, + "a project with no index of its own was not measured at all (Skip) - #1008: \ + a gate that can only run where an index happens to exist cannot run in CI. \ + Message was: {}", + without.message + ); + + // B: the same tree, with the index `pmat query` would have built. + build_in_project_index(project.path()); + let with = check_tdg_grade_gate(project.path(), &config); + + assert_eq!( + without.status, with.status, + "the same tree got two different verdicts:\n without: {}\n with: {}", + without.message, with.message + ); + assert_eq!( + verdict_core(&without), + verdict_core(&with), + "the same tree was measured differently depending on where its index lived" + ); + assert!( + with.message.split(" [measured against").count() == 1, + "a project's own index is read in place and says nothing about a cache: {}", + with.message + ); + assert!( + without.message.contains("[measured against"), + "a verdict from outside the tree must name the file it read: {}", + without.message + ); + } + + /// The fixture is load-bearing: if it graded clean, the A/B above would be + /// comparing two Passes and could not tell a measurement from a shrug. + #[test] + #[serial_test::serial] + fn the_out_of_tree_measurement_can_actually_fail() { + let cache = tempfile::tempdir().expect("create cache dir"); + let project = project_with_code(); + + let verdict = check_tdg_grade_gate_with_index( + project.path(), + &ComplyConfig::default(), + &cache.path().join("context.idx"), + ); + + assert_eq!( + verdict.status, + CheckStatus::Fail, + "a definition below the floor must fail from the out-of-tree index too: {}", + verdict.message + ); + assert!( + verdict.message.contains("awful"), + "the offender must be named: {}", + verdict.message + ); + } + + /// THE counter-test. Building an index must not manufacture a verdict out + /// of a tree pmat never read a line of. + /// + /// Zero definitions violate no floor, so the naive version of this fix + /// answers "no functions below minimum grade A" — a clean bill of health + /// for a directory with no code in it, which is worse than the skip it + /// replaced. An empty measurement is not a measurement. + #[test] + fn a_tree_with_no_definitions_is_not_measured_into_a_pass() { + let cache = tempfile::tempdir().expect("create cache dir"); + let project = tempfile::tempdir().expect("create tempdir"); + std::fs::write(project.path().join("README.md"), "# no code here\n").expect("write"); + + let verdict = check_tdg_grade_gate_with_index( + project.path(), + &ComplyConfig::default(), + &cache.path().join("context.idx"), + ); + + assert_eq!( + verdict.status, + CheckStatus::Skip, + "a tree with nothing to grade must not report a pass: {}", + verdict.message + ); + assert!( + verdict.message.contains("no definitions"), + "the reason must say what was missing: {}", + verdict.message + ); + assert!( + !cache.path().join("context.db").exists(), + "an empty index must not be cached as if it were a measurement" + ); + + // And the same nothing, with a ratchet recorded against it, fails + // rather than passes: an unmeasured baseline has not held. + std::fs::write( + project.path().join(".pmat-gates.toml"), + "[tdg]\nbaseline = 7\n", + ) + .expect("write gates toml"); + let recorded = check_tdg_grade_gate_with_index( + project.path(), + &ComplyConfig::default(), + &cache.path().join("context.idx"), + ); + assert_eq!(recorded.status, CheckStatus::Fail, "{}", recorded.message); + assert!( + recorded.message.contains("Not measured"), + "{}", + recorded.message + ); + } + + /// The property the refusal to build was protecting, asked of git rather + /// than of a filename: after CB-200 has measured a repository, that + /// repository's `git status` is empty. + #[test] + #[serial_test::serial] + fn measuring_a_repository_leaves_its_git_status_clean() { + let cache = tempfile::tempdir().expect("create cache dir"); + let _env = CacheDirGuard::pointing_at(cache.path()); + let project = git_project_with_code(); + assert_eq!(porcelain(project.path()), "", "fixture must start clean"); + + let verdict = check_tdg_grade_gate(project.path(), &ComplyConfig::default()); + + assert_ne!( + verdict.status, + CheckStatus::Skip, + "the point is that it measured: {}", + verdict.message + ); + assert_eq!( + porcelain(project.path()), + "", + "CB-200 dirtied the repository it audited" + ); + assert!( + !project.path().join(".pmat").exists(), + "not even an ignored .pmat/ - the tree is not pmat's to write to" + ); + } + + /// The other counter-test: a project that VERSIONS its own `.pmat/` must + /// keep it. The out-of-tree index is a fallback for a tree that has none, + /// never a replacement for one that does — nothing here may ignore, move, + /// clobber or shadow a committed index. + #[test] + #[serial_test::serial] + fn a_committed_index_is_read_in_place_and_left_alone() { + let cache = tempfile::tempdir().expect("create cache dir"); + let _env = CacheDirGuard::pointing_at(cache.path()); + let project = git_project_with_code(); + build_in_project_index(project.path()); + git(project.path(), &["add", "-f", ".pmat/context.db"]); + assert!( + git( + project.path(), + &[ + "-c", + "user.email=t@t", + "-c", + "user.name=t", + "commit", + "--no-verify", + "-qm", + "we version our index", + ], + ) + .status + .success(), + "commit failed" + ); + let before = std::fs::metadata(project.path().join(".pmat/context.db")) + .and_then(|m| m.modified()) + .expect("db mtime"); + + let verdict = check_tdg_grade_gate(project.path(), &ComplyConfig::default()); + + assert!( + git( + project.path(), + &["ls-files", "--error-unmatch", ".pmat/context.db"], + ) + .status + .success(), + "the committed index was dropped from the index" + ); + let after = std::fs::metadata(project.path().join(".pmat/context.db")) + .and_then(|m| m.modified()) + .expect("db mtime"); + assert_eq!(before, after, "the committed index was rewritten"); + assert!( + !verdict.message.contains("[measured against"), + "a committed index must be READ, not shadowed by a cached copy: {}", + verdict.message + ); + assert!( + !crate::utils::pmat_cache_dir::comply_index_path(project.path()) + .with_extension("db") + .exists(), + "no out-of-tree index may be built for a project that has one" + ); + // Asked of THIS project's cache entry and not of the cache root: the + // root is shared, so `/comply` merely existing says only that + // some other project was measured, which is not this test's business + // and made it fail under a full parallel run. + } + + /// The cache is a cache: a second run reads what the first one built + /// instead of paying for the walk again. + #[test] + fn a_built_index_is_reused_by_the_next_run() { + let cache = tempfile::tempdir().expect("create cache dir"); + let index_path = cache.path().join("context.idx"); + let project = project_with_code(); + let config = ComplyConfig::default(); + + let first = check_tdg_grade_gate_with_index(project.path(), &config, &index_path); + let built = std::fs::metadata(index_path.with_extension("db")) + .and_then(|m| m.modified()) + .expect("db mtime"); + + let second = check_tdg_grade_gate_with_index(project.path(), &config, &index_path); + let after = std::fs::metadata(index_path.with_extension("db")) + .and_then(|m| m.modified()) + .expect("db mtime"); + + assert_eq!( + built, after, + "the second run rebuilt an index it could reuse" + ); + assert_eq!(first.status, second.status); + assert_eq!(first.message, second.message); + } + + /// …and it is a cache of THIS tree. Edit the sources and the next run + /// measures the edit, or the gate would hold a baseline against a tree that + /// no longer exists — which is how `.pmat-gates.toml`'s own baseline was + /// first banked 216 units too high. + #[test] + fn an_edit_after_the_build_is_measured_not_ignored() { + let cache = tempfile::tempdir().expect("create cache dir"); + let index_path = cache.path().join("context.idx"); + let project = project_with_code(); + let config = ComplyConfig::default(); + + let before = check_tdg_grade_gate_with_index(project.path(), &config, &index_path); + assert_eq!(before.status, CheckStatus::Fail, "{}", before.message); + + // The offending definition is deleted, and the CACHED INDEX is aged a + // clear minute rather than the edit being raced against it: filesystem + // timestamp granularity is coarser than the microseconds between two + // writes, which is the kind of thing that passes locally and fails in + // CI. Ageing the index rather than post-dating the source also keeps + // the fixture honest — a source file with an mtime in the future is + // newer than any index that could ever be built from it. + let src = project.path().join("src/lib.rs"); + std::fs::write(&src, "pub fn fine(a: u32) -> u32 { a + 1 }\n").expect("write source"); + let db = index_path.with_extension("db"); + let db_mtime = std::fs::metadata(&db) + .and_then(|m| m.modified()) + .expect("db mtime"); + std::fs::File::options() + .write(true) + .open(&db) + .and_then(|f| f.set_modified(db_mtime - std::time::Duration::from_secs(60))) + .expect("age the cached index a clear minute"); + assert!( + is_index_stale(project.path(), &db), + "fixture must be stale before the run under test" + ); + + let after = check_tdg_grade_gate_with_index(project.path(), &config, &index_path); + assert_eq!( + after.status, + CheckStatus::Pass, + "the fix was not seen: a stale cached index was reported as the current tree: {}", + after.message + ); + } +} + +#[cfg_attr(coverage_nightly, coverage(off))] +#[cfg(test)] +mod tests_the_reported_ab { + use super::super::check::compute_compliance_report; + use super::super::types::{CheckStatus, ComplianceReport}; + use super::tests_index_outside_the_audited_tree as fixture; + + fn names(report: &ComplianceReport, want: CheckStatus) -> Vec { + let mut found: Vec = report + .checks + .iter() + .filter(|c| c.status == want) + .map(|c| c.name.clone()) + .collect(); + found.sort(); + found + } + + /// #1008 as it was reported: the WHOLE compliance report, one tree, one + /// commit, the index the only variable. The set of FAILING checks must be + /// the same on both sides. + /// + /// The report on `paiml/rmedia` at `ffe4f68` was 2 failing checks without + /// an index and 3 with — CB-200 the difference. The exit code happened to + /// be 1 either way there, because other checks were failing; had CB-200 + /// been the only violation, the same commit would have exited 0 in CI and 1 + /// on a developer's machine. + /// + /// Measured here, on this fixture, RED against the code this replaces: + /// ```text + /// A fail=3 B fail=4 only_with_index: ["CB-200: TDG Grade Gate"] + /// ``` + /// and after: + /// ```text + /// A fail=4 B fail=4 only_with_index: [] + /// ``` + /// + /// It asserts the SET and not the count, and it names the differing checks + /// when it fails, because the interesting failure is a new check that can + /// only be measured where somebody has run `pmat query` — this defect + /// arriving again somewhere else. + #[test] + #[serial_test::serial] + fn the_failing_checks_do_not_depend_on_whether_an_index_exists() { + let cache = tempfile::tempdir().expect("create cache dir"); + let _env = fixture::cache_dir_guard(cache.path()); + let project = fixture::committed_cargo_project(); + + let without = compute_compliance_report(project.path()).expect("report without an index"); + let without_failing = names(&without, CheckStatus::Fail); + assert_eq!( + fixture::porcelain_of(project.path()), + "", + "auditing the project dirtied it" + ); + + fixture::build_index_in(project.path()); + let with = compute_compliance_report(project.path()).expect("report with an index"); + let with_failing = names(&with, CheckStatus::Fail); + + let only_with: Vec<&String> = with_failing + .iter() + .filter(|n| !without_failing.contains(n)) + .collect(); + let only_without: Vec<&String> = without_failing + .iter() + .filter(|n| !with_failing.contains(n)) + .collect(); + assert!( + only_with.is_empty() && only_without.is_empty(), + "the same tree failed a different set of checks depending on whether an index \ + happened to exist.\n fails only WITH an index: {only_with:?}\n \ + fails only WITHOUT an index: {only_without:?}" + ); + assert!( + without_failing.iter().any(|n| n.starts_with("CB-200")), + "the fixture must actually violate CB-200, or this proves nothing. Failing: \ + {without_failing:?}" + ); + } +} diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/cli/handlers/comply_handlers/muda_handlers.rs.txt b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/cli/handlers/comply_handlers/muda_handlers.rs.txt new file mode 100644 index 0000000000..5736f169a9 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/cli/handlers/comply_handlers/muda_handlers.rs.txt @@ -0,0 +1,160 @@ +//! CB-300: Muda (Seven Wastes) Score for Code Quality +//! +//! Maps Toyota Production System's Seven Wastes to code quality metrics: +//! +//! | Toyota Waste | Code Equivalent | Detection | +//! |------------------|---------------------------|------------------------------| +//! | Overproduction | Dead Code | `dead_code` analysis (CB-128)| +//! | Waiting | Slow Tests / Builds | Test time (CB-126/CB-127) | +//! | Inventory | Stale SATD markers | SATD age > 90 days | +//! | Transport | Excessive cloning | `.clone()` in hot paths | +//! | Over-processing | High complexity | Cyclomatic > 15 | +//! | Motion | Dependency sprawl | Dep count / graph depth | +//! | Defects | Bugs / Test failures | Panic count / stub count | +//! +//! Score: 0-100 (lower is better, 0 = zero waste) + +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::path::Path; + +/// Muda Waste Report aggregating all seven wastes +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MudaReport { + /// Overproduction: dead code percentage (0-100) + pub overproduction: f64, + /// Waiting: slow test/build score (0-100) + pub waiting: f64, + /// Inventory: stale SATD/branch score (0-100) + pub inventory: f64, + /// Transport: excessive copying score (0-100) + pub transport: f64, + /// Over-processing: complexity waste (0-100) + pub over_processing: f64, + /// Motion: dependency sprawl (0-100) + pub motion: f64, + /// Defects: bug/panic indicators (0-100) + pub defects: f64, + /// Total aggregate score (0-100, lower is better) + pub total_score: f64, + /// Grade based on total score + pub grade: MudaGrade, + /// Maps each Muda category to its top contributing files (up to 5 per category) + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub file_details: HashMap>, +} + +/// Muda grade classification +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub enum MudaGrade { + /// 0-20: Lean (minimal waste) + Lean, + /// 21-40: Efficient + Efficient, + /// 41-60: Moderate waste + Moderate, + /// 61-80: High waste + High, + /// 81-100: Critical waste + Critical, +} + +impl std::fmt::Display for MudaGrade { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + MudaGrade::Lean => write!(f, "Lean"), + MudaGrade::Efficient => write!(f, "Efficient"), + MudaGrade::Moderate => write!(f, "Moderate"), + MudaGrade::High => write!(f, "High"), + MudaGrade::Critical => write!(f, "Critical"), + } + } +} + +impl MudaGrade { + fn from_score(score: f64) -> Self { + match score as u32 { + 0..=20 => MudaGrade::Lean, + 21..=40 => MudaGrade::Efficient, + 41..=60 => MudaGrade::Moderate, + 61..=80 => MudaGrade::High, + _ => MudaGrade::Critical, + } + } +} + +/// Calculate the Muda Waste Score for a project. +/// +/// Weights: Defects (25%), Inventory (20%), Over-processing (15%), +/// Overproduction (15%), Waiting (15%), Motion (5%), Transport (5%) +/// +/// Inventory (SATD) elevated to 20% — stale TODO/FIXME/HACK accumulation +/// is a primary signal of unmaintained code and must not be masked. +#[provable_contracts_macros::contract("pmat-core.yaml", equation = "path_exists")] +pub fn calculate_muda_score(project_path: &Path) -> MudaReport { + let overproduction = measure_overproduction(project_path); + let waiting = measure_waiting(project_path); + let inventory = measure_inventory(project_path); + let transport = measure_transport(project_path); + let over_processing = measure_over_processing(project_path); + let motion = measure_motion(project_path); + let defects = measure_defects(project_path); + + // Collect file details for categories with concrete file mappings + let mut file_details = HashMap::new(); + let overproduction_files = collect_overproduction_files(project_path); + if !overproduction_files.is_empty() { + file_details.insert("Overproduction".to_string(), overproduction_files); + } + let inventory_files = collect_inventory_files(project_path); + if !inventory_files.is_empty() { + file_details.insert("Inventory".to_string(), inventory_files); + } + let over_processing_files = collect_over_processing_files(project_path); + if !over_processing_files.is_empty() { + file_details.insert("Over-processing".to_string(), over_processing_files); + } + let defect_files = collect_defect_files(project_path); + if !defect_files.is_empty() { + file_details.insert("Defects".to_string(), defect_files); + } + + // Weighted average (weights sum to 1.0) + // Inventory elevated: stale SATD is a primary waste signal + let total_score = (defects * 0.25) + + (inventory * 0.20) + + (over_processing * 0.15) + + (overproduction * 0.15) + + (waiting * 0.15) + + (motion * 0.05) + + (transport * 0.05); + + let total_score = total_score.clamp(0.0, 100.0); + let grade = MudaGrade::from_score(total_score); + + MudaReport { + overproduction, + waiting, + inventory, + transport, + over_processing, + motion, + defects, + total_score, + grade, + file_details, + } +} + +// --- Include split submodules --- + +// SATD measurement: overproduction, waiting, inventory, SATD counting helpers +include!("muda_handlers_measurement.rs"); + +// Project metrics: transport, over-processing, motion, defects +include!("muda_handlers_metrics.rs"); + +// Unit tests +include!("muda_handlers_tests.rs"); +// #[requires(project_path.exists())] +// #[ensures(result.is_ok())] diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/cli/handlers/work_contract_scoring.rs.txt b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/cli/handlers/work_contract_scoring.rs.txt new file mode 100644 index 0000000000..be523ec6f8 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/cli/handlers/work_contract_scoring.rs.txt @@ -0,0 +1,468 @@ +// Contract Scoring: 5-dimension quality scoring for pmat work contracts +// Spec: docs/specifications/dbc.md §13.4-13.5 +// +// Adapted from provable-contracts scoring (spec_depth, falsification, kani, lean, binding) +// to pmat work contract dimensions (spec_depth, falsification, invariant_health, +// subcontracting, traceability). + +/// 5-dimension contract quality score +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ContractScore { + /// How comprehensive are the contract clauses? (weight: 0.20) + pub spec_depth: f64, + /// What fraction of claims have been verified? (weight: 0.25) + pub falsification_coverage: f64, + /// Invariant pass rate across checkpoints (weight: 0.25) + pub invariant_health: f64, + /// Monotonic postcondition strengthening score (weight: 0.10) + pub subcontracting: f64, + /// Coverage of require/ensure/invariant triad (weight: 0.20) + pub traceability: f64, + /// Weighted total score (0.0 to 1.0) + pub total: f64, + /// Letter grade (A-F) + pub grade: ScoreGrade, +} + +/// Scoring weights (sum = 1.0) +#[derive(Debug, Clone)] +pub struct ScoringWeights { + pub spec_depth: f64, + pub falsification: f64, + pub invariant_health: f64, + pub subcontracting: f64, + pub traceability: f64, +} + +impl Default for ScoringWeights { + fn default() -> Self { + Self { + spec_depth: 0.20, + falsification: 0.25, + invariant_health: 0.25, + subcontracting: 0.10, + traceability: 0.20, + } + } +} + +/// Letter grades for contract scores +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum ScoreGrade { + A, + B, + C, + D, + F, +} + +impl ScoreGrade { + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "score_range")] + /// From score. + pub fn from_score(score: f64) -> Self { + if score >= 0.90 { + ScoreGrade::A + } else if score >= 0.75 { + ScoreGrade::B + } else if score >= 0.60 { + ScoreGrade::C + } else if score >= 0.40 { + ScoreGrade::D + } else { + ScoreGrade::F + } + } +} + +impl std::fmt::Display for ScoreGrade { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ScoreGrade::A => write!(f, "A"), + ScoreGrade::B => write!(f, "B"), + ScoreGrade::C => write!(f, "C"), + ScoreGrade::D => write!(f, "D"), + ScoreGrade::F => write!(f, "F"), + } + } +} + +/// Compute the 5-dimension contract score for a work contract. +/// +/// Each dimension is scored 0.0..1.0, then weighted and summed. +#[provable_contracts_macros::contract("pmat-core.yaml", equation = "path_exists")] +pub fn score_contract(contract: &WorkContract, project_path: &Path) -> ContractScore { + debug_assert!(project_path.exists(), "project_path must exist: {}", project_path.display()); + let weights = ScoringWeights::default(); + + let spec_depth = compute_spec_depth(contract); + let falsification_coverage = compute_falsification_coverage(contract); + let invariant_health = compute_invariant_health(contract, project_path); + let subcontracting = compute_subcontracting_score(contract); + let traceability = compute_traceability(contract); + + let total = weights.spec_depth * spec_depth + + weights.falsification * falsification_coverage + + weights.invariant_health * invariant_health + + weights.subcontracting * subcontracting + + weights.traceability * traceability; + + // All dimensions must be in [0.0, 1.0] + debug_assert!((0.0..=1.0).contains(&spec_depth), "spec_depth out of range: {}", spec_depth); + debug_assert!((0.0..=1.0).contains(&falsification_coverage), "falsification out of range: {}", falsification_coverage); + debug_assert!((0.0..=1.0).contains(&invariant_health), "invariant_health out of range: {}", invariant_health); + debug_assert!((0.0..=1.0).contains(&subcontracting), "subcontracting out of range: {}", subcontracting); + debug_assert!((0.0..=1.0).contains(&traceability), "traceability out of range: {}", traceability); + debug_assert!((0.0..=1.0).contains(&total), "total score out of range: {}", total); + + let grade = ScoreGrade::from_score(total); + + ContractScore { + spec_depth, + falsification_coverage, + invariant_health, + subcontracting, + traceability, + total, + grade, + } +} + +/// Spec depth: ratio of defined clauses to expected minimum. +/// +/// A well-specified contract has at least 3 require, 5 ensure, 3 invariant clauses. +fn compute_spec_depth(contract: &WorkContract) -> f64 { + if !contract.is_dbc() { + // v4.0 flat contracts: score based on claim count vs expected 22 + let ratio = contract.claims.len() as f64 / 22.0; + return ratio.min(1.0); + } + + let require_score = (contract.require.len() as f64 / 3.0).min(1.0); + let ensure_score = (contract.ensure.len() as f64 / 5.0).min(1.0); + let invariant_score = (contract.invariant.len() as f64 / 3.0).min(1.0); + + (require_score + ensure_score + invariant_score) / 3.0 +} + +/// Falsification coverage: fraction of claims with verification results. +fn compute_falsification_coverage(contract: &WorkContract) -> f64 { + if contract.claims.is_empty() { + return 0.0; + } + + let verified = contract + .claims + .iter() + .filter(|c| c.result.is_some()) + .count(); + verified as f64 / contract.claims.len() as f64 +} + +/// Invariant health: pass rate across checkpoint history. +/// +/// Loads all checkpoints and computes the fraction of invariants that passed. +fn compute_invariant_health(contract: &WorkContract, project_path: &Path) -> f64 { + debug_assert!(project_path.exists(), "project_path must exist: {}", project_path.display()); + let checkpoints = CheckpointRecord::load_all(project_path, &contract.work_item_id); + if checkpoints.is_empty() { + // No checkpoints yet — neutral score (not penalized) + return if contract.invariant.is_empty() { + 1.0 + } else { + 0.5 + }; + } + + let total_invariants: usize = checkpoints + .iter() + .map(|cp| cp.invariant_results.len()) + .sum(); + if total_invariants == 0 { + return 1.0; + } + + let passed: usize = checkpoints + .iter() + .flat_map(|cp| &cp.invariant_results) + .filter(|r| r.passed) + .count(); + + passed as f64 / total_invariants as f64 +} + +/// Subcontracting score: 1.0 if postconditions are monotonically non-weakening. +fn compute_subcontracting_score(contract: &WorkContract) -> f64 { + if contract.iteration <= 1 || contract.inherited_postconditions.is_empty() { + return 1.0; // First iteration or no inherited — full score + } + + match validate_subcontracting(&contract.inherited_postconditions, &contract.ensure) { + Ok(()) => 1.0, + Err(_) => 0.0, + } +} + +/// Traceability: coverage of the require/ensure/invariant triad. +/// +/// Full traceability = all three triad legs are non-empty. +fn compute_traceability(contract: &WorkContract) -> f64 { + if !contract.is_dbc() { + // v4.0: traceability is binary — claims exist or not + return if contract.claims.is_empty() { 0.0 } else { 0.8 }; + } + + let mut score: f64 = 0.0; + if !contract.require.is_empty() { + score += 1.0 / 3.0; + } + if !contract.ensure.is_empty() { + score += 1.0 / 3.0; + } + if !contract.invariant.is_empty() { + score += 1.0 / 3.0; + } + + // Bonus for exclusion transparency (up to 1.0 total) + if !contract.excluded_claims.is_empty() && score > 0.0 { + score = (score + 0.05).min(1.0); + } + + score +} + +// === Drift Detection (DBC spec §13.5, §14.3) === + +/// Drift metrics for a contract — measures staleness and divergence. +/// +/// Based on ABC drift bounds theorem (arXiv:2602.22302): +/// D* = alpha / gamma, where alpha = drift rate, gamma = recovery rate. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DriftMetrics { + /// Hours since last checkpoint + pub hours_since_checkpoint: f64, + /// Hours since contract creation + pub hours_since_creation: f64, + /// Estimated drift rate alpha (0.0..1.0) + pub drift_rate: f64, + /// Recovery rate gamma from checkpoint frequency + pub recovery_rate: f64, + /// Bounded drift D* = alpha / gamma (lower is better) + pub bounded_drift: f64, + /// Whether the contract is considered stale + pub is_stale: bool, +} + +/// Compute drift metrics for a contract. +/// +/// Staleness threshold: 24 hours without a checkpoint. +#[provable_contracts_macros::contract("pmat-core.yaml", equation = "path_exists")] +pub fn compute_drift_metrics(contract: &WorkContract, project_path: &Path) -> DriftMetrics { + debug_assert!(project_path.exists(), "project_path must exist: {}", project_path.display()); + let now = chrono::Utc::now(); + let hours_since_creation = (now - contract.created_at).num_minutes() as f64 / 60.0; + + let checkpoints = CheckpointRecord::load_all(project_path, &contract.work_item_id); + let hours_since_checkpoint = if let Some(last) = checkpoints.last() { + (now - last.timestamp).num_minutes() as f64 / 60.0 + } else { + hours_since_creation + }; + + // Drift rate: increases with time since last checkpoint + // alpha = min(1.0, hours_since_checkpoint / 48.0) + let drift_rate = (hours_since_checkpoint / 48.0).min(1.0); + + // Recovery rate: based on checkpoint frequency + // gamma = checkpoints_per_24h (capped at 1.0) + let recovery_rate = if hours_since_creation > 0.0 { + let cps_per_day = (checkpoints.len() as f64 / hours_since_creation) * 24.0; + cps_per_day.min(1.0).max(0.01) // floor at 0.01 to avoid div-by-zero + } else { + 0.01 + }; + + // ABC theorem: D* = alpha / gamma + let bounded_drift = (drift_rate / recovery_rate).min(1.0); + let is_stale = hours_since_checkpoint > 24.0; + + DriftMetrics { + hours_since_checkpoint, + hours_since_creation, + drift_rate, + recovery_rate, + bounded_drift, + is_stale, + } +} + +// === Trend Tracking (DBC spec §13.6) === + +/// Point-in-time quality snapshot for trend tracking. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct QualityTrendSnapshot { + /// Timestamp of the snapshot + pub timestamp: chrono::DateTime, + /// Contract score at this point + pub score: f64, + /// Grade at this point + pub grade: ScoreGrade, + /// Number of active claims + pub active_claims: usize, + /// Number of verified claims + pub verified_claims: usize, + /// Drift bound at this point + pub bounded_drift: f64, + /// Git SHA at snapshot time + pub git_sha: String, +} + +/// Trend analysis result with drift detection. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct QualityTrend { + /// All snapshots, sorted chronologically + pub snapshots: Vec, + /// Rolling average score (7-snapshot window) + pub rolling_average: f64, + /// Score delta from rolling average + pub delta_from_average: f64, + /// Whether drift is detected (>5% drop from rolling average) + pub drift_detected: bool, + /// Trend direction + pub direction: TrendDirection, +} + +/// Trend direction indicator +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum TrendDirection { + Improving, + Stable, + Declining, +} + +impl std::fmt::Display for TrendDirection { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TrendDirection::Improving => write!(f, "improving"), + TrendDirection::Stable => write!(f, "stable"), + TrendDirection::Declining => write!(f, "declining"), + } + } +} + +/// Record a quality trend snapshot to .pmat-work/{item-id}/trend/ +#[provable_contracts_macros::contract("pmat-core.yaml", equation = "path_exists")] +pub fn record_trend_snapshot( + contract: &WorkContract, + score: &ContractScore, + drift: &DriftMetrics, + git_sha: &str, + project_path: &Path, +) -> Result { + debug_assert!(project_path.exists(), "project_path must exist: {}", project_path.display()); + let verified = contract + .claims + .iter() + .filter(|c| c.result.is_some()) + .count(); + + let snapshot = QualityTrendSnapshot { + timestamp: chrono::Utc::now(), + score: score.total, + grade: score.grade, + active_claims: contract.claims.len(), + verified_claims: verified, + bounded_drift: drift.bounded_drift, + git_sha: git_sha.to_string(), + }; + + let trend_dir = project_path + .join(".pmat-work") + .join(&contract.work_item_id) + .join("trend"); + std::fs::create_dir_all(&trend_dir)?; + + let filename = format!( + "snapshot-{}.json", + snapshot.timestamp.format("%Y%m%dT%H%M%S") + ); + let path = trend_dir.join(filename); + let json = serde_json::to_string_pretty(&snapshot)?; + std::fs::write(&path, json)?; + + Ok(path) +} + +/// Load and analyze quality trend for a work item. +/// +/// Uses a 7-snapshot rolling window. Drift is detected when the current +/// score drops more than 5% below the rolling average. +#[provable_contracts_macros::contract("pmat-core.yaml", equation = "path_exists")] +pub fn load_quality_trend(project_path: &Path, work_item_id: &str) -> QualityTrend { + debug_assert!(project_path.exists(), "project_path must exist: {}", project_path.display()); + let trend_dir = project_path + .join(".pmat-work") + .join(work_item_id) + .join("trend"); + + let mut snapshots = Vec::new(); + if let Ok(entries) = std::fs::read_dir(&trend_dir) { + for entry in entries.flatten() { + if let Ok(content) = std::fs::read_to_string(entry.path()) { + if let Ok(snap) = serde_json::from_str::(&content) { + snapshots.push(snap); + } + } + } + } + snapshots.sort_by_key(|s| s.timestamp); + + analyze_trend(snapshots) +} + +/// Analyze a list of snapshots into a QualityTrend. +fn analyze_trend(snapshots: Vec) -> QualityTrend { + if snapshots.is_empty() { + return QualityTrend { + snapshots, + rolling_average: 0.0, + delta_from_average: 0.0, + drift_detected: false, + direction: TrendDirection::Stable, + }; + } + + // Rolling average over last 7 snapshots + let window_size = 7.min(snapshots.len()); + let window_start = snapshots.len() - window_size; + let rolling_sum: f64 = snapshots[window_start..].iter().map(|s| s.score).sum(); + let rolling_average = rolling_sum / window_size as f64; + + let current_score = snapshots.last().map(|s| s.score).unwrap_or(0.0); + let delta_from_average = current_score - rolling_average; + + // Drift: >5% drop from rolling average + let drift_detected = delta_from_average < -0.05; + + // Direction based on last 2 snapshots + let direction = if snapshots.len() < 2 { + TrendDirection::Stable + } else { + let prev = snapshots[snapshots.len() - 2].score; + let diff = current_score - prev; + if diff > 0.02 { + TrendDirection::Improving + } else if diff < -0.02 { + TrendDirection::Declining + } else { + TrendDirection::Stable + } + }; + + QualityTrend { + snapshots, + rolling_average, + delta_from_average, + drift_detected, + direction, + } +} diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/services/file_health_types.rs.txt b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/services/file_health_types.rs.txt new file mode 100644 index 0000000000..ea6a340e34 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/services/file_health_types.rs.txt @@ -0,0 +1,240 @@ +/// Thresholds for file size classification +pub mod thresholds { + pub const IDEAL_MAX: usize = 200; + pub const ACCEPTABLE_MAX: usize = 500; + pub const WARNING_MAX: usize = 1000; + pub const PROBLEM_MAX: usize = 2000; + // >2000 is CRITICAL +} + +/// File size classification based on line count +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum FileSizeClass { + /// 0-200 lines: Optimal cognitive chunk + Ideal, + /// 201-500 lines: Within SRP tolerance + Acceptable, + /// 501-1000 lines: Approaching limit + Warning, + /// 1001-2000 lines: Exceeds cognitive capacity + Problem, + /// >2000 lines: Untestable monolith + Critical, +} + +impl FileSizeClass { + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + /// From lines. + pub fn from_lines(lines: usize) -> Self { + match lines { + 0..=200 => Self::Ideal, + 201..=500 => Self::Acceptable, + 501..=1000 => Self::Warning, + 1001..=2000 => Self::Problem, + _ => Self::Critical, + } + } + + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + /// As str. + pub fn as_str(&self) -> &'static str { + match self { + Self::Ideal => "ideal", + Self::Acceptable => "acceptable", + Self::Warning => "warning", + Self::Problem => "problem", + Self::Critical => "critical", + } + } +} + +/// Health grade based on composite score +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum HealthGrade { + A, // 90-100 + B, // 80-89 + C, // 70-79 + D, // 60-69 + E, // 50-59 + F, // 0-49 +} + +impl HealthGrade { + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "score_range")] + /// From score. + pub fn from_score(score: u8) -> Self { + match score { + 90..=100 => Self::A, + 80..=89 => Self::B, + 70..=79 => Self::C, + 60..=69 => Self::D, + 50..=59 => Self::E, + _ => Self::F, + } + } + + // ── Kani proof harnesses (GH-276) ──────────────────────────────────── + // See kani/README.md for instructions on running these proofs. + // Total proof effort: verify `from_score` is a total function on u8 + // and respects monotonic grade ordering. + + /// Kani proof: `from_score` is total on all u8 inputs (0..=255). + /// i.e. it never panics and always returns a defined variant. + #[cfg(kani)] + #[kani::proof] + fn kani_from_score_total() { + let score: u8 = kani::any(); + let g = HealthGrade::from_score(score); + // Every u8 maps to exactly one of the six variants. + let _ok = matches!( + g, + HealthGrade::A + | HealthGrade::B + | HealthGrade::C + | HealthGrade::D + | HealthGrade::E + | HealthGrade::F + ); + assert!(_ok); + } + + /// Kani proof: passing grades correspond to score >= 70. + /// `is_passing()` <=> grade is A, B, or C <=> score >= 70. + #[cfg(kani)] + #[kani::proof] + fn kani_is_passing_iff_score_ge_70() { + let score: u8 = kani::any(); + // Bound to the documented range so proofs are crisp. + kani::assume(score <= 100); + let g = HealthGrade::from_score(score); + assert!(g.is_passing() == (score >= 70)); + } + + /// Kani proof: grade ordering is monotonic (higher score -> not-worse grade). + /// Specifically, a score in the A band (>=90) is never classified below C. + #[cfg(kani)] + #[kani::proof] + fn kani_high_score_yields_high_grade() { + let score: u8 = kani::any(); + kani::assume(score >= 90 && score <= 100); + let g = HealthGrade::from_score(score); + assert!(matches!(g, HealthGrade::A)); + assert!(g.is_passing()); + } + + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + /// As str. + pub fn as_str(&self) -> &'static str { + match self { + Self::A => "A", + Self::B => "B", + Self::C => "C", + Self::D => "D", + Self::E => "E", + Self::F => "F", + } + } + + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + /// Is passing. + pub fn is_passing(&self) -> bool { + matches!(self, Self::A | Self::B | Self::C) + } +} + +/// Health metrics for a single file +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FileHealthMetrics { + pub path: PathBuf, + pub lines: usize, + pub test_lines: usize, + pub tlr: f32, + pub required_tlr: f32, + pub avg_complexity: f32, + pub churn_30d: usize, + pub health_score: u8, + pub grade: HealthGrade, + pub size_class: FileSizeClass, +} + +impl FileHealthMetrics { + /// Calculate health score using the composite formula + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "path_exists")] + pub fn calculate( + path: PathBuf, + lines: usize, + test_lines: usize, + avg_complexity: f32, + churn_30d: usize, + ) -> Self { + let required_tlr = Self::required_tlr_for_size(lines); + let tlr = if lines > 0 { + test_lines as f32 / lines as f32 + } else { + 1.0 + }; + + // Size Score (30 points max) + let size_score: u8 = match lines { + 0..=200 => 30, + 201..=500 => 25, + 501..=1000 => 15, + 1001..=2000 => 5, + _ => 0, + }; + + // TLR Score (40 points max) + let tlr_ratio = (tlr / required_tlr).min(1.0); + let tlr_score = (tlr_ratio * 40.0) as u8; + + // Complexity Score (20 points max) + let complexity_score: u8 = if avg_complexity <= 5.0 { + 20 + } else if avg_complexity <= 10.0 { + 15 + } else if avg_complexity <= 15.0 { + 10 + } else if avg_complexity <= 20.0 { + 5 + } else { + 0 + }; + + // Stability Score (10 points max) + let stability_score: u8 = match churn_30d { + 0..=2 => 10, + 3..=5 => 7, + 6..=10 => 4, + _ => 0, + }; + + let health_score = size_score + tlr_score + complexity_score + stability_score; + let grade = HealthGrade::from_score(health_score); + let size_class = FileSizeClass::from_lines(lines); + + Self { + path, + lines, + test_lines, + tlr, + required_tlr, + avg_complexity, + churn_30d, + health_score, + grade, + size_class, + } + } + + /// Get required TLR based on file size (scaling thresholds) + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + pub fn required_tlr_for_size(lines: usize) -> f32 { + match lines { + 0..=100 => 0.3, + 101..=300 => 0.5, + 301..=500 => 0.7, + 501..=1000 => 1.0, + _ => 1.5, + } + } +} diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/services/infra_score/models.rs.txt b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/services/infra_score/models.rs.txt new file mode 100644 index 0000000000..1c9678e1e3 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/services/infra_score/models.rs.txt @@ -0,0 +1,549 @@ +#![cfg_attr(coverage_nightly, coverage(off))] +//! Data models for pmat infra-score +//! Implements the scoring system defined in docs/specifications/components/infra-score.md +//! +//! 5 dimensions, 100 points total. Hard cutoff: <90 = auto-fail. + +use crate::services::normalized_score::NormalizedScore; +use serde::{Deserialize, Serialize}; +use std::fmt; +use std::path::PathBuf; + +/// Maximum possible raw points for Infra Score (no bonuses — strict 100) +pub const INFRA_SCORE_MAX_POINTS: f64 = 100.0; + +/// Maximum bonus points from the Provable Contracts category. +/// +/// This MUST equal the sum of the PV-01..PV-05 check weights (3+3+2+2+2), which +/// is what `ProvableContractsScorer` actually awards. Three places used to carry +/// their own copy of this number — the scorer said 12, the model default said 10 +/// and the text renderer hardcoded a "/110.0" denominator — so a perfect run +/// printed a bonus larger than the maximum the same output advertised. +pub const INFRA_SCORE_BONUS_MAX_POINTS: f64 = 12.0; + +/// Overall infrastructure score result +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InfraScore { + /// Raw score (0-100) + pub total_score: f64, + pub grade: InfraGrade, + pub auto_fail: bool, + pub categories: InfraCategoryScores, + pub recommendations: Vec, + pub metadata: InfraScoreMetadata, +} + +/// Infra-specific grade with hard cutoff semantics +/// A+ (95-100), A (90-94), B (80-89) AUTO-FAIL, C (60-79) AUTO-FAIL, D (40-59) AUTO-FAIL, F (0-39) AUTO-FAIL +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub enum InfraGrade { + APlus, // 95-100 + A, // 90-94 + B, // 80-89 — AUTO-FAIL + C, // 60-79 — AUTO-FAIL + D, // 40-59 — AUTO-FAIL + F, // 0-39 — AUTO-FAIL +} + +/// Category scores (100 points total) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InfraCategoryScores { + pub workflow_architecture: InfraCategoryScore, // 25 points + pub build_reliability: InfraCategoryScore, // 25 points + pub quality_pipeline: InfraCategoryScore, // 20 points + pub deployment_release: InfraCategoryScore, // 15 points + pub supply_chain: InfraCategoryScore, // 15 points + pub provable_contracts: InfraCategoryScore, // 12 points (bonus) +} + +/// Individual category score (mirrors repo_score CategoryScore) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InfraCategoryScore { + pub score: f64, + pub max_score: f64, + pub percentage: f64, + pub checks: Vec, + pub findings: Vec, +} + +/// Individual check result (e.g., WA-01, BR-03) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InfraCheck { + pub id: String, // "WA-01", "BR-03", etc. + pub name: String, // Human-readable name + pub score: f64, // Earned points + pub max_score: f64, // Maximum possible + pub passed: bool, + pub evidence: Vec, +} + +/// Finding with severity (mirrors repo_score Finding) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InfraFinding { + pub severity: InfraSeverity, + pub check_id: String, + pub message: String, + pub location: Option, + pub impact_points: f64, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +/// Severity level classification for infra. +pub enum InfraSeverity { + Pass, // Check passed + Warning, // Partial compliance + Fail, // Check failed + Info, // Informational +} + +/// Recommendation for improvement +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InfraRecommendation { + pub priority: InfraPriority, + pub check_id: String, + pub title: String, + pub description: String, + pub impact_points: f64, + pub estimated_effort: String, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +/// Priority level for infra. +pub enum InfraPriority { + Low = 1, + Medium = 2, + High = 3, + Critical = 4, +} + +/// Metadata about the scoring run +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InfraScoreMetadata { + pub timestamp: String, + pub repository_path: PathBuf, + pub git_branch: Option, + pub git_commit: Option, + pub pmat_version: String, + pub spec_version: String, + pub execution_time_ms: u64, +} + +// --- impl blocks --- + +impl NormalizedScore for InfraScore { + fn raw(&self) -> f64 { + self.total_score + } + + fn max_raw(&self) -> f64 { + INFRA_SCORE_MAX_POINTS + } +} + +impl fmt::Display for InfraScore { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let fail_marker = if self.auto_fail { " [AUTO-FAIL]" } else { "" }; + write!( + f, + "Infra Score: {:.1}/100 ({}){}", + self.total_score, + self.grade.as_str(), + fail_marker, + ) + } +} + +impl InfraGrade { + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "score_range")] + /// From score. + pub fn from_score(score: f64) -> Self { + match score { + s if s >= 95.0 => InfraGrade::APlus, + s if s >= 90.0 => InfraGrade::A, + s if s >= 80.0 => InfraGrade::B, + s if s >= 60.0 => InfraGrade::C, + s if s >= 40.0 => InfraGrade::D, + _ => InfraGrade::F, + } + } + + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + /// As str. + pub fn as_str(&self) -> &'static str { + match self { + InfraGrade::APlus => "A+", + InfraGrade::A => "A", + InfraGrade::B => "B", + InfraGrade::C => "C", + InfraGrade::D => "D", + InfraGrade::F => "F", + } + } + + /// Returns true if this grade is an auto-fail (<90) + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + pub fn is_auto_fail(&self) -> bool { + !matches!(self, InfraGrade::APlus | InfraGrade::A) + } +} + +// ── Kani proof harnesses (GH-276) ──────────────────────────────────────────── +// Prove the auto-fail cutoff semantics that the specification documents: +// scores < 90 MUST auto-fail; scores >= 90 MUST NOT auto-fail. +#[cfg(kani)] +mod kani_proofs { + use super::InfraGrade; + + /// Any finite score < 90 must classify as an auto-fail grade. + /// This is the central safety property of the infra-score spec. + #[kani::proof] + fn auto_fail_iff_below_90() { + let s: f64 = kani::any(); + kani::assume(s.is_finite()); + kani::assume((-1000.0..=1000.0).contains(&s)); + let g = InfraGrade::from_score(s); + // s < 90 => is_auto_fail + // s >= 90 => !is_auto_fail + assert!(g.is_auto_fail() == (s < 90.0)); + } + + /// `from_score` is total on bounded finite inputs. + #[kani::proof] + fn from_score_total() { + let s: f64 = kani::any(); + kani::assume(s.is_finite()); + kani::assume((-1000.0..=1000.0).contains(&s)); + let g = InfraGrade::from_score(s); + let _ok = matches!( + g, + InfraGrade::APlus + | InfraGrade::A + | InfraGrade::B + | InfraGrade::C + | InfraGrade::D + | InfraGrade::F + ); + assert!(_ok); + } +} + +impl InfraCategoryScores { + /// Total base score (100 points max, excluding bonus) + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + pub fn total(&self) -> f64 { + self.workflow_architecture.score + + self.build_reliability.score + + self.quality_pipeline.score + + self.deployment_release.score + + self.supply_chain.score + } + + /// Total including provable contracts bonus + /// (`INFRA_SCORE_MAX_POINTS` + `INFRA_SCORE_BONUS_MAX_POINTS` max) + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + pub fn total_with_bonus(&self) -> f64 { + self.total() + self.provable_contracts.score + } +} + +impl Default for InfraCategoryScores { + fn default() -> Self { + Self { + workflow_architecture: InfraCategoryScore::empty(25.0), + build_reliability: InfraCategoryScore::empty(25.0), + quality_pipeline: InfraCategoryScore::empty(20.0), + deployment_release: InfraCategoryScore::empty(15.0), + supply_chain: InfraCategoryScore::empty(15.0), + provable_contracts: InfraCategoryScore::empty(INFRA_SCORE_BONUS_MAX_POINTS), + } + } +} + +impl InfraCategoryScore { + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + /// Empty. + pub fn empty(max_score: f64) -> Self { + Self { + score: 0.0, + max_score, + percentage: 0.0, + checks: vec![], + findings: vec![], + } + } + + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + /// Create a new instance. + pub fn new(max_score: f64, checks: Vec, findings: Vec) -> Self { + let score: f64 = checks.iter().map(|c| c.score).sum(); + let percentage = if max_score > 0.0 { + (score / max_score) * 100.0 + } else { + 0.0 + }; + + Self { + score, + max_score, + percentage, + checks, + findings, + } + } +} + +impl InfraCheck { + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + /// Pass. + pub fn pass(id: &str, name: &str, max_score: f64, evidence: Vec) -> Self { + Self { + id: id.to_string(), + name: name.to_string(), + score: max_score, + max_score, + passed: true, + evidence, + } + } + + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + /// Fail. + pub fn fail(id: &str, name: &str, max_score: f64, evidence: Vec) -> Self { + Self { + id: id.to_string(), + name: name.to_string(), + score: 0.0, + max_score, + passed: false, + evidence, + } + } + + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + /// Partial. + pub fn partial( + id: &str, + name: &str, + score: f64, + max_score: f64, + evidence: Vec, + ) -> Self { + Self { + id: id.to_string(), + name: name.to_string(), + score, + max_score, + passed: score >= max_score, + evidence, + } + } +} + +impl InfraScoreMetadata { + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "path_exists")] + /// Create a new instance. + pub fn new(repository_path: PathBuf) -> Self { + Self { + timestamp: chrono::Utc::now().to_rfc3339(), + repository_path, + git_branch: None, + git_commit: None, + pmat_version: env!("CARGO_PKG_VERSION").to_string(), + spec_version: "0.1.0".to_string(), + execution_time_ms: 0, + } + } +} + +#[cfg_attr(coverage_nightly, coverage(off))] +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_infra_grade_from_score() { + assert_eq!(InfraGrade::from_score(100.0), InfraGrade::APlus); + assert_eq!(InfraGrade::from_score(95.0), InfraGrade::APlus); + assert_eq!(InfraGrade::from_score(94.0), InfraGrade::A); + assert_eq!(InfraGrade::from_score(90.0), InfraGrade::A); + assert_eq!(InfraGrade::from_score(89.0), InfraGrade::B); + assert_eq!(InfraGrade::from_score(80.0), InfraGrade::B); + assert_eq!(InfraGrade::from_score(79.0), InfraGrade::C); + assert_eq!(InfraGrade::from_score(60.0), InfraGrade::C); + assert_eq!(InfraGrade::from_score(59.0), InfraGrade::D); + assert_eq!(InfraGrade::from_score(40.0), InfraGrade::D); + assert_eq!(InfraGrade::from_score(39.0), InfraGrade::F); + assert_eq!(InfraGrade::from_score(0.0), InfraGrade::F); + } + + #[test] + fn test_infra_grade_auto_fail() { + assert!(!InfraGrade::APlus.is_auto_fail()); + assert!(!InfraGrade::A.is_auto_fail()); + assert!(InfraGrade::B.is_auto_fail()); + assert!(InfraGrade::C.is_auto_fail()); + assert!(InfraGrade::D.is_auto_fail()); + assert!(InfraGrade::F.is_auto_fail()); + } + + #[test] + fn test_infra_grade_as_str() { + assert_eq!(InfraGrade::APlus.as_str(), "A+"); + assert_eq!(InfraGrade::A.as_str(), "A"); + assert_eq!(InfraGrade::B.as_str(), "B"); + assert_eq!(InfraGrade::C.as_str(), "C"); + assert_eq!(InfraGrade::D.as_str(), "D"); + assert_eq!(InfraGrade::F.as_str(), "F"); + } + + #[test] + fn test_infra_category_scores_total() { + let scores = InfraCategoryScores { + workflow_architecture: InfraCategoryScore { + score: 20.0, + max_score: 25.0, + percentage: 80.0, + checks: vec![], + findings: vec![], + }, + build_reliability: InfraCategoryScore { + score: 22.0, + max_score: 25.0, + percentage: 88.0, + checks: vec![], + findings: vec![], + }, + quality_pipeline: InfraCategoryScore { + score: 18.0, + max_score: 20.0, + percentage: 90.0, + checks: vec![], + findings: vec![], + }, + deployment_release: InfraCategoryScore { + score: 12.0, + max_score: 15.0, + percentage: 80.0, + checks: vec![], + findings: vec![], + }, + supply_chain: InfraCategoryScore { + score: 10.0, + max_score: 15.0, + percentage: 66.7, + checks: vec![], + findings: vec![], + }, + provable_contracts: InfraCategoryScore { + score: 5.0, + max_score: 10.0, + percentage: 50.0, + checks: vec![], + findings: vec![], + }, + }; + assert!((scores.total() - 82.0).abs() < f64::EPSILON); + } + + #[test] + fn test_infra_check_pass() { + let check = InfraCheck::pass( + "WA-01", + "Reusable workflow", + 5.0, + vec!["found uses: org/.github".to_string()], + ); + assert!(check.passed); + assert!((check.score - 5.0).abs() < f64::EPSILON); + } + + #[test] + fn test_infra_check_fail() { + let check = InfraCheck::fail( + "WA-01", + "Reusable workflow", + 5.0, + vec!["no reusable workflow found".to_string()], + ); + assert!(!check.passed); + assert!((check.score - 0.0).abs() < f64::EPSILON); + } + + #[test] + fn test_infra_check_partial() { + let check = InfraCheck::partial( + "BR-01", + "CI success rate", + 3.0, + 5.0, + vec!["7/10 passed".to_string()], + ); + assert!(!check.passed); + assert!((check.score - 3.0).abs() < f64::EPSILON); + } + + #[test] + fn test_infra_category_score_new() { + let checks = vec![ + InfraCheck::pass("WA-01", "Reusable", 5.0, vec![]), + InfraCheck::fail("WA-02", "Self-hosted", 5.0, vec![]), + ]; + let cat = InfraCategoryScore::new(25.0, checks, vec![]); + assert!((cat.score - 5.0).abs() < f64::EPSILON); + assert!((cat.percentage - 20.0).abs() < f64::EPSILON); + } + + #[test] + fn test_infra_score_display() { + let score = InfraScore { + total_score: 92.0, + grade: InfraGrade::A, + auto_fail: false, + categories: InfraCategoryScores::default(), + recommendations: vec![], + metadata: InfraScoreMetadata::new(PathBuf::from("/tmp/test")), + }; + let display = format!("{}", score); + assert!(display.contains("92.0")); + assert!(display.contains("A")); + assert!(!display.contains("AUTO-FAIL")); + } + + #[test] + fn test_infra_score_display_auto_fail() { + let score = InfraScore { + total_score: 75.0, + grade: InfraGrade::C, + auto_fail: true, + categories: InfraCategoryScores::default(), + recommendations: vec![], + metadata: InfraScoreMetadata::new(PathBuf::from("/tmp/test")), + }; + let display = format!("{}", score); + assert!(display.contains("AUTO-FAIL")); + } + + #[test] + fn test_normalized_score_trait() { + let score = InfraScore { + total_score: 85.0, + grade: InfraGrade::B, + auto_fail: true, + categories: InfraCategoryScores::default(), + recommendations: vec![], + metadata: InfraScoreMetadata::new(PathBuf::from("/tmp/test")), + }; + assert!((score.raw() - 85.0).abs() < f64::EPSILON); + assert!((score.max_raw() - 100.0).abs() < f64::EPSILON); + assert!((score.normalized() - 85.0).abs() < f64::EPSILON); + } + + #[test] + fn test_metadata_new() { + let meta = InfraScoreMetadata::new(PathBuf::from("/tmp/repo")); + assert_eq!(meta.repository_path, PathBuf::from("/tmp/repo")); + assert_eq!(meta.spec_version, "0.1.0"); + assert!(meta.git_branch.is_none()); + assert!(meta.git_commit.is_none()); + } +} diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/services/normalized_score.rs.txt b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/services/normalized_score.rs.txt new file mode 100644 index 0000000000..0c95396b5a --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/services/normalized_score.rs.txt @@ -0,0 +1,234 @@ +#![cfg_attr(coverage_nightly, coverage(off))] +//! Normalized Score System (PMAT-454) +//! +//! All PMAT scoring systems MUST output values in the 0-100 range. +//! This module provides the trait and utilities to ensure consistent scoring. +//! +//! # Design Principles +//! - All scores are normalized to 0.0-100.0 range +//! - Raw scores can use any internal scale (106, 110, 200 points) +//! - `normalized()` method always returns 0-100 +//! - Clamping ensures no out-of-range values + +use std::fmt; + +/// Trait for all scoring systems in PMAT. +/// +/// Implementors MUST ensure `normalized()` returns values in [0.0, 100.0]. +pub trait NormalizedScore: fmt::Display { + /// Returns the raw score value (internal scale). + fn raw(&self) -> f64; + + /// Returns the maximum possible raw score. + fn max_raw(&self) -> f64; + + /// Returns the normalized score in 0-100 range. + /// + /// # Guarantees + /// - Always returns a value in [0.0, 100.0] + /// - Values are clamped if raw calculation exceeds bounds + fn normalized(&self) -> f64 { + let max = self.max_raw(); + if max <= 0.0 { + return 0.0; + } + let normalized = (self.raw() / max) * 100.0; + normalized.clamp(0.0, 100.0) + } + + /// Returns the letter grade based on normalized score. + fn grade(&self) -> Grade { + Grade::from_score(self.normalized()) + } + + /// Returns true if score meets the given threshold (0-100). + fn meets_threshold(&self, threshold: f64) -> bool { + self.normalized() >= threshold.clamp(0.0, 100.0) + } +} + +/// Universal letter grades for all scoring systems. +/// Ordering: A > B > C > D > F (higher grade = better) +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Grade { + /// 90-100: Excellent + A, + /// 80-89: Good + B, + /// 70-79: Satisfactory + C, + /// 60-69: Needs Improvement + D, + /// 0-59: Failing + F, +} + +impl PartialOrd for Grade { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Grade { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.min_score() + .partial_cmp(&other.min_score()) + .unwrap_or(std::cmp::Ordering::Equal) + } +} + +impl Grade { + /// Convert a normalized score (0-100) to a grade. + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "score_range")] + pub fn from_score(score: f64) -> Self { + match score { + s if s >= 90.0 => Grade::A, + s if s >= 80.0 => Grade::B, + s if s >= 70.0 => Grade::C, + s if s >= 60.0 => Grade::D, + _ => Grade::F, + } + } + + /// Returns the minimum score for this grade. + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "score_range")] + pub fn min_score(&self) -> f64 { + match self { + Grade::A => 90.0, + Grade::B => 80.0, + Grade::C => 70.0, + Grade::D => 60.0, + Grade::F => 0.0, + } + } + + /// Returns the grade as a string with description. + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + pub fn description(&self) -> &'static str { + match self { + Grade::A => "A (Excellent)", + Grade::B => "B (Good)", + Grade::C => "C (Satisfactory)", + Grade::D => "D (Needs Improvement)", + Grade::F => "F (Failing)", + } + } +} + +// ── Kani proof harnesses (GH-276) ──────────────────────────────────────────── +// These prove pure arithmetic invariants about the grade classifier. +// See kani/README.md for how to run. +#[cfg(kani)] +mod kani_proofs { + use super::Grade; + + /// `from_score` is total on a bounded f64 input and never panics. + /// We bound the input to a "reasonable" range since Kani reasons poorly + /// about NaN/Inf and we only care about realistic scores here. + #[kani::proof] + fn from_score_total_in_bounds() { + let s: f64 = kani::any(); + // Exclude NaN/Inf explicitly — the contract is about finite scores. + kani::assume(s.is_finite()); + kani::assume((-1000.0..=1000.0).contains(&s)); + let g = Grade::from_score(s); + let _ok = matches!(g, Grade::A | Grade::B | Grade::C | Grade::D | Grade::F); + assert!(_ok); + } + + /// `min_score` is a left-inverse of `from_score`'s band structure: + /// for any grade G, `from_score(G.min_score())` returns G or better. + /// (We can't prove full inverse since `from_score` is many-to-one.) + /// This proves the band boundaries are internally consistent. + #[kani::proof] + fn min_score_consistent_at_A_boundary() { + let s: f64 = kani::any(); + kani::assume(s.is_finite()); + kani::assume((90.0..=100.0).contains(&s)); + // Any score in [90, 100] must classify as A. + assert!(matches!(Grade::from_score(s), Grade::A)); + } + + /// `min_score` is a left-inverse at the F boundary: any score < 60 is F. + #[kani::proof] + fn below_D_boundary_is_F() { + let s: f64 = kani::any(); + kani::assume(s.is_finite()); + kani::assume((-1000.0..60.0).contains(&s)); + assert!(matches!(Grade::from_score(s), Grade::F)); + } +} + +impl fmt::Display for Grade { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Grade::A => write!(f, "A"), + Grade::B => write!(f, "B"), + Grade::C => write!(f, "C"), + Grade::D => write!(f, "D"), + Grade::F => write!(f, "F"), + } + } +} + +/// Helper struct for creating normalized scores from raw values. +#[derive(Debug, Clone, Copy)] +pub struct SimpleScore { + raw: f64, + max: f64, + name: &'static str, +} + +impl SimpleScore { + /// Create a new simple score. + /// + /// # Panics + /// Panics if max <= 0. + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + pub fn new(raw: f64, max: f64, name: &'static str) -> Self { + assert!(max > 0.0, "max must be positive"); + Self { + raw: raw.max(0.0), + max, + name, + } + } + + /// Create from a percentage (0-100). + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + pub fn from_percentage(pct: f64, name: &'static str) -> Self { + Self { + raw: pct.clamp(0.0, 100.0), + max: 100.0, + name, + } + } +} + +impl NormalizedScore for SimpleScore { + fn raw(&self) -> f64 { + self.raw + } + + fn max_raw(&self) -> f64 { + self.max + } +} + +impl fmt::Display for SimpleScore { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{}: {:.1}/100 ({})", + self.name, + self.normalized(), + self.grade() + ) + } +} + +// Aggregate scoring: AggregateScore, NormalizedScoreClone trait +include!("normalized_score_aggregate.rs"); + +// Tests +include!("normalized_score_tests.rs"); diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/services/repo_score/models_impls.rs.txt b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/services/repo_score/models_impls.rs.txt new file mode 100644 index 0000000000..4737bc3e06 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/services/repo_score/models_impls.rs.txt @@ -0,0 +1,216 @@ +// RepoScore: NormalizedScore trait impl +impl NormalizedScore for RepoScore { + fn raw(&self) -> f64 { + self.total_score + } + + fn max_raw(&self) -> f64 { + REPO_SCORE_MAX_POINTS + } +} + +// RepoScore: Display impl +impl fmt::Display for RepoScore { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "Repo Score: {:.1}/100 ({}) [raw: {:.1}/{}]", + self.normalized(), + self.grade.as_str(), + self.total_score, + REPO_SCORE_MAX_POINTS as u32 + ) + } +} + +// Grade: conversion and formatting methods +impl Grade { + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "score_range")] + /// From score. + pub fn from_score(score: f64) -> Self { + match score { + s if s >= 95.0 => Grade::APlus, + s if s >= 90.0 => Grade::A, + s if s >= 85.0 => Grade::AMinus, + s if s >= 80.0 => Grade::BPlus, + s if s >= 70.0 => Grade::B, + s if s >= 60.0 => Grade::C, + s if s >= 50.0 => Grade::D, + _ => Grade::F, + } + } + + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + /// As str. + pub fn as_str(&self) -> &'static str { + match self { + Grade::APlus => "A+", + Grade::A => "A", + Grade::AMinus => "A-", + Grade::BPlus => "B+", + Grade::B => "B", + Grade::C => "C", + Grade::D => "D", + Grade::F => "F", + } + } +} + +// CategoryScores: aggregate scoring +impl CategoryScores { + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + /// Total. + pub fn total(&self) -> f64 { + self.documentation.score + + self.precommit_hooks.score + + self.repository_hygiene.score + + self.build_test_automation.score + + self.continuous_integration.score + + self.pmat_compliance.score + } +} + +impl Default for CategoryScores { + fn default() -> Self { + Self { + documentation: CategoryScore::default_with_max(15.0), + precommit_hooks: CategoryScore::default_with_max(20.0), + repository_hygiene: CategoryScore::default_with_max(15.0), + build_test_automation: CategoryScore::default_with_max(25.0), + continuous_integration: CategoryScore::default_with_max(20.0), + pmat_compliance: CategoryScore::default_with_max(5.0), + } + } +} + +// CategoryScore: construction and defaults +impl CategoryScore { + fn default_with_max(max_score: f64) -> Self { + Self { + score: 0.0, + max_score, + percentage: 0.0, + status: ScoreStatus::Fail, + subcategories: vec![], + findings: vec![], + } + } + + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + /// Create a new instance. + pub fn new( + score: f64, + max_score: f64, + subcategories: Vec, + findings: Vec, + ) -> Self { + let percentage = if max_score > 0.0 { + (score / max_score) * 100.0 + } else { + 0.0 + }; + + let status = if percentage >= 90.0 { + ScoreStatus::Pass + } else if percentage >= 70.0 { + ScoreStatus::Warning + } else { + ScoreStatus::Fail + }; + + Self { + score, + max_score, + percentage, + status, + subcategories, + findings, + } + } +} + +// BonusScores: aggregate and defaults +impl BonusScores { + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + /// Total. + pub fn total(&self) -> f64 { + self.property_tests.points + + self.fuzzing.points + + self.mutation_testing.points + + self.living_docs.points + } +} + +impl Default for BonusScores { + fn default() -> Self { + Self { + property_tests: BonusItem { + points: 0.0, + max_points: 3.0, + detected: false, + evidence: vec![], + }, + fuzzing: BonusItem { + points: 0.0, + max_points: 2.0, + detected: false, + evidence: vec![], + }, + mutation_testing: BonusItem { + points: 0.0, + max_points: 2.0, + detected: false, + evidence: vec![], + }, + living_docs: BonusItem { + points: 0.0, + max_points: 3.0, + detected: false, + evidence: vec![], + }, + } + } +} + +// Priority: manual PartialOrd/Ord for correct ordering +// Critical > High > Medium > Low +impl PartialOrd for Priority { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Priority { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + let self_rank = match self { + Priority::Critical => 4, + Priority::High => 3, + Priority::Medium => 2, + Priority::Low => 1, + }; + let other_rank = match other { + Priority::Critical => 4, + Priority::High => 3, + Priority::Medium => 2, + Priority::Low => 1, + }; + self_rank.cmp(&other_rank) + } +} + +// ScoreMetadata: construction +impl ScoreMetadata { + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "path_exists")] + /// Create a new instance. + pub fn new(repository_path: PathBuf) -> Self { + Self { + timestamp: chrono::Utc::now().to_rfc3339(), + repository_path, + git_branch: None, + git_commit: None, + pmat_version: env!("CARGO_PKG_VERSION").to_string(), + spec_version: "1.0.0".to_string(), + execution_time_ms: 0, + } + } +} diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/services/rust_project_score/models_score.rs.txt b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/services/rust_project_score/models_score.rs.txt new file mode 100644 index 0000000000..4f761b2ff5 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/services/rust_project_score/models_score.rs.txt @@ -0,0 +1,155 @@ +// ============================================================================ +// RustProjectScore - Main Score Container +// ============================================================================ + +/// Comprehensive Rust project quality score (v1.1) +/// +/// Total score: 0-106 points across 6 categories +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RustProjectScore { + /// Total score (0-106 points) + pub total_score: f64, + + /// Letter grade (A+ to F) + pub grade: Grade, + + /// Breakdown by category + pub categories: CategoryScores, + + /// Actionable recommendations + pub recommendations: Vec, + + /// Metadata (timestamp, project, version) + pub metadata: ScoreMetadata, + + /// Score velocity (Kaizen tracking) - NEW in v1.1 + pub velocity: Option, +} + +impl RustProjectScore { + /// Create a new score with zero values + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + pub fn new() -> Self { + Self { + total_score: 0.0, + grade: Grade::F, + categories: CategoryScores::default(), + recommendations: Vec::new(), + metadata: ScoreMetadata::new("unknown".to_string(), "1.1.0".to_string()), + velocity: None, + } + } +} + +impl Default for RustProjectScore { + fn default() -> Self { + Self::new() + } +} + +impl NormalizedScore for RustProjectScore { + fn raw(&self) -> f64 { + self.total_score + } + + fn max_raw(&self) -> f64 { + RUST_PROJECT_MAX_POINTS + } +} + +impl fmt::Display for RustProjectScore { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "Rust Project Score: {:.1}/100 ({}) [raw: {:.1}/{}]", + self.normalized(), + self.grade, + self.total_score, + RUST_PROJECT_MAX_POINTS as u32 + ) + } +} + +// ============================================================================ +// Grade - Letter Grade Enum +// ============================================================================ + +/// Letter grade based on NORMALIZED percentage (0-100 scale) +/// +/// PMAT-454: All grading now uses normalized 0-100 percentages +/// +/// Thresholds (normalized 0-100): +/// - A+ : 95-100% +/// - A : 90-94% +/// - A- : 85-89% +/// - B+ : 80-84% +/// - B : 70-79% +/// - C : 60-69% +/// - D : 50-59% +/// - F : 0-49% +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum Grade { + APlus, + A, + AMinus, + BPlus, + B, + C, + D, + F, +} + +impl Grade { + /// Calculate grade from raw score and max possible points + /// + /// PMAT-454: Now properly normalizes to 0-100 before grading + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "score_range")] + pub fn from_score(score: f64, max: f64) -> Self { + // Normalize to 0-100 percentage + let normalized = if max > 0.0 { + (score / max * 100.0).clamp(0.0, 100.0) + } else { + 0.0 + }; + + if normalized >= 95.0 { + Grade::APlus + } else if normalized >= 90.0 { + Grade::A + } else if normalized >= 85.0 { + Grade::AMinus + } else if normalized >= 80.0 { + Grade::BPlus + } else if normalized >= 70.0 { + Grade::B + } else if normalized >= 60.0 { + Grade::C + } else if normalized >= 50.0 { + Grade::D + } else { + Grade::F + } + } + + /// Calculate grade from already-normalized percentage (0-100) + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + pub fn from_normalized(normalized: f64) -> Self { + Self::from_score(normalized, 100.0) + } +} + +impl fmt::Display for Grade { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Grade::APlus => write!(f, "A+"), + Grade::A => write!(f, "A"), + Grade::AMinus => write!(f, "A-"), + Grade::BPlus => write!(f, "B+"), + Grade::B => write!(f, "B"), + Grade::C => write!(f, "C"), + Grade::D => write!(f, "D"), + Grade::F => write!(f, "F"), + } + } +} + diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/tdg/cuda_simd_scores.rs.txt b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/tdg/cuda_simd_scores.rs.txt new file mode 100644 index 0000000000..d603d82649 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/tdg/cuda_simd_scores.rs.txt @@ -0,0 +1,293 @@ +//! CUDA-SIMD Score Types +//! +//! Extracted from cuda_simd.rs for file health compliance (CB-040). +//! Contains the 100-point Karl Popper falsification scoring structures. + +use serde::{Deserialize, Serialize}; + +/// Category A: Falsifiability & Testability (25 points) - GATEWAY +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct FalsifiabilityScore { + /// A.1: All bar.sync reachable from all threads (5 pts) + pub barrier_safety: f64, + /// A.2: Shared memory indices within tile dimensions (5 pts) + pub bounds_verification: f64, + /// A.3: Branch coverage includes warp divergence cases (5 pts) + pub divergence_testing: f64, + /// A.4: ThreadSanitizer or equivalent analysis (5 pts) + pub memory_race_detection: f64, + /// A.5: Register/shared memory within SM limits (5 pts) + pub occupancy_bounds: f64, +} + +impl FalsifiabilityScore { + /// Calculate total for Category A + #[must_use] + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + pub fn total(&self) -> f64 { + self.barrier_safety + + self.bounds_verification + + self.divergence_testing + + self.memory_race_detection + + self.occupancy_bounds + } + + /// Maximum possible score for Category A + pub const MAX: f64 = 25.0; + + /// Gateway threshold - if below this, total score is 0 + pub const GATEWAY_THRESHOLD: f64 = 15.0; +} + +/// Category B: Reproducibility Infrastructure (25 points) +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ReproducibilityScore { + /// B.1: Bitwise reproducible results (8 pts) + pub deterministic_output: f64, + /// B.2: CUDA/Driver/SM version locked (5 pts) + pub version_pinning: f64, + /// B.3: GPU model, compute capability documented (5 pts) + pub hardware_specification: f64, + /// B.4: Criterion-style statistical benchmarking (4 pts) + pub benchmark_harness: f64, + /// B.5: Automated regression on GPU hardware (3 pts) + pub ci_cd_integration: f64, +} + +impl ReproducibilityScore { + /// Calculate total for Category B + #[must_use] + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + pub fn total(&self) -> f64 { + self.deterministic_output + + self.version_pinning + + self.hardware_specification + + self.benchmark_harness + + self.ci_cd_integration + } + + /// Maximum possible score for Category B + pub const MAX: f64 = 25.0; +} + +/// Category C: Transparency & Openness (20 points) +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct TransparencyScore { + /// C.1: Generated PTX accessible and documented (6 pts) + pub ptx_inspection: f64, + /// C.2: --ptxas-options=-v output analyzed (5 pts) + pub register_allocation: f64, + /// C.3: SM occupancy explicitly computed (5 pts) + pub occupancy_calculation: f64, + /// C.4: Shared memory bank mapping documented (4 pts) + pub memory_layout: f64, +} + +impl TransparencyScore { + /// Calculate total for Category C + #[must_use] + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + pub fn total(&self) -> f64 { + self.ptx_inspection + + self.register_allocation + + self.occupancy_calculation + + self.memory_layout + } + + /// Maximum possible score for Category C + pub const MAX: f64 = 20.0; +} + +/// Category D: Statistical Rigor (15 points) +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct StatisticalRigorScore { + /// D.1: ≥3s warmup before measurement (4 pts) + pub warmup_iterations: f64, + /// D.2: ≥100 samples for statistical significance (4 pts) + pub sample_count: f64, + /// D.3: IQR-based outlier detection reported (4 pts) + pub outlier_analysis: f64, + /// D.4: 95% CI on throughput metrics (3 pts) + pub confidence_intervals: f64, +} + +impl StatisticalRigorScore { + /// Calculate total for Category D + #[must_use] + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + pub fn total(&self) -> f64 { + self.warmup_iterations + + self.sample_count + + self.outlier_analysis + + self.confidence_intervals + } + + /// Maximum possible score for Category D + pub const MAX: f64 = 15.0; +} + +/// Category E: Historical Integrity (10 points) +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct HistoricalIntegrityScore { + /// E.1: PARITY/PAR ticket references (4 pts) + pub fault_lineage: f64, + /// E.2: Tests derived from historical bugs (3 pts) + pub regression_tests: f64, + /// E.3: 5-Why analysis for each P0 defect (3 pts) + pub root_cause_documentation: f64, +} + +impl HistoricalIntegrityScore { + /// Calculate total for Category E + #[must_use] + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + pub fn total(&self) -> f64 { + self.fault_lineage + self.regression_tests + self.root_cause_documentation + } + + /// Maximum possible score for Category E + pub const MAX: f64 = 10.0; +} + +/// Category F: GPU/SIMD Specific (5 points) +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct GpuSimdSpecificScore { + /// F.1: Active threads / warp size ratio (2 pts) + pub warp_efficiency: f64, + /// F.2: Achieved vs theoretical bandwidth (2 pts) + pub memory_throughput: f64, + /// F.3: FMA/memory instruction ratio (1 pt) + pub instruction_mix: f64, +} + +impl GpuSimdSpecificScore { + /// Calculate total for Category F + #[must_use] + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + pub fn total(&self) -> f64 { + self.warp_efficiency + self.memory_throughput + self.instruction_mix + } + + /// Maximum possible score for Category F + pub const MAX: f64 = 5.0; +} + +/// Complete 100-point Popper Falsification Score +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct PopperScore { + /// Category A: Falsifiability & Testability (25 pts) - GATEWAY + pub falsifiability: FalsifiabilityScore, + /// Category B: Reproducibility Infrastructure (25 pts) + pub reproducibility: ReproducibilityScore, + /// Category C: Transparency & Openness (20 pts) + pub transparency: TransparencyScore, + /// Category D: Statistical Rigor (15 pts) + pub statistical_rigor: StatisticalRigorScore, + /// Category E: Historical Integrity (10 pts) + pub historical_integrity: HistoricalIntegrityScore, + /// Category F: GPU/SIMD Specific (5 pts) + pub gpu_simd_specific: GpuSimdSpecificScore, + /// Total score (0-100, or 0 if gateway fails) + pub total: f64, + /// Whether the gateway (Category A ≥ 15) passed + pub gateway_passed: bool, + /// Grade interpretation + pub grade: CudaTdgGrade, +} + +impl PopperScore { + /// Calculate total score with gateway rule + #[must_use] + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "check_compliance")] + pub fn calculate( + falsifiability: FalsifiabilityScore, + reproducibility: ReproducibilityScore, + transparency: TransparencyScore, + statistical_rigor: StatisticalRigorScore, + historical_integrity: HistoricalIntegrityScore, + gpu_simd_specific: GpuSimdSpecificScore, + ) -> Self { + let category_a = falsifiability.total(); + let gateway_passed = category_a >= FalsifiabilityScore::GATEWAY_THRESHOLD; + + let raw_total = category_a + + reproducibility.total() + + transparency.total() + + statistical_rigor.total() + + historical_integrity.total() + + gpu_simd_specific.total(); + + // Gateway rule: if Category A < 15, total = 0 + let total = if gateway_passed { raw_total } else { 0.0 }; + let grade = CudaTdgGrade::from_score(total, gateway_passed); + + Self { + falsifiability, + reproducibility, + transparency, + statistical_rigor, + historical_integrity, + gpu_simd_specific, + total, + gateway_passed, + grade, + } + } +} + +/// Grade interpretation for CUDA-SIMD TDG scores +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +pub enum CudaTdgGrade { + /// 90-100: Production-ready, minimal debt + /// + /// Spelled with a capital L up to and including v3.29.0. Nothing persists + /// this enum (it is only ever rendered into a report), so the rename needs + /// no compatibility alias. + APlus, + /// 80-89: Production-ready with monitoring + A, + /// 70-79: Acceptable, prioritize improvements + B, + /// 60-69: Technical debt accumulating + C, + /// 50-59: Significant remediation needed + D, + /// 0-49: Not production-ready + #[default] + F, + /// Gateway failure: Falsifiability requirements not met + GatewayFail, +} + +impl CudaTdgGrade { + /// Convert score to grade + #[must_use] + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "score_range")] + pub fn from_score(score: f64, gateway_passed: bool) -> Self { + if !gateway_passed { + return Self::GatewayFail; + } + match score { + s if s >= 90.0 => Self::APlus, + s if s >= 80.0 => Self::A, + s if s >= 70.0 => Self::B, + s if s >= 60.0 => Self::C, + s if s >= 50.0 => Self::D, + _ => Self::F, + } + } +} + +impl std::fmt::Display for CudaTdgGrade { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::APlus => write!(f, "A+"), + Self::A => write!(f, "A"), + Self::B => write!(f, "B"), + Self::C => write!(f, "C"), + Self::D => write!(f, "D"), + Self::F => write!(f, "F"), + Self::GatewayFail => write!(f, "FAIL (Gateway)"), + } + } +} diff --git a/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/tdg/grade.rs.txt b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/tdg/grade.rs.txt new file mode 100644 index 0000000000..c9c8654b93 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/fixtures/pvl/obligations/pmat/src/tdg/grade.rs.txt @@ -0,0 +1,502 @@ +#![cfg_attr(coverage_nightly, coverage(off))] + +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; + +/// Grade. +/// +/// One serialization, everywhere: the symbolic form (`"A+"`, `"B-"`), which is +/// what `Display`, SARIF, `pmat tdg --format json` and the `--min-grade` +/// argument parser already used. `Serialize` was left DERIVED, so the Rust +/// variant names leaked onto every serde-rendered surface and the SAME binary +/// reported the SAME score as `"grade": "A+"` from `pmat tdg --format json` +/// and `"grade": "APlus"` from `pmat analyze tdg --format json` and from the +/// MCP `quality_gate` tool — no machine consumer could match on one string +/// (GH #703, #669). `Serialize` is therefore written by hand below and emits +/// exactly what `Display` does. The old variant-name spellings are kept as +/// deserialization aliases so stored baselines still load. +// `Serialize`/`Deserialize` are both implemented by hand below (deserialization +// accepts the wire spelling AND the historical variant names), so neither may +// also be derived -- two fix agents each solved this and the merge kept both, +// which is a conflicting-impl compile error. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Grade { + APlus, + A, + AMinus, + BPlus, + B, + BMinus, + CPlus, + #[default] + C, + CMinus, + D, + F, +} + +/// Grade spellings as `Serialize` emits them, for error messages. +/// CANONICAL, and the only list of grade spellings anything may enumerate. +/// Ordered worst-last, matching `Grade`'s own `Ord`. CB-200 kept a private +/// `["A","B","C","D","F"]` and was blind to every modified grade for a release. +pub(crate) const GRADE_VARIANTS: &[&str] = + &["A+", "A", "A-", "B+", "B", "B-", "C+", "C", "C-", "D", "F"]; + +impl Serialize for Grade { + /// Emits the symbolic form -- byte-identical to `Display`. See the note on + /// `Grade`: a derived `Serialize` here is what made one binary print two + /// spellings of one grade. + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.collect_str(self) + } +} + +impl<'de> Deserialize<'de> for Grade { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let name = String::deserialize(deserializer)?; + Grade::from_variant_name(&name) + .ok_or_else(|| serde::de::Error::unknown_variant(&name, GRADE_VARIANTS)) + } +} + +impl Grade { + /// Parse a serialized variant name. Case-insensitive on purpose: see the + /// note on `Grade::APlus`. + pub(crate) fn from_variant_name(name: &str) -> Option { + // Accepts BOTH spellings. `Serialize` emits the variant name (what + // every stored baseline contains), but symbolic forms reach this from + // user input and from output written while the two round-3 fixes + // disagreed about the wire format. One serialization, two accepted + // inputs -- which is what #669's "two spellings" finding asked for. + Some(match name.to_ascii_lowercase().as_str() { + "aplus" | "a+" => Grade::APlus, + "a" => Grade::A, + "aminus" | "a-" => Grade::AMinus, + "bplus" | "b+" => Grade::BPlus, + "b" => Grade::B, + "bminus" | "b-" => Grade::BMinus, + "cplus" | "c+" => Grade::CPlus, + "c" => Grade::C, + "cminus" | "c-" => Grade::CMinus, + "d" => Grade::D, + "f" => Grade::F, + _ => return None, + }) + } + + /// Returns `true` if this grade is at least as good as `threshold`. + /// + /// `Grade`'s derived `Ord` follows declaration order (`APlus` first, + /// `F` last), so BETTER grades compare as SMALLER. Threshold checks + /// must use this helper instead of a raw `>=`/`<=` so call sites never + /// have to reason about that inversion (see v3.18.2 quality_gate fix). + #[must_use] + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "tdg_grade_monotonic")] + pub fn meets_threshold(self, threshold: Grade) -> bool { + // Smaller discriminant == better grade, so "at least as good" is `<=`. + self <= threshold + } + + /// The ONE score → grade mapping. + /// + /// Every grade pmat prints — per file, per project, in every renderer — + /// must come through here, and it must depend on nothing but the score. + /// + /// v3.29.0 had a second, stricter mapping layered on top of this one: the + /// CB-1400 "no provable contracts ⇒ cap at A-" override in + /// `TdgScore::calculate_total`. Because contract coverage is unmeasured for + /// any project without a `contracts/binding.yaml` — the flag simply keeps + /// its `false` default — that override fired unconditionally, and after + /// #680 unified the aggregate path onto it the entire top of the scale went + /// dead: a fixture scoring a perfect **100.0 reported `AMinus`**, and + /// `pmat tdg` printed the self-contradicting line + /// `Overall Score: 100.0/100 (A-)`. `APlus` and `A` were unreachable at any + /// score. The override is gone; contract coverage is still reported on + /// `TdgScore::has_contract_coverage` and still enforced by + /// `pmat comply` (CB-1400), where an unmeasured signal cannot silently + /// rewrite a measurement. + #[must_use] + #[provable_contracts_macros::contract("pmat-core.yaml", equation = "score_range")] + pub fn from_score(score: f32) -> Self { + GRADE_BANDS + .iter() + .find(|(floor, _)| score >= *floor) + .map_or(Grade::F, |(_, grade)| *grade) + } + + /// The half-open score band this grade covers, `[floor, ceiling)` — the top + /// grade's band is closed at 100.0. + /// + /// Exists so nothing has to restate the bands in prose. `pmat explain` + /// used to carry a hand-written five-grade table (A = "Score 85-94") that + /// contradicted this one (A = 90..95, and A-/B+/B-/C+/C-/D were not listed + /// at all), so `explain TDG-A-` answered "No checks matching 'TDG-A-'" for a + /// grade `pmat tdg` prints routinely. + #[must_use] + pub fn score_band(self) -> (f32, f32) { + match GRADE_BANDS.iter().position(|(_, grade)| *grade == self) { + // `F` is everything below the last floor. + None => (0.0, GRADE_BANDS[GRADE_BANDS.len() - 1].0), + Some(0) => (GRADE_BANDS[0].0, 100.0), + Some(i) => (GRADE_BANDS[i].0, GRADE_BANDS[i - 1].0), + } + } + + /// Every grade, best first — the population `pmat explain` must document. + #[must_use] + pub fn all() -> [Grade; 11] { + [ + Grade::APlus, + Grade::A, + Grade::AMinus, + Grade::BPlus, + Grade::B, + Grade::BMinus, + Grade::CPlus, + Grade::C, + Grade::CMinus, + Grade::D, + Grade::F, + ] + } +} + +/// Score floors, best grade first. `F` is everything below the last floor. +/// +/// A table rather than a chain of guard arms: the bands are then contiguous +/// and monotonic by construction (each band starts where the next-worse one +/// ends), and the eleven guards no longer push `from_score` to a cognitive +/// complexity of 32 against a ceiling of 25. +const GRADE_BANDS: [(f32, Grade); 10] = [ + (95.0, Grade::APlus), + (90.0, Grade::A), + (85.0, Grade::AMinus), + (80.0, Grade::BPlus), + (75.0, Grade::B), + (70.0, Grade::BMinus), + (65.0, Grade::CPlus), + (60.0, Grade::C), + (55.0, Grade::CMinus), + (50.0, Grade::D), +]; + +impl std::fmt::Display for Grade { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Grade::APlus => write!(f, "A+"), + Grade::A => write!(f, "A"), + Grade::AMinus => write!(f, "A-"), + Grade::BPlus => write!(f, "B+"), + Grade::B => write!(f, "B"), + Grade::BMinus => write!(f, "B-"), + Grade::CPlus => write!(f, "C+"), + Grade::C => write!(f, "C"), + Grade::CMinus => write!(f, "C-"), + Grade::D => write!(f, "D"), + Grade::F => write!(f, "F"), + } + } +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)] +/// Category classification for metric. +pub enum MetricCategory { + StructuralComplexity, + SemanticComplexity, + Duplication, + Coupling, + Documentation, + Consistency, + /// Not a scored component — the source of the critical-defect penalty + /// `TdgScore::calculate_total` applies on top of the components. + /// + /// The penalty was the single largest term in the score (up to ~91 points) + /// and appeared in `penalties_applied` nowhere at all: a file could read + /// `total: 25.16, grade: F, critical_defects_count: 3` with + /// `penalties_applied: ["Duplication"]` worth 9 points, so any consumer + /// reconstructing the grade from components-minus-penalties was wrong by the + /// whole difference and concluded the defects had cost nothing. A penalty + /// that moves the score must be attributable like every other penalty. + CriticalDefect, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +/// Penalty attribution. +pub struct PenaltyAttribution { + pub source_metric: MetricCategory, + pub amount: f32, + pub applied_to: HashSet, + pub issue: String, +} + +#[cfg(test)] +mod tests { + use super::*; + + /// All grades from best to worst (declaration order). + const BEST_TO_WORST: [Grade; 11] = [ + Grade::APlus, + Grade::A, + Grade::AMinus, + Grade::BPlus, + Grade::B, + Grade::BMinus, + Grade::CPlus, + Grade::C, + Grade::CMinus, + Grade::D, + Grade::F, + ]; + + #[test] + fn test_derived_ord_makes_better_grades_smaller() { + // Documents the inversion that meets_threshold exists to hide: + // declaration order means APlus < F under the derived Ord. + assert!(Grade::APlus < Grade::F); + assert!(Grade::AMinus < Grade::BPlus); + } + + #[test] + fn test_meets_threshold_better_grade_passes() { + // A- is better than B+, so it must meet a B+ threshold + assert!(Grade::AMinus.meets_threshold(Grade::BPlus)); + // Best grade meets every threshold + for threshold in BEST_TO_WORST { + assert!(Grade::APlus.meets_threshold(threshold)); + } + } + + #[test] + fn test_meets_threshold_worse_grade_fails() { + // C is worse than B+, so it must NOT meet a B+ threshold + assert!(!Grade::C.meets_threshold(Grade::BPlus)); + // Worst grade only meets the F threshold + for threshold in &BEST_TO_WORST[..BEST_TO_WORST.len() - 1] { + assert!(!Grade::F.meets_threshold(*threshold)); + } + assert!(Grade::F.meets_threshold(Grade::F)); + } + + /// Regression (GH #703, #669): ONE serialization for one grade. + /// `pmat tdg --format json` and `pmat analyze tdg --format json` disagreed + /// ("A+" vs "APlus") on the identical score, so no machine consumer could + /// match on a string. + /// + /// The wire form is the SYMBOLIC one -- what `Display`, SARIF, + /// `pmat tdg --format json` and `--min-grade` all already used. Only the + /// derived `Serialize` spoke variant names. `Deserialize` still accepts + /// both, so every baseline already on disk keeps its meaning. + #[test] + fn test_grade_has_exactly_one_json_form_and_accepts_both() { + for grade in BEST_TO_WORST { + let json = serde_json::to_string(&grade).expect("serialize"); + assert_eq!( + json, + format!("\"{grade}\""), + "JSON form must be the symbolic form Display prints" + ); + + let back: Grade = serde_json::from_str(&json).expect("round trip"); + assert_eq!(back, grade); + + // The historical variant-name form must still parse. + let variant = format!("\"{grade:?}\""); + let from_variant: Grade = + serde_json::from_str(&variant).expect("variant-name form must load"); + assert_eq!(from_variant, grade); + } + } + + /// Baselines written with the old variant names must still load. + #[test] + fn test_grade_deserializes_legacy_variant_names() { + let legacy = [ + ("\"APLus\"", Grade::APlus), + ("\"APlus\"", Grade::APlus), + ("\"AMinus\"", Grade::AMinus), + ("\"BPlus\"", Grade::BPlus), + ("\"BMinus\"", Grade::BMinus), + ("\"CPlus\"", Grade::CPlus), + ("\"CMinus\"", Grade::CMinus), + ]; + for (json, expected) in legacy { + let parsed: Grade = serde_json::from_str(json).expect("legacy grade must deserialize"); + assert_eq!(parsed, expected, "legacy form {json}"); + } + // ... and so must the new symbolic form, including as a map key. + let map: std::collections::BTreeMap = + serde_json::from_str("{\"A+\":2,\"B-\":1,\"APLus\":3}").expect("map of grades"); + assert_eq!(map.get(&Grade::APlus), Some(&3)); + assert_eq!(map.get(&Grade::BMinus), Some(&1)); + } + + #[test] + fn test_meets_threshold_exhaustive_direction() { + // For every pair: grade meets threshold iff it sits at or before + // the threshold in best-to-worst order. + for (gi, grade) in BEST_TO_WORST.iter().enumerate() { + for (ti, threshold) in BEST_TO_WORST.iter().enumerate() { + assert_eq!( + grade.meets_threshold(*threshold), + gi <= ti, + "{grade} vs threshold {threshold}" + ); + } + } + } + + /// GH #680 (second round): every grade must be reachable from some score. + /// + /// The v3.29.0 binary could not print `APlus` or `A` at all — the CB-1400 + /// cap rewrote both to `AMinus`, so the observed grade for a perfect 100.0 + /// was `AMinus`. A sweep of the whole scale must hit all 11 bands. + #[test] + fn test_every_grade_is_reachable_from_some_score() { + let mut seen = std::collections::HashSet::new(); + for tenth in 0..=1000 { + #[allow(clippy::cast_precision_loss)] + seen.insert(Grade::from_score(tenth as f32 / 10.0)); + } + for grade in BEST_TO_WORST { + assert!( + seen.contains(&grade), + "no score in 0.0..=100.0 maps to {grade}; the scale has a dead band" + ); + } + } + + /// A perfect score must take the top grade, not a capped one. + #[test] + fn test_perfect_score_is_the_top_grade() { + assert_eq!(Grade::from_score(100.0), Grade::APlus); + assert_eq!(Grade::from_score(95.0), Grade::APlus); + } + + /// Bands are contiguous and monotonic: sweeping the score upward may only + /// ever improve the grade, and never skips a band. + #[test] + fn test_bands_are_contiguous_and_monotonic() { + let mut previous = Grade::from_score(0.0); + assert_eq!(previous, Grade::F); + let mut transitions = 0; + for tenth in 0..=1000 { + #[allow(clippy::cast_precision_loss)] + let grade = Grade::from_score(tenth as f32 / 10.0); + if grade != previous { + // Better grades have smaller discriminants, so an improving + // score must step to exactly the next-better variant. + let prev_idx = BEST_TO_WORST.iter().position(|g| *g == previous).unwrap(); + let idx = BEST_TO_WORST.iter().position(|g| *g == grade).unwrap(); + assert_eq!( + idx + 1, + prev_idx, + "score {} jumped from {previous} to {grade}", + f64::from(tenth) / 10.0 + ); + transitions += 1; + previous = grade; + } + } + assert_eq!( + transitions, + BEST_TO_WORST.len() - 1, + "expected one transition per band boundary" + ); + } + + /// GH #680 (second round): the enum was misspelled `APlus`. What pmat + /// emits from now on is `"A+"`; what it accepts still includes the variant + /// names, so baselines written by <= v3.29.0 keep loading. + #[test] + fn test_a_plus_spelling_serialises_correctly_and_accepts_the_old_typo() { + assert_eq!(serde_json::to_string(&Grade::APlus).unwrap(), "\"A+\""); + assert_eq!(format!("{:?}", Grade::APlus), "APlus"); + // The old spelling is written out of two `char`s so this test file + // does not reintroduce the literal the fix removed. + let old = format!("\"AP{}us\"", 'L'); + let from_old: Grade = serde_json::from_str(&old).expect("old spelling must load"); + assert_eq!(from_old, Grade::APlus); + let from_new: Grade = serde_json::from_str("\"APlus\"").expect("new spelling must load"); + assert_eq!(from_new, Grade::APlus); + } + + /// Round-trip every variant, including as a `HashMap` key — `ProjectScore` + /// serialises `grade_distribution: HashMap`, and a hand-written + /// `Deserialize` has to keep working in map-key position. + #[test] + fn test_grade_round_trips_as_value_and_as_map_key() { + for grade in BEST_TO_WORST { + let json = serde_json::to_string(&grade).unwrap(); + let back: Grade = serde_json::from_str(&json).unwrap(); + assert_eq!(back, grade); + + let map: std::collections::HashMap = + std::collections::HashMap::from([(grade, 3)]); + let encoded = serde_json::to_string(&map).unwrap(); + let decoded: std::collections::HashMap = + serde_json::from_str(&encoded).unwrap(); + assert_eq!(decoded.get(&grade), Some(&3)); + } + } + + /// `score_band` must agree with `from_score` at every boundary, or the + /// documentation generated from it would restate the same contradiction + /// `pmat explain` used to carry. + #[test] + fn test_score_band_agrees_with_from_score() { + for grade in Grade::all() { + let (floor, ceiling) = grade.score_band(); + assert_eq!( + Grade::from_score(floor), + grade, + "{grade}'s floor {floor} does not map back to {grade}" + ); + // Just under the ceiling is still this grade; at the ceiling it is + // the next better one (except for the top band, closed at 100). + assert_eq!(Grade::from_score(ceiling - 0.1), grade, "{grade} ceiling"); + if grade != Grade::APlus { + assert_ne!(Grade::from_score(ceiling), grade, "{grade} ceiling is open"); + } + } + assert_eq!(Grade::APlus.score_band(), (95.0, 100.0)); + assert_eq!(Grade::F.score_band(), (0.0, 50.0)); + } + + #[test] + fn test_unknown_variant_name_is_rejected() { + let err = serde_json::from_str::("\"Z\"").unwrap_err(); + assert!(err.to_string().contains("unknown variant"), "{err}"); + } +} + +/// Every canonical spelling parses, and `GRADE_VARIANTS` is in `Ord` order. +/// +/// A TEST, not a `const` block: `from_variant_name` lowercases, and +/// `str::to_ascii_lowercase` is not a `const fn`, so this cannot run in a const +/// context. It runs under `cargo test --lib`, which is the rung this repository +/// actually executes. The anchoring property — that rank tracks the score band +/// — is proved in `contracts/lean/Theorems/Tdg/Grade.lean`, because no +/// self-referential assertion can catch a scale reversed together with its own +/// array. +#[test] +fn grade_order_is_parseable() { + let parsed: Vec = GRADE_VARIANTS + .iter() + .map(|s| Grade::from_variant_name(s).expect("every canonical spelling must parse")) + .collect(); + let mut sorted = parsed.clone(); + sorted.sort(); + assert_eq!(parsed, sorted, "GRADE_VARIANTS is not in Ord order"); + assert_eq!( + parsed.len(), + Grade::all().len(), + "GRADE_VARIANTS and Grade::all() disagree" + ); +} +const GRADE_ORDER_IS_PARSEABLE: () = (); diff --git a/crates/aprender-contracts-cli/tests/includes/dispatch_tests.rs b/crates/aprender-contracts-cli/tests/includes/dispatch_tests.rs index 88b0289f1d..580725950a 100644 --- a/crates/aprender-contracts-cli/tests/includes/dispatch_tests.rs +++ b/crates/aprender-contracts-cli/tests/includes/dispatch_tests.rs @@ -141,7 +141,13 @@ fn dispatch_proof_status_with_binding() { kind: None, verify_bindings: None, }); - assert!(result.is_ok()); + // PVL-001 EV-2: `--binding` RESOLVES every binding. This registry names five + // functions that exist nowhere in the tree (swap_axes, validate_element_count, + // map_tensor_name, bidirectional_attention, mint_test_token; #4094), so the + // honest answer is a reject. This test asserted `is_ok()` while proof-status + // counted entries without resolving them: it encoded the defect. + let err = result.expect_err("a registry holding ghost bindings must be rejected"); + assert!(err.to_string().contains("ghost binding"), "{err}"); } #[test] @@ -454,3 +460,24 @@ fn dispatch_pipeline_json() { }); assert!(result.is_ok()); } + +/// G1.3 (#4476): `pv lint --format` is an enumerated clap value, so a typo is a +/// usage error (clap exits 2 and lists the values), not a run that silently +/// prints text. Every value `print_report` handles still parses. +#[test] +fn g1_3_lint_format_is_a_clap_value_parser() { + let err = match Cli::try_parse_from(["pv", "lint", "contracts", "--format", "nosuch"]) { + Ok(_) => panic!("--format nosuch parsed"), + Err(e) => e, + }; + assert_eq!(err.kind(), clap::error::ErrorKind::InvalidValue); + assert_eq!(err.exit_code(), 2); + let msg = err.to_string(); + for v in ["text", "json", "sarif", "github", "html"] { + assert!(msg.contains(v), "error must name `{v}`:\n{msg}"); + assert!( + Cli::try_parse_from(["pv", "lint", "contracts", "--format", v]).is_ok(), + "--format {v} must parse" + ); + } +} diff --git a/crates/aprender-contracts-cli/tests/includes/version_identity_unit.rs b/crates/aprender-contracts-cli/tests/includes/version_identity_unit.rs index 7bd901dccc..8ee4d248ee 100644 --- a/crates/aprender-contracts-cli/tests/includes/version_identity_unit.rs +++ b/crates/aprender-contracts-cli/tests/includes/version_identity_unit.rs @@ -102,3 +102,28 @@ fn semver_stays_the_second_field_of_the_first_line() { fn short_and_long_agree_on_the_first_line() { assert_eq!(long().lines().next(), short().lines().next()); } + +/// G0.1 (#4476): the version names the commit it was built from, as the third +/// field `(<9 hex>)`. Two worktrees share one workspace semver, so without the +/// sha a stale pv reads as HEAD. `v+no-git` is the only other allowed value, and +/// it never passes for a sha. +#[test] +fn g0_1_the_version_names_the_commit() { + let first = long().lines().next().unwrap_or_default().to_string(); + let sha = env!("APR_GIT_SHA"); + assert!( + first.contains(&format!("({sha})")), + "pv --version must carry the build sha; got `{first}`" + ); + // git's --short grows with the repo (9 hex until 2026-09, 10 after), so accept any + // abbreviation git can print (7..=40 hex), never one fixed length. + let short_sha = (7..=40).contains(&sha.len()) && sha.chars().all(|c| c.is_ascii_hexdigit()); + assert!( + short_sha || sha.ends_with("+no-git"), + "APR_GIT_SHA must be 7-40 hex or v+no-git; got `{sha}`" + ); + assert_eq!( + first.split_whitespace().nth(2), + Some(format!("({sha})").as_str()) + ); +} diff --git a/crates/aprender-contracts-cli/tests/ont2c_owl_tbox.rs b/crates/aprender-contracts-cli/tests/ont2c_owl_tbox.rs new file mode 100644 index 0000000000..0d55ff9955 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/ont2c_owl_tbox.rs @@ -0,0 +1,196 @@ +//! ONT-2c (PMAT-4071, aprender#4071) — `pv ontology export --owl`, `pv ontology tbox`, `pv lint --gate tbox`, +//! driven through the BUILT binary. The lib tests prove the writer and the told-closure. This file proves the +//! CLI surface the row's probe calls, and the one mapping the row names at the dispatch layer: the gate maps +//! the advisory report to `Unknown{Advisory}` (`decline: Advisory`, exit 2) and has NO arm that exits 0 (R-7). +//! +//! DISCRIMINATION: a fresh corpus declines with exit 2, and a stale or missing artifact is exit 3. A build +//! that mapped Advisory to a pass (exit 0) fails `the_repo_corpus_is_advisory_never_zero`. One that stopped +//! checking freshness fails `a_stale_report_is_refused_with_exit_3`. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn pv_bin() -> PathBuf { + PathBuf::from(env!("CARGO_BIN_EXE_pv")) +} + +struct Run { + code: i32, + stdout: String, + stderr: String, +} + +fn pv(args: &[&str]) -> Run { + let scratch = tempfile::tempdir().expect("scratch cwd is creatable"); + let out = Command::new(pv_bin()) + .current_dir(scratch.path()) + .args(args) + .output() + .expect("failed to spawn pv"); + Run { + code: out.status.code().unwrap_or(-1), + stdout: String::from_utf8_lossy(&out.stdout).into_owned(), + stderr: String::from_utf8_lossy(&out.stderr).into_owned(), + } +} + +fn show(r: &Run) -> String { + format!( + "exit {}\n--- stdout\n{}\n--- stderr\n{}", + r.code, r.stdout, r.stderr + ) +} + +fn repo(rel: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .join(rel) +} + +fn s(p: &Path) -> String { + p.to_str().expect("utf-8 path").to_string() +} + +/// A corpus directory holding the fixture Σ plus the two artifacts `pv ontology … --write` produces. +fn fresh_fixture_corpus() -> tempfile::TempDir { + let d = tempfile::tempdir().expect("tempdir"); + std::fs::copy( + repo("tests/fixtures/ont/owl/ontology.yaml"), + d.path().join("ontology.yaml"), + ) + .expect("copy Σ"); + // `pv lint` declines a corpus with 0 contracts before any gate runs; one minimal contract makes it a corpus. + std::fs::copy( + repo("tests/fixtures/ont/sigma-ok/fixture-kernel-v1.yaml"), + d.path().join("fixture-kernel-v1.yaml"), + ) + .expect("copy contract"); + let sigma = s(&d.path().join("ontology.yaml")); + let r = pv(&["ontology", "export", "--owl", "--write", &sigma]); + assert_eq!(r.code, 0, "{}", show(&r)); + let r = pv(&["ontology", "tbox", "--write", &sigma]); + assert_eq!(r.code, 0, "{}", show(&r)); + d +} + +#[test] +fn export_prints_exactly_the_committed_fixture_axiom_set() { + let r = pv(&[ + "ontology", + "export", + "--owl", + &s(&repo("tests/fixtures/ont/owl/ontology.yaml")), + ]); + assert_eq!(r.code, 0, "{}", show(&r)); + let want = + std::fs::read_to_string(repo("tests/fixtures/ont/owl/expected.ofn")).expect("expected.ofn"); + assert_eq!(r.stdout, want); +} + +#[test] +fn export_without_a_format_is_refused() { + let r = pv(&[ + "ontology", + "export", + &s(&repo("tests/fixtures/ont/owl/ontology.yaml")), + ]); + assert_eq!(r.code, 2, "{}", show(&r)); +} + +#[test] +fn the_repo_export_equals_the_tracked_ofn() { + // The row's probe: `"$PV" ontology export --owl contracts/ontology.yaml | cmp - contracts/ontology.ofn`. + let r = pv(&[ + "ontology", + "export", + "--owl", + &s(&repo("contracts/ontology.yaml")), + ]); + assert_eq!(r.code, 0, "{}", show(&r)); + let tracked = + std::fs::read_to_string(repo("contracts/ontology.ofn")).expect("contracts/ontology.ofn"); + assert_eq!( + r.stdout, tracked, + "contracts/ontology.ofn is not what the writer produces" + ); +} + +#[test] +fn write_writes_both_artifacts_next_to_sigma() { + let d = fresh_fixture_corpus(); + let ofn = + std::fs::read_to_string(d.path().join("ontology.ofn")).expect("--write wrote ontology.ofn"); + assert!(ofn.contains("SymmetricObjectProperty("), "{ofn}"); + let rep: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(d.path().join("tbox-report.json")).expect("report"), + ) + .expect("report is JSON"); + assert_eq!(rep["advisory"], true); + assert_eq!(rep["method"], "told-closure"); + assert_eq!(rep["consistent"], true); +} + +#[test] +fn the_repo_corpus_is_advisory_never_zero() { + let r = pv(&["lint", &s(&repo("contracts")), "--gate", "tbox"]); + assert_eq!( + r.code, + 2, + "the tbox gate must DECLINE (Unknown{{Advisory}}), never pass: {}", + show(&r) + ); + assert!( + r.stderr.contains("decline: Advisory") || r.stdout.contains("decline: Advisory"), + "{}", + show(&r) + ); +} + +#[test] +fn a_fresh_fixture_corpus_is_advisory() { + let d = fresh_fixture_corpus(); + let r = pv(&["lint", &s(d.path()), "--gate", "tbox"]); + assert_eq!(r.code, 2, "{}", show(&r)); + assert!( + format!("{}{}", r.stdout, r.stderr).contains("Advisory"), + "{}", + show(&r) + ); +} + +#[test] +fn a_stale_report_is_refused_with_exit_3() { + let d = fresh_fixture_corpus(); + let p = d.path().join("tbox-report.json"); + let body = std::fs::read_to_string(&p).expect("report"); + std::fs::write(&p, body.replace("\"classes\": 3", "\"classes\": 9")).expect("tamper"); + let r = pv(&["lint", &s(d.path()), "--gate", "tbox"]); + assert_eq!(r.code, 3, "{}", show(&r)); + assert!(r.stderr.contains("tbox-report.json"), "{}", show(&r)); +} + +#[test] +fn a_missing_ofn_is_refused_with_exit_3() { + let d = fresh_fixture_corpus(); + std::fs::remove_file(d.path().join("ontology.ofn")).expect("rm"); + let r = pv(&["lint", &s(d.path()), "--gate", "tbox"]); + assert_eq!(r.code, 3, "{}", show(&r)); + assert!(r.stderr.contains("ontology.ofn"), "{}", show(&r)); +} + +#[test] +fn an_undeclared_unexpressed_key_is_exit_3() { + let d = tempfile::tempdir().expect("tempdir"); + let sigma = std::fs::read_to_string(repo("tests/fixtures/ont/owl/ontology.yaml")).expect("Σ"); + let broken = sigma.replace(" - {key: symbols, reader: ontology/owl.rs}\n", ""); + assert_ne!(broken, sigma, "the mutation must change Σ"); + std::fs::write(d.path().join("ontology.yaml"), broken).expect("write"); + let r = pv(&[ + "ontology", + "export", + "--owl", + &s(&d.path().join("ontology.yaml")), + ]); + assert_eq!(r.code, 3, "{}", show(&r)); + assert!(r.stderr.contains("symbols"), "{}", show(&r)); +} diff --git a/crates/aprender-contracts-cli/tests/ont4b_shapes_gate.rs b/crates/aprender-contracts-cli/tests/ont4b_shapes_gate.rs index 1057520206..b71dddc7cd 100644 --- a/crates/aprender-contracts-cli/tests/ont4b_shapes_gate.rs +++ b/crates/aprender-contracts-cli/tests/ont4b_shapes_gate.rs @@ -88,11 +88,14 @@ fn the_repo_corpus_passes_with_the_plant_fired_and_the_whole_corpus_as_focus_nod ); assert_eq!(v["extra"]["pc_shape"], "fired", "{}", show(&r)); // ONT-4b: exactly one, from ont:id minCount. ONT-4c1 plants a bare model:Model too, which draws - // ladder-measured's two minCounts: three on this corpus, and never zero. + // ladder-measured's two minCounts: three on this corpus, and never zero. ONT-4f (#4330) arms the four GitHub + // shapes, and the plant draws every minCount they declare: github-repo 4 + github-issue 6 + + // github-pull-request 7 + github-milestone 6 = 23 more, so 26. ONT-4c (#3847) arms readme-root, claude-md, + // model-setfit-slice and csv-train, whose plants add sixteen: 42. assert_eq!( v["extra"]["plant_violations"], - 3, - "ont:id minCount + ladder-measured's two (ONT-4c1)\n{}", + 42, + "ont:id minCount + ladder-measured's two (ONT-4c1) + the four GitHub shapes' 23 (ONT-4f) + the ONT-4c plants against readme-root, claude-md, model-setfit-slice, csv-train (16)\n{}", show(&r) ); } @@ -230,11 +233,20 @@ fn the_tracked_repo_graph_is_fresh() { // that is the ratchet working, not a conflict to route around. It did: the // 0.69 batch folded #3600 in and the count went 6 -> 9 with its three shapes. #3715 added the nine-shape // `release-readiness-v1` family (shapes_n=18, triples=15863, measured on its branch); it contributes no focus - // node to a PR's graph — the release evidence is extracted only under `--release-*`. + // node to a PR's graph — the release evidence is extracted only under `--release-*`. ONT-4f (#4330) adds + // `github-entities-v1`'s four armed shapes: 22. #3560 R1 adds `examples-well-formed` (reported, not armed): 23. R4 adds `examples-model-current` (reported): 24. + // ONT-4c (#3847) adds the four armed non-code shapes readme-root, claude-md, model-setfit-slice, csv-train: 28. + // ONT-4g/ONT-10 (#4502) add binary-target, surface-audit-ledger, binary-apr-http-mcp and fifteen + // binary--surface shapes (apr-cli-apr, apr-corpus-ingest, aprender-cbtop, aprender-compute-xtask, aprender-db, + // aprender-explain, aprender-orchestrate-aprender-orchestrate, aprender-profile, aprender-ptx-debug, aprender-test-cli, aprender-zram-generator, + // apr-qa-readme-sync, pv, score, trueno-rag): 46. Each new ONT-10 slice raises this — name it here. + // The 0.70.0 car (#4429) added, disjoint from these: #3745 S2 (#3777) six DERIVED cell-class shapes (effect, + // probe, model-cell, effect-cell, crux-verb, crux) and a8's `sampling`, ONT-4c5 (PMAT-3972) `capability-cells`, + // and ONT-4c4 (#4069) kernel-parity, kernel-timing and kernel-safety: 46 + 8 + 3 = 57. assert_eq!( v["shapes_n"], - 18, - "ont-shapes-v1 + ladder-measured + ladder-green (ONT-4c1) + bound-symbols-resolve + lean-statements-grounded (ONT-4b2) + refusal-receipt-v1 (#3605) + parity-receipt-complete + parity-comparator-self + parity-comparator-oracle (parity-receipt-v2, #3600) + release-readiness-v1{{,.release,.host,.context,.model,.coverage,.tokenizer,.kernel,.refusal}} (#3715)\n{}", + 57, + "ont-shapes-v1 + ladder-measured + ladder-green (ONT-4c1) + bound-symbols-resolve + lean-statements-grounded (ONT-4b2) + refusal-receipt-v1 (#3605) + parity-receipt-complete + parity-comparator-self + parity-comparator-oracle (parity-receipt-v2, #3600) + release-readiness-v1{{,.release,.host,.context,.model,.coverage,.tokenizer,.kernel,.refusal}} (#3715) + github-{{repo,issue,pull-request,milestone}} (#4330) + examples-well-formed (#3560 R1) + examples-model-current (#3560 R4) + readme-root + claude-md + model-setfit-slice + csv-train (ONT-4c, #3847) + binary-target + surface-audit-ledger + binary-apr-http-mcp + 15 binary-*-surface (ONT-10, #4502) + release-readiness-v1{{.effect,.probe,.model-cell,.effect-cell,.crux-verb,.crux,.sampling}} (#3745 S2) + capability-cells (ONT-4c5) + kernel-parity + kernel-timing + kernel-safety (ONT-4c4)\n{}", show(&r) ); } diff --git a/crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs b/crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs index c2452e7955..856a672b9b 100644 --- a/crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs +++ b/crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs @@ -87,9 +87,18 @@ fn a_complete_receipt_passes_with_one_focus_node() { assert_eq!(r.code, 0, "{}", r.all()); assert_eq!(violations(&r), 0, "{}", r.all()); assert_eq!(focus_nodes(&r), 1, "{}", r.all()); - // The shapes are ARMED in the fixture (no lint-baseline.json narrows them), so a violation here would be - // a real Fail rather than a reported one — which is what makes the two FAIL cases below meaningful. + // The two shapes this receipt reaches are ARMED (the fixture's lint-baseline.json), so a violation here + // would be a real Fail rather than a reported one — which is what makes the two FAIL cases below + // meaningful. `parity-comparator-oracle` is left UNARMED on purpose: a self-comparator receipt gives it + // zero focus nodes, and an ARMED shape that grades nothing declines (#3610) rather than passing. assert!(r.stdout.contains("parity-receipt-complete"), "{}", r.all()); + let v: serde_json::Value = serde_json::from_str(&r.stdout).expect("json report"); + assert_eq!( + names(&v["extra"]["declines"]), + vec!["parity-comparator-oracle"] as Vec<&str>, + "the unarmed vacuity is still NAMED\n{}", + r.all() + ); } #[test] @@ -242,3 +251,10 @@ fn the_committed_tree_agrees_with_its_own_denominator() { String::from_utf8_lossy(&out.stderr) ); } + +/// A JSON array of strings as `Vec<&str>` (`json!` array literals expand to a disallowed `unwrap`). +fn names(v: &serde_json::Value) -> Vec<&str> { + v.as_array() + .map(|a| a.iter().filter_map(serde_json::Value::as_str).collect()) + .unwrap_or_default() +} diff --git a/crates/aprender-contracts-cli/tests/ont4c5_capability_cells.rs b/crates/aprender-contracts-cli/tests/ont4c5_capability_cells.rs new file mode 100644 index 0000000000..6af3402cad --- /dev/null +++ b/crates/aprender-contracts-cli/tests/ont4c5_capability_cells.rs @@ -0,0 +1,371 @@ +//! ONT-4c5 (PMAT-3972; aprender #3972 + #4047) — capability cells, on the CLI. +//! +//! ONT-001 v4.12 §5 ONT-4c5 RED, on `pv lint --gate shapes`: a required cell of the model-capability ladder is +//! Pass or Fail at the current release, and `Unknown{NotRun}` — a missing row, DEFER, MANUAL, NO-VERDICT, a label +//! nobody knows — is a violation of the ARMED `capability-cells` shape: exit 1, naming the cell. An empty domain +//! is a decline (exit 2), never Pass. A Fail cell is ADMITTED by this shape (ladder-green is what refuses it). +//! +//! DISCRIMINATION: `capcells-clean/` must PASS with `pc_shapes` fired and the whole domain reported, so a build +//! that declines or rejects every ladder fails this file; each NotRun fixture must FAIL naming exactly its cell, +//! so a build that arms nothing, or folds an absence into Fail, fails it too; `capcells-failadmit/` must PASS, +//! so a build that treats Fail as NotRun fails it. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +struct Run { + code: i32, + stdout: String, + stderr: String, +} + +fn lint(fixture: &str) -> Run { + let contracts = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../tests/fixtures/ont") + .join(fixture) + .join("contracts"); + let scratch = tempfile::tempdir().expect("scratch cwd is creatable"); + let out = Command::new(PathBuf::from(env!("CARGO_BIN_EXE_pv"))) + .current_dir(scratch.path()) + .args([ + "lint", + &contracts.to_string_lossy(), + "--gate", + "shapes", + "--format", + "json", + ]) + .output() + .expect("failed to spawn pv"); + Run { + code: out.status.code().unwrap_or(-1), + stdout: String::from_utf8_lossy(&out.stdout).into_owned(), + stderr: String::from_utf8_lossy(&out.stderr).into_owned(), + } +} + +fn show(r: &Run) -> String { + format!( + "exit {}\n--- stdout\n{}\n--- stderr\n{}", + r.code, r.stdout, r.stderr + ) +} + +fn json(r: &Run) -> serde_json::Value { + serde_json::from_str(&r.stdout).unwrap_or_else(|e| panic!("not JSON ({e}):\n{}", show(r))) +} + +fn strings(v: &serde_json::Value) -> Vec { + v.as_array() + .unwrap_or_else(|| panic!("not an array: {v}")) + .iter() + .map(|x| x.as_str().expect("string").to_string()) + .collect() +} + +const DOMAIN: [&str; 4] = [ + "rung-a@gx10", + "rung-a@lambda", + "rung-b@gx10", + "rung-b@lambda", +]; + +#[test] +fn a_clean_ladder_passes_with_the_plant_fired_and_the_whole_domain_reported() { + let r = lint("capcells-clean"); + assert_eq!(r.code, 0, "{}", show(&r)); + let v = json(&r); + assert_eq!(v["verdict"], "Pass", "{}", show(&r)); + assert_eq!(v["pc_shapes"]["capability-cells"], "fired", "{}", show(&r)); + assert!(strings(&v["armed_shapes"]).contains(&"capability-cells".to_string())); + assert_eq!(strings(&v["capability_cells"]["domain"]), DOMAIN); + assert!(strings(&v["capability_cells"]["not_run"]).is_empty()); + assert_eq!(v["capability_cells"]["v_star"], "0.69.1"); +} + +/// Every NotRun fixture: exit 1, `not_run` is exactly its cell, and a finding names that cell. +#[test] +fn every_not_run_required_cell_rejects_naming_exactly_that_cell() { + for (fixture, cell) in [ + ("capcells-defer", "rung-a@lambda"), + ("capcells-manual", "rung-a@gx10"), + ("capcells-noverdict", "rung-b@lambda"), + ("capcells-missing", "rung-b@gx10"), + ("capcells-badlabel", "rung-a@lambda"), + ] { + let r = lint(fixture); + assert_eq!(r.code, 1, "{fixture}: {}", show(&r)); + let v = json(&r); + assert_eq!(v["verdict"], "Fail", "{fixture}"); + assert_eq!(v["pc_shapes"]["capability-cells"], "fired", "{fixture}"); + assert_eq!( + strings(&v["capability_cells"]["not_run"]), + [cell], + "{fixture}" + ); + let named = v["findings"].as_array().expect("findings").iter().any(|f| { + f["rule_id"] == "PV-ONT-013" + && f["message"] + .as_str() + .is_some_and(|m| m.contains(&format!("capability cell {cell} "))) + }); + assert!( + named, + "{fixture}: no PV-ONT-013 finding names {cell}\n{}", + show(&r) + ); + } +} + +#[test] +fn an_unknown_label_is_refused_by_name() { + let r = lint("capcells-badlabel"); + let v = json(&r); + let refused = strings(&v["capability_cells"]["refused_labels"]); + assert_eq!(refused.len(), 1, "{}", show(&r)); + assert!(refused[0].ends_with(": rung-a: DEFERRED"), "{refused:?}"); +} + +#[test] +fn a_fail_cell_is_admitted_by_this_shape() { + let r = lint("capcells-failadmit"); + assert_eq!(r.code, 0, "{}", show(&r)); + let v = json(&r); + assert_eq!(v["verdict"], "Pass"); + assert!(strings(&v["capability_cells"]["not_run"]).is_empty()); +} + +#[test] +fn an_empty_domain_declines_never_passes() { + let r = lint("capcells-emptydomain"); + assert_eq!(r.code, 2, "{}", show(&r)); + assert!( + r.stderr.contains("capability-cells domain D is empty"), + "{}", + show(&r) + ); + assert!(r.stderr.contains("decline:"), "{}", show(&r)); +} + +// ── The row's own probe on THIS repository, and the row's data mutations (each must turn it RED) ────────────── +// +// The probe is ONT-4c5's jq predicate, restated: verdict Pass, `capability-cells` armed, `pc_shapes` fired, no +// NotRun cell, and every id of the baseline set B in the domain. The mutations run in a scratch tree that is +// symlinks all the way down except on the path to the one file a mutation edits, which is a real copy — the +// checkout is never written. The row's code mutation ("fold a missing cell into Fail") is held by the unit tests +// and the plant, not here. + +/// ONT-4c5's baseline set B, verbatim from the row's probe (paiml/infra paiml-ontology.md v4.12 §5). +const B: [&str; 14] = [ + "qwen2-1.5b-q4km@lambda", + "qwen2-1.5b-q4km@gx10", + "qwen3-1.7b-q4km@lambda", + "qwen3-1.7b-q4km@gx10", + "qwen35-0.8b-q4km@lambda", + "qwen35-0.8b-q4km@gx10", + "qwen35-2b-q4km@lambda", + "qwen35-2b-q4km@gx10", + "qwen35-4b-q4km@lambda", + "qwen35-4b-q4km@gx10", + "qwen35-9b-q4km@lambda", + "qwen35-9b-q4km@gx10", + "qwen35-27b-q4km@lambda", + "qwen35-27b-q4km@gx10", +]; + +fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../..") +} + +fn lint_root(root: &Path) -> Run { + let out = Command::new(PathBuf::from(env!("CARGO_BIN_EXE_pv"))) + .current_dir(root) + .args(["lint", "contracts", "--gate", "shapes", "--format", "json"]) + .output() + .expect("failed to spawn pv"); + Run { + code: out.status.code().unwrap_or(-1), + stdout: String::from_utf8_lossy(&out.stdout).into_owned(), + stderr: String::from_utf8_lossy(&out.stderr).into_owned(), + } +} + +/// The row's probe predicate over one run: `Ok(())` green, `Err(why)` red. +fn probe(r: &Run) -> Result<(), String> { + let v: serde_json::Value = + serde_json::from_str(&r.stdout).map_err(|_| format!("no JSON (exit {})", r.code))?; + let has = |key: &str, id: &str| { + v[key] + .as_array() + .is_some_and(|a| a.iter().any(|x| x.as_str() == Some(id))) + }; + if v["verdict"] != "Pass" { + return Err(format!("verdict {}", v["verdict"])); + } + if !has("armed_shapes", "capability-cells") { + return Err("capability-cells not armed".into()); + } + if v["pc_shapes"]["capability-cells"] != "fired" { + return Err("pc_shapes not fired".into()); + } + let cc = &v["capability_cells"]; + if cc["not_run"].as_array().is_none_or(|a| !a.is_empty()) { + return Err(format!("not_run {}", cc["not_run"])); + } + let domain: Vec<&str> = cc["domain"] + .as_array() + .map(|a| a.iter().filter_map(serde_json::Value::as_str).collect()) + .unwrap_or_default(); + let missing: Vec<&str> = B.iter().copied().filter(|b| !domain.contains(b)).collect(); + if missing.is_empty() { + Ok(()) + } else { + Err(format!("B ⊄ D: {missing:?}")) + } +} + +/// The receipt of `host` at the current release V*, as the gate itself reports V* on this repository — never a +/// pinned version, so the mutations follow the tree to whichever release it has receipts for (the branch was +/// parked for a week because these paths named 0.69.1 and its base had only 0.68.2). +fn current_receipt(host: &str) -> String { + let v = json(&lint_root(&repo_root())); + let v_star = v["capability_cells"]["v_star"] + .as_str() + .unwrap_or_else(|| panic!("no v_star on this repository: {v}")); + format!("evidence/dogfood/models/{v_star}/{host}.json") +} + +/// A scratch copy of the repository in which `rel` (and only it) is a real, writable file. +fn scratch_with(rel: &str) -> (tempfile::TempDir, PathBuf) { + let dir = tempfile::tempdir().expect("scratch"); + let repo = repo_root().canonicalize().expect("repo root"); + let rel = Path::new(rel); + let mut src = repo.clone(); + let mut dst = dir.path().to_path_buf(); + let comps: Vec<_> = rel.components().collect(); + for (i, comp) in comps.iter().enumerate() { + // at this level: symlink every sibling, descend into (or copy) the one on the path + for entry in std::fs::read_dir(&src).expect("read_dir") { + let entry = entry.expect("entry"); + let name = entry.file_name(); + if name == comp.as_os_str() || name == ".git" || name == "target" || name == ".pv" { + continue; + } + std::os::unix::fs::symlink(entry.path(), dst.join(&name)).expect("symlink"); + } + src = src.join(comp); + dst = dst.join(comp); + if i + 1 == comps.len() { + std::fs::copy(&src, &dst).expect("copy the mutated file"); + } else { + std::fs::create_dir(&dst).expect("mkdir on the path"); + } + } + (dir, dst) +} + +fn edit_json(path: &Path, f: impl FnOnce(&mut serde_json::Value)) { + let mut v: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("read")).expect("json"); + f(&mut v); + std::fs::write(path, serde_json::to_string_pretty(&v).expect("ser")).expect("write"); +} + +fn rung_row<'a>(v: &'a mut serde_json::Value, id: &str) -> &'a mut serde_json::Value { + v["rungs"] + .as_array_mut() + .expect("rungs") + .iter_mut() + .find(|r| r["id"] == id) + .unwrap_or_else(|| panic!("mutation anchor missing: {id}")) +} + +#[test] +fn the_rows_probe_is_green_on_this_repository() { + let r = lint_root(&repo_root()); + probe(&r).unwrap_or_else(|why| panic!("ONT-4c5 probe RED on the tree: {why}\n{}", show(&r))); +} + +#[test] +fn disarming_capability_cells_turns_the_probe_red() { + let (dir, f) = scratch_with("contracts/lint-baseline.json"); + edit_json(&f, |v| { + let a = v["armed_shapes"].as_array_mut().expect("armed_shapes"); + let n = a.len(); + a.retain(|x| x != "capability-cells"); + assert_eq!(a.len(), n - 1, "mutation anchor missing"); + }); + let r = lint_root(dir.path()); + assert!(probe(&r).is_err(), "disarmed and still green\n{}", show(&r)); + assert_eq!( + r.code, + 2, + "the plant cannot fire through an unarmed shape\n{}", + show(&r) + ); +} + +#[test] +fn flipping_a_required_rung_to_optional_leaves_b_outside_the_domain() { + let (dir, f) = scratch_with("contracts/model-capability-ladder-v1.yaml"); + let text = std::fs::read_to_string(&f).expect("read"); + let at = text + .find("- id: qwen35-2b-q4km") + .expect("mutation anchor missing"); + let tail = &text[at..]; + let next = tail[1..].find("\n - id: ").map_or(tail.len(), |n| n + 1); + let off = tail[..next] + .find("required: true") + .expect("the rung has its own required: true"); + let mut out = text.clone(); + out.replace_range( + at + off..at + off + "required: true".len(), + "required: false", + ); + std::fs::write(&f, out).expect("write"); + let r = lint_root(dir.path()); + let why = probe(&r).expect_err("an optional rung left B ⊆ D intact"); + assert!(why.contains("qwen35-2b-q4km@lambda"), "{why}"); +} + +#[test] +fn a_not_run_label_on_a_current_release_row_rejects_naming_the_cell() { + for label in ["DEFER", "MANUAL", "NO-VERDICT"] { + let (dir, f) = scratch_with(¤t_receipt("lambda")); + edit_json(&f, |v| { + rung_row(v, "qwen35-4b-q4km")["verdict"] = label.into() + }); + let r = lint_root(dir.path()); + assert_eq!(r.code, 1, "{label}\n{}", show(&r)); + assert!(probe(&r).is_err(), "{label}"); + assert!( + r.stdout.contains("qwen35-4b-q4km@lambda"), + "{label}\n{}", + show(&r) + ); + } +} + +#[test] +fn a_deleted_current_release_row_rejects_naming_the_cell() { + let (dir, f) = scratch_with(¤t_receipt("gx10")); + edit_json(&f, |v| { + let rows = v["rungs"].as_array_mut().expect("rungs"); + let n = rows.len(); + rows.retain(|r| r["id"] != "qwen3-1.7b-q4km"); + assert_eq!(rows.len(), n - 1, "mutation anchor missing"); + }); + let r = lint_root(dir.path()); + assert_eq!(r.code, 1, "{}", show(&r)); + assert!(r.stdout.contains("qwen3-1.7b-q4km@gx10"), "{}", show(&r)); +} + +/// Review lane C: the non-numeral-version refusal must reach the PROCESS exit, not only `compute()`'s `Err`. +#[test] +fn a_receipt_version_that_is_not_dotted_numerals_exits_3_naming_the_file() { + let r = lint("capcells-badversion"); + assert_eq!(r.code, 3, "{}", show(&r)); + assert!(r.stderr.contains("0.69.1-rc1"), "{}", show(&r)); + assert!(r.stderr.contains("lambda.json"), "{}", show(&r)); + assert!(r.stderr.contains("refused by name"), "{}", show(&r)); +} diff --git a/crates/aprender-contracts-cli/tests/ont4c_doc_contracts.rs b/crates/aprender-contracts-cli/tests/ont4c_doc_contracts.rs new file mode 100644 index 0000000000..9345ed5a1a --- /dev/null +++ b/crates/aprender-contracts-cli/tests/ont4c_doc_contracts.rs @@ -0,0 +1,462 @@ +//! ONT-4c (PMAT-3847, #3847) — README.md, CLAUDE.md and a CSV as graded documents, on the CLI. +//! +//! ONT-001 v4.16 §5 ONT-4c RED, on `pv lint --gate shapes`: `tests/fixtures/ont/docs-green/` is a whole repo in +//! miniature (a merge-path workflow, a README and a CLAUDE.md with claim fences, a CSV and its producer, and a +//! baseline carrying the measured sets). Every other case is ONE edit to a copy of it, so each RED is caused by +//! that edit and nothing else: +//! +//! - a claim no merge-path step runs is refused by name (README and CLAUDE.md), whatever the fence's spelling of +//! `bash`/`sh`/`shell`, and a step under `if: false` runs nothing; +//! - a path CLAUDE.md names that does not exist is refused by name; +//! - a ragged CSV row is refused naming the file and the row; +//! - F-33: a committed `verified_commands[]` that disagrees with the live set is RED naming the command; +//! - F-34 (git): a verified claim relabelled away with no new `withdrawn[]` entry is RED naming it, and passes +//! with one; a `withdrawn[]` entry for a claim still live is RED; a baseline whose keys are absent at the +//! comparand is checked against ∅ (bootstrap), not exempt; a work tree with no comparand ref is RED, and a +//! corpus outside any work tree is `not-checked`. +//! +//! DISCRIMINATION: the green fixture must PASS with every doc type counted, so a build that refuses everything +//! fails this file; each RED asserts the refusal TEXT, so a build that fails for another reason fails it too. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn pv_bin() -> PathBuf { + PathBuf::from(env!("CARGO_BIN_EXE_pv")) +} + +struct Run { + code: i32, + stdout: String, + stderr: String, +} + +fn pv_in(cwd: &Path, args: &[&str]) -> Run { + let out = Command::new(pv_bin()) + .current_dir(cwd) + .args(args) + .env_remove("GIT_DIR") + .env_remove("GIT_WORK_TREE") + .env_remove("GIT_INDEX_FILE") + .output() + .expect("failed to spawn pv"); + Run { + code: out.status.code().unwrap_or(-1), + stdout: String::from_utf8_lossy(&out.stdout).into_owned(), + stderr: String::from_utf8_lossy(&out.stderr).into_owned(), + } +} + +fn show(r: &Run) -> String { + format!( + "exit {}\n--- stdout\n{}\n--- stderr\n{}", + r.code, r.stdout, r.stderr + ) +} + +fn json_of(r: &Run) -> serde_json::Value { + serde_json::from_str(&r.stdout).unwrap_or_else(|e| panic!("stdout is JSON: {e}\n{}", show(r))) +} + +fn fixture_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../tests/fixtures/ont/docs-green") +} + +fn copy_tree(src: &Path, dst: &Path) { + std::fs::create_dir_all(dst).expect("mkdir"); + for e in std::fs::read_dir(src).expect("read fixture dir") { + let p = e.expect("entry").path(); + let to = dst.join(p.file_name().expect("name")); + if p.is_dir() { + copy_tree(&p, &to); + } else { + std::fs::copy(&p, &to).expect("copy"); + } + } +} + +/// A fresh copy of the green fixture, OUTSIDE any git work tree (tempdir), so F-34 reads `not-checked`. +fn copy() -> tempfile::TempDir { + let tmp = tempfile::tempdir().expect("tempdir"); + copy_tree(&fixture_root(), tmp.path()); + tmp +} + +fn gate_at(repo: &Path, extra: &[&str]) -> Run { + let mut args = vec!["lint", "contracts", "--gate", "shapes", "--format", "json"]; + args.extend_from_slice(extra); + pv_in(repo, &args) +} + +fn append(p: &Path, text: &str) { + let mut s = std::fs::read_to_string(p).expect("read"); + s.push_str(text); + std::fs::write(p, s).expect("write"); +} + +fn replace(p: &Path, from: &str, to: &str) { + let s = std::fs::read_to_string(p).expect("read"); + assert_eq!( + s.matches(from).count(), + 1, + "exactly one `{from}` in {}", + p.display() + ); + std::fs::write(p, s.replacen(from, to, 1)).expect("write"); +} + +fn baseline(repo: &Path, readme: &[&str], readme_w: &[&str], claude: &[&str], claude_w: &[&str]) { + let arr = |xs: &[&str]| serde_json::to_string(xs).expect("json"); + std::fs::write( + repo.join("contracts/lint-baseline.json"), + format!( + "{{\n \"armed_gates\": [\"shapes\"],\n \"armed_shapes\": [\"shape-holder\", \"readme-doc\", \"claude-doc\", \"data-csv\"],\n \"readme\": {{\"verified_commands\":{},\"withdrawn\":{}}},\n \"claude_md\": {{\"verified_commands\":{},\"withdrawn\":{}}}\n}}\n", + arr(readme), + arr(readme_w), + arr(claude), + arr(claude_w) + ), + ) + .expect("baseline"); +} + +fn git(repo: &Path, args: &[&str]) { + let out = Command::new("git") + .arg("-C") + .arg(repo) + .args(args) + .env_remove("GIT_DIR") + .env_remove("GIT_WORK_TREE") + .env_remove("GIT_INDEX_FILE") + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); +} + +/// A copy of the green fixture committed as the comparand. +fn committed_copy() -> tempfile::TempDir { + let tmp = copy(); + let repo = tmp.path(); + git(repo, &["init", "-q", "-b", "main"]); + git(repo, &["config", "user.email", "t@example.com"]); + git(repo, &["config", "user.name", "t"]); + git(repo, &["add", "-A"]); + git(repo, &["commit", "-q", "-m", "comparand"]); + tmp +} + +fn strs(v: &serde_json::Value) -> Vec<&str> { + v.as_array() + .map(|a| a.iter().filter_map(serde_json::Value::as_str).collect()) + .unwrap_or_default() +} + +const CHECK: &str = "bash scripts/check.sh"; +const TEST: &str = "cargo test --lib"; + +#[test] +fn the_green_fixture_passes_with_every_doc_type_counted_and_the_sets_reported() { + let t = copy(); + let r = gate_at(t.path(), &[]); + assert_eq!(r.code, 0, "{}", show(&r)); + let v = json_of(&r); + assert_eq!(v["verdict"], "Pass", "{}", show(&r)); + for ty in ["readme", "llm-context", "csv"] { + assert_eq!(v["by_entity_type"][ty], 1, "{ty}\n{}", show(&r)); + assert_eq!(v["pc_extract"][ty], "fired", "{ty}\n{}", show(&r)); + } + assert_eq!(strs(&v["readme"]["verified_commands"]), [CHECK, TEST]); + assert_eq!(strs(&v["claude_md"]["verified_commands"]), [TEST]); + assert_eq!( + v["ratchets"]["measured_sets"], + "not-checked", + "outside a git work tree there is nothing to compare\n{}", + show(&r) + ); +} + +#[test] +fn readme_bad_command_is_refused_by_name() { + let t = copy(); + append(&t.path().join("README.md"), "\n```bash\nmake ghost\n```\n"); + let r = gate_at(t.path(), &[]); + assert_eq!(r.code, 1, "{}", show(&r)); + assert!( + r.stdout + .contains("readme:verifiedCommand unresolved: make ghost"), + "{}", + show(&r) + ); +} + +#[test] +fn readme_tag_variants_are_claims_and_console_is_not() { + for (open, close, claim) in [ + ("```{.Bash}", "```", true), + ("~~~ shell", "~~~", true), + ("```SH title=x", "```", true), + ("```console", "```", false), + ("```text", "```", false), + ] { + let t = copy(); + append( + &t.path().join("README.md"), + &format!("\n{open}\nmake ghost\n{close}\n"), + ); + let r = gate_at(t.path(), &[]); + let refused = r + .stdout + .contains("readme:verifiedCommand unresolved: make ghost"); + assert_eq!(refused, claim, "{open}\n{}", show(&r)); + assert_eq!(r.code, if claim { 1 } else { 0 }, "{open}\n{}", show(&r)); + } +} + +#[test] +fn a_step_under_if_false_runs_nothing_so_its_command_is_not_a_verified_claim() { + let t = copy(); + append( + &t.path().join("README.md"), + "\n```bash\nmake disabled-step\n```\n", + ); + let r = gate_at(t.path(), &[]); + assert_eq!(r.code, 1, "{}", show(&r)); + assert!( + r.stdout + .contains("readme:verifiedCommand unresolved: make disabled-step"), + "{}", + show(&r) + ); +} + +#[test] +fn claude_md_bad_command_and_bad_path_are_refused_by_name() { + let t = copy(); + append( + &t.path().join("CLAUDE.md"), + "\nSee `no/such/dir/`.\n\n```shell\nmake nope\n```\n", + ); + let r = gate_at(t.path(), &[]); + assert_eq!(r.code, 1, "{}", show(&r)); + assert!( + r.stdout.contains("llm:command unresolved: make nope"), + "{}", + show(&r) + ); + assert!( + r.stdout + .contains("llm:referencedPath unresolved: no/such/dir/"), + "{}", + show(&r) + ); +} + +#[test] +fn claude_md_without_its_purpose_section_fails_the_shape() { + let t = copy(); + replace( + &t.path().join("CLAUDE.md"), + "## Project Overview", + "## Overview", + ); + let r = gate_at(t.path(), &[]); + assert_eq!(r.code, 1, "{}", show(&r)); + assert!(r.stdout.contains("purposeSection"), "{}", show(&r)); +} + +#[test] +fn csv_column_mismatch_is_refused_naming_file_and_row() { + let t = copy(); + append(&t.path().join("data/train.csv"), "1,2,3,4\n"); + let r = gate_at(t.path(), &[]); + assert_eq!(r.code, 1, "{}", show(&r)); + assert!(r.stdout.contains("data/train.csv"), "{}", show(&r)); + assert!( + r.stdout.contains("row 4 has 4 column(s), the header has 3"), + "{}", + show(&r) + ); +} + +#[test] +fn f33_a_committed_set_that_disagrees_with_the_live_one_is_red_naming_the_command() { + let t = copy(); + baseline(t.path(), &[TEST], &[], &[TEST], &[]); + let r = gate_at(t.path(), &[]); + assert_eq!(r.code, 1, "{}", show(&r)); + assert!(r.stdout.contains("PV-ONT-013"), "{}", show(&r)); + assert!( + r.stdout + .contains(&format!("lacks the live claim `{CHECK}`")), + "{}", + show(&r) + ); +} + +#[test] +fn f34_relabel_drops_claim_without_withdrawn_is_red_and_with_it_passes() { + let t = committed_copy(); + let repo = t.path(); + replace( + &repo.join("README.md"), + "```bash\nbash scripts/check.sh", + "```text\nbash scripts/check.sh", + ); + // restamped verified_commands (F-33 green), but no withdrawal + baseline(repo, &[TEST], &[], &[TEST], &[]); + let r = gate_at(repo, &["--armed-baseline-ref", "HEAD"]); + assert_eq!(r.code, 1, "{}", show(&r)); + let v = json_of(&r); + assert_eq!(v["ratchets"]["measured_sets"], "checked", "{}", show(&r)); + assert!(r.stdout.contains("PV-ONT-014"), "{}", show(&r)); + assert!( + r.stdout + .contains(&format!("verified claim `{CHECK}` was dropped")), + "{}", + show(&r) + ); + // the same drop, recorded + baseline(repo, &[TEST], &[CHECK], &[TEST], &[]); + let r = gate_at(repo, &["--armed-baseline-ref", "HEAD"]); + assert_eq!(r.code, 0, "{}", show(&r)); + assert_eq!(json_of(&r)["ratchets"]["measured_sets"], "checked"); +} + +#[test] +fn f34_withdrawal_without_drop_is_red_naming_it() { + let t = committed_copy(); + let repo = t.path(); + baseline(repo, &[CHECK, TEST], &[TEST], &[TEST], &[]); + let r = gate_at(repo, &["--armed-baseline-ref", "HEAD"]); + assert_eq!(r.code, 1, "{}", show(&r)); + assert!( + r.stdout.contains(&format!( + "withdrawn[] names `{TEST}`, which was not dropped" + )), + "{}", + show(&r) + ); +} + +#[test] +fn f34_ratchet_bootstrap_key_absent_at_the_comparand_is_checked_against_the_empty_set() { + let t = copy(); + let repo = t.path(); + // the comparand predates the keys + std::fs::write( + repo.join("contracts/lint-baseline.json"), + "{\n \"armed_gates\": [\"shapes\"],\n \"armed_shapes\": [\"shape-holder\", \"readme-doc\", \"claude-doc\", \"data-csv\"]\n}\n", + ) + .expect("baseline"); + git(repo, &["init", "-q", "-b", "main"]); + git(repo, &["config", "user.email", "t@example.com"]); + git(repo, &["config", "user.name", "t"]); + git(repo, &["add", "-A"]); + git(repo, &["commit", "-q", "-m", "comparand without the keys"]); + baseline(repo, &[CHECK, TEST], &[], &[TEST], &[]); + let r = gate_at(repo, &["--armed-baseline-ref", "HEAD"]); + assert_eq!(r.code, 0, "{}", show(&r)); + assert_eq!(json_of(&r)["ratchets"]["measured_sets"], "checked"); + // a withdrawal at bootstrap names a claim that was never in set(comparand) = ∅ + baseline(repo, &[CHECK, TEST], &["make gone"], &[TEST], &[]); + let r = gate_at(repo, &["--armed-baseline-ref", "HEAD"]); + assert_eq!(r.code, 1, "{}", show(&r)); + assert!( + r.stdout + .contains("withdrawn[] names `make gone`, which was not dropped"), + "{}", + show(&r) + ); +} + +#[test] +fn f34_a_work_tree_with_no_comparand_ref_is_red_not_unchecked() { + let t = committed_copy(); + // no origin/main, no explicit ref + let r = gate_at(t.path(), &[]); + assert_eq!(r.code, 1, "{}", show(&r)); + assert_eq!(json_of(&r)["ratchets"]["measured_sets"], "checked"); + assert!( + r.stdout.contains("no comparand in a git work tree"), + "{}", + show(&r) + ); +} + +// ── Σ `entity_type_target_class` is cross-checked, not hand-kept (v4.16 D-T1) ────────────────────────────────── + +fn sigma_at(repo: &Path) -> Run { + pv_in( + repo, + &["lint", "contracts", "--gate", "sigma", "--format", "json"], + ) +} + +const SIGMA: &str = "contracts/ontology.yaml"; + +#[test] +fn sigma_passes_the_green_fixture_whose_map_covers_every_implemented_type() { + let t = copy(); + let r = sigma_at(t.path()); + assert_eq!(r.code, 0, "{}", show(&r)); +} + +#[test] +fn sigma_refuses_a_target_class_key_that_no_entity_type_declares() { + let t = copy(); + replace( + &t.path().join(SIGMA), + " gguf: model:Model\n", + " gguf: model:Model\n ghost-type: ghost:Thing\n", + ); + let r = sigma_at(t.path()); + assert_eq!(r.code, 3, "{}", show(&r)); + assert!(r.stderr.starts_with("error: "), "{}", show(&r)); + assert!( + r.stderr + .contains("entity_type_target_class key `ghost-type`"), + "{}", + show(&r) + ); +} + +#[test] +fn sigma_refuses_an_implemented_type_with_neither_a_mapping_nor_an_explicit_unmapped_entry() { + let t = copy(); + replace(&t.path().join(SIGMA), " json: ~\n", ""); + let r = sigma_at(t.path()); + assert_eq!(r.code, 3, "{}", show(&r)); + assert!( + r.stderr.contains("implemented entity_type `json`") + && r.stderr.contains("neither a class nor an explicit `~`"), + "{}", + show(&r) + ); +} + +/// v4.16 §5 ONT-4c Mutation, verbatim: "remove an `entity.type` from Σ `entity_type_target_class` → the shape +/// naming that type exits 3 malformed, named". Both readers see it: the shapes gate names the SHAPE, and the Σ +/// gate names the TYPE (the map no longer covers an implemented one). +#[test] +fn removing_an_entity_type_from_the_target_class_map_exits_3_naming_the_shape_and_the_type() { + let t = copy(); + replace(&t.path().join(SIGMA), " readme: readme:Readme\n", ""); + let r = gate_at(t.path(), &[]); + assert_eq!(r.code, 3, "{}", show(&r)); + assert!( + r.stderr.contains("readme-doc") && r.stderr.contains("no targetClass"), + "{}", + show(&r) + ); + let s = sigma_at(t.path()); + assert_eq!(s.code, 3, "{}", show(&s)); + assert!( + s.stderr.contains("implemented entity_type `readme`"), + "{}", + show(&s) + ); +} diff --git a/crates/aprender-contracts-cli/tests/ont4d_subsumption.rs b/crates/aprender-contracts-cli/tests/ont4d_subsumption.rs new file mode 100644 index 0000000000..116135b7f2 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/ont4d_subsumption.rs @@ -0,0 +1,230 @@ +//! ONT-4d (PMAT-4070, aprender#4070): subsumption in Σ; shapes inherit down the hierarchy. Every RED clause of +//! the row, driven through the BUILT `pv`. +//! +//! - cycle → exit 3 `error: subsumes cycle ` `subsumption-cycle/` +//! - a shape on `Code` rejects a `Kernel` instance; focus_nodes_n == instances of Code ∪ all sub-concepts +//! `subsumption-inherit/` +//! - a sub-concept shape removing a super constraint → exit 1 `reject: weakens .` +//! `subsumption-weaken/` +//! - rdf:type closure present in contracts.nt for every super-concept (tracked corpus, and a fixture extract) +//! - export byte-identical across two runs +//! +//! DISCRIMINATION: `subsumption-ok/` (the same hierarchy, a valid kernel) is Pass at exit 0, so a build that +//! rejects every inherited instance fails this file. The row's MUTATION (drop the closure materialization) turns +//! `a_code_shape_rejects_a_kernel_instance_through_the_closure` RED: with no closure, the Code shape has no +//! focus node, and the gate declines instead of rejecting. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn pv_bin() -> PathBuf { + PathBuf::from(env!("CARGO_BIN_EXE_pv")) +} + +struct Run { + code: i32, + stdout: String, + stderr: String, +} + +fn pv(args: &[&str]) -> Run { + let scratch = tempfile::tempdir().expect("scratch cwd is creatable"); + let out = Command::new(pv_bin()) + .current_dir(scratch.path()) + .args(args) + .output() + .expect("failed to spawn pv"); + Run { + code: out.status.code().unwrap_or(-1), + stdout: String::from_utf8_lossy(&out.stdout).into_owned(), + stderr: String::from_utf8_lossy(&out.stderr).into_owned(), + } +} + +fn show(r: &Run) -> String { + format!( + "exit {}\n--- stdout\n{}\n--- stderr\n{}", + r.code, r.stdout, r.stderr + ) +} + +fn repo(rel: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .join(rel) +} + +fn fixture(name: &str) -> String { + s(&repo(&format!("tests/fixtures/ont/{name}"))) +} + +fn s(p: &Path) -> String { + p.to_str().expect("utf-8 path").to_string() +} + +fn shapes_json(dir: &str) -> (Run, serde_json::Value) { + let r = pv(&["lint", dir, "--gate", "shapes", "--format", "json"]); + let v = serde_json::from_str(&r.stdout) + .unwrap_or_else(|e| panic!("stdout is JSON: {e}\n{}", show(&r))); + (r, v) +} + +#[test] +fn a_subsumption_cycle_is_exit_3_naming_the_path() { + let r = pv(&["lint", &fixture("subsumption-cycle"), "--gate", "sigma"]); + assert_eq!(r.code, 3, "{}", show(&r)); + assert!(r.stderr.contains("error: subsumes cycle "), "{}", show(&r)); + for c in ["Kernel", "Code", "Contract"] { + assert!( + r.stderr.contains(c), + "the cycle path names {c}: {}", + show(&r) + ); + } +} + +#[test] +fn a_code_shape_rejects_a_kernel_instance_through_the_closure() { + let (r, v) = shapes_json(&fixture("subsumption-inherit")); + assert_eq!(r.code, 1, "{}", show(&r)); + assert_eq!(v["verdict"], "Fail", "{}", show(&r)); + // focus_nodes_n == instances of Code ∪ all sub-concepts: no contract is typed Code directly; the one kernel + // contract reaches the Code shape ONLY through Kernel ⊑ Code. + assert_eq!(v["focus_nodes_n"], 1, "{}", show(&r)); + assert_eq!( + v["by_shape"], + serde_json::Value::from(vec!["code-shape=1"]), + "{}", + show(&r) + ); + assert_eq!(v["inherited_shapes_applied"], 1, "{}", show(&r)); + assert_eq!(v["violations"], 1, "{}", show(&r)); +} + +#[test] +fn the_same_hierarchy_with_a_valid_kernel_passes() { + let (r, v) = shapes_json(&fixture("subsumption-ok")); + assert_eq!(r.code, 0, "{}", show(&r)); + assert_eq!(v["verdict"], "Pass", "{}", show(&r)); + assert_eq!(v["inherited_shapes_applied"], 1, "{}", show(&r)); + assert_eq!( + v["inherited_by_shape"], + serde_json::Value::from(vec!["code-shape <- Kernel=1"]), + "{}", + show(&r) + ); +} + +#[test] +fn a_sub_shape_that_weakens_a_super_shape_is_rejected() { + let (r, v) = shapes_json(&fixture("subsumption-weaken")); + assert_eq!(r.code, 1, "{}", show(&r)); + let msgs: Vec = v["findings"] + .as_array() + .expect("findings") + .iter() + .map(|f| f["message"].as_str().unwrap_or_default().to_string()) + .collect(); + assert!( + msgs.iter() + .any(|m| m.starts_with("reject: kernel-shape weakens contract-shape.name.minCount")), + "{msgs:?}\n{}", + show(&r) + ); + // The kernel instance satisfies BOTH shapes, so the weakening is the ONLY violation. + assert_eq!(v["violations"], 1, "{}", show(&r)); +} + +#[test] +fn the_repo_corpus_passes_with_shapes_applied_down_the_hierarchy() { + // The row's probe. + let sigma = std::fs::read_to_string(repo("contracts/ontology.yaml")).expect("Σ"); + assert!( + sigma.lines().any(|l| l.starts_with("subsumes:")), + "Σ declares subsumes" + ); + let (r, v) = shapes_json(&s(&repo("contracts"))); + assert_eq!(v["verdict"], "Pass", "{}", show(&r)); + assert!( + v["inherited_shapes_applied"].as_u64().unwrap_or(0) > 0, + "{}", + show(&r) + ); + let ofn = std::fs::read_to_string(repo("contracts/ontology.ofn")).expect("ontology.ofn"); + assert!(ofn.contains("SubClassOf( )")); + assert!(ofn.contains( + "SubClassOf( )" + )); +} + +/// Subjects typed `class` in an N-Triples text. +fn typed(nt: &str, class: &str) -> std::collections::BTreeSet { + let needle = format!("> ."); + nt.lines() + .filter(|l| l.ends_with(&needle)) + .map(|l| l.split(' ').next().unwrap_or_default().to_string()) + .collect() +} + +#[test] +fn the_tracked_contracts_nt_carries_the_type_closure() { + let nt = std::fs::read_to_string(repo("contracts/contracts.nt")).expect("contracts.nt"); + let kernels = typed(&nt, "Kernel"); + let symbols = typed(&nt, "Symbol"); + assert!( + !kernels.is_empty() && !symbols.is_empty(), + "both sub-concepts have instances" + ); + assert!( + kernels.is_subset(&typed(&nt, "Contract")), + "every Kernel is typed Contract" + ); + assert!( + symbols.is_subset(&typed(&nt, "Code")), + "every Symbol is typed Code (closure-only: no extractor types Code)" + ); +} + +#[test] +fn a_fixture_extract_carries_the_closure_and_is_byte_identical_twice() { + let d = tempfile::tempdir().expect("tempdir"); + for f in ["ontology.yaml", "code-shape.yaml", "kernel-noname.yaml"] { + std::fs::copy( + repo(&format!("tests/fixtures/ont/subsumption-inherit/{f}")), + d.path().join(f), + ) + .expect("copy"); + } + let dir = s(d.path()); + let r = pv(&["extract", &dir]); + assert_eq!(r.code, 0, "{}", show(&r)); + let first = std::fs::read(d.path().join("contracts.nt")).expect("contracts.nt written"); + let r = pv(&["extract", &dir]); + assert_eq!(r.code, 0, "{}", show(&r)); + let second = std::fs::read(d.path().join("contracts.nt")).expect("contracts.nt written"); + assert_eq!(first, second, "export byte-identical across two runs"); + let nt = String::from_utf8(first).expect("utf-8"); + let k = typed(&nt, "Kernel"); + assert_eq!(k.len(), 1, "{nt}"); + assert!( + k.is_subset(&typed(&nt, "Code")) && k.is_subset(&typed(&nt, "Contract")), + "closure over two levels: {nt}" + ); +} + +#[test] +fn census_reports_by_concept_over_the_closure() { + let r = pv(&["census", &s(&repo("contracts")), "--format", "json"]); + assert_eq!(r.code, 0, "{}", show(&r)); + let v: serde_json::Value = serde_json::from_str(&r.stdout).expect("census JSON"); + let n = |c: &str| v["by_concept"][c].as_u64().unwrap_or(0); + assert!(n("Kernel") > 0 && n("Symbol") > 0, "{}", show(&r)); + assert!( + n("Contract") >= n("Kernel"), + "Contract counts its Kernel sub-instances" + ); + assert!( + n("Code") >= n("Symbol"), + "Code counts its Symbol sub-instances" + ); +} diff --git a/crates/aprender-contracts-cli/tests/ont4e_refines_gate.rs b/crates/aprender-contracts-cli/tests/ont4e_refines_gate.rs new file mode 100644 index 0000000000..994c546c78 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/ont4e_refines_gate.rs @@ -0,0 +1,240 @@ +//! ONT-4e (PMAT-4075) — `refines` is Liskov or it is rejected (R-20), on the CLI. +//! +//! ONT-001 §5 ONT-4e probe, verbatim in `the_spec_probe_holds_on_the_repo_corpus`. Around it, each RED line of the +//! row: a strengthened precondition and a weakened postcondition are `reject:` lines naming the clause, a dropped +//! invariant is exit 1, a `prose` clause is `Unknown{Prose}` naming the clause, a legacy `invariants[]`-only pair is +//! `Pass` with `liskov_pairs_checked: 0` on the verdict line. Every witness is written by the real `pv-sat` binary, so +//! the reasoner and the checker are tested against each other; a tampered witness is PV-ONT-025, never a verdict. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +struct Run { + code: i32, + stdout: String, + stderr: String, +} + +fn run(bin: &str, args: &[&str]) -> Run { + let out = Command::new(bin).args(args).output().expect("spawn"); + Run { + code: out.status.code().unwrap_or(-1), + stdout: String::from_utf8_lossy(&out.stdout).into_owned(), + stderr: String::from_utf8_lossy(&out.stderr).into_owned(), + } +} + +fn s(p: &Path) -> &str { + p.to_str().expect("utf-8 path") +} + +fn show(r: &Run) -> String { + format!( + "exit {}\n--- stdout\n{}\n--- stderr\n{}", + r.code, r.stdout, r.stderr + ) +} + +fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../..") +} + +fn pv_sat(dir: &Path) -> Run { + run(env!("CARGO_BIN_EXE_pv-sat"), &[s(dir)]) +} + +fn gate(dir: &Path) -> Run { + run( + env!("CARGO_BIN_EXE_pv"), + &["lint", s(dir), "--gate", "refines", "--format", "json"], + ) +} + +fn json_of(r: &Run) -> serde_json::Value { + serde_json::from_str(&r.stdout).unwrap_or_else(|e| panic!("stdout is JSON: {e}\n{}", show(r))) +} + +/// A copy of `tests/fixtures/ont/`. +fn fixture(name: &str) -> tempfile::TempDir { + let dir = tempfile::tempdir().expect("tempdir"); + for entry in std::fs::read_dir(repo().join("tests/fixtures/ont").join(name)).expect("fixture") { + let entry = entry.expect("entry"); + std::fs::copy(entry.path(), dir.path().join(entry.file_name())).expect("copy"); + } + dir +} + +/// The fixture, with the witness the real pv-sat wrote for it, gated. +fn reasoned(name: &str) -> (tempfile::TempDir, Run) { + let dir = fixture(name); + let sat = pv_sat(dir.path()); + assert_eq!(sat.code, 0, "pv-sat on {name}: {}", show(&sat)); + let r = gate(dir.path()); + (dir, r) +} + +fn liskov_witnesses(dir: &Path) -> Vec { + std::fs::read_dir(dir.join("witness/liskov")) + .map(|rd| rd.flatten().map(|e| e.path()).collect()) + .unwrap_or_default() +} + +#[test] +fn the_spec_probe_holds_on_the_repo_corpus() { + // ONT-001 §5 ONT-4e probe: `"$PV" lint contracts/ --gate refines --format json >"$TMP/lk.json" + // && json_object "$TMP/lk.json" && jq -e '.liskov_pairs_checked>0 and .pc_checker=="fired" and .verdict=="Pass"'` + let r = gate(&repo().join("contracts")); + assert_eq!(r.code, 0, "{}", show(&r)); + let v = json_of(&r); + assert!(v.is_object(), "json_object: {v}"); + assert!( + v["liskov_pairs_checked"].as_u64().unwrap_or(0) > 0, + "liskov_pairs_checked>0: {v}" + ); + assert_eq!(v["pc_checker"], "fired", "{v}"); + assert_eq!(v["verdict"], "Pass", "{v}"); + assert_eq!(v["witness"]["pc_reasoner"], "fired", "{v}"); +} + +#[test] +fn a_liskov_pair_passes_and_the_witness_is_a_chain() { + let (dir, r) = reasoned("refines-ok"); + assert_eq!(r.code, 0, "{}", show(&r)); + let v = json_of(&r); + assert_eq!(v["verdict"], "Pass", "{v}"); + assert_eq!(v["liskov_pairs_checked"], 1, "{v}"); + assert_eq!(v["violations"], 0, "{v}"); + let w = liskov_witnesses(dir.path()); + assert_eq!(w.len(), 1, "one Liskov witness: {w:?}"); + let text = std::fs::read_to_string(&w[0]).expect("witness"); + assert!(text.contains("\"chain\""), "{text}"); + assert!(!text.contains("counter_model"), "{text}"); + + // Byte-stable: a second pv-sat run confirms, it does not rewrite. + let again = pv_sat(dir.path()); + assert_eq!(again.code, 0, "{}", show(&again)); + assert_eq!( + std::fs::read_to_string(&w[0]).expect("witness"), + text, + "rerun rewrote the witness" + ); +} + +#[test] +fn a_strengthened_precondition_is_rejected_naming_the_clause() { + let (_dir, r) = reasoned("refines-pre-strengthened"); + assert_eq!(r.code, 1, "{}", show(&r)); + assert!( + r.stderr + .contains("reject: a refines b: precondition strengthened (PRE-1)"), + "{}", + show(&r) + ); + let v = json_of(&r); + assert_eq!(v["verdict"], "Fail", "{v}"); + assert_eq!(v["findings"][0]["rule_id"], "PV-ONT-024", "{v}"); +} + +#[test] +fn a_weakened_postcondition_is_rejected_naming_the_clause() { + let (_dir, r) = reasoned("refines-post-weakened"); + assert_eq!(r.code, 1, "{}", show(&r)); + assert!( + r.stderr + .contains("reject: a refines b: postcondition weakened (POST-1)"), + "{}", + show(&r) + ); +} + +#[test] +fn a_dropped_invariant_is_rejected() { + let (_dir, r) = reasoned("refines-inv-dropped"); + assert_eq!(r.code, 1, "{}", show(&r)); + assert!( + r.stderr + .contains("reject: a refines b: invariant dropped (INV-1)"), + "{}", + show(&r) + ); +} + +#[test] +fn a_prose_clause_is_unknown_prose_naming_the_clause() { + let (_dir, r) = reasoned("refines-prose"); + assert_eq!(r.code, 2, "{}", show(&r)); + let v = json_of(&r); + assert_eq!(v["verdict"], "Unknown(Prose)", "{v}"); + assert_eq!(v["liskov_pairs_checked"], 0, "{v}"); + assert_eq!(v["liskov_prose"], 1, "{v}"); + assert_eq!(v["prose_clauses"][0], "a.requires PRE-1", "{v}"); + assert!(r.stderr.contains("a.requires PRE-1"), "{}", show(&r)); + assert!(r.stderr.contains("decline: Prose"), "{}", show(&r)); +} + +#[test] +fn a_legacy_pair_passes_and_says_nothing_was_checked() { + // No witness needed: nothing to check, and the verdict line says so. + let dir = fixture("refines-legacy"); + let r = gate(dir.path()); + assert_eq!(r.code, 0, "{}", show(&r)); + let v = json_of(&r); + assert_eq!(v["verdict"], "Pass", "{v}"); + assert_eq!(v["liskov_pairs_checked"], 0, "{v}"); + assert_eq!(v["liskov_pairs_legacy"], 1, "{v}"); + assert_eq!(v["pc_checker"], "fired", "{v}"); +} + +#[test] +fn no_witness_is_a_stale_decline_naming_make_contracts() { + let dir = fixture("refines-ok"); + let r = gate(dir.path()); + assert_eq!(r.code, 2, "{}", show(&r)); + assert!(r.stderr.contains("make contracts"), "{}", show(&r)); + assert!(r.stderr.contains("decline: WitnessStale"), "{}", show(&r)); +} + +#[test] +fn a_tampered_witness_is_refused_not_believed() { + // pv-sat proves the pre-strengthened pair violates PRE-1; a hand edit turns the counter-model into a chain + // that derives A's PRE-1 from nothing B requires. The checker must refuse it (PV-ONT-025), not pass it. + let dir = fixture("refines-pre-strengthened"); + assert_eq!(pv_sat(dir.path()).code, 0); + let w = liskov_witnesses(dir.path()); + assert_eq!(w.len(), 1, "{w:?}"); + let mut doc: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&w[0]).expect("witness")).expect("json"); + let obligations = doc["pairs"][0]["obligations"] + .as_array_mut() + .expect("obligations"); + for ob in obligations.iter_mut() { + if ob["kind"] == "pre" { + *ob = serde_json::from_str( + r#"{"kind": "pre", "chain": [{"clause": "PRE-1", "from": "PRE-1"}]}"#, + ) + .expect("obligation"); + } + } + std::fs::write(&w[0], serde_json::to_string_pretty(&doc).expect("ser")).expect("write"); + let r = gate(dir.path()); + assert_eq!(r.code, 1, "{}", show(&r)); + let v = json_of(&r); + assert_eq!(v["findings"][0]["rule_id"], "PV-ONT-025", "{v}"); + assert!( + !r.stderr.contains("precondition strengthened"), + "a refused witness names no verdict: {}", + show(&r) + ); +} + +#[test] +fn pv_sat_self_test_fires_the_liskov_controls() { + let r = run(env!("CARGO_BIN_EXE_pv-sat"), &["--self-test"]); + assert_eq!(r.code, 0, "{}", show(&r)); + assert!( + r.stdout.contains("pc_liskov_reasoner fired"), + "{}", + show(&r) + ); + assert!(r.stdout.contains("pc_liskov_checker fired"), "{}", show(&r)); +} diff --git a/crates/aprender-contracts-cli/tests/ont4f_github_entities.rs b/crates/aprender-contracts-cli/tests/ont4f_github_entities.rs new file mode 100644 index 0000000000..bebc5af3fd --- /dev/null +++ b/crates/aprender-contracts-cli/tests/ont4f_github_entities.rs @@ -0,0 +1,167 @@ +//! ONT-4f (aprender#4330) — GitHub repos, issues, pull requests and milestones as focus nodes, on the CLI. +//! +//! Each fixture is the real Σ, the real `contracts/github-entities-v1.yaml`, and committed snapshots under +//! `evidence/github//`: +//! +//! | fixture | exit | why | +//! |---|---|---| +//! | `github-green` | 0 | one tracked snapshot per type; `pr:baseRepo` and `issue:milestone` resolve to IRIs | +//! | `github-merged-no-mergedat` | 1 | `state: MERGED` with no `mergedAt` — refused by the extractor, named | +//! | `github-untracked-milestone` | 1 | an issue naming a milestone no snapshot tracks — FAIL CLOSED, never Unknown | +//! | `github-sha-mismatch` | 1 | a repo whose `sha` disagrees with its `ref` — refused naming both shas | +//! +//! DISCRIMINATION, in both directions: `github-green` must PASS, so a build that refuses every snapshot fails +//! here; the three mutation fixtures must FAIL, so a build that accepts anything fails here too. The fixture copies +//! of the contract are asserted byte-identical to the real one, so weakening the real shapes without the fixtures +//! is caught by the drift test instead. No test here touches the network: the snapshots are the whole input. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn pv_bin() -> PathBuf { + PathBuf::from(env!("CARGO_BIN_EXE_pv")) +} + +fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../..") +} + +fn fixture(name: &str) -> PathBuf { + repo_root().join("tests/fixtures/ont").join(name) +} + +const FIXTURES: [&str; 4] = [ + "github-green", + "github-merged-no-mergedat", + "github-untracked-milestone", + "github-sha-mismatch", +]; + +struct Run { + code: i32, + stdout: String, + stderr: String, +} + +impl Run { + fn all(&self) -> String { + format!( + "exit {}\n--- stdout\n{}\n--- stderr\n{}", + self.code, self.stdout, self.stderr + ) + } + + fn json(&self) -> serde_json::Value { + serde_json::from_str(&self.stdout).expect("json report") + } +} + +fn shapes_on(contracts: &Path, cwd: Option<&Path>) -> Run { + let mut cmd = Command::new(pv_bin()); + cmd.args([ + "lint", + contracts.to_str().expect("utf-8 path"), + "--gate", + "shapes", + "--format", + "json", + ]); + if let Some(dir) = cwd { + cmd.current_dir(dir); + } + let out = cmd.output().expect("failed to spawn pv"); + Run { + code: out.status.code().unwrap_or(-1), + stdout: String::from_utf8_lossy(&out.stdout).into_owned(), + stderr: String::from_utf8_lossy(&out.stderr).into_owned(), + } +} + +fn on_fixture(name: &str) -> Run { + shapes_on(&fixture(name).join("contracts"), None) +} + +fn violations(r: &Run) -> u64 { + r.json()["extra"]["violations"].as_u64().unwrap_or(0) +} + +#[test] +fn one_tracked_snapshot_per_type_passes_and_every_type_is_counted() { + let r = on_fixture("github-green"); + assert_eq!(r.code, 0, "{}", r.all()); + assert_eq!(violations(&r), 0, "{}", r.all()); + let by = &r.json()["extra"]["by_entity_type"]; + for t in ["repo", "issue", "pull-request", "milestone"] { + assert_eq!(by[t], 1, "by_entity_type[{t}]\n{}", r.all()); + } +} + +#[test] +fn mutation_a_merged_pull_request_with_no_merged_at_fails_naming_the_field() { + let r = on_fixture("github-merged-no-mergedat"); + assert_eq!(r.code, 1, "{}", r.all()); + assert!(r.stdout.contains("PV-ONT-012"), "{}", r.all()); + assert!(r.stdout.contains("`mergedAt` is absent"), "{}", r.all()); + assert!( + r.stdout.contains("paiml__aprender__3706.json"), + "{}", + r.all() + ); +} + +#[test] +fn mutation_an_issue_naming_an_untracked_milestone_fails_closed_never_unknown() { + let r = on_fixture("github-untracked-milestone"); + assert_eq!(r.code, 1, "Fail, not a decline:\n{}", r.all()); + assert_eq!(violations(&r), 1, "{}", r.all()); + assert!(r.stdout.contains("github-issue"), "{}", r.all()); + assert!(r.stdout.contains("milestoneUnresolved"), "{}", r.all()); + assert!(r.stdout.contains("paiml/aprender#3"), "{}", r.all()); +} + +#[test] +fn mutation_a_repo_whose_sha_disagrees_with_its_ref_fails_naming_both() { + let r = on_fixture("github-sha-mismatch"); + assert_eq!(r.code, 1, "{}", r.all()); + assert!(r.stdout.contains("PV-ONT-012"), "{}", r.all()); + assert!( + r.stdout + .contains("aa7c6ef03ee7b7f8d8dc09dc393e97619952d95f") + && r.stdout + .contains("0000000000000000000000000000000000000000"), + "both shas are named\n{}", + r.all() + ); +} + +#[test] +fn the_real_corpus_counts_every_github_type_and_fires_every_control() { + let r = shapes_on(Path::new("contracts"), Some(&repo_root())); + let v = r.json(); + for t in ["repo", "issue", "pull-request", "milestone"] { + assert!( + v["extra"]["by_entity_type"][t].as_u64().unwrap_or(0) >= 1, + "by_entity_type[{t}] is not >= 1\n{}", + r.all() + ); + assert_eq!( + v["pc_extract"][t], "fired", + "pc_extract[{t}]: {}", + v["pc_extract"] + ); + } +} + +#[test] +fn every_fixture_carries_the_real_contract_byte_for_byte() { + let real = std::fs::read(repo_root().join("contracts/github-entities-v1.yaml")) + .expect("the real contract is in the tree"); + for name in FIXTURES { + let copy = std::fs::read(fixture(name).join("contracts/github-entities-v1.yaml")) + .unwrap_or_else(|e| panic!("{name} carries the contract: {e}")); + assert_eq!( + copy, real, + "{name}'s copy of github-entities-v1.yaml has drifted from contracts/" + ); + } +} diff --git a/crates/aprender-contracts-cli/tests/ont5_consistency_gate.rs b/crates/aprender-contracts-cli/tests/ont5_consistency_gate.rs new file mode 100644 index 0000000000..7b8d7b4cd0 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/ont5_consistency_gate.rs @@ -0,0 +1,178 @@ +//! ONT-5 (PMAT-4074) — `pv-sat` writes the witness, `pv lint --gate ont-consistency` checks it, on the CLI. +//! +//! ONT-001 §5 ONT-5 probe, verbatim in `the_spec_probe_holds_on_the_repo_corpus`. Around it, every exit the gate +//! can give: 0 on the repo's consistent relations, 1 (PV-ONT-022) on `relations-ok` — where `a contradicts d` and +//! both are live, so a gate that never reasons would pass it — 2 for no witness / no Σ / no relation, and 3 for a +//! Σ that does not parse. The witness in every non-repo case is written by the real `pv-sat` binary, so the two +//! halves are tested against each other, not against a hand-built file. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +struct Run { + code: i32, + stdout: String, + stderr: String, +} + +fn run(bin: &str, args: &[&str]) -> Run { + let out = Command::new(bin).args(args).output().expect("spawn"); + Run { + code: out.status.code().unwrap_or(-1), + stdout: String::from_utf8_lossy(&out.stdout).into_owned(), + stderr: String::from_utf8_lossy(&out.stderr).into_owned(), + } +} + +fn pv(args: &[&str]) -> Run { + run(env!("CARGO_BIN_EXE_pv"), args) +} + +fn pv_sat(dir: &Path) -> Run { + run(env!("CARGO_BIN_EXE_pv-sat"), &[s(dir)]) +} + +fn s(p: &Path) -> &str { + p.to_str().expect("utf-8 path") +} + +fn show(r: &Run) -> String { + format!( + "exit {}\n--- stdout\n{}\n--- stderr\n{}", + r.code, r.stdout, r.stderr + ) +} + +fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../..") +} + +fn json_of(r: &Run) -> serde_json::Value { + serde_json::from_str(&r.stdout).unwrap_or_else(|e| panic!("stdout is JSON: {e}\n{}", show(r))) +} + +fn gate(dir: &Path) -> Run { + pv(&[ + "lint", + s(dir), + "--gate", + "ont-consistency", + "--format", + "json", + ]) +} + +/// A copy of `tests/fixtures/ont/relations-ok`. +fn corpus() -> tempfile::TempDir { + let dir = tempfile::tempdir().expect("tempdir"); + for entry in std::fs::read_dir(repo().join("tests/fixtures/ont/relations-ok")).expect("fixture") + { + let entry = entry.expect("entry"); + std::fs::copy(entry.path(), dir.path().join(entry.file_name())).expect("copy"); + } + dir +} + +#[test] +fn the_spec_probe_holds_on_the_repo_corpus() { + // F-7: the reasoner is not reachable from the library. + let lib = std::fs::read_to_string(repo().join("crates/aprender-contracts/src/lib.rs")) + .expect("lib.rs"); + assert!(!lib.contains("mod sat"), "the lib reaches the reasoner"); + let r = gate(&repo().join("contracts")); + assert_eq!(r.code, 0, "{}", show(&r)); + let v = json_of(&r); + assert!( + v["checkable_n"].as_u64().is_some_and(|n| n > 0), + "{}", + show(&r) + ); + assert_eq!(v["pc_checker"], "fired", "{}", show(&r)); + assert_eq!(v["witness"]["pc_reasoner"], "fired", "{}", show(&r)); + assert_eq!(v["witness"]["stale"], false, "{}", show(&r)); + assert_eq!(v["verdict"], "Pass", "{}", show(&r)); +} + +#[test] +fn an_inconsistent_corpus_fails_on_the_witness_pv_sat_wrote() { + let d = corpus(); + let w = pv_sat(d.path()); + assert_eq!(w.code, 0, "{}", show(&w)); + assert!(w.stdout.contains("unsat_core"), "{}", show(&w)); + let r = gate(d.path()); + assert_eq!(r.code, 1, "{}", show(&r)); + assert!(r.stdout.contains("PV-ONT-022"), "{}", show(&r)); + assert_eq!( + json_of(&r)["core"], + serde_json::Value::from(vec!["a", "d"]), + "{}", + show(&r) + ); +} + +#[test] +fn an_edit_after_the_witness_is_stale_and_says_make_contracts() { + let d = corpus(); + assert_eq!(pv_sat(d.path()).code, 0); + let a = d.path().join("a.yaml"); + let text = std::fs::read_to_string(&a).expect("a.yaml"); + std::fs::write(&a, text.replace(" contradicts: [d]\n", "")).expect("edit"); + let r = gate(d.path()); + assert_eq!(r.code, 2, "{}", show(&r)); + assert!(r.stderr.contains("make contracts"), "{}", show(&r)); + assert!(r.stderr.contains("WitnessStale"), "{}", show(&r)); + + // Regenerating is the whole remedy: the edited corpus is consistent, and says so. + assert_eq!(pv_sat(d.path()).code, 0); + let r = gate(d.path()); + assert_eq!(r.code, 0, "{}", show(&r)); + let files = std::fs::read_dir(d.path().join("witness")) + .expect("witness dir") + .count(); + assert_eq!(files, 1, "the old graph's witness was not pruned"); +} + +#[test] +fn nothing_to_check_declines_and_a_broken_sigma_is_exit_3() { + let d = corpus(); + for name in ["a.yaml", "b.yaml"] { + let p = d.path().join(name); + let text = std::fs::read_to_string(&p).expect("read"); + let cut = text.find("relations:").expect("relations block"); + std::fs::write(&p, &text[..cut]).expect("write"); + } + let r = gate(d.path()); + assert_eq!(r.code, 2, "{}", show(&r)); + assert!(r.stderr.contains("NoCheckable"), "{}", show(&r)); + assert_eq!( + pv_sat(d.path()).code, + 2, + "pv-sat must not write a witness for zero clauses" + ); + + let d = corpus(); + std::fs::remove_file(d.path().join("ontology.yaml")).expect("rm"); + assert_eq!(gate(d.path()).code, 2); + + let d = corpus(); + std::fs::write(d.path().join("ontology.yaml"), "roles: [unclosed\n").expect("write"); + let r = gate(d.path()); + assert_eq!(r.code, 3, "{}", show(&r)); +} + +/// R-8: computed in every `pv lint` run, and not armed until the cop arms it. +#[test] +fn the_full_run_computes_the_gate_unarmed() { + let r = pv(&["lint", s(&repo().join("contracts")), "--format", "json"]); + let v = json_of(&r); + let gate = v["gates"] + .as_array() + .and_then(|g| g.iter().find(|x| x["name"] == "ont-consistency")) + .unwrap_or_else(|| panic!("ont-consistency not computed\n{}", show(&r))); + assert_eq!(gate["verdict"], "Pass", "{gate}"); + let not_armed: Vec<&str> = v["not_armed"] + .as_array() + .map(|a| a.iter().filter_map(|x| x.as_str()).collect()) + .unwrap_or_default(); + assert!(not_armed.contains(&"ont-consistency"), "{not_armed:?}"); +} diff --git a/crates/aprender-contracts-cli/tests/ont6_lint_verdict.rs b/crates/aprender-contracts-cli/tests/ont6_lint_verdict.rs index a3388229d1..97166dcbd8 100644 --- a/crates/aprender-contracts-cli/tests/ont6_lint_verdict.rs +++ b/crates/aprender-contracts-cli/tests/ont6_lint_verdict.rs @@ -47,9 +47,27 @@ fn repo_contracts() -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")).join("../../contracts") } +struct Corpus { + dir: PathBuf, + _root: tempfile::TempDir, +} + +impl Corpus { + fn path(&self) -> &Path { + &self.dir + } +} + /// A corpus directory holding the PVL-1 control contract, and optionally a `lint-baseline.json`. -fn corpus(baseline: Option<&str>) -> tempfile::TempDir { - let d = tempfile::tempdir().expect("corpus dir is creatable"); +/// Nested one level inside a private tempdir: `pv lint` takes the dir's PARENT as the +/// project root, and a bare tempdir's parent is the shared `/tmp` (#4207). +fn corpus(baseline: Option<&str>) -> Corpus { + let root = tempfile::tempdir().expect("corpus dir is creatable"); + let d = Corpus { + dir: root.path().join("contracts"), + _root: root, + }; + std::fs::create_dir(d.path()).expect("nested corpus dir is creatable"); std::fs::copy( repo_contracts().join("softmax-kernel-v1.yaml"), d.path().join("softmax-kernel-v1.yaml"), @@ -109,12 +127,9 @@ fn explicit_empty_armed_set_declines_r2() { #[test] fn a_failing_armed_gate_rejects_at_exit_1() { - let d = tempfile::tempdir().expect("corpus dir"); - std::fs::copy( - repo_contracts().join("softmax-kernel-v1.yaml"), - d.path().join("softmax-kernel-v1.yaml"), - ) - .expect("control copies"); + // Through `corpus()`, not a bare tempdir: on a host with a stray `/tmp/scripts/` + // baseline, PV-DUP-002 would also reject, and this test could pass for the wrong reason. + let d = corpus(None); std::fs::write( d.path().join("broken-v1.yaml"), "metadata:\n version: not-a-contract\n", @@ -265,6 +280,22 @@ fn json_report_carries_the_lattice() { serde_json::Value::String("shapes".into()), // ONT-7's gate: same R-8 shape. serde_json::Value::String("valid-under".into()), + // PVL-001 EV-11's two ratchets: same R-8 shape. + serde_json::Value::String("theorem-pairing".into()), + serde_json::Value::String("depends-on-present".into()), + // PVL-001 EV-8a's ratchet: same R-8 shape. + serde_json::Value::String("proved-is-derived".into()), + // PVL-001 EV-7b's gate: same R-8 shape. + serde_json::Value::String("challenge-fresh".into()), + // ONT-5 (gate 19) likewise: computed everywhere, armed by nobody yet. + serde_json::Value::String("ont-consistency".into()), + // ONT-4e (gate 18) likewise. + serde_json::Value::String("refines".into()), + // ONT-3a's bindings gate and ONT-3b's refinement gate likewise. + serde_json::Value::String("bindings".into()), + serde_json::Value::String("refinement".into()), + // ONT-8's gate: same R-8 shape. + serde_json::Value::String("evidence".into()), ]), "{}", show(&r) @@ -318,9 +349,11 @@ fn repo_baseline_arms_the_eight_ruled_gates_and_every_later_row_that_armed_one() "composition", "sigma", "relations", - "shapes" + "shapes", + "proved-is-derived" ] .to_vec(), - "the ruled 8 plus the gates later rows armed (ONT-2b: sigma; ONT-4: relations; ONT-4b: shapes)" + "the ruled 8 plus the gates later rows armed (ONT-2b: sigma; ONT-4: relations; ONT-4b: shapes; \ + PVL-001 EV-8a: proved-is-derived)" ); } diff --git a/crates/aprender-contracts-cli/tests/ont8_evidence_gate.rs b/crates/aprender-contracts-cli/tests/ont8_evidence_gate.rs new file mode 100644 index 0000000000..192576e4e2 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/ont8_evidence_gate.rs @@ -0,0 +1,213 @@ +//! ONT-8 (PMAT-4077) — `pv lint --gate evidence`: one evidence block, PROV-O names, one L-enum, every entity type. +//! +//! The row's probe, verbatim (paiml/infra `docs/specifications/paiml-ontology.md` v4.14 :667): +//! `pv lint contracts/ --gate evidence --format json … jq -e '.verdict=="Pass" and .levels_source=="enum" and +//! .entity_types_checked>=3'`. The first test is that probe on the real corpus. +//! +//! The rest build a corpus in a tempdir around the repo's own Σ, so every rule is judged against the agents and +//! levels the corpus really declares: +//! +//! | corpus | expected | +//! |---|---| +//! | readme + apr-model + code, one identical block | exit 0, Pass, `entity_types_checked` 3 — R-17 | +//! | `author:` under `provenance` | exit 1, PV-ONT-017 — F-16 | +//! | `level: L0` | exit 1, PV-ONT-018 — EV-3's one enum has no L0 | +//! | `wasAttributedTo: orchestrator` | exit 1, PV-ONT-021 — not a Σ agent | +//! | no evidence block anywhere | exit 2, decline — R-2 | +//! | no Σ | exit 2, decline | +//! +//! DISCRIMINATION: the R-17 corpus passes at exit 0 and every reject names its own rule id, so a build that +//! refuses everything, or refuses with the wrong rule, fails this file. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +struct Run { + code: i32, + stdout: String, + stderr: String, +} + +fn pv(args: &[&str]) -> Run { + let scratch = tempfile::tempdir().expect("scratch cwd is creatable"); + let out = Command::new(env!("CARGO_BIN_EXE_pv")) + .current_dir(scratch.path()) + .args(args) + .output() + .expect("failed to spawn pv"); + Run { + code: out.status.code().unwrap_or(-1), + stdout: String::from_utf8_lossy(&out.stdout).into_owned(), + stderr: String::from_utf8_lossy(&out.stderr).into_owned(), + } +} + +fn show(r: &Run) -> String { + format!( + "exit {}\n--- stdout\n{}\n--- stderr\n{}", + r.code, r.stdout, r.stderr + ) +} + +fn repo_contracts() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../contracts") +} + +fn gate(dir: &Path) -> Run { + pv(&[ + "lint", + dir.to_str().expect("utf-8 path"), + "--gate", + "evidence", + "--format", + "json", + ]) +} + +fn json_of(r: &Run) -> serde_json::Value { + serde_json::from_str(&r.stdout).unwrap_or_else(|e| panic!("stdout is JSON: {e}\n{}", show(r))) +} + +const FULL: &str = "level: L2\nmark: C\nprovenance:\n wasGeneratedBy: {command: \"pv extract README.md\"}\n wasAttributedTo: pv\n generatedAtTime: \"2026-09-24T00:00:00Z\"\n"; + +fn contract(entity_type: &str, evidence: &str) -> String { + let indented: String = evidence.lines().map(|l| format!(" {l}\n")).collect(); + format!( + "metadata:\n version: \"1.0.0\"\nentity:\n type: {entity_type}\nevidence:\n{indented}" + ) +} + +/// A corpus around the repo's Σ. `with_sigma: false` leaves Σ out. +fn corpus(with_sigma: bool, files: &[(&str, String)]) -> tempfile::TempDir { + let tmp = tempfile::tempdir().expect("corpus dir is creatable"); + if with_sigma { + std::fs::copy( + repo_contracts().join("ontology.yaml"), + tmp.path().join("ontology.yaml"), + ) + .expect("Σ copies"); + } + for (name, body) in files { + std::fs::write(tmp.path().join(name), body).expect("contract writes"); + } + tmp +} + +/// The rejects: exit 1, verdict Fail, and the report names exactly this rule. +fn assert_rejects(evidence: &str, rule: &str) { + let tmp = corpus(true, &[("c-v1.yaml", contract("code", evidence))]); + let r = gate(tmp.path()); + assert_eq!(r.code, 1, "{}", show(&r)); + let v = json_of(&r); + assert_eq!(v["verdict"], "Fail", "{}", show(&r)); + let rules: Vec<_> = v["findings"] + .as_array() + .expect("findings is an array") + .iter() + .map(|f| f["rule_id"].clone()) + .collect(); + assert_eq!( + rules, + [serde_json::Value::String(rule.into())], + "{}", + show(&r) + ); +} + +#[test] +fn the_row_probe_passes_on_the_real_corpus() { + let r = gate(&repo_contracts()); + assert_eq!(r.code, 0, "{}", show(&r)); + let v = json_of(&r); + assert_eq!(v["gate"], "evidence", "{}", show(&r)); + assert_eq!(v["verdict"], "Pass", "{}", show(&r)); + assert_eq!(v["levels_source"], "enum", "{}", show(&r)); + assert!( + v["entity_types_checked"] + .as_u64() + .expect("entity_types_checked is a number") + >= 3, + "{}", + show(&r) + ); +} + +#[test] +fn r17_readme_model_and_code_contracts_pass_the_same_gate() { + let tmp = corpus( + true, + &[ + ("readme-v1.yaml", contract("readme", FULL)), + ("model-v1.yaml", contract("apr-model", FULL)), + ("code-v1.yaml", contract("code", FULL)), + ], + ); + let r = gate(tmp.path()); + assert_eq!(r.code, 0, "{}", show(&r)); + let v = json_of(&r); + assert_eq!(v["verdict"], "Pass", "{}", show(&r)); + assert_eq!(v["entity_types_checked"], 3, "{}", show(&r)); + assert_eq!(v["contracts_with_evidence"], 3, "{}", show(&r)); +} + +#[test] +fn f16_author_is_rejected() { + assert_rejects( + &FULL.replace( + " wasAttributedTo: pv\n", + " wasAttributedTo: pv\n author: noah\n", + ), + "PV-ONT-017", + ); +} + +#[test] +fn level_l0_is_rejected() { + assert_rejects(&FULL.replace("L2", "L0"), "PV-ONT-018"); +} + +#[test] +fn an_undeclared_agent_is_rejected() { + assert_rejects( + &FULL.replace("wasAttributedTo: pv", "wasAttributedTo: orchestrator"), + "PV-ONT-021", + ); +} + +#[test] +fn no_evidence_anywhere_is_a_decline() { + let tmp = corpus( + true, + &[("c-v1.yaml", "metadata:\n version: \"1.0.0\"\n".to_string())], + ); + let r = gate(tmp.path()); + assert_eq!(r.code, 2, "{}", show(&r)); + assert!(r.stderr.contains("no evidence block"), "{}", show(&r)); +} + +#[test] +fn no_sigma_is_a_decline() { + let tmp = corpus(false, &[("c-v1.yaml", contract("code", FULL))]); + let r = gate(tmp.path()); + assert_eq!(r.code, 2, "{}", show(&r)); +} + +/// R-8: computed in every `pv lint` run, and reported as not armed until the baseline names it. +#[test] +fn a_full_run_computes_the_gate() { + let r = pv(&[ + "lint", + repo_contracts().to_str().expect("utf-8 path"), + "--format", + "json", + ]); + assert_eq!(r.code, 0, "{}", show(&r)); + let v = json_of(&r); + let g = v["gates"] + .as_array() + .expect("gates is an array") + .iter() + .find(|g| g["name"] == "evidence") + .unwrap_or_else(|| panic!("no evidence gate in a full run\n{}", show(&r))); + assert_eq!(g["passed"], true, "{}", show(&r)); +} diff --git a/crates/aprender-contracts-cli/tests/ont9_self_contract.rs b/crates/aprender-contracts-cli/tests/ont9_self_contract.rs new file mode 100644 index 0000000000..ef3970677a --- /dev/null +++ b/crates/aprender-contracts-cli/tests/ont9_self_contract.rs @@ -0,0 +1,233 @@ +//! ONT-9 (#4078) — the ontology has a contract on itself: `contracts/ont-self-v1.yaml`, `entity.type: pv-contract`, +//! `ref: contracts/`. +//! +//! ONT-001 §5 ONT-9 probe, minus the ledger's `merged`, in `the_spec_probe_holds_on_the_repo`. Around it: the +//! contract names a real test for every falsifier it lists (a contract about contracts that cites a ghost is the +//! defect it exists to catch); it does not claim the L3 rung before `cargo kani` has run; and the corpus-level +//! mutation the row names — introduce a cycle — turns the `relations` gate RED on the CLI, including the mixed +//! `depends_on ∪ supersedes` cycle the per-role sweep used to pass. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +struct Run { + code: i32, + stdout: String, + stderr: String, +} + +fn pv(args: &[&str]) -> Run { + let out = Command::new(env!("CARGO_BIN_EXE_pv")) + .args(args) + .output() + .expect("spawn pv"); + Run { + code: out.status.code().unwrap_or(-1), + stdout: String::from_utf8_lossy(&out.stdout).into_owned(), + stderr: String::from_utf8_lossy(&out.stderr).into_owned(), + } +} + +fn show(r: &Run) -> String { + format!( + "exit {}\n--- stdout\n{}\n--- stderr\n{}", + r.code, r.stdout, r.stderr + ) +} + +fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../..") +} + +fn s(p: &Path) -> &str { + p.to_str().expect("utf-8 path") +} + +const CONTRACT: &str = "contracts/ont-self-v1.yaml"; +const WITNESS_RS: &str = "crates/aprender-contracts/src/ontology/witness.rs"; + +fn contract() -> serde_yaml::Value { + let text = std::fs::read_to_string(repo().join(CONTRACT)) + .unwrap_or_else(|e| panic!("{CONTRACT} is readable: {e}")); + serde_yaml::from_str(&text).unwrap_or_else(|e| panic!("{CONTRACT} parses: {e}")) +} + +fn text(rel: &str) -> String { + std::fs::read_to_string(repo().join(rel)).unwrap_or_else(|e| panic!("{rel}: {e}")) +} + +/// §5 ONT-9's probe: `tracked contracts/ont-self-v1.yaml && rc_is 0 "$PV" validate contracts/ont-self-v1.yaml && +/// present 'fn ont_planted' …/witness.rs && present 'KANI-ONT-9-1' …/witness.rs` (`merged ONT-9` is the ledger's). +#[test] +fn the_spec_probe_holds_on_the_repo() { + // CI runs this in a container whose uid does not own the checkout, and actions/checkout marked it safe only in + // the host's git config — git refuses the repo ("dubious ownership") and the probe read "not tracked". git 2.34 + // honours `safe.directory` only from global/system config (not `-c`, not GIT_CONFIG_COUNT), so hand it one. + let cfg = tempfile::NamedTempFile::new().expect("temp gitconfig"); + std::fs::write(cfg.path(), "[safe]\n\tdirectory = *\n").expect("write temp gitconfig"); + let tracked = Command::new("git") + .args(["ls-files", "--error-unmatch", CONTRACT]) + .env("GIT_CONFIG_GLOBAL", cfg.path()) + .current_dir(repo()) + .output() + .expect("spawn git"); + assert!( + tracked.status.success(), + "{CONTRACT} is not tracked: {}", + String::from_utf8_lossy(&tracked.stderr) + ); + let r = pv(&["validate", s(&repo().join(CONTRACT))]); + assert_eq!(r.code, 0, "{}", show(&r)); + let w = text(WITNESS_RS); + assert!( + w.contains("fn ont_planted"), + "no `fn ont_planted` in {WITNESS_RS}" + ); + assert!( + w.contains("KANI-ONT-9-1"), + "no `KANI-ONT-9-1` in {WITNESS_RS}" + ); +} + +/// The contract is about the corpus itself (§5 ONT-9, B.16), and binds the checker and the gates it governs. +#[test] +fn the_contract_is_anchored_on_the_corpus_and_depends_on_what_it_governs() { + let c = contract(); + assert_eq!(c["entity"]["type"].as_str(), Some("pv-contract")); + assert_eq!(c["entity"]["ref"].as_str(), Some("contracts/")); + let deps: Vec<&str> = c["relations"]["depends_on"] + .as_sequence() + .expect("relations.depends_on is a list") + .iter() + .filter_map(serde_yaml::Value::as_str) + .collect(); + for d in [ + "ont-consistency-v1", + "ont-relations-v1", + "ont-verdict-lattice-v1", + ] { + assert!( + deps.contains(&d), + "ont-self-v1 must depend_on {d}: {deps:?}" + ); + } +} + +/// Every falsifier names a test that exists. `cargo test … ::` whose `` is no `fn` anywhere is +/// a filter that matches zero tests and exits 0 — a ghost binding, the defect a contract on contracts must not +/// carry itself. +#[test] +fn every_falsifier_names_a_test_function_that_exists() { + let c = contract(); + let tests = c["falsification_tests"] + .as_sequence() + .expect("falsification_tests is a list"); + assert!(tests.len() >= 7, "one falsifier per obligation at least"); + let src = crate_sources(); + for t in tests { + let cmd = t["test"].as_str().expect("test is a string"); + let last = cmd.split_whitespace().last().expect("non-empty"); + if last.starts_with("--") || !cmd.contains("--lib") { + // `--test ` or `--bin` — the target is the binding; check the file exists. + let target = cmd.split_whitespace().skip_while(|w| *w != "--test").nth(1); + if let Some(target) = target { + assert!( + src.iter() + .any(|(p, _)| p.ends_with(&format!("tests/{target}.rs"))), + "{cmd}: no tests/{target}.rs" + ); + if let Some(name) = cmd.split_whitespace().nth(6) { + assert!( + src.iter() + .any(|(_, body)| body.contains(&format!("fn {name}("))), + "{cmd}: no `fn {name}(`" + ); + } + } + continue; + } + let name = last.rsplit("::").next().expect("a path"); + assert!( + src.iter() + .any(|(_, body)| body.contains(&format!("fn {name}("))), + "{cmd}: no `fn {name}(` in aprender-contracts{{,-cli}} — a ghost binding" + ); + } +} + +fn crate_sources() -> Vec<(String, String)> { + let mut out = Vec::new(); + let mut stack = vec![ + repo().join("crates/aprender-contracts/src"), + repo().join("crates/aprender-contracts-cli/src"), + repo().join("crates/aprender-contracts-cli/tests"), + ]; + while let Some(d) = stack.pop() { + for e in std::fs::read_dir(&d).into_iter().flatten().flatten() { + let p = e.path(); + if p.is_dir() { + stack.push(p); + } else if p.extension().is_some_and(|x| x == "rs") { + let body = std::fs::read_to_string(&p).unwrap_or_default(); + out.push((p.to_string_lossy().into_owned(), body)); + } + } + } + out +} + +/// The L3 rung is not claimed before it ran. `KANI-ONT-9-1` is a real `#[kani::proof]` in witness.rs and is +/// listed, and while `proof.status` is `declared` the summary counts zero Kani-proved obligations. Flipping one +/// without the other — a proof claimed on a run nobody made — is RED here. +#[test] +fn the_kani_rung_is_declared_not_claimed() { + let c = contract(); + let h = c["kani_harnesses"] + .as_sequence() + .expect("kani_harnesses is a list"); + assert!( + h.iter().any(|x| x["id"].as_str() == Some("KANI-ONT-9-1") + && x["harness"].as_str() == Some("kani_ont_9_1")), + "KANI-ONT-9-1 → kani_ont_9_1 is listed" + ); + let w = text(WITNESS_RS); + assert!( + w.contains("#[kani::proof]") && w.contains("fn kani_ont_9_1()"), + "the harness is a real #[kani::proof] fn" + ); + let status = c["proof"]["status"].as_str().expect("proof.status"); + let l3 = c["verification_summary"]["l3_kani_proved"] + .as_u64() + .expect("l3_kani_proved"); + match status { + "declared" => assert_eq!(l3, 0, "declared, so nothing is Kani-proved yet"), + "proved" => assert!(l3 >= 1, "proved, so the summary counts it"), + other => panic!("proof.status `{other}` is neither declared nor proved"), + } +} + +/// Mutation "introduce a cycle → RED", at the CLI: the repo's relations (ont-self-v1 included) are acyclic and +/// exit 0; a per-role cycle and a `depends_on ∪ supersedes` cycle each exit 1 with PV-ONT-009. +#[test] +fn a_cycle_in_the_corpus_turns_the_relations_gate_red() { + let live = pv(&["lint", s(&repo().join("contracts")), "--gate", "relations"]); + assert_eq!(live.code, 0, "{}", show(&live)); + for (fixture, path) in [ + ("relations-cycle", "a -> b -> c -> a"), + ("relations-mixed-cycle", "a -> b -> a"), + ] { + let r = pv(&[ + "lint", + s(&repo().join("tests/fixtures/ont").join(fixture)), + "--gate", + "relations", + ]); + let all = format!("{}{}", r.stdout, r.stderr); + assert_eq!(r.code, 1, "{fixture}: {}", show(&r)); + assert!( + all.contains("PV-ONT-009") && all.contains(path), + "{fixture}: {}", + show(&r) + ); + } +} diff --git a/crates/aprender-contracts-cli/tests/ont_release_readiness.rs b/crates/aprender-contracts-cli/tests/ont_release_readiness.rs index 7f60586a5b..94d2755ccb 100644 --- a/crates/aprender-contracts-cli/tests/ont_release_readiness.rs +++ b/crates/aprender-contracts-cli/tests/ont_release_readiness.rs @@ -1,22 +1,25 @@ -//! aprender#3715 — `release-readiness-v1` on the CLI: the red-turning case table. +//! aprender#3715 + #3745 S2 (#3777) — `release-readiness-v1` on the CLI: the red-turning case table over DERIVED +//! cells. //! -//! Every case copies `tests/fixtures/ont/release-green/` AND the real `contracts/release-readiness-v1.yaml` (so the -//! fixture cannot drift from the shapes it proves) into a scratch repo, changes exactly ONE thing, and runs -//! `pv lint contracts --gate shapes --shape release-readiness-v1 --release-version 0.69.1 --release-commit MC …`. +//! Every case copies `tests/fixtures/ont/release-green/` (a measured inventory, a GENERIC surface — `gen` generates, +//! `look` reads a model, `list` takes none; not apr verbs, so S3's hand-list guard has nothing to find) and the +//! real `contracts/release-readiness-v1.yaml` into a scratch repo. The green base then asks pv for the cells it +//! derives (`pv extract … --surface --cells-out`) and writes one passing row per cell, keyed by `cell_id` — so the +//! fixture cannot drift from the derivation — and each case changes exactly ONE thing. //! -//! DISCRIMINATION: the green base must PASS with every cell named (48/48, 4 kernel cells) — so a build that -//! rejects everything fails this file — and each single-cell change must FAIL naming that cell — so a build that -//! accepts everything fails it too. The ONT-001 rule: a shape that has never gone red is not armed. +//! DISCRIMINATION: the green base must PASS with every derived cell carrying a row, so a build that rejects +//! everything fails this file; each single change must FAIL naming what it broke, so a build that accepts +//! everything fails it too. The ONT-001 rule: a shape that has never gone red is not armed. use std::path::{Path, PathBuf}; use std::process::Command; -use serde_json::Value; +use serde_json::{Map, Value}; const MC: &str = "1111111111111111111111111111111111111111"; const PARENT: &str = "3333333333333333333333333333333333333333"; const BUMP: &str = "2222222222222222222222222222222222222222"; -const MODEL: &str = "tiny-qwen35.gguf"; +const GB: u64 = 1_000_000_000; struct Run { code: i32, @@ -27,7 +30,9 @@ struct Run { fn show(r: &Run) -> String { format!( "exit {}\n--- stdout\n{}\n--- stderr\n{}", - r.code, r.stdout, r.stderr + r.code, + r.stdout.chars().take(6000).collect::(), + r.stderr ) } @@ -48,21 +53,6 @@ fn copy_dir(from: &Path, to: &Path) { } } -/// The green base: the fixture plus the REAL shape contract. -fn green() -> tempfile::TempDir { - let t = tempfile::tempdir().expect("tempdir"); - copy_dir( - &repo_root().join("tests/fixtures/ont/release-green"), - t.path(), - ); - std::fs::copy( - repo_root().join("contracts/release-readiness-v1.yaml"), - t.path().join("contracts/release-readiness-v1.yaml"), - ) - .expect("copy the real contract"); - t -} - fn pv(cwd: &Path, args: &[&str]) -> Run { let out = Command::new(env!("CARGO_BIN_EXE_pv")) .current_dir(cwd) @@ -80,35 +70,12 @@ fn s(p: &Path) -> String { p.to_str().expect("utf-8 path").to_string() } -/// The release gate as T-1 calls it, plus `extra` flags. -fn gate(t: &Path, extra: &[&str]) -> Run { - let contracts = s(&t.join("contracts")); - let dogfood = s(&t.join("dogfood-receipt.json")); - let mut args = vec![ - "lint", - &contracts, - "--gate", - "shapes", - "--shape", - "release-readiness-v1", - "--release-version", - "0.69.1", - "--release-commit", - MC, - "--dogfood-receipt", - &dogfood, - ]; - args.extend_from_slice(extra); - pv(t, &args) -} - -fn json_of(r: &Run) -> Value { - serde_json::from_str(&r.stdout).unwrap_or_else(|e| panic!("stdout is JSON: {e}\n{}", show(r))) +fn read(path: &Path) -> Value { + serde_json::from_str(&std::fs::read_to_string(path).expect("read")).expect("json") } fn edit(path: &Path, f: impl FnOnce(&mut Value)) { - let mut v: Value = - serde_json::from_str(&std::fs::read_to_string(path).expect("read")).expect("json"); + let mut v = read(path); f(&mut v); std::fs::write(path, serde_json::to_string_pretty(&v).expect("ser")).expect("write"); } @@ -121,23 +88,218 @@ fn kernels(t: &Path, host: &str) -> PathBuf { t.join(format!("evidence/dogfood/kernels/0.69.1/{host}.json")) } -/// Index of the one row for (verb, thinking, rung) in a model receipt. -fn row_index(v: &Value, verb: &str, thinking: &str, rung: &str) -> usize { - v["cells"] +fn tokenizer(t: &Path) -> PathBuf { + t.join("evidence/dogfood/tokenizer/0.69.1/parity.json") +} + +fn json_of(r: &Run) -> Value { + serde_json::from_str(&r.stdout).unwrap_or_else(|e| panic!("stdout is JSON: {e}\n{}", show(r))) +} + +/// The cells pv derives for the scratch repo, written to `/cells.json` and returned. +fn derive(t: &Path) -> Vec { + let contracts = s(&t.join("contracts")); + let surface = s(&t.join("surface.json")); + let out = s(&t.join("release.nt")); + let cells = s(&t.join("cells.json")); + let r = pv( + t, + &[ + "extract", + &contracts, + "--release-version", + "0.69.1", + "--release-commit", + MC, + "--surface", + &surface, + "--out", + &out, + "--cells-out", + &cells, + ], + ); + assert_eq!(r.code, 0, "{}", show(&r)); + read(&t.join("cells.json"))["cells"] + .as_array() + .expect("cells") + .clone() +} + +/// One PASSING row for a derived cell — whatever its class needs to pass, and nothing it does not. +fn pass_row(c: &Value) -> Value { + let mut r = Map::new(); + r.insert("cell_id".into(), c["id"].clone()); + r.insert("verdict".into(), "pass".into()); + r.insert("answer_chars".into(), 42.into()); + r.insert("rc".into(), 0.into()); + r.insert("wall_ms".into(), 1500.into()); + let generating = c["generates"] == true && c["kind"] == "matrix"; + r.insert( + "backend".into(), + if generating { "cuda" } else { "cpu" }.into(), + ); + r.insert("fallback".into(), false.into()); + if let Some(t) = c["rung_tokens"].as_u64() { + let declared = c["rung"] == "declared"; + r.insert("max_tokens".into(), 256.into()); + let p = if declared { + t.saturating_sub(256) + } else { + t + 1 + }; + r.insert("prompt_tokens".into(), p.into()); + } + if c["thinking"] == "on" { + r.insert("think_closed".into(), true.into()); + } + if generating { + r.insert("f2_source".into(), "fresh".into()); + } + if matches!(c["kind"].as_str(), Some("base" | "effect")) { + let id = c["id"].as_str().unwrap_or_default(); + r.insert("output_sha256".into(), format!("out:{id}").into()); + } + // a8's controls, coherent: T 0 and top-k 1 are the same greedy bytes; seed A twice is identical; seed B differs + if c["kind"] == "sampling" { + let out = match c["control"].as_str() { + Some("t0" | "topk1") => "greedy", + Some("seed-a" | "seed-a-again") => "sampled-a", + _ => "sampled-b", + }; + r.insert("output_sha256".into(), out.into()); + } + Value::Object(r) +} + +/// The green base: the fixture, the REAL contract, and one passing row per derived cell. +fn green() -> tempfile::TempDir { + let t = tempfile::tempdir().expect("tempdir"); + copy_dir( + &repo_root().join("tests/fixtures/ont/release-green"), + t.path(), + ); + std::fs::copy( + repo_root().join("contracts/release-readiness-v1.yaml"), + t.path().join("contracts/release-readiness-v1.yaml"), + ) + .expect("copy the real contract"); + synthesize(t.path()); + t +} + +/// (Re)write every host's rows from the cells pv derives now. +fn synthesize(t: &Path) { + let cells = derive(t); + for h in ["lambda", "gx10"] { + let rows: Vec = cells + .iter() + .filter(|c| c["host"] == h) + .map(pass_row) + .collect(); + edit(&models(t, h), |v| v["cells"] = Value::Array(rows)); + let mut keys: Vec = cells + .iter() + .filter(|c| c["host"] == h && c["command"] == "gen" && c["kind"] == "matrix") + .map(|c| { + format!( + r#"{{"key":{{"model_sha256":{},"host":"{h}","verb":"gen","thinking":{},"rung":{},"prompt_id":"p1"}},"verdict":"GREEN"}}"#, + c["model_sha256"], c["thinking"], c["rung"] + ) + }) + .collect(); + keys.sort(); + keys.dedup(); + // the LAST row is the set's positive control (#3739): one per model per host + if let Some(last) = keys.last_mut() { + *last = last.replace( + r#""verdict":"GREEN""#, + r#""verdict":"GREEN","positive_control":true"#, + ); + } + let crux_dir = t.join("evidence/crux/0.69.1"); + std::fs::create_dir_all(&crux_dir).expect("crux dir"); + let receipt = format!( + r#"{{"schema":"crux-inference-receipt/v1","version":"0.69.1","host":"{h}","apr":{{"version_line":"apr 0.69.1 (111111111)"}},"cells":[{}]}}"#, + keys.join(",") + ); + std::fs::write(crux_dir.join(format!("{h}-gpu.json")), receipt).expect("crux receipt"); + } +} + +/// The id of the first derived cell `pred` accepts. +fn pick(t: &Path, pred: impl Fn(&Value) -> bool) -> String { + read(&t.join("cells.json"))["cells"] .as_array() .expect("cells") .iter() - .position(|c| c["verb"] == verb && c["thinking"] == thinking && c["context"] == rung) - .unwrap_or_else(|| panic!("no row {verb}/{thinking}/{rung}")) + .find(|c| pred(c)) + .and_then(|c| c["id"].as_str()) + .expect("a derived cell matches") + .to_string() +} + +/// The IRI tail a finding names for a cell id: each segment percent-encoded as the graph does. +fn named(id: &str) -> String { + let enc = |seg: &str| { + seg.bytes() + .map(|b| match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' | b':' => { + (b as char).to_string() + } + _ => format!("%{b:02X}"), + }) + .collect::() + }; + let segs: Vec = id.split('/').map(enc).collect(); + format!("release-cell/0.69.1/{}", segs.join("/")) +} + +/// Edit the one row keyed to `id` on `host`. +fn edit_row(t: &Path, host: &str, id: &str, f: impl FnOnce(&mut Value)) { + edit(&models(t, host), |v| { + let row = v["cells"] + .as_array_mut() + .expect("cells") + .iter_mut() + .find(|r| r["cell_id"] == id) + .expect("the row exists"); + f(row); + }); } -fn row<'a>(v: &'a mut Value, verb: &str, thinking: &str, rung: &str) -> &'a mut Value { - let i = row_index(v, verb, thinking, rung); - &mut v["cells"][i] +fn drop_row(t: &Path, host: &str, id: &str) { + edit(&models(t, host), |v| { + v["cells"] + .as_array_mut() + .expect("cells") + .retain(|r| r["cell_id"] != id); + }); } -fn cell(host: &str, verb: &str, thinking: &str, rung: &str) -> String { - format!("release-cell/0.69.1/{host}/{MODEL}/{verb}/think-{thinking}/{rung}") +/// The release gate as T-1 calls it, plus `extra` flags. +fn gate(t: &Path, extra: &[&str]) -> Run { + let contracts = s(&t.join("contracts")); + let dogfood = s(&t.join("dogfood-receipt.json")); + let surface = s(&t.join("surface.json")); + let mut args = vec![ + "lint", + &contracts, + "--gate", + "shapes", + "--shape", + "release-readiness-v1", + "--release-version", + "0.69.1", + "--release-commit", + MC, + "--dogfood-receipt", + &dogfood, + "--surface", + &surface, + ]; + args.extend_from_slice(extra); + pv(t, &args) } /// RED, and the findings name `needle`. @@ -154,13 +316,29 @@ fn assert_red_naming(r: &Run, needle: &str) -> Value { assert!( msgs.iter().any(|m| m.contains(needle)), "no finding names {needle}:\n{}", - msgs.join("\n") + msgs.iter().take(40).copied().collect::>().join("\n") ); v } +fn gen_cell( + host: &'static str, + thinking: &'static str, + rung: &'static str, +) -> impl Fn(&Value) -> bool { + move |c| { + c["command"] == "gen" + && c["kind"] == "matrix" + && c["host"] == host + && c["thinking"] == thinking + && c["rung"] == rung + } +} + +// ── the green base ─────────────────────────────────────────────────────────────────────────────────────── + #[test] -fn the_green_release_passes_with_every_cell_named() { +fn the_green_release_passes_with_every_derived_cell_carrying_a_row() { let t = green(); let r = gate(t.path(), &[]); assert_eq!(r.code, 0, "{}", show(&r)); @@ -170,96 +348,96 @@ fn the_green_release_passes_with_every_cell_named() { v["pc_shape"], "fired", "the plant fired from the named family" ); + assert_eq!(v["pc_extract"]["release-evidence"], "fired"); + assert_eq!(v["pc_extract"]["cli-surface"], "fired"); + let rel = &v["release"]; + let derived = read(&t.path().join("cells.json"))["cells"] + .as_array() + .map(Vec::len); assert_eq!( - v["pc_extract"]["release-evidence"], "fired", - "the extractor's own control (PMAT-3704)" + rel["cells"].as_u64().map(|n| n as usize), + derived, + "N/N: every derived cell is a node" ); - let rel = &v["release"]; + assert_eq!(rel["cells_with_row"], rel["cells"]); + assert_eq!(rel["orphan_rows"], 0); assert_eq!( - rel["cells"], 48, - "2 hosts × 4 verbs × 2 thinking × (4k, consumer-max, declared)" + rel["probe_cells"], 2, + "`list` takes no model: one probe per host" ); - assert_eq!(rel["cells_with_row"], 48); - assert_eq!(rel["cell_names"].as_array().map(Vec::len), Some(48)); + // `--shape` arms the whole family of sixteen whatever armed_shapes says. The green release has no cell that + // fails to fit, so `.refusal` grades zero focus nodes: it declares `allowEmpty`, so the verdict stands, and + // like every vacuity it is named in `declines` and in neither shape list (#3610). + let armed = v["armed_shapes"].as_array().expect("armed").len(); assert_eq!( - rel["kernel_cells"], 4, - "(q4k_gemv@q4_k, rmsnorm@f32) × 2 hosts" + armed, 15, + "sixteen armed, less the one that measured nothing" ); - assert_eq!(rel["orphan_rows"], 0); assert_eq!( - rel["tokenizer_cells"], 1, - "one distinct model across both hosts (#3726)" + names(&v["declines"]), + vec!["release-readiness-v1.refusal"] as Vec<&str> ); - let armed = v["armed_shapes"].as_array().expect("armed").len(); + assert!( + rel["mode_effects"].as_u64() >= Some(1), + "`gen --chat` is judged by the flag-effect oracle" + ); + assert_eq!(rel["surface"]["generating_commands"], 1); + assert_eq!(rel["kernel_cells"], 4); + assert_eq!(rel["tokenizer_cells"], 1); assert_eq!( - armed, 9, - "--shape arms the whole family whatever armed_shapes says" + rel["sampling_checks"], 2, + "gen's typed sampling args, one model arch, two hosts" ); -} - -#[test] -fn a_missing_cell_is_red_naming_it() { - let t = green(); - edit(&models(t.path(), "lambda"), |v| { - let i = row_index(v, "chat", "on", "consumer-max"); - v["cells"].as_array_mut().expect("cells").remove(i); - }); - assert_red_naming( - &gate(t.path(), &[]), - &cell("lambda", "chat", "on", "consumer-max"), + assert_eq!(rel["crux"]["verbs"], 3, "every derived leaf verb"); + assert_eq!(rel["crux"]["mapped_verbs"], 1, "only gen has a counterpart"); + assert!(rel["crux"]["obligations"].as_u64() > Some(0)); + let lambda = &rel["projection"]["lambda"]; + assert_eq!( + lambda["unmeasured_cells"], 0, + "every class has a measured median" ); + assert!(lambda["projected_secs"].as_u64() > Some(0), "{lambda}"); } -#[test] -fn a_skipped_cell_is_red_naming_it() { - let t = green(); - edit(&models(t.path(), "gx10"), |v| { - row(v, "serve", "off", "4k")["verdict"] = "skip".into(); - }); - assert_red_naming(&gate(t.path(), &[]), &cell("gx10", "serve", "off", "4k")); -} +// ── #3715's cases, on derived cells ────────────────────────────────────────────────────────────────────── #[test] -fn a_fallback_cell_is_red_naming_it() { +fn a_missing_cell_is_red_naming_it() { let t = green(); - edit(&models(t.path(), "lambda"), |v| { - row(v, "run", "off", "4k")["fallback"] = true.into(); - }); - assert_red_naming(&gate(t.path(), &[]), &cell("lambda", "run", "off", "4k")); + let id = pick(t.path(), gen_cell("lambda", "on", "consumer-max")); + drop_row(t.path(), "lambda", &id); + assert_red_naming(&gate(t.path(), &[]), &named(&id)); } #[test] -fn a_cell_that_does_not_say_whether_it_fell_back_is_red() { - let t = green(); - edit(&models(t.path(), "lambda"), |v| { - row(v, "code", "off", "4k") - .as_object_mut() - .expect("row") - .remove("fallback"); - }); - assert_red_naming(&gate(t.path(), &[]), &cell("lambda", "code", "off", "4k")); +fn a_skipped_a_fallback_or_a_silent_fallback_cell_is_red_naming_it() { + for change in ["skip", "fallback", "silent"] { + let t = green(); + let id = pick(t.path(), gen_cell("gx10", "off", "4k")); + edit_row(t.path(), "gx10", &id, |r| match change { + "skip" => r["verdict"] = "skip".into(), + "fallback" => r["fallback"] = true.into(), + _ => { + r.as_object_mut().expect("row").remove("fallback"); + } + }); + assert_red_naming(&gate(t.path(), &[]), &named(&id)); + } } #[test] -fn a_receipt_measured_at_another_commit_is_stale_and_red() { +fn a_receipt_of_another_commit_or_version_is_stale_and_red() { let t = green(); edit(&models(t.path(), "lambda"), |v| { v["apr_sha"] = PARENT.into() }); let v = assert_red_naming(&gate(t.path(), &[]), "release-host/0.69.1/lambda"); - let msgs = v["findings"].to_string(); - assert!( - msgs.contains(&cell("lambda", "run", "off", "4k")), - "every lambda cell is stale" - ); assert!( - !msgs.contains("release-host/0.69.1/gx10"), + !v["findings"] + .to_string() + .contains("release-host/0.69.1/gx10"), "gx10 is untouched" ); -} - -#[test] -fn a_receipt_for_another_version_is_stale_and_red() { let t = green(); edit(&models(t.path(), "gx10"), |v| { v["version"] = "0.69.0".into() @@ -270,56 +448,25 @@ fn a_receipt_for_another_version_is_stale_and_red() { #[test] fn a_think_on_row_with_an_unclosed_block_and_an_empty_answer_is_red() { let t = green(); - edit(&models(t.path(), "lambda"), |v| { - let r = row(v, "chat", "on", "4k"); + let id = pick(t.path(), gen_cell("lambda", "on", "4k")); + edit_row(t.path(), "lambda", &id, |r| { r["think_closed"] = false.into(); r["answer_chars"] = 0.into(); }); - let v = assert_red_naming(&gate(t.path(), &[]), &cell("lambda", "chat", "on", "4k")); + let v = assert_red_naming(&gate(t.path(), &[]), &named(&id)); let msgs = v["findings"].to_string(); - assert!(msgs.contains("thinkOk"), "names the unclosed block: {msgs}"); - assert!(msgs.contains("answered"), "names the empty answer: {msgs}"); -} - -#[test] -fn a_missing_think_on_cell_is_red_naming_it() { - let t = green(); - edit(&models(t.path(), "gx10"), |v| { - let i = row_index(v, "code", "on", "declared"); - v["cells"].as_array_mut().expect("cells").remove(i); - }); - assert_red_naming( - &gate(t.path(), &[]), - &cell("gx10", "code", "on", "declared"), + assert!( + msgs.contains("thinkOk") && msgs.contains("answered"), + "{msgs}" ); } #[test] -fn an_absent_declared_length_rung_is_red_and_so_is_an_unmeasured_length() { - let t = green(); - edit(&models(t.path(), "lambda"), |v| { - let i = row_index(v, "run", "off", "declared"); - v["cells"].as_array_mut().expect("cells").remove(i); - }); - assert_red_naming( - &gate(t.path(), &[]), - &cell("lambda", "run", "off", "declared"), - ); - - // the length is a property of the FILE (one sha256): gx10 dropping it leaves lambda's measurement standing, - // but gx10's own declared cells have no size to fill and go red … +fn a_declared_length_cell_missing_or_an_unmeasured_length_is_red() { let t = green(); - edit(&models(t.path(), "gx10"), |v| { - v["inventory"][0] - .as_object_mut() - .expect("item") - .remove("context_length"); - }); - assert_red_naming( - &gate(t.path(), &[]), - &cell("gx10", "run", "off", "declared"), - ); - // … and with no host measuring it, the model itself is named + let id = pick(t.path(), gen_cell("lambda", "off", "declared")); + drop_row(t.path(), "lambda", &id); + assert_red_naming(&gate(t.path(), &[]), &named(&id)); let t = green(); for h in ["lambda", "gx10"] { edit(&models(t.path(), h), |v| { @@ -333,73 +480,50 @@ fn an_absent_declared_length_rung_is_red_and_so_is_an_unmeasured_length() { } #[test] -fn two_hosts_that_disagree_on_one_files_length_are_red() { +fn a_prompt_short_of_its_rung_or_a_duplicate_row_is_red() { let t = green(); - edit(&models(t.path(), "gx10"), |v| { - v["inventory"][0]["context_length"] = 4096.into(); + let id = pick(t.path(), gen_cell("lambda", "on", "consumer-max")); + edit_row(t.path(), "lambda", &id, |r| { + r["prompt_tokens"] = 1000.into() }); - let v = assert_red_naming(&gate(t.path(), &[]), "contextLength"); - assert!(v["findings"].to_string().contains("maxCount")); -} - -#[test] -fn a_prompt_short_of_its_rung_is_red() { - let t = green(); - edit(&models(t.path(), "lambda"), |v| { - row(v, "serve", "on", "consumer-max")["prompt_tokens"] = 1000.into(); - }); - let v = assert_red_naming( - &gate(t.path(), &[]), - &cell("lambda", "serve", "on", "consumer-max"), - ); + let v = assert_red_naming(&gate(t.path(), &[]), &named(&id)); assert!(v["findings"].to_string().contains("contextMet")); -} - -#[test] -fn two_rows_for_one_cell_are_ambiguous_and_red() { let t = green(); + let id = pick(t.path(), gen_cell("lambda", "off", "4k")); edit(&models(t.path(), "lambda"), |v| { - let dup = row(v, "run", "on", "4k").clone(); - v["cells"].as_array_mut().expect("cells").push(dup); + let cells = v["cells"].as_array_mut().expect("cells"); + let dup = cells + .iter() + .find(|r| r["cell_id"] == id.as_str()) + .cloned() + .expect("row"); + cells.push(dup); }); - let v = assert_red_naming(&gate(t.path(), &[]), &cell("lambda", "run", "on", "4k")); + let v = assert_red_naming(&gate(t.path(), &[]), &named(&id)); assert!(v["findings"].to_string().contains("maxCount")); } #[test] -fn a_kernel_green_on_sm_121_and_red_on_sm_89_is_named_on_lambda_before_any_model_cell() { +fn a_kernel_green_on_sm_121_and_red_on_sm_89_is_named_before_any_cell() { let t = green(); edit(&kernels(t.path(), "lambda"), |v| { v["rows"][0]["verdict"] = "fail".into(); v["rows"][0]["max_err"] = 0.5.into(); }); - // and one model cell too, so the ORDER is observable - edit(&models(t.path(), "lambda"), |v| { - row(v, "run", "off", "4k")["verdict"] = "fail".into(); - }); + let id = pick(t.path(), gen_cell("lambda", "off", "4k")); + edit_row(t.path(), "lambda", &id, |r| r["verdict"] = "fail".into()); let v = assert_red_naming( &gate(t.path(), &[]), "release-kernel/0.69.1/lambda/q4k_gemv/q4_k", ); - let findings = v["findings"].as_array().expect("findings"); - let first = findings[0]["message"].as_str().unwrap_or(""); + let first = v["findings"][0]["message"].as_str().unwrap_or_default(); assert!( first.contains("release-kernel/0.69.1/lambda"), "kernel first: {first}" ); - assert!( - !v["findings"] - .to_string() - .contains("release-kernel/0.69.1/gx10"), - "gx10's kernel is green" - ); -} - -#[test] -fn a_kernel_measured_on_one_arch_only_is_a_missing_row_on_the_other() { let t = green(); edit(&kernels(t.path(), "gx10"), |v| { - v["rows"].as_array_mut().expect("rows").remove(1); // rmsnorm@f32 unmeasured on gx10 + v["rows"].as_array_mut().expect("rows").remove(1); }); assert_red_naming( &gate(t.path(), &[]), @@ -408,22 +532,27 @@ fn a_kernel_measured_on_one_arch_only_is_a_missing_row_on_the_other() { } #[test] -fn a_required_host_with_no_receipt_is_red_naming_the_host() { +fn a_required_host_with_no_receipt_or_a_v1_receipt_is_red_naming_the_host() { let t = green(); std::fs::remove_file(models(t.path(), "gx10")).expect("rm"); assert_red_naming(&gate(t.path(), &[]), "release-host/0.69.1/gx10"); + let t = green(); + edit(&models(t.path(), "gx10"), |v| { + v["schema"] = "apr-model-ladder-receipt/v1".into() + }); + assert_red_naming(&gate(t.path(), &[]), "release-host/0.69.1/gx10"); } #[test] fn an_inherited_dogfood_or_none_at_all_is_red() { let t = green(); edit(&t.path().join("dogfood-receipt.json"), |v| { - v["commit"] = PARENT.into(); + v["commit"] = PARENT.into() }); assert_red_naming(&gate(t.path(), &[]), "release-dogfood/0.69.1"); - let t = green(); let contracts = s(&t.path().join("contracts")); + let surface = s(&t.path().join("surface.json")); let r = pv( t.path(), &[ @@ -437,48 +566,34 @@ fn an_inherited_dogfood_or_none_at_all_is_red() { "0.69.1", "--release-commit", MC, + "--surface", + &surface, ], ); assert_red_naming(&r, "dogfoodReceipt"); } #[test] -fn a_consumer_with_no_recorded_number_is_red_on_the_derived_rung() { +fn a_consumer_with_no_number_or_a_thinking_contradiction_is_red() { let t = green(); edit(&t.path().join("evidence/release/context-rungs.json"), |v| { - v["consumers"].as_array_mut().expect("consumers").push( - serde_json::from_str::( - r#"{"consumer": "pending", "basis": "", "source": "fixture://pending"}"#, - ) - .expect("a consumer record"), - ); + let rec = serde_json::from_str::( + r#"{"consumer":"pending","basis":"","source":"fixture://p"}"#, + ) + .expect("record"); + v["consumers"].as_array_mut().expect("consumers").push(rec); }); assert_red_naming(&gate(t.path(), &[]), "release-context/consumer-max"); -} - -#[test] -fn thinking_that_contradicts_its_template_markers_is_red() { let t = green(); edit(&models(t.path(), "lambda"), |v| { - // off only, though the template carries the `enable_thinking` switch (#3723's derivation says both) v["inventory"][0]["thinking_modes"] = serde_json::from_str::(r#"["off"]"#).expect("modes"); }); assert_red_naming(&gate(t.path(), &[]), "thinkingContradiction"); } -#[test] -fn a_v1_receipt_cannot_stand_for_a_host() { - let t = green(); - edit(&models(t.path(), "gx10"), |v| { - v["schema"] = "apr-model-ladder-receipt/v1".into(); - }); - assert_red_naming(&gate(t.path(), &[]), "release-host/0.69.1/gx10"); -} - #[test] fn t4_receipts_measured_at_the_bump_head_pass_only_with_receipts_commit() { - // T-4: the committed receipts were measured at the bump head; R7 proved it equal to MC modulo evidence/ let t = green(); let other = t.path().join("t4-receipts"); std::fs::create_dir_all(&other).expect("mkdir"); @@ -487,19 +602,171 @@ fn t4_receipts_measured_at_the_bump_head_pass_only_with_receipts_commit() { std::fs::rename(models(t.path(), h), &dst).expect("mv"); edit(&dst, |v| v["apr_sha"] = BUMP.into()); edit(&kernels(t.path(), h), |v| v["apr_sha"] = BUMP.into()); - edit(&tokenizer(t.path()), |v| v["apr_sha"] = BUMP.into()); + } + edit(&tokenizer(t.path()), |v| v["apr_sha"] = BUMP.into()); + for h in ["lambda", "gx10"] { + edit( + &t.path().join(format!("evidence/crux/0.69.1/{h}-gpu.json")), + |v| { + v["apr"]["version_line"] = format!("apr 0.69.1 ({})", &BUMP[..9]).into(); + }, + ); } let dir = s(&other); let r = gate(t.path(), &["--receipts", &dir, "--receipts-commit", BUMP]); assert_eq!(r.code, 0, "{}", show(&r)); - let r = gate(t.path(), &["--receipts", &dir]); - assert_red_naming(&r, "release-host/0.69.1/lambda"); + assert_red_naming( + &gate(t.path(), &["--receipts", &dir]), + "release-host/0.69.1/lambda", + ); } #[test] -fn no_release_subject_declines_it_never_passes() { +fn one_token_id_off_llama_cpp_or_no_comparison_is_red_naming_the_model() { + for change in ["mismatch", "empty", "roundtrip", "absent"] { + let t = green(); + match change { + "mismatch" => edit(&tokenizer(t.path()), |v| { + v["rows"][0]["mismatches"] = 1.into() + }), + "empty" => edit(&tokenizer(t.path()), |v| { + v["rows"][0]["tokens_compared"] = 0.into() + }), + "roundtrip" => edit(&tokenizer(t.path()), |v| { + v["rows"][0]["roundtrip_ok"] = false.into() + }), + _ => std::fs::remove_file(tokenizer(t.path())).expect("rm"), + } + assert_red_naming( + &gate(t.path(), &[]), + "release-tokenizer/0.69.1/tiny-qwen35.gguf", + ); + } +} + +// ── the memory rule (#3710, operator decision "a") ─────────────────────────────────────────────────────── +// 16 GB weights + 1 MB/token KV + 1 GB workspace: `4k` (21.1 GB) fits the 24 GB card; `consumer-max` (25 GB) +// and `declared` (27 GB) do not, and are owed on lambda as honest pre-load refusals — and on gx10 as passes. + +fn memory_split(t: &Path) { + for (h, gpu) in [("lambda", 24 * GB), ("gx10", 119 * GB)] { + edit(&models(t, h), |v| { + v["gpu_mem_total_bytes"] = gpu.into(); + let item = &mut v["inventory"][0]; + item["weights_bytes"] = (16 * GB).into(); + item["kv_bytes_per_token"] = 1_000_000u64.into(); + item["workspace_bytes"] = GB.into(); + }); + } + let cells = read(&t.join("cells.json"))["cells"] + .as_array() + .expect("cells") + .clone(); + edit(&models(t, "lambda"), |v| { + for r in v["cells"].as_array_mut().expect("cells") { + let spec = cells + .iter() + .find(|c| c["id"] == r["cell_id"]) + .expect("spec"); + let need = match (spec["kind"].as_str(), spec["rung"].as_str()) { + (Some("matrix"), Some("consumer-max")) => 25 * GB, + (Some("matrix"), Some("declared")) => 27 * GB, + _ => continue, + }; + r["verdict"] = "refused".into(); + r["required_bytes"] = need.into(); + r["available_bytes"] = (24 * GB).into(); + } + }); +} + +#[test] +fn an_honest_refusal_on_lambda_and_a_pass_on_gx10_is_green() { + let t = green(); + memory_split(t.path()); + let r = gate(t.path(), &[]); + assert_eq!(r.code, 0, "{}", show(&r)); + assert!(json_of(&r)["release"]["refusal_cells"].as_u64() > Some(0)); +} + +#[test] +fn an_oom_a_co_tenant_refusal_or_a_rung_that_fits_nowhere_is_red() { + let t = green(); + memory_split(t.path()); + let id = pick(t.path(), gen_cell("lambda", "on", "declared")); + edit_row(t.path(), "lambda", &id, |r| { + r["verdict"] = "error".into(); + r["reason"] = "CUDA_ERROR_OUT_OF_MEMORY after 41 s of prefill".into(); + }); + assert_red_naming(&gate(t.path(), &[]), &named(&id)); + let t = green(); + memory_split(t.path()); + let id = pick(t.path(), gen_cell("lambda", "off", "consumer-max")); + edit_row(t.path(), "lambda", &id, |r| { + r["required_bytes"] = (23 * GB).into() + }); + let v = assert_red_naming(&gate(t.path(), &[]), &named(&id)); + assert!(v["findings"].to_string().contains("arithmeticNamed")); + let t = green(); + memory_split(t.path()); + edit(&models(t.path(), "gx10"), |v| { + v["gpu_mem_total_bytes"] = (24 * GB).into() + }); + assert_red_naming( + &gate(t.path(), &[]), + "release-coverage/0.69.1/tiny-qwen35.gguf/declared", + ); +} + +// ── #3745 S2: the cells are DERIVED ─────────────────────────────────────────────────────────────────────── + +#[test] +fn issue_mutant_1_a_new_flag_on_a_generating_command_owes_cells_nobody_has_run() { + let t = green(); + edit(&t.path().join("surface.json"), |v| { + let flag = serde_json::from_str::( + r#"{"id":"extra","long":"extra","value_type":"flag","values":["true","false"],"role":"mode"}"#, + ) + .expect("flag"); + v["commands"][0]["args"] + .as_array_mut() + .expect("args") + .push(flag); + }); + let v = assert_red_naming(&gate(t.path(), &[]), "--extra"); + assert!( + v["findings"] + .to_string() + .contains("release-cell/0.69.1/gen/"), + "the owed cells are gen's" + ); +} + +#[test] +fn issue_mutant_2_the_prompt_with_chat_cell_missing_is_red_naming_it() { + // #3743's shape: `-i file --chat` was covered, the `--prompt` shape was not + let t = green(); + let id = pick(t.path(), |c| { + c["command"] == "gen" + && c["kind"] == "matrix" + && c["host"] == "lambda" + && c["shape"] == "--prompt" + && c["args"].to_string().contains("--chat") + }); + drop_row(t.path(), "lambda", &id); + let v = assert_red_naming(&gate(t.path(), &[]), &named(&id)); + assert!( + id.contains("--prompt") && id.contains("--chat"), + "the cell names both: {id}" + ); + let _ = v; +} + +#[test] +fn no_surface_or_a_surface_that_does_not_say_what_generates_is_red() { let t = green(); let contracts = s(&t.path().join("contracts")); + let dogfood = s(&t.path().join("dogfood-receipt.json")); let r = pv( t.path(), &[ @@ -509,37 +776,117 @@ fn no_release_subject_declines_it_never_passes() { "shapes", "--shape", "release-readiness-v1", + "--release-version", + "0.69.1", + "--release-commit", + MC, + "--dogfood-receipt", + &dogfood, ], ); + assert_red_naming(&r, "release/surface"); + let t = green(); + edit(&t.path().join("surface.json"), |v| { + v["commands"][0] + .as_object_mut() + .expect("cmd") + .remove("generates"); + }); + assert_red_naming(&gate(t.path(), &[]), "generatesUndeclared"); +} + +#[test] +fn an_unknown_role_arg_beyond_its_ceiling_grows_the_ratchet_and_is_red() { + let t = green(); + edit(&t.path().join("surface.json"), |v| { + let arg = serde_json::from_str::( + r#"{"id":"mystery","long":"mystery","value_type":"text","role":"unknown"}"#, + ) + .expect("arg"); + v["commands"][1]["args"] + .as_array_mut() + .expect("args") + .push(arg); + }); + assert_red_naming(&gate(t.path(), &[]), "ratchetGrew"); +} + +#[test] +fn a_flag_whose_output_never_changes_is_red_naming_the_setting() { + // S2.5: the effect cell prints exactly what its base printed + let t = green(); + let cells = read(&t.path().join("cells.json"))["cells"] + .as_array() + .expect("cells") + .clone(); + for h in ["lambda", "gx10"] { + edit(&models(t.path(), h), |v| { + for r in v["cells"].as_array_mut().expect("cells") { + let spec = cells + .iter() + .find(|c| c["id"] == r["cell_id"]) + .expect("spec"); + if spec["kind"] == "effect" { + r["output_sha256"] = + format!("out:{}", spec["base"].as_str().unwrap_or_default()).into(); + } + } + }); + } + assert_red_naming( + &gate(t.path(), &[]), + "release-effect/0.69.1/gen/--chat%3Dtrue", + ); +} + +#[test] +fn a_modelless_command_must_run_or_refuse_by_name_and_a_model_command_must_pass_or_refuse() { + let t = green(); + let id = pick(t.path(), |c| { + c["command"] == "list" && c["host"] == "lambda" + }); + edit_row(t.path(), "lambda", &id, |r| r["verdict"] = "refused".into()); assert_eq!( - r.code, - 2, - "zero cells is a decline, not a pass: {}", - show(&r) + gate(t.path(), &[]).code, + 0, + "a typed refusal by name is an answer" ); + edit_row(t.path(), "lambda", &id, |r| r["verdict"] = "fail".into()); + assert_red_naming(&gate(t.path(), &[]), &named(&id)); + let t = green(); + let id = pick(t.path(), |c| { + c["command"] == "look" && c["kind"] == "matrix" && c["host"] == "gx10" + }); + drop_row(t.path(), "gx10", &id); + assert_red_naming(&gate(t.path(), &[]), &named(&id)); } +// ── the CLI's own answers ──────────────────────────────────────────────────────────────────────────────── + #[test] -fn a_malformed_subject_or_a_misplaced_flag_is_the_callers_error() { +fn no_release_subject_declines_and_a_malformed_one_is_the_callers_error() { let t = green(); let contracts = s(&t.path().join("contracts")); - let partial = pv( + let r = pv( t.path(), &[ "lint", &contracts, "--gate", "shapes", - "--release-version", - "0.69.1", + "--shape", + "release-readiness-v1", ], ); - assert_eq!(partial.code, 3, "{}", show(&partial)); - let short = pv( - t.path(), - &[ - "lint", - &contracts, + assert_eq!( + r.code, + 2, + "zero cells is a decline, not a pass: {}", + show(&r) + ); + for bad in [ + vec!["--gate", "shapes", "--release-version", "0.69.1"], + vec![ "--gate", "shapes", "--release-version", @@ -547,266 +894,266 @@ fn a_malformed_subject_or_a_misplaced_flag_is_the_callers_error() { "--release-commit", "225b2a9ab", ], - ); - assert_eq!( - short.code, - 3, - "a prefix is refused, never matched: {}", - show(&short) - ); - let misplaced = pv( - t.path(), - &["lint", &contracts, "--gate", "sigma", "--shape", "x"], - ); - assert_eq!(misplaced.code, 3, "{}", show(&misplaced)); - let unknown = pv( - t.path(), - &[ - "lint", - &contracts, - "--gate", - "shapes", - "--shape", - "no-such-shape", - ], - ); - assert_eq!(unknown.code, 3, "{}", show(&unknown)); + vec!["--gate", "sigma", "--shape", "x"], + vec!["--gate", "shapes", "--shape", "no-such-shape"], + ] { + let mut args = vec!["lint", contracts.as_str()]; + args.extend(bad.iter().copied()); + let r = pv(t.path(), &args); + assert_eq!(r.code, 3, "{bad:?}: {}", show(&r)); + } } #[test] -fn pv_extract_writes_the_release_graph_only_where_it_is_told() { +fn pv_extract_writes_the_release_graph_and_the_cell_list_only_where_told() { let t = green(); let contracts = s(&t.path().join("contracts")); - let out = t.path().join("release.nt"); - let dogfood = s(&t.path().join("dogfood-receipt.json")); - let args = [ - "extract", - &contracts, - "--release-version", - "0.69.1", - "--release-commit", - MC, - "--dogfood-receipt", - &dogfood, - ]; - let refused = pv(t.path(), &args); + let refused = pv( + t.path(), + &[ + "extract", + &contracts, + "--release-version", + "0.69.1", + "--release-commit", + MC, + ], + ); assert_eq!(refused.code, 3, "no --out: {}", show(&refused)); + let cells = read(&t.path().join("cells.json")); + assert_eq!(cells["schema"], "apr-release-cells/v1"); + assert_eq!(cells["release_commit"], MC); + let nt = std::fs::read_to_string(t.path().join("release.nt")).expect("written"); + assert!( + nt.contains("release/Cell") && nt.contains("cli/Command"), + "cells and the surface are in the graph" + ); + assert!(!nt.contains("_:"), "no blank nodes (R-15)"); assert!( !t.path().join("contracts/contracts.nt").exists(), "the tracked file is untouched" ); - let out_s = s(&out); - let mut with_out = args.to_vec(); - with_out.extend_from_slice(&["--out", &out_s]); - let r = pv(t.path(), &with_out); - assert_eq!(r.code, 0, "{}", show(&r)); - let v = json_of(&r); - assert_eq!(v["release"]["cells"], 48); - let nt = std::fs::read_to_string(&out).expect("written"); - assert!(nt.contains("release/Cell"), "the cells are in the graph"); - assert!(nt.contains("release/KernelDiffReceipt")); - assert!(!nt.contains("_:"), "no blank nodes (R-15)"); - assert!(!t.path().join("contracts/contracts.nt").exists()); } -fn tokenizer(t: &Path) -> PathBuf { - t.join("evidence/dogfood/tokenizer/0.69.1/parity.json") +// ── S2.4: CRUX for every derived verb (#3739) ─────────────────────────────────────────────────────────── + +fn crux_map(t: &Path) -> PathBuf { + t.join("evidence/crux/verb-correspondence.yaml") } -const TOK_CELL: &str = "release-tokenizer/0.69.1/tiny-qwen35.gguf"; +fn crux_receipt(t: &Path, host: &str) -> PathBuf { + t.join(format!("evidence/crux/0.69.1/{host}-gpu.json")) +} -#[test] -fn one_token_id_that_disagrees_with_llama_cpp_is_red_naming_the_model() { - // #3726: apr mapped every non-ASCII byte to id 0 and `apr parity` could not see it (CPU and GPU share it) - let t = green(); - edit(&tokenizer(t.path()), |v| { - v["rows"][0]["mismatches"] = 1.into() - }); - let v = assert_red_naming(&gate(t.path(), &[]), TOK_CELL); - assert!(v["findings"].to_string().contains("parityExact")); +fn rewrite(path: &Path, from: &str, to: &str) { + let text = std::fs::read_to_string(path).expect("read"); + assert!(text.contains(from), "{from:?} is in {}", path.display()); + std::fs::write(path, text.replace(from, to)).expect("write"); } #[test] -fn a_tokenizer_comparison_over_nothing_or_a_broken_round_trip_or_no_receipt_is_red() { +fn a_verb_without_an_entry_an_engine_it_omits_or_a_none_without_reason_is_red() { let t = green(); - edit(&tokenizer(t.path()), |v| { - v["rows"][0]["tokens_compared"] = 0.into() - }); - assert_red_naming(&gate(t.path(), &[]), TOK_CELL); - + rewrite( + &crux_map(t.path()), + " - verb: look\n comparator: none\n reason: \"fixture: reading a model's header has no comparator counterpart\"\n", + "", + ); + assert_red_naming(&gate(t.path(), &[]), "release-crux-verb/0.69.1/look"); let t = green(); - edit(&tokenizer(t.path()), |v| { - v["rows"][0]["roundtrip_ok"] = false.into() - }); - let v = assert_red_naming(&gate(t.path(), &[]), TOK_CELL); - assert!(v["findings"].to_string().contains("roundTrip")); - + rewrite( + &crux_map(t.path()), + " llamafile: {none: \"fixture: no llamafile build pinned\"}\n", + "", + ); + assert_red_naming(&gate(t.path(), &[]), "engineMissing"); let t = green(); - std::fs::remove_file(tokenizer(t.path())).expect("rm"); - assert_red_naming(&gate(t.path(), &[]), TOK_CELL); + rewrite( + &crux_map(t.path()), + "reason: \"fixture: listing is apr's own\"", + "reason: \"\"", + ); + assert_red_naming(&gate(t.path(), &[]), "noneWithoutReason"); } -// ── the memory rule (#3710, operator decision "a") ────────────────────────────────────────────────────── -// weights 16 GB + 1 MB/token KV + 1 GB workspace: `4k` (21.1 GB) fits the 24 GB card; `consumer-max` (25 GB) and -// `declared` (27 GB) do not, and are owed on lambda as honest pre-load refusals — and on gx10 (119 GB) as passes. - -const GB: u64 = 1_000_000_000; - -fn memory_split(t: &Path) { - for (h, gpu) in [("lambda", 24 * GB), ("gx10", 119 * GB)] { - edit(&models(t, h), |v| { - v["gpu_mem_total_bytes"] = gpu.into(); - let item = &mut v["inventory"][0]; - item["weights_bytes"] = (16 * GB).into(); - item["kv_bytes_per_token"] = 1_000_000u64.into(); - item["workspace_bytes"] = GB.into(); +#[test] +fn a_red_an_unjudged_or_an_absent_crux_cell_is_red_and_all_wrong_is_named_not_failed() { + for verdict in ["RED", "UNJUDGED"] { + let t = green(); + edit(&crux_receipt(t.path(), "lambda"), |v| { + v["cells"][0]["verdict"] = verdict.into() }); + assert_red_naming( + &gate(t.path(), &[]), + "release-crux/0.69.1/lambda/tiny-qwen35.gguf/gen", + ); } - edit(&models(t, "lambda"), |v| { - for c in v["cells"].as_array_mut().expect("cells") { - let need = match c["context"].as_str() { - Some("consumer-max") => 25 * GB, - Some("declared") => 27 * GB, - _ => continue, - }; - c["verdict"] = "refused".into(); - c["required_bytes"] = need.into(); - c["available_bytes"] = (24 * GB).into(); - c["reason"] = - "pre-load refusal: KV for this context needs more than the device holds".into(); - } - }); -} - -#[test] -fn a_27b_that_refuses_honestly_on_lambda_and_passes_on_gx10_is_green() { let t = green(); - memory_split(t.path()); + std::fs::remove_file(crux_receipt(t.path(), "gx10")).expect("rm"); + assert_red_naming( + &gate(t.path(), &[]), + "release-crux/0.69.1/gx10/tiny-qwen35.gguf/gen", + ); + let t = green(); + edit(&crux_receipt(t.path(), "lambda"), |v| { + v["cells"][0]["verdict"] = "ALL_WRONG".into() + }); let r = gate(t.path(), &[]); - assert_eq!(r.code, 0, "{}", show(&r)); - let v = json_of(&r); assert_eq!( - v["release"]["refusal_cells"], 16, - "4 verbs × 2 modes × (consumer-max, declared) on lambda" + r.code, + 0, + "ALL_WRONG is named, not a violation: {}", + show(&r) ); - assert_eq!(v["release"]["cells"], 48, "a refusal is still an owed cell"); + assert_eq!(json_of(&r)["release"]["crux"]["all_wrong"], 1); } +// ── #3748 and the CRUX harness guard ───────────────────────────────────────────────────────────────────── + #[test] -fn the_gx10_cell_that_covers_the_rung_missing_is_red() { +fn a_correctness_cell_validated_by_another_binarys_f2_receipt_is_unmeasured_and_red() { let t = green(); - memory_split(t.path()); - edit(&models(t.path(), "gx10"), |v| { - let i = row_index(v, "chat", "off", "declared"); - v["cells"].as_array_mut().expect("cells").remove(i); + let id = pick(t.path(), gen_cell("lambda", "off", "4k")); + edit_row(t.path(), "lambda", &id, |r| { + r["f2_source"] = "receipt".into(); + r["f2_receipt_binary_sha"] = PARENT.into(); }); - assert_red_naming( - &gate(t.path(), &[]), - &cell("gx10", "chat", "off", "declared"), - ); -} - -#[test] -fn an_oom_mid_run_where_a_refusal_is_owed_is_red() { + let v = assert_red_naming(&gate(t.path(), &[]), &named(&id)); + assert!(v["findings"].to_string().contains("f2Measured")); let t = green(); - memory_split(t.path()); - edit(&models(t.path(), "lambda"), |v| { - let r = row(v, "serve", "on", "declared"); - r["verdict"] = "error".into(); - r["reason"] = "CUDA_ERROR_OUT_OF_MEMORY after 41 s of prefill".into(); - r.as_object_mut().expect("row").remove("required_bytes"); - r.as_object_mut().expect("row").remove("available_bytes"); + let id = pick(t.path(), gen_cell("gx10", "on", "4k")); + edit_row(t.path(), "gx10", &id, |r| { + r["f2_source"] = "receipt".into(); + r["f2_receipt_binary_sha"] = MC.into(); }); - let v = assert_red_naming( - &gate(t.path(), &[]), - &cell("lambda", "serve", "on", "declared"), + assert_eq!( + gate(t.path(), &[]).code, + 0, + "this binary's own F2 receipt counts" ); - assert!(v["findings"] - .to_string() - .contains("release-readiness-v1.refusal")); + edit_row(t.path(), "gx10", &id, |r| { + r.as_object_mut().expect("row").remove("f2_source"); + }); + assert_red_naming(&gate(t.path(), &[]), &named(&id)); } #[test] -fn a_refusal_whose_arithmetic_shows_no_shortfall_is_red() { +fn a_broken_crux_harness_declines_the_gate_naming_the_model() { let t = green(); - memory_split(t.path()); - edit(&models(t.path(), "lambda"), |v| { - row(v, "code", "off", "consumer-max")["required_bytes"] = (20 * GB).into(); + edit(&crux_receipt(t.path(), "lambda"), |v| { + for c in v["cells"].as_array_mut().expect("cells") { + if c["positive_control"] == true { + c["verdict"] = "ALL_WRONG".into(); + } + } }); - let v = assert_red_naming( - &gate(t.path(), &[]), - &cell("lambda", "code", "off", "consumer-max"), + let r = gate(t.path(), &[]); + assert_eq!( + r.code, + 2, + "a positive control that is ALL_WRONG declines: {}", + show(&r) + ); + assert!( + r.stderr.contains("tiny-qwen35.gguf") && r.stderr.contains("ALL_WRONG"), + "{}", + show(&r) + ); + let t = green(); + for h in ["lambda", "gx10"] { + edit(&crux_receipt(t.path(), h), |v| { + for c in v["cells"].as_array_mut().expect("cells") { + c["positive_control"] = false.into(); + } + }); + } + let r = gate(t.path(), &[]); + assert_eq!(r.code, 2, "no measured control declines: {}", show(&r)); + assert!( + r.stderr.contains("no measured positive-control cell"), + "{}", + show(&r) ); - assert!(v["findings"].to_string().contains("arithmeticNamed")); } -#[test] -fn a_cell_that_fits_by_the_arithmetic_but_is_missing_is_red() { - let t = green(); - memory_split(t.path()); - edit(&models(t.path(), "lambda"), |v| { - let i = row_index(v, "run", "on", "4k"); - v["cells"].as_array_mut().expect("cells").remove(i); +// ── S2.5: a8's sampling controls over typed `sampling` args ───────────────────────────────────────────── + +/// Set the output digest of `host`'s sampling control `control`. +fn set_control(t: &Path, host: &str, control: &str, out: &str) { + let id = pick(t, |c| { + c["kind"] == "sampling" && c["host"] == host && c["control"] == control }); - assert_red_naming(&gate(t.path(), &[]), &cell("lambda", "run", "on", "4k")); + edit_row(t, host, &id, |r| r["output_sha256"] = out.into()); } #[test] -fn a_rung_that_fits_on_no_required_host_is_red_naming_the_rung() { - let t = green(); - memory_split(t.path()); - edit(&models(t.path(), "gx10"), |v| { - v["gpu_mem_total_bytes"] = (24 * GB).into(); - }); - assert_red_naming( - &gate(t.path(), &[]), - &format!("release-coverage/0.69.1/{MODEL}/declared"), - ); +fn each_of_a8s_sampling_controls_turns_the_gate_red_when_broken() { + let cases: [(&str, &str, &str); 4] = [ + ("seed-a-again", "drifted", "seedRepeatable"), + ("seed-b", "sampled-a", "seedsDiffer"), + ("topk1", "not-greedy", "greedyAgrees"), + ("seed-a", "greedy", "sampledDiffers"), + ]; + for (control, out, check) in cases { + let t = green(); + set_control(t.path(), "lambda", control, out); + if check == "sampledDiffers" { + set_control(t.path(), "lambda", "seed-a-again", "greedy"); + set_control(t.path(), "lambda", "seed-b", "greedy"); + } + let v = assert_red_naming(&gate(t.path(), &[]), "release-sampling/0.69.1/lambda/gen"); + assert!(v["findings"].to_string().contains(check), "{check}"); + } } #[test] -fn a_refusal_because_a_co_tenant_holds_memory_is_red() { - // required 23 GB ≤ the card's 24 GB total, refused because only 20 GB were free: a busy GPU, not the - // arithmetic — it must not shrink what the release proves (#3712, aprender-62) +fn a_control_the_command_cannot_express_is_named_never_passed() { let t = green(); - memory_split(t.path()); - edit(&models(t.path(), "lambda"), |v| { - let r = row(v, "chat", "off", "consumer-max"); - r["required_bytes"] = (23 * GB).into(); - r["available_bytes"] = (20 * GB).into(); + edit(&t.path().join("surface.json"), |v| { + v["commands"][0]["args"] + .as_array_mut() + .expect("args") + .retain(|a| a["sampling_kind"] != "seed"); }); - let v = assert_red_naming( - &gate(t.path(), &[]), - &cell("lambda", "chat", "off", "consumer-max"), + synthesize(t.path()); + let v = assert_red_naming(&gate(t.path(), &[]), "underivable"); + assert!( + v["findings"].to_string().contains("seed-a"), + "names the control it cannot run" ); - assert!(v["findings"].to_string().contains("arithmeticNamed")); } #[test] -fn a_model_whose_template_always_opens_think_owes_only_the_on_cells() { - // Qwen3-style prefill that always opens (#3723): ON only — the OFF cells are not owed, so removing - // every OFF row leaves the release green, and removing an ON row does not +fn a_verb_whose_entry_declares_request_modes_owes_a_crux_cell_in_each() { + // the FILE says which verb has modes (#3739 slice 4); pv never names one let t = green(); + rewrite( + &crux_map(t.path()), + " judged_by: golden\n", + " judged_by: golden\n modes: [nonstream, stream]\n", + ); + let v = assert_red_naming(&gate(t.path(), &[]), "modeMissing"); + assert!(v["findings"].to_string().contains("stream")); for h in ["lambda", "gx10"] { - edit(&models(t.path(), h), |v| { - v["inventory"][0]["thinking_modes"] = - serde_json::from_str::(r#"["on"]"#).expect("modes"); - v["inventory"][0]["thinking_markers"] = - serde_json::from_str::(r#"[""]"#).expect("markers"); - let cells = v["cells"].as_array_mut().expect("cells"); - cells.retain(|c| c["thinking"] == "on"); + edit(&crux_receipt(t.path(), h), |v| { + let rows: Vec = v["cells"].as_array().expect("cells").clone(); + let mut out = Vec::new(); + for r in rows { + for m in ["nonstream", "stream"] { + let mut r2 = r.clone(); + r2["key"]["mode"] = m.into(); + out.push(r2); + } + } + v["cells"] = Value::Array(out); }); } - let r = gate(t.path(), &[]); - assert_eq!(r.code, 0, "{}", show(&r)); - assert_eq!( - json_of(&r)["release"]["cells"], - 24, - "2 hosts × 4 verbs × ON × 3 rungs" - ); - edit(&models(t.path(), "gx10"), |v| { - let i = row_index(v, "code", "on", "4k"); - v["cells"].as_array_mut().expect("cells").remove(i); - }); - assert_red_naming(&gate(t.path(), &[]), &cell("gx10", "code", "on", "4k")); + assert_eq!(gate(t.path(), &[]).code, 0, "a row in each declared mode"); +} + +/// A JSON array of strings as `Vec<&str>` (`json!` array literals expand to a disallowed `unwrap`). +fn names(v: &serde_json::Value) -> Vec<&str> { + v.as_array() + .map(|a| a.iter().filter_map(serde_json::Value::as_str).collect()) + .unwrap_or_default() } diff --git a/crates/aprender-contracts-cli/tests/pv_surface_gate.rs b/crates/aprender-contracts-cli/tests/pv_surface_gate.rs index be97935aac..5c87d68d80 100644 --- a/crates/aprender-contracts-cli/tests/pv_surface_gate.rs +++ b/crates/aprender-contracts-cli/tests/pv_surface_gate.rs @@ -691,23 +691,14 @@ fn validate_reaches_every_rule_at_the_right_severity() { /// /// The validator source is read at RUN time from the sibling crate (#4129). It was an /// `include_str!("../../aprender-contracts/…")`, a path OUTSIDE this crate, so `cargo test` from -/// the published aprender-contracts-cli tarball could not compile this test file. In tree, a -/// missing validator.rs FAILS; only a build with no sibling `aprender-contracts/` (the crates.io -/// tarball) returns `None`, and the caller skips by name. +/// the published aprender-contracts-cli tarball could not compile this test file. The in-tree +/// decision is the shared rule, `provable_contracts::workspace_file_or_skip!`: in tree a missing +/// validator.rs FAILS; out of tree it returns `None` and the caller skips by name. fn declared_rule_ids() -> Option> { - let sibling = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../aprender-contracts"); - if !sibling.is_dir() { - eprintln!( - "SKIP every_rule_in_the_validator_source_appears_in_the_table: out of tree (no {} \ - beside this crate) - the rule universe is read from the sibling crate's source, \ - which a published crate does not carry (#4129)", - sibling.display() - ); - return None; - } - let path = sibling.join("src/schema/validator.rs"); - let src = std::fs::read_to_string(&path) - .unwrap_or_else(|e| panic!("in tree, {} must be readable: {e}", path.display())); + let src = provable_contracts::workspace_file_or_skip!( + "every_rule_in_the_validator_source_appears_in_the_table", + "crates/aprender-contracts/src/schema/validator.rs", + )?; let mut ids = BTreeSet::new(); for chunk in src.split('"').skip(1).step_by(2) { if is_rule_id_shaped(chunk) { diff --git a/crates/aprender-contracts-cli/tests/pvl_challenge.rs b/crates/aprender-contracts-cli/tests/pvl_challenge.rs new file mode 100644 index 0000000000..3c0a56e35f --- /dev/null +++ b/crates/aprender-contracts-cli/tests/pvl_challenge.rs @@ -0,0 +1,199 @@ +//! PVL-001 EV-7a (#4200) — `pv challenge gen | check`, end to end on a built-in-a-tempdir Lean tree. +//! +//! The RED rows carry the spec's mutation: a hand-edited statement in `Challenge/`. The Lean elaboration of the +//! generated files is not run here (CI has no Mathlib); it was measured on lambda (see #4200). + +use std::path::PathBuf; +use std::process::Command; + +struct Run { + code: i32, + stdout: String, + stderr: String, +} + +impl Run { + fn show(&self) -> String { + format!( + "rc {}\n--- stdout\n{}\n--- stderr\n{}", + self.code, self.stdout, self.stderr + ) + } +} + +const CHALLENGE: &str = "lean/Challenge/gelu-v1.lean"; + +struct Fx { + dir: tempfile::TempDir, +} + +impl Fx { + /// `gelu_bound` is bound by exact name; `unbound` by nothing. + fn new() -> Self { + let fx = Self { + dir: tempfile::tempdir().expect("tempdir"), + }; + fx.write( + "lean/ProvableContracts.lean", + "import ProvableContracts.Theorems.Gelu.Bound\n", + ); + fx.write( + "lean/ProvableContracts/Theorems/Gelu/Bound.lean", + "namespace ProvableContracts.Gelu\n\ + theorem gelu_bound (x : Nat) :\n x ≤ x + 1 := Nat.le_succ x\n\ + theorem unbound : True := trivial\n\ + end ProvableContracts.Gelu\n", + ); + fx.write( + "contracts/gelu-v1.yaml", + "equations:\n e:\n lean_theorem: ProvableContracts.Gelu.gelu_bound\n", + ); + fx + } + + fn path(&self, rel: &str) -> PathBuf { + self.dir.path().join(rel) + } + + fn write(&self, rel: &str, text: &str) { + let p = self.path(rel); + std::fs::create_dir_all(p.parent().expect("parent")).expect("mkdir"); + std::fs::write(p, text).expect("write"); + } + + fn read(&self, rel: &str) -> String { + std::fs::read_to_string(self.path(rel)).expect("read") + } + + fn pv(&self, action: &str) -> Run { + let out = Command::new(env!("CARGO_BIN_EXE_pv")) + .current_dir(self.dir.path()) + .args(["challenge", action, "contracts", "lean"]) + .output() + .expect("spawn pv"); + Run { + code: out.status.code().unwrap_or(-1), + stdout: String::from_utf8_lossy(&out.stdout).into_owned(), + stderr: String::from_utf8_lossy(&out.stderr).into_owned(), + } + } + + fn generated(&self) { + let g = self.pv("gen"); + assert_eq!(g.code, 0, "{}", g.show()); + } +} + +#[test] +fn gen_writes_the_statement_with_sorry_and_check_is_then_fresh() { + let fx = Fx::new(); + fx.generated(); + let text = fx.read(CHALLENGE); + assert!( + text.contains( + "theorem _root_.PvlChallenge.ProvableContracts.Gelu.gelu_bound (x : Nat) : x ≤ x + 1 := sorry\n" + ), + "{text}" + ); + assert!( + text.contains("namespace ProvableContracts.Gelu\n"), + "{text}" + ); + assert!(!text.contains("unbound"), "{text}"); + let c = fx.pv("check"); + assert_eq!(c.code, 0, "{}", c.show()); + assert!( + c.stdout.contains("challenge-fresh: 1 file(s)"), + "{}", + c.show() + ); +} + +#[test] +fn gen_is_deterministic() { + let fx = Fx::new(); + fx.generated(); + let first = fx.read(CHALLENGE); + fx.generated(); + assert_eq!(fx.read(CHALLENGE), first); +} + +/// EV-7a's mutation: hand-edit a statement → RED, naming the file and the line. +#[test] +fn a_hand_edited_statement_is_red() { + let fx = Fx::new(); + fx.generated(); + let text = fx.read(CHALLENGE); + fx.write(CHALLENGE, &text.replace("x ≤ x + 1", "x ≤ x + 2")); + let c = fx.pv("check"); + assert_eq!(c.code, 1, "{}", c.show()); + assert!( + c.stdout + .contains("FAIL STALE differs Challenge/gelu-v1.lean:"), + "{}", + c.show() + ); +} + +/// The solution's statement changing (a weakened theorem) leaves the committed challenge stale → RED. +#[test] +fn a_changed_solution_statement_is_red_until_regenerated() { + let fx = Fx::new(); + fx.generated(); + let src = "lean/ProvableContracts/Theorems/Gelu/Bound.lean"; + let text = fx.read(src); + fx.write(src, &text.replace("x ≤ x + 1", "0 ≤ x")); + assert_eq!(fx.pv("check").code, 1); + fx.generated(); + assert_eq!(fx.pv("check").code, 0); +} + +#[test] +fn an_extra_or_missing_challenge_file_is_red() { + let fx = Fx::new(); + fx.generated(); + fx.write("lean/Challenge/Sub/stray.lean", ""); + let c = fx.pv("check"); + assert_eq!(c.code, 1, "{}", c.show()); + assert!( + c.stdout.contains("extra Challenge/Sub/stray.lean"), + "{}", + c.show() + ); + fx.generated(); + assert!(!fx.path("lean/Challenge/Sub/stray.lean").exists()); + std::fs::remove_file(fx.path(CHALLENGE)).expect("rm"); + let c = fx.pv("check"); + assert_eq!(c.code, 1, "{}", c.show()); + assert!( + c.stdout.contains("missing Challenge/gelu-v1.lean"), + "{}", + c.show() + ); +} + +/// Zero challenges is a decline (rc 2), never "fresh": nothing is pinned. +#[test] +fn zero_challenges_declines() { + let fx = Fx::new(); + fx.write( + "contracts/gelu-v1.yaml", + "equations:\n e:\n formula: x\n", + ); + for action in ["gen", "check"] { + let r = fx.pv(action); + assert_eq!(r.code, 2, "{action}: {}", r.show()); + assert!( + r.stderr.contains("zero challenges"), + "{action}: {}", + r.show() + ); + } +} + +#[test] +fn an_unloadable_tree_declines() { + let fx = Fx::new(); + std::fs::remove_file(fx.path("lean/ProvableContracts.lean")).expect("rm"); + assert_eq!(fx.pv("check").code, 2); +} diff --git a/crates/aprender-contracts-cli/tests/pvl_comparator.rs b/crates/aprender-contracts-cli/tests/pvl_comparator.rs new file mode 100644 index 0000000000..001b7df07d --- /dev/null +++ b/crates/aprender-contracts-cli/tests/pvl_comparator.rs @@ -0,0 +1,295 @@ +//! PVL-001 EV-7b (#4201) — `pv discharge check --comparator`, end to end with a stub `lake` on PATH. +//! +//! The spec's RED: a solution that proves a WEAKER statement than its Challenge pins → rc 1 (MISMATCH), and a +//! solution on `sorry` → rc 1. No Challenge file, or zero rows → rc 2, never a pass. The stub replays rows the +//! committed `scripts/Comparator.lean` produced on a real Lean tree; the real run needs the built tree and +//! Mathlib, and is measured on lambda (see #4201). The last test pins the committed script to the row shape the +//! parser reads, so the two cannot drift apart silently. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use provable_contracts::discharge::comparator::{CHALLENGE_NS, COMPARATOR, SORRY_AXIOM}; + +fn pv_bin() -> PathBuf { + PathBuf::from(env!("CARGO_BIN_EXE_pv")) +} + +/// sha256 hashes `scripts/Comparator.lean` printed for `gelu_pos (x : Nat) (h : 0 < x) : 0 < g x`, and for the +/// same theorem with its hypothesis strengthened to `1 < x` (measured with Lean v4.29.0-rc4, 2026-09-24). +const PINNED: &str = "902d0aa1835a8af24db87013bf140f155ab57b9002d3665f37294f2a3623b6fd"; +const WEAKER: &str = "66afc7f44f64333e7cc3a4c495e91ce8d3e376d8e813accc9542f2a9f5397224"; +const NAME: &str = "ProvableContracts.Gelu.gelu_pos"; + +struct Fx { + dir: tempfile::TempDir, +} + +impl Fx { + /// A one-theorem tree with `Axioms.lean` generated, a Challenge file and the comparator script in place. + fn new() -> Self { + let fx = Self { + dir: tempfile::tempdir().expect("tempdir"), + }; + fx.write( + "lean/ProvableContracts.lean", + "import ProvableContracts.Theorems.Gelu.Bound\n", + ); + fx.write( + "lean/ProvableContracts/Theorems/Gelu/Bound.lean", + "namespace ProvableContracts.Gelu\ntheorem gelu_bound : True := trivial\nend ProvableContracts.Gelu\n", + ); + fx.write( + "contracts/gelu-v1.yaml", + "equations:\n e:\n lean_theorem: Theorems.Gelu\n", + ); + fx.write( + "lean/Challenge/gelu-v1.lean", + &format!("-- EV-7a writes this\ntheorem _root_.{CHALLENGE_NS}.{NAME} (x : Nat) (h : 0 < x) : 0 < g x := sorry\n"), + ); + fx.write( + &format!("lean/{COMPARATOR}"), + "-- the real one is committed\n", + ); + let g = fx.pv(&["discharge", "gen-axioms", "lean"]); + assert_eq!(g.0, 0, "gen-axioms: {}", g.1); + fx + } + + fn path(&self, rel: &str) -> PathBuf { + self.dir.path().join(rel) + } + + fn write(&self, rel: &str, text: &str) { + let p = self.path(rel); + std::fs::create_dir_all(p.parent().expect("parent")).expect("mkdir"); + std::fs::write(p, text).expect("write"); + } + + /// `bin/lake` on the test's PATH: `env lean --run