Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,27 @@ fn the_entity_type_is_counted_in_by_entity_type_not_merely_registered() {
);
}

#[test]
fn the_parity_extractor_control_is_drawn_by_the_gate_every_run() {
// PMAT-3704. ONT-001 v4.10's probe also asks `.pc_extract["parity-receipt"] == "fired"`, at the TOP level
// of the report, exactly as read here. The control existed from #3600 on — `parity_receipt::positive_control`
// — but only a unit test called it, so the gate never reported it and an extractor that stopped reading the
// v2 layout would have been counted, never refused. R-3: one planted defect per registered extractor, every
// run.
let out = Command::new(pv_bin())
.args(["lint", "contracts", "--gate", "shapes", "--format", "json"])
.current_dir(repo_root())
.output()
.expect("failed to spawn pv");
let v: serde_json::Value =
serde_json::from_str(&String::from_utf8_lossy(&out.stdout)).expect("json report");
assert_eq!(
v["pc_extract"]["parity-receipt"], "fired",
"pc_extract carries no fired `parity-receipt` control: {}",
v["pc_extract"]
);
}

#[test]
fn every_fixture_carries_the_real_contract_byte_for_byte() {
// A fixture copy that drifts from `contracts/parity-receipt-v2.yaml` would let the real shape be mutated
Expand Down
54 changes: 50 additions & 4 deletions crates/aprender-contracts/src/lint/shapes_gate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,20 @@
//! proves nothing about the gate.
//!
//! **The positive controls.** `pc_shape`: a bare focus node of each shape's target class, drawn every run, must
//! violate at least one armed shape. `pc_extract.gguf`: a corrupt magic is refused. `pc_extract["apr-model"]`:
//! a header whose tensor count disagrees with its index is refused. All three every run, in memory.
//! violate at least one armed shape. `pc_extract` — one planted defect per extractor Σ marks implemented (R-3):
//! `pv-contract`, a contract stripped of `metadata` carries no `ont:kind`; `json`, a nested key the vocabulary does
//! not map is refused naming it; `gguf`, a corrupt magic is refused; `apr-model`, a header whose tensor count
//! disagrees with its index is refused; `code` and `lean`, as their modules state; `parity-receipt`, a record
//! stripped of `comparator` loses its comparator edge. All of them every run, in memory.

use std::collections::{BTreeMap, BTreeSet};
use std::path::Path;
use std::time::Instant;

use crate::ontology::arming::ArmedShapes;
use crate::ontology::extract::{self, apr_model, code, gguf, lean, pv_contract, ExtractFailure};
use crate::ontology::extract::{
self, apr_model, code, gguf, json, lean, parity_receipt, pv_contract, ExtractFailure,
};
use crate::ontology::rdf::{iri, Graph, Term, RDF_TYPE};
use crate::ontology::receipts;
use crate::ontology::shapes::{self, NodeShape, Report, Severity, ShapeError};
Expand Down Expand Up @@ -378,14 +383,22 @@ fn decline(
})
}

/// The extractor positive controls (R-3), run in memory every gate run.
/// The extractor positive controls (R-3: one planted defect per registered extractor), run in memory every gate
/// run, keyed by the Σ entity type each extractor reads. The key set must equal Σ's implemented entity types —
/// [`tests::every_implemented_entity_type_in_sigma_has_an_extract_control_and_it_fires`] holds the two together.
fn extract_controls() -> BTreeMap<String, String> {
let apr_sample = apr_model::minimal_container(2);
[
("pv-contract", pv_contract::positive_control()),
("json", json::positive_control()),
("gguf", gguf::positive_control()),
("apr-model", apr_model::positive_control(&apr_sample)),
("code", code::positive_control()),
("lean", lean::positive_control()),
(
"parity-receipt",
parity_receipt::positive_control(&parity_receipt::control_sample()),
),
]
.into_iter()
.map(|(k, fired)| {
Expand Down Expand Up @@ -798,4 +811,37 @@ mod tests {
ShapesOutcome::PositiveControlFailed { .. }
));
}

#[test]
fn every_implemented_entity_type_in_sigma_has_an_extract_control_and_it_fires() {
// PMAT-3704 — R-3: `pc_extract`, one planted defect per registered extractor, every run. The control set
// was a hand-written array beside Σ with nothing tying the two together, so `parity-receipt` (#3600) and
// `json` (#3516) shipped as implemented entity types the gate never controlled — the same shape as #3624
// for `by_entity_type`, and as bashrs#266's two lists. Σ is this build's registry (its `extractors[]`
// name this crate's readers), so the two sets must be EQUAL: an implemented type without a control fails,
// and so does a control for a type Σ does not implement.
let sigma_path =
std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../contracts/ontology.yaml");
let text = std::fs::read_to_string(&sigma_path).expect("contracts/ontology.yaml");
let sigma = crate::ontology::sigma::Sigma::from_yaml(&text).expect("Σ parses");
let implemented: BTreeSet<String> = sigma
.entity_types
.iter()
.filter(|e| e.implemented)
.map(|e| e.name.clone())
.collect();
assert!(
!implemented.is_empty(),
"Σ implements nothing — the comparison would be vacuous"
);
let controls = extract_controls();
let controlled: BTreeSet<String> = controls.keys().cloned().collect();
assert_eq!(
controlled, implemented,
"pc_extract keys (left) must equal Σ's implemented entity types (right)"
);
for (k, v) in &controls {
assert_eq!(v, "fired", "pc_extract.{k}");
}
}
}
59 changes: 55 additions & 4 deletions crates/aprender-contracts/src/ontology/extract/json.rs
Original file line number Diff line number Diff line change
Expand Up @@ -160,19 +160,64 @@ pub fn extract_into(
path: path.clone(),
why: e.to_string(),
})?;
extract_text(g, stem, &path, &text, &vocab)
}

/// Everything after the read: the document's text as nodes. Shared by [`extract_into`] and [`positive_control`],
/// so the control exercises the code the gate runs rather than a copy of it.
fn extract_text(
g: &mut Graph,
stem: &str,
path: &str,
text: &str,
vocab: &Vocabulary,
) -> Result<Vec<Warning>, ExtractError> {
if path.ends_with(".jsonl") {
return extract_jsonl(g, stem, &path, &text, &vocab);
return extract_jsonl(g, stem, path, text, vocab);
}
let value: serde_json::Value =
serde_json::from_str(&text).map_err(|e| ExtractError::NotJson {
serde_json::from_str(text).map_err(|e| ExtractError::NotJson {
contract: stem.to_string(),
path: path.clone(),
path: path.to_string(),
why: e.to_string(),
})?;
node(g, stem, stem, &vocab.root_class, true, &value, &vocab)?;
node(g, stem, stem, &vocab.root_class, true, &value, vocab)?;
Ok(Vec::new())
}

/// The positive control (R-3, PMAT-3704), in memory every gate run: a document whose nested key the vocabulary
/// maps must extract with that child typed by its class, and the planted copy whose nested key the vocabulary
/// does NOT map must be refused naming the key — the "no inference" rule this module states, measured.
#[must_use]
pub fn positive_control() -> bool {
let vocab = Vocabulary {
prefix: "pc".into(),
root_class: "pc:Root".into(),
nested: vec![("child".into(), "pc:Child".into())],
};
let mut g = Graph::new();
let mapped = extract_text(
&mut g,
"__pc_extract__",
"pc.json",
r#"{"a":1,"child":{"b":true}}"#,
&vocab,
)
.is_ok()
&& g.objects(&iri("pc", "__pc_extract__.child"), RDF_TYPE)
.iter()
.any(|t| t.as_iri() == Some(expand("pc:Child").as_str()));
let planted = extract_text(
&mut Graph::new(),
"__pc_extract__",
"pc.json",
r#"{"a":1,"orphan":{"b":true}}"#,
&vocab,
);
let refused = matches!(planted, Err(ExtractError::Unmapped { ref key, .. }) if key == "orphan");
mapped && refused
}

fn extract_jsonl(
g: &mut Graph,
stem: &str,
Expand Down Expand Up @@ -516,4 +561,10 @@ mod tests {
let none: serde_yaml::Value = serde_yaml::from_str("name: x").unwrap();
assert!(!applies(&none));
}

#[test]
fn the_positive_control_fires() {
// PMAT-3704: drawn by the shapes gate every run as pc_extract.json.
assert!(positive_control());
}
}
27 changes: 27 additions & 0 deletions crates/aprender-contracts/src/ontology/extract/parity_receipt.rs
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,33 @@ fn emit(g: &mut Graph, root: &Path, rel: &str, v: &serde_json::Value, stats: &mu
}
}

/// The body of [`control_sample`], less its `schema` (which is [`SCHEMA`], inserted so the two cannot drift).
const CONTROL_SAMPLE: &str = r#"{
"cell": {"model": "pc", "file": "./pc.gguf", "quant": "Q4_K_M"},
"host": "pc-host",
"backend": "cuda",
"apr_version": "0.0.0",
"generated_at": "1970-01-01",
"comparator": {"kind": "self", "reason": "positive control"},
"partially_receipted": true,
"threshold_source": "evidence/parity/thresholds.yaml",
"unmeasured": ["ORACLE ARM: not measured."],
"result": {"positions": 1, "parity": true},
"raw": {"model": "./pc.gguf", "metrics": []}
}"#;

/// The record the gate hands [`positive_control`] every run (PMAT-3704): a minimal [`SCHEMA`] record in the
/// layout the seven back-filled receipts carry, self-compared, so its comparator edge is the one thing the
/// plant removes. Fails closed: a sample that did not parse is `Null`, on which the control cannot fire.
#[must_use]
pub fn control_sample() -> serde_json::Value {
let mut v: serde_json::Value = serde_json::from_str(CONTROL_SAMPLE).unwrap_or_default();
if let Some(o) = v.as_object_mut() {
o.insert("schema".into(), SCHEMA.into());
}
v
}

/// The positive control (R-3): a copy of a real record with `comparator` removed must lose its comparator
/// edge, every run. Drawn beside the corpus so "the extractor still reads this layout" is measured rather
/// than assumed.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,21 @@ fn the_plant_removes_the_comparator_edge_every_run() {
assert!(positive_control(&record("self", serde_json::json!({}))));
}

#[test]
fn the_gate_sample_is_a_record_the_control_fires_on() {
// PMAT-3704: the gate draws the control on control_sample(), not on a test helper's record.
assert!(positive_control(&control_sample()));
}

#[test]
fn a_sample_without_a_comparator_cannot_fire_the_control() {
// Discrimination: the control is a claim about the comparator edge, so a sample that has none to lose must
// not fire — else "fired" would say nothing about the extractor.
let mut bare = control_sample();
bare.as_object_mut().expect("object").remove("comparator");
assert!(!positive_control(&bare));
}

#[test]
fn an_unmigrated_legacy_record_is_refused_by_name_and_never_skipped() {
// The exact layout the seven records carried before #3577: no schema, metrics[] at the top level.
Expand Down
37 changes: 37 additions & 0 deletions crates/aprender-contracts/src/ontology/extract/pv_contract.rs
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,37 @@ pub fn extract_one(g: &mut Graph, stem: &str, file: &str, doc: &serde_yaml::Valu
}
}

/// The positive control (R-3, PMAT-3704), in memory every gate run: a contract carrying `metadata.kind` and a
/// `depends_on` relation must come out with `ont:kind` and the typed edge, and the planted copy with `metadata`
/// removed must carry NO `ont:kind` — "nothing is inferred", measured rather than stated.
#[must_use]
pub fn positive_control() -> bool {
const STEM: &str = "__pc_extract__";
let Ok(doc) = serde_yaml::from_str::<serde_yaml::Value>(
"metadata: {kind: kernel}\nrelations: {depends_on: [contracts/__pc_target__-v1.yaml]}\n",
) else {
return false;
};
let s = iri("contract", STEM);
let mut g = Graph::new();
extract_one(&mut g, STEM, "contracts/__pc_extract__.yaml", &doc);
let kind = g
.objects(&s, &ont("kind"))
.iter()
.any(|t| t.as_literal().is_some_and(|l| l.0 == "kernel"));
let edge = g
.objects(&s, &ont("depends_on"))
.iter()
.any(|t| t.as_iri() == Some(iri("contract", "__pc_target__-v1").as_str()));
let mut planted = doc;
if let Some(m) = planted.as_mapping_mut() {
m.remove("metadata");
}
let mut g2 = Graph::new();
extract_one(&mut g2, STEM, "contracts/__pc_extract__.yaml", &planted);
kind && edge && g2.objects(&s, &ont("kind")).is_empty()
}

/// §4.2 `entity: {type, ref, properties}` — the entity's OWN properties, as `<type>:<key>` on the contract node.
///
/// A contract may carry the properties of the thing it is a contract FOR, and for a pre-registration the
Expand Down Expand Up @@ -292,4 +323,10 @@ mod tests {
assert_eq!(a, b);
assert!(a.contains("/contract/a> <https://ont.paiml.dev/v1alpha1/contradicts> <https://ont.paiml.dev/v1alpha1/contract/d>"), "{a}");
}

#[test]
fn the_positive_control_fires() {
// PMAT-3704: drawn by the shapes gate every run as pc_extract["pv-contract"].
assert!(positive_control());
}
}
Loading
Loading