diff --git a/.claude/skills/apr-dogfood/SKILL.md b/.claude/skills/apr-dogfood/SKILL.md index 327d53f2fa..b8b6f68e2d 100644 --- a/.claude/skills/apr-dogfood/SKILL.md +++ b/.claude/skills/apr-dogfood/SKILL.md @@ -1594,7 +1594,7 @@ The nine clusters at zero, largest first: `http-orchestrate-banco` (95), `test-harness` (49), `rag-eval` (44), `qa-cgp` (37), `simulation` (18), `orchestrate-pacha-secrets` (17). -**Report both numbers or neither (T2).** "5 of 14 clusters gated (35.7%)" without "144 of 911 features gated (15.8%)" beside it is a proxy masquerading as coverage, and the gate refuses to emit it — *on this line too*. +**Report both numbers or neither (T2).** "5 of 14 clusters gated (35.7%)" without "144 of 912 features gated (15.8%)" beside it is a proxy masquerading as coverage, and the gate refuses to emit it — *on this line too*. The rule is about the NUMBER, not about a phrasing, and it is enforced on every surface that can emit one: the gate's own report, the receipt, the output of diff --git a/contracts/apr-dogfood-coverage-v1.yaml b/contracts/apr-dogfood-coverage-v1.yaml index afc082013b..6e28be2311 100644 --- a/contracts/apr-dogfood-coverage-v1.yaml +++ b/contracts/apr-dogfood-coverage-v1.yaml @@ -3,7 +3,7 @@ metadata: created: '2026-08-22' author: PAIML Engineering references: - - 'docs/audits/surface_audit.csv — 142 of 830 features gated, across 28 binaries and 14 clusters, measured 2026-08-22 @ 4bbfeb07f; 142 of 832 features gated after the PERF-046 partial re-audit (PR #2742); 143 of 833 features gated after the PP-LLAMA-001 re-audit (GET /v1/effective-config added and swept by the dogfood route enumeration, router.rs rows re-pointed); 144 of 837 features gated after PP-066 L0-1b added the three `apr parity` flag rows (--per-op, --out, --threshold), which the dogfood sweep does not reach -- the denominator grew and the numerator did not; 144 of 838 features gated after #3726 added `apr tokenize encode` (the tokenizer-parity gate''s id source), which the dogfood sweep does not reach either; 144 of 839 features gated after ONT-10 S19 ledgered the bare `aprender-cbtop` root surface (interactive TUI / --headless), which the dogfood sweep does not reach; 144 of 851 features gated after ONT-4g (#4476) added the twelve pv/apr command rows the binary snapshot found unledgered, none of them gated; 144 of 853 features gated after the ONT-4g bin gate (surface_audit_bins_gate.sh) found `pv-sat` shipping with no row and its two surfaces were ledgered, neither gated; 144 of 857 features gated after ONT-10 S4 ledgered the 4 hf_* tools `aprender-orchestrate mcp` lists, none of them gated; 144 of 868 features gated after ONT-10 S9/S7 mirrored the eleven ONT-4g pv rows as `apr pv` rows (the apr node joins binary=apr), none of them gated; 144 of 894 features gated once main (with #4219''s 26 ` --version` rows, each run and observed printing its SHA, which the dogfood sweep does not reach) merged into ONT-10''s 868; 144 of 904 features gated after the bin gate found `aprender-ci-tools` (the C301 port of the CI helpers) shipping with no row and its ten surfaces (nine subcommands and --version) were ledgered, none of them gated (#5170); 144 of 906 features gated after #5382 added the `pv ontology readme` and `apr pv ontology readme` rows, ungated; 144 of 907 features gated after #5225 added the `aprender-ci-tools ladder-cells-produce` row, ungated; 144 of 911 features gated after the `sim repo-flow` and `sim course-flow` rows (apr and aprender-simulate, four rows) were ledgered, each run on x86_64, ungated' + - 'docs/audits/surface_audit.csv — 142 of 830 features gated, across 28 binaries and 14 clusters, measured 2026-08-22 @ 4bbfeb07f; 142 of 832 features gated after the PERF-046 partial re-audit (PR #2742); 143 of 833 features gated after the PP-LLAMA-001 re-audit (GET /v1/effective-config added and swept by the dogfood route enumeration, router.rs rows re-pointed); 144 of 837 features gated after PP-066 L0-1b added the three `apr parity` flag rows (--per-op, --out, --threshold), which the dogfood sweep does not reach -- the denominator grew and the numerator did not; 144 of 838 features gated after #3726 added `apr tokenize encode` (the tokenizer-parity gate''s id source), which the dogfood sweep does not reach either; 144 of 839 features gated after ONT-10 S19 ledgered the bare `aprender-cbtop` root surface (interactive TUI / --headless), which the dogfood sweep does not reach; 144 of 851 features gated after ONT-4g (#4476) added the twelve pv/apr command rows the binary snapshot found unledgered, none of them gated; 144 of 853 features gated after the ONT-4g bin gate (surface_audit_bins_gate.sh) found `pv-sat` shipping with no row and its two surfaces were ledgered, neither gated; 144 of 857 features gated after ONT-10 S4 ledgered the 4 hf_* tools `aprender-orchestrate mcp` lists, none of them gated; 144 of 868 features gated after ONT-10 S9/S7 mirrored the eleven ONT-4g pv rows as `apr pv` rows (the apr node joins binary=apr), none of them gated; 144 of 894 features gated once main (with #4219''s 26 ` --version` rows, each run and observed printing its SHA, which the dogfood sweep does not reach) merged into ONT-10''s 868; 144 of 904 features gated after the bin gate found `aprender-ci-tools` (the C301 port of the CI helpers) shipping with no row and its ten surfaces (nine subcommands and --version) were ledgered, none of them gated (#5170); 144 of 906 features gated after #5382 added the `pv ontology readme` and `apr pv ontology readme` rows, ungated; 144 of 907 features gated after #5225 added the `aprender-ci-tools ladder-cells-produce` row, ungated; 144 of 912 features gated after the `sim repo-flow` and `sim course-flow` rows (apr and aprender-simulate, four rows) were ledgered, each run on x86_64, ungated' - 'docs/audits/surface_audit_elbow.png — k-selection evidence: inertia knee and silhouette over k in [2,45]' - 'scripts/dogfood_cluster.py — the k-sweep that produced the elbow plot (k-selection ONLY; it does not write the columns)' - 'scripts/check_no_cluster_id_keys.sh — T1 enforcement: nothing may key on `cluster_id`' @@ -87,9 +87,9 @@ metadata: overall: command: "python3 scripts/dogfood_baseline.py # section: totals" - rows: 911 + rows: 912 covered: 144 - ratio: 0.1581 # 144/911 -- 15.8%, NOT "about 16%" + ratio: 0.1579 # 144/912 -- 15.8%, NOT "about 16%" binaries: 30 binaries_at_zero_coverage: 29 clusters: 14 @@ -104,7 +104,7 @@ metadata: note: > covered may never DECREASE for any binary; total may never decrease without a reconciliation finding recording which direction moved. - apr: {total: 390, covered: 144} + apr: {total: 391, covered: 144} aprender-orchestrate: {total: 189, covered: 0} aprender-test-cli: {total: 40, covered: 0} pv: {total: 50, covered: 0} @@ -178,7 +178,7 @@ metadata: apr-lint-diag: {total: 71, covered: 55} # 77.5% of cluster, 38.2% of gate effort orchestrate-agent-stack: {total: 56, covered: 0} # ZERO test-harness: {total: 62, covered: 0} # ZERO - http-apr-serve: {total: 45, covered: 40} # 88.9% of cluster, 27.8% of gate effort + http-apr-serve: {total: 46, covered: 40} # 87.0% of cluster, 27.8% of gate effort profiling: {total: 46, covered: 1} # 2.2% of cluster, 0.7% of gate effort rag-eval: {total: 46, covered: 0} # ZERO qa-cgp: {total: 38, covered: 0} # ZERO @@ -197,7 +197,7 @@ metadata: q3_4: {total: 49, covered: 27} # 55.1% q5_6: {total: 723, covered: 29} # 4.1% q7_8: {total: 38, covered: 10} # 26.3% - q9_10: {total: 21, covered: 20} # 95.2% + q9_10: {total: 22, covered: 20} # 90.9% hardware_provenance: command: "python3 scripts/dogfood_baseline.py # section: totals" @@ -306,7 +306,7 @@ equations: codomain: bool invariants: - "T2. Cluster coverage is a PROXY: one gate in a 95-member cluster is 1%, not covered" - - "5/14 clusters (35.7%) and 144/911 features (15.8%) are reported together, always" + - "5/14 clusters (35.7%) and 144/912 features (15.8%) are reported together, always" - "Enforced mechanically by enforce_pairing(); deleting the feature fraction from the emitter turns the gate RED" - "This is the single most important trap: reporting the proxy alone is how a coverage gate becomes theatre while looking stricter" @@ -695,7 +695,7 @@ falsification_tests: Every emitted line stating a CLUSTER-level ratio also states the underlying FEATURE-level ratio, on EVERY channel one can be emitted through: this gate's report, the receipt, scripts/dogfood_baseline.py and - .claude/skills/apr-dogfood/SKILL.md. 5 of 14 clusters (35.7%) is a proxy and 144 of 911 features (15.8%) is the thing. + .claude/skills/apr-dogfood/SKILL.md. 5 of 14 clusters (35.7%) is a proxy and 144 of 912 features (15.8%) is the thing. One gate in a 95-member cluster is 1%. The rule keys on the NUMBER -- a denominator equal to the cluster count -- not on a phrase, because a phrase can be reworded and a denominator cannot; diff --git a/contracts/contracts.nt b/contracts/contracts.nt index 70db0aa4b0..eb72dcac1f 100644 --- a/contracts/contracts.nt +++ b/contracts/contracts.nt @@ -7,6 +7,7 @@ "GET /api/tags"^^ . "GET /api/version"^^ . "GET /health"^^ . + "GET /health binary_sha256"^^ . "GET /health/live"^^ . "GET /health/ready"^^ . "GET /metrics"^^ . @@ -2066,6 +2067,7 @@ "GET /api/tags"^^ . "GET /api/version"^^ . "GET /health"^^ . + "GET /health binary_sha256"^^ . "GET /health/live"^^ . "GET /health/ready"^^ . "GET /metrics"^^ . diff --git a/crates/aprender-serve/src/api/router.rs b/crates/aprender-serve/src/api/router.rs index 04098479c3..16224c4c4d 100644 --- a/crates/aprender-serve/src/api/router.rs +++ b/crates/aprender-serve/src/api/router.rs @@ -710,10 +710,34 @@ fn health_status_code(body: &HealthResponse) -> StatusCode { } } +/// SHA-256 (lowercase hex) of the running executable, computed once (#5348 R-8: +/// one gateway per host, pinned by binary sha; consumers refuse a mismatch). +/// `None` when the executable cannot be read: the field is then omitted, never faked. +fn binary_sha256() -> Option<&'static str> { + static SHA: std::sync::OnceLock> = std::sync::OnceLock::new(); + SHA.get_or_init(|| { + use sha2::{Digest, Sha256}; + let bytes = std::fs::read(std::env::current_exe().ok()?).ok()?; + Some(format!("{:x}", Sha256::digest(&bytes))) + }) + .as_deref() +} + +/// `/health` body: the contract fields plus the R-8 binary pin. +#[derive(Serialize)] +struct PinnedHealthResponse { + #[serde(flatten)] + health: HealthResponse, + #[serde(skip_serializing_if = "Option::is_none")] + binary_sha256: Option<&'static str>, +} + /// `GET /health` — vLLM / llama.cpp-parity liveness probe. /// /// Discharges FALSIFY-CRUX-C-34-001/002/003. -async fn health_handler(State(state): State) -> (StatusCode, Json) { +async fn health_handler( + State(state): State, +) -> (StatusCode, Json) { if state.is_verbose() { eprintln!("[VERBOSE] GET /health"); } @@ -722,7 +746,14 @@ async fn health_handler(State(state): State) -> (StatusCode, Json {} status={}", code, body.status); } - (code, Json(body)) + let binary_sha256 = binary_sha256(); + ( + code, + Json(PinnedHealthResponse { + health: body, + binary_sha256, + }), + ) } /// `GET /health/live` — k8s liveness probe. diff --git a/crates/aprender-serve/src/api/tests/usage_serde_health.rs b/crates/aprender-serve/src/api/tests/usage_serde_health.rs index 89529c109d..fbfd3f55b8 100644 --- a/crates/aprender-serve/src/api/tests/usage_serde_health.rs +++ b/crates/aprender-serve/src/api/tests/usage_serde_health.rs @@ -98,3 +98,26 @@ async fn test_chat_completions_with_trace_header() { // MOUNTED route is 503, and that is now what is asserted. crate::api::test_helpers::assert_no_model_status(response.status()); } + +#[tokio::test] +async fn test_health_reports_binary_sha256() { + // #5348 R-8: /health pins the gateway by the sha256 of the running binary. + use sha2::{Digest, Sha256}; + let app = create_test_app(); + let request = Request::builder() + .method("GET") + .uri("/health") + .body(Body::empty()) + .expect("test value should be present"); + let response = app.oneshot(request).await.expect("test value should be present"); + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .expect("body should read"); + let body: serde_json::Value = serde_json::from_slice(&bytes).expect("health body is JSON"); + let want = format!( + "{:x}", + Sha256::digest(std::fs::read(std::env::current_exe().expect("exe")).expect("read exe")) + ); + assert_eq!(body["binary_sha256"].as_str(), Some(want.as_str())); + assert_eq!(body["status"], "ok", "contract fields stay flattened alongside the pin"); +} diff --git a/crates/aprender-train/src/train/transformer_trainer/step_profiler.rs b/crates/aprender-train/src/train/transformer_trainer/step_profiler.rs index c002a6c5e2..a7735cb66d 100644 --- a/crates/aprender-train/src/train/transformer_trainer/step_profiler.rs +++ b/crates/aprender-train/src/train/transformer_trainer/step_profiler.rs @@ -304,86 +304,95 @@ impl StepProfiler { ); println!("└────────────┴──────────┴────────┴──────────┘"); - // Per-layer breakdown (PMAT-480) - if self.num_layers > 0 && self.step_count > 0 { - println!( - "\n┌─ Per-Layer Profile ({} layers, {} steps) ─┐", - self.num_layers, self.step_count - ); - println!( - "│ {:>5} │ {:>8} │ {:>8} │ {:>8} │ {:>8} │", - "layer", "fwd_ms", "bwd_ms", "fwd_avg", "bwd_avg" - ); - println!("│ {:->5} │ {:->8} │ {:->8} │ {:->8} │ {:->8} │", "", "", "", "", ""); - let mut fwd_total = Duration::ZERO; - let mut bwd_total = Duration::ZERO; - for i in 0..self.num_layers { - let fwd = self.layer_fwd_totals[i]; - let bwd = self.layer_bwd_totals[i]; - fwd_total += fwd; - bwd_total += bwd; - let fwd_ms = fwd.as_micros() as f64 / 1000.0; - let bwd_ms = bwd.as_micros() as f64 / 1000.0; - let fwd_avg = fwd_ms / self.step_count as f64; - let bwd_avg = bwd_ms / self.step_count as f64; - println!( - "│ {i:>5} │ {fwd_ms:>8.1} │ {bwd_ms:>8.1} │ {fwd_avg:>8.2} │ {bwd_avg:>8.2} │" - ); - } - let fwd_total_ms = fwd_total.as_micros() as f64 / 1000.0; - let bwd_total_ms = bwd_total.as_micros() as f64 / 1000.0; + self.print_layer_profile(); + self.print_latency_percentiles(); + } + + /// Per-layer breakdown and hotspot layers (PMAT-480). + fn print_layer_profile(&self) { + if self.num_layers == 0 || self.step_count == 0 { + return; + } + println!( + "\n┌─ Per-Layer Profile ({} layers, {} steps) ─┐", + self.num_layers, self.step_count + ); + println!( + "│ {:>5} │ {:>8} │ {:>8} │ {:>8} │ {:>8} │", + "layer", "fwd_ms", "bwd_ms", "fwd_avg", "bwd_avg" + ); + println!("│ {:->5} │ {:->8} │ {:->8} │ {:->8} │ {:->8} │", "", "", "", "", ""); + let mut fwd_total = Duration::ZERO; + let mut bwd_total = Duration::ZERO; + for i in 0..self.num_layers { + let fwd = self.layer_fwd_totals[i]; + let bwd = self.layer_bwd_totals[i]; + fwd_total += fwd; + bwd_total += bwd; + let fwd_ms = fwd.as_micros() as f64 / 1000.0; + let bwd_ms = bwd.as_micros() as f64 / 1000.0; + let fwd_avg = fwd_ms / self.step_count as f64; + let bwd_avg = bwd_ms / self.step_count as f64; println!( - "│ {:>5} │ {:>8.1} │ {:>8.1} │ {:>8.2} │ {:>8.2} │", - "TOTAL", - fwd_total_ms, - bwd_total_ms, - fwd_total_ms / self.step_count as f64, - bwd_total_ms / self.step_count as f64 + "│ {i:>5} │ {fwd_ms:>8.1} │ {bwd_ms:>8.1} │ {fwd_avg:>8.2} │ {bwd_avg:>8.2} │" ); - println!("└───────┴──────────┴──────────┴──────────┴──────────┘"); - - // Identify hotspot layers (>1.5x average) - let avg_fwd = fwd_total / self.num_layers as u32; - let avg_bwd = bwd_total / self.num_layers as u32; - let mut hotspots = Vec::new(); - for i in 0..self.num_layers { - let fwd_ratio = if avg_fwd.as_nanos() > 0 { - self.layer_fwd_totals[i].as_nanos() as f64 / avg_fwd.as_nanos() as f64 - } else { - 0.0 - }; - let bwd_ratio = if avg_bwd.as_nanos() > 0 { - self.layer_bwd_totals[i].as_nanos() as f64 / avg_bwd.as_nanos() as f64 - } else { - 0.0 - }; - if fwd_ratio > 1.5 || bwd_ratio > 1.5 { - hotspots.push((i, fwd_ratio, bwd_ratio)); - } + } + let fwd_total_ms = fwd_total.as_micros() as f64 / 1000.0; + let bwd_total_ms = bwd_total.as_micros() as f64 / 1000.0; + println!( + "│ {:>5} │ {:>8.1} │ {:>8.1} │ {:>8.2} │ {:>8.2} │", + "TOTAL", + fwd_total_ms, + bwd_total_ms, + fwd_total_ms / self.step_count as f64, + bwd_total_ms / self.step_count as f64 + ); + println!("└───────┴──────────┴──────────┴──────────┴──────────┘"); + + // Identify hotspot layers (>1.5x average) + let avg_fwd = fwd_total / self.num_layers as u32; + let avg_bwd = bwd_total / self.num_layers as u32; + let mut hotspots = Vec::new(); + for i in 0..self.num_layers { + let fwd_ratio = if avg_fwd.as_nanos() > 0 { + self.layer_fwd_totals[i].as_nanos() as f64 / avg_fwd.as_nanos() as f64 + } else { + 0.0 + }; + let bwd_ratio = if avg_bwd.as_nanos() > 0 { + self.layer_bwd_totals[i].as_nanos() as f64 / avg_bwd.as_nanos() as f64 + } else { + 0.0 + }; + if fwd_ratio > 1.5 || bwd_ratio > 1.5 { + hotspots.push((i, fwd_ratio, bwd_ratio)); } - if !hotspots.is_empty() { - println!(" Hotspot layers (>1.5x average):"); - for (layer, fwd_r, bwd_r) in &hotspots { - println!(" L{layer}: fwd {fwd_r:.1}x, bwd {bwd_r:.1}x"); - } + } + if !hotspots.is_empty() { + println!(" Hotspot layers (>1.5x average):"); + for (layer, fwd_r, bwd_r) in &hotspots { + println!(" L{layer}: fwd {fwd_r:.1}x, bwd {bwd_r:.1}x"); } } + } - // Percentiles for step wall-clock - if self.step_durations.len() >= 10 { - let mut sorted: Vec = - self.step_durations.iter().map(std::time::Duration::as_micros).collect(); - sorted.sort_unstable(); - let p50 = sorted[sorted.len() / 2]; - let p95 = sorted[sorted.len() * 95 / 100]; - let p99 = sorted[sorted.len() * 99 / 100]; - println!( - " Step latency: p50={:.1}ms p95={:.1}ms p99={:.1}ms", - p50 as f64 / 1000.0, - p95 as f64 / 1000.0, - p99 as f64 / 1000.0 - ); + /// Print p50/p95/p99 of step wall-clock once at least 10 steps exist. + fn print_latency_percentiles(&self) { + if self.step_durations.len() < 10 { + return; } + let mut sorted: Vec = + self.step_durations.iter().map(std::time::Duration::as_micros).collect(); + sorted.sort_unstable(); + let p50 = sorted[sorted.len() / 2]; + let p95 = sorted[sorted.len() * 95 / 100]; + let p99 = sorted[sorted.len() * 99 / 100]; + println!( + " Step latency: p50={:.1}ms p95={:.1}ms p99={:.1}ms", + p50 as f64 / 1000.0, + p95 as f64 / 1000.0, + p99 as f64 / 1000.0 + ); } /// PMAT-483/entrenar#328: Start timing a per-operation phase within a layer. diff --git a/docs/audits/surface_audit.csv b/docs/audits/surface_audit.csv index 0a25cff4e4..4fd70fdd95 100644 --- a/docs/audits/surface_audit.csv +++ b/docs/audits/surface_audit.csv @@ -41,6 +41,7 @@ apr,GET /api/version,9,x86_64-linux,ollama,yes,3,http-apr-serve,crates/aprender- apr,GET /health,9,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:89,medium apr,GET /health/live,9,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:90,medium apr,GET /health/ready,9,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:91,medium +apr,GET /health binary_sha256,9,x86_64-linux,llama.cpp,no,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:89,medium apr,GET /metrics,6,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:116,medium apr,GET /metrics/dispatch,7,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:117,medium apr,GET /models,6,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:95,high