From 906cc33f40a822fda0845870a687b121e52612f4 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Wed, 21 Oct 2026 10:43:13 +0200 Subject: [PATCH 1/8] test(serve): crux_c_34_003 bounds uptime delta by measured wall clock (Refs #5783) (fold of #5784) Refs PMAT-1702 Co-Authored-By: Claude Sonnet 5.5 --- .../aprender-serve/tests/falsification_crux_c_34.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/aprender-serve/tests/falsification_crux_c_34.rs b/crates/aprender-serve/tests/falsification_crux_c_34.rs index a1308dca16..e2c95e7b6b 100644 --- a/crates/aprender-serve/tests/falsification_crux_c_34.rs +++ b/crates/aprender-serve/tests/falsification_crux_c_34.rs @@ -144,6 +144,7 @@ async fn falsify_crux_c_34_003_uptime_monotonic() { // Two sequential oneshots on the SAME process must see uptime_sec advance. // Router is rebuilt each call but the OnceLock is process-wide, // so start time is shared. + let wall = std::time::Instant::now(); let a = router_with_model() .oneshot(get("/health")) .await @@ -159,15 +160,21 @@ async fn falsify_crux_c_34_003_uptime_monotonic() { .expect("oneshot 2"); let json_b = json_body(b).await; let u2 = json_b["uptime_sec"].as_f64().expect("u2"); + let wall_s = wall.elapsed().as_secs_f64(); assert!( u2 > u1, "FALSIFY-CRUX-C-34-003: uptime must strictly increase, got u1={u1} u2={u2}" ); + // #5783: the upper bound is the wall clock measured around the two reads, not a fixed + // 5.0 s, so host load (the reads can be seconds apart) cannot fail it. A constant or reset + // clock fails the lower bound. A clock running at rate r fails this bound only when + // r * gap > wall_s + 0.05 (gap = real time between the two server samples), i.e. when + // r > (wall_s + 0.05) / gap; setup or parse time outside the gap widens the window. let delta = u2 - u1; assert!( - (0.01..=5.0).contains(&delta), - "FALSIFY-CRUX-C-34-003: delta {delta}s outside plausible [0.01, 5.0] window" + (0.01..=wall_s + 0.05).contains(&delta), + "FALSIFY-CRUX-C-34-003: delta {delta}s outside [0.01, measured wall {wall_s}s + 0.05]" ); } From d021fe05156931a223c7f1d95efed42901871972 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Wed, 21 Oct 2026 10:43:34 +0200 Subject: [PATCH 2/8] batch 7: crux setsid, #5639, #5636, #5601 tests, contracts_resolve fixes (fold of #5781) Refs PMAT-1702 Co-Authored-By: Claude Sonnet 5.5 --- contracts/bin-aprender-qa-cli--apr-qa-v1.yaml | 6 + contracts/distributed-training-v1.yaml | 5 + crates/aprender-explain/src/tui/mod.rs | 3 + .../src/tui/source_pane_tests.rs | 107 +++++++++++++++ .../src/certification_processing.rs | 82 ++++++++++++ .../src/finetune/classify_trainer_tests.rs | 125 ++++++++++++++++++ 6 files changed, 328 insertions(+) create mode 100644 crates/aprender-explain/src/tui/source_pane_tests.rs diff --git a/contracts/bin-aprender-qa-cli--apr-qa-v1.yaml b/contracts/bin-aprender-qa-cli--apr-qa-v1.yaml index 3d687dfd53..06f4d63b94 100644 --- a/contracts/bin-aprender-qa-cli--apr-qa-v1.yaml +++ b/contracts/bin-aprender-qa-cli--apr-qa-v1.yaml @@ -135,3 +135,9 @@ falsification_tests: test: 'cargo test -p aprender-qa-cli --test cli_surface_gate unusable_input_is_rejected' if_fails: 'MEASURED 2026-09-26 (aprender-01): GREEN on an unmutated tree; no source mutation was run for this binary' mutation: 'make a rejection path return Ok(()) or unwrap the read; must turn RED on that row' + - id: FALSIFY-BIN-APR-QA-007 + rule: 'profile throughput minimums resolve per backend, then legacy, then 5 (cpu) / 50 (gpu)' + prediction: 'check_profiling_assertions records a failed assertion exactly when a measured tok/s is below the resolved minimum, and records nothing when profile_ci is absent' + test: 'cargo test -p aprender-qa-cli --lib profiling_assertion_tests' + if_fails: 'a throughput regression passes certification, or a legacy min_throughput stops applying' + mutation: 'change unwrap_or(5.0) to unwrap_or(6.0) in check_profiling_assertions; defaults_are_5_cpu_and_50_gpu must turn RED (run 2026-10-20)' diff --git a/contracts/distributed-training-v1.yaml b/contracts/distributed-training-v1.yaml index db3ab68e71..21346cb607 100644 --- a/contracts/distributed-training-v1.yaml +++ b/contracts/distributed-training-v1.yaml @@ -33,3 +33,8 @@ falsification: description: "loss(distributed, N) ≈ loss(single, N) within ε" test: "Related tests in crates/aprender-distribute/src/" evidence: "cargo test --lib" + + - id: FALSIFY-DISTRIBUTED_TRAINING_V1_003 + description: "run_worker clamps an out-of-range shard to the training split and fails closed on a bad coordinator (Shutdown mid-AllReduce, non-shard message, refused connect)" + test: "crates/aprender-train/src/finetune/classify_trainer_tests.rs::run_worker_tests" + evidence: "cargo test -p aprender-train --lib run_worker_tests" diff --git a/crates/aprender-explain/src/tui/mod.rs b/crates/aprender-explain/src/tui/mod.rs index 112e5dd65d..93e6979237 100644 --- a/crates/aprender-explain/src/tui/mod.rs +++ b/crates/aprender-explain/src/tui/mod.rs @@ -431,3 +431,6 @@ fn render_status_bar(canvas: &mut DirectTerminalCanvas<'_>, width: u16, height: mod scrollbar_tests; #[cfg(test)] mod tests; + +#[cfg(test)] +mod source_pane_tests; diff --git a/crates/aprender-explain/src/tui/source_pane_tests.rs b/crates/aprender-explain/src/tui/source_pane_tests.rs new file mode 100644 index 0000000000..bdaa95f44f --- /dev/null +++ b/crates/aprender-explain/src/tui/source_pane_tests.rs @@ -0,0 +1,107 @@ +//! `render_source_pane` (#5601): title, line numbers, scroll offset, truncation, borders. + +use super::*; +use crate::analyzer::AnalysisReport; + +const W: u16 = 30; + +fn app_with(src: &str, scroll: u16) -> TuiApp { + let report = AnalysisReport { + name: "k".to_string(), + ..Default::default() + }; + let mut app = TuiApp::new(src.to_string(), report); + app.source_scroll = scroll; + app +} + +/// Render the pane at (0,0) with the given height into a W x 12 buffer. +fn rendered(app: &TuiApp, width: u16, height: u16) -> CellBuffer { + let mut buf = CellBuffer::new(W, 12); + { + let mut canvas = DirectTerminalCanvas::new(&mut buf); + render_source_pane(&mut canvas, app, 0.0, 0.0, width, height); + } + buf +} + +fn cell(buf: &CellBuffer, x: u16, y: u16) -> String { + buf.get(x, y).expect("cell").symbol.to_string() +} + +fn row(buf: &CellBuffer, y: u16) -> String { + (0..W) + .map(|x| buf.get(x, y).expect("cell").symbol.to_string()) + .collect::() + .trim_end() + .to_string() +} + +#[test] +fn title_carries_the_report_name_and_corners() { + let buf = rendered(&app_with("ret;", 0), 20, 5); + let top = row(&buf, 0); + assert!(top.starts_with("┌ PTX: k "), "{top}"); + assert!(top.ends_with('┐'), "{top}"); + assert_eq!(top.chars().count(), 20, "{top}"); + let bottom = row(&buf, 4); + assert_eq!(bottom, format!("└{}┘", "─".repeat(18))); +} + +#[test] +fn rows_show_one_based_line_numbers_and_text_between_borders() { + let buf = rendered(&app_with("a\nb\nc", 0), 20, 5); + let r1 = row(&buf, 1); + assert!(r1.starts_with("│ 1 a"), "{r1}"); + assert_eq!(cell(&buf, 19, 1), "│", "right border"); + assert!(row(&buf, 2).starts_with("│ 2 b"), "{}", row(&buf, 2)); + assert!(row(&buf, 3).starts_with("│ 3 c"), "{}", row(&buf, 3)); +} + +#[test] +fn scroll_offsets_the_first_visible_line() { + let src = "l1\nl2\nl3\nl4\nl5"; + let buf = rendered(&app_with(src, 2), 20, 5); + assert!(row(&buf, 1).starts_with("│ 3 l3"), "{}", row(&buf, 1)); + assert!(row(&buf, 2).starts_with("│ 4 l4"), "{}", row(&buf, 2)); + assert!(row(&buf, 3).starts_with("│ 5 l5"), "{}", row(&buf, 3)); +} + +#[test] +fn rows_past_the_end_keep_borders_but_have_no_line_number() { + let buf = rendered(&app_with("only", 0), 20, 5); + let r = row(&buf, 2); + assert!(r.starts_with('│'), "{r}"); + assert!(!r.chars().any(|c| c.is_ascii_digit()), "{r}"); + assert_eq!(cell(&buf, 19, 2), "│", "right border"); +} + +#[test] +fn long_lines_are_truncated_to_the_inner_width() { + // width 20 -> inner 18 -> text budget 18 - 6 = 12 chars + let buf = rendered(&app_with("ABCDEFGHIJKLMNOPQRST", 0), 20, 4); + let r = row(&buf, 1); + // text occupies cols 6..=17; col 18 is the scrollbar, col 19 the border + assert_eq!(cell(&buf, 6, 1), "A"); + assert_eq!(cell(&buf, 17, 1), "L"); + assert_eq!(cell(&buf, 19, 1), "│"); + assert!((20..W).all(|x| cell(&buf, x, 1) == " "), "{r}"); +} + +#[test] +fn offset_origin_moves_the_whole_pane() { + let app = app_with("a", 0); + let mut buf = CellBuffer::new(W, 12); + { + let mut canvas = DirectTerminalCanvas::new(&mut buf); + render_source_pane(&mut canvas, &app, 2.0, 3.0, 20, 4); + } + assert_eq!(cell(&buf, 2, 3), "┌"); + assert_eq!(cell(&buf, 2, 6), "└"); + assert_eq!(cell(&buf, 0, 3), " "); + // content row 4: left border at x, number at x+1, text at x+6, right border at x+width-1 + assert_eq!(cell(&buf, 2, 4), "│"); + assert_eq!(cell(&buf, 6, 4), "1"); + assert_eq!(cell(&buf, 8, 4), "a"); + assert_eq!(cell(&buf, 21, 4), "│"); +} diff --git a/crates/aprender-qa-cli/src/certification_processing.rs b/crates/aprender-qa-cli/src/certification_processing.rs index 3b448eb795..024e917dd7 100644 --- a/crates/aprender-qa-cli/src/certification_processing.rs +++ b/crates/aprender-qa-cli/src/certification_processing.rs @@ -511,3 +511,85 @@ fn print_ticket_line(ticket: &aprender_qa_report::ticket::UpstreamTicket) { println!(" Fixture: {fixture}"); } } + +#[cfg(test)] +mod profiling_assertion_tests { + use super::*; + use aprender_qa_runner::{Playbook, SixColumnProfile}; + + fn playbook(assertions: &str) -> Playbook { + let yaml = format!( + "name: pa\nversion: \"1.0.0\"\nmodel:\n hf_repo: \"t/m\"\ntest_matrix:\n modalities: [run]\n backends: [cpu]\n scenario_count: 1\nprofile_ci:\n enabled: true\n assertions:\n{assertions}\n" + ); + Playbook::from_yaml(&yaml).expect("playbook yaml parses") + } + + fn profile(cpu: f64, gpu: f64) -> SixColumnProfile { + SixColumnProfile { + tps_gguf_cpu: Some(cpu), + tps_gguf_gpu: Some(gpu), + ..SixColumnProfile::default() + } + } + + fn failures(p: &SixColumnProfile) -> Vec<(String, String, f64)> { + p.failed_assertions + .iter() + .map(|a| (a.format.clone(), a.backend.clone(), a.min_threshold)) + .collect() + } + + #[test] + fn no_profile_ci_records_nothing() { + let mut pb = playbook(" max_p99_ms: 1.0"); + pb.profile_ci = None; + let mut p = profile(0.1, 0.1); + check_profiling_assertions(&mut p, &pb); + assert!(p.failed_assertions.is_empty()); + } + + #[test] + fn defaults_are_5_cpu_and_50_gpu() { + let pb = playbook(" max_p99_ms: 1.0"); + let mut p = profile(4.9, 49.9); + check_profiling_assertions(&mut p, &pb); + assert_eq!( + failures(&p), + vec![ + ("gguf".to_string(), "cpu".to_string(), 5.0), + ("gguf".to_string(), "gpu".to_string(), 50.0), + ] + ); + } + + #[test] + fn throughput_at_default_threshold_passes() { + let pb = playbook(" max_p99_ms: 1.0"); + let mut p = profile(5.0, 50.0); + check_profiling_assertions(&mut p, &pb); + assert!(p.failed_assertions.is_empty()); + } + + #[test] + fn legacy_min_throughput_applies_to_both_backends() { + let pb = playbook(" min_throughput: 10.0"); + let mut p = profile(8.0, 8.0); + check_profiling_assertions(&mut p, &pb); + assert_eq!( + failures(&p), + vec![ + ("gguf".to_string(), "cpu".to_string(), 10.0), + ("gguf".to_string(), "gpu".to_string(), 10.0), + ] + ); + } + + #[test] + fn per_backend_minimum_overrides_legacy_minimum() { + let pb = playbook(" min_throughput: 10.0\n min_throughput_cpu: 1.0\n min_throughput_gpu: 100.0"); + let mut p = profile(8.0, 8.0); + check_profiling_assertions(&mut p, &pb); + assert_eq!(failures(&p), vec![("gguf".to_string(), "gpu".to_string(), 100.0)]); + assert_eq!(p.failed_assertions[0].actual_tps, 8.0); + } +} diff --git a/crates/aprender-train/src/finetune/classify_trainer_tests.rs b/crates/aprender-train/src/finetune/classify_trainer_tests.rs index e07109a10a..3af44ac004 100644 --- a/crates/aprender-train/src/finetune/classify_trainer_tests.rs +++ b/crates/aprender-train/src/finetune/classify_trainer_tests.rs @@ -5099,3 +5099,128 @@ fn test_coverage_card_confusion_raw_formatting() { assert!(out.contains("```")); assert!(out.contains("Predicted")); } + +// ========================================================================= +// run_worker: distributed worker loop (#5636) +// ========================================================================= + +mod run_worker_tests { + use super::*; + use crate::finetune::distributed::{DistributedConfig, WireMessage}; + use crate::finetune::gradient_server::{read_wire_message, send_wire_message}; + use std::net::TcpListener; + + fn worker_trainer(addr: Option) -> ClassifyTrainer { + let config = TrainingConfig { + distributed: addr.map(DistributedConfig::worker), + ..TrainingConfig::default() + }; + ClassifyTrainer::new(tiny_pipeline(3), make_corpus(8, 3), config) + .expect("config should be valid") + } + + fn expect_config_error(result: crate::Result, needle: &str) { + match result { + Err(crate::Error::ConfigError(msg)) => { + assert!(msg.contains(needle), "expected {needle:?} in {msg:?}"); + } + other => panic!("expected ConfigError containing {needle:?}, got {other:?}"), + } + } + + /// Accept one worker, answer its join, then play `script` against the stream. + fn fake_coordinator( + script: impl FnOnce(&std::net::TcpStream) -> T + Send + 'static, + ) -> (std::net::SocketAddr, std::thread::JoinHandle) { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind"); + let addr = listener.local_addr().expect("addr"); + let handle = std::thread::spawn(move || { + let (stream, _) = listener.accept().expect("accept"); + let join = read_wire_message(&stream).expect("join request"); + assert!(matches!(join, WireMessage::JoinRequest { gpu_count: 1, .. })); + let accepted = WireMessage::JoinAccepted { worker_id: 7, total_workers: 2 }; + send_wire_message(&stream, &accepted).expect("join accepted"); + script(&stream) + }); + (addr, handle) + } + + #[test] + fn missing_distributed_config_is_a_config_error() { + let mut trainer = worker_trainer(None); + expect_config_error(trainer.run_worker(), "distributed config required"); + } + + #[test] + fn unreachable_coordinator_is_a_connect_error() { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind"); + let addr = listener.local_addr().expect("addr"); + drop(listener); + let mut trainer = worker_trainer(Some(addr)); + expect_config_error(trainer.run_worker(), "worker connect failed"); + } + + #[test] + fn immediate_shutdown_returns_an_empty_result() { + let (addr, coordinator) = fake_coordinator(|s| { + send_wire_message(s, &WireMessage::Shutdown).expect("shutdown"); + }); + let result = worker_trainer(Some(addr)).run_worker().expect("clean shutdown"); + coordinator.join().expect("coordinator thread"); + assert!(result.epoch_metrics.is_empty()); + assert_eq!(result.best_epoch, 0); + assert!(result.best_val_loss.is_infinite()); + assert!(!result.stopped_early); + } + + #[test] + fn one_step_round_trips_shard_gradients_and_averaged_update() { + let (addr, coordinator) = fake_coordinator(|s| { + // Shard end past the training split must be clamped to its length. + let shard = WireMessage::ShardAssignment { step: 3, shard_start: 2, shard_end: 99 }; + send_wire_message(s, &shard).expect("shard"); + let payload = read_wire_message(s).expect("gradient payload"); + let WireMessage::GradientPayload { step, worker_id, gradients, total, .. } = payload + else { + panic!("expected GradientPayload, got {payload:?}"); + }; + let averaged = WireMessage::AveragedGradient { + step, + gradients: gradients.clone(), + global_loss: 0.5, + }; + send_wire_message(s, &averaged).expect("averaged"); + send_wire_message(s, &WireMessage::Shutdown).expect("shutdown"); + (step, worker_id, gradients.len(), total) + }); + let mut trainer = worker_trainer(Some(addr)); + let expected_total = trainer.train_data.len() - 2; + let result = trainer.run_worker().expect("one step then shutdown"); + let (step, worker_id, grad_len, total) = coordinator.join().expect("coordinator thread"); + assert_eq!((step, worker_id, total), (3, 7, expected_total), "shard 2..len"); + assert!(grad_len > 0, "worker must send its LoRA gradients"); + assert!(result.epoch_metrics.is_empty()); + } + + #[test] + fn shutdown_during_allreduce_is_an_averaged_receive_error() { + let (addr, coordinator) = fake_coordinator(|s| { + let shard = WireMessage::ShardAssignment { step: 0, shard_start: 0, shard_end: 2 }; + send_wire_message(s, &shard).expect("shard"); + read_wire_message(s).expect("gradient payload"); + send_wire_message(s, &WireMessage::Shutdown).expect("shutdown"); + }); + expect_config_error(worker_trainer(Some(addr)).run_worker(), "averaged receive failed"); + coordinator.join().expect("coordinator thread"); + } + + #[test] + fn unexpected_message_instead_of_shard_is_a_shard_receive_error() { + let (addr, coordinator) = fake_coordinator(|s| { + let hb = WireMessage::Heartbeat { node_id: "x".into(), timestamp_ms: 1 }; + send_wire_message(s, &hb).expect("heartbeat"); + }); + expect_config_error(worker_trainer(Some(addr)).run_worker(), "shard receive failed"); + coordinator.join().expect("coordinator thread"); + } +} From dfc9aa33496e45172f4f23e83adcab323533d761 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Wed, 21 Oct 2026 10:44:15 +0200 Subject: [PATCH 3/8] test: fold branch-only lines #5622 #5632 #5626 (fold of #5728) Refs PMAT-1702 Co-Authored-By: Claude Sonnet 5.5 --- ...7-contracts-cli-certify-analyze-config.cmd | 1 + .../bin-aprender-contracts-cli--pv-v1.yaml | 6 ++ contracts/contracts.nt | 5 + .../test-lib-loadtest-run-phase-max-v1.yaml | 47 ++++++++ ...5f0be2f90923120a3027c176ad56723d0d850.json | 2 +- .../tests/certify_analyze_config_5622.rs | 101 ++++++++++++++++++ .../aprender-sparse/src/bsr_new_5632_tests.rs | 91 ++++++++++++++++ crates/aprender-sparse/src/lib.rs | 3 + crates/aprender-test-lib/src/llm/loadtest.rs | 75 +++++++++++++ 9 files changed, 330 insertions(+), 1 deletion(-) create mode 100644 ci/explicit-test-commands.d/817-contracts-cli-certify-analyze-config.cmd create mode 100644 contracts/test-lib-loadtest-run-phase-max-v1.yaml create mode 100644 crates/aprender-contracts-cli/tests/certify_analyze_config_5622.rs create mode 100644 crates/aprender-sparse/src/bsr_new_5632_tests.rs diff --git a/ci/explicit-test-commands.d/817-contracts-cli-certify-analyze-config.cmd b/ci/explicit-test-commands.d/817-contracts-cli-certify-analyze-config.cmd new file mode 100644 index 0000000000..6460edb736 --- /dev/null +++ b/ci/explicit-test-commands.d/817-contracts-cli-certify-analyze-config.cmd @@ -0,0 +1 @@ +cargo test -p aprender-contracts-cli --test certify_analyze_config_5622 diff --git a/contracts/bin-aprender-contracts-cli--pv-v1.yaml b/contracts/bin-aprender-contracts-cli--pv-v1.yaml index c4358221b9..f2e048b2a9 100644 --- a/contracts/bin-aprender-contracts-cli--pv-v1.yaml +++ b/contracts/bin-aprender-contracts-cli--pv-v1.yaml @@ -174,3 +174,9 @@ falsification_tests: test: 'cargo test -p aprender-contracts-cli --test coverage_report_render' if_fails: 'the coverage report renderer drifts without a test noticing' mutation: 'print binding_partial in the Binding implemented line; coverage_with_binding_adds_binding_totals must turn RED (MEASURED 2026-10-19)' + - id: FALSIFY-BIN-PV-007 + rule: '`certify` config analysis flags the algebraic constraints a transformer needs' + prediction: 'analyze_config derives head_dim = hidden/heads, defaults kv heads to heads, and names each failing check (positivity, hidden%heads, heads%kv, even head_dim) without dividing by zero' + test: 'cargo test -p aprender-contracts-cli --test certify_analyze_config_5622' + if_fails: 'a malformed config.json is certified, or the report names the wrong constraint' + mutation: 'change `head_dim % 2 == 0` to `head_dim % 2 == 1`; odd_head_dim_fails_only_the_rope_check must turn RED' diff --git a/contracts/contracts.nt b/contracts/contracts.nt index 87447a2049..a0b20c44eb 100644 --- a/contracts/contracts.nt +++ b/contracts/contracts.nt @@ -13961,6 +13961,11 @@ . "contracts/tensor-transpose-roundtrip-v1.yaml"^^ . "tensor-transpose-roundtrip-v1"^^ . + . + . + "contracts/test-lib-loadtest-run-phase-max-v1.yaml"^^ . + "test-lib-loadtest-run-phase-max-v1"^^ . + "pattern"^^ . . . . diff --git a/contracts/test-lib-loadtest-run-phase-max-v1.yaml b/contracts/test-lib-loadtest-run-phase-max-v1.yaml new file mode 100644 index 0000000000..5ecdbb320d --- /dev/null +++ b/contracts/test-lib-loadtest-run-phase-max-v1.yaml @@ -0,0 +1,47 @@ +metadata: + version: 1.0.0 + created: '2026-10-19' + author: PAIML Engineering + description: "aprender-test-lib LoadTest::run_phase_max (#5626) — closed-loop phase: every worker sends back-to-back requests until the deadline and the records carry the server's token counts; a dead server yields failed records, a zero-length phase sends nothing." + kind: pattern + references: + - "crates/aprender-test-lib/src/llm/loadtest.rs — run_phase_max" + - "aprender#5626 — coverage ticket" + +equations: + closed_loop_phase: + formula: "records(run_phase_max(d)) = [] when d = 0; otherwise >= concurrency records; success iff the server answered 2xx with a parseable body; tokens = usage.completion_tokens" + domain: "LoadTest with Max rate, a duration, and an HTTP endpoint" + codomain: "Vec" + invariants: + - "response content is not captured unless validation needs it" + - "an unreachable server never yields a successful record" + +proof_obligations: + - id: LOADTEST-MAX-INV-001 + type: invariant + property: "Records carry the server's token counts and finish reason" + formal: "∀ run: cargo test -p aprender-test-lib --features llm --lib test_run_phase_max_records_server_token_counts exits 0" + applies_to: closed_loop_phase + - id: LOADTEST-MAX-INV-002 + type: invariant + property: "Zero-length phase sends nothing" + formal: "∀ run: cargo test -p aprender-test-lib --features llm --lib test_run_phase_max_zero_duration_sends_nothing exits 0" + applies_to: closed_loop_phase + +falsification_tests: + - id: FALSIFY-LOADTEST-MAX-001 + rule: "deadline loop disabled" + prediction: "replacing the while condition with false turns test_run_phase_max_records_server_token_counts RED" + test: "cargo test -p aprender-test-lib --features llm --lib test_run_phase_max_records_server_token_counts" + if_fails: "worker loop is unguarded" + - id: FALSIFY-LOADTEST-MAX-002 + rule: "workers not spawned" + prediction: "spawning 0 workers turns test_run_phase_max_unreachable_server_yields_failed_records RED" + test: "cargo test -p aprender-test-lib --features llm --lib test_run_phase_max_unreachable_server_yields_failed_records" + if_fails: "worker spawn count is unguarded" + - id: FALSIFY-LOADTEST-MAX-003 + rule: "content capture forced on" + prediction: "capture_content = true turns test_run_phase_max_records_server_token_counts RED" + test: "cargo test -p aprender-test-lib --features llm --lib test_run_phase_max_records_server_token_counts" + if_fails: "content capture gating is unguarded" diff --git a/contracts/witness/811c78184eb2609244c59534f015f0be2f90923120a3027c176ad56723d0d850.json b/contracts/witness/811c78184eb2609244c59534f015f0be2f90923120a3027c176ad56723d0d850.json index 5ca9abb1de..cd088777f2 100644 --- a/contracts/witness/811c78184eb2609244c59534f015f0be2f90923120a3027c176ad56723d0d850.json +++ b/contracts/witness/811c78184eb2609244c59534f015f0be2f90923120a3027c176ad56723d0d850.json @@ -1,5 +1,5 @@ { - "census_id_set_sha256": "db20faa02774eeffec52d05d3e579c0ec7047e2f0bed655de26b7e3eeff34151", + "census_id_set_sha256": "a3ed7f39b829b351f611f40bcdd1f5d2193f087021ff4941712d41f762526b6e", "relations_sha256": "811c78184eb2609244c59534f015f0be2f90923120a3027c176ad56723d0d850", "checkable_n": 87, "result": { diff --git a/crates/aprender-contracts-cli/tests/certify_analyze_config_5622.rs b/crates/aprender-contracts-cli/tests/certify_analyze_config_5622.rs new file mode 100644 index 0000000000..0a7f11c803 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/certify_analyze_config_5622.rs @@ -0,0 +1,101 @@ +//! `certify::analyze_config` derives geometry from a HF `config.json` and flags +//! the algebraic constraints a transformer needs (#5622). + +use aprender_contracts_cli::commands::certify::analyze_config; + +fn analyze(json: &str) -> serde_json::Value { + let dir = tempfile::tempdir().expect("tempdir"); + let p = dir.path().join("config.json"); + std::fs::write(&p, json).expect("write"); + analyze_config(&p).expect("analyze") +} + +fn failed_checks(v: &serde_json::Value) -> Vec { + v["algebra_checks"] + .as_array() + .expect("array") + .iter() + .filter(|c| c["passed"] == false) + .map(|c| c["check"].as_str().expect("name").to_string()) + .collect() +} + +#[test] +fn valid_config_derives_geometry_and_passes_every_check() { + let v = analyze( + r#"{"hidden_size":64,"num_hidden_layers":2,"num_attention_heads":8, + "num_key_value_heads":2,"vocab_size":100}"#, + ); + assert_eq!(v["head_dim"], 8); + assert_eq!(v["num_kv_heads"], 2); + // embed + 9 tensors per layer + final norm + lm_head + assert_eq!(v["expected_tensors"], 1 + 2 * 9 + 2); + assert_eq!(v["algebra_checks"].as_array().expect("array").len(), 7); + assert_eq!(v["all_checks_pass"], true); + assert!(failed_checks(&v).is_empty()); +} + +#[test] +fn kv_heads_default_to_attention_heads() { + let v = analyze( + r#"{"hidden_size":32,"num_hidden_layers":1,"num_attention_heads":4,"vocab_size":10}"#, + ); + assert_eq!(v["num_kv_heads"], 4); + assert_eq!(v["all_checks_pass"], true); +} + +#[test] +fn empty_config_fails_the_positivity_checks_without_dividing_by_zero() { + let v = analyze("{}"); + assert_eq!(v["all_checks_pass"], false); + assert_eq!(v["head_dim"], 0); + assert_eq!(v["expected_tensors"], 0); + let failed = failed_checks(&v); + for name in [ + "hidden_size > 0", + "num_layers > 0", + "num_heads > 0", + "vocab_size > 0", + ] { + assert!( + failed.iter().any(|f| f == name), + "{name} should fail: {failed:?}" + ); + } +} + +#[test] +fn indivisible_heads_are_named() { + let v = analyze( + r#"{"hidden_size":30,"num_hidden_layers":1,"num_attention_heads":4, + "num_key_value_heads":3,"vocab_size":10}"#, + ); + assert_eq!(v["all_checks_pass"], false); + let failed = failed_checks(&v); + assert!( + failed.contains(&"hidden_size % num_heads == 0".to_string()), + "{failed:?}" + ); + assert!( + failed.contains(&"num_heads % num_kv_heads == 0".to_string()), + "{failed:?}" + ); +} + +#[test] +fn odd_head_dim_fails_only_the_rope_check() { + let v = analyze( + r#"{"hidden_size":12,"num_hidden_layers":1,"num_attention_heads":4,"vocab_size":10}"#, + ); + assert_eq!(v["head_dim"], 3); + assert_eq!(failed_checks(&v), vec!["head_dim % 2 == 0".to_string()]); +} + +#[test] +fn missing_file_and_bad_json_are_errors() { + let dir = tempfile::tempdir().expect("tempdir"); + assert!(analyze_config(&dir.path().join("absent.json")).is_err()); + let p = dir.path().join("bad.json"); + std::fs::write(&p, "{not json").expect("write"); + assert!(analyze_config(&p).is_err()); +} diff --git a/crates/aprender-sparse/src/bsr_new_5632_tests.rs b/crates/aprender-sparse/src/bsr_new_5632_tests.rs new file mode 100644 index 0000000000..b42f13bb61 --- /dev/null +++ b/crates/aprender-sparse/src/bsr_new_5632_tests.rs @@ -0,0 +1,91 @@ +//! Coverage for `BsrMatrix::new` validation (#5632). + +use crate::{BsrMatrix, SparseError}; + +#[test] +fn accepts_consistent_structure_and_reports_shape() { + // 2x3 block grid, block_size 2, two stored blocks. + let m = BsrMatrix::new(2, 3, 2, vec![0, 1, 2], vec![0, 2], vec![1.0; 8]).expect("valid"); + assert_eq!(m.rows(), 4); + assert_eq!(m.cols(), 6); + assert_eq!(m.block_size(), 2); + assert_eq!(m.nnz_blocks(), 2); +} + +#[test] +fn accepts_empty_matrix() { + let m = BsrMatrix::new(1, 1, 4, vec![0, 0], vec![], vec![]).expect("empty is valid"); + assert_eq!(m.nnz_blocks(), 0); + assert_eq!(m.rows(), 4); +} + +#[test] +fn rejects_offsets_of_wrong_length() { + let err = BsrMatrix::new(2, 2, 1, vec![0, 1], vec![0], vec![1.0]).expect_err("short offsets"); + assert!( + matches!( + err, + SparseError::InvalidOffsetsLength { + actual: 2, + expected: 3 + } + ), + "{err}" + ); + let err = BsrMatrix::new(1, 1, 1, vec![0, 0, 0], vec![], vec![]).expect_err("long offsets"); + assert!( + matches!( + err, + SparseError::InvalidOffsetsLength { + actual: 3, + expected: 2 + } + ), + "{err}" + ); +} + +#[test] +fn rejects_values_not_matching_block_count_times_block_area() { + // 1 block of 2x2 needs 4 values. + let err = BsrMatrix::new(1, 1, 2, vec![0, 1], vec![0], vec![1.0; 3]).expect_err("short"); + assert!( + matches!( + err, + SparseError::LengthMismatch { + col_len: 4, + val_len: 3 + } + ), + "{err}" + ); + let err = BsrMatrix::new(1, 1, 2, vec![0, 1], vec![0], vec![1.0; 5]).expect_err("long"); + assert!( + matches!( + err, + SparseError::LengthMismatch { + col_len: 4, + val_len: 5 + } + ), + "{err}" + ); +} + +#[test] +fn offsets_length_is_checked_before_values_length() { + let err = BsrMatrix::new(2, 2, 2, vec![0], vec![0], vec![]).expect_err("both wrong"); + assert!( + matches!(err, SparseError::InvalidOffsetsLength { .. }), + "{err}" + ); +} + +#[test] +fn new_matrix_round_trips_through_to_csr() { + // One 2x2 block [[1,2],[3,4]] at block (0,1) of a 1x2 block grid. + let m = BsrMatrix::new(1, 2, 2, vec![0, 1], vec![1], vec![1.0, 2.0, 3.0, 4.0]).expect("valid"); + let csr = m.to_csr().expect("csr"); + assert_eq!((csr.rows(), csr.cols()), (2, 4)); + assert_eq!(csr.nnz(), 4); +} diff --git a/crates/aprender-sparse/src/lib.rs b/crates/aprender-sparse/src/lib.rs index bedd6b75a5..41a444f1a8 100644 --- a/crates/aprender-sparse/src/lib.rs +++ b/crates/aprender-sparse/src/lib.rs @@ -69,3 +69,6 @@ pub use validate::validate_csr_invariants; #[cfg(test)] mod tests; + +#[cfg(test)] +mod bsr_new_5632_tests; diff --git a/crates/aprender-test-lib/src/llm/loadtest.rs b/crates/aprender-test-lib/src/llm/loadtest.rs index a80361e12f..f2c2348f2b 100644 --- a/crates/aprender-test-lib/src/llm/loadtest.rs +++ b/crates/aprender-test-lib/src/llm/loadtest.rs @@ -2378,4 +2378,79 @@ mod tests { assert_eq!(tail.jitter.spike_count, 0); assert!(!tail.drift.degradation_detected); } + + // ===== run_phase_max (FALSIFY-LOADTEST-MAX-001..003) ===== + + /// Serve `body` to every connection until the returned handle is aborted. + async fn spawn_json_server(body: &'static str) -> (String, tokio::task::JoinHandle<()>) { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let handle = tokio::spawn(async move { + loop { + let Ok((mut sock, _)) = listener.accept().await else { + return; + }; + tokio::spawn(async move { + let mut buf = vec![0u8; 8192]; + let _ = sock.read(&mut buf).await; + let resp = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _ = sock.write_all(resp.as_bytes()).await; + }); + } + }); + (url, handle) + } + + const CANNED: &str = r#"{"id":"x","object":"chat.completion","created":1,"model":"m","choices":[{"index":0,"message":{"role":"assistant","content":"4"},"finish_reason":"stop"}],"usage":{"prompt_tokens":7,"completion_tokens":3,"total_tokens":10}}"#; + + fn max_config(concurrency: usize) -> LoadTestConfig { + LoadTestConfig { + concurrency, + ..LoadTestConfig::default() + } + } + + #[tokio::test] + async fn test_run_phase_max_records_server_token_counts() { + let (url, server) = spawn_json_server(CANNED).await; + let lt = LoadTest::new(LlmClient::new(url, "m"), max_config(2)); + let records = lt.run_phase_max(Duration::from_millis(200)).await.unwrap(); + server.abort(); + assert!(records.len() >= 2, "each worker sends at least once"); + for r in &records { + assert!(r.success); + assert_eq!(r.tokens, 3); + assert_eq!(r.prompt_tokens, 7); + assert_eq!(r.finish_reason.as_deref(), Some("stop")); + assert!( + r.response_content.is_none(), + "content not captured by default" + ); + } + } + + #[tokio::test] + async fn test_run_phase_max_unreachable_server_yields_failed_records() { + // Bind then drop so the port is closed. + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", l.local_addr().unwrap()); + drop(l); + let lt = LoadTest::new(LlmClient::new(url, "m"), max_config(1)); + let records = lt.run_phase_max(Duration::from_millis(100)).await.unwrap(); + assert!(!records.is_empty()); + assert!(records.iter().all(|r| !r.success && r.tokens == 0)); + } + + #[tokio::test] + async fn test_run_phase_max_zero_duration_sends_nothing() { + let (url, server) = spawn_json_server(CANNED).await; + let lt = LoadTest::new(LlmClient::new(url, "m"), max_config(3)); + let records = lt.run_phase_max(Duration::ZERO).await.unwrap(); + server.abort(); + assert!(records.is_empty()); + } } From 2da32800d46f371b09053a356724c450431d2662 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Wed, 21 Oct 2026 10:44:35 +0200 Subject: [PATCH 4/8] test(train): StepProfiler print_report split + json_report extracted and covered (split c of #5590, Refs #5577) (fold of #5827) Refs PMAT-1702 Co-Authored-By: Claude Sonnet 5.5 --- .../transformer_trainer/step_profiler.rs | 176 +++++++++++++----- 1 file changed, 126 insertions(+), 50 deletions(-) diff --git a/crates/aprender-train/src/train/transformer_trainer/step_profiler.rs b/crates/aprender-train/src/train/transformer_trainer/step_profiler.rs index a7735cb66d..10e5536206 100644 --- a/crates/aprender-train/src/train/transformer_trainer/step_profiler.rs +++ b/crates/aprender-train/src/train/transformer_trainer/step_profiler.rs @@ -439,13 +439,36 @@ impl StepProfiler { /// PMAT-483: Emit structured JSON profiling report to stderr. /// Parseable by canary scripts for scientific analysis. pub fn print_json_report(&self) { - if self.step_count == 0 { - return; + if let Some(json) = self.json_report() { + eprintln!("{json}"); } + } + /// PMAT-483: The structured JSON profiling report, or `None` before any step finished. + pub fn json_report(&self) -> Option { + if self.step_count == 0 { + return None; + } let total_us = self.total_wall.as_micros() as f64; let avg_step_ms = total_us / self.step_count as f64 / 1000.0; + let (phases, accounted_us) = self.phases_json(total_us); + let wall_coverage = if total_us > 0.0 { accounted_us as f64 / total_us } else { 0.0 }; + let (ops_json, gemm_pct) = self.ops_json(); + Some(format!( + "{{\"_profiler\":\"step_profiler_v2\",\"steps\":{},\"avg_step_ms\":{:.2},\"wall_coverage\":{:.3},\"bottleneck\":\"{}\",\"gemm_pct\":{:.1},\"phases\":{{{}}},\"per_layer\":[{}],\"ops\":{{{}}}}}", + self.step_count, + avg_step_ms, + wall_coverage, + self.bottleneck(total_us), + gemm_pct, + phases.join(","), + self.layers_json().join(","), + ops_json.join(","), + )) + } + /// Per-phase JSON entries and the microseconds they account for. + fn phases_json(&self, total_us: f64) -> (Vec, u128) { let mut accounted_us = 0u128; let mut phases = Vec::new(); for i in 0..NUM_PHASES { @@ -459,73 +482,59 @@ impl StepProfiler { PHASE_NAMES[i], ms, pct, avg )); } + (phases, accounted_us) + } - let wall_coverage = if total_us > 0.0 { accounted_us as f64 / total_us } else { 0.0 }; + fn layers_json(&self) -> Vec { + (0..self.num_layers) + .map(|i| { + let fwd_ms = self.layer_fwd_totals[i].as_micros() as f64 / 1000.0; + let bwd_ms = self.layer_bwd_totals[i].as_micros() as f64 / 1000.0; + format!("{{\"layer\":{i},\"fwd_ms\":{fwd_ms:.1},\"bwd_ms\":{bwd_ms:.1}}}") + }) + .collect() + } - let mut layers_json = Vec::new(); - for i in 0..self.num_layers { - let fwd_ms = self.layer_fwd_totals[i].as_micros() as f64 / 1000.0; - let bwd_ms = self.layer_bwd_totals[i].as_micros() as f64 / 1000.0; - layers_json - .push(format!("{{\"layer\":{i},\"fwd_ms\":{fwd_ms:.1},\"bwd_ms\":{bwd_ms:.1}}}")); + /// Classify the bottleneck from the phase distribution. + fn bottleneck(&self, total_us: f64) -> &'static str { + if total_us <= 0.0 { + return "launch"; } - - // Classify bottleneck based on phase distribution - let forward_pct = if total_us > 0.0 { - self.totals[FORWARD].as_micros() as f64 / total_us * 100.0 - } else { - 0.0 - }; - let transfer_pct = if total_us > 0.0 { + let forward_pct = self.totals[FORWARD].as_micros() as f64 / total_us * 100.0; + let transfer_pct = (self.totals[H2D].as_micros() + self.totals[GRAD_H2D].as_micros()) as f64 / total_us - * 100.0 - } else { - 0.0 - }; - let bottleneck = if transfer_pct > 30.0 { + * 100.0; + if transfer_pct > 30.0 { "transfer" } else if forward_pct < 20.0 { "launch" } else { "memory_bw" - }; + } + } - // PMAT-483: Per-operation breakdown + /// Non-zero per-operation JSON entries and the GEMM share of operation time. + fn ops_json(&self) -> (Vec, f64) { let mut ops_json = Vec::new(); let mut total_op_us = 0u128; for i in 0..NUM_OPS { - let t = self.op_totals[i]; - let us = t.as_micros(); + let us = self.op_totals[i].as_micros(); total_op_us += us; if us > 0 { - let ms = us as f64 / 1000.0; - ops_json.push(format!("\"{}\":{:.1}", OP_NAMES[i], ms)); + ops_json.push(format!("\"{}\":{:.1}", OP_NAMES[i], us as f64 / 1000.0)); } } - - // Classify which operation type dominates - let gemm_us = self.op_totals[OP_QKV_GEMM].as_micros() - + self.op_totals[OP_O_PROJ].as_micros() - + self.op_totals[OP_GATE_UP_GEMM].as_micros() - + self.op_totals[OP_DOWN_GEMM].as_micros(); - let gemm_bwd_us = self.op_totals[OP_QKV_BWD].as_micros() - + self.op_totals[OP_GATE_UP_BWD].as_micros() - + self.op_totals[OP_DOWN_BWD].as_micros(); - let total_gemm_us = gemm_us + gemm_bwd_us; + let us = |op: usize| self.op_totals[op].as_micros(); + let total_gemm_us = us(OP_QKV_GEMM) + + us(OP_O_PROJ) + + us(OP_GATE_UP_GEMM) + + us(OP_DOWN_GEMM) + + us(OP_QKV_BWD) + + us(OP_GATE_UP_BWD) + + us(OP_DOWN_BWD); let gemm_pct = if total_op_us > 0 { total_gemm_us as f64 / total_op_us as f64 * 100.0 } else { 0.0 }; - - eprintln!( - "{{\"_profiler\":\"step_profiler_v2\",\"steps\":{},\"avg_step_ms\":{:.2},\"wall_coverage\":{:.3},\"bottleneck\":\"{}\",\"gemm_pct\":{:.1},\"phases\":{{{}}},\"per_layer\":[{}],\"ops\":{{{}}}}}", - self.step_count, - avg_step_ms, - wall_coverage, - bottleneck, - gemm_pct, - phases.join(","), - layers_json.join(","), - ops_json.join(","), - ); + (ops_json, gemm_pct) } /// PMAT-483: Feed per-layer timing data from InstructGpuTrainingState. @@ -901,4 +910,71 @@ mod tests { p.print_report(); assert_eq!(p.num_layers, 3); } + + fn profiler_with_wall(steps: usize, wall_ms: u64) -> StepProfiler { + let mut p = StepProfiler::new(true, 0); + p.step_count = steps; + p.total_wall = Duration::from_millis(wall_ms); + p + } + + #[test] + fn test_json_report_is_none_before_any_step() { + assert!(StepProfiler::new(true, 0).json_report().is_none()); + } + + #[test] + fn test_json_report_header_steps_avg_and_coverage() { + let mut p = profiler_with_wall(4, 400); + p.totals[FORWARD] = Duration::from_millis(200); + let j = p.json_report().expect("report"); + assert!(j.starts_with("{\"_profiler\":\"step_profiler_v2\",\"steps\":4,")); + assert!(j.contains("\"avg_step_ms\":100.00"), "{j}"); + assert!(j.contains("\"wall_coverage\":0.500"), "{j}"); + assert!(j.contains("\"forward\":{\"total_ms\":200.0,\"pct\":50.0,\"avg_ms\":50.00}")); + } + + #[test] + fn test_json_report_bottleneck_classification() { + let mut p = profiler_with_wall(1, 100); + p.totals[H2D] = Duration::from_millis(20); + p.totals[GRAD_H2D] = Duration::from_millis(15); + assert!(p.json_report().expect("report").contains("\"bottleneck\":\"transfer\"")); + + let mut p = profiler_with_wall(1, 100); + p.totals[FORWARD] = Duration::from_millis(10); + assert!(p.json_report().expect("report").contains("\"bottleneck\":\"launch\"")); + + let mut p = profiler_with_wall(1, 100); + p.totals[FORWARD] = Duration::from_millis(60); + assert!(p.json_report().expect("report").contains("\"bottleneck\":\"memory_bw\"")); + } + + #[test] + fn test_json_report_gemm_pct_and_nonzero_ops_only() { + let mut p = profiler_with_wall(1, 100); + p.op_totals[OP_QKV_GEMM] = Duration::from_millis(30); + p.op_totals[OP_QKV_BWD] = Duration::from_millis(10); + p.op_totals[OP_ATTENTION] = Duration::from_millis(60); + let j = p.json_report().expect("report"); + assert!(j.contains("\"gemm_pct\":40.0"), "{j}"); + assert!(j.contains(&format!("\"{}\":30.0", OP_NAMES[OP_QKV_GEMM]))); + assert!(!j.contains(&format!("\"{}\":", OP_NAMES[OP_SILU])), "zero ops omitted"); + } + + #[test] + fn test_json_report_per_layer_rows_from_recorded_times() { + let mut p = profiler_with_wall(1, 100); + p.record_layer_times(&[2_000, 4_000], &[0, 6_000]); + let j = p.json_report().expect("report"); + assert!(j.contains("\"per_layer\":[{\"layer\":0,\"fwd_ms\":2.0,\"bwd_ms\":0.0},{\"layer\":1,\"fwd_ms\":4.0,\"bwd_ms\":6.0}]"), "{j}"); + } + + #[test] + fn test_json_report_zero_wall_does_not_divide_by_zero() { + let mut p = profiler_with_wall(1, 0); + p.totals[FORWARD] = Duration::from_millis(5); + let j = p.json_report().expect("report"); + assert!(j.contains("\"wall_coverage\":0.000") && j.contains("\"gemm_pct\":0.0"), "{j}"); + } } From 207a6b92a095c33b959c5da177961b026324614e Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Wed, 21 Oct 2026 11:42:56 +0200 Subject: [PATCH 5/8] fix(explicit-tests): move certify-analyze-config to free ordinal 826 (817 is taken by harness-server-cleanup) Quorum FAIL 2da32800d4. Refs PMAT-1702 Co-Authored-By: Claude Sonnet 5.5 --- ...ze-config.cmd => 826-contracts-cli-certify-analyze-config.cmd} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ci/explicit-test-commands.d/{817-contracts-cli-certify-analyze-config.cmd => 826-contracts-cli-certify-analyze-config.cmd} (100%) diff --git a/ci/explicit-test-commands.d/817-contracts-cli-certify-analyze-config.cmd b/ci/explicit-test-commands.d/826-contracts-cli-certify-analyze-config.cmd similarity index 100% rename from ci/explicit-test-commands.d/817-contracts-cli-certify-analyze-config.cmd rename to ci/explicit-test-commands.d/826-contracts-cli-certify-analyze-config.cmd From dbd2eaa1ad4deff71c9cf7f7a554abc86e2fc6b6 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Wed, 21 Oct 2026 12:29:24 +0200 Subject: [PATCH 6/8] docs(contracts): QA-007 mutation receipt + witness regen command QA-007 mutation (unwrap_or(5.0)->6.0 at certification_processing.rs:280), re-run 2026-10-21T10:15Z: cargo test -p aprender-qa-cli --lib profiling_assertion_tests mutant: rc=101, 2 failed (defaults_are_5_cpu_and_50_gpu, throughput_at_default_threshold_passes) restored: rc=0, 5 passed. Witness 811c78... census_id_set_sha256 db20faa0->a3ed7f39: the census id set gained the new falsification ids (FALSIFY-BIN-PV-006, test-lib-loadtest contract). Regen: cargo run -p aprender-contracts-cli --bin pv-sat -- contracts (run 2026-10-21T09:59Z rc=0: witness 'is fresh', git status --porcelain contracts/witness empty). Refs PMAT-1702 Co-Authored-By: Claude Sonnet 5.5 --- contracts/bin-aprender-qa-cli--apr-qa-v1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts/bin-aprender-qa-cli--apr-qa-v1.yaml b/contracts/bin-aprender-qa-cli--apr-qa-v1.yaml index 06f4d63b94..9b941bed5f 100644 --- a/contracts/bin-aprender-qa-cli--apr-qa-v1.yaml +++ b/contracts/bin-aprender-qa-cli--apr-qa-v1.yaml @@ -140,4 +140,4 @@ falsification_tests: prediction: 'check_profiling_assertions records a failed assertion exactly when a measured tok/s is below the resolved minimum, and records nothing when profile_ci is absent' test: 'cargo test -p aprender-qa-cli --lib profiling_assertion_tests' if_fails: 'a throughput regression passes certification, or a legacy min_throughput stops applying' - mutation: 'change unwrap_or(5.0) to unwrap_or(6.0) in check_profiling_assertions; defaults_are_5_cpu_and_50_gpu must turn RED (run 2026-10-20)' + mutation: 'change unwrap_or(5.0) to unwrap_or(6.0) in check_profiling_assertions; defaults_are_5_cpu_and_50_gpu must turn RED (run 2026-10-21T10:15Z on lambda-labs: mutant rc=101, 2 of 5 RED incl. defaults_are_5_cpu_and_50_gpu; restored rc=0, 5 passed)' From 4dfa9f77e697378f05cd150af3c94730773bc62e Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Wed, 21 Oct 2026 12:54:34 +0200 Subject: [PATCH 7/8] docs(audits): raw logs for #5840 QA-007 mutation and contracts.nt regen (diff rc=0) contracts.nt/shapes.ttl regen: pv extract contracts on a clean tree, git diff --exit-code rc=0. QA-007 mutation is a manual cargo-test mutation, not cargo-mutants. Sub-PRs folded: #5784 #5781 #5728 #5827. Refs PMAT-1702 Co-Authored-By: Claude Sonnet 5.5 --- docs/audits/5840-contracts-nt-regen.txt | 10 ++++++++++ docs/audits/5840-qa-007-mutation.txt | 19 +++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 docs/audits/5840-contracts-nt-regen.txt create mode 100644 docs/audits/5840-qa-007-mutation.txt diff --git a/docs/audits/5840-contracts-nt-regen.txt b/docs/audits/5840-contracts-nt-regen.txt new file mode 100644 index 0000000000..7f842cf646 --- /dev/null +++ b/docs/audits/5840-contracts-nt-regen.txt @@ -0,0 +1,10 @@ +2026-10-21T10:46:43Z +HEAD dbd2eaa1ad4deff71c9cf7f7a554abc86e2fc6b6 (clean tree: 0 changes) +$ cargo run -q -p aprender-contracts-cli --bin pv -- extract contracts +extract_rc=0 + "written": [], + "check": null +} +$ git diff --exit-code contracts/contracts.nt contracts/shapes.ttl +diff_rc=0 +2026-10-21T10:54:18Z diff --git a/docs/audits/5840-qa-007-mutation.txt b/docs/audits/5840-qa-007-mutation.txt new file mode 100644 index 0000000000..9a7e1cdd27 --- /dev/null +++ b/docs/audits/5840-qa-007-mutation.txt @@ -0,0 +1,19 @@ +NOTE: manual mutation via 'cargo test', NOT cargo-mutants; raw output below. +2026-10-21T09:59:33Z +test cli::profiling_assertion_tests::no_profile_ci_records_nothing ... ok +test cli::profiling_assertion_tests::legacy_min_throughput_applies_to_both_backends ... ok +test cli::profiling_assertion_tests::per_backend_minimum_overrides_legacy_minimum ... ok +test cli::profiling_assertion_tests::defaults_are_5_cpu_and_50_gpu ... FAILED +test cli::profiling_assertion_tests::throughput_at_default_threshold_passes ... FAILED +---- cli::profiling_assertion_tests::defaults_are_5_cpu_and_50_gpu stdout ---- +thread 'cli::profiling_assertion_tests::defaults_are_5_cpu_and_50_gpu' (1561049) panicked at crates/aprender-qa-cli/src/certification_processing.rs:556:9: +---- cli::profiling_assertion_tests::throughput_at_default_threshold_passes stdout ---- +thread 'cli::profiling_assertion_tests::throughput_at_default_threshold_passes' (1561053) panicked at crates/aprender-qa-cli/src/certification_processing.rs:570:9: + cli::profiling_assertion_tests::defaults_are_5_cpu_and_50_gpu + cli::profiling_assertion_tests::throughput_at_default_threshold_passes +test result: FAILED. 3 passed; 2 failed; 0 ignored; 0 measured; 122 filtered out; finished in 0.00s +mutant_rc=101 +test cli::profiling_assertion_tests::defaults_are_5_cpu_and_50_gpu ... ok +test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 122 filtered out; finished in 0.00s +restored_rc=0 +2026-10-21T10:29:13Z From e170e60d20d7a6a3a37714cf2a0f6b113d61e50f Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Wed, 21 Oct 2026 13:27:33 +0200 Subject: [PATCH 8/8] batch: regenerate scripts/contracts_resolve.sh on 4dfa9f77e6 (rc=0, 2026-10-21T11:19-11:26Z): contracts/ clean, README CONTRACT_COUNT 1996 -> 1997 (batch adds contracts/test-lib-loadtest-run-phase-max-v1.yaml). Refs PMAT-1702 Co-Authored-By: Claude Sonnet 5.5 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 68106cff29..01c71588c7 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ publishing — all backed by YAML provable contracts that fail CI on drift. | Metric | Count | Source of truth | |-------:|------:|---| | Workspace crates | **81** workspace crates | `cargo metadata --no-deps` (NOT `ls crates/` — 4 are `exclude`d, 1 has no Cargo.toml) | -| Provable contracts | **1996** provable contracts | `contracts/census.json` `.n_files` — the set `pv lint` walks (`pv census`, ONT-001 ONT-1; regenerated by `make contracts`, written by `make readme-sync`, guarded by `scripts/check_readme_claims.sh`) | +| Provable contracts | **1997** provable contracts | `contracts/census.json` `.n_files` — the set `pv lint` walks (`pv census`, ONT-001 ONT-1; regenerated by `make contracts`, written by `make readme-sync`, guarded by `scripts/check_readme_claims.sh`) | | CLI commands | **113** CLI commands | `contracts/apr-cli-commands-v1.yaml` §`commands` (parse the list; `apr --help` also lists `help` itself, and `grep -c '^ - name:'` gives 119 — other same-indent `name:` keys exist in the file) | | Book CLI chapters | **114** chapters | `ls book/src/cli/*.md` | | Book lib chapters | **73** chapters | `ls book/src/lib/*.md` (parity with `pub mod`) | @@ -403,7 +403,7 @@ falsification_tests: prediction: apr validate bad-model.apr exits non-zero ``` -The tree carries 1996 contracts across inference, training, quantization, attention, FFN, +The tree carries 1997 contracts across inference, training, quantization, attention, FFN, tokenization, model formats, CLI safety — and this README itself. ## Migration from old crates