Skip to content
Merged
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
cargo test -p aprender-contracts-cli --test coverage_report_render
6 changes: 6 additions & 0 deletions contracts/bin-aprender-contracts-cli--pv-v1.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -168,3 +168,9 @@ falsification_tests:
test: 'scripts/check_surface_audit_bins.sh (aprender#4476)'
if_fails: 'a pv command ships unaudited (RED 2026-09-26: 11 of 49 paths had no row; GREEN 2026-09-27: 49 of 49)'
mutation: 'delete every pv row; must turn RED naming pv'
- id: FALSIFY-BIN-PV-006
rule: '`coverage` prints per-contract rows, totals and the overall percentage; binding totals only with --binding (#5621)'
prediction: 'without --binding no Binding lines print; with it, implemented/partial/missing totals and impl=N/M rows appear'
test: 'cargo test -p aprender-contracts-cli --test coverage_report_render'
if_fails: 'the coverage report renderer drifts without a test noticing'
mutation: 'print binding_partial in the Binding implemented line; coverage_with_binding_adds_binding_totals must turn RED (MEASURED 2026-10-19)'
6 changes: 6 additions & 0 deletions contracts/bin-aprender-ptx-debug--aprender-ptx-debug-v1.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -126,3 +126,9 @@ falsification_tests:
test: 'cargo test -p aprender-ptx-debug --test cli_surface_gate unusable_input_is_rejected'
if_fails: 'MEASURED 2026-09-27 (aprender-01): before the fix `analyze` on an empty file printed "Score: 97.1/100" with F001–F003 failed and exited 0; with the fix reverted this test is RED; fixed (a module with no .version is refused), GREEN'
mutation: 'make a rejection path return Ok(()) or unwrap the read; must turn RED on that row'
- id: FALSIFY-BIN-APRENDER-PTX-DEBUG-007
rule: 'gen-fkr emits one test per failed falsification result and per bug, named from the normalised module'
prediction: 'generate_fkr_tests emits fkr_<id>_<module> only for Fail results, fkr_bug_<class>_<module> per bug, and fkr_summary_<module> with score and confidence; "My-Kernel" yields idents my_kernel but keeps My-Kernel in the header'
test: 'cargo test -p aprender-ptx-debug --lib fkr_generator'
if_fails: 'a generated probar suite would miss a failed check, test a passing one, or carry an ident that does not compile'
mutation: 'flip result.is_fail(), drop the replace of - with _, or drop the *100 on confidence in fkr_generator.rs; fkr_generator tests must turn RED (all three shown RED 2026-10-19)'
86 changes: 86 additions & 0 deletions crates/aprender-contracts-cli/tests/coverage_report_render.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
//! `pv coverage` report rendering (#5621), driven through
//! the binary on a two-contract corpus.

#![allow(clippy::disallowed_methods)]

use std::path::Path;
use std::process::{Command, Output};

fn corpus() -> tempfile::TempDir {
let dir = tempfile::tempdir().expect("tempdir");
let fx = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures");
std::fs::copy(
fx.join("relu-kernel-v1.yaml"),
dir.path().join("relu-kernel-v1.yaml"),
)
.expect("copy relu");
let dot = std::fs::read_to_string(fx.join("dot-kernel-v1.yaml")).expect("read dot");
let dot = dot.replacen(
"metadata:\n",
"metadata:\n depends_on: [\"relu-kernel-v1\"]\n",
1,
);
std::fs::write(dir.path().join("dot-kernel-v1.yaml"), dot).expect("write dot");
dir
}

fn pv(args: &[&str], dir: &Path) -> Output {
Command::new(env!("CARGO_BIN_EXE_pv"))
.arg(args[0])
.arg(dir)
.args(&args[1..])
.output()
.expect("run pv")
}

fn stdout(o: &Output) -> String {
assert!(
o.status.success(),
"pv failed: {}",
String::from_utf8_lossy(&o.stderr)
);
String::from_utf8_lossy(&o.stdout).into_owned()
}

#[test]
fn coverage_report_prints_per_contract_rows_totals_and_percentage() {
let d = corpus();
let out = stdout(&pv(&["coverage"], d.path()));
assert!(
out.starts_with("Obligation Coverage Report\n=========================="),
"{out}"
);
assert!(
out.contains("dot-kernel-v1") && out.contains("relu-kernel-v1"),
"{out}"
);
assert!(out.contains("Contracts: 2"), "{out}");
assert!(out.contains("Overall obligation coverage: "), "{out}");
assert!(
out.contains("Falsification tests:") && out.contains("Kani harnesses:"),
"{out}"
);
assert!(
!out.contains("Binding implemented"),
"binding lines only with --binding: {out}"
);
}

#[test]
fn coverage_with_binding_adds_binding_totals() {
let d = corpus();
let b = d.path().join("binding.yaml");
std::fs::write(
&b,
"version: 1.0.0\ntarget_crate: x\nbindings:\n- contract: dot-kernel-v1.yaml\n equation: dot\n \
module_path: x::d\n function: d\n signature: 'fn d()'\n status: implemented\n",
)
.expect("binding");
let out = stdout(&pv(
&["coverage", "--binding", b.to_str().expect("utf8")],
d.path(),
));
assert!(out.contains("Binding implemented: 1"), "{out}");
assert!(out.contains("Binding partial: 0"), "{out}");
assert!(out.contains("impl=1/1"), "{out}");
}
2 changes: 2 additions & 0 deletions crates/aprender-ptx-debug/src/bugs/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

mod registry;

#[cfg(test)]
pub(crate) use registry::Bug;
pub use registry::{BugClass, BugPattern, BugRegistry};

/// Severity levels for bugs
Expand Down
115 changes: 112 additions & 3 deletions crates/aprender-ptx-debug/src/output/fkr_generator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -98,9 +98,9 @@ fn fkr_summary_{module}() {{
#[cfg(test)]
mod tests {
use super::*;
use crate::bugs::BugRegistry;
use crate::falsification::FalsificationRegistry;
use crate::parser::Parser;
use crate::bugs::{Bug, BugClass, BugPattern, BugRegistry};
use crate::falsification::{Category, FalsificationRegistry, FalsificationReport, TestResult};
use crate::parser::{Parser, SourceLocation};

#[test]
fn test_fkr_generation() {
Expand All @@ -123,4 +123,113 @@ mod tests {
assert!(fkr.contains("test_kernel"));
assert!(fkr.contains("fkr_summary_"));
}

fn report_with(
results: Vec<(String, Category, String, TestResult)>,
score: f64,
confidence: f64,
) -> FalsificationReport {
FalsificationReport {
results,
score,
earned_points: 0,
total_points: 0,
confidence,
}
}

fn fail(id: &str, desc: &str) -> (String, Category, String, TestResult) {
let result = TestResult::Fail {
evidence: "e".to_string(),
location: None,
};
(
id.to_string(),
Category::BarrierSafety,
desc.to_string(),
result,
)
}

fn sample_bug(class: BugClass) -> Bug {
Bug {
class,
location: SourceLocation {
line: 7,
column: 3,
file: None,
},
pattern: BugPattern::Other("p".to_string()),
message: "msg-xyz".to_string(),
}
}

#[test]
fn only_failed_results_get_a_test() {
let results = vec![
fail("F036", "barrier missing"),
(
"F037".to_string(),
Category::BarrierSafety,
"ok".to_string(),
TestResult::Pass,
),
(
"F038".to_string(),
Category::BarrierSafety,
"na".to_string(),
TestResult::NotApplicable,
),
];
let result = AnalysisResult::new("k", report_with(results, 80.0, 0.5), BugRegistry::new());
let fkr = generate_fkr_tests(&result);
assert!(fkr.contains("fn fkr_f036_k()"));
assert!(fkr.contains("/// FKR test for F036: barrier missing"));
assert!(fkr.contains("/// Category: BarrierSafety"));
assert!(!fkr.contains("fkr_f037"));
assert!(!fkr.contains("fkr_f038"));
}

#[test]
fn each_bug_gets_a_test_with_severity_and_mitigation() {
let mut bugs = BugRegistry::new();
bugs.add(sample_bug(BugClass::MissingBarrierSync));
let result = AnalysisResult::new("k", report_with(vec![], 80.0, 0.5), bugs);
let fkr = generate_fkr_tests(&result);
assert!(fkr.contains("fn fkr_bug_missingbarriersync_k()"));
assert!(fkr.contains("/// Severity: HIGH"));
assert!(fkr.contains("// Issue: msg-xyz"));
assert!(fkr.contains(BugClass::MissingBarrierSync.mitigation()));
}

#[test]
fn module_name_is_normalised_for_idents_but_not_the_header() {
let results = vec![fail("F036", "d")];
let result = AnalysisResult::new(
"My-Kernel",
report_with(results, 80.0, 0.5),
BugRegistry::new(),
);
let fkr = generate_fkr_tests(&result);
assert!(fkr.contains("//! Auto-generated PTX FKR tests for My-Kernel"));
assert!(fkr.contains("fn fkr_f036_my_kernel()"));
assert!(fkr.contains("fn fkr_summary_my_kernel()"));
}

#[test]
fn summary_reports_score_confidence_and_critical_flag() {
let ok = AnalysisResult::new("k", report_with(vec![], 85.0, 0.5), BugRegistry::new());
let fkr = generate_fkr_tests(&ok);
assert!(fkr.contains("// Falsification Score: 85.0/100"));
assert!(fkr.contains("// Confidence: 50.0%"));
assert!(fkr.contains("// Critical bugs absent: true"));
assert!(fkr.contains("assert!(85.0 >= 70.0"));

let bad = AnalysisResult::new(
"k",
report_with(vec![fail("F082", "critical")], 85.0, 0.5),
BugRegistry::new(),
);
assert!(generate_fkr_tests(&bad).contains("// Critical bugs absent: false"));
}
}
3 changes: 3 additions & 0 deletions crates/aprender-test-lib/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1560,3 +1560,6 @@ mod tests {
}
}
}

#[cfg(all(test, feature = "browser"))]
mod validators_partials_5625_tests;
46 changes: 23 additions & 23 deletions crates/aprender-test-lib/src/validators.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1043,6 +1043,28 @@ pub struct StreamingValidationResult {
pub total_frames: usize,
}

/// Turn the raw `[timestamp, text]` rows of `window.__probar_partials` into
/// results; malformed rows are dropped and the last survivor is marked final.
#[cfg(feature = "browser")]
pub(crate) fn partials_from_raw(rows: Vec<Vec<serde_json::Value>>) -> Vec<PartialResult> {
let mut partials: Vec<PartialResult> = rows
.into_iter()
.filter_map(|arr| {
let ts = arr.first()?.as_f64()?;
let text = arr.get(1)?.as_str()?.to_string();
Some(PartialResult {
timestamp_ms: ts,
text,
is_final: false,
})
})
.collect();
if let Some(last) = partials.last_mut() {
last.is_final = true;
}
partials
}

// CDP-based streaming UX tracking (PROBAR-SPEC-011)
#[cfg(feature = "browser")]
impl StreamingUxValidator {
Expand Down Expand Up @@ -1313,29 +1335,7 @@ impl StreamingUxValidator {
.into_value()
.map_err(|_| StreamingValidationError::EndedInError)?;

let partials: Vec<PartialResult> = result
.into_iter()
.filter_map(|arr| {
let ts = arr.first()?.as_f64()?;
let text = arr.get(1)?.as_str()?.to_string();
Some(PartialResult {
timestamp_ms: ts,
text,
is_final: false,
})
})
.collect();

// Mark the last one as final if present
if partials.is_empty() {
return Ok(partials);
}

let mut result = partials;
if let Some(last) = result.last_mut() {
last.is_final = true;
}
Ok(result)
Ok(partials_from_raw(result))
}

/// Assert state sequence occurred in order
Expand Down
55 changes: 55 additions & 0 deletions crates/aprender-test-lib/src/validators_partials_5625_tests.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
//! Coverage for the post-processing inside `collect_partials_cdp` (#5625).
//!
//! The CDP call itself needs a live chromium; the row filtering and the
//! "last row is final" marking are the branches, and they live in
//! `partials_from_raw`.

use crate::validators::partials_from_raw;
use serde_json::json;

#[test]
fn empty_input_yields_no_partials() {
assert!(partials_from_raw(vec![]).is_empty());
}

#[test]
fn only_the_last_row_is_final() {
let out = partials_from_raw(vec![
vec![json!(10.0), json!("he")],
vec![json!(20.0), json!("hel")],
vec![json!(30.0), json!("hello")],
]);
let finals: Vec<bool> = out.iter().map(|p| p.is_final).collect();
assert_eq!(finals, vec![false, false, true]);
assert_eq!(out[2].text, "hello");
assert!((out[1].timestamp_ms - 20.0).abs() < f64::EPSILON);
}

#[test]
fn single_row_is_final() {
let out = partials_from_raw(vec![vec![json!(1.5), json!("a")]]);
assert_eq!(out.len(), 1);
assert!(out[0].is_final);
}

#[test]
fn malformed_rows_are_dropped_and_finality_goes_to_last_survivor() {
let out = partials_from_raw(vec![
vec![json!(1.0), json!("ok")],
vec![], // no timestamp
vec![json!("x"), json!("bad")], // non-numeric timestamp
vec![json!(2.0)], // no text
vec![json!(3.0), json!(7)], // non-string text
vec![json!(4.0), json!("last")],
vec![json!(5.0), json!(null)], // trailing malformed row
]);
let texts: Vec<&str> = out.iter().map(|p| p.text.as_str()).collect();
assert_eq!(texts, vec!["ok", "last"]);
assert!(!out[0].is_final);
assert!(out[1].is_final);
}

#[test]
fn all_malformed_yields_empty_not_final() {
assert!(partials_from_raw(vec![vec![json!(null), json!(null)], vec![]]).is_empty());
}
1 change: 1 addition & 0 deletions scripts/tree_reader_tests.txt
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ aprender-contracts-cli --lib commands::obligations
aprender-contracts-cli --lib commands::verify_pipeline
aprender-contracts-cli --lib contract_walk
aprender-contracts-cli --test cli_integration
aprender-contracts-cli --test coverage_report_render
aprender-contracts-cli --test ev11_lint_ratchets
aprender-contracts-cli --test ground_truth
aprender-contracts-cli --test ont2b_sigma_gate
Expand Down
Loading