test: fold OPS-015..023 chain and wire repl_run + cli_entrypoint_dispatch (HELD)

ci/explicit-test-commands.d/805-aprender-orchestrate-repl-run.cmd

@@ -0,0 +1,3 @@
+
+# #5706: run_repl loop tests (stdin EOF, quit, blank lines, max-turns, zero budget); dark in CI until wired here.
+cargo test -p aprender-orchestrate --test repl_run

ci/explicit-test-commands.d/806-aprender-serve-cli-entrypoint-dispatch.cmd

@@ -0,0 +1,3 @@
+
+# #5707: CLI entrypoint dispatch tests for aprender-serve; dark in CI until wired here.
+cargo test -p aprender-serve --test cli_entrypoint_dispatch

contracts/apr-cli-operations-v1.yaml

@@ -265,6 +265,51 @@ falsification_tests:
   prediction: The timestamp in a cbtop report matches the UTC date at the moment it was generated, on both headless paths
   test: commands::cbtop::tests::test_chrono_timestamp_is_current_utc_date
   if_fails: Persisted CI provenance is stamped with a stale hardcoded date
+- id: FALSIFY-OPS-015
+  rule: benchmark summaries use the sample standard deviation and the t interval
+  prediction: stat_summary of [2,4,6] has mean 4, stddev 2 and a 95% margin of t(3)*2/sqrt(3); one sample collapses the interval; an empty slice yields zeros
+  test: llm::benchmark::tests::stat_summary_computes_mean_stddev_and_t_interval
+  if_fails: benchmark confidence intervals are wrong, so a speed claim rests on a wrong error bar
+- id: FALSIFY-OPS-016
+  rule: pv lint layers CLI flags over the config file
+  prediction: build_config takes scalars from the flag when set else the file, merges suppression lists, and lets a CLI rule override beat the file; a missing config yields defaults
+  test: commands::lint::tests::build_config_merges_cli_flags_over_the_config_file
+  if_fails: pv lint silently ignores a flag or a config setting
+- id: FALSIFY-OPS-021
+  rule: Mock page coverage cannot be read before it is started, and stopping ends collection
+  prediction: take_coverage before start_coverage is an InvalidState error; after start it returns the page url and added functions; stop_coverage returns the final report and later takes fail
+  test: mock_page_take_coverage_5695::stop_coverage_returns_final_report_then_disables
+  if_fails: Coverage reads succeed on a page that never enabled collection, or keep succeeding after stop
+- id: FALSIFY-OPS-022
+  rule: Resuming a classifier checkpoint restores what was saved and refuses a foreign dataset
+  prediction: resume_from_apr_checkpoint returns the saved epoch and the saved classifier weights, and a checkpoint from a different corpus is rejected with F-CKPT-006 before any weight is touched
+  test: finetune::classify_trainer::tests::test_resume_rejects_data_hash_mismatch
+  if_fails: A resume silently continues from weights trained on different data, or loses the epoch
+- id: FALSIFY-OPS-017
+  rule: batch inference never aborts on a failing prompt
+  prediction: run_batch_loop emits one JSONL error row and counts one failure per bad-JSON, encode-failed or model-less prompt, skips blank lines, and returns an IoError when the sink is unwritable
+  test: infer::tests_batch_loop_5691::bad_json_encode_failure_and_missing_model_each_emit_one_error_row_5691
+  if_fails: a bad prompt aborts or silently vanishes from a batch run
+- id: FALSIFY-OPS-019
+  rule: oracle recommender emits a distinct code example per stack component
+  prediction: generate_code_example returns Some for every known component with a distinct non-empty body, honours a recommended aprender path, and returns None for an unknown component
+  test: oracle::recommender::tests::code_example_exists_for_every_stack_component_and_is_distinct
+  if_fails: the oracle shows the wrong or no example for a recommended component
+- id: FALSIFY-OPS-020
+  rule: extract-frames fails loudly and dry-runs without side effects
+  prediction: run_extract_frames rejects a missing path and a video-less or fully excluded directory with distinct errors, rejects a malformed exclude glob, lists videos on dry-run without creating frame dirs, and leaves a populated frames dir untouched under skip-existing
+  test: extract::tests::skip_existing_treats_a_populated_frames_dir_as_done_but_an_empty_one_as_pending
+  if_fails: extract-frames re-extracts finished videos or hides why nothing was processed
+- id: FALSIFY-OPS-023
+  rule: /v1/predict computes from the loaded tensor and refuses what it cannot compute
+  prediction: apr_predict_handler rejects empty features with 400, reports a missing weights/output tensor as 400, returns the dot product for matching-length weights, and argmax class plus ranked top-k for logits
+  test: api::tests::apr_predict_5698::predict_classification_takes_argmax_and_ranks_top_k
+  if_fails: The predict endpoint returns a plausible number or class that is not derived from the loaded model
+- id: FALSIFY-OPS-024
+  rule: The MoE FFN layer refuses malformed router input and routes k=1 to exactly one expert
+  prediction: moe_ffn_forward_layer rejects a wrong hidden length, a quantized router and a short router byte_size, and with top-k 1 returns the favoured expert's SwiGLU output unchanged
+  test: gguf::qwen3_moe_load::moe_ffn_layer_5699_tests::moe_ffn_layer_5699_k1_output_is_the_routed_experts_swiglu
+  if_fails: A mis-sized router or hidden vector reads past its tensor, or routing mixes experts that top-k 1 should exclude
 kani_harnesses:
 - id: KANI-OPS-001
   obligation: ReadOnly commands have no side effects

crates/aprender-contracts-cli/src/commands/lint.rs

@@ -1244,4 +1244,96 @@ mod tests {
             assert!(!parses(refused), "{refused:?} must be refused");
         }
     }
+
+    /// FALSIFY-OPS-016: `build_config` layers CLI flags over the config file —
+    /// flags win for scalars, lists are merged, and a CLI rule override beats the file's.
+    #[test]
+    fn build_config_merges_cli_flags_over_the_config_file() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let cfg = dir.path().join("pv.toml");
+        std::fs::write(
+            &cfg,
+            "[lint]\nmin_score = 0.5\nseverity = \"warn\"\nstrict = true\n\
+             [lint.rules]\nPV-A = \"error\"\nPV-B = \"info\"\n\
+             [lint.suppress]\nfindings = [\"f1\"]\nrules = [\"r1\"]\nfiles = [\"x.yaml\"]\n",
+        )
+        .expect("write config");
+        let overrides = vec!["PV-A=off".to_string(), "bad-no-equals".to_string()];
+        let lc = build_config(
+            dir.path(),
+            None,
+            0.0,
+            None,
+            None,
+            false,
+            Some("f2, ,f3"),
+            Some("r2"),
+            None,
+            &overrides,
+            Some(&cfg),
+            true,
+            true,
+            None,
+            Some("STRICT"),
+            true,
+        );
+        // file supplies what the flags leave unset
+        assert!((lc.min_score - 0.5).abs() < f64::EPSILON && lc.strict);
+        assert_eq!(lc.severity_filter, RuleSeverity::from_str_opt("warn"));
+        // lists: CLI entries first (blanks dropped), then the file's
+        assert_eq!(lc.suppressed_findings, ["f2", "f3", "f1"]);
+        assert_eq!(lc.suppressed_rules, ["r2", "r1"]);
+        assert_eq!(lc.suppressed_files, ["x.yaml"]);
+        // CLI override beats the file for PV-A; the file's PV-B survives; malformed ignored
+        assert_eq!(lc.severity_overrides["PV-A"], RuleSeverity::Off);
+        assert_eq!(lc.severity_overrides["PV-B"], RuleSeverity::Info);
+        assert_eq!(lc.severity_overrides.len(), 2);
+        assert!(lc.no_cache && lc.cache_stats && lc.strict_test_binding);
+        assert!(!lc.strict_test_binding_only);
+        assert!(lc.min_level.is_some());
+
+        // a CLI score and severity beat the file; a missing config falls back to defaults
+        let lc = build_config(
+            dir.path(),
+            None,
+            0.9,
+            None,
+            Some("error"),
+            false,
+            None,
+            None,
+            None,
+            &[],
+            Some(&cfg),
+            false,
+            false,
+            None,
+            Some("nonsense"),
+            false,
+        );
+        assert!((lc.min_score - 0.9).abs() < f64::EPSILON);
+        assert_eq!(lc.severity_filter, RuleSeverity::from_str_opt("error"));
+        assert!(lc.min_level.is_none());
+        let bare = tempfile::tempdir().expect("tempdir");
+        let lc = build_config(
+            bare.path(),
+            None,
+            0.0,
+            None,
+            None,
+            false,
+            None,
+            None,
+            None,
+            &[],
+            Some(&bare.path().join("missing.toml")),
+            false,
+            false,
+            None,
+            None,
+            false,
+        );
+        assert!(lc.min_score.abs() < f64::EPSILON && !lc.strict);
+        assert!(lc.severity_overrides.is_empty() && lc.severity_filter.is_none());
+    }
 }

crates/aprender-orchestrate/src/oracle/recommender_tests.rs

@@ -521,3 +521,55 @@ fn test_code_example_none_for_unknown() {
         "fallback component should not produce a code example"
     );
 }
+
+// =========================================================================
+// generate_code_example (#5693)
+// =========================================================================
+
+fn example_for(component: &str, path: Option<&str>) -> Option<String> {
+    let primary = match path {
+        Some(p) => ComponentRecommendation::with_path(component, 0.9, "r", p.to_string()),
+        None => ComponentRecommendation::new(component, 0.9, "r"),
+    };
+    let parsed = QueryEngine::new().parse("train a random forest");
+    Recommender::new().generate_code_example(&primary, &[], &parsed)
+}
+
+#[test]
+fn code_example_names_the_recommended_aprender_path_or_the_default() {
+    let custom = example_for("aprender", Some("aprender::linear::LinearRegression"))
+        .expect("aprender always has an example");
+    assert!(custom.contains("use aprender::linear::LinearRegression;"));
+    assert!(custom.contains("LinearRegression::new()"));
+    let default = example_for("aprender", None).expect("default path");
+    assert!(default.contains("use aprender::tree::RandomForestClassifier;"));
+    assert!(default.contains("RandomForestClassifier::new()"));
+}
+
+#[test]
+fn code_example_exists_for_every_stack_component_and_is_distinct() {
+    let components = [
+        "trueno",
+        "depyler",
+        "realizar",
+        "whisper-apr",
+        "provable-contracts",
+        "tiny-model-ground-truth",
+        "rmedia",
+        "repartir",
+    ];
+    let mut seen = std::collections::HashSet::new();
+    for c in components {
+        let code = example_for(c, None).unwrap_or_else(|| panic!("no example for {c}"));
+        assert!(!code.trim().is_empty(), "{c} example is empty");
+        assert!(seen.insert(code), "{c} shares an example with another component");
+    }
+    assert!(example_for("trueno", None).expect("trueno").contains("trueno::prelude"));
+    assert!(example_for("rmedia", None).expect("rmedia").contains("rmedia::prelude"));
+}
+
+#[test]
+fn code_example_is_none_for_an_unknown_component() {
+    assert!(example_for("not-a-component", None).is_none());
+    assert!(example_for("", None).is_none());
+}

crates/aprender-orchestrate/tests/repl_run.rs

@@ -0,0 +1,95 @@
+//! `run_repl` loop behaviour, driven through real stdin.
+//!
+//! `run_repl` reads `io::stdin()` directly, so each case re-executes this test
+//! binary as a child with piped stdin (`HOME` pointed at a temp dir so session
+//! files never touch the real one) and asserts on the child's stdout.
+//!
+//! Refs: #5706
+
+#![cfg(feature = "agents")]
+
+use std::io::Write;
+use std::process::{Command, Stdio};
+
+use batuta::agent::driver::mock::MockDriver;
+use batuta::agent::manifest::AgentManifest;
+use batuta::agent::memory::InMemorySubstrate;
+use batuta::agent::repl::run_repl;
+use batuta::agent::tool::ToolRegistry;
+
+const CHILD_ENV: &str = "APR_REPL_RUN_CHILD";
+
+/// Child entry point: a no-op unless re-executed by `run_child`.
+#[test]
+fn child_entry() {
+    let Ok(spec) = std::env::var(CHILD_ENV) else { return };
+    let mut parts = spec.split(',');
+    let max_turns: u32 = parts.next().unwrap().parse().unwrap();
+    let budget: f64 = parts.next().unwrap().parse().unwrap();
+    let driver = MockDriver::single_response("hello from mock");
+    let mut manifest = AgentManifest::default();
+    manifest.name = "repl-run-test".to_string();
+    run_repl(
+        &manifest,
+        &driver,
+        &ToolRegistry::new(),
+        &InMemorySubstrate::new(),
+        max_turns,
+        budget,
+        None,
+    )
+    .expect("run_repl returns Ok");
+}
+
+/// Run `run_repl` in a child with `stdin` as input; returns its stdout.
+fn run_child(max_turns: u32, budget: f64, stdin: &str) -> String {
+    let home = tempfile::tempdir().expect("tempdir");
+    let mut child = Command::new(std::env::current_exe().unwrap())
+        .args(["child_entry", "--exact", "--nocapture", "--test-threads=1"])
+        .env(CHILD_ENV, format!("{max_turns},{budget}"))
+        .env("HOME", home.path())
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::null())
+        .spawn()
+        .expect("spawn child");
+    child.stdin.take().unwrap().write_all(stdin.as_bytes()).unwrap();
+    let out = child.wait_with_output().expect("child output");
+    assert!(out.status.success(), "child failed: {}", String::from_utf8_lossy(&out.stdout));
+    String::from_utf8_lossy(&out.stdout).into_owned()
+}
+
+#[test]
+fn eof_ends_the_loop_with_no_summary_when_no_turn_ran() {
+    let out = run_child(10, 100.0, "");
+    assert!(!out.contains("Session Summary"), "{out}");
+    assert!(!out.contains("tok]"), "{out}");
+}
+
+#[test]
+fn a_prompt_runs_one_turn_then_quit_prints_the_summary() {
+    let out = run_child(10, 100.0, "hi\n/quit\n");
+    assert!(out.contains("10/5 tok"), "{out}");
+    assert!(out.contains("Session Summary"), "{out}");
+    assert!(out.contains("Turns: 1,"), "{out}");
+}
+
+#[test]
+fn blank_lines_do_not_consume_a_turn() {
+    let out = run_child(10, 100.0, "\n   \n/quit\n");
+    assert!(!out.contains("Session Summary"), "{out}");
+}
+
+#[test]
+fn max_turns_stops_the_loop_before_reading_more_input() {
+    let out = run_child(1, 100.0, "one\ntwo\n");
+    assert!(out.contains("Max turns (1) reached"), "{out}");
+    assert!(out.contains("Turns: 1,"), "{out}");
+}
+
+#[test]
+fn a_zero_budget_stops_before_the_first_turn() {
+    let out = run_child(10, 0.0, "hi\n");
+    assert!(out.contains("Budget ($0.00) exhausted"), "{out}");
+    assert!(!out.contains("tok]"), "{out}");
+}