diff --git a/ci/explicit-test-commands.d/805-aprender-orchestrate-repl-run.cmd b/ci/explicit-test-commands.d/805-aprender-orchestrate-repl-run.cmd new file mode 100644 index 0000000000..2950c818dd --- /dev/null +++ b/ci/explicit-test-commands.d/805-aprender-orchestrate-repl-run.cmd @@ -0,0 +1,3 @@ + +# #5706: run_repl loop tests (stdin EOF, quit, blank lines, max-turns, zero budget); dark in CI until wired here. +cargo test -p aprender-orchestrate --test repl_run diff --git a/ci/explicit-test-commands.d/806-aprender-serve-cli-entrypoint-dispatch.cmd b/ci/explicit-test-commands.d/806-aprender-serve-cli-entrypoint-dispatch.cmd new file mode 100644 index 0000000000..a824d87743 --- /dev/null +++ b/ci/explicit-test-commands.d/806-aprender-serve-cli-entrypoint-dispatch.cmd @@ -0,0 +1,3 @@ + +# #5707: CLI entrypoint dispatch tests for aprender-serve; dark in CI until wired here. +cargo test -p aprender-serve --test cli_entrypoint_dispatch diff --git a/contracts/apr-cli-operations-v1.yaml b/contracts/apr-cli-operations-v1.yaml index 96ec9fb715..5152982b25 100644 --- a/contracts/apr-cli-operations-v1.yaml +++ b/contracts/apr-cli-operations-v1.yaml @@ -265,6 +265,51 @@ falsification_tests: prediction: The timestamp in a cbtop report matches the UTC date at the moment it was generated, on both headless paths test: commands::cbtop::tests::test_chrono_timestamp_is_current_utc_date if_fails: Persisted CI provenance is stamped with a stale hardcoded date +- id: FALSIFY-OPS-015 + rule: benchmark summaries use the sample standard deviation and the t interval + prediction: stat_summary of [2,4,6] has mean 4, stddev 2 and a 95% margin of t(3)*2/sqrt(3); one sample collapses the interval; an empty slice yields zeros + test: llm::benchmark::tests::stat_summary_computes_mean_stddev_and_t_interval + if_fails: benchmark confidence intervals are wrong, so a speed claim rests on a wrong error bar +- id: FALSIFY-OPS-016 + rule: pv lint layers CLI flags over the config file + prediction: build_config takes scalars from the flag when set else the file, merges suppression lists, and lets a CLI rule override beat the file; a missing config yields defaults + test: commands::lint::tests::build_config_merges_cli_flags_over_the_config_file + if_fails: pv lint silently ignores a flag or a config setting +- id: FALSIFY-OPS-021 + rule: Mock page coverage cannot be read before it is started, and stopping ends collection + prediction: take_coverage before start_coverage is an InvalidState error; after start it returns the page url and added functions; stop_coverage returns the final report and later takes fail + test: mock_page_take_coverage_5695::stop_coverage_returns_final_report_then_disables + if_fails: Coverage reads succeed on a page that never enabled collection, or keep succeeding after stop +- id: FALSIFY-OPS-022 + rule: Resuming a classifier checkpoint restores what was saved and refuses a foreign dataset + prediction: resume_from_apr_checkpoint returns the saved epoch and the saved classifier weights, and a checkpoint from a different corpus is rejected with F-CKPT-006 before any weight is touched + test: finetune::classify_trainer::tests::test_resume_rejects_data_hash_mismatch + if_fails: A resume silently continues from weights trained on different data, or loses the epoch +- id: FALSIFY-OPS-017 + rule: batch inference never aborts on a failing prompt + prediction: run_batch_loop emits one JSONL error row and counts one failure per bad-JSON, encode-failed or model-less prompt, skips blank lines, and returns an IoError when the sink is unwritable + test: infer::tests_batch_loop_5691::bad_json_encode_failure_and_missing_model_each_emit_one_error_row_5691 + if_fails: a bad prompt aborts or silently vanishes from a batch run +- id: FALSIFY-OPS-019 + rule: oracle recommender emits a distinct code example per stack component + prediction: generate_code_example returns Some for every known component with a distinct non-empty body, honours a recommended aprender path, and returns None for an unknown component + test: oracle::recommender::tests::code_example_exists_for_every_stack_component_and_is_distinct + if_fails: the oracle shows the wrong or no example for a recommended component +- id: FALSIFY-OPS-020 + rule: extract-frames fails loudly and dry-runs without side effects + prediction: run_extract_frames rejects a missing path and a video-less or fully excluded directory with distinct errors, rejects a malformed exclude glob, lists videos on dry-run without creating frame dirs, and leaves a populated frames dir untouched under skip-existing + test: extract::tests::skip_existing_treats_a_populated_frames_dir_as_done_but_an_empty_one_as_pending + if_fails: extract-frames re-extracts finished videos or hides why nothing was processed +- id: FALSIFY-OPS-023 + rule: /v1/predict computes from the loaded tensor and refuses what it cannot compute + prediction: apr_predict_handler rejects empty features with 400, reports a missing weights/output tensor as 400, returns the dot product for matching-length weights, and argmax class plus ranked top-k for logits + test: api::tests::apr_predict_5698::predict_classification_takes_argmax_and_ranks_top_k + if_fails: The predict endpoint returns a plausible number or class that is not derived from the loaded model +- id: FALSIFY-OPS-024 + rule: The MoE FFN layer refuses malformed router input and routes k=1 to exactly one expert + prediction: moe_ffn_forward_layer rejects a wrong hidden length, a quantized router and a short router byte_size, and with top-k 1 returns the favoured expert's SwiGLU output unchanged + test: gguf::qwen3_moe_load::moe_ffn_layer_5699_tests::moe_ffn_layer_5699_k1_output_is_the_routed_experts_swiglu + if_fails: A mis-sized router or hidden vector reads past its tensor, or routing mixes experts that top-k 1 should exclude kani_harnesses: - id: KANI-OPS-001 obligation: ReadOnly commands have no side effects diff --git a/crates/aprender-contracts-cli/src/commands/lint.rs b/crates/aprender-contracts-cli/src/commands/lint.rs index b505e098a9..2a59a1c58b 100644 --- a/crates/aprender-contracts-cli/src/commands/lint.rs +++ b/crates/aprender-contracts-cli/src/commands/lint.rs @@ -1244,4 +1244,96 @@ mod tests { assert!(!parses(refused), "{refused:?} must be refused"); } } + + /// FALSIFY-OPS-016: `build_config` layers CLI flags over the config file — + /// flags win for scalars, lists are merged, and a CLI rule override beats the file's. + #[test] + fn build_config_merges_cli_flags_over_the_config_file() { + let dir = tempfile::tempdir().expect("tempdir"); + let cfg = dir.path().join("pv.toml"); + std::fs::write( + &cfg, + "[lint]\nmin_score = 0.5\nseverity = \"warn\"\nstrict = true\n\ + [lint.rules]\nPV-A = \"error\"\nPV-B = \"info\"\n\ + [lint.suppress]\nfindings = [\"f1\"]\nrules = [\"r1\"]\nfiles = [\"x.yaml\"]\n", + ) + .expect("write config"); + let overrides = vec!["PV-A=off".to_string(), "bad-no-equals".to_string()]; + let lc = build_config( + dir.path(), + None, + 0.0, + None, + None, + false, + Some("f2, ,f3"), + Some("r2"), + None, + &overrides, + Some(&cfg), + true, + true, + None, + Some("STRICT"), + true, + ); + // file supplies what the flags leave unset + assert!((lc.min_score - 0.5).abs() < f64::EPSILON && lc.strict); + assert_eq!(lc.severity_filter, RuleSeverity::from_str_opt("warn")); + // lists: CLI entries first (blanks dropped), then the file's + assert_eq!(lc.suppressed_findings, ["f2", "f3", "f1"]); + assert_eq!(lc.suppressed_rules, ["r2", "r1"]); + assert_eq!(lc.suppressed_files, ["x.yaml"]); + // CLI override beats the file for PV-A; the file's PV-B survives; malformed ignored + assert_eq!(lc.severity_overrides["PV-A"], RuleSeverity::Off); + assert_eq!(lc.severity_overrides["PV-B"], RuleSeverity::Info); + assert_eq!(lc.severity_overrides.len(), 2); + assert!(lc.no_cache && lc.cache_stats && lc.strict_test_binding); + assert!(!lc.strict_test_binding_only); + assert!(lc.min_level.is_some()); + + // a CLI score and severity beat the file; a missing config falls back to defaults + let lc = build_config( + dir.path(), + None, + 0.9, + None, + Some("error"), + false, + None, + None, + None, + &[], + Some(&cfg), + false, + false, + None, + Some("nonsense"), + false, + ); + assert!((lc.min_score - 0.9).abs() < f64::EPSILON); + assert_eq!(lc.severity_filter, RuleSeverity::from_str_opt("error")); + assert!(lc.min_level.is_none()); + let bare = tempfile::tempdir().expect("tempdir"); + let lc = build_config( + bare.path(), + None, + 0.0, + None, + None, + false, + None, + None, + None, + &[], + Some(&bare.path().join("missing.toml")), + false, + false, + None, + None, + false, + ); + assert!(lc.min_score.abs() < f64::EPSILON && !lc.strict); + assert!(lc.severity_overrides.is_empty() && lc.severity_filter.is_none()); + } } diff --git a/crates/aprender-orchestrate/src/oracle/recommender_tests.rs b/crates/aprender-orchestrate/src/oracle/recommender_tests.rs index c15e96323e..ebd72d18b9 100644 --- a/crates/aprender-orchestrate/src/oracle/recommender_tests.rs +++ b/crates/aprender-orchestrate/src/oracle/recommender_tests.rs @@ -521,3 +521,55 @@ fn test_code_example_none_for_unknown() { "fallback component should not produce a code example" ); } + +// ========================================================================= +// generate_code_example (#5693) +// ========================================================================= + +fn example_for(component: &str, path: Option<&str>) -> Option { + let primary = match path { + Some(p) => ComponentRecommendation::with_path(component, 0.9, "r", p.to_string()), + None => ComponentRecommendation::new(component, 0.9, "r"), + }; + let parsed = QueryEngine::new().parse("train a random forest"); + Recommender::new().generate_code_example(&primary, &[], &parsed) +} + +#[test] +fn code_example_names_the_recommended_aprender_path_or_the_default() { + let custom = example_for("aprender", Some("aprender::linear::LinearRegression")) + .expect("aprender always has an example"); + assert!(custom.contains("use aprender::linear::LinearRegression;")); + assert!(custom.contains("LinearRegression::new()")); + let default = example_for("aprender", None).expect("default path"); + assert!(default.contains("use aprender::tree::RandomForestClassifier;")); + assert!(default.contains("RandomForestClassifier::new()")); +} + +#[test] +fn code_example_exists_for_every_stack_component_and_is_distinct() { + let components = [ + "trueno", + "depyler", + "realizar", + "whisper-apr", + "provable-contracts", + "tiny-model-ground-truth", + "rmedia", + "repartir", + ]; + let mut seen = std::collections::HashSet::new(); + for c in components { + let code = example_for(c, None).unwrap_or_else(|| panic!("no example for {c}")); + assert!(!code.trim().is_empty(), "{c} example is empty"); + assert!(seen.insert(code), "{c} shares an example with another component"); + } + assert!(example_for("trueno", None).expect("trueno").contains("trueno::prelude")); + assert!(example_for("rmedia", None).expect("rmedia").contains("rmedia::prelude")); +} + +#[test] +fn code_example_is_none_for_an_unknown_component() { + assert!(example_for("not-a-component", None).is_none()); + assert!(example_for("", None).is_none()); +} diff --git a/crates/aprender-orchestrate/tests/repl_run.rs b/crates/aprender-orchestrate/tests/repl_run.rs new file mode 100644 index 0000000000..6903c92972 --- /dev/null +++ b/crates/aprender-orchestrate/tests/repl_run.rs @@ -0,0 +1,95 @@ +//! `run_repl` loop behaviour, driven through real stdin. +//! +//! `run_repl` reads `io::stdin()` directly, so each case re-executes this test +//! binary as a child with piped stdin (`HOME` pointed at a temp dir so session +//! files never touch the real one) and asserts on the child's stdout. +//! +//! Refs: #5706 + +#![cfg(feature = "agents")] + +use std::io::Write; +use std::process::{Command, Stdio}; + +use batuta::agent::driver::mock::MockDriver; +use batuta::agent::manifest::AgentManifest; +use batuta::agent::memory::InMemorySubstrate; +use batuta::agent::repl::run_repl; +use batuta::agent::tool::ToolRegistry; + +const CHILD_ENV: &str = "APR_REPL_RUN_CHILD"; + +/// Child entry point: a no-op unless re-executed by `run_child`. +#[test] +fn child_entry() { + let Ok(spec) = std::env::var(CHILD_ENV) else { return }; + let mut parts = spec.split(','); + let max_turns: u32 = parts.next().unwrap().parse().unwrap(); + let budget: f64 = parts.next().unwrap().parse().unwrap(); + let driver = MockDriver::single_response("hello from mock"); + let mut manifest = AgentManifest::default(); + manifest.name = "repl-run-test".to_string(); + run_repl( + &manifest, + &driver, + &ToolRegistry::new(), + &InMemorySubstrate::new(), + max_turns, + budget, + None, + ) + .expect("run_repl returns Ok"); +} + +/// Run `run_repl` in a child with `stdin` as input; returns its stdout. +fn run_child(max_turns: u32, budget: f64, stdin: &str) -> String { + let home = tempfile::tempdir().expect("tempdir"); + let mut child = Command::new(std::env::current_exe().unwrap()) + .args(["child_entry", "--exact", "--nocapture", "--test-threads=1"]) + .env(CHILD_ENV, format!("{max_turns},{budget}")) + .env("HOME", home.path()) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn child"); + child.stdin.take().unwrap().write_all(stdin.as_bytes()).unwrap(); + let out = child.wait_with_output().expect("child output"); + assert!(out.status.success(), "child failed: {}", String::from_utf8_lossy(&out.stdout)); + String::from_utf8_lossy(&out.stdout).into_owned() +} + +#[test] +fn eof_ends_the_loop_with_no_summary_when_no_turn_ran() { + let out = run_child(10, 100.0, ""); + assert!(!out.contains("Session Summary"), "{out}"); + assert!(!out.contains("tok]"), "{out}"); +} + +#[test] +fn a_prompt_runs_one_turn_then_quit_prints_the_summary() { + let out = run_child(10, 100.0, "hi\n/quit\n"); + assert!(out.contains("10/5 tok"), "{out}"); + assert!(out.contains("Session Summary"), "{out}"); + assert!(out.contains("Turns: 1,"), "{out}"); +} + +#[test] +fn blank_lines_do_not_consume_a_turn() { + let out = run_child(10, 100.0, "\n \n/quit\n"); + assert!(!out.contains("Session Summary"), "{out}"); +} + +#[test] +fn max_turns_stops_the_loop_before_reading_more_input() { + let out = run_child(1, 100.0, "one\ntwo\n"); + assert!(out.contains("Max turns (1) reached"), "{out}"); + assert!(out.contains("Turns: 1,"), "{out}"); +} + +#[test] +fn a_zero_budget_stops_before_the_first_turn() { + let out = run_child(10, 0.0, "hi\n"); + assert!(out.contains("Budget ($0.00) exhausted"), "{out}"); + assert!(!out.contains("tok]"), "{out}"); +} diff --git a/crates/aprender-rag-cli/src/extract.rs b/crates/aprender-rag-cli/src/extract.rs index a8e05abd14..fda9fcffa2 100644 --- a/crates/aprender-rag-cli/src/extract.rs +++ b/crates/aprender-rag-cli/src/extract.rs @@ -173,3 +173,110 @@ fn extract_frames_ffmpeg(video: &Path, threshold: f64, min_interval: f64) -> Res Ok(frame_count) } + +#[cfg(test)] +mod tests { + use super::*; + + fn ffmpeg_present() -> bool { + std::process::Command::new("ffmpeg") + .arg("-version") + .output() + .is_ok_and(|o| o.status.success()) + } + + /// Run with the common arguments; returns None (after asserting the ffmpeg + /// error) when the host has no ffmpeg, so the test stays honest on any runner. + fn run( + path: &Path, + skip_existing: bool, + dry_run: bool, + exclude: &[String], + ) -> Option> { + let r = run_extract_frames( + &path.to_string_lossy(), + true, + 0.3, + 1.0, + 1, + skip_existing, + dry_run, + exclude, + ); + if ffmpeg_present() { + return Some(r); + } + let err = r.expect_err("no ffmpeg must fail first"); + assert!(err.to_string().contains("ffmpeg not found"), "{err}"); + None + } + + #[test] + fn missing_path_and_videoless_dir_fail_with_distinct_errors() { + let dir = tempfile::tempdir().expect("tempdir"); + if let Some(r) = run(&dir.path().join("nope"), false, true, &[]) { + assert!(r + .expect_err("missing") + .to_string() + .contains("Path not found")); + } + if let Some(r) = run(dir.path(), false, true, &[]) { + assert!(r + .expect_err("empty") + .to_string() + .contains("No video files found")); + } + } + + #[test] + fn dry_run_lists_videos_without_creating_frame_dirs() { + let dir = tempfile::tempdir().expect("tempdir"); + let video = dir.path().join("lecture.mp4"); + fs::write(&video, b"not a real video").expect("write"); + if let Some(r) = run(dir.path(), false, true, &[]) { + r.expect("dry run succeeds"); + assert!( + !video.with_extension("frames").exists(), + "dry run wrote frames" + ); + } + } + + #[test] + fn skip_existing_treats_a_populated_frames_dir_as_done_but_an_empty_one_as_pending() { + let dir = tempfile::tempdir().expect("tempdir"); + let video = dir.path().join("lecture.mp4"); + fs::write(&video, b"x").expect("write"); + let frames = video.with_extension("frames"); + fs::create_dir(&frames).expect("mkdir"); + // empty frames dir: pending, so a dry run still succeeds (and lists it) + if let Some(r) = run(dir.path(), true, true, &[]) { + r.expect("pending video, dry run"); + } + fs::write(frames.join("0001.jpg"), b"j").expect("frame"); + // populated: nothing to do, returns Ok before any ffmpeg work even without dry-run + if let Some(r) = run(dir.path(), true, false, &[]) { + r.expect("all already extracted"); + assert_eq!( + fs::read_dir(&frames).expect("ls").count(), + 1, + "frames were touched" + ); + } + } + + #[test] + fn exclude_patterns_can_remove_every_video() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("skip.mp4"), b"x").expect("write"); + if let Some(r) = run(dir.path(), false, true, &["*.mp4".to_string()]) { + assert!(r + .expect_err("all excluded") + .to_string() + .contains("No video files found")); + } + if let Some(r) = run(dir.path(), false, true, &["[".to_string()]) { + assert!(r.is_err(), "a malformed glob must be rejected"); + } + } +} diff --git a/crates/aprender-serve/src/api/tests/apr_predict_5698.rs b/crates/aprender-serve/src/api/tests/apr_predict_5698.rs new file mode 100644 index 0000000000..ae7a5813ec --- /dev/null +++ b/crates/aprender-serve/src/api/tests/apr_predict_5698.rs @@ -0,0 +1,128 @@ +//! #5698: `apr_predict_handler` branch coverage, called directly (no router). + +use axum::{extract::State, http::StatusCode, Json}; +use std::sync::Arc; + +use super::super::{apr_predict_handler, AppState, PredictRequest}; +use crate::apr::{AprModel, HEADER_SIZE, MAGIC}; + +/// Minimal APR v2 file holding one F32 tensor called `name`. +fn apr_with_tensor(name: &str, values: &[f32]) -> AprModel { + let metadata = r#"{"model_type":"test","name":"predict-5698"}"#; + // binary tensor index: name_len u16, name, dtype u8 (0 = F32), ndim u8, dims u64, offset u64, size u64 + let mut index = Vec::new(); + index.extend_from_slice(&(name.len() as u16).to_le_bytes()); + index.extend_from_slice(name.as_bytes()); + index.push(0); + index.push(1); + index.extend_from_slice(&(values.len() as u64).to_le_bytes()); + index.extend_from_slice(&0u64.to_le_bytes()); + index.extend_from_slice(&((values.len() * 4) as u64).to_le_bytes()); + let bytes: Vec = values.iter().flat_map(|f| f.to_le_bytes()).collect(); + let meta_off = HEADER_SIZE as u64; + let index_off = ((HEADER_SIZE + metadata.len()).div_ceil(64) * 64) as u64; + let data_off = ((index_off as usize + index.len()).div_ceil(64) * 64) as u64; + let mut data = vec![0u8; data_off as usize + bytes.len()]; + data[0..4].copy_from_slice(&MAGIC); + data[4] = 2; + data[8..12].copy_from_slice(&1u32.to_le_bytes()); + data[12..20].copy_from_slice(&meta_off.to_le_bytes()); + data[20..24].copy_from_slice(&(metadata.len() as u32).to_le_bytes()); + data[24..32].copy_from_slice(&index_off.to_le_bytes()); + data[32..40].copy_from_slice(&data_off.to_le_bytes()); + data[HEADER_SIZE..HEADER_SIZE + metadata.len()].copy_from_slice(metadata.as_bytes()); + data[index_off as usize..index_off as usize + index.len()].copy_from_slice(&index); + data[data_off as usize..].copy_from_slice(&bytes); + AprModel::from_bytes(data).expect("test APR parses") +} + +fn state_with(name: &str, values: &[f32]) -> AppState { + let mut state = AppState::demo().expect("demo state"); + state.apr_model = Some(Arc::new(apr_with_tensor(name, values))); + state +} + +fn request(features: Vec, top_k: Option, confidence: bool) -> PredictRequest { + PredictRequest { + model: None, + features, + feature_names: None, + top_k, + include_confidence: confidence, + } +} + +#[tokio::test] +async fn predict_rejects_empty_features() { + let state = state_with("weights", &[1.0, 2.0]); + let err = apr_predict_handler(State(state), Json(request(vec![], None, false))) + .await + .expect_err("empty features"); + assert_eq!(err.0, StatusCode::BAD_REQUEST); + assert_eq!(err.1 .0.error, "Input features cannot be empty"); +} + +#[tokio::test] +async fn predict_without_a_tensor_named_weights_or_output_is_400() { + // the demo tensor is called "weight" (singular): neither lookup name matches + let state = state_with("weight", &[1.0, 2.0]); + let err = apr_predict_handler(State(state), Json(request(vec![1.0, 2.0], None, false))) + .await + .expect_err("no weights/output tensor"); + assert_eq!(err.0, StatusCode::BAD_REQUEST); + assert!( + err.1 .0.error.contains("Inference failed"), + "{}", + err.1 .0.error + ); +} + +#[tokio::test] +async fn predict_regression_is_the_dot_product_with_confidence_one() { + let state = state_with("weights", &[2.0, 3.0]); + let resp = match apr_predict_handler( + State(state), + Json(request(vec![1.0, 10.0], Some(3), true)), + ) + .await + { + Ok(Json(r)) => r, + Err((s, e)) => panic!("regression predict failed: {s} {}", e.0.error), + }; + assert_eq!(resp.prediction, serde_json::json!(32.0)); + assert_eq!(resp.confidence, Some(1.0)); + assert_eq!(resp.model, "default"); + let top = resp.top_k_predictions.expect("top_k requested"); + assert_eq!(top.len(), 1); + assert_eq!(top[0].label, "32.0000"); + assert_eq!(top[0].score, 1.0); +} + +#[tokio::test] +async fn predict_classification_takes_argmax_and_ranks_top_k() { + // 3 weights vs 2 features: length mismatch, so the weights are the logits + let state = state_with("output", &[0.1, 2.0, 0.5]); + let Ok(Json(resp)) = + apr_predict_handler(State(state), Json(request(vec![1.0, 1.0], Some(2), false))).await + else { + panic!("classification predict failed"); + }; + assert_eq!(resp.prediction, serde_json::json!("class_1")); + assert_eq!(resp.confidence, None, "include_confidence=false hides it"); + let top = resp.top_k_predictions.expect("top_k requested"); + assert_eq!( + top.iter().map(|p| p.label.as_str()).collect::>(), + ["class_1", "class_2"] + ); + assert!(top[0].score > top[1].score && top[0].score < 1.0); +} + +#[tokio::test] +async fn predict_with_no_apr_model_loaded_is_an_error() { + let mut state = AppState::demo().expect("demo state"); + state.apr_model = None; + let err = apr_predict_handler(State(state), Json(request(vec![1.0], None, false))) + .await + .expect_err("no model"); + assert!(err.0.is_client_error() || err.0.is_server_error()); +} diff --git a/crates/aprender-serve/src/api/tests/mod.rs b/crates/aprender-serve/src/api/tests/mod.rs index 842f97dcee..9da5403a65 100644 --- a/crates/aprender-serve/src/api/tests/mod.rs +++ b/crates/aprender-serve/src/api/tests/mod.rs @@ -83,3 +83,4 @@ mod logprobs_4971; // aprender#4971: chat logprobs/top_logprobs honoured on the #[cfg(feature = "cuda")] mod logprobs_cuda_4971; // aprender#4971: chat logprobs on the dense CUDA backend, direct, through the batch scheduler and on the iteration scheduler (needs a device + tinyllama; SKIP = not_measured) mod embed_kind_eg3_4902; // APR-EMBED-001 EG-3 (#4902): embedding routes serve the CLI vector, generative routes name the kind, media refused (I-7) +mod apr_predict_5698; // #5698: apr_predict_handler branches, called directly diff --git a/crates/aprender-serve/src/gguf/qwen3_moe_ffn_layer_5699_tests.rs b/crates/aprender-serve/src/gguf/qwen3_moe_ffn_layer_5699_tests.rs new file mode 100644 index 0000000000..885eee33d6 --- /dev/null +++ b/crates/aprender-serve/src/gguf/qwen3_moe_ffn_layer_5699_tests.rs @@ -0,0 +1,133 @@ +// #5699 — `moe_ffn_forward_layer` coverage: input guards and the routed +// weighted-sum path (one expert selected, so output == that expert's SwiGLU). +// +// Named `moe_ffn_layer_5699_*` so `cargo test -p aprender-serve --lib +// moe_ffn_layer_5699` selects exactly this file. + +#[cfg(test)] +mod moe_ffn_layer_5699_tests { + use super::*; + use crate::gguf::types::GGUF_TYPE_Q4_K; + + const H: usize = 256; + const I: usize = 256; + const E: usize = 2; + const SB: usize = 144; + + /// Q4_K bytes for `n_blocks` super-blocks: d = 1.0 (f16), dmin = 0, the + /// rest a pattern that depends on `seed`, so different experts differ. + fn q4k_blocks(n_blocks: usize, seed: usize) -> Vec { + let mut out = Vec::with_capacity(n_blocks * SB); + for b in 0..n_blocks { + out.extend_from_slice(&0x3C00u16.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + for i in 4..SB { + out.push(((b * 31 + i * 7 + seed * 13 + 3) % 251) as u8); + } + } + out + } + + struct Fixture { + layer: Qwen3MoeQuantizedLayer, + data: Vec, + } + + /// Layout: [router F32 E×H][gate E×I rows][up E×I rows][down E×H rows]. + /// `favoured` is the expert the router scores highest for an all-ones input. + fn fixture(favoured: usize) -> Fixture { + let mut data = Vec::new(); + let router_off = data.len(); + for e in 0..E { + let w: f32 = if e == favoured { 1.0 } else { -1.0 }; + for _ in 0..H { + data.extend_from_slice(&w.to_le_bytes()); + } + } + let router_len = data.len() - router_off; + let mut tensor = |rows_per_expert: usize, seed: usize| { + let off = data.len(); + for e in 0..E { + data.extend(q4k_blocks(rows_per_expert, seed + e)); + } + QuantizedTensorRef { + offset: off, + byte_size: E * rows_per_expert * SB, + num_elements: E * rows_per_expert * 256, + qtype: GGUF_TYPE_Q4_K, + } + }; + let gate_exps = tensor(I, 1); + let up_exps = tensor(I, 5); + let down_exps = tensor(H, 9); + let router = QuantizedTensorRef { + offset: router_off, + byte_size: router_len, + num_elements: E * H, + qtype: crate::gguf::types::GGUF_TYPE_F32, + }; + Fixture { + layer: Qwen3MoeQuantizedLayer { + router, + gate_exps, + up_exps, + down_exps, + }, + data, + } + } + + #[test] + fn moe_ffn_layer_5699_rejects_hidden_dim_mismatch() { + let f = fixture(0); + let err = + moe_ffn_forward_layer(&vec![1.0; H - 1], &f.layer, E, 1, I, H, &f.data).unwrap_err(); + assert!( + format!("{err}").contains(&format!("hidden.len() = {} but hidden_dim = {H}", H - 1)), + "{err}" + ); + } + + #[test] + fn moe_ffn_layer_5699_rejects_quantized_router() { + let mut f = fixture(0); + f.layer.router.qtype = GGUF_TYPE_Q4_K; + let err = moe_ffn_forward_layer(&vec![1.0; H], &f.layer, E, 1, I, H, &f.data).unwrap_err(); + let msg = format!("{err}"); + assert!( + msg.contains("moe_router_quantized_read") && msg.contains("not F32"), + "{msg}" + ); + } + + #[test] + fn moe_ffn_layer_5699_rejects_router_byte_size_mismatch() { + let mut f = fixture(0); + f.layer.router.byte_size -= 4; + let err = moe_ffn_forward_layer(&vec![1.0; H], &f.layer, E, 1, I, H, &f.data).unwrap_err(); + assert!(format!("{err}").contains("router byte_size"), "{err}"); + } + + #[test] + fn moe_ffn_layer_5699_k1_output_is_the_routed_experts_swiglu() { + let hidden = vec![1.0f32; H]; + for favoured in 0..E { + let f = fixture(favoured); + let out = moe_ffn_forward_layer(&hidden, &f.layer, E, 1, I, H, &f.data).unwrap(); + let want = + expert_swiglu_quantized(&hidden, &f.layer, favoured, E, I, H, &f.data).unwrap(); + assert_eq!(out.len(), H); + assert!(out.iter().all(|v| v.is_finite())); + assert_eq!( + out, want, + "k=1 weight is 1.0, so output must equal expert {favoured}" + ); + let other = + expert_swiglu_quantized(&hidden, &f.layer, 1 - favoured, E, I, H, &f.data).unwrap(); + assert_ne!( + out, other, + "the fixture's experts must differ or routing is untested" + ); + } + } +} diff --git a/crates/aprender-serve/src/gguf/qwen3_moe_load.rs b/crates/aprender-serve/src/gguf/qwen3_moe_load.rs index 2d037388d0..7ac2bb7780 100644 --- a/crates/aprender-serve/src/gguf/qwen3_moe_load.rs +++ b/crates/aprender-serve/src/gguf/qwen3_moe_load.rs @@ -987,3 +987,7 @@ mod tests { // #3341 load-time contract tests (see the module doc in the included file). #[cfg(test)] include!("qwen3_moe_load_contract_tests.rs"); + +// #5699 moe_ffn_forward_layer input guards + routed path. +#[cfg(test)] +include!("qwen3_moe_ffn_layer_5699_tests.rs"); diff --git a/crates/aprender-serve/src/infer/mod.rs b/crates/aprender-serve/src/infer/mod.rs index 0f83791b3a..3326571653 100644 --- a/crates/aprender-serve/src/infer/mod.rs +++ b/crates/aprender-serve/src/infer/mod.rs @@ -998,6 +998,10 @@ mod tests_dense_session_4268; #[cfg(test)] #[path = "tests_encoded_ids_5056.rs"] mod tests_encoded_ids_5056; +// #5691: run_batch_loop failure rows. +#[cfg(test)] +#[path = "tests_batch_loop_5691.rs"] +mod tests_batch_loop_5691; #[cfg(test)] mod sibling_tokenizer_config_3990 { diff --git a/crates/aprender-serve/src/infer/tests_batch_loop_5691.rs b/crates/aprender-serve/src/infer/tests_batch_loop_5691.rs new file mode 100644 index 0000000000..6a8f862522 --- /dev/null +++ b/crates/aprender-serve/src/infer/tests_batch_loop_5691.rs @@ -0,0 +1,128 @@ +//! #5691: `run_batch_loop` turns every failing prompt into one JSONL error row +//! and counts it, never aborting the batch. + +use super::{run_batch_loop, BatchInferenceConfig, BatchModel}; +use std::path::PathBuf; + +fn config() -> BatchInferenceConfig { + BatchInferenceConfig { + model_path: PathBuf::from("unused.gguf"), + max_tokens: 4, + temperature: 0.0, + top_k: 1, + no_gpu: true, + verbose: false, + stop_tokens: vec![], + } +} + +/// A model with no backend: `generate` fails with "No model available". +fn empty_model() -> BatchModel { + BatchModel { + #[cfg(feature = "cuda")] + gpu: None, + #[cfg(feature = "gpu")] + wgpu: None, + cpu: None, + } +} + +fn run( + input: &str, + encode: fn(&str) -> Option>, +) -> (Vec, usize, usize, usize) { + let mut out = Vec::new(); + let stats = run_batch_loop( + &config(), + input.as_bytes(), + &mut out, + &[], + 12.5, + empty_model(), + &encode, + &|_: &[u32]| String::new(), + ) + .expect("loop survives per-prompt failures"); + let rows = String::from_utf8(out) + .expect("utf8") + .lines() + .map(|l| serde_json::from_str(l).expect("each row is JSON")) + .collect(); + (rows, stats.total_prompts, stats.successful, stats.failed) +} + +#[test] +fn bad_json_encode_failure_and_missing_model_each_emit_one_error_row_5691() { + let input = "\n \nnot json\n{\"prompt\":\"hi\",\"task_id\":\"enc\"}\n"; + let (rows, total, ok, failed) = run(input, |_| None); + // blank lines are skipped and uncounted; the other two fail + assert_eq!((total, ok, failed), (2, 0, 2)); + assert_eq!(rows.len(), 2); + assert!(rows[0]["error"] + .as_str() + .expect("err") + .starts_with("JSON parse error")); + assert!(rows[0]["task_id"].is_null()); + assert_eq!(rows[1]["error"], "Tokenizer encode failed"); + assert_eq!(rows[1]["task_id"], "enc"); + + // encode succeeds, the model has no backend: the error names it, the task id survives + let (rows, total, ok, failed) = run("{\"prompt\":\"hi\",\"task_id\":\"t1\"}\n", |_| { + Some(vec![1, 2]) + }); + assert_eq!((total, ok, failed), (1, 0, 1)); + assert_eq!(rows[0]["task_id"], "t1"); + assert_eq!(rows[0]["tokens_generated"], 0); + assert!(rows[0]["error"] + .as_str() + .expect("err") + .contains("No model available")); +} + +#[test] +fn stats_carry_model_load_and_every_tenth_prompt_is_counted_5691() { + let input = "x\n".repeat(10); + let mut out = Vec::new(); + let stats = run_batch_loop( + &config(), + input.as_bytes(), + &mut out, + &[], + 12.5, + empty_model(), + &|_: &str| None, + &|_: &[u32]| String::new(), + ) + .expect("ok"); + assert_eq!( + (stats.total_prompts, stats.failed, stats.successful), + (10, 10, 0) + ); + assert!((stats.model_load_ms - 12.5).abs() < f64::EPSILON); + assert!(stats.total_inference_ms.abs() < f64::EPSILON); +} + +#[test] +fn an_unwritable_sink_is_an_io_error_not_a_panic_5691() { + struct Broken; + impl std::io::Write for Broken { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::Error::other("disk full")) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let err = run_batch_loop( + &config(), + "oops\n".as_bytes(), + &mut Broken, + &[], + 0.0, + empty_model(), + &|_: &str| None, + &|_: &[u32]| String::new(), + ) + .expect_err("write failure surfaces"); + assert!(format!("{err}").contains("Write error"), "{err}"); +} diff --git a/crates/aprender-serve/tests/cli_entrypoint_dispatch.rs b/crates/aprender-serve/tests/cli_entrypoint_dispatch.rs new file mode 100644 index 0000000000..6964456bd3 --- /dev/null +++ b/crates/aprender-serve/tests/cli_entrypoint_dispatch.rs @@ -0,0 +1,69 @@ +//! `cli::entrypoint` dispatch: each arm reaches its handler and surfaces the handler's result. +//! +//! Complements the in-crate Info/Viz/Bench-list tests with the arms they leave +//! uncovered: List, BenchCompare (error propagation) and Run/Chat on a model +//! path that does not exist. +//! +//! Refs: #5707 + +use realizar::cli::{entrypoint, Cli, Commands}; + +const MISSING: &str = "/nonexistent-5707/model.gguf"; + +#[tokio::test] +async fn list_with_remote_returns_ok_without_touching_the_local_registry() { + let cli = Cli { + command: Commands::List { + remote: Some("http://registry.invalid".to_string()), + format: "table".to_string(), + }, + }; + assert!(entrypoint(cli).await.is_ok()); +} + +#[tokio::test] +async fn bench_compare_propagates_the_read_error_naming_the_file() { + let cli = Cli { + command: Commands::BenchCompare { + file1: "/nonexistent-5707/a.json".to_string(), + file2: "/nonexistent-5707/b.json".to_string(), + threshold: 5.0, + }, + }; + let err = entrypoint(cli).await.expect_err("missing files must fail"); + assert!( + err.to_string().contains("/nonexistent-5707/a.json"), + "{err}" + ); +} + +#[tokio::test] +async fn run_on_a_missing_local_model_is_an_error() { + let cli = Cli { + command: Commands::Run { + model: MISSING.to_string(), + prompt: Some("hi".to_string()), + max_tokens: 1, + temperature: 0.0, + format: "text".to_string(), + system: None, + raw: true, + gpu: false, + verbose: false, + trace: None, + }, + }; + assert!(entrypoint(cli).await.is_err()); +} + +#[tokio::test] +async fn chat_on_a_missing_local_model_is_an_error() { + let cli = Cli { + command: Commands::Chat { + model: MISSING.to_string(), + system: None, + history: None, + }, + }; + assert!(entrypoint(cli).await.is_err()); +} diff --git a/crates/aprender-test-lib/src/llm/benchmark.rs b/crates/aprender-test-lib/src/llm/benchmark.rs index 458f9c8582..573d3e3159 100644 --- a/crates/aprender-test-lib/src/llm/benchmark.rs +++ b/crates/aprender-test-lib/src/llm/benchmark.rs @@ -832,4 +832,33 @@ mod tests { assert!(exited(server), "{sig}: server {server} outlived the run"); } } + + #[test] + fn stat_summary_computes_mean_stddev_and_t_interval() { + let empty = stat_summary(&[]); + assert_eq!( + (empty.mean, empty.stddev, empty.ci_95_lower), + (0.0, 0.0, 0.0) + ); + assert!(empty.values.is_empty()); + + // one sample: no spread, and t_critical_95(1) = 0 so the interval collapses + let one = stat_summary(&[7.0]); + assert_eq!((one.mean, one.stddev), (7.0, 0.0)); + assert_eq!((one.ci_95_lower, one.ci_95_upper), (7.0, 7.0)); + + // [2,4,6]: mean 4, sample variance 4, so stddev 2; margin = t(3) * 2 / sqrt(3) + let s = stat_summary(&[2.0, 4.0, 6.0]); + let margin = 4.303 * 2.0 / 3.0_f64.sqrt(); + assert!((s.mean - 4.0).abs() < 1e-12 && (s.stddev - 2.0).abs() < 1e-12); + assert!((s.ci_95_lower - (4.0 - margin)).abs() < 1e-9); + assert!((s.ci_95_upper - (4.0 + margin)).abs() < 1e-9); + assert_eq!(s.values, vec![2.0, 4.0, 6.0]); + + // the table's bucket edges + assert_eq!(t_critical_95(10), 2.262); + assert_eq!(t_critical_95(11), 2.145); + assert_eq!(t_critical_95(30), 2.042); + assert_eq!(t_critical_95(31), 1.96); + } } diff --git a/crates/aprender-test-lib/tests/mock_page_take_coverage_5695.rs b/crates/aprender-test-lib/tests/mock_page_take_coverage_5695.rs new file mode 100644 index 0000000000..74b978bc46 --- /dev/null +++ b/crates/aprender-test-lib/tests/mock_page_take_coverage_5695.rs @@ -0,0 +1,63 @@ +//! #5695: the mock `Page::take_coverage` / `stop_coverage` contract. +//! +//! These run against the mock `Page` (the `browser` feature off), which is the +//! build every default-feature consumer gets. +#![cfg(not(feature = "browser"))] + +use jugar_probar::{FunctionCoverage, Page}; + +fn func(name: &str) -> FunctionCoverage { + FunctionCoverage { + function_name: name.to_string(), + ranges: Vec::new(), + is_block_coverage: false, + } +} + +#[test] +fn take_coverage_before_start_is_invalid_state() { + let page = Page::new(800, 600); + assert!(!page.is_coverage_enabled()); + let err = page.take_coverage().expect_err("coverage is not enabled"); + assert!( + err.to_string().contains("Coverage not enabled"), + "unexpected error: {err}" + ); +} + +#[test] +fn take_coverage_reports_page_url_and_added_functions() { + let mut page = Page::new(800, 600); + page.start_coverage().expect("mock start never fails"); + assert!(page.is_coverage_enabled()); + page.add_mock_coverage(func("alpha")); + page.add_mock_coverage(func("beta")); + + let report = page.take_coverage().expect("enabled"); + assert_eq!(report.scripts.len(), 1); + let script = &report.scripts[0]; + assert_eq!(script.script_id, "mock-script-1"); + assert_eq!(script.url, "about:blank"); + let names: Vec<&str> = script + .functions + .iter() + .map(|f| f.function_name.as_str()) + .collect(); + assert_eq!(names, ["alpha", "beta"]); + assert!(report.timestamp_ms > 0); + // take_coverage is a snapshot: it must not disable collection. + assert!(page.is_coverage_enabled()); +} + +#[test] +fn stop_coverage_returns_final_report_then_disables() { + let mut page = Page::new(800, 600); + page.start_coverage().expect("start"); + page.add_mock_coverage(func("gamma")); + + let report = page.stop_coverage().expect("enabled"); + assert_eq!(report.scripts[0].functions.len(), 1); + assert!(!page.is_coverage_enabled()); + assert!(page.take_coverage().is_err()); + assert!(page.stop_coverage().is_err()); +} diff --git a/docs/findings/aprender-w5404.jsonl b/docs/findings/aprender-w5404.jsonl new file mode 100644 index 0000000000..ffa41e6139 --- /dev/null +++ b/docs/findings/aprender-w5404.jsonl @@ -0,0 +1 @@ +{"session":"aprender-w5404","ref":"#5651","note":"stat_summary already had 4 tests; added exact t-interval + bucket-edge test, FALSIFY-OPS-015; no pmat ticket (needs GH issue)"}