diff --git a/README.md b/README.md index 68106cff29..f9b9b475de 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ publishing — all backed by YAML provable contracts that fail CI on drift. | Metric | Count | Source of truth | |-------:|------:|---| | Workspace crates | **81** workspace crates | `cargo metadata --no-deps` (NOT `ls crates/` — 4 are `exclude`d, 1 has no Cargo.toml) | -| Provable contracts | **1996** provable contracts | `contracts/census.json` `.n_files` — the set `pv lint` walks (`pv census`, ONT-001 ONT-1; regenerated by `make contracts`, written by `make readme-sync`, guarded by `scripts/check_readme_claims.sh`) | +| Provable contracts | **1999** provable contracts | `contracts/census.json` `.n_files` — the set `pv lint` walks (`pv census`, ONT-001 ONT-1; regenerated by `make contracts`, written by `make readme-sync`, guarded by `scripts/check_readme_claims.sh`) | | CLI commands | **113** CLI commands | `contracts/apr-cli-commands-v1.yaml` §`commands` (parse the list; `apr --help` also lists `help` itself, and `grep -c '^ - name:'` gives 119 — other same-indent `name:` keys exist in the file) | | Book CLI chapters | **114** chapters | `ls book/src/cli/*.md` | | Book lib chapters | **73** chapters | `ls book/src/lib/*.md` (parity with `pub mod`) | @@ -403,7 +403,7 @@ falsification_tests: prediction: apr validate bad-model.apr exits non-zero ``` -The tree carries 1996 contracts across inference, training, quantization, attention, FFN, +The tree carries 1999 contracts across inference, training, quantization, attention, FFN, tokenization, model formats, CLI safety — and this README itself. ## Migration from old crates diff --git a/ci/explicit-test-commands.d/812-aprender-cgp-quant-profile-output.cmd b/ci/explicit-test-commands.d/812-aprender-cgp-quant-profile-output.cmd new file mode 100644 index 0000000000..7a69cb826f --- /dev/null +++ b/ci/explicit-test-commands.d/812-aprender-cgp-quant-profile-output.cmd @@ -0,0 +1 @@ +cargo test -p aprender-cgp --test quant_profile_output diff --git a/ci/explicit-test-commands.d/814-aprender-profile-run-config-errors.cmd b/ci/explicit-test-commands.d/814-aprender-profile-run-config-errors.cmd new file mode 100644 index 0000000000..e7afc92a0b --- /dev/null +++ b/ci/explicit-test-commands.d/814-aprender-profile-run-config-errors.cmd @@ -0,0 +1 @@ +cargo test -p aprender-profile --test run_config_errors_5713 diff --git a/ci/explicit-test-commands.d/815-aprender-serve-qwen35-load-failclosed.cmd b/ci/explicit-test-commands.d/815-aprender-serve-qwen35-load-failclosed.cmd new file mode 100644 index 0000000000..ac97df4f5e --- /dev/null +++ b/ci/explicit-test-commands.d/815-aprender-serve-qwen35-load-failclosed.cmd @@ -0,0 +1 @@ +cargo test -p aprender-serve --test qwen35_load_failclosed_5678 diff --git a/ci/explicit-test-commands.d/816-aprender-train-execute-plan-guards.cmd b/ci/explicit-test-commands.d/816-aprender-train-execute-plan-guards.cmd new file mode 100644 index 0000000000..5cbae16e4d --- /dev/null +++ b/ci/explicit-test-commands.d/816-aprender-train-execute-plan-guards.cmd @@ -0,0 +1 @@ +cargo test -p aprender-train --test execute_plan_guards_5674 diff --git a/ci/explicit-test-commands.d/817-aprender-orchestrate-repl-slash.cmd b/ci/explicit-test-commands.d/817-aprender-orchestrate-repl-slash.cmd new file mode 100644 index 0000000000..1283ff055d --- /dev/null +++ b/ci/explicit-test-commands.d/817-aprender-orchestrate-repl-slash.cmd @@ -0,0 +1 @@ +cargo test -p aprender-orchestrate --test repl_slash diff --git a/contracts/bin-aprender-profile--aprender-profile-v1.yaml b/contracts/bin-aprender-profile--aprender-profile-v1.yaml index bd2e34fd88..e3d8041ac7 100644 --- a/contracts/bin-aprender-profile--aprender-profile-v1.yaml +++ b/contracts/bin-aprender-profile--aprender-profile-v1.yaml @@ -133,3 +133,9 @@ falsification_tests: test: 'cargo test -p aprender-profile --test cli_surface_gate unusable_input_is_rejected' if_fails: 'MEASURED 2026-09-27 (aprender-01): each exits 2 ("Baseline not found", the documented code) with no panic, GREEN; an existing FILE is also reported "not found" because a baseline is a directory' mutation: 'make a rejection path return Ok(()) or unwrap the read; must turn RED on that row' + - id: FALSIFY-BIN-APRENDER-PROFILE-007 + rule: 'run rejects bad configuration before tracing, naming the flag' + prediction: '`--ml-clusters 1`, a missing or malformed `--transpiler-map`, an unknown `--chaos` preset and an unparsable `--chaos-memory-limit` each exit 1 with a stderr message naming the offending value' + test: 'cargo test -p aprender-profile --test run_config_errors_5713' + if_fails: 'a bad flag is silently accepted and a trace starts with a default (#5713)' + mutation: 'change the ml_clusters guard from < 2 to < 1; ml_clusters_below_two_is_rejected_with_the_value must turn RED' diff --git a/contracts/classification-finetune-v1.yaml b/contracts/classification-finetune-v1.yaml index 95b02bae8e..08d8af17bf 100644 --- a/contracts/classification-finetune-v1.yaml +++ b/contracts/classification-finetune-v1.yaml @@ -146,6 +146,11 @@ falsification_tests: prediction: 'pass + warn + fail equals the number of pre-flight checks, so none is counted twice or dropped' test: 'aprender-train finetune::training_plan::tests::check_counts_partitions_every_preflight_check' if_fails: 'A status bucket was dropped or a filter loosened; the displayed summary and the verdict would then disagree about the same checks. MUTANT: fail = 0 turns 2 rows RED.' +- id: FALSIFY-CLASS-010 + rule: Apply preconditions fail loudly, by kind + prediction: 'execute_plan returns ConfigError naming a missing model dir, Io naming missing training data, and ConfigError for an unknown model size (GH-377) after creating the output dir; none fall through to training' + test: 'aprender-train --test execute_plan_guards_5674 (4 tests)' + if_fails: 'A precondition was dropped or its error kind changed; apply would start training against a missing model/data or silently default the model size. MUTANT: execute_plan missing-data guard returning ConfigError instead of Io turns missing_training_data_is_an_io_error RED.' kani_harnesses: - id: KANI-FT-001 obligation: FT-INV-001 @@ -204,5 +209,5 @@ qa_gate: - softmax_sum_invariant - nan_inf_rejection - config_factory_contract_sync - pass_criteria: All 9 falsification tests pass + pass_criteria: All 10 falsification tests pass falsification: Bypass ValidatedClassLogits::new() with unsafe or wrong constructor diff --git a/contracts/contracts.nt b/contracts/contracts.nt index 87447a2049..8c39543ec3 100644 --- a/contracts/contracts.nt +++ b/contracts/contracts.nt @@ -11788,6 +11788,11 @@ "contracts/explain-tui-draw-scrollbar-v1.yaml"^^ . "explain-tui-draw-scrollbar-v1"^^ . "pattern"^^ . + . + . + "contracts/explain-tui-register-panel-v1.yaml"^^ . + "explain-tui-register-panel-v1"^^ . + "pattern"^^ . . . . @@ -12550,6 +12555,11 @@ "contracts/model-capability-ladder-v1.yaml"^^ . "model-capability-ladder-v1"^^ . "pattern"^^ . + . + . + "contracts/model-card-paint-v1.yaml"^^ . + "model-card-paint-v1"^^ . + "pattern"^^ . . . . @@ -14078,6 +14088,11 @@ "tracing-observability-v1"^^ . "tracing-observability"^^ . "1.0.0"^^ . + . + . + "contracts/train-classify-train-loop-v1.yaml"^^ . + "train-classify-train-loop-v1"^^ . + "pattern"^^ . . . "contracts/train-loader-try-load-apr-v1.yaml"^^ . diff --git a/contracts/explain-tui-register-panel-v1.yaml b/contracts/explain-tui-register-panel-v1.yaml new file mode 100644 index 0000000000..7750dd5f41 --- /dev/null +++ b/contracts/explain-tui-register-panel-v1.yaml @@ -0,0 +1,47 @@ +metadata: + id: C-EXPLAIN-TUI-REGISTER-PANEL + version: 1.0.0 + created: '2026-10-19' + author: PAIML Engineering + kind: pattern + status: ACTIVE_ALGORITHM_LEVEL + description: | + Pins the register panel drawn by the aprender-explain TUI sidebar (#5594): + one line per register class with its limit, a total/occupancy line, and a + status colour that is green below 64 registers, yellow from 64 to 127 and + red from 128. + references: + - 'crates/aprender-explain/src/tui/widgets.rs (render_register_panel)' + +equations: + register_panel_status_colour: + formula: | + colour(total) = green if total < 64, yellow if total < 128, else red + domain: RegisterUsage totals passed through render_sidebar + invariants: + - 'the thresholds are 64 and 128, lower bound inclusive' + +proof_obligations: +- id: ERP-INV-001 + type: invariant + property: register panel text and status colour follow the total register count + formal: 'colour ≡ threshold' + applies_to: register_panel_status_colour + +falsification_tests: +- id: FALSIFY-EXPLAIN-TUI-REGISTER-PANEL-001 + rule: render_register_panel draws exact lines and the threshold colour + prediction: 'totals 63, 64, 127 and 128 give green, yellow, yellow and red borders; the lines show the .f32/.b32/.b64/.pred counts and the Total/occ text' + test: cargo test -p aprender-explain --lib register_panel + if_fails: the TUI misreports register pressure on a kernel + ship_blocking: false + mutation: 'change either < to <=, or print the wrong register field; each must turn RED' + +qa_gate: + id: QA-EXPLAIN-TUI-REGISTER-PANEL-001 + name: Explain TUI register panel gate + description: register panel text and colour thresholds are exact + checks: + - register_panel_status_colour + pass_criteria: FALSIFY-EXPLAIN-TUI-REGISTER-PANEL-001 passes + falsification: Any failure fails the gate diff --git a/contracts/model-card-paint-v1.yaml b/contracts/model-card-paint-v1.yaml new file mode 100644 index 0000000000..b87c4d239f --- /dev/null +++ b/contracts/model-card-paint-v1.yaml @@ -0,0 +1,47 @@ +metadata: + version: 1.0.0 + created: '2026-10-19' + author: PAIML Engineering + description: "presentar-widgets ModelCard::paint (#5539) — draws background, border and status badge, then title and version, then optional description, framework/task line, parameters, at most 4 metrics (only when the chart is on) and at most 5 tags, each row below the previous." + kind: pattern + references: + - "crates/aprender-present-widgets/src/model_card.rs — paint" + - "aprender#5539 — coverage ticket" + +equations: + paint_draw_list: + formula: "paint emits 3 rects (bg, border, badge) + texts [status, name, vN]; description adds 24 to y of later rows; framework/task text is 'f • t', f or t; metrics emit 2 texts each for min(n,4) when show_metrics_chart; tags emit rect(len*6+12 x 18)+text for min(n,5)" + domain: "a laid-out ModelCard and a RecordingCanvas" + codomain: "the recorded DrawCommand list" + invariants: + - "metrics beyond the 4th and tags beyond the 5th are never drawn" + - "show_metrics_chart=false suppresses all metric text" + +proof_obligations: + - id: MODELCARD-PAINT-INV-001 + type: invariant + property: "Metrics are capped at 4 and gated by show_metrics_chart" + formal: "∀ run: cargo test -p aprender-present-widgets --lib test_paint_metrics_cap_at_four_and_toggle exits 0" + applies_to: paint_draw_list + - id: MODELCARD-PAINT-INV-002 + type: invariant + property: "Tags are capped at 5 and sized from their length" + formal: "∀ run: cargo test -p aprender-present-widgets --lib test_paint_tags_cap_at_five_with_width_from_len exits 0" + applies_to: paint_draw_list + +falsification_tests: + - id: FALSIFY-MODELCARD-PAINT-001 + rule: "metric cap raised" + prediction: "changing take(4) to take(5) turns test_paint_metrics_cap_at_four_and_toggle RED" + test: "cargo test -p aprender-present-widgets --lib test_paint_metrics_cap_at_four_and_toggle" + if_fails: "metric cap is unguarded" + - id: FALSIFY-MODELCARD-PAINT-002 + rule: "tag width formula changed" + prediction: "using len*5 instead of len*6 turns test_paint_tags_cap_at_five_with_width_from_len RED" + test: "cargo test -p aprender-present-widgets --lib test_paint_tags_cap_at_five_with_width_from_len" + if_fails: "tag sizing is unguarded" + - id: FALSIFY-MODELCARD-PAINT-003 + rule: "description offset changed" + prediction: "y_offset += 20 after the description turns test_paint_description_shifts_following_rows_by_24 RED" + test: "cargo test -p aprender-present-widgets --lib test_paint_description_shifts_following_rows_by_24" + if_fails: "row layout is unguarded" diff --git a/contracts/qwen35-shapes-v1.yaml b/contracts/qwen35-shapes-v1.yaml index 880d8c371f..65c549928f 100644 --- a/contracts/qwen35-shapes-v1.yaml +++ b/contracts/qwen35-shapes-v1.yaml @@ -128,6 +128,11 @@ falsification_tests: prediction: SIMD shapes match scalar shapes test: 'proptest: compare scalar vs SIMD projection shapes' if_fails: SIMD implementation uses different dimensions +- id: FALSIFY-Q35-008 + rule: Layer loader fails closed + prediction: 'load_qwen35_layers errors naming {arch}.block_count when the key is absent, returns 0 layers for block_count = 0, and errors (not a truncated model) when a declared layer lacks its tensors' + test: 'aprender-serve --test qwen35_load_failclosed_5678 (4 tests, header-only GGUF)' + if_fails: 'A missing depth is guessed or a missing tensor is skipped, yielding a silently truncated Qwen3.5. MUTANT: missing block_count falling back to unwrap_or(0) turns missing_block_count_is_refused_not_guessed RED.' kani_harnesses: - id: KANI-Q35-001 obligation: Q35-INV-001 @@ -181,5 +186,5 @@ qa_gate: - swiglu_ratio - o_projection - rope_frequencies - pass_criteria: All 7 falsification tests pass + pass_criteria: All 8 falsification tests pass falsification: Change n_kv from 4 to 3 to break GQA divisibility diff --git a/contracts/train-classify-train-loop-v1.yaml b/contracts/train-classify-train-loop-v1.yaml new file mode 100644 index 0000000000..a9152f9a54 --- /dev/null +++ b/contracts/train-classify-train-loop-v1.yaml @@ -0,0 +1,47 @@ +metadata: + id: C-TRAIN-CLASSIFY-TRAIN-LOOP + version: 1.0.0 + created: '2026-10-19' + author: PAIML Engineering + kind: pattern + status: ACTIVE_ALGORITHM_LEVEL + description: | + Pins ClassifyTrainer::train on the single-process path (#5686): early + stopping fires when epochs without improvement reach the patience, best + epoch tracks the lowest validation loss, best and periodic checkpoints are + written, and an attached monitor writer receives the run. + references: + - 'crates/aprender-train/src/finetune/classify_trainer.rs (train)' + +equations: + classify_train_loop: + formula: | + stop after epoch e iff epochs_without_improvement(e) >= patience; checkpoint epoch-e iff (e + 1) mod save_every == 0 (save_every is 1 when epochs <= save_every); best/ is written on every new lowest val_loss + domain: single-process ClassifyTrainer runs + invariants: + - 'epoch_metrics are recorded in epoch order' + +proof_obligations: +- id: TCT-INV-001 + type: invariant + property: patience bounds the run and best_epoch is the lowest val_loss + formal: 'stopped ⇒ without_improvement ≡ patience' + applies_to: classify_train_loop + +falsification_tests: +- id: FALSIFY-TRAIN-CLASSIFY-TRAIN-LOOP-001 + rule: train stops at the patience, tracks the best epoch and writes checkpoints + prediction: 'patience 0 gives one epoch and stopped_early; patience 100 runs every epoch with best_epoch at the lowest val_loss; best, epoch-0 and epoch-1 checkpoints exist' + test: cargo test -p aprender-train --lib _5686 + if_fails: training overruns its patience, reports the wrong best epoch, or leaves no checkpoint to resume from + ship_blocking: false + mutation: 'change >= to > in the early-stopping check, or disable the periodic checkpoint branch; each must turn RED' + +qa_gate: + id: QA-TRAIN-CLASSIFY-TRAIN-LOOP-001 + name: Classify train loop gate + description: early stopping, best epoch, checkpoints and monitor hold + checks: + - classify_train_loop + pass_criteria: FALSIFY-TRAIN-CLASSIFY-TRAIN-LOOP-001 passes + falsification: Any failure fails the gate diff --git a/contracts/witness/811c78184eb2609244c59534f015f0be2f90923120a3027c176ad56723d0d850.json b/contracts/witness/811c78184eb2609244c59534f015f0be2f90923120a3027c176ad56723d0d850.json index 5ca9abb1de..428db74ad0 100644 --- a/contracts/witness/811c78184eb2609244c59534f015f0be2f90923120a3027c176ad56723d0d850.json +++ b/contracts/witness/811c78184eb2609244c59534f015f0be2f90923120a3027c176ad56723d0d850.json @@ -1,5 +1,5 @@ { - "census_id_set_sha256": "db20faa02774eeffec52d05d3e579c0ec7047e2f0bed655de26b7e3eeff34151", + "census_id_set_sha256": "97d2981c4166b0ad4b9557a36ed59d891f92a64092bc63c7c7241832d9c4cbae", "relations_sha256": "811c78184eb2609244c59534f015f0be2f90923120a3027c176ad56723d0d850", "checkable_n": 87, "result": { diff --git a/crates/aprender-cbtop/src/app/render/panels.rs b/crates/aprender-cbtop/src/app/render/panels.rs index 16da23d96a..e40972309f 100644 --- a/crates/aprender-cbtop/src/app/render/panels.rs +++ b/crates/aprender-cbtop/src/app/render/panels.rs @@ -400,6 +400,94 @@ impl CbtopApp { } } +#[cfg(test)] +mod render_sparklines_tests { + use super::*; + use presentar_terminal::direct::CellBuffer; + + const W: u16 = 80; + const H: u16 = 40; + + fn row_text(buf: &CellBuffer, y: u16) -> String { + (0..W) + .map(|x| buf.get(x, y).map_or(" ", |c| c.symbol.as_str())) + .collect() + } + + /// Run `render_sparklines` on a blank buffer; returns (buffer, final current_y). + fn run(start_y: f32, height: u16, cpu: &[f64], bricks: &[f64]) -> (CellBuffer, f32) { + let mut buf = CellBuffer::new(W, H); + let theme = Theme::default(); + let style = TextStyle::default(); + let mut y = start_y; + { + let mut canvas = DirectTerminalCanvas::new(&mut buf); + CbtopApp::render_sparklines( + &mut canvas, + &mut y, + height, + cpu, + bricks, + 60, + 62, + 40, + &theme, + &style, + &style, + ); + } + (buf, y) + } + + #[test] + fn empty_data_draws_nothing_and_keeps_y() { + let (buf, y) = run(2.0, H, &[], &[]); + assert_eq!(y, 2.0); + for row in 0..H { + assert!(!row_text(&buf, row).contains('┌'), "row {row} drew a box"); + } + } + + #[test] + fn cpu_box_draws_header_and_advances_four() { + let (buf, y) = run(2.0, H, &[0.5, 0.9], &[]); + assert_eq!(y, 6.0); + assert!(row_text(&buf, 2).contains("CPU History (braille)")); + assert!(row_text(&buf, 3).contains('│')); + assert!(row_text(&buf, 4).contains('└')); + assert!(!row_text(&buf, 6).contains("Bricks")); + } + + #[test] + fn cpu_box_skipped_when_too_close_to_bottom() { + // threshold is current_y < height - 8 + let (_, y) = run(32.0, H, &[0.5], &[]); + assert_eq!(y, 32.0); + let (buf, y) = run(31.0, H, &[0.5], &[]); + assert_eq!(y, 35.0); + assert!(row_text(&buf, 31).contains("CPU History")); + } + + #[test] + fn bricks_box_has_own_threshold_and_no_trailing_advance() { + // threshold is current_y < height - 5 + let (_, y) = run(35.0, H, &[], &[1.0]); + assert_eq!(y, 35.0); + let (buf, y) = run(34.0, H, &[], &[1.0]); + assert_eq!(y, 36.0); + assert!(row_text(&buf, 34).contains("Bricks/sec (braille)")); + assert!(row_text(&buf, 36).contains('└')); + } + + #[test] + fn both_boxes_stack_cpu_then_bricks() { + let (buf, y) = run(2.0, H, &[0.5], &[2.0]); + assert!(row_text(&buf, 2).contains("CPU History")); + assert!(row_text(&buf, 6).contains("Bricks/sec")); + assert_eq!(y, 8.0); + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/aprender-cgp/tests/quant_profile_output.rs b/crates/aprender-cgp/tests/quant_profile_output.rs new file mode 100644 index 0000000000..0653e86edf --- /dev/null +++ b/crates/aprender-cgp/tests/quant_profile_output.rs @@ -0,0 +1,110 @@ +//! `profile_quant` prints to stdout and discovers the benchmark binary relative to the +//! NOTE: the label branch is `bw_pct > compute_pct => COMPUTE-BOUND`; these tests pin that behaviour as-is. +//! working directory, so the output tests re-run this test binary as a child process in a +//! scratch directory (where `cargo metadata` fails and `./target/release/examples/...` is +//! the only candidate) and assert on its stdout. + +#![allow(clippy::disallowed_methods)] + +use std::os::unix::fs::PermissionsExt; +use std::path::Path; +use std::process::Command; + +const ENV_ARGS: &str = "CGP_QUANT_CHILD_ARGS"; + +/// Child entry: a no-op unless the parent set the env var. +#[test] +fn child_entry() { + let Ok(args) = std::env::var(ENV_ARGS) else { + return; + }; + let (kernel, size) = args.split_once(' ').expect("kernel and size"); + cgp::profilers::quant::profile_quant(kernel, size).expect("profile_quant"); +} + +fn run_child(cwd: &Path, kernel: &str, size: &str) -> String { + let out = Command::new(std::env::current_exe().expect("current_exe")) + .args(["--exact", "child_entry", "--nocapture", "--test-threads=1"]) + .env(ENV_ARGS, format!("{kernel} {size}")) + .current_dir(cwd) + .output() + .expect("spawn child"); + assert!(out.status.success(), "child failed: {out:?}"); + String::from_utf8_lossy(&out.stdout).into_owned() +} + +fn install_bench_shim(cwd: &Path, line: &str) { + let dir = cwd.join("target/release/examples"); + std::fs::create_dir_all(&dir).expect("mkdir"); + let bin = dir.join("benchmark_matrix_suite"); + std::fs::write(&bin, format!("#!/bin/sh\necho '{line}'\n")).expect("write shim"); + std::fs::set_permissions(&bin, std::fs::Permissions::from_mode(0o755)).expect("chmod"); +} + +#[test] +fn unknown_kernel_and_bad_size_are_errors() { + let err = cgp::profilers::quant::profile_quant("q9_gemv", "4096x1x4096").unwrap_err(); + assert!(err.to_string().contains("Unknown quant kernel"), "{err}"); + let err = cgp::profilers::quant::profile_quant("q4k", "4096x4096").unwrap_err(); + assert!(err.to_string().contains("MxNxK"), "{err}"); +} + +#[test] +fn without_bench_binary_prints_static_metrics_and_no_timing() { + let tmp = tempfile::tempdir().expect("tempdir"); + let out = run_child(tmp.path(), "q4k", "4096x1x4096"); + assert!(out.contains("Kernel: Q4kGemv"), "{out}"); + assert!( + out.contains("Super-block: 256 elements, 144 bytes"), + "{out}" + ); + assert!(out.contains("Total weights: 16777216"), "{out}"); + assert!(out.contains("Super-blocks: 65536"), "{out}"); + assert!(out.contains("Compressed size: 9.44 MB"), "{out}"); + assert!(out.contains("FP32 equivalent: 67.11 MB"), "{out}"); + assert!(out.contains("Compression ratio: 7.1x"), "{out}"); + assert!(out.contains("No timing data"), "{out}"); + assert!(!out.contains("Roofline"), "{out}"); +} + +#[test] +fn timing_line_drives_roofline_and_token_estimate() { + let tmp = tempfile::tempdir().expect("tempdir"); + install_bench_shim( + tmp.path(), + "Q4K GEMV 1536x8960 ... 100.0 us (50.0 GFLOPS, 20.0 GB/s)", + ); + let out = run_child(tmp.path(), "q4k_gemv", "1536x1x8960"); + assert!(out.contains("Time: 100.0 us"), "{out}"); + assert!(out.contains("Compute util: 33% of AVX-512 peak"), "{out}"); + assert!( + out.contains("Bandwidth util: 50% of practical DRAM"), + "{out}" + ); + assert!(out.contains("Bottleneck: COMPUTE-BOUND"), "{out}"); + assert!(out.contains("Est. 192 GEMVs/token: 19.2 ms"), "{out}"); + assert!(out.contains("Est. tokens/sec: 52.1"), "{out}"); + assert!(!out.contains("No timing data"), "{out}"); +} + +#[test] +fn higher_compute_than_bandwidth_pct_flips_label() { + let tmp = tempfile::tempdir().expect("tempdir"); + install_bench_shim( + tmp.path(), + "Q4K GEMV 1536x8960 ... 100.0 us (90.0 GFLOPS, 10.0 GB/s)", + ); + let out = run_child(tmp.path(), "q4k", "1536x1x8960"); + assert!(out.contains("Bottleneck: MEMORY-BOUND"), "{out}"); +} + +#[test] +fn timing_for_other_dimensions_is_ignored() { + let tmp = tempfile::tempdir().expect("tempdir"); + install_bench_shim( + tmp.path(), + "Q4K GEMV 1536x8960 ... 100.0 us (50.0 GFLOPS, 20.0 GB/s)", + ); + let out = run_child(tmp.path(), "q4k", "4096x1x4096"); + assert!(out.contains("No timing data"), "{out}"); +} diff --git a/crates/aprender-explain/src/tui/tests.rs b/crates/aprender-explain/src/tui/tests.rs index 0748b87243..80c8c7af74 100644 --- a/crates/aprender-explain/src/tui/tests.rs +++ b/crates/aprender-explain/src/tui/tests.rs @@ -236,3 +236,5 @@ fn f028_sync_scroll_source_asm() { app.handle_key(KeyCode::PageUp); assert_eq!(app.source_scroll, 0, "Should scroll back to top"); } + +include!("tests_register_panel.rs"); diff --git a/crates/aprender-explain/src/tui/tests_register_panel.rs b/crates/aprender-explain/src/tui/tests_register_panel.rs new file mode 100644 index 0000000000..3e3809d3f9 --- /dev/null +++ b/crates/aprender-explain/src/tui/tests_register_panel.rs @@ -0,0 +1,102 @@ +// #5594: behavioural tests for widgets::render_register_panel, driven through +// the pub(super) render_sidebar entry point (the panel occupies rows 0..8). +// Included from tests.rs so `super::*` items (TuiApp, sample_report) are in scope. + +fn render_panel_rows(regs: RegisterUsage, occupancy: f32) -> (CellBuffer, Vec) { + let mut report = sample_report(); + report.registers = regs; + report.estimated_occupancy = occupancy; + let app = TuiApp::new(".entry t() { ret; }".to_string(), report); + let mut buffer = CellBuffer::new(44, 30); + { + let mut canvas = DirectTerminalCanvas::new(&mut buffer); + widgets::render_sidebar(&mut canvas, &app, 0.0, 0.0, 44, 30); + } + let rows = (0..8u16) + .map(|y| { + (0..44u16) + .filter_map(|x| buffer.get(x, y).map(|c| c.symbol.to_string())) + .collect::() + }) + .collect(); + (buffer, rows) +} + +fn regs_with_total(f32_regs: u32) -> RegisterUsage { + RegisterUsage { + f32_regs, + ..Default::default() + } +} + +fn border_fg(buffer: &CellBuffer) -> Color { + buffer.get(0, 0).expect("cell (0,0) exists").fg +} + +const PANEL_GREEN: Color = Color { + r: 0.3, + g: 1.0, + b: 0.3, + a: 1.0, +}; +const PANEL_YELLOW: Color = Color { + r: 1.0, + g: 1.0, + b: 0.3, + a: 1.0, +}; +const PANEL_RED: Color = Color { + r: 1.0, + g: 0.3, + b: 0.3, + a: 1.0, +}; + +#[test] +fn register_panel_lines_show_each_register_class() { + let regs = RegisterUsage { + f32_regs: 24, + b32_regs: 18, + b64_regs: 12, + pred_regs: 4, + ..Default::default() + }; + let (_, rows) = render_panel_rows(regs, 0.875); + assert!(rows[0].starts_with("┌─ Registers "), "{}", rows[0]); + assert!(rows[0].trim_end().ends_with('┐'), "{}", rows[0]); + assert!(rows[1].contains(".f32: 24 / 255"), "{}", rows[1]); + assert!(rows[2].contains(".b32: 18 / 255"), "{}", rows[2]); + assert!(rows[3].contains(".b64: 12 / 255"), "{}", rows[3]); + assert!(rows[4].contains(".pred: 4 / 8"), "{}", rows[4]); + assert!(rows[5].contains("Total: 58 -> 88% occ"), "{}", rows[5]); + assert!( + rows[7].starts_with('└') && rows[7].trim_end().ends_with('┘'), + "{}", + rows[7] + ); +} + +#[test] +fn register_panel_total_below_64_is_green() { + let (buf, _) = render_panel_rows(regs_with_total(63), 0.5); + assert!(color_approx_eq(border_fg(&buf), PANEL_GREEN)); +} + +#[test] +fn register_panel_total_64_is_yellow() { + let (buf, _) = render_panel_rows(regs_with_total(64), 0.5); + assert!(color_approx_eq(border_fg(&buf), PANEL_YELLOW)); +} + +#[test] +fn register_panel_total_127_is_yellow() { + let (buf, _) = render_panel_rows(regs_with_total(127), 0.5); + assert!(color_approx_eq(border_fg(&buf), PANEL_YELLOW)); +} + +#[test] +fn register_panel_total_128_is_red() { + let (buf, rows) = render_panel_rows(regs_with_total(128), 0.25); + assert!(color_approx_eq(border_fg(&buf), PANEL_RED)); + assert!(rows[5].contains("Total: 128 -> 25% occ"), "{}", rows[5]); +} diff --git a/crates/aprender-orchestrate/tests/repl_slash.rs b/crates/aprender-orchestrate/tests/repl_slash.rs new file mode 100644 index 0000000000..51873de793 --- /dev/null +++ b/crates/aprender-orchestrate/tests/repl_slash.rs @@ -0,0 +1,137 @@ +//! Slash-command handling in the REPL, driven through real stdin. +//! +//! `handle_slash_command` is private and prints to stdout, so each case re-executes this +//! test binary as a child running `run_repl` with piped stdin (`HOME` and `NO_COLOR` set) +//! and asserts on the child's stdout. +//! +//! Refs: #5540 + +#![cfg(feature = "agents")] + +use std::io::Write; +use std::process::{Command, Stdio}; + +use batuta::agent::driver::mock::MockDriver; +use batuta::agent::manifest::AgentManifest; +use batuta::agent::memory::InMemorySubstrate; +use batuta::agent::repl::run_repl; +use batuta::agent::tool::ToolRegistry; + +const CHILD_ENV: &str = "APR_REPL_SLASH_CHILD"; + +/// Child entry point: a no-op unless re-executed by `run_child`. +#[test] +fn child_entry() { + if std::env::var(CHILD_ENV).is_err() { + return; + } + let driver = MockDriver::single_response("ok"); + let mut manifest = AgentManifest::default(); + manifest.name = "repl-slash-test".to_string(); + run_repl(&manifest, &driver, &ToolRegistry::new(), &InMemorySubstrate::new(), 10, 100.0, None) + .expect("run_repl returns Ok"); +} + +/// Feed `stdin` to a fresh REPL and return its stdout. +fn run_child(stdin: &str) -> String { + let home = tempfile::tempdir().expect("tempdir"); + let mut child = Command::new(std::env::current_exe().unwrap()) + .args(["child_entry", "--exact", "--nocapture", "--test-threads=1"]) + .env(CHILD_ENV, "1") + .env("HOME", home.path()) + .env("NO_COLOR", "1") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn child"); + child.stdin.take().unwrap().write_all(stdin.as_bytes()).unwrap(); + let out = child.wait_with_output().expect("child output"); + assert!(out.status.success(), "child failed: {}", String::from_utf8_lossy(&out.stdout)); + String::from_utf8_lossy(&out.stdout).into_owned() +} + +#[test] +fn cost_on_a_fresh_session_is_free_with_zero_counters() { + let out = run_child("/cost\n/quit\n"); + assert!(out.contains("(local inference)") && !out.contains("% )"), "{out}"); + assert!(out.contains("Tokens: 0 in / 0 out"), "{out}"); + assert!(out.contains("Turns: 0, Tool calls: 0"), "{out}"); +} + +#[test] +fn cost_and_context_count_the_turn_that_ran() { + let out = run_child("hello\n/cost\n/context\n/quit\n"); + assert!(out.contains("Tokens: 10 in / 5 out"), "{out}"); + assert!(out.contains("Turns: 1, Tool calls: 0"), "{out}"); + assert!(out.contains("History: 2 messages (1 user, 1 assistant, 0 tool)"), "{out}"); +} + +#[test] +fn context_on_empty_history_reports_zero_and_no_limit_warning() { + let out = run_child("/context\n/quit\n"); + assert!(out.contains("History: 0 messages (0 user, 0 assistant, 0 tool)"), "{out}"); + assert!(out.contains("Context: ~0 / 32768 tokens (0%)"), "{out}"); + assert!(!out.contains("Near context limit"), "{out}"); +} + +#[test] +fn clear_empties_the_history() { + let out = run_child("hello\n/clear\n/context\n/quit\n"); + assert!(out.contains("Screen and conversation history cleared."), "{out}"); + let after = out.rsplit("cleared.").next().unwrap(); + assert!(after.contains("History: 0 messages"), "{after}"); +} + +#[test] +fn compact_reports_before_and_after_message_counts() { + let out = run_child("/compact\n/quit\n"); + assert!(out.contains("Compacted: 0 -> 0 messages"), "{out}"); +} + +#[test] +fn session_reports_the_id_and_message_count() { + let out = run_child("hello\n/session\n/quit\n"); + assert!(out.contains("Turns: 1, Messages: 2"), "{out}"); + assert!(out.matches("Session: ").count() >= 2, "banner and /session both print the id:\n{out}"); +} + +#[test] +fn every_placeholder_command_prints_its_own_message() { + let cases = [ + ("/model", "Model switching not yet implemented."), + ("/mcp", "Project-root .mcp.json loader: PMAT-CODE-MCP-JSON-LOADER-001"), + ("/config", "User-global ladder tracked in PMAT-CODE-CONFIG-LADDER-001."), + ("/review", "tracked by PMAT-CODE-REVIEW-001."), + ("/memory", "/memory TUI: PMAT-CODE-MEMORY-TUI-001."), + ("/permissions", "tracked by PMAT-CODE-PERMISSIONS-001."), + ("/hooks", "tracked by PMAT-CODE-HOOKS-001."), + ("/init", "tracked by PMAT-CODE-INIT-001."), + ("/resume", "CLI `apr code --resume [id]` works today."), + ("/add-dir", "tracked by PMAT-CODE-ADDDIR-001."), + ("/agents", "tracked by PMAT-CODE-CUSTOM-AGENTS-001."), + ("/debug", "tracked by PMAT-CODE-SLASH-DEBUG-001"), + ("/rename", "tracked by PMAT-CODE-SLASH-RENAME-001"), + ("/upgrade", "tracked by PMAT-CODE-SLASH-UPGRADE-001"), + ]; + let script: String = + cases.iter().map(|(c, _)| format!("{c}\n")).collect::() + "/quit\n"; + let out = run_child(&script); + for (cmd, expect) in cases { + assert!(out.contains(expect), "{cmd}: missing {expect:?} in:\n{out}"); + } +} + +#[test] +fn an_unknown_command_is_named_and_does_not_start_a_turn() { + let out = run_child("/bogus\n/quit\n"); + assert!(out.contains("Unknown command: /bogus. Type /help for commands."), "{out}"); + assert!(!out.contains("Session Summary"), "{out}"); +} + +#[test] +fn quit_prints_goodbye_and_ends_the_loop() { + let out = run_child("/quit\n/cost\n"); + assert!(out.contains("Goodbye."), "{out}"); + assert!(!out.contains("Cost:"), "input after /quit must not be read:\n{out}"); +} diff --git a/crates/aprender-present-widgets/src/model_card.rs b/crates/aprender-present-widgets/src/model_card.rs index 3d1709521a..33014228ac 100644 --- a/crates/aprender-present-widgets/src/model_card.rs +++ b/crates/aprender-present-widgets/src/model_card.rs @@ -1354,4 +1354,130 @@ mod tests { let card = ModelCard::new("test").parameters(1_000_000_000); assert_eq!(card.formatted_parameters(), Some("1.0B".to_string())); } + + // ===== paint (FALSIFY-MODELCARD-PAINT-001..006) ===== + + use presentar_core::draw::DrawCommand; + use presentar_core::RecordingCanvas; + + fn painted(card: &mut ModelCard) -> Vec { + card.layout(Rect::new(10.0, 20.0, 400.0, 300.0)); + let mut canvas = RecordingCanvas::new(); + card.paint(&mut canvas); + canvas.take_commands() + } + + fn texts(cmds: &[DrawCommand]) -> Vec { + cmds.iter() + .filter_map(|c| match c { + DrawCommand::Text { content, .. } => Some(content.clone()), + _ => None, + }) + .collect() + } + + fn rect_count(cmds: &[DrawCommand]) -> usize { + cmds.iter() + .filter(|c| matches!(c, DrawCommand::Rect { .. })) + .count() + } + + #[test] + fn test_paint_bare_card_draws_chrome_badge_title_version() { + let mut card = ModelCard::new("bert") + .version("2.1") + .status(ModelStatus::Published); + let cmds = painted(&mut card); + // background + border + badge rect, then 3 texts + assert_eq!(rect_count(&cmds), 3); + assert_eq!(texts(&cmds), vec!["Published", "bert", "v2.1"]); + } + + #[test] + fn test_paint_badge_is_anchored_right_with_status_color() { + let mut card = ModelCard::new("m").status(ModelStatus::Review); + let cmds = painted(&mut card); + match &cmds[2] { + DrawCommand::Rect { bounds, style, .. } => { + assert_eq!(bounds.x, 10.0 + 400.0 - 80.0); + assert_eq!(bounds.y, 20.0 + 16.0); + assert_eq!(style.fill, Some(ModelStatus::Review.color())); + } + other => panic!("expected badge rect, got {other:?}"), + } + } + + #[test] + fn test_paint_description_shifts_following_rows_by_24() { + let y_of = |cmds: &[DrawCommand], want: &str| { + cmds.iter().find_map(|c| match c { + DrawCommand::Text { + content, position, .. + } if content == want => Some(position.y), + _ => None, + }) + }; + let mut plain = ModelCard::new("m").framework("pt"); + let mut described = ModelCard::new("m").framework("pt").description("hello"); + let a = y_of(&painted(&mut plain), "pt").unwrap(); + let b_cmds = painted(&mut described); + let b = y_of(&b_cmds, "pt").unwrap(); + assert!(texts(&b_cmds).contains(&"hello".to_string())); + assert_eq!(b - a, 24.0); + } + + #[test] + fn test_paint_framework_task_join_variants() { + let mut both = ModelCard::new("m").framework("pytorch").task("nlp"); + assert!(texts(&painted(&mut both)).contains(&"pytorch • nlp".to_string())); + let mut f = ModelCard::new("m").framework("pytorch"); + assert!(texts(&painted(&mut f)).contains(&"pytorch".to_string())); + let mut t = ModelCard::new("m").task("nlp"); + assert!(texts(&painted(&mut t)).contains(&"nlp".to_string())); + let mut none = ModelCard::new("m"); + assert_eq!(texts(&painted(&mut none)).len(), 3); + } + + #[test] + fn test_paint_parameters_row() { + let mut card = ModelCard::new("m").parameters(1_500_000); + assert!(texts(&painted(&mut card)).contains(&"Parameters: 1.5M".to_string())); + } + + #[test] + fn test_paint_metrics_cap_at_four_and_toggle() { + let metrics: Vec = (0..6) + .map(|i| ModelMetric::new(format!("m{i}"), 0.5)) + .collect(); + let mut card = ModelCard::new("x").metrics(metrics.clone()); + let t = texts(&painted(&mut card)); + assert!(t.contains(&"m3".to_string())); + assert!(!t.contains(&"m4".to_string())); + assert_eq!(t.iter().filter(|s| *s == "50.00%").count(), 4); + + let mut off = ModelCard::new("x") + .metrics(metrics) + .show_metrics_chart(false); + assert!(!texts(&painted(&mut off)).contains(&"m0".to_string())); + } + + #[test] + fn test_paint_tags_cap_at_five_with_width_from_len() { + let mut card = ModelCard::new("x").tags(["aa", "bbbb", "c", "d", "e", "f"]); + let cmds = painted(&mut card); + let t = texts(&cmds); + assert!(t.contains(&"e".to_string())); + assert!(!t.contains(&"f".to_string())); + // chrome (3) + 5 tag backgrounds + assert_eq!(rect_count(&cmds), 8); + let widths: Vec = cmds + .iter() + .filter_map(|c| match c { + DrawCommand::Rect { bounds, .. } if bounds.height == 18.0 => Some(bounds.width), + _ => None, + }) + .collect(); + assert_eq!(widths[0], 2.0 * 6.0 + 12.0); + assert_eq!(widths[1], 4.0 * 6.0 + 12.0); + } } diff --git a/crates/aprender-profile/tests/run_config_errors_5713.rs b/crates/aprender-profile/tests/run_config_errors_5713.rs new file mode 100644 index 0000000000..30341ac844 --- /dev/null +++ b/crates/aprender-profile/tests/run_config_errors_5713.rs @@ -0,0 +1,51 @@ +//! `run` rejects bad configuration before it traces anything, with a message that +//! names the offending flag (#5713). No ptrace is needed: every case fails early. +#![allow(deprecated)] // suppress assert_cmd::Command::cargo_bin deprecation in tests + +use predicates::prelude::*; + +fn profile(args: &[&str]) -> assert_cmd::assert::Assert { + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("aprender-profile"); + cmd.args(args).args(["--", "/bin/true"]); + cmd.assert() +} + +#[test] +fn ml_clusters_below_two_is_rejected_with_the_value() { + profile(&["--ml-clusters", "1"]) + .code(1) + .stderr(predicate::str::contains("Invalid value for --ml-clusters: 1 (must be >= 2)")); +} + +#[test] +fn missing_transpiler_map_names_the_path() { + profile(&["--transpiler-map", "/nonexistent/map.json"]) + .code(1) + .stderr(predicate::str::contains("Source map file not found: /nonexistent/map.json")); +} + +#[test] +fn malformed_transpiler_map_is_an_error_not_an_empty_map() { + let dir = tempfile::tempdir().expect("tempdir"); + let map = dir.path().join("bad.json"); + std::fs::write(&map, "{bad").expect("write"); + profile(&["--transpiler-map", map.to_str().expect("utf8")]) + .code(1) + .stderr(predicate::str::contains("Invalid source map JSON")); +} + +#[test] +fn unknown_chaos_preset_lists_the_valid_ones() { + profile(&["--chaos", "nosuchpreset"]) + .code(1) + .stderr(predicate::str::contains("Chaos config error")) + .stderr(predicate::str::contains("gentle")) + .stderr(predicate::str::contains("aggressive")); +} + +#[test] +fn unparsable_chaos_memory_limit_is_rejected() { + profile(&["--chaos-memory-limit", "zzz"]) + .code(1) + .stderr(predicate::str::contains("Chaos config error")); +} diff --git a/crates/aprender-serve/tests/qwen35_load_failclosed_5678.rs b/crates/aprender-serve/tests/qwen35_load_failclosed_5678.rs new file mode 100644 index 0000000000..b6f4fac9ea --- /dev/null +++ b/crates/aprender-serve/tests/qwen35_load_failclosed_5678.rs @@ -0,0 +1,74 @@ +//! `load_qwen35_layers` fails closed on a GGUF that does not describe its depth or +//! carry its tensors, instead of returning a truncated model (#5678). +//! Header-only GGUFs are hand-built so no model file is needed. + +use realizar::gguf::qwen35_load::load_qwen35_layers; +use realizar::gguf::GGUFModel; + +fn gguf_string(s: &str) -> Vec { + let mut v = (s.len() as u64).to_le_bytes().to_vec(); + v.extend_from_slice(s.as_bytes()); + v +} + +/// GGUF v3, zero tensors, `arch` plus optional `{arch}.block_count`. +fn header_only(arch: &str, block_count: Option) -> Vec { + let mut kvs = Vec::new(); + kvs.extend(gguf_string("general.architecture")); + kvs.extend(8u32.to_le_bytes()); + kvs.extend(gguf_string(arch)); + let mut n = 1u64; + if let Some(bc) = block_count { + kvs.extend(gguf_string(&format!("{arch}.block_count"))); + kvs.extend(4u32.to_le_bytes()); + kvs.extend(bc.to_le_bytes()); + n += 1; + } + let mut out = b"GGUF".to_vec(); + out.extend(3u32.to_le_bytes()); + out.extend(0u64.to_le_bytes()); + out.extend(n.to_le_bytes()); + out.extend(kvs); + out +} + +fn load(data: &[u8]) -> Result { + let model = GGUFModel::from_bytes(data).map_err(|e| format!("parse: {e}"))?; + load_qwen35_layers(&model, data) + .map(|l| l.len()) + .map_err(|e| e.to_string()) +} + +#[test] +fn missing_block_count_is_refused_not_guessed() { + let err = load(&header_only("qwen35", None)).expect_err("no depth, no model"); + assert!( + err.contains("block_count is missing"), + "error must name the missing key: {err}" + ); + assert!( + err.contains("qwen35"), + "error must name the architecture: {err}" + ); +} + +#[test] +fn zero_layers_loads_as_empty_not_an_error() { + assert_eq!(load(&header_only("qwen35", Some(0))).expect("empty"), 0); +} + +#[test] +fn declared_layer_without_its_tensors_is_an_error() { + // block_count = 1 -> layer 0 is DeltaNet (interval 4); its tensors are absent. + let err = load(&header_only("qwen35", Some(1))).expect_err("tensors absent"); + assert!( + !err.contains("block_count"), + "failure is the tensor, not the depth: {err}" + ); +} + +#[test] +fn full_attention_layer_without_tensors_is_an_error() { + // block_count = 4 -> the 4th layer is full attention, but even layer 0 is missing. + assert!(load(&header_only("qwen35", Some(4))).is_err()); +} diff --git a/crates/aprender-train/src/finetune/classify_trainer_tests.rs b/crates/aprender-train/src/finetune/classify_trainer_tests.rs index e07109a10a..a12dc6f423 100644 --- a/crates/aprender-train/src/finetune/classify_trainer_tests.rs +++ b/crates/aprender-train/src/finetune/classify_trainer_tests.rs @@ -5099,3 +5099,78 @@ fn test_coverage_card_confusion_raw_formatting() { assert!(out.contains("```")); assert!(out.contains("Predicted")); } + +// #5686: ClassifyTrainer::train, single-process path. +fn train_5686(epochs: usize, patience: usize, tag: &str) -> (TrainResult, PathBuf) { + let dir = std::env::temp_dir().join(format!("aprender_5686_{tag}_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let config = TrainingConfig { + epochs, + val_split: 0.3, + early_stopping_patience: patience, + checkpoint_dir: dir.clone(), + warmup_fraction: 0.0, + ..TrainingConfig::default() + }; + let mut trainer = + ClassifyTrainer::new(tiny_pipeline(2), make_corpus(20, 2), config).expect("valid"); + (trainer.train(), dir) +} + +#[test] +fn train_zero_patience_stops_after_first_epoch_5686() { + let (result, dir) = train_5686(4, 0, "patience0"); + assert!(result.stopped_early, "patience 0 must stop at once"); + assert_eq!(result.epoch_metrics.len(), 1); + let _ = std::fs::remove_dir_all(dir); +} + +#[test] +fn train_runs_every_epoch_when_patience_is_not_reached_5686() { + let (result, dir) = train_5686(2, 100, "full"); + assert!(!result.stopped_early); + assert_eq!(result.epoch_metrics.len(), 2); + let want = result + .epoch_metrics + .iter() + .filter(|m| m.val_loss.is_finite()) + .min_by(|a, b| a.val_loss.total_cmp(&b.val_loss)) + .expect("a finite epoch"); + assert_eq!(result.best_epoch, want.epoch, "best_epoch is the lowest val_loss"); + assert_eq!(result.best_val_loss, want.val_loss); + for (i, m) in result.epoch_metrics.iter().enumerate() { + assert_eq!(m.epoch, i, "metrics are recorded in epoch order"); + } + let _ = std::fs::remove_dir_all(dir); +} + +#[test] +fn train_saves_best_and_periodic_checkpoints_5686() { + // epochs (2) <= save_every (5) => a checkpoint every epoch, plus `best`. + let (_result, dir) = train_5686(2, 100, "ckpt"); + assert!(dir.join("best").join("metadata.json").exists(), "best checkpoint"); + assert!(dir.join("epoch-0").join("metadata.json").exists(), "epoch-0 checkpoint"); + assert!(dir.join("epoch-1").join("metadata.json").exists(), "epoch-1 checkpoint"); + let _ = std::fs::remove_dir_all(dir); +} + +#[test] +fn train_with_monitor_writer_completes_the_state_file_5686() { + let dir = std::env::temp_dir().join(format!("aprender_5686_mon_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let config = + TrainingConfig { epochs: 1, checkpoint_dir: dir.join("ckpt"), ..TrainingConfig::default() }; + let mut trainer = + ClassifyTrainer::new(tiny_pipeline(2), make_corpus(20, 2), config).expect("valid"); + let writer = crate::monitor::tui::TrainingStateWriter::new(&dir, "exp-5686", "tiny"); + trainer.set_monitor_writer(writer); + let result = trainer.train(); + assert_eq!(result.epoch_metrics.len(), 1); + let state: Vec<_> = std::fs::read_dir(&dir) + .expect("monitor dir") + .filter_map(Result::ok) + .filter(|e| e.path().extension().is_some_and(|x| x == "json")) + .collect(); + assert!(!state.is_empty(), "the monitor wrote a state file"); + let _ = std::fs::remove_dir_all(dir); +} diff --git a/crates/aprender-train/tests/execute_plan_guards_5674.rs b/crates/aprender-train/tests/execute_plan_guards_5674.rs new file mode 100644 index 0000000000..da5246e268 --- /dev/null +++ b/crates/aprender-train/tests/execute_plan_guards_5674.rs @@ -0,0 +1,102 @@ +//! `execute_plan` refuses to start when a precondition fails, and says which (#5674). +//! These guards run before any weight load, so they need no model or GPU. + +use entrenar::finetune::{ + execute_plan, training_plan, ApplyConfig, PlanConfig, PlanVerdict, TrainingPlan, +}; +use entrenar::Error; +use std::path::Path; + +fn make_plan(dir: &Path) -> TrainingPlan { + let data = dir.join("train.jsonl"); + std::fs::write( + &data, + "{\"input\":\"echo hi\",\"label\":0}\n{\"input\":\"rm -rf /\",\"label\":1}\n", + ) + .expect("write corpus"); + let cfg = PlanConfig { + task: "classify".to_string(), + data_path: data, + val_path: None, + test_path: None, + model_size: "0.5B".to_string(), + model_path: None, + num_classes: 2, + output_dir: dir.join("out"), + strategy: "manual".to_string(), + budget: 1, + scout: true, + max_epochs: 1, + manual_lr: Some(1e-4), + manual_lora_rank: Some(4), + manual_batch_size: Some(2), + manual_lora_alpha: None, + manual_warmup: None, + manual_gradient_clip: None, + manual_lr_min_ratio: None, + manual_class_weights: None, + manual_target_modules: None, + }; + training_plan(&cfg).expect("plan builds from a valid corpus") +} + +fn apply(model: &Path, data: &Path, out: &Path) -> ApplyConfig { + ApplyConfig { + model_path: model.to_path_buf(), + data_path: data.to_path_buf(), + output_dir: out.to_path_buf(), + on_trial_complete: None, + } +} + +fn config_msg(e: Error) -> String { + match e { + Error::ConfigError(m) | Error::Io(m) => m, + other => format!("unexpected variant: {other}"), + } +} + +#[test] +fn blocked_plan_is_refused_before_touching_the_filesystem() { + let tmp = tempfile::tempdir().expect("tempdir"); + let mut plan = make_plan(tmp.path()); + plan.verdict = PlanVerdict::Blocked; + let out = tmp.path().join("apply-out"); + let a = apply(tmp.path(), &tmp.path().join("train.jsonl"), &out); + let err = execute_plan(&plan, &a).expect_err("blocked plan must not run"); + assert!(config_msg(err).contains("blocked plan")); + assert!(!out.exists(), "no output dir for a refused plan"); +} + +#[test] +fn missing_model_dir_is_a_config_error_naming_the_path() { + let tmp = tempfile::tempdir().expect("tempdir"); + let plan = make_plan(tmp.path()); + let missing = tmp.path().join("no-such-model"); + let a = apply(&missing, &tmp.path().join("train.jsonl"), &tmp.path().join("o")); + let err = execute_plan(&plan, &a).expect_err("missing model dir"); + assert!(matches!(err, Error::ConfigError(_))); + assert!(config_msg(execute_plan(&plan, &a).expect_err("again")).contains("no-such-model")); +} + +#[test] +fn missing_training_data_is_an_io_error() { + let tmp = tempfile::tempdir().expect("tempdir"); + let plan = make_plan(tmp.path()); + let a = apply(tmp.path(), &tmp.path().join("gone.jsonl"), &tmp.path().join("o")); + let err = execute_plan(&plan, &a).expect_err("missing data"); + assert!(matches!(err, Error::Io(_)), "data absence is Io, not Config"); + assert!(config_msg(err).contains("gone.jsonl")); +} + +#[test] +fn unknown_model_size_is_rejected_not_silently_defaulted() { + let tmp = tempfile::tempdir().expect("tempdir"); + let mut plan = make_plan(tmp.path()); + plan.model.size = "not-a-size".to_string(); + let out = tmp.path().join("o"); + let a = apply(tmp.path(), &tmp.path().join("train.jsonl"), &out); + let err = execute_plan(&plan, &a).expect_err("GH-377: unknown size must error"); + assert!(matches!(err, Error::ConfigError(_))); + assert!(out.is_dir(), "output dir is created before the size check"); +}