From 7545eb3500ad09ea9c6f3802a8deca604142e89e Mon Sep 17 00:00:00 2001 From: almondsun Date: Sun, 12 Jul 2026 23:05:43 -0500 Subject: [PATCH] Add multi-seed robustness study --- AGENTS.md | 8 +- README.md | 1 + ...12_5k_lr1e-3_ctx37_earlystop_seed2027.yaml | 38 +++++ ...12_5k_lr1e-3_ctx37_earlystop_seed4242.yaml | 38 +++++ docs/codex/build-and-test.md | 2 +- docs/codex/experiments.md | 4 + docs/experiments.md | 6 + docs/training.md | 12 ++ experiments/023-multi-seed-robustness.md | 91 ++++++++++++ notes/06-reproducibility.md | 21 +++ scripts/summarize_runs.py | 139 ++++++++++++++++++ src/smallm/config.py | 6 + src/smallm/evaluation/robustness.py | 61 ++++++++ tests/test_config.py | 3 + tests/test_robustness.py | 55 +++++++ tests/test_summarize_runs.py | 88 +++++++++++ 16 files changed, 568 insertions(+), 5 deletions(-) create mode 100644 configs/gptiny_bytebpe512_5k_lr1e-3_ctx37_earlystop_seed2027.yaml create mode 100644 configs/gptiny_bytebpe512_5k_lr1e-3_ctx37_earlystop_seed4242.yaml create mode 100644 experiments/023-multi-seed-robustness.md create mode 100644 scripts/summarize_runs.py create mode 100644 src/smallm/evaluation/robustness.py create mode 100644 tests/test_robustness.py create mode 100644 tests/test_summarize_runs.py diff --git a/AGENTS.md b/AGENTS.md index 4990c8f..709c3db 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -90,10 +90,10 @@ If a relevant check cannot be run, state why and what remains unverified. ## Current Technical Status -Milestone 022 is the latest modeling evidence. Patience-3 validation early stopping reproduces -ByteBPE512's best full-validation BPC `2.0083`, stops at step 2,500, and roughly halves runtime. -Weight decay `0.01` reaches `2.0080`, too small a difference to claim improvement. Future work should -test the tokenizer result across seeds or corpus splits rather than add another narrow hyperparameter. +Milestone 023 is the latest modeling evidence. Across preregistered seeds 1337, 2027, and 4242, +ByteBPE512 early stopping reaches best full-validation BPC `2.0225 ± 0.0124` (population SD), range +`2.0083–2.0384`; all seeds beat the character control. Future work should test split or corpus +robustness rather than select a favorable seed or add another narrow hyperparameter. ## Safety And Security diff --git a/README.md b/README.md index d6359a2..9ac8cd0 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,7 @@ chronological 90/10 split unless noted otherwise. | [020](experiments/020-bpe-context-and-learning-rate.md) | BPE context/LR controls | Best BPE128 bits/character | `2.0976` remains best | Matched character context and `5e-4` LR changed timing/diversity but did not improve held-out BPC. | | [021](experiments/021-boundary-aware-byte-bpe.md) | Boundary-aware ByteBPE320/512 | Full-validation best bits/character | `2.0286` / `2.0083` | Both beat the corrected character and BPE128 controls; ByteBPE512 overfit after step 1750. | | [022](experiments/022-early-stopping-and-regularization.md) | ByteBPE512 early stopping / weight decay | Actual steps and best BPC | `2500`, `2.0083` / `2.0080` | Early stopping halves runtime and limits final degradation; weight decay `0.01` is effectively neutral. | +| [023](experiments/023-multi-seed-robustness.md) | ByteBPE512 across three seeds | Best BPC mean ± population SD | `2.0225 ± 0.0124` | All tested seeds beat the character control; stopping varies from 2500–3000 steps. | Token-level loss and perplexity are not directly comparable between character and BPE tokenizers because they predict different units. The tokenizer diff --git a/configs/gptiny_bytebpe512_5k_lr1e-3_ctx37_earlystop_seed2027.yaml b/configs/gptiny_bytebpe512_5k_lr1e-3_ctx37_earlystop_seed2027.yaml new file mode 100644 index 0000000..d18db8e --- /dev/null +++ b/configs/gptiny_bytebpe512_5k_lr1e-3_ctx37_earlystop_seed2027.yaml @@ -0,0 +1,38 @@ +data: + input_path: data/raw/input.txt + prepared_path: data/processed/corpus.txt + manifest_path: data/processed/corpus_manifest.json + tokenizer_path: data/processed/tokenizer_bytebpe512.json + tokenizer_type: byte_bpe + bpe_vocab_size: 512 + bpe_min_frequency: 2 + block_size: 37 + train_split: 0.9 + +model: + vocab_size: 512 + block_size: 37 + n_layer: 4 + n_head: 4 + n_embd: 128 + dropout: 0.1 + +train: + run_name: gptiny_bytebpe512_5k_lr1e-3_ctx37_earlystop_seed2027 + runs_dir: runs + batch_size: 27 + max_steps: 5000 + learning_rate: 0.001 + weight_decay: 0.0 + log_interval: 100 + eval_interval: 250 + eval_batches: null + early_stopping_patience: 3 + early_stopping_min_delta: 0.0 + sample_prompt: Once + sample_max_new_tokens: 100 + sample_temperature: 1.0 + sample_top_k: null + sample_seed: 1337 + sample_greedy: false + seed: 2027 diff --git a/configs/gptiny_bytebpe512_5k_lr1e-3_ctx37_earlystop_seed4242.yaml b/configs/gptiny_bytebpe512_5k_lr1e-3_ctx37_earlystop_seed4242.yaml new file mode 100644 index 0000000..39316cb --- /dev/null +++ b/configs/gptiny_bytebpe512_5k_lr1e-3_ctx37_earlystop_seed4242.yaml @@ -0,0 +1,38 @@ +data: + input_path: data/raw/input.txt + prepared_path: data/processed/corpus.txt + manifest_path: data/processed/corpus_manifest.json + tokenizer_path: data/processed/tokenizer_bytebpe512.json + tokenizer_type: byte_bpe + bpe_vocab_size: 512 + bpe_min_frequency: 2 + block_size: 37 + train_split: 0.9 + +model: + vocab_size: 512 + block_size: 37 + n_layer: 4 + n_head: 4 + n_embd: 128 + dropout: 0.1 + +train: + run_name: gptiny_bytebpe512_5k_lr1e-3_ctx37_earlystop_seed4242 + runs_dir: runs + batch_size: 27 + max_steps: 5000 + learning_rate: 0.001 + weight_decay: 0.0 + log_interval: 100 + eval_interval: 250 + eval_batches: null + early_stopping_patience: 3 + early_stopping_min_delta: 0.0 + sample_prompt: Once + sample_max_new_tokens: 100 + sample_temperature: 1.0 + sample_top_k: null + sample_seed: 1337 + sample_greedy: false + seed: 4242 diff --git a/docs/codex/build-and-test.md b/docs/codex/build-and-test.md index 3e6d5ea..7ad31b6 100644 --- a/docs/codex/build-and-test.md +++ b/docs/codex/build-and-test.md @@ -50,7 +50,7 @@ The individual commands remain canonical and are listed below. python -m pytest ``` -Current expected result after milestone 022+: at least 121 tests passing with at least 90% coverage. +Current expected result after milestone 023+: at least 141 tests passing with at least 90% coverage. ### Compile Check diff --git a/docs/codex/experiments.md b/docs/codex/experiments.md index fcf908d..6b55645 100644 --- a/docs/codex/experiments.md +++ b/docs/codex/experiments.md @@ -157,3 +157,7 @@ controlled question is early stopping or modest regularization rather than more Milestone 022 adds patience-3 validation early stopping. It reproduces ByteBPE512's step-1,750 best checkpoint and stops at step 2,500, roughly halving runtime. Weight decay `0.01` is effectively neutral at best BPC `2.0080`; future work should test robustness across seeds or corpus splits. + +Milestone 023 runs the unregularized ByteBPE512 early-stopping setup at seeds 1337, 2027, and 4242. +Best BPC is `2.0225 ± 0.0124` (population SD), range `2.0083–2.0384`; every seed beats the +character control. The next modeling question should test split/corpus robustness, not select a seed. diff --git a/docs/experiments.md b/docs/experiments.md index 37a8c7e..b8a315a 100644 --- a/docs/experiments.md +++ b/docs/experiments.md @@ -29,6 +29,7 @@ not a replacement for the original reports. | [020 BPE Context and Learning Rate](../experiments/020-bpe-context-and-learning-rate.md) | Tokenizer diagnostics | Matching character context and lowering BPE learning rate did not beat the corrected BPE128 control or character model. | | [021 Boundary-Aware Byte BPE](../experiments/021-boundary-aware-byte-bpe.md) | Tokenizer design | Lossless boundary-aware ByteBPE320/512 beat both corrected controls on best BPC; ByteBPE512 reached `2.0083` but overfit early. | | [022 Early Stopping and Regularization](../experiments/022-early-stopping-and-regularization.md) | Training control | Patience-3 stopping reproduced the step-1750 optimum and halved runtime; weight decay `0.01` was effectively neutral. | +| [023 Multi-Seed Robustness](../experiments/023-multi-seed-robustness.md) | Robustness | Three preregistered seeds average best BPC `2.0225 ± 0.0124`; every seed beats the corrected character control. | ## Topic Shortcuts @@ -79,3 +80,8 @@ The 512-token run's final BPC rose to `2.2450`, making early best-checkpoint sel Milestone 022 operationalizes that result. Patience-3 early stopping terminates at step 2,500 and retains best BPC `2.0083`, while stopped-final BPC improves from `2.2450` to `2.0554`. Weight decay `0.01` reaches best BPC `2.0080`; the `0.00025` difference is too small to interpret as a real gain. + +Milestone 023 measures seed sensitivity directly. Across seeds 1337, 2027, and 4242, best BPC is +`2.0225 ± 0.0124` with range `2.0083–2.0384`; best step ranges 1,750–2,250 and stop step +2,500–3,000. The tokenizer result survives all tested seeds, while the observed seed spread confirms +that milestone 022's tiny weight-decay delta was not decision-grade evidence. diff --git a/docs/training.md b/docs/training.md index ac8a606..951f4d0 100644 --- a/docs/training.md +++ b/docs/training.md @@ -117,6 +117,15 @@ the saved sample. inspectable. The final checkpoint is the model at the stop step, while `best_checkpoint.pt` remains the lowest observed validation-loss model. +Aggregate completed runs without selecting a winner: + +```bash +python scripts/summarize_runs.py runs// runs// runs// +``` + +The command reads each run's config and summary, requires distinct training seeds, and prints every +observation plus mean, population standard deviation, minimum, and maximum. + ## Generation Controls ```bash @@ -167,6 +176,9 @@ control, though ByteBPE512 overfit sharply after step 1,750. Experiment 022 adds patience-3 early stopping: it ends at step 2,500, halves runtime, and avoids most final-checkpoint degradation. Weight decay `0.01` changes best BPC by only `0.00025`, which is not meaningful evidence of improvement. +Experiment 023 repeats the unregularized early-stopping run across seeds 1337, 2027, and 4242. +Best BPC is `2.0225 ± 0.0124` (population SD), and all three runs remain better than the corrected +character control. Stop steps vary from 2,500 to 3,000. ## Artifact Policy diff --git a/experiments/023-multi-seed-robustness.md b/experiments/023-multi-seed-robustness.md new file mode 100644 index 0000000..aaf88d5 --- /dev/null +++ b/experiments/023-multi-seed-robustness.md @@ -0,0 +1,91 @@ +# 023 — Multi-Seed Robustness + +## Goal + +Measure whether ByteBPE512's advantage and early-stopping behavior survive training randomness. +Seeds 1337, 2027, and 4242 were fixed before the additional runs; all completed seeds are reported, +and no result is selected or discarded based on quality. + +## Setup + +Every run uses the milestone-022 unregularized configuration: the same training-only ByteBPE512 +tokenizer, corpus checksum `a4c81ef23eb9…`, chronological 90/10 split, 37-token context, batch 27, +4-layer/4-head/128-wide GPTiny, dropout 0.1, AdamW `lr=1e-3`, zero weight decay, full validation +every 250 steps, and patience-3 early stopping under a 5,000-step ceiling. Training seed is the only +changed field. Generation sampling holds seed 1337 fixed to isolate model-training variation. + +Population standard deviation describes this complete preregistered seed set; with only three seeds, +it is descriptive uncertainty, not a confidence interval or population estimate. +The aggregation CLI verified canonical experiment fingerprint +`ed9650e463830d0d85dbbc32a40164b6699d7033017386a2cb22f2ab035431b0` across all runs. + +## Validation Results + +| seed | actual steps | best step | best loss | best BPC | stopped-final BPC | duration | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 1337 | 2,500 | 1,750 | 2.375155 | 2.008269 | 2.055447 | 209.3s | +| 2027 | 3,000 | 2,250 | 2.410806 | 2.038414 | 2.099879 | 308.3s | +| 4242 | 2,500 | 1,750 | 2.390150 | 2.020948 | 2.052917 | 244.1s | + +| metric | mean | population SD | minimum | maximum | +| --- | ---: | ---: | ---: | ---: | +| actual steps | 2,666.67 | 235.70 | 2,500 | 3,000 | +| best step | 1,916.67 | 235.70 | 1,750 | 2,250 | +| best BPC | **2.022544** | **0.012358** | 2.008269 | 2.038414 | +| stopped-final BPC | 2.069414 | 0.021567 | 2.052917 | 2.099879 | +| duration seconds | 253.93 | 41.01 | 209.32 | 308.33 | + +All three seeds beat the corrected character control (`2.075981`) and BPE128 control (`2.097552`) +on best BPC. The worst tested ByteBPE512 seed retains a 0.03757 BPC advantage over character. The +tokenizer conclusion therefore survives this seed set, although its effect is smaller than the +single best seed suggested. + +Seed-to-seed best-BPC SD (`0.012358`) is about 49 times milestone 022's apparent weight-decay gain +(`0.000251`). That comparison strengthens the earlier conclusion that weight decay `0.01` was +effectively neutral. Early stopping is stable in direction but not timing: seed 2027 improves later +and needs 500 additional steps. + +## Controlled Generation + +Prompt `Once`, 100 new tokens. Seeded decoding uses temperature 0.8, top-k 10, seed 1337. + +| training seed | best greedy d-2 | best seeded d-2 | final greedy d-2 | final seeded d-2 | +| ---: | ---: | ---: | ---: | ---: | +| 1337 | 0.5217 | 0.5417 | 0.5723 | 0.5980 | +| 2027 | 0.5562 | 0.6836 | 0.5595 | 0.6591 | +| 4242 | 0.3657 | 0.6548 | 0.5056 | 0.5670 | +| mean | 0.4812 | 0.6267 | 0.5458 | 0.6080 | +| population SD | 0.0829 | 0.0612 | 0.0289 | 0.0383 | + +Generation varies materially across training seeds. Seeded distinct-2 is usually higher than greedy, +but neither it nor final-checkpoint diversity tracks best validation BPC. Samples remain locally +plausible and globally incoherent, so the robustness claim is limited to held-out likelihood. + +## Exact Commands + +```bash +uv run --frozen --extra dev python scripts/prepare_data.py --config configs/gptiny_bytebpe512_5k_lr1e-3_ctx37_earlystop_seed2027.yaml +uv run --frozen --extra dev python scripts/train.py --config configs/gptiny_bytebpe512_5k_lr1e-3_ctx37_earlystop_seed2027.yaml +uv run --frozen --extra dev python scripts/train.py --config configs/gptiny_bytebpe512_5k_lr1e-3_ctx37_earlystop_seed4242.yaml +uv run --frozen --extra dev python scripts/summarize_runs.py \ + runs/gptiny_bytebpe512_5k_lr1e-3_ctx37_earlystop/2026-07-12_22-12-08 \ + runs/gptiny_bytebpe512_5k_lr1e-3_ctx37_earlystop_seed2027/2026-07-12_22-37-30 \ + runs/gptiny_bytebpe512_5k_lr1e-3_ctx37_earlystop_seed4242/2026-07-12_22-43-16 +uv run --frozen --extra dev make check +``` + +Generation used `scripts/generate.py` for seeds 2027 and 4242 at `best` and `final`, once with +`--greedy --diagnostics` and once with +`--temperature 0.8 --top-k 10 --seed 1337 --diagnostics`. Seed-1337 diagnostics are the controlled +milestone-022 values. + +## Limitations And Next Step + +- Three seeds reveal variation but do not estimate a broad seed distribution precisely. +- Every seed shares one chronological validation split and one small English corpus. +- Repeated validation drives early stopping and model selection. +- Distinct-n is not semantic or human evaluation. + +The next strong test changes the data axis: evaluate the fixed ByteBPE512 early-stopping protocol on +multiple deterministic corpus splits or an additional public-domain corpus. That would test whether +the tokenizer advantage is distribution-robust rather than merely seed-robust. diff --git a/notes/06-reproducibility.md b/notes/06-reproducibility.md index e9eeb31..dcbf78b 100644 --- a/notes/06-reproducibility.md +++ b/notes/06-reproducibility.md @@ -60,3 +60,24 @@ Implementation: [`artifacts.py`](../src/smallm/training/artifacts.py), Checks: reconstruct a result from its summary; identify which failures occur before any run directory exists; explain why old and corrected BPC values cannot be compared as one series. +### Seed ensembles and descriptive uncertainty + +A random seed fixes initialization, minibatch order, dropout masks, and sampling streams; it is an +experimental condition, not a hyperparameter to optimize. For preregistered seeds +(s_1,\ldots,s_n) and metric (x_i), report every observation plus + +\[ +\bar{x}=\frac1n\sum_{i=1}^n x_i, +\qquad +\sigma_{\mathrm{pop}}=\sqrt{\frac1n\sum_{i=1}^n(x_i-\bar{x})^2}. +\] + +smaLLM uses population standard deviation because the report describes the complete, explicitly +chosen seed set; it does not pretend three seeds estimate a universal sampling distribution. The +minimum and maximum expose asymmetry that a mean and deviation can hide. Stop step is itself a +random outcome under early stopping and must be summarized alongside model quality. + +Never discard a completed seed because it weakens the conclusion, and never report the best seed as +the expected result. A hyperparameter difference much smaller than seed-to-seed spread is not robust +evidence. Decoding randomness is held fixed when comparing training seeds so observed generation +variation comes from model training rather than a second uncontrolled random stream. diff --git a/scripts/summarize_runs.py b/scripts/summarize_runs.py new file mode 100644 index 0000000..8be1e73 --- /dev/null +++ b/scripts/summarize_runs.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +from dataclasses import asdict +from math import isfinite +from pathlib import Path +from typing import Any + +from smallm.config import load_config +from smallm.evaluation.robustness import RunObservation, summarize_observations + +_MAX_RUNS = 32 +_MAX_ARTIFACT_BYTES = 1_000_000 + + +def _read_bounded_text(path: Path) -> str: + if not path.is_file(): + raise ValueError(f"required run artifact is missing: {path}") + if path.stat().st_size > _MAX_ARTIFACT_BYTES: + raise ValueError(f"run artifact exceeds {_MAX_ARTIFACT_BYTES} bytes: {path}") + return path.read_text(encoding="utf-8") + + +def _load_summary(run_dir: Path) -> dict[str, Any]: + path = run_dir / "summary.json" + try: + payload = json.loads(_read_bounded_text(path)) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise ValueError(f"invalid run summary at {path}: {exc}") from exc + if not isinstance(payload, dict): + raise ValueError(f"run summary must be a mapping: {path}") + if payload.get("schema_version") != 2 or payload.get("status") != "complete": + raise ValueError(f"run summary is not a complete schema-v2 artifact: {path}") + return payload + + +def _required_float(summary: dict[str, Any], field: str, *, positive: bool = False) -> float: + value = summary.get(field) + if not isinstance(value, int | float) or isinstance(value, bool): + raise ValueError(f"run summary field {field!r} must be numeric") + if isinstance(value, int) and abs(value) > 1_000_000_000: + raise ValueError(f"run summary field {field!r} is outside supported bounds") + try: + number = float(value) + except OverflowError as exc: + raise ValueError(f"run summary field {field!r} is outside supported bounds") from exc + if not isfinite(number) or number < 0 or (positive and number == 0) or number > 1_000_000_000: + raise ValueError(f"run summary field {field!r} is outside supported bounds") + return number + + +def _required_int(summary: dict[str, Any], field: str) -> int: + value = summary.get(field) + if not isinstance(value, int) or isinstance(value, bool) or value < 0 or value > 1_000_000_000: + raise ValueError(f"run summary field {field!r} must be a bounded non-negative integer") + return value + + +def _comparison_fingerprint(config: Any, summary: dict[str, Any]) -> str: + config_state = asdict(config) + train_state = config_state["train"] + for field in ("run_name", "runs_dir", "seed"): + train_state.pop(field) + dataset = summary.get("dataset") + if not isinstance(dataset, dict) or not isinstance(dataset.get("prepared_sha256"), str): + raise ValueError("run summary dataset must contain prepared_sha256") + identity = { + "config": config_state, + "dataset": { + "prepared_sha256": dataset["prepared_sha256"], + "train_split": dataset.get("train_split"), + "train_characters": dataset.get("train_characters"), + "validation_characters": dataset.get("validation_characters"), + }, + "summary_schema_version": summary["schema_version"], + "validation_mode": summary.get("validation_mode"), + "validation_coverage": summary.get("validation_coverage"), + "tokenizer_type": summary.get("tokenizer_type"), + "tokenizer_vocab_size": summary.get("tokenizer_vocab_size"), + } + encoded = json.dumps(identity, sort_keys=True, allow_nan=False).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _observation(run_dir: Path) -> RunObservation: + summary = _load_summary(run_dir) + config_path = run_dir / "config.yaml" + try: + _read_bounded_text(config_path) + config = load_config(config_path) + except (OSError, UnicodeError, TypeError, AttributeError, ValueError) as exc: + raise ValueError(f"invalid run config at {config_path}: {exc}") from exc + actual_steps = _required_int(summary, "actual_steps") + best_step = _required_int(summary, "best_val_step") + if best_step > actual_steps: + raise ValueError("run summary best_val_step cannot exceed actual_steps") + return RunObservation( + seed=config.train.seed, + actual_steps=actual_steps, + best_step=best_step, + best_bpc=_required_float(summary, "best_val_bits_per_char", positive=True), + final_bpc=_required_float(summary, "final_val_bits_per_char", positive=True), + duration_seconds=_required_float(summary, "duration_seconds"), + comparison_fingerprint=_comparison_fingerprint(config, summary), + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("runs", nargs="+", type=Path) + args = parser.parse_args() + if not 2 <= len(args.runs) <= _MAX_RUNS: + parser.error(f"provide between 2 and {_MAX_RUNS} run directories") + observations = [_observation(run_dir) for run_dir in args.runs] + summaries = summarize_observations(observations) + + print(f"verified_experiment_fingerprint_sha256 {observations[0].comparison_fingerprint}") + print("standard_deviation population_descriptive") + print() + print("seed actual_steps best_step best_bpc final_bpc duration_seconds") + for observation in observations: + print( + f"{observation.seed} {observation.actual_steps} {observation.best_step} " + f"{observation.best_bpc:.6f} {observation.final_bpc:.6f} " + f"{observation.duration_seconds:.1f}" + ) + print() + print("metric mean population_stddev min max") + for metric, summary in summaries.items(): + print( + f"{metric} {summary.mean:.6f} {summary.population_stddev:.6f} " + f"{summary.minimum:.6f} {summary.maximum:.6f}" + ) + + +if __name__ == "__main__": + main() diff --git a/src/smallm/config.py b/src/smallm/config.py index a2ecb9d..f17a807 100644 --- a/src/smallm/config.py +++ b/src/smallm/config.py @@ -136,6 +136,12 @@ def __post_init__(self) -> None: raise ValueError("train.sample_temperature must be finite and positive") if self.sample_top_k is not None and self.sample_top_k <= 0: raise ValueError("train.sample_top_k must be positive or null") + if ( + not isinstance(self.seed, int) + or isinstance(self.seed, bool) + or not 0 <= self.seed <= 9_223_372_036_854_775_807 + ): + raise ValueError("train.seed must be an integer between 0 and 2^63-1") @dataclass(frozen=True) diff --git a/src/smallm/evaluation/robustness.py b/src/smallm/evaluation/robustness.py new file mode 100644 index 0000000..6ea3360 --- /dev/null +++ b/src/smallm/evaluation/robustness.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from dataclasses import dataclass +from math import isfinite +from statistics import fmean, pstdev + + +@dataclass(frozen=True) +class RunObservation: + seed: int + actual_steps: int + best_step: int + best_bpc: float + final_bpc: float + duration_seconds: float + comparison_fingerprint: str = "test-fixture" + + +@dataclass(frozen=True) +class DistributionSummary: + mean: float + population_stddev: float + minimum: float + maximum: float + + +def summarize_values(values: list[float]) -> DistributionSummary: + if not values: + raise ValueError("at least one value is required") + if not all(isfinite(value) for value in values): + raise ValueError("summary values must be finite") + return DistributionSummary( + mean=fmean(values), + population_stddev=pstdev(values), + minimum=min(values), + maximum=max(values), + ) + + +def summarize_observations( + observations: list[RunObservation], +) -> dict[str, DistributionSummary]: + if len(observations) < 2: + raise ValueError("at least two run observations are required") + if len({observation.seed for observation in observations}) != len(observations): + raise ValueError("run observations must have distinct seeds") + if len({observation.comparison_fingerprint for observation in observations}) != 1: + raise ValueError("run observations do not share an experiment fingerprint") + return { + "actual_steps": summarize_values( + [float(observation.actual_steps) for observation in observations] + ), + "best_step": summarize_values( + [float(observation.best_step) for observation in observations] + ), + "best_bpc": summarize_values([observation.best_bpc for observation in observations]), + "final_bpc": summarize_values([observation.final_bpc for observation in observations]), + "duration_seconds": summarize_values( + [observation.duration_seconds for observation in observations] + ), + } diff --git a/tests/test_config.py b/tests/test_config.py index efb38ab..4868ce7 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -114,6 +114,9 @@ def test_data_config_rejects_invalid_bpe_vocab_size(): (lambda: TrainConfig(sample_temperature=0), "sample_temperature"), (lambda: TrainConfig(sample_temperature=float("inf")), "sample_temperature"), (lambda: TrainConfig(sample_top_k=0), "sample_top_k"), + (lambda: TrainConfig(seed=True), "train.seed"), + (lambda: TrainConfig(seed=-1), "train.seed"), + (lambda: TrainConfig(seed=2**63), "train.seed"), ( lambda: ExperimentConfig( data=DataConfig(block_size=8), model=ModelConfig(block_size=4) diff --git a/tests/test_robustness.py b/tests/test_robustness.py new file mode 100644 index 0000000..6c9352f --- /dev/null +++ b/tests/test_robustness.py @@ -0,0 +1,55 @@ +import pytest + +from smallm.evaluation.robustness import RunObservation, summarize_observations, summarize_values + + +def test_summarize_values_reports_population_statistics(): + summary = summarize_values([1.0, 2.0, 3.0]) + + assert summary.mean == 2.0 + assert summary.population_stddev == pytest.approx(0.81649658) + assert summary.minimum == 1.0 + assert summary.maximum == 3.0 + + +def test_summarize_values_rejects_empty_or_non_finite_input(): + with pytest.raises(ValueError, match="at least one"): + summarize_values([]) + with pytest.raises(ValueError, match="finite"): + summarize_values([float("nan")]) + + +def test_summarize_observations_requires_distinct_seeds(): + observation = RunObservation(1, 10, 5, 2.0, 2.1, 3.0) + + with pytest.raises(ValueError, match="distinct seeds"): + summarize_observations([observation, observation]) + + +def test_summarize_observations_requires_multiple_comparable_runs(): + one = RunObservation(1, 10, 5, 2.0, 2.1, 3.0, "same") + other = RunObservation(2, 10, 5, 2.0, 2.1, 3.0, "different") + + with pytest.raises(ValueError, match="at least two"): + summarize_observations([one]) + with pytest.raises(ValueError, match="fingerprint"): + summarize_observations([one, other]) + + +def test_summarize_observations_covers_declared_metrics(): + observations = [ + RunObservation(1, 10, 5, 2.0, 2.1, 3.0), + RunObservation(2, 20, 7, 2.2, 2.3, 5.0), + ] + + summaries = summarize_observations(observations) + + assert set(summaries) == { + "actual_steps", + "best_step", + "best_bpc", + "final_bpc", + "duration_seconds", + } + assert summaries["actual_steps"].mean == 15.0 + assert summaries["best_bpc"].mean == pytest.approx(2.1) diff --git a/tests/test_summarize_runs.py b/tests/test_summarize_runs.py new file mode 100644 index 0000000..893672e --- /dev/null +++ b/tests/test_summarize_runs.py @@ -0,0 +1,88 @@ +import json + +import pytest + +from scripts.summarize_runs import _observation +from smallm.config import ExperimentConfig, TrainConfig +from smallm.evaluation.robustness import summarize_observations +from smallm.training.artifacts import write_config_snapshot + + +def _write_run(tmp_path, name, *, seed=1, learning_rate=1e-3, summary_updates=None): + run_dir = tmp_path / name + run_dir.mkdir() + write_config_snapshot( + run_dir / "config.yaml", + ExperimentConfig( + train=TrainConfig( + run_name=name, + runs_dir=str(tmp_path), + seed=seed, + learning_rate=learning_rate, + ) + ), + ) + summary = { + "schema_version": 2, + "status": "complete", + "actual_steps": 100, + "best_val_step": 80, + "best_val_bits_per_char": 2.0, + "final_val_bits_per_char": 2.1, + "duration_seconds": 5.0, + "validation_mode": "full", + "validation_coverage": 1.0, + "tokenizer_type": "char", + "tokenizer_vocab_size": 10, + "dataset": { + "prepared_sha256": "abc", + "train_split": 0.9, + "train_characters": 90, + "validation_characters": 10, + }, + } + summary.update(summary_updates or {}) + (run_dir / "summary.json").write_text(json.dumps(summary), encoding="utf-8") + return run_dir + + +def test_observations_verify_training_comparability(tmp_path): + first = _observation(_write_run(tmp_path, "first", seed=1)) + second = _observation(_write_run(tmp_path, "second", seed=2)) + different = _observation(_write_run(tmp_path, "different", seed=3, learning_rate=2e-3)) + + assert first.comparison_fingerprint == second.comparison_fingerprint + summarize_observations([first, second]) + with pytest.raises(ValueError, match="fingerprint"): + summarize_observations([first, different]) + + +@pytest.mark.parametrize( + "updates", + [ + {"actual_steps": 12.5}, + {"actual_steps": -1}, + {"actual_steps": 10, "best_val_step": 11}, + {"best_val_bits_per_char": float("nan")}, + {"final_val_bits_per_char": float("inf")}, + {"duration_seconds": -1}, + {"actual_steps": 10**100}, + {"best_val_bits_per_char": 10**1000}, + {"final_val_bits_per_char": 10**1000}, + {"duration_seconds": 10**1000}, + ], +) +def test_observation_rejects_corrupt_numeric_fields(tmp_path, updates): + run_dir = _write_run(tmp_path, "invalid", summary_updates=updates) + + with pytest.raises(ValueError): + _observation(run_dir) + + +def test_observation_reports_malformed_artifact_path(tmp_path): + run_dir = tmp_path / "malformed" + run_dir.mkdir() + (run_dir / "summary.json").write_text("[]", encoding="utf-8") + + with pytest.raises(ValueError, match="summary"): + _observation(run_dir)