diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f3e00c5..1de0949 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,12 @@ jobs: - name: Physical design (levelization) run: ./scripts/levelize.py src + - name: Benchmark inputs are committed + # A benchmark whose input is gitignored works perfectly for its + # author and for nobody else, and running it proves nothing - + # the only way to catch it is to ask whether the input ships. + run: python3 scripts/check_bench_inputs.py + - name: Contracts script unit tests # Pure parsing and date arithmetic only. The network half is not # exercised here on purpose: a CI job that needs Polymarket to be diff --git a/docs/bench/allocator.md b/docs/bench/allocator.md index c329276..e5c7734 100644 --- a/docs/bench/allocator.md +++ b/docs/bench/allocator.md @@ -20,9 +20,23 @@ cmake --build build-bde -j ``` That synthetic session is 724,298 records (124 MB) in real wire formats. -The live capture referenced below is a 60 s Polymarket market-channel -recording (877 records) used for allocation profiles only; it is too short -for stable timing percentiles. +The live capture referenced below is the committed 60 s Polymarket +market-channel recording in this directory, 877 records. It is used for +allocation profiles only: it is too short for stable timing, and the +pipeline figures below all come from the synthetic session instead. + +``` +gunzip -c docs/bench/live-poly-60s.feedlog.gz > /tmp/live-poly-60s.feedlog +./build/src/basis replay /tmp/live-poly-60s.feedlog \ + --config configs/contracts.toml --alloc count +``` + +It was not committed until 2026-09-01, which made the live row here the +one figure in docs/bench/ that nobody could check. `/captures/` is +gitignored as large and regenerable, and this file was sitting in it, so +the row was measured on an input that shipped no copy of itself. The +repo's rule is that a figure is either measured on a committed capture or +labelled as not yet measured, and this was neither. 85 KB gzipped. Modes, all via the same binary: diff --git a/docs/bench/live-poly-60s.feedlog.gz b/docs/bench/live-poly-60s.feedlog.gz new file mode 100644 index 0000000..02ec550 Binary files /dev/null and b/docs/bench/live-poly-60s.feedlog.gz differ diff --git a/scripts/check_bench_inputs.py b/scripts/check_bench_inputs.py new file mode 100755 index 0000000..25f5e6a --- /dev/null +++ b/scripts/check_bench_inputs.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Fail if a doc under docs/bench/ names an input that is not committed. + +The rule this enforces is the repo's own, from the README: a figure is +either measured on a committed capture or it is labelled as not yet +measured. There was one exception and nobody noticed, because the +exception is invisible from inside a working tree - `/captures/` is +gitignored, the file sat there, every command ran fine, and the row it +produced could not be reproduced by anyone who cloned the repo. + +That is the failure mode worth guarding: a benchmark that works perfectly +for its author and for nobody else. It cannot be caught by running the +benchmark, only by asking whether its input ships. + + scripts/check_bench_inputs.py +""" + +import pathlib +import re +import subprocess +import sys + +ROOT = pathlib.Path(__file__).resolve().parent.parent + +# Paths that are meant to be produced, not shipped: scratch files a command +# in the doc generates first. Referencing one is correct. +GENERATED = re.compile(r"^(/tmp/|out\.|.*\bmktemp\b)") + + +def tracked_files(): + out = subprocess.run(["git", "ls-files"], cwd=ROOT, capture_output=True, + text=True, check=True).stdout + return set(out.split()) + + +def main(): + tracked = tracked_files() + failures = 0 + checked = 0 + + for doc in sorted((ROOT / "docs" / "bench").glob("*.md")): + text = doc.read_text(encoding="utf-8") + # Angle brackets mark a usage metavariable - `basis xvenue-lead + # ` names an argument, not a file. Strip those + # first so the check does not demand that a placeholder ship. + prose = re.sub(r"<[^<>\n]*\.feedlog(?:\.gz)?>", "", text) + # Any path ending in .feedlog or .feedlog.gz that the prose names. + for ref in sorted(set(re.findall(r"[\w./-]+\.feedlog(?:\.gz)?", prose))): + if GENERATED.match(ref): + continue + checked += 1 + # A doc may name either the .gz that ships or the file a reader + # gunzips it into; the .gz is what has to be tracked. + gz = ref if ref.endswith(".gz") else ref + ".gz" + candidates = {ref, gz, + f"docs/bench/{pathlib.PurePath(gz).name}"} + if candidates & tracked: + continue + print(f" {doc.relative_to(ROOT)} references {ref}, " + f"which is not committed") + failures += 1 + + print(f"\n{checked} capture references across docs/bench/") + if failures: + print(f"FAIL: {failures} reference(s) to inputs that do not ship") + return 1 + print("ok: every referenced capture is committed") + return 0 + + +if __name__ == "__main__": + sys.exit(main())