From 82ae5482106b1179865eb3a89398223e1422135c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 17:06:05 +0000 Subject: [PATCH 1/5] Fix textual floor (>=2.0), add dev extra, test on 3.10-3.13 `memeval[tui]` declared textual>=0.80, but render/tui.py imports textual.markup, which first shipped in textual 2.0.0. Installing the extra at the declared floor gave ModuleNotFoundError (verified against 0.80.0, 1.0.0 and 2.0.0). matplotlib>=3.8 was verified good as is. Add a `dev` extra (pytest, pytest-asyncio, ruff) so CI and contributors install the same thing, and run the test job on every supported minor (requires-python is >=3.10; CI only exercised 3.12). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Nc5DZiN4NhcQ9mBX2SL2yP --- .github/workflows/ci.yml | 10 +++++++--- pyproject.toml | 5 +++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f4027e..507b91b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,15 +12,19 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - - run: pip install "ruff>=0.8,<1" + - run: pip install -e ".[dev]" - run: ruff check . test: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.12" - - run: pip install -e ".[all]" pytest pytest-asyncio + python-version: ${{ matrix.python-version }} + - run: pip install -e ".[all,dev]" - run: pytest -q diff --git a/pyproject.toml b/pyproject.toml index b478126..fc65c21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,9 +9,10 @@ dependencies = [] keywords = ["agent-memory", "evaluation", "rag", "hindsight", "mem0", "vector-search"] [project.optional-dependencies] -tui = ["textual>=0.80"] +tui = ["textual>=2.0"] charts = ["matplotlib>=3.8"] -all = ["textual>=0.80", "matplotlib>=3.8"] +all = ["textual>=2.0", "matplotlib>=3.8"] +dev = ["pytest", "pytest-asyncio", "ruff>=0.8,<1"] [project.urls] Repository = "https://github.com/nuk3s/memeval" From 5d5b370fc4cec73128fa85d858626484cbd2c947 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 17:06:05 +0000 Subject: [PATCH 2/5] Readiness poll samples the whole corpus; retry only transient HTTP errors wait_ready checked only the first 10 docs. Stores retain in batches (Hindsight: 20 per POST), so a batch dropped at the tail of the 125-doc bundled corpus was invisible to the very check that exists to catch silent write loss. Sample up to 10 docs spread evenly across the corpus, first and last always included. The `#i` fallback label in `missing` stays the corpus index. Runner.query retried on OSError, and urllib's HTTPError is an OSError, so a 401/404/422 was re-issued --retries times per query: no chance of success, an inflated latency on the error row, and N extra requests to a misconfigured store. Retry HTTPError only for 5xx/408/425/429. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Nc5DZiN4NhcQ9mBX2SL2yP --- docs/run-log.md | 7 ++++- memeval/runner.py | 39 ++++++++++++++++++----- tests/test_runner.py | 75 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 8 deletions(-) diff --git a/docs/run-log.md b/docs/run-log.md index e2ca967..5905355 100644 --- a/docs/run-log.md +++ b/docs/run-log.md @@ -19,7 +19,8 @@ line, in the order things happened. Same-second collisions get a `-2`, `-3`… s Notes. The `ready` event is the readiness poll that replaced a fixed sleep: after retain, memeval -polls recall on a sample of corpus docs (word-overlap match, ≥50% of the doc's content +polls recall on up to 10 docs spread evenly across the corpus, first and last always +included so a dropped tail batch is caught (word-overlap match, ≥50% of the doc's content words, because extracting stores rewrite text) until each appears or `--ready-timeout` runs out. A doc in `missing` was accepted by the store but never became recallable — silent write loss. The run continues; the event keeps the evidence. @@ -28,5 +29,9 @@ Hit text is truncated in the log for size; fabrication verdicts are computed on recalled text before truncation, so a `verdict` can reference a match you can't fully see in the neighbouring `query` event. +`--retries N` re-issues a recall only on transport errors and HTTP 5xx/408/425/429. Any +other 4xx (bad token, unknown route) fails identically every time, so it becomes an error row +on the first attempt. + Latency percentiles count only error-free queries; `errors` counts the rest. Error messages are never empty strings — a bare exception records its class name. diff --git a/memeval/runner.py b/memeval/runner.py index 590e921..743009a 100644 --- a/memeval/runner.py +++ b/memeval/runner.py @@ -2,18 +2,32 @@ readiness polling, event emission. Hardening lives here, written once.""" import re import time +import urllib.error HIT_TEXT_LIMIT = 300 +READY_SAMPLE = 10 # ConnectionError and TimeoutError are OSError subclasses — listed anyway for # readability. urllib.error.URLError is also an OSError subclass, so plain-http # adapter failures land here too. RETRYABLE = (ConnectionError, TimeoutError, OSError) +# HTTP statuses that can plausibly clear on a second attempt. Any other 4xx (401, +# 404, 422...) is a configuration or request problem that fails identically every +# time; retrying it only delays the error row and inflates its recorded latency. +RETRYABLE_HTTP = frozenset({408, 425, 429}) def _ms(t0): return round((time.monotonic() - t0) * 1000) +def _retryable(e): + # HTTPError is an OSError subclass, so without this check a bad token would be + # retried --retries times per query. + if isinstance(e, urllib.error.HTTPError): + return e.code >= 500 or e.code in RETRYABLE_HTTP + return isinstance(e, RETRYABLE) + + def _coerce_score(v): try: return None if v is None else float(v) @@ -40,6 +54,18 @@ def _overlaps(source, text, threshold=0.5): return found / len(words) >= threshold +def _spread(corpus, sample): + """Up to `sample` (index, doc) pairs spaced evenly across the corpus, first and + last always included. Stores retain in batches, and a batch dropped at the tail + of a big corpus is invisible to a head-only sample — which is exactly the silent + write loss the readiness poll exists to catch.""" + n = len(corpus) + if n <= sample: + return list(enumerate(corpus)) + picks = sorted({round(i * (n - 1) / max(sample - 1, 1)) for i in range(sample)}) + return [(i, corpus[i]) for i in picks] + + class Runner: def __init__(self, adapter, log, retries=1, ready_timeout=60.0, poll_interval=1.0, echo=None): @@ -76,12 +102,12 @@ def consolidate(self): self.log.emit("consolidate", ms=_ms(t0), ok=True, error=None) return True - def wait_ready(self, corpus, sample=10): + def wait_ready(self, corpus, sample=READY_SAMPLE): """Poll until each sampled doc is recallable; report the ones that never are. A doc missing at timeout is the 'success:true but not persisted' failure — recorded, warned about, and the run continues.""" - docs = corpus[:sample] - pending = dict(enumerate(docs)) + pending = dict(_spread(corpus, sample)) # corpus index -> doc + n_sampled = len(pending) t0 = time.monotonic() while True: for idx in list(pending): @@ -100,7 +126,7 @@ def wait_ready(self, corpus, sample=10): time.sleep(min(self.poll_interval, remaining)) missing = [pending[i].get("id") or f"#{i}" for i in sorted(pending)] self.log.emit("ready", polled_s=round(time.monotonic() - t0, 1), - verified_n=len(docs) - len(missing), sampled_n=len(docs), + verified_n=n_sampled - len(missing), sampled_n=n_sampled, missing=missing) if missing: self.echo(f"WARNING: {len(missing)} doc(s) never became recallable " @@ -118,11 +144,10 @@ def query(self, query, k, phase, rep=1, pattern=None): hits = _normalize_hits(self.adapter.recall(query, k)) ms, err = _ms(t0), None break - except RETRYABLE as e: - ms, err = _ms(t0), str(e) or e.__class__.__name__ except Exception as e: ms, err = _ms(t0), str(e) or e.__class__.__name__ - break + if not _retryable(e): + break rank = None if err is None and pattern: try: diff --git a/tests/test_runner.py b/tests/test_runner.py index 7e0d73d..d2f139a 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -144,3 +144,78 @@ def recall(self, query, k=10): row = r.query("checkout", 5, phase="rank") assert row["hits"][0]["score"] == 0.9 assert row["hits"][1]["score"] is None + + +def _distinct_corpus(n): + # every >3-char word is unique to its doc (trailing "x" blocks prefix substrings), + # so word-overlap readiness cannot be satisfied by a neighbouring doc + return [{"id": f"d{i}", "content": f"fact {i}: token{i}x alpha{i}x beta{i}x gamma{i}x"} + for i in range(n)] + + +def test_wait_ready_samples_across_whole_corpus(tmp_path): + # a doc dropped at the tail of a big corpus must be as visible as one at the head; + # a head-only sample (the old behaviour) reported [] here + corpus = _distinct_corpus(30) + r, log = make(tmp_path, FakeAdapter(drop_ids={"d0", "d29"}), + ready_timeout=0.3, poll_interval=0.1) + r.retain(corpus) + assert r.wait_ready(corpus) == ["d0", "d29"] + ready = [e for e in read_events(log.path) if e["ev"] == "ready"][0] + assert ready["sampled_n"] == 10 and ready["verified_n"] == 8 + + +def test_wait_ready_missing_label_uses_corpus_index(tmp_path): + corpus = [{"content": d["content"]} for d in _distinct_corpus(30)] # no ids + r, log = make(tmp_path, FakeAdapter(), ready_timeout=0.3, poll_interval=0.1) + r.adapter.docs = [] # nothing stored at all -> every sampled doc is missing + missing = r.wait_ready(corpus) + assert missing[0] == "#0" and missing[-1] == "#29" and len(missing) == 10 + + +def test_spread_sample_is_even_and_bounded(): + from memeval.runner import _spread + idx = [i for i, _ in _spread(list(range(125)), 10)] + assert idx[0] == 0 and idx[-1] == 124 and len(idx) == 10 and idx == sorted(idx) + assert [i for i, _ in _spread(list(range(3)), 10)] == [0, 1, 2] + assert _spread([], 10) == [] + assert [i for i, _ in _spread(list(range(5)), 1)] == [0] + + +def test_query_does_not_retry_client_http_errors(tmp_path): + import urllib.error + + class Unauthorized(FakeAdapter): + def __init__(self): + super().__init__() + self.calls = 0 + + def recall(self, query, k=10): + self.calls += 1 + raise urllib.error.HTTPError("http://x", 401, "Unauthorized", {}, None) + a = Unauthorized() + r, log = make(tmp_path, a, retries=3) + row = r.query("q", 5, phase="rank") + assert a.calls == 1, "a 401 fails identically every time; retrying it is waste" + assert "401" in row["error"] + + +def test_query_retries_server_and_rate_limit_http_errors(tmp_path): + import urllib.error + + class Flaky(FakeAdapter): + def __init__(self, codes): + super().__init__() + self.codes = list(codes) + self.calls = 0 + + def recall(self, query, k=10): + self.calls += 1 + if self.codes: + raise urllib.error.HTTPError("http://x", self.codes.pop(0), "err", {}, None) + return super().recall(query, k) + a = Flaky([503, 429]) + a.retain(CORPUS) + r, log = make(tmp_path, a, retries=2) + row = r.query("what port does the checkout service use?", 5, phase="rank", pattern="8080") + assert a.calls == 3 and row["error"] is None and row["rank"] == 1 From 726e03d8a1305e9d7eab179d644d1d5eeaafad53 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 17:06:05 +0000 Subject: [PATCH 3/5] read_events: skip JSON lines that are not objects A foreign .jsonl in runs/ whose line parses to an array or scalar took down `memeval runs`, `memeval chart` and the replay TUI with AttributeError on .get(). Every event is an object, so drop anything else the same way a truncated line is dropped. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Nc5DZiN4NhcQ9mBX2SL2yP --- memeval/events.py | 4 +++- tests/test_events.py | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/memeval/events.py b/memeval/events.py index cbdfae0..77a3d0d 100644 --- a/memeval/events.py +++ b/memeval/events.py @@ -87,7 +87,9 @@ def read_events(path): if not line.strip(): continue try: - events.append(json.loads(line)) + rec = json.loads(line) except json.JSONDecodeError: continue + if isinstance(rec, dict): # an event is always an object; a foreign file's + events.append(rec) # arrays/scalars would crash every consumer's .get return events diff --git a/tests/test_events.py b/tests/test_events.py index 1952178..09f1f9e 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -74,3 +74,10 @@ def test_runlog_context_manager(tmp_path): with RunLog(p) as log: log.emit("run_meta", run_id="r1") assert read_events(p) == [{"ev": "run_meta", "run_id": "r1"}] + + +def test_read_events_skips_non_object_lines(tmp_path): + # a foreign .jsonl in runs/ can hold arrays or scalars; every consumer does .get() + p = tmp_path / "run.jsonl" + p.write_text('[1, 2]\n"text"\n42\nnull\n{"ev": "run_meta", "run_id": "r1"}\n') + assert read_events(p) == [{"ev": "run_meta", "run_id": "r1"}] From ee519b4296020974f5ecd2d0c9a6561cf4aabf08 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 17:06:05 +0000 Subject: [PATCH 4/5] Validate corpus/gold/probe at load time; clean CLI exits; python -m memeval A gold row without `answer`, a corpus row without `content`, or a probe without `checks` was a KeyError traceback after retain had already written into the store. load_corpus/load_gold/validate_probe now reject malformed input with a row number before anything is written, and load_jsonl reports the line of a JSON syntax error. A malformed gold regex is still an error row at query time, per the README contract; probe regexes are compiled at load since the probe is one small file. `memeval replay` and `memeval chart` on a missing path were tracebacks; they exit with a message. `chart RUN --ab A B` silently ignored RUN; it is now rejected. `memeval adapters` crashed on an adapter with an empty docstring. `python -m memeval` works (there was no __main__). Replace test_negation_correction_not_flagged_as_fabrication, which only asserted the negation regex matched a string, with a run against a store that recalls the negated correction (must stay clean) and one that recalls the affirmative claim (must be flagged). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Nc5DZiN4NhcQ9mBX2SL2yP --- memeval/__main__.py | 5 +++ memeval/adversarial.py | 41 ++++++++++++++++++- memeval/cli.py | 17 +++++--- memeval/rank.py | 34 +++++++++++++++- tests/test_adversarial.py | 56 ++++++++++++++++++++++---- tests/test_charts.py | 12 ++++++ tests/test_cli.py | 84 +++++++++++++++++++++++++++++++++++++++ tests/test_rank.py | 31 +++++++++++++++ 8 files changed, 266 insertions(+), 14 deletions(-) create mode 100644 memeval/__main__.py diff --git a/memeval/__main__.py b/memeval/__main__.py new file mode 100644 index 0000000..dad7b45 --- /dev/null +++ b/memeval/__main__.py @@ -0,0 +1,5 @@ +"""`python -m memeval` entry point; the console script is memeval.cli:main.""" +from .cli import main + +if __name__ == "__main__": + main() diff --git a/memeval/adversarial.py b/memeval/adversarial.py index bb9bf15..f657cbe 100644 --- a/memeval/adversarial.py +++ b/memeval/adversarial.py @@ -14,7 +14,46 @@ def load_probe(path): with open(path, encoding="utf-8") as f: - return json.load(f) + return validate_probe(json.load(f)) + + +def _nonempty_str(v): + return isinstance(v, str) and bool(v.strip()) + + +def validate_probe(probe): + """Shape-check the probe at load time, so a typo fails before retain writes into + the store rather than as a KeyError or re.error halfway through the run.""" + if not isinstance(probe, dict): + raise ValueError("probe must be a JSON object") + corpus = probe.get("corpus") + if not isinstance(corpus, list) or not corpus: + raise ValueError("probe 'corpus' must be a non-empty list") + for n, d in enumerate(corpus, 1): + if not isinstance(d, dict) or not _nonempty_str(d.get("content")): + raise ValueError(f"probe corpus item {n}: missing or empty 'content'") + checks = probe.get("checks") + if not isinstance(checks, list) or not checks: + raise ValueError("probe 'checks' must be a non-empty list") + for n, c in enumerate(checks, 1): + if not isinstance(c, dict) or not _nonempty_str(c.get("query")): + raise ValueError(f"probe check {n}: missing or empty 'query'") + for key in ("forbidden", "expected"): + pats = c.get(key, []) + if not isinstance(pats, list) or not all(isinstance(p, str) for p in pats): + raise ValueError(f"probe check {n}: {key!r} must be a list of regex strings") + for p in pats: + try: + re.compile(p) + except re.error as e: + raise ValueError(f"probe check {n}: bad {key} regex {p!r}: {e}") from None + corrections = probe.get("corrections", []) + if not isinstance(corrections, list) or not all(isinstance(x, str) for x in corrections): + raise ValueError("probe 'corrections' must be a list of strings") + wait = probe.get("consolidate_wait", 2) + if isinstance(wait, bool) or not isinstance(wait, (int, float)) or wait < 0: + raise ValueError("probe 'consolidate_wait' must be a non-negative number of seconds") + return probe def _check(runner, checks, phase): diff --git a/memeval/cli.py b/memeval/cli.py index 98f01f6..f3c6828 100644 --- a/memeval/cli.py +++ b/memeval/cli.py @@ -5,7 +5,7 @@ from .adapters import REGISTRY from .adversarial import load_probe, run_adversarial from .events import SCHEMA_VERSION, RunLog, file_sha256, read_events, redact, run_path -from .rank import load_jsonl, run_rank +from .rank import load_corpus, load_gold, run_rank from .render.text import format_ab, format_adversarial, format_summary from .runner import Runner @@ -46,8 +46,8 @@ def _start_run(args, adapter, kind, files): def _rank_one(args, name, kind): - corpus = _load_input(args.corpus, load_jsonl, "corpus") - gold = _load_input(args.gold, load_jsonl, "gold") + corpus = _load_input(args.corpus, load_corpus, "corpus") + gold = _load_input(args.gold, load_gold, "gold") adapter = build_adapter(name) path, log, runner = _start_run(args, adapter, kind, {"corpus": args.corpus, "gold": args.gold}) @@ -116,7 +116,7 @@ def cmd_runs(args): def cmd_adapters(args): for name, cls in sorted(REGISTRY.items()): - doc = (cls.__doc__ or "").strip().splitlines()[0] + doc = next(iter((cls.__doc__ or "").strip().splitlines()), "") print(f"{name:>14} {cls.status:>8} {doc}") @@ -197,7 +197,10 @@ def cmd_replay(args): from .render.tui import ReplayApp except ImportError: sys.exit("replay needs textual — pip install 'memeval[tui]'") - evs = read_events(args.run) + try: + evs = read_events(args.run) + except OSError as e: + sys.exit(f"cannot read run log {args.run}: {e}") if not evs: sys.exit(f"empty run log: {args.run}") ReplayApp(evs).run() @@ -208,6 +211,8 @@ def cmd_chart(args): from .render.charts import render_ab_chart, render_chart except ImportError: sys.exit("chart needs matplotlib — pip install 'memeval[charts]'") + if args.ab and args.run: + sys.exit("pass one run file, or --ab RUN_A RUN_B — not both") try: if args.ab: out = render_ab_chart(args.ab[0], args.ab[1], out=args.out, png=args.png) @@ -217,6 +222,8 @@ def cmd_chart(args): sys.exit("pass one run file, or --ab RUN_A RUN_B") except ValueError as e: sys.exit(str(e)) + except OSError as e: + sys.exit(f"chart failed: {e}") print(f"wrote {out}") diff --git a/memeval/rank.py b/memeval/rank.py index e0281e6..bf03daa 100644 --- a/memeval/rank.py +++ b/memeval/rank.py @@ -4,8 +4,40 @@ def load_jsonl(path): + rows = [] with open(path, encoding="utf-8") as f: - return [json.loads(line) for line in f if line.strip()] + for n, line in enumerate(f, 1): + if not line.strip(): + continue + try: + rows.append(json.loads(line)) + except json.JSONDecodeError as e: + raise ValueError(f"line {n}: {e}") from None + return rows + + +def _validate_rows(rows, label, fields): + """Every row is an object whose `fields` are non-empty strings. Fails loud at + load time with a row number, instead of a KeyError mid-run after retain.""" + for n, row in enumerate(rows, 1): + if not isinstance(row, dict): + raise ValueError( + f"{label} row {n}: expected a JSON object, got {type(row).__name__}") + for field in fields: + v = row.get(field) + if not isinstance(v, str) or not v.strip(): + raise ValueError(f"{label} row {n}: missing or empty {field!r}") + return rows + + +def load_corpus(path): + return _validate_rows(load_jsonl(path), "corpus", ("content",)) + + +def load_gold(path): + """`answer` is checked for presence only: a malformed regex is an error row at + query time (Runner.query), not a rejected file — see the README.""" + return _validate_rows(load_jsonl(path), "gold", ("query", "answer")) # nearest-rank with round(); differs from numpy's linear interpolation — diff --git a/tests/test_adversarial.py b/tests/test_adversarial.py index 6d00f55..05426ed 100644 --- a/tests/test_adversarial.py +++ b/tests/test_adversarial.py @@ -1,3 +1,7 @@ +import json +from pathlib import Path + +import pytest from conftest import FakeAdapter from memeval.adversarial import run_adversarial @@ -54,13 +58,22 @@ def test_clean_store_stays_clean(tmp_path): def test_negation_correction_not_flagged_as_fabrication(tmp_path): - # the correction itself contains "media ... 8080" but with a negation cue - a = FakeAdapter() - r, _ = run(tmp_path, a) - a.retain([{"content": "the media service is not on port 8080"}]) - # direct check: a store returning only the negated correction must stay clean - from memeval.adversarial import _NEG - assert _NEG.search("the media service is not on port 8080") + # a store that recalls only the negated correction restates "media ... 8080" in + # order to deny it: the forbidden regex matches that text, and the check must + # skip it or every correction would read as a fabrication + class RecallsCorrection(FakeAdapter): + def recall(self, query, k=10): + return [{"text": "the media service is not on port 8080", "score": 1.0}] + r, _ = run(tmp_path, RecallsCorrection(), consolidate=False) + assert r["before"][0]["fabricated"] == [] + assert r["after_corrections"][0]["fabricated"] == [] + + # ...while the same binding stated affirmatively is still caught + class Affirms(FakeAdapter): + def recall(self, query, k=10): + return [{"text": "the media service is on port 8080", "score": 1.0}] + r, _ = run(tmp_path, Affirms(), consolidate=False) + assert r["before"][0]["fabricated"] == ["media service[^.]{0,30}8080"] def test_no_corrections_probe(tmp_path): @@ -104,3 +117,32 @@ def consolidate(self): assert r is not None and not r["before"][0]["fabricated"] cons = [e for e in evs if e["ev"] == "consolidate"] assert cons and cons[0]["ok"] is False + + +EXAMPLE_PROBE = Path(__file__).resolve().parent.parent / "examples" / "adversarial.json" + + +@pytest.mark.parametrize("bad, msg", [ + ({"checks": [{"query": "q"}]}, "corpus"), + ({"corpus": [{"content": "x"}], "checks": []}, "checks"), + ({"corpus": [{"content": "x"}], "checks": [{"forbidden": []}]}, "check 1: missing"), + ({"corpus": [{"content": "x"}], "checks": [{"query": "q", "forbidden": ["("]}]}, + "bad forbidden regex"), + ({"corpus": [{"content": "x"}], "checks": [{"query": "q"}], "consolidate_wait": "soon"}, + "consolidate_wait"), + ({"corpus": [{"content": "x"}], "checks": [{"query": "q"}], "corrections": "nope"}, + "corrections"), + ([], "JSON object"), +]) +def test_load_probe_rejects_malformed_shapes(tmp_path, bad, msg): + from memeval.adversarial import load_probe + p = tmp_path / "probe.json" + p.write_text(json.dumps(bad)) + with pytest.raises(ValueError, match=msg): + load_probe(p) + + +def test_bundled_probe_passes_validation(): + from memeval.adversarial import load_probe + probe = load_probe(EXAMPLE_PROBE) + assert probe["checks"] and probe["corrections"] diff --git a/tests/test_charts.py b/tests/test_charts.py index 3281194..e9ca6b3 100644 --- a/tests/test_charts.py +++ b/tests/test_charts.py @@ -112,3 +112,15 @@ def test_chart_all_errors_still_renders(tmp_path, monkeypatch): from memeval.render.charts import render_chart render_chart(str(next(runs.glob("*.jsonl"))), out=str(tmp_path / "e.svg")) assert " Date: Fri, 25 Sep 2026 17:06:06 +0000 Subject: [PATCH 5/5] Adapters: one HTTP helper; hindsight batch floor; record consolidate failures The five bundled adapters each carried a copy of the urllib request/ decode code, and only mem0, letta and zep folded the server's error body into the exception; hindsight and example-rest surfaced a bare status line. adapters/_http.request_json is the one copy, so a 4xx from any store says why in the run log. Request shapes are unchanged (existing FakeHTTP tests pass as-is); example-rest gains its first test. HINDSIGHT_BATCH=0 raised `range() arg 3 must not be zero` on the first retain; clamp to >= 1. HindsightAdapter.consolidate swallowed every exception, so an adversarial run against an unreachable server, or a bank that rejects the call, logged consolidate ok=true and reported the fabrication check as if consolidation had happened. Runner.consolidate already guards the call, records ok=false with the error, and continues, and doctor reports it as informational; let it propagate so the log tells the truth. Verified with mocked HTTP only, not against a live instance. README: quote the extras in the install lines so they work under zsh, and point adapter authors at the shared helper. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Nc5DZiN4NhcQ9mBX2SL2yP --- README.md | 12 +++--- memeval/adapters/_http.py | 40 +++++++++++++++++++ memeval/adapters/example_rest.py | 11 ++---- memeval/adapters/hindsight.py | 26 ++++++------- memeval/adapters/letta.py | 26 ++----------- memeval/adapters/mem0.py | 25 ++---------- memeval/adapters/zep.py | 24 ++---------- tests/test_adapters_http.py | 66 ++++++++++++++++++++++++++++++-- 8 files changed, 134 insertions(+), 96 deletions(-) create mode 100644 memeval/adapters/_http.py diff --git a/README.md b/README.md index b192bfc..945e88d 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ That's 119 paraphrased queries, and the two rank-1 misses are the dataset doing **Adversarial.** Retain a small set of facts chosen to invite a bad merge, trigger the store's consolidation, and check whether recall now returns a statement no source made. Then feed corrections and check whether the fabrication goes away. A store that invents a fact and won't retract it is worse than one that misses. A check that errors is reported as an error, never as "clean". -**Silent write loss.** After retain, memeval polls until the stored docs are actually recallable, instead of trusting the write acknowledgment. This caught a real case during this repo's own release verification — the store returned success and the fact never became findable: +**Silent write loss.** After retain, memeval polls a sample of docs spread across the whole corpus until each is actually recallable, instead of trusting the write acknowledgment. This caught a real case during this repo's own release verification — the store returned success and the fact never became findable: ``` WARNING: 1 doc(s) never became recallable (silent write loss?): nw-06 @@ -41,9 +41,9 @@ Rank scores can come out perfect while a store quietly drops writes. This check git clone https://github.com/nuk3s/memeval cd memeval pip install -e . # core: stdlib only - pip install -e .[tui] # + replay TUI (Textual) - pip install -e .[charts] # + SVG/PNG charts (matplotlib) - pip install -e .[all] + pip install -e ".[tui]" # + replay TUI (Textual) + pip install -e ".[charts]" # + SVG/PNG charts (matplotlib) + pip install -e ".[all]" Python 3.10 or newer. @@ -95,7 +95,9 @@ numbers from it, and send a fix if the docs lied. Writing an adapter is ~40 lines: subclass `MemoryAdapter` from `memeval/adapters/base.py`, implement `retain(items)` and `recall(query, k)`, optionally `consolidate()`, `supersede()`, `prepare()`, and `config()`, then register it in -`memeval/adapters/__init__.py`. Copy `example_rest.py` and start from there. +`memeval/adapters/__init__.py`. Copy `example_rest.py` and start from there; the bundled adapters share +`memeval/adapters/_http.py` for JSON requests, which folds the server's error body into +the exception message so a 4xx in the run log says why. `memeval doctor` is the conformance check: it writes three sentinel docs, polls the recall roundtrip, and probes the optional capabilities. Partial visibility is an exit-1 PARTIAL, not a pass. diff --git a/memeval/adapters/_http.py b/memeval/adapters/_http.py new file mode 100644 index 0000000..152705e --- /dev/null +++ b/memeval/adapters/_http.py @@ -0,0 +1,40 @@ +"""Shared urllib plumbing for the bundled REST adapters. Stdlib only. + +Every adapter used to carry its own copy of this, and only some of them folded the +server's error body into the exception. Keeping it here means a 4xx in the run log +says *why* for all of them. +""" +import json +import urllib.error +import urllib.request + +ERROR_BODY_LIMIT = 200 + + +def request_json(url, body=None, method=None, headers=None, timeout=120, context=None): + """Send `body` as JSON (POST by default when a body is given, GET otherwise) and + decode the JSON reply; an empty reply decodes to {}. + + An HTTPError is re-raised with the first ERROR_BODY_LIMIT chars of the response + body in its message, so the error row reads "400 Bad Request — {detail...}" + instead of just the status line. + """ + data = json.dumps(body).encode() if body is not None else None + if method is None: + method = "POST" if data is not None else "GET" + req = urllib.request.Request( + url, data=data, method=method, + headers={"Content-Type": "application/json", **(headers or {})}) + try: + with urllib.request.urlopen(req, timeout=timeout, context=context) as r: + raw = r.read().decode() + return json.loads(raw) if raw else {} + except urllib.error.HTTPError as e: + detail = "" + try: + detail = e.read().decode()[:ERROR_BODY_LIMIT] + except Exception: + pass + raise urllib.error.HTTPError( + e.url, e.code, f"{e.reason} — {detail}" if detail else e.reason, + e.headers, None) from None diff --git a/memeval/adapters/example_rest.py b/memeval/adapters/example_rest.py index 2557812..5234416 100644 --- a/memeval/adapters/example_rest.py +++ b/memeval/adapters/example_rest.py @@ -1,8 +1,7 @@ -import json import os -import urllib.request from typing import Dict, List +from ._http import request_json from .base import MemoryAdapter @@ -27,12 +26,8 @@ def __init__(self, url=None, token=None, user=None): self.user = user or os.environ.get("MEMEVAL_REST_USER", "memeval") def _post(self, path: str, body: Dict, timeout: int = 120): - req = urllib.request.Request( - self.url + path, data=json.dumps(body).encode(), method="POST", - headers={"Authorization": f"Bearer {self.token}", "Content-Type": "application/json"}) - with urllib.request.urlopen(req, timeout=timeout) as r: - raw = r.read().decode() - return json.loads(raw) if raw else {} + return request_json(self.url + path, body=body, method="POST", timeout=timeout, + headers={"Authorization": f"Bearer {self.token}"}) def retain(self, items: List[Dict]) -> None: for i in items: diff --git a/memeval/adapters/hindsight.py b/memeval/adapters/hindsight.py index 1a46eb4..0767cd1 100644 --- a/memeval/adapters/hindsight.py +++ b/memeval/adapters/hindsight.py @@ -1,9 +1,8 @@ -import json import os import ssl -import urllib.request from typing import Dict, List, Optional +from ._http import request_json from .base import MemoryAdapter @@ -15,7 +14,7 @@ class HindsightAdapter(MemoryAdapter): HINDSIGHT_TOKEN bearer token (default empty) HINDSIGHT_BANK bank name (default memeval-scratch) HINDSIGHT_INSECURE any non-empty value skips TLS verification (for internal CAs) - HINDSIGHT_BATCH items per retain POST (default 20) + HINDSIGHT_BATCH items per retain POST (default 20, minimum 1) HINDSIGHT_TIMEOUT seconds per retain POST (default 600) Point HINDSIGHT_BANK at a throwaway bank. The eval writes into it. @@ -31,7 +30,8 @@ def __init__(self, url: Optional[str] = None, token: Optional[str] = None, self.bank = bank or os.environ.get("HINDSIGHT_BANK", "memeval-scratch") if insecure is None: insecure = bool(os.environ.get("HINDSIGHT_INSECURE", "")) - self.batch = int(os.environ.get("HINDSIGHT_BATCH", "20")) + # a batch of 0 would make range() raise on the first retain + self.batch = max(1, int(os.environ.get("HINDSIGHT_BATCH", "20"))) self.timeout = int(os.environ.get("HINDSIGHT_TIMEOUT", "600")) self.ctx = ssl.create_default_context() if insecure: @@ -39,13 +39,9 @@ def __init__(self, url: Optional[str] = None, token: Optional[str] = None, self.ctx.verify_mode = ssl.CERT_NONE def _call(self, method: str, path: str, body=None, timeout: int = 300): - data = json.dumps(body).encode() if body is not None else None - req = urllib.request.Request( - self.url + path, data=data, method=method, - headers={"Authorization": f"Bearer {self.token}", "Content-Type": "application/json"}) - with urllib.request.urlopen(req, timeout=timeout, context=self.ctx) as r: - raw = r.read().decode() - return json.loads(raw) if raw else {} + return request_json(self.url + path, body=body, method=method, timeout=timeout, + headers={"Authorization": f"Bearer {self.token}"}, + context=self.ctx) def _bank(self, suffix: str) -> str: return f"/v1/default/banks/{self.bank}{suffix}" @@ -82,10 +78,10 @@ def recall(self, query: str, k: int = 10) -> List[Dict]: return out def consolidate(self) -> None: - try: - self._call("POST", self._bank("/consolidate"), {}) - except Exception: - pass # bank may have observations off; nothing to consolidate + # Deliberately unguarded. Runner.consolidate records a failure in the run log + # and continues, and doctor reports it; swallowing it here would log ok=True + # for a consolidation that never ran (observations off, server down, 4xx). + self._call("POST", self._bank("/consolidate"), {}) def supersede(self, doc_id: str, content: str, tags: Optional[list] = None) -> None: self.retain([{"content": content, "id": doc_id, "tags": tags or []}]) diff --git a/memeval/adapters/letta.py b/memeval/adapters/letta.py index aafd86e..e0a4eb8 100644 --- a/memeval/adapters/letta.py +++ b/memeval/adapters/letta.py @@ -1,9 +1,7 @@ -import json import os -import urllib.error import urllib.parse -import urllib.request +from ._http import request_json from .base import MemoryAdapter @@ -36,26 +34,8 @@ def _agent_path(self, suffix=""): return f"{self.url}/v1/agents/{agent}/archival-memory{suffix}" def _call(self, url, body=None, timeout=120): - headers = {"Content-Type": "application/json"} - if self.token: - headers["Authorization"] = f"Bearer {self.token}" - data = json.dumps(body).encode() if body is not None else None - req = urllib.request.Request(url, data=data, - method="POST" if body is not None else "GET", - headers=headers) - try: - with urllib.request.urlopen(req, timeout=timeout) as r: - raw = r.read().decode() - return json.loads(raw) if raw else {} - except urllib.error.HTTPError as e: - detail = "" - try: - detail = e.read().decode()[:200] - except Exception: - pass - raise urllib.error.HTTPError( - e.url, e.code, f"{e.reason} — {detail}" if detail else e.reason, - e.headers, None) from None + headers = {"Authorization": f"Bearer {self.token}"} if self.token else {} + return request_json(url, body=body, headers=headers, timeout=timeout) def retain(self, items): for i in items: diff --git a/memeval/adapters/mem0.py b/memeval/adapters/mem0.py index f0637a1..80ceb13 100644 --- a/memeval/adapters/mem0.py +++ b/memeval/adapters/mem0.py @@ -1,8 +1,6 @@ -import json import os -import urllib.error -import urllib.request +from ._http import request_json from .base import MemoryAdapter @@ -30,24 +28,9 @@ def __init__(self): self.infer = os.environ.get("MEM0_INFER", "1") not in ("0", "false", "no") def _call(self, path, body, timeout=120): - headers = {"Content-Type": "application/json"} - if self.api_key: - headers["X-API-Key"] = self.api_key - req = urllib.request.Request(self.url + path, data=json.dumps(body).encode(), - method="POST", headers=headers) - try: - with urllib.request.urlopen(req, timeout=timeout) as r: - raw = r.read().decode() - return json.loads(raw) if raw else {} - except urllib.error.HTTPError as e: - detail = "" - try: - detail = e.read().decode()[:200] - except Exception: - pass - raise urllib.error.HTTPError( - e.url, e.code, f"{e.reason} — {detail}" if detail else e.reason, - e.headers, None) from None + headers = {"X-API-Key": self.api_key} if self.api_key else {} + return request_json(self.url + path, body=body, method="POST", + headers=headers, timeout=timeout) def retain(self, items): for i in items: diff --git a/memeval/adapters/zep.py b/memeval/adapters/zep.py index 38f9d09..88805b4 100644 --- a/memeval/adapters/zep.py +++ b/memeval/adapters/zep.py @@ -1,9 +1,8 @@ -import json import os import urllib.error import urllib.parse -import urllib.request +from ._http import request_json from .base import MemoryAdapter @@ -34,24 +33,9 @@ def __init__(self): self.session = os.environ.get("ZEP_SESSION", "memeval-scratch") def _call(self, path, body, timeout=120): - headers = {"Content-Type": "application/json"} - if self.api_key: - headers["Authorization"] = f"Api-Key {self.api_key}" - req = urllib.request.Request(self.url + path, data=json.dumps(body).encode(), - method="POST", headers=headers) - try: - with urllib.request.urlopen(req, timeout=timeout) as r: - raw = r.read().decode() - return json.loads(raw) if raw else {} - except urllib.error.HTTPError as e: - detail = "" - try: - detail = e.read().decode()[:200] - except Exception: - pass - raise urllib.error.HTTPError( - e.url, e.code, f"{e.reason} — {detail}" if detail else e.reason, - e.headers, None) from None + headers = {"Authorization": f"Api-Key {self.api_key}"} if self.api_key else {} + return request_json(self.url + path, body=body, method="POST", + headers=headers, timeout=timeout) def prepare(self): for path, body in (("/api/v2/users", {"user_id": self.user}), diff --git a/tests/test_adapters_http.py b/tests/test_adapters_http.py index 5b4d9cd..def5a06 100644 --- a/tests/test_adapters_http.py +++ b/tests/test_adapters_http.py @@ -151,14 +151,72 @@ def test_hindsight_retain_batches(http, monkeypatch): assert len(posts[0][2]["items"]) == 20 and len(posts[2][2]["items"]) == 5 -def test_http_error_body_surfaces_in_message(monkeypatch): +@pytest.mark.parametrize("name", ["hindsight", "mem0", "letta", "zep", "example-rest"]) +def test_http_error_body_surfaces_for_every_adapter(monkeypatch, name): + # one shared helper: a 4xx from any store says why, not just the status line import urllib.error - monkeypatch.setenv("MEM0_URL", "http://mem0:8888") - from memeval.adapters.mem0 import Mem0Adapter + + from memeval.adapters import REGISTRY + monkeypatch.setenv("LETTA_AGENT_ID", "agent-1") def bad_request(req, timeout=None, context=None): raise urllib.error.HTTPError(req.full_url, 400, "Bad Request", {}, io.BytesIO(b'{"detail": "user_id required"}')) monkeypatch.setattr(urllib.request, "urlopen", bad_request) with pytest.raises(urllib.error.HTTPError, match="user_id required"): - Mem0Adapter().recall("q", 5) + REGISTRY[name]().recall("q", 5) + + +def test_example_rest_retain_and_recall(http, monkeypatch): + monkeypatch.setenv("MEMEVAL_REST_URL", "http://rest:8000/") + monkeypatch.setenv("MEMEVAL_REST_TOKEN", "tok") + monkeypatch.setenv("MEMEVAL_REST_USER", "u9") + from memeval.adapters.example_rest import ExampleRestAdapter + fake = http([{}]) + a = ExampleRestAdapter() + a.retain([{"content": "fact one"}]) + method, url, body, headers = fake.requests[0] + assert method == "POST" and url == "http://rest:8000/memories" + assert body == {"text": "fact one", "user_id": "u9"} + assert headers.get("Authorization") == "Bearer tok" + assert headers.get("Content-type") == "application/json" + + fake = http([{"results": [{"memory": "fact one", "score": 0.5}, {"text": "alt"}]}]) + hits = a.recall("fact", 1) + assert fake.requests[0][2] == {"query": "fact", "user_id": "u9"} + assert hits == [{"text": "fact one", "score": 0.5}] # k applied client-side + + +def test_request_json_defaults_to_get_without_body(http): + from memeval.adapters._http import request_json + fake = http([{"ok": True}]) + assert request_json("http://x/ping") == {"ok": True} + assert fake.requests[0][0] == "GET" and fake.requests[0][2] is None + fake = http([{}]) + request_json("http://x/put", body={"a": 1}, method="PUT") + assert fake.requests[0][0] == "PUT" and fake.requests[0][2] == {"a": 1} + + +def test_hindsight_batch_floor(monkeypatch): + monkeypatch.setenv("HINDSIGHT_BATCH", "0") + from memeval.adapters.hindsight import HindsightAdapter + assert HindsightAdapter().batch == 1 # used to raise: range() arg 3 must not be zero + + +def test_hindsight_consolidate_failure_reaches_the_run_log(monkeypatch, tmp_path): + import urllib.error + + from memeval.adapters.hindsight import HindsightAdapter + from memeval.events import RunLog, read_events + from memeval.runner import Runner + + def refuse(req, timeout=None, context=None): + raise urllib.error.URLError("connection refused") + monkeypatch.setattr(urllib.request, "urlopen", refuse) + with pytest.raises(urllib.error.URLError): + HindsightAdapter().consolidate() # adapter no longer swallows it... + log = RunLog(tmp_path / "run.jsonl") + assert Runner(HindsightAdapter(), log).consolidate() is False # ...the Runner guards it + log.close() + ev = [e for e in read_events(log.path) if e["ev"] == "consolidate"][0] + assert ev["ok"] is False and "refused" in ev["error"] # and the log tells the truth