diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f4027e..507b91b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,15 +12,19 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - - run: pip install "ruff>=0.8,<1" + - run: pip install -e ".[dev]" - run: ruff check . test: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.12" - - run: pip install -e ".[all]" pytest pytest-asyncio + python-version: ${{ matrix.python-version }} + - run: pip install -e ".[all,dev]" - run: pytest -q diff --git a/README.md b/README.md index b192bfc..945e88d 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ That's 119 paraphrased queries, and the two rank-1 misses are the dataset doing **Adversarial.** Retain a small set of facts chosen to invite a bad merge, trigger the store's consolidation, and check whether recall now returns a statement no source made. Then feed corrections and check whether the fabrication goes away. A store that invents a fact and won't retract it is worse than one that misses. A check that errors is reported as an error, never as "clean". -**Silent write loss.** After retain, memeval polls until the stored docs are actually recallable, instead of trusting the write acknowledgment. This caught a real case during this repo's own release verification — the store returned success and the fact never became findable: +**Silent write loss.** After retain, memeval polls a sample of docs spread across the whole corpus until each is actually recallable, instead of trusting the write acknowledgment. This caught a real case during this repo's own release verification — the store returned success and the fact never became findable: ``` WARNING: 1 doc(s) never became recallable (silent write loss?): nw-06 @@ -41,9 +41,9 @@ Rank scores can come out perfect while a store quietly drops writes. This check git clone https://github.com/nuk3s/memeval cd memeval pip install -e . # core: stdlib only - pip install -e .[tui] # + replay TUI (Textual) - pip install -e .[charts] # + SVG/PNG charts (matplotlib) - pip install -e .[all] + pip install -e ".[tui]" # + replay TUI (Textual) + pip install -e ".[charts]" # + SVG/PNG charts (matplotlib) + pip install -e ".[all]" Python 3.10 or newer. @@ -95,7 +95,9 @@ numbers from it, and send a fix if the docs lied. Writing an adapter is ~40 lines: subclass `MemoryAdapter` from `memeval/adapters/base.py`, implement `retain(items)` and `recall(query, k)`, optionally `consolidate()`, `supersede()`, `prepare()`, and `config()`, then register it in -`memeval/adapters/__init__.py`. Copy `example_rest.py` and start from there. +`memeval/adapters/__init__.py`. Copy `example_rest.py` and start from there; the bundled adapters share +`memeval/adapters/_http.py` for JSON requests, which folds the server's error body into +the exception message so a 4xx in the run log says why. `memeval doctor` is the conformance check: it writes three sentinel docs, polls the recall roundtrip, and probes the optional capabilities. Partial visibility is an exit-1 PARTIAL, not a pass. diff --git a/docs/run-log.md b/docs/run-log.md index e2ca967..5905355 100644 --- a/docs/run-log.md +++ b/docs/run-log.md @@ -19,7 +19,8 @@ line, in the order things happened. Same-second collisions get a `-2`, `-3`… s Notes. The `ready` event is the readiness poll that replaced a fixed sleep: after retain, memeval -polls recall on a sample of corpus docs (word-overlap match, ≥50% of the doc's content +polls recall on up to 10 docs spread evenly across the corpus, first and last always +included so a dropped tail batch is caught (word-overlap match, ≥50% of the doc's content words, because extracting stores rewrite text) until each appears or `--ready-timeout` runs out. A doc in `missing` was accepted by the store but never became recallable — silent write loss. The run continues; the event keeps the evidence. @@ -28,5 +29,9 @@ Hit text is truncated in the log for size; fabrication verdicts are computed on recalled text before truncation, so a `verdict` can reference a match you can't fully see in the neighbouring `query` event. +`--retries N` re-issues a recall only on transport errors and HTTP 5xx/408/425/429. Any +other 4xx (bad token, unknown route) fails identically every time, so it becomes an error row +on the first attempt. + Latency percentiles count only error-free queries; `errors` counts the rest. Error messages are never empty strings — a bare exception records its class name. diff --git a/memeval/__main__.py b/memeval/__main__.py new file mode 100644 index 0000000..dad7b45 --- /dev/null +++ b/memeval/__main__.py @@ -0,0 +1,5 @@ +"""`python -m memeval` entry point; the console script is memeval.cli:main.""" +from .cli import main + +if __name__ == "__main__": + main() diff --git a/memeval/adapters/_http.py b/memeval/adapters/_http.py new file mode 100644 index 0000000..152705e --- /dev/null +++ b/memeval/adapters/_http.py @@ -0,0 +1,40 @@ +"""Shared urllib plumbing for the bundled REST adapters. Stdlib only. + +Every adapter used to carry its own copy of this, and only some of them folded the +server's error body into the exception. Keeping it here means a 4xx in the run log +says *why* for all of them. +""" +import json +import urllib.error +import urllib.request + +ERROR_BODY_LIMIT = 200 + + +def request_json(url, body=None, method=None, headers=None, timeout=120, context=None): + """Send `body` as JSON (POST by default when a body is given, GET otherwise) and + decode the JSON reply; an empty reply decodes to {}. + + An HTTPError is re-raised with the first ERROR_BODY_LIMIT chars of the response + body in its message, so the error row reads "400 Bad Request — {detail...}" + instead of just the status line. + """ + data = json.dumps(body).encode() if body is not None else None + if method is None: + method = "POST" if data is not None else "GET" + req = urllib.request.Request( + url, data=data, method=method, + headers={"Content-Type": "application/json", **(headers or {})}) + try: + with urllib.request.urlopen(req, timeout=timeout, context=context) as r: + raw = r.read().decode() + return json.loads(raw) if raw else {} + except urllib.error.HTTPError as e: + detail = "" + try: + detail = e.read().decode()[:ERROR_BODY_LIMIT] + except Exception: + pass + raise urllib.error.HTTPError( + e.url, e.code, f"{e.reason} — {detail}" if detail else e.reason, + e.headers, None) from None diff --git a/memeval/adapters/example_rest.py b/memeval/adapters/example_rest.py index 2557812..5234416 100644 --- a/memeval/adapters/example_rest.py +++ b/memeval/adapters/example_rest.py @@ -1,8 +1,7 @@ -import json import os -import urllib.request from typing import Dict, List +from ._http import request_json from .base import MemoryAdapter @@ -27,12 +26,8 @@ def __init__(self, url=None, token=None, user=None): self.user = user or os.environ.get("MEMEVAL_REST_USER", "memeval") def _post(self, path: str, body: Dict, timeout: int = 120): - req = urllib.request.Request( - self.url + path, data=json.dumps(body).encode(), method="POST", - headers={"Authorization": f"Bearer {self.token}", "Content-Type": "application/json"}) - with urllib.request.urlopen(req, timeout=timeout) as r: - raw = r.read().decode() - return json.loads(raw) if raw else {} + return request_json(self.url + path, body=body, method="POST", timeout=timeout, + headers={"Authorization": f"Bearer {self.token}"}) def retain(self, items: List[Dict]) -> None: for i in items: diff --git a/memeval/adapters/hindsight.py b/memeval/adapters/hindsight.py index 1a46eb4..0767cd1 100644 --- a/memeval/adapters/hindsight.py +++ b/memeval/adapters/hindsight.py @@ -1,9 +1,8 @@ -import json import os import ssl -import urllib.request from typing import Dict, List, Optional +from ._http import request_json from .base import MemoryAdapter @@ -15,7 +14,7 @@ class HindsightAdapter(MemoryAdapter): HINDSIGHT_TOKEN bearer token (default empty) HINDSIGHT_BANK bank name (default memeval-scratch) HINDSIGHT_INSECURE any non-empty value skips TLS verification (for internal CAs) - HINDSIGHT_BATCH items per retain POST (default 20) + HINDSIGHT_BATCH items per retain POST (default 20, minimum 1) HINDSIGHT_TIMEOUT seconds per retain POST (default 600) Point HINDSIGHT_BANK at a throwaway bank. The eval writes into it. @@ -31,7 +30,8 @@ def __init__(self, url: Optional[str] = None, token: Optional[str] = None, self.bank = bank or os.environ.get("HINDSIGHT_BANK", "memeval-scratch") if insecure is None: insecure = bool(os.environ.get("HINDSIGHT_INSECURE", "")) - self.batch = int(os.environ.get("HINDSIGHT_BATCH", "20")) + # a batch of 0 would make range() raise on the first retain + self.batch = max(1, int(os.environ.get("HINDSIGHT_BATCH", "20"))) self.timeout = int(os.environ.get("HINDSIGHT_TIMEOUT", "600")) self.ctx = ssl.create_default_context() if insecure: @@ -39,13 +39,9 @@ def __init__(self, url: Optional[str] = None, token: Optional[str] = None, self.ctx.verify_mode = ssl.CERT_NONE def _call(self, method: str, path: str, body=None, timeout: int = 300): - data = json.dumps(body).encode() if body is not None else None - req = urllib.request.Request( - self.url + path, data=data, method=method, - headers={"Authorization": f"Bearer {self.token}", "Content-Type": "application/json"}) - with urllib.request.urlopen(req, timeout=timeout, context=self.ctx) as r: - raw = r.read().decode() - return json.loads(raw) if raw else {} + return request_json(self.url + path, body=body, method=method, timeout=timeout, + headers={"Authorization": f"Bearer {self.token}"}, + context=self.ctx) def _bank(self, suffix: str) -> str: return f"/v1/default/banks/{self.bank}{suffix}" @@ -82,10 +78,10 @@ def recall(self, query: str, k: int = 10) -> List[Dict]: return out def consolidate(self) -> None: - try: - self._call("POST", self._bank("/consolidate"), {}) - except Exception: - pass # bank may have observations off; nothing to consolidate + # Deliberately unguarded. Runner.consolidate records a failure in the run log + # and continues, and doctor reports it; swallowing it here would log ok=True + # for a consolidation that never ran (observations off, server down, 4xx). + self._call("POST", self._bank("/consolidate"), {}) def supersede(self, doc_id: str, content: str, tags: Optional[list] = None) -> None: self.retain([{"content": content, "id": doc_id, "tags": tags or []}]) diff --git a/memeval/adapters/letta.py b/memeval/adapters/letta.py index aafd86e..e0a4eb8 100644 --- a/memeval/adapters/letta.py +++ b/memeval/adapters/letta.py @@ -1,9 +1,7 @@ -import json import os -import urllib.error import urllib.parse -import urllib.request +from ._http import request_json from .base import MemoryAdapter @@ -36,26 +34,8 @@ def _agent_path(self, suffix=""): return f"{self.url}/v1/agents/{agent}/archival-memory{suffix}" def _call(self, url, body=None, timeout=120): - headers = {"Content-Type": "application/json"} - if self.token: - headers["Authorization"] = f"Bearer {self.token}" - data = json.dumps(body).encode() if body is not None else None - req = urllib.request.Request(url, data=data, - method="POST" if body is not None else "GET", - headers=headers) - try: - with urllib.request.urlopen(req, timeout=timeout) as r: - raw = r.read().decode() - return json.loads(raw) if raw else {} - except urllib.error.HTTPError as e: - detail = "" - try: - detail = e.read().decode()[:200] - except Exception: - pass - raise urllib.error.HTTPError( - e.url, e.code, f"{e.reason} — {detail}" if detail else e.reason, - e.headers, None) from None + headers = {"Authorization": f"Bearer {self.token}"} if self.token else {} + return request_json(url, body=body, headers=headers, timeout=timeout) def retain(self, items): for i in items: diff --git a/memeval/adapters/mem0.py b/memeval/adapters/mem0.py index f0637a1..80ceb13 100644 --- a/memeval/adapters/mem0.py +++ b/memeval/adapters/mem0.py @@ -1,8 +1,6 @@ -import json import os -import urllib.error -import urllib.request +from ._http import request_json from .base import MemoryAdapter @@ -30,24 +28,9 @@ def __init__(self): self.infer = os.environ.get("MEM0_INFER", "1") not in ("0", "false", "no") def _call(self, path, body, timeout=120): - headers = {"Content-Type": "application/json"} - if self.api_key: - headers["X-API-Key"] = self.api_key - req = urllib.request.Request(self.url + path, data=json.dumps(body).encode(), - method="POST", headers=headers) - try: - with urllib.request.urlopen(req, timeout=timeout) as r: - raw = r.read().decode() - return json.loads(raw) if raw else {} - except urllib.error.HTTPError as e: - detail = "" - try: - detail = e.read().decode()[:200] - except Exception: - pass - raise urllib.error.HTTPError( - e.url, e.code, f"{e.reason} — {detail}" if detail else e.reason, - e.headers, None) from None + headers = {"X-API-Key": self.api_key} if self.api_key else {} + return request_json(self.url + path, body=body, method="POST", + headers=headers, timeout=timeout) def retain(self, items): for i in items: diff --git a/memeval/adapters/zep.py b/memeval/adapters/zep.py index 38f9d09..88805b4 100644 --- a/memeval/adapters/zep.py +++ b/memeval/adapters/zep.py @@ -1,9 +1,8 @@ -import json import os import urllib.error import urllib.parse -import urllib.request +from ._http import request_json from .base import MemoryAdapter @@ -34,24 +33,9 @@ def __init__(self): self.session = os.environ.get("ZEP_SESSION", "memeval-scratch") def _call(self, path, body, timeout=120): - headers = {"Content-Type": "application/json"} - if self.api_key: - headers["Authorization"] = f"Api-Key {self.api_key}" - req = urllib.request.Request(self.url + path, data=json.dumps(body).encode(), - method="POST", headers=headers) - try: - with urllib.request.urlopen(req, timeout=timeout) as r: - raw = r.read().decode() - return json.loads(raw) if raw else {} - except urllib.error.HTTPError as e: - detail = "" - try: - detail = e.read().decode()[:200] - except Exception: - pass - raise urllib.error.HTTPError( - e.url, e.code, f"{e.reason} — {detail}" if detail else e.reason, - e.headers, None) from None + headers = {"Authorization": f"Api-Key {self.api_key}"} if self.api_key else {} + return request_json(self.url + path, body=body, method="POST", + headers=headers, timeout=timeout) def prepare(self): for path, body in (("/api/v2/users", {"user_id": self.user}), diff --git a/memeval/adversarial.py b/memeval/adversarial.py index bb9bf15..f657cbe 100644 --- a/memeval/adversarial.py +++ b/memeval/adversarial.py @@ -14,7 +14,46 @@ def load_probe(path): with open(path, encoding="utf-8") as f: - return json.load(f) + return validate_probe(json.load(f)) + + +def _nonempty_str(v): + return isinstance(v, str) and bool(v.strip()) + + +def validate_probe(probe): + """Shape-check the probe at load time, so a typo fails before retain writes into + the store rather than as a KeyError or re.error halfway through the run.""" + if not isinstance(probe, dict): + raise ValueError("probe must be a JSON object") + corpus = probe.get("corpus") + if not isinstance(corpus, list) or not corpus: + raise ValueError("probe 'corpus' must be a non-empty list") + for n, d in enumerate(corpus, 1): + if not isinstance(d, dict) or not _nonempty_str(d.get("content")): + raise ValueError(f"probe corpus item {n}: missing or empty 'content'") + checks = probe.get("checks") + if not isinstance(checks, list) or not checks: + raise ValueError("probe 'checks' must be a non-empty list") + for n, c in enumerate(checks, 1): + if not isinstance(c, dict) or not _nonempty_str(c.get("query")): + raise ValueError(f"probe check {n}: missing or empty 'query'") + for key in ("forbidden", "expected"): + pats = c.get(key, []) + if not isinstance(pats, list) or not all(isinstance(p, str) for p in pats): + raise ValueError(f"probe check {n}: {key!r} must be a list of regex strings") + for p in pats: + try: + re.compile(p) + except re.error as e: + raise ValueError(f"probe check {n}: bad {key} regex {p!r}: {e}") from None + corrections = probe.get("corrections", []) + if not isinstance(corrections, list) or not all(isinstance(x, str) for x in corrections): + raise ValueError("probe 'corrections' must be a list of strings") + wait = probe.get("consolidate_wait", 2) + if isinstance(wait, bool) or not isinstance(wait, (int, float)) or wait < 0: + raise ValueError("probe 'consolidate_wait' must be a non-negative number of seconds") + return probe def _check(runner, checks, phase): diff --git a/memeval/cli.py b/memeval/cli.py index 98f01f6..f3c6828 100644 --- a/memeval/cli.py +++ b/memeval/cli.py @@ -5,7 +5,7 @@ from .adapters import REGISTRY from .adversarial import load_probe, run_adversarial from .events import SCHEMA_VERSION, RunLog, file_sha256, read_events, redact, run_path -from .rank import load_jsonl, run_rank +from .rank import load_corpus, load_gold, run_rank from .render.text import format_ab, format_adversarial, format_summary from .runner import Runner @@ -46,8 +46,8 @@ def _start_run(args, adapter, kind, files): def _rank_one(args, name, kind): - corpus = _load_input(args.corpus, load_jsonl, "corpus") - gold = _load_input(args.gold, load_jsonl, "gold") + corpus = _load_input(args.corpus, load_corpus, "corpus") + gold = _load_input(args.gold, load_gold, "gold") adapter = build_adapter(name) path, log, runner = _start_run(args, adapter, kind, {"corpus": args.corpus, "gold": args.gold}) @@ -116,7 +116,7 @@ def cmd_runs(args): def cmd_adapters(args): for name, cls in sorted(REGISTRY.items()): - doc = (cls.__doc__ or "").strip().splitlines()[0] + doc = next(iter((cls.__doc__ or "").strip().splitlines()), "") print(f"{name:>14} {cls.status:>8} {doc}") @@ -197,7 +197,10 @@ def cmd_replay(args): from .render.tui import ReplayApp except ImportError: sys.exit("replay needs textual — pip install 'memeval[tui]'") - evs = read_events(args.run) + try: + evs = read_events(args.run) + except OSError as e: + sys.exit(f"cannot read run log {args.run}: {e}") if not evs: sys.exit(f"empty run log: {args.run}") ReplayApp(evs).run() @@ -208,6 +211,8 @@ def cmd_chart(args): from .render.charts import render_ab_chart, render_chart except ImportError: sys.exit("chart needs matplotlib — pip install 'memeval[charts]'") + if args.ab and args.run: + sys.exit("pass one run file, or --ab RUN_A RUN_B — not both") try: if args.ab: out = render_ab_chart(args.ab[0], args.ab[1], out=args.out, png=args.png) @@ -217,6 +222,8 @@ def cmd_chart(args): sys.exit("pass one run file, or --ab RUN_A RUN_B") except ValueError as e: sys.exit(str(e)) + except OSError as e: + sys.exit(f"chart failed: {e}") print(f"wrote {out}") diff --git a/memeval/events.py b/memeval/events.py index cbdfae0..77a3d0d 100644 --- a/memeval/events.py +++ b/memeval/events.py @@ -87,7 +87,9 @@ def read_events(path): if not line.strip(): continue try: - events.append(json.loads(line)) + rec = json.loads(line) except json.JSONDecodeError: continue + if isinstance(rec, dict): # an event is always an object; a foreign file's + events.append(rec) # arrays/scalars would crash every consumer's .get return events diff --git a/memeval/rank.py b/memeval/rank.py index e0281e6..bf03daa 100644 --- a/memeval/rank.py +++ b/memeval/rank.py @@ -4,8 +4,40 @@ def load_jsonl(path): + rows = [] with open(path, encoding="utf-8") as f: - return [json.loads(line) for line in f if line.strip()] + for n, line in enumerate(f, 1): + if not line.strip(): + continue + try: + rows.append(json.loads(line)) + except json.JSONDecodeError as e: + raise ValueError(f"line {n}: {e}") from None + return rows + + +def _validate_rows(rows, label, fields): + """Every row is an object whose `fields` are non-empty strings. Fails loud at + load time with a row number, instead of a KeyError mid-run after retain.""" + for n, row in enumerate(rows, 1): + if not isinstance(row, dict): + raise ValueError( + f"{label} row {n}: expected a JSON object, got {type(row).__name__}") + for field in fields: + v = row.get(field) + if not isinstance(v, str) or not v.strip(): + raise ValueError(f"{label} row {n}: missing or empty {field!r}") + return rows + + +def load_corpus(path): + return _validate_rows(load_jsonl(path), "corpus", ("content",)) + + +def load_gold(path): + """`answer` is checked for presence only: a malformed regex is an error row at + query time (Runner.query), not a rejected file — see the README.""" + return _validate_rows(load_jsonl(path), "gold", ("query", "answer")) # nearest-rank with round(); differs from numpy's linear interpolation — diff --git a/memeval/runner.py b/memeval/runner.py index 590e921..743009a 100644 --- a/memeval/runner.py +++ b/memeval/runner.py @@ -2,18 +2,32 @@ readiness polling, event emission. Hardening lives here, written once.""" import re import time +import urllib.error HIT_TEXT_LIMIT = 300 +READY_SAMPLE = 10 # ConnectionError and TimeoutError are OSError subclasses — listed anyway for # readability. urllib.error.URLError is also an OSError subclass, so plain-http # adapter failures land here too. RETRYABLE = (ConnectionError, TimeoutError, OSError) +# HTTP statuses that can plausibly clear on a second attempt. Any other 4xx (401, +# 404, 422...) is a configuration or request problem that fails identically every +# time; retrying it only delays the error row and inflates its recorded latency. +RETRYABLE_HTTP = frozenset({408, 425, 429}) def _ms(t0): return round((time.monotonic() - t0) * 1000) +def _retryable(e): + # HTTPError is an OSError subclass, so without this check a bad token would be + # retried --retries times per query. + if isinstance(e, urllib.error.HTTPError): + return e.code >= 500 or e.code in RETRYABLE_HTTP + return isinstance(e, RETRYABLE) + + def _coerce_score(v): try: return None if v is None else float(v) @@ -40,6 +54,18 @@ def _overlaps(source, text, threshold=0.5): return found / len(words) >= threshold +def _spread(corpus, sample): + """Up to `sample` (index, doc) pairs spaced evenly across the corpus, first and + last always included. Stores retain in batches, and a batch dropped at the tail + of a big corpus is invisible to a head-only sample — which is exactly the silent + write loss the readiness poll exists to catch.""" + n = len(corpus) + if n <= sample: + return list(enumerate(corpus)) + picks = sorted({round(i * (n - 1) / max(sample - 1, 1)) for i in range(sample)}) + return [(i, corpus[i]) for i in picks] + + class Runner: def __init__(self, adapter, log, retries=1, ready_timeout=60.0, poll_interval=1.0, echo=None): @@ -76,12 +102,12 @@ def consolidate(self): self.log.emit("consolidate", ms=_ms(t0), ok=True, error=None) return True - def wait_ready(self, corpus, sample=10): + def wait_ready(self, corpus, sample=READY_SAMPLE): """Poll until each sampled doc is recallable; report the ones that never are. A doc missing at timeout is the 'success:true but not persisted' failure — recorded, warned about, and the run continues.""" - docs = corpus[:sample] - pending = dict(enumerate(docs)) + pending = dict(_spread(corpus, sample)) # corpus index -> doc + n_sampled = len(pending) t0 = time.monotonic() while True: for idx in list(pending): @@ -100,7 +126,7 @@ def wait_ready(self, corpus, sample=10): time.sleep(min(self.poll_interval, remaining)) missing = [pending[i].get("id") or f"#{i}" for i in sorted(pending)] self.log.emit("ready", polled_s=round(time.monotonic() - t0, 1), - verified_n=len(docs) - len(missing), sampled_n=len(docs), + verified_n=n_sampled - len(missing), sampled_n=n_sampled, missing=missing) if missing: self.echo(f"WARNING: {len(missing)} doc(s) never became recallable " @@ -118,11 +144,10 @@ def query(self, query, k, phase, rep=1, pattern=None): hits = _normalize_hits(self.adapter.recall(query, k)) ms, err = _ms(t0), None break - except RETRYABLE as e: - ms, err = _ms(t0), str(e) or e.__class__.__name__ except Exception as e: ms, err = _ms(t0), str(e) or e.__class__.__name__ - break + if not _retryable(e): + break rank = None if err is None and pattern: try: diff --git a/pyproject.toml b/pyproject.toml index b478126..fc65c21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,9 +9,10 @@ dependencies = [] keywords = ["agent-memory", "evaluation", "rag", "hindsight", "mem0", "vector-search"] [project.optional-dependencies] -tui = ["textual>=0.80"] +tui = ["textual>=2.0"] charts = ["matplotlib>=3.8"] -all = ["textual>=0.80", "matplotlib>=3.8"] +all = ["textual>=2.0", "matplotlib>=3.8"] +dev = ["pytest", "pytest-asyncio", "ruff>=0.8,<1"] [project.urls] Repository = "https://github.com/nuk3s/memeval" diff --git a/tests/test_adapters_http.py b/tests/test_adapters_http.py index 5b4d9cd..def5a06 100644 --- a/tests/test_adapters_http.py +++ b/tests/test_adapters_http.py @@ -151,14 +151,72 @@ def test_hindsight_retain_batches(http, monkeypatch): assert len(posts[0][2]["items"]) == 20 and len(posts[2][2]["items"]) == 5 -def test_http_error_body_surfaces_in_message(monkeypatch): +@pytest.mark.parametrize("name", ["hindsight", "mem0", "letta", "zep", "example-rest"]) +def test_http_error_body_surfaces_for_every_adapter(monkeypatch, name): + # one shared helper: a 4xx from any store says why, not just the status line import urllib.error - monkeypatch.setenv("MEM0_URL", "http://mem0:8888") - from memeval.adapters.mem0 import Mem0Adapter + + from memeval.adapters import REGISTRY + monkeypatch.setenv("LETTA_AGENT_ID", "agent-1") def bad_request(req, timeout=None, context=None): raise urllib.error.HTTPError(req.full_url, 400, "Bad Request", {}, io.BytesIO(b'{"detail": "user_id required"}')) monkeypatch.setattr(urllib.request, "urlopen", bad_request) with pytest.raises(urllib.error.HTTPError, match="user_id required"): - Mem0Adapter().recall("q", 5) + REGISTRY[name]().recall("q", 5) + + +def test_example_rest_retain_and_recall(http, monkeypatch): + monkeypatch.setenv("MEMEVAL_REST_URL", "http://rest:8000/") + monkeypatch.setenv("MEMEVAL_REST_TOKEN", "tok") + monkeypatch.setenv("MEMEVAL_REST_USER", "u9") + from memeval.adapters.example_rest import ExampleRestAdapter + fake = http([{}]) + a = ExampleRestAdapter() + a.retain([{"content": "fact one"}]) + method, url, body, headers = fake.requests[0] + assert method == "POST" and url == "http://rest:8000/memories" + assert body == {"text": "fact one", "user_id": "u9"} + assert headers.get("Authorization") == "Bearer tok" + assert headers.get("Content-type") == "application/json" + + fake = http([{"results": [{"memory": "fact one", "score": 0.5}, {"text": "alt"}]}]) + hits = a.recall("fact", 1) + assert fake.requests[0][2] == {"query": "fact", "user_id": "u9"} + assert hits == [{"text": "fact one", "score": 0.5}] # k applied client-side + + +def test_request_json_defaults_to_get_without_body(http): + from memeval.adapters._http import request_json + fake = http([{"ok": True}]) + assert request_json("http://x/ping") == {"ok": True} + assert fake.requests[0][0] == "GET" and fake.requests[0][2] is None + fake = http([{}]) + request_json("http://x/put", body={"a": 1}, method="PUT") + assert fake.requests[0][0] == "PUT" and fake.requests[0][2] == {"a": 1} + + +def test_hindsight_batch_floor(monkeypatch): + monkeypatch.setenv("HINDSIGHT_BATCH", "0") + from memeval.adapters.hindsight import HindsightAdapter + assert HindsightAdapter().batch == 1 # used to raise: range() arg 3 must not be zero + + +def test_hindsight_consolidate_failure_reaches_the_run_log(monkeypatch, tmp_path): + import urllib.error + + from memeval.adapters.hindsight import HindsightAdapter + from memeval.events import RunLog, read_events + from memeval.runner import Runner + + def refuse(req, timeout=None, context=None): + raise urllib.error.URLError("connection refused") + monkeypatch.setattr(urllib.request, "urlopen", refuse) + with pytest.raises(urllib.error.URLError): + HindsightAdapter().consolidate() # adapter no longer swallows it... + log = RunLog(tmp_path / "run.jsonl") + assert Runner(HindsightAdapter(), log).consolidate() is False # ...the Runner guards it + log.close() + ev = [e for e in read_events(log.path) if e["ev"] == "consolidate"][0] + assert ev["ok"] is False and "refused" in ev["error"] # and the log tells the truth diff --git a/tests/test_adversarial.py b/tests/test_adversarial.py index 6d00f55..05426ed 100644 --- a/tests/test_adversarial.py +++ b/tests/test_adversarial.py @@ -1,3 +1,7 @@ +import json +from pathlib import Path + +import pytest from conftest import FakeAdapter from memeval.adversarial import run_adversarial @@ -54,13 +58,22 @@ def test_clean_store_stays_clean(tmp_path): def test_negation_correction_not_flagged_as_fabrication(tmp_path): - # the correction itself contains "media ... 8080" but with a negation cue - a = FakeAdapter() - r, _ = run(tmp_path, a) - a.retain([{"content": "the media service is not on port 8080"}]) - # direct check: a store returning only the negated correction must stay clean - from memeval.adversarial import _NEG - assert _NEG.search("the media service is not on port 8080") + # a store that recalls only the negated correction restates "media ... 8080" in + # order to deny it: the forbidden regex matches that text, and the check must + # skip it or every correction would read as a fabrication + class RecallsCorrection(FakeAdapter): + def recall(self, query, k=10): + return [{"text": "the media service is not on port 8080", "score": 1.0}] + r, _ = run(tmp_path, RecallsCorrection(), consolidate=False) + assert r["before"][0]["fabricated"] == [] + assert r["after_corrections"][0]["fabricated"] == [] + + # ...while the same binding stated affirmatively is still caught + class Affirms(FakeAdapter): + def recall(self, query, k=10): + return [{"text": "the media service is on port 8080", "score": 1.0}] + r, _ = run(tmp_path, Affirms(), consolidate=False) + assert r["before"][0]["fabricated"] == ["media service[^.]{0,30}8080"] def test_no_corrections_probe(tmp_path): @@ -104,3 +117,32 @@ def consolidate(self): assert r is not None and not r["before"][0]["fabricated"] cons = [e for e in evs if e["ev"] == "consolidate"] assert cons and cons[0]["ok"] is False + + +EXAMPLE_PROBE = Path(__file__).resolve().parent.parent / "examples" / "adversarial.json" + + +@pytest.mark.parametrize("bad, msg", [ + ({"checks": [{"query": "q"}]}, "corpus"), + ({"corpus": [{"content": "x"}], "checks": []}, "checks"), + ({"corpus": [{"content": "x"}], "checks": [{"forbidden": []}]}, "check 1: missing"), + ({"corpus": [{"content": "x"}], "checks": [{"query": "q", "forbidden": ["("]}]}, + "bad forbidden regex"), + ({"corpus": [{"content": "x"}], "checks": [{"query": "q"}], "consolidate_wait": "soon"}, + "consolidate_wait"), + ({"corpus": [{"content": "x"}], "checks": [{"query": "q"}], "corrections": "nope"}, + "corrections"), + ([], "JSON object"), +]) +def test_load_probe_rejects_malformed_shapes(tmp_path, bad, msg): + from memeval.adversarial import load_probe + p = tmp_path / "probe.json" + p.write_text(json.dumps(bad)) + with pytest.raises(ValueError, match=msg): + load_probe(p) + + +def test_bundled_probe_passes_validation(): + from memeval.adversarial import load_probe + probe = load_probe(EXAMPLE_PROBE) + assert probe["checks"] and probe["corrections"] diff --git a/tests/test_charts.py b/tests/test_charts.py index 3281194..e9ca6b3 100644 --- a/tests/test_charts.py +++ b/tests/test_charts.py @@ -112,3 +112,15 @@ def test_chart_all_errors_still_renders(tmp_path, monkeypatch): from memeval.render.charts import render_chart render_chart(str(next(runs.glob("*.jsonl"))), out=str(tmp_path / "e.svg")) assert "3-char word is unique to its doc (trailing "x" blocks prefix substrings), + # so word-overlap readiness cannot be satisfied by a neighbouring doc + return [{"id": f"d{i}", "content": f"fact {i}: token{i}x alpha{i}x beta{i}x gamma{i}x"} + for i in range(n)] + + +def test_wait_ready_samples_across_whole_corpus(tmp_path): + # a doc dropped at the tail of a big corpus must be as visible as one at the head; + # a head-only sample (the old behaviour) reported [] here + corpus = _distinct_corpus(30) + r, log = make(tmp_path, FakeAdapter(drop_ids={"d0", "d29"}), + ready_timeout=0.3, poll_interval=0.1) + r.retain(corpus) + assert r.wait_ready(corpus) == ["d0", "d29"] + ready = [e for e in read_events(log.path) if e["ev"] == "ready"][0] + assert ready["sampled_n"] == 10 and ready["verified_n"] == 8 + + +def test_wait_ready_missing_label_uses_corpus_index(tmp_path): + corpus = [{"content": d["content"]} for d in _distinct_corpus(30)] # no ids + r, log = make(tmp_path, FakeAdapter(), ready_timeout=0.3, poll_interval=0.1) + r.adapter.docs = [] # nothing stored at all -> every sampled doc is missing + missing = r.wait_ready(corpus) + assert missing[0] == "#0" and missing[-1] == "#29" and len(missing) == 10 + + +def test_spread_sample_is_even_and_bounded(): + from memeval.runner import _spread + idx = [i for i, _ in _spread(list(range(125)), 10)] + assert idx[0] == 0 and idx[-1] == 124 and len(idx) == 10 and idx == sorted(idx) + assert [i for i, _ in _spread(list(range(3)), 10)] == [0, 1, 2] + assert _spread([], 10) == [] + assert [i for i, _ in _spread(list(range(5)), 1)] == [0] + + +def test_query_does_not_retry_client_http_errors(tmp_path): + import urllib.error + + class Unauthorized(FakeAdapter): + def __init__(self): + super().__init__() + self.calls = 0 + + def recall(self, query, k=10): + self.calls += 1 + raise urllib.error.HTTPError("http://x", 401, "Unauthorized", {}, None) + a = Unauthorized() + r, log = make(tmp_path, a, retries=3) + row = r.query("q", 5, phase="rank") + assert a.calls == 1, "a 401 fails identically every time; retrying it is waste" + assert "401" in row["error"] + + +def test_query_retries_server_and_rate_limit_http_errors(tmp_path): + import urllib.error + + class Flaky(FakeAdapter): + def __init__(self, codes): + super().__init__() + self.codes = list(codes) + self.calls = 0 + + def recall(self, query, k=10): + self.calls += 1 + if self.codes: + raise urllib.error.HTTPError("http://x", self.codes.pop(0), "err", {}, None) + return super().recall(query, k) + a = Flaky([503, 429]) + a.retain(CORPUS) + r, log = make(tmp_path, a, retries=2) + row = r.query("what port does the checkout service use?", 5, phase="rank", pattern="8080") + assert a.calls == 3 and row["error"] is None and row["rank"] == 1