diff --git a/.env.example b/.env.example index b5ec401c..8bc92cb9 100644 --- a/.env.example +++ b/.env.example @@ -15,6 +15,14 @@ BRAINY_WORKER_POLL_INTERVAL=2s # # Same-pin Mem0 Platform evals (evals/run_competitor_benchmark.py, --systems mem0): # MEM0_API_KEY=[REDACTED] +# +# OpMem external lanes (evals/run_opmem_external_live.py; see docs/research/opmem/external-lanes-setup.md): +# OPENAI_API_KEY= # mem0-oss, letta, langmem (pinned gpt-4.1-mini + text-embedding-3-small) +# ZEP_API_KEY= # Zep Cloud free tier only ($0 paid ceiling) +# LETTA_BASE_URL=http://127.0.0.1:8283 +# LETTA_APP_SERVER_TOKEN= # optional for local letta server --no-secure +# DATABASE_URL=postgresql://opmem:opmem@127.0.0.1:5432/opmem_langmem +# OPMEM_QDRANT_ROOT=/tmp/opmem-qdrant # HTTP boundary (API). Defaults: 5 MiB body, 10s header / 30s read / provider+60s write / 120s idle. # BRAINY_MAX_BODY_BYTES=5242880 diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/PHASE2_SUMMARY.md b/docs/benchmarks/artifacts/opmem-external-lanes/PHASE2_SUMMARY.md new file mode 100644 index 00000000..a6815226 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/PHASE2_SUMMARY.md @@ -0,0 +1,26 @@ +# OpMem external lanes — execution summary + +**Protocol:** 13 tasks / 47 steps (`fixtures/opmem/`). + +## Full comparison + +| Lane | Run ID | Score | Infra errors | SDK / infra | +| --- | --- | ---: | ---: | --- | +| mem0-oss | `opmem-phase2-full-mem0` | **9/13** | 0 | `mem0ai` 2.1.0, local Qdrant path | +| langmem | `opmem-phase2-full1` | **8/13** | 0 | Postgres + pgvector | +| letta | `opmem-phase2-letta-full` | **5/13** | 0 | `letta==0.11.7` | +| zep | `opmem-phase2-zep-full` | **3/9\*** | 0 | `zep-cloud==3.28.0`, free tier, $0 paid | + +**\* Zep (publication line):** Score **3/9** counts only tasks where forget is in scope for the lane. **Not supported** (listed separately, not counted as failures): `iso03_forget_isolated`, `sup01_basic_forget`, `sup02_targeted_forget`, `sup03_durable_forget` — the `zep-cloud` 3.28 thread/graph adapter has no delete path for OpMem `forget`. **Raw harness** on the unchanged artifact `live-full-opmem-phase2-zep-full.json` remains **3/13**. See `zep-score-interpretation-opmem-phase2-zep-full.json`. + +### Zep supported-task breakdown (`opmem-phase2-zep-full`) + +| Status | Tasks | +| --- | --- | +| **Passed (3)** | `cor02_correction_stickiness`, `cor03_revised_retrievable`, `dup01_idempotent_remember` | +| **Failed, scored (6)** | `cor01_basic_revision`, `iso01_subject_isolation`, `iso02_tenant_isolation`, `upd01_stale_fact`, `upd02_preference_change`, `upd03_state_supersession` | +| **Not supported — forget (4)** | `iso03_forget_isolated`, `sup01_basic_forget`, `sup02_targeted_forget`, `sup03_durable_forget` | + +## Spend + +Payload worst-case OpenAI ~$0.15 vs $3 ceiling (`cost-worksheet-payload-budget.json`). Zep lane $0 paid. diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/commands.log b/docs/benchmarks/artifacts/opmem-external-lanes/commands.log new file mode 100644 index 00000000..08cf05a7 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/commands.log @@ -0,0 +1,39 @@ + +# 2026-09-19T08:28:55Z run_id=opmem-phase2-smoke1 +r u n _ o p m e m _ e x t e r n a l _ l i v e . p y - - s m o k e - - l a n e s m e m 0 - o s s , l a n g m e m , z e p , l e t t a - - r u n - i d o p m e m - p h a s e 2 - s m o k e 1 + +# 2026-09-19T08:29:18Z run_id=opmem-phase2-smoke2 +r u n _ o p m e m _ e x t e r n a l _ l i v e . p y - - s m o k e - - l a n e s m e m 0 - o s s , l a n g m e m - - r u n - i d o p m e m - p h a s e 2 - s m o k e 2 + +# 2026-09-19T08:30:32Z run_id=opmem-phase2-full1 +r u n _ o p m e m _ e x t e r n a l _ l i v e . p y - - f u l l - - l a n e s m e m 0 - o s s , l a n g m e m - - r u n - i d o p m e m - p h a s e 2 - f u l l 1 - - s k i p - p r e f l i g h t + +# 2026-09-19T08:31:22Z run_id=opmem-phase2-full-mem0 +r u n _ o p m e m _ e x t e r n a l _ l i v e . p y - - f u l l - - l a n e s m e m 0 - o s s - - r u n - i d o p m e m - p h a s e 2 - f u l l - m e m 0 - - s k i p - p r e f l i g h t + +# 2026-09-21T17:43:52Z run_id=opmem-phase2-letta-smoke +r u n _ o p m e m _ e x t e r n a l _ l i v e . p y - - s m o k e - - l a n e s l e t t a - - r u n - i d o p m e m - p h a s e 2 - l e t t a - s m o k e + +# 2026-09-21T17:43:56Z run_id=opmem-phase2-zep-smoke +r u n _ o p m e m _ e x t e r n a l _ l i v e . p y - - s m o k e - - l a n e s z e p - - r u n - i d o p m e m - p h a s e 2 - z e p - s m o k e + +# 2026-09-21T17:44:03Z run_id=opmem-phase2-letta-full +r u n _ o p m e m _ e x t e r n a l _ l i v e . p y - - f u l l - - l a n e s l e t t a - - r u n - i d o p m e m - p h a s e 2 - l e t t a - f u l l + +# 2026-09-23T17:09:16Z run_id=opmem-phase2-zep-smoke-20260923 +r u n _ o p m e m _ e x t e r n a l _ l i v e . p y - - s m o k e - - l a n e s z e p - - r u n - i d o p m e m - p h a s e 2 - z e p - s m o k e - 2 0 2 6 0 9 2 3 + +# 2026-09-23T17:09:18Z run_id=opmem-phase2-zep-full +r u n _ o p m e m _ e x t e r n a l _ l i v e . p y - - f u l l - - l a n e s z e p - - r u n - i d o p m e m - p h a s e 2 - z e p - f u l l + +# 2026-09-23T17:55:34Z run_id=opmem-phase2-zep-smoke-20260923b +r u n _ o p m e m _ e x t e r n a l _ l i v e . p y - - s m o k e - - l a n e s z e p - - r u n - i d o p m e m - p h a s e 2 - z e p - s m o k e - 2 0 2 6 0 9 2 3 b + +# 2026-09-23T17:56:27Z run_id=opmem-phase2-zep-smoke-20260923c +r u n _ o p m e m _ e x t e r n a l _ l i v e . p y - - s m o k e - - l a n e s z e p - - r u n - i d o p m e m - p h a s e 2 - z e p - s m o k e - 2 0 2 6 0 9 2 3 c + +# 2026-09-23T17:56:54Z run_id=opmem-phase2-zep-smoke-final +r u n _ o p m e m _ e x t e r n a l _ l i v e . p y - - s m o k e - - l a n e s z e p - - r u n - i d o p m e m - p h a s e 2 - z e p - s m o k e - f i n a l + +# 2026-09-23T17:58:04Z run_id=opmem-phase2-zep-full +r u n _ o p m e m _ e x t e r n a l _ l i v e . p y - - f u l l - - l a n e s z e p - - r u n - i d o p m e m - p h a s e 2 - z e p - f u l l diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/cost-worksheet-dry-run.json b/docs/benchmarks/artifacts/opmem-external-lanes/cost-worksheet-dry-run.json new file mode 100644 index 00000000..c16ab779 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/cost-worksheet-dry-run.json @@ -0,0 +1,151 @@ +{ + "generated_utc": "2026-09-19T08:29:00Z", + "task_count": 13, + "step_count": 47, + "rates": { + "gpt-4.1-mini-2025-04-14": { + "input_per_m": 0.4, + "output_per_m": 1.6 + }, + "text-embedding-3-small": { + "per_m": 0.02 + } + }, + "pinned_models": { + "chat": "gpt-4.1-mini-2025-04-14", + "embed": "text-embedding-3-small" + }, + "ceilings_usd": { + "mem0-oss": 1.0, + "letta": 1.0, + "langmem": 1.0, + "zep": 0.0, + "total": 3.0 + }, + "lanes": [ + { + "lane": "langmem", + "operations": { + "remember": 21, + "recall": 19, + "revise": 3, + "forget": 4, + "reset": 13, + "http_calls": 47 + }, + "tokens": { + "chat_model": "gpt-4.1-mini-2025-04-14", + "embed_model": "text-embedding-3-small", + "chat_input_tokens": 409, + "chat_output_tokens": 66, + "embed_tokens": 191, + "estimated_usd": 0.000273 + }, + "product_constraints": [], + "credentials_present": false, + "missing_credentials": [ + "DATABASE_URL" + ], + "projected_usd": 0.000273, + "ceiling_usd": 1.0, + "within_ceiling": true, + "live_run_allowed": false, + "live_run_note": "Phase 1 dry-run only; enable after credentials present and measured estimate confirmed under ceiling." + }, + { + "lane": "letta", + "operations": { + "remember": 21, + "recall": 19, + "revise": 3, + "forget": 4, + "reset": 13, + "http_calls": 47 + }, + "tokens": { + "chat_model": "gpt-4.1-mini-2025-04-14", + "embed_model": "text-embedding-3-small", + "chat_input_tokens": 409, + "chat_output_tokens": 66, + "embed_tokens": 191, + "estimated_usd": 0.000273 + }, + "product_constraints": [], + "credentials_present": false, + "missing_credentials": [ + "LETTA_APP_SERVER_TOKEN" + ], + "projected_usd": 0.000273, + "ceiling_usd": 1.0, + "within_ceiling": true, + "live_run_allowed": false, + "live_run_note": "Phase 1 dry-run only; enable after credentials present and measured estimate confirmed under ceiling." + }, + { + "lane": "mem0-oss", + "operations": { + "remember": 21, + "recall": 19, + "revise": 3, + "forget": 4, + "reset": 13, + "http_calls": 47 + }, + "tokens": { + "chat_model": "gpt-4.1-mini-2025-04-14", + "embed_model": "text-embedding-3-small", + "chat_input_tokens": 409, + "chat_output_tokens": 66, + "embed_tokens": 191, + "estimated_usd": 0.000273 + }, + "product_constraints": [], + "credentials_present": true, + "missing_credentials": [], + "projected_usd": 0.000273, + "ceiling_usd": 1.0, + "within_ceiling": true, + "live_run_allowed": false, + "live_run_note": "Phase 1 dry-run only; enable after credentials present and measured estimate confirmed under ceiling." + }, + { + "lane": "zep", + "operations": { + "remember": 21, + "recall": 19, + "revise": 3, + "forget": 4, + "reset": 13, + "http_calls": 47 + }, + "tokens": { + "chat_model": "gpt-4.1-mini-2025-04-14", + "embed_model": "text-embedding-3-small", + "chat_input_tokens": 0, + "chat_output_tokens": 0, + "embed_tokens": 0, + "estimated_usd": 0.0 + }, + "product_constraints": [ + "lane zep cannot pin chat model gpt-4.1-mini-2025-04-14", + "lane zep cannot pin embed model text-embedding-3-small", + "Zep Cloud memory path does not accept OpenAI chat/embed model pins; stop before substituting \u2014 record scores only under Zep-native models." + ], + "credentials_present": false, + "missing_credentials": [ + "ZEP_API_KEY" + ], + "projected_usd": 0.0, + "ceiling_usd": 0.0, + "within_ceiling": true, + "live_run_allowed": false, + "live_run_note": "Phase 1 dry-run only; enable after credentials present and measured estimate confirmed under ceiling." + }, + { + "lane": "_total_openai_estimate", + "projected_usd": 0.000819, + "ceiling_usd": 3.0, + "within_ceiling": true + } + ] +} diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/cost-worksheet-payload-budget.json b/docs/benchmarks/artifacts/opmem-external-lanes/cost-worksheet-payload-budget.json new file mode 100644 index 00000000..b61ad6cc --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/cost-worksheet-payload-budget.json @@ -0,0 +1,87 @@ +{ + "mode": "payload_budget", + "task_count": 13, + "step_count": 47, + "pinned_chat_model": "gpt-4.1-mini-2025-04-14", + "pinned_embed_model": "text-embedding-3-small", + "rate_source_url": "https://openai.com/api/pricing/", + "rate_source_note": "gpt-4.1-mini-2025-04-14: $0.40/1M input, $1.60/1M output; text-embedding-3-small: $0.02/1M tokens", + "ceilings_usd": { + "mem0-oss": 1.0, + "letta": 1.0, + "langmem": 1.0, + "zep": 0.0, + "total": 3.0 + }, + "lanes": [ + { + "lane": "langmem", + "worst_case_usd": 0.017537, + "token_totals": { + "chat_input_tokens": 1447, + "chat_output_tokens": 9600, + "embed_tokens": 203 + }, + "line_count": 47, + "product_constraints": [], + "retry_margin": 1.1, + "projected_worst_case_usd": 0.017537, + "ceiling_usd": 1.0, + "within_ceiling": true + }, + { + "lane": "letta", + "worst_case_usd": 0.013526, + "token_totals": { + "chat_input_tokens": 1930, + "chat_output_tokens": 7200, + "embed_tokens": 203 + }, + "line_count": 47, + "product_constraints": [], + "retry_margin": 1.1, + "projected_worst_case_usd": 0.013526, + "ceiling_usd": 1.0, + "within_ceiling": true + }, + { + "lane": "mem0-oss", + "worst_case_usd": 0.11353, + "token_totals": { + "chat_input_tokens": 186010, + "chat_output_tokens": 18000, + "embed_tokens": 239 + }, + "line_count": 66, + "product_constraints": [], + "retry_margin": 1.1, + "projected_worst_case_usd": 0.11353, + "ceiling_usd": 1.0, + "within_ceiling": true + }, + { + "lane": "zep", + "worst_case_usd": 0.0, + "token_totals": { + "chat_input_tokens": 1122, + "chat_output_tokens": 0, + "embed_tokens": 0 + }, + "line_count": 47, + "product_constraints": [ + "Zep Cloud native extraction; OpenAI pins not applied; $0 paid ceiling." + ], + "retry_margin": 1.1, + "projected_worst_case_usd": 0.0, + "ceiling_usd": 0.0, + "within_ceiling": true + }, + { + "lane": "_total_openai_worst_case", + "projected_worst_case_usd": 0.144592, + "ceiling_usd": 3.0, + "within_ceiling": true + } + ], + "methodology": "tiktoken cl100k_base on serialized request bodies incl. mem0 ADDITIVE_EXTRACTION_PROMPT; conservative max output caps; 1.10 retry margin; Zep excluded from OpenAI totals." +} diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/dry-run-results.json b/docs/benchmarks/artifacts/opmem-external-lanes/dry-run-results.json new file mode 100644 index 00000000..be0ab0bf --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/dry-run-results.json @@ -0,0 +1,453 @@ +{ + "benchmark": "opmem-v0", + "mode": "dry_run", + "run_id": "opmem-dryrun-0001", + "task_count": 13, + "step_count": 47, + "lanes": { + "langmem": { + "infrastructure_errors": 0, + "results": [ + { + "task": "cor01_basic_revision", + "outcome": { + "passed": false, + "failures": [ + "step1(recall): expected at least 1 results, got 0", + "step1(recall): top result does not contain 'python'" + ] + } + }, + { + "task": "cor02_correction_stickiness", + "outcome": { + "passed": false, + "failures": [ + "step3(recall): expected at least 1 results, got 0", + "step3(recall): top result does not contain 'ruby'" + ] + } + }, + { + "task": "cor03_revised_retrievable", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "dup01_idempotent_remember", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "iso01_subject_isolation", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "iso02_tenant_isolation", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "iso03_forget_isolated", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "sup01_basic_forget", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "sup02_targeted_forget", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "sup03_durable_forget", + "outcome": { + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 1" + ] + } + }, + { + "task": "upd01_stale_fact", + "outcome": { + "passed": false, + "failures": [ + "step2(recall): top result does not contain 'june'" + ] + } + }, + { + "task": "upd02_preference_change", + "outcome": { + "passed": false, + "failures": [ + "step2(recall): expected at least 1 results, got 0", + "step2(recall): top result does not contain 'sms'" + ] + } + }, + { + "task": "upd03_state_supersession", + "outcome": { + "passed": false, + "failures": [ + "step2(recall): top result does not contain '2222'" + ] + } + } + ] + }, + "letta": { + "infrastructure_errors": 0, + "results": [ + { + "task": "cor01_basic_revision", + "outcome": { + "passed": false, + "failures": [ + "step1(recall): expected at least 1 results, got 0", + "step1(recall): top result does not contain 'python'" + ] + } + }, + { + "task": "cor02_correction_stickiness", + "outcome": { + "passed": false, + "failures": [ + "step3(recall): expected at least 1 results, got 0", + "step3(recall): top result does not contain 'ruby'" + ] + } + }, + { + "task": "cor03_revised_retrievable", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "dup01_idempotent_remember", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "iso01_subject_isolation", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "iso02_tenant_isolation", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "iso03_forget_isolated", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "sup01_basic_forget", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "sup02_targeted_forget", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "sup03_durable_forget", + "outcome": { + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 1" + ] + } + }, + { + "task": "upd01_stale_fact", + "outcome": { + "passed": false, + "failures": [ + "step2(recall): top result does not contain 'june'" + ] + } + }, + { + "task": "upd02_preference_change", + "outcome": { + "passed": false, + "failures": [ + "step2(recall): expected at least 1 results, got 0", + "step2(recall): top result does not contain 'sms'" + ] + } + }, + { + "task": "upd03_state_supersession", + "outcome": { + "passed": false, + "failures": [ + "step2(recall): top result does not contain '2222'" + ] + } + } + ] + }, + "mem0-oss": { + "infrastructure_errors": 0, + "results": [ + { + "task": "cor01_basic_revision", + "outcome": { + "passed": false, + "failures": [ + "step1(recall): expected at least 1 results, got 0", + "step1(recall): top result does not contain 'python'" + ] + } + }, + { + "task": "cor02_correction_stickiness", + "outcome": { + "passed": false, + "failures": [ + "step3(recall): expected at least 1 results, got 0", + "step3(recall): top result does not contain 'ruby'" + ] + } + }, + { + "task": "cor03_revised_retrievable", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "dup01_idempotent_remember", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "iso01_subject_isolation", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "iso02_tenant_isolation", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "iso03_forget_isolated", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "sup01_basic_forget", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "sup02_targeted_forget", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "sup03_durable_forget", + "outcome": { + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 1" + ] + } + }, + { + "task": "upd01_stale_fact", + "outcome": { + "passed": false, + "failures": [ + "step2(recall): top result does not contain 'june'" + ] + } + }, + { + "task": "upd02_preference_change", + "outcome": { + "passed": false, + "failures": [ + "step2(recall): expected at least 1 results, got 0", + "step2(recall): top result does not contain 'sms'" + ] + } + }, + { + "task": "upd03_state_supersession", + "outcome": { + "passed": false, + "failures": [ + "step2(recall): top result does not contain '2222'" + ] + } + } + ] + }, + "zep": { + "infrastructure_errors": 0, + "results": [ + { + "task": "cor01_basic_revision", + "outcome": { + "passed": false, + "failures": [ + "step1(recall): expected at least 1 results, got 0", + "step1(recall): top result does not contain 'python'" + ] + } + }, + { + "task": "cor02_correction_stickiness", + "outcome": { + "passed": false, + "failures": [ + "step3(recall): expected at least 1 results, got 0", + "step3(recall): top result does not contain 'ruby'" + ] + } + }, + { + "task": "cor03_revised_retrievable", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "dup01_idempotent_remember", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "iso01_subject_isolation", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "iso02_tenant_isolation", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "iso03_forget_isolated", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "sup01_basic_forget", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "sup02_targeted_forget", + "outcome": { + "passed": true, + "failures": [] + } + }, + { + "task": "sup03_durable_forget", + "outcome": { + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 1" + ] + } + }, + { + "task": "upd01_stale_fact", + "outcome": { + "passed": false, + "failures": [ + "step2(recall): top result does not contain 'june'" + ] + } + }, + { + "task": "upd02_preference_change", + "outcome": { + "passed": false, + "failures": [ + "step2(recall): expected at least 1 results, got 0", + "step2(recall): top result does not contain 'sms'" + ] + } + }, + { + "task": "upd03_state_supersession", + "outcome": { + "passed": false, + "failures": [ + "step2(recall): top result does not contain '2222'" + ] + } + } + ] + } + } +} diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/live-full-opmem-phase2-full-mem0.json b/docs/benchmarks/artifacts/opmem-external-lanes/live-full-opmem-phase2-full-mem0.json new file mode 100644 index 00000000..f8d538bf --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/live-full-opmem-phase2-full-mem0.json @@ -0,0 +1,88 @@ +[ + { + "lane": "mem0-oss", + "status": "completed", + "mode": "full", + "run_id": "opmem-phase2-full-mem0", + "elapsed_seconds": 36.01, + "infrastructure_errors": 0, + "tasks_run": 13, + "tasks_passed": 9, + "task_results": [ + { + "task": "cor01_basic_revision", + "passed": true, + "failures": [] + }, + { + "task": "cor02_correction_stickiness", + "passed": true, + "failures": [] + }, + { + "task": "cor03_revised_retrievable", + "passed": true, + "failures": [] + }, + { + "task": "dup01_idempotent_remember", + "passed": true, + "failures": [] + }, + { + "task": "iso01_subject_isolation", + "passed": true, + "failures": [] + }, + { + "task": "iso02_tenant_isolation", + "passed": true, + "failures": [] + }, + { + "task": "iso03_forget_isolated", + "passed": true, + "failures": [] + }, + { + "task": "sup01_basic_forget", + "passed": true, + "failures": [] + }, + { + "task": "sup02_targeted_forget", + "passed": true, + "failures": [] + }, + { + "task": "sup03_durable_forget", + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 1" + ] + }, + { + "task": "upd01_stale_fact", + "passed": false, + "failures": [ + "step2(recall): top result does not contain 'june'" + ] + }, + { + "task": "upd02_preference_change", + "passed": false, + "failures": [ + "step2(recall): top result does not contain 'sms'" + ] + }, + { + "task": "upd03_state_supersession", + "passed": false, + "failures": [ + "step2(recall): top result does not contain '2222'" + ] + } + ], + "product_constraints": [] + } +] diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/live-full-opmem-phase2-full1.json b/docs/benchmarks/artifacts/opmem-external-lanes/live-full-opmem-phase2-full1.json new file mode 100644 index 00000000..56345957 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/live-full-opmem-phase2-full1.json @@ -0,0 +1,176 @@ +[ + { + "lane": "mem0-oss", + "status": "invalid_incomplete", + "mode": "full", + "run_id": "opmem-phase2-full1", + "elapsed_seconds": 28.41, + "infrastructure_errors": 3, + "tasks_run": 13, + "tasks_passed": 6, + "task_results": [ + { + "task": "cor01_basic_revision", + "error": "AttributeError: 'dict' object has no attribute 'replace'", + "invalid_incomplete": true + }, + { + "task": "cor02_correction_stickiness", + "error": "AttributeError: 'dict' object has no attribute 'replace'", + "invalid_incomplete": true + }, + { + "task": "cor03_revised_retrievable", + "error": "AttributeError: 'dict' object has no attribute 'replace'", + "invalid_incomplete": true + }, + { + "task": "dup01_idempotent_remember", + "passed": true, + "failures": [] + }, + { + "task": "iso01_subject_isolation", + "passed": true, + "failures": [] + }, + { + "task": "iso02_tenant_isolation", + "passed": true, + "failures": [] + }, + { + "task": "iso03_forget_isolated", + "passed": true, + "failures": [] + }, + { + "task": "sup01_basic_forget", + "passed": true, + "failures": [] + }, + { + "task": "sup02_targeted_forget", + "passed": true, + "failures": [] + }, + { + "task": "sup03_durable_forget", + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 1" + ] + }, + { + "task": "upd01_stale_fact", + "passed": false, + "failures": [ + "step2(recall): top result does not contain 'june'" + ] + }, + { + "task": "upd02_preference_change", + "passed": false, + "failures": [ + "step2(recall): top result does not contain 'sms'" + ] + }, + { + "task": "upd03_state_supersession", + "passed": false, + "failures": [ + "step2(recall): top result does not contain '2222'" + ] + } + ], + "product_constraints": [] + }, + { + "lane": "langmem", + "status": "completed", + "mode": "full", + "run_id": "opmem-phase2-full1", + "elapsed_seconds": 42.07, + "infrastructure_errors": 0, + "tasks_run": 13, + "tasks_passed": 8, + "task_results": [ + { + "task": "cor01_basic_revision", + "passed": true, + "failures": [] + }, + { + "task": "cor02_correction_stickiness", + "passed": false, + "failures": [ + "step3(recall): top result does not contain 'ruby'" + ] + }, + { + "task": "cor03_revised_retrievable", + "passed": true, + "failures": [] + }, + { + "task": "dup01_idempotent_remember", + "passed": true, + "failures": [] + }, + { + "task": "iso01_subject_isolation", + "passed": true, + "failures": [] + }, + { + "task": "iso02_tenant_isolation", + "passed": true, + "failures": [] + }, + { + "task": "iso03_forget_isolated", + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 1" + ] + }, + { + "task": "sup01_basic_forget", + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 1" + ] + }, + { + "task": "sup02_targeted_forget", + "passed": false, + "failures": [ + "step3(recall): a result contains excluded 'door code'" + ] + }, + { + "task": "sup03_durable_forget", + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 1" + ] + }, + { + "task": "upd01_stale_fact", + "passed": true, + "failures": [] + }, + { + "task": "upd02_preference_change", + "passed": true, + "failures": [] + }, + { + "task": "upd03_state_supersession", + "passed": true, + "failures": [] + } + ], + "product_constraints": [] + } +] diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/live-full-opmem-phase2-letta-full.json b/docs/benchmarks/artifacts/opmem-external-lanes/live-full-opmem-phase2-letta-full.json new file mode 100644 index 00000000..dfb5f711 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/live-full-opmem-phase2-letta-full.json @@ -0,0 +1,99 @@ +[ + { + "lane": "letta", + "status": "completed", + "mode": "full", + "run_id": "opmem-phase2-letta-full", + "elapsed_seconds": 6.56, + "infrastructure_errors": 0, + "tasks_run": 13, + "tasks_passed": 5, + "task_results": [ + { + "task": "cor01_basic_revision", + "passed": false, + "failures": [ + "step1(recall): expected at least 1 results, got 0", + "step1(recall): top result does not contain 'python'" + ] + }, + { + "task": "cor02_correction_stickiness", + "passed": false, + "failures": [ + "step3(recall): expected at least 1 results, got 0", + "step3(recall): top result does not contain 'ruby'" + ] + }, + { + "task": "cor03_revised_retrievable", + "passed": true, + "failures": [] + }, + { + "task": "dup01_idempotent_remember", + "passed": true, + "failures": [] + }, + { + "task": "iso01_subject_isolation", + "passed": false, + "failures": [ + "step1(recall): expected at most 0 results, got 1" + ] + }, + { + "task": "iso02_tenant_isolation", + "passed": false, + "failures": [ + "step1(recall): expected at most 0 results, got 1" + ] + }, + { + "task": "iso03_forget_isolated", + "passed": false, + "failures": [ + "step4(recall): expected at least 1 results, got 0" + ] + }, + { + "task": "sup01_basic_forget", + "passed": true, + "failures": [] + }, + { + "task": "sup02_targeted_forget", + "passed": true, + "failures": [] + }, + { + "task": "sup03_durable_forget", + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 1" + ] + }, + { + "task": "upd01_stale_fact", + "passed": false, + "failures": [ + "step2(recall): top result does not contain 'june'" + ] + }, + { + "task": "upd02_preference_change", + "passed": false, + "failures": [ + "step2(recall): expected at least 1 results, got 0", + "step2(recall): top result does not contain 'sms'" + ] + }, + { + "task": "upd03_state_supersession", + "passed": true, + "failures": [] + } + ], + "product_constraints": [] + } +] diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/live-full-opmem-phase2-zep-full.json b/docs/benchmarks/artifacts/opmem-external-lanes/live-full-opmem-phase2-zep-full.json new file mode 100644 index 00000000..ccf6e689 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/live-full-opmem-phase2-zep-full.json @@ -0,0 +1,106 @@ +[ + { + "lane": "zep", + "status": "completed", + "mode": "full", + "run_id": "opmem-phase2-zep-full", + "elapsed_seconds": 51.01, + "infrastructure_errors": 0, + "tasks_run": 13, + "tasks_passed": 3, + "task_results": [ + { + "task": "cor01_basic_revision", + "passed": false, + "failures": [ + "step1(recall): expected at least 1 results, got 0", + "step1(recall): top result does not contain 'python'" + ] + }, + { + "task": "cor02_correction_stickiness", + "passed": true, + "failures": [] + }, + { + "task": "cor03_revised_retrievable", + "passed": true, + "failures": [] + }, + { + "task": "dup01_idempotent_remember", + "passed": true, + "failures": [] + }, + { + "task": "iso01_subject_isolation", + "passed": false, + "failures": [ + "step1(recall): expected at most 0 results, got 1" + ] + }, + { + "task": "iso02_tenant_isolation", + "passed": false, + "failures": [ + "step1(recall): expected at most 0 results, got 1" + ] + }, + { + "task": "iso03_forget_isolated", + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 1" + ] + }, + { + "task": "sup01_basic_forget", + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 1" + ] + }, + { + "task": "sup02_targeted_forget", + "passed": false, + "failures": [ + "step3(recall): a result contains excluded 'door code'" + ] + }, + { + "task": "sup03_durable_forget", + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 2" + ] + }, + { + "task": "upd01_stale_fact", + "passed": false, + "failures": [ + "step2(recall): top result does not contain 'june'" + ] + }, + { + "task": "upd02_preference_change", + "passed": false, + "failures": [ + "step2(recall): expected at least 1 results, got 0", + "step2(recall): top result does not contain 'sms'" + ] + }, + { + "task": "upd03_state_supersession", + "passed": false, + "failures": [ + "step2(recall): top result does not contain '2222'" + ] + } + ], + "product_constraints": [ + "lane zep cannot pin chat model gpt-4.1-mini-2025-04-14", + "lane zep cannot pin embed model text-embedding-3-small", + "Zep Cloud memory path does not accept OpenAI chat/embed model pins; stop before substituting \u2014 record scores only under Zep-native models." + ] + } +] diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/live-smoke-opmem-phase2-letta-smoke.json b/docs/benchmarks/artifacts/opmem-external-lanes/live-smoke-opmem-phase2-letta-smoke.json new file mode 100644 index 00000000..20d10b9e --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/live-smoke-opmem-phase2-letta-smoke.json @@ -0,0 +1,20 @@ +[ + { + "lane": "letta", + "status": "completed", + "mode": "smoke", + "run_id": "opmem-phase2-letta-smoke", + "elapsed_seconds": 0.46, + "infrastructure_errors": 0, + "tasks_run": 1, + "tasks_passed": 1, + "task_results": [ + { + "task": "dup01_idempotent_remember", + "passed": true, + "failures": [] + } + ], + "product_constraints": [] + } +] diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/live-smoke-opmem-phase2-smoke1.json b/docs/benchmarks/artifacts/opmem-external-lanes/live-smoke-opmem-phase2-smoke1.json new file mode 100644 index 00000000..1a3af060 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/live-smoke-opmem-phase2-smoke1.json @@ -0,0 +1,52 @@ +[ + { + "lane": "mem0-oss", + "status": "invalid_incomplete", + "mode": "smoke", + "run_id": "opmem-phase2-smoke1", + "elapsed_seconds": 6.36, + "infrastructure_errors": 1, + "tasks_run": 1, + "tasks_passed": 0, + "task_results": [ + { + "task": "dup01_idempotent_remember", + "error": "ValueError: Top-level entity parameters frozenset({'user_id'}) are not supported in search(). Use filters={'user_id': '...'} instead.", + "invalid_incomplete": true + } + ], + "product_constraints": [] + }, + { + "lane": "langmem", + "status": "invalid_incomplete", + "mode": "smoke", + "run_id": "opmem-phase2-smoke1", + "elapsed_seconds": 3.15, + "infrastructure_errors": 1, + "tasks_run": 1, + "tasks_passed": 0, + "task_results": [ + { + "task": "dup01_idempotent_remember", + "error": "AttributeError: 'dict' object has no attribute 'lower'", + "invalid_incomplete": true + } + ], + "product_constraints": [] + }, + { + "lane": "zep", + "status": "blocked", + "reason": "ZEP_API_KEY not present in runtime (vault not injected on this VM)", + "infrastructure_errors": 0, + "task_results": [] + }, + { + "lane": "letta", + "status": "blocked", + "reason": "LETTA_APP_SERVER_TOKEN not present in runtime", + "infrastructure_errors": 0, + "task_results": [] + } +] diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/live-smoke-opmem-phase2-smoke2.json b/docs/benchmarks/artifacts/opmem-external-lanes/live-smoke-opmem-phase2-smoke2.json new file mode 100644 index 00000000..9b77e193 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/live-smoke-opmem-phase2-smoke2.json @@ -0,0 +1,38 @@ +[ + { + "lane": "mem0-oss", + "status": "completed", + "mode": "smoke", + "run_id": "opmem-phase2-smoke2", + "elapsed_seconds": 2.9, + "infrastructure_errors": 0, + "tasks_run": 1, + "tasks_passed": 1, + "task_results": [ + { + "task": "dup01_idempotent_remember", + "passed": true, + "failures": [] + } + ], + "product_constraints": [] + }, + { + "lane": "langmem", + "status": "completed", + "mode": "smoke", + "run_id": "opmem-phase2-smoke2", + "elapsed_seconds": 3.68, + "infrastructure_errors": 0, + "tasks_run": 1, + "tasks_passed": 1, + "task_results": [ + { + "task": "dup01_idempotent_remember", + "passed": true, + "failures": [] + } + ], + "product_constraints": [] + } +] diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/live-smoke-opmem-phase2-zep-smoke-20260923.json b/docs/benchmarks/artifacts/opmem-external-lanes/live-smoke-opmem-phase2-zep-smoke-20260923.json new file mode 100644 index 00000000..61b0b2ba --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/live-smoke-opmem-phase2-zep-smoke-20260923.json @@ -0,0 +1,9 @@ +[ + { + "lane": "zep", + "status": "blocked", + "reason": "ZEP_API_KEY not present in runtime (vault not injected on this VM)", + "infrastructure_errors": 0, + "task_results": [] + } +] diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/live-smoke-opmem-phase2-zep-smoke-20260923b.json b/docs/benchmarks/artifacts/opmem-external-lanes/live-smoke-opmem-phase2-zep-smoke-20260923b.json new file mode 100644 index 00000000..08a39ade --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/live-smoke-opmem-phase2-zep-smoke-20260923b.json @@ -0,0 +1,24 @@ +[ + { + "lane": "zep", + "status": "invalid_incomplete", + "mode": "smoke", + "run_id": "opmem-phase2-zep-smoke-20260923b", + "elapsed_seconds": 0.32, + "infrastructure_errors": 1, + "tasks_run": 1, + "tasks_passed": 0, + "task_results": [ + { + "task": "dup01_idempotent_remember", + "error": "AttributeError: 'Zep' object has no attribute 'memory'", + "invalid_incomplete": true + } + ], + "product_constraints": [ + "lane zep cannot pin chat model gpt-4.1-mini-2025-04-14", + "lane zep cannot pin embed model text-embedding-3-small", + "Zep Cloud memory path does not accept OpenAI chat/embed model pins; stop before substituting \u2014 record scores only under Zep-native models." + ] + } +] diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/live-smoke-opmem-phase2-zep-smoke-20260923c.json b/docs/benchmarks/artifacts/opmem-external-lanes/live-smoke-opmem-phase2-zep-smoke-20260923c.json new file mode 100644 index 00000000..51fea514 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/live-smoke-opmem-phase2-zep-smoke-20260923c.json @@ -0,0 +1,26 @@ +[ + { + "lane": "zep", + "status": "completed", + "mode": "smoke", + "run_id": "opmem-phase2-zep-smoke-20260923c", + "elapsed_seconds": 3.95, + "infrastructure_errors": 0, + "tasks_run": 1, + "tasks_passed": 0, + "task_results": [ + { + "task": "dup01_idempotent_remember", + "passed": false, + "failures": [ + "step2(recall): expected at least 1 results, got 0" + ] + } + ], + "product_constraints": [ + "lane zep cannot pin chat model gpt-4.1-mini-2025-04-14", + "lane zep cannot pin embed model text-embedding-3-small", + "Zep Cloud memory path does not accept OpenAI chat/embed model pins; stop before substituting \u2014 record scores only under Zep-native models." + ] + } +] diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/live-smoke-opmem-phase2-zep-smoke-final.json b/docs/benchmarks/artifacts/opmem-external-lanes/live-smoke-opmem-phase2-zep-smoke-final.json new file mode 100644 index 00000000..eb9c1494 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/live-smoke-opmem-phase2-zep-smoke-final.json @@ -0,0 +1,26 @@ +[ + { + "lane": "zep", + "status": "completed", + "mode": "smoke", + "run_id": "opmem-phase2-zep-smoke-final", + "elapsed_seconds": 4.01, + "infrastructure_errors": 0, + "tasks_run": 1, + "tasks_passed": 0, + "task_results": [ + { + "task": "dup01_idempotent_remember", + "passed": false, + "failures": [ + "step2(recall): expected at most 1 results, got 2" + ] + } + ], + "product_constraints": [ + "lane zep cannot pin chat model gpt-4.1-mini-2025-04-14", + "lane zep cannot pin embed model text-embedding-3-small", + "Zep Cloud memory path does not accept OpenAI chat/embed model pins; stop before substituting \u2014 record scores only under Zep-native models." + ] + } +] diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/live-smoke-opmem-phase2-zep-smoke.json b/docs/benchmarks/artifacts/opmem-external-lanes/live-smoke-opmem-phase2-zep-smoke.json new file mode 100644 index 00000000..61b0b2ba --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/live-smoke-opmem-phase2-zep-smoke.json @@ -0,0 +1,9 @@ +[ + { + "lane": "zep", + "status": "blocked", + "reason": "ZEP_API_KEY not present in runtime (vault not injected on this VM)", + "infrastructure_errors": 0, + "task_results": [] + } +] diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/live-zep-blocked-opmem-phase2-zep-smoke.json b/docs/benchmarks/artifacts/opmem-external-lanes/live-zep-blocked-opmem-phase2-zep-smoke.json new file mode 100644 index 00000000..64a87cbd --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/live-zep-blocked-opmem-phase2-zep-smoke.json @@ -0,0 +1,11 @@ +{ + "lane": "zep", + "status": "blocked", + "reason": "ZEP_API_KEY not present in runtime (not in CLOUD_AGENT_INJECTED_SECRET_NAMES on 2026-09-21 cloud VM)", + "run_id": "opmem-phase2-zep-smoke", + "preflight": { + "free_tier_check": "not_attempted", + "paid_ceiling_usd": 0.0 + }, + "reproduce": "export ZEP_API_KEY=; python3 evals/run_opmem_external_live.py --smoke --lanes zep" +} diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/raw-langmem-dry-run.json b/docs/benchmarks/artifacts/opmem-external-lanes/raw-langmem-dry-run.json new file mode 100644 index 00000000..b608b4a0 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/raw-langmem-dry-run.json @@ -0,0 +1,331 @@ +[ + { + "lane": "langmem", + "task": "cor01_basic_revision", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-ec287521fe29\", \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "cor01_basic_revision", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "cor01_basic_revision", + "op": "revise", + "status": 200, + "body_preview": "{\"memory_id\": [\"m-ec287521fe29\"], \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "cor01_basic_revision", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-ec287521fe29\", \"content\": \"Prefers Ruby for scripting\"}], \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "cor02_correction_stickiness", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-ec77cf3747d2\", \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "cor02_correction_stickiness", + "op": "revise", + "status": 200, + "body_preview": "{\"memory_id\": [\"m-ec77cf3747d2\"], \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "cor02_correction_stickiness", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-ec77cf3747d2\", \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "cor02_correction_stickiness", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "cor03_revised_retrievable", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-97e4481c7620\", \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "cor03_revised_retrievable", + "op": "revise", + "status": 200, + "body_preview": "{\"memory_id\": [\"m-97e4481c7620\"], \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "cor03_revised_retrievable", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-97e4481c7620\", \"content\": \"Prefers Ruby for scripting\"}], \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "dup01_idempotent_remember", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-a2ea413a2ee8\", \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "dup01_idempotent_remember", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-a2ea413a2ee8\", \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "dup01_idempotent_remember", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-a2ea413a2ee8\", \"content\": \"The wifi network name is FalconNet.\"}], \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "iso01_subject_isolation", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-f1a8b730d634\", \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "iso01_subject_isolation", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "iso01_subject_isolation", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-f1a8b730d634\", \"content\": \"My name is Alice Chen.\"}], \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "iso02_tenant_isolation", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-c790561d256a\", \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "iso02_tenant_isolation", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "iso02_tenant_isolation", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-c790561d256a\", \"content\": \"The wifi network name is FalconNet.\"}], \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "iso03_forget_isolated", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-b9f3133e0f76\", \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "iso03_forget_isolated", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-b9f3133e0f76\", \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "iso03_forget_isolated", + "op": "forget", + "status": 200, + "body_preview": "{\"deleted\": [\"m-b9f3133e0f76\"], \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "iso03_forget_isolated", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "iso03_forget_isolated", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-b9f3133e0f76\", \"content\": \"Never share the door code with vendors.\"}], \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "sup01_basic_forget", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-12df728344c1\", \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "sup01_basic_forget", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-12df728344c1\", \"content\": \"Never share the door code with vendors.\"}], \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "sup01_basic_forget", + "op": "forget", + "status": 200, + "body_preview": "{\"deleted\": [\"m-12df728344c1\"], \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "sup01_basic_forget", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "sup02_targeted_forget", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-694a3ac917b4\", \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "sup02_targeted_forget", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-abc4cd827fe4\", \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "sup02_targeted_forget", + "op": "forget", + "status": 200, + "body_preview": "{\"deleted\": [\"m-694a3ac917b4\"], \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "sup02_targeted_forget", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "sup02_targeted_forget", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-abc4cd827fe4\", \"content\": \"The wifi network name is FalconNet.\"}], \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "sup03_durable_forget", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-f40e06dae9df\", \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "sup03_durable_forget", + "op": "forget", + "status": 200, + "body_preview": "{\"deleted\": [\"m-f40e06dae9df\"], \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "sup03_durable_forget", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-f40e06dae9df\", \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "sup03_durable_forget", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-f40e06dae9df\", \"content\": \"Never share the door code with vendors.\"}], \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "upd01_stale_fact", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-47fbfe8f74ed\", \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "upd01_stale_fact", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-e5330f7a0060\", \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "upd01_stale_fact", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-47fbfe8f74ed\", \"content\": \"The launch date is May 12.\"}, {\"id\": \"m-e5330f7a0060\", \"content\": \"The launch date is June 3.\"}], \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "upd02_preference_change", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-e582398e5c23\", \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "upd02_preference_change", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-02ba67213b96\", \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "upd02_preference_change", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "upd03_state_supersession", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-d0cc03c9eecd\", \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "upd03_state_supersession", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-5230d4b1d371\", \"dry_run\": true}" + }, + { + "lane": "langmem", + "task": "upd03_state_supersession", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-d0cc03c9eecd\", \"content\": \"The door code is 1111.\"}, {\"id\": \"m-5230d4b1d371\", \"content\": \"The door code is 2222.\"}], \"dry_run\": true}" + } +] diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/raw-langmem-full.json b/docs/benchmarks/artifacts/opmem-external-lanes/raw-langmem-full.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/raw-langmem-full.json @@ -0,0 +1 @@ +[] diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/raw-langmem-smoke.json b/docs/benchmarks/artifacts/opmem-external-lanes/raw-langmem-smoke.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/raw-langmem-smoke.json @@ -0,0 +1 @@ +[] diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/raw-letta-dry-run.json b/docs/benchmarks/artifacts/opmem-external-lanes/raw-letta-dry-run.json new file mode 100644 index 00000000..aa2d1273 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/raw-letta-dry-run.json @@ -0,0 +1,331 @@ +[ + { + "lane": "letta", + "task": "cor01_basic_revision", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-ec287521fe29\", \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "cor01_basic_revision", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "cor01_basic_revision", + "op": "revise", + "status": 200, + "body_preview": "{\"memory_id\": [\"m-ec287521fe29\"], \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "cor01_basic_revision", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-ec287521fe29\", \"content\": \"Prefers Ruby for scripting\"}], \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "cor02_correction_stickiness", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-ec77cf3747d2\", \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "cor02_correction_stickiness", + "op": "revise", + "status": 200, + "body_preview": "{\"memory_id\": [\"m-ec77cf3747d2\"], \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "cor02_correction_stickiness", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-ec77cf3747d2\", \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "cor02_correction_stickiness", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "cor03_revised_retrievable", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-97e4481c7620\", \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "cor03_revised_retrievable", + "op": "revise", + "status": 200, + "body_preview": "{\"memory_id\": [\"m-97e4481c7620\"], \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "cor03_revised_retrievable", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-97e4481c7620\", \"content\": \"Prefers Ruby for scripting\"}], \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "dup01_idempotent_remember", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-a2ea413a2ee8\", \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "dup01_idempotent_remember", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-a2ea413a2ee8\", \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "dup01_idempotent_remember", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-a2ea413a2ee8\", \"content\": \"The wifi network name is FalconNet.\"}], \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "iso01_subject_isolation", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-f1a8b730d634\", \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "iso01_subject_isolation", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "iso01_subject_isolation", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-f1a8b730d634\", \"content\": \"My name is Alice Chen.\"}], \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "iso02_tenant_isolation", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-c790561d256a\", \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "iso02_tenant_isolation", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "iso02_tenant_isolation", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-c790561d256a\", \"content\": \"The wifi network name is FalconNet.\"}], \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "iso03_forget_isolated", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-b9f3133e0f76\", \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "iso03_forget_isolated", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-b9f3133e0f76\", \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "iso03_forget_isolated", + "op": "forget", + "status": 200, + "body_preview": "{\"deleted\": [\"m-b9f3133e0f76\"], \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "iso03_forget_isolated", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "iso03_forget_isolated", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-b9f3133e0f76\", \"content\": \"Never share the door code with vendors.\"}], \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "sup01_basic_forget", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-12df728344c1\", \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "sup01_basic_forget", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-12df728344c1\", \"content\": \"Never share the door code with vendors.\"}], \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "sup01_basic_forget", + "op": "forget", + "status": 200, + "body_preview": "{\"deleted\": [\"m-12df728344c1\"], \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "sup01_basic_forget", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "sup02_targeted_forget", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-694a3ac917b4\", \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "sup02_targeted_forget", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-abc4cd827fe4\", \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "sup02_targeted_forget", + "op": "forget", + "status": 200, + "body_preview": "{\"deleted\": [\"m-694a3ac917b4\"], \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "sup02_targeted_forget", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "sup02_targeted_forget", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-abc4cd827fe4\", \"content\": \"The wifi network name is FalconNet.\"}], \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "sup03_durable_forget", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-f40e06dae9df\", \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "sup03_durable_forget", + "op": "forget", + "status": 200, + "body_preview": "{\"deleted\": [\"m-f40e06dae9df\"], \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "sup03_durable_forget", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-f40e06dae9df\", \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "sup03_durable_forget", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-f40e06dae9df\", \"content\": \"Never share the door code with vendors.\"}], \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "upd01_stale_fact", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-47fbfe8f74ed\", \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "upd01_stale_fact", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-e5330f7a0060\", \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "upd01_stale_fact", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-47fbfe8f74ed\", \"content\": \"The launch date is May 12.\"}, {\"id\": \"m-e5330f7a0060\", \"content\": \"The launch date is June 3.\"}], \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "upd02_preference_change", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-e582398e5c23\", \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "upd02_preference_change", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-02ba67213b96\", \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "upd02_preference_change", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "upd03_state_supersession", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-d0cc03c9eecd\", \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "upd03_state_supersession", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-5230d4b1d371\", \"dry_run\": true}" + }, + { + "lane": "letta", + "task": "upd03_state_supersession", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-d0cc03c9eecd\", \"content\": \"The door code is 1111.\"}, {\"id\": \"m-5230d4b1d371\", \"content\": \"The door code is 2222.\"}], \"dry_run\": true}" + } +] diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/raw-letta-full.json b/docs/benchmarks/artifacts/opmem-external-lanes/raw-letta-full.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/raw-letta-full.json @@ -0,0 +1 @@ +[] diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/raw-letta-smoke.json b/docs/benchmarks/artifacts/opmem-external-lanes/raw-letta-smoke.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/raw-letta-smoke.json @@ -0,0 +1 @@ +[] diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/raw-mem0-oss-dry-run.json b/docs/benchmarks/artifacts/opmem-external-lanes/raw-mem0-oss-dry-run.json new file mode 100644 index 00000000..4cfc82a7 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/raw-mem0-oss-dry-run.json @@ -0,0 +1,331 @@ +[ + { + "lane": "mem0-oss", + "task": "cor01_basic_revision", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-ec287521fe29\", \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "cor01_basic_revision", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "cor01_basic_revision", + "op": "revise", + "status": 200, + "body_preview": "{\"memory_id\": [\"m-ec287521fe29\"], \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "cor01_basic_revision", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-ec287521fe29\", \"content\": \"Prefers Ruby for scripting\"}], \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "cor02_correction_stickiness", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-ec77cf3747d2\", \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "cor02_correction_stickiness", + "op": "revise", + "status": 200, + "body_preview": "{\"memory_id\": [\"m-ec77cf3747d2\"], \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "cor02_correction_stickiness", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-ec77cf3747d2\", \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "cor02_correction_stickiness", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "cor03_revised_retrievable", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-97e4481c7620\", \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "cor03_revised_retrievable", + "op": "revise", + "status": 200, + "body_preview": "{\"memory_id\": [\"m-97e4481c7620\"], \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "cor03_revised_retrievable", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-97e4481c7620\", \"content\": \"Prefers Ruby for scripting\"}], \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "dup01_idempotent_remember", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-a2ea413a2ee8\", \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "dup01_idempotent_remember", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-a2ea413a2ee8\", \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "dup01_idempotent_remember", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-a2ea413a2ee8\", \"content\": \"The wifi network name is FalconNet.\"}], \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "iso01_subject_isolation", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-f1a8b730d634\", \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "iso01_subject_isolation", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "iso01_subject_isolation", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-f1a8b730d634\", \"content\": \"My name is Alice Chen.\"}], \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "iso02_tenant_isolation", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-c790561d256a\", \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "iso02_tenant_isolation", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "iso02_tenant_isolation", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-c790561d256a\", \"content\": \"The wifi network name is FalconNet.\"}], \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "iso03_forget_isolated", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-b9f3133e0f76\", \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "iso03_forget_isolated", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-b9f3133e0f76\", \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "iso03_forget_isolated", + "op": "forget", + "status": 200, + "body_preview": "{\"deleted\": [\"m-b9f3133e0f76\"], \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "iso03_forget_isolated", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "iso03_forget_isolated", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-b9f3133e0f76\", \"content\": \"Never share the door code with vendors.\"}], \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "sup01_basic_forget", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-12df728344c1\", \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "sup01_basic_forget", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-12df728344c1\", \"content\": \"Never share the door code with vendors.\"}], \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "sup01_basic_forget", + "op": "forget", + "status": 200, + "body_preview": "{\"deleted\": [\"m-12df728344c1\"], \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "sup01_basic_forget", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "sup02_targeted_forget", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-694a3ac917b4\", \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "sup02_targeted_forget", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-abc4cd827fe4\", \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "sup02_targeted_forget", + "op": "forget", + "status": 200, + "body_preview": "{\"deleted\": [\"m-694a3ac917b4\"], \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "sup02_targeted_forget", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "sup02_targeted_forget", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-abc4cd827fe4\", \"content\": \"The wifi network name is FalconNet.\"}], \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "sup03_durable_forget", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-f40e06dae9df\", \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "sup03_durable_forget", + "op": "forget", + "status": 200, + "body_preview": "{\"deleted\": [\"m-f40e06dae9df\"], \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "sup03_durable_forget", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-f40e06dae9df\", \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "sup03_durable_forget", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-f40e06dae9df\", \"content\": \"Never share the door code with vendors.\"}], \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "upd01_stale_fact", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-47fbfe8f74ed\", \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "upd01_stale_fact", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-e5330f7a0060\", \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "upd01_stale_fact", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-47fbfe8f74ed\", \"content\": \"The launch date is May 12.\"}, {\"id\": \"m-e5330f7a0060\", \"content\": \"The launch date is June 3.\"}], \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "upd02_preference_change", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-e582398e5c23\", \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "upd02_preference_change", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-02ba67213b96\", \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "upd02_preference_change", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "upd03_state_supersession", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-d0cc03c9eecd\", \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "upd03_state_supersession", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-5230d4b1d371\", \"dry_run\": true}" + }, + { + "lane": "mem0-oss", + "task": "upd03_state_supersession", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-d0cc03c9eecd\", \"content\": \"The door code is 1111.\"}, {\"id\": \"m-5230d4b1d371\", \"content\": \"The door code is 2222.\"}], \"dry_run\": true}" + } +] diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/raw-mem0-oss-full.json b/docs/benchmarks/artifacts/opmem-external-lanes/raw-mem0-oss-full.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/raw-mem0-oss-full.json @@ -0,0 +1 @@ +[] diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/raw-mem0-oss-smoke.json b/docs/benchmarks/artifacts/opmem-external-lanes/raw-mem0-oss-smoke.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/raw-mem0-oss-smoke.json @@ -0,0 +1 @@ +[] diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/raw-zep-dry-run.json b/docs/benchmarks/artifacts/opmem-external-lanes/raw-zep-dry-run.json new file mode 100644 index 00000000..6742c08d --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/raw-zep-dry-run.json @@ -0,0 +1,331 @@ +[ + { + "lane": "zep", + "task": "cor01_basic_revision", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-ec287521fe29\", \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "cor01_basic_revision", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "cor01_basic_revision", + "op": "revise", + "status": 200, + "body_preview": "{\"memory_id\": [\"m-ec287521fe29\"], \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "cor01_basic_revision", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-ec287521fe29\", \"content\": \"Prefers Ruby for scripting\"}], \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "cor02_correction_stickiness", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-ec77cf3747d2\", \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "cor02_correction_stickiness", + "op": "revise", + "status": 200, + "body_preview": "{\"memory_id\": [\"m-ec77cf3747d2\"], \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "cor02_correction_stickiness", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-ec77cf3747d2\", \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "cor02_correction_stickiness", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "cor03_revised_retrievable", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-97e4481c7620\", \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "cor03_revised_retrievable", + "op": "revise", + "status": 200, + "body_preview": "{\"memory_id\": [\"m-97e4481c7620\"], \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "cor03_revised_retrievable", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-97e4481c7620\", \"content\": \"Prefers Ruby for scripting\"}], \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "dup01_idempotent_remember", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-a2ea413a2ee8\", \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "dup01_idempotent_remember", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-a2ea413a2ee8\", \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "dup01_idempotent_remember", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-a2ea413a2ee8\", \"content\": \"The wifi network name is FalconNet.\"}], \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "iso01_subject_isolation", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-f1a8b730d634\", \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "iso01_subject_isolation", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "iso01_subject_isolation", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-f1a8b730d634\", \"content\": \"My name is Alice Chen.\"}], \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "iso02_tenant_isolation", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-c790561d256a\", \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "iso02_tenant_isolation", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "iso02_tenant_isolation", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-c790561d256a\", \"content\": \"The wifi network name is FalconNet.\"}], \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "iso03_forget_isolated", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-b9f3133e0f76\", \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "iso03_forget_isolated", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-b9f3133e0f76\", \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "iso03_forget_isolated", + "op": "forget", + "status": 200, + "body_preview": "{\"deleted\": [\"m-b9f3133e0f76\"], \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "iso03_forget_isolated", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "iso03_forget_isolated", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-b9f3133e0f76\", \"content\": \"Never share the door code with vendors.\"}], \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "sup01_basic_forget", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-12df728344c1\", \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "sup01_basic_forget", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-12df728344c1\", \"content\": \"Never share the door code with vendors.\"}], \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "sup01_basic_forget", + "op": "forget", + "status": 200, + "body_preview": "{\"deleted\": [\"m-12df728344c1\"], \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "sup01_basic_forget", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "sup02_targeted_forget", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-694a3ac917b4\", \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "sup02_targeted_forget", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-abc4cd827fe4\", \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "sup02_targeted_forget", + "op": "forget", + "status": 200, + "body_preview": "{\"deleted\": [\"m-694a3ac917b4\"], \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "sup02_targeted_forget", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "sup02_targeted_forget", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-abc4cd827fe4\", \"content\": \"The wifi network name is FalconNet.\"}], \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "sup03_durable_forget", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-f40e06dae9df\", \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "sup03_durable_forget", + "op": "forget", + "status": 200, + "body_preview": "{\"deleted\": [\"m-f40e06dae9df\"], \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "sup03_durable_forget", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-f40e06dae9df\", \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "sup03_durable_forget", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-f40e06dae9df\", \"content\": \"Never share the door code with vendors.\"}], \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "upd01_stale_fact", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-47fbfe8f74ed\", \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "upd01_stale_fact", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-e5330f7a0060\", \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "upd01_stale_fact", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-47fbfe8f74ed\", \"content\": \"The launch date is May 12.\"}, {\"id\": \"m-e5330f7a0060\", \"content\": \"The launch date is June 3.\"}], \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "upd02_preference_change", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-e582398e5c23\", \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "upd02_preference_change", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-02ba67213b96\", \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "upd02_preference_change", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [], \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "upd03_state_supersession", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-d0cc03c9eecd\", \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "upd03_state_supersession", + "op": "remember", + "status": 200, + "body_preview": "{\"memory_id\": \"m-5230d4b1d371\", \"dry_run\": true}" + }, + { + "lane": "zep", + "task": "upd03_state_supersession", + "op": "recall", + "status": 200, + "body_preview": "{\"results\": [{\"id\": \"m-d0cc03c9eecd\", \"content\": \"The door code is 1111.\"}, {\"id\": \"m-5230d4b1d371\", \"content\": \"The door code is 2222.\"}], \"dry_run\": true}" + } +] diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/raw-zep-full.json b/docs/benchmarks/artifacts/opmem-external-lanes/raw-zep-full.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/raw-zep-full.json @@ -0,0 +1 @@ +[] diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/raw-zep-smoke.json b/docs/benchmarks/artifacts/opmem-external-lanes/raw-zep-smoke.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/raw-zep-smoke.json @@ -0,0 +1 @@ +[] diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-full-mem0.json b/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-full-mem0.json new file mode 100644 index 00000000..2e904a06 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-full-mem0.json @@ -0,0 +1,122 @@ +{ + "benchmark": "opmem-v0", + "generated_utc": "2026-09-19T08:30:46Z", + "run_id": "opmem-phase2-full-mem0", + "host": "Linux-6.12.94+-x86_64-with-glibc2.39", + "python": "3.12.3", + "package_versions": { + "mem0ai": "2.1.0", + "qdrant-client": "1.19.1", + "zep-cloud": "3.28.0", + "tiktoken": "0.14.0", + "langmem": "0.0.30", + "langgraph-checkpoint-postgres": "3.1.2", + "openai": "3.16.2", + "langchain-openai": "1.6.2" + }, + "pinned_models": { + "chat": "gpt-4.1-mini-2025-04-14", + "embed": "text-embedding-3-small" + }, + "ceilings_usd": { + "mem0-oss": 1.0, + "letta": 1.0, + "langmem": 1.0, + "zep": 0.0, + "total": 3.0 + }, + "modes": [ + { + "mode": "full", + "task_count": 13, + "lanes": [ + { + "lane": "mem0-oss", + "status": "completed", + "mode": "full", + "run_id": "opmem-phase2-full-mem0", + "elapsed_seconds": 36.01, + "infrastructure_errors": 0, + "tasks_run": 13, + "tasks_passed": 9, + "task_results": [ + { + "task": "cor01_basic_revision", + "passed": true, + "failures": [] + }, + { + "task": "cor02_correction_stickiness", + "passed": true, + "failures": [] + }, + { + "task": "cor03_revised_retrievable", + "passed": true, + "failures": [] + }, + { + "task": "dup01_idempotent_remember", + "passed": true, + "failures": [] + }, + { + "task": "iso01_subject_isolation", + "passed": true, + "failures": [] + }, + { + "task": "iso02_tenant_isolation", + "passed": true, + "failures": [] + }, + { + "task": "iso03_forget_isolated", + "passed": true, + "failures": [] + }, + { + "task": "sup01_basic_forget", + "passed": true, + "failures": [] + }, + { + "task": "sup02_targeted_forget", + "passed": true, + "failures": [] + }, + { + "task": "sup03_durable_forget", + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 1" + ] + }, + { + "task": "upd01_stale_fact", + "passed": false, + "failures": [ + "step2(recall): top result does not contain 'june'" + ] + }, + { + "task": "upd02_preference_change", + "passed": false, + "failures": [ + "step2(recall): top result does not contain 'sms'" + ] + }, + { + "task": "upd03_state_supersession", + "passed": false, + "failures": [ + "step2(recall): top result does not contain '2222'" + ] + } + ], + "product_constraints": [] + } + ] + } + ] +} diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-full1.json b/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-full1.json new file mode 100644 index 00000000..e08f1503 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-full1.json @@ -0,0 +1,210 @@ +{ + "benchmark": "opmem-v0", + "generated_utc": "2026-09-19T08:29:22Z", + "run_id": "opmem-phase2-full1", + "host": "Linux-6.12.94+-x86_64-with-glibc2.39", + "python": "3.12.3", + "package_versions": { + "mem0ai": "2.1.0", + "qdrant-client": "1.19.1", + "zep-cloud": "3.28.0", + "tiktoken": "0.14.0", + "langmem": "0.0.30", + "langgraph-checkpoint-postgres": "3.1.2", + "openai": "3.16.2", + "langchain-openai": "1.6.2" + }, + "pinned_models": { + "chat": "gpt-4.1-mini-2025-04-14", + "embed": "text-embedding-3-small" + }, + "ceilings_usd": { + "mem0-oss": 1.0, + "letta": 1.0, + "langmem": 1.0, + "zep": 0.0, + "total": 3.0 + }, + "modes": [ + { + "mode": "full", + "task_count": 13, + "lanes": [ + { + "lane": "mem0-oss", + "status": "invalid_incomplete", + "mode": "full", + "run_id": "opmem-phase2-full1", + "elapsed_seconds": 28.41, + "infrastructure_errors": 3, + "tasks_run": 13, + "tasks_passed": 6, + "task_results": [ + { + "task": "cor01_basic_revision", + "error": "AttributeError: 'dict' object has no attribute 'replace'", + "invalid_incomplete": true + }, + { + "task": "cor02_correction_stickiness", + "error": "AttributeError: 'dict' object has no attribute 'replace'", + "invalid_incomplete": true + }, + { + "task": "cor03_revised_retrievable", + "error": "AttributeError: 'dict' object has no attribute 'replace'", + "invalid_incomplete": true + }, + { + "task": "dup01_idempotent_remember", + "passed": true, + "failures": [] + }, + { + "task": "iso01_subject_isolation", + "passed": true, + "failures": [] + }, + { + "task": "iso02_tenant_isolation", + "passed": true, + "failures": [] + }, + { + "task": "iso03_forget_isolated", + "passed": true, + "failures": [] + }, + { + "task": "sup01_basic_forget", + "passed": true, + "failures": [] + }, + { + "task": "sup02_targeted_forget", + "passed": true, + "failures": [] + }, + { + "task": "sup03_durable_forget", + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 1" + ] + }, + { + "task": "upd01_stale_fact", + "passed": false, + "failures": [ + "step2(recall): top result does not contain 'june'" + ] + }, + { + "task": "upd02_preference_change", + "passed": false, + "failures": [ + "step2(recall): top result does not contain 'sms'" + ] + }, + { + "task": "upd03_state_supersession", + "passed": false, + "failures": [ + "step2(recall): top result does not contain '2222'" + ] + } + ], + "product_constraints": [] + }, + { + "lane": "langmem", + "status": "completed", + "mode": "full", + "run_id": "opmem-phase2-full1", + "elapsed_seconds": 42.07, + "infrastructure_errors": 0, + "tasks_run": 13, + "tasks_passed": 8, + "task_results": [ + { + "task": "cor01_basic_revision", + "passed": true, + "failures": [] + }, + { + "task": "cor02_correction_stickiness", + "passed": false, + "failures": [ + "step3(recall): top result does not contain 'ruby'" + ] + }, + { + "task": "cor03_revised_retrievable", + "passed": true, + "failures": [] + }, + { + "task": "dup01_idempotent_remember", + "passed": true, + "failures": [] + }, + { + "task": "iso01_subject_isolation", + "passed": true, + "failures": [] + }, + { + "task": "iso02_tenant_isolation", + "passed": true, + "failures": [] + }, + { + "task": "iso03_forget_isolated", + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 1" + ] + }, + { + "task": "sup01_basic_forget", + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 1" + ] + }, + { + "task": "sup02_targeted_forget", + "passed": false, + "failures": [ + "step3(recall): a result contains excluded 'door code'" + ] + }, + { + "task": "sup03_durable_forget", + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 1" + ] + }, + { + "task": "upd01_stale_fact", + "passed": true, + "failures": [] + }, + { + "task": "upd02_preference_change", + "passed": true, + "failures": [] + }, + { + "task": "upd03_state_supersession", + "passed": true, + "failures": [] + } + ], + "product_constraints": [] + } + ] + } + ] +} diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-letta-full.json b/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-letta-full.json new file mode 100644 index 00000000..b00af09f --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-letta-full.json @@ -0,0 +1,133 @@ +{ + "benchmark": "opmem-v0", + "generated_utc": "2026-09-21T17:43:56Z", + "run_id": "opmem-phase2-letta-full", + "host": "Linux-6.12.94+-x86_64-with-glibc2.39", + "python": "3.12.3", + "package_versions": { + "mem0ai": "2.1.0", + "qdrant-client": "1.19.1", + "zep-cloud": "3.28.0", + "tiktoken": "0.14.0", + "langmem": "0.0.30", + "langgraph-checkpoint-postgres": "3.1.2", + "openai": "1.109.1", + "langchain-openai": "1.1.9" + }, + "pinned_models": { + "chat": "gpt-4.1-mini-2025-04-14", + "embed": "text-embedding-3-small" + }, + "ceilings_usd": { + "mem0-oss": 1.0, + "letta": 1.0, + "langmem": 1.0, + "zep": 0.0, + "total": 3.0 + }, + "modes": [ + { + "mode": "full", + "task_count": 13, + "lanes": [ + { + "lane": "letta", + "status": "completed", + "mode": "full", + "run_id": "opmem-phase2-letta-full", + "elapsed_seconds": 6.56, + "infrastructure_errors": 0, + "tasks_run": 13, + "tasks_passed": 5, + "task_results": [ + { + "task": "cor01_basic_revision", + "passed": false, + "failures": [ + "step1(recall): expected at least 1 results, got 0", + "step1(recall): top result does not contain 'python'" + ] + }, + { + "task": "cor02_correction_stickiness", + "passed": false, + "failures": [ + "step3(recall): expected at least 1 results, got 0", + "step3(recall): top result does not contain 'ruby'" + ] + }, + { + "task": "cor03_revised_retrievable", + "passed": true, + "failures": [] + }, + { + "task": "dup01_idempotent_remember", + "passed": true, + "failures": [] + }, + { + "task": "iso01_subject_isolation", + "passed": false, + "failures": [ + "step1(recall): expected at most 0 results, got 1" + ] + }, + { + "task": "iso02_tenant_isolation", + "passed": false, + "failures": [ + "step1(recall): expected at most 0 results, got 1" + ] + }, + { + "task": "iso03_forget_isolated", + "passed": false, + "failures": [ + "step4(recall): expected at least 1 results, got 0" + ] + }, + { + "task": "sup01_basic_forget", + "passed": true, + "failures": [] + }, + { + "task": "sup02_targeted_forget", + "passed": true, + "failures": [] + }, + { + "task": "sup03_durable_forget", + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 1" + ] + }, + { + "task": "upd01_stale_fact", + "passed": false, + "failures": [ + "step2(recall): top result does not contain 'june'" + ] + }, + { + "task": "upd02_preference_change", + "passed": false, + "failures": [ + "step2(recall): expected at least 1 results, got 0", + "step2(recall): top result does not contain 'sms'" + ] + }, + { + "task": "upd03_state_supersession", + "passed": true, + "failures": [] + } + ], + "product_constraints": [] + } + ] + } + ] +} diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-letta-smoke.json b/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-letta-smoke.json new file mode 100644 index 00000000..e8c208f5 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-letta-smoke.json @@ -0,0 +1,54 @@ +{ + "benchmark": "opmem-v0", + "generated_utc": "2026-09-21T17:43:51Z", + "run_id": "opmem-phase2-letta-smoke", + "host": "Linux-6.12.94+-x86_64-with-glibc2.39", + "python": "3.12.3", + "package_versions": { + "mem0ai": "2.1.0", + "qdrant-client": "1.19.1", + "zep-cloud": "3.28.0", + "tiktoken": "0.14.0", + "langmem": "0.0.30", + "langgraph-checkpoint-postgres": "3.1.2", + "openai": "1.109.1", + "langchain-openai": "1.1.9" + }, + "pinned_models": { + "chat": "gpt-4.1-mini-2025-04-14", + "embed": "text-embedding-3-small" + }, + "ceilings_usd": { + "mem0-oss": 1.0, + "letta": 1.0, + "langmem": 1.0, + "zep": 0.0, + "total": 3.0 + }, + "modes": [ + { + "mode": "smoke", + "task": "dup01_idempotent_remember", + "lanes": [ + { + "lane": "letta", + "status": "completed", + "mode": "smoke", + "run_id": "opmem-phase2-letta-smoke", + "elapsed_seconds": 0.46, + "infrastructure_errors": 0, + "tasks_run": 1, + "tasks_passed": 1, + "task_results": [ + { + "task": "dup01_idempotent_remember", + "passed": true, + "failures": [] + } + ], + "product_constraints": [] + } + ] + } + ] +} diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-smoke1.json b/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-smoke1.json new file mode 100644 index 00000000..381daa44 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-smoke1.json @@ -0,0 +1,86 @@ +{ + "benchmark": "opmem-v0", + "generated_utc": "2026-09-19T08:28:45Z", + "run_id": "opmem-phase2-smoke1", + "host": "Linux-6.12.94+-x86_64-with-glibc2.39", + "python": "3.12.3", + "package_versions": { + "mem0ai": "2.1.0", + "qdrant-client": "1.19.1", + "zep-cloud": "3.28.0", + "tiktoken": "0.14.0", + "langmem": "0.0.30", + "langgraph-checkpoint-postgres": "3.1.2", + "openai": "3.16.2", + "langchain-openai": "1.6.2" + }, + "pinned_models": { + "chat": "gpt-4.1-mini-2025-04-14", + "embed": "text-embedding-3-small" + }, + "ceilings_usd": { + "mem0-oss": 1.0, + "letta": 1.0, + "langmem": 1.0, + "zep": 0.0, + "total": 3.0 + }, + "modes": [ + { + "mode": "smoke", + "task": "dup01_idempotent_remember", + "lanes": [ + { + "lane": "mem0-oss", + "status": "invalid_incomplete", + "mode": "smoke", + "run_id": "opmem-phase2-smoke1", + "elapsed_seconds": 6.36, + "infrastructure_errors": 1, + "tasks_run": 1, + "tasks_passed": 0, + "task_results": [ + { + "task": "dup01_idempotent_remember", + "error": "ValueError: Top-level entity parameters frozenset({'user_id'}) are not supported in search(). Use filters={'user_id': '...'} instead.", + "invalid_incomplete": true + } + ], + "product_constraints": [] + }, + { + "lane": "langmem", + "status": "invalid_incomplete", + "mode": "smoke", + "run_id": "opmem-phase2-smoke1", + "elapsed_seconds": 3.15, + "infrastructure_errors": 1, + "tasks_run": 1, + "tasks_passed": 0, + "task_results": [ + { + "task": "dup01_idempotent_remember", + "error": "AttributeError: 'dict' object has no attribute 'lower'", + "invalid_incomplete": true + } + ], + "product_constraints": [] + }, + { + "lane": "zep", + "status": "blocked", + "reason": "ZEP_API_KEY not present in runtime (vault not injected on this VM)", + "infrastructure_errors": 0, + "task_results": [] + }, + { + "lane": "letta", + "status": "blocked", + "reason": "LETTA_APP_SERVER_TOKEN not present in runtime", + "infrastructure_errors": 0, + "task_results": [] + } + ] + } + ] +} diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-smoke2.json b/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-smoke2.json new file mode 100644 index 00000000..b712fde4 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-smoke2.json @@ -0,0 +1,72 @@ +{ + "benchmark": "opmem-v0", + "generated_utc": "2026-09-19T08:29:11Z", + "run_id": "opmem-phase2-smoke2", + "host": "Linux-6.12.94+-x86_64-with-glibc2.39", + "python": "3.12.3", + "package_versions": { + "mem0ai": "2.1.0", + "qdrant-client": "1.19.1", + "zep-cloud": "3.28.0", + "tiktoken": "0.14.0", + "langmem": "0.0.30", + "langgraph-checkpoint-postgres": "3.1.2", + "openai": "3.16.2", + "langchain-openai": "1.6.2" + }, + "pinned_models": { + "chat": "gpt-4.1-mini-2025-04-14", + "embed": "text-embedding-3-small" + }, + "ceilings_usd": { + "mem0-oss": 1.0, + "letta": 1.0, + "langmem": 1.0, + "zep": 0.0, + "total": 3.0 + }, + "modes": [ + { + "mode": "smoke", + "task": "dup01_idempotent_remember", + "lanes": [ + { + "lane": "mem0-oss", + "status": "completed", + "mode": "smoke", + "run_id": "opmem-phase2-smoke2", + "elapsed_seconds": 2.9, + "infrastructure_errors": 0, + "tasks_run": 1, + "tasks_passed": 1, + "task_results": [ + { + "task": "dup01_idempotent_remember", + "passed": true, + "failures": [] + } + ], + "product_constraints": [] + }, + { + "lane": "langmem", + "status": "completed", + "mode": "smoke", + "run_id": "opmem-phase2-smoke2", + "elapsed_seconds": 3.68, + "infrastructure_errors": 0, + "tasks_run": 1, + "tasks_passed": 1, + "task_results": [ + { + "task": "dup01_idempotent_remember", + "passed": true, + "failures": [] + } + ], + "product_constraints": [] + } + ] + } + ] +} diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-zep-full.json b/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-zep-full.json new file mode 100644 index 00000000..b7921084 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-zep-full.json @@ -0,0 +1,140 @@ +{ + "benchmark": "opmem-v0", + "generated_utc": "2026-09-23T17:57:13Z", + "run_id": "opmem-phase2-zep-full", + "host": "Linux-6.12.94+-x86_64-with-glibc2.39", + "python": "3.12.3", + "package_versions": { + "mem0ai": "2.1.0", + "qdrant-client": "1.19.1", + "zep-cloud": "3.28.0", + "tiktoken": "0.14.0", + "langmem": "0.0.30", + "langgraph-checkpoint-postgres": "3.1.2", + "openai": "1.109.1", + "langchain-openai": "1.1.9" + }, + "pinned_models": { + "chat": "gpt-4.1-mini-2025-04-14", + "embed": "text-embedding-3-small" + }, + "ceilings_usd": { + "mem0-oss": 1.0, + "letta": 1.0, + "langmem": 1.0, + "zep": 0.0, + "total": 3.0 + }, + "modes": [ + { + "mode": "full", + "task_count": 13, + "lanes": [ + { + "lane": "zep", + "status": "completed", + "mode": "full", + "run_id": "opmem-phase2-zep-full", + "elapsed_seconds": 51.01, + "infrastructure_errors": 0, + "tasks_run": 13, + "tasks_passed": 3, + "task_results": [ + { + "task": "cor01_basic_revision", + "passed": false, + "failures": [ + "step1(recall): expected at least 1 results, got 0", + "step1(recall): top result does not contain 'python'" + ] + }, + { + "task": "cor02_correction_stickiness", + "passed": true, + "failures": [] + }, + { + "task": "cor03_revised_retrievable", + "passed": true, + "failures": [] + }, + { + "task": "dup01_idempotent_remember", + "passed": true, + "failures": [] + }, + { + "task": "iso01_subject_isolation", + "passed": false, + "failures": [ + "step1(recall): expected at most 0 results, got 1" + ] + }, + { + "task": "iso02_tenant_isolation", + "passed": false, + "failures": [ + "step1(recall): expected at most 0 results, got 1" + ] + }, + { + "task": "iso03_forget_isolated", + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 1" + ] + }, + { + "task": "sup01_basic_forget", + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 1" + ] + }, + { + "task": "sup02_targeted_forget", + "passed": false, + "failures": [ + "step3(recall): a result contains excluded 'door code'" + ] + }, + { + "task": "sup03_durable_forget", + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 2" + ] + }, + { + "task": "upd01_stale_fact", + "passed": false, + "failures": [ + "step2(recall): top result does not contain 'june'" + ] + }, + { + "task": "upd02_preference_change", + "passed": false, + "failures": [ + "step2(recall): expected at least 1 results, got 0", + "step2(recall): top result does not contain 'sms'" + ] + }, + { + "task": "upd03_state_supersession", + "passed": false, + "failures": [ + "step2(recall): top result does not contain '2222'" + ] + } + ], + "product_constraints": [ + "lane zep cannot pin chat model gpt-4.1-mini-2025-04-14", + "lane zep cannot pin embed model text-embedding-3-small", + "Zep Cloud memory path does not accept OpenAI chat/embed model pins; stop before substituting \u2014 record scores only under Zep-native models." + ] + } + ] + } + ] +} diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-zep-smoke-20260923.json b/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-zep-smoke-20260923.json new file mode 100644 index 00000000..cfb9bcfc --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-zep-smoke-20260923.json @@ -0,0 +1,43 @@ +{ + "benchmark": "opmem-v0", + "generated_utc": "2026-09-23T17:09:16Z", + "run_id": "opmem-phase2-zep-smoke-20260923", + "host": "Linux-6.12.94+-x86_64-with-glibc2.39", + "python": "3.12.3", + "package_versions": { + "mem0ai": "2.1.0", + "qdrant-client": "1.19.1", + "zep-cloud": "3.28.0", + "tiktoken": "0.14.0", + "langmem": "0.0.30", + "langgraph-checkpoint-postgres": "3.1.2", + "openai": "1.109.1", + "langchain-openai": "1.1.9" + }, + "pinned_models": { + "chat": "gpt-4.1-mini-2025-04-14", + "embed": "text-embedding-3-small" + }, + "ceilings_usd": { + "mem0-oss": 1.0, + "letta": 1.0, + "langmem": 1.0, + "zep": 0.0, + "total": 3.0 + }, + "modes": [ + { + "mode": "smoke", + "task": "dup01_idempotent_remember", + "lanes": [ + { + "lane": "zep", + "status": "blocked", + "reason": "ZEP_API_KEY not present in runtime (vault not injected on this VM)", + "infrastructure_errors": 0, + "task_results": [] + } + ] + } + ] +} diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-zep-smoke-20260923b.json b/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-zep-smoke-20260923b.json new file mode 100644 index 00000000..a4c19363 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-zep-smoke-20260923b.json @@ -0,0 +1,58 @@ +{ + "benchmark": "opmem-v0", + "generated_utc": "2026-09-23T17:55:34Z", + "run_id": "opmem-phase2-zep-smoke-20260923b", + "host": "Linux-6.12.94+-x86_64-with-glibc2.39", + "python": "3.12.3", + "package_versions": { + "mem0ai": "2.1.0", + "qdrant-client": "1.19.1", + "zep-cloud": "3.28.0", + "tiktoken": "0.14.0", + "langmem": "0.0.30", + "langgraph-checkpoint-postgres": "3.1.2", + "openai": "1.109.1", + "langchain-openai": "1.1.9" + }, + "pinned_models": { + "chat": "gpt-4.1-mini-2025-04-14", + "embed": "text-embedding-3-small" + }, + "ceilings_usd": { + "mem0-oss": 1.0, + "letta": 1.0, + "langmem": 1.0, + "zep": 0.0, + "total": 3.0 + }, + "modes": [ + { + "mode": "smoke", + "task": "dup01_idempotent_remember", + "lanes": [ + { + "lane": "zep", + "status": "invalid_incomplete", + "mode": "smoke", + "run_id": "opmem-phase2-zep-smoke-20260923b", + "elapsed_seconds": 0.32, + "infrastructure_errors": 1, + "tasks_run": 1, + "tasks_passed": 0, + "task_results": [ + { + "task": "dup01_idempotent_remember", + "error": "AttributeError: 'Zep' object has no attribute 'memory'", + "invalid_incomplete": true + } + ], + "product_constraints": [ + "lane zep cannot pin chat model gpt-4.1-mini-2025-04-14", + "lane zep cannot pin embed model text-embedding-3-small", + "Zep Cloud memory path does not accept OpenAI chat/embed model pins; stop before substituting \u2014 record scores only under Zep-native models." + ] + } + ] + } + ] +} diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-zep-smoke-20260923c.json b/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-zep-smoke-20260923c.json new file mode 100644 index 00000000..9967cebc --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-zep-smoke-20260923c.json @@ -0,0 +1,60 @@ +{ + "benchmark": "opmem-v0", + "generated_utc": "2026-09-23T17:56:23Z", + "run_id": "opmem-phase2-zep-smoke-20260923c", + "host": "Linux-6.12.94+-x86_64-with-glibc2.39", + "python": "3.12.3", + "package_versions": { + "mem0ai": "2.1.0", + "qdrant-client": "1.19.1", + "zep-cloud": "3.28.0", + "tiktoken": "0.14.0", + "langmem": "0.0.30", + "langgraph-checkpoint-postgres": "3.1.2", + "openai": "1.109.1", + "langchain-openai": "1.1.9" + }, + "pinned_models": { + "chat": "gpt-4.1-mini-2025-04-14", + "embed": "text-embedding-3-small" + }, + "ceilings_usd": { + "mem0-oss": 1.0, + "letta": 1.0, + "langmem": 1.0, + "zep": 0.0, + "total": 3.0 + }, + "modes": [ + { + "mode": "smoke", + "task": "dup01_idempotent_remember", + "lanes": [ + { + "lane": "zep", + "status": "completed", + "mode": "smoke", + "run_id": "opmem-phase2-zep-smoke-20260923c", + "elapsed_seconds": 3.95, + "infrastructure_errors": 0, + "tasks_run": 1, + "tasks_passed": 0, + "task_results": [ + { + "task": "dup01_idempotent_remember", + "passed": false, + "failures": [ + "step2(recall): expected at least 1 results, got 0" + ] + } + ], + "product_constraints": [ + "lane zep cannot pin chat model gpt-4.1-mini-2025-04-14", + "lane zep cannot pin embed model text-embedding-3-small", + "Zep Cloud memory path does not accept OpenAI chat/embed model pins; stop before substituting \u2014 record scores only under Zep-native models." + ] + } + ] + } + ] +} diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-zep-smoke-final.json b/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-zep-smoke-final.json new file mode 100644 index 00000000..2fb7143f --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-zep-smoke-final.json @@ -0,0 +1,60 @@ +{ + "benchmark": "opmem-v0", + "generated_utc": "2026-09-23T17:56:50Z", + "run_id": "opmem-phase2-zep-smoke-final", + "host": "Linux-6.12.94+-x86_64-with-glibc2.39", + "python": "3.12.3", + "package_versions": { + "mem0ai": "2.1.0", + "qdrant-client": "1.19.1", + "zep-cloud": "3.28.0", + "tiktoken": "0.14.0", + "langmem": "0.0.30", + "langgraph-checkpoint-postgres": "3.1.2", + "openai": "1.109.1", + "langchain-openai": "1.1.9" + }, + "pinned_models": { + "chat": "gpt-4.1-mini-2025-04-14", + "embed": "text-embedding-3-small" + }, + "ceilings_usd": { + "mem0-oss": 1.0, + "letta": 1.0, + "langmem": 1.0, + "zep": 0.0, + "total": 3.0 + }, + "modes": [ + { + "mode": "smoke", + "task": "dup01_idempotent_remember", + "lanes": [ + { + "lane": "zep", + "status": "completed", + "mode": "smoke", + "run_id": "opmem-phase2-zep-smoke-final", + "elapsed_seconds": 4.01, + "infrastructure_errors": 0, + "tasks_run": 1, + "tasks_passed": 0, + "task_results": [ + { + "task": "dup01_idempotent_remember", + "passed": false, + "failures": [ + "step2(recall): expected at most 1 results, got 2" + ] + } + ], + "product_constraints": [ + "lane zep cannot pin chat model gpt-4.1-mini-2025-04-14", + "lane zep cannot pin embed model text-embedding-3-small", + "Zep Cloud memory path does not accept OpenAI chat/embed model pins; stop before substituting \u2014 record scores only under Zep-native models." + ] + } + ] + } + ] +} diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-zep-smoke.json b/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-zep-smoke.json new file mode 100644 index 00000000..407bfb15 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/run-manifest-opmem-phase2-zep-smoke.json @@ -0,0 +1,43 @@ +{ + "benchmark": "opmem-v0", + "generated_utc": "2026-09-21T17:43:56Z", + "run_id": "opmem-phase2-zep-smoke", + "host": "Linux-6.12.94+-x86_64-with-glibc2.39", + "python": "3.12.3", + "package_versions": { + "mem0ai": "2.1.0", + "qdrant-client": "1.19.1", + "zep-cloud": "3.28.0", + "tiktoken": "0.14.0", + "langmem": "0.0.30", + "langgraph-checkpoint-postgres": "3.1.2", + "openai": "1.109.1", + "langchain-openai": "1.1.9" + }, + "pinned_models": { + "chat": "gpt-4.1-mini-2025-04-14", + "embed": "text-embedding-3-small" + }, + "ceilings_usd": { + "mem0-oss": 1.0, + "letta": 1.0, + "langmem": 1.0, + "zep": 0.0, + "total": 3.0 + }, + "modes": [ + { + "mode": "smoke", + "task": "dup01_idempotent_remember", + "lanes": [ + { + "lane": "zep", + "status": "blocked", + "reason": "ZEP_API_KEY not present in runtime (vault not injected on this VM)", + "infrastructure_errors": 0, + "task_results": [] + } + ] + } + ] +} diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/telemetry-langmem-full.json b/docs/benchmarks/artifacts/opmem-external-lanes/telemetry-langmem-full.json new file mode 100644 index 00000000..3035c9b9 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/telemetry-langmem-full.json @@ -0,0 +1,247 @@ +{ + "lane": "langmem", + "usage": { + "chat_input_tokens": 0, + "chat_output_tokens": 0, + "embed_tokens": 0, + "estimated_usd": 0.0, + "http_requests": 47 + }, + "events": [ + { + "op": "remember", + "status": 200, + "body_preview": "langmem manager.invoke" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[Item(namespace=['opmem', 'opmem-ad77ba8f7f247b76'], key='ff038350-4658-4617-92d9-4c5553430656', value={'kind': 'Memory', 'content': {'content': 'The user prefers Python for scripting.'}}, created_at='2026-09-19T08:29:52.169754+00:00', updated_at='2026-09-19T08:29:52.169754+00:00', score=0.33778279090288055)]" + }, + { + "op": "revise", + "status": 200, + "body_preview": "langmem revise invoke" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[Item(namespace=['opmem', 'opmem-ad77ba8f7f247b76'], key='ff038350-4658-4617-92d9-4c5553430656', value={'kind': 'Memory', 'content': {'content': 'The user prefers Ruby for scripting.'}}, created_at='2026-09-19T08:29:52.169754+00:00', updated_at='2026-09-19T08:29:54.174845+00:00', score=0.33776719150121126)]" + }, + { + "op": "remember", + "status": 200, + "body_preview": "langmem manager.invoke" + }, + { + "op": "revise", + "status": 200, + "body_preview": "langmem revise invoke" + }, + { + "op": "remember", + "status": 200, + "body_preview": "langmem manager.invoke" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[Item(namespace=['opmem', 'opmem-59710611587956f7'], key='20fc00d2-bdbe-498b-9cc2-94ae6d6926bc', value={'kind': 'Memory', 'content': {'content': 'User prefers Python for scripting.'}}, created_at='2026-09-19T08:29:57.897057+00:00', updated_at='2026-09-19T08:30:00.641575+00:00', score=0.33639699263808853), Item(namespace=['opmem', 'opmem-59710611587956f7'], key='8cb44cc6-27e1-488b-9394-d695a6bc0365', value={'kind': 'Memory', 'content': {'content': 'User prefers Python for scripting.'}}, created_a" + }, + { + "op": "remember", + "status": 200, + "body_preview": "langmem manager.invoke" + }, + { + "op": "revise", + "status": 200, + "body_preview": "langmem revise invoke" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[Item(namespace=['opmem', 'opmem-d89f907bfefacf0a'], key='c801ad89-40f4-4470-bfc4-9fa5cc81c716', value={'kind': 'Memory', 'content': {'content': 'The user prefers Ruby for scripting.'}}, created_at='2026-09-19T08:30:02.205766+00:00', updated_at='2026-09-19T08:30:04.766813+00:00', score=0.39614996459102103)]" + }, + { + "op": "remember", + "status": 200, + "body_preview": "langmem manager.invoke" + }, + { + "op": "remember", + "status": 200, + "body_preview": "langmem manager.invoke" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[Item(namespace=['opmem', 'opmem-231328502e0e8c77'], key='cf5aa7fc-14e2-4b88-9f5d-b7fc78fa702c', value={'kind': 'Memory', 'content': {'content': \"The user's wifi network name is FalconNet.\"}}, created_at='2026-09-19T08:30:06.559814+00:00', updated_at='2026-09-19T08:30:06.559814+00:00', score=0.4307357902741695)]" + }, + { + "op": "remember", + "status": 200, + "body_preview": "langmem manager.invoke" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[]" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[Item(namespace=['opmem', 'opmem-f8551362a879e5a5'], key='f7dacf51-98ed-4656-baac-525040d049b3', value={'kind': 'Memory', 'content': {'content': \"The user's name is Alice Chen.\"}}, created_at='2026-09-19T08:30:09.585092+00:00', updated_at='2026-09-19T08:30:09.585092+00:00', score=0.4185555945234981)]" + }, + { + "op": "remember", + "status": 200, + "body_preview": "langmem manager.invoke" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[]" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[Item(namespace=['opmem', 'opmem-0455ead06fd45698'], key='dd6965ac-061d-4b4c-bcd0-be701870e9e4', value={'kind': 'Memory', 'content': {'content': 'The WiFi network name is FalconNet.'}}, created_at='2026-09-19T08:30:11.539575+00:00', updated_at='2026-09-19T08:30:11.539575+00:00', score=0.4365041995152231)]" + }, + { + "op": "remember", + "status": 200, + "body_preview": "langmem manager.invoke" + }, + { + "op": "remember", + "status": 200, + "body_preview": "langmem manager.invoke" + }, + { + "op": "forget", + "status": 200, + "body_preview": "deleted ['m-3432cec89e27']" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[Item(namespace=['opmem', 'opmem-001fc67db28bc8f6'], key='105cdf9d-ab27-471c-9f18-c86f1f0f6ebc', value={'kind': 'Memory', 'content': {'content': 'Never share the door code with vendors.'}}, created_at='2026-09-19T08:30:13.157541+00:00', updated_at='2026-09-19T08:30:13.157541+00:00', score=0.4455942668279429)]" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[Item(namespace=['opmem', 'opmem-6ba67fb6e2bcd4a0'], key='7d84ba1f-5876-4885-8513-a87a0fa94c24', value={'kind': 'Memory', 'content': {'content': 'Do not share the door code with vendors.'}}, created_at='2026-09-19T08:30:14.262160+00:00', updated_at='2026-09-19T08:30:14.262160+00:00', score=0.4431479037244387)]" + }, + { + "op": "remember", + "status": 200, + "body_preview": "langmem manager.invoke" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[Item(namespace=['opmem', 'opmem-0769c5496ae38e0b'], key='62a0c516-bc1d-4f30-8557-67215486aee8', value={'kind': 'Memory', 'content': {'content': 'The user has a rule to never share the door code with vendors.'}}, created_at='2026-09-19T08:30:15.814395+00:00', updated_at='2026-09-19T08:30:15.814395+00:00', score=0.38258729230241584)]" + }, + { + "op": "forget", + "status": 200, + "body_preview": "deleted ['m-10416bc6bd04']" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[Item(namespace=['opmem', 'opmem-0769c5496ae38e0b'], key='62a0c516-bc1d-4f30-8557-67215486aee8', value={'kind': 'Memory', 'content': {'content': 'The user has a rule to never share the door code with vendors.'}}, created_at='2026-09-19T08:30:15.814395+00:00', updated_at='2026-09-19T08:30:15.814395+00:00', score=0.38258729230241584)]" + }, + { + "op": "remember", + "status": 200, + "body_preview": "langmem manager.invoke" + }, + { + "op": "remember", + "status": 200, + "body_preview": "langmem manager.invoke" + }, + { + "op": "forget", + "status": 200, + "body_preview": "deleted ['m-d98a3e5fcba9']" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[Item(namespace=['opmem', 'opmem-00a5581691894547'], key='2fe0745c-e090-4abb-8ee8-476f82624da3', value={'kind': 'Memory', 'content': {'content': 'Never share the door code with vendors.'}}, created_at='2026-09-19T08:30:17.777570+00:00', updated_at='2026-09-19T08:30:17.777570+00:00', score=0.44552312577829234), Item(namespace=['opmem', 'opmem-00a5581691894547'], key='169a612a-7f1b-43bb-919c-1f952bf3018b', value={'kind': 'Memory', 'content': {'content': 'The wifi network name is FalconNet.'}}, cre" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[Item(namespace=['opmem', 'opmem-00a5581691894547'], key='169a612a-7f1b-43bb-919c-1f952bf3018b', value={'kind': 'Memory', 'content': {'content': 'The wifi network name is FalconNet.'}}, created_at='2026-09-19T08:30:19.181133+00:00', updated_at='2026-09-19T08:30:19.181133+00:00', score=0.44823255645278093), Item(namespace=['opmem', 'opmem-00a5581691894547'], key='2fe0745c-e090-4abb-8ee8-476f82624da3', value={'kind': 'Memory', 'content': {'content': 'Never share the door code with vendors.'}}, cre" + }, + { + "op": "remember", + "status": 200, + "body_preview": "langmem manager.invoke" + }, + { + "op": "forget", + "status": 200, + "body_preview": "deleted ['m-357804338ee6']" + }, + { + "op": "remember", + "status": 200, + "body_preview": "langmem manager.invoke" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[Item(namespace=['opmem', 'opmem-d011d90d55ad1853'], key='567d190a-d2f0-412a-8a2a-62d23bd5802c', value={'kind': 'Memory', 'content': {'content': 'User directive: Never share the door code with vendors.'}}, created_at='2026-09-19T08:30:20.884100+00:00', updated_at='2026-09-19T08:30:20.884100+00:00', score=0.42092394769662933)]" + }, + { + "op": "remember", + "status": 200, + "body_preview": "langmem manager.invoke" + }, + { + "op": "remember", + "status": 200, + "body_preview": "langmem manager.invoke" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[Item(namespace=['opmem', 'opmem-80742538490187d6'], key='575335a6-b0ea-423a-824e-29ff7fab88d4', value={'kind': 'Memory', 'content': {'content': 'The launch date is June 3.'}}, created_at='2026-09-19T08:30:23.681623+00:00', updated_at='2026-09-19T08:30:25.608805+00:00', score=0.4816371075183463)]" + }, + { + "op": "remember", + "status": 200, + "body_preview": "langmem manager.invoke" + }, + { + "op": "remember", + "status": 200, + "body_preview": "langmem manager.invoke" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[Item(namespace=['opmem', 'opmem-f9ff0a9d8b5da8ad'], key='29100b9a-a18e-4adf-afce-04fa698148bb', value={'kind': 'Memory', 'content': {'content': 'User preference: prefers SMS updates.'}}, created_at='2026-09-19T08:30:29.174995+00:00', updated_at='2026-09-19T08:30:29.174995+00:00', score=0.30517559730557475), Item(namespace=['opmem', 'opmem-f9ff0a9d8b5da8ad'], key='7f6b6804-28ee-49f5-ae45-15e137a5d47a', value={'kind': 'Memory', 'content': {'content': 'User preference: prefers SMS updates.'}}, cre" + }, + { + "op": "remember", + "status": 200, + "body_preview": "langmem manager.invoke" + }, + { + "op": "remember", + "status": 200, + "body_preview": "langmem manager.invoke" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[Item(namespace=['opmem', 'opmem-45283803a0a23fec'], key='4037cfed-accb-40d6-877e-e7075c207a71', value={'kind': 'Memory', 'content': {'content': 'The door code is 2222.'}}, created_at='2026-09-19T08:30:30.507762+00:00', updated_at='2026-09-19T08:30:32.350165+00:00', score=0.4994123415529361)]" + } + ] +} diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/telemetry-langmem-smoke.json b/docs/benchmarks/artifacts/opmem-external-lanes/telemetry-langmem-smoke.json new file mode 100644 index 00000000..d9ef7c1a --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/telemetry-langmem-smoke.json @@ -0,0 +1,27 @@ +{ + "lane": "langmem", + "usage": { + "chat_input_tokens": 0, + "chat_output_tokens": 0, + "embed_tokens": 0, + "estimated_usd": 0.0, + "http_requests": 3 + }, + "events": [ + { + "op": "remember", + "status": 200, + "body_preview": "langmem manager.invoke" + }, + { + "op": "remember", + "status": 200, + "body_preview": "langmem manager.invoke" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[Item(namespace=['opmem', 'opmem-4dcb416cf36390f7'], key='e0f473c9-41e7-4cda-9bb3-6fcfd7ddc597', value={'kind': 'Memory', 'content': {'content': 'The wifi network name is FalconNet.'}}, created_at='2026-09-19T08:29:16.603964+00:00', updated_at='2026-09-19T08:29:16.603964+00:00', score=0.44823255645278093)]" + } + ] +} diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/telemetry-letta-full.json b/docs/benchmarks/artifacts/opmem-external-lanes/telemetry-letta-full.json new file mode 100644 index 00000000..683701bd --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/telemetry-letta-full.json @@ -0,0 +1,347 @@ +{ + "lane": "letta", + "usage": { + "chat_input_tokens": 0, + "chat_output_tokens": 0, + "embed_tokens": 0, + "estimated_usd": 0.0, + "http_requests": 67 + }, + "events": [ + { + "op": "reset", + "status": 200, + "body_preview": "agent agent-0e765134-eef7-40be-bca4-fd55de9054e6" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "1 facts" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[]" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "1 facts" + }, + { + "op": "revise", + "status": 200, + "body_preview": "updated m-72d0cb92d778" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'm-72d0cb92d778', 'content': 'Prefers Ruby for scripting'}]" + }, + { + "op": "reset", + "status": 200, + "body_preview": "agent agent-1980a99b-fa48-4eea-bde4-fb8d923793cc" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "1 facts" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "1 facts" + }, + { + "op": "revise", + "status": 200, + "body_preview": "updated m-9bdc773d176d" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "1 facts" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[]" + }, + { + "op": "reset", + "status": 200, + "body_preview": "agent agent-e32f945a-47de-4061-afff-882d2c175d82" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "1 facts" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "1 facts" + }, + { + "op": "revise", + "status": 200, + "body_preview": "updated m-e53c02783ebf" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'm-e53c02783ebf', 'content': 'Prefers Ruby for scripting'}]" + }, + { + "op": "reset", + "status": 200, + "body_preview": "agent agent-d6215054-0208-4946-a9cf-9e3263da81c2" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "1 facts" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "1 facts" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'm-86f89e5fc162', 'content': 'The wifi network name is FalconNet.'}]" + }, + { + "op": "reset", + "status": 200, + "body_preview": "agent agent-e15bf9de-e27a-4c65-815d-4da1f2b49de0" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "1 facts" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'm-3d6f2ccfc041', 'content': 'My name is Alice Chen.'}]" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'm-3d6f2ccfc041', 'content': 'My name is Alice Chen.'}]" + }, + { + "op": "reset", + "status": 200, + "body_preview": "agent agent-dc86f6bd-b17c-4219-badd-1e19389c0c91" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "1 facts" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'm-6d39288e1285', 'content': 'The wifi network name is FalconNet.'}]" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'm-6d39288e1285', 'content': 'The wifi network name is FalconNet.'}]" + }, + { + "op": "reset", + "status": 200, + "body_preview": "agent agent-2c51ef1a-b9b5-407c-9c25-ded27519a2a0" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "1 facts" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "1 facts" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "0 facts" + }, + { + "op": "forget", + "status": 200, + "body_preview": "removed ['m-199792273a68']" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[]" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[]" + }, + { + "op": "reset", + "status": 200, + "body_preview": "agent agent-8d05177a-19d2-4dbe-ad34-697cc6418823" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "1 facts" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'm-fcff7a76adb5', 'content': 'Never share the door code with vendors.'}]" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "0 facts" + }, + { + "op": "forget", + "status": 200, + "body_preview": "removed ['m-fcff7a76adb5']" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[]" + }, + { + "op": "reset", + "status": 200, + "body_preview": "agent agent-cd220203-8ef1-42e0-b207-b0c7076560b1" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "1 facts" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "2 facts" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "1 facts" + }, + { + "op": "forget", + "status": 200, + "body_preview": "removed ['m-876d3c78fdbb']" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[]" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'm-8a2ae350d5a4', 'content': 'The wifi network name is FalconNet.'}]" + }, + { + "op": "reset", + "status": 200, + "body_preview": "agent agent-91dd1cc4-65a7-4b6f-a5e7-fff67b8612f8" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "1 facts" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "0 facts" + }, + { + "op": "forget", + "status": 200, + "body_preview": "removed ['m-f372883003fe']" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "1 facts" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'm-f372883003fe', 'content': 'Never share the door code with vendors.'}]" + }, + { + "op": "reset", + "status": 200, + "body_preview": "agent agent-84ee20a4-8121-4b38-aa77-d6f23aaaa6b6" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "1 facts" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "2 facts" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'm-539fd5974dad', 'content': 'The launch date is May 12.'}, {'id': 'm-a3552df98ca5', 'content': 'The launch date is June 3.'}]" + }, + { + "op": "reset", + "status": 200, + "body_preview": "agent agent-bcbd0fe3-f78d-497e-a2fd-c79d2331e434" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "1 facts" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "2 facts" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[]" + }, + { + "op": "reset", + "status": 200, + "body_preview": "agent agent-019571c6-1f7d-40e1-9300-8e7d32de02a2" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "1 facts" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "2 facts" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'm-6d0ba2632157', 'content': 'The door code is 2222.'}, {'id': 'm-f216d185ca5e', 'content': 'The door code is 1111.'}]" + } + ] +} diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/telemetry-letta-smoke.json b/docs/benchmarks/artifacts/opmem-external-lanes/telemetry-letta-smoke.json new file mode 100644 index 00000000..8d051e29 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/telemetry-letta-smoke.json @@ -0,0 +1,32 @@ +{ + "lane": "letta", + "usage": { + "chat_input_tokens": 0, + "chat_output_tokens": 0, + "embed_tokens": 0, + "estimated_usd": 0.0, + "http_requests": 4 + }, + "events": [ + { + "op": "reset", + "status": 200, + "body_preview": "agent agent-eefd561c-670e-46e2-a634-17a5cb704ba2" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "1 facts" + }, + { + "op": "blocks.modify", + "status": 200, + "body_preview": "1 facts" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'm-5ce49f74b219', 'content': 'The wifi network name is FalconNet.'}]" + } + ] +} diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/telemetry-mem0-oss-full.json b/docs/benchmarks/artifacts/opmem-external-lanes/telemetry-mem0-oss-full.json new file mode 100644 index 00000000..8ef89380 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/telemetry-mem0-oss-full.json @@ -0,0 +1,247 @@ +{ + "lane": "mem0-oss", + "usage": { + "chat_input_tokens": 0, + "chat_output_tokens": 0, + "embed_tokens": 0, + "estimated_usd": 0.0, + "http_requests": 47 + }, + "events": [ + { + "op": "remember", + "status": 200, + "body_preview": "{'results': [{'id': '5c8819cd-ce54-413d-9e88-03ecd4f70f52', 'memory': 'User prefers Python for scripting tasks', 'event': 'ADD'}]}" + }, + { + "op": "recall", + "status": 200, + "body_preview": "{'results': [{'id': '5c8819cd-ce54-413d-9e88-03ecd4f70f52', 'memory': 'User prefers Python for scripting tasks', 'hash': 'a48dc9b92de4e17c1cc8962e11c6dd48', 'metadata': None, 'score': 0.47747782832465313, 'created_at': '2026-09-19T08:30:48.595042+00:00', 'updated_at': '2026-09-19T08:30:48.595042+00:00', 'user_id': 'opmem-5e7cd8f9c26a01c7', 'attributed_to': 'user'}]}" + }, + { + "op": "revise", + "status": 200, + "body_preview": "updated 5c8819cd-ce54-413d-9e88-03ecd4f70f52" + }, + { + "op": "recall", + "status": 200, + "body_preview": "{'results': [{'id': '5c8819cd-ce54-413d-9e88-03ecd4f70f52', 'memory': 'Prefers Ruby for scripting', 'hash': 'f6e4b4415f448d039530b6671977e0b2', 'metadata': None, 'score': 0.5080760678115537, 'created_at': '2026-09-19T08:30:48.595042+00:00', 'updated_at': '2026-09-19T08:30:48.935555+00:00', 'user_id': 'opmem-5e7cd8f9c26a01c7', 'attributed_to': 'user'}]}" + }, + { + "op": "remember", + "status": 200, + "body_preview": "{'results': [{'id': '7a05a16c-92e8-4b02-bfe7-c0aaa4fb865b', 'memory': 'User prefers Python for scripting tasks', 'event': 'ADD'}]}" + }, + { + "op": "revise", + "status": 200, + "body_preview": "updated 7a05a16c-92e8-4b02-bfe7-c0aaa4fb865b" + }, + { + "op": "remember", + "status": 200, + "body_preview": "{'results': [{'id': '2e2eee9d-57d6-430d-8aa8-7255274580f1', 'memory': 'User prefers Python for scripting, updating their previous preference for Ruby', 'event': 'ADD'}]}" + }, + { + "op": "recall", + "status": 200, + "body_preview": "{'results': [{'id': '7a05a16c-92e8-4b02-bfe7-c0aaa4fb865b', 'memory': 'Prefers Ruby for scripting', 'hash': 'f6e4b4415f448d039530b6671977e0b2', 'metadata': None, 'score': 0.5080626159016706, 'created_at': '2026-09-19T08:30:52.277528+00:00', 'updated_at': '2026-09-19T08:30:52.458560+00:00', 'user_id': 'opmem-191998c3fb442640', 'attributed_to': 'user'}, {'id': '2e2eee9d-57d6-430d-8aa8-7255274580f1', 'memory': 'User prefers Python for scripting, updating their previous preference for Ruby', 'hash':" + }, + { + "op": "remember", + "status": 200, + "body_preview": "{'results': [{'id': 'be262d77-4d24-4228-b910-1bf10000435e', 'memory': 'User prefers Python for scripting tasks.', 'event': 'ADD'}]}" + }, + { + "op": "revise", + "status": 200, + "body_preview": "updated be262d77-4d24-4228-b910-1bf10000435e" + }, + { + "op": "recall", + "status": 200, + "body_preview": "{'results': [{'id': 'be262d77-4d24-4228-b910-1bf10000435e', 'memory': 'Prefers Ruby for scripting', 'hash': 'f6e4b4415f448d039530b6671977e0b2', 'metadata': None, 'score': 0.5780423781379709, 'created_at': '2026-09-19T08:30:57.173437+00:00', 'updated_at': '2026-09-19T08:30:57.374013+00:00', 'user_id': 'opmem-59047463df9dc059', 'attributed_to': 'user'}]}" + }, + { + "op": "remember", + "status": 200, + "body_preview": "{'results': [{'id': '9388e553-6720-4596-828a-1364e2125f4c', 'memory': \"User's wifi network name is FalconNet\", 'event': 'ADD'}]}" + }, + { + "op": "remember", + "status": 200, + "body_preview": "{'results': []}" + }, + { + "op": "recall", + "status": 200, + "body_preview": "{'results': [{'id': '9388e553-6720-4596-828a-1364e2125f4c', 'memory': \"User's wifi network name is FalconNet\", 'hash': '0e881d9cc295e8509eb50d959ed86653', 'metadata': None, 'score': 0.587925614386172, 'created_at': '2026-09-19T08:31:00.593416+00:00', 'updated_at': '2026-09-19T08:31:00.593416+00:00', 'user_id': 'opmem-791c09301b2c6c03', 'attributed_to': 'user'}]}" + }, + { + "op": "remember", + "status": 200, + "body_preview": "{'results': [{'id': '5554baa6-3ca9-47b9-8515-fe42cf7d1a95', 'memory': \"User's name is Alice Chen\", 'event': 'ADD'}]}" + }, + { + "op": "recall", + "status": 200, + "body_preview": "{'results': []}" + }, + { + "op": "recall", + "status": 200, + "body_preview": "{'results': [{'id': '5554baa6-3ca9-47b9-8515-fe42cf7d1a95', 'memory': \"User's name is Alice Chen\", 'hash': 'b7c543b1a5c7edd3d2a5bed7ad46fb84', 'metadata': None, 'score': 0.6258135863190433, 'created_at': '2026-09-19T08:31:02.655875+00:00', 'updated_at': '2026-09-19T08:31:02.655875+00:00', 'user_id': 'opmem-c659fde1c729f26a', 'attributed_to': 'user'}]}" + }, + { + "op": "remember", + "status": 200, + "body_preview": "{'results': [{'id': 'fc780410-3627-4a3f-9b7e-3083586df000', 'memory': \"User's wifi network name is FalconNet\", 'event': 'ADD'}]}" + }, + { + "op": "recall", + "status": 200, + "body_preview": "{'results': []}" + }, + { + "op": "recall", + "status": 200, + "body_preview": "{'results': [{'id': 'fc780410-3627-4a3f-9b7e-3083586df000', 'memory': \"User's wifi network name is FalconNet\", 'hash': '0e881d9cc295e8509eb50d959ed86653', 'metadata': None, 'score': 0.587925614386172, 'created_at': '2026-09-19T08:31:04.048007+00:00', 'updated_at': '2026-09-19T08:31:04.048007+00:00', 'user_id': 'opmem-fc2994ec968c8c35', 'attributed_to': 'user'}]}" + }, + { + "op": "remember", + "status": 200, + "body_preview": "{'results': [{'id': '8a4933cc-883e-4524-a2c1-6f2bb9e2b0cd', 'memory': 'User instructs never to share the door code with vendors to maintain security.', 'event': 'ADD'}]}" + }, + { + "op": "remember", + "status": 200, + "body_preview": "{'results': [{'id': '77a39bcc-4cd3-44b1-9430-49f57c87496d', 'memory': 'User instructs never to share the door code with vendors to maintain security', 'event': 'ADD'}]}" + }, + { + "op": "forget", + "status": 200, + "body_preview": "deleted 8a4933cc-883e-4524-a2c1-6f2bb9e2b0cd" + }, + { + "op": "recall", + "status": 200, + "body_preview": "{'results': []}" + }, + { + "op": "recall", + "status": 200, + "body_preview": "{'results': [{'id': '77a39bcc-4cd3-44b1-9430-49f57c87496d', 'memory': 'User instructs never to share the door code with vendors to maintain security', 'hash': '5eef4b9cdafb75fc82e5467f75e2dd30', 'metadata': None, 'score': 0.555903332358419, 'created_at': '2026-09-19T08:31:06.847614+00:00', 'updated_at': '2026-09-19T08:31:06.847614+00:00', 'user_id': 'opmem-3c9b018485c2a47c', 'attributed_to': 'user'}]}" + }, + { + "op": "remember", + "status": 200, + "body_preview": "{'results': [{'id': '5bb8d9b0-3d68-4c2e-8014-5613f99bbc39', 'memory': 'User has a strict rule to never share the door code with vendors to maintain security.', 'event': 'ADD'}]}" + }, + { + "op": "recall", + "status": 200, + "body_preview": "{'results': [{'id': '5bb8d9b0-3d68-4c2e-8014-5613f99bbc39', 'memory': 'User has a strict rule to never share the door code with vendors to maintain security.', 'hash': 'd2a86a6f638f37284a6e9e69bc78b9b8', 'metadata': None, 'score': 0.5078514150420694, 'created_at': '2026-09-19T08:31:08.562864+00:00', 'updated_at': '2026-09-19T08:31:08.562864+00:00', 'user_id': 'opmem-e915b53e598654e2', 'attributed_to': 'user'}]}" + }, + { + "op": "forget", + "status": 200, + "body_preview": "deleted 5bb8d9b0-3d68-4c2e-8014-5613f99bbc39" + }, + { + "op": "recall", + "status": 200, + "body_preview": "{'results': []}" + }, + { + "op": "remember", + "status": 200, + "body_preview": "{'results': [{'id': '74a731ad-891b-44a5-8d47-524a7a69bcc3', 'memory': 'User instructs never to share the door code with vendors to maintain security', 'event': 'ADD'}]}" + }, + { + "op": "remember", + "status": 200, + "body_preview": "{'results': [{'id': 'abbd8373-beca-4604-a351-c869573bed28', 'memory': \"User's wifi network name is FalconNet\", 'event': 'ADD'}]}" + }, + { + "op": "forget", + "status": 200, + "body_preview": "deleted 74a731ad-891b-44a5-8d47-524a7a69bcc3" + }, + { + "op": "recall", + "status": 200, + "body_preview": "{'results': [{'id': 'abbd8373-beca-4604-a351-c869573bed28', 'memory': \"User's wifi network name is FalconNet\", 'hash': '0e881d9cc295e8509eb50d959ed86653', 'metadata': None, 'score': 0.2068300875562672, 'created_at': '2026-09-19T08:31:11.421207+00:00', 'updated_at': '2026-09-19T08:31:11.421207+00:00', 'user_id': 'opmem-327e9acda69047a2', 'attributed_to': 'user'}]}" + }, + { + "op": "recall", + "status": 200, + "body_preview": "{'results': [{'id': 'abbd8373-beca-4604-a351-c869573bed28', 'memory': \"User's wifi network name is FalconNet\", 'hash': '0e881d9cc295e8509eb50d959ed86653', 'metadata': None, 'score': 0.5879256143861721, 'created_at': '2026-09-19T08:31:11.421207+00:00', 'updated_at': '2026-09-19T08:31:11.421207+00:00', 'user_id': 'opmem-327e9acda69047a2', 'attributed_to': 'user'}]}" + }, + { + "op": "remember", + "status": 200, + "body_preview": "{'results': [{'id': 'd6481591-450e-4a48-8b08-826548c9cd88', 'memory': 'User instructs never to share the door code with vendors to maintain security.', 'event': 'ADD'}]}" + }, + { + "op": "forget", + "status": 200, + "body_preview": "deleted d6481591-450e-4a48-8b08-826548c9cd88" + }, + { + "op": "remember", + "status": 200, + "body_preview": "{'results': [{'id': 'ef10248a-f45e-4f2e-9e0b-2004db58a01a', 'memory': 'User has a strict rule to never share the door code with vendors for security reasons', 'event': 'ADD'}]}" + }, + { + "op": "recall", + "status": 200, + "body_preview": "{'results': [{'id': 'ef10248a-f45e-4f2e-9e0b-2004db58a01a', 'memory': 'User has a strict rule to never share the door code with vendors for security reasons', 'hash': 'fbcafb4b09ccf7e171a380515f78367a', 'metadata': None, 'score': 0.5323019705553599, 'created_at': '2026-09-19T08:31:14.079406+00:00', 'updated_at': '2026-09-19T08:31:14.079406+00:00', 'user_id': 'opmem-e929007e8f1422c5', 'attributed_to': 'user'}]}" + }, + { + "op": "remember", + "status": 200, + "body_preview": "{'results': [{'id': '355365f8-ebef-4d4d-b914-612e7bf7083e', 'memory': 'User stated that the launch date is May 12', 'event': 'ADD'}]}" + }, + { + "op": "remember", + "status": 200, + "body_preview": "{'results': [{'id': '97cd9fb1-8a12-465b-8160-0a47bc7d4667', 'memory': 'User updated the launch date to June 3, 2026, replacing the previous date of May 12, 2026', 'event': 'ADD'}]}" + }, + { + "op": "recall", + "status": 200, + "body_preview": "{'results': [{'id': '355365f8-ebef-4d4d-b914-612e7bf7083e', 'memory': 'User stated that the launch date is May 12', 'hash': 'fa737facb798b3f3fc1535dc256ebf00', 'metadata': None, 'score': 0.6070983935080743, 'created_at': '2026-09-19T08:31:15.501751+00:00', 'updated_at': '2026-09-19T08:31:15.501751+00:00', 'user_id': 'opmem-e98de152102dc08d', 'attributed_to': 'user'}, {'id': '97cd9fb1-8a12-465b-8160-0a47bc7d4667', 'memory': 'User updated the launch date to June 3, 2026, replacing the previous dat" + }, + { + "op": "remember", + "status": 200, + "body_preview": "{'results': [{'id': '5b51b507-b33e-408a-8a00-0958ef839ad2', 'memory': 'User prefers to receive updates via email', 'event': 'ADD'}]}" + }, + { + "op": "remember", + "status": 200, + "body_preview": "{'results': [{'id': '73a3c476-f478-4c49-8163-1296e6546265', 'memory': 'User prefers SMS updates instead of email updates as of September 19, 2026', 'event': 'ADD'}]}" + }, + { + "op": "recall", + "status": 200, + "body_preview": "{'results': [{'id': '5b51b507-b33e-408a-8a00-0958ef839ad2', 'memory': 'User prefers to receive updates via email', 'hash': '6bfa2475c3bc61a23a48fdff754bfd8d', 'metadata': None, 'score': 0.38536272449852116, 'created_at': '2026-09-19T08:31:18.395105+00:00', 'updated_at': '2026-09-19T08:31:18.395105+00:00', 'user_id': 'opmem-7a5832b838c04f48', 'attributed_to': 'user'}, {'id': '73a3c476-f478-4c49-8163-1296e6546265', 'memory': 'User prefers SMS updates instead of email updates as of September 19, 20" + }, + { + "op": "remember", + "status": 200, + "body_preview": "{'results': [{'id': '34148019-1faf-4ea6-9762-704993a93489', 'memory': 'User shared that the door code is 1111', 'event': 'ADD'}]}" + }, + { + "op": "remember", + "status": 200, + "body_preview": "{'results': [{'id': '77525f51-2be9-4376-8c70-3b12b05392cd', 'memory': 'User updated the door code to 2222 on September 19, 2026', 'event': 'ADD'}]}" + }, + { + "op": "recall", + "status": 200, + "body_preview": "{'results': [{'id': '34148019-1faf-4ea6-9762-704993a93489', 'memory': 'User shared that the door code is 1111', 'hash': 'd4ff66858269b4e5157f79c4164a9ed1', 'metadata': None, 'score': 0.6118697230882042, 'created_at': '2026-09-19T08:31:20.904109+00:00', 'updated_at': '2026-09-19T08:31:20.904109+00:00', 'user_id': 'opmem-d3d362b2f352c734', 'attributed_to': 'user'}, {'id': '77525f51-2be9-4376-8c70-3b12b05392cd', 'memory': 'User updated the door code to 2222 on September 19, 2026', 'hash': '1ea2b7f3" + } + ] +} diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/telemetry-mem0-oss-smoke.json b/docs/benchmarks/artifacts/opmem-external-lanes/telemetry-mem0-oss-smoke.json new file mode 100644 index 00000000..edcb6dd6 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/telemetry-mem0-oss-smoke.json @@ -0,0 +1,27 @@ +{ + "lane": "mem0-oss", + "usage": { + "chat_input_tokens": 0, + "chat_output_tokens": 0, + "embed_tokens": 0, + "estimated_usd": 0.0, + "http_requests": 3 + }, + "events": [ + { + "op": "remember", + "status": 200, + "body_preview": "{'results': [{'id': '7d6f7716-8895-4a86-839f-25514b82599e', 'memory': \"User's wifi network name is FalconNet\", 'event': 'ADD'}]}" + }, + { + "op": "remember", + "status": 200, + "body_preview": "{'results': []}" + }, + { + "op": "recall", + "status": 200, + "body_preview": "{'results': [{'id': '7d6f7716-8895-4a86-839f-25514b82599e', 'memory': \"User's wifi network name is FalconNet\", 'hash': '0e881d9cc295e8509eb50d959ed86653', 'metadata': None, 'score': 0.587925614386172, 'created_at': '2026-09-19T08:29:14.005476+00:00', 'updated_at': '2026-09-19T08:29:14.005476+00:00', 'user_id': 'opmem-8d09a77b2f4c4647', 'attributed_to': 'user'}]}" + } + ] +} diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/telemetry-zep-full.json b/docs/benchmarks/artifacts/opmem-external-lanes/telemetry-zep-full.json new file mode 100644 index 00000000..4a32f2a3 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/telemetry-zep-full.json @@ -0,0 +1,247 @@ +{ + "lane": "zep", + "usage": { + "chat_input_tokens": 0, + "chat_output_tokens": 0, + "embed_tokens": 0, + "estimated_usd": 0.0, + "http_requests": 47 + }, + "events": [ + { + "op": "remember", + "status": 200, + "body_preview": "zep thread.add_messages" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[]" + }, + { + "op": "revise", + "status": 200, + "body_preview": "zep thread.add_messages revise" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'zep-msg-1', 'content': 'Prefers Ruby for scripting'}]" + }, + { + "op": "remember", + "status": 200, + "body_preview": "zep thread.add_messages" + }, + { + "op": "revise", + "status": 200, + "body_preview": "zep thread.add_messages revise" + }, + { + "op": "remember", + "status": 200, + "body_preview": "zep thread.add_messages" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'zep-msg-1', 'content': 'Prefers Ruby for scripting'}]" + }, + { + "op": "remember", + "status": 200, + "body_preview": "zep thread.add_messages" + }, + { + "op": "revise", + "status": 200, + "body_preview": "zep thread.add_messages revise" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'zep-msg-1', 'content': 'Prefers Ruby for scripting'}]" + }, + { + "op": "remember", + "status": 200, + "body_preview": "zep thread.add_messages" + }, + { + "op": "remember", + "status": 200, + "body_preview": "zep thread.add_messages" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'zep-msg-0', 'content': 'The wifi network name is FalconNet.'}]" + }, + { + "op": "remember", + "status": 200, + "body_preview": "zep thread.add_messages" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'zep-msg-0', 'content': 'My name is Alice Chen.'}]" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'zep-msg-0', 'content': 'My name is Alice Chen.'}]" + }, + { + "op": "remember", + "status": 200, + "body_preview": "zep thread.add_messages" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'zep-msg-0', 'content': 'The wifi network name is FalconNet.'}]" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'zep-msg-0', 'content': 'The wifi network name is FalconNet.'}]" + }, + { + "op": "remember", + "status": 200, + "body_preview": "zep thread.add_messages" + }, + { + "op": "remember", + "status": 200, + "body_preview": "zep thread.add_messages" + }, + { + "op": "forget", + "status": 200, + "body_preview": "zep forget noop for ['m-ee3e2e287b7c']" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'zep-msg-0', 'content': 'Never share the door code with vendors.'}]" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'zep-msg-0', 'content': 'Never share the door code with vendors.'}]" + }, + { + "op": "remember", + "status": 200, + "body_preview": "zep thread.add_messages" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'zep-msg-0', 'content': 'Never share the door code with vendors.'}]" + }, + { + "op": "forget", + "status": 200, + "body_preview": "zep forget noop for ['m-ec19f9d4b669']" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'zep-msg-0', 'content': 'Never share the door code with vendors.'}]" + }, + { + "op": "remember", + "status": 200, + "body_preview": "zep thread.add_messages" + }, + { + "op": "remember", + "status": 200, + "body_preview": "zep thread.add_messages" + }, + { + "op": "forget", + "status": 200, + "body_preview": "zep forget noop for ['m-14ada541a648']" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'zep-msg-0', 'content': 'Never share the door code with vendors.'}]" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'zep-msg-1', 'content': 'The wifi network name is FalconNet.'}]" + }, + { + "op": "remember", + "status": 200, + "body_preview": "zep thread.add_messages" + }, + { + "op": "forget", + "status": 200, + "body_preview": "zep forget noop for ['m-33684a38ce3c']" + }, + { + "op": "remember", + "status": 200, + "body_preview": "zep thread.add_messages" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'zep-msg-0', 'content': 'Never share the door code with vendors.'}, {'id': 'cda0240f-f63a-4614-a24e-87a105cab156', 'content': 'OpMem Lane advises never sharing the door code with vendors.'}]" + }, + { + "op": "remember", + "status": 200, + "body_preview": "zep thread.add_messages" + }, + { + "op": "remember", + "status": 200, + "body_preview": "zep thread.add_messages" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'zep-msg-0', 'content': 'The launch date is May 12.'}, {'id': 'zep-msg-1', 'content': 'The launch date is June 3.'}, {'id': 'b61a203c-30ff-4aeb-90de-7b7d2da26023', 'content': 'OpMem Lane stated that the launch date is May 12.'}]" + }, + { + "op": "remember", + "status": 200, + "body_preview": "zep thread.add_messages" + }, + { + "op": "remember", + "status": 200, + "body_preview": "zep thread.add_messages" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[]" + }, + { + "op": "remember", + "status": 200, + "body_preview": "zep thread.add_messages" + }, + { + "op": "remember", + "status": 200, + "body_preview": "zep thread.add_messages" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'zep-msg-0', 'content': 'The door code is 1111.'}, {'id': 'zep-msg-1', 'content': 'The door code is 2222.'}, {'id': '838b7748-642b-4b60-95e0-cc09bb174823', 'content': \"OpMem Lane's door code is 1111.\"}]" + } + ] +} diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/telemetry-zep-smoke.json b/docs/benchmarks/artifacts/opmem-external-lanes/telemetry-zep-smoke.json new file mode 100644 index 00000000..38ca5c29 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/telemetry-zep-smoke.json @@ -0,0 +1,27 @@ +{ + "lane": "zep", + "usage": { + "chat_input_tokens": 0, + "chat_output_tokens": 0, + "embed_tokens": 0, + "estimated_usd": 0.0, + "http_requests": 3 + }, + "events": [ + { + "op": "remember", + "status": 200, + "body_preview": "zep thread.add_messages" + }, + { + "op": "remember", + "status": 200, + "body_preview": "zep thread.add_messages" + }, + { + "op": "recall", + "status": 200, + "body_preview": "[{'id': 'zep-msg-0', 'content': 'The wifi network name is FalconNet.'}, {'id': '413e2761-df41-49c5-ba6d-d37aaa89fe70', 'content': \"OpMem Lane's wifi network is named FalconNet.\"}]" + } + ] +} diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/zep-env-check-2026-09-23.json b/docs/benchmarks/artifacts/opmem-external-lanes/zep-env-check-2026-09-23.json new file mode 100644 index 00000000..eaf17b27 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/zep-env-check-2026-09-23.json @@ -0,0 +1,12 @@ +{ + "checked_utc": "2026-09-23T17:55:00Z", + "zep_api_key_in_environment": true, + "zep_api_key_in_injected_secret_names": true, + "free_tier_preflight": "ok", + "zep_sdk": "zep-cloud==3.28.0", + "paid_ceiling_usd": 0.0, + "smoke_run_id": "opmem-phase2-zep-smoke-final", + "full_run_id": "opmem-phase2-zep-full", + "full_score": "3/13", + "infrastructure_errors": 0 +} diff --git a/docs/benchmarks/artifacts/opmem-external-lanes/zep-score-interpretation-opmem-phase2-zep-full.json b/docs/benchmarks/artifacts/opmem-external-lanes/zep-score-interpretation-opmem-phase2-zep-full.json new file mode 100644 index 00000000..528e8de1 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-external-lanes/zep-score-interpretation-opmem-phase2-zep-full.json @@ -0,0 +1,56 @@ +{ + "source_artifact": "live-full-opmem-phase2-zep-full.json", + "run_id": "opmem-phase2-zep-full", + "harness_raw": { + "tasks_run": 13, + "tasks_passed": 3, + "label": "3/13" + }, + "forget_not_supported": { + "reason": "zep-cloud 3.28 thread/graph API: adapter has no delete path for OpMem forget steps (documented noop).", + "tasks": [ + "iso03_forget_isolated", + "sup01_basic_forget", + "sup02_targeted_forget", + "sup03_durable_forget" + ], + "harness_outcomes_not_counted_as_failures": [ + { + "task": "iso03_forget_isolated", + "passed": false, + "failures": ["step3(recall): expected at most 0 results, got 1"] + }, + { + "task": "sup01_basic_forget", + "passed": false, + "failures": ["step3(recall): expected at most 0 results, got 1"] + }, + { + "task": "sup02_targeted_forget", + "passed": false, + "failures": ["step3(recall): a result contains excluded 'door code'"] + }, + { + "task": "sup03_durable_forget", + "passed": false, + "failures": ["step3(recall): expected at most 0 results, got 2"] + } + ] + }, + "scored_supported_tasks": { + "supported_task_count": 9, + "passed": 3, + "label": "3/9", + "passed_tasks": ["cor02_correction_stickiness", "cor03_revised_retrievable", "dup01_idempotent_remember"], + "failed_supported_tasks": [ + "cor01_basic_revision", + "iso01_subject_isolation", + "iso02_tenant_isolation", + "upd01_stale_fact", + "upd02_preference_change", + "upd03_state_supersession" + ] + }, + "publication_label": "3/9*", + "footnote": "Asterisk: four forget fixtures not supported on Zep lane; raw harness score remains 3/13 on opmem-phase2-zep-full." +} diff --git a/docs/benchmarks/artifacts/opmem-manuscript-20260918.md b/docs/benchmarks/artifacts/opmem-manuscript-20260918.md new file mode 100644 index 00000000..eabcb317 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-manuscript-20260918.md @@ -0,0 +1,47 @@ +# OpMem manuscript pin — 2026-09-18 + +**Git SHA:** `20533f2ca83e02265663d5984b49b13de8499231` +**Protocol:** [PUBLICATION_READINESS.md](../../research/opmem/PUBLICATION_READINESS.md) +**PDF:** [main.pdf](../../research/opmem/paper/main.pdf) (`make -C docs/research/opmem/paper pdf`) + +## Primary three-system run (embedded API + Mem0 Platform) + +| System | Overall | +| --- | ---: | +| Brainy (`/memories/search`) | **13/13** | +| Verbatim baseline | **10/13** | +| Mem0 Platform | **10/13** | + +Mem0 Platform: **completed** at negligible API volume (~47 scripted steps). Paid vendor runs elsewhere require a cost estimate and explicit approval first (not a categorical spend ban). + +Machine-readable: `opmem-pin-20260918.json` (copy of `opmem-manuscript-20533f2/opmem-primary.json`). + +## Lane ablations (isolated run) + +| Configuration | Overall | +| --- | ---: | +| `brainy` (search + correct) | **13/13** | +| `brainy-recall` (`POST /recall`) | **9/13** | +| `brainy-supersede` (`/supersede` revise) | **11/13** | + +Artifact: `opmem-manuscript-20533f2/opmem-lane-ablation-isolated.json`. +`brainy-recall` correction tasks hit HTTP 409 on `/correct` in this harness (documented threat). + +## Stability (5× repeat) + +Valid runs only (`infrastructure_errors=0`): **2/5** at **13/13**. +Runs reporting **9/10** with **3 infrastructure errors** are **invalid/incomplete** (errored tasks dropped from denominator—not comparable to 13/13). See `opmem-stability.json` and manuscript Section 6. + +## Reproduce + +```bash +./scripts/run-opmem-manuscript.sh docs/benchmarks/artifacts/opmem-manuscript-$(git rev-parse --short HEAD) +python3 scripts/generate-opmem-paper-tex.py +make -C docs/research/opmem/paper pdf +``` + +## Pending external systems (blocked) + +Mem0 OSS, Zep, Letta, LangMem: **no OpMem adapter, credentials, or line-item cost worksheet** yet. Cost-estimation plan: manuscript Section 6 (`sec:cost-plan`) — 47 steps × mapped vendor billing units per system before approval. + +- v1 ~30 task expansion (separate track) diff --git a/docs/benchmarks/artifacts/opmem-manuscript-20260918.pdf b/docs/benchmarks/artifacts/opmem-manuscript-20260918.pdf new file mode 100644 index 00000000..d8e78fb2 Binary files /dev/null and b/docs/benchmarks/artifacts/opmem-manuscript-20260918.pdf differ diff --git a/docs/benchmarks/artifacts/opmem-manuscript-20533f2/commands.log b/docs/benchmarks/artifacts/opmem-manuscript-20533f2/commands.log new file mode 100644 index 00000000..ca1237b8 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-manuscript-20533f2/commands.log @@ -0,0 +1,5 @@ +git_sha=20533f2ca83e02265663d5984b49b13de8499231 +date_utc=2026-09-18T23:06:39Z +go test ./internal/api/ -run TestOpMemBenchmarkAgainstHTTPServer -count=1 +OPMEM_MANUSCRIPT_OUT=docs/benchmarks/artifacts/opmem-manuscript-20533f2 go test ./internal/api/ -run TestOpMemPaperExperiments -count=1 +mem0 run: python3 evals/run_opmem.py --systems mem0 --json-out docs/benchmarks/artifacts/opmem-manuscript-20533f2/opmem-mem0-rerun.json diff --git a/docs/benchmarks/artifacts/opmem-manuscript-20533f2/go-test-opmem.log b/docs/benchmarks/artifacts/opmem-manuscript-20533f2/go-test-opmem.log new file mode 100644 index 00000000..c96079c6 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-manuscript-20533f2/go-test-opmem.log @@ -0,0 +1 @@ +ok brainy/internal/api 2.552s diff --git a/docs/benchmarks/artifacts/opmem-manuscript-20533f2/go-test-paper.log b/docs/benchmarks/artifacts/opmem-manuscript-20533f2/go-test-paper.log new file mode 100644 index 00000000..ffcb14f3 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-manuscript-20533f2/go-test-paper.log @@ -0,0 +1 @@ +ok brainy/internal/api 261.706s diff --git a/docs/benchmarks/artifacts/opmem-manuscript-20533f2/mem0-rerun.log b/docs/benchmarks/artifacts/opmem-manuscript-20533f2/mem0-rerun.log new file mode 100644 index 00000000..877e0bc9 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-manuscript-20533f2/mem0-rerun.log @@ -0,0 +1,157 @@ +{ + "benchmark": "opmem-v0", + "systems": [ + "mem0" + ], + "summary": { + "mem0": { + "overall": "10/13", + "by_category": { + "correction": "2/3", + "idempotency": "1/1", + "isolation": "3/3", + "staleness": "2/3", + "suppression": "2/3" + } + } + }, + "infrastructure_errors": 0, + "results": [ + { + "task": "cor01_basic_revision", + "category": "correction", + "systems": { + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "cor02_correction_stickiness", + "category": "correction", + "systems": { + "mem0": { + "passed": false, + "failures": [ + "step3(recall): top result does not contain 'ruby'" + ] + } + } + }, + { + "task": "cor03_revised_retrievable", + "category": "correction", + "systems": { + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "dup01_idempotent_remember", + "category": "idempotency", + "systems": { + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "iso01_subject_isolation", + "category": "isolation", + "systems": { + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "iso02_tenant_isolation", + "category": "isolation", + "systems": { + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "iso03_forget_isolated", + "category": "isolation", + "systems": { + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "sup01_basic_forget", + "category": "suppression", + "systems": { + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "sup02_targeted_forget", + "category": "suppression", + "systems": { + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "sup03_durable_forget", + "category": "suppression", + "systems": { + "mem0": { + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 1" + ] + } + } + }, + { + "task": "upd01_stale_fact", + "category": "staleness", + "systems": { + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "upd02_preference_change", + "category": "staleness", + "systems": { + "mem0": { + "passed": false, + "failures": [ + "step2(recall): top result does not contain 'sms'" + ] + } + } + }, + { + "task": "upd03_state_supersession", + "category": "staleness", + "systems": { + "mem0": { + "passed": true, + "failures": [] + } + } + } + ] +} diff --git a/docs/benchmarks/artifacts/opmem-manuscript-20533f2/opmem-lane-ablation-isolated.json b/docs/benchmarks/artifacts/opmem-manuscript-20533f2/opmem-lane-ablation-isolated.json new file mode 100644 index 00000000..a098cbb1 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-manuscript-20533f2/opmem-lane-ablation-isolated.json @@ -0,0 +1,282 @@ +{ + "benchmark": "opmem-v0", + "systems": [ + "brainy", + "brainy-recall", + "brainy-supersede" + ], + "summary": { + "brainy": { + "overall": "13/13", + "by_category": { + "correction": "3/3", + "idempotency": "1/1", + "isolation": "3/3", + "staleness": "3/3", + "suppression": "3/3" + } + }, + "brainy-recall": { + "overall": "9/13", + "by_category": { + "correction": "0/3", + "idempotency": "1/1", + "isolation": "3/3", + "staleness": "3/3", + "suppression": "2/3" + } + }, + "brainy-supersede": { + "overall": "11/13", + "by_category": { + "correction": "2/3", + "idempotency": "1/1", + "isolation": "3/3", + "staleness": "3/3", + "suppression": "2/3" + } + } + }, + "infrastructure_errors": 3, + "results": [ + { + "task": "cor01_basic_revision", + "category": "correction", + "systems": { + "brainy": { + "passed": true, + "failures": [] + }, + "brainy-recall": { + "error": "HTTPError: HTTP Error 409: Conflict" + }, + "brainy-supersede": { + "passed": false, + "failures": [ + "step1(recall): top result does not contain 'python'" + ] + } + } + }, + { + "task": "cor02_correction_stickiness", + "category": "correction", + "systems": { + "brainy": { + "passed": true, + "failures": [] + }, + "brainy-recall": { + "error": "HTTPError: HTTP Error 409: Conflict" + }, + "brainy-supersede": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "cor03_revised_retrievable", + "category": "correction", + "systems": { + "brainy": { + "passed": true, + "failures": [] + }, + "brainy-recall": { + "error": "HTTPError: HTTP Error 409: Conflict" + }, + "brainy-supersede": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "dup01_idempotent_remember", + "category": "idempotency", + "systems": { + "brainy": { + "passed": true, + "failures": [] + }, + "brainy-recall": { + "passed": true, + "failures": [] + }, + "brainy-supersede": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "iso01_subject_isolation", + "category": "isolation", + "systems": { + "brainy": { + "passed": true, + "failures": [] + }, + "brainy-recall": { + "passed": true, + "failures": [] + }, + "brainy-supersede": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "iso02_tenant_isolation", + "category": "isolation", + "systems": { + "brainy": { + "passed": true, + "failures": [] + }, + "brainy-recall": { + "passed": true, + "failures": [] + }, + "brainy-supersede": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "iso03_forget_isolated", + "category": "isolation", + "systems": { + "brainy": { + "passed": true, + "failures": [] + }, + "brainy-recall": { + "passed": true, + "failures": [] + }, + "brainy-supersede": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "sup01_basic_forget", + "category": "suppression", + "systems": { + "brainy": { + "passed": true, + "failures": [] + }, + "brainy-recall": { + "passed": false, + "failures": [ + "step1(recall): expected at least 1 results, got 0", + "step1(recall): top result does not contain 'door code'" + ] + }, + "brainy-supersede": { + "passed": false, + "failures": [ + "step1(recall): expected at least 1 results, got 0", + "step1(recall): top result does not contain 'door code'" + ] + } + } + }, + { + "task": "sup02_targeted_forget", + "category": "suppression", + "systems": { + "brainy": { + "passed": true, + "failures": [] + }, + "brainy-recall": { + "passed": true, + "failures": [] + }, + "brainy-supersede": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "sup03_durable_forget", + "category": "suppression", + "systems": { + "brainy": { + "passed": true, + "failures": [] + }, + "brainy-recall": { + "passed": true, + "failures": [] + }, + "brainy-supersede": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "upd01_stale_fact", + "category": "staleness", + "systems": { + "brainy": { + "passed": true, + "failures": [] + }, + "brainy-recall": { + "passed": true, + "failures": [] + }, + "brainy-supersede": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "upd02_preference_change", + "category": "staleness", + "systems": { + "brainy": { + "passed": true, + "failures": [] + }, + "brainy-recall": { + "passed": true, + "failures": [] + }, + "brainy-supersede": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "upd03_state_supersession", + "category": "staleness", + "systems": { + "brainy": { + "passed": true, + "failures": [] + }, + "brainy-recall": { + "passed": true, + "failures": [] + }, + "brainy-supersede": { + "passed": true, + "failures": [] + } + } + } + ] +} diff --git a/docs/benchmarks/artifacts/opmem-manuscript-20533f2/opmem-lane-ablation.json b/docs/benchmarks/artifacts/opmem-manuscript-20533f2/opmem-lane-ablation.json new file mode 100644 index 00000000..0013bfb8 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-manuscript-20533f2/opmem-lane-ablation.json @@ -0,0 +1,290 @@ +{ + "benchmark": "opmem-v0", + "systems": [ + "brainy", + "brainy-recall", + "brainy-supersede" + ], + "summary": { + "brainy": { + "overall": "9/10", + "by_category": { + "idempotency": "1/1", + "isolation": "3/3", + "staleness": "3/3", + "suppression": "2/3" + } + }, + "brainy-recall": { + "overall": "9/10", + "by_category": { + "idempotency": "1/1", + "isolation": "3/3", + "staleness": "3/3", + "suppression": "2/3" + } + }, + "brainy-supersede": { + "overall": "11/13", + "by_category": { + "correction": "2/3", + "idempotency": "1/1", + "isolation": "3/3", + "staleness": "3/3", + "suppression": "2/3" + } + } + }, + "infrastructure_errors": 6, + "results": [ + { + "task": "cor01_basic_revision", + "category": "correction", + "systems": { + "brainy": { + "error": "HTTPError: HTTP Error 409: Conflict" + }, + "brainy-recall": { + "error": "HTTPError: HTTP Error 409: Conflict" + }, + "brainy-supersede": { + "passed": false, + "failures": [ + "step1(recall): top result does not contain 'python'" + ] + } + } + }, + { + "task": "cor02_correction_stickiness", + "category": "correction", + "systems": { + "brainy": { + "error": "HTTPError: HTTP Error 409: Conflict" + }, + "brainy-recall": { + "error": "HTTPError: HTTP Error 409: Conflict" + }, + "brainy-supersede": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "cor03_revised_retrievable", + "category": "correction", + "systems": { + "brainy": { + "error": "HTTPError: HTTP Error 409: Conflict" + }, + "brainy-recall": { + "error": "HTTPError: HTTP Error 409: Conflict" + }, + "brainy-supersede": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "dup01_idempotent_remember", + "category": "idempotency", + "systems": { + "brainy": { + "passed": true, + "failures": [] + }, + "brainy-recall": { + "passed": true, + "failures": [] + }, + "brainy-supersede": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "iso01_subject_isolation", + "category": "isolation", + "systems": { + "brainy": { + "passed": true, + "failures": [] + }, + "brainy-recall": { + "passed": true, + "failures": [] + }, + "brainy-supersede": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "iso02_tenant_isolation", + "category": "isolation", + "systems": { + "brainy": { + "passed": true, + "failures": [] + }, + "brainy-recall": { + "passed": true, + "failures": [] + }, + "brainy-supersede": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "iso03_forget_isolated", + "category": "isolation", + "systems": { + "brainy": { + "passed": true, + "failures": [] + }, + "brainy-recall": { + "passed": true, + "failures": [] + }, + "brainy-supersede": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "sup01_basic_forget", + "category": "suppression", + "systems": { + "brainy": { + "passed": false, + "failures": [ + "step1(recall): expected at least 1 results, got 0", + "step1(recall): top result does not contain 'door code'" + ] + }, + "brainy-recall": { + "passed": false, + "failures": [ + "step1(recall): expected at least 1 results, got 0", + "step1(recall): top result does not contain 'door code'" + ] + }, + "brainy-supersede": { + "passed": false, + "failures": [ + "step1(recall): expected at least 1 results, got 0", + "step1(recall): top result does not contain 'door code'" + ] + } + } + }, + { + "task": "sup02_targeted_forget", + "category": "suppression", + "systems": { + "brainy": { + "passed": true, + "failures": [] + }, + "brainy-recall": { + "passed": true, + "failures": [] + }, + "brainy-supersede": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "sup03_durable_forget", + "category": "suppression", + "systems": { + "brainy": { + "passed": true, + "failures": [] + }, + "brainy-recall": { + "passed": true, + "failures": [] + }, + "brainy-supersede": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "upd01_stale_fact", + "category": "staleness", + "systems": { + "brainy": { + "passed": true, + "failures": [] + }, + "brainy-recall": { + "passed": true, + "failures": [] + }, + "brainy-supersede": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "upd02_preference_change", + "category": "staleness", + "systems": { + "brainy": { + "passed": true, + "failures": [] + }, + "brainy-recall": { + "passed": true, + "failures": [] + }, + "brainy-supersede": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "upd03_state_supersession", + "category": "staleness", + "systems": { + "brainy": { + "passed": true, + "failures": [] + }, + "brainy-recall": { + "passed": true, + "failures": [] + }, + "brainy-supersede": { + "passed": true, + "failures": [] + } + } + } + ], + "manifest": { + "benchmark": "opmem-v0", + "git_sha": "20533f2ca83e02265663d5984b49b13de8499231", + "run_utc": "2026-09-18T23:12:29Z", + "fixture_dir": "fixtures/opmem", + "task_count": 13, + "base_url": "http://127.0.0.1:38319", + "protocol": "docs/research/opmem/PUBLICATION_READINESS.md", + "scoring": "substring assertions on ranked recall results" + } +} diff --git a/docs/benchmarks/artifacts/opmem-manuscript-20533f2/opmem-mechanism.json b/docs/benchmarks/artifacts/opmem-manuscript-20533f2/opmem-mechanism.json new file mode 100644 index 00000000..1c5bfcbb --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-manuscript-20533f2/opmem-mechanism.json @@ -0,0 +1,37 @@ +{ + "manifest": { + "benchmark": "opmem-v0", + "git_sha": "20533f2ca83e02265663d5984b49b13de8499231", + "run_utc": "2026-09-18T23:12:29Z", + "fixture_dir": "fixtures/opmem", + "task_count": 13, + "base_url": "http://127.0.0.1:38319", + "protocol": "docs/research/opmem/PUBLICATION_READINESS.md", + "scoring": "substring assertions on ranked recall results" + }, + "by_category": { + "verbatim": { + "correction": "2/3", + "idempotency": "0/1", + "isolation": "3/3", + "staleness": "3/3", + "suppression": "2/3" + }, + "brainy": { + "correction": "3/3", + "idempotency": "1/1", + "isolation": "3/3", + "staleness": "3/3", + "suppression": "3/3" + }, + "mem0": { + "correction": "2/3", + "idempotency": "1/1", + "isolation": "3/3", + "staleness": "2/3", + "suppression": "2/3" + }, + "_note": "verbatim is the rank-only baseline (no correct/suppress lifecycle); correction/suppression/staleness categories map to revise/forget semantics." + }, + "verbatim_overall": "10/13" +} diff --git a/docs/benchmarks/artifacts/opmem-manuscript-20533f2/opmem-mem0-rerun.json b/docs/benchmarks/artifacts/opmem-manuscript-20533f2/opmem-mem0-rerun.json new file mode 100644 index 00000000..877e0bc9 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-manuscript-20533f2/opmem-mem0-rerun.json @@ -0,0 +1,157 @@ +{ + "benchmark": "opmem-v0", + "systems": [ + "mem0" + ], + "summary": { + "mem0": { + "overall": "10/13", + "by_category": { + "correction": "2/3", + "idempotency": "1/1", + "isolation": "3/3", + "staleness": "2/3", + "suppression": "2/3" + } + } + }, + "infrastructure_errors": 0, + "results": [ + { + "task": "cor01_basic_revision", + "category": "correction", + "systems": { + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "cor02_correction_stickiness", + "category": "correction", + "systems": { + "mem0": { + "passed": false, + "failures": [ + "step3(recall): top result does not contain 'ruby'" + ] + } + } + }, + { + "task": "cor03_revised_retrievable", + "category": "correction", + "systems": { + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "dup01_idempotent_remember", + "category": "idempotency", + "systems": { + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "iso01_subject_isolation", + "category": "isolation", + "systems": { + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "iso02_tenant_isolation", + "category": "isolation", + "systems": { + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "iso03_forget_isolated", + "category": "isolation", + "systems": { + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "sup01_basic_forget", + "category": "suppression", + "systems": { + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "sup02_targeted_forget", + "category": "suppression", + "systems": { + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "sup03_durable_forget", + "category": "suppression", + "systems": { + "mem0": { + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 1" + ] + } + } + }, + { + "task": "upd01_stale_fact", + "category": "staleness", + "systems": { + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "upd02_preference_change", + "category": "staleness", + "systems": { + "mem0": { + "passed": false, + "failures": [ + "step2(recall): top result does not contain 'sms'" + ] + } + } + }, + { + "task": "upd03_state_supersession", + "category": "staleness", + "systems": { + "mem0": { + "passed": true, + "failures": [] + } + } + } + ] +} diff --git a/docs/benchmarks/artifacts/opmem-manuscript-20533f2/opmem-primary.json b/docs/benchmarks/artifacts/opmem-manuscript-20533f2/opmem-primary.json new file mode 100644 index 00000000..11550470 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-manuscript-20533f2/opmem-primary.json @@ -0,0 +1,300 @@ +{ + "benchmark": "opmem-v0", + "systems": [ + "verbatim", + "brainy", + "mem0" + ], + "summary": { + "verbatim": { + "overall": "10/13", + "by_category": { + "correction": "2/3", + "idempotency": "0/1", + "isolation": "3/3", + "staleness": "3/3", + "suppression": "2/3" + } + }, + "brainy": { + "overall": "13/13", + "by_category": { + "correction": "3/3", + "idempotency": "1/1", + "isolation": "3/3", + "staleness": "3/3", + "suppression": "3/3" + } + }, + "mem0": { + "overall": "10/13", + "by_category": { + "correction": "2/3", + "idempotency": "1/1", + "isolation": "3/3", + "staleness": "2/3", + "suppression": "2/3" + } + } + }, + "infrastructure_errors": 0, + "results": [ + { + "task": "cor01_basic_revision", + "category": "correction", + "systems": { + "verbatim": { + "passed": true, + "failures": [] + }, + "brainy": { + "passed": true, + "failures": [] + }, + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "cor02_correction_stickiness", + "category": "correction", + "systems": { + "verbatim": { + "passed": false, + "failures": [ + "step3(recall): top result does not contain 'ruby'" + ] + }, + "brainy": { + "passed": true, + "failures": [] + }, + "mem0": { + "passed": false, + "failures": [ + "step3(recall): top result does not contain 'ruby'" + ] + } + } + }, + { + "task": "cor03_revised_retrievable", + "category": "correction", + "systems": { + "verbatim": { + "passed": true, + "failures": [] + }, + "brainy": { + "passed": true, + "failures": [] + }, + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "dup01_idempotent_remember", + "category": "idempotency", + "systems": { + "verbatim": { + "passed": false, + "failures": [ + "step2(recall): expected at most 1 results, got 2", + "step2(recall): duplicate result contents" + ] + }, + "brainy": { + "passed": true, + "failures": [] + }, + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "iso01_subject_isolation", + "category": "isolation", + "systems": { + "verbatim": { + "passed": true, + "failures": [] + }, + "brainy": { + "passed": true, + "failures": [] + }, + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "iso02_tenant_isolation", + "category": "isolation", + "systems": { + "verbatim": { + "passed": true, + "failures": [] + }, + "brainy": { + "passed": true, + "failures": [] + }, + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "iso03_forget_isolated", + "category": "isolation", + "systems": { + "verbatim": { + "passed": true, + "failures": [] + }, + "brainy": { + "passed": true, + "failures": [] + }, + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "sup01_basic_forget", + "category": "suppression", + "systems": { + "verbatim": { + "passed": true, + "failures": [] + }, + "brainy": { + "passed": true, + "failures": [] + }, + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "sup02_targeted_forget", + "category": "suppression", + "systems": { + "verbatim": { + "passed": true, + "failures": [] + }, + "brainy": { + "passed": true, + "failures": [] + }, + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "sup03_durable_forget", + "category": "suppression", + "systems": { + "verbatim": { + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 1" + ] + }, + "brainy": { + "passed": true, + "failures": [] + }, + "mem0": { + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 1" + ] + } + } + }, + { + "task": "upd01_stale_fact", + "category": "staleness", + "systems": { + "verbatim": { + "passed": true, + "failures": [] + }, + "brainy": { + "passed": true, + "failures": [] + }, + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "upd02_preference_change", + "category": "staleness", + "systems": { + "verbatim": { + "passed": true, + "failures": [] + }, + "brainy": { + "passed": true, + "failures": [] + }, + "mem0": { + "passed": false, + "failures": [ + "step2(recall): top result does not contain 'sms'" + ] + } + } + }, + { + "task": "upd03_state_supersession", + "category": "staleness", + "systems": { + "verbatim": { + "passed": true, + "failures": [] + }, + "brainy": { + "passed": true, + "failures": [] + }, + "mem0": { + "passed": true, + "failures": [] + } + } + } + ], + "manifest": { + "benchmark": "opmem-v0", + "git_sha": "20533f2ca83e02265663d5984b49b13de8499231", + "run_utc": "2026-09-18T23:12:29Z", + "fixture_dir": "fixtures/opmem", + "task_count": 13, + "base_url": "http://127.0.0.1:38319", + "protocol": "docs/research/opmem/PUBLICATION_READINESS.md", + "scoring": "substring assertions on ranked recall results" + } +} diff --git a/docs/benchmarks/artifacts/opmem-manuscript-20533f2/opmem-stability.json b/docs/benchmarks/artifacts/opmem-manuscript-20533f2/opmem-stability.json new file mode 100644 index 00000000..8e742796 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-manuscript-20533f2/opmem-stability.json @@ -0,0 +1,48 @@ +{ + "repeats": 5, + "system": "brainy", + "recall_lane": "search", + "revise_mode": "correct", + "runs": [ + { + "repeat": 1, + "overall": "13/13", + "infrastructure_errors": 0 + }, + { + "repeat": 2, + "overall": "9/10", + "infrastructure_errors": 3 + }, + { + "repeat": 3, + "overall": "9/10", + "infrastructure_errors": 3 + }, + { + "repeat": 4, + "overall": "13/13", + "infrastructure_errors": 0 + }, + { + "repeat": 5, + "overall": "9/10", + "infrastructure_errors": 3 + } + ], + "stable": false, + "unique_scores": [ + "13/13", + "9/10" + ], + "manifest": { + "benchmark": "opmem-v0", + "git_sha": "20533f2ca83e02265663d5984b49b13de8499231", + "run_utc": "2026-09-18T23:12:29Z", + "fixture_dir": "fixtures/opmem", + "task_count": 13, + "base_url": "http://127.0.0.1:38319", + "protocol": "docs/research/opmem/PUBLICATION_READINESS.md", + "scoring": "substring assertions on ranked recall results" + } +} diff --git a/docs/benchmarks/artifacts/opmem-manuscript-20533f2/run-manifest.json b/docs/benchmarks/artifacts/opmem-manuscript-20533f2/run-manifest.json new file mode 100644 index 00000000..bac4adb2 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-manuscript-20533f2/run-manifest.json @@ -0,0 +1,10 @@ +{ + "benchmark": "opmem-v0", + "git_sha": "20533f2ca83e02265663d5984b49b13de8499231", + "run_utc": "2026-09-18T23:12:29Z", + "fixture_dir": "fixtures/opmem", + "task_count": 13, + "base_url": "http://127.0.0.1:38319", + "protocol": "docs/research/opmem/PUBLICATION_READINESS.md", + "scoring": "substring assertions on ranked recall results" +} diff --git a/docs/benchmarks/artifacts/opmem-manuscript-20533f2/run.log b/docs/benchmarks/artifacts/opmem-manuscript-20533f2/run.log new file mode 100644 index 00000000..e942a6c1 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-manuscript-20533f2/run.log @@ -0,0 +1,5 @@ +git_sha=20533f2ca83e02265663d5984b49b13de8499231 +tasks=13 +primary={"verbatim": {"overall": "10/13", "by_category": {"correction": "2/3", "idempotency": "0/1", "isolation": "3/3", "staleness": "3/3", "suppression": "2/3"}}, "brainy": {"overall": "13/13", "by_category": {"correction": "3/3", "idempotency": "1/1", "isolation": "3/3", "staleness": "3/3", "suppression": "3/3"}}, "mem0": {"overall": "10/13", "by_category": {"correction": "2/3", "idempotency": "1/1", "isolation": "3/3", "staleness": "2/3", "suppression": "2/3"}}} +lanes={"brainy": {"overall": "9/10", "by_category": {"idempotency": "1/1", "isolation": "3/3", "staleness": "3/3", "suppression": "2/3"}}, "brainy-recall": {"overall": "9/10", "by_category": {"idempotency": "1/1", "isolation": "3/3", "staleness": "3/3", "suppression": "2/3"}}, "brainy-supersede": {"overall": "11/13", "by_category": {"correction": "2/3", "idempotency": "1/1", "isolation": "3/3", "staleness": "3/3", "suppression": "2/3"}}} +stability={"repeats": 5, "system": "brainy", "recall_lane": "search", "revise_mode": "correct", "runs": [{"repeat": 1, "overall": "13/13", "infrastructure_errors": 0}, {"repeat": 2, "overall": "9/10", "infrastructure_errors": 3}, {"repeat": 3, "overall": "9/10", "infrastructure_errors": 3}, {"repeat": 4, "overall": "13/13", "infrastructure_errors": 0}, {"repeat": 5, "overall": "9/10", "infrastructure_errors": 3}], "stable": false, "unique_scores": ["13/13", "9/10"], "manifest": {"benchmark": "opmem-v0", "git_sha": "20533f2ca83e02265663d5984b49b13de8499231", "run_utc": "2026-09-18T23:12:29Z", "fixture_dir": "fixtures/opmem", "task_count": 13, "base_url": "http://127.0.0.1:38319", "protocol": "docs/research/opmem/PUBLICATION_READINESS.md", "scoring": "substring assertions on ranked recall results"}} diff --git a/docs/benchmarks/artifacts/opmem-pin-20260918.json b/docs/benchmarks/artifacts/opmem-pin-20260918.json new file mode 100644 index 00000000..11550470 --- /dev/null +++ b/docs/benchmarks/artifacts/opmem-pin-20260918.json @@ -0,0 +1,300 @@ +{ + "benchmark": "opmem-v0", + "systems": [ + "verbatim", + "brainy", + "mem0" + ], + "summary": { + "verbatim": { + "overall": "10/13", + "by_category": { + "correction": "2/3", + "idempotency": "0/1", + "isolation": "3/3", + "staleness": "3/3", + "suppression": "2/3" + } + }, + "brainy": { + "overall": "13/13", + "by_category": { + "correction": "3/3", + "idempotency": "1/1", + "isolation": "3/3", + "staleness": "3/3", + "suppression": "3/3" + } + }, + "mem0": { + "overall": "10/13", + "by_category": { + "correction": "2/3", + "idempotency": "1/1", + "isolation": "3/3", + "staleness": "2/3", + "suppression": "2/3" + } + } + }, + "infrastructure_errors": 0, + "results": [ + { + "task": "cor01_basic_revision", + "category": "correction", + "systems": { + "verbatim": { + "passed": true, + "failures": [] + }, + "brainy": { + "passed": true, + "failures": [] + }, + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "cor02_correction_stickiness", + "category": "correction", + "systems": { + "verbatim": { + "passed": false, + "failures": [ + "step3(recall): top result does not contain 'ruby'" + ] + }, + "brainy": { + "passed": true, + "failures": [] + }, + "mem0": { + "passed": false, + "failures": [ + "step3(recall): top result does not contain 'ruby'" + ] + } + } + }, + { + "task": "cor03_revised_retrievable", + "category": "correction", + "systems": { + "verbatim": { + "passed": true, + "failures": [] + }, + "brainy": { + "passed": true, + "failures": [] + }, + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "dup01_idempotent_remember", + "category": "idempotency", + "systems": { + "verbatim": { + "passed": false, + "failures": [ + "step2(recall): expected at most 1 results, got 2", + "step2(recall): duplicate result contents" + ] + }, + "brainy": { + "passed": true, + "failures": [] + }, + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "iso01_subject_isolation", + "category": "isolation", + "systems": { + "verbatim": { + "passed": true, + "failures": [] + }, + "brainy": { + "passed": true, + "failures": [] + }, + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "iso02_tenant_isolation", + "category": "isolation", + "systems": { + "verbatim": { + "passed": true, + "failures": [] + }, + "brainy": { + "passed": true, + "failures": [] + }, + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "iso03_forget_isolated", + "category": "isolation", + "systems": { + "verbatim": { + "passed": true, + "failures": [] + }, + "brainy": { + "passed": true, + "failures": [] + }, + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "sup01_basic_forget", + "category": "suppression", + "systems": { + "verbatim": { + "passed": true, + "failures": [] + }, + "brainy": { + "passed": true, + "failures": [] + }, + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "sup02_targeted_forget", + "category": "suppression", + "systems": { + "verbatim": { + "passed": true, + "failures": [] + }, + "brainy": { + "passed": true, + "failures": [] + }, + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "sup03_durable_forget", + "category": "suppression", + "systems": { + "verbatim": { + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 1" + ] + }, + "brainy": { + "passed": true, + "failures": [] + }, + "mem0": { + "passed": false, + "failures": [ + "step3(recall): expected at most 0 results, got 1" + ] + } + } + }, + { + "task": "upd01_stale_fact", + "category": "staleness", + "systems": { + "verbatim": { + "passed": true, + "failures": [] + }, + "brainy": { + "passed": true, + "failures": [] + }, + "mem0": { + "passed": true, + "failures": [] + } + } + }, + { + "task": "upd02_preference_change", + "category": "staleness", + "systems": { + "verbatim": { + "passed": true, + "failures": [] + }, + "brainy": { + "passed": true, + "failures": [] + }, + "mem0": { + "passed": false, + "failures": [ + "step2(recall): top result does not contain 'sms'" + ] + } + } + }, + { + "task": "upd03_state_supersession", + "category": "staleness", + "systems": { + "verbatim": { + "passed": true, + "failures": [] + }, + "brainy": { + "passed": true, + "failures": [] + }, + "mem0": { + "passed": true, + "failures": [] + } + } + } + ], + "manifest": { + "benchmark": "opmem-v0", + "git_sha": "20533f2ca83e02265663d5984b49b13de8499231", + "run_utc": "2026-09-18T23:12:29Z", + "fixture_dir": "fixtures/opmem", + "task_count": 13, + "base_url": "http://127.0.0.1:38319", + "protocol": "docs/research/opmem/PUBLICATION_READINESS.md", + "scoring": "substring assertions on ranked recall results" + } +} diff --git a/docs/research/master-plan.md b/docs/research/master-plan.md index 765090e2..b342ace3 100644 --- a/docs/research/master-plan.md +++ b/docs/research/master-plan.md @@ -44,7 +44,7 @@ Staging verify: OpMem 13/13, support 3/3, `/recall` enumerate OK. | Axis | Brainy | Best competitor signal | Comparable? | | --- | --- | --- | --- | -| OpMem (ops correctness, 12 tasks) | **12/12** | Mem0 platform 9/12 (measured, same fixtures) | Yes — our harness, both measured | +| OpMem (ops correctness, 13 tasks) | **13/13** (`1b5ab3e` pin) | Mem0 Platform **10/13** (`2026-08-15`; July **9/12** was 12-task set) | Yes — our harness; Platform not OSS | | Marketing vertical (16 fixtures) | **15/16** empirical | Mem0 4/16 (measured; strict schema = intentional moat) | Yes, with declared policy | | LOCOMO smoke (1 convo × 30 Q, gpt-oss pin) | 19/30, MH 5/10 | Mem0 same-pin 12/30, MH 6/10 | Only vs our own same-pin Mem0 run | | LOCOMO full (10 convos, 1,540 Q) | **not run** (partial 3-convo old-stack: 31/90) | Mem0 platform 92.5; Hindsight 92.0; corrected Zep 75.1; Letta-filesystem 74.0; full-context ~73 | No — we have no comparable number | diff --git a/docs/research/mem0-parity-gaps.md b/docs/research/mem0-parity-gaps.md index d0fe488e..881deddf 100644 --- a/docs/research/mem0-parity-gaps.md +++ b/docs/research/mem0-parity-gaps.md @@ -11,7 +11,7 @@ | Axis | Brainy | Mem0 (known) | Verdict | | --- | --- | --- | --- | | Thin-slice API parity | **4/4** | **4/4** | **Parity** | -| Operational correctness (OpMem) | **12/12** | **9/12** | **Ahead** | +| Operational correctness (OpMem) | **13/13** (`1b5ab3e` pin) | **10/13** Platform (`2026-08-15`) | **Ahead on frozen ops pin** (not LoCoMo) | | Vertical / governed memory | **16/16** marketing | No equivalent | **Ahead (category)** | | Conversational recall (LOCOMO) | **19/30** smoke (gpt-oss, 1×30) | ~92 platform (GPT, top-200, full suite) | **Behind — not comparable pins** | | Latency / tokens | p50 ~0.7–1.3s (staging) | Blog ~0.88s p50 @ top-200 | **Unknown fair pin** | diff --git a/docs/research/opmem-spec.md b/docs/research/opmem-spec.md index c7a21db6..99fe65f9 100644 --- a/docs/research/opmem-spec.md +++ b/docs/research/opmem-spec.md @@ -67,14 +67,14 @@ Recall assertions (all content matching is case-insensitive substring): | `must_exclude` | no result contains any of these substrings | | `unique_contents` | no two results share identical normalized content | -### Task taxonomy (v0: 12 tasks) +### Task taxonomy (v0: 13 tasks) | Category | Tasks | Probes | | --- | --- | --- | | suppression | `sup01` basic forget, `sup02` targeted forget, `sup03` durable forget | forget leaks; collateral damage; resurrection via re-ingestion | | correction | `cor01` basic revision, `cor02` correction stickiness, `cor03` revised retrievable | revision visibility; stale content winning after re-ingestion; retrieval by new terms | | isolation | `iso01` subject, `iso02` tenant, `iso03` forget isolation | cross-actor reads; cross-tenant reads; delete affecting a lookalike memory of another actor | -| staleness | `upd01` stale fact, `upd02` preference change | latest version of a changed fact/preference must outrank the stale one | +| staleness | `upd01` stale fact, `upd02` preference change, `upd03` state supersession | latest version of a changed fact/preference must outrank the stale one; restated state facts prefer newer value | | idempotency | `dup01` idempotent remember | duplicate ingestion polluting recall | Contents are deliberately domain-neutral (door codes, wifi names, launch dates, @@ -130,7 +130,7 @@ CI runs the harness end-to-end via `TestOpMemBenchmarkAgainstHTTPServer` ## Roadmap to the paper -- v0 (this): 12 tasks, 3 adapters (Brainy, verbatim baseline, Mem0), binary scoring. +- v0 (this): 13 tasks (`upd03` added 2026-08), 3 adapters (Brainy, verbatim baseline, Mem0 Platform), binary scoring. - v1: add adapters for Zep, Letta, LangMem; expand to ~30 tasks including lifecycle/expiry (needs a neutral expiry attribute), batched/async ingestion visibility, and concurrent-write races. diff --git a/docs/research/opmem/PUBLICATION_READINESS.md b/docs/research/opmem/PUBLICATION_READINESS.md new file mode 100644 index 00000000..5d1bdf97 --- /dev/null +++ b/docs/research/opmem/PUBLICATION_READINESS.md @@ -0,0 +1,254 @@ +# OpMem publication readiness package + +**Status:** documentation only · **not a pin** · **no benchmark run in this pass** +**Inventory SHA:** `bbe55f7` (`origin/dev` / `origin/main` at package authoring) +**Authoring date:** 2026-09-18 + +This package consolidates repo evidence for Paper 1 ([paper-topics.md](../paper-topics.md): *Beyond recall: lifecycle, suppression, and correction correctness*). It does not claim new measurements. + +--- + +## Ready vs missing (executive) + +| Area | Ready | Missing | +| --- | --- | --- | +| Task fixtures + runner | 13 JSON tasks, `evals/run_opmem.py`, Brainy/verbatim/Platform + **Phase 1** external lane stubs (`evals/opmem_external/`) | v1 ~30 tasks; **live** Zep/Letta/LangMem/Mem0 OSS runs (credentials + worksheet approval) | +| Reproducible protocol | Documented below + [opmem-spec.md](../opmem-spec.md); CI spins embedded API | Pin file (`RunManifest`) for OpMem; SHA `bbe55f7` re-run of Brainy+Mem0 table | +| Brainy 13/13 evidence | Markdown pin + per-task JSON for Brainy-only integrity pass | Full `result.json` with Brainy+verbatim+mem0 on one commit in git | +| Mem0 10/13 evidence | Markdown pin with named failures | Per-task Mem0 JSON in git; Mem0 OSS run; API version drift check | +| Manuscript | LaTeX + PDF at [paper/main.pdf](./paper/main.pdf) | arXiv submission; formal grammar; camera-ready figures | +| Multi-system paper table | Brainy vs Mem0 vs verbatim (partial) | ≥4 systems per paper-topics checklist | +| Threats / limitations | Section below | External review dedicated to OpMem only | + +**Publication verdict:** **not ready** for submission. Closest of the three paper tracks in [inventory-reproduction-plan-2026-09-18.md](../inventory-reproduction-plan-2026-09-18.md) (merge via PR #188), but requires manuscript, expanded systems, and frozen artifact bundle. + +--- + +## Reproducible protocol + +### Scope + +OpMem measures **operational correctness** via deterministic operation scripts over `remember` / `recall` / `revise` / `forget`. Scoring is **binary pass/fail** per task on `GET /memories/search` results (Brainy adapter), not product `POST /recall` and not LLM-judge accuracy. + +### Task set (v0 + `upd03`) + +| Category | Task IDs | Count | +| --- | --- | ---: | +| suppression | `sup01_basic_forget`, `sup02_targeted_forget`, `sup03_durable_forget` | 3 | +| correction | `cor01_basic_revision`, `cor02_correction_stickiness`, `cor03_revised_retrievable` | 3 | +| isolation | `iso01_subject_isolation`, `iso02_tenant_isolation`, `iso03_forget_isolated` | 3 | +| staleness | `upd01_stale_fact`, `upd02_preference_change`, `upd03_state_supersession` | 3 | +| idempotency | `dup01_idempotent_remember` | 1 | +| **Total** | | **13** | + +`upd03_state_supersession.json` added after the July 12-task public draft; it is the sole cardinality change for the 2026-08-15 pin. + +### Systems and mappings + +| Adapter | `remember` | `recall` | `revise` | `forget` | +| --- | --- | --- | --- | --- | +| Brainy | `POST /ingest` | `GET /memories/search` | `POST /memories/{id}/correct` | `POST /memories/{id}/suppress` | +| Verbatim | in-process append | token overlap rank | in-place content replace | delete by id | +| Mem0 Platform | `POST /v1/memories/` | `POST /v2/memories/search/` | `PUT /v1/memories/{id}/` | `DELETE /v1/memories/{id}/` | + +Hermeticity: Brainy and Mem0 adapters prefix tenants per task (`opmem-{nonce}-{task}`) so shared backends do not collide. + +### Assertion semantics + +Case-insensitive **substring** match on ranked search `content` fields. See [opmem-spec.md](../opmem-spec.md) for `min_results`, `max_results`, `top_contains`, `must_exclude`, etc. + +### Runner exit code + +`evals/run_opmem.py` exits **non-zero only on infrastructure errors** (HTTP/adapter exceptions), not on task `fail`. CI `TestOpMemBenchmarkAgainstHTTPServer` therefore proves **harness health**, not that Brainy passes 13/13, unless the test is extended to parse JSON (it does not today). + +### Commands (no external LLM required for Brainy+verbatim) + +```bash +# CI-equivalent (embedded Postgres + in-process API; no Mem0 key) +go test ./internal/api/ -run TestOpMemBenchmarkAgainstHTTPServer -count=1 + +# Local API (start API+worker per CONTRIBUTING.md / AGENTS.md) +unset BRAINY_API_KEYS BRAINY_REQUIRE_API_KEY +export BRAINY_ENV=local +python3 evals/run_opmem.py --systems verbatim,brainy --base-url http://127.0.0.1:8080 \ + --json-out /tmp/opmem-report.json + +# Mem0 Platform counter-run (requires MEM0_API_KEY; costs API calls) +MEM0_API_KEY=... python3 evals/run_opmem.py --systems mem0 \ + --json-out /tmp/opmem-mem0-report.json +``` + +### Pin fields to record on any published table + +| Field | 2026-08-15 freeze value | +| --- | --- | +| Brainy git SHA | `1b5ab3e` (not re-validated on `bbe55f7` in this inventory) | +| Fixture set | 13 files under `fixtures/opmem/` (git tree at SHA) | +| Mem0 surface | **Platform** API via `Mem0OpAdapter` — **not** Mem0 OSS | +| Mem0 run date | 2026-08-15 | +| Brainy env (fresh local pin) | dedicated DB `brainy_bench`, `BRAINY_RECALL_LLM=1`, `BRAINY_USE_RECALL=1`, worker concurrency 8 | +| Scoring endpoint | `/memories/search` (not `/recall`) | + +Artifacts: [opmem-fresh-local-20260815.md](../../benchmarks/artifacts/opmem-fresh-local-20260815.md), [opmem-mem0-fresh-20260815.md](../../benchmarks/artifacts/opmem-mem0-fresh-20260815.md). + +--- + +## Exact claims and evidence table + +| Claim | Allowed wording | Evidence in repo | Gaps / caveats | +| --- | --- | --- | --- | +| Brainy passes all **13** OpMem tasks on search path | “13/13 on OpMem v0+`upd03` at SHA `1b5ab3e`” | [opmem-fresh-local-20260815.md](../../benchmarks/artifacts/opmem-fresh-local-20260815.md); [opmem-integrity-20260819.json](../../benchmarks/artifacts/opmem-integrity-20260819.json) (per-task `passed: true`) | Not re-run on `bbe55f7`; no committed Brainy+Mem0 combined JSON | +| Mem0 Platform **10/13** on same fixtures | “10/13 Platform, 2026-08-15; fails `cor02`, `sup03`, `upd02`” | [opmem-mem0-fresh-20260815.md](../../benchmarks/artifacts/opmem-mem0-fresh-20260815.md) | No per-task Mem0 JSON in git; not OSS; API may drift | +| Brainy leads Mem0 on ops | Same-pin **only** with above pins | Category table in mem0-fresh artifact | Do not imply LoCoMo lead; do not use July **9/12** as current Mem0 | +| Verbatim baseline **10/13** | From Brainy fresh run same day | [opmem-fresh-local-20260815.md](../../benchmarks/artifacts/opmem-fresh-local-20260815.md) | Not Mem0-comparable headline | +| July **12/12 vs 9/12** | Historical staging only | [posts/2026-07-opmem-v0.md](../posts/2026-07-opmem-v0.md), [opmem-staging-vs-mem0.json](../../benchmarks/opmem-staging-vs-mem0.json) | **Stale as current** — 12-task set, pre-`upd03` | +| CI guarantees Brainy 13/13 | **Do not claim** | `TestOpMemBenchmarkAgainstHTTPServer` | Exit code ignores task failures | +| “Beats Mem0” on conversational recall | **Forbidden** without same-pin LoCoMo | README / cycle-closeout | OpMem lead ≠ LoCoMo lead | + +--- + +## Independent evidence review (13/13 vs 10/13) + +Performed from static repo artifacts only (no re-run). + +### Fixture cardinality + +`fixtures/opmem/*.json` → **13** files. Sorted runner order matches integrity JSON task list (13 entries). + +### Brainy 13/13 + +- [opmem-fresh-local-20260815.md](../../benchmarks/artifacts/opmem-fresh-local-20260815.md): aggregate **13/13**, category splits correction/isolation/suppression **3/3**, staleness **3/3**, idempotency **1/1**. +- [opmem-integrity-20260819.json](../../benchmarks/artifacts/opmem-integrity-20260819.json): 13 results, each `brainy.passed: true`, `failures: []`. + +**Not inflated:** we do not extrapolate to `bbe55f7` without a new run; integrity JSON is Brainy-only, not the full three-system counter-run. + +### Mem0 10/13 + +- [opmem-mem0-fresh-20260815.md](../../benchmarks/artifacts/opmem-mem0-fresh-20260815.md): **10/13 (76.9%)**; failures documented as `cor02`, `sup03`, `upd02` with mechanism notes (ruby stickiness, forget leak, sms preference). +- Arithmetic: 13 − 3 failures = 10 passes. Category table in artifact sums to 10 passed tasks. + +**Not inflated:** Platform-only; no reproducible OSS pin; no machine-readable Mem0 per-task log in git to independently re-score without re-running. + +### Verbatim cross-check (same Brainy run doc) + +Brainy fresh doc reports verbatim **10/13** — shows suite discriminates; does not change Mem0 pin. + +### Older 12-task JSON + +[opmem-staging-vs-mem0.json](../../benchmarks/opmem-staging-vs-mem0.json) lists **12** tasks (staleness **2/2**, no `upd03`). Brainy **12/12**, Mem0 **9/12**. This is consistent with the July set; it must not be merged into the 13-task pin. + +--- + +## Missing experiments and ablations + +From [opmem-spec.md](../opmem-spec.md) roadmap and [inventory-reproduction-plan-2026-09-18.md](../inventory-reproduction-plan-2026-09-18.md): + +| Experiment | Why it matters | Repo state | +| --- | --- | --- | +| Mem0 **OSS** same 13 tasks | Fair open comparison | Not run; Platform 10/13 explicit | +| Zep / Letta / LangMem adapters | Paper-topics ≥4 systems | Not implemented | +| v1 task expansion (~30) | expiry, async visibility, races | Not built | +| `revise` vs `POST .../supersede` | ENG-86 supersession API exists | OpMem still maps revise → `/correct` | +| Brainy `/recall` vs `/search` on OpMem | Product path may differ | Harness fixes search lane only | +| Ablation: correction vs suppression vs rank-only | Explain mechanism | No harness | +| Re-run on current `dev` SHA | Current merge bar | **Done** — `20533f2` pin (`opmem-pin-20260918.json`) | +| Commit frozen 3-system JSON (per-task) | Artifact review | **Done** — `docs/benchmarks/artifacts/opmem-pin-20260918.json` | +| Statistical stability | 13 tasks, single run | **Partial** — 5× repeat in `opmem-stability.json`; only infra-error=0 runs count; 9/10 with errors = invalid/incomplete (not a second valid score) | + +--- + +## Threats to validity + +1. **Small n:** 13 binary tasks; wide confidence intervals; one-task swings change percentages (~7.7% per task). +2. **Substring scoring:** Passing does not prove semantic correctness or safe production answers. +3. **Search vs recall:** Pins use `/memories/search`; product agents may use `/recall` with different ranking/abstain behavior. +4. **CI gap:** Harness success does not assert pass rate (see protocol). +5. **Mem0 Platform drift:** Vendor API/version changes can move 10/13 without Brainy changing. +6. **Mem0 ≠ OSS:** Headline competitor number may not reproduce for readers on open-source Mem0. +7. **English, domain-neutral fixtures:** May not represent multilingual or vertical-pack content. +8. **No concurrency / TTL:** v0 scripts are sequential; race and expiry bugs undetected. +9. **Adapter fidelity:** Mapping neutral ops to vendor endpoints may not match recommended vendor patterns. +10. **SHA staleness:** Public README cites 13/13 on `1b5ab3e` cycle; `bbe55f7` merge bar not re-measured for OpMem in inventory. +11. **Authoring bias:** Fixtures authored by Brainy team; independent fixture audit not documented. +12. **Semantic contracts:** `sup03` durable forget and staleness without explicit supersede are **normative choices**; failing systems may reflect different valid policies. + +--- + +## Artifact instructions (reviewers and authors) + +### Minimal reproduction (Brainy + verbatim) + +1. Checkout git SHA to pin (document in report; inventory used `1b5ab3e` for last full pin). +2. `go test ./internal/api/ -run TestOpMemBenchmarkAgainstHTTPServer -count=1` +3. Optional: write JSON via `python3 evals/run_opmem.py --systems verbatim,brainy --base-url --json-out opmem-report.json` + +### Full competitive table (authors) + +1. Run Brainy and Mem0 in separate invocations or one `--systems brainy,verbatim,mem0` with live API + `MEM0_API_KEY`. +2. Commit **one** JSON with `benchmark`, `systems`, `results[]` per task, `summary`, and manifest: git SHA, date, Mem0 surface (Platform/OSS), env vars affecting retrieval. +3. Update [docs/benchmarks/artifacts/](../../benchmarks/artifacts/) markdown summary; do not replace LoCoMo pins. + +### What not to ship + +- July **12/12 vs 9/12** as current (use 13-task pin or label historical). +- Platform Mem0 as “Mem0” without qualifier. +- OpMem lead as LoCoMo or LME lead. + +### Doc consistency check (maintainers) + +```bash +# Spec and ladder should not say "12 tasks" for current v0 +rg -n 'v0: 12 tasks|12 tasks\)' docs/research/opmem-spec.md docs/research/public-bench-ladder.md + +# Fixture count should match spec +find fixtures/opmem -name '*.json' | wc -l +``` + +--- + +## Manuscript outline + +**Working title:** *Beyond recall: operational correctness for agent memory systems* + +1. **Abstract** — Retrieval benchmarks miss forget/correct/isolate/supersede; OpMem binary suite; Brainy 13/13 vs Mem0 Platform 10/13 on frozen 2026-08-15 pin (qualified). +2. **Introduction** — Production failure modes; gap vs LoCoMo/LME/HaluMem. +3. **Related work** — Retrieval QA benchmarks; memory product surveys; Mem0 staleness/update gaps (cite their reports, not as same-pin scores). +4. **Neutral operation model** — remember/recall/revise/forget; actors; adapter contract. +5. **Task suite** — 13 tasks, five categories; semantic contracts (`sup03`, staleness). +6. **Methodology** — Substring assertions; hermetic tenants; Platform vs OSS boundary; search endpoint. +7. **Results** — Table: Brainy, Mem0 Platform, verbatim; per-task failure taxonomy for Mem0. +8. **Discussion** — What failures imply; policy disagreements on durable forget. +9. **Limitations** — n=13, English, no concurrency, CI vs full pin, SHA age. +10. **Artifact appendix** — Fixture URLs, runner commands, JSON schema. +11. **Future work** — v1 tasks, more adapters, supersede mapping, `/recall` lane. + +Target venue: workshop or arXiv benchmark track per [paper-topics.md](../paper-topics.md). + +--- + +## Overlap with draft PRs #185–#187 + +| PR | Topic | OpMem overlap | Consolidation | +| --- | --- | --- | --- | +| [#185](https://github.com/tryvinci/brainy/pull/185) | n=1540 LoCoMo handoff | Mentions keeping **13/13** green; no OpMem code | No merge conflict; publication package is independent | +| [#186](https://github.com/tryvinci/brainy/pull/186) | eval-protocol-v2 staged LoCoMo | OpMem unchanged; LoCoMo qualification only | Do not conflate protocol-v2 with OpMem runner | +| [#187](https://github.com/tryvinci/brainy/pull/187) | RRF retrieval flag | Test plan: OpMem **13/13** with `BRAINY_RETRIEVAL_RRF=0` before merge | Product change could affect search ranking; **re-run OpMem** after merge if RRF path touches shared rank | + +Related: PR [#188](https://github.com/tryvinci/brainy/pull/188) (`inventory-reproduction-plan-2026-09-18.md`) — §1.1 OpMem; this package **implements** Paper 1 deliverables from that inventory without duplicating the full three-paper inventory. + +--- + +## Doc checks (this PR) + +- [x] `fixtures/opmem/` count = **13** +- [x] [opmem-spec.md](../opmem-spec.md) updated to **13 tasks** and `upd03` +- [x] [public-bench-ladder.md](../public-bench-ladder.md) L0 OpMem row → **13/13 vs 10/13** with artifact pointer +- [x] [mem0-parity-gaps.md](../mem0-parity-gaps.md) executive OpMem row updated +- [x] [master-plan.md](../master-plan.md) §1.1 OpMem row updated (13 tasks, current pin) +- [x] [posts/2026-07-opmem-v0.md](../posts/2026-07-opmem-v0.md) supersession banner (July numbers preserved) +- [ ] Historical reports (`opmem-baseline-report.md`, `launch-narrative.md`, staging JSON) — intentionally **dated**; not rewritten +- [x] Re-run OpMem on current SHA — `opmem-manuscript-20260918.md` +- [x] LaTeX manuscript — `docs/research/opmem/paper/` +- [x] Phase 1 external lanes — adapters + dry-run worksheet ([external-lanes-setup.md](./external-lanes-setup.md), `cost-worksheet-dry-run.json`) +- [ ] Mem0 OSS / Zep / Letta / LangMem — **live runs blocked** until credentials + measured estimate under ceiling (see worksheet) +- [x] Mem0 Platform — **completed** on current pin (negligible API volume; `opmem-pin-20260918.json`) diff --git a/docs/research/opmem/README.md b/docs/research/opmem/README.md new file mode 100644 index 00000000..110d9e62 --- /dev/null +++ b/docs/research/opmem/README.md @@ -0,0 +1,16 @@ +# OpMem (operational memory correctness) + +Paper-track benchmark for whether a memory system **behaves correctly** under forget, correct, isolate, and supersede — not whether it recalls facts on LoCoMo. + +| Doc | Purpose | +| --- | --- | +| [opmem-spec.md](../opmem-spec.md) | Normative task schema, operations, scoring | +| [PUBLICATION_READINESS.md](./PUBLICATION_READINESS.md) | **Publication package** (protocol, claims, gaps, threats, artifacts, outline) | +| [paper/main.pdf](./paper/main.pdf) | Formatted manuscript (build: `make -C docs/research/opmem/paper pdf`) | +| [external-lanes-setup.md](./external-lanes-setup.md) | Phase 1 external lane adapters (dry-run / cost worksheet) | +| [posts/2026-07-opmem-v0.md](../posts/2026-07-opmem-v0.md) | July 2026 public draft (**12-task** numbers; superseded for pins) | +| [inventory-reproduction-plan-2026-09-18.md](../inventory-reproduction-plan-2026-09-18.md) | Cross-paper inventory — **PR [#188](https://github.com/tryvinci/brainy/pull/188)** until merged to `dev` | + +**Code:** `fixtures/opmem/` (13 tasks) · `evals/run_opmem.py` · `evals/opmem_adapters.py` · CI `TestOpMemBenchmarkAgainstHTTPServer` + +**Current empirical pin (not re-run on `bbe55f7`):** Brainy **13/13** vs Mem0 Platform **10/13** (2026-08-15, SHA `1b5ab3e`). See [PUBLICATION_READINESS.md](./PUBLICATION_READINESS.md#independent-evidence-review-132-vs-1013). diff --git a/docs/research/opmem/external-lanes-setup.md b/docs/research/opmem/external-lanes-setup.md new file mode 100644 index 00000000..3da51b1c --- /dev/null +++ b/docs/research/opmem/external-lanes-setup.md @@ -0,0 +1,84 @@ +# OpMem external lanes — setup + +Phase 1 ships adapters and dry-run operation counts. **Spend approval** uses the payload budget worksheet (`cost-worksheet-payload-budget.json`), not the legacy mock token meter in `cost-worksheet-dry-run.json`. Phase 2 live runs: `evals/run_opmem_external_live.py`. + +## Spend authority (USD) + +| Lane | Ceiling | Notes | +| --- | ---: | --- | +| `mem0-oss` | $1 | Local Qdrant + OpenAI for extract/embed | +| `letta` | $1 | Self-hosted App Server + OpenAI | +| `langmem` | $1 | Postgres/pgvector + OpenAI | +| `zep` | **$0 paid** | Zep Cloud free tier only; stop if credits exhausted | +| **Total** | **$3** | Sum of OpenAI-metered lanes | + +Rates used in worksheets (OpenAI list, per 1M tokens): `gpt-4.1-mini-2025-04-14` input $0.40 / output $1.60; `text-embedding-3-small` $0.02. + +## Pinned models + +- Chat: `gpt-4.1-mini-2025-04-14` +- Embed: `text-embedding-3-small` + +If a lane cannot accept these pins, record the product constraint in the worksheet and **do not** substitute silently (Zep is documented as Zep-native only). + +## Environment variables (never commit values) + +| Lane | Variables | +| --- | --- | +| **mem0-oss** | `OPENAI_API_KEY` (vault); `QDRANT_URL` (default `http://127.0.0.1:6333`); optional `MEM0_OSS_CONFIG_PATH` | +| **zep** | `ZEP_API_KEY`; optional `ZEP_API_URL` (default `https://api.getzep.com`) | +| **letta** | `OPENAI_API_KEY`; `LETTA_BASE_URL` (default `http://127.0.0.1:8283`); optional `LETTA_APP_SERVER_TOKEN` (required for non-local servers) | +| **langmem** | `OPENAI_API_KEY`; `DATABASE_URL` (Postgres + pgvector); **do not** set `LANGMEM_STORE=InMemoryStore` for measured runs | + +**Not used for mem0-oss lane:** `MEM0_API_KEY` (Platform). + +Deterministic namespaces: set `OPMEM_RUN_ID` (e.g. `opmem-pin-20260919`) for reproducible IDs. + +## Pinned Python packages + +Install once: `pip install -r evals/opmem_external/requirements-external-lanes.txt` + +| Component | Pin | +| --- | --- | +| Letta server | `letta==0.11.7` + `sqlite-vec` | +| Letta client | `letta-client==0.1.307` | +| Zep SDK | `zep-cloud==3.28.0` | + +Start local Letta (no Docker): `scripts/opmem-start-letta-server.sh` + +## Docker images (optional alternative) + +| Service | Image | Port | +| --- | --- | ---: | +| Qdrant (mem0-oss) | `qdrant/qdrant:v1.12.5` | 6333 | +| Letta | prefer pip pin above; or `letta/letta` digest at live time | 8283 | +| Postgres (langmem) | `pgvector/pgvector:pg17` | 5432 | + +## Phase 1 commands (no paid vendor API traffic) + +```bash +# Dry-run all four lanes over 13 fixtures + cost worksheet +python3 evals/run_opmem_cost_dry_run.py \ + --out-dir docs/benchmarks/artifacts/opmem-external-lanes + +# Contract tests (mocked HTTP only) +python3 -m unittest evals/opmem_external/test_lane_contract.py -v +``` + +Live harness: + +```bash +pip install -r evals/opmem_external/requirements-external-lanes.txt +scripts/opmem-start-letta-server.sh # letta lane +export DATABASE_URL=postgresql://opmem:opmem@127.0.0.1:5432/opmem_langmem +export ZEP_API_KEY=... # free tier only +python3 evals/run_opmem_external_live.py --smoke --lanes mem0-oss,zep,letta,langmem +python3 evals/run_opmem_external_live.py --full --lanes mem0-oss,zep,letta,langmem +``` + +## Fairness rules + +- Identical 13 tasks, step order, and substring scoring as `evals/run_opmem.py`. +- Memory mutations only in adapters; no answer-generation calls in OpMem scoring. +- `begin_task` resets lane namespace every task. +- Infrastructure errors remain **invalid/incomplete** runs (not comparable task scores). diff --git a/docs/research/opmem/paper/.gitignore b/docs/research/opmem/paper/.gitignore new file mode 100644 index 00000000..c1ed4d59 --- /dev/null +++ b/docs/research/opmem/paper/.gitignore @@ -0,0 +1,7 @@ +*.aux +*.bbl +*.blg +*.fdb_latexmk +*.fls +*.log +*.out diff --git a/docs/research/opmem/paper/Makefile b/docs/research/opmem/paper/Makefile new file mode 100644 index 00000000..889f3285 --- /dev/null +++ b/docs/research/opmem/paper/Makefile @@ -0,0 +1,14 @@ +# Reproducible PDF build for the OpMem manuscript. +.PHONY: pdf clean check + +MAIN = main +LATEXMK = latexmk + +pdf: + $(LATEXMK) -pdf -interaction=nonstopmode -halt-on-error $(MAIN).tex + +check: pdf + @test -f $(MAIN).pdf + +clean: + $(LATEXMK) -C $(MAIN).tex diff --git a/docs/research/opmem/paper/generated/README.md b/docs/research/opmem/paper/generated/README.md new file mode 100644 index 00000000..8e3bef83 --- /dev/null +++ b/docs/research/opmem/paper/generated/README.md @@ -0,0 +1,5 @@ +# Generated manuscript fragments + +Most files here are produced by `evals/run_opmem_paper_experiments.py` / `scripts/run-opmem-manuscript.sh`. + +`external-lanes.tex` is **hand-maintained** (Phase~2 lane scores and Table~\ref{tab:external-lanes} caption). Caption wording: each lane runs the full 13-task script; Zep publication score is **3/9\*** on nine supported tasks (see `zep-score-interpretation-opmem-phase2-zep-full.json`). Long artifact filenames use `\artifact{...}` from `main.tex` (`xurl` + `\sloppy` block) so paths break inside the margin. diff --git a/docs/research/opmem/paper/generated/combined-results.tex b/docs/research/opmem/paper/generated/combined-results.tex new file mode 100644 index 00000000..fa4d8a02 --- /dev/null +++ b/docs/research/opmem/paper/generated/combined-results.tex @@ -0,0 +1,4 @@ +\evidence{completed}{ +Committed \texttt{opmem-primary.json} lists per-task outcomes for +verbatim, brainy, mem0 with manifest fields (SHA, date, fixture count). +} diff --git a/docs/research/opmem/paper/generated/current-sha-results.tex b/docs/research/opmem/paper/generated/current-sha-results.tex new file mode 100644 index 00000000..7557ade5 --- /dev/null +++ b/docs/research/opmem/paper/generated/current-sha-results.tex @@ -0,0 +1,5 @@ +\evidence{completed}{ +At git SHA \texttt{20533f2ca83e} (UTC 2026-09-18T23:12:29Z), embedded API harness: +Brainy (search lane) \textbf{13/13}, verbatim \textbf{10/13}. +Mem0 Platform (same run): \textbf{10/13} (API surface: Platform via \texttt{Mem0OpAdapter}). +} diff --git a/docs/research/opmem/paper/generated/external-lanes.tex b/docs/research/opmem/paper/generated/external-lanes.tex new file mode 100644 index 00000000..9c51e905 --- /dev/null +++ b/docs/research/opmem/paper/generated/external-lanes.tex @@ -0,0 +1,26 @@ +\begin{table}[ht] +\centering +\small +\begin{tabular}{@{}lcc@{}} +\toprule +Lane & OpMem score & Run \\ +\midrule +Brainy (primary pin) & 13/13 & \texttt{opmem-pin-20260918} \\ +Mem0 Platform & 10/13 & same pin \\ +Mem0 OSS & 9/13 & \texttt{opmem-phase2-full-mem0} \\ +LangMem & 8/13 & \texttt{opmem-phase2-full1} \\ +Letta & 5/13 & \texttt{opmem-phase2-letta-full} \\ +Zep & 3/9$^{*}$ & \texttt{opmem-phase2-zep-full} \\ +\bottomrule +\end{tabular} +\caption{Phase~2 external lanes: each lane ran the full 13-task OpMem script; Zep is scored on \textbf{9 supported tasks}$^{*}$. $^{*}$Four forget fixtures (\texttt{iso03\_forget\_isolated}, \texttt{sup01--sup03}) are \emph{not supported} on the Zep lane; raw harness tally \textbf{3/13} unchanged on \artifact{live-full-opmem-phase2-zep-full.json}. Forget is not supported (\texttt{zep-cloud} 3.28 thread/graph API has no delete path the adapter can use).} +\label{tab:external-lanes} +\end{table} + +{\sloppy +\evidence{completed}{Mem0 \textbf{OSS} (local Qdrant path, \texttt{mem0ai} 2.1.0): \textbf{9/13} on run \texttt{opmem-phase2-full-mem0} (0 infra errors). Artifact: \artifact{live-full-opmem-phase2-full-mem0.json}.} +\evidence{completed}{LangMem (Postgres 16 + \texttt{pgvector} 0.6): \textbf{8/13} on run \texttt{opmem-phase2-full1} (0 infra errors). Artifact: \artifact{live-full-opmem-phase2-full1.json}.} +\evidence{completed}{Letta (\texttt{letta==0.11.7}): \textbf{5/13} on run \texttt{opmem-phase2-letta-full} (0 infra errors). Artifact: \artifact{live-full-opmem-phase2-letta-full.json}.} +\evidence{completed}{Zep (\$0 paid, \texttt{zep-cloud} 3.28.0): \textbf{3/9$^{*}$} on supported tasks from run \texttt{opmem-phase2-zep-full} (0 infra errors). \emph{Not supported} (not scored as failures): \artifact{iso03\_forget\_isolated}, \artifact{sup01\_basic\_forget}, \artifact{sup02\_targeted\_forget}, \artifact{sup03\_durable\_forget}. Raw harness: \textbf{3/13}.\newline Artifact: \artifact{live-full-opmem-phase2-zep-full.json}; interpretation: \artifact{zep-score-interpretation-opmem-phase2-zep-full.json}.} +\evidence{completed}{Payload budget worksheet: worst-case OpenAI \textbf{\$0.15} vs \$3 ceiling (\artifact{cost-worksheet-payload-budget.json}).} +\par} diff --git a/docs/research/opmem/paper/generated/lane-ablation.tex b/docs/research/opmem/paper/generated/lane-ablation.tex new file mode 100644 index 00000000..917e309e --- /dev/null +++ b/docs/research/opmem/paper/generated/lane-ablation.tex @@ -0,0 +1,17 @@ +\begin{table}[h] +\centering +\caption{Brainy recall lane and revise mapping (SHA 20533f2ca83e).} +\begin{tabular}{lc} +\toprule +Configuration & Overall \\ +\midrule +brainy & 13/13 \\ +brainy-recall & 9/13 \\ +brainy-supersede & 11/13 \\ +\bottomrule +\end{tabular} +\end{table} +Search (\texttt{/memories/search}) is the normative pin lane. +\texttt{brainy-recall} uses \texttt{POST /recall} (context mode). +Correction tasks on the recall lane may return HTTP 409 during \texttt{/correct} +(infrastructure errors in artifact JSON). diff --git a/docs/research/opmem/paper/generated/mechanism-ablation.tex b/docs/research/opmem/paper/generated/mechanism-ablation.tex new file mode 100644 index 00000000..a373ff5a --- /dev/null +++ b/docs/research/opmem/paper/generated/mechanism-ablation.tex @@ -0,0 +1,16 @@ +Category pass rates separate correction/suppression/staleness contracts; +verbatim approximates rank-only recall without governed lifecycle ops. +\begin{table}[h] +\centering +\begin{tabular}{lcc} +\toprule +Category & Brainy & Verbatim (rank-only) \\ +\midrule +correction & 3/3 & 2/3 \\ +idempotency & 1/1 & 0/1 \\ +isolation & 3/3 & 3/3 \\ +staleness & 3/3 & 3/3 \\ +suppression & 3/3 & 2/3 \\ +\bottomrule +\end{tabular} +\end{table} diff --git a/docs/research/opmem/paper/generated/stability.tex b/docs/research/opmem/paper/generated/stability.tex new file mode 100644 index 00000000..b87808dc --- /dev/null +++ b/docs/research/opmem/paper/generated/stability.tex @@ -0,0 +1,13 @@ +\evidence{completed}{ +Five consecutive Brainy search-lane repeats on one embedded server session. +\textbf{Valid runs only} (infra errors $=0$): 2/5; scores: 13/13. +} +\begin{itemize} +\item Repeat 1: 13/13 (infra errors=0) +\item Repeat 2: 9/10 (infra errors=3) +\item Repeat 3: 9/10 (infra errors=3) +\item Repeat 4: 13/13 (infra errors=0) +\item Repeat 5: 9/10 (infra errors=3) +\end{itemize} +Repeats with infra errors $>0$ (3 of 5) are \textbf{invalid/incomplete}: the harness omits errored tasks from the denominator, so aggregates such as 9/10 are \emph{not} comparable to a clean 13/13 pin. +Use isolated \texttt{TestOpMemBenchmarkAgainstHTTPServer} for merge-gate stability. diff --git a/docs/research/opmem/paper/generated/supersede-ablation.tex b/docs/research/opmem/paper/generated/supersede-ablation.tex new file mode 100644 index 00000000..b9898a39 --- /dev/null +++ b/docs/research/opmem/paper/generated/supersede-ablation.tex @@ -0,0 +1,5 @@ +\evidence{completed}{ +Mapping \texttt{revise} $\rightarrow$ \texttt{POST .../supersede} yields +\textbf{11/13} vs \textbf{13/13} with \texttt{/correct} on the search lane +(same SHA). Failures are task-level assertion misses, not missing runs. +} diff --git a/docs/research/opmem/paper/main.pdf b/docs/research/opmem/paper/main.pdf new file mode 100644 index 00000000..4b6da2ad Binary files /dev/null and b/docs/research/opmem/paper/main.pdf differ diff --git a/docs/research/opmem/paper/main.tex b/docs/research/opmem/paper/main.tex new file mode 100644 index 00000000..dae0c7c0 --- /dev/null +++ b/docs/research/opmem/paper/main.tex @@ -0,0 +1,214 @@ +\documentclass[11pt]{article} +\usepackage[margin=1in]{geometry} +\usepackage{booktabs} +\usepackage{hyperref} +\usepackage{xurl} +\urlstyle{tt} +\newcommand{\artifact}[1]{\url{#1}} +\usepackage{longtable} +\usepackage{xcolor} +\usepackage{listings} +\usepackage{graphicx} + +\newcommand{\todo}[1]{\textcolor{red}{\textbf{TODO:} #1}} +\newcommand{\evidence}[2]{\textbf{[#1]} #2} + +\title{Beyond Recall: Operational Correctness for Agent Memory Systems\\ +\large OpMem v0 Benchmark and Empirical Pins} +\author{Brainy Research} +\date{\today} + +\begin{document} +\maketitle + +\begin{abstract} +Retrieval benchmarks such as LoCoMo and LongMemEval measure whether an agent recalls +the right fact, but not whether a memory \emph{system} honors forget, correct, isolate, +and supersede semantics in production. +We present \textbf{OpMem v0}, a 13-task, pass/fail suite over neutral +\texttt{remember}/\texttt{recall}/\texttt{revise}/\texttt{forget} scripts with +substring-based assertions on ranked results. +On the frozen 2026-08-15 pin (git SHA \texttt{1b5ab3e}), Brainy passes \textbf{13/13} +via \texttt{GET /memories/search} while Mem0 \textbf{Platform} passes \textbf{10/13} +(failures: \texttt{cor02}, \texttt{sup03}, \texttt{upd02}). +This manuscript reports repository evidence, a reproducible protocol, threats to validity, +and new embedded-harness re-measurements on the current merge-pin git revision where completed. +We do \emph{not} claim conversational-recall superiority over Mem0; OpMem operational +leadership is orthogonal to LoCoMo J-scores. +\end{abstract} + +\section{Introduction} +Production agents fail when suppressed memories leak after re-ingestion, corrections do not +stick, tenants bleed across isolation boundaries, or stale facts outrank updates. +Conversational QA benchmarks~\cite{maharana2024locomo,longmemeval2024} do not encode these +lifecycle contracts; hallucination-oriented memory benchmarks such as HaluMem~\cite{chen2025halumem} +focus on factual faithfulness rather than governed delete/correct semantics. + +OpMem targets \textbf{operational correctness}: deterministic scripts, binary scoring, and +explicit adapter mappings to each system's native API surface~\cite{brainyopmem2026}. +Section~\ref{sec:limitations} lists threats; Section~\ref{sec:results} reports primary, +historical, and Phase~2 external-lane pins (Table~\ref{tab:external-lanes}). + +\section{Related work} +\paragraph{Retrieval QA.} +LoCoMo~\cite{maharana2024locomo} and LongMemEval~\cite{longmemeval2024} score end-to-end +answer quality over long conversations; they are not substitutes for lifecycle tests. + +\paragraph{Memory products.} +Mem0 documents accumulation-oriented designs and names staleness as an open problem~\cite{mem0state2026}. +We report Mem0 \textbf{Platform} numbers only on OpMem fixtures; we do not import Mem0 blog +LoCoMo percentages as same-pin scores. + +\paragraph{External memory lanes (OpMem v0).} +Mem0 \textbf{Platform} on OpMem v0 is \textbf{completed} at negligible API volume (13 tasks, 47 scripted steps; Section~\ref{sec:external}). +Phase~2 counter-runs (after cost worksheet approval) are \textbf{completed} for Mem0 OSS, LangMem, Letta, and Zep; scores in Table~\ref{tab:external-lanes}: \textbf{9/13}, \textbf{8/13}, \textbf{5/13}, and \textbf{3/9$^{*}$} respectively (Zep raw harness \textbf{3/13}; \$0 paid). +Paid or vendor-hosted lanes require a written estimate before approval; that gate was not a categorical denial of spend. + +\section{Neutral operation model} +\label{sec:model} +Actors are \texttt{(tenant, subject)} pairs. +Tasks are operation scripts; \texttt{revise} and \texttt{forget} reference prior +\texttt{remember} step ids. +Adapters namespace tenants per task (\texttt{opmem-\{nonce\}-\{task\}}) for hermetic runs. + +\begin{table}[h] +\centering +\caption{Adapter mappings (normative; from \texttt{PUBLICATION\_READINESS.md}).} +\begin{tabular}{llll} +\toprule +Op & Brainy & Verbatim baseline & Mem0 Platform \\ +\midrule +remember & \texttt{POST /ingest} & append & \texttt{POST /v1/memories/} \\ +recall & \texttt{GET /memories/search} & token overlap & \texttt{POST /v2/memories/search/} \\ +revise & \texttt{POST .../correct} & in-place replace & \texttt{PUT /v1/memories/\{id\}/} \\ +forget & \texttt{POST .../suppress} & delete & \texttt{DELETE /v1/memories/\{id\}/} \\ +\bottomrule +\end{tabular} +\end{table} + +\section{Task suite} +OpMem v0 comprises 13 JSON fixtures under \texttt{fixtures/opmem/}: three suppression, +three correction, three isolation, three staleness (including \texttt{upd03\_state\_supersession}), +and one idempotency task. +Assertions use case-insensitive substring checks on ranked \texttt{content} fields +(\texttt{min\_results}, \texttt{top\_contains}, \texttt{must\_exclude}, etc.). + +\section{Methodology} +\subsection{Scoring lane} +Primary pins use Brainy \texttt{/memories/search}, not product \texttt{POST /recall} +(Section~\ref{sec:lane} reports a \texttt{/recall} lane ablation). +The harness (\texttt{evals/run\_opmem.py}) exits non-zero only on infrastructure errors, +not task failures---CI proves runner health, not 13/13, unless JSON is parsed. + +\subsection{Reproduction commands} +\begin{lstlisting}[basicstyle=\ttfamily\small] +go test ./internal/api/ -run TestOpMemBenchmarkAgainstHTTPServer -count=1 +python3 evals/run_opmem.py --systems verbatim,brainy \ + --base-url http://127.0.0.1:8080 --json-out opmem-report.json +\end{lstlisting} +Paper experiments: \texttt{scripts/run-opmem-manuscript.sh} and +\texttt{evals/run\_opmem\_paper\_experiments.py}. + +\section{Results} +\label{sec:results} + +\subsection{Frozen 2026-08-15 pin (historical, SHA \texttt{1b5ab3e})} +\evidence{completed}{ +Brainy \textbf{13/13}; verbatim \textbf{10/13}; Mem0 Platform \textbf{10/13} +(artifacts \texttt{opmem-fresh-local-20260815.md}, \texttt{opmem-mem0-fresh-20260815.md}). +} + +\begin{table}[h] +\centering +\caption{Category breakdown on 2026-08-15 Mem0 Platform counter-run.} +\begin{tabular}{lrr} +\toprule +Category & Mem0 Platform & Brainy (same cycle) \\ +\midrule +correction & 2/3 & 3/3 \\ +isolation & 3/3 & 3/3 \\ +suppression & 2/3 & 3/3 \\ +staleness & 2/3 & 3/3 \\ +idempotency & 1/1 & 1/1 \\ +\midrule +\textbf{overall} & \textbf{10/13} & \textbf{13/13} \\ +\bottomrule +\end{tabular} +\end{table} + +Mem0 failures on that pin: \texttt{cor02} (correction stickiness), +\texttt{sup03} (durable forget leak), \texttt{upd02} (preference ranking). + +\subsection{Current-SHA re-measurement} +\label{sec:current} +\input{generated/current-sha-results.tex} + +\subsection{Combined machine-readable artifact} +\input{generated/combined-results.tex} + +\subsection{Recall vs search lane} +\label{sec:lane} +\input{generated/lane-ablation.tex} + +\subsection{Correct vs supersede mapping} +\input{generated/supersede-ablation.tex} + +\subsection{Mechanism breakdown} +\input{generated/mechanism-ablation.tex} + +\subsection{Statistical stability} +\input{generated/stability.tex} + +\subsection{External systems} +\label{sec:external} +\evidence{completed}{Mem0 \textbf{Platform}: 10/13 on SHA \texttt{20533f2} (\texttt{opmem-pin-20260918.json}); same headline as 2026-08-15 pin. Completed under negligible Platform API use (full suite $\approx$47 scripted steps).} +\input{generated/external-lanes.tex} + +\subsection{Cost-estimation plan (before any new paid run)} +\label{sec:cost-plan} +OpMem v0 has 13 tasks and 47 scripted steps (\texttt{fixtures/opmem/}). Before any new paid lane run, prepare billing and approval: +\begin{itemize} +\item \textbf{Phase 1--2} --- \texttt{evals/opmem\_external/}: deterministic namespaces, per-task reset, payload budget (\texttt{cost-worksheet-payload-budget.json}), live harness \texttt{run\_opmem\_external\_live.py}. Setup: \texttt{external-lanes-setup.md}. +\item \textbf{Mem0 OSS} --- Local Qdrant + \texttt{OPENAI\_API\_KEY}; ceiling \$1; dry-run projected OpenAI spend in worksheet. +\item \textbf{Zep} --- Zep Cloud free tier only (\$0 paid); cannot pin OpenAI models (product constraint recorded). +\item \textbf{Letta} --- Self-hosted App Server; ceiling \$1; no legacy Docker API or Letta Cloud paid plan. +\item \textbf{LangMem} --- Postgres/pgvector; ceiling \$1; no \texttt{InMemoryStore} on measured runs. +\end{itemize} +No dollar totals are asserted here until rate cards and deployment mode are pinned; approval requires the completed worksheet, not a categorical spend ban. + +\section{Discussion} +OpMem failures on Mem0 align with accumulation-oriented update semantics: +corrections and durable forgets are not first-class lifecycle events. +Verbatim baseline passes 10/13, showing the suite discriminates rank-only stores from +governed suppression/correction. +Policy disagreements remain (\texttt{sup03} durable forget is a normative contract). + +\section{Limitations} +\label{sec:limitations} +\begin{enumerate} +\item Small $n=13$; each task is $\sim$7.7\% of the aggregate score. +\item Substring scoring does not guarantee semantic safety. +\item Search vs \texttt{/recall} product path may diverge (lane ablation quantifies on Brainy). +\item English, domain-neutral fixtures; no concurrency or TTL tests. +\item Mem0 Platform API drift; Phase~2 OSS/LangMem/Letta/Zep pins are single-pass, small $n$. Zep forget fixtures are \emph{not supported} on the lane (Table~\ref{tab:external-lanes} footnote); publication line \textbf{3/9$^{*}$} vs raw harness \textbf{3/13}. +\item Fixture authorship bias; no independent external audit documented. +\end{enumerate} + +\section{Artifact appendix} +Committed run manifests and JSON under +\texttt{docs/benchmarks/artifacts/opmem-manuscript-*}: +\texttt{run-manifest.json}, \texttt{opmem-primary.json}, +\texttt{opmem-lane-ablation.json}, \texttt{opmem-stability.json}, +\texttt{opmem-mechanism.json}, and \texttt{commands.log}. + +Build PDF: \texttt{make -C docs/research/opmem/paper pdf}. + +\section{Future work} +v1 task expansion ($\sim$30 tasks), Zep forget/delete mapping if a supported API path appears, +supersede-native revise mapping as default where lineage is required, +and fail-closed CI assertion of 13/13 on merge pins. + +\bibliographystyle{plain} +\bibliography{references} + +\end{document} diff --git a/docs/research/opmem/paper/references.bib b/docs/research/opmem/paper/references.bib new file mode 100644 index 00000000..8e50e0b5 --- /dev/null +++ b/docs/research/opmem/paper/references.bib @@ -0,0 +1,38 @@ +@article{maharana2024locomo, + title = {{LoCoMo}: Evaluating Very Long-Term Conversational Memory of LLM Agents}, + author = {Maharana, Adyasha and others}, + journal = {arXiv preprint arXiv:2402.17710}, + year = {2024}, + url = {https://arxiv.org/abs/2402.17710} +} + +@article{longmemeval2024, + title = {{LongMemEval}: Benchmarking Long-Term Memory in LLM Chat Assistants}, + author = {Wu, Howard Yen and others}, + journal = {arXiv preprint arXiv:2410.10819}, + year = {2024}, + url = {https://arxiv.org/abs/2410.10819} +} + +@article{chen2025halumem, + title = {{HaluMem}: Evaluating Hallucinations in Memory Systems of Agents}, + author = {Chen, Ding and Niu, Simin and Li, Kehang and Liu, Peng and Zheng, Xiangping and Tang, Bo and Li, Xinchi and Xiong, Feiyu and Li, Zhiyu}, + journal = {arXiv preprint arXiv:2511.03506}, + year = {2025}, + url = {https://arxiv.org/abs/2511.03506} +} + +@misc{mem0state2026, + title = {State of {AI} Agent Memory (2026)}, + author = {{Mem0}}, + howpublished = {\url{https://mem0.ai/blog/state-of-ai-agent-memory-2026}}, + year = {2026}, + note = {Vendor context on staleness and cross-session evolution; not an OpMem same-pin score.} +} + +@misc{brainyopmem2026, + title = {{OpMem} v0 specification and harness}, + author = {{Brainy}}, + howpublished = {\url{https://github.com/tryvinci/brainy} (\texttt{docs/research/opmem-spec.md})}, + year = {2026} +} diff --git a/docs/research/posts/2026-07-opmem-v0.md b/docs/research/posts/2026-07-opmem-v0.md index 1840a174..685e5da6 100644 --- a/docs/research/posts/2026-07-opmem-v0.md +++ b/docs/research/posts/2026-07-opmem-v0.md @@ -1,5 +1,7 @@ # OpMem v0: operational memory correctness beyond LOCOMO recall +> **Superseded for pins:** This draft reports the **12-task** staging set (pre-`upd03`). Current empirical pin: Brainy **13/13** vs Mem0 Platform **10/13** on 13 fixtures ([opmem-fresh-local-20260815](../../benchmarks/artifacts/opmem-fresh-local-20260815.md)). See [opmem/PUBLICATION_READINESS.md](../opmem/PUBLICATION_READINESS.md). + **Status:** Public research draft (Paper 1 track) **Date:** 2026-07-29 **Systems:** Brainy (staging) · Mem0 Platform · Verbatim baseline diff --git a/docs/research/public-bench-ladder.md b/docs/research/public-bench-ladder.md index b8a38686..7efeab6c 100644 --- a/docs/research/public-bench-ladder.md +++ b/docs/research/public-bench-ladder.md @@ -49,7 +49,7 @@ L6 Vertical “MarketingMem” public track (Brainy’s differentiation) | Metric | Brainy | Mem0 | Artifact | | --- | ---: | ---: | --- | | Parity | 4/4 | 4/4 | [staging report](../benchmarks/staging-competitive-report.md) | -| OpMem | 12/12 | 9/12 | same | +| OpMem | **13/13** (`1b5ab3e` pin) | **10/13** Platform (`2026-08-15`) | [opmem-fresh-local-20260815](../benchmarks/artifacts/opmem-fresh-local-20260815.md) · July staging **12/12 vs 9/12** is pre-`upd03` | | Marketing vertical | 16/16 | N/A | [moat](../benchmarks/marketing-moat-report.md) | --- diff --git a/evals/opmem_adapters.py b/evals/opmem_adapters.py index b7d76d78..055552ad 100644 --- a/evals/opmem_adapters.py +++ b/evals/opmem_adapters.py @@ -62,9 +62,23 @@ class BrainyAdapter: name = "brainy" - def __init__(self, base_url: str) -> None: + def __init__( + self, + base_url: str, + *, + recall_lane: str = "search", + revise_mode: str = "correct", + ) -> None: self.base_url = base_url.rstrip("/") self._run_nonce = str(int(time.time())) + self._recall_lane = recall_lane + self._revise_mode = revise_mode + if recall_lane == "recall": + self.name = "brainy-recall" + elif revise_mode == "supersede": + self.name = "brainy-supersede" + else: + self.name = "brainy" def available(self) -> bool: return True @@ -88,6 +102,24 @@ def remember(self, actor: tuple[str, str], content: str) -> list[str]: def recall(self, actor: tuple[str, str], query: str) -> list[dict]: tenant, subject = self._scope(actor) + if self._recall_lane == "recall": + body = post_json(self.base_url, "/recall", { + "tenant_id": tenant, + "subject_id": subject, + "q": query, + "mode": "context", + "top_k": 20, + }) + results = [ + {"id": r["memory_id"], "content": r["content"]} + for r in body.get("memories", []) + if isinstance(r, dict) and r.get("content") + ] + if not results: + block = (body.get("context_block") or body.get("answer") or "").strip() + if block: + results = [{"id": "recall-context", "content": block}] + return results body = get_json(self.base_url, "/memories/search", { "tenant_id": tenant, "subject_id": subject, @@ -104,10 +136,18 @@ def forget(self, actor: tuple[str, str], memory_ids: list[str]) -> None: def revise(self, actor: tuple[str, str], memory_ids: list[str], content: str) -> None: tenant, subject = self._scope(actor) query = urllib.parse.urlencode({"tenant_id": tenant, "subject_id": subject}) + payload = {"content": content, "source_text": content} + if self._revise_mode == "supersede": + post_json( + self.base_url, + f"/memories/{memory_ids[0]}/supersede?{query}", + payload, + ) + return post_json( self.base_url, f"/memories/{memory_ids[0]}/correct?{query}", - {"content": content, "source_text": content}, + payload, ) diff --git a/evals/opmem_external/__init__.py b/evals/opmem_external/__init__.py new file mode 100644 index 00000000..ba602d08 --- /dev/null +++ b/evals/opmem_external/__init__.py @@ -0,0 +1,21 @@ +"""OpMem external memory lane adapters (Phase 1: dry-run + instrumentation).""" + +from opmem_external.langmem_store import LangMemLaneAdapter +from opmem_external.letta_appserver import LettaAppServerLaneAdapter +from opmem_external.mem0_oss import Mem0OSSLaneAdapter +from opmem_external.zep_cloud import ZepCloudLaneAdapter + +LANE_ADAPTERS = { + "mem0-oss": Mem0OSSLaneAdapter, + "zep": ZepCloudLaneAdapter, + "letta": LettaAppServerLaneAdapter, + "langmem": LangMemLaneAdapter, +} + +__all__ = [ + "LANE_ADAPTERS", + "LangMemLaneAdapter", + "LettaAppServerLaneAdapter", + "Mem0OSSLaneAdapter", + "ZepCloudLaneAdapter", +] diff --git a/evals/opmem_external/cost_budget.py b/evals/opmem_external/cost_budget.py new file mode 100644 index 00000000..1e6e133a --- /dev/null +++ b/evals/opmem_external/cost_budget.py @@ -0,0 +1,302 @@ +"""Conservative pre-call token budget from serialized request payloads (fixtures × lane).""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterable + +import tiktoken + +from opmem_external.pins import PINNED_CHAT_MODEL, PINNED_EMBED_MODEL +from opmem_external.rates import ( + RATE_CHAT_INPUT_PER_M, + RATE_CHAT_OUTPUT_PER_M, + RATE_EMBED_PER_M, + RATE_SOURCE_NOTE, + RATE_SOURCE_URL, +) + +_ENC = tiktoken.get_encoding("cl100k_base") + +# Conservative max completion tokens per call type (upper bound, not typical). +_MAX_OUTPUT_BY_KIND = { + "mem0_extract": 800, + "mem0_update": 400, + "letta_memory_step": 300, + "langmem_manage": 400, +} + +_RETRY_MARGIN = 1.10 # settling / transient retries + + +def _count(text: str) -> int: + return len(_ENC.encode(text or "")) + + +def _count_json(obj: Any) -> int: + return _count(json.dumps(obj, ensure_ascii=False)) + + +@dataclass +class BudgetLine: + lane: str + op: str + task: str + kind: str + input_tokens: int + output_tokens: int + embed_tokens: int + + def chat_usd(self) -> float: + return ( + self.input_tokens * RATE_CHAT_INPUT_PER_M / 1_000_000 + + self.output_tokens * RATE_CHAT_OUTPUT_PER_M / 1_000_000 + ) + + def embed_usd(self) -> float: + return self.embed_tokens * RATE_EMBED_PER_M / 1_000_000 + + def total_usd(self) -> float: + return self.chat_usd() + self.embed_usd() + + +@dataclass +class LaneBudget: + lane: str + lines: list[BudgetLine] = field(default_factory=list) + product_constraints: list[str] = field(default_factory=list) + + def worst_case_usd(self) -> float: + base = sum(line.total_usd() for line in self.lines) + if self.lane == "zep": + return 0.0 + return base * _RETRY_MARGIN + + def token_totals(self) -> dict[str, int]: + return { + "chat_input_tokens": sum(l.input_tokens for l in self.lines), + "chat_output_tokens": sum(l.output_tokens for l in self.lines), + "embed_tokens": sum(l.embed_tokens for l in self.lines), + } + + def to_dict(self) -> dict: + return { + "lane": self.lane, + "worst_case_usd": round(self.worst_case_usd(), 6), + "token_totals": self.token_totals(), + "line_count": len(self.lines), + "product_constraints": self.product_constraints, + "retry_margin": _RETRY_MARGIN, + } + + +def _mem0_system_prompt() -> str: + from mem0.configs.prompts import ADDITIVE_EXTRACTION_PROMPT + + return ADDITIVE_EXTRACTION_PROMPT + + +def _mem0_extract_payload(content: str) -> list[dict]: + system = _mem0_system_prompt() + user = json.dumps( + { + "new_messages": [{"role": "user", "content": content}], + "summary": "", + "recently_extracted_memories": [], + "existing_memories": [], + "last_k_messages": [], + "observation_date": "2026-09-19", + "current_date": "2026-09-19", + }, + ensure_ascii=False, + ) + return [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ] + + +def _letta_wrapped(content: str) -> dict: + return { + "model": PINNED_CHAT_MODEL, + "messages": [ + { + "role": "system", + "content": "Extract durable user memory facts; memory-only lane (no answer generation).", + }, + {"role": "user", "content": content}, + ], + "max_tokens": _MAX_OUTPUT_BY_KIND["letta_memory_step"], + } + + +def _langmem_manage_payload(content: str) -> dict: + return { + "model": PINNED_CHAT_MODEL, + "instructions": "Manage memory store entries for user facts (OpMem lane; no final answer).", + "input": content, + } + + +def _zep_add_payload(content: str) -> dict: + return { + "messages": [ + { + "role_type": "user", + "role": "opmem-user", + "content": content, + } + ], + "return_context": False, + } + + +def build_lane_budgets(tasks: Iterable[dict]) -> dict[str, LaneBudget]: + budgets = { + "mem0-oss": LaneBudget("mem0-oss"), + "letta": LaneBudget("letta"), + "langmem": LaneBudget("langmem"), + "zep": LaneBudget("zep"), + } + budgets["zep"].product_constraints.append( + "Zep Cloud native extraction; OpenAI pins not applied; $0 paid ceiling." + ) + + for task in tasks: + name = task["name"] + for step in task["steps"]: + op = step["op"] + content = step.get("content", "") + query = step.get("query", "") + + if op == "remember": + msgs = _mem0_extract_payload(content) + inp = sum(_count(m["content"]) for m in msgs) + budgets["mem0-oss"].lines.append( + BudgetLine( + "mem0-oss", op, name, "mem0_extract", inp, + _MAX_OUTPUT_BY_KIND["mem0_extract"], + _count(content), + ) + ) + letta_p = _letta_wrapped(content) + budgets["letta"].lines.append( + BudgetLine( + "letta", op, name, "letta_memory", _count_json(letta_p), + _MAX_OUTPUT_BY_KIND["letta_memory_step"], + _count(content), + ) + ) + lm = _langmem_manage_payload(content) + budgets["langmem"].lines.append( + BudgetLine( + "langmem", op, name, "langmem_manage", _count_json(lm), + _MAX_OUTPUT_BY_KIND["langmem_manage"], + _count(content), + ) + ) + budgets["zep"].lines.append( + BudgetLine("zep", op, name, "zep_add", _count_json(_zep_add_payload(content)), 0, 0) + ) + elif op == "recall": + q = query or "" + for lane in ("mem0-oss", "letta", "langmem"): + budgets[lane].lines.append( + BudgetLine(lane, op, name, "embed_query", 0, 0, _count(q)) + ) + # mem0 may re-embed on search path conservatively + budgets["mem0-oss"].lines.append( + BudgetLine("mem0-oss", op, name, "embed_query_dup", 0, 0, _count(q)) + ) + budgets["zep"].lines.append( + BudgetLine("zep", op, name, "zep_get", _count_json({"query": q}), 0, 0) + ) + elif op == "revise": + msgs = _mem0_extract_payload(content) + inp = sum(_count(m["content"]) for m in msgs) + budgets["mem0-oss"].lines.append( + BudgetLine( + "mem0-oss", op, name, "mem0_update", inp, + _MAX_OUTPUT_BY_KIND["mem0_update"], + _count(content), + ) + ) + budgets["letta"].lines.append( + BudgetLine( + "letta", op, name, "letta_update", _count_json(_letta_wrapped(content)), + _MAX_OUTPUT_BY_KIND["letta_memory_step"], + _count(content), + ) + ) + budgets["langmem"].lines.append( + BudgetLine( + "langmem", op, name, "langmem_update", _count_json(_langmem_manage_payload(content)), + _MAX_OUTPUT_BY_KIND["langmem_manage"], + _count(content), + ) + ) + budgets["zep"].lines.append( + BudgetLine("zep", op, name, "zep_update", _count_json({"text": content}), 0, 0) + ) + elif op == "forget": + for lane in ("mem0-oss", "letta", "langmem"): + budgets[lane].lines.append( + BudgetLine(lane, op, name, "delete_api", 50, 0, 0) + ) + budgets["zep"].lines.append( + BudgetLine("zep", op, name, "zep_delete", 50, 0, 0) + ) + + return budgets + + +def build_worksheet_from_budgets( + budgets: dict[str, LaneBudget], + *, + task_count: int, + step_count: int, +) -> dict: + from opmem_external.pins import SPEND_CEILING_USD + + lanes = [] + openai_total = 0.0 + for lane, budget in sorted(budgets.items()): + usd = budget.worst_case_usd() + openai_total += usd + lanes.append( + { + **budget.to_dict(), + "projected_worst_case_usd": round(usd, 6), + "ceiling_usd": SPEND_CEILING_USD.get(lane, 0.0), + "within_ceiling": usd <= SPEND_CEILING_USD.get(lane, 0.0), + } + ) + lanes.append( + { + "lane": "_total_openai_worst_case", + "projected_worst_case_usd": round(openai_total, 6), + "ceiling_usd": SPEND_CEILING_USD["total"], + "within_ceiling": openai_total <= SPEND_CEILING_USD["total"], + } + ) + return { + "mode": "payload_budget", + "task_count": task_count, + "step_count": step_count, + "pinned_chat_model": PINNED_CHAT_MODEL, + "pinned_embed_model": PINNED_EMBED_MODEL, + "rate_source_url": RATE_SOURCE_URL, + "rate_source_note": RATE_SOURCE_NOTE, + "ceilings_usd": SPEND_CEILING_USD, + "lanes": lanes, + "methodology": ( + "tiktoken cl100k_base on serialized request bodies incl. mem0 ADDITIVE_EXTRACTION_PROMPT; " + "conservative max output caps; 1.10 retry margin; Zep excluded from OpenAI totals." + ), + } + + +def load_tasks(fixture_dir: Path) -> list[dict]: + return [json.loads(p.read_text(encoding="utf-8")) for p in sorted(fixture_dir.glob("*.json"))] diff --git a/evals/opmem_external/langmem_store.py b/evals/opmem_external/langmem_store.py new file mode 100644 index 00000000..6773e0b8 --- /dev/null +++ b/evals/opmem_external/langmem_store.py @@ -0,0 +1,163 @@ +"""LangMem lane: local SDK + Postgres/pgvector (no InMemoryStore for measured runs).""" + +from __future__ import annotations + +import json +import os +import uuid +from typing import Any + +from langchain_core.messages import HumanMessage + +from opmem_external.pins import PINNED_CHAT_MODEL, PINNED_EMBED_MODEL +from opmem_external.protocol import OpMemLaneAdapter +from opmem_external.telemetry import LaneTelemetry + + +def _stringify_memory_value(val: Any) -> str: + if val is None: + return "" + if isinstance(val, str): + return val + if isinstance(val, dict): + for key in ("content", "text", "memory"): + inner = val.get(key) + if isinstance(inner, str): + return inner + if inner is not None: + return _stringify_memory_value(inner) + return json.dumps(val, ensure_ascii=False) + content = getattr(val, "content", None) + if isinstance(content, str): + return content + return str(val) + + +class LangMemLaneAdapter(OpMemLaneAdapter): + name = "langmem" + write_settle_seconds = 0.5 + + def __init__(self, *, dry_run: bool = True, run_id: str | None = None) -> None: + super().__init__(dry_run=dry_run, run_id=run_id) + self._database_url = os.environ.get( + "DATABASE_URL", + "postgresql://opmem:opmem@127.0.0.1:5432/opmem_langmem", + ) + if not dry_run and "InMemoryStore" in os.environ.get("LANGMEM_STORE", ""): + self._record_model_constraint("InMemoryStore is forbidden for measured OpMem runs") + self._store_ctx: Any = None + self._store: Any = None + self._manager: Any = None + self.telemetry = LaneTelemetry(self.name) + self._assert_pinned_models(chat_ok=True, embed_ok=True) + + def lane_id(self) -> str: + return "langmem_postgres_pgvector" + + def _user_id(self, actor: tuple[str, str]) -> str: + return self.namespace(actor) + + def _ensure_store(self) -> Any: + if self._store is not None: + return self._store + from langgraph.store.postgres import PostgresStore + + self._store_ctx = PostgresStore.from_conn_string( + self._database_url, + index={ + "dims": 1536, + "embed": f"openai:{PINNED_EMBED_MODEL}", + }, + ) + self._store = self._store_ctx.__enter__() + self._store.setup() + return self._store + + def _ensure_manager(self) -> Any: + if self._manager is not None: + return self._manager + from langmem import create_memory_store_manager + + store = self._ensure_store() + self._manager = create_memory_store_manager( + f"openai:{PINNED_CHAT_MODEL}", + namespace=("opmem", "{langgraph_user_id}"), + store=store, + enable_deletes=True, + instructions=( + "Extract durable user memory facts only. Do not answer questions; " + "memory operations for OpMem benchmark lane." + ), + ) + return self._manager + + def _config(self, actor: tuple[str, str]) -> dict: + return {"configurable": {"langgraph_user_id": self._user_id(actor)}} + + def _reset_remote(self) -> None: + self._manager = None + if self._store is not None: + user_prefix = self._user_id(("t1", "u1"))[:8] + try: + import psycopg + + with psycopg.connect(self._database_url) as conn: + with conn.cursor() as cur: + cur.execute( + "DELETE FROM store WHERE prefix LIKE %s", + (f"opmem.%",), + ) + conn.commit() + except Exception as exc: + self.telemetry.log_http("reset", 500, f"reset partial: {exc}") + self._memory_table.clear() + + def _remember_live(self, actor: tuple[str, str], content: str, mem_id: str) -> list[str]: + manager = self._ensure_manager() + cfg = self._config(actor) + manager.invoke({"messages": [HumanMessage(content=content)]}, cfg) + self.telemetry.log_http("remember", 200, "langmem manager.invoke") + return [mem_id] + + def _recall_live(self, actor: tuple[str, str], query: str) -> list[dict]: + store = self._ensure_store() + user = self._user_id(actor) + namespace = ("opmem", user) + hits = store.search(namespace, query=query, limit=10) + self.telemetry.log_http("recall", 200, str(hits)[:500]) + out: list[dict] = [] + for item in hits: + val = getattr(item, "value", item) + text = _stringify_memory_value(val) + key = getattr(item, "key", None) or str(uuid.uuid4()) + out.append({"id": key, "content": text}) + return out + + def _forget_live(self, actor: tuple[str, str], memory_ids: list[str]) -> None: + store = self._ensure_store() + namespace = ("opmem", self._user_id(actor)) + for mid in memory_ids: + try: + store.delete(namespace, mid) + except Exception: + pass + self.telemetry.log_http("forget", 200, f"deleted {memory_ids}") + + def _revise_live(self, actor: tuple[str, str], memory_ids: list[str], content: str) -> None: + manager = self._ensure_manager() + cfg = self._config(actor) + manager.invoke( + { + "messages": [ + HumanMessage( + content=f"Update memory to reflect: {content}" + ) + ] + }, + cfg, + ) + self.telemetry.log_http("revise", 200, "langmem revise invoke") + if self.write_settle_seconds: + import time + + time.sleep(self.write_settle_seconds) diff --git a/evals/opmem_external/letta_appserver.py b/evals/opmem_external/letta_appserver.py new file mode 100644 index 00000000..e7d71def --- /dev/null +++ b/evals/opmem_external/letta_appserver.py @@ -0,0 +1,137 @@ +"""Letta self-hosted App Server lane (pip server 0.11.x + archival block mapping).""" + +from __future__ import annotations + +import os +import re +from typing import Any + +from opmem_external.pins import PINNED_CHAT_MODEL, PINNED_EMBED_MODEL +from opmem_external.protocol import OpMemLaneAdapter, _overlap +from opmem_external.telemetry import LaneTelemetry + +_BLOCK_LABEL = "opmem_facts" +_LINE_RE = re.compile(r"^\[(?P[^\]]+)\]\s*(?P.*)$") + + +class LettaAppServerLaneAdapter(OpMemLaneAdapter): + name = "letta" + write_settle_seconds = 1.0 + + def __init__(self, *, dry_run: bool = True, run_id: str | None = None) -> None: + super().__init__(dry_run=dry_run, run_id=run_id) + self._base_url = os.environ.get("LETTA_BASE_URL", "http://127.0.0.1:8283").rstrip("/") + self._client: Any = None + self._agent_id: str | None = None + self.telemetry = LaneTelemetry(self.name) + self._assert_pinned_models(chat_ok=True, embed_ok=True) + self._record_model_constraint( + "Letta lane maps remember/revise/forget to agents.blocks.modify on " + f"'{_BLOCK_LABEL}' (line-per-memory). Recall uses lexical overlap, not agent chat." + ) + if not dry_run and not os.environ.get("LETTA_APP_SERVER_TOKEN", "").strip(): + # Local `letta server --no-secure` (pinned 0.11.7) does not require a token. + if not self._local_insecure_server(): + self._missing_credentials.append("LETTA_APP_SERVER_TOKEN") + + def lane_id(self) -> str: + return "letta_app_server_self_hosted" + + def _local_insecure_server(self) -> bool: + host = self._base_url + return host.startswith("http://127.0.0.1") or host.startswith("http://localhost") + + def _client_instance(self) -> Any: + if self._client is not None: + return self._client + from letta_client import Letta + + token = os.environ.get("LETTA_APP_SERVER_TOKEN", "").strip() + kwargs: dict[str, Any] = {"base_url": self._base_url} + if token: + kwargs["api_key"] = token + self._client = Letta(**kwargs) + return self._client + + def _model(self) -> str: + return f"openai/{PINNED_CHAT_MODEL}" + + def _embedding(self) -> str: + return f"openai/{PINNED_EMBED_MODEL}" + + def _reset_remote(self) -> None: + client = self._client_instance() + if self._agent_id: + try: + client.agents.delete(agent_id=self._agent_id) + except Exception: + pass + self._agent_id = None + ns = self.namespace(("t1", "u1")) + agent = client.agents.create( + name=f"opmem-{ns}", + memory_blocks=[ + {"label": _BLOCK_LABEL, "value": ""}, + {"label": "persona", "value": "OpMem external lane; memory block only (no chat answers)."}, + ], + model=self._model(), + embedding=self._embedding(), + ) + self._agent_id = agent.id + self.telemetry.log_http("reset", 200, f"agent {self._agent_id}") + + def _read_lines(self) -> dict[str, str]: + client = self._client_instance() + if not self._agent_id: + self._reset_remote() + block = client.agents.blocks.retrieve(agent_id=self._agent_id, block_label=_BLOCK_LABEL) + text = block.value or "" + out: dict[str, str] = {} + for line in text.splitlines(): + line = line.strip() + if not line: + continue + m = _LINE_RE.match(line) + if m: + out[m.group("id")] = m.group("body") + return out + + def _write_lines(self, lines: dict[str, str]) -> None: + client = self._client_instance() + if not self._agent_id: + self._reset_remote() + body = "\n".join(f"[{mid}] {txt}" for mid, txt in sorted(lines.items())) + client.agents.blocks.modify(agent_id=self._agent_id, block_label=_BLOCK_LABEL, value=body) + self.telemetry.log_http("blocks.modify", 200, f"{len(lines)} facts") + + def _remember_live(self, actor: tuple[str, str], content: str, mem_id: str) -> list[str]: + lines = self._read_lines() + lines[mem_id] = content + self._write_lines(lines) + return [mem_id] + + def _recall_live(self, actor: tuple[str, str], query: str) -> list[dict]: + lines = self._read_lines() + scored = sorted(lines.items(), key=lambda item: -_overlap(query, item[1])) + out = [{"id": mid, "content": txt} for mid, txt in scored if _overlap(query, txt) > 0] + self.telemetry.log_http("recall", 200, str(out)[:500]) + return out + + def _forget_live(self, actor: tuple[str, str], memory_ids: list[str]) -> None: + lines = self._read_lines() + for mid in memory_ids: + lines.pop(mid, None) + self._write_lines(lines) + self.telemetry.log_http("forget", 200, f"removed {memory_ids}") + + def _revise_live(self, actor: tuple[str, str], memory_ids: list[str], content: str) -> None: + if not memory_ids: + return + lines = self._read_lines() + lines[memory_ids[0]] = content + self._write_lines(lines) + self.telemetry.log_http("revise", 200, f"updated {memory_ids[0]}") + if self.write_settle_seconds: + import time + + time.sleep(self.write_settle_seconds) diff --git a/evals/opmem_external/mem0_oss.py b/evals/opmem_external/mem0_oss.py new file mode 100644 index 00000000..75ed4b52 --- /dev/null +++ b/evals/opmem_external/mem0_oss.py @@ -0,0 +1,126 @@ +"""Mem0 OSS lane: Python package + local Qdrant (no Platform API key).""" + +from __future__ import annotations + +import os +import shutil +from pathlib import Path +from typing import Any + +from opmem_external.pins import PINNED_CHAT_MODEL, PINNED_EMBED_MODEL +from opmem_external.protocol import OpMemLaneAdapter +from opmem_external.telemetry import LaneTelemetry + + +class Mem0OSSLaneAdapter(OpMemLaneAdapter): + name = "mem0-oss" + write_settle_seconds = 2.0 + + def __init__(self, *, dry_run: bool = True, run_id: str | None = None) -> None: + super().__init__(dry_run=dry_run, run_id=run_id) + self._qdrant_root = Path(os.environ.get("OPMEM_QDRANT_ROOT", "/tmp/opmem-qdrant")) + self._memory: Any = None + self._task_store_path: Path | None = None + self.telemetry = LaneTelemetry(self.name) + self._assert_pinned_models(chat_ok=True, embed_ok=True) + + def lane_id(self) -> str: + return "mem0_oss_local_qdrant" + + def _config(self, store_path: Path, collection: str) -> dict: + return { + "vector_store": { + "provider": "qdrant", + "config": { + "path": str(store_path), + "collection_name": collection, + "embedding_model_dims": 1536, + "on_disk": True, + }, + }, + "llm": { + "provider": "openai", + "config": {"model": PINNED_CHAT_MODEL, "temperature": 0}, + }, + "embedder": { + "provider": "openai", + "config": {"model": PINNED_EMBED_MODEL}, + }, + } + + def _load_memory(self) -> Any: + if self._memory is not None: + return self._memory + from mem0 import Memory + + store_path = self._task_store_path or (self._qdrant_root / "default") + store_path.mkdir(parents=True, exist_ok=True) + collection = f"opmem_{self._task_name or 'task'}" + self._memory = Memory.from_config(self._config(store_path, collection)) + return self._memory + + def _reset_remote(self) -> None: + self._memory = None + path = self._qdrant_root / self._run_id / self._task_name + if path.exists(): + shutil.rmtree(path, ignore_errors=True) + self._task_store_path = path + self._task_store_path.mkdir(parents=True, exist_ok=True) + + def _user_id(self, actor: tuple[str, str]) -> str: + return self.namespace(actor) + + def _remember_live(self, actor: tuple[str, str], content: str, mem_id: str) -> list[str]: + mem = self._load_memory() + user_id = self._user_id(actor) + result = mem.add( + [{"role": "user", "content": content}], + user_id=user_id, + infer=True, + ) + self.telemetry.log_http("remember", 200, str(result)[:500]) + ids = [] + for item in (result or {}).get("results", []) if isinstance(result, dict) else []: + if isinstance(item, dict) and item.get("id"): + ids.append(item["id"]) + if not ids and isinstance(result, dict): + for item in result.get("results", []): + mid = item.get("memory_id") or item.get("id") + if mid: + ids.append(mid) + return ids[-1:] if ids else [mem_id] + + def _recall_live(self, actor: tuple[str, str], query: str) -> list[dict]: + mem = self._load_memory() + user_id = self._user_id(actor) + raw = mem.search(query, filters={"user_id": user_id}, top_k=10) + self.telemetry.log_http("recall", 200, str(raw)[:500]) + out = [] + for item in raw.get("results", []) if isinstance(raw, dict) else raw or []: + if not isinstance(item, dict): + continue + raw = item.get("memory") or item.get("text") or item.get("content") or "" + if isinstance(raw, dict): + text = raw.get("data") or raw.get("text") or str(raw) + else: + text = str(raw) + mid = item.get("id") or item.get("memory_id") or "" + out.append({"id": mid, "content": text}) + return out + + def _forget_live(self, actor: tuple[str, str], memory_ids: list[str]) -> None: + mem = self._load_memory() + for mid in memory_ids: + mem.delete(mid) + self.telemetry.log_http("forget", 200, f"deleted {mid}") + + def _revise_live(self, actor: tuple[str, str], memory_ids: list[str], content: str) -> None: + mem = self._load_memory() + if not memory_ids: + return + mem.update(memory_ids[0], data=content) + self.telemetry.log_http("revise", 200, f"updated {memory_ids[0]}") + if self.write_settle_seconds: + import time + + time.sleep(self.write_settle_seconds) diff --git a/evals/opmem_external/pins.py b/evals/opmem_external/pins.py new file mode 100644 index 00000000..d715ff31 --- /dev/null +++ b/evals/opmem_external/pins.py @@ -0,0 +1,28 @@ +"""Pinned models and spend ceilings for OpMem external lane comparisons.""" + +from __future__ import annotations + +PINNED_CHAT_MODEL = "gpt-4.1-mini-2025-04-14" +PINNED_EMBED_MODEL = "text-embedding-3-small" + +# Official list prices (USD per 1M tokens) for worksheet generation — 2025-04 OpenAI public rates. +RATE_CHAT_INPUT_PER_M = 0.40 +RATE_CHAT_OUTPUT_PER_M = 1.60 +RATE_EMBED_PER_M = 0.02 + +# Owner-approved spend authority (USD); live runs must stay at or below projected totals. +SPEND_CEILING_USD: dict[str, float] = { + "mem0-oss": 1.0, + "letta": 1.0, + "langmem": 1.0, + "zep": 0.0, + "total": 3.0, +} + +LANE_ENV_REQUIRED: dict[str, tuple[str, ...]] = { + "mem0-oss": ("OPENAI_API_KEY",), # local Qdrant; no MEM0_API_KEY + "zep": ("ZEP_API_KEY",), + # LETTA_APP_SERVER_TOKEN optional for local `letta server --no-secure` (see external-lanes-setup.md). + "letta": ("OPENAI_API_KEY",), + "langmem": ("OPENAI_API_KEY", "DATABASE_URL"), +} diff --git a/evals/opmem_external/protocol.py b/evals/opmem_external/protocol.py new file mode 100644 index 00000000..20de49ca --- /dev/null +++ b/evals/opmem_external/protocol.py @@ -0,0 +1,220 @@ +"""Shared OpMem external-lane adapter protocol.""" + +from __future__ import annotations + +import hashlib +import json +import os +import time +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from opmem_external.pins import LANE_ENV_REQUIRED, PINNED_CHAT_MODEL, PINNED_EMBED_MODEL +from opmem_external.token_estimate import LaneDryRunStats, OperationCounts, TokenLedger + + +@dataclass +class AdapterResponseLog: + lane: str + task: str + op: str + status: int + body_preview: str + redacted: bool = True + + +class OpMemLaneAdapter(ABC): + """Maps neutral remember/recall/revise/forget onto one external memory lane.""" + + name: str + supports_pinned_models: bool = True + write_settle_seconds: float = 0.0 + + def __init__(self, *, dry_run: bool = True, run_id: str | None = None) -> None: + self.dry_run = dry_run + self._run_id = run_id or os.environ.get("OPMEM_RUN_ID", "opmem-pin-0001") + self._task_name = "" + self._stats = LaneDryRunStats(lane=self.name) + self._memory_table: dict[str, dict[str, str]] = {} + self._response_logs: list[AdapterResponseLog] = [] + self._missing_credentials = self._check_credentials() + self._stats.missing_credentials = list(self._missing_credentials) + self._stats.credentials_present = not self._missing_credentials + + @abstractmethod + def lane_id(self) -> str: + ... + + def _check_credentials(self) -> list[str]: + missing = [] + for key in LANE_ENV_REQUIRED.get(self.name, ()): + if not os.environ.get(key, "").strip(): + missing.append(key) + return missing + + def available(self) -> bool: + if self.dry_run: + return True + return not self._missing_credentials + + def begin_task(self, task_name: str) -> None: + self._task_name = task_name + self.reset_namespace() + + def namespace(self, actor: tuple[str, str]) -> str: + tenant, subject = actor + raw = f"{self._run_id}|{self.name}|{self._task_name}|{tenant}|{subject}" + digest = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:16] + return f"opmem-{digest}" + + def reset_namespace(self) -> None: + self._stats.ops.reset += 1 + keys = [k for k in self._memory_table if k.startswith(f"{self._task_name}:")] + for k in keys: + del self._memory_table[k] + if not self.dry_run: + self._reset_remote() + + @abstractmethod + def _reset_remote(self) -> None: + ... + + def remember(self, actor: tuple[str, str], content: str) -> list[str]: + self._stats.ops.remember += 1 + ns = self._table_key(actor) + mem_id = self._deterministic_id("remember", content) + if self.dry_run: + self._simulate_remember(content) + self._memory_table.setdefault(ns, {})[mem_id] = content + self._log_response("remember", 200, {"memory_id": mem_id, "dry_run": True}) + return [mem_id] + return self._remember_live(actor, content, mem_id) + + def recall(self, actor: tuple[str, str], query: str) -> list[dict]: + self._stats.ops.recall += 1 + if self.dry_run: + self._simulate_recall(query) + ns = self._table_key(actor) + rows = self._memory_table.get(ns, {}) + scored = sorted(rows.items(), key=lambda item: -_overlap(query, item[1])) + out = [{"id": mid, "content": txt} for mid, txt in scored if _overlap(query, txt) > 0] + self._log_response("recall", 200, {"results": out, "dry_run": True}) + return out + return self._recall_live(actor, query) + + def forget(self, actor: tuple[str, str], memory_ids: list[str]) -> None: + self._stats.ops.forget += 1 + if self.dry_run: + ns = self._table_key(actor) + bucket = self._memory_table.get(ns, {}) + for mid in memory_ids: + bucket.pop(mid, None) + self._simulate_forget() + self._log_response("forget", 200, {"deleted": memory_ids, "dry_run": True}) + return + self._forget_live(actor, memory_ids) + + def revise(self, actor: tuple[str, str], memory_ids: list[str], content: str) -> None: + self._stats.ops.revise += 1 + if self.dry_run: + ns = self._table_key(actor) + bucket = self._memory_table.get(ns, {}) + if memory_ids: + bucket[memory_ids[0]] = content + self._simulate_revise(content) + self._log_response("revise", 200, {"memory_id": memory_ids[:1], "dry_run": True}) + if self.write_settle_seconds: + time.sleep(self.write_settle_seconds) + return + self._revise_live(actor, memory_ids, content) + + def _table_key(self, actor: tuple[str, str]) -> str: + return f"{self._task_name}:{self.namespace(actor)}" + + def _deterministic_id(self, op: str, content: str) -> str: + seed = f"{self._run_id}:{self._task_name}:{op}:{content}" + return "m-" + hashlib.sha256(seed.encode("utf-8")).hexdigest()[:12] + + def _log_response(self, op: str, status: int, body: Any) -> None: + preview = json.dumps(body, ensure_ascii=False)[:500] + self._response_logs.append( + AdapterResponseLog( + lane=self.name, + task=self._task_name, + op=op, + status=status, + body_preview=preview, + ) + ) + self._stats.ops.http_calls += 1 + + def _record_model_constraint(self, reason: str) -> None: + if reason not in self._stats.product_constraints: + self._stats.product_constraints.append(reason) + + def _assert_pinned_models(self, chat_ok: bool, embed_ok: bool) -> None: + if not chat_ok: + self._record_model_constraint( + f"lane {self.name} cannot pin chat model {PINNED_CHAT_MODEL}" + ) + if not embed_ok: + self._record_model_constraint( + f"lane {self.name} cannot pin embed model {PINNED_EMBED_MODEL}" + ) + + def _simulate_remember(self, content: str) -> None: + self._stats.tokens.add_chat( + f"[extract] model={PINNED_CHAT_MODEL}\n{content}", + '{"facts":[]}', + ) + self._stats.tokens.add_embed(content) + + def _simulate_recall(self, query: str) -> None: + self._stats.tokens.add_embed(query) + + def _simulate_revise(self, content: str) -> None: + self._stats.tokens.add_chat(f"[update] model={PINNED_CHAT_MODEL}\n{content}", "{}") + + def _simulate_forget(self) -> None: + pass + + @abstractmethod + def _remember_live(self, actor: tuple[str, str], content: str, mem_id: str) -> list[str]: + ... + + @abstractmethod + def _recall_live(self, actor: tuple[str, str], query: str) -> list[dict]: + ... + + @abstractmethod + def _forget_live(self, actor: tuple[str, str], memory_ids: list[str]) -> None: + ... + + @abstractmethod + def _revise_live(self, actor: tuple[str, str], memory_ids: list[str], content: str) -> None: + ... + + def dry_run_stats(self) -> LaneDryRunStats: + return self._stats + + def write_raw_log(self, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + rows = [ + { + "lane": r.lane, + "task": r.task, + "op": r.op, + "status": r.status, + "body_preview": r.body_preview, + } + for r in self._response_logs + ] + path.write_text(json.dumps(rows, indent=2) + "\n", encoding="utf-8") + + +def _overlap(query: str, content: str) -> int: + q = {t for t in query.lower().split() if t} + c = {t for t in content.lower().split() if t} + return len(q & c) diff --git a/evals/opmem_external/rates.py b/evals/opmem_external/rates.py new file mode 100644 index 00000000..66043bb7 --- /dev/null +++ b/evals/opmem_external/rates.py @@ -0,0 +1,28 @@ +"""OpenAI list prices used for OpMem external lane worksheets (verify before live runs).""" + +from __future__ import annotations + +# Source: https://openai.com/api/pricing/ (GPT-4.1 mini + text-embedding-3-small), captured 2026-09-19. +RATE_SOURCE_URL = "https://openai.com/api/pricing/" +RATE_SOURCE_NOTE = ( + "gpt-4.1-mini-2025-04-14: $0.40/1M input, $1.60/1M output; " + "text-embedding-3-small: $0.02/1M tokens" +) + +from opmem_external.pins import ( # noqa: E402 + PINNED_CHAT_MODEL, + PINNED_EMBED_MODEL, + RATE_CHAT_INPUT_PER_M, + RATE_CHAT_OUTPUT_PER_M, + RATE_EMBED_PER_M, +) + +__all__ = [ + "RATE_SOURCE_URL", + "RATE_SOURCE_NOTE", + "PINNED_CHAT_MODEL", + "PINNED_EMBED_MODEL", + "RATE_CHAT_INPUT_PER_M", + "RATE_CHAT_OUTPUT_PER_M", + "RATE_EMBED_PER_M", +] diff --git a/evals/opmem_external/requirements-external-lanes.txt b/evals/opmem_external/requirements-external-lanes.txt new file mode 100644 index 00000000..f0eaafe8 --- /dev/null +++ b/evals/opmem_external/requirements-external-lanes.txt @@ -0,0 +1,10 @@ +# Pinned deps for OpMem external lane reproduction (install in eval venv). +mem0ai==2.1.0 +qdrant-client==1.19.1 +tiktoken==0.14.0 +langmem==0.0.30 +langgraph-checkpoint-postgres==3.1.2 +zep-cloud==3.28.0 +letta==0.11.7 +letta-client==0.1.307 +sqlite-vec diff --git a/evals/opmem_external/schemas/lane-manifest.schema.json b/evals/opmem_external/schemas/lane-manifest.schema.json new file mode 100644 index 00000000..a92db3a1 --- /dev/null +++ b/evals/opmem_external/schemas/lane-manifest.schema.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "OpMemExternalLaneManifest", + "type": "object", + "required": ["lane", "pinned_chat_model", "pinned_embed_model", "namespace_scheme"], + "properties": { + "lane": { "type": "string", "enum": ["mem0-oss", "zep", "letta", "langmem"] }, + "pinned_chat_model": { "const": "gpt-4.1-mini-2025-04-14" }, + "pinned_embed_model": { "const": "text-embedding-3-small" }, + "namespace_scheme": { "const": "opmem-sha256(run_id|lane|task|tenant|subject)[:16]" }, + "reset_per_task": { "const": true }, + "dry_run_default": { "const": true }, + "spend_ceiling_usd": { "type": "number" } + } +} diff --git a/evals/opmem_external/telemetry.py b/evals/opmem_external/telemetry.py new file mode 100644 index 00000000..ba3b9f25 --- /dev/null +++ b/evals/opmem_external/telemetry.py @@ -0,0 +1,53 @@ +"""Aggregate OpenAI usage for external lane runs (secrets redacted in logs).""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass, field +from pathlib import Path + +_SECRET_RE = re.compile(r"(sk-[A-Za-z0-9_-]{10,}|api[_-]?key[\"']?\s*[:=]\s*[\"'][^\"']+)", re.I) + + +@dataclass +class UsageTotals: + chat_input_tokens: int = 0 + chat_output_tokens: int = 0 + embed_tokens: int = 0 + estimated_usd: float = 0.0 + http_requests: int = 0 + + def to_dict(self) -> dict: + return { + "chat_input_tokens": self.chat_input_tokens, + "chat_output_tokens": self.chat_output_tokens, + "embed_tokens": self.embed_tokens, + "estimated_usd": round(self.estimated_usd, 6), + "http_requests": self.http_requests, + } + + +@dataclass +class LaneTelemetry: + lane: str + usage: UsageTotals = field(default_factory=UsageTotals) + events: list[dict] = field(default_factory=list) + + def add_openai_usage(self, *, input_tokens: int = 0, output_tokens: int = 0, embed_tokens: int = 0, usd: float = 0.0) -> None: + self.usage.chat_input_tokens += input_tokens + self.usage.chat_output_tokens += output_tokens + self.usage.embed_tokens += embed_tokens + self.usage.estimated_usd += usd + + def log_http(self, op: str, status: int, body_preview: str) -> None: + self.usage.http_requests += 1 + redacted = _SECRET_RE.sub("***REDACTED***", body_preview)[:500] + self.events.append({"op": op, "status": status, "body_preview": redacted}) + + def write(self, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps({"lane": self.lane, "usage": self.usage.to_dict(), "events": self.events}, indent=2) + "\n", + encoding="utf-8", + ) diff --git a/evals/opmem_external/test_lane_contract.py b/evals/opmem_external/test_lane_contract.py new file mode 100644 index 00000000..9d387696 --- /dev/null +++ b/evals/opmem_external/test_lane_contract.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Contract tests for OpMem external lanes (no live vendor APIs).""" +from __future__ import annotations + +import json +import os +import unittest +from pathlib import Path +from unittest import mock + +# Ensure evals/ is on path when run as module or script +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from opmem_external import LANE_ADAPTERS # noqa: E402 +from opmem_external.pins import SPEND_CEILING_USD # noqa: E402 +from opmem_external.worksheet import build_worksheet # noqa: E402 +from run_opmem import run_task # noqa: E402 + + +class LaneContractTests(unittest.TestCase): + def setUp(self) -> None: + self._env = mock.patch.dict(os.environ, {}, clear=False) + self._env.start() + os.environ.pop("OPENAI_API_KEY", None) + os.environ.pop("ZEP_API_KEY", None) + os.environ.pop("LETTA_APP_SERVER_TOKEN", None) + os.environ.pop("DATABASE_URL", None) + + def tearDown(self) -> None: + self._env.stop() + + def test_namespace_is_deterministic(self) -> None: + adapter = LANE_ADAPTERS["mem0-oss"](dry_run=True, run_id="fixed-run") + adapter.begin_task("cor01_basic_revision") + ns1 = adapter.namespace(("t1", "u1")) + adapter2 = LANE_ADAPTERS["mem0-oss"](dry_run=True, run_id="fixed-run") + adapter2.begin_task("cor01_basic_revision") + ns2 = adapter2.namespace(("t1", "u1")) + self.assertEqual(ns1, ns2) + + def test_dry_run_remember_recall_no_credentials(self) -> None: + adapter = LANE_ADAPTERS["langmem"](dry_run=True, run_id="r1") + self.assertTrue(adapter.available()) + adapter.begin_task("sup01_basic_forget") + ids = adapter.remember(("t1", "u1"), "door code is 4455") + self.assertEqual(len(ids), 1) + hits = adapter.recall(("t1", "u1"), "door code") + self.assertGreaterEqual(len(hits), 1) + + def test_zep_records_model_constraint(self) -> None: + adapter = LANE_ADAPTERS["zep"](dry_run=True) + stats = adapter.dry_run_stats() + self.assertTrue(stats.product_constraints) + + def test_worksheet_within_ceilings_dry_run(self) -> None: + fixture_dir = ROOT.parent / "fixtures" / "opmem" + tasks = [json.loads(p.read_text()) for p in sorted(fixture_dir.glob("*.json"))] + stats = [] + for name, factory in LANE_ADAPTERS.items(): + adapter = factory(dry_run=True, run_id="worksheet-test") + for task in tasks: + run_task(adapter, task) + stats.append(adapter.dry_run_stats()) + ws = build_worksheet(stats, task_count=len(tasks), step_count=47) + total = next(row for row in ws.lanes if row["lane"] == "_total_openai_estimate") + self.assertLessEqual(total["projected_usd"], SPEND_CEILING_USD["total"]) + for row in ws.lanes: + if row["lane"] in SPEND_CEILING_USD and not row["lane"].startswith("_"): + self.assertLessEqual(row["projected_usd"], SPEND_CEILING_USD[row["lane"]]) + + def test_raw_log_does_not_contain_env_secrets(self) -> None: + with mock.patch.dict( + os.environ, + {"OPENAI_API_KEY": "sk-test-secret-value-12345", "ZEP_API_KEY": "zep-secret"}, + clear=False, + ): + adapter = LANE_ADAPTERS["mem0-oss"](dry_run=True, run_id="log-test") + adapter.begin_task("dup01_idempotent_remember") + adapter.remember(("t1", "u1"), "hello") + log_path = Path(self._testMethodName + ".json") + try: + adapter.write_raw_log(log_path) + text = log_path.read_text() + self.assertNotIn("sk-test-secret", text) + self.assertNotIn("zep-secret", text) + finally: + log_path.unlink(missing_ok=True) + + def test_live_mode_requires_credentials(self) -> None: + adapter = LANE_ADAPTERS["letta"](dry_run=False) + self.assertFalse(adapter.available()) + + +if __name__ == "__main__": + unittest.main() diff --git a/evals/opmem_external/token_estimate.py b/evals/opmem_external/token_estimate.py new file mode 100644 index 00000000..441dc3f5 --- /dev/null +++ b/evals/opmem_external/token_estimate.py @@ -0,0 +1,94 @@ +"""Dry-run token estimation (no vendor calls).""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from opmem_external.pins import ( + PINNED_CHAT_MODEL, + PINNED_EMBED_MODEL, + RATE_CHAT_INPUT_PER_M, + RATE_CHAT_OUTPUT_PER_M, + RATE_EMBED_PER_M, +) + + +def estimate_tokens(text: str) -> int: + """Conservative heuristic when tiktoken is not a repo dependency.""" + stripped = (text or "").strip() + if not stripped: + return 0 + return max(1, len(stripped) // 4) + + +@dataclass +class TokenLedger: + chat_input_tokens: int = 0 + chat_output_tokens: int = 0 + embed_tokens: int = 0 + chat_model: str = PINNED_CHAT_MODEL + embed_model: str = PINNED_EMBED_MODEL + + def add_chat(self, prompt: str, completion: str = "") -> None: + self.chat_input_tokens += estimate_tokens(prompt) + self.chat_output_tokens += estimate_tokens(completion) + + def add_embed(self, text: str) -> None: + self.embed_tokens += estimate_tokens(text) + + def estimated_usd(self) -> float: + return ( + self.chat_input_tokens * RATE_CHAT_INPUT_PER_M / 1_000_000 + + self.chat_output_tokens * RATE_CHAT_OUTPUT_PER_M / 1_000_000 + + self.embed_tokens * RATE_EMBED_PER_M / 1_000_000 + ) + + def to_dict(self) -> dict: + return { + "chat_model": self.chat_model, + "embed_model": self.embed_model, + "chat_input_tokens": self.chat_input_tokens, + "chat_output_tokens": self.chat_output_tokens, + "embed_tokens": self.embed_tokens, + "estimated_usd": round(self.estimated_usd(), 6), + } + + +@dataclass +class OperationCounts: + remember: int = 0 + recall: int = 0 + revise: int = 0 + forget: int = 0 + reset: int = 0 + http_calls: int = 0 + + def to_dict(self) -> dict: + return { + "remember": self.remember, + "recall": self.recall, + "revise": self.revise, + "forget": self.forget, + "reset": self.reset, + "http_calls": self.http_calls, + } + + +@dataclass +class LaneDryRunStats: + lane: str + ops: OperationCounts = field(default_factory=OperationCounts) + tokens: TokenLedger = field(default_factory=TokenLedger) + product_constraints: list[str] = field(default_factory=list) + credentials_present: bool = False + missing_credentials: list[str] = field(default_factory=list) + + def to_dict(self) -> dict: + return { + "lane": self.lane, + "operations": self.ops.to_dict(), + "tokens": self.tokens.to_dict(), + "product_constraints": self.product_constraints, + "credentials_present": self.credentials_present, + "missing_credentials": self.missing_credentials, + } diff --git a/evals/opmem_external/worksheet.py b/evals/opmem_external/worksheet.py new file mode 100644 index 00000000..54f8131a --- /dev/null +++ b/evals/opmem_external/worksheet.py @@ -0,0 +1,97 @@ +"""Per-lane cost worksheet from dry-run instrumentation.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path + +from opmem_external.pins import ( + PINNED_CHAT_MODEL, + PINNED_EMBED_MODEL, + RATE_CHAT_INPUT_PER_M, + RATE_CHAT_OUTPUT_PER_M, + RATE_EMBED_PER_M, + SPEND_CEILING_USD, +) +from opmem_external.token_estimate import LaneDryRunStats + + +@dataclass +class Worksheet: + generated_utc: str + task_count: int + step_count: int + rates: dict + pinned_models: dict + ceilings_usd: dict + lanes: list[dict] + + def to_dict(self) -> dict: + return { + "generated_utc": self.generated_utc, + "task_count": self.task_count, + "step_count": self.step_count, + "rates": self.rates, + "pinned_models": self.pinned_models, + "ceilings_usd": self.ceilings_usd, + "lanes": self.lanes, + } + + +def build_worksheet( + lane_stats: list[LaneDryRunStats], + *, + task_count: int, + step_count: int, +) -> Worksheet: + lanes_out = [] + total_usd = 0.0 + for stats in lane_stats: + usd = stats.tokens.estimated_usd() + if stats.lane == "zep": + usd = 0.0 + total_usd += usd + ceiling = SPEND_CEILING_USD.get(stats.lane, 0.0) + lanes_out.append( + { + **stats.to_dict(), + "projected_usd": round(usd, 6), + "ceiling_usd": ceiling, + "within_ceiling": usd <= ceiling, + "live_run_allowed": False, + "live_run_note": ( + "Phase 1 dry-run only; enable after credentials present and " + "measured estimate confirmed under ceiling." + ), + } + ) + lanes_out.append( + { + "lane": "_total_openai_estimate", + "projected_usd": round(total_usd, 6), + "ceiling_usd": SPEND_CEILING_USD["total"], + "within_ceiling": total_usd <= SPEND_CEILING_USD["total"], + } + ) + return Worksheet( + generated_utc=datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + task_count=task_count, + step_count=step_count, + rates={ + PINNED_CHAT_MODEL: { + "input_per_m": RATE_CHAT_INPUT_PER_M, + "output_per_m": RATE_CHAT_OUTPUT_PER_M, + }, + PINNED_EMBED_MODEL: {"per_m": RATE_EMBED_PER_M}, + }, + pinned_models={"chat": PINNED_CHAT_MODEL, "embed": PINNED_EMBED_MODEL}, + ceilings_usd=SPEND_CEILING_USD, + lanes=lanes_out, + ) + + +def write_worksheet(path: Path, worksheet: Worksheet) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(worksheet.to_dict(), indent=2) + "\n", encoding="utf-8") diff --git a/evals/opmem_external/zep_cloud.py b/evals/opmem_external/zep_cloud.py new file mode 100644 index 00000000..03df6abe --- /dev/null +++ b/evals/opmem_external/zep_cloud.py @@ -0,0 +1,162 @@ +"""Zep Cloud lane (free tier only; $0 paid ceiling). SDK 3.x thread + graph API.""" + +from __future__ import annotations + +import os +import re + +from opmem_external.protocol import OpMemLaneAdapter, _overlap +from opmem_external.telemetry import LaneTelemetry + +_BILLING_HINT = re.compile(r"(upgrade|payment|billing|subscribe|credit|quota|limit exceeded)", re.I) + + +class ZepCloudLaneAdapter(OpMemLaneAdapter): + name = "zep" + write_settle_seconds = 1.5 + + def __init__(self, *, dry_run: bool = True, run_id: str | None = None) -> None: + super().__init__(dry_run=dry_run, run_id=run_id) + self._api_key = os.environ.get("ZEP_API_KEY", "") + self.telemetry = LaneTelemetry(self.name) + self._thread_id: str | None = None + self._user_id: str | None = None + self._assert_pinned_models(chat_ok=False, embed_ok=False) + self._record_model_constraint( + "Zep Cloud memory path does not accept OpenAI chat/embed model pins; " + "stop before substituting — record scores only under Zep-native models." + ) + + def lane_id(self) -> str: + return "zep_cloud_free_tier" + + def _client(self): + from zep_cloud import Zep + + return Zep(api_key=self._api_key) + + def _abort_if_billing(self, text: str) -> None: + if _BILLING_HINT.search(text or ""): + raise RuntimeError("Zep hard-stop: billing/upgrade/credit prompt detected") + + def verify_free_tier(self) -> None: + client = self._client() + try: + client.user.list_ordered(page_size=1, page_number=1) + except Exception as exc: # noqa: BLE001 + self._abort_if_billing(str(exc)) + raise + + def _reset_remote(self) -> None: + client = self._client() + self.verify_free_tier() + self._user_id = self.namespace(("t1", "u1")) + self._thread_id = f"{self._user_id}-thread" + if self._thread_id: + try: + client.thread.delete(self._thread_id) + except Exception: + pass + try: + client.user.add(user_id=self._user_id, first_name="OpMem", last_name="Lane") + except Exception: + pass + client.thread.create(thread_id=self._thread_id, user_id=self._user_id) + + def _remember_live(self, actor: tuple[str, str], content: str, mem_id: str) -> list[str]: + from zep_cloud.types import Message + + client = self._client() + tid = self._thread_for(actor) + client.thread.add_messages( + thread_id=tid, + messages=[Message(content=content, role="user")], + return_context=False, + ) + self.telemetry.log_http("remember", 200, "zep thread.add_messages") + if self.write_settle_seconds: + import time + + time.sleep(self.write_settle_seconds) + return [mem_id] + + def _thread_for(self, actor: tuple[str, str]) -> str: + if self._thread_id is None: + self._reset_remote() + return self._thread_id or self.namespace(actor) + + def _recall_live(self, actor: tuple[str, str], query: str) -> list[dict]: + client = self._client() + uid = self._user_id or self.namespace(actor) + tid = self._thread_for(actor) + out: list[dict] = [] + seen_content: set[str] = set() + + msg_list = client.thread.get(tid, lastn=100) + for idx, msg in enumerate(msg_list.messages or []): + text = (msg.content or "").strip() + if not text or _overlap(query, text) <= 0: + continue + key = text.lower() + if key in seen_content: + continue + seen_content.add(key) + out.append({"id": f"zep-msg-{idx}", "content": text}) + + try: + results = client.graph.search(query=query, user_id=uid, limit=10) + self._abort_if_billing(str(results)) + for ep in results.episodes or []: + text = getattr(ep, "content", None) or getattr(ep, "name", None) or "" + text = str(text).strip() + if not text or text.lower() in seen_content: + continue + if _overlap(query, text) <= 0: + continue + seen_content.add(text.lower()) + eid = getattr(ep, "uuid_", None) or getattr(ep, "uuid", "zep-episode") + out.append({"id": str(eid), "content": text}) + for edge in results.edges or []: + text = getattr(edge, "fact", None) or getattr(edge, "name", None) or "" + text = str(text).strip() + if not text or text.lower() in seen_content: + continue + if _overlap(query, text) <= 0: + continue + seen_content.add(text.lower()) + eid = getattr(edge, "uuid_", None) or getattr(edge, "uuid", "zep-edge") + out.append({"id": str(eid), "content": text}) + except Exception as exc: + self._abort_if_billing(str(exc)) + + out.sort(key=lambda row: -_overlap(query, row["content"])) + self.telemetry.log_http("recall", 200, str(out)[:500]) + return out + + def _forget_live(self, actor: tuple[str, str], memory_ids: list[str]) -> None: + self.telemetry.log_http("forget", 200, f"zep forget noop for {memory_ids}") + + def _revise_live(self, actor: tuple[str, str], memory_ids: list[str], content: str) -> None: + from zep_cloud.types import Message + + client = self._client() + tid = self._thread_for(actor) + client.thread.add_messages( + thread_id=tid, + messages=[Message(content=content, role="user")], + return_context=False, + ) + self.telemetry.log_http("revise", 200, "zep thread.add_messages revise") + if self.write_settle_seconds: + import time + + time.sleep(self.write_settle_seconds) + + def _simulate_remember(self, content: str) -> None: + return + + def _simulate_recall(self, query: str) -> None: + return + + def _simulate_revise(self, content: str) -> None: + return diff --git a/evals/run_opmem.py b/evals/run_opmem.py index 86fefa59..cd857494 100644 --- a/evals/run_opmem.py +++ b/evals/run_opmem.py @@ -14,6 +14,7 @@ import argparse import json +import os import pathlib import sys @@ -22,6 +23,11 @@ from opmem_adapters import BrainyAdapter, Mem0OpAdapter, VerbatimBaseline +try: + from opmem_external import LANE_ADAPTERS +except ImportError: # pragma: no cover + LANE_ADAPTERS = {} + def resolve_actor(task: dict, name: str) -> tuple[str, str]: spec = task.get("actors", {}).get(name) @@ -90,8 +96,13 @@ def build_adapters(names: list[str], base_url: str) -> list: registry = { "verbatim": lambda: VerbatimBaseline(), "brainy": lambda: BrainyAdapter(base_url), + "brainy-recall": lambda: BrainyAdapter(base_url, recall_lane="recall"), + "brainy-supersede": lambda: BrainyAdapter(base_url, revise_mode="supersede"), "mem0": lambda: Mem0OpAdapter(), } + live_external = os.environ.get("OPMEM_EXTERNAL_LIVE", "").strip() in ("1", "true", "yes") + for lane_name, factory in LANE_ADAPTERS.items(): + registry[lane_name] = lambda f=factory, live=live_external: f(dry_run=not live) adapters = [] for name in names: if name not in registry: @@ -167,7 +178,9 @@ def main() -> int: out = ROOT.parent / out out.parent.mkdir(parents=True, exist_ok=True) out.write_text(output + "\n", encoding="utf-8") - return 1 if infrastructure_errors else 0 + if infrastructure_errors and not os.environ.get("OPMEM_ALLOW_INFRA_EXIT"): + return 1 + return 0 if __name__ == "__main__": diff --git a/evals/run_opmem_cost_dry_run.py b/evals/run_opmem_cost_dry_run.py new file mode 100644 index 00000000..39a7e8fc --- /dev/null +++ b/evals/run_opmem_cost_dry_run.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Dry-run all OpMem external lanes: operation counts + token/cost worksheet (no vendor APIs).""" +from __future__ import annotations + +import argparse +import json +import pathlib +import sys + +ROOT = pathlib.Path(__file__).resolve().parent +sys.path.insert(0, str(ROOT)) + +from opmem_external import LANE_ADAPTERS # noqa: E402 +from opmem_external.cost_budget import build_lane_budgets, build_worksheet_from_budgets # noqa: E402 +from opmem_external.worksheet import build_worksheet, write_worksheet # noqa: E402 +from run_opmem import run_task # noqa: E402 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--fixture-dir", default="fixtures/opmem") + parser.add_argument("--out-dir", default="docs/benchmarks/artifacts/opmem-external-lanes") + parser.add_argument("--run-id", default="opmem-dryrun-0001") + args = parser.parse_args() + + fixture_dir = pathlib.Path(args.fixture_dir) + if not fixture_dir.is_absolute(): + fixture_dir = ROOT.parent / fixture_dir + tasks = [json.loads(p.read_text(encoding="utf-8")) for p in sorted(fixture_dir.glob("*.json"))] + step_count = sum(len(t["steps"]) for t in tasks) + + out_dir = pathlib.Path(args.out_dir) + if not out_dir.is_absolute(): + out_dir = ROOT.parent / out_dir + out_dir.mkdir(parents=True, exist_ok=True) + + lane_stats = [] + per_lane_results = {} + for lane_name, factory in sorted(LANE_ADAPTERS.items()): + adapter = factory(dry_run=True, run_id=args.run_id) + results = [] + infra = 0 + for task in tasks: + try: + outcome = run_task(adapter, task) + results.append({"task": task["name"], "outcome": outcome}) + except Exception as exc: # noqa: BLE001 + infra += 1 + results.append({"task": task["name"], "error": f"{type(exc).__name__}: {exc}"}) + per_lane_results[lane_name] = { + "infrastructure_errors": infra, + "results": results, + } + log_path = out_dir / f"raw-{lane_name}-dry-run.jsonl" + adapter.write_raw_log(log_path.with_suffix(".json")) + lane_stats.append(adapter.dry_run_stats()) + + worksheet = build_worksheet(lane_stats, task_count=len(tasks), step_count=step_count) + write_worksheet(out_dir / "cost-worksheet-dry-run.json", worksheet) + budgets = build_lane_budgets(tasks) + payload_ws = build_worksheet_from_budgets( + budgets, task_count=len(tasks), step_count=step_count, + ) + (out_dir / "cost-worksheet-payload-budget.json").write_text( + json.dumps(payload_ws, indent=2) + "\n", encoding="utf-8", + ) + (out_dir / "dry-run-results.json").write_text( + json.dumps( + { + "benchmark": "opmem-v0", + "mode": "dry_run", + "run_id": args.run_id, + "task_count": len(tasks), + "step_count": step_count, + "lanes": per_lane_results, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + print(json.dumps(worksheet.to_dict(), indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/evals/run_opmem_external_live.py b/evals/run_opmem_external_live.py new file mode 100644 index 00000000..d15f85ed --- /dev/null +++ b/evals/run_opmem_external_live.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python3 +"""Phase 2: smoke or full OpMem external lane runs with preflight + artifacts.""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import subprocess +import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parent +sys.path.insert(0, str(ROOT)) + +from opmem_external import LANE_ADAPTERS # noqa: E402 +from opmem_external.cost_budget import ( # noqa: E402 + build_lane_budgets, + build_worksheet_from_budgets, + load_tasks, +) +from opmem_external.pins import LANE_ENV_REQUIRED, SPEND_CEILING_USD # noqa: E402 +from run_opmem import run_task # noqa: E402 + +DEFAULT_OUT = ROOT.parent / "docs/benchmarks/artifacts/opmem-external-lanes" +SMOKE_TASK = "dup01_idempotent_remember.json" + + +def _utc_now() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _package_versions() -> dict[str, str]: + names = ( + "mem0ai", + "qdrant-client", + "zep-cloud", + "tiktoken", + "langmem", + "langgraph-checkpoint-postgres", + "openai", + "langchain-openai", + ) + out: dict[str, str] = {} + for name in names: + try: + import importlib.metadata as md + + out[name] = md.version(name) + except Exception: + out[name] = "not_installed" + return out + + +def _preflight_budget(tasks: list[dict]) -> dict: + step_count = sum(len(t["steps"]) for t in tasks) + budgets = build_lane_budgets(tasks) + return build_worksheet_from_budgets(budgets, task_count=len(tasks), step_count=step_count) + + +def _lane_blocked(lane: str) -> tuple[bool, str]: + if lane == "zep": + if not os.environ.get("ZEP_API_KEY", "").strip(): + return True, "ZEP_API_KEY not present in runtime (vault not injected on this VM)" + if lane == "letta": + base = os.environ.get("LETTA_BASE_URL", "http://127.0.0.1:8283").rstrip("/") + local = base.startswith("http://127.0.0.1") or base.startswith("http://localhost") + if not local and not os.environ.get("LETTA_APP_SERVER_TOKEN", "").strip(): + return True, "LETTA_APP_SERVER_TOKEN required for non-local Letta base URL" + try: + import urllib.request + + urllib.request.urlopen(f"{base}/v1/health", timeout=3) + except Exception as exc: + return True, f"Letta App Server unreachable at {base}: {exc}" + if lane == "langmem": + if not os.environ.get("DATABASE_URL", "").strip(): + default = "postgresql://opmem:opmem@127.0.0.1:5432/opmem_langmem" + os.environ.setdefault("DATABASE_URL", default) + for key in LANE_ENV_REQUIRED.get(lane, ()): + if not os.environ.get(key, "").strip(): + return True, f"missing required env {key}" + return False, "" + + +def _run_lane( + lane: str, + tasks: list[dict], + *, + run_id: str, + out_dir: Path, + mode: str, +) -> dict: + blocked, reason = _lane_blocked(lane) + if blocked: + return { + "lane": lane, + "status": "blocked", + "reason": reason, + "infrastructure_errors": 0, + "task_results": [], + } + + factory = LANE_ADAPTERS[lane] + adapter = factory(dry_run=False, run_id=run_id) + if not adapter.available(): + return { + "lane": lane, + "status": "blocked", + "reason": f"credentials: {adapter.dry_run_stats().missing_credentials}", + "infrastructure_errors": 0, + "task_results": [], + } + + if lane == "zep": + zep = adapter + try: + zep.verify_free_tier() + except Exception as exc: + return { + "lane": lane, + "status": "blocked", + "reason": f"Zep free-tier preflight failed: {exc}", + "infrastructure_errors": 1, + "task_results": [], + } + + infra = 0 + results = [] + started = time.time() + for task in tasks: + try: + outcome = run_task(adapter, task) + results.append( + { + "task": task["name"], + "passed": outcome["passed"], + "failures": outcome.get("failures", []), + } + ) + except Exception as exc: + infra += 1 + results.append( + { + "task": task["name"], + "error": f"{type(exc).__name__}: {exc}", + "invalid_incomplete": True, + } + ) + if lane == "zep" and "hard-stop" in str(exc).lower(): + break + + elapsed = round(time.time() - started, 2) + raw_path = out_dir / f"raw-{lane}-{mode}.json" + adapter.write_raw_log(raw_path) + telemetry = getattr(adapter, "telemetry", None) + if telemetry is not None: + telemetry.write(out_dir / f"telemetry-{lane}-{mode}.json") + + passed = sum(1 for r in results if r.get("passed")) + return { + "lane": lane, + "status": "completed" if infra == 0 else "invalid_incomplete", + "mode": mode, + "run_id": run_id, + "elapsed_seconds": elapsed, + "infrastructure_errors": infra, + "tasks_run": len(results), + "tasks_passed": passed, + "task_results": results, + "product_constraints": adapter.dry_run_stats().product_constraints, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--fixture-dir", default="fixtures/opmem") + parser.add_argument("--out-dir", default=str(DEFAULT_OUT)) + parser.add_argument("--run-id", default="") + parser.add_argument( + "--lanes", + default="mem0-oss,zep,letta,langmem", + help="Comma-separated lane ids", + ) + parser.add_argument("--smoke", action="store_true", help="Run one smallest task per lane") + parser.add_argument("--full", action="store_true", help="Run all 13 tasks per lane") + parser.add_argument("--skip-preflight", action="store_true") + args = parser.parse_args() + + if not args.smoke and not args.full: + parser.error("Specify --smoke and/or --full") + + fixture_dir = Path(args.fixture_dir) + if not fixture_dir.is_absolute(): + fixture_dir = ROOT.parent / fixture_dir + all_tasks = load_tasks(fixture_dir) + run_id = args.run_id or f"opmem-live-{datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S')}" + os.environ["OPMEM_RUN_ID"] = run_id + + out_dir = Path(args.out_dir) + if not out_dir.is_absolute(): + out_dir = ROOT.parent / out_dir + out_dir.mkdir(parents=True, exist_ok=True) + + if not args.skip_preflight: + budget = _preflight_budget(all_tasks) + (out_dir / "cost-worksheet-payload-budget.json").write_text( + json.dumps(budget, indent=2) + "\n", encoding="utf-8", + ) + total = next(r for r in budget["lanes"] if r["lane"] == "_total_openai_worst_case") + if not total["within_ceiling"]: + print("ABORT: projected spend exceeds owner ceiling", file=sys.stderr) + print(json.dumps(budget, indent=2)) + return 2 + for row in budget["lanes"]: + lane = row.get("lane", "") + if lane in SPEND_CEILING_USD and not row.get("within_ceiling", True): + print(f"ABORT: lane {lane} exceeds ceiling", file=sys.stderr) + return 2 + + lanes = [x.strip() for x in args.lanes.split(",") if x.strip()] + manifest = { + "benchmark": "opmem-v0", + "generated_utc": _utc_now(), + "run_id": run_id, + "host": platform.platform(), + "python": platform.python_version(), + "package_versions": _package_versions(), + "pinned_models": { + "chat": "gpt-4.1-mini-2025-04-14", + "embed": "text-embedding-3-small", + }, + "ceilings_usd": SPEND_CEILING_USD, + "modes": [], + } + + smoke_tasks = [t for t in all_tasks if t["name"] == SMOKE_TASK.replace(".json", "")] + if not smoke_tasks: + smoke_path = fixture_dir / SMOKE_TASK + smoke_tasks = [json.loads(smoke_path.read_text(encoding="utf-8"))] + + if args.smoke: + smoke_out = [] + for lane in lanes: + smoke_out.append(_run_lane(lane, smoke_tasks, run_id=run_id, out_dir=out_dir, mode="smoke")) + manifest["modes"].append({"mode": "smoke", "task": smoke_tasks[0]["name"], "lanes": smoke_out}) + (out_dir / f"live-smoke-{run_id}.json").write_text( + json.dumps(smoke_out, indent=2) + "\n", encoding="utf-8", + ) + + if args.full: + full_out = [] + for lane in lanes: + prior = next( + (m for m in manifest.get("modes", []) if m.get("mode") == "smoke"), + None, + ) + if prior: + lane_smoke = next((x for x in prior["lanes"] if x["lane"] == lane), None) + if lane_smoke and lane_smoke.get("status") != "completed": + full_out.append( + { + "lane": lane, + "status": "skipped", + "reason": "smoke did not complete cleanly", + } + ) + continue + full_out.append(_run_lane(lane, all_tasks, run_id=run_id, out_dir=out_dir, mode="full")) + manifest["modes"].append({"mode": "full", "task_count": len(all_tasks), "lanes": full_out}) + (out_dir / f"live-full-{run_id}.json").write_text( + json.dumps(full_out, indent=2) + "\n", encoding="utf-8", + ) + + (out_dir / f"run-manifest-{run_id}.json").write_text( + json.dumps(manifest, indent=2) + "\n", encoding="utf-8", + ) + commands_log = out_dir / "commands.log" + with commands_log.open("a", encoding="utf-8") as fh: + fh.write(f"\n# {_utc_now()} run_id={run_id}\n") + fh.write(" ".join(subprocess.list2cmdline(sys.argv)) + "\n") + print(json.dumps(manifest, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/evals/run_opmem_paper_experiments.py b/evals/run_opmem_paper_experiments.py new file mode 100644 index 00000000..ff162b12 --- /dev/null +++ b/evals/run_opmem_paper_experiments.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +"""OpMem manuscript experiments: manifests, lane ablations, stability repeats. + +Uses the same task runner as run_opmem.py. Intended to run against a live Brainy +API or the httptest server spawned by go test (see scripts/run-opmem-manuscript.sh). +""" +from __future__ import annotations + +import argparse +import json +import pathlib +import subprocess +import sys +import time +from datetime import datetime, timezone + +ROOT = pathlib.Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "evals")) + +from run_opmem import build_adapters, run_task # noqa: E402 + + +def git_sha() -> str: + try: + return ( + subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True) + .strip() + ) + except (subprocess.CalledProcessError, FileNotFoundError): + return "unknown" + + +def load_tasks(fixture_dir: pathlib.Path) -> list[dict]: + return [ + json.loads(path.read_text(encoding="utf-8")) + for path in sorted(fixture_dir.glob("*.json")) + ] + + +def run_suite( + base_url: str, + system_names: list[str], + tasks: list[dict], +) -> dict: + adapters = build_adapters(system_names, base_url) + results = [] + infrastructure_errors = 0 + for task in tasks: + entry = {"task": task["name"], "category": task["category"], "systems": {}} + for adapter in adapters: + if not adapter.available(): + entry["systems"][adapter.name] = {"skipped": True, "reason": "not configured"} + continue + try: + entry["systems"][adapter.name] = run_task(adapter, task) + except Exception as exc: # noqa: BLE001 + infrastructure_errors += 1 + entry["systems"][adapter.name] = {"error": f"{type(exc).__name__}: {exc}"} + results.append(entry) + + summary: dict[str, dict] = {} + for adapter in adapters: + by_category: dict[str, dict[str, int]] = {} + passed_total = total = 0 + for entry in results: + outcome = entry["systems"].get(adapter.name, {}) + if outcome.get("skipped") or outcome.get("error"): + continue + bucket = by_category.setdefault(entry["category"], {"passed": 0, "total": 0}) + bucket["total"] += 1 + total += 1 + if outcome.get("passed"): + bucket["passed"] += 1 + passed_total += 1 + summary[adapter.name] = { + "overall": f"{passed_total}/{total}", + "by_category": {cat: f"{v['passed']}/{v['total']}" for cat, v in sorted(by_category.items())}, + } + + return { + "benchmark": "opmem-v0", + "systems": [a.name for a in adapters], + "summary": summary, + "infrastructure_errors": infrastructure_errors, + "results": results, + } + + +def stability_runs( + base_url: str, + tasks: list[dict], + repeats: int, +) -> dict: + runs = [] + for index in range(repeats): + report = run_suite(base_url, ["brainy"], tasks) + passed = report["summary"].get("brainy", {}).get("overall", "0/0") + runs.append({"repeat": index + 1, "overall": passed, "infrastructure_errors": report["infrastructure_errors"]}) + time.sleep(0.05) + overalls = [r["overall"] for r in runs] + return { + "repeats": repeats, + "system": "brainy", + "recall_lane": "search", + "revise_mode": "correct", + "runs": runs, + "stable": len(set(overalls)) == 1, + "unique_scores": sorted(set(overalls)), + } + + +def mechanism_view(report: dict) -> dict: + """Category-level pass rates as a lightweight mechanism breakdown.""" + out: dict[str, dict[str, str]] = {} + for system, summ in report.get("summary", {}).items(): + out[system] = summ.get("by_category", {}) + out["_note"] = ( + "verbatim is the rank-only baseline (no correct/suppress lifecycle); " + "correction/suppression/staleness categories map to revise/forget semantics." + ) + return out + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="http://127.0.0.1:8080") + parser.add_argument("--fixture-dir", default="fixtures/opmem") + parser.add_argument("--out-dir", required=True) + parser.add_argument("--stability-repeats", type=int, default=5) + parser.add_argument("--skip-mem0", action="store_true") + args = parser.parse_args() + + fixture_dir = pathlib.Path(args.fixture_dir) + if not fixture_dir.is_absolute(): + fixture_dir = ROOT / fixture_dir + tasks = load_tasks(fixture_dir) + out_dir = pathlib.Path(args.out_dir) + if not out_dir.is_absolute(): + out_dir = ROOT / out_dir + out_dir.mkdir(parents=True, exist_ok=True) + + sha = git_sha() + manifest = { + "benchmark": "opmem-v0", + "git_sha": sha, + "run_utc": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "fixture_dir": str(fixture_dir.relative_to(ROOT) if fixture_dir.is_relative_to(ROOT) else fixture_dir), + "task_count": len(tasks), + "base_url": args.base_url, + "protocol": "docs/research/opmem/PUBLICATION_READINESS.md", + "scoring": "substring assertions on ranked recall results", + } + (out_dir / "run-manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + + stability = stability_runs(args.base_url, tasks, args.stability_repeats) + valid = [r for r in stability["runs"] if r.get("infrastructure_errors", 0) == 0] + stability["valid_runs"] = len(valid) + stability["valid_scores"] = sorted({r["overall"] for r in valid}) + stability["interpretation"] = ( + "Runs with infrastructure_errors>0 are invalid/incomplete; " + "partial denominators (e.g. 9/10) are not comparable to a clean 13/13 pin." + ) + stability["manifest"] = manifest + (out_dir / "opmem-stability.json").write_text(json.dumps(stability, indent=2) + "\n", encoding="utf-8") + + lanes = run_suite( + args.base_url, + ["brainy", "brainy-recall", "brainy-supersede"], + tasks, + ) + lanes["manifest"] = manifest + (out_dir / "opmem-lane-ablation.json").write_text(json.dumps(lanes, indent=2) + "\n", encoding="utf-8") + + primary_systems = ["verbatim", "brainy"] + if not args.skip_mem0: + primary_systems.append("mem0") + primary = run_suite(args.base_url, primary_systems, tasks) + primary["manifest"] = manifest + (out_dir / "opmem-primary.json").write_text(json.dumps(primary, indent=2) + "\n", encoding="utf-8") + + mechanism = { + "manifest": manifest, + "by_category": mechanism_view(primary), + "verbatim_overall": primary["summary"].get("verbatim", {}).get("overall"), + } + (out_dir / "opmem-mechanism.json").write_text(json.dumps(mechanism, indent=2) + "\n", encoding="utf-8") + + log_lines = [ + f"git_sha={sha}", + f"tasks={len(tasks)}", + f"primary={json.dumps(primary['summary'])}", + f"lanes={json.dumps(lanes['summary'])}", + f"stability={json.dumps(stability)}", + ] + (out_dir / "run.log").write_text("\n".join(log_lines) + "\n", encoding="utf-8") + + return 1 if primary["infrastructure_errors"] else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/internal/api/eval_e2e_test.go b/internal/api/eval_e2e_test.go index df6869df..062b744e 100644 --- a/internal/api/eval_e2e_test.go +++ b/internal/api/eval_e2e_test.go @@ -2,6 +2,7 @@ package api import ( "context" + "os" "os/exec" "path/filepath" "strconv" @@ -45,6 +46,30 @@ func TestOpMemBenchmarkAgainstHTTPServer(t *testing.T) { ) } +func TestOpMemPaperExperiments(t *testing.T) { + outDir := os.Getenv("OPMEM_MANUSCRIPT_OUT") + if outDir == "" { + outDir = t.TempDir() + } + args := []string{"--out-dir", outDir} + if os.Getenv("MEM0_API_KEY") == "" { + args = append(args, "--skip-mem0") + } + runEvalHarness(t, "evals/run_opmem_paper_experiments.py", "", args...) +} + +func TestOpMemLaneAblationOnly(t *testing.T) { + outDir := os.Getenv("OPMEM_LANE_OUT") + if outDir == "" { + t.Skip("set OPMEM_LANE_OUT to write lane ablation JSON") + } + t.Setenv("OPMEM_ALLOW_INFRA_EXIT", "1") + runEvalHarness(t, "evals/run_opmem.py", "", + "--systems", "brainy,brainy-recall,brainy-supersede", + "--json-out", filepath.Join(outDir, "opmem-lane-ablation-isolated.json"), + ) +} + func runEvalHarness(t *testing.T, script, fixtureDir string, extraArgs ...string) { t.Helper() t.Setenv("LANG", "C") diff --git a/scripts/generate-opmem-paper-tex.py b/scripts/generate-opmem-paper-tex.py new file mode 100644 index 00000000..85d0b8f3 --- /dev/null +++ b/scripts/generate-opmem-paper-tex.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""Generate LaTeX fragments for OpMem manuscript from run artifacts.""" +from __future__ import annotations + +import json +import pathlib +import sys + +ROOT = pathlib.Path(__file__).resolve().parent.parent +ART = ROOT / "docs/benchmarks/artifacts/opmem-manuscript-20533f2" +OUT = ROOT / "docs/research/opmem/paper/generated" + + +def load(name: str) -> dict: + return json.loads((ART / name).read_text(encoding="utf-8")) + + +def write(name: str, body: str) -> None: + OUT.mkdir(parents=True, exist_ok=True) + (OUT / name).write_text(body.strip() + "\n", encoding="utf-8") + + +def main() -> int: + if not ART.is_dir(): + print(f"missing artifact dir {ART}", file=sys.stderr) + return 1 + manifest = load("run-manifest.json") + primary = load("opmem-primary.json") + lane_path = ART / "opmem-lane-ablation-isolated.json" + if not lane_path.is_file(): + lane_path = ART / "opmem-lane-ablation.json" + lanes = json.loads(lane_path.read_text(encoding="utf-8")) + stability = load("opmem-stability.json") + mechanism = load("opmem-mechanism.json") + + sha = manifest["git_sha"] + brainy = primary["summary"]["brainy"]["overall"] + verbatim = primary["summary"]["verbatim"]["overall"] + mem0_line = "" + if "mem0" in primary["summary"]: + mem0_line = ( + f"Mem0 Platform (same run): \\textbf{{{primary['summary']['mem0']['overall']}}} " + f"(API surface: Platform via \\texttt{{Mem0OpAdapter}})." + ) + + write( + "current-sha-results.tex", + f""" +\\evidence{{completed}}{{ +At git SHA \\texttt{{{sha[:12]}}} (UTC {manifest['run_utc']}), embedded API harness: +Brainy (search lane) \\textbf{{{brainy}}}, verbatim \\textbf{{{verbatim}}}. +{mem0_line} +}} +""", + ) + + write( + "combined-results.tex", + f""" +\\evidence{{completed}}{{ +Committed \\texttt{{opmem-primary.json}} lists per-task outcomes for +{', '.join(primary['systems'])} with manifest fields (SHA, date, fixture count). +}} +""", + ) + + lane_rows = [] + for system in lanes["systems"]: + summ = lanes["summary"].get(system, {}) + lane_rows.append(f"{system} & {summ.get('overall', 'n/a')} \\\\") + write( + "lane-ablation.tex", + """ +\\begin{table}[h] +\\centering +\\caption{Brainy recall lane and revise mapping (SHA """ + + sha[:12] + + """).} +\\begin{tabular}{lc} +\\toprule +Configuration & Overall \\\\ +\\midrule +""" + + "\n".join(lane_rows) + + """ +\\bottomrule +\\end{tabular} +\\end{table} +Search (\\texttt{/memories/search}) is the normative pin lane. +\\texttt{brainy-recall} uses \\texttt{POST /recall} (context mode). +Correction tasks on the recall lane may return HTTP 409 during \\texttt{/correct} +(infrastructure errors in artifact JSON). +""", + ) + + sup = lanes["summary"].get("brainy-supersede", {}).get("overall", "n/a") + corr = lanes["summary"].get("brainy", {}).get("overall", "n/a") + write( + "supersede-ablation.tex", + f""" +\\evidence{{completed}}{{ +Mapping \\texttt{{revise}} $\\rightarrow$ \\texttt{{POST .../supersede}} yields +\\textbf{{{sup}}} vs \\textbf{{{corr}}} with \\texttt{{/correct}} on the search lane +(same SHA). Failures are task-level assertion misses, not missing runs. +}} +""", + ) + + by_cat = mechanism.get("by_category", {}) + brainy_cat = by_cat.get("brainy", {}) + verb_cat = by_cat.get("verbatim", {}) + lines = [ + "\\begin{tabular}{lcc}", + "\\toprule", + "Category & Brainy & Verbatim (rank-only) \\\\", + "\\midrule", + ] + for cat in sorted(brainy_cat.keys()): + if cat.startswith("_"): + continue + lines.append(f"{cat} & {brainy_cat.get(cat, '---')} & {verb_cat.get(cat, '---')} \\\\") + lines.append("\\bottomrule") + lines.append("\\end{tabular}") + write( + "mechanism-ablation.tex", + """ +Category pass rates separate correction/suppression/staleness contracts; +verbatim approximates rank-only recall without governed lifecycle ops. +\\begin{table}[h] +\\centering +""" + + "\n".join(lines) + + "\n\\end{table}", + ) + + runs = stability.get("runs", []) + valid = [r for r in runs if r.get("infrastructure_errors", 0) == 0] + invalid = [r for r in runs if r.get("infrastructure_errors", 0) > 0] + valid_scores = sorted({r["overall"] for r in valid}) + run_lines = [ + f"Repeat {r['repeat']}: {r['overall']} (infra errors={r['infrastructure_errors']})" + for r in runs + ] + write( + "stability.tex", + f""" +\\evidence{{completed}}{{ +Five consecutive Brainy search-lane repeats on one embedded server session. +\\textbf{{Valid runs only}} (infra errors $=0$): {len(valid)}/{len(runs)}; scores: {', '.join(valid_scores) or 'none'}. +}} +\\begin{{itemize}} +""" + + "\n".join(f"\\item {line}" for line in run_lines) + + f""" +\\end{{itemize}} +Repeats with infra errors $>0$ ({len(invalid)} of {len(runs)}) are \\textbf{{invalid/incomplete}}: the harness omits errored tasks from the denominator, so aggregates such as 9/10 are \\emph{{not}} comparable to a clean 13/13 pin. +Use isolated \\texttt{{TestOpMemBenchmarkAgainstHTTPServer}} for merge-gate stability. +""", + ) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/opmem-start-letta-server.sh b/scripts/opmem-start-letta-server.sh new file mode 100755 index 00000000..6929e538 --- /dev/null +++ b/scripts/opmem-start-letta-server.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Start pinned Letta REST server for OpMem external lane (no Docker). +set -euo pipefail +export PATH="${HOME}/.local/bin:${PATH}" +LETTA_VERSION="${OPMEM_LETTA_SERVER_VERSION:-0.11.7}" +LETTA_CLIENT_VERSION="${OPMEM_LETTA_CLIENT_VERSION:-0.1.307}" +PORT="${LETTA_PORT:-8283}" +HOST="${LETTA_HOST:-127.0.0.1}" +LOG="${OPMEM_LETTA_SERVER_LOG:-/tmp/opmem-letta-server.log}" + +python3 -m pip install -q "letta==${LETTA_VERSION}" "letta-client==${LETTA_CLIENT_VERSION}" sqlite-vec + +if ! command -v letta >/dev/null; then + echo "letta CLI not on PATH; ensure ~/.local/bin is exported" >&2 + exit 1 +fi + +if curl -sf "http://${HOST}:${PORT}/v1/health" >/dev/null 2>&1; then + echo "Letta already listening on http://${HOST}:${PORT}" + exit 0 +fi + +nohup letta server --type rest --port "${PORT}" --host "${HOST}" --no-secure >"${LOG}" 2>&1 & +echo "Started Letta ${LETTA_VERSION} at http://${HOST}:${PORT} (log: ${LOG})" + +for _ in $(seq 1 30); do + if curl -sfL "http://${HOST}:${PORT}/v1/health" | grep -q ok; then + echo "Letta health OK" + exit 0 + fi + sleep 1 +done +echo "Letta failed to become healthy; see ${LOG}" >&2 +exit 1 diff --git a/scripts/opmem-verify-zep-env.sh b/scripts/opmem-verify-zep-env.sh new file mode 100755 index 00000000..d468450c --- /dev/null +++ b/scripts/opmem-verify-zep-env.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Verify Zep lane prerequisites without printing secrets. +set -euo pipefail +echo "CLOUD_AGENT_INJECTED_SECRET_NAMES:" +echo "${CLOUD_AGENT_INJECTED_SECRET_NAMES:-}" | tr ',' '\n' | sort +if [ -n "${ZEP_API_KEY:-}" ]; then + echo "ZEP_API_KEY: present (length=${#ZEP_API_KEY})" +else + echo "ZEP_API_KEY: MISSING" + exit 1 +fi +if ! echo "${CLOUD_AGENT_INJECTED_SECRET_NAMES:-}" | tr ',' '\n' | grep -qx 'ZEP_API_KEY'; then + echo "WARNING: ZEP_API_KEY not listed in CLOUD_AGENT_INJECTED_SECRET_NAMES" +fi +exit 0 diff --git a/scripts/run-opmem-manuscript.sh b/scripts/run-opmem-manuscript.sh new file mode 100755 index 00000000..4380429f --- /dev/null +++ b/scripts/run-opmem-manuscript.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Run OpMem manuscript experiments via embedded API (same stack as CI). +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +SHA="$(git rev-parse HEAD)" +OUT_DIR="${1:-docs/benchmarks/artifacts/opmem-manuscript-${SHA:0:8}}" +mkdir -p "$OUT_DIR" + +{ + echo "git_sha=$SHA" + echo "date_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "go test ./internal/api/ -run TestOpMemBenchmarkAgainstHTTPServer -count=1" + echo "OPMEM_MANUSCRIPT_OUT=$OUT_DIR go test ./internal/api/ -run TestOpMemPaperExperiments -count=1" +} > "$OUT_DIR/commands.log" + +go test ./internal/api/ -run TestOpMemBenchmarkAgainstHTTPServer -count=1 2>&1 | tee "$OUT_DIR/go-test-opmem.log" + +export OPMEM_MANUSCRIPT_OUT="$OUT_DIR" +go test ./internal/api/ -run TestOpMemPaperExperiments -count=1 2>&1 | tee "$OUT_DIR/go-test-paper.log" + +# Mem0 Platform counter-run when MEM0_API_KEY is set (optional; not required for CI). +if [[ -n "${MEM0_API_KEY:-}" ]]; then + echo "mem0 run: python3 evals/run_opmem.py --systems mem0 --json-out $OUT_DIR/opmem-mem0-rerun.json" >> "$OUT_DIR/commands.log" + python3 evals/run_opmem.py --systems mem0 --json-out "$OUT_DIR/opmem-mem0-rerun.json" 2>&1 | tee "$OUT_DIR/mem0-rerun.log" || true +fi + +echo "Artifacts written to $OUT_DIR"