docs(opmem): formatted manuscript and SHA 20533f2 experiment pin

.env.example

@@ -15,6 +15,14 @@ BRAINY_WORKER_POLL_INTERVAL=2s
 #
 # Same-pin Mem0 Platform evals (evals/run_competitor_benchmark.py, --systems mem0):
 # MEM0_API_KEY=[REDACTED]
+#
+# OpMem external lanes (evals/run_opmem_external_live.py; see docs/research/opmem/external-lanes-setup.md):
+# OPENAI_API_KEY=          # mem0-oss, letta, langmem (pinned gpt-4.1-mini + text-embedding-3-small)
+# ZEP_API_KEY=             # Zep Cloud free tier only ($0 paid ceiling)
+# LETTA_BASE_URL=http://127.0.0.1:8283
+# LETTA_APP_SERVER_TOKEN=  # optional for local letta server --no-secure
+# DATABASE_URL=postgresql://opmem:opmem@127.0.0.1:5432/opmem_langmem
+# OPMEM_QDRANT_ROOT=/tmp/opmem-qdrant
 
 # HTTP boundary (API). Defaults: 5 MiB body, 10s header / 30s read / provider+60s write / 120s idle.
 # BRAINY_MAX_BODY_BYTES=5242880

docs/benchmarks/artifacts/opmem-external-lanes/PHASE2_SUMMARY.md

@@ -0,0 +1,26 @@
+# OpMem external lanes — execution summary
+
+**Protocol:** 13 tasks / 47 steps (`fixtures/opmem/`).
+
+## Full comparison
+
+| Lane | Run ID | Score | Infra errors | SDK / infra |
+| --- | --- | ---: | ---: | --- |
+| mem0-oss | `opmem-phase2-full-mem0` | **9/13** | 0 | `mem0ai` 2.1.0, local Qdrant path |
+| langmem | `opmem-phase2-full1` | **8/13** | 0 | Postgres + pgvector |
+| letta | `opmem-phase2-letta-full` | **5/13** | 0 | `letta==0.11.7` |
+| zep | `opmem-phase2-zep-full` | **3/9\*** | 0 | `zep-cloud==3.28.0`, free tier, $0 paid |
+
+**\* Zep (publication line):** Score **3/9** counts only tasks where forget is in scope for the lane. **Not supported** (listed separately, not counted as failures): `iso03_forget_isolated`, `sup01_basic_forget`, `sup02_targeted_forget`, `sup03_durable_forget` — the `zep-cloud` 3.28 thread/graph adapter has no delete path for OpMem `forget`. **Raw harness** on the unchanged artifact `live-full-opmem-phase2-zep-full.json` remains **3/13**. See `zep-score-interpretation-opmem-phase2-zep-full.json`.
+
+### Zep supported-task breakdown (`opmem-phase2-zep-full`)
+
+| Status | Tasks |
+| --- | --- |
+| **Passed (3)** | `cor02_correction_stickiness`, `cor03_revised_retrievable`, `dup01_idempotent_remember` |
+| **Failed, scored (6)** | `cor01_basic_revision`, `iso01_subject_isolation`, `iso02_tenant_isolation`, `upd01_stale_fact`, `upd02_preference_change`, `upd03_state_supersession` |
+| **Not supported — forget (4)** | `iso03_forget_isolated`, `sup01_basic_forget`, `sup02_targeted_forget`, `sup03_durable_forget` |
+
+## Spend
+
+Payload worst-case OpenAI ~$0.15 vs $3 ceiling (`cost-worksheet-payload-budget.json`). Zep lane $0 paid.

docs/benchmarks/artifacts/opmem-external-lanes/commands.log

@@ -0,0 +1,39 @@
+
+# 2026-09-19T08:28:55Z run_id=opmem-phase2-smoke1
+r u n _ o p m e m _ e x t e r n a l _ l i v e . p y   - - s m o k e   - - l a n e s   m e m 0 - o s s , l a n g m e m , z e p , l e t t a   - - r u n - i d   o p m e m - p h a s e 2 - s m o k e 1
+
+# 2026-09-19T08:29:18Z run_id=opmem-phase2-smoke2
+r u n _ o p m e m _ e x t e r n a l _ l i v e . p y   - - s m o k e   - - l a n e s   m e m 0 - o s s , l a n g m e m   - - r u n - i d   o p m e m - p h a s e 2 - s m o k e 2
+
+# 2026-09-19T08:30:32Z run_id=opmem-phase2-full1
+r u n _ o p m e m _ e x t e r n a l _ l i v e . p y   - - f u l l   - - l a n e s   m e m 0 - o s s , l a n g m e m   - - r u n - i d   o p m e m - p h a s e 2 - f u l l 1   - - s k i p - p r e f l i g h t
+
+# 2026-09-19T08:31:22Z run_id=opmem-phase2-full-mem0
+r u n _ o p m e m _ e x t e r n a l _ l i v e . p y   - - f u l l   - - l a n e s   m e m 0 - o s s   - - r u n - i d   o p m e m - p h a s e 2 - f u l l - m e m 0   - - s k i p - p r e f l i g h t
+
+# 2026-09-21T17:43:52Z run_id=opmem-phase2-letta-smoke
+r u n _ o p m e m _ e x t e r n a l _ l i v e . p y   - - s m o k e   - - l a n e s   l e t t a   - - r u n - i d   o p m e m - p h a s e 2 - l e t t a - s m o k e
+
+# 2026-09-21T17:43:56Z run_id=opmem-phase2-zep-smoke
+r u n _ o p m e m _ e x t e r n a l _ l i v e . p y   - - s m o k e   - - l a n e s   z e p   - - r u n - i d   o p m e m - p h a s e 2 - z e p - s m o k e
+
+# 2026-09-21T17:44:03Z run_id=opmem-phase2-letta-full
+r u n _ o p m e m _ e x t e r n a l _ l i v e . p y   - - f u l l   - - l a n e s   l e t t a   - - r u n - i d   o p m e m - p h a s e 2 - l e t t a - f u l l
+
+# 2026-09-23T17:09:16Z run_id=opmem-phase2-zep-smoke-20260923
+r u n _ o p m e m _ e x t e r n a l _ l i v e . p y   - - s m o k e   - - l a n e s   z e p   - - r u n - i d   o p m e m - p h a s e 2 - z e p - s m o k e - 2 0 2 6 0 9 2 3
+
+# 2026-09-23T17:09:18Z run_id=opmem-phase2-zep-full
+r u n _ o p m e m _ e x t e r n a l _ l i v e . p y   - - f u l l   - - l a n e s   z e p   - - r u n - i d   o p m e m - p h a s e 2 - z e p - f u l l
+
+# 2026-09-23T17:55:34Z run_id=opmem-phase2-zep-smoke-20260923b
+r u n _ o p m e m _ e x t e r n a l _ l i v e . p y   - - s m o k e   - - l a n e s   z e p   - - r u n - i d   o p m e m - p h a s e 2 - z e p - s m o k e - 2 0 2 6 0 9 2 3 b
+
+# 2026-09-23T17:56:27Z run_id=opmem-phase2-zep-smoke-20260923c
+r u n _ o p m e m _ e x t e r n a l _ l i v e . p y   - - s m o k e   - - l a n e s   z e p   - - r u n - i d   o p m e m - p h a s e 2 - z e p - s m o k e - 2 0 2 6 0 9 2 3 c
+
+# 2026-09-23T17:56:54Z run_id=opmem-phase2-zep-smoke-final
+r u n _ o p m e m _ e x t e r n a l _ l i v e . p y   - - s m o k e   - - l a n e s   z e p   - - r u n - i d   o p m e m - p h a s e 2 - z e p - s m o k e - f i n a l
+
+# 2026-09-23T17:58:04Z run_id=opmem-phase2-zep-full
+r u n _ o p m e m _ e x t e r n a l _ l i v e . p y   - - f u l l   - - l a n e s   z e p   - - r u n - i d   o p m e m - p h a s e 2 - z e p - f u l l

docs/benchmarks/artifacts/opmem-external-lanes/cost-worksheet-dry-run.json

@@ -0,0 +1,151 @@
+{
+  "generated_utc": "2026-09-19T08:29:00Z",
+  "task_count": 13,
+  "step_count": 47,
+  "rates": {
+    "gpt-4.1-mini-2025-04-14": {
+      "input_per_m": 0.4,
+      "output_per_m": 1.6
+    },
+    "text-embedding-3-small": {
+      "per_m": 0.02
+    }
+  },
+  "pinned_models": {
+    "chat": "gpt-4.1-mini-2025-04-14",
+    "embed": "text-embedding-3-small"
+  },
+  "ceilings_usd": {
+    "mem0-oss": 1.0,
+    "letta": 1.0,
+    "langmem": 1.0,
+    "zep": 0.0,
+    "total": 3.0
+  },
+  "lanes": [
+    {
+      "lane": "langmem",
+      "operations": {
+        "remember": 21,
+        "recall": 19,
+        "revise": 3,
+        "forget": 4,
+        "reset": 13,
+        "http_calls": 47
+      },
+      "tokens": {
+        "chat_model": "gpt-4.1-mini-2025-04-14",
+        "embed_model": "text-embedding-3-small",
+        "chat_input_tokens": 409,
+        "chat_output_tokens": 66,
+        "embed_tokens": 191,
+        "estimated_usd": 0.000273
+      },
+      "product_constraints": [],
+      "credentials_present": false,
+      "missing_credentials": [
+        "DATABASE_URL"
+      ],
+      "projected_usd": 0.000273,
+      "ceiling_usd": 1.0,
+      "within_ceiling": true,
+      "live_run_allowed": false,
+      "live_run_note": "Phase 1 dry-run only; enable after credentials present and measured estimate confirmed under ceiling."
+    },
+    {
+      "lane": "letta",
+      "operations": {
+        "remember": 21,
+        "recall": 19,
+        "revise": 3,
+        "forget": 4,
+        "reset": 13,
+        "http_calls": 47
+      },
+      "tokens": {
+        "chat_model": "gpt-4.1-mini-2025-04-14",
+        "embed_model": "text-embedding-3-small",
+        "chat_input_tokens": 409,
+        "chat_output_tokens": 66,
+        "embed_tokens": 191,
+        "estimated_usd": 0.000273
+      },
+      "product_constraints": [],
+      "credentials_present": false,
+      "missing_credentials": [
+        "LETTA_APP_SERVER_TOKEN"
+      ],
+      "projected_usd": 0.000273,
+      "ceiling_usd": 1.0,
+      "within_ceiling": true,
+      "live_run_allowed": false,
+      "live_run_note": "Phase 1 dry-run only; enable after credentials present and measured estimate confirmed under ceiling."
+    },
+    {
+      "lane": "mem0-oss",
+      "operations": {
+        "remember": 21,
+        "recall": 19,
+        "revise": 3,
+        "forget": 4,
+        "reset": 13,
+        "http_calls": 47
+      },
+      "tokens": {
+        "chat_model": "gpt-4.1-mini-2025-04-14",
+        "embed_model": "text-embedding-3-small",
+        "chat_input_tokens": 409,
+        "chat_output_tokens": 66,
+        "embed_tokens": 191,
+        "estimated_usd": 0.000273
+      },
+      "product_constraints": [],
+      "credentials_present": true,
+      "missing_credentials": [],
+      "projected_usd": 0.000273,
+      "ceiling_usd": 1.0,
+      "within_ceiling": true,
+      "live_run_allowed": false,
+      "live_run_note": "Phase 1 dry-run only; enable after credentials present and measured estimate confirmed under ceiling."
+    },
+    {
+      "lane": "zep",
+      "operations": {
+        "remember": 21,
+        "recall": 19,
+        "revise": 3,
+        "forget": 4,
+        "reset": 13,
+        "http_calls": 47
+      },
+      "tokens": {
+        "chat_model": "gpt-4.1-mini-2025-04-14",
+        "embed_model": "text-embedding-3-small",
+        "chat_input_tokens": 0,
+        "chat_output_tokens": 0,
+        "embed_tokens": 0,
+        "estimated_usd": 0.0
+      },
+      "product_constraints": [
+        "lane zep cannot pin chat model gpt-4.1-mini-2025-04-14",
+        "lane zep cannot pin embed model text-embedding-3-small",
+        "Zep Cloud memory path does not accept OpenAI chat/embed model pins; stop before substituting \u2014 record scores only under Zep-native models."
+      ],
+      "credentials_present": false,
+      "missing_credentials": [
+        "ZEP_API_KEY"
+      ],
+      "projected_usd": 0.0,
+      "ceiling_usd": 0.0,
+      "within_ceiling": true,
+      "live_run_allowed": false,
+      "live_run_note": "Phase 1 dry-run only; enable after credentials present and measured estimate confirmed under ceiling."
+    },
+    {
+      "lane": "_total_openai_estimate",
+      "projected_usd": 0.000819,
+      "ceiling_usd": 3.0,
+      "within_ceiling": true
+    }
+  ]
+}

docs/benchmarks/artifacts/opmem-external-lanes/cost-worksheet-payload-budget.json

@@ -0,0 +1,87 @@
+{
+  "mode": "payload_budget",
+  "task_count": 13,
+  "step_count": 47,
+  "pinned_chat_model": "gpt-4.1-mini-2025-04-14",
+  "pinned_embed_model": "text-embedding-3-small",
+  "rate_source_url": "https://openai.com/api/pricing/",
+  "rate_source_note": "gpt-4.1-mini-2025-04-14: $0.40/1M input, $1.60/1M output; text-embedding-3-small: $0.02/1M tokens",
+  "ceilings_usd": {
+    "mem0-oss": 1.0,
+    "letta": 1.0,
+    "langmem": 1.0,
+    "zep": 0.0,
+    "total": 3.0
+  },
+  "lanes": [
+    {
+      "lane": "langmem",
+      "worst_case_usd": 0.017537,
+      "token_totals": {
+        "chat_input_tokens": 1447,
+        "chat_output_tokens": 9600,
+        "embed_tokens": 203
+      },
+      "line_count": 47,
+      "product_constraints": [],
+      "retry_margin": 1.1,
+      "projected_worst_case_usd": 0.017537,
+      "ceiling_usd": 1.0,
+      "within_ceiling": true
+    },
+    {
+      "lane": "letta",
+      "worst_case_usd": 0.013526,
+      "token_totals": {
+        "chat_input_tokens": 1930,
+        "chat_output_tokens": 7200,
+        "embed_tokens": 203
+      },
+      "line_count": 47,
+      "product_constraints": [],
+      "retry_margin": 1.1,
+      "projected_worst_case_usd": 0.013526,
+      "ceiling_usd": 1.0,
+      "within_ceiling": true
+    },
+    {
+      "lane": "mem0-oss",
+      "worst_case_usd": 0.11353,
+      "token_totals": {
+        "chat_input_tokens": 186010,
+        "chat_output_tokens": 18000,
+        "embed_tokens": 239
+      },
+      "line_count": 66,
+      "product_constraints": [],
+      "retry_margin": 1.1,
+      "projected_worst_case_usd": 0.11353,
+      "ceiling_usd": 1.0,
+      "within_ceiling": true
+    },
+    {
+      "lane": "zep",
+      "worst_case_usd": 0.0,
+      "token_totals": {
+        "chat_input_tokens": 1122,
+        "chat_output_tokens": 0,
+        "embed_tokens": 0
+      },
+      "line_count": 47,
+      "product_constraints": [
+        "Zep Cloud native extraction; OpenAI pins not applied; $0 paid ceiling."
+      ],
+      "retry_margin": 1.1,
+      "projected_worst_case_usd": 0.0,
+      "ceiling_usd": 0.0,
+      "within_ceiling": true
+    },
+    {
+      "lane": "_total_openai_worst_case",
+      "projected_worst_case_usd": 0.144592,
+      "ceiling_usd": 3.0,
+      "within_ceiling": true
+    }
+  ],
+  "methodology": "tiktoken cl100k_base on serialized request bodies incl. mem0 ADDITIVE_EXTRACTION_PROMPT; conservative max output caps; 1.10 retry margin; Zep excluded from OpenAI totals."
+}