diff --git a/.well-known/agent-memory.json b/.well-known/agent-memory.json index 7dc1687..dee7bbc 100644 --- a/.well-known/agent-memory.json +++ b/.well-known/agent-memory.json @@ -1,20 +1,19 @@ { "schema_version": "memory-vault-discovery/v1", "name": "Memory Vault", - "implementation_version": "0.28.0-alpha.0.37", + "implementation_version": "0.28.0-alpha.0.40", "implementation_status": "alpha_preview", "license": "Apache-2.0", "repository": "https://github.com/qh-work/memory-vault-sync", - "release": "https://github.com/qh-work/memory-vault-sync/releases/tag/v0.28.0-alpha.0.37", - "release_publication_verified": true, + "release": "https://github.com/qh-work/memory-vault-sync/releases/tag/v0.28.0-alpha.0.40", "release_assets": { - "client": "https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.37/memory-vault-client-v0.28.0-alpha.0.37.zip", - "protocol": "https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.37/memory-vault-protocol-v0.28.0-alpha.0.37.zip", - "review": "https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.37/memory-vault-review-v0.28.0-alpha.0.37.zip", - "network_trial": "https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.37/memory-vault-network-test-v0.28.0-alpha.0.37.zip", - "checksums": "https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.37/SHA256SUMS" + "client": "https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.40/memory-vault-client-v0.28.0-alpha.0.40.zip", + "protocol": "https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.40/memory-vault-protocol-v0.28.0-alpha.0.40.zip", + "review": "https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.40/memory-vault-review-v0.28.0-alpha.0.40.zip", + "network_trial": "https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.40/memory-vault-network-test-v0.28.0-alpha.0.40.zip", + "checksums": "https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.40/SHA256SUMS" }, - "previous_release": "https://github.com/qh-work/memory-vault-sync/releases/tag/v0.28.0-alpha.0.36", + "previous_release": "https://github.com/qh-work/memory-vault-sync/releases/tag/v0.28.0-alpha.0.37", "relative_links_base": "this discovery document", "start_here": "../AI_START_HERE.md", "specification": "../PROTOCOL.md", @@ -46,7 +45,7 @@ "automatic_sync_default": false, "readme_is_installation": false, "runtime_tests_run_for_release": true, - "runtime_test_scope": "Twelve checks passed using the extracted alpha.0.37 client: automatic native cold-memory receipt return, lost-reply restart, independent original-owner confirmation, cross-client completion, local repeat history, retained revocation and expired-use refusal, remote mailbox reception and saved receipt recovery. Exact-source cloud regression passed 1238 tests across 139 modules, with all 316 reported source hashes matching fe943c6466a6ef459ccb0d03e309b207b3b0fb95. All three Python 3.10 suites and cross-platform conformance passed. All eight public assets were downloaded through the authenticated GitHub client and matched. These results do not establish independent external adoption, global reliability or thousand-agent capacity.", + "runtime_test_scope": "63 local targeted cases passed; isolated native 48-message/2-per-second trial completed 39 -> 48 chains. Full native message-replica source-loss/COPY/READ and finite-work refusal are verified. See release CI and capacity report for final exact-source validation. CPU/storage ceilings and independent-host/WAN capacity remain unverified.", "runtime_test_report": "../docs/RELEASE.md", "full_runtime_test_suite_run": true, "runtime_parity_verified": false, @@ -124,5 +123,6 @@ "runnable_package": "review", "source_owned_reads_required": true, "external_model_trial_verified": false - } + }, + "publication_verification": "Use the published GitHub release record and exact-source release manifest; this static discovery document is not live publication evidence." } diff --git a/AI_START_HERE.md b/AI_START_HERE.md index d7f880e..008fe1a 100644 --- a/AI_START_HERE.md +++ b/AI_START_HERE.md @@ -1,5 +1,7 @@ # Memory Vault: connect, remember, exchange, continue +Alpha.0.40 adds bounded HTTP admission, WAL read concurrency, separately approved resource distribution and native full message-replica COPY/READ recovery. The isolated 48-Memory/2-per-second workload completes 39 → 48 chains; this proves authorized quota use, not a CPU/SQLite ceiling. Replica cold reads remain slow and stop at the existing finite work allowance. See [capacity evidence and limits](docs/CAPACITY_HANDOFF.md). + Native TypeScript recipients can now retain an explicitly authorized receipt destination for a cold-mailbox message. Ordinary Agent `receive` returns its actual saved receipt, reusing the exact upload after a lost reply or restart. Python and TypeScript share jobs and completed history; revocation stops retries and expired uncertain uploads stop for reconciliation. See [native mailbox receipt returns](clients/typescript/network/README.md). Native TypeScript recipients can now register an authorized remote mailbox and use ordinary Agent `receive` to recover messages and selected memories after old delivery records disappear. Current READ permission, complete original authority and durable inbox checks remain required. Python and TypeScript share receiver configuration, status history and saved inbox evidence. See [native mailbox recovery](clients/typescript/network/README.md). @@ -10,7 +12,7 @@ Sources can now republish an original ACK location through another independently Python and native TypeScript senders can now locate an original-source ACK receipt through an explicitly selected directory using `connect/recover_discovered_receipt`. A separate authorized source read must match the actual recipient receipt, published custody and original send. New opt-in `receipt-index` sources fund the whole workflow; existing signed limits stay unchanged. See [directory receipt recovery](docs/OPEN_ACK_DIRECTORY.md#agent-directory-recovery-development-after-alpha032). -Native TypeScript senders can now recover an independent original-source ACK receipt through `connect/recover_receipt`, or by repeating an unchanged original `send` with an already retained preparation while the delivery node is offline. Python and native clients share saved receipts and original-root revocation history, including older installations. Preparation and replica reception still use Python; native ordinary mailbox reception and explicit saved receipt return are available. See [native ACK confirmation](docs/OPEN_ACK_PROVISIONING.md#native-agent-receipt-recovery-source-after-alpha031). +Native TypeScript senders can now recover an independent original-source ACK receipt through `connect/recover_receipt`, or by repeating an unchanged original `send` with an already retained preparation while the delivery node is offline. Python and native clients share saved receipts and original-root revocation history, including older installations. Preparation and native ACK/root/feed replica reception still use Python; native ordinary mailbox reception and explicit saved receipt return are available. See [native ACK confirmation](docs/OPEN_ACK_PROVISIONING.md#native-agent-receipt-recovery-source-after-alpha031). Python maintainers can retain exact authorized mailbox copies in a [finite persistent worker](docs/OPEN_ACK_RECOVERY.md). Senders can retain selected ACK replicas; repeating an unchanged original `send` confirms the recipient receipt after both original delivery and ACK nodes stop. Exact requests and signed refusal history survive restart. Destination selection and independent permissions remain explicit. See [ACK replica confirmation](docs/OPEN_ACK_PROVISIONING.md). diff --git a/README.md b/README.md index 8c92fc3..b392414 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Memory Vault — an agent communication and memory network +Alpha.0.40 adds bounded HTTP admission, WAL read concurrency, separately approved resource distribution and native full message-replica COPY/READ recovery. The isolated 48-Memory/2-per-second workload completes 39 → 48 chains; this proves authorized quota use, not a CPU/SQLite ceiling. Replica cold reads remain slow and stop at the existing finite work allowance. See [capacity evidence and limits](docs/CAPACITY_HANDOFF.md). + Native TypeScript recipients can now retain an explicitly authorized receipt destination for a cold-mailbox message. Ordinary Agent `receive` returns its actual saved receipt, reusing the exact upload after a lost reply or restart. Python and TypeScript share jobs and completed history; revocation stops retries and expired uncertain uploads stop for reconciliation. See [native mailbox receipt returns](clients/typescript/network/README.md). Native TypeScript recipients can now register an authorized remote mailbox and use ordinary Agent `receive` to recover messages and selected memories after old delivery records disappear. Current READ permission, complete original authority and durable inbox checks remain required. Python and TypeScript share receiver configuration, status history and saved inbox evidence. See [native mailbox recovery](clients/typescript/network/README.md). @@ -10,7 +12,7 @@ Sources can now republish an original ACK location through another independently Python and native TypeScript senders can now locate an original-source ACK receipt through an explicitly selected directory using `connect/recover_discovered_receipt`. A separate authorized source read must match the actual recipient receipt, published custody and original send. New opt-in `receipt-index` sources fund the whole workflow; existing signed limits stay unchanged. See [directory receipt recovery](docs/OPEN_ACK_DIRECTORY.md#agent-directory-recovery-development-after-alpha032). -Native TypeScript senders can now recover an independent original-source ACK receipt through `connect/recover_receipt`, or by repeating an unchanged original `send` with an already retained preparation while the delivery node is offline. Python and native clients share saved receipts and original-root revocation history, including older installations. Preparation and replica reception still use Python; native ordinary mailbox reception and explicit saved receipt return are available. See [native ACK confirmation](docs/OPEN_ACK_PROVISIONING.md#native-agent-receipt-recovery-source-after-alpha031). +Native TypeScript senders can now recover an independent original-source ACK receipt through `connect/recover_receipt`, or by repeating an unchanged original `send` with an already retained preparation while the delivery node is offline. Python and native clients share saved receipts and original-root revocation history, including older installations. Preparation and native ACK/root/feed replica reception still use Python; native ordinary mailbox reception and explicit saved receipt return are available. See [native ACK confirmation](docs/OPEN_ACK_PROVISIONING.md#native-agent-receipt-recovery-source-after-alpha031). Python maintainers can retain exact authorized mailbox copies in a [finite persistent worker](docs/OPEN_ACK_RECOVERY.md). Senders can retain selected ACK replicas; repeating an unchanged original `send` confirms the recipient receipt after both original delivery and ACK nodes stop. Exact requests and signed refusal history survive restart. Destination selection and independent permissions remain explicit. See [ACK replica confirmation](docs/OPEN_ACK_PROVISIONING.md). @@ -74,7 +76,7 @@ provenance and exchange contract in their preferred language and storage. ## Join the open network: encrypted messages and shared memories **[Open-network quickstart](docs/OPEN_NETWORK_QUICKSTART.md)** is the entry for -agents and node operators. Python and native TypeScript in **0.28.0-alpha.0.37** support +agents and node operators. Python and native TypeScript in **0.28.0-alpha.0.40** support encrypted `send`, `receive` and local message reads after explicit first-contact approval. An agent can send text or select original memories for sharing. The recipient saves accepted content locally before signing a saved receipt. @@ -178,15 +180,15 @@ The pre-existing MCP memory interface remains for existing users. ## Download the current preview -Use the assets for **[v0.28.0-alpha.0.37](https://github.com/qh-work/memory-vault-sync/releases/tag/v0.28.0-alpha.0.37)**. +Use the assets for **[v0.28.0-alpha.0.40](https://github.com/qh-work/memory-vault-sync/releases/tag/v0.28.0-alpha.0.40)**. The [open-network quickstart](docs/OPEN_NETWORK_QUICKSTART.md) works from the full client archive without installing a plugin. The release manifest identifies its exact source; historical test reports do not validate this new preview. -- **[Protocol-only package](https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.37/memory-vault-protocol-v0.28.0-alpha.0.37.zip):** specification, schemas and synthetic examples; no executable. -- **[Full plugin package](https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.37/memory-vault-client-v0.28.0-alpha.0.37.zip):** local memory, opt-in capture, optional encrypted network, recovery and a local marketplace catalog. -- **[Independent review kit](https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.37/memory-vault-review-v0.28.0-alpha.0.37.zip):** public source and synthetic tests; nothing runs automatically. -- **[Synthetic network trial](https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.37/memory-vault-network-test-v0.28.0-alpha.0.37.zip):** retained private-profile endpoint template, no Docker or plugin; operator-provisioned service, with service trust unconfigured in this release. +- **[Protocol-only package](https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.40/memory-vault-protocol-v0.28.0-alpha.0.40.zip):** specification, schemas and synthetic examples; no executable. +- **[Full plugin package](https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.40/memory-vault-client-v0.28.0-alpha.0.40.zip):** local memory, opt-in capture, optional encrypted network, recovery and a local marketplace catalog. +- **[Independent review kit](https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.40/memory-vault-review-v0.28.0-alpha.0.40.zip):** public source and synthetic tests; nothing runs automatically. +- **[Synthetic network trial](https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.40/memory-vault-network-test-v0.28.0-alpha.0.40.zip):** retained private-profile endpoint template, no Docker or plugin; operator-provisioned service, with service trust unconfigured in this release. - **Core source:** [`memory_vault.py`](memory_vault.py); use the full client or review package for the Experience module and complete runtime. Alpha.3 makes current records deterministically rank before superseded/resolved diff --git a/clients/typescript/network/README.md b/clients/typescript/network/README.md index b8ed1df..ef1d15b 100644 --- a/clients/typescript/network/README.md +++ b/clients/typescript/network/README.md @@ -1,5 +1,7 @@ # Independent network-v1 endpoint candidate +Alpha.0.40 adds bounded HTTP admission, WAL read concurrency, separately approved resource distribution and native full message-replica COPY/READ recovery. The isolated 48-Memory/2-per-second workload completes 39 → 48 chains; this proves authorized quota use, not a CPU/SQLite ceiling. Replica cold reads remain slow and stop at the existing finite work allowance. See [capacity evidence and limits](../../../docs/CAPACITY_HANDOFF.md). + This optional Node TypeScript package implements the existing Memory Vault network-v1 envelope without invoking Python. It is separate from the zero dependency HTTP SDK in the parent directory. It uses `jose` 6.2.10 for X25519 @@ -332,3 +334,55 @@ history. Re-registering a completed selection can reuse authenticated local history without another upload. Direct `return_mailbox_receipt` supports the same message and source selection. Native replica inbox support remains unfinished. These additions are development source after the frozen alpha.0.36 candidate. + +### Receiver-issued mailbox authorization (development after alpha.0.37) + +After registering an already provisioned mailbox and explicitly approving the +sender's contact request, a native recipient can export the existing mailbox +admission invitation with ordinary `connect`: + +```ts +await agent.handle({op: 'connect', invitation: { + schema_version: 'memory-vault-open-mailbox-connect/v1', action: 'authorize', + receiver_id, contact_request_ref, expires_at, status_revision, status_until +}}); +``` + +Choose the expiry windows within the original contact and mailbox permissions. +The owner supplies and coordinates `status_revision`; a new selection must not +reuse or roll back an issued revision. Authorization binds the exact approved +contact, sender and recipient keys, mailbox slot, selected source and existing +storage resources. It does not reserve new storage or create ACK-return grants. +The native client signs with the recipient's own existing identity and persists +the exact destination and status originals before returning them. + +The response supplies a base64 `authorization_chunk` of at most 3,072 bytes, +`authorization_sha256`, `total_bytes`, `offset`, and `next_cursor`. Repeat the +same invitation with `cursor: next_cursor` until it is null, join decoded chunks +in order, and verify the complete byte count and SHA-256 before parsing the +sender's `retain` authorization. A cursor cannot be reused for different bytes. +`inspect` with `receiver_id` similarly pages `configuration_chunk` with +`configuration_sha256`; both operations reauthenticate the retained receiver +and perform no HTTP requests. + +Python and native clients share the bounded destination journal. Restarting or +switching clients returns the same signed originals; a conflicting request or +reused older revision fails. Both clients reauthenticate retained receive-status +observations inside the signing transaction. Revoked slot, parent permission or +storage-resource authority blocks both cached output and fresh issuance even +after that status expires; higher remembered document floors also remain in force. A completed local authorization does not prove that +the source is currently available or that a message was delivered. The sender +must still prepare and admit its exact ciphertext under current source checks; +source provisioning currently uses Python. The native sender supports the same +`prepare`, `admit`, and combined `retain` invitations in development after +alpha.0.38. It freezes the original outbox ciphertext, disclosure and attempt in +the shared Python/native journal before HTTP. A lost reply or restart reuses the +same request, and changed selections fail instead of silently renewing grants. + +Pass `ack_request_id` to `prepare` or `retain` only when selecting an existing +independently prepared receipt destination. Its six complete original controls +and statuses are authenticated and carried in the mailbox member. The node +still performs current permission, storage and resource checks before admission. +`retained_at_mailbox` means the source signed actual custody; +`recipient_acknowledged` remains false until a separate saved receipt is verified. +Neither local preparation nor source custody means the recipient saved a memory. diff --git a/clients/typescript/network/open-ack-status.ts b/clients/typescript/network/open-ack-status.ts index 966d3ea..e4ce1d6 100644 --- a/clients/typescript/network/open-ack-status.ts +++ b/clients/typescript/network/open-ack-status.ts @@ -14,9 +14,9 @@ function fail(code:string):never{throw new NetworkError(code);} * Transactions are synchronous and finish before any network await. */ export class OriginalAckStatusJournal{ readonly root:Buffer;readonly key:string;private readonly participant:OpenParticipant; - constructor(participant:OpenParticipant,root:unknown){ + constructor(participant:OpenParticipant,root:unknown,originalSource=true){ this.participant=participant; - this.root=bytes(root);this.key=hash(Buffer.concat([Buffer.from('original-source\0'),this.root])); + this.root=bytes(root);this.key=hash(originalSource?Buffer.concat([Buffer.from('original-source\0'),this.root]):this.root); participant.providerStorage(db=>db.exec(` CREATE TABLE IF NOT EXISTS open_ack_replica_roots(root_id TEXT PRIMARY KEY,root BLOB NOT NULL,blocked INTEGER NOT NULL); CREATE TABLE IF NOT EXISTS open_ack_replica_statuses(root_id TEXT NOT NULL,ref_id TEXT NOT NULL,issuer TEXT NOT NULL,raw BLOB NOT NULL,ref BLOB NOT NULL,PRIMARY KEY(root_id,ref_id));`)); @@ -79,3 +79,7 @@ export class OriginalAckStatusJournal{ if(full)fail('repair_status_history_capacity'); } } +/** Exact Python replica domain; selection removal cannot erase observations. */ +export class ReplicaStatusJournal extends OriginalAckStatusJournal{ + constructor(participant:OpenParticipant,root:unknown){super(participant,root,false);} +} diff --git a/clients/typescript/network/open-client.ts b/clients/typescript/network/open-client.ts index a3abe78..53b23f3 100644 --- a/clients/typescript/network/open-client.ts +++ b/clients/typescript/network/open-client.ts @@ -6,6 +6,7 @@ import {absolutePath,readPrivate,NetworkError} from './io.ts'; import {loadClient} from './client-config.ts'; import type {Client} from './client-config.ts'; import {OpenDeliveryClient} from './open-delivery-client.ts'; +import {MailboxSender,MAILBOX_SEND_ACTIONS} from './open-mailbox-send.ts'; import {RegisteredMailboxReceivers,MAILBOX_CONNECT_SCHEMA} from './open-mailbox-receivers.ts'; import {MailboxReceiptJobs,MAILBOX_RECEIPT_ACTIONS,returnMailboxReceipt} from './open-mailbox-receipts.ts'; import {OpenParticipant} from './open-participant.ts'; @@ -38,8 +39,10 @@ export class OpenNetworkClient{ private result(value:Record):Record{if(canonicalBytes(value).length>8192)fail('network_response_too_large');return value;} async connect(invitation?:unknown,requestId?:string):Promise>{ if(invitation!=null){ - if(typeof invitation==='object'&&!Array.isArray(invitation)&&(invitation as any).schema_version===MAILBOX_CONNECT_SCHEMA) - return this.result(new RegisteredMailboxReceivers(this.participant,this.encryption,this.delivery()).connect(invitation)); + if(typeof invitation==='object'&&!Array.isArray(invitation)&&(invitation as any).schema_version===MAILBOX_CONNECT_SCHEMA){ + const delivery=this.delivery(); + return this.result(MAILBOX_SEND_ACTIONS.has((invitation as any).action)?await new MailboxSender(this.participant,this.encryption).connect(invitation):await new RegisteredMailboxReceivers(this.participant,this.encryption,delivery).connect(invitation)); + } if(typeof invitation==='object'&&!Array.isArray(invitation)&&(invitation as any).schema_version===ACK_CONNECT_SCHEMA){ if(MAILBOX_RECEIPT_ACTIONS.has((invitation as any).action))return this.result((invitation as any).action==='return_mailbox_receipt'?await returnMailboxReceipt(this.participant,this.encryption,this.delivery(),invitation):new MailboxReceiptJobs(this.participant,this.encryption,this.delivery()).connect(invitation)); return this.result(await ((invitation as any).action==='return_receipt'?returnSavedReceipt:recoverReceipt)(this.participant,this.encryption,this.delivery(),invitation)); diff --git a/clients/typescript/network/open-contact-client.ts b/clients/typescript/network/open-contact-client.ts index bbcc9ad..2f79805 100644 --- a/clients/typescript/network/open-contact-client.ts +++ b/clients/typescript/network/open-contact-client.ts @@ -57,6 +57,13 @@ export class OpenContactClient{ if(!row||Buffer.from(row.body).equals(Buffer.from(LOCAL_RESERVATION)))throw new ContactError('contact_local_missing'); if(row.expires_at<=now())throw new ContactError('contact_expired');return document(row.body,24576); } + approvedIncomingOriginals(reference:string):Obj{ + const incoming=this.load('incoming',reference),decision=this.load('decision',reference).decision; + if(decision.payload.decision!=='approved')throw new ContactError('open_delivery_not_authorized'); + const grant=decision.payload.grant,docs={node:incoming.node,policy:incoming.policy,request:incoming.request, + decision,knock_lease:incoming.lease,grant,delivery_lease:grant.payload.resource_lease}; + return Object.fromEntries(Object.entries(docs).map(([name,value])=>[name,canonicalBytes(value)])); + } private savePolicy(reservationRef:string,leaseId:string,value:Obj,expires:number):void{ const raw=canonicalBytes(document(value,24576)); this.participant.contactStorage(db=>transaction(db,()=>{ diff --git a/clients/typescript/network/open-delivery-client.ts b/clients/typescript/network/open-delivery-client.ts index 8ddb1f3..d1e1b29 100644 --- a/clients/typescript/network/open-delivery-client.ts +++ b/clients/typescript/network/open-delivery-client.ts @@ -3,6 +3,8 @@ */ import {randomBytes} from 'node:crypto'; import {verifyMailboxInboxEvidence} from './open-repair-mailbox-inbox.ts'; +import {MailboxMessageReplicaRecoveryClient} from './open-mailbox-replica-client.ts'; +import type {MessageReplicaRecoverOptions} from './open-mailbox-replica-client.ts'; import {MailboxFeedRecoveryClient} from './open-mailbox-client.ts'; import type {MailboxRecoverOptions} from './open-mailbox-client.ts'; import {performance} from 'node:perf_hooks'; @@ -188,7 +190,7 @@ export class OpenDeliveryClient{ const checked=verifyDecision(session.decision,{request:session.request,policy:session.policy,lease:session.lease,node}); if(checked.decision!=='approved'||!checked.grant)fail('open_contact_approval_required');return {...session,node}; } - private async sendingSession(recipient:string,budget:DeliveryBudget):Promise{ + private async sendingSession(recipient:string,budget:DeliveryBudget,row?:Obj):Promise{ let sessions=this.contactSessions(true,recipient); if(!sessions.length){ const rows=this.db(db=>db.prepare("SELECT body FROM open_contact_local WHERE category='outgoing' AND expires_at>? ORDER BY expires_at DESC,reference LIMIT 128").all(now())) as Obj[]; @@ -208,8 +210,58 @@ export class OpenDeliveryClient{ }finally{budget.merge(child);}break; }sessions=this.contactSessions(true,recipient); } - for(const session of sessions){try{return await this.checkedSession(session,budget);}catch(error){if(!['contact_expired','contact_policy_mismatch'].includes((error as any)?.code))throw error;}} - fail('open_contact_approval_required'); + const eligible:Obj[]=[],unavailable:unknown[]=[]; + for(const session of sessions){try{const checked=await this.checkedSession(session,budget); + if(!row)return checked;eligible.push(checked); + }catch(error){const code=(error as any)?.code; + if(['contact_expired','contact_policy_mismatch'].includes(code))continue; + if(row&&code==='contact_resource_unresolved'){unavailable.push(error);continue;}throw error;}} + if(!eligible.length){if(unavailable.length)throw unavailable[0];fail('open_contact_approval_required');} + const frozen=await this.freezeDispatch(row!,eligible);return document(frozen.session,MAX_SESSION_BYTES) as Obj; + } + private static sessionScope(session:Obj):string{ + // Exact signed resource lease, identical to Python. Aliases share a quota. + return sha256(canonicalBytes(session.decision.payload.grant.payload.resource_lease)); + } + private async freezeDispatch(row:Obj,sessions:Obj[]):Promise{ + const keys=OpenDeliveryClient.keys(sessions[0]); + sessions=sessions.filter(session=>same(OpenDeliveryClient.keys(session),keys)); + const envelope=await createEnvelope(row.body,{signer:this.participant.identity, + sender_encryption_key:validateEncryptionIdentity(this.encryption), + recipient_signing_key:keys.recipient_signing_key,recipient_encryption_key:keys.recipient_encryption_key, + message_id:row.message_id,object_key:randomBytes(32).toString('hex'),created_at:row.created_at}); + const encoded=canonicalBytes(envelope,MAX_ENVELOPE_BYTES); + return this.db(db=>transaction(db,()=>{ + const prior=db.prepare('SELECT * FROM open_delivery_outbox WHERE request_id=?').get(row.request_id) as Obj|undefined; + if(!prior)fail('open_delivery_outbox_missing');if(prior.envelope)return prior; + const used=new Map(); + // Admission already limits the local outbox to 1024 records. + for(const saved of db.prepare('SELECT session,length(envelope) AS bytes FROM open_delivery_outbox WHERE session IS NOT NULL LIMIT 1024').all() as Obj[]){ + const scope=OpenDeliveryClient.sessionScope(document(saved.session,MAX_SESSION_BYTES) as Obj),count=used.get(scope)??{items:0,bytes:0}; + used.set(scope,{items:count.items+1,bytes:count.bytes+(saved.bytes||MAX_ENVELOPE_BYTES)}); + } + const available=sessions.map(session=>{const lease=session.decision.payload.grant.payload.resource_lease.payload, + scope=OpenDeliveryClient.sessionScope(session),count=used.get(scope)??{items:0,bytes:0}; + return {session,scope,lease,count}; + }).filter(v=>v.count.itemsencoded.length>session.decision.payload.grant.payload.resource_lease.payload.max_bytes))fail('open_delivery_approval_bytes_insufficient'); + fail('open_delivery_approval_capacity');} + available.sort((a,b)=>a.count.items/a.lease.max_items-b.count.items/b.lease.max_items|| + a.count.bytes/a.lease.max_bytes-b.count.bytes/b.lease.max_bytes||(a.scopeb.scope?1:0)); + const proof=canonicalBytes(document(available[0].session,MAX_SESSION_BYTES)),size=this.size(db,'open_delivery_outbox'); + if(size.bytes-MAX_ENVELOPE_BYTES-MAX_SESSION_BYTES+encoded.length+proof.length>MAX_LOCAL_BYTES)fail('network_outbox_capacity'); + db.prepare('UPDATE open_delivery_outbox SET envelope=?,session=? WHERE request_id=?').run(encoded,proof,row.request_id); + return db.prepare('SELECT * FROM open_delivery_outbox WHERE request_id=?').get(row.request_id) as Obj; + })); + } + private receivingSessions():Obj[]{ + const sessions=this.contactSessions(false),row=this.db(db=>db.prepare("SELECT body FROM open_delivery_client_state WHERE key='receive_session_cursor'").get()) as Obj|undefined; + if(row){const previous=raw(row.body).toString('ascii'),index=sessions.findIndex(session=>OpenDeliveryClient.sessionScope(session)===previous); + if(index>=0)return [...sessions.slice(index+1),...sessions.slice(0,index+1)];} + return sessions; + } + private advanceReceivingSession(session:Obj):void{ + this.db(db=>db.prepare("INSERT INTO open_delivery_client_state VALUES('receive_session_cursor',?) ON CONFLICT(key) DO UPDATE SET body=excluded.body").run(Buffer.from(OpenDeliveryClient.sessionScope(session),'ascii'))); } private uploadIntent(row:Obj,session:Obj):Obj{ if(row.intent){const prior=document(row.intent,MAX_INTENT_BYTES) as Obj;if(prior.payload.expires_at>now())return prior;} @@ -281,7 +333,7 @@ export class OpenDeliveryClient{ } private async sendInternal(requestId:string,recipient:string,hash:string,text:string,memoryIds:string[],recoverAck?:AckRecovery):Promise{ let row=this.prepareOutbox(requestId,recipient,hash,text,memoryIds);const budget=new DeliveryBudget();let session:Obj; - if(!row.session){session=await this.sendingSession(recipient,budget);row=await this.encryptOutbox(row,session);} + if(!row.session){session=await this.sendingSession(recipient,budget,row);row=this.outbox(requestId)!;} session=document(row.session,MAX_SESSION_BYTES) as Obj;row=await this.encryptOutbox(row,session); let recovery:Obj={network_accessed:false}; if(row.result&&!row.acknowledgement&&recoverAck){ @@ -478,6 +530,21 @@ export class OpenDeliveryClient{ return {messages,errors:[],state:'observed',body_transport:'mailbox_retained_copy',receipt_state:'retained_for_independent_return'}; }); } + async receiveMailboxReplica(reader:MailboxMessageReplicaRecoveryClient,baseUrl:string,options:MessageReplicaRecoverOptions):Promise{ + if(!(reader instanceof MailboxMessageReplicaRecoveryClient)||!same(reader.subject,{signing_key:validateSigningIdentity(this.participant.identity),encryption_key:validateEncryptionIdentity(this.encryption)}))fail('open_delivery_key_binding_mismatch'); + return this.serial(async()=>{ + const deadline=performance.now()/1000+(options.timeout??60),recovered=await reader.recover(baseUrl,options),selected=recovered.replica.source.message_scope.envelope_ref; + const message=recovered.replica.source.graph.members.find((v:Obj)=>same(v.attempt.envelope_ref,selected))?.attempt.message_id??(recovered.replica.source.custody.payload.message_id as string); + const prior=this.inbox(message);let result:Obj,fresh:boolean; + if(prior){if(prior.envelope_sha256!==selected.raw_sha256)fail('network_inbox_identity_conflict');fresh=prior.phase==='staged';result=await this.finishInbox(message);} + else{ + const remaining=Math.min(30,deadline-performance.now()/1000);if(remaining<=0)fail('open_delivery_budget_exhausted',true);const verified=await reader.readMessage(baseUrl,recovered,remaining),roles=verified.setup.roles; + const authority=Object.fromEntries([['request','contact.request'],['policy','contact.policy']].map(([n,r])=>[n,document(roles[r].raw)])),session={authority,source_node:document((options.targetNodeEntry as Obj).raw),mailbox:reader.inboxEvidence(recovered)},envelope=verified.envelope; + const body=await decryptEnvelope(envelope,{...OpenDeliveryClient.keys(session),encryption_identity:this.encryption} as any);this.stageInbox(message,authority.request.payload.signing_key.key_id,envelope,body,session);result=await this.finishInbox(message);fresh=true; + } + if(result.state==='validated_saved')this.savedReceipt(message);return {messages:fresh?[result]:[],errors:[],state:'observed',network_accessed:true,body_transport:'mailbox_message_replica',receipt_state:'retained_for_independent_return'}; + }); + } private pendingReceiptBatch():Obj[]{return this.db(db=>{ const select="SELECT message_id,phase,created_at FROM open_delivery_inbox WHERE (phase='staged' OR (phase='saved' AND receipt_sent=0))", order=' ORDER BY created_at,message_id LIMIT ?', @@ -500,7 +567,7 @@ export class OpenDeliveryClient{ if(result.state==='validated_saved')await this.sendReceipt(row.message_id,budget);}catch(error){errors.push({message_id:row.message_id,...errorData(error)});} finally{this.advancePendingReceipt(row);}} if(options.pendingOnly)return {messages,errors:errors.slice(0,4),network_accessed:budget.requests>0}; - for(const candidate of this.contactSessions(false)){if(messages.length>=limit)break; + for(const candidate of this.receivingSessions()){if(messages.length>=limit)break; try{const session=await this.checkedSession(candidate,budget),node=session.node,leaseId=session.decision.payload.grant.payload.resource_lease.payload.lease_id; let [key,after]=this.cursor(session);const intent=readerIntent('list',{lease_id:leaseId,caller_key_id:this.participant.keyId}); const prepared=await this.call(node,'list.prepare',{lease_id:leaseId},budget),challenge=prepared.challenge,answer=await this.answer(challenge,intent,node); @@ -511,7 +578,8 @@ export class OpenDeliveryClient{ const [result,fresh]=await this.receiveEntry(session,entry,budget);after=entry.sequence;this.advance(key,after);if(fresh)messages.push(result); if(result.state==='validated_saved')try{await this.sendReceipt(entry.message_id,budget,node);}catch(error){errors.push({message_id:entry.message_id,...errorData(error)});}} if(safeInteger(listing.next_sequence)!==after)fail('open_delivery_sequence_mismatch'); - }catch(error){errors.push(errorData(error));if(errors.length>=4)break;}} + }catch(error){errors.push(errorData(error));if(errors.length>=4)break;} + finally{this.advanceReceivingSession(candidate);}} return {messages,errors:errors.slice(0,4),network_accessed:budget.requests>0}; } readMessage(messageId:string,offset=0):Obj{ diff --git a/clients/typescript/network/open-mailbox-destination.ts b/clients/typescript/network/open-mailbox-destination.ts new file mode 100644 index 0000000..c7fab85 --- /dev/null +++ b/clients/typescript/network/open-mailbox-destination.ts @@ -0,0 +1,99 @@ +/** Receiver-issued original mailbox admission, retained across client changes. */ +import {canonicalBytes,sha256,encodeBase64url,decodeBase64url,validateSigningIdentity,validateEncryptionIdentity} from './crypto.ts'; +import type {EncryptionIdentityDocument} from './crypto.ts'; +import type {OpenParticipant} from './open-participant.ts'; +import {transaction} from './io.ts'; +import {RepairBudget,RepairError,buildNewWire,parseNewWire,objectFields,rawRef,u53} from './open-repair-wire.ts'; +import {DEFAULT_REPAIR_CLIENT_POLICY} from './open-repair-client.ts'; +import {verifyMailboxFeedBootstrap} from './open-repair-mailbox-authority.ts'; +import {originalPublicDescriptor,verifyContactOriginals,verifyBoundedControlSignature} from './open-repair-original.ts'; +import {signBoundedBootstrapOriginal} from './open-repair-probe.ts'; +import {MailboxSetupJournal} from './open-mailbox-journal.ts'; +import {authenticateStatusOriginal,statusScope} from './open-repair-status.ts'; +import {issueStatus} from './open-provider.ts'; +type Obj=Record; +function fail(code:string):never{throw new RepairError(code);} +/** Inputs are the locally reauthenticated receiver and approved contact originals. */ +export function prepareMailboxDestination(participant:OpenParticipant,encryption:EncryptionIdentityDocument,config:Obj,slots:Obj,contact:Obj, + value:{expires_at:number;status_revision:number;status_until:number}):Obj{ + objectFields(value,['expires_at','status_revision','status_until']); + const policy={...DEFAULT_REPAIR_CLIENT_POLICY,max_signature_checks:512},budget=new RepairBudget(policy),at=Math.floor(Date.now()/1000); + const owner={signing_key:validateSigningIdentity(participant.identity),encryption_key:validateEncryptionIdentity(encryption)}; + const encode=(entry:Obj)=>{const ref=rawRef(entry.ref),raw=parseNewWire(entry.raw,policy,budget).raw;if(raw.length!==ref.size||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch');return {ref,raw_base64url:encodeBase64url(raw)};}; + const decode=(entry:unknown)=>{const e=objectFields(entry,['ref','raw_base64url']),ref=rawRef(e.ref),raw=decodeBase64url(e.raw_base64url,65536);if(raw.length!==ref.size||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch');return {ref,raw};}; + const setup=verifyMailboxFeedBootstrap(slots,{expectedSlot:config.expected_slot,expectedOwner:owner,expectedTarget:config.expected_target, + targetStorageEpoch:config.expected_slot.writer_storage_epoch,limitPolicy:config.limit_policy,at,policy,budget}),slot=setup.slot.payload as Obj,root=slot.slot_key.root_key; + const sender=objectFields(config.expected_sender,['signing_key','encryption_key']); + originalPublicDescriptor(sender.signing_key,budget);originalPublicDescriptor(sender.encryption_key,budget,true); + if(sender.signing_key.key_id!==slot.sender.signing_key_id||sender.encryption_key.key_id!==slot.sender.encryption_key_id)fail('open_invalid_mailbox_receiver'); + const plan={root_key:root,slot_key:slot.slot_key,sender:slot.sender,target:config.expected_target,limits:config.limit_policy, + ...Object.fromEntries(['budget','windows','max_appends','max_live_items'].map(n=>[n,slot[n]]))}; + u53(value.status_revision,1);u53(value.expires_at);u53(value.status_until); + const binding=budget.hash(buildNewWire({plan,slots:Object.fromEntries(Object.entries(slots).map(([n,e])=>[n,encode(e)])), + contact:Object.fromEntries(Object.entries(contact).map(([n,raw])=>[n,budget.hash(raw as Uint8Array)])),...value},policy,budget).raw); + const rootDigest=budget.hash(buildNewWire(root,policy,budget).raw); + const journal=new MailboxSetupJournal(participant),journalKey=journal.start({kind:'mailbox.feed_recovery',slot_key:slot.slot_key,owner, + sender:config.expected_sender,target:config.expected_target,entries:Object.fromEntries(Object.entries(setup).map(([n,e])=>[n,e.ref]))}); + const required=new Map(); + for(const [name,mask] of [['slot',65],['read',2],['maintenance',65],['bootstrap',10]] as const){ + const e=setup[name],kind=name==='slot'?'mailbox_slot':'authority',scope=statusScope(root,kind,name==='slot'?slot.slot_key:{authority_kind:e.payload.kind,authority_sha256:e.ref.raw_sha256},policy,budget); + required.set(owner.signing_key.key_id+':'+kind+':'+scope,{revision:e.payload.revision as number,mask}); + } + for(const name of ['data','metadata'])required.set(config.expected_target.signing_key.key_id+':resource:'+statusScope(root,'resource',slot[name+'_resource_ref'],policy,budget),{revision:Infinity,mask:65}); + return participant.providerStorage(db=>transaction(db,()=>{ + const revisions=new Map();let observedOwnerRevision=-1; + for(const entry of journal.statuses(journalKey)){ + const signed=objectFields(parseNewWire(entry.raw,policy,budget).value,['payload','proof']),preview=signed.payload as Obj; + const signer=[owner.signing_key,config.expected_target.signing_key,config.expected_sender.signing_key].find(v=>v.key_id===preview.signing_key?.key_id); + if(!signer||u53(preview.issued_at)>at||!Array.isArray(preview.entries)||preview.entries.length>16)fail('repair_status_disclosure'); + const item=authenticateStatusOriginal(entry,{expectedRoot:root,expectedSigningKey:signer,at:preview.issued_at, + allowedScopes:preview.entries.map((e:Obj)=>({scope_kind:e.scope_kind,scope_id:e.scope_id})),policy,budget}); + const status=item.payload as Obj,key=signer.key_id+':'+status.revision; + if(revisions.has(key)&&revisions.get(key)!==item.canonical_sha256)fail('repair_status_conflict');revisions.set(key,item.canonical_sha256); + if(signer.key_id===owner.signing_key.key_id)observedOwnerRevision=Math.max(observedOwnerRevision,status.revision); + for(const row of status.entries){const need=required.get(signer.key_id+':'+row.scope_kind+':'+row.scope_id);if(!need)continue; + if(row.status==='revoked'&&(row.operation_mask&need.mask))fail('repair_authority_revoked'); + if(row.minimum_document_revision>need.revision)fail('repair_status_revision'); + } + } + db.exec('CREATE TABLE IF NOT EXISTS open_mailbox_destinations(root_digest TEXT NOT NULL,revision INTEGER NOT NULL,binding TEXT NOT NULL,bundle BLOB NOT NULL,bundle_sha256 TEXT NOT NULL,PRIMARY KEY(root_digest,revision))'); + const old=db.prepare('SELECT binding,bundle,bundle_sha256 FROM open_mailbox_destinations WHERE root_digest=? AND revision=?').get(rootDigest,value.status_revision) as Obj|undefined; + if(old){ + if(old.binding!==binding)fail('repair_destination_conflict');if(old.bundle.length>65536||budget.hash(old.bundle)!==old.bundle_sha256)fail('repair_destination_journal_corrupt'); + const saved=objectFields(parseNewWire(old.bundle,policy,budget).value,['destination','owner_status']),result=Object.fromEntries(Object.entries(saved).map(([n,e])=>[n,decode(e)])); + for(const entry of Object.values(result)){const signed=objectFields(parseNewWire(entry.raw,policy,budget).value,['payload','proof']);verifyBoundedControlSignature(signed.payload,signed.proof,owner.signing_key,budget);} + return result; + } + const latest=(db.prepare('SELECT max(revision) AS revision FROM open_mailbox_destinations WHERE root_digest=?').get(rootDigest) as Obj).revision; + if(value.status_revision<=observedOwnerRevision||(latest!==null&&value.status_revision<=latest))fail('repair_destination_revision_rollback'); + if((db.prepare('SELECT count(*) AS n FROM open_mailbox_destinations').get() as Obj).n>=128)fail('repair_destination_capacity'); + const held=verifyContactOriginals(contact,{senderKeyId:slot.sender.signing_key_id,senderEncryptionKeyId:slot.sender.encryption_key_id, + recipientKeyId:owner.signing_key.key_id,recipientEncryptionKeyId:owner.encryption_key.key_id,nodeKeyId:config.expected_target.signing_key.key_id, + storageEpoch:slot.slot_key.writer_storage_epoch,at,policy,budget}); + if(!(atsetup[n].payload.expires_at as number),held.originals.grant.payload.expires_at as number)) + ||!(ate.payload.expires_at as number))))fail('repair_resource_expired'); + const refs=Object.fromEntries([['contact_request_ref','request'],['contact_policy_ref','policy'],['contact_knock_lease_ref','knock_lease'],['contact_decision_ref','decision'],['store_grant_ref','grant']].map(([field,role])=>{ + const raw=held.originals[role].document.raw,digest=budget.hash(raw);return [field,{namespace:'meta',key:digest,raw_sha256:digest,size:raw.length}];})); + const destinationId='destination_'+sha256(Buffer.concat([Buffer.from('memory-vault-mailbox-setup/v1\0'),buildNewWire(root,policy,budget).raw,Buffer.from('\0destination')])); + const signed=signBoundedBootstrapOriginal({schema_version:'memory-vault-open-repair/v1',kind:'delivery.destination',signing_key:owner.signing_key, + issued_at:at,expires_at:value.expires_at,destination_id:destinationId,sender:slot.sender,recipient:slot.recipient,slot_key:slot.slot_key,slot_ref:setup.slot.ref,...refs, + ...Object.fromEntries(['data_resource_ref','data_resource_offer_ref','metadata_resource_ref','metadata_resource_offer_ref','read_grant_ref','maintenance_root_ref','budget','windows'].map(n=>[n,slot[n]]))},participant.identity,policy,budget); + const destination={raw:signed.raw,ref:signed.ref},scopes=[['destination',destination],...Object.entries(slots)].map(([name,entry])=>{ + const e=entry as Obj,p=(parseNewWire(e.raw,policy,budget).value as Obj).payload,kind=name==='slot'?'mailbox_slot':'authority'; + return {scope_kind:kind,scope_id:statusScope(root,kind,name==='slot'?slot.slot_key:{authority_kind:p.kind,authority_sha256:e.ref.raw_sha256},policy,budget), + minimum_document_revision:p.revision??1,status:'active',operation_mask:127}; + }).sort((a,b)=>a.scope_kindb.scope_kind?1:a.scope_idb.scope_id?1:0); + const raw=buildNewWire(issueStatus(participant.identity,{root,revision:value.status_revision,entries:scopes,issued_at:at,valid_until:value.status_until}),policy,budget).raw,digest=budget.hash(raw); + const result={destination,owner_status:{raw,ref:{namespace:'meta',key:digest,raw_sha256:digest,size:raw.length}}}; + const bundle=buildNewWire(Object.fromEntries(Object.entries(result).map(([n,e])=>[n,encode(e)])),policy,budget).raw;if(bundle.length>65536)fail('repair_destination_capacity'); + db.prepare('INSERT INTO open_mailbox_destinations VALUES(?,?,?,?,?)').run(rootDigest,value.status_revision,binding,bundle,budget.hash(bundle));return result; + })); +} +/** Finite output shares the existing Python cursor and exact authorization bytes. */ +export function mailboxPage(value:unknown,receiver:string,cursor:unknown,kind:'authorization'|'configuration'):Obj{ + const raw=canonicalBytes(value,65536),digest=sha256(raw);let offset=0; + if(cursor!==undefined&&cursor!==null){const v=objectFields(cursor,['sha256','offset']);offset=v.offset;if(v.sha256!==digest||!Number.isSafeInteger(offset)||offset<=0||offset>=raw.length||offset%3072)fail('open_invalid_mailbox_cursor');} + const end=Math.min(offset+3072,raw.length);return {state:'mailbox_'+kind,receiver_id:receiver,[kind+'_sha256']:digest,total_bytes:raw.length,offset, + [kind+'_chunk']:Buffer.from(raw.subarray(offset,end)).toString('base64'),next_cursor:end===raw.length?null:{sha256:digest,offset:end}, + network_accessed:false,source_rechecked:false,receipt_return:'separate_authority_required'}; +} diff --git a/clients/typescript/network/open-mailbox-receivers.ts b/clients/typescript/network/open-mailbox-receivers.ts index faf0b7b..89554dd 100644 --- a/clients/typescript/network/open-mailbox-receivers.ts +++ b/clients/typescript/network/open-mailbox-receivers.ts @@ -12,7 +12,10 @@ import {verifyMailboxFeedBootstrap} from './open-repair-mailbox-authority.ts'; import {verifySourceNodeOriginal} from './open-repair-original.ts'; import {endpoint} from './open-transport.ts'; import {transaction} from './io.ts'; +import {prepareMailboxDestination,mailboxPage} from './open-mailbox-destination.ts'; +import {OpenContactClient} from './open-contact-client.ts'; import {MailboxReceiptJobs} from './open-mailbox-receipts.ts'; +import {RegisteredMailboxReplicas} from './open-mailbox-replicas.ts'; type Obj=Record; export const MAILBOX_CONNECT_SCHEMA='memory-vault-open-mailbox-connect/v1'; function fail(code:string):never{throw new RepairError(code);} @@ -36,10 +39,25 @@ export class RegisteredMailboxReceivers{ const node=verifySourceNodeOriginal(raw,{expectedSigningKey:value.expected_target.signing_key,expectedStorageEpoch:value.expected_slot.writer_storage_epoch,at,policy,budget}); if(!same(endpoint(value.base_url,this.#participant.transport.allow_loopback),endpoint(node.payload.base_url,this.#participant.transport.allow_loopback)))fail('open_invalid_mailbox_receiver');return options; } - connect(invitation:unknown):Obj{ + async connect(invitation:unknown):Promise{ const value=document(invitation as any,65536) as Obj;if(value.schema_version!==MAILBOX_CONNECT_SCHEMA)fail('open_invalid_mailbox_receiver'); + if(['register_replica','receive_replica','list_replicas','remove_replica','inspect_replica'].includes(value.action))return new RegisteredMailboxReplicas(this.#participant,this.#encryption,this.#delivery).connect(value); if(value.action==='list'){objectFields(value,['schema_version','action']);const rows=this.#participant.providerStorage(db=>db.prepare('SELECT receiver_id FROM open_mailbox_receivers ORDER BY receiver_id LIMIT 17').all()) as Obj[];if(rows.length>16)fail('open_mailbox_receiver_capacity');return {state:'configured',mailboxes:rows.map(v=>v.receiver_id),network_accessed:false};} if(value.action==='remove'){objectFields(value,['schema_version','action','receiver_id']);opaqueId(value.receiver_id);this.#participant.providerStorage(db=>db.prepare('DELETE FROM open_mailbox_receivers WHERE receiver_id=?').run(value.receiver_id));return {state:'removed',receiver_id:value.receiver_id,network_accessed:false};} + if(value.action==='inspect'||value.action==='authorize'){ + objectFields(value,['schema_version','action','receiver_id',...(value.action==='authorize'?['contact_request_ref','expires_at','status_revision','status_until']:[]),...(Object.hasOwn(value,'cursor')?['cursor']:[])]); + opaqueId(value.receiver_id);const row=this.#participant.providerStorage(db=>db.prepare('SELECT body FROM open_mailbox_receivers WHERE receiver_id=?').get(value.receiver_id)) as Obj|undefined; + if(!row)fail('open_mailbox_receiver_missing');const config=document(row.body,65536) as Obj; + if(value.receiver_id!=='mailbox_'+sha256(canonicalBytes(config.expected_slot)))fail('open_invalid_mailbox_receiver');const options=this.#validate(config); + if(value.action==='inspect')return mailboxPage(config,value.receiver_id,value.cursor,'configuration'); + const contact=new OpenContactClient(this.#participant,this.#encryption).approvedIncomingOriginals(value.contact_request_ref); + const bundle=prepareMailboxDestination(this.#participant,this.#encryption,config,options.slotEntries,contact, + {expires_at:value.expires_at,status_revision:value.status_revision,status_until:value.status_until}); + const encoded=(e:Obj)=>({raw:Buffer.from(e.raw).toString('utf8'),ref:e.ref}); + return mailboxPage({destination_entry:encoded(bundle.destination),owner_status_entry:encoded(bundle.owner_status), + slot_entries:Object.fromEntries(['slot','read','maintenance'].map(n=>[n,config.slot_entries[n]])),target:config.expected_target, + target_node_entry:config.target_node_entry,base_url:config.base_url},value.receiver_id,value.cursor,'authorization'); + } this.#validate(value);const receiver='mailbox_'+sha256(canonicalBytes(value.expected_slot)),raw=canonicalBytes(value); this.#participant.providerStorage(db=>transaction(db,()=>{const old=db.prepare('SELECT body FROM open_mailbox_receivers WHERE receiver_id=?').get(receiver) as Obj|undefined; if(old){if(!Buffer.from(old.body).equals(Buffer.from(raw)))fail('open_mailbox_receiver_conflict');return;} @@ -50,12 +68,18 @@ export class RegisteredMailboxReceivers{ const deadline=performance.now()/1000+60,result=await this.#delivery.receive(limit,{pendingOnly:true,deadline}); const jobs=new MailboxReceiptJobs(this.#participant,this.#encryption,this.#delivery),attempted=new Set(); await jobs.poll(result,Math.min(deadline,performance.now()/1000+30),attempted); - const rows=this.#participant.providerStorage(db=>db.prepare('SELECT receiver_id,body FROM open_mailbox_receivers ORDER BY last_attempt,receiver_id LIMIT 17').all()) as Obj[]; - if(rows.length>16)fail('open_mailbox_receiver_capacity'); + const replicas=new RegisteredMailboxReplicas(this.#participant,this.#encryption,this.#delivery); + const rows=this.#participant.providerStorage(db=>db.prepare(`SELECT receiver_id,body,receiver_kind,body_sha256 FROM (SELECT receiver_id,body,last_attempt,'original' AS receiver_kind,NULL AS body_sha256 FROM open_mailbox_receivers UNION ALL + SELECT receiver_id,body,last_attempt,'replica' AS receiver_kind,body_sha256 FROM open_mailbox_replica_receivers AS r WHERE NOT EXISTS( + SELECT 1 FROM open_delivery_inbox AS i WHERE i.envelope_sha256=r.envelope_sha256 AND i.phase!='staged')) ORDER BY last_attempt,receiver_id LIMIT 4`).all()) as Obj[]; for(const row of rows){let remaining=deadline-performance.now()/1000;if(result.messages.length>=limit||remaining<=0)break; - this.#participant.providerStorage(db=>db.prepare('UPDATE open_mailbox_receivers SET last_attempt=? WHERE receiver_id=?').run(BigInt(Date.now())*1000000n,row.receiver_id)); + this.#participant.providerStorage(db=>db.prepare('UPDATE '+(row.receiver_kind==='replica'?'open_mailbox_replica_receivers':'open_mailbox_receivers')+' SET last_attempt=? WHERE receiver_id=?').run(BigInt(Date.now())*1000000n,row.receiver_id)); let reader:MailboxFeedRecoveryClient|undefined; try{ + if(row.receiver_kind==='replica'){ + if(sha256(row.body)!==row.body_sha256)fail('open_invalid_mailbox_receiver'); + const received=await replicas.receive(document(row.body,65536) as Obj,deadline,()=>{result.network_accessed=true;});result.messages.push(...received.messages);result.errors.push(...received.errors);continue; + } const config=document(row.body,65536) as Obj;if(row.receiver_id!=='mailbox_'+sha256(canonicalBytes(config.expected_slot)))fail('open_invalid_mailbox_receiver'); const options=this.#validate(config),journal=new MailboxSetupJournal(this.#participant); reader=new MailboxFeedRecoveryClient(this.#participant.identity,this.#encryption,{limitPolicy:config.limit_policy,allowLoopback:this.#participant.transport.allow_loopback,transport:this.#participant.transport,networkObserver:()=>{result.network_accessed=true;}}); diff --git a/clients/typescript/network/open-mailbox-replica-client.ts b/clients/typescript/network/open-mailbox-replica-client.ts new file mode 100644 index 0000000..bf0c48f --- /dev/null +++ b/clients/typescript/network/open-mailbox-replica-client.ts @@ -0,0 +1,112 @@ +/** Native bounded replica recovery. All controls and original ciphertext are + * obtained through the selected endpoint, with independent dual possession. */ +import {performance} from 'node:perf_hooks'; +import {OpenHTTPTransport,endpoint} from './open-transport.ts'; +import {RepairBudget,buildNewWire,parseNewWire,rawRef,u53,LocalRawResolver,parseRawPack} from './open-repair-wire.ts'; +import type {RawOriginal,RepairPolicy} from './open-repair-wire.ts'; +import {DEFAULT_REPAIR_CLIENT_POLICY,DEFAULT_REPAIR_CLIENT_LIMITS} from './open-repair-client.ts'; +import {validateSigningIdentity,validateEncryptionIdentity,encodeBase64url} from './crypto.ts'; +import {verifySourceNodeOriginal} from './open-repair-original.ts'; +import {verifyMailboxFeedBootstrap} from './open-repair-mailbox-authority.ts'; +import {resolveHistoricalInputs} from './open-repair-history.ts'; +import {makeBootstrapProbe,solveBootstrapChallenge} from './open-repair-probe.ts'; +import {verifyBootstrapProofResponse,makeBootstrapChildRequest,makeMailboxBodyRequest} from './open-repair-proof.ts'; +import {authenticateStatusOriginal,statusScope} from './open-repair-status.ts'; +import {readMailboxIndex,verifyMailboxAdmissionMetadata} from './open-repair-mailbox-read.ts'; +import {ReplicaStatusJournal} from './open-ack-status.ts'; +import {replicaFields as fields,replicaFail as fail,replicaSame as same,replicaRefKey as id,replicaEntry,replicaFloors,verifyMailboxMessageReplica,checkMailboxMessageReturn} from './open-repair-mailbox-replica.ts'; +type Obj=Record; +export const MAILBOX_REPLICA_LIMITS=Object.freeze({...DEFAULT_REPAIR_CLIENT_LIMITS,max_signature_checks:4096,max_proof_bytes:4194304,max_requests:256,max_replay_records:256,max_proof_items:128}); +const monotonic=()=>performance.now()/1000; +function boundedTimeout(v:unknown):number{if(typeof v!=='number'||!Number.isFinite(v)||v<=0||v>60)fail('repair_invalid_deadline');return v;} +/** Authenticate retained observations before considering current access. */ +export function knownReplicaStatuses(entries:unknown,e:Obj,owner:Obj,at:number,policy:RepairPolicy,budget:RepairBudget,observe?: (v:any)=>void):Obj[]{ + if(!Array.isArray(entries)||entries.length>32)fail('repair_status_history_capacity');const parties={...e,owner},signers=new Map(Object.values(parties).filter((v:Obj)=>v?.signing_key).map((v:Obj)=>[v.signing_key.key_id,v.signing_key])),result:Obj[]=[];let denial:unknown; + for(const input of entries)try{const entry=replicaEntry(input,policy,budget),p=(parseNewWire(entry.raw,policy,budget).value as Obj).payload,signer=p.signing_key; + if(!same(signers.get(signer.key_id),signer)||u53(p.issued_at)>at||!Array.isArray(p.entries))fail('repair_status_disclosure'); + const permitted=new Set();if(same(signer,owner.signing_key)){permitted.add('authority');permitted.add('mailbox_slot');}if(same(signer,e.sender.signing_key)||same(signer,e.source.signing_key)||same(signer,e.maintainer.signing_key))permitted.add('authority');if(same(signer,e.source.signing_key)||same(signer,e.target.signing_key))permitted.add('resource');if(same(signer,e.maintainer.signing_key))permitted.add('assignment'); + if(p.entries.some((v:Obj)=>!permitted.has(v.scope_kind)))fail('repair_status_disclosure'); + result.push(authenticateStatusOriginal(entry,{expectedRoot:e.slot.root_key,expectedSigningKey:signer,at:p.issued_at,allowedScopes:p.entries.map((v:Obj)=>({scope_kind:v.scope_kind,scope_id:v.scope_id})),policy,budget},observe)); + }catch(error){denial??=error;} + if(denial)throw denial;return result; +} +export interface MessageReplicaRecoverOptions{targetNodeEntry:unknown;expectedTarget:unknown;expectedSender:unknown;expectedSlot:unknown;expectedSource:unknown;sourceStorageEpoch:string;expectedMaintainer:unknown;expectedEnvelopeRef:unknown;slotEntries:unknown;journal:ReplicaStatusJournal;knownStatuses?:readonly unknown[];archiveStatuses?:readonly unknown[];timeout?:number;} +export class MailboxMessageReplicaRecoveryClient{ + readonly #identity:Obj;readonly #encryption:Obj;readonly #subject:Obj;readonly #policy:RepairPolicy;readonly #transport:OpenHTTPTransport;readonly #loopback:boolean;readonly #clock:()=>number;readonly #observer?:()=>void; + readonly #recovered=new WeakMap(); + constructor(identity:unknown,encryption:unknown,a:{transport:OpenHTTPTransport;allowLoopback:boolean;clock?:()=>number;networkObserver?:()=>void}){ + this.#policy=Object.freeze({...DEFAULT_REPAIR_CLIENT_POLICY,max_signature_checks:512});const budget=new RepairBudget(this.#policy),e=buildNewWire({identity,encryption},this.#policy,budget).value as Obj; + this.#identity=e.identity;this.#encryption=e.encryption;this.#subject={signing_key:validateSigningIdentity(e.identity),encryption_key:validateEncryptionIdentity(e.encryption)};this.#transport=a.transport;this.#loopback=a.allowLoopback;this.#clock=a.clock??(()=>Date.now()/1000);this.#observer=a.networkObserver;Object.freeze(this); + } + get subject():Readonly{return this.#subject;} + close():void{} + #now():number{return u53(Math.floor(this.#clock()));} + async recover(baseUrl:string,a:MessageReplicaRecoverOptions):Promise>{ + if(!(a.journal instanceof ReplicaStatusJournal))fail('repair_invalid_context'); + const policy=this.#policy,budget=new RepairBudget(policy),started=this.#now(),timeout=boundedTimeout(a.timeout??60),deadline=monotonic()+timeout; + const e=buildNewWire({slot:a.expectedSlot,sender:a.expectedSender,target:a.expectedTarget,source:a.expectedSource,maintainer:a.expectedMaintainer},policy,budget).value as Obj; + if(e.slot.writer_storage_epoch!==a.sourceStorageEpoch)fail('repair_proof_mismatch');const envelopeRef=rawRef(a.expectedEnvelopeRef); + const setup=verifyMailboxFeedBootstrap(a.slotEntries,{expectedSlot:e.slot,expectedOwner:this.#subject,expectedTarget:e.source,targetStorageEpoch:a.sourceStorageEpoch,limitPolicy:MAILBOX_REPLICA_LIMITS,at:started,policy,budget}); + if(!same(setup.slot.payload.sender,{signing_key_id:e.sender.signing_key.key_id,encryption_key_id:e.sender.encryption_key.key_id}))fail('repair_proof_mismatch'); + const nodeEntry=replicaEntry(a.targetNodeEntry,policy,budget),np=(parseNewWire(nodeEntry.raw,policy,budget).value as Obj).payload,node=verifySourceNodeOriginal(nodeEntry.raw,{expectedSigningKey:e.target.signing_key,expectedStorageEpoch:np.storage_epoch,at:started,policy,budget}); + if(!same(endpoint(baseUrl,this.#loopback),endpoint(node.payload.base_url,this.#loopback)))fail('repair_proof_mismatch');const epoch=node.payload.storage_epoch as string; + const retained=new Map();for(const group of [a.knownStatuses??[],a.archiveStatuses??[],a.journal.load(new Set([this.#subject,...[e.sender,e.target,e.source,e.maintainer]].map(v=>v.signing_key.key_id)))]){if(!Array.isArray(group)||group.length>32)fail('repair_status_history_capacity');for(const item of group){const entry=replicaEntry(item,policy,budget);retained.set(id(entry.ref),entry);}} + const known=knownReplicaStatuses([...retained.values()],e,this.#subject,started,policy,budget,v=>a.journal.observe(v)); + const probeNeeds=['slot','read','maintenance','bootstrap'].map(n=>({signer:this.#subject.signing_key,scope_kind:n==='slot'?'mailbox_slot':'authority',scope_id:statusScope(e.slot.root_key,n==='slot'?'mailbox_slot':'authority',n==='slot'?e.slot:{authority_kind:setup[n].payload.kind,authority_sha256:setup[n].ref.raw_sha256},policy,budget),revision:setup[n].payload.revision,mask:n==='read'?2:10}));replicaFloors(known,[],probeNeeds); + const grant=setup.bootstrap.payload,expiry=Math.min(started+Math.min(60,Math.max(1,Math.floor(timeout))),node.payload.expires_at as number,grant.probe_until,grant.proof_until,...Object.values(setup).map(v=>v.payload.expires_at as number));if(expiry<=started)fail('repair_access_expired'); + const counts={requests:0,wire_bytes:0,proof_bytes:0},request=async(raw:Uint8Array,child=false)=>{if(counts.requests>=grant.limits.max_requests||monotonic()>=deadline)fail('repair_over_budget');counts.requests++;this.#observer?.();const reply=await this.#transport.requestRepair(baseUrl,raw,deadline,{child});counts.wire_bytes+=raw.length+reply.length;return reply;}; + const binding={expectedSubject:this.#subject,expectedTarget:e.target,targetStorageEpoch:epoch,bootstrapGrantSha256:setup.bootstrap.ref.raw_sha256,selector:grant.selector,consumer:'mailbox_feed' as const,policy,budget}; + const outgoing=await makeBootstrapProbe(this.#identity,{...binding,at:started,expiresAt:expiry});if(outgoing.original.raw.length>grant.limits.max_probe_bytes)fail('repair_over_budget'); + const challengeRaw=await request(outgoing.original.raw),digest=budget.hash(challengeRaw),challenge={raw:challengeRaw,ref:rawRef({namespace:'meta',key:digest,raw_sha256:digest,size:challengeRaw.length})}; + const preview=(parseNewWire(challengeRaw,policy,budget).value as Obj).payload,answer=await solveBootstrapChallenge({raw:outgoing.original.raw,ref:outgoing.original.ref},challenge,{...binding,signer:this.#identity,encryptionIdentity:this.#encryption,targetNonce:outgoing.nonce,at:this.#now(),expiresAt:Math.min(expiry,u53(preview.expires_at))}); + const response=await request(answer.raw),held=verifyBootstrapProofResponse(response,{expectedSubject:this.#subject,expectedTarget:e.target,targetStorageEpoch:epoch,selector:grant.selector,bootstrapGrantRef:setup.bootstrap.ref,probeRef:outgoing.original.ref,challengeRef:challenge.ref,answerRef:answer.ref,at:this.#now(),maxProofItems:grant.limits.max_proof_items,maxProofBytes:grant.limits.max_proof_bytes,expectedSourceState:'replica_message',consumer:'mailbox_feed',policy,budget}); + counts.proof_bytes=response.length+held.handle.raw.length+held.manifest.raw.length; + const originals=new Map(),roles=new Map(),available=new Map(),children=(held.manifest.value as Obj).children; + for(const item of [...Object.values(setup),...known])available.set(id(item.ref),item.raw);for(const item of children)roles.set(item.role,[...(roles.get(item.role)??[]),rawRef(item.ref)]); + const download=async(item:Obj)=>{const ref=rawRef(item.ref),k=id(ref);if(originals.has(k))return;if(ref.size>policy.max_document_bytes||counts.proof_bytes+ref.size>grant.limits.max_proof_bytes)fail('repair_over_budget');let raw=available.get(k); + if(raw)budget.input(raw.length);else{let offset=0;const parts:Uint8Array[]=[];while(offsetv.role==='replica.read_pack');if(packedChildren.length!==1)fail('repair_proof_mismatch');await download(packedChildren[0]);const packedItem=originals.get(id(packedChildren[0].ref))!,pack=parseRawPack(packedItem.raw,packedItem.ref,policy,budget); + const packRoles=new Set(['replica.manifest','replica.custody','copy.allocation','copy.offer','copy.assignment','copy.reservation_consent','copy.owner_disclosure','copy.source_disclosure','copy.current_status','return.owner','return.source','return.maintainer','current.status.replica_read','copy.sender_reservation_consent','copy.sender_disclosure','return.sender']); + const wanted=new Map();for(const item of children)if(packRoles.has(item.role)){const ref=rawRef(item.ref);wanted.set(ref.raw_sha256+':'+ref.size,ref);} + if(wanted.size!==pack.entries.length||pack.entries.some(v=>!wanted.has(v.raw_sha256+':'+v.size)))fail('repair_proof_mismatch'); + for(const [i,item] of pack.entries.entries()){const ref=wanted.get(item.raw_sha256+':'+item.size),raw=pack.entry(i,ref).raw,old=available.get(id(ref));if(old&&!Buffer.from(old).equals(Buffer.from(raw)))fail('repair_ref_conflict');available.set(id(ref),raw);} + for(const item of children)if(['history.raw_pack','history.mailbox_feed'].includes(item.role))await download(item); + const packed=new LocalRawResolver(policy,budget);for(const ref of roles.get('history.raw_pack')??[]){const item=originals.get(id(ref))!;if(!same(packed.put(ref.namespace,ref.key,item.raw).ref,ref))fail('repair_ref_mismatch');} + const reuse=(tree:Obj)=>{for(const row of tree.roles){const ref=row.original.ref,raw=row.original.raw,old=available.get(id(ref));if(old&&!Buffer.from(old).equals(Buffer.from(raw)))fail('repair_ref_conflict');available.set(id(ref),raw);}for(const prior of tree.predecessors)reuse(prior);}; + for(const ref of roles.get('history.mailbox_feed')??[]){const tree=resolveHistoricalInputs(originals.get(id(ref))!.raw,packed,policy,budget);if((tree.manifest.value as Obj).variant!=='mailbox_feed')fail('repair_proof_mismatch');reuse(tree);}for(const item of children)await download(item); + const entry=(ref:unknown)=>{const v=originals.get(id(ref));if(!v)fail('repair_original_missing');return v;},one=(role:string)=>{const refs=roles.get(role);if(refs?.length!==1)fail('repair_proof_mismatch');return entry(refs[0]);}; + const resolver=new LocalRawResolver(policy,budget);for(const item of originals.values())if(!same(resolver.put(item.ref.namespace,item.ref.key,item.raw).ref,item.ref))fail('repair_ref_mismatch'); + const context={expectedSlot:e.slot,expectedOwner:this.#subject,expectedSender:e.sender,expectedSource:e.source,sourceStorageEpoch:a.sourceStorageEpoch,expectedMaintainer:e.maintainer,expectedTarget:e.target,targetStorageEpoch:epoch,expectedEnvelopeRef:envelopeRef,limitPolicy:MAILBOX_REPLICA_LIMITS,policy,budget}; + const replica=verifyMailboxMessageReplica(one('replica.manifest'),resolver,one('replica.custody'),context); + for(const member of replica.source.graph.members)for(const [n,parent] of Object.entries(setup)){if(!same(member.originals[n].ref,parent.ref)||!Buffer.from(member.originals[n].raw).equals(Buffer.from(parent.raw)))fail('repair_proof_mismatch');} + const extras=new Set(['replica.manifest','replica.custody','replica.read_pack','return.owner','return.source','return.maintainer','return.sender','current.status.replica_read']),actual=new Set(),expected=new Set(); + for(const [role,refs] of roles)if(!extras.has(role))for(const ref of refs)actual.add(role+'|'+id(ref));for(const [role,items] of replica.entries)for(const item of items)expected.add(role+'|'+id(item.ref));if(actual.size!==expected.size||[...expected].some(v=>!actual.has(v)))fail('repair_proof_mismatch'); + const consents=Object.fromEntries(['owner','source','maintainer','sender'].map(n=>[n,one('return.'+n)])),current=(roles.get('current.status.replica_read')??[]).map(entry),permission=checkMailboxMessageReturn(replica,consents,{...context,currentStatuses:current,at:this.#now(),action:'proof',onObserved:(v:any)=>a.journal.observe(v)}); + const previous=[...replica.source.graph.statuses,...replica.source.graph.members.flatMap((v:Obj)=>v.statuses),...replica.authority.statuses,...known];replicaFloors(previous,permission.statuses,permission.obligations); + const members=await readMailboxIndex(one('feed.head'),one('feed.checkpoint'),{expectedSlot:e.slot,expectedSigningKey:e.source.signing_key,encryptionIdentity:this.#encryption as any,readOriginal:ref=>entry(ref).raw,at:this.#now(),maxMessages:setup.slot.payload.max_appends,policy,budget}); + const until=Math.min(expiry,held.handle.payload.expires_at as number,permission.expires_at);if(this.#now()>=until||monotonic()>=deadline)fail('repair_access_expired'); + const archive=new Map([...previous,...permission.statuses].map(v=>[id(v.ref),v]));if(archive.size>32)fail('repair_status_history_capacity'); + const result=Object.freeze({replica,proof:held,current_statuses:permission.statuses,archive_statuses:[...archive.values()],entries:members,metrics:{...counts,...budget.snapshot()},base_url:baseUrl});this.#recovered.set(result,{e,context,setup,held,originals,consents,current,archive:[...archive.values()],counts,until,journal:a.journal});return result; + } + async readMessage(baseUrl:string,recovered:object,timeout=30):Promise>{ + const c=this.#recovered.get(recovered);if(!c||!same(endpoint(baseUrl,this.#loopback),endpoint((recovered as Obj).base_url,this.#loopback)))fail('repair_invalid_context');const deadline=monotonic()+boundedTimeout(timeout),policy=this.#policy,budget=new RepairBudget(policy),context={...c.context,policy,budget}; + const resolver=new LocalRawResolver(policy,budget);for(const item of c.originals.values())if(!same(resolver.put(item.ref.namespace,item.ref.key,item.raw).ref,item.ref))fail('repair_ref_mismatch'); + const child=(role:string)=>{const rows=(c.held.manifest.value as Obj).children.filter((v:Obj)=>v.role===role);if(rows.length!==1)fail('repair_proof_mismatch');return c.originals.get(id(rows[0].ref));}; + const replica=verifyMailboxMessageReplica(child('replica.manifest'),resolver,child('replica.custody'),context),permission=checkMailboxMessageReturn(replica,c.consents,{...context,currentStatuses:c.current,at:this.#now(),action:'child',onObserved:(v:any)=>c.journal.observe(v)}); + const retained=knownReplicaStatuses(c.journal.load(new Set([this.#subject,c.e.sender,c.e.target,c.e.source,c.e.maintainer].map(v=>v.signing_key.key_id))),c.e,this.#subject,this.#now(),policy,budget,v=>c.journal.observe(v));replicaFloors([...c.archive,...retained],permission.statuses,permission.obligations); + const checked=await verifyMailboxAdmissionMetadata(replica.source.message_member,{expectedSlot:c.e.slot,expectedSigningKey:c.e.source.signing_key,expectedOwner:this.#subject,expectedSender:c.e.sender,expectedTarget:c.e.source,encryptionIdentity:this.#encryption as any,readOriginal:ref=>{const v=c.originals.get(id(ref));if(!v)fail('repair_original_missing');return v.raw;},at:this.#now(),limitPolicy:MAILBOX_REPLICA_LIMITS,policy,budget,currentStatuses:[],knownStatuses:[],statusObligations:[]}); + const ref=rawRef(context.expectedEnvelopeRef),children=(c.held.manifest.value as Obj).children.filter((v:Obj)=>v.role==='member.core'&&same(v.ref,checked.link.core_ref));if(children.length!==1||!same(ref,checked.core.envelope_ref))fail('repair_proof_mismatch'); + if(ref.size+c.counts.proof_bytes>MAILBOX_REPLICA_LIMITS.max_proof_bytes)fail('repair_over_budget');let offset=0;const parts:Uint8Array[]=[]; + const guard=()=>{if(this.#now()>=Math.min(c.until,permission.expires_at)||monotonic()>=deadline)fail('repair_access_expired');}; + while(offset=MAILBOX_REPLICA_LIMITS.max_requests)fail('repair_over_budget');const count=Math.min(65536,ref.size-offset),packet=makeMailboxBodyRequest(this.#identity,c.held,ref,{subject:this.#subject,target:c.e.target,at:this.#now(),expiresAt:c.held.handle.payload.expires_at,childIndex:children[0].index,offset,requestedBytes:count,policy,budget});c.counts.requests++;this.#observer?.();const raw=await this.#transport.requestRepair(baseUrl,packet.raw,deadline,{child:true});if(raw.length!==count)fail('repair_ref_mismatch');budget.input(raw.length);parts.push(raw);offset+=count;} + guard();budget.output(ref.size);const envelope=Buffer.concat(parts);if(budget.hash(envelope)!==ref.raw_sha256)fail('repair_ref_mismatch');return Object.freeze({...checked,envelope,replica,current_statuses:permission.statuses}); + } + inboxEvidence(recovered:object):Readonly{ + const c=this.#recovered.get(recovered);if(!c)fail('repair_invalid_context');const reference=(role:string)=>{const found=(c.held.manifest.value as Obj).children.filter((v:Obj)=>v.role===role);if(found.length!==1)fail('repair_proof_mismatch');return found[0].ref;}; + const selected=new Map();for(const items of (recovered as Obj).replica.entries.values())for(const item of items)selected.set(id(item.ref),item); + for(const role of ['replica.manifest','replica.custody','return.owner','return.source','return.maintainer','return.sender']){const ref=reference(role);selected.set(id(ref),c.originals.get(id(ref)));} + for(const item of [...(recovered as Obj).current_statuses,...c.archive])selected.set(id(item.ref),item); + return Object.freeze({schema_version:'memory-vault-mailbox-replica-inbox/v1',received_at:this.#now(),slot:c.e.slot,sender:c.e.sender,source:c.e.source,target:c.e.target,maintainer:c.e.maintainer,target_storage_epoch:c.context.targetStorageEpoch,envelope_ref:c.context.expectedEnvelopeRef,limits:MAILBOX_REPLICA_LIMITS,manifest_ref:reference('replica.manifest'),custody_ref:reference('replica.custody'),consent_refs:Object.fromEntries(['owner','source','maintainer','sender'].map(n=>[n,reference('return.'+n)])),originals:[...selected.values()].sort((a,b)=>id(a.ref)({ref:v.ref,raw_base64url:encodeBase64url(v.raw)})),status_refs:(recovered as Obj).current_statuses.map((v:Obj)=>v.ref),archive_status_refs:c.archive.map((v:Obj)=>v.ref)}); + } +} diff --git a/clients/typescript/network/open-mailbox-replicas.ts b/clients/typescript/network/open-mailbox-replicas.ts new file mode 100644 index 0000000..23bf65a --- /dev/null +++ b/clients/typescript/network/open-mailbox-replicas.ts @@ -0,0 +1,69 @@ +/** Explicit finite replica placements, shared with Python's protected state. + * No global membership, authority minting, or automatic unconsented movement. */ +import {performance} from 'node:perf_hooks'; +import {canonicalBytes,document,sha256,opaqueId,validateSigningIdentity,validateEncryptionIdentity,validateSigningPublic,validateEncryptionPublic,encodeBase64url} from './crypto.ts'; +import type {EncryptionIdentityDocument} from './crypto.ts'; +import type {OpenParticipant} from './open-participant.ts'; +import {transaction} from './io.ts'; +import {verifyNode} from './open-control.ts'; +import {endpoint} from './open-transport.ts'; +import {RepairBudget,parseNewWire} from './open-repair-wire.ts'; +import {DEFAULT_REPAIR_CLIENT_POLICY} from './open-repair-client.ts'; +import {verifySourceNodeOriginal} from './open-repair-original.ts'; +import {statusScope} from './open-repair-status.ts'; +import {verifyMailboxFeedBootstrap} from './open-repair-mailbox-authority.ts'; +import {MailboxMessageReplicaRecoveryClient,MAILBOX_REPLICA_LIMITS,knownReplicaStatuses} from './open-mailbox-replica-client.ts'; +import {ReplicaStatusJournal} from './open-ack-status.ts'; +import {replicaFields as fields,replicaFail as fail,replicaSame as same,replicaEntry,replicaFloors} from './open-repair-mailbox-replica.ts'; +import type {OpenDeliveryClient} from './open-delivery-client.ts'; +type Obj=Record; +const SCHEMA='memory-vault-open-mailbox-connect/v1',TABLE='open_mailbox_replica_receivers'; +const REQUEST='expected_slot expected_sender expected_target expected_source source_storage_epoch expected_maintainer expected_envelope_ref target_node_entry slot_entries known_statuses archive_statuses'.split(' '); +const decode=(v:unknown)=>{const e=fields(v,['raw','ref']);if(typeof e.raw!=='string')fail('open_invalid_mailbox_receiver');return {raw:Buffer.from(e.raw,'utf8'),ref:e.ref};}; +export class RegisteredMailboxReplicas{ + readonly #participant:OpenParticipant;readonly #encryption:EncryptionIdentityDocument;readonly #delivery:OpenDeliveryClient; + constructor(participant:OpenParticipant,encryption:EncryptionIdentityDocument,delivery:OpenDeliveryClient){this.#participant=participant;this.#encryption=encryption;this.#delivery=delivery; + participant.providerStorage(db=>db.exec(`CREATE TABLE IF NOT EXISTS ${TABLE}(receiver_id TEXT PRIMARY KEY,selection_digest TEXT NOT NULL,envelope_sha256 TEXT NOT NULL,body BLOB NOT NULL,body_sha256 TEXT NOT NULL,last_attempt INTEGER NOT NULL DEFAULT 0)`));} + #checked(value:Obj,current=true):Obj{ + fields(value,['schema_version','action','base_url','repair_profile','request']);if(value.schema_version!==SCHEMA||value.action!=='register_replica'||value.repair_profile!=='mailbox')fail('open_invalid_mailbox_receiver');const request=fields(value.request,REQUEST); + const policy=Object.freeze({...DEFAULT_REPAIR_CLIENT_POLICY,max_signature_checks:512}),budget=new RepairBudget(policy),at=Math.floor(Date.now()/1000),owner={signing_key:validateSigningIdentity(this.#participant.identity),encryption_key:validateEncryptionIdentity(this.#encryption)}; + const e={slot:request.expected_slot,sender:request.expected_sender,target:request.expected_target,source:request.expected_source,maintainer:request.expected_maintainer}; + for(const keys of [owner,e.sender,e.target,e.source,e.maintainer]){fields(keys,['signing_key','encryption_key']);validateSigningPublic(keys.signing_key);validateEncryptionPublic(keys.encryption_key);} + if(same(e.target,e.source)||same(e.target,e.maintainer)||request.source_storage_epoch!==e.slot.writer_storage_epoch)fail('open_invalid_mailbox_receiver'); + fields(request.slot_entries,['slot','read','maintenance','bootstrap']);const slots=Object.fromEntries(Object.entries(request.slot_entries).map(([n,v])=>[n,decode(v)])); + const setup=verifyMailboxFeedBootstrap(slots,{expectedSlot:e.slot,expectedOwner:owner,expectedTarget:e.source,targetStorageEpoch:request.source_storage_epoch,limitPolicy:MAILBOX_REPLICA_LIMITS,at,policy,budget}); + if(!same(setup.slot.payload.sender,{signing_key_id:e.sender.signing_key.key_id,encryption_key_id:e.sender.encryption_key.key_id})||!setup.maintenance.payload.maintainers.some((v:Obj)=>same(v,{signing_key_id:e.maintainer.signing_key.key_id,encryption_key_id:e.maintainer.encryption_key.key_id})))fail('open_invalid_mailbox_receiver'); + const envelope=request.expected_envelope_ref;if(envelope.namespace!=='object')fail('open_invalid_mailbox_receiver');const entry=replicaEntry(decode(request.target_node_entry),policy,budget),p=(parseNewWire(entry.raw,policy,budget).value as Obj).payload; + const node=verifySourceNodeOriginal(entry.raw,{expectedSigningKey:e.target.signing_key,expectedStorageEpoch:p.storage_epoch,at:current?at:p.issued_at,policy,budget});if(!same(endpoint(value.base_url,this.#participant.transport.allow_loopback),endpoint(node.payload.base_url,this.#participant.transport.allow_loopback)))fail('open_invalid_mailbox_receiver'); + for(const [n,max] of [['known_statuses',16],['archive_statuses',32]] as const)if(!Array.isArray(request[n])||request[n].length>max)fail('repair_status_history_capacity'); + const journal=new ReplicaStatusJournal(this.#participant,e.slot.root_key),observe=(v:any)=>journal.observe(v); + // The independent journal survives invalid later inputs and removals. + const retained=journal.load(new Set([owner,e.sender,e.target,e.source,e.maintainer].map(v=>v.signing_key.key_id))),known=knownReplicaStatuses([...request.known_statuses.map(decode),...request.archive_statuses.map(decode),...retained],e,owner,at,policy,budget,observe); + return {e,owner,setup,node,slots,journal,known,policy,budget,request}; + } + async connect(value:Obj):Promise{ + if(value.action==='list_replicas'){fields(value,['schema_version','action']);const rows=this.#participant.providerStorage(db=>db.prepare(`SELECT receiver_id FROM ${TABLE} ORDER BY receiver_id LIMIT 17`).all()) as Obj[];if(rows.length>16)fail('open_mailbox_receiver_capacity');return {state:'configured',replicas:rows.map(v=>v.receiver_id),network_accessed:false};} + if(['remove_replica','inspect_replica'].includes(value.action)){ + fields(value,['schema_version','action','receiver_id',...(Object.hasOwn(value,'cursor')?['cursor']:[])]);opaqueId(value.receiver_id); + if(value.action==='remove_replica'){this.#participant.providerStorage(db=>db.prepare(`DELETE FROM ${TABLE} WHERE receiver_id=?`).run(value.receiver_id));return {state:'removed',receiver_id:value.receiver_id,network_accessed:false};} + const row=this.#participant.providerStorage(db=>db.prepare(`SELECT body,body_sha256 FROM ${TABLE} WHERE receiver_id=?`).get(value.receiver_id)) as Obj|undefined;if(!row)fail('open_mailbox_receiver_missing');if(sha256(row.body)!==row.body_sha256)fail('open_invalid_mailbox_receiver'); + let offset=0;if(value.cursor!=null){fields(value.cursor,['sha256','offset']);offset=value.cursor.offset;if(value.cursor.sha256!==row.body_sha256||!Number.isSafeInteger(offset)||offset<=0||offset>=row.body.length||offset%3072)fail('open_invalid_mailbox_cursor');}const stop=Math.min(offset+3072,row.body.length); + return {state:'mailbox_replica_configuration',receiver_id:value.receiver_id,replica_configuration_sha256:row.body_sha256,total_bytes:row.body.length,offset,replica_configuration_chunk:Buffer.from(row.body.subarray(offset,stop)).toString('base64'),next_cursor:stoptransaction(db,()=>{const old=db.prepare(`SELECT body FROM ${TABLE} WHERE receiver_id=?`).get(receiver) as Obj|undefined;if(old){if(!Buffer.from(old.body).equals(Buffer.from(raw)))fail('open_mailbox_receiver_conflict');return;} + if((db.prepare(`SELECT count(*) AS n FROM ${TABLE}`).get() as Obj).n>=16||(db.prepare(`SELECT count(*) AS n FROM ${TABLE} WHERE envelope_sha256=?`).get(c.request.expected_envelope_ref.raw_sha256) as Obj).n>=2)fail('open_mailbox_receiver_capacity'); + db.prepare(`INSERT INTO ${TABLE}(receiver_id,selection_digest,envelope_sha256,body,body_sha256) VALUES(?,?,?,?,?)`).run(receiver,selection,c.request.expected_envelope_ref.raw_sha256,raw,sha256(raw));})); + return {state:'registered',receiver_id:receiver,network_accessed:false,recovery:'ordinary_receive',receipt_return:'separate_authority_required'}; + } + async receive(value:Obj,deadline:number,accessed?:()=>void):Promise{ + const c=this.#checked(value,false),{e,owner,setup,journal,known,policy,budget}=c; + const needs=['slot','read','maintenance','bootstrap'].map(n=>({signer:owner.signing_key,scope_kind:n==='slot'?'mailbox_slot':'authority',scope_id:statusScope(e.slot.root_key,n==='slot'?'mailbox_slot':'authority',n==='slot'?e.slot:{authority_kind:setup[n].payload.kind,authority_sha256:setup[n].ref.raw_sha256},policy,budget),revision:setup[n].payload.revision,mask:n==='read'?2:10}));replicaFloors(known,[],needs); + const saved=this.#participant.providerStorage(db=>db.prepare("SELECT 1 FROM open_delivery_inbox WHERE envelope_sha256=? AND phase!='staged'").get(c.request.expected_envelope_ref.raw_sha256));if(saved)return {messages:[],errors:[],state:'observed',network_accessed:false,body_transport:'mailbox_message_replica'}; + accessed?.();const introduction=(await this.#participant.transport.requestNode(value.base_url,deadline)).response,node=verifyNode(introduction); + if(node.status!=='active'||!same(node.signing_key,e.target.signing_key)||node.storage_epoch!==c.node.payload.storage_epoch||node.revision; +const SCHEMA='memory-vault-open-mailbox-connect/v1',REPAIR='memory-vault-open-repair/v1'; +export const MAILBOX_SEND_ACTIONS=new Set(['prepare','admit','retain']); +const SLOT_FIELDS={ + slot:'issued_at expires_at revision slot_key feed_ref sender recipient data_resource_ref data_resource_offer_ref metadata_resource_ref metadata_resource_offer_ref read_grant_ref maintenance_root_ref max_appends max_live_items budget windows', + read:'issued_at expires_at grant_id root_key slot_key reader serving_authority_id operation_mask budget windows revision', + maintenance:'issued_at expires_at root_key authority_id slot_key sender recipient maintainers operation_mask allowed_roles budget windows max_delegate_depth max_destinations_per_job max_concurrent_jobs revision'}; +const ACK_ROLES=['ack.root_authority','ack.write_grant','bootstrap.ack_offer','historical.status.ack_root','historical.status.ack_write','historical.status.ack_offer_bootstrap']; +function fail(code:string):never{throw new RepairError(code);} +function same(a:unknown,b:unknown):boolean{return Buffer.from(canonicalBytes(a)).equals(Buffer.from(canonicalBytes(b)));} +/** Shares Python's original-byte journals. Network retries never renew a grant. */ +export class MailboxSender{ + readonly #participant:OpenParticipant;readonly #encryption:EncryptionIdentityDocument; + readonly #policy={...DEFAULT_REPAIR_CLIENT_POLICY,max_signature_checks:512}; + constructor(participant:OpenParticipant,encryption:EncryptionIdentityDocument){this.#participant=participant;this.#encryption=encryption;} + #entry(value:unknown,budget:RepairBudget):Obj{ + const e=objectFields(value,['raw','ref']);if(typeof e.raw!=='string')fail('open_invalid_mailbox_request'); + return this.#checked(Buffer.from(e.raw,'utf8'),e.ref,budget); + } + #checked(raw:Uint8Array,reference:unknown,budget:RepairBudget):Obj{ + const ref=rawRef(reference);if(raw.length>524288||raw.length!==ref.size||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch');return {raw,ref}; + } + #decode(value:unknown,budget:RepairBudget):Obj{const e=objectFields(value,['ref','raw_base64url']);return this.#checked(decodeBase64url(e.raw_base64url,524288),e.ref,budget);} + #encode(e:Obj):Obj{return {ref:e.ref,raw_base64url:encodeBase64url(e.raw)};} + #raw(raw:Uint8Array,budget:RepairBudget):Obj{const digest=budget.hash(raw);return {raw,ref:{namespace:'meta',key:digest,raw_sha256:digest,size:raw.length}};} + #dual(value:Obj,budget:RepairBudget):Obj{objectFields(value,['signing_key','encryption_key']);originalPublicDescriptor(value.signing_key,budget);originalPublicDescriptor(value.encryption_key,budget,true);return {signing_key_id:value.signing_key.key_id,encryption_key_id:value.encryption_key.key_id};} + #ack(requestId:string,messageId:string,budget:RepairBudget):Obj{ + opaqueId(requestId);const row=this.#participant.providerStorage(db=>{ + if(!db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='open_ack_agent_preparations'").get())return undefined; + return db.prepare('SELECT result,result_sha256 FROM open_ack_agent_preparations WHERE request_id=?').get(requestId); + }) as Obj|undefined; + if(!row?.result)fail('open_ack_preparation_incomplete');if(row.result.length>1048576||sha256(row.result)!==row.result_sha256)fail('open_ack_preparation_corrupt'); + const prepared=document(row.result,1048576) as Obj;if(prepared.message_id!==messageId)fail('open_ack_preparation_conflict'); + const artifact=objectFields(prepared.recipient_request,['schema_version','base_url','repair_profile','request']) as Obj; + if(artifact.schema_version!=='memory-vault-open-saved-ack-request/v1'||!['receipt','receipt-index'].includes(artifact.repair_profile))fail('repair_invalid_request_bundle'); + const bundle=objectFields(artifact.request,['message_id','envelope_ref','ack_slot','owner','target','target_node_entry','root_entry','write_entry','bootstrap_entry','binding_entry','current_statuses','read_until','retain_until']) as Obj,roles:Obj={}; + if(bundle.message_id!==messageId)fail('open_ack_preparation_conflict'); + for(const [name,field] of [['ack.root_authority','root_entry'],['ack.write_grant','write_entry'],['bootstrap.ack_offer','bootstrap_entry']])roles[name]=this.#decode(bundle[field],budget); + if(!Array.isArray(bundle.current_statuses)||bundle.current_statuses.length>64)fail('open_ack_preparation_corrupt'); + const statuses=bundle.current_statuses.map((v:Obj)=>this.#decode(v,budget)); + for(const [name,role] of [['ack.root_authority','ack_root'],['ack.write_grant','ack_write'],['bootstrap.ack_offer','ack_offer_bootstrap']]){ + const entry=roles[name],p=(parseNewWire(entry.raw,this.#policy,budget).value as Obj).payload; + const scope=statusScope(bundle.ack_slot.root_key,'authority',{authority_kind:p.kind,authority_sha256:entry.ref.raw_sha256},this.#policy,budget); + const matches=statuses.filter((v:Obj)=>(parseNewWire(v.raw,this.#policy,budget).value as Obj).payload.entries.some((r:Obj)=>r.scope_kind==='authority'&&r.scope_id===scope)); + if(matches.length!==1)fail('open_ack_configuration_status_missing');roles['historical.status.'+role]=matches[0]; + } + return roles; + } + #prepare(value:Obj):Obj{ + objectFields(value,['schema_version','action','message_id','slot_entries','destination_entry','attempt_until','consent_until',...(Object.hasOwn(value,'ack_request_id')?['ack_request_id']:[])]); + opaqueId(value.message_id);const budget=new RepairBudget(this.#policy),at=Math.floor(Date.now()/1000); + const row=this.#participant.providerStorage(db=>db.prepare('SELECT envelope,session FROM open_delivery_outbox WHERE message_id=?').get(value.message_id)) as Obj|undefined; + if(!row?.envelope||!row.session)fail('open_delivery_outbox_missing'); + const session=document(row.session,65536) as Obj,request=session.request.payload,recipient={signing_key:session.policy.payload.signing_key,encryption_key:session.policy.payload.encryption_key}; + const sender={signing_key:validateSigningIdentity(this.#participant.identity),encryption_key:validateEncryptionIdentity(this.#encryption)},senderIds=this.#dual(sender,budget),recipientIds=this.#dual(recipient,budget); + const envelope=verifyEnvelope(row.envelope,{sender_signing_key:sender.signing_key,sender_encryption_key:sender.encryption_key,recipient_signing_key:recipient.signing_key,recipient_encryption_key:recipient.encryption_key,now:at}); + if(!same(request.signing_key,sender.signing_key)||!same(request.encryption_key,sender.encryption_key)||envelope.context.message_id!==value.message_id)fail('repair_message_mismatch'); + const reference={...this.#raw(row.envelope,budget).ref,namespace:'object',key:envelope.context.object_ref.key}; + objectFields(value.slot_entries,['slot','read','maintenance']);const slots=Object.fromEntries(Object.entries(value.slot_entries).map(([n,e])=>[n,this.#entry(e,budget)])); + const destinationEntry=this.#entry(value.destination_entry,budget); + const check=(e:Obj,kind:string,names:string):Obj=>{ + if(e.ref.namespace!=='meta')fail('repair_ref_mismatch');const v=objectFields(parseNewWire(e.raw,this.#policy,budget).value,['payload','proof']),p=objectFields(v.payload,['schema_version','kind','signing_key',...names.split(' ')]) as Obj; + if(p.schema_version!==REPAIR||p.kind!==kind)fail('repair_invalid_message');verifyBoundedControlSignature(p,v.proof,recipient.signing_key,budget);if(u53(p.issued_at)>=u53(p.expires_at))fail('repair_resource_expired');return p; + }; + const slot=check(slots.slot,'mailbox.slot',SLOT_FIELDS.slot),read=check(slots.read,'mailbox.read_grant',SLOT_FIELDS.read),maintenance=check(slots.maintenance,'mailbox.maintenance_root',SLOT_FIELDS.maintenance); + const key=slot.slot_key,root=key.root_key;objectFields(key,['root_key','slot_id','writer','writer_storage_epoch']);opaqueId(key.slot_id);opaqueId(key.writer_storage_epoch); + objectFields(root,['root_kind','owner','owner_epoch','root_id','anchor_ref']);opaqueId(root.owner_epoch);opaqueId(root.root_id);objectFields(root.anchor_ref,['namespace','key']); + if(root.root_kind!=='mailbox'||root.anchor_ref.namespace!=='anchor'||!/^[a-f0-9]{64}$/.test(root.anchor_ref.key))fail('repair_message_mismatch'); + const destination=check(destinationEntry,'delivery.destination','issued_at expires_at destination_id sender recipient contact_request_ref contact_policy_ref contact_knock_lease_ref contact_decision_ref store_grant_ref slot_key slot_ref data_resource_ref data_resource_offer_ref metadata_resource_ref metadata_resource_offer_ref read_grant_ref maintenance_root_ref budget windows'); + if(!same(root.owner,recipientIds)||[slot,maintenance,destination].some(p=>!same(p.sender,senderIds)||!same(p.recipient,recipientIds))||!same(read.reader,recipientIds)|| + [read,maintenance,destination].some(p=>!same(p.slot_key,key))||!same(read.root_key,root)||!same(maintenance.root_key,root)||read.serving_authority_id!==maintenance.authority_id|| + !same(destination.slot_ref,slots.slot.ref)||!same(slot.read_grant_ref,slots.read.ref)||!same(slot.maintenance_root_ref,slots.maintenance.ref))fail('repair_message_mismatch'); + for(const n of ['data_resource_ref','data_resource_offer_ref','metadata_resource_ref','metadata_resource_offer_ref','read_grant_ref','maintenance_root_ref'])if(!same(destination[n],slot[n]))fail('repair_message_mismatch'); + for(const [field,names] of [['budget','max_live_bytes max_meta_bytes max_items max_requests max_pending max_replay_records max_jobs max_job_bytes'],['windows','admit_until read_until copy_until publish_until retain_until']]){ + objectFields(destination[field],names.split(' '));for(const n of names.split(' '))if(u53(destination[field][n])>u53(slot[field][n])||(field==='windows'&&destination[field][n]>destination.windows.retain_until))fail('repair_message_mismatch'); + } + const docs:Obj=Object.fromEntries(['node','policy','request','decision'].map(n=>[n,session[n]]));docs.knock_lease=session.lease;docs.grant=session.decision.payload.grant;docs.delivery_lease=docs.grant.payload.resource_lease; + const contact=Object.fromEntries(Object.entries(docs).map(([n,v])=>[n,canonicalBytes(v)])); + const held=verifyContactOriginals(contact,{senderKeyId:sender.signing_key.key_id,senderEncryptionKeyId:sender.encryption_key.key_id,recipientKeyId:recipient.signing_key.key_id,recipientEncryptionKeyId:recipient.encryption_key.key_id,nodeKeyId:key.writer.signing_key_id,storageEpoch:key.writer_storage_epoch,at:destination.issued_at,policy:this.#policy,budget}); + for(const [n,role] of [['contact_request_ref','request'],['contact_policy_ref','policy'],['contact_knock_lease_ref','knock_lease'],['contact_decision_ref','decision'],['store_grant_ref','grant']]){ + const ref=rawRef(destination[n]),raw=held.originals[role].document.raw;if(ref.raw_sha256!==budget.hash(raw)||ref.size!==raw.length)fail('repair_ref_mismatch'); + } + u53(value.attempt_until);u53(value.consent_until);const ack=Object.hasOwn(value,'ack_request_id')?this.#ack(value.ack_request_id,value.message_id,budget):undefined; + if(ack)verifyMailboxAckConfiguration(ack,{sender,recipient,messageId:value.message_id,envelopeRef:reference,at,policy:this.#policy,budget}); + const encodedAck=ack?Object.fromEntries(Object.entries(ack).map(([n,e])=>[n,this.#encode(e)])):undefined; + const binding=sha256(canonicalBytes({envelope:reference,recipient,slot_refs:Object.fromEntries(Object.entries(slots).map(([n,e])=>[n,e.ref])),destination_ref:destinationEntry.ref, + contact_hashes:Object.fromEntries(Object.entries(contact).map(([n,raw])=>[n,sha256(raw)])),attempt_until:value.attempt_until,consent_until:value.consent_until,...(ack?{ack_configuration:encodedAck}:{})})); + const raw=this.#participant.providerStorage(db=>transaction(db,()=>{ + db.exec('CREATE TABLE IF NOT EXISTS open_mailbox_message_drafts(sender TEXT NOT NULL,message_id TEXT NOT NULL,binding TEXT NOT NULL,root_digest TEXT NOT NULL,status_revision INTEGER NOT NULL,envelope BLOB NOT NULL,originals BLOB NOT NULL,PRIMARY KEY(sender,message_id))'); + const old=db.prepare('SELECT binding,envelope,originals FROM open_mailbox_message_drafts WHERE sender=? AND message_id=?').get(sender.signing_key.key_id,value.message_id) as Obj|undefined; + if(old){if(old.binding!==binding||!Buffer.from(old.envelope).equals(Buffer.from(row.envelope)))fail('repair_message_conflict');return old.originals;} + if((db.prepare('SELECT count(*) AS n FROM open_mailbox_message_drafts').get() as Obj).n>=16)fail('repair_message_capacity'); + if(!(envelope.context.created_at<=at&&at!(p.issued_at<=at&&atsignBoundedBootstrapOriginal({schema_version:REPAIR,kind,signing_key:sender.signing_key,issued_at:at,expires_at:until,...fields},this.#participant.identity,this.#policy,budget); + const consent=sign('message.disclosure',{consent_id:'consent_'+binding,root_key:root,slot_key:key,sender:senderIds,recipient:recipientIds,envelope_ref:reference,maintenance_root_ref:slots.maintenance.ref, + allowed_roles:['contact.request','delivery.attempt','message.disclosure','authority.status.disclosure',...(ack?ACK_ROLES:[])].sort(),operation_mask:127,consent_until:value.consent_until, + bootstrap_return:{subject:recipientIds,consumer:'mailbox_feed',roles:['authority.status.disclosure','message.disclosure'],until:value.consent_until},revision:1},value.consent_until); + const attempt=sign('delivery.attempt',{attempt_id:'attempt_'+binding,message_id:value.message_id,envelope_ref:reference,sender:senderIds,recipient:recipientIds,destination_ref:destinationEntry.ref,slot_key:key, + operation:'message.store',disclosure_ref:consent.ref,ack_grant_ref:ack?.['ack.write_grant'].ref??null},value.attempt_until); + const scope=statusScope(root,'authority',{authority_kind:'message.disclosure',authority_sha256:consent.ref.raw_sha256},this.#policy,budget); + const observed=this.#raw(buildNewWire(issueStatus(this.#participant.identity,{root,revision,entries:[{scope_kind:'authority',scope_id:scope,minimum_document_revision:1,status:'active',operation_mask:127}],issued_at:at,valid_until:value.consent_until}),this.#policy,budget).raw,budget); + const bundle={disclosure:this.#encode(consent),disclosure_status:this.#encode(observed),attempt:this.#encode(attempt),destination:this.#encode(destinationEntry),slot:Object.fromEntries(Object.entries(slots).map(([n,e])=>[n,this.#encode(e)])), + contact:Object.fromEntries(Object.entries(contact).map(([n,bytes])=>[n,this.#encode(this.#raw(bytes,budget))])),...(ack?{ack_configuration:encodedAck}:{})}; + const encoded=buildNewWire(bundle,this.#policy,budget).raw;if(encoded.length>131072)fail('repair_message_capacity'); + db.prepare('INSERT OR REPLACE INTO open_mailbox_message_status_revisions VALUES(?,?,?)').run(sender.signing_key.key_id,rootDigest,revision); + db.prepare('INSERT INTO open_mailbox_message_drafts VALUES(?,?,?,?,?,?,?)').run(sender.signing_key.key_id,value.message_id,binding,rootDigest,revision,row.envelope,encoded);return encoded; + })); + const saved=parseNewWire(raw,this.#policy,new RepairBudget(this.#policy)).value as Obj; + return {state:'mailbox_draft_saved',message_id:value.message_id,attempt_ref:this.#decode(saved.attempt,budget).ref,disclosure_ref:this.#decode(saved.disclosure,budget).ref,network_accessed:false}; + } + async #admit(value:Obj):Promise{ + objectFields(value,['schema_version','action','base_url','message_id','target','target_node_entry','owner_status_entry','expires_at','object_until','enum_until']);opaqueId(value.message_id); + const budget=new RepairBudget(this.#policy),at=Math.floor(Date.now()/1000),sender={signing_key:validateSigningIdentity(this.#participant.identity),encryption_key:validateEncryptionIdentity(this.#encryption)}; + const nodeEntry=this.#entry(value.target_node_entry,budget),nodePreview=(parseNewWire(nodeEntry.raw,this.#policy,budget).value as Obj).payload; + const node=verifySourceNodeOriginal(nodeEntry.raw,{expectedSigningKey:value.target.signing_key,expectedStorageEpoch:nodePreview.storage_epoch,at,policy:this.#policy,budget}); + if(nodeEntry.ref.namespace!=='meta'||!same(endpoint(value.base_url,this.#participant.transport.allow_loopback),endpoint(node.payload.base_url as string,this.#participant.transport.allow_loopback)))fail('repair_probe_mismatch'); + const packet=this.#participant.providerStorage(db=>transaction(db,()=>{ + if(!db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='open_mailbox_message_drafts'").get())fail('repair_message_delivery_missing'); + const row=db.prepare('SELECT originals FROM open_mailbox_message_drafts WHERE sender=? AND message_id=?').get(sender.signing_key.key_id,value.message_id) as Obj|undefined;if(!row)fail('repair_message_delivery_missing'); + const raw=Buffer.from(row.originals);if(raw.length>131072)fail('repair_message_capacity');const draft=parseNewWire(raw,this.#policy,budget).value as Obj; + const slot=this.#decode(draft.slot.slot,budget),key=(parseNewWire(slot.raw,this.#policy,budget).value as Obj).payload.slot_key; + if(!same(this.#dual(value.target,budget),key.writer)||node.payload.storage_epoch!==key.writer_storage_epoch)fail('repair_probe_mismatch'); + const plain={raw:raw.toString('utf8'),ref:this.#raw(raw,budget).ref};let encoded:Obj=plain; + if(draft.ack_configuration){const compressed=deflateSync(raw);budget.output(compressed.length);encoded={codec:'zlib-base64url-v1',compressed_size:compressed.length,raw_base64url:encodeBase64url(compressed),ref:plain.ref};} + const payload={schema_version:REPAIR,kind:'mailbox.source_message',signing_key:sender.signing_key,subject:sender,target:value.target,target_storage_epoch:key.writer_storage_epoch,slot_key:key,message_id:value.message_id, + draft:encoded,owner_status:this.#encode(this.#entry(value.owner_status_entry,budget)),object_until:value.object_until,enum_until:value.enum_until}; + const binding=budget.hash(buildNewWire({...payload,draft:plain},this.#policy,budget).raw); + db.exec('CREATE TABLE IF NOT EXISTS open_mailbox_message_requests(sender TEXT NOT NULL,message_id TEXT NOT NULL,binding TEXT NOT NULL,raw BLOB NOT NULL,PRIMARY KEY(sender,message_id))'); + const prior=db.prepare('SELECT binding,raw FROM open_mailbox_message_requests WHERE sender=? AND message_id=?').get(sender.signing_key.key_id,value.message_id) as Obj|undefined; + let requestRaw:Uint8Array; + if(prior){if(prior.binding!==binding)fail('repair_message_conflict');requestRaw=prior.raw;} + else{ + if(!(at65536)fail('repair_remote_setup_too_large');db.prepare('INSERT INTO open_mailbox_message_requests VALUES(?,?,?,?)').run(sender.signing_key.key_id,value.message_id,binding,requestRaw); + } + return {raw:requestRaw,payload,draft}; + })); + const response=await this.#participant.transport.requestRepair(value.base_url,packet.raw,performance.now()/1000+30); + if(response.length>65536)fail('repair_remote_setup_too_large');const signed=objectFields(parseNewWire(response,this.#policy,budget).value,['payload','proof']); + const p=objectFields(signed.payload,['schema_version','kind','signing_key','request_sha256','slot_key','message_id','stored_at','originals']) as Obj; + if(p.schema_version!==REPAIR||p.kind!=='mailbox.source_message_stored'||p.request_sha256!==budget.hash(packet.raw)||!same(p.slot_key,packet.payload.slot_key)||p.message_id!==value.message_id||!same(p.signing_key,value.target.signing_key))fail('repair_message_mismatch'); + u53(p.stored_at);verifyBoundedControlSignature(p,signed.proof,value.target.signing_key,budget); + const kinds={core:'admission.core',link:'admission.link',custody:'message.custody',head:'mailbox.feed_head',feed_custody:'feed.custody'},entries:Obj={},payloads:Obj={};objectFields(p.originals,Object.keys(kinds)); + for(const [n,kind] of Object.entries(kinds)){ + const e=this.#decode(p.originals[n],budget),v=objectFields(parseNewWire(e.raw,this.#policy,budget).value,['payload','proof']),item=v.payload as Obj; + if(item.schema_version!==REPAIR||item.kind!==kind||!same(item.slot_key,packet.payload.slot_key))fail('repair_message_mismatch');verifyBoundedControlSignature(item,v.proof,value.target.signing_key,budget);entries[n]=e;payloads[n]=item; + } + const core=payloads.core,custody=payloads.custody,attempt=this.#decode(packet.draft.attempt,budget),ap=(parseNewWire(attempt.raw,this.#policy,budget).value as Obj).payload; + if(core.message_id!==value.message_id||custody.message_id!==value.message_id||core.object_until!==value.object_until||core.enum_until!==value.enum_until||!same(core.attempt_ref,attempt.ref)||!same(core.envelope_ref,ap.envelope_ref)|| + !same(custody.envelope_ref,core.envelope_ref)||!same(payloads.link.envelope_ref,core.envelope_ref)||payloads.link.message_id!==value.message_id||!same(payloads.feed_custody.feed_head_ref,entries.head.ref)||!same(custody.admission_link_ref,entries.link.ref)||!same(custody.feed_head_ref,entries.head.ref))fail('repair_message_mismatch'); + this.#participant.providerStorage(db=>transaction(db,()=>{ + db.exec('CREATE TABLE IF NOT EXISTS open_mailbox_message_results(sender TEXT NOT NULL,message_id TEXT NOT NULL,raw BLOB NOT NULL,PRIMARY KEY(sender,message_id))'); + const old=db.prepare('SELECT raw FROM open_mailbox_message_results WHERE sender=? AND message_id=?').get(sender.signing_key.key_id,value.message_id) as Obj|undefined; + if(old&&!Buffer.from(old.raw).equals(Buffer.from(response)))fail('repair_message_conflict');db.prepare('INSERT OR IGNORE INTO open_mailbox_message_results VALUES(?,?,?)').run(sender.signing_key.key_id,value.message_id,response); + })); + return {state:'retained_at_mailbox',message_id:value.message_id,stored_at:p.stored_at,network_accessed:true,custody_ref:entries.custody.ref,feed_custody_ref:entries.feed_custody.ref,recipient_acknowledged:false}; + } + async connect(invitation:unknown):Promise{ + const v=document(invitation as any,65536) as Obj;if(v.schema_version!==SCHEMA)fail('open_invalid_mailbox_request'); + if(v.action==='prepare')return this.#prepare(v);if(v.action==='admit')return this.#admit(v); + objectFields(v,['schema_version','action','message_id','authorization','attempt_until','consent_until','expires_at','object_until','enum_until',...(Object.hasOwn(v,'ack_request_id')?['ack_request_id']:[])]); + if(v.action!=='retain')fail('open_invalid_mailbox_request');const a=objectFields(v.authorization,['destination_entry','owner_status_entry','slot_entries','target','target_node_entry','base_url']); + this.#prepare({schema_version:SCHEMA,action:'prepare',message_id:v.message_id,slot_entries:a.slot_entries,destination_entry:a.destination_entry,attempt_until:v.attempt_until,consent_until:v.consent_until,...(Object.hasOwn(v,'ack_request_id')?{ack_request_id:v.ack_request_id}:{})}); + return this.#admit({schema_version:SCHEMA,action:'admit',message_id:v.message_id,...Object.fromEntries(['base_url','target','target_node_entry','owner_status_entry'].map(n=>[n,a[n]])),...Object.fromEntries(['expires_at','object_until','enum_until'].map(n=>[n,v[n]]))}); + } +} diff --git a/clients/typescript/network/open-repair-mailbox-inbox.ts b/clients/typescript/network/open-repair-mailbox-inbox.ts index c2d1c63..5e406c3 100644 --- a/clients/typescript/network/open-repair-mailbox-inbox.ts +++ b/clients/typescript/network/open-repair-mailbox-inbox.ts @@ -6,12 +6,14 @@ import {decodeBase64url} from './crypto.ts'; import type {EncryptionIdentityDocument} from './crypto.ts'; import {verifyMailboxFeedSourceEvent} from './open-repair-mailbox-feed.ts'; import {readMailboxIndex,readMailboxAdmission} from './open-repair-mailbox-read.ts'; +import {verifyMailboxReplicaInboxEvidence} from './open-repair-mailbox-replica-inbox.ts'; type Obj=Record; const fields=(v:unknown,n:readonly string[]):Obj=>objectFields(v,n); const key=(value:unknown)=>{const r=rawRef(value);return [r.namespace,r.key,r.raw_sha256,r.size].join(':');}; function fail(code:string):never{throw new RepairError(code);} function same(a:unknown,b:unknown):boolean{if(a===b)return true;if(!a||!b||typeof a!=='object'||typeof b!=='object'||Array.isArray(a)!==Array.isArray(b))return false;const keys=Object.keys(a);return keys.length===Object.keys(b).length&&keys.every(k=>Object.hasOwn(b,k)&&same((a as Obj)[k],(b as Obj)[k]));} export async function verifyMailboxInboxEvidence(evidence:unknown,envelope:Uint8Array,options:{owner:unknown;encryptionIdentity:EncryptionIdentityDocument;stagedAt:number}):Promise>{ + if((evidence as Obj)?.schema_version==='memory-vault-mailbox-replica-inbox/v1')return verifyMailboxReplicaInboxEvidence(evidence,envelope,options); const a=fields(options,['owner','encryptionIdentity','stagedAt']); const value=fields(evidence,['schema_version','received_at','slot','sender','target','limits','member','manifest_ref','custody_ref','originals','status_refs']); if(value.schema_version!=='memory-vault-mailbox-inbox/v1')fail('repair_invalid_context'); diff --git a/clients/typescript/network/open-repair-mailbox-member.ts b/clients/typescript/network/open-repair-mailbox-member.ts index aea9499..937dfab 100644 --- a/clients/typescript/network/open-repair-mailbox-member.ts +++ b/clients/typescript/network/open-repair-mailbox-member.ts @@ -90,6 +90,12 @@ function ackConfiguration(roles:Obj,e:Obj,m:Obj,policy:RepairPolicy,budget:Repai duties.push({role,scope_kind:'authority',scope_id:statusScope(root,'authority',{authority_kind:item.payload.kind,authority_sha256:item.ref.raw_sha256},policy,budget),document_revision:item.payload.revision,operation_mask:mask,signer:e.sender.signing_key});} return Object.freeze({originals:checked.originals,statuses:historicalStatuses(roles,duties,root,e.at,policy,budget)}); } +/** Sender preparation verifies the same complete independent receipt originals. */ +export function verifyMailboxAckConfiguration(roles:Obj,options:{sender:Obj;recipient:Obj;messageId:string;envelopeRef:unknown;at:number;policy:RepairPolicy;budget:RepairBudget}):Obj{ + if(!roleSet(roles,ACK))fail('repair_original_missing'); + return ackConfiguration(roles,{sender:options.sender,owner:options.recipient,at:options.at}, + {message_id:options.messageId,envelope_ref:options.envelopeRef},options.policy,options.budget); +} function member(resolved:DraftHistoryInputs,e:Obj,policy:RepairPolicy,budget:RepairBudget):Obj{ const graph=setup(resolved,e,policy,budget),{roles,originals}=graph,m=resolved.manifest.value as Obj,root=m.root_key,key=m.slot_key,at=e.at; if(!roleSet(roles,BASE)&&!roleSet(roles,[...BASE,...ACK]))fail(); diff --git a/clients/typescript/network/open-repair-mailbox-read.ts b/clients/typescript/network/open-repair-mailbox-read.ts index 8f7c534..b9cbfcf 100644 --- a/clients/typescript/network/open-repair-mailbox-read.ts +++ b/clients/typescript/network/open-repair-mailbox-read.ts @@ -105,6 +105,18 @@ function currentRead(setup:Obj,entries:readonly unknown[],known:readonly unknown /** Authenticate/decrypt metadata and check present authority before fetching * the envelope. The enclosing verified feed determines which member to select. */ export async function readMailboxAdmission(member:unknown,options:MailboxAdmissionReadOptions):Promise>{ + const metadata=await verifyMailboxAdmissionMetadata(member,options); + const {policy,budget}=options; + const statuses=currentRead(metadata.setup,options.currentStatuses,options.knownStatuses,options.statusObligations,options.at,policy,budget,options.onStatusAuthenticated); + const ref=rawRef(metadata.core.envelope_ref),raw=await options.readOriginal(ref); + if(!isUint8Array(raw)||ref.namespace!=='object'||ref.size>6291456||raw.length!==ref.size||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch'); + budget.input(raw.length);budget.retain(raw.length);const envelope=Uint8Array.from(raw); + return Object.freeze({...metadata,current_statuses:statuses,originals:Object.freeze([...metadata.originals,{ref,raw:envelope}]),get envelope(){budget.output(envelope.length);return Uint8Array.from(envelope);}}); +} +/** Original source authentication and recipient decryption, without asserting + * current source READ. Replica callers must establish independent live READ + * before fetching the envelope; this function performs no object fetch. */ +export async function verifyMailboxAdmissionMetadata(member:unknown,options:MailboxAdmissionReadOptions):Promise>{ const names=['expectedSlot','expectedSigningKey','encryptionIdentity','readOriginal','at','policy','budget','expectedOwner','expectedSender','expectedTarget','limitPolicy','currentStatuses','knownStatuses','statusObligations']; const a=fields(options,Object.hasOwn(options,'onStatusAuthenticated')?[...names,'onStatusAuthenticated']:names),{policy,budget}=a; const e=buildNewWire({slot:a.expectedSlot,key:a.expectedSigningKey,owner:a.expectedOwner,sender:a.expectedSender,target:a.expectedTarget,limits:a.limitPolicy,at:a.at,identity:a.encryptionIdentity,member},policy,budget).value as Obj; @@ -144,8 +156,6 @@ export async function readMailboxAdmission(member:unknown,options:MailboxAdmissi for(const n of ['slot','read','maintenance','bootstrap']){const p=setup.originals[n].payload;deadlines.push(p.expires_at);if(p.windows)deadlines.push(p.windows.read_until,p.windows.retain_until);if(n==='bootstrap')deadlines.push(p.probe_until,p.proof_until,p.upload_until);} for(const n of ['data','metadata']){const p=setup.resources[n].active.payload;deadlines.push(p.windows.read_until,p.windows.retain_until);} if(core.enum_until>Math.min(...deadlines))fail('repair_mailbox_member_mismatch'); - const statuses=currentRead(setup,a.currentStatuses,a.knownStatuses,a.statusObligations,e.at,policy,budget,a.onStatusAuthenticated); - const envelope=await load(core.envelope_ref,'object'); const retained=Object.freeze([...originals.values()].map(v=>Object.freeze({ref:v.ref,get raw(){budget.output(v.raw.length);return Uint8Array.from(v.raw);}}))); - return Object.freeze({core,link,checkpoint,setup,history:manifest,current_statuses:statuses,originals:retained,get envelope(){budget.output(envelope.length);return Uint8Array.from(envelope);}}); + return Object.freeze({core,link,checkpoint,setup,history:manifest,originals:retained}); } diff --git a/clients/typescript/network/open-repair-mailbox-replica-inbox.ts b/clients/typescript/network/open-repair-mailbox-replica-inbox.ts new file mode 100644 index 0000000..41a4b92 --- /dev/null +++ b/clients/typescript/network/open-repair-mailbox-replica-inbox.ts @@ -0,0 +1,27 @@ +/** Verify the exact replica proof retained with an inbox entry at receipt time. + * Local reopening grants no later network access or renewed COPY/READ rights. */ +import {RepairBudget,LocalRawResolver,buildNewWire,parseNewWire,rawRef,u53} from './open-repair-wire.ts'; +import {DEFAULT_REPAIR_CLIENT_POLICY} from './open-repair-client.ts'; +import {decodeBase64url} from './crypto.ts'; +import {authenticateStatusOriginal} from './open-repair-status.ts'; +import {verifyMailboxAdmissionMetadata} from './open-repair-mailbox-read.ts'; +import {MAILBOX_REPLICA_LIMITS} from './open-mailbox-replica-client.ts'; +import {replicaFields as fields,replicaFail as fail,replicaSame as same,replicaRefKey as key,replicaFloors,verifyMailboxMessageReplica,checkMailboxMessageReturn} from './open-repair-mailbox-replica.ts'; +type Obj=Record; +export async function verifyMailboxReplicaInboxEvidence(evidence:unknown,envelope:Uint8Array,a:Obj):Promise>{ + fields(a,['owner','encryptionIdentity','stagedAt']);const v=fields(evidence,'schema_version received_at slot sender source target maintainer target_storage_epoch envelope_ref limits manifest_ref custody_ref consent_refs originals status_refs archive_status_refs'.split(' ')); + if(v.schema_version!=='memory-vault-mailbox-replica-inbox/v1'||u53(v.received_at)>u53(a.stagedAt))fail('repair_invalid_context');const at=v.received_at,policy=Object.freeze({...DEFAULT_REPAIR_CLIENT_POLICY,max_signature_checks:512}),budget=new RepairBudget(policy); + const limits=buildNewWire(v.limits,policy,budget).value as Obj;fields(limits,Object.keys(MAILBOX_REPLICA_LIMITS));for(const [name,maximum] of Object.entries(MAILBOX_REPLICA_LIMITS))if(u53(limits[name],1)>maximum)fail('repair_over_budget'); + if(!Array.isArray(v.originals)||v.originals.length<1||v.originals.length>limits.max_proof_items||!Array.isArray(v.status_refs)||v.status_refs.length<1||v.status_refs.length>16||!Array.isArray(v.archive_status_refs)||v.archive_status_refs.length>32)fail('repair_over_budget'); + const resolver=new LocalRawResolver(policy,budget),originals=new Map();let total=envelope.length; + for(const item of v.originals){fields(item,['ref','raw_base64url']);const ref=rawRef(item.ref);if(ref.namespace!=='meta'||ref.size>policy.max_document_bytes||originals.has(key(ref)))fail('repair_ref_mismatch');const raw=decodeBase64url(item.raw_base64url,policy.max_document_bytes);total+=raw.length;if(total>limits.max_proof_bytes)fail('repair_over_budget');if(!same(resolver.put(ref.namespace,ref.key,raw).ref,ref))fail('repair_ref_mismatch');originals.set(key(ref),{ref,raw});} + const entry=(ref:unknown)=>{const item=originals.get(key(ref));if(!item)fail('repair_original_missing');return item;}; + const context={expectedSlot:v.slot,expectedOwner:a.owner,expectedSender:v.sender,expectedSource:v.source,sourceStorageEpoch:v.slot.writer_storage_epoch,expectedMaintainer:v.maintainer,expectedTarget:v.target,targetStorageEpoch:v.target_storage_epoch,expectedEnvelopeRef:v.envelope_ref,limitPolicy:limits,policy,budget}; + const replica=verifyMailboxMessageReplica(entry(v.manifest_ref),resolver,entry(v.custody_ref),context);if(replica.custody.payload.stored_at>at)fail('repair_access_expired');fields(v.consent_refs,['owner','source','maintainer','sender']); + const permission=checkMailboxMessageReturn(replica,Object.fromEntries(Object.entries(v.consent_refs).map(([n,ref])=>[n,entry(ref)])),{...context,currentStatuses:v.status_refs.map(entry),at,action:'child'}); + const signers=new Map([a.owner,v.sender,v.source,v.maintainer,v.target].map(v=>[v.signing_key.key_id,v.signing_key])),archive:Obj[]=[]; + for(const ref of v.archive_status_refs){const saved=entry(ref),p=(parseNewWire(saved.raw,policy,budget).value as Obj).payload;if(!same(signers.get(p.signing_key.key_id),p.signing_key)||u53(p.issued_at)>at)fail('repair_status_disclosure');archive.push(authenticateStatusOriginal(saved,{expectedRoot:v.slot.root_key,expectedSigningKey:p.signing_key,at:p.issued_at,allowedScopes:p.entries.map((v:Obj)=>({scope_kind:v.scope_kind,scope_id:v.scope_id})),policy,budget}));} + replicaFloors(archive,permission.statuses,permission.obligations); + const checked=await verifyMailboxAdmissionMetadata(replica.source.message_member,{expectedSlot:v.slot,expectedSigningKey:v.source.signing_key,encryptionIdentity:a.encryptionIdentity,readOriginal:ref=>entry(ref).raw,at,policy,budget,expectedOwner:a.owner,expectedSender:v.sender,expectedTarget:v.source,limitPolicy:limits,currentStatuses:[],knownStatuses:[],statusObligations:[]}); + const ref=rawRef(checked.core.envelope_ref);if(!same(ref,v.envelope_ref)||envelope.length!==ref.size||budget.hash(envelope)!==ref.raw_sha256)fail('repair_ref_mismatch');return Object.freeze({...checked,envelope,originals:[...checked.originals,{ref,raw:envelope}],current_statuses:permission.statuses}); +} diff --git a/clients/typescript/network/open-repair-mailbox-replica.ts b/clients/typescript/network/open-repair-mailbox-replica.ts new file mode 100644 index 0000000..df51ade --- /dev/null +++ b/clients/typescript/network/open-repair-mailbox-replica.ts @@ -0,0 +1,170 @@ +/** Complete original message COPY closure and independent present READ. + * A target custody signature alone never authorizes disclosure or decryption. + * This verifier has no storage, transport, admission or publication effects. */ +import {buildNewWire,parseNewWire,objectFields,rawRef,u53,RepairError} from './open-repair-wire.ts'; +import type {RepairPolicy,RepairBudget,RawOriginal,LocalRawResolver} from './open-repair-wire.ts'; +import {verifyBoundedControlSignature} from './open-repair-original.ts'; +import {statusScope,authenticateStatusOriginal} from './open-repair-status.ts'; +import {verifyMailboxFeedSourceEvent} from './open-repair-mailbox-feed.ts'; +type Obj=Record; +const COMMON=['schema_version','kind','signing_key'],SCHEMA='memory-vault-open-repair/v1'; +const BUDGET='max_live_bytes max_meta_bytes max_items max_requests max_pending max_replay_records max_jobs max_job_bytes'.split(' '); +const WINDOWS='admit_until read_until copy_until publish_until retain_until'.split(' '); +const ALLOCATE='issued_at expires_at request_id target_node_key_id target_storage_epoch intent intent_sha256'; +const OFFER='issued_at reservation_until offer_id allocation_request_ref intent intent_sha256 resource target_encryption_key reservation_generation budget windows'; +const ASSIGNMENT='issued_at expires_at assignment_id job_id root_key parent_root_ref parent_assignment_ref depth subject target_node_key_id target_storage_epoch operation_mask scope resource_intent_sha256 resource_offer_ref resource bootstrap_grant_refs budget windows'; +const RESERVATION='issued_at expires_at consent_id revision variant root_authority_ref source_custody_ref historical_manifest_ref maintainer target target_storage_epoch reservation_disclosure'; +const DISCLOSURE='issued_at expires_at consent_id revision variant root_key assignment_ref source_custody_ref historical_manifest_ref target target_storage_epoch disclosure'; +const RETURN='issued_at expires_at consent_id revision variant root_key source_custody_ref historical_manifest_ref assignment_ref subject target target_storage_epoch bootstrap_grant_ref return_permission'; +const CUSTODY='root_key scope original_custody_ref replica_manifest_ref assignment_ref resource_offer_ref resource reservation_generation stored_at read_until retain_until'; +export const replicaFields=objectFields; +export function replicaFail(code='repair_copy_authority_mismatch'):never{throw new RepairError(code);} +export function replicaSame(a:unknown,b:unknown):boolean{if(a===b)return true;if(!a||!b||typeof a!=='object'||typeof b!=='object'||Array.isArray(a)!==Array.isArray(b))return false;const keys=Object.keys(a);return keys.length===Object.keys(b).length&&keys.every(k=>Object.hasOwn(b,k)&&replicaSame((a as Obj)[k],(b as Obj)[k]));} +export const replicaRefKey=(v:unknown)=>{const r=rawRef(v);return [r.namespace,r.key,r.raw_sha256,r.size].join(':');}; +const pair=(role:string,ref:unknown)=>role+'|'+replicaRefKey(ref); +const sorted=(values:T[],key:(v:T)=>string)=>values.sort((a,b)=>key(a)key(b)?1:0); +const require=(v:unknown,code='repair_copy_authority_mismatch')=>{if(!v)replicaFail(code);}; +const opaque=(v:unknown)=>{require(typeof v==='string'&&/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.exec(v)?.[0]===v);return v;}; +const ids=(v:Obj)=>({signing_key_id:v.signing_key.key_id,encryption_key_id:v.encryption_key.key_id}); +function windows(v:Obj,at?:number):void{objectFields(v,WINDOWS);for(const n of WINDOWS){u53(v[n]);require(v[n]<=v.retain_until&&(at===undefined||v[n]>at));}} +function capacity(v:Obj):void{objectFields(v,BUDGET);for(const n of BUDGET)u53(v[n]);} +function lifetime(v:Obj,at?:number):void{u53(v.issued_at);u53(v.expires_at);require(v.issued_atstatusScope(root,'authority',{authority_kind:item.payload.kind,authority_sha256:item.ref.raw_sha256},policy,budget); +const assignment=(root:unknown,item:Obj,policy:RepairPolicy,budget:RepairBudget)=>statusScope(root,'assignment',{assignment_kind:'maintenance.assignment',assignment_sha256:item.ref.raw_sha256},policy,budget); +const duty=(signer:Obj,scope_kind:string,scope_id:string,revision:number,mask:number)=>({signer,scope_kind,scope_id,revision,mask}); +const scopes=(values:Obj[])=>sorted([...new Map(values.map(v=>[v.scope_kind+':'+v.scope_id,{scope_kind:v.scope_kind,scope_id:v.scope_id}])).values()],v=>v.scope_kind+':'+v.scope_id); +/** Conflict, monotonically increasing floors, stale current views, and known + * revocations are checked across every authentic observation. */ +export function replicaFloors(previous:readonly Obj[],current:readonly Obj[],obligations:readonly Obj[]=[]):void{ + const required=new Map(obligations.map(v=>[[v.signer.key_id,v.scope_kind,v.scope_id].join(':'),v])),seen=new Map(),values=new Map(); + for(const item of [...previous,...current]){const p=item.payload,issuer=p.scope_key.issuer_key_id,revision=p.revision,k=issuer+':'+revision; + if(seen.has(k)&&seen.get(k)!==item.canonical_sha256)replicaFail('repair_status_conflict');seen.set(k,item.canonical_sha256); + for(const row of p.entries){const key=[issuer,row.scope_kind,row.scope_id].join(':'),need=required.get(key); + if(need&&row.status==='revoked'&&(row.operation_mask&need.mask))replicaFail('repair_authority_revoked'); + if(need&&row.minimum_document_revision>need.revision)replicaFail('repair_status_revision'); + const held=values.get(key)??[];held.push([revision,row.minimum_document_revision]);values.set(key,held);}} + for(const held of values.values()){let minimum=0;for(const [,v] of held.sort((a,b)=>a[0]-b[0]||a[1]-b[1])){if(vv[0]>p.revision))replicaFail('repair_status_rollback');} +} +function observations(entries:unknown,parties:Obj,allowed:Map,needs:Obj[],root:unknown,at:number,previous:Obj[],policy:RepairPolicy,budget:RepairBudget,onObserved?: (v:Obj)=>void):Obj[]{ + if(!Array.isArray(entries)||entries.length<1||entries.length>16)replicaFail('repair_status_missing');const checked:Obj[]=[],signers=new Map(Object.values(parties).map((v:Obj)=>[v.signing_key.key_id,v.signing_key]));let denial:unknown; + // Persist each authentic status even when a later supplied document is bad. + for(const entry of entries)try{const p=(parseNewWire((entry as Obj).raw,policy,budget).value as Obj).payload,key=p.signing_key.key_id; + if(!signers.has(key))replicaFail('repair_status_disclosure');authenticateStatusOriginal(entry,{expectedRoot:root,expectedSigningKey:signers.get(key),at,allowedScopes:scopes(allowed.get(key)??[]),policy,budget},v=>{checked.push(v);onObserved?.(v);}); + }catch(error){denial??=error;} + if(new Set(checked.map(v=>replicaRefKey(v.ref))).size!==checked.length)denial??=new RepairError('repair_duplicate_status'); + try{replicaFloors(previous,checked);}catch(error){denial??=error;} + for(const need of needs){const rows=checked.filter(v=>replicaSame(v.payload.signing_key,need.signer)).flatMap(v=>v.payload.entries).filter(v=>v.scope_kind===need.scope_kind&&v.scope_id===need.scope_id); + if(!rows.length)denial??=new RepairError('repair_status_missing');for(const row of rows){if(row.status==='revoked')denial??=new RepairError('repair_authority_revoked');else if(row.minimum_document_revision>need.revision)denial??=new RepairError('repair_status_revision');else if((row.operation_mask&need.mask)!==need.mask)denial??=new RepairError('repair_status_operation');}} + if(denial)throw denial;return checked; +} +function inventory(source:Obj,reservations:Obj,policy:RepairPolicy,budget:RepairBudget):Obj[]{ + const rows=new Map(),add=(role:string,item:Obj)=>{const p=(parseNewWire(item.raw,policy,budget).value as Obj).payload;rows.set(pair(role,item.ref),{role,original:item,issuer:p?.signing_key.key_id??''});}; + for(const tree of [source.manifest,...source.manifest.predecessors])for(const row of tree.roles)add(row.role,row.original); + add('feed.custody',source.feed_custody);add('message.custody',source.custody);add('copy.reservation_consent',reservations.owner);add('copy.sender_reservation_consent',reservations.sender); + return sorted([...rows.values()],v=>pair(v.role,v.original.ref)); +} +function permissions(rows:Obj[],issuer:string,root:unknown,mode:'copy'|'return',policy:RepairPolicy,budget:RepairBudget):Obj{ + const selected=rows.filter(v=>v.issuer===issuer),allowed:Obj[]=[]; + for(const row of selected){const p=(parseNewWire(row.original.raw,policy,budget).value as Obj).payload,item={...row.original,payload:p}; + if(p.kind==='authority.status')allowed.push(...p.entries.map((v:Obj)=>({scope_kind:v.scope_kind,scope_id:v.scope_id}))); + if(['copy.reservation_consent','copy.sender_reservation_consent'].includes(row.role)||mode==='return'&&['mailbox.message_copy_disclosure','mailbox.feed_copy_disclosure'].includes(p.kind))allowed.push({scope_kind:'authority',scope_id:authority(root,item,policy,budget)}); + if(mode==='return'&&p.kind==='maintenance.assignment')allowed.push({scope_kind:'assignment',scope_id:assignment(root,item,policy,budget)});} + return {originals:selected.map(v=>({role:v.role,ref:v.original.ref})),status_scopes:scopes(allowed)}; +} +export interface MessageReplicaOptions {expectedSlot:unknown;expectedOwner:unknown;expectedSender:unknown;expectedSource:unknown;sourceStorageEpoch:string;expectedMaintainer:unknown;expectedTarget:unknown;targetStorageEpoch:string;expectedEnvelopeRef:unknown;limitPolicy:unknown;policy:RepairPolicy;budget:RepairBudget;} +export function verifyMailboxMessageReplica(manifestEntry:unknown,resolver:LocalRawResolver,custodyEntry:unknown,a:MessageReplicaOptions):Readonly{ + const {policy,budget}=a;const e=buildNewWire({slot:a.expectedSlot,owner:a.expectedOwner,sender:a.expectedSender,source:a.expectedSource,maintainer:a.expectedMaintainer,target:a.expectedTarget},policy,budget).value as Obj; + const envelope=rawRef(a.expectedEnvelopeRef);require(envelope.namespace==='object'&&envelope.size>0&&envelope.size<=6291456); + const custody=signed(custodyEntry,e.target.signing_key,'replica.custody',CUSTODY,policy,budget),p=custody.payload;resource(p.resource); + const at=u53(p.stored_at);require(at0&&v.original_roles.length<=128,'repair_copy_commit_mismatch'); + const entries=new Map();let last=''; + for(const row of v.original_roles){objectFields(row,['role','ref']);require(typeof row.role==='string');const ref=rawRef(row.ref),k=pair(row.role,ref);require(!last||k>last,'repair_copy_commit_mismatch');last=k; + if(row.role==='message.envelope'){require(replicaSame(ref,envelope),'repair_copy_commit_mismatch');continue;} + const item=resolver.resolve(ref);entries.set(row.role,[...(entries.get(row.role)??[]),{raw:item.raw,ref:item.ref}]);} + const one=(role:string):RawOriginal=>{const values=entries.get(role);if(values?.length!==1)replicaFail('repair_copy_commit_mismatch');return values[0];}; + const feed=verifyMailboxFeedSourceEvent(one('history.mailbox_feed'),resolver,one('feed.custody'),{expectedSlot:e.slot,expectedOwner:e.owner,expectedSender:e.sender,expectedTarget:e.source,limitPolicy:a.limitPolicy,policy,budget}); + const selected=(feed.manifest.manifest.value as Obj).members.filter((v:Obj)=>replicaSame(v.envelope_ref,envelope));require(selected.length===1);const member=selected[0]; + const role=(name:string,ref:unknown):RawOriginal=>{const rows=feed.manifest.roles.filter((v:Obj)=>v.role===name&&replicaSame(v.original.ref,ref));require(rows.length===1);return rows[0].original;}; + const messageCustody=role('member.custody',member.source_custody_ref),mc=(parseNewWire(messageCustody.raw,policy,budget).value as Obj).payload,core=(parseNewWire(role('member.core',member.admission_core_ref).raw,policy,budget).value as Obj).payload; + const setup=feed.graph.members[member.sequence];require(setup.attempt.message_id===member.message_id&&replicaSame(mc.envelope_ref,envelope)); + const active=setup.resources.data.active.payload,readUntil=Math.min(feed.read_until,core.object_until,active.windows.read_until),retainUntil=Math.min(feed.retain_until,core.object_until,active.windows.retain_until); + const data=setup.obligations.find((v:Obj)=>v.role==='data_resource');require(data); + const scope={kind:'mailbox_member',slot_key:e.slot,attempt_ref:core.attempt_ref,envelope_ref:envelope,admission_link_ref:member.admission_link_ref,source_custody_ref:member.source_custody_ref}; + const source={...feed,feed_custody:feed.custody,custody:{...messageCustody,payload:mc},message_scope:scope,message_history_ref:member.historical_manifest_ref,message_core_ref:member.admission_core_ref, + message_member:{sequence:member.sequence,admission_link_ref:member.admission_link_ref,sealed_core_ref:(parseNewWire(role('member.link',member.admission_link_ref).raw,policy,budget).value as Obj).payload.sealed_core_ref},graph:{...feed.graph,obligations:[...feed.graph.obligations,{...data,operation_mask:70}]},read_until:Math.min(readUntil,retainUntil),retain_until:retainUntil}; + const originals=feed.graph.members[0].originals,parent=originals.maintenance,read=originals.read,bootstrap=originals.bootstrap,slot=originals.slot,root=e.slot.root_key,pr=parent.payload; + require(e.slot.writer_storage_epoch===a.sourceStorageEpoch&&pr.maintainers.some((v:Obj)=>replicaSame(v,ids(e.maintainer)))&&(pr.operation_mask&78)===78&&pr.max_delegate_depth>=2&&pr.max_destinations_per_job>0&&pr.max_concurrent_jobs>0&&!replicaSame(ids(e.target),ids(e.source))&&!replicaSame(ids(e.target),ids(e.maintainer))); + const allocation=signed(one('copy.allocation'),e.maintainer.signing_key,'resource.allocate',ALLOCATE,policy,budget),offer=signed(one('copy.offer'),e.target.signing_key,'resource.offer',OFFER,policy,budget),assign=signed(one('copy.assignment'),e.maintainer.signing_key,'maintenance.assignment',ASSIGNMENT,policy,budget),ap=allocation.payload,op=offer.payload,mp=assign.payload; + lifetime(ap,at);lifetime(mp,at);for(const [n,v] of [['request_id',ap],['offer_id',op],['assignment_id',mp]] as const)opaque(v[n]); + const intent=objectFields(ap.intent,'kind allocation_id job_id root_key caller target target_storage_epoch purpose scope historical_manifest_ref budget windows'.split(' '));capacity(intent.budget);windows(intent.windows,at);resource(op.resource); + for(const n of ['allocation_id','job_id','target_storage_epoch'])opaque(intent[n]);const digest=budget.hash(buildNewWire(intent,policy,budget).raw); + require(intent.kind==='resource.copy_intent'&&intent.purpose==='message_replica'&&intent.budget.max_live_bytes>=envelope.size&&replicaSame(intent.caller,e.maintainer)&&replicaSame(intent.target,e.target)&&intent.target_storage_epoch===a.targetStorageEpoch&&replicaSame(intent.root_key,root)&&replicaSame(intent.scope,scope)&&replicaSame(intent.historical_manifest_ref,member.historical_manifest_ref)&&replicaSame(op.intent,intent)&&ap.intent_sha256===digest&&op.intent_sha256===digest&&replicaSame(op.allocation_request_ref,allocation.ref)&&replicaSame(op.target_encryption_key,e.target.encryption_key)&&op.resource.node_key_id===e.target.signing_key.key_id&&op.resource.storage_epoch===a.targetStorageEpoch&&u53(op.reservation_generation,1)===1&&replicaSame(op.budget,intent.budget)&&replicaSame(op.windows,intent.windows)&&ap.issued_at<=u53(op.issued_at)&&op.issued_at<=mp.issued_at&&atintent.budget[n]<=control.payload.budget[n])&&WINDOWS.every(n=>intent.windows[n]<=control.payload.windows[n])); + const parentUntil=Math.min(...[slot,parent,read,bootstrap].map(v=>v.payload.expires_at),read.payload.windows.read_until,bootstrap.payload.probe_until,bootstrap.payload.proof_until,bootstrap.payload.upload_until); + require(mp.job_id===intent.job_id&&replicaSame(mp.root_key,root)&&replicaSame(mp.parent_root_ref,parent.ref)&&mp.parent_assignment_ref===null&&u53(mp.depth)===2&&replicaSame(mp.subject,ids(e.target))&&u53(mp.operation_mask)===70&&replicaSame(mp.scope,scope)&&mp.resource_intent_sha256===digest&&replicaSame(mp.resource_offer_ref,offer.ref)&&replicaSame(mp.resource,op.resource)&&replicaSame(mp.bootstrap_grant_refs,[bootstrap.ref])&&replicaSame(mp.budget,op.budget)&&replicaSame(mp.windows,op.windows)&&Object.values(mp.windows).every(v=>u53(v)<=mp.expires_at)&&mp.expires_at<=parentUntil); + for(const v of [ap,mp])require(v.target_node_key_id===e.target.signing_key.key_id&&v.target_storage_epoch===a.targetStorageEpoch); + let maximum=Math.min(source.read_until,source.retain_until,parentUntil,active.windows.copy_until,slot.payload.windows.copy_until,pr.windows.copy_until,intent.windows.copy_until); + for(const item of feed.graph.members){const d=item.disclosure;require((d.operation_mask&70)===70&&['maintenance','slot','read','bootstrap'].every(n=>replicaSame(item.originals[n].ref,originals[n].ref)));maximum=Math.min(maximum,d.consent_until,d.expires_at);} + const reservations:Obj={};for(const variant of ['owner','sender']){const item=signed(one(variant==='owner'?'copy.reservation_consent':'copy.sender_reservation_consent'),e[variant].signing_key,'mailbox.message_copy_reservation_consent',RESERVATION,policy,budget),c=item.payload;lifetime(c,at);opaque(c.consent_id);u53(c.revision,1);const d=objectFields(c.reservation_disclosure,['intent_sha256','until']); + require(c.variant===variant&&replicaSame(c.root_authority_ref,parent.ref)&&replicaSame(c.source_custody_ref,source.custody.ref)&&replicaSame(c.historical_manifest_ref,intent.historical_manifest_ref)&&replicaSame(c.maintainer,ids(e.maintainer))&&replicaSame(c.target,e.target)&&c.target_storage_epoch===a.targetStorageEpoch&&Math.max(mc.stored_at,feed.custody.payload.stored_at)<=c.issued_at&&c.issued_at<=ap.issued_at&&d.intent_sha256===digest&&at(),consents:Obj[]=[];for(const name of ['owner','sender','source','maintainer'])allowed.set(e[name].signing_key.key_id,[]);let until=Math.min(maximum,mp.expires_at); + for(const variant of ['owner','source','sender']){const item=signed(one('copy.'+variant+'_disclosure'),e[variant].signing_key,'mailbox.message_copy_disclosure',DISCLOSURE,policy,budget),c=item.payload;lifetime(c,at);opaque(c.consent_id);u53(c.revision,1); + require(c.variant===variant&&replicaSame(c.root_key,root)&&replicaSame(c.assignment_ref,assign.ref)&&replicaSame(c.source_custody_ref,source.custody.ref)&&replicaSame(c.historical_manifest_ref,intent.historical_manifest_ref)&&replicaSame(c.target,e.target)&&c.target_storage_epoch===a.targetStorageEpoch&&mp.issued_at<=c.issued_at); + const d=objectFields(c.disclosure,['originals','status_scopes','until']),expected=permissions(rows,e[variant].signing_key.key_id,root,'copy',policy,budget);require(replicaSame(d.originals,expected.originals)&&replicaSame(d.status_scopes,expected.status_scopes)&&atduty(v.signer,v.scope_kind,v.scope_id,v.document_revision,v.operation_mask|(['slot','maintenance','disclosure','metadata_resource','data_resource'].includes(v.role)?4:0))); + for(const variant of ['owner','sender']){const item=reservations[variant],id=authority(root,item,policy,budget);allowed.get(e[variant].signing_key.key_id)!.push({scope_kind:'authority',scope_id:id});needs.push(duty(e[variant].signing_key,'authority',id,item.payload.revision,4));} + needs.push(duty(e.owner.signing_key,'authority',authority(root,parent,policy,budget),pr.revision,78));const as=assignment(root,assign,policy,budget);allowed.get(e.maintainer.signing_key.key_id)!.push({scope_kind:'assignment',scope_id:as});needs.push(duty(e.maintainer.signing_key,'assignment',as,0,70)); + for(const [i,variant] of ['owner','source','sender'].entries())needs.push(duty(e[variant].signing_key,'authority',authority(root,consents[i],policy,budget),consents[i].payload.revision,4)); + const previous=[...feed.graph.statuses,...feed.graph.members.flatMap((v:Obj)=>v.statuses)],statuses=observations(entries.get('copy.current_status'),{owner:e.owner,sender:e.sender,source:e.source,maintainer:e.maintainer},allowed,needs,root,at,previous,policy,budget); + const read_until=Math.min(until,intent.windows.read_until,...statuses.map(v=>v.payload.valid_until)),retain_until=Math.min(until,intent.windows.retain_until);require(at(),edges=new Map(),add=(role:string,ref:unknown)=>{const r=rawRef(ref);closure.set(pair(role,r),{role,ref:r});},edge=(parent:unknown,relation:string,child:unknown)=>{const v={parent_ref:rawRef(parent),relation,child_ref:rawRef(child)};edges.set(replicaRefKey(parent)+'|'+relation+'|'+replicaRefKey(child),v);}; + for(const row of rows)add(row.role,row.original.ref);for(const [role,item] of [['copy.allocation',allocation],['copy.offer',offer],['copy.assignment',assign],...consents.map((item,i)=>['copy.'+['owner','source','sender'][i]+'_disclosure',item])] as [string,Obj][])add(role,item.ref); + add('message.envelope',envelope);for(const item of statuses)add('copy.current_status',item.ref); + for(const tree of [feed.manifest,...feed.manifest.predecessors]){ + let ref=one('history.mailbox_feed').ref; + if(tree!==feed.manifest){const rows=feed.manifest.roles.filter((v:Obj)=>v.role==='history.member'&&Buffer.from(v.original.raw).equals(Buffer.from(tree.manifest.raw)));require(rows.length===1,'repair_copy_commit_mismatch');ref=rows[0].original.ref;} + add(tree===feed.manifest?'history.mailbox_feed':'history.member',ref); + for(const row of tree.manifest.value.roles){add('history.raw_pack',row.pack_ref);edge(ref,'manifest-member',row.document_ref);edge(ref,'manifest-member',row.pack_ref);} + } + const fm=feed.manifest.manifest.value;edge(fm.slot_ref,'slot-feed',fm.feed_head_ref);edge(fm.feed_head_ref,'head-checkpoint',feed.graph.head.checkpoint_ref);edge(fm.feed_head_ref,'head-range',feed.graph.head.range_root_ref); + for(const item of fm.members){edge(item.admission_link_ref,'link-core',item.admission_core_ref);edge(item.admission_link_ref,'link-history',item.historical_manifest_ref);edge(item.source_custody_ref,'custody-link',item.admission_link_ref);} + for(const row of feed.manifest.roles)if(['range.index','range.repair_page'].includes(row.role)){const p=(parseNewWire(row.original.raw,policy,budget).value as Obj);if(row.role==='range.index')for(const child of p.children)edge(row.original.ref,'range-child',child.ref);else{edge(row.original.ref,'page-sealed',p.sealed_page_ref);for(const child of p.entries)edge(row.original.ref,'page-link',child.admission_link_ref);}} + edge(member.admission_core_ref,'core-envelope',envelope); + const expected={schema_version:SCHEMA,kind:'replica.manifest',root_key:root,scope,original_roles:sorted([...closure.values()],v=>pair(v.role,v.ref)),physical_objects:sorted([...closure.values()],v=>pair(v.role,v.ref)),edges:sorted([...edges.values()],v=>replicaRefKey(v.parent_ref)+'|'+v.relation+'|'+replicaRefKey(v.child_ref))}; + require(replicaSame(v,expected)&&new Set(v.original_roles.map((v:Obj)=>replicaRefKey(v.ref))).size<=op.budget.max_items,'repair_copy_commit_mismatch'); + return Object.freeze({state:'historical_replica',custody,source,authority:plan,entries}); +} +export function checkMailboxMessageReturn(replica:Obj,consentEntries:Obj,a:Obj):Readonly{ + const {policy,budget}=a,source=replica.source,plan=replica.authority,custody=replica.custody,root=plan.assignment.payload.root_key,setup=source.graph.members[0].originals,bootstrap=setup.bootstrap; + require(['challenge','proof','child'].includes(a.action),'repair_access_action');u53(a.at); + const e=buildNewWire({owner:a.expectedOwner,sender:a.expectedSender,source:a.expectedSource,maintainer:a.expectedMaintainer,target:a.expectedTarget},policy,budget).value as Obj,variants=['owner','source','maintainer','sender'];objectFields(consentEntries,variants); + const allowed=new Map();for(const p of Object.values(e) as Obj[])allowed.set(p.signing_key.key_id,[]);const needs:Obj[]=[],consents:Obj[]=[]; + let expires=Math.min(custody.payload.read_until,custody.payload.retain_until,source.read_until,bootstrap.payload.expires_at,bootstrap.payload[a.action==='challenge'?'probe_until':'proof_until']); + for(const item of source.graph.members){for(const n of ['slot','read','maintenance']){const p=item.originals[n].payload;expires=Math.min(expires,p.expires_at,p.windows.read_until,p.windows.retain_until);}const d=item.disclosure;expires=Math.min(expires,d.expires_at,d.consent_until,d.bootstrap_return.until);} + const rows=new Map(plan.originals.map((v:Obj)=>[pair(v.role,v.original.ref),v]));for(const [role,item] of [['copy.allocation',plan.allocation],['copy.offer',plan.offer],['copy.assignment',plan.assignment],...plan.disclosures.map((v:Obj,i:number)=>['copy.'+['owner','source','sender'][i]+'_disclosure',v])] as [string,Obj][])rows.set(pair(role,item.ref),{role,original:item,issuer:item.payload.signing_key.key_id}); + for(const variant of variants){const item=signed(consentEntries[variant],e[variant].signing_key,'mailbox.message_replica_return_consent',RETURN,policy,budget),p=item.payload;lifetime(p,a.at);opaque(p.consent_id);u53(p.revision,1); + require(p.variant===variant&&replicaSame(p.root_key,root)&&replicaSame(p.source_custody_ref,source.custody.ref)&&replicaSame(p.historical_manifest_ref,plan.allocation.payload.intent.historical_manifest_ref)&&replicaSame(p.assignment_ref,plan.assignment.ref)&&replicaSame(p.subject,ids(e.owner))&&replicaSame(p.target,e.target)&&p.target_storage_epoch===a.targetStorageEpoch&&replicaSame(p.bootstrap_grant_ref,bootstrap.ref)&&p.issued_at>=plan.assignment.payload.issued_at); + const d=objectFields(p.return_permission,['originals','status_scopes','until']),expected=permissions(sorted([...rows.values()],v=>pair(v.role,v.original.ref)),e[variant].signing_key.key_id,root,'return',policy,budget); + require(replicaSame(d.originals,expected.originals)&&replicaSame(d.status_scopes,expected.status_scopes)&&p.issued_atv.scope_kind+':'+v.scope_id));if(item.payload.entries.some((v:Obj)=>!permitted.has(v.scope_kind+':'+v.scope_id)))replicaFail('repair_status_disclosure');} + const previous=[...source.graph.statuses,...plan.statuses,...source.graph.members.flatMap((v:Obj)=>v.statuses)],statuses=observations(a.currentStatuses,e,allowed,needs,root,a.at,previous,policy,budget,a.onObserved);expires=Math.min(expires,...statuses.map(v=>v.payload.valid_until));require(a.at=22.19; `MEMORY_VAULT_NODE` may select it. The tests/adapter +accept `MEMORY_VAULT_JOSE_MODULE` for an existing locked jose distribution. +Dependency installation needs an approved source. For comparison, restore a +second baseline checkout and copy only the three files in `baseline-adapters.json`; +leave production code unchanged. Measure separately from regressions. + +## Remaining work + +- Frozen-message recovery requires retained original COPY/READ and independent + return authority. It is not automatic universal replication, authority renewal, + silent frozen-send relocation or an unlimited backlog. +- This native verifier covers message replicas. Native ACK, root and whole-feed + replica clients remain separate work. +- Further old demand needs independently funded targets. Restart cannot reset + quota. Profiling/caching must preserve exact immutable context, current status, + revocation floors, epochs and transaction snapshots. +- Next: bounded higher-rate sweep, first-contact/SQLite/CPU attribution, a second + independently consented old-message replica, and independent hosts/WAN. A local + trial is not a maximum supported-agent count or global reliability result. +- No paid API, real model, host deployment or real grant/trust changes are needed. + Repository publication follows the authorized normal CI/merge/release workflow. + Existing gpt-6.1-sol/high task settings were accepted earlier; tools do not expose + independently verifiable actual inference model metadata. diff --git a/docs/DELIVERY_CAPACITY.md b/docs/DELIVERY_CAPACITY.md new file mode 100644 index 0000000..9b4984a --- /dev/null +++ b/docs/DELIVERY_CAPACITY.md @@ -0,0 +1,176 @@ +> This records the batch3 Python measurement. The native completion and existing +> full native replica recovery and fresh alpha.0.39 comparison are documented in +> [CAPACITY_HANDOFF.md](CAPACITY_HANDOFF.md). + +# Approved resource distribution for real message delivery + +The Python Agent `send/receive` path now distributes **new** messages over +independently approved, currently valid contact resource leases. Selection and +ciphertext/session freezing happen in one SQLite writer transaction. All frozen +outbox records, including unfinished uploads, count against their exact signed +resource lease. Among slots with enough item/byte capacity, selection uses relative +reserved item count, then byte usage and a stable scope tie-breaker. The server +still independently enforces its live policy, grant, resource budget, possession +proof and per-node capacity. Aliases of one lease cannot multiply its quota. + +The pool includes only local, actual recipient-approved decisions for the same +sender and recipient encryption identities. Discovering a node does not authorize +sending it messages. The recipient must explicitly obtain that node's resource, +receive the sender's first-contact request and approve its finite grant. No real +user permission or signing/encryption identity is created by the dispatch change. +The synthetic fixture makes those separate owner decisions using existing protocol +operations. New grants are learned by the ordinary explicit result workflow. + +Existing frozen messages keep their immutable envelope, node/epoch, exact authority +and request/message IDs on retries, interruption or revocation. The implementation +does not silently migrate a frozen send after ambiguous remote commit. A known +revoked old node can be excluded for a **new** send when another independent valid +grant exists; the old frozen send is still bound to its original node. Existing +repair callers retain their former first-valid-session behavior. + +Receive polling rotates approved sessions with one persisted bounded cursor. +Per-resource sequence cursors, signature and possession checks, receipt identity, +content validation, explicit Memory selection/import and author trust stay intact. +The same Agent API reads the saved message and recalls imported Memory. Approval +does not infer author trust: the fixture separately and explicitly trusts A's +Memory author key in B's test TrustStore. No model calls participate. + +## Fixed-load complete-chain comparison + +Mac mini / Apple M1 / eight CPU cores / 8 GiB / macOS 27.2. Python 3.12.0b4, +SQLite 3.42.0, cryptography 50.0.1. Two independent loopback HTTP processes share +one physical computer/SSD and one source group. This is not proof of independent +failure domains or hardware scalability. + +Both runs offer **48 messages at 2/s for 24 seconds**, without dropping or pausing +scheduled offers. There is one bounded pipeline worker; measured latency starts +at the intended offer time, so driver queueing and join delays are included. Each +message selects a different pre-created signed Memory observation. Local source +Memory creation is outside the offer window; export, encryption, remote storage, +decryption/validation, recipient Memory import, signed saved receipt, message read +and Memory recall are inside the measured chain. Success requires every one of +those stages, not just a server accepting ciphertext. + +R0 initially has a separately approved lease for 24 items and 2 MiB. R1 starts at +second 6, joins using R0 and receives another explicitly approved lease for exactly +24 items and 2 MiB. A and B only have R0 in their initial network configuration. +The observer never assigns individual messages to nodes. Offers continue while +the new resource becomes usable. Both runs use the same fixture, timing, payload +shape, limits and protocol; synthetic key material is newly generated per run. +Baseline is preserved batch2 source with the original first-session selection. +After is the measured final dispatch implementation. + +| Whole 24-second offer window, including drain | Before | After | +| --- | ---: | ---: | +| Offered messages | 48 | 48 | +| Complete verified chains | 40 | 48 | +| Success rate | 83.33% | 100.00% | +| Complete chains/s | 1.667 | 2.000 | +| Successful p95, ms | 398.24 | 406.25 | +| Successful p99, ms | 400.94 | 417.96 | +| All outcomes p95, ms | 388.23 | 406.25 | +| All outcomes p99, ms | 400.94 | 417.96 | +| Offer queue lag p99, ms | 10.34 | 10.16 | +| Recipient saved inbox + sent receipt | 40 | 48 | +| Imported and recalled distinct Memories | 40 | 48 | + +Elapsed including drain: 24.001678s before, 24.003349s after. R1 approval was usable at 7.953s / 7.946s. + +| Node | Stored/read/saved-receipt chains before / after | Stored ciphertext bytes before / after | CPU seconds before / after | Peak HTTP requests/s before / after | +| --- | ---: | ---: | ---: | ---: | +| 0 | 16 / 24 | 113234 / 169866 | 2.199 / 2.785 | 32 / 33 | +| 1 | 24 / 24 | 169896 / 169896 | 2.960 / 2.897 | 26 / 26 | + +Before has eight `open_delivery_quota` failures: the newest first session fills R1's +24-item grant while eight R0 slots remain unused. After uses all 24+24 individually +approved slots, and all 48 chains complete. The additional storage/receipt/read work +on R0 is visible in node counters and actual rows; R1 also stores 24 complete +messages. This proves finite authorized storage capacity is usable across nodes +with automatic assignment. It does **not** measure higher CPU or SQLite maximum +throughput. There are zero HTTP admission/queue refusals in either delivery run, +and observed peak traffic is below the unchanged 64/s per-source cap. + +Tail latency increases slightly. Before's successful-only percentiles exclude its +eight failed messages; all-outcome percentiles and every stage failure are retained. +This result describes all 48 offers, including joining; it is not a best tail window. +The new node's first-contact setup has two `contact_unavailable` attempts before a +successful third attempt in each run. These are visible in setup_attempts and wire +counts. Fixture retries are at most six, 0.5 seconds apart, use the same logical +request ID, and do not auto-approve anything. They are not hidden successful message +chains. The eight-message pilot had one capacity failure before its second resource +was approved; it is excluded from the selected capacity comparison. + +## Interpreting the prior routing result and reducing discovery failures + +The prior 80/s public-query workload exceeded one node's 64/s same-source HTTP +admission allowance. Distributing queries among node-level rate gates improved +admission, not proven compute/storage throughput. Those guards remain per-node; +there is no invented network-wide identity rate service. Routing/source bounds and +the lookup/application request and byte budgets were not increased. Message grants +remain independently node/epoch bound, and their finite quotas are never multiplied +by routing to another node. A retargeted, freshly signed sender request still gets +an exact signed `contact_wrong_node` denial and no upload challenge/storage. + +The routing API consumed a five-second probe delay even when a reply had no new +neighbor. It now consumes the delay only for an actual eligible challenge, with the +same maximum of one new proof per five seconds, at most three find attempts and the +same shared finite lookup budget. It does not add an unbounded retry queue. + +| Full public-query run: 1920 offers, 80/s, 24s, join at 6s | Batch2 | Final fix | +| --- | ---: | ---: | +| Successful queries | 1766 | 1828 | +| Failures | 154 | 92 | +| Success rate | 91.98% | 95.21% | +| Successful p95, ms | 29.76 | 30.04 | +| Successful p99, ms | 44.91 | 37.24 | + +First useful new-peer offer times from window start: + +| Joining peer | Batch2, seconds | Final, seconds | +| --- | ---: | ---: | +| R1 | 15.088 | 6.112 | +| R2 | 10.038 | 11.125 | + +The remaining 92 failures are explicit node0 HTTP admission refusals before enough +proven peers are available. A single allowed 64/s entry cannot complete a sustained +80/s offer without deferral or rejection. The fix reduces unnecessary convergence +delay; it does not hide errors, relax rate gates or claim that all startup offers +succeed. A finite caller retry/admission policy would need a separately measured +latency and request budget; it is not added to disguise overload here. + +## Verification, delivery and remaining scope + +The final-code combined regression run passed all 51 distinct cases (225.143s), +including Python/native contact state interoperability, existing mailbox HTTP +flows, delivery, consent, revocation, quotas, saved receipts and repair +compatibility. Focused intermediate logs are also preserved. The final run covers +actual HTTP, late independent approval, all-chain Memory saving/recall, receive +fairness, atomic concurrent freezing, duplicate aliases, capacity refusal, policy +and known node revocation, immutable retries, and foreign-node grant rejection. The final measured protocol +runtime hashes match the delivered code. The benchmark's added finite watchdog +caps the offer window plus 30s drain and stops only its disposable child processes. +That watchdog branch was not triggered in measured runs. + +```sh +python -m unittest tests.test_open_delivery_distribution tests.test_open_delivery_http tests.test_open_repair_provision tests.test_open_neighbors tests.test_open_contact_state -v +python scripts/benchmark_open_delivery.py --messages 48 --rate 2 --join-after 12 --output after.json +python scripts/benchmark_open_neighbors.py --rate 80 --seconds 24 --join-after 6 --output discovery-after.json +``` + +The bundle plus cumulative patch gives the complete source without fetching Git. +All selected per-offer traces, setup failures, RPC/blob counts, stored bytes, +receipts, resource leases, CPU/RSS and source hashes are in delivery-evidence/. +Earlier two batches remain archived in this same Library artifact/version history. + +This implementation covers the Python common Agent direct delivery carrier. Native +TypeScript selection/receive fairness was unchanged in that batch3 measurement; +the completion batch aligns it with Python and records its own full fixed-load run. It does not move an +already frozen message to a replica, dynamically copy all stored Memory, prove a +multi-host compute/storage capacity increase, or demonstrate thousand-agent scale. +Independent hosts/SSDs with owner-operated HTTPS are needed for WAN and resource +scaling validation, not for the local implementation delivered here. Real-model +conversation/semantic reuse remains untested; no paid API was called. No persistent +real credential, new user grant, trust change, push, merge or deployment occurred. +Actual deployment/new real grants or credential setup need user authorization. +The thread's gpt-6.1-sol/high setting was accepted earlier; the runtime does not +expose actual inference model metadata for independent confirmation. diff --git a/docs/HTTP_CAPACITY.md b/docs/HTTP_CAPACITY.md new file mode 100644 index 0000000..dc59d32 --- /dev/null +++ b/docs/HTTP_CAPACITY.md @@ -0,0 +1,59 @@ +> Recorded batch1 transport/index measurements. Later code and complete message-chain results are in DELIVERY_CAPACITY.md. + +# Bounded HTTP directory read capacity — development evidence + +Baseline: `c840617eb50e8856695234b482fed5560c0c72bf` (main, rechecked against the remote). Development branch: `feat/bounded-directory-read-capacity`. No version bump, commit, push, deployment or live user configuration change was performed. + +## Implemented behavior + +- Each HTTP node uses eight fixed daemon execution workers and a queue of at most sixteen accepted sockets. No worker is created per incoming connection. Queue wait consumes the original three-second whole-input deadline; a queued request never receives a fresh input window. Expired/overflow work receives a best-effort 503 with Retry-After. Transport resets can still occur on an overloaded socket; existing transport errors remain retryable. Active repair-response deadlines retain their existing separate finite bound. +- An initialized transport database verifies its durable configuration binding on every open without acquiring the writer lock. First initialization and legacy migration still acquire and recheck under the writer lock. Existing protected-path and sidecar validation remain on each open. +- Directory `get` uses a consistent WAL read snapshot. Lease/floor/replay expiry and capacity checks use the same logical live rows without mutating storage. The next allocation prunes expired rows; reads cannot create unbounded state. Committed revocation/conflict floors remain effective, and an uncommitted concurrent writer is not treated as committed authority. + +The eight execution slots, global 128 requests/s gate and per-source 64 requests/s gate remain. Owner consent, signed identity/node/epoch binding, retry receipts, resource ceilings and the six Agent operations retain their existing contracts. + +## Actual experiment + +One Mac mini, Apple M1, eight cores, 8 GiB RAM; macOS 27.2 (26B5091g). Python 3.12.0b4, SQLite 3.42.0, cryptography 50.0.1, joserfc 1.7.5, httpx 0.28.1. Node 22.19.0 / jose 6.2.10 were used for selected native SDK regression. All keys, contacts, messages and storage were disposable synthetic fixtures. No real model API was called. + +`scripts/benchmark_open_http.py` runs real independent node processes and actual signed HTTP requests/responses. There are 64 deterministic synthetic contact owners, 32 concurrent requester threads and four rounds of 64 directory reads per topology (256 requests each). The same owners select their explicit directory shard by XOR distance over the configured experiment nodes. The observer's full node list is not installed into any routing participant; all server participants have empty initial introductions. Placement in this benchmark is explicit, not automatic migration or a global discovery acceptance claim. Per-node contact counts were `[64]`, `[34,30]`, and `[8,14,32,10]`. + +| Node processes | Successful reads | Error rate | Burst service reads/s | Mean of per-round successful-read p95, ms | +| --- | --- | --- | --- | --- | +| 1 | 32/256 → 117/256 | 87.50% → 54.30% | 99.1 → 195.3 | 79.4 → 125.1 | +| 2 | 64/256 → 256/256 | 75.00% → 0.00% | 199.5 → 271.6 | 78.4 → 139.1 | +| 4 | 140/256 → 256/256 | 45.31% → 0.00% | 281.6 → 299.0 | 100.2 → 135.6 | + +Burst service rate excludes setup and the 1.1-second quiet intervals required to keep the ordinary rate gates intact. It is not a sustained requests/s capacity claim. Successful-read latency omits failed requests, so the baseline percentiles describe a smaller surviving subset. More admitted work increases queue latency while greatly increasing completion. Two to four processes do not show linear throughput growth on this one host and one requester process. The experiment does not isolate a final CPU bottleneck. + +A separate paced single-node control submitted 256 requests at 48 requests/s without inter-round quiet intervals (about 5.3 seconds). Both versions completed 256/256. Per-round p95 was approximately 14 ms in both; there is no demonstrated steady-load latency gain. JSON evidence retains every round, status count, source hash and measurement definition. Timing samples that overlapped regression workloads were discarded. + +## Validation and limits + +- Existing/index/network-race/transport/multihop/routing/contact regression: 68 reported tests, OK with one initially skipped native-state class, in 126.067 seconds. This includes real eight-process multihop lookup after bootstrap exit and independent committed owner revocation. It is a targeted campaign, not the full repository suite. +- Final boundary campaign: six tests pass. They exercise real WAL readers with an uncommitted writer then committed revocation, expired-row invisibility and later reclamation, binding changes on later opens, bounded live socket overload, preserved whole-input timeout after queuing, and bind-failure cleanup. +- Selected native Agent campaign: two tests pass in 9.314 seconds with real Node / JOSE and real Python HTTP resources. Each Node driver prevents subprocess delegation before importing production SDK code. Native approval/encrypted delivery/saved receipts and cold-sender/owner/resource restart are exercised. Python Agent separately saves selected canonical memory, restarts, confirms the original receipt and recalls the original evidence. + +The broad and native campaigns preceded the final small listener-construction/cleanup guard; the final boundary campaign and the published performance JSON pin the delivered runtime sources. There is no combined whole-suite certification on one exact source. No load tests have been run against the unmerged alpha39 branch; its versions of the three touched runtime files match the baseline and the independent patch is applicable. + +This batch improves finite burst admission and reader/writer coexistence. Single-node overflow still occurs. Process-local first-contact CPU slots, other service writer transactions, hot-shard migration, automatic placement/rebalancing, multi-host fault domains, long-duration demand growth and 1,000-model communication remain unverified or unfinished. Additional nodes demonstrably serve independently selected useful directory reads here; automatic redistribution is not implemented by this batch. + +## Agent access and the next acceptance step + +Existing Python hosts call `Agent(client_config, network_config).handle(request)` or use the six-operation NDJSON `serve` interface in `docs/OPEN_NETWORK_QUICKSTART.md`. Native Node hosts call `await Agent.handle(request)` using the same six-operation objects. A model host must expose that already-authorized endpoint as its tool and preserve the actual signed identities, independent recipient consent and current resource permissions. SDK tests do not establish a model's understanding or autonomous follow-through. No installed MCP/capture configuration was altered. + +A harmless local inventory found no Ollama CLI and no listener on its ordinary loopback port. No model was downloaded and no external paid API was used. To run a real-model sustained acceptance next, select authorized existing Agent/model sessions or local models and the endpoints/resources they may use, with explicit permission for that integration. Keep new spending and external deployment separately authorized. + +## Reproduce + +Use an isolated Python environment with the repository's network/server dependency files. From the checkout, run: + +```sh +python scripts/benchmark_open_http.py --output before-or-after.json +python scripts/benchmark_open_http.py --arrival-rate 48 --nodes 1 --output paced.json +python -m unittest -v tests.test_open_read_concurrency +``` + +The base bundle permits offline cloning of exact main and alpha36/alpha37 history. Apply `capacity.patch` in that main clone; new source files are included by the patch. `delivery-manifest.json` records SHA256 for all changed sources and evidence. The old dirty Mac checkout and separate unmerged alpha39 worktree remain intact. + +Model selection was explicitly submitted and accepted for this same task using `gpt-6.1-sol` with high reasoning. The execution API does not return actual inference-model metadata, so independent runtime confirmation of the selected model is unavailable; no different model was silently selected. diff --git a/docs/OPEN_NETWORK_CONTACT.md b/docs/OPEN_NETWORK_CONTACT.md index 783ca92..1c3dc52 100644 --- a/docs/OPEN_NETWORK_CONTACT.md +++ b/docs/OPEN_NETWORK_CONTACT.md @@ -197,6 +197,17 @@ signed destination and slot controls; the sender reuses its own frozen outbox ciphertext and contact originals. Preparation never invents permission or sends plaintext to the mailbox node. +Native TypeScript also supports receiver `authorize` and paginated `inspect` in +development source after alpha.0.37. It shares the exact destination journal with +Python, signs only with the receiver's own existing identity, and performs no HTTP +requests during authorization. Both clients apply authenticated retained +revocations before returning or issuing authorization. Source provisioning still +uses Python. In development after alpha.0.38, native senders also support the `prepare`, `admit` and `retain` invitations below. They share +the frozen ciphertext, disclosure, exact HTTP request and verified custody +result with Python. Restarting or changing clients does not renew authority. +An optional `ack_request_id` carries only the explicitly selected, independently +prepared original receipt grants; custody does not imply recipient acknowledgement. + A receiver using the Agent interface can authorize from an existing registered mailbox and its locally approved contact, without opening a database or loading private key objects in the calling program: diff --git a/docs/RELEASE.md b/docs/RELEASE.md index 884e53a..740c335 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -1,5 +1,9 @@ # Distribution scope and publication gates +Alpha.0.40 adds bounded HTTP admission, WAL read concurrency, separately approved resource distribution and native full message-replica COPY/READ recovery. The isolated 48-Memory/2-per-second workload completes 39 → 48 chains; this proves authorized quota use, not a CPU/SQLite ceiling. Replica cold reads remain slow and stop at the existing finite work allowance. See [capacity evidence and limits](CAPACITY_HANDOFF.md). + +Alpha.0.40 candidate assets are validated against their exact source commit by `release-manifest.json` and `SHA256SUMS`; use the [release record](https://github.com/qh-work/memory-vault-sync/releases/tag/v0.28.0-alpha.0.40) for publication and CI evidence. Earlier release results below remain historical. + Published **[alpha.0.37](https://github.com/qh-work/memory-vault-sync/releases/tag/v0.28.0-alpha.0.37)** is fixed at `fe943c6466a6ef459ccb0d03e309b207b3b0fb95`. Native TypeScript recipients can now retain an explicitly authorized receipt destination for a cold-mailbox message. Ordinary Agent `receive` returns its actual saved receipt, reusing the exact upload after a lost reply or restart. Python and TypeScript share jobs and completed history; revocation stops retries and expired uncertain uploads stop for reconciliation. See [native mailbox receipt returns](../clients/typescript/network/README.md). diff --git a/docs/ROUTING_CAPACITY.md b/docs/ROUTING_CAPACITY.md new file mode 100644 index 0000000..49c19a8 --- /dev/null +++ b/docs/ROUTING_CAPACITY.md @@ -0,0 +1,135 @@ +> Recorded batch2 routing measurements. DELIVERY_CAPACITY.md documents the later probe fix and the separate real delivery implementation. + +# Bounded public routing query distribution + +`OpenParticipant.find_neighbors(target, view="general", budget=None)` distributes +one public routing step across a bounded pool of endpoint-proven routers. A caller +starts with its configured one or two introductions, receives a signed `find` +response and can challenge one returned new router with `hello` at most once per +five seconds. Future calls rotate proven peers. At most three peers are tried per +operation; any extra challenge uses the same finite lookup budget. New keys have +probe priority over temporarily failed old peers. Existing routing source, bucket, +expiry and durable rollback/revocation/fork limits apply. No global roster or new +admission authority is introduced, and the first entry is not required after other +peers have been proven. + +```python +hints = await participant.find_neighbors(contact_key(owner_key_id), view="directory") +# hints["nodes"] are introductions. They do not grant access or certify a contact. +contact = await participant.find_contact(owner_key_id) +``` + +This is a public neighborhood query API, not a replacement for iterative lookup. +`find_contact` still observes all selected shards before choosing an eligible +contact, so a later shard's revocation or conflict cannot be bypassed by choosing a +convenient replica. The current Python/TypeScript message send and storage paths +have not been switched to this new API. This batch does not move memory records, +replicate ciphertext, redistribute private reads/writes, or raise the eight-worker +HTTP limit. It does not establish full contact resolution or message delivery +capacity. A future client can use these hints in a routing stage, but that complete +path needs separate implementation and measurement. + +## Controlled before/after workload + +The benchmark uses deterministic synthetic Ed25519 identities and real HTTP on +127.0.0.1. All three nodes run on one Mac mini (Apple M1, eight CPU cores, 8 GiB RAM, +macOS 27.2). The driver has sixteen threads and sends **80 queries/s for 24 seconds**, +1920 offered queries per run, with no removed idle gaps. Both runs have the same +three server processes. Nodes 1 and 2 enter the network at second 6 using existing +join and maintenance exchanges. The client receives only node 0; the observer's +full node list is used only to label measurements. New node descriptors are not +injected into the client. Requests query the same routing coordinate, and responses +are signature/request/node verified. No real memories, paid model calls or external +services participate. + +Baseline is the prior one-hop `find` call at a fixed entry, executed on the preserved +batch1 runtime. After uses the new `find_neighbors` API. **This baseline is not the +existing complete `find_contact` operation.** The core source hashes are retained +in both reports. Success means a verified, non-error public routing response; hints +remain unverified until separately challenged. Endpoint proofs, maintenance and +failed attempts are additional wire work and are recorded. + +| Full offer window plus drain | Before | After | +| --- | ---: | ---: | +| Offered queries | 1920 | 1920 | +| Successful queries | 1509 | 1766 | +| Success rate | 78.59% | 91.98% | +| Successful queries/s (including drain) | 62.85 | 73.55 | +| Successful p95, ms | 23.89 | 29.76 | +| Successful p99, ms | 27.80 | 44.91 | +| All outcomes p95, ms | 23.69 | 29.52 | +| All outcomes p99, ms | 27.47 | 44.66 | + +Elapsed including drain: 24.0098s before, 24.0113s after. + +| Last eight-second offered cohort (seconds 16..24) | Before | After | +| --- | ---: | ---: | +| Total / offered | 640 | 640 | +| Successful queries | 504 | 640 | +| Success rate | 78.75% | 100.00% | +| Successful queries/s | 63.00 | 80.00 | +| Successful p95, ms | 23.75 | 33.05 | +| Successful p99, ms | 26.43 | 45.56 | + +Completions physically inside seconds 16..24: 504 before and 640 after (the same 8-second completion rate). Offered-cohort tail latencies include queueing, scheduling delay and final drain; every failed outcome is retained. + +| Node | Driver successes before / after | Total server find work before / after | CPU seconds before / after | Max RSS MiB before / after | +| --- | ---: | ---: | ---: | ---: | +| 0 | 1509 / 1088 | 1522 / 1106 | 7.348 / 5.302 | 40.17 / 39.81 | +| 1 | 0 / 238 | 16 / 254 | 0.867 / 2.012 | 38.47 / 38.78 | +| 2 | 0 / 440 | 18 / 456 | 0.911 / 2.974 | 39.11 / 39.70 | + +Driver CPU seconds: 17.948 before, 26.027 after. The last offered cohort uses nodes 0/1/2 for {'2': 214, '0': 213, '1': 213}. + +## Interpretation and boundaries + +The additional routers carry real client requests after being discovered through +normal protocol traffic. They reduce the original node's client workload without +raising its policy limit. At this offered rate the baseline's main observed +constraint is the unchanged per-source 64/s HTTP admission policy, plus maintenance +traffic sharing that allowance. This is **policy-bound public query capacity**; +CPU saturation, SQLite write capacity, wide-area limits and maximum sustainable +throughput have not been established. The new API increases client CPU work and +p95/p99 latency. During join/proof convergence the full after run still has 154 +failed queries (8.02%); the final 12 seconds have zero failures, and the final eight +seconds distribute successful requests 213/213/214 across nodes 0/1/2. This short +finite observation is not a long-run reliability guarantee. + +The trace shows the first useful query on node 2 around second 10 and node 1 around +second 15. These are bounded client probes, not immediate topology convergence. +A router can return a different local neighborhood than another router; this API +only returns hints and never labels them globally closest or authoritative. + +No contact/control permission was weakened. Known revocation and conflict remain +blocking. Independently authorized replica placement and end-to-end storage or +message load balancing remain separate work. The next useful environment is at +least three independently hosted nodes with owner-operated HTTPS and explicit +finite resource grants, followed by a measured complete authorized read/write or +mailbox path. One shared Mac cannot demonstrate independent CPU/storage/failure +domains or thousand-agent capacity. Native SDK integration also remains. + +## Validation and reproducibility + +29 existing/new routing, real HTTP, directory revocation/conflict and concurrency +regression cases passed in 109.716 seconds before the final newcomer-priority +correction. Five neighbor boundary cases passed with the final correction; 30 +distinct cases are covered. The latter includes corrupted signatures, late join, +old-peer failure, current revocation, failover and shared budget enforcement. +The final measured runtime hashes match the delivered source. The alpha39 native +mailbox branch (`3df5cf8e58c94c0c2f703d574321f3c2561ffcec`) changes none of the four +runtime files involved here; the cumulative patch applies without changing that +original worktree. Its new mailbox paths have not been performance certified here. + +```sh +python scripts/benchmark_open_neighbors.py --rate 80 --seconds 24 --join-after 6 --output after.json +python -m unittest tests.test_open_neighbors -v +``` + +For baseline, use the batch1 source in the delivery and the same benchmark script; +it detects absence of `find_neighbors` and uses the prior fixed-entry signed find. +Per-request traces, four-second windows, all failure attempts, CPU/RSS and wire byte +counts are in the delivery's `neighbor-evidence/before.json` and `after.json`. +Pilot and pre-correction runs are excluded from the selected evidence. The runtime +model setting was accepted as `gpt-6.1-sol` with high reasoning; actual inference +model metadata is not exposed for independent confirmation. No model-switch +request was repeated in this batch. diff --git a/docs/STATUS.md b/docs/STATUS.md index cbf66a1..bf41769 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -1,5 +1,9 @@ # Memory Vault development status +Alpha.0.40 adds bounded HTTP admission, WAL read concurrency, separately approved resource distribution and native full message-replica COPY/READ recovery. The isolated 48-Memory/2-per-second workload completes 39 → 48 chains; this proves authorized quota use, not a CPU/SQLite ceiling. Replica cold reads remain slow and stop at the existing finite work allowance. See [capacity evidence and limits](CAPACITY_HANDOFF.md). + +Alpha.0.40 candidate assets are validated against their exact source commit by `release-manifest.json` and `SHA256SUMS`; use the [release record](https://github.com/qh-work/memory-vault-sync/releases/tag/v0.28.0-alpha.0.40) for publication and CI evidence. Earlier release results below remain historical. + Published **[alpha.0.37](https://github.com/qh-work/memory-vault-sync/releases/tag/v0.28.0-alpha.0.37)** is fixed at `fe943c6466a6ef459ccb0d03e309b207b3b0fb95`. Native TypeScript recipients can now retain an explicitly authorized receipt destination for a cold-mailbox message. Ordinary Agent `receive` returns its actual saved receipt, reusing the exact upload after a lost reply or restart. Python and TypeScript share jobs and completed history; revocation stops retries and expired uncertain uploads stop for reconciliation. See [native mailbox receipt returns](../clients/typescript/network/README.md). diff --git a/docs/evidence/capacity-alpha40/after.json b/docs/evidence/capacity-alpha40/after.json new file mode 100644 index 0000000..832b03e --- /dev/null +++ b/docs/evidence/capacity-alpha40/after.json @@ -0,0 +1,1818 @@ +{ + "synthetic": true, + "real_models": 0, + "physical_hosts": 1, + "cpu_count": 8, + "python": "3.12.0b4 (v3.12.0b4:97a6a41816, Jul 11 2023, 11:19:02) [Clang 13.0.0 (clang-1300.0.29.30)]", + "platform": "macOS-27.2-arm64-arm-64bit", + "scope": "independently approved direct message storage, encrypted read, recipient saved receipt, Vault import/recall", + "source_hashes": { + "memory_vault_network.py": "a83d457d41d305048f1277f735a29af061607ef53c6e4eca5c585d812ea5063b", + "memory_vault_open_node.py": "aad46af7bbdfef59b9317bb9732322a323afbbd9783dad36d1a6bbe359ce38a4", + "memory_vault_open_delivery_client.py": "14a72a6926163e41a9ae0a051d0e39992921382a49e1796e5a99c0cc77e4fcad", + "memory_vault_open_delivery_state.py": "cc473f5b102523b16cc3352bf25042e8ae0b86b80eb3f4f641b84d54be4d8d07", + "memory_vault_open_index.py": "c637cc96eba5a83653d2d368b4198906d7448a3f30c388f563af99452dd13367", + "clients/typescript/network/agent.ts": "4e2610e314c63fb6e33c3baae96ff6c96d33808277614c52001ddc7164b296b9", + "clients/typescript/network/client-config.ts": "c414ccdb6c95d099c670c5c8fb5582239d7ec9d179e7a318069cc997cf113817", + "clients/typescript/network/content.ts": "dd214415cf8eee1a26d5f0c6b3e08ca8f5b63066fc0ab0f627f6671611402212", + "clients/typescript/network/control.ts": "0ad7cf773ba5be25ab67b31c11dcbe65d89f38d6d8cd0c9664e8c548b49e0f1c", + "clients/typescript/network/crypto.ts": "7b538dd6d334cb2c42ea24103e6930d38063440d16eafc455716b3570fee438a", + "clients/typescript/network/hints.ts": "cc42d41c87544ae3374359cbcba4b3ab14e7d82738fab2629c5991d2d3c96562", + "clients/typescript/network/io.ts": "8be85ce42d1d68ba9169e1a639e10c77f3dffa274094638446bd3821215a805b", + "clients/typescript/network/nodes.ts": "f77c5b901ef43b3d8d4f2813af6c9c0ad61372900c07e416cd5232b23c71988b", + "clients/typescript/network/open-ack-client.ts": "d923bda9bd53462c22432d5c3e0f202f8426924189a341037456a25165fbfe3a", + "clients/typescript/network/open-ack-discovery.ts": "8afad511e4eb1645b62caafc5c22fc2284c94c965e8f15f5e6ea0390fc530a21", + "clients/typescript/network/open-ack-status.ts": "ab56e0bb37057060910da4b140e2f278c74ff67243d989c23299e80e2afddde0", + "clients/typescript/network/open-blob.ts": "00d9f845f34d8d309ed8910c46177df3c0aeb1939e35fb26a6dfa23584433fbd", + "clients/typescript/network/open-capacity.ts": "514a163013cc1bf20551dba9bd858a56c0fa1abd720054e61048e3101d4522e8", + "clients/typescript/network/open-client.ts": "087b28d765c6b946ee6a38d428082db0f7a63f36d376e92c8b758515cb79c4eb", + "clients/typescript/network/open-contact-client.ts": "bbd520337e54c1e5fea656129cabc8c0e6879ee4f03326ff920ccf665ab70068", + "clients/typescript/network/open-contact-state.ts": "d7deef9625f140edc8c31d2b7b1480db3da9f5967820a29ca7a605de0592167c", + "clients/typescript/network/open-contact.ts": "38fc88aea51e9fbcaf3f7c19c25a5cf0704660dcbd54fca02191d2ec629371da", + "clients/typescript/network/open-control.ts": "863f9a0a29d9bd85cec16dfd87f331198fd44ef7dce8e93620f6cd15a39d2b0e", + "clients/typescript/network/open-delivery-client.ts": "9aba3143c937cdfe806a2528241cd82b4f1a36becffee4607de85f684de41885", + "clients/typescript/network/open-delivery-control.ts": "839fd6106e851e2a722e9c394c09e7acab9d2505d0c40eb76e91085656b9e970", + "clients/typescript/network/open-delivery.ts": "6582e3de1909ceb65c1dbe35aefe9f08cee0270b745927459ecbaf3c1df43892", + "clients/typescript/network/open-mailbox-client.ts": "01a27d6f8abd8a9ebe229b11b7dfcd08b95c38b74d6ceec0373a82c476c41dc9", + "clients/typescript/network/open-mailbox-destination.ts": "7f5c3c515f51c7bfb676068abbb43db7b1beb696ec748f788f636cc28adaec09", + "clients/typescript/network/open-mailbox-journal.ts": "0929088f3f4c1b647b6599f57be23cb921f70db2e5fc97dc36a853217979b3a0", + "clients/typescript/network/open-mailbox-receipts.ts": "a3df0b7aac2701001b6672dde6d11ce5e885c7f2d9554d431b5a2bbe8cb4c86e", + "clients/typescript/network/open-mailbox-receivers.ts": "6b7e7cf14f43e4be27a61bb8d97221e73d216b8c2c96bad28a8cded908230fb1", + "clients/typescript/network/open-mailbox-replica-client.ts": "57c869cfbb88dadb2f10e525a20b2068cdddc91e39ce6f9bcb3c729cbdd35cc4", + "clients/typescript/network/open-mailbox-replicas.ts": "58130254ff4bafcf9cd190f5f9d19c73a22ad8a810adc73552115702aa71adaa", + "clients/typescript/network/open-mailbox-send.ts": "560ad4a9b49a6070cb7381ef851fe7f555b45745f1cdb97f2ce8d09749164cae", + "clients/typescript/network/open-node.ts": "4d5326f2ebb63fee44f268125d25697ccb47b448357cd6ea1a7209bb497de417", + "clients/typescript/network/open-participant.ts": "20327310486a282274b501a29f2ec67794df25bfe97b0820b1fe7e312bc1cc67", + "clients/typescript/network/open-provider-client.ts": "0dda9dc2e48f3d05fc9cb77a02f75267c5b1d9b5cabc4924ec821c64fac3cc05", + "clients/typescript/network/open-provider.ts": "7f382152585b133d4ef91491b238b109b8fe8b1648ebf923d0ee038b606b37ee", + "clients/typescript/network/open-repair-ack.ts": "312e27ff0e0729a3ee34306f44ad3de3414323cf54b8902b74d035613005ab01", + "clients/typescript/network/open-repair-admin.ts": "3bf9f2c4826b9cf4b305cdfb36e6d1e077435bc9899e0fac61067dcb71716b64", + "clients/typescript/network/open-repair-bootstrap.ts": "b8831454095299d7ed34c8d56e43aaf55cf6f38ec4e94fad57886e506eaf82ca", + "clients/typescript/network/open-repair-bound.ts": "e762e43fb2c0d570227f38b20dbcbebf078134c414693472c6ff2d18a6857fbb", + "clients/typescript/network/open-repair-client.ts": "73f439160f9a597f2b94a43a9d5abda734efcf236b14eae5e421b8898144e4ee", + "clients/typescript/network/open-repair-empty.ts": "309547b4b281b34ac8a9ce84f5d434d3e5cb72dccba0585ddca0a57d6897ef93", + "clients/typescript/network/open-repair-history.ts": "5f88d2335c552b4edb3eaa84f397ab6ce294261f1a98e97810800346914c7bf2", + "clients/typescript/network/open-repair-mailbox-authority.ts": "2098c3eaa3af6636751781c2f9129e8ab96b660437cc080742f68a98bf4dd1be", + "clients/typescript/network/open-repair-mailbox-feed.ts": "3996efc4e76fa199ae0ea109e31cc5702a65db282d54334d1777d1ef8d9d4ad6", + "clients/typescript/network/open-repair-mailbox-inbox.ts": "89f3ebc2d98233786b80713228dc02adc746b3c011d2bb89988060aed600b9ba", + "clients/typescript/network/open-repair-mailbox-member.ts": "25d16b4437bed2c3984116487088e94b51a75259ab9a9bc36a0408e7274c7784", + "clients/typescript/network/open-repair-mailbox-range.ts": "c6d6b6078f8632784abc5ee8b52dd2ed7e531d1631b68491c568c9a5646c9527", + "clients/typescript/network/open-repair-mailbox-read.ts": "4498b2b33d5d0a02d028ac80d3c3c733f28620f2545bdf5f2b14e0a1bacb3039", + "clients/typescript/network/open-repair-mailbox-replica-inbox.ts": "ea4394a6e2abcaf9c79c50fc4b363ef8fbc6daa5de9fa043556c6ba7ee5ae70c", + "clients/typescript/network/open-repair-mailbox-replica.ts": "61d90adcf745e864359566c22e9a0512af1a1fce98f4c7d67fd6e5f014c88533", + "clients/typescript/network/open-repair-occupied.ts": "9c43836e6957dfe7319dad7deffabe87730ec4190c3fc921b94b75c0239c6fb1", + "clients/typescript/network/open-repair-offer-client.ts": "8e0ec6d63b4c26274d709c76ca9f396d0c6863292324bb9c05b179c208265d21", + "clients/typescript/network/open-repair-original.ts": "d289733c05d0b6a0691fdfe7458aac4fb0b577a9297676d05cafe245eff4aa01", + "clients/typescript/network/open-repair-probe.ts": "3720d5a45b1354a4c0e7f6cb8cbb4e8a8d6ec998c1ac52f883fe90621420eb7b", + "clients/typescript/network/open-repair-proof.ts": "40ee00f74fd3147434bb54a7f978340d06da9ef448177b4d2d2b6aa288b88502", + "clients/typescript/network/open-repair-receipt.ts": "25e345006f8e5cd1f8a807c024c2319ef1aad9177582975ea2d91a05b7bd0a36", + "clients/typescript/network/open-repair-resource.ts": "b42c85fddd32672687cb3d2a613b11b223dcc04d9326b70967da43fc20484b43", + "clients/typescript/network/open-repair-status.ts": "21880a57c7523664e58b43c705558df5a4aaee0dd18aab2730b037709a8def36", + "clients/typescript/network/open-repair-wire.ts": "82f92918da5ba4975e22a9021e5985ed7760ba9145ba359c6578e311f7c02862", + "clients/typescript/network/open-routing.ts": "ce59007a2b3703fab81d97a01d298ea825e34884ef0b694600f6af206a3e880b", + "clients/typescript/network/open-state.ts": "00d37b59b8db9b5d7749249d535e53ea2f2e0a0d0722ab9dae03579f9065801c", + "clients/typescript/network/open-transport.ts": "64b765bb76427375ad07e84d8ae2e3ce17d580593ef5e89ef2def6ab3fba949f", + "clients/typescript/network/peer.ts": "1e6e1614aa51584db194582f819af6cbc8d2179c3e803da8406c4dd58c4ac781", + "clients/typescript/network/privacy.ts": "b4a21f3d95f2fdb16518e881dc930be764f8cd52b96991e6eab41feb231d52af", + "clients/typescript/network/ranking_math.ts": "b509bec4360d23c39429d17d491766ed172334f10ca71667659eb9b7e5c1af6b", + "clients/typescript/network/records.ts": "6e0f2d66f128db7593e9ca4b53bba2b037ae682f32cc852660c1cfe5af76408c", + "clients/typescript/network/retrieval.ts": "ee5baa5fddbf6d780ce5a3ef27bb91cd98985693040852bf484b425e5b6cd47d", + "clients/typescript/network/retrieval_text.ts": "98e007279ad065aea7bdc29ebbb846a1709205016143cc75860b583feaca041f", + "clients/typescript/network/setup.ts": "5bce583aaf60f164f5fad2e694c0f88f6cfe4cc5be3c3e2402ed20754425da44", + "clients/typescript/network/topics.ts": "b1ae2e39490b3126f2b2090a6fbea3dcf3828192dab8545f88193f909e7e3fae", + "clients/typescript/network/transport-state.ts": "814ea3808a1242cba010df02272c4a12b836e515820b82c6b0af21663ef63323", + "clients/typescript/network/transport.ts": "ff986ed1378bcb531773257e5857798d58d3bf3fb80178a36fdcaf98d42430df", + "clients/typescript/network/vault.ts": "8a0465bbd6aa7dae73d80b3ccc0e6ee43ae1690587024e46c60c7f141be42adc" + }, + "result": { + "mode": "approved_resource_distribution", + "agent_runtime": "native", + "native_before": [ + { + "cpu": { + "user": 704530, + "system": 97998 + }, + "rss": 139575296, + "subprocessCalls": 0, + "distribution": true + }, + { + "cpu": { + "user": 469817, + "system": 23781 + }, + "rss": 122912768, + "subprocessCalls": 0, + "distribution": true + } + ], + "native_after": [ + { + "cpu": { + "user": 7143337, + "system": 596788 + }, + "rss": 159645696, + "subprocessCalls": 0, + "distribution": true + }, + { + "cpu": { + "user": 5167527, + "system": 418393 + }, + "rss": 129499136, + "subprocessCalls": 0, + "distribution": true + } + ], + "offered_messages": 48, + "offered_messages_per_second": 2, + "offer_window_seconds": 24.0, + "elapsed_including_drain": 24.01034599999548, + "successful_chains": 48, + "success_rate": 1.0, + "successful_chains_per_second": 1.9991382048392405, + "statuses": { + "ok": 48 + }, + "successful_p95_ms": 365.48066698014736, + "successful_p99_ms": 386.7010419780854, + "all_p95_ms": 365.48066698014736, + "all_p99_ms": 386.7010419780854, + "offer_lag_p99_ms": 10.209166997810826, + "successful_by_node": { + "0": 24, + "1": 24 + }, + "recipient_saved_inbox": 48, + "recipient_vault_memories": 48, + "per_node": [ + { + "node": 0, + "counts": { + "http_requests": 363, + "second_1790947809": 0, + "admitted": 363, + "rpc_lease": 0, + "request_bytes": 1099546, + "response_bytes": 1047124, + "rpc_policy.put": 0, + "rpc_find": 14, + "rpc_put": 1, + "rpc_get": 2, + "rpc_policy.get": 0, + "rpc_challenge": 0, + "rpc_submit": 0, + "rpc_poll": 0, + "rpc_decide": 0, + "rpc_result": 0, + "second_1790947810": 12, + "rpc_prepare": 24, + "rpc_start": 24, + "blob_upload.chunk": 24, + "blob_input_bytes": 169602, + "blob_output_bytes": 169602, + "rpc_commit": 24, + "rpc_receipt.get": 48, + "rpc_list.prepare": 48, + "rpc_list": 48, + "rpc_read.prepare": 24, + "rpc_read.start": 24, + "blob_download.chunk": 24, + "rpc_receipt.put": 24, + "second_1790947811": 24, + "second_1790947812": 24, + "second_1790947813": 24, + "second_1790947814": 24, + "second_1790947815": 24, + "second_1790947816": 29, + "rpc_hello": 10, + "second_1790947817": 30, + "second_1790947818": 6, + "second_1790947819": 4, + "second_1790947820": 6, + "second_1790947821": 4, + "second_1790947822": 5, + "second_1790947823": 5, + "second_1790947824": 4, + "second_1790947825": 16, + "second_1790947826": 14, + "second_1790947827": 16, + "second_1790947828": 14, + "second_1790947829": 16, + "second_1790947830": 14, + "second_1790947831": 15, + "second_1790947832": 15, + "second_1790947833": 16, + "second_1790947834": 2 + }, + "peak_http_requests_per_second": 30, + "stored_items": 24, + "stored_ciphertext_bytes": 169602, + "saved_receipts": 24, + "delivery_leases": [ + { + "lease_id": "lease_e47eb55d44a406d04303aa271a93f541f99e3b5aa80c5950e669434444399130", + "max_items": 24, + "max_bytes": 2097152 + } + ], + "user_cpu_seconds": 2.4404559999999997, + "system_cpu_seconds": 0.49677099999999996, + "max_rss_bytes": 46104576 + }, + { + "node": 1, + "counts": { + "http_requests": 342, + "second_1790947816": 4, + "admitted": 342, + "rpc_lease": 2, + "request_bytes": 1059323, + "response_bytes": 1001747, + "rpc_policy.put": 1, + "rpc_find": 12, + "rpc_put": 1, + "second_1790947817": 16, + "rpc_hello": 10, + "rpc_get": 1, + "rpc_policy.get": 1, + "rpc_challenge": 2, + "rpc_submit": 1, + "rpc_poll": 1, + "rpc_decide": 1, + "rpc_list.prepare": 34, + "rpc_list": 34, + "rpc_result": 1, + "second_1790947818": 24, + "rpc_prepare": 24, + "rpc_start": 24, + "blob_upload.chunk": 24, + "blob_input_bytes": 169632, + "blob_output_bytes": 169632, + "rpc_commit": 24, + "rpc_receipt.get": 48, + "rpc_read.prepare": 24, + "rpc_read.start": 24, + "blob_download.chunk": 24, + "rpc_receipt.put": 24, + "second_1790947819": 26, + "second_1790947820": 24, + "second_1790947821": 26, + "second_1790947822": 24, + "second_1790947823": 26, + "second_1790947824": 24, + "second_1790947825": 16, + "second_1790947826": 14, + "second_1790947827": 16, + "second_1790947828": 14, + "second_1790947829": 16, + "second_1790947830": 14, + "second_1790947831": 16, + "second_1790947832": 14, + "second_1790947833": 16, + "second_1790947834": 12 + }, + "peak_http_requests_per_second": 26, + "stored_items": 24, + "stored_ciphertext_bytes": 169632, + "saved_receipts": 24, + "delivery_leases": [ + { + "lease_id": "lease_ccdebdd9c93d367f1e9204c096bd961256586d5b3cede101a631ca1827abc994", + "max_items": 24, + "max_bytes": 2097152 + } + ], + "user_cpu_seconds": 2.5197089999999998, + "system_cpu_seconds": 0.49668999999999996, + "max_rss_bytes": 46399488 + } + ], + "join": { + "started_seconds": 6.0085583749751095, + "authorized_seconds": 7.262500332988566 + }, + "join_errors": [], + "setup_attempts": [ + { + "node": 0, + "attempt": 1, + "ok": true, + "code": null, + "time": 251363.438165 + }, + { + "node": 1, + "attempt": 1, + "ok": false, + "code": "contact_unavailable", + "time": 251370.853902333 + }, + { + "node": 1, + "attempt": 2, + "ok": true, + "code": null, + "time": 251371.562567041 + } + ], + "requests": [ + { + "index": 0, + "offered_seconds": 0.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 0.23058298393152654, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_8f68eae307b23e62a9e746c40719579102c5c7b04228839cbbc8ca844440991a", + "latency_ms": 334.8842079867609, + "completed_seconds": 0.3348869579785969 + }, + { + "index": 1, + "offered_seconds": 0.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 9.003916988149285, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_a20adb9b55fd9960821d04bf9d57bc908624fc7d78e73ae09eacc8031b007b33", + "latency_ms": 330.7371669798158, + "completed_seconds": 0.8307382499915548 + }, + { + "index": 2, + "offered_seconds": 1.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 6.948791997274384, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_51079f314b955ec3103a3f847d027de7e735306ad8ff18099f00b8cffdcfb7b9", + "latency_ms": 318.36741699953564, + "completed_seconds": 1.3183682499802671 + }, + { + "index": 3, + "offered_seconds": 1.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 0.6570419936906546, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_dfbed31a8f6f121ff0808ee23c757724d64c3a189cc7fa9ccbeb232724af3de2", + "latency_ms": 311.82333297329023, + "completed_seconds": 1.8118242499767803 + }, + { + "index": 4, + "offered_seconds": 2.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 7.732582977041602, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_3edd694af8280f7edef9f9a62308979ba9109356e4b51e47036e817f64a5d842", + "latency_ms": 320.5242079857271, + "completed_seconds": 2.3205252079933416 + }, + { + "index": 5, + "offered_seconds": 2.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 8.429207984590903, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_e13f90410bdd501b9ebd568f46207f63e59e6ee81f98c32ecd70b979dae22c61", + "latency_ms": 319.3186249991413, + "completed_seconds": 2.8193196669744793 + }, + { + "index": 6, + "offered_seconds": 3.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 4.502374998992309, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_1d876059381da1350cad3c0f33f85388d1004b9d5cc62019057eadc2860690b5", + "latency_ms": 314.9886249739211, + "completed_seconds": 3.314989666978363 + }, + { + "index": 7, + "offered_seconds": 3.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.13999999850057, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_a06f50a0f1662bf7153ff79a54583330a944209c6d3ca7f25d1ae0a5594b16c8", + "latency_ms": 307.65933298971504, + "completed_seconds": 3.8076602079963777 + }, + { + "index": 8, + "offered_seconds": 4.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.20937497378327, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_437d398340428d1bc9cc96c72a59151431fd9b9c460cb4dca53cd9e962003c23", + "latency_ms": 323.81549998535775, + "completed_seconds": 4.32381637499202 + }, + { + "index": 9, + "offered_seconds": 4.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 1.7387080006301403, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_74d30a671f3e7932eb972ed34a79e81633a07b0539a645a6037054ad4c514763", + "latency_ms": 310.2499999804422, + "completed_seconds": 4.810250916983932 + }, + { + "index": 10, + "offered_seconds": 5.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 8.471124980133027, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_666863c7897d3ef1d9bfa220f8678d2e4f132881abb9b856b18a7026c67ade8d", + "latency_ms": 317.6245829963591, + "completed_seconds": 5.317625457973918 + }, + { + "index": 11, + "offered_seconds": 5.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 4.97562499367632, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_ab0e453dcbc6ba073f28d6c3e56b800062303ac20ee77072a566fd3cb157ccfa", + "latency_ms": 314.9044999736361, + "completed_seconds": 5.8149053329834715 + }, + { + "index": 12, + "offered_seconds": 6.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 8.762874989770353, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_185bc8a1475c8286182ae633ab3ab9622dedede9cc537618df2604e3a62e3c60", + "latency_ms": 436.67341698892415, + "completed_seconds": 6.436678541998845 + }, + { + "index": 13, + "offered_seconds": 6.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 1.6159169899765402, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_eccee2dba147170d42d905510d4819fe56a69d6dd3787f2bd62fa755f359df98", + "latency_ms": 286.38141698320396, + "completed_seconds": 6.786382332997164 + }, + { + "index": 14, + "offered_seconds": 7.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 8.336291997693479, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_d457b6417293e17eaf841912b3633f7d0993675137b3a24ec2bc2b16995fe1fc", + "latency_ms": 375.04083299427293, + "completed_seconds": 7.375041624996811 + }, + { + "index": 15, + "offered_seconds": 7.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 8.000207977602258, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_9a82c3138b7ab7cde245d54f6e54a2939bd9611f2f1493881d8e1c642d5848fc", + "latency_ms": 341.68499999213964, + "completed_seconds": 7.84168591699563 + }, + { + "index": 16, + "offered_seconds": 8.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.140082973521203, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_db4116acef9632bc77f6dc15d7796d5233605daab42a4c90bb0477c5e0185b44", + "latency_ms": 354.4743749953341, + "completed_seconds": 8.354475332977017 + }, + { + "index": 17, + "offered_seconds": 8.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.117749974597245, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_d7bc3a1c9907d5b186d1eb26f873d807b6177f84b633c1d2cc362d06058dee2a", + "latency_ms": 348.2023329997901, + "completed_seconds": 8.8482033329783 + }, + { + "index": 18, + "offered_seconds": 9.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.097541991854087, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_d1374b1b2b4e2fd9bbbab84c290cd381b0f8a7ce0a146887c2b4711de84d4ca6", + "latency_ms": 347.0013329933863, + "completed_seconds": 9.347002291993704 + }, + { + "index": 19, + "offered_seconds": 9.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.111124982358888, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_f2a1e86139017dc37294a4cb1c89f363c439c3e3ee44fe5b6df7b0d28743da7e", + "latency_ms": 349.66454198001884, + "completed_seconds": 9.849665666988585 + }, + { + "index": 20, + "offered_seconds": 10.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.102916974574327, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_a3e6c6bf3fc10432ced2c9fbe7bf2fde9c36d9ab62b41d3fa1b8079537d5ded1", + "latency_ms": 350.2269579912536, + "completed_seconds": 10.350228041992523 + }, + { + "index": 21, + "offered_seconds": 10.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.118999984115362, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_6e87e5a518bc8cdbea2b2a8220b32db0a094912e260ecde73d1a62641388afff", + "latency_ms": 351.75004199845716, + "completed_seconds": 10.851750874979189 + }, + { + "index": 22, + "offered_seconds": 11.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 9.16408299235627, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_7f227c080877002396eb0e9a83f17f1b76c66491e8c8c801598565d89f3330c6", + "latency_ms": 341.4394579885993, + "completed_seconds": 11.341440124990186 + }, + { + "index": 23, + "offered_seconds": 11.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 9.105499979341403, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_d71270a81ecc6d594150ed1669067a2937709e19882afdf9d9a81d690a54e04b", + "latency_ms": 351.6734579752665, + "completed_seconds": 11.851674374978757 + }, + { + "index": 24, + "offered_seconds": 12.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.098082973854616, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_f952404541296c0a4036fab199361db41f6146c345411608652b374939113b60", + "latency_ms": 338.92299997387454, + "completed_seconds": 12.33892383298371 + }, + { + "index": 25, + "offered_seconds": 12.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 1.5290829760488123, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_a0db30f5dca9e51aee7fe8151d23666aada67ad9e15f2e4f47164ad6c48900fe", + "latency_ms": 321.081541973399, + "completed_seconds": 12.82108229197911 + }, + { + "index": 26, + "offered_seconds": 13.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.068291972856969, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_f0cf4c7bc0c9756d98a011bc8eae4ff89c8f56c729141ff095760f3dfd045871", + "latency_ms": 323.6810419766698, + "completed_seconds": 13.323682082991581 + }, + { + "index": 27, + "offered_seconds": 13.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.209166997810826, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_6506b47c00f7685c1bff3a0658ff88859ca2a0c75bc50c2ab83fc8ab27a5d3d2", + "latency_ms": 355.78766698017716, + "completed_seconds": 13.855788499990012 + }, + { + "index": 28, + "offered_seconds": 14.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.111332987435162, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_c1dabd6c8184279b63ffc1c7b7cb888e34103b506cc704d4f2c18744137c93a5", + "latency_ms": 347.3817919730209, + "completed_seconds": 14.347383041982539 + }, + { + "index": 29, + "offered_seconds": 14.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 1.191874995129183, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_46c69a1fa56547ea5c353ba2e210733f5dfad0f0c2d0567d5583c03cb63fd5c5", + "latency_ms": 312.5214579922613, + "completed_seconds": 14.812522291991627 + }, + { + "index": 30, + "offered_seconds": 15.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 4.433541995240375, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_6805fc3863c762d2988954bb5b2588403550cefc514d1fbafede2404940ccb39", + "latency_ms": 311.48958299309015, + "completed_seconds": 15.311490457999753 + }, + { + "index": 31, + "offered_seconds": 15.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.173249989748001, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_c1de781e9558e17e27a59a984b779d93cfb83c5a5251c376db6f708173bd1e03", + "latency_ms": 365.48066698014736, + "completed_seconds": 15.865481666987762 + }, + { + "index": 32, + "offered_seconds": 16.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 0.7221669948194176, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_bdf3be969ae3e676cec9ced950867c0f130076956d4c6221f8b951836dda5daa", + "latency_ms": 337.5981249846518, + "completed_seconds": 16.337599041988142 + }, + { + "index": 33, + "offered_seconds": 16.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 3.206166991731152, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_4d350be3cf68164789467efc6d25f65324683058239b9f7d8169bb3bbdb655bb", + "latency_ms": 323.84795797406696, + "completed_seconds": 16.82384895798168 + }, + { + "index": 34, + "offered_seconds": 17.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 2.369875001022592, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_8176c9a2c7f13b73b53353445943e51aee0ff76a96e021f3169ac49a468339ce", + "latency_ms": 351.5385829959996, + "completed_seconds": 17.35153962500044 + }, + { + "index": 35, + "offered_seconds": 17.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 9.737457992741838, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_c771ff07892bceeac3c7b648d07f68b41296fdeaab47172cfe9d39f08194e571", + "latency_ms": 356.5667919756379, + "completed_seconds": 17.85656783299055 + }, + { + "index": 36, + "offered_seconds": 18.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 5.107457982376218, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_e5935d16e1a9053be726018013e44f3f454079369c914a3e2b9027b233069e2c", + "latency_ms": 350.6079579819925, + "completed_seconds": 18.350609332992462 + }, + { + "index": 37, + "offered_seconds": 18.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.139416990568861, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_49897b1f107b99444bf140af62fa23635698b6e720c23288525fbf73811a4045", + "latency_ms": 361.6230829793494, + "completed_seconds": 18.861624041979667 + }, + { + "index": 38, + "offered_seconds": 19.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 0.8077919774223119, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_3f89f102c6a4dff46454691697648ddb1639e22dc846ab0db4177ae8c477a831", + "latency_ms": 346.1954579979647, + "completed_seconds": 19.346196250000503 + }, + { + "index": 39, + "offered_seconds": 19.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.125082975719124, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_c51c7f08e63f50cc1e9b8ae7dea171b9febb7e396c6a75456e0e7ec0d33aae5c", + "latency_ms": 360.6920829915907, + "completed_seconds": 19.86069299999508 + }, + { + "index": 40, + "offered_seconds": 20.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.09474997408688, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_e2271f1b38b54cb74d21e99e28af38ecfa549ee4a6a18183fbb3fb1f9a02389c", + "latency_ms": 360.6581249914598, + "completed_seconds": 20.36065904199495 + }, + { + "index": 41, + "offered_seconds": 20.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 1.5711249725427479, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_b32986ba3cd178923c9f1dfb711eb22d3c927691473949aa17ab641bdc174fba", + "latency_ms": 346.64495798642747, + "completed_seconds": 20.84664570799214 + }, + { + "index": 42, + "offered_seconds": 21.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 8.437416981905699, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_f40a17272dc974685cff2229dab923559598c5930a1c6629616bb039ecbb4cb4", + "latency_ms": 386.7010419780854, + "completed_seconds": 21.38670187498792 + }, + { + "index": 43, + "offered_seconds": 21.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 8.386374975088984, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_9a2e7d1fb91c564012172c1287a8405da148a336d0680e9f306a39a8b7d44665", + "latency_ms": 344.43137497873977, + "completed_seconds": 21.8444319999835 + }, + { + "index": 44, + "offered_seconds": 22.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 8.440041972789913, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_dbc8bee6be4d2cedbe47fcfc5ce72f54284f27b85d91320403bd62f0e4380256", + "latency_ms": 356.54854198219255, + "completed_seconds": 22.356549457996152 + }, + { + "index": 45, + "offered_seconds": 22.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 1.5702499949838966, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_79bf32db7b9431366d577281c686f5f40e7699b700c76ff2695d34e2d148ec38", + "latency_ms": 347.0553749939427, + "completed_seconds": 22.847056332975626 + }, + { + "index": 46, + "offered_seconds": 23.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 6.759124982636422, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_d1780a59fe8cb4327cc3be7d8789c3cdd97bb390d18c362028c36a81fa2d069f", + "latency_ms": 360.992832982447, + "completed_seconds": 23.36099383299006 + }, + { + "index": 47, + "offered_seconds": 23.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 8.816749992547557, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_c563d592118349ab32caea80477955621fd04f8b5ce12e5d961781f5c0519384", + "latency_ms": 350.80141699290834, + "completed_seconds": 23.850802332977764 + } + ], + "driver": { + "user_cpu_seconds": 0.14187100000000002, + "system_cpu_seconds": 0.07647099999999998, + "max_rss_bytes": 44007424 + } + } +} diff --git a/docs/evidence/capacity-alpha40/before.json b/docs/evidence/capacity-alpha40/before.json new file mode 100644 index 0000000..a1c13f4 --- /dev/null +++ b/docs/evidence/capacity-alpha40/before.json @@ -0,0 +1,1714 @@ +{ + "synthetic": true, + "real_models": 0, + "physical_hosts": 1, + "cpu_count": 8, + "python": "3.12.0b4 (v3.12.0b4:97a6a41816, Jul 11 2023, 11:19:02) [Clang 13.0.0 (clang-1300.0.29.30)]", + "platform": "macOS-27.2-arm64-arm-64bit", + "scope": "independently approved direct message storage, encrypted read, recipient saved receipt, Vault import/recall", + "source_hashes": { + "memory_vault_network.py": "5f1f8d6c6ca08eadc63efe13ff7e77077707f01d2b2dc7791e691c5ef474e60a", + "memory_vault_open_node.py": "b6d05a333e173331f3d0466d2c8f2ba4a8454295a554a8b708d93faf081f4562", + "memory_vault_open_delivery_client.py": "73fb48e1d30e120e7d852202ffdb097cc83f4d77753cd8a40abdb781ccc4e0a5", + "memory_vault_open_delivery_state.py": "cc473f5b102523b16cc3352bf25042e8ae0b86b80eb3f4f641b84d54be4d8d07", + "memory_vault_open_index.py": "f13616488e3c0af507e1cb011a7faea10fea1346446213803138be0b369dfcc0", + "clients/typescript/network/agent.ts": "4e2610e314c63fb6e33c3baae96ff6c96d33808277614c52001ddc7164b296b9", + "clients/typescript/network/client-config.ts": "c414ccdb6c95d099c670c5c8fb5582239d7ec9d179e7a318069cc997cf113817", + "clients/typescript/network/content.ts": "dd214415cf8eee1a26d5f0c6b3e08ca8f5b63066fc0ab0f627f6671611402212", + "clients/typescript/network/control.ts": "0ad7cf773ba5be25ab67b31c11dcbe65d89f38d6d8cd0c9664e8c548b49e0f1c", + "clients/typescript/network/crypto.ts": "7b538dd6d334cb2c42ea24103e6930d38063440d16eafc455716b3570fee438a", + "clients/typescript/network/hints.ts": "cc42d41c87544ae3374359cbcba4b3ab14e7d82738fab2629c5991d2d3c96562", + "clients/typescript/network/io.ts": "8be85ce42d1d68ba9169e1a639e10c77f3dffa274094638446bd3821215a805b", + "clients/typescript/network/nodes.ts": "f77c5b901ef43b3d8d4f2813af6c9c0ad61372900c07e416cd5232b23c71988b", + "clients/typescript/network/open-ack-client.ts": "d923bda9bd53462c22432d5c3e0f202f8426924189a341037456a25165fbfe3a", + "clients/typescript/network/open-ack-discovery.ts": "8afad511e4eb1645b62caafc5c22fc2284c94c965e8f15f5e6ea0390fc530a21", + "clients/typescript/network/open-ack-status.ts": "cd5f45005a32f0cedde730b078d9a5127879719170a89eea213e1b8d77401bdc", + "clients/typescript/network/open-blob.ts": "00d9f845f34d8d309ed8910c46177df3c0aeb1939e35fb26a6dfa23584433fbd", + "clients/typescript/network/open-capacity.ts": "514a163013cc1bf20551dba9bd858a56c0fa1abd720054e61048e3101d4522e8", + "clients/typescript/network/open-client.ts": "060c4db30804867636da447f1273fcd425ba9258e3fc3b7845322dd42164c236", + "clients/typescript/network/open-contact-client.ts": "bbd520337e54c1e5fea656129cabc8c0e6879ee4f03326ff920ccf665ab70068", + "clients/typescript/network/open-contact-state.ts": "d7deef9625f140edc8c31d2b7b1480db3da9f5967820a29ca7a605de0592167c", + "clients/typescript/network/open-contact.ts": "38fc88aea51e9fbcaf3f7c19c25a5cf0704660dcbd54fca02191d2ec629371da", + "clients/typescript/network/open-control.ts": "863f9a0a29d9bd85cec16dfd87f331198fd44ef7dce8e93620f6cd15a39d2b0e", + "clients/typescript/network/open-delivery-client.ts": "009b61e3df8ede4fdbb47702061f241b664190c5d123dcc6c1d86653816d87f8", + "clients/typescript/network/open-delivery-control.ts": "839fd6106e851e2a722e9c394c09e7acab9d2505d0c40eb76e91085656b9e970", + "clients/typescript/network/open-delivery.ts": "6582e3de1909ceb65c1dbe35aefe9f08cee0270b745927459ecbaf3c1df43892", + "clients/typescript/network/open-mailbox-client.ts": "01a27d6f8abd8a9ebe229b11b7dfcd08b95c38b74d6ceec0373a82c476c41dc9", + "clients/typescript/network/open-mailbox-destination.ts": "7f5c3c515f51c7bfb676068abbb43db7b1beb696ec748f788f636cc28adaec09", + "clients/typescript/network/open-mailbox-journal.ts": "0929088f3f4c1b647b6599f57be23cb921f70db2e5fc97dc36a853217979b3a0", + "clients/typescript/network/open-mailbox-receipts.ts": "a3df0b7aac2701001b6672dde6d11ce5e885c7f2d9554d431b5a2bbe8cb4c86e", + "clients/typescript/network/open-mailbox-receivers.ts": "1b7f63d6e8583d48c768b1924191c9a0a0c3949e534f7655d93e09b6554900d3", + "clients/typescript/network/open-mailbox-send.ts": "560ad4a9b49a6070cb7381ef851fe7f555b45745f1cdb97f2ce8d09749164cae", + "clients/typescript/network/open-node.ts": "4d5326f2ebb63fee44f268125d25697ccb47b448357cd6ea1a7209bb497de417", + "clients/typescript/network/open-participant.ts": "20327310486a282274b501a29f2ec67794df25bfe97b0820b1fe7e312bc1cc67", + "clients/typescript/network/open-provider-client.ts": "0dda9dc2e48f3d05fc9cb77a02f75267c5b1d9b5cabc4924ec821c64fac3cc05", + "clients/typescript/network/open-provider.ts": "7f382152585b133d4ef91491b238b109b8fe8b1648ebf923d0ee038b606b37ee", + "clients/typescript/network/open-repair-ack.ts": "312e27ff0e0729a3ee34306f44ad3de3414323cf54b8902b74d035613005ab01", + "clients/typescript/network/open-repair-admin.ts": "3bf9f2c4826b9cf4b305cdfb36e6d1e077435bc9899e0fac61067dcb71716b64", + "clients/typescript/network/open-repair-bootstrap.ts": "b8831454095299d7ed34c8d56e43aaf55cf6f38ec4e94fad57886e506eaf82ca", + "clients/typescript/network/open-repair-bound.ts": "e762e43fb2c0d570227f38b20dbcbebf078134c414693472c6ff2d18a6857fbb", + "clients/typescript/network/open-repair-client.ts": "73f439160f9a597f2b94a43a9d5abda734efcf236b14eae5e421b8898144e4ee", + "clients/typescript/network/open-repair-empty.ts": "309547b4b281b34ac8a9ce84f5d434d3e5cb72dccba0585ddca0a57d6897ef93", + "clients/typescript/network/open-repair-history.ts": "5f88d2335c552b4edb3eaa84f397ab6ce294261f1a98e97810800346914c7bf2", + "clients/typescript/network/open-repair-mailbox-authority.ts": "2098c3eaa3af6636751781c2f9129e8ab96b660437cc080742f68a98bf4dd1be", + "clients/typescript/network/open-repair-mailbox-feed.ts": "3996efc4e76fa199ae0ea109e31cc5702a65db282d54334d1777d1ef8d9d4ad6", + "clients/typescript/network/open-repair-mailbox-inbox.ts": "51f8c3274ebb603b729dd5dbf082fc3fd37a817c028d99de5ae48eef4fc349f0", + "clients/typescript/network/open-repair-mailbox-member.ts": "25d16b4437bed2c3984116487088e94b51a75259ab9a9bc36a0408e7274c7784", + "clients/typescript/network/open-repair-mailbox-range.ts": "c6d6b6078f8632784abc5ee8b52dd2ed7e531d1631b68491c568c9a5646c9527", + "clients/typescript/network/open-repair-mailbox-read.ts": "9f5fe5d369ae4db353c0a9c1332caabf7218bf37155376035777d1ead9eba091", + "clients/typescript/network/open-repair-occupied.ts": "9c43836e6957dfe7319dad7deffabe87730ec4190c3fc921b94b75c0239c6fb1", + "clients/typescript/network/open-repair-offer-client.ts": "8e0ec6d63b4c26274d709c76ca9f396d0c6863292324bb9c05b179c208265d21", + "clients/typescript/network/open-repair-original.ts": "d289733c05d0b6a0691fdfe7458aac4fb0b577a9297676d05cafe245eff4aa01", + "clients/typescript/network/open-repair-probe.ts": "3720d5a45b1354a4c0e7f6cb8cbb4e8a8d6ec998c1ac52f883fe90621420eb7b", + "clients/typescript/network/open-repair-proof.ts": "40ee00f74fd3147434bb54a7f978340d06da9ef448177b4d2d2b6aa288b88502", + "clients/typescript/network/open-repair-receipt.ts": "25e345006f8e5cd1f8a807c024c2319ef1aad9177582975ea2d91a05b7bd0a36", + "clients/typescript/network/open-repair-resource.ts": "b42c85fddd32672687cb3d2a613b11b223dcc04d9326b70967da43fc20484b43", + "clients/typescript/network/open-repair-status.ts": "3e98f1e2b9374730c18ae37ba6605c66948235f0c45d04965672b822a9f7fcdb", + "clients/typescript/network/open-repair-wire.ts": "82f92918da5ba4975e22a9021e5985ed7760ba9145ba359c6578e311f7c02862", + "clients/typescript/network/open-routing.ts": "ce59007a2b3703fab81d97a01d298ea825e34884ef0b694600f6af206a3e880b", + "clients/typescript/network/open-state.ts": "00d37b59b8db9b5d7749249d535e53ea2f2e0a0d0722ab9dae03579f9065801c", + "clients/typescript/network/open-transport.ts": "64b765bb76427375ad07e84d8ae2e3ce17d580593ef5e89ef2def6ab3fba949f", + "clients/typescript/network/peer.ts": "1e6e1614aa51584db194582f819af6cbc8d2179c3e803da8406c4dd58c4ac781", + "clients/typescript/network/privacy.ts": "b4a21f3d95f2fdb16518e881dc930be764f8cd52b96991e6eab41feb231d52af", + "clients/typescript/network/ranking_math.ts": "b509bec4360d23c39429d17d491766ed172334f10ca71667659eb9b7e5c1af6b", + "clients/typescript/network/records.ts": "6e0f2d66f128db7593e9ca4b53bba2b037ae682f32cc852660c1cfe5af76408c", + "clients/typescript/network/retrieval.ts": "ee5baa5fddbf6d780ce5a3ef27bb91cd98985693040852bf484b425e5b6cd47d", + "clients/typescript/network/retrieval_text.ts": "98e007279ad065aea7bdc29ebbb846a1709205016143cc75860b583feaca041f", + "clients/typescript/network/setup.ts": "5bce583aaf60f164f5fad2e694c0f88f6cfe4cc5be3c3e2402ed20754425da44", + "clients/typescript/network/topics.ts": "b1ae2e39490b3126f2b2090a6fbea3dcf3828192dab8545f88193f909e7e3fae", + "clients/typescript/network/transport-state.ts": "814ea3808a1242cba010df02272c4a12b836e515820b82c6b0af21663ef63323", + "clients/typescript/network/transport.ts": "ff986ed1378bcb531773257e5857798d58d3bf3fb80178a36fdcaf98d42430df", + "clients/typescript/network/vault.ts": "8a0465bbd6aa7dae73d80b3ccc0e6ee43ae1690587024e46c60c7f141be42adc" + }, + "result": { + "mode": "prior_first_approved_session", + "agent_runtime": "native", + "native_before": [ + { + "cpu": { + "user": 693131, + "system": 158666 + }, + "rss": 133808128, + "subprocessCalls": 0, + "distribution": false + }, + { + "cpu": { + "user": 458167, + "system": 26637 + }, + "rss": 124485632, + "subprocessCalls": 0, + "distribution": false + } + ], + "native_after": [ + { + "cpu": { + "user": 6338370, + "system": 628028 + }, + "rss": 153665536, + "subprocessCalls": 0, + "distribution": false + }, + { + "cpu": { + "user": 4263213, + "system": 339086 + }, + "rss": 171638784, + "subprocessCalls": 0, + "distribution": false + } + ], + "offered_messages": 48, + "offered_messages_per_second": 2, + "offer_window_seconds": 24.0, + "elapsed_including_drain": 24.002671291993465, + "successful_chains": 39, + "success_rate": 0.8125, + "successful_chains_per_second": 1.6248191514004182, + "statuses": { + "ok": 39, + "open_delivery_quota": 9 + }, + "successful_p95_ms": 345.6919579766691, + "successful_p99_ms": 403.5438749997411, + "all_p95_ms": 343.1104579940438, + "all_p99_ms": 403.5438749997411, + "offer_lag_p99_ms": 10.21012497949414, + "successful_by_node": { + "0": 15, + "1": 24 + }, + "recipient_saved_inbox": 39, + "recipient_vault_memories": 39, + "per_node": [ + { + "node": 0, + "counts": { + "http_requests": 255, + "second_1790947783": 0, + "admitted": 255, + "rpc_lease": 0, + "request_bytes": 744667, + "response_bytes": 716419, + "rpc_policy.put": 0, + "rpc_find": 14, + "rpc_put": 1, + "rpc_get": 2, + "rpc_policy.get": 0, + "rpc_challenge": 0, + "rpc_submit": 0, + "rpc_poll": 0, + "rpc_decide": 0, + "rpc_result": 0, + "second_1790947784": 12, + "rpc_prepare": 15, + "rpc_start": 15, + "blob_upload.chunk": 15, + "blob_input_bytes": 105990, + "blob_output_bytes": 105990, + "rpc_commit": 15, + "rpc_receipt.get": 30, + "rpc_list.prepare": 39, + "rpc_list": 39, + "rpc_read.prepare": 15, + "rpc_read.start": 15, + "blob_download.chunk": 15, + "rpc_receipt.put": 15, + "second_1790947785": 24, + "second_1790947786": 24, + "second_1790947787": 24, + "second_1790947788": 24, + "second_1790947789": 24, + "second_1790947790": 32, + "rpc_hello": 10, + "second_1790947791": 25, + "second_1790947792": 8, + "second_1790947793": 4, + "second_1790947794": 6, + "second_1790947795": 4, + "second_1790947796": 6, + "second_1790947797": 4, + "second_1790947798": 5, + "second_1790947799": 5, + "second_1790947800": 4, + "second_1790947801": 6, + "second_1790947802": 4, + "second_1790947803": 6, + "second_1790947805": 2, + "second_1790947807": 2 + }, + "peak_http_requests_per_second": 32, + "stored_items": 15, + "stored_ciphertext_bytes": 105990, + "saved_receipts": 15, + "delivery_leases": [ + { + "lease_id": "lease_94d8f931025e2e831c51ab8a6d719e265a7e2299efcc6c351bc4aef7bb8cd7cd", + "max_items": 24, + "max_bytes": 2097152 + } + ], + "user_cpu_seconds": 1.7946000000000002, + "system_cpu_seconds": 0.377117, + "max_rss_bytes": 45858816 + }, + { + "node": 1, + "counts": { + "http_requests": 342, + "second_1790947790": 4, + "admitted": 342, + "rpc_lease": 2, + "request_bytes": 1232447, + "response_bytes": 1001603, + "rpc_policy.put": 1, + "rpc_find": 12, + "rpc_put": 1, + "second_1790947791": 16, + "rpc_hello": 10, + "rpc_get": 1, + "rpc_policy.get": 1, + "rpc_challenge": 2, + "rpc_submit": 1, + "rpc_poll": 1, + "rpc_decide": 1, + "rpc_list.prepare": 25, + "rpc_list": 25, + "rpc_result": 1, + "second_1790947792": 24, + "rpc_prepare": 33, + "rpc_start": 33, + "blob_upload.chunk": 24, + "blob_input_bytes": 169632, + "blob_output_bytes": 169632, + "rpc_commit": 24, + "rpc_receipt.get": 48, + "rpc_read.prepare": 24, + "rpc_read.start": 24, + "blob_download.chunk": 24, + "rpc_receipt.put": 24, + "second_1790947793": 26, + "second_1790947794": 24, + "second_1790947795": 26, + "second_1790947796": 24, + "second_1790947797": 26, + "second_1790947798": 24, + "second_1790947799": 26, + "second_1790947800": 24, + "second_1790947801": 26, + "second_1790947802": 24, + "second_1790947803": 26, + "second_1790947804": 4, + "error_open_delivery_quota": 9, + "second_1790947805": 6, + "second_1790947806": 4, + "second_1790947807": 6, + "second_1790947808": 2 + }, + "peak_http_requests_per_second": 26, + "stored_items": 24, + "stored_ciphertext_bytes": 169632, + "saved_receipts": 24, + "delivery_leases": [ + { + "lease_id": "lease_c26b23bc12ba25c55edee7bcab86726c83db16bb9e59d40a80d46edecb2bdcd4", + "max_items": 24, + "max_bytes": 2097152 + } + ], + "user_cpu_seconds": 2.637645, + "system_cpu_seconds": 0.5067849999999999, + "max_rss_bytes": 46743552 + } + ], + "join": { + "started_seconds": 6.001860874996055, + "authorized_seconds": 7.375652207992971 + }, + "join_errors": [], + "setup_attempts": [ + { + "node": 0, + "attempt": 1, + "ok": true, + "code": null, + "time": 251337.198063125 + }, + { + "node": 1, + "attempt": 1, + "ok": false, + "code": "contact_unavailable", + "time": 251344.87656325 + }, + { + "node": 1, + "attempt": 2, + "ok": true, + "code": null, + "time": 251345.60706525 + } + ], + "requests": [ + { + "index": 0, + "offered_seconds": 0.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 0.20462498650886118, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_27df0135c4a68a719a062b4ced61dcfc2e55e59f95fdf4d7cd032b724a1b9d99", + "latency_ms": 339.797249995172, + "completed_seconds": 0.33979829199961387 + }, + { + "index": 1, + "offered_seconds": 0.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.086583002703264, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_ffa70736e3608cfb681fb6a94b9054c6e9d0de5112bf401decc10a3ab4c82bf4", + "latency_ms": 320.2794169774279, + "completed_seconds": 0.8202804579923395 + }, + { + "index": 2, + "offered_seconds": 1.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 6.47133297752589, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_f58a6bf9ad321ec92960dd4fe86c19248b9f96b59b6644a2d727254792d241bd", + "latency_ms": 319.5926249900367, + "completed_seconds": 1.3195935419935267 + }, + { + "index": 3, + "offered_seconds": 1.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 4.80662498739548, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_a5217d85bbd2ff77dc81e85ad8ffecc59f37618ce702f83f71a4fa16a5bfe069", + "latency_ms": 319.14745797985233, + "completed_seconds": 1.8191484579874668 + }, + { + "index": 4, + "offered_seconds": 2.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 2.2973750019446015, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_c57db38380cbf8727a01bd3d7b0a6f031d69b3a436f387afdf2b9d974500a7a3", + "latency_ms": 310.9174580022227, + "completed_seconds": 2.310918582981685 + }, + { + "index": 5, + "offered_seconds": 2.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 7.529583002906293, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_ecadd2e92e06443b922a6c6a032c9e2a1630200bc063ad2fd0a63b93958bea3e", + "latency_ms": 306.01429200032726, + "completed_seconds": 2.806015166977886 + }, + { + "index": 6, + "offered_seconds": 3.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 6.65191697771661, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_c2eb3c6c3a3200e208b44dab56d70899076f341f7868ff59ec3797ff0fa327e0", + "latency_ms": 316.6577919910196, + "completed_seconds": 3.3166586669976823 + }, + { + "index": 7, + "offered_seconds": 3.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 8.552249986678362, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_3766e217ae94365a7032037389608af075d3033c8672bbfba2346a542c36331c", + "latency_ms": 322.04458297928795, + "completed_seconds": 3.822045499982778 + }, + { + "index": 8, + "offered_seconds": 4.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 3.5538749943953007, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_8bef2ee3d725e507077c229f3ee8c264cf1796d222874e812e1d158e296e3c06", + "latency_ms": 319.44520797696896, + "completed_seconds": 4.3194462079845835 + }, + { + "index": 9, + "offered_seconds": 4.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 4.788541991729289, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_7835afcda13eb85aed8cdd08cb5c0076f90d62523f977a69d1c2f739f4b16217", + "latency_ms": 316.006249981001, + "completed_seconds": 4.816007207991788 + }, + { + "index": 10, + "offered_seconds": 5.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 4.770667001139373, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_a644cd02d8a1fab9a9e755a4f1f227e5e556309869f7bc0d23285aeee2cbefc6", + "latency_ms": 309.7827919991687, + "completed_seconds": 5.3097836249799 + }, + { + "index": 11, + "offered_seconds": 5.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 2.776332985376939, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_2dadd8c9544712231f9ed07137953c34fe959b9fd49a9b866fda7510539bbddf", + "latency_ms": 305.1470420032274, + "completed_seconds": 5.805147957988083 + }, + { + "index": 12, + "offered_seconds": 6.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 2.1234579908195883, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_77d7b6feb44a2841df9e73dbfd33757501258c96a1f40e7f8637c811726721c5", + "latency_ms": 403.5438749997411, + "completed_seconds": 6.403544999979204 + }, + { + "index": 13, + "offered_seconds": 6.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 9.79362498037517, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_7fbe58a73e14529a05be4324d7eaa122ffa2d2b5bdb486a182532e21f0a972a9", + "latency_ms": 289.8156250012107, + "completed_seconds": 6.789816417003749 + }, + { + "index": 14, + "offered_seconds": 7.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 7.083916978444904, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_4991843b15a58ef7a6dc83d698f3f498b30f4f81be52a528442bdbbb43aefa5f", + "latency_ms": 477.58395798155107, + "completed_seconds": 7.477584666979965 + }, + { + "index": 15, + "offered_seconds": 7.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.067249997518957, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_0a734ee4802ce5ea218ced84aa40b0bf93c2e60b8944a97cdfe6b847ee556f2b", + "latency_ms": 312.6083329843823, + "completed_seconds": 7.8126094169856515 + }, + { + "index": 16, + "offered_seconds": 8.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 1.357000001007691, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_ad023554aefb10e03b55f2b52f31d2e6820c3be5f5287b11cc9b09db0ad361d7", + "latency_ms": 336.31695798248984, + "completed_seconds": 8.336318041983759 + }, + { + "index": 17, + "offered_seconds": 8.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.172457987209782, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_86ae290d10d79650bcb34020b8eb7bc7416e9b5bb2d5038763e9482fc7c55373", + "latency_ms": 340.3323329985142, + "completed_seconds": 8.840333208005177 + }, + { + "index": 18, + "offered_seconds": 9.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 6.302750000031665, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_a780bfbb49273276b12e8b62305d2e419cb1a809c09a519b5efebf440bba4eef", + "latency_ms": 338.7631249788683, + "completed_seconds": 9.33876416698331 + }, + { + "index": 19, + "offered_seconds": 9.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 6.850457983091474, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_68f45bdd93e96d0cfa67d480e77cbbaa26f02b71c3cd306b5aedbc21f16e0d36", + "latency_ms": 336.3378749927506, + "completed_seconds": 9.836338833003538 + }, + { + "index": 20, + "offered_seconds": 10.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 1.0559170041233301, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_2abaabf907992676747afc0d25bcbdad76f074edf9ed73c31924e8b2e28216f5", + "latency_ms": 332.36912498250604, + "completed_seconds": 10.332369791984092 + }, + { + "index": 21, + "offered_seconds": 10.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 9.241041989298537, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_0d2a1275d34c061f7be6f9d623a74a24095f72501261315b1994a3f877897762", + "latency_ms": 342.3760829900857, + "completed_seconds": 10.842377041990403 + }, + { + "index": 22, + "offered_seconds": 11.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.11741699767299, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_59cb3054b18d13111cb746e7220e145e4741a6bf50c5e3b3655bcc8cb8805e3b", + "latency_ms": 341.231832979247, + "completed_seconds": 11.34123270798591 + }, + { + "index": 23, + "offered_seconds": 11.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.11216698680073, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_c3192d88307a2d2a432b1130eba64bda30b38456036a89adc38062b7e074c5d5", + "latency_ms": 341.6482500033453, + "completed_seconds": 11.841649166977732 + }, + { + "index": 24, + "offered_seconds": 12.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.14450000366196, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_f36a21866aeeaf29f0b6c70a65461617ff99bee1c6ab2d9d711556310da163df", + "latency_ms": 340.23412500391714, + "completed_seconds": 12.340234999981476 + }, + { + "index": 25, + "offered_seconds": 12.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.141416976694018, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_43067bd3e3487fa80490ca1db114dd2727565dd93a9c6e5ef950cbe3c9f3cb63", + "latency_ms": 326.22299998183735, + "completed_seconds": 12.8262238749885 + }, + { + "index": 26, + "offered_seconds": 13.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 6.301874993368983, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_3131c42d99f94713439e60b9c3871e968744add7ad826d0f63fe6102fe915015", + "latency_ms": 338.44145797775127, + "completed_seconds": 13.338442457985366 + }, + { + "index": 27, + "offered_seconds": 13.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 0.256957981036976, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_8622e70e0fb0a4a65133a8e76e0122dc90516518964fa3041ca72df917c16891", + "latency_ms": 327.36766699235886, + "completed_seconds": 13.82736841699807 + }, + { + "index": 28, + "offered_seconds": 14.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 3.043374978005886, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_40feb22168dbe4e2e7b6060cd781bc2c0d807a5afe9d24400cc836443f543e62", + "latency_ms": 332.9546669847332, + "completed_seconds": 14.332955457997741 + }, + { + "index": 29, + "offered_seconds": 14.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.117500001797453, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_65319104848cacd7343d4bc1f34c64eaea3298a9524554bb18a5beefb5536ff5", + "latency_ms": 310.37187497713603, + "completed_seconds": 14.810372624982847 + }, + { + "index": 30, + "offered_seconds": 15.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.176541982218623, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_cc3748ce14473c67c09053f27d1cef260d58782945761d4df97fa443ad984232", + "latency_ms": 343.1104579940438, + "completed_seconds": 15.343111374997534 + }, + { + "index": 31, + "offered_seconds": 15.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 9.896250005112961, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_45a152b134c8d65a4798c5cc77a6c53962d530503d6c4ba0893d2d4e73b4ed90", + "latency_ms": 342.4699169991072, + "completed_seconds": 15.842470791976666 + }, + { + "index": 32, + "offered_seconds": 16.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 9.048332984093577, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_dbf498d76e602ea9f47927e24429a1260d1f67a74a848abf72e3034cbfce7e50", + "latency_ms": 338.04800000507385, + "completed_seconds": 16.338048957986757 + }, + { + "index": 33, + "offered_seconds": 16.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.0802079832647, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_44f6486596a6c2053ebdbfeecdd2f4ff70fdf6994e2b0ceab6b5f7de7d22ce29", + "latency_ms": 314.3101669847965, + "completed_seconds": 16.81431104199146 + }, + { + "index": 34, + "offered_seconds": 17.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 6.945207976968959, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_83ac2f8885762f3e4311dbf16f35eff90f0c7c6fc2d6eabfd629d5bc539f1596", + "latency_ms": 329.1951249993872, + "completed_seconds": 17.32919595798012 + }, + { + "index": 35, + "offered_seconds": 17.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.249249986372888, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_e5c13de4d730243b331529cb0ff9e936495a11349b87626fec76288ec333105c", + "latency_ms": 345.6919579766691, + "completed_seconds": 17.84569279200514 + }, + { + "index": 36, + "offered_seconds": 18.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 7.744000002276152, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_c1c50e1ffd4f0789bc4edf1042b487d6f59d771f3b58b3fe12478d586ea5cd0a", + "latency_ms": 336.4784169825725, + "completed_seconds": 18.336479416990187 + }, + { + "index": 37, + "offered_seconds": 18.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 7.602208002936095, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_5b619bfa04109036e8921bf709f310b00dfbff8cfc44ac1e3a9c441bf3ff51b2", + "latency_ms": 340.51829198142514, + "completed_seconds": 18.84051929198904 + }, + { + "index": 38, + "offered_seconds": 19.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.137124976608902, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_a315da76dce866f791c33e216ba539c22a907b901434a280941bb832746871b4", + "latency_ms": 315.02437498420477, + "completed_seconds": 19.31502520799404 + }, + { + "index": 39, + "offered_seconds": 19.5, + "status": "open_delivery_quota", + "stages": [ + { + "stage": "send", + "ok": false, + "code": "open_delivery_quota" + } + ], + "offer_lag_ms": 3.6427919985726476, + "node": null, + "storage_accepted": false, + "saved_receipt": false, + "local_read": false, + "memory_recalled": false, + "latency_ms": 117.27504199370742, + "completed_seconds": 19.617276042001322 + }, + { + "index": 40, + "offered_seconds": 20.0, + "status": "open_delivery_quota", + "stages": [ + { + "stage": "send", + "ok": false, + "code": "open_delivery_quota" + } + ], + "offer_lag_ms": 0.5479169776663184, + "node": null, + "storage_accepted": false, + "saved_receipt": false, + "local_read": false, + "memory_recalled": false, + "latency_ms": 115.80499997944571, + "completed_seconds": 20.115806124988012 + }, + { + "index": 41, + "offered_seconds": 20.5, + "status": "open_delivery_quota", + "stages": [ + { + "stage": "send", + "ok": false, + "code": "open_delivery_quota" + } + ], + "offer_lag_ms": 6.151374982437119, + "node": null, + "storage_accepted": false, + "saved_receipt": false, + "local_read": false, + "memory_recalled": false, + "latency_ms": 122.4948329909239, + "completed_seconds": 20.622496041993145 + }, + { + "index": 42, + "offered_seconds": 21.0, + "status": "open_delivery_quota", + "stages": [ + { + "stage": "send", + "ok": false, + "code": "open_delivery_quota" + } + ], + "offer_lag_ms": 8.526624995283782, + "node": null, + "storage_accepted": false, + "saved_receipt": false, + "local_read": false, + "memory_recalled": false, + "latency_ms": 87.92895800434053, + "completed_seconds": 21.08792995798285 + }, + { + "index": 43, + "offered_seconds": 21.5, + "status": "open_delivery_quota", + "stages": [ + { + "stage": "send", + "ok": false, + "code": "open_delivery_quota" + } + ], + "offer_lag_ms": 6.70687499223277, + "node": null, + "storage_accepted": false, + "saved_receipt": false, + "local_read": false, + "memory_recalled": false, + "latency_ms": 121.67724999017082, + "completed_seconds": 21.62167891699937 + }, + { + "index": 44, + "offered_seconds": 22.0, + "status": "open_delivery_quota", + "stages": [ + { + "stage": "send", + "ok": false, + "code": "open_delivery_quota" + } + ], + "offer_lag_ms": 10.189332999289036, + "node": null, + "storage_accepted": false, + "saved_receipt": false, + "local_read": false, + "memory_recalled": false, + "latency_ms": 126.61791700520553, + "completed_seconds": 22.126618791982764 + }, + { + "index": 45, + "offered_seconds": 22.5, + "status": "open_delivery_quota", + "stages": [ + { + "stage": "send", + "ok": false, + "code": "open_delivery_quota" + } + ], + "offer_lag_ms": 8.597291976911947, + "node": null, + "storage_accepted": false, + "saved_receipt": false, + "local_read": false, + "memory_recalled": false, + "latency_ms": 125.30191699624993, + "completed_seconds": 22.62530320798396 + }, + { + "index": 46, + "offered_seconds": 23.0, + "status": "open_delivery_quota", + "stages": [ + { + "stage": "send", + "ok": false, + "code": "open_delivery_quota" + } + ], + "offer_lag_ms": 9.995375003200024, + "node": null, + "storage_accepted": false, + "saved_receipt": false, + "local_read": false, + "memory_recalled": false, + "latency_ms": 126.90704199485481, + "completed_seconds": 23.12690812497749 + }, + { + "index": 47, + "offered_seconds": 23.5, + "status": "open_delivery_quota", + "stages": [ + { + "stage": "send", + "ok": false, + "code": "open_delivery_quota" + } + ], + "offer_lag_ms": 10.21012497949414, + "node": null, + "storage_accepted": false, + "saved_receipt": false, + "local_read": false, + "memory_recalled": false, + "latency_ms": 125.14387498958968, + "completed_seconds": 23.625145041994983 + } + ], + "driver": { + "user_cpu_seconds": 0.11804799999999999, + "system_cpu_seconds": 0.064217, + "max_rss_bytes": 60915712 + } + } +} diff --git a/docs/evidence/capacity-alpha40/native-replica.json b/docs/evidence/capacity-alpha40/native-replica.json new file mode 100644 index 0000000..3fdd52c --- /dev/null +++ b/docs/evidence/capacity-alpha40/native-replica.json @@ -0,0 +1,52 @@ +{ + "synthetic_replica_closure": { + "message_id": "msg_2ebb8cf80c810667c7b47075c6c7f7d98c8868f06d29a9b4920c427ab5c56a6d", + "envelope_sha256": "a5ef1acfee4e12d00904e9fe3dccdb18b832fc5b8b1aeb127a71e03681b558d1", + "copied_ciphertext_bytes": 7059, + "recipient_saved": true, + "native_offline_recall": true, + "retained_revocation_code": "repair_authority_revoked", + "revoked_repair_http_calls": 0, + "recipient_runtime": "native", + "cold_known_message_load": [ + { + "offered": 1, + "completed": 1, + "elapsed_ms": 17902.777874987805, + "repair_http_calls": 12, + "copy_work_before": 46, + "copy_work_after": 58, + "copied_ciphertext_bytes": 7059, + "subprocess_calls": 0 + }, + { + "offered": 1, + "completed": 0, + "elapsed_ms": 9330.418333003763, + "repair_http_calls": 7, + "copy_work_before": 58, + "copy_work_after": 64, + "copied_ciphertext_bytes": 0, + "subprocess_calls": 0, + "errors": [ + "open_request_rejected" + ] + }, + { + "offered": 1, + "completed": 0, + "elapsed_ms": 577.4701249902137, + "repair_http_calls": 1, + "copy_work_before": 64, + "copy_work_after": 64, + "copied_ciphertext_bytes": 0, + "subprocess_calls": 0, + "errors": [ + "open_request_rejected" + ] + } + ] + }, + "original_send_native_confirmed": true, + "frozen_send_replay_network_calls": 0 +} diff --git a/docs/evidence/capacity-alpha40/replica-profile.json b/docs/evidence/capacity-alpha40/replica-profile.json new file mode 100644 index 0000000..caff251 --- /dev/null +++ b/docs/evidence/capacity-alpha40/replica-profile.json @@ -0,0 +1,115 @@ +{ + "synthetic": true, + "real_models": 0, + "physical_hosts": 1, + "cpu": "Apple M1", + "scope": "additional known-message demand; full COPY and current READ preparation only; timing-only instrumentation; excludes HTTP/packing/Node client CPU", + "request_work_ceiling": 64, + "passed": true, + "requests": [ + { + "kind": "challenge", + "wall_seconds": 1.1301507920143194, + "thread_cpu_seconds": 1.1269313749999998 + }, + { + "kind": "proof", + "wall_seconds": 1.122289249993628, + "thread_cpu_seconds": 1.120069167 + }, + { + "kind": "child", + "wall_seconds": 1.1198066660144832, + "thread_cpu_seconds": 1.117508125 + }, + { + "kind": "child", + "wall_seconds": 1.1400847500190139, + "thread_cpu_seconds": 1.1351167920000003 + }, + { + "kind": "child", + "wall_seconds": 1.1360727910068817, + "thread_cpu_seconds": 1.1338473750000002 + }, + { + "kind": "child", + "wall_seconds": 1.1336871660023462, + "thread_cpu_seconds": 1.1317049999999993 + }, + { + "kind": "child", + "wall_seconds": 1.137808249972295, + "thread_cpu_seconds": 1.1355606670000005 + }, + { + "kind": "child", + "wall_seconds": 1.1339372500078753, + "thread_cpu_seconds": 1.1312616249999996 + }, + { + "kind": "child", + "wall_seconds": 1.132088250014931, + "thread_cpu_seconds": 1.1297155830000003 + }, + { + "kind": "child", + "wall_seconds": 1.1340743750042748, + "thread_cpu_seconds": 1.1320607919999999 + }, + { + "kind": "child", + "wall_seconds": 1.1349039579799864, + "thread_cpu_seconds": 1.1327826250000004 + }, + { + "kind": "child", + "wall_seconds": 1.1311838749970775, + "thread_cpu_seconds": 1.1274406659999998 + }, + { + "kind": "challenge", + "wall_seconds": 1.1351207080006134, + "thread_cpu_seconds": 1.132373167 + }, + { + "kind": "proof", + "wall_seconds": 1.127089750021696, + "thread_cpu_seconds": 1.1238766670000002 + }, + { + "kind": "child", + "wall_seconds": 1.128002541983733, + "thread_cpu_seconds": 1.1249620420000008 + }, + { + "kind": "child", + "wall_seconds": 1.127287166978931, + "thread_cpu_seconds": 1.1240482500000004 + }, + { + "kind": "child", + "wall_seconds": 1.12175745802233, + "thread_cpu_seconds": 1.1187367499999992 + }, + { + "kind": "child", + "wall_seconds": 1.1303392910049297, + "thread_cpu_seconds": 1.1272125830000004 + }, + { + "kind": "child", + "wall_seconds": 0.0043163750087842345, + "thread_cpu_seconds": 0.004315875000000524 + }, + { + "kind": "challenge", + "wall_seconds": 0.004289208009140566, + "thread_cpu_seconds": 0.004282499999999523 + } + ], + "total_server_thread_cpu_seconds": 20.313807626000003, + "total_server_request_wall_seconds": 20.36428987205727, + "first_successful_cold_read_server_preparation_thread_cpu_seconds": 13.553999792, + "first_successful_cold_read_server_preparation_wall_seconds": 13.586087373027112 +} diff --git a/llms.txt b/llms.txt index 364a487..7da1570 100644 --- a/llms.txt +++ b/llms.txt @@ -1,7 +1,7 @@ # Memory Vault Model-neutral encrypted agent communication and persistent original memories. -Published prerelease: v0.28.0-alpha.0.37. Check the published release record for exact +Current release target: v0.28.0-alpha.0.40. Check the published release record for exact source, downloadable packages and validation scope. Start with docs/OPEN_NETWORK_QUICKSTART.md and AI_START_HERE.md. A participant @@ -30,7 +30,7 @@ Mailbox consent preparation (Python): memory_vault_open_repair_mailbox_consent.p Ordinary replica reception (Python): connect/register_replica retains an explicitly selected authorized message replica; ordinary receive recovers it through the durable inbox after source loss and restart, with current destination checks and no duplicate import. -Native original-source ACK confirmation: connect/recover_receipt and unchanged original send can confirm independently retained receipts from an existing preparation. Original-root revocations survive restart, language changes and replacement bootstraps. Preparation and replica reception remain Python. +Native original-source ACK confirmation: connect/recover_receipt and unchanged original send can confirm independently retained receipts from an existing preparation. Original-root revocations survive restart, language changes and replacement bootstraps. Preparation and native ACK/root/feed replica reception remain Python. Directory-discovered ACK confirmation: Python and native TypeScript connect/recover_discovered_receipt use an explicit independent directory, then independently read the exact receipt. Missing facts and wrong custody cannot confirm a send. Directory replacement and publication still require explicit independent permission. @@ -38,15 +38,15 @@ Routed ACK recovery: Python and native TypeScript connect/recover_routed_receipt Native saved receipt return: connect/return_receipt authenticates the actual saved inbox and separate original return authority, shares the pending/completed journal with Python, and resumes the same upload after restart without a new grant. Native provisioning and full mailbox parity remain unfinished. -Native remote mailbox recovery: register_mailbox selects an explicitly authorized source; ordinary receive verifies current READ permission, full original feed and ciphertext, then imports through the durable inbox. Receiver configuration and refusal history are shared with Python. Native provisioning and replica inbox recovery remain unfinished. +Native remote mailbox recovery: register_mailbox selects an explicitly authorized source; ordinary receive verifies current READ permission, full original feed and ciphertext, then imports through the durable inbox. Receiver configuration and refusal history are shared with Python. Native provisioning and ACK/root/feed replica inbox recovery remain unfinished; native message replica recovery is available. -Native cold-mailbox receipt returns: register_mailbox_receipt_return retains an independently selected ACK destination. Ordinary receive returns the saved receipt with original permission, same-upload restart and cross-client history. Expired uncertain returns stop for reconciliation. Native provisioning and replica reception remain unfinished. +Native cold-mailbox receipt returns: register_mailbox_receipt_return retains an independently selected ACK destination. Ordinary receive returns the saved receipt with original permission, same-upload restart and cross-client history. Expired uncertain returns stop for reconciliation. Native provisioning and ACK/root/feed replica reception remain unfinished; native message replica recovery is available. Ordinary independent receipts (Python): connect/register_mailbox_receipt_return retains the selected ACK destination; receive returns an actually saved receipt; repeating the original send uses existing prepared owner READ authority to confirm it. Lost replies and restarts preserve the same requests. Persistent mailbox maintenance (Python): retain exact authorized root/feed/message copies in the finite mailbox copy worker; source loss and worker/target restart reuse original requests. ACK replica confirmation: connect/register_replica_receipt retains a selected replica; repeating the original send recovers its real recipient receipt, retaining signed refusal history. -Mailbox replicas (Python Agent): docs/OPEN_ACK_RECOVERY.md — explicit separately authorized directory/feed/message copies; original-key message and memory reception through the durable inbox after source loss; independent receipt return when the original message includes ACK authority. Automatic replacement and native replica reception remain unfinished. +Mailbox replicas (Python Agent): docs/OPEN_ACK_RECOVERY.md — explicit separately authorized directory/feed/message copies; original-key message and memory reception through the durable inbox after source loss; independent receipt return when the original message includes ACK authority. Automatic replacement and native ACK/root/feed replica reception remain unfinished. Receipt recovery: - docs/OPEN_ACK_PROVISIONING.md — independent remote source setup with only the sender's own config and the selected source's public origin/key; original unbound source precedes ciphertext. @@ -72,3 +72,5 @@ Reference: Use generic implementation and synthetic fixtures in public contributions. Never publish private memories, credentials or identity files. Reading this repository enables no relay, hook or background service. + +Bounded alpha40 capacity: docs/CAPACITY_HANDOFF.md. Native message replicas verify full historical COPY and current READ with shared protected receiver/status state. Same-host 48-message load completes 39 -> 48; independent-host/global capacity is unverified. Finite work grants and slow cold reads remain explicit limits. diff --git a/memory_vault.py b/memory_vault.py index d7b0e14..6949a29 100644 --- a/memory_vault.py +++ b/memory_vault.py @@ -60,7 +60,7 @@ from typing import Any, Callable, Iterable, Mapping, Sequence -VERSION = "0.28.0-alpha.0.37" +VERSION = "0.28.0-alpha.0.40" REQUEST_SCHEMA = "universal-agent-memory-request/v1" RESULT_SCHEMA = "universal-agent-memory-result/v1" RECORD_SCHEMA = "universal-memory-record/v1" diff --git a/memory_vault_network.py b/memory_vault_network.py index 6303f9e..fc2e8e8 100644 --- a/memory_vault_network.py +++ b/memory_vault_network.py @@ -287,22 +287,30 @@ def db(self): CREATE TABLE IF NOT EXISTS quarantine(message_id TEXT PRIMARY KEY,digest TEXT NOT NULL UNIQUE, sender TEXT NOT NULL,envelope BLOB NOT NULL,code TEXT NOT NULL); """) - with connection: - connection.execute("BEGIN IMMEDIATE") - if "recipients" not in {row[1] for row in connection.execute("PRAGMA table_info(outbox)")}: - # Older completely offline rows did not preserve routing. - # Keep them intact; only their original caller can supply - # missing recipients when no frozen envelope exists. - connection.execute("ALTER TABLE outbox ADD COLUMN recipients BLOB") - binding = connection.execute("SELECT value FROM state WHERE key='configuration_binding'").fetchone() - if binding is None: - occupied = any(connection.execute("SELECT 1 FROM " + table + " LIMIT 1").fetchone() - for table in ("state", "outbox", "inbox", "acknowledgements", "quarantine")) - if occupied: - raise MemoryError("network_state_binding_missing") - connection.execute("INSERT INTO state VALUES('configuration_binding',?)", (canonical_bytes(self._binding).decode(),)) - elif strict_json_loads(binding["value"]) != self._binding: - raise MemoryError("network_state_configuration_mismatch") + binding = connection.execute("SELECT value FROM state WHERE key='configuration_binding'").fetchone() + migrated = "recipients" in {row[1] for row in connection.execute("PRAGMA table_info(outbox)")} + if binding is not None and strict_json_loads(binding["value"]) != self._binding: + raise MemoryError("network_state_configuration_mismatch") + # Normal opens verify the current durable binding without becoming + # writers. Initialization/migration still rechecks under the writer + # lock so two first opens cannot admit different configurations. + if binding is None or not migrated: + with connection: + connection.execute("BEGIN IMMEDIATE") + if "recipients" not in {row[1] for row in connection.execute("PRAGMA table_info(outbox)")}: + # Older completely offline rows did not preserve routing. + # Keep them intact; only their original caller can supply + # missing recipients when no frozen envelope exists. + connection.execute("ALTER TABLE outbox ADD COLUMN recipients BLOB") + binding = connection.execute("SELECT value FROM state WHERE key='configuration_binding'").fetchone() + if binding is None: + occupied = any(connection.execute("SELECT 1 FROM " + table + " LIMIT 1").fetchone() + for table in ("state", "outbox", "inbox", "acknowledgements", "quarantine")) + if occupied: + raise MemoryError("network_state_binding_missing") + connection.execute("INSERT INTO state VALUES('configuration_binding',?)", (canonical_bytes(self._binding).decode(),)) + elif strict_json_loads(binding["value"]) != self._binding: + raise MemoryError("network_state_configuration_mismatch") with connection: yield connection finally: diff --git a/memory_vault_open_delivery_client.py b/memory_vault_open_delivery_client.py index 287c5c2..e525bb2 100644 --- a/memory_vault_open_delivery_client.py +++ b/memory_vault_open_delivery_client.py @@ -351,11 +351,12 @@ async def send(self, request_id, recipients, text="", memory_ids=None, control=N budget = _DeliveryBudget() if _deadline is not None:budget.deadline=min(budget.deadline,_deadline) if row["session"] is None: - session = await self._sending_session(recipients[0], budget) - row = self._encrypt_outbox(row, session) + session = await self._sending_session(recipients[0], budget, row=row) + row = self._outbox(request_id) else: session = document(bytes(row["session"]), maximum=MAX_SESSION_BYTES) - self._encrypt_outbox(row, session) + row = self._encrypt_outbox(row, session) + session = document(bytes(row["session"]), maximum=MAX_SESSION_BYTES) node, pending_code = None, None if row["result"] is None: # Recover a committed upload even if its final response was lost @@ -515,7 +516,7 @@ def _keys(session): "recipient_signing_key": policy["signing_key"], "recipient_encryption_key": policy["encryption_key"]} - async def _sending_session(self, recipient, budget): + async def _sending_session(self, recipient, budget, *, row=None): sessions = self._contact_sessions(outgoing=True, recipient=recipient) if not sessions: # Fetch only an existing explicit request's decision. This neither @@ -543,13 +544,101 @@ async def _sending_session(self, recipient, budget): budget.merge_routing(route_budget) break sessions = self._contact_sessions(outgoing=True, recipient=recipient) + eligible, unavailable = [], [] for session in sessions: try: - return await self._checked_session(session, budget) + checked = await self._checked_session(session, budget) + if row is None: + # Repair callers retain their original first valid choice. + return checked + eligible.append(checked) except ContactError as exc: - if exc.code not in {"contact_expired", "contact_policy_mismatch"}: + if exc.code in {"contact_expired", "contact_policy_mismatch"}: + continue + if row is not None and exc.code == "contact_resource_unresolved": + unavailable.append(exc) + continue + else: raise - raise MemoryError("open_contact_approval_required") + if not eligible: + if unavailable: + raise unavailable[0] + raise MemoryError("open_contact_approval_required") + frozen = self._freeze_dispatch(row, eligible) + return document(bytes(frozen["session"]), maximum=MAX_SESSION_BYTES) + + @staticmethod + def _session_scope(session): + """The exact signed resource lease; aliases cannot multiply its quota.""" + lease = session["decision"]["payload"]["grant"]["payload"]["resource_lease"] + return hashlib.sha256(canonical_bytes(lease)).hexdigest() + + def _freeze_dispatch(self, row, sessions): + """Atomically place a new immutable envelope in a finite approved slot. + + Count every local frozen send, including unfinished uploads, against its + exact grant. The server independently rechecks live authority and quota. + Existing frozen sends never move to a different node on retry or failure. + """ + keys = self._keys(sessions[0]) + # Preserve the formerly selected recipient key incarnation. Never pool + # approvals for different encryption identities under one ciphertext. + sessions = [session for session in sessions if self._keys(session) == keys] + envelope = create_envelope(bytes(row["body"]), signer=self.identity, + sender_encryption_key=self.encryption.public_descriptor(), + recipient_signing_key=keys["recipient_signing_key"], + recipient_encryption_key=keys["recipient_encryption_key"], + message_id=row["message_id"], object_key=secrets.token_hex(32), created_at=row["created_at"]) + encoded = canonical_bytes(document(envelope, maximum=MAX_ENVELOPE_BYTES)) + with self.participant.state.db() as db: + db.execute("BEGIN IMMEDIATE") + prior = db.execute("SELECT * FROM open_delivery_outbox WHERE request_id=?", (row["request_id"],)).fetchone() + if prior is None: + raise MemoryError("open_delivery_outbox_missing") + if prior["envelope"] is not None: + return dict(prior) + used = {} + # MAX_OUTBOX_RECORDS is enforced before admission; no new roster or + # unbounded historical archive is introduced by this local scan. + for saved in db.execute("SELECT session,length(envelope) AS bytes FROM open_delivery_outbox WHERE session IS NOT NULL LIMIT ?", (MAX_OUTBOX_RECORDS,)): + session = document(bytes(saved["session"]), maximum=MAX_SESSION_BYTES) + scope = self._session_scope(session) + items, size = used.get(scope, (0, 0)) + used[scope] = (items+1, size+(saved["bytes"] or MAX_ENVELOPE_BYTES)) + available = [] + for session in sessions: + lease = session["decision"]["payload"]["grant"]["payload"]["resource_lease"]["payload"] + scope = self._session_scope(session) + items, size = used.get(scope, (0, 0)) + if items < lease["max_items"] and size+len(encoded) <= lease["max_bytes"]: + available.append((items/lease["max_items"], size/lease["max_bytes"], scope, session)) + if not available: + if all(len(encoded) > session["decision"]["payload"]["grant"]["payload"]["resource_lease"]["payload"]["max_bytes"] for session in sessions): + raise MemoryError("open_delivery_approval_bytes_insufficient") + raise MemoryError("open_delivery_approval_capacity") + session = min(available, key=lambda x: x[:3])[3] + proof = canonical_bytes(document(session, maximum=MAX_SESSION_BYTES)) + _, size = self._local_size(db, "open_delivery_outbox") + if size-MAX_ENVELOPE_BYTES-MAX_SESSION_BYTES+len(encoded)+len(proof) > MAX_LOCAL_BYTES: + raise MemoryError("network_outbox_capacity") + db.execute("UPDATE open_delivery_outbox SET envelope=?,session=? WHERE request_id=?", (encoded,proof,row["request_id"])) + return dict(db.execute("SELECT * FROM open_delivery_outbox WHERE request_id=?", (row["request_id"],)).fetchone()) + + def _receiving_sessions(self): + sessions = self._contact_sessions(outgoing=False) + with self.participant.state.db() as db: + row = db.execute("SELECT body FROM open_delivery_client_state WHERE key='receive_session_cursor'").fetchone() + if row is not None: + previous = bytes(row[0]).decode("ascii") + for index, session in enumerate(sessions): + if self._session_scope(session) == previous: + return sessions[index+1:]+sessions[:index+1] + return sessions + + def _advance_receiving_session(self, session): + with self.participant.state.db() as db: + db.execute("INSERT INTO open_delivery_client_state VALUES('receive_session_cursor',?) ON CONFLICT(key) DO UPDATE SET body=excluded.body", + (self._session_scope(session).encode("ascii"),)) def _encrypt_outbox(self, row, session): if row["envelope"] is not None: @@ -955,7 +1044,7 @@ async def receive(self, limit=4, *, _pending_only=False, _skip_pending=False, _d self._advance_pending_receipt(pending_row) if _pending_only: return dict(messages=messages,errors=errors[:4],network_accessed=budget.requests>0) - for candidate in self._contact_sessions(outgoing=False): + for candidate in self._receiving_sessions(): if len(messages) >= limit: break try: @@ -994,6 +1083,8 @@ async def receive(self, limit=4, *, _pending_only=False, _skip_pending=False, _d errors.append({"code": exc.code, "retryable": exc.retryable}) if len(errors) >= 4: break + finally: + self._advance_receiving_session(candidate) return {"messages": messages, "errors": errors[:4], "network_accessed": budget.requests > 0} def read_message(self, message_id, offset=0): diff --git a/memory_vault_open_index.py b/memory_vault_open_index.py index 9073017..cd04892 100644 --- a/memory_vault_open_index.py +++ b/memory_vault_open_index.py @@ -84,23 +84,33 @@ def _prune(self, now: int) -> None: self.connection.execute("DELETE FROM open_contact_floors WHERE retain_until<=? AND owner NOT IN (SELECT owner FROM open_contacts)", (now,)) self.connection.execute("DELETE FROM open_index_replay WHERE retain_until<=?", (now,)) - def _capacity(self) -> None: + def _capacity(self, *, now: int | None = None) -> None: db = self.connection - records = db.execute("SELECT count(*) FROM open_contact_floors").fetchone()[0] - replays = db.execute("SELECT count(*) FROM open_index_replay").fetchone()[0] + # A read snapshot applies the same logical expiry as _prune without + # turning every directory lookup into a serialized garbage-collection + # write. Allocation still prunes and checks physical obligations. + floors_where = "" if now is None else " WHERE retain_until>? OR owner IN (SELECT owner FROM open_contacts WHERE expires_at>?)" + floor_args = () if now is None else (now, now) + replay_where = "" if now is None else " WHERE retain_until>?" + contact_where = "" if now is None else " WHERE expires_at>?" + live_args = () if now is None else (now,) + records = db.execute("SELECT count(*) FROM open_contact_floors" + floors_where, floor_args).fetchone()[0] + replays = db.execute("SELECT count(*) FROM open_index_replay" + replay_where, live_args).fetchone()[0] # Reserve two maximum descriptors per floor so observing a same-revision # fork can always invalidate a previously accepted contact, even when # all unreserved capacity is full. This is reserved capacity, not a # claim that padding or two records physically exist on disk. floors = records * (2 * 4096 + 512) - contacts = db.execute("SELECT coalesce(sum(length(record)+length(lease)+256),0) FROM open_contacts").fetchone()[0] - replay_bytes = db.execute("SELECT coalesce(sum(length(response)+256),0) FROM open_index_replay").fetchone()[0] + contacts = db.execute("SELECT coalesce(sum(length(record)+length(lease)+256),0) FROM open_contacts" + contact_where, live_args).fetchone()[0] + replay_bytes = db.execute("SELECT coalesce(sum(length(response)+256),0) FROM open_index_replay" + replay_where, live_args).fetchone()[0] if records > self.maximum_records or replays > self.maximum_replays or floors + contacts + replay_bytes > self.maximum_bytes: raise OpenControlError("open_index_capacity", retryable=True) def _get(self, key: str, now: int) -> dict[str, Any]: floor = self.connection.execute( - "SELECT owner,status FROM open_contact_floors WHERE lookup_key=?", (key,)).fetchone() + """SELECT owner,status FROM open_contact_floors WHERE lookup_key=? AND + (retain_until>? OR owner IN (SELECT owner FROM open_contacts WHERE expires_at>?))""", + (key, now, now)).fetchone() if floor is None: return {"state": "not_found"} if floor[1] in {"revoked", "conflict"}: @@ -141,17 +151,18 @@ def handle(self, request: Mapping[str, Any] | bytes, *, now: int | None = None) db = self.connection if db.in_transaction: raise OpenControlError("open_storage_transaction") - db.execute("BEGIN IMMEDIATE") + reading = checked["action"] == "get" + db.execute("BEGIN" if reading else "BEGIN IMMEDIATE") try: # A wait for the writer lock must not extend request authority. current = int(time.time()) if now is None else integer(now) checked = verify_request(signed_request, node=self.node, now=current) - self._prune(current) - if checked["action"] == "get": + if reading: result = self._get(checked["body"]["key"], current) else: + self._prune(current) result, failure = self._write(signed_request, checked, current) - self._capacity() + self._capacity(now=current if reading else None) db.commit() except BaseException: db.rollback() diff --git a/memory_vault_open_node.py b/memory_vault_open_node.py index 49144a4..18f2acd 100644 --- a/memory_vault_open_node.py +++ b/memory_vault_open_node.py @@ -15,6 +15,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path import secrets +import queue import socket from socketserver import TCPServer import sqlite3 @@ -24,7 +25,7 @@ from memory_vault import MemoryError, canonical_bytes from memory_vault_network import NetworkClient, _read_private -from memory_vault_network_crypto import document, document_sha256, integer, object_fields +from memory_vault_network_crypto import document, document_sha256, integer, object_fields, digest from memory_vault_open_control import ( contact_key, coordinate, issue_node, sign_request, sign_response, verify_contact, verify_node, verify_response, verify_request, @@ -220,6 +221,9 @@ def __init__(self, identity: Identity, state_directory: Path, *, seeds: list, self._publication = None self._maintenance_cycle = 0 self._seed_refresh_after = 0.0 + self._neighbor_lock = threading.Lock() + self._neighbor_cursor = 0 + self._neighbor_probe_after = 0.0 with self.state.db() as db: OpenCheckpoints(db).initialize() db.execute("CREATE TABLE IF NOT EXISTS open_peer_cache (key_id TEXT PRIMARY KEY,node BLOB NOT NULL,seen_at INTEGER NOT NULL)") @@ -557,6 +561,73 @@ async def _lookup(self, target, view, budget): result["partial"] = result["partial"] or bool(refresh_errors) return result + async def find_neighbors(self, target: str, *, view="general", budget=None): + """One distributed routing step, returning introductions, not a contact. + + Repeated callers share a bounded, endpoint-proven router pool instead + of pinning every routing query to the initial seed. A reply can introduce + one new router for a fresh hello proof, at most once per five seconds. + This does not replace iterative lookup or its multi-shard contact checks. + No remote node receives the caller's authority or private message data. + """ + digest(target) + if view not in ("general", "directory"): + raise MemoryError("open_invalid_view") + budget = budget or LookupBudget() + errors = await self._refresh_seed_introductions(budget) + introductions = [node for lane in self._initial(target, "general") for node in lane] + proven = {node["payload"]["signing_key"]["key_id"]: node for node in introductions + if "router" in node["payload"]["roles"] and self.table.has_verified(node)} + nodes = sorted(proven.values(), key=lambda n: n["payload"]["signing_key"]["key_id"]) + with self._neighbor_lock: + offset = self._neighbor_cursor % len(nodes) if nodes else 0 + self._neighbor_cursor += 1 + nodes = nodes[offset:] + nodes[:offset] + # Cold starts and failed pools retain independently configured/cache + # introductions. They still need a valid response from the exact peer. + seen = set(proven) + for node in introductions: + key = node["payload"]["signing_key"]["key_id"] + if key not in seen and "router" in node["payload"]["roles"]: + nodes.append(node) + seen.add(key) + for node in nodes[:3]: + try: + body = await self._call(node, "find", {"target": target, "view": view}, budget) + except (MemoryError, OSError, TimeoutError) as exc: + self.table.mark_failed(node["payload"]["signing_key"]["key_id"]) + errors.append(getattr(exc, "code", "open_rpc_unreachable")) + continue + # The responder's proof never transfers to a returned descriptor. + # Use normal checkpoint and source/bucket limits on each new proof. + # An empty/fully proven reply must not consume the five-second + # probe allowance before a joining peer becomes discoverable. + candidates = [] + for introduced in sorted(body["nodes"], key=lambda n: + n["payload"]["signing_key"]["key_id"] in seen): + try: + if (introduced["payload"]["signing_key"]["key_id"] != self.identity.key_id + and "router" in introduced["payload"]["roles"] + and not self.table.has_verified(introduced)): + candidates.append(introduced) + except MemoryError: + continue + with self._neighbor_lock: + probe = bool(candidates) and time.monotonic() >= self._neighbor_probe_after + if probe: + self._neighbor_probe_after = time.monotonic() + 5 + if probe: + try: + await self._call(candidates[0], "hello", {"node": None}, budget) + except (MemoryError, OSError, TimeoutError) as exc: + errors.append(getattr(exc, "code", "open_rpc_unreachable")) + return {"state": "introduced", "responder": node, "nodes": body["nodes"], + "metrics": self._metrics(budget), "errors": errors, + "introductions_verified": False, "discovery_grants_access": False} + return {"state": "unreachable", "responder": None, "nodes": [], + "metrics": self._metrics(budget), "errors": errors, + "introductions_verified": False, "discovery_grants_access": False} + @staticmethod def _metrics(budget): return {"requests": budget.requests, "request_bytes": budget.request_bytes, @@ -829,7 +900,11 @@ def handle_blob(self, request, chunk): class OpenHTTPServer(ThreadingHTTPServer): - """Bounded HTTP admission. No plaintext, HTTP redirects or secret logging.""" + """Eight fixed workers and a finite queue; no per-connection worker spawn. + + Queue time is part of the existing input deadline. Overload is an explicit + retryable HTTP refusal, never a signed acceptance or a new protocol grant. + """ daemon_threads = True request_queue_size = 16 @@ -841,27 +916,85 @@ def server_bind(self): def __init__(self, address, participant): self.participant = participant - self._slots = threading.BoundedSemaphore(8) + self._requests = queue.Queue(maxsize=16) + self._closing = threading.Event() + self._deadlines = {} + self._deadlines_lock = threading.Lock() + self._workers = [] self._rate_lock = threading.Lock() self._rate = OrderedDict() self._global_rate = (0, 0) super().__init__(address, _Handler) + self._workers = [threading.Thread(target=self._work, daemon=True) for _ in range(8)] + for worker in self._workers: + worker.start() def process_request(self, request, client_address): - if not self._slots.acquire(blocking=False): - request.close() + if self._closing.is_set(): + self.shutdown_request(request) return try: - super().process_request(request, client_address) - except BaseException: - self._slots.release() - raise + self._requests.put_nowait((request, client_address, time.monotonic() + HTTP_REQUEST_SECONDS)) + except queue.Full: + self._busy(request) - def process_request_thread(self, request, client_address): + def _busy(self, request): + # A tiny fixed response, bounded send time, no input parsing or echo. try: - super().process_request_thread(request, client_address) + request.settimeout(.05) + request.sendall(b"HTTP/1.0 503 Service Unavailable\r\nContent-Length: 0\r\n" + b"Retry-After: 1\r\nConnection: close\r\n\r\n") + request.shutdown(socket.SHUT_WR) + # Closing with a normal client's unread upload can turn the 503 + # into a TCP reset. Drain only bytes already buffered, with a fixed + # ceiling; a peer cannot make admission wait for more input. + request.setblocking(False) + for _ in range(4): + if not request.recv(4096): + break + except OSError: + pass finally: - self._slots.release() + self.shutdown_request(request) + + def _work(self): + while not self._closing.is_set(): + try: + request, address, deadline = self._requests.get(timeout=.1) + except queue.Empty: + continue + try: + if self._closing.is_set() or time.monotonic() >= deadline: + self._busy(request) + continue + with self._deadlines_lock: + self._deadlines[request] = deadline + self.process_request_thread(request, address) + finally: + with self._deadlines_lock: + self._deadlines.pop(request, None) + self._requests.task_done() + + def input_deadline(self, request): + with self._deadlines_lock: + return self._deadlines[request] + + def server_close(self): + self._closing.set() + super().server_close() + while True: + try: + request, _, _ = self._requests.get_nowait() + except queue.Empty: + break + self.shutdown_request(request) + self._requests.task_done() + # Idle workers stop promptly. Active finite handlers retain the same + # daemon/timeout semantics as the previous server. + deadline = time.monotonic() + .2 + for worker in self._workers: + if worker is not threading.current_thread(): + worker.join(max(0, deadline-time.monotonic())) def admitted(self, address): now = int(time.monotonic()) @@ -886,13 +1019,17 @@ class _Handler(BaseHTTPRequestHandler): protocol_version = "HTTP/1.0" def setup(self): - self.request.settimeout(HTTP_REQUEST_SECONDS) + deadline = self.server.input_deadline(self.request) + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("open_request_timeout") + self.request.settimeout(remaining) super().setup() self.rfile = _HeaderBudget(self.rfile) # Bound the complete incoming headers/body, including drip-fed input. self._deadline_lock = threading.Lock() - self._deadline_at = time.monotonic() + HTTP_REQUEST_SECONDS - self._deadline = threading.Timer(HTTP_REQUEST_SECONDS, self._abort) + self._deadline_at = deadline + self._deadline = threading.Timer(remaining, self._abort) self._deadline.daemon = True self._deadline.start() diff --git a/memory_vault_open_repair_client.py b/memory_vault_open_repair_client.py index 7b41212..e939b76 100644 --- a/memory_vault_open_repair_client.py +++ b/memory_vault_open_repair_client.py @@ -1348,6 +1348,47 @@ class MailboxDestinationStore: def __init__(self, db, identity, encryption_identity, *, policy=DEFAULT_POLICY): self.db,self.identity,self.encryption_identity,self.policy=db,identity,encryption_identity,policy + def _retained_authority(self, builder, slots, contact, at, budget): + """Authenticate durable receive observations before issuing more authority.""" + from memory_vault_open_repair_mailbox_activation import verify_mailbox_feed_bootstrap + p=builder.plan;root=p['root_key'];owner=builder.owner;target=p['target'] + request=original.parse_original_control(contact['request'],self.policy,budget).value['payload'] + sender=dict(signing_key=request['signing_key'],encryption_key=request['encryption_key']) + plan=wire.build_new_wire(dict(kind='mailbox.feed_recovery',slot_key=p['slot_key'],owner=owner,sender=sender,target=target, + entries={name:value['ref'] for name,value in slots.items()}),self.policy,budget).raw + key=budget._hash(plan) + exists=self.db.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='open_mailbox_setup_jobs'").fetchone() + if exists is None or self.db.execute('SELECT 1 FROM open_mailbox_setup_jobs WHERE job_key=?',(key,)).fetchone() is None:return -1 + setup=verify_mailbox_feed_bootstrap(slots,expected_slot=p['slot_key'],expected_owner=owner, + expected_target=target,target_storage_epoch=p['slot_key']['writer_storage_epoch'], + limit_policy=p['limits'],at=at,policy=self.policy,budget=budget) + required={} + for name,bits in (('slot',65),('read',2),('maintenance',65),('bootstrap',10)): + entry=setup[name];kind='mailbox_slot' if name=='slot' else 'authority' + subject=p['slot_key'] if name=='slot' else dict(authority_kind=entry.payload['kind'],authority_sha256=entry.ref.raw_sha256) + required[(owner['signing_key']['key_id'],kind,status.status_scope(root,kind,subject,self.policy,budget))]=(entry.payload['revision'],bits) + for name in ('data','metadata'): + resource=setup['slot'].payload[name+'_resource_ref'] + required[(target['signing_key']['key_id'],'resource',status.status_scope(root,'resource',resource,self.policy,budget))]=(None,65) + revisions={};owner_revision=-1 + for entry in MailboxSetupJournal(self.db).statuses(key): + preview=original.parse_original_control(entry['raw'],self.policy,budget).value['payload'] + signer=next((v['signing_key'] for v in (owner,target,sender) if v['signing_key']==preview['signing_key']),None) + if signer is None or wire.u53(preview['issued_at'])>at:_fail('repair_status_disclosure') + observed=status.authenticate_status_original(entry,expected_root=root,expected_signing_key=signer,at=preview['issued_at'], + allowed_scopes=[dict(scope_kind=v['scope_kind'],scope_id=v['scope_id']) for v in preview['entries']],policy=self.policy,budget=budget) + value=observed.payload;identity=(signer['key_id'],value['revision']) + if identity in revisions and revisions[identity]!=observed.canonical_sha256:_fail('repair_status_conflict') + revisions[identity]=observed.canonical_sha256 + if signer==owner['signing_key']:owner_revision=max(owner_revision,value['revision']) + for row in value['entries']: + needed=required.get((signer['key_id'],row['scope_kind'],row['scope_id'])) + if needed is None:continue + revision,bits=needed + if row['status']=='revoked' and row['operation_mask']&bits:_fail('repair_authority_revoked') + if revision is not None and row['minimum_document_revision']>revision:_fail('repair_status_revision') + return owner_revision + def prepare(self, plan, slot_entries, contact_originals, *, at, expires_at, status_revision, status_until): from memory_vault_open_repair_bind import encode_entry,decode_entry import memory_vault_open_repair_resource as resource @@ -1365,6 +1406,7 @@ def prepare(self, plan, slot_entries, contact_originals, *, at, expires_at, stat if self.db.in_transaction:_fail('repair_destination_transaction_active') self.db.execute('BEGIN IMMEDIATE') try: + observed_revision=self._retained_authority(builder,slots,contact_originals,at,budget) self.db.execute('CREATE TABLE IF NOT EXISTS open_mailbox_destinations(root_digest TEXT NOT NULL,revision INTEGER NOT NULL,binding TEXT NOT NULL,bundle BLOB NOT NULL,bundle_sha256 TEXT NOT NULL,PRIMARY KEY(root_digest,revision))') row=self.db.execute('SELECT binding,bundle,bundle_sha256 FROM open_mailbox_destinations WHERE root_digest=? AND revision=?',(root,status_revision)).fetchone() if row is not None: @@ -1378,7 +1420,7 @@ def prepare(self, plan, slot_entries, contact_originals, *, at, expires_at, stat original._verify_control_signature(signed['payload'],signed['proof'],builder.owner['signing_key'],budget) else: latest=self.db.execute('SELECT max(revision) FROM open_mailbox_destinations WHERE root_digest=?',(root,)).fetchone()[0] - if latest is not None and status_revision<=latest:_fail('repair_destination_revision_rollback') + if status_revision<=observed_revision or (latest is not None and status_revision<=latest):_fail('repair_destination_revision_rollback') if self.db.execute('SELECT count(*) FROM open_mailbox_destinations').fetchone()[0]>=128:_fail('repair_destination_capacity') result=builder.destination_bundle(slots,contact_originals,at=at,expires_at=expires_at, status_revision=status_revision,status_until=status_until) diff --git a/packaging/CLIENT_README.md b/packaging/CLIENT_README.md index 0fa2a7f..0a32271 100644 --- a/packaging/CLIENT_README.md +++ b/packaging/CLIENT_README.md @@ -1,4 +1,44 @@ -# Memory Vault v0.28.0-alpha.0.37 — authorized full client +# Memory Vault v0.28.0-alpha.0.40 — authorized full client + +Alpha.0.40 distributes new messages across separately approved item and byte +resources in Python and native TypeScript. Existing frozen ciphertext is never +retargeted. Receive polling rotates across approved sessions; bounded HTTP +admission and concurrent SQLite reads preserve the existing execution limits. + +Native recipients can recover an explicitly selected message replica after its +original source disappears, with complete historical COPY and current READ +verification, retained revocation and independent receipt confirmation. +Source provisioning and native ACK/root/feed replica reception still use Python. + +In the synthetic same-host workload, 48 selected Memories offered at 2/second +complete 39 to 48 chains; each approved node completes 24. This measures use of +authorized capacity, not maximum throughput. Cold replica recovery remains +about 17.9 seconds and the existing 64-work allowance remains finite. Independent +hosts, WAN capacity and automatic universal repair are unverified. See +`docs/CAPACITY_HANDOFF.md` for the workload and explicit limits. + +Native TypeScript senders can prepare and retain an existing outgoing message +in the recipient-authorized mailbox through Agent `connect`. The sender keeps +the original ciphertext and persists its exact disclosure, attempt and upload +request before HTTP. A lost reply, restart or switch to Python reuses those same +bytes. The source returns signed custody that the client verifies and retains; +this does not claim the recipient saved the message or acknowledged it. + +An optional `ack_request_id` selects an already prepared independent receipt +permission. Complete signed controls and statuses travel with the message, +including bounded compressed transport for the larger draft. No new permission +or storage allocation is inferred. Native source provisioning and ACK/root/feed replica inbox +recovery still use Python. See `clients/typescript/network/README.md` and +`docs/OPEN_NETWORK_CONTACT.md` for the existing invitations. + +Native TypeScript recipients can issue an original mailbox destination from +an explicitly approved contact and a registered mailbox. The receiver signs with +its own existing identity and exports bounded authorization pages for sender +admission. Python and native clients reopen the same persisted signed originals; +conflicting selections and stale revisions fail. Both clients enforce retained +parent/resource revocations before cached or fresh authorization, even after the +status expires. Authorization creates no new storage or receipt-return grants. +See `clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md`. Native TypeScript recipients can retain an independently authorized receipt return for a cold-mailbox message. Ordinary Agent `receive` then returns the @@ -18,7 +58,7 @@ inbox. Retained revocations survive restart. Python and TypeScript share receive configuration, status history and saved inbox evidence; either can resume an interrupted import. The native path performs its own HTTP and cryptography. Saved receipts remain available through independently authorized return requests. -Native provisioning and replica inbox recovery remain unfinished. See +Native provisioning and ACK/root/feed replica inbox recovery remain unfinished. See `clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md`. Native TypeScript recipients can return an actually saved message or memory @@ -157,7 +197,7 @@ remain unfinished. There is no project-operated public seed or verified thousand-agent/global-availability result. Use the source-bound release record for actual validation and compare archive bytes with published checksums. -This full-client package targets **v0.28.0-alpha.0.37 open-delivery source**, +This full-client package targets **v0.28.0-alpha.0.40 open-delivery source**, not a stable-release or complete runtime-certification claim. Existing published versions remain immutable. Match the artifact's source and hashes to its manifest; this README does not establish installation or publication. The plugin is under diff --git a/packaging/PROTOCOL_README.md b/packaging/PROTOCOL_README.md index 4c230e7..2968508 100644 --- a/packaging/PROTOCOL_README.md +++ b/packaging/PROTOCOL_README.md @@ -1,4 +1,44 @@ -# Memory Vault v0.28.0-alpha.0.37 — independent protocol +# Memory Vault v0.28.0-alpha.0.40 — independent protocol + +Alpha.0.40 distributes new messages across separately approved item and byte +resources in Python and native TypeScript. Existing frozen ciphertext is never +retargeted. Receive polling rotates across approved sessions; bounded HTTP +admission and concurrent SQLite reads preserve the existing execution limits. + +Native recipients can recover an explicitly selected message replica after its +original source disappears, with complete historical COPY and current READ +verification, retained revocation and independent receipt confirmation. +Source provisioning and native ACK/root/feed replica reception still use Python. + +In the synthetic same-host workload, 48 selected Memories offered at 2/second +complete 39 to 48 chains; each approved node completes 24. This measures use of +authorized capacity, not maximum throughput. Cold replica recovery remains +about 17.9 seconds and the existing 64-work allowance remains finite. Independent +hosts, WAN capacity and automatic universal repair are unverified. See +`docs/CAPACITY_HANDOFF.md` for the workload and explicit limits. + +Native TypeScript senders can prepare and retain an existing outgoing message +in the recipient-authorized mailbox through Agent `connect`. The sender keeps +the original ciphertext and persists its exact disclosure, attempt and upload +request before HTTP. A lost reply, restart or switch to Python reuses those same +bytes. The source returns signed custody that the client verifies and retains; +this does not claim the recipient saved the message or acknowledged it. + +An optional `ack_request_id` selects an already prepared independent receipt +permission. Complete signed controls and statuses travel with the message, +including bounded compressed transport for the larger draft. No new permission +or storage allocation is inferred. Native source provisioning and ACK/root/feed replica inbox +recovery still use Python. See `clients/typescript/network/README.md` and +`docs/OPEN_NETWORK_CONTACT.md` for the existing invitations. + +Native TypeScript recipients can issue an original mailbox destination from +an explicitly approved contact and a registered mailbox. The receiver signs with +its own existing identity and exports bounded authorization pages for sender +admission. Python and native clients reopen the same persisted signed originals; +conflicting selections and stale revisions fail. Both clients enforce retained +parent/resource revocations before cached or fresh authorization, even after the +status expires. Authorization creates no new storage or receipt-return grants. +See `clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md`. Native TypeScript recipients can retain an independently authorized receipt return for a cold-mailbox message. Ordinary Agent `receive` then returns the @@ -18,7 +58,7 @@ inbox. Retained revocations survive restart. Python and TypeScript share receive configuration, status history and saved inbox evidence; either can resume an interrupted import. The native path performs its own HTTP and cryptography. Saved receipts remain available through independently authorized return requests. -Native provisioning and replica inbox recovery remain unfinished. See +Native provisioning and ACK/root/feed replica inbox recovery remain unfinished. See `clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md`. Native TypeScript recipients can return an actually saved message or memory @@ -174,7 +214,7 @@ require independently configured providers; reading metadata cannot grant authority or enroll keys. The complete Python client and executable synthetic review kit are separate -artifacts described in `docs/RELEASE.md`. This package targets v0.28.0-alpha.0.37; +artifacts described in `docs/RELEASE.md`. This package targets v0.28.0-alpha.0.40; previous published versions remain immutable. The optional native network adds communication around existing records without changing canonical record/v1 or share-v1. It has no MCP, A2A, Matrix, Nostr or Graphiti adapter or compatibility diff --git a/packaging/RELEASE_NOTES.md b/packaging/RELEASE_NOTES.md index 4b49e7a..d5aa5cb 100644 --- a/packaging/RELEASE_NOTES.md +++ b/packaging/RELEASE_NOTES.md @@ -1,4 +1,44 @@ -# Memory Vault v0.28.0-alpha.0.37 — automatic native mailbox receipt returns +# Memory Vault v0.28.0-alpha.0.40 — bounded delivery distribution and native message replicas + +Alpha.0.40 distributes new messages across separately approved item and byte +resources in Python and native TypeScript. Existing frozen ciphertext is never +retargeted. Receive polling rotates across approved sessions; bounded HTTP +admission and concurrent SQLite reads preserve the existing execution limits. + +Native recipients can recover an explicitly selected message replica after its +original source disappears, with complete historical COPY and current READ +verification, retained revocation and independent receipt confirmation. +Source provisioning and native ACK/root/feed replica reception still use Python. + +In the synthetic same-host workload, 48 selected Memories offered at 2/second +complete 39 to 48 chains; each approved node completes 24. This measures use of +authorized capacity, not maximum throughput. Cold replica recovery remains +about 17.9 seconds and the existing 64-work allowance remains finite. Independent +hosts, WAN capacity and automatic universal repair are unverified. See +`docs/CAPACITY_HANDOFF.md` for the workload and explicit limits. + +Native TypeScript senders can prepare and retain an existing outgoing message +in the recipient-authorized mailbox through Agent `connect`. The sender keeps +the original ciphertext and persists its exact disclosure, attempt and upload +request before HTTP. A lost reply, restart or switch to Python reuses those same +bytes. The source returns signed custody that the client verifies and retains; +this does not claim the recipient saved the message or acknowledged it. + +An optional `ack_request_id` selects an already prepared independent receipt +permission. Complete signed controls and statuses travel with the message, +including bounded compressed transport for the larger draft. No new permission +or storage allocation is inferred. Native source provisioning and ACK/root/feed replica inbox +recovery still use Python. See `clients/typescript/network/README.md` and +`docs/OPEN_NETWORK_CONTACT.md` for the existing invitations. + +Native TypeScript recipients can issue an original mailbox destination from +an explicitly approved contact and a registered mailbox. The receiver signs with +its own existing identity and exports bounded authorization pages for sender +admission. Python and native clients reopen the same persisted signed originals; +conflicting selections and stale revisions fail. Both clients enforce retained +parent/resource revocations before cached or fresh authorization, even after the +status expires. Authorization creates no new storage or receipt-return grants. +See `clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md`. Native TypeScript recipients can retain an independently authorized receipt return for a cold-mailbox message. Ordinary Agent `receive` then returns the @@ -18,7 +58,7 @@ inbox. Retained revocations survive restart. Python and TypeScript share receive configuration, status history and saved inbox evidence; either can resume an interrupted import. The native path performs its own HTTP and cryptography. Saved receipts remain available through independently authorized return requests. -Native provisioning and replica inbox recovery remain unfinished. See +Native provisioning and ACK/root/feed replica inbox recovery remain unfinished. See `clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md`. Native TypeScript recipients can return an actually saved message or memory @@ -183,7 +223,7 @@ TypeScript recognizes the mailbox replica proof and status profiles. Participant operate their own authorized nodes; no central authority or project-operated public seed is required or provided. -Use the [quickstart](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.37/docs/OPEN_NETWORK_QUICKSTART.md) -and [mailbox replica guide](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.37/docs/OPEN_ACK_RECOVERY.md#receive-messages-and-shared-memories-from-a-replica-development-after-alpha025). +Use the [quickstart](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.40/docs/OPEN_NETWORK_QUICKSTART.md) +and [mailbox replica guide](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.40/docs/OPEN_ACK_RECOVERY.md#receive-messages-and-shared-memories-from-a-replica-development-after-alpha025). Preserve existing identity and state files when installing. The archives contain implementation, public documentation and wholly synthetic fixtures only. diff --git a/packaging/REVIEW_README.md b/packaging/REVIEW_README.md index 0374631..431cdbb 100644 --- a/packaging/REVIEW_README.md +++ b/packaging/REVIEW_README.md @@ -1,4 +1,44 @@ -# Memory Vault v0.28.0-alpha.0.37 independent review kit +# Memory Vault v0.28.0-alpha.0.40 independent review kit + +Alpha.0.40 distributes new messages across separately approved item and byte +resources in Python and native TypeScript. Existing frozen ciphertext is never +retargeted. Receive polling rotates across approved sessions; bounded HTTP +admission and concurrent SQLite reads preserve the existing execution limits. + +Native recipients can recover an explicitly selected message replica after its +original source disappears, with complete historical COPY and current READ +verification, retained revocation and independent receipt confirmation. +Source provisioning and native ACK/root/feed replica reception still use Python. + +In the synthetic same-host workload, 48 selected Memories offered at 2/second +complete 39 to 48 chains; each approved node completes 24. This measures use of +authorized capacity, not maximum throughput. Cold replica recovery remains +about 17.9 seconds and the existing 64-work allowance remains finite. Independent +hosts, WAN capacity and automatic universal repair are unverified. See +`docs/CAPACITY_HANDOFF.md` for the workload and explicit limits. + +Native TypeScript senders can prepare and retain an existing outgoing message +in the recipient-authorized mailbox through Agent `connect`. The sender keeps +the original ciphertext and persists its exact disclosure, attempt and upload +request before HTTP. A lost reply, restart or switch to Python reuses those same +bytes. The source returns signed custody that the client verifies and retains; +this does not claim the recipient saved the message or acknowledged it. + +An optional `ack_request_id` selects an already prepared independent receipt +permission. Complete signed controls and statuses travel with the message, +including bounded compressed transport for the larger draft. No new permission +or storage allocation is inferred. Native source provisioning and ACK/root/feed replica inbox +recovery still use Python. See `clients/typescript/network/README.md` and +`docs/OPEN_NETWORK_CONTACT.md` for the existing invitations. + +Native TypeScript recipients can issue an original mailbox destination from +an explicitly approved contact and a registered mailbox. The receiver signs with +its own existing identity and exports bounded authorization pages for sender +admission. Python and native clients reopen the same persisted signed originals; +conflicting selections and stale revisions fail. Both clients enforce retained +parent/resource revocations before cached or fresh authorization, even after the +status expires. Authorization creates no new storage or receipt-return grants. +See `clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md`. Native TypeScript recipients can retain an independently authorized receipt return for a cold-mailbox message. Ordinary Agent `receive` then returns the @@ -18,7 +58,7 @@ inbox. Retained revocations survive restart. Python and TypeScript share receive configuration, status history and saved inbox evidence; either can resume an interrupted import. The native path performs its own HTTP and cryptography. Saved receipts remain available through independently authorized return requests. -Native provisioning and replica inbox recovery remain unfinished. See +Native provisioning and ACK/root/feed replica inbox recovery remain unfinished. See `clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md`. Native TypeScript recipients can return an actually saved message or memory diff --git a/plugins/memory-vault-client/.codex-plugin/plugin.json b/plugins/memory-vault-client/.codex-plugin/plugin.json index fd07619..8550a68 100644 --- a/plugins/memory-vault-client/.codex-plugin/plugin.json +++ b/plugins/memory-vault-client/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "memory-vault-client", - "version": "0.28.0-alpha.0.37", + "version": "0.28.0-alpha.0.40", "description": "Full authorized client for the open memory protocol: MCP, visible-turn capture, queued signed sync, host adapters and recovery.", "author": { "name": "Memory Vault contributors" diff --git a/plugins/memory-vault-client/scripts/launcher.py b/plugins/memory-vault-client/scripts/launcher.py index 90ef8fa..1277095 100644 --- a/plugins/memory-vault-client/scripts/launcher.py +++ b/plugins/memory-vault-client/scripts/launcher.py @@ -98,6 +98,9 @@ } REQUIRED_MODULES = ALLOWED_MODULES MAX_MODULE_BYTES = 1024 * 1024 +# The closed 153-module inventory exceeds 16 KiB with ordinary indentation. +# Bound metadata independently; module names, file set and every hash stay strict. +MAX_INVENTORY_BYTES = 32 * 1024 def _regular(path: Path) -> bool: @@ -140,7 +143,7 @@ def main() -> int: try: if not _regular(Path(__file__).absolute()): raise ValueError("runtime_not_built") - inventory = json.loads(_read(manifest, 16 * 1024).decode("utf-8"), object_pairs_hook=_unique) + inventory = json.loads(_read(manifest, MAX_INVENTORY_BYTES).decode("utf-8"), object_pairs_hook=_unique) if not isinstance(inventory, dict) or set(inventory) != {"schema_version", "modules"} or inventory["schema_version"] != "memory-vault-client-runtime/v1": raise ValueError("invalid_runtime_inventory") modules = inventory["modules"] diff --git a/scripts/benchmark_native_runtime.py b/scripts/benchmark_native_runtime.py new file mode 100644 index 0000000..90c6889 --- /dev/null +++ b/scripts/benchmark_native_runtime.py @@ -0,0 +1,92 @@ +"""Two persistent native Agents for the finite synthetic capacity harness only.""" +import json +import os +from pathlib import Path +import select +import shutil +import subprocess +import tempfile +import threading + +DRIVER = r''' +import child from 'node:child_process'; +import {syncBuiltinESMExports} from 'node:module'; +import {createInterface} from 'node:readline'; +let subprocessCalls=0; +const deny=()=>{subprocessCalls++;throw Error('native Agent must not delegate to a subprocess');}; +for(const name of ['spawn','spawnSync','exec','execSync','execFile','execFileSync','fork'])child[name]=deny; +syncBuiltinESMExports(); +const {Agent}=await import('./agent.ts'); +const {OpenDeliveryClient}=await import('./open-delivery-client.ts'); +let agent; +for await(const line of createInterface({input:process.stdin,crlfDelay:Infinity})){ + if(Buffer.byteLength(line)>1048576)throw Error('finite synthetic input limit'); + let result; + try{ + const request=JSON.parse(line); + if(!agent){agent=new Agent(request.client_config,request.network_config);result={ready:true};} + else if(request.capacity_metrics){result={cpu:process.cpuUsage(),rss:process.memoryUsage().rss,subprocessCalls, + distribution:typeof OpenDeliveryClient.prototype.freezeDispatch==='function'};} + else result=await agent.handle(request); + }catch(error){result={synthetic_driver_error:error.code??error.name};} + process.stdout.write(JSON.stringify(result)+'\n'); +} +''' + + +class NativeAgent: + def __init__(self,node,directory,agent): + self.lock=threading.Lock() + self.process=subprocess.Popen([node,'--experimental-strip-types',str(directory/'driver.mjs')], + cwd=directory,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + self.handle(dict(client_config=str(agent.client_config),network_config=str(agent.network_config))) + + def handle(self,request): + encoded=json.dumps(request,separators=(',',':')).encode()+b'\n' + if len(encoded)>1048576:raise RuntimeError('finite native input limit') + with self.lock: + self.process.stdin.write(encoded);self.process.stdin.flush() + if not select.select([self.process.stdout],[],[],60)[0]:raise TimeoutError('native synthetic reply deadline') + reply=self.process.stdout.readline(1048577) + if not reply or len(reply)>1048576:raise RuntimeError('native synthetic process/reply failure') + result=json.loads(reply) + if 'synthetic_driver_error' in result:raise RuntimeError(result['synthetic_driver_error']) + return result + + def close(self): + if self.process.poll() is None: + self.process.terminate() + try:self.process.wait(2) + except subprocess.TimeoutExpired:self.process.kill();self.process.wait(2) + for pipe in (self.process.stdin,self.process.stdout,self.process.stderr):pipe.close() + + +class NativeRuntime: + def __init__(self,fixture,root): + node=os.environ.get('MEMORY_VAULT_NODE') or shutil.which('node') + if not node:raise RuntimeError('existing Node >=22.19 required') + selected=os.environ.get('MEMORY_VAULT_JOSE_MODULE') + package=Path(selected).expanduser().resolve().parents[2] if selected else root/'clients/typescript/network/node_modules/jose' + metadata=json.loads((package/'package.json').read_text()) + if metadata.get('name')!='jose' or metadata.get('version')!='6.2.10':raise RuntimeError('locked existing jose 6.2.10 required') + self.temporary=tempfile.TemporaryDirectory(prefix='synthetic-native-capacity-') + directory=Path(self.temporary.name) + source=root/'clients/typescript/network' + for path in [*source.glob('*.ts'),source/'package.json']:shutil.copyfile(path,directory/path.name) + shutil.copyfile(root/'memory_vault_open_capacity_schema.json',directory/'memory_vault_open_capacity_schema.json') + (directory/'node_modules').mkdir();(directory/'node_modules/jose').symlink_to(package,target_is_directory=True) + (directory/'driver.mjs').write_text(DRIVER) + self.agents=[] + try: + for agent in (fixture.a,fixture.b): + native=NativeAgent(node,directory,agent);self.agents.append(native);agent.handle=native.handle + except BaseException:self.close();raise + + def usage(self): + result=[agent.handle(dict(capacity_metrics=True)) for agent in self.agents] + assert all(value['subprocessCalls']==0 for value in result) + return result + + def close(self): + for agent in self.agents:agent.close() + self.temporary.cleanup() diff --git a/scripts/benchmark_open_delivery.py b/scripts/benchmark_open_delivery.py new file mode 100644 index 0000000..069c57c --- /dev/null +++ b/scripts/benchmark_open_delivery.py @@ -0,0 +1,254 @@ +"""Finite real encrypted delivery -> saved receipt -> local read/Memory recall. + +Two loopback node processes, one sender and one recipient. Each node receives an +independent explicit finite resource approval; no grant is copied to another node. +Offers continue at a fixed rate while the second node starts and is authorized. +At most 48 offers, 48 seconds and one pipeline worker; all failures/lag are retained. +""" +from __future__ import annotations + +import argparse +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +import os +from pathlib import Path +import platform +import resource +import signal +import sys +import tempfile +import threading +import time + +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT)) + + +def usage(): + r=resource.getrusage(resource.RUSAGE_SELF) + return dict(user_cpu_seconds=r.ru_utime,system_cpu_seconds=r.ru_stime, + max_rss_bytes=r.ru_maxrss if sys.platform=='darwin' else r.ru_maxrss*1024) + + +def serve(config_path,metrics_path): + import memory_vault_open_node as node + from memory_vault import canonical_bytes + from memory_vault_storage import atomic_write + lock=threading.RLock();counts=Counter();serial=0 + OriginalParticipant=node.OpenParticipant;OriginalServer=node.OpenHTTPServer + class Participant(OriginalParticipant): + def handle(self,request): + response=super().handle(request) + with lock: + action=request['payload']['action'];counts['rpc_'+action]+=1 + counts['request_bytes']+=len(canonical_bytes(request)) + counts['response_bytes']+=len(canonical_bytes(response)) + error=response.get('payload',{}).get('body',{}).get('error') + if error:counts['error_'+error['code']]+=1 + return response + def handle_blob(self,request,chunk): + result=super().handle_blob(request,chunk) + with lock: + counts['blob_'+request['payload']['action']]+=1 + counts['blob_input_bytes']+=len(chunk) + counts['blob_output_bytes']+=len(result[1]) + return result + class Server(OriginalServer): + def admitted(self,address): + accepted=super().admitted(address) + with lock: + counts['http_requests']+=1;counts['second_'+str(int(time.time()))]+=1 + counts['admitted' if accepted else 'rate_refused']+=1 + return accepted + def _busy(self,request): + with lock:counts['queue_refused']+=1 + return super()._busy(request) + node.OpenParticipant=Participant;node.OpenHTTPServer=Server + def snapshot(*_): + nonlocal serial + with lock: + serial+=1;value=dict(serial=serial,counts=dict(counts),usage=usage(),time=time.monotonic()) + atomic_write(metrics_path,json.dumps(value).encode(),replace=True) + signal.signal(signal.SIGUSR1,snapshot) + with node._publication_lock(config_path): + node._run_node(config_path) + + +def percentile(values,p): + ordered=sorted(values) + return ordered[min(len(ordered)-1,int((len(ordered)-1)*p))] if ordered else None + + +def main_run(args): + from memory_vault_client import ClientConfig + from memory_vault_open_delivery_client import OpenDeliveryClient + from tests.test_open_delivery_distribution import DeliveryDistributionFixture + from tests.test_open_node import HTTPNodes + class InstrumentedNodes(HTTPNodes): + def command(self,index): + return [sys.executable,str(Path(__file__).resolve()),'--serve-config',str(self.configs[index]), + '--metrics',str(self.root/f'node_{index}.metrics.json')] + def snapshot(self,index): + if index not in self.processes: + return dict(counts={},usage=dict(user_cpu_seconds=0,system_cpu_seconds=0,max_rss_bytes=0)) + path=self.root/f'node_{index}.metrics.json' + previous=json.loads(path.read_bytes())['serial'] if path.exists() else 0 + os.kill(self.processes[index].pid,signal.SIGUSR1) + deadline=time.monotonic()+2 + while time.monotonic()previous:return value + time.sleep(.01) + raise RuntimeError('bounded synthetic snapshot deadline') + with tempfile.TemporaryDirectory(prefix='synthetic-delivery-capacity-') as temporary: + fixture=DeliveryDistributionFixture(temporary,host_class=InstrumentedNodes,late_node=True) + finished=threading.Event() + native=None + try: + items=args.messages//2 + fixture.approve(0,items=items,bytes_limit=2*1024*1024) + if args.runtime=='native': + from benchmark_native_runtime import NativeRuntime + native=NativeRuntime(fixture,ROOT) + memory=[] + for i in range(args.messages): + text=f'Synthetic observation {i}: independent evidence must be checked before reuse.' + value=fixture.call(fixture.a,op='remember',request_id=f'req_capacity_memory_{i}', + kind='observation',text=text) + memory.append(dict(memory_id=value['memory_id'],text=text)) + snapshots=[fixture.host.snapshot(i) for i in range(2)] + native_before=native.usage() if native else None + before=usage();start=time.perf_counter();offers=[];join={};join_error=[] + def hard_deadline(): + if not finished.wait(args.messages/args.rate+30): + # Terminate only this disposable harness and its two children, + # including queued work. Never let socket failure multiply + # the ordinary per-operation deadline into an unbounded run. + if native:native.close() + fixture.close() + os.write(2,b'synthetic benchmark exceeded its finite drain allowance\n') + os._exit(124) + threading.Thread(target=hard_deadline,daemon=True).start() + def join_node(): + join['started_seconds']=time.perf_counter()-start + try: + fixture.host.start(1) + fixture.approve(1,items=items,bytes_limit=2*1024*1024) + join['authorized_seconds']=time.perf_counter()-start + except Exception as exc: + join_error.append(str(exc));join['failed_seconds']=time.perf_counter()-start + def pipeline(i,offered): + row=dict(index=i,offered_seconds=offered-start,status='unknown',stages=[], + offer_lag_ms=(time.perf_counter()-offered)*1000,node=None,storage_accepted=False, + saved_receipt=False,local_read=False,memory_recalled=False) + try: + sent=fixture.send(i,memory_ids=[memory[i]['memory_id']]) + row['stages'].append(dict(stage='send',ok=sent['ok'],code=sent.get('error',{}).get('code'))) + if not sent['ok']: + row['status']=sent['error']['code'];return row + result=sent['result'];row['message_id']=result['message_id'];row['storage_accepted']=result['storage_accepted'] + with fixture.a._network() as network: + with network.participant.state.db() as db: + session=db.execute('SELECT session FROM open_delivery_outbox WHERE request_id=?', + (f'req_synthetic_distribution_send_{i}',)).fetchone()[0] + key=json.loads(session)['node']['payload']['signing_key']['key_id'] + row['node']=next(j for j,n in enumerate(fixture.host.nodes) if n['payload']['signing_key']['key_id']==key) + received=fixture.b.handle({'op':'receive','limit':4}) + row['stages'].append(dict(stage='receive',ok=received['ok'],errors=received.get('result',{}).get('errors',[]))) + if not received['ok'] or received['result']['errors']: + row['status']=received.get('error',{}).get('code','receive_errors');return row + ack=fixture.send(i,memory_ids=[memory[i]['memory_id']]) + row['stages'].append(dict(stage='receipt',ok=ack['ok'],code=ack.get('error',{}).get('code'))) + row['saved_receipt']=bool(ack['ok'] and ack['result']['endpoint_validated']) + read=fixture.b.handle({'op':'receive','message_id':result['message_id'],'offset':0}) + row['local_read']=bool(read['ok'] and f'payload {i}' in read['result']['text']) + recall=fixture.b.handle({'op':'recall','memory_id':memory[i]['memory_id']}) + row['memory_recalled']=bool(recall['ok'] and any(x['text']==memory[i]['text'] for x in recall['result']['hits'])) + row['status']='ok' if all(row[n] for n in ('storage_accepted','saved_receipt','local_read','memory_recalled')) else 'incomplete_chain' + return row + except Exception as exc: + row['status']=type(exc).__name__;row['detail']=str(exc)[:256];return row + finally: + row['latency_ms']=(time.perf_counter()-offered)*1000 + row['completed_seconds']=time.perf_counter()-start + worker=None + with ThreadPoolExecutor(max_workers=1) as pool: + for i in range(args.messages): + offered=start+i/args.rate + time.sleep(max(0,offered-time.perf_counter())) + if worker is None and i>=args.join_after: + worker=threading.Thread(target=join_node,daemon=True);worker.start() + offers.append(pool.submit(pipeline,i,offered)) + offer_window=args.messages/args.rate + time.sleep(max(0,start+offer_window-time.perf_counter())) + rows=[future.result() for future in offers] + if worker:worker.join(8) + elapsed=time.perf_counter()-start;after=usage();per_node=[] + for i in range(2): + snapshot=fixture.host.snapshot(i) + counts={k:v-snapshots[i]['counts'].get(k,0) for k,v in snapshot['counts'].items()} + with __import__('sqlite3').connect(fixture.root/f'node_{i}'/'transport/network.sqlite3') as db: + stored=db.execute('SELECT count(*),coalesce(sum(length(envelope)),0) FROM open_delivery_messages').fetchone() + receipts=db.execute('SELECT count(*) FROM open_delivery_receipts').fetchone()[0] + leases=[dict(lease_id=r[0],max_items=r[1],max_bytes=r[2]) for r in db.execute( + "SELECT lease_id,max_items,max_bytes FROM open_contact_resource_leases WHERE purpose='delivery'")] + per_node.append(dict(node=i,counts=counts,peak_http_requests_per_second=max( + [v for k,v in counts.items() if k.startswith('second_')] or [0]), + stored_items=stored[0],stored_ciphertext_bytes=stored[1],saved_receipts=receipts,delivery_leases=leases, + user_cpu_seconds=snapshot['usage']['user_cpu_seconds']-snapshots[i]['usage']['user_cpu_seconds'], + system_cpu_seconds=snapshot['usage']['system_cpu_seconds']-snapshots[i]['usage']['system_cpu_seconds'], + max_rss_bytes=snapshot['usage']['max_rss_bytes'])) + ok=[r for r in rows if r['status']=='ok'] + with fixture.b._network() as network: + with network.participant.state.db() as db: + inbox=db.execute("SELECT count(*) FROM open_delivery_inbox WHERE phase='saved' AND receipt_sent=1").fetchone()[0] + with __import__('sqlite3').connect(ClientConfig.load(fixture.b.client_config).vault_path) as db: + vault_memories=db.execute('SELECT count(*) FROM memories').fetchone()[0] + native_after=native.usage() if native else None + return dict(mode='approved_resource_distribution' if (native_after[0]['distribution'] if native else hasattr(OpenDeliveryClient,'_freeze_dispatch')) else 'prior_first_approved_session', + agent_runtime=args.runtime,native_before=native_before,native_after=native_after, + offered_messages=args.messages,offered_messages_per_second=args.rate,offer_window_seconds=offer_window, + elapsed_including_drain=elapsed,successful_chains=len(ok),success_rate=len(ok)/len(rows), + successful_chains_per_second=len(ok)/elapsed,statuses=dict(Counter(r['status'] for r in rows)), + successful_p95_ms=percentile([r['latency_ms'] for r in ok],.95),successful_p99_ms=percentile([r['latency_ms'] for r in ok],.99), + all_p95_ms=percentile([r['latency_ms'] for r in rows],.95),all_p99_ms=percentile([r['latency_ms'] for r in rows],.99), + offer_lag_p99_ms=percentile([r['offer_lag_ms'] for r in rows],.99),successful_by_node=dict(Counter(str(r['node']) for r in ok)), + recipient_saved_inbox=inbox,recipient_vault_memories=vault_memories,per_node=per_node,join=join,join_errors=join_error, + setup_attempts=fixture.setup_attempts,requests=rows,driver=dict( + user_cpu_seconds=after['user_cpu_seconds']-before['user_cpu_seconds'], + system_cpu_seconds=after['system_cpu_seconds']-before['system_cpu_seconds'],max_rss_bytes=after['max_rss_bytes'])) + finally: + finished.set() + if native:native.close() + fixture.close() + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--messages',type=int,default=48) + parser.add_argument('--rate',type=int,default=2) + parser.add_argument('--join-after',type=int,default=12,help='offer index at which second node starts') + parser.add_argument('--output',type=Path) + parser.add_argument('--runtime',choices=('python','native'),default='python') + parser.add_argument('--serve-config',type=Path) + parser.add_argument('--metrics',type=Path) + args=parser.parse_args() + if args.serve_config: + if not args.metrics:parser.error('metrics path required for synthetic child') + serve(args.serve_config,args.metrics);return + if args.messages not in (8,48) or args.rate not in (1,2) or not 1<=args.join_after= args.join_after: + for pipe in pipes[1:]: + pipe.send("join"); pipe.recv() + joined = True + futures.append(pool.submit(operation, i, offered, start)) + # Include the complete sustained offer window, without removing + # scheduling gaps. Drain is reported separately and in total rate. + time.sleep(max(0, start+args.seconds-time.perf_counter())) + rows = [future.result() for future in futures] + elapsed = time.perf_counter()-start + driver_after = usage() + per_node = [] + for i, pipe in enumerate(pipes): + pipe.send("snapshot"); end = pipe.recv() + per_node.append(dict(node=i, counts={key: value-snapshots[i]["counts"].get(key,0) + for key,value in end["counts"].items()}, routing=end["routing"], + user_cpu_seconds=end["usage"]["user_cpu_seconds"]-snapshots[i]["usage"]["user_cpu_seconds"], + system_cpu_seconds=end["usage"]["system_cpu_seconds"]-snapshots[i]["usage"]["system_cpu_seconds"], + max_rss_bytes=end["usage"]["max_rss_bytes"])) + client.close() + return dict(mode="automatic_neighbors" if adaptive else "prior_fixed_entry_find", + offered_rate=args.rate, offer_window_seconds=args.seconds, elapsed_including_drain=elapsed, + join_after_seconds=args.join_after, caller_initial_introductions=1, + overall=summary(rows, elapsed), + completed_during_offer_window=sum(r["status"] == "ok" and + r["offered_seconds"]+r["latency_ms"]/1000 <= args.seconds for r in rows), + completed_during_last_eight_seconds=sum(r["status"] == "ok" and + args.seconds-8 <= r["offered_seconds"]+r["latency_ms"]/1000 <= args.seconds for r in rows), + steady_last_eight_seconds=summary( + [r for r in rows if r["offered_seconds"] >= args.seconds-8],8), + windows=[summary([r for r in rows if second <= r["offered_seconds"] < second+4],4) + for second in range(0,args.seconds,4)], per_node=per_node, + driver=dict(user_cpu_seconds=driver_after["user_cpu_seconds"]-driver_before["user_cpu_seconds"], + system_cpu_seconds=driver_after["system_cpu_seconds"]-driver_before["system_cpu_seconds"], + max_rss_bytes=driver_after["max_rss_bytes"]), requests=rows) + finally: + for pipe in pipes: + try: pipe.send("stop") + except (OSError, BrokenPipeError): pass + for process in processes: + process.join(3) + if process.is_alive(): process.terminate(); process.join(2) + if process.is_alive(): process.kill(); process.join(2) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--rate", type=int, default=80) + parser.add_argument("--seconds", type=int, default=24) + parser.add_argument("--join-after", type=int, default=6) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if not 1<=args.rate<=96 or args.seconds not in (16,20,24) or not 2<=args.join_after<=6: + parser.error("bounded load: rate <=96; seconds 16,20,24; join-after 2..6") + report=dict(synthetic=True, real_models=0, physical_hosts=1, cpu_count=multiprocessing.cpu_count(), + python=sys.version, platform=platform.platform(), source_hashes={name:hashlib.sha256((ROOT/name).read_bytes()).hexdigest() + for name in ("memory_vault_network.py","memory_vault_open_index.py","memory_vault_open_node.py","memory_vault_open_routing.py")}, + scope="signed public neighbor query, not full contact resolution or message I/O", result=run(args)) + args.output.parent.mkdir(parents=True,exist_ok=True) + args.output.write_text(json.dumps(report,indent=2)+"\n") + print(json.dumps({key:value for key,value in report["result"].items() if key!="requests"}),flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/build_client_plugin.py b/scripts/build_client_plugin.py index 184be1f..6018b6b 100644 --- a/scripts/build_client_plugin.py +++ b/scripts/build_client_plugin.py @@ -146,7 +146,11 @@ "clients/typescript/network/open-repair-mailbox-member.ts", "clients/typescript/network/open-repair-mailbox-feed.ts", "clients/typescript/network/open-repair-mailbox-read.ts", - "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", + "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", "clients/typescript/network/open-mailbox-destination.ts", "clients/typescript/network/open-mailbox-send.ts", + "clients/typescript/network/open-repair-mailbox-replica.ts", + "clients/typescript/network/open-mailbox-replica-client.ts", + "clients/typescript/network/open-repair-mailbox-replica-inbox.ts", + "clients/typescript/network/open-mailbox-replicas.ts", "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", diff --git a/scripts/build_release.py b/scripts/build_release.py index 18887cf..56802f1 100644 --- a/scripts/build_release.py +++ b/scripts/build_release.py @@ -56,6 +56,9 @@ "docs/CONTINUATION_TRIAL.md", "docs/NATIVE_OPEN_PROVIDER.md", "docs/DEPENDENCIES_NETWORK.md", + "docs/HTTP_CAPACITY.md", "docs/ROUTING_CAPACITY.md", "docs/DELIVERY_CAPACITY.md", "docs/CAPACITY_HANDOFF.md", + "docs/evidence/capacity-alpha40/before.json", "docs/evidence/capacity-alpha40/after.json", + "docs/evidence/capacity-alpha40/native-replica.json", "docs/evidence/capacity-alpha40/replica-profile.json", "docs/NETWORK_RECOVERY.md", "docs/NETWORK_NODE_TRANSFER.md", "docs/NETWORK_TYPESCRIPT.md", ) # Each executable network review fixture is selected deliberately. A matching @@ -70,6 +73,7 @@ "tests/test_open_contact_directory_maintenance.py", "tests/test_open_contact_typescript.py", "tests/test_open_contact_typescript_http.py", "tests/test_open_delivery_http.py", + "tests/test_open_mailbox_replica_typescript.py", "tests/test_open_read_concurrency.py", "tests/test_open_neighbors.py", "tests/test_open_delivery_distribution.py", "tests/test_open_delivery_distribution_typescript.py", "tests/test_capacity_replica_closure.py", "tests/test_open_provider_typescript_http.py", "tests/test_open_provider_typescript.py", "tests/test_open_provider_status.py", "tests/test_open_provider_status_typescript.py", @@ -123,7 +127,7 @@ "tests/test_open_repair_roundtrip.py", "tests/test_open_repair_receipt.py", "tests/test_open_repair_receipt_typescript.py", - "tests/test_open_repair_mailbox_authority_typescript.py", "tests/test_open_repair_mailbox_member_typescript.py", "tests/test_open_mailbox_client_typescript.py", "tests/test_open_mailbox_receipts_typescript.py", + "tests/test_open_repair_mailbox_authority_typescript.py", "tests/test_open_repair_mailbox_member_typescript.py", "tests/test_open_mailbox_client_typescript.py", "tests/test_open_mailbox_receipts_typescript.py", "tests/test_open_mailbox_destination_typescript.py", "tests/test_open_mailbox_send_typescript.py", "tests/open_repair_index_fixtures.py", "tests/test_open_repair_stage.py", "tests/test_open_repair_index.py", @@ -248,7 +252,11 @@ "clients/typescript/network/open-repair-mailbox-member.ts", "clients/typescript/network/open-repair-mailbox-feed.ts", "clients/typescript/network/open-repair-mailbox-read.ts", - "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", + "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", "clients/typescript/network/open-mailbox-destination.ts", "clients/typescript/network/open-mailbox-send.ts", + "clients/typescript/network/open-repair-mailbox-replica.ts", + "clients/typescript/network/open-mailbox-replica-client.ts", + "clients/typescript/network/open-repair-mailbox-replica-inbox.ts", + "clients/typescript/network/open-mailbox-replicas.ts", "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", @@ -326,6 +334,8 @@ def review_sources(material: list[Path], source_tree: ReleaseSource) -> list[Pat "scripts/build_client_plugin.py", "scripts/build_release.py", "scripts/release_source.py", "scripts/verify_client_package.py", "scripts/continuation_trial.py", + "scripts/benchmark_open_http.py", "scripts/benchmark_open_neighbors.py", + "scripts/benchmark_open_delivery.py", "scripts/benchmark_native_runtime.py", "docs/proposals/OPEN_DELIVERY_REPAIR_R3.md", "docs/OPEN_NETWORK_ARCHITECTURE.md", "docs/OPEN_NETWORK_LIFECYCLE.md", "docs/OPEN_NETWORK_ACCEPTANCE.md", "docs/OPEN_NETWORK_ROADMAP.md", diff --git a/scripts/release_source.py b/scripts/release_source.py index 1c8c0fa..bec6389 100644 --- a/scripts/release_source.py +++ b/scripts/release_source.py @@ -20,10 +20,10 @@ MAX_SOURCE_BYTES = 2 * 1024 * 1024 MAX_TREE_BYTES = 2 * 1024 * 1024 MAX_TREE_ENTRIES = 8192 -# The explicit native mailbox runtime/review inventory adds eleven files -# to the alpha35 inventory and now exceeds 624 selected source files. -# Keep a finite source-count ceiling independent of the unchanged byte caps. -MAX_SELECTED_FILES = 640 +# Alpha40's explicit native replica modules, tests, benchmark adapters and +# evidence bring the declared review inventory to 649 files. Keep a finite +# count ceiling independent of the unchanged per-file/tree/aggregate byte caps. +MAX_SELECTED_FILES = 664 MAX_SELECTED_BYTES = 32 * 1024 * 1024 GIT_TIMEOUT_SECONDS = 15 _SHA = re.compile(r"[0-9a-f]{40}") diff --git a/scripts/run_open_network_ci.py b/scripts/run_open_network_ci.py index 5ef5617..2a3f2aa 100644 --- a/scripts/run_open_network_ci.py +++ b/scripts/run_open_network_ci.py @@ -54,7 +54,8 @@ "tests.test_open_repair_index_prepare", "tests.test_open_repair_index_prepare_admin", "tests.test_open_repair_provision", "tests.test_open_repair_onboarding", "tests.test_open_repair_bind_recovery", "tests.test_open_repair_remote_setup", "tests.test_open_repair_remote_provision", - "tests.test_open_repair_status_observer", "tests.test_open_repair_mailbox_resources", "tests.test_open_repair_copy_resources", "tests.test_open_repair_copy_prepare", "tests.test_open_repair_copy_state", "tests.test_open_repair_copy_service", "tests.test_open_repair_copy_upload", "tests.test_open_repair_copy_empty", "tests.test_open_repair_copy_occupied", "tests.test_open_repair_mailbox_activation", "tests.test_open_repair_mailbox_range", "tests.test_open_repair_mailbox_authority_typescript", "tests.test_open_repair_mailbox_member_typescript", "tests.test_open_mailbox_client_typescript", "tests.test_open_mailbox_receipts_typescript", "tests.test_open_repair_mailbox_root", "tests.test_open_repair_mailbox_status", "tests.test_open_repair_mailbox_source", "tests.test_open_repair_mailbox_snapshot", "tests.test_open_repair_mailbox_copy", "tests.test_open_repair_mailbox_copy_authority", "tests.test_open_repair_mailbox_copy_upload", "tests.test_open_repair_mailbox_reservation", "tests.test_open_repair_mailbox_consent", "tests.test_open_mailbox_replica_receive", "tests.test_open_mailbox_receipt_jobs", "tests.test_open_ack_replica_send", "tests.test_open_ack_typescript_agent", "tests.test_open_ack_discovery_typescript", "tests.test_open_ack_discovery_agent", "tests.test_open_mailbox_copy_jobs", "tests.test_open_repair_mailbox_copy_admin", "tests.test_open_repair_mailbox_feed_copy", "tests.test_open_repair_mailbox_message_copy", + "tests.test_open_repair_status_observer", "tests.test_open_repair_mailbox_resources", "tests.test_open_repair_copy_resources", "tests.test_open_repair_copy_prepare", "tests.test_open_repair_copy_state", "tests.test_open_repair_copy_service", "tests.test_open_repair_copy_upload", "tests.test_open_repair_copy_empty", "tests.test_open_repair_copy_occupied", "tests.test_open_repair_mailbox_activation", "tests.test_open_repair_mailbox_range", "tests.test_open_repair_mailbox_authority_typescript", "tests.test_open_repair_mailbox_member_typescript", "tests.test_open_mailbox_client_typescript", "tests.test_open_mailbox_receipts_typescript", "tests.test_open_mailbox_destination_typescript", "tests.test_open_mailbox_send_typescript", "tests.test_open_repair_mailbox_root", "tests.test_open_repair_mailbox_status", "tests.test_open_repair_mailbox_source", "tests.test_open_repair_mailbox_snapshot", "tests.test_open_repair_mailbox_copy", "tests.test_open_repair_mailbox_copy_authority", "tests.test_open_repair_mailbox_copy_upload", "tests.test_open_repair_mailbox_reservation", "tests.test_open_repair_mailbox_consent", "tests.test_open_mailbox_replica_receive", "tests.test_open_mailbox_receipt_jobs", "tests.test_open_ack_replica_send", "tests.test_open_ack_typescript_agent", "tests.test_open_ack_discovery_typescript", "tests.test_open_ack_discovery_agent", "tests.test_open_mailbox_copy_jobs", "tests.test_open_repair_mailbox_copy_admin", "tests.test_open_repair_mailbox_feed_copy", "tests.test_open_repair_mailbox_message_copy", + "tests.test_open_mailbox_replica_typescript", "tests.test_open_read_concurrency", "tests.test_open_neighbors", "tests.test_open_delivery_distribution", "tests.test_open_delivery_distribution_typescript", "tests.test_capacity_replica_closure", "tests.test_open_provider_merge", "tests.test_continuation_trial", "tests.test_network_typescript_agent_network", "tests.test_network_packaging") EXPECTED = { @@ -244,7 +245,7 @@ def initialize(reports, mode, seed, partition_index=0, partition_count=1): "open-control.ts", "open-routing.ts", "open-state.ts", "client-config.ts", "transport-state.ts", "open-contact.ts", "open-contact-state.ts", "open-contact-client.ts", "open-provider.ts", "open-provider-client.ts", "open-blob.ts", "open-repair-wire.ts", "open-repair-history.ts", "open-repair-original.ts", "open-repair-resource.ts", - "open-repair-bootstrap.ts", "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-mailbox-client.ts", "open-mailbox-journal.ts", "open-mailbox-receivers.ts", "open-mailbox-receipts.ts", "open-repair-ack.ts", "open-capacity.ts", + "open-repair-bootstrap.ts", "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-mailbox-client.ts", "open-mailbox-journal.ts", "open-mailbox-receivers.ts", "open-mailbox-receipts.ts", "open-mailbox-destination.ts", "open-mailbox-send.ts", "open-repair-mailbox-replica.ts", "open-mailbox-replica-client.ts", "open-repair-mailbox-replica-inbox.ts", "open-mailbox-replicas.ts", "open-repair-ack.ts", "open-capacity.ts", "open-repair-probe.ts", "open-repair-proof.ts", "open-repair-client.ts", "open-repair-offer-client.ts", "open-repair-receipt.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", "open-repair-bound.ts", "open-repair-empty.ts", "open-repair-occupied.ts", "open-delivery.ts", "open-delivery-control.ts", "open-delivery-client.ts", diff --git a/tests/test_capacity_replica_closure.py b/tests/test_capacity_replica_closure.py new file mode 100644 index 0000000..5a63f38 --- /dev/null +++ b/tests/test_capacity_replica_closure.py @@ -0,0 +1,169 @@ +"""Native frozen send, full replica verification and independent saved receipt. + +Python serves the independently approved replica. Native sender and recipient +execute their Agent operations without subprocess delegation. Finite replica +work limits are measured, retained and never increased for the load test. +""" +import hashlib +import json +import os +from pathlib import Path +import time +import unittest +from unittest.mock import patch + +from tests import test_open_delivery_http as delivery_fixtures +from tests import test_open_repair_mailbox_feed_copy as feed_fixtures +from tests import test_open_mailbox_replica_receive as registered_fixtures +from tests import test_open_ack_typescript_agent as native_ack +from tests import test_network_typescript_agent_network as native_runtime + + +class CrossLanguageReplicaClosureTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + native_runtime.TypeScriptAgentNetworkTests.setUpClass.__func__(cls) + (cls.fixture/'driver.mjs').write_text(native_ack.DRIVER.replace('for(const name of', + "if(process.env.MEMORY_VAULT_FIXTURE_NOW)Date.now=()=>Number(process.env.MEMORY_VAULT_FIXTURE_NOW)*1000;\nfor(const name of",1)) + + def native(self, agent, requests, **options): + return native_ack.NativeAckAgentTests.native(self,agent,requests,**options) + + def run_fixture(self, inspect): + fixture=delivery_fixtures.MailboxStagingHTTPTests('test_ack_configuration_survives_mailbox_custody') + fixture.setUp();self.addCleanup(fixture.doCleanups);fixture.ack_cold_return=True + original=fixture.call + def call(agent,**request): + if (agent is fixture.a and request['op']=='send') or (getattr(self,'native_recipient',False) and agent is fixture.b): + response=self.native(agent,[request])['results'][0] + self.assertTrue(response['ok'],response) + return response['result'] + return original(agent,**request) + fixture.call=call;self.delivery_fixture=fixture + def committed(staging,slot,head): + h=feed_fixtures.FeedCopyFixture(self,fixture,staging,slot,head,message=True) + # Both runtimes use the same fixture clock. Real monotonic deadlines + # remain active; the protocol rejects future-dated probes strictly. + with patch('time.time',return_value=h.now),patch.dict(os.environ,{'MEMORY_VAULT_FIXTURE_NOW':str(h.now)}):inspect(h) + fixture.inspect_committed_mailbox=committed + fixture.test_ack_configuration_survives_mailbox_custody() + + def inspect_registration(self,h,registration,configured): + # Includes wrong-epoch refusal and bounded independent destination count. + registered_fixtures.RegisteredReplicaTests.inspect_registration(self,h,registration,configured) + self.original_envelope=h.snapshot.read(h.envelope_ref) + self.assertEqual(hashlib.sha256(self.original_envelope).hexdigest(),h.envelope_ref['raw_sha256']) + if getattr(self,'native_recipient',False): + from memory_vault_open_control import issue_node + changed=issue_node(h.identity,base_url=registration['base_url'],storage_epoch='synthetic_replaced_native_target', + roles=['directory','router'],revision=100,issued_at=h.now,expires_at=h.now+300) + wrong=self.native(h.recipient_agent,[dict(op='receive')],node_override=changed) + self.assertTrue(wrong['results'][0]['ok'],wrong) + self.assertTrue(any(e['code']=='open_invalid_mailbox_receiver' for e in wrong['results'][0]['result']['errors']),wrong) + self.assertFalse(any(c['kind']=='requestRepair' and c['base']==registration['base_url'] for c in wrong['calls'])) + + def cold_load(self,h,registration): + """Three additional empty Vaults reuse explicit existing permission. + + This measures a bounded known-message demand on the added replica. It + does not inject unknown future IDs into a pre-message frozen recipient, + or represent three independent owners/failure domains. + """ + from memory_vault_agent import Agent + from memory_vault import canonical_bytes + from memory_vault_trust import _write_new_private + client=json.loads(Path(h.recipient_agent.client_config).read_bytes()) + network=json.loads(Path(h.recipient_agent.network_config).read_bytes()) + self.cold_results=[] + for i in range(3): + folder=h.directory/('synthetic-native-cold-'+str(i));folder.mkdir(mode=0o700) + # A new private state binds to its own empty Vault. Copying another + # state's frozen Vault binding would correctly be refused. + state=folder/'transport';state.mkdir(mode=0o700) + vault=folder/'vault';vault.mkdir(mode=0o700) + copied_client=dict(client,vault_path=str(vault/'memory.sqlite3')) + client_path=folder/'client.json';config_path=folder/'open.json' + _write_new_private(client_path,canonical_bytes(copied_client)) + _write_new_private(config_path,canonical_bytes(dict(network,client_config_path=str(client_path),state_directory=str(state)))) + agent=Agent(client_path,config_path) + work_before=h.db.execute('SELECT requests FROM open_repair_copy_work WHERE resource_id=?',(h.rid,)).fetchone()[0] + started=time.perf_counter() + out=self.native(agent,[dict(op='connect',invitation=registration),dict(op='receive')]);elapsed=(time.perf_counter()-started)*1000 + self.assertTrue(out['results'][0]['ok'],out) + received=out['results'][1];self.assertTrue(received['ok'],received) + repair=[c for c in out['calls'] if c['kind']=='requestRepair'];self.assertTrue(repair) + self.assertTrue(all(c['base']==registration['base_url'] for c in repair)) + work_after=h.db.execute('SELECT requests FROM open_repair_copy_work WHERE resource_id=?',(h.rid,)).fetchone()[0] + self.assertLessEqual(work_after,64) + evidence=dict(offered=1,completed=len(received['result']['messages']),elapsed_ms=elapsed, + repair_http_calls=len(repair),copy_work_before=work_before,copy_work_after=work_after, + copied_ciphertext_bytes=h.envelope_ref['size'] if received['result']['messages'] else 0,subprocess_calls=0) + self.cold_results.append(evidence) + if not received['result']['messages']: + # Upload and complete reads use this resource's original finite + # work grant. Additional demand must fail closed, rather + # than reset the durable count or silently expand permission. + self.assertEqual(received['result']['messages'],[],received) + self.assertEqual(work_after,64) + self.assertTrue(received['result']['errors'],received) + evidence['errors']=[v['code'] for v in received['result']['errors']] + continue + self.assertEqual(len(received['result']['messages']),1,received) + message=received['result']['messages'][0];self.assertEqual(message['share']['records_added'],1) + offline=self.native(agent,[dict(op='receive',message_id=message['message_id']),dict(op='recall',query='Synthetic mailbox memory')],no_network=True) + self.assertTrue(all(row['ok'] for row in offline['results']),offline) + self.assertGreaterEqual(sum(v['completed'] for v in self.cold_results),1,self.cold_results) + self.assertTrue(any(not v['completed'] for v in self.cold_results),self.cold_results) + + def inspect_received_registration(self,h,registration,configured): + f=self.delivery_fixture + message=json.loads(self.original_envelope)['payload']['context']['message_id'] + with f.b._network() as network: + delivery=network._delivery();row=delivery._inbox(message) + self.assertEqual(bytes(row['envelope']),self.original_envelope) + self.assertEqual(row['phase'],'saved');self.assertIsNotNone(row['receipt']) + # Native reopens the imported Memory and saved message with zero network. + read=self.native(f.b,[dict(op='receive',message_id=message), + dict(op='recall',query='Synthetic mailbox memory')],no_network=True)['results'] + self.assertTrue(all(value['ok'] for value in read),read) + self.assertTrue(any(hit['text']=='Synthetic mailbox memory: consult current evidence before reuse.' + for hit in read[1]['result']['hits'])) + if getattr(self,'native_recipient',False):self.cold_load(h,registration) + # Fixture's existing explicit ACK authority carries the endpoint receipt + # independently; original send identity/envelope/session stay unchanged. + registered_fixtures.RegisteredReplicaTests.inspect_received_registration(self,h,registration,configured) + # Removing a selection cannot erase the authenticated denial. A direct + # retry after client restart must stop before any repair proof HTTP. + from memory_vault_open_transport import OpenHTTPTransport + with patch.object(OpenHTTPTransport,'request_repair',side_effect=AssertionError('retained revoked authority attempted repair HTTP')): + refused=f.b.handle(dict(op='connect',invitation=dict(registration,action='receive_replica'))) + self.assertFalse(refused['ok'],refused) + self.assertEqual(refused['error']['code'],'repair_authority_revoked') + if getattr(self,'native_recipient',False): + denied=self.native(f.b,[dict(op='connect',invitation=dict(registration,action='receive_replica'))],no_network=True)['results'][0] + self.assertFalse(denied['ok'],denied) + self.assertEqual(denied['error']['code'],'repair_authority_revoked') + self.closure_evidence=dict(message_id=message,envelope_sha256=h.envelope_ref['raw_sha256'], + copied_ciphertext_bytes=h.envelope_ref['size'],recipient_saved=True,native_offline_recall=True, + retained_revocation_code=refused['error']['code'],revoked_repair_http_calls=0) + self.closure_evidence['recipient_runtime']='native' if getattr(self,'native_recipient',False) else 'python' + if hasattr(self,'cold_results'):self.closure_evidence['cold_known_message_load']=self.cold_results + + def test_native_send_python_replica_receive_native_offline_reuse_and_independent_receipt(self): + self.with_independent_return=True + feed_fixtures.MailboxFeedCopyTests.http_roundtrip(self,message=True,commands=True,agent=True,registered=True) + f=self.delivery_fixture + confirmed=self.native(f.a,[f.ack_original_send_request],no_network=True)['results'][0] + self.assertTrue(confirmed['ok'],confirmed) + self.assertTrue(confirmed['result']['endpoint_validated']) + self.assertEqual(confirmed['result']['state'],'validated_saved') + self.assertEqual(confirmed['result']['message_id'],self.closure_evidence['message_id']) + print(json.dumps(dict(synthetic_replica_closure=self.closure_evidence, + original_send_native_confirmed=True,frozen_send_replay_network_calls=0))) + + def test_native_recipient_verifies_full_replica_proof_after_original_source_loss(self): + self.native_recipient=True + self.test_native_send_python_replica_receive_native_offline_reuse_and_independent_receipt() + + +if __name__=='__main__':unittest.main() diff --git a/tests/test_network_packaging.py b/tests/test_network_packaging.py index 6a19020..6ea5887 100644 --- a/tests/test_network_packaging.py +++ b/tests/test_network_packaging.py @@ -46,8 +46,8 @@ "open-repair-original.ts", "open-repair-resource.ts", "open-repair-bootstrap.ts", - "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-mailbox-client.ts", "open-mailbox-journal.ts", "open-mailbox-receivers.ts", "open-mailbox-receipts.ts", - "open-repair-ack.ts", + "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-mailbox-client.ts", "open-mailbox-journal.ts", "open-mailbox-receivers.ts", "open-mailbox-receipts.ts", "open-mailbox-destination.ts", "open-mailbox-send.ts", + "open-repair-mailbox-replica.ts", "open-mailbox-replica-client.ts", "open-repair-mailbox-replica-inbox.ts", "open-mailbox-replicas.ts", "open-repair-ack.ts", "open-repair-probe.ts", "open-repair-proof.ts", "open-repair-client.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", "open-repair-bound.ts", "open-repair-empty.ts", "open-repair-occupied.ts", "open-repair-offer-client.ts", "open-repair-receipt.ts")} TS_ENDPOINT_TESTS = {"tests/test_network_typescript_" + name + ".py" for name in @@ -102,7 +102,7 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): self.assertEqual(len(documents), len(set(documents))) self.assertEqual(len(review), len(set(review))) self.assertGreaterEqual(len(review), 39) - self.assertEqual(len(TS_NETWORK), 68) + self.assertEqual(len(TS_NETWORK), 74) self.assertTrue(RUNTIME_DATA <= set(documents)) self.assertTrue(TS_NETWORK <= set(documents)) self.assertTrue(TS_ENDPOINT_TESTS <= set(review)) @@ -122,7 +122,7 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): "tests/test_open_repair_status.py", "tests/test_open_repair_status_typescript.py", "tests/open_repair_ack_fixtures.py", "tests/test_open_repair_ack.py", "tests/test_open_repair_ack_typescript.py", "tests/test_open_repair_state.py", "tests/test_open_repair_mailbox_resources.py", "tests/test_open_repair_copy_resources.py", - "tests/test_open_repair_copy_prepare.py", "tests/test_open_repair_copy_state.py", "tests/test_open_repair_copy_service.py", "tests/test_open_repair_copy_upload.py", "tests/test_open_repair_copy_empty.py", "tests/test_open_repair_mailbox_activation.py", "tests/test_open_repair_mailbox_range.py", "tests/test_open_repair_mailbox_authority_typescript.py", "tests/test_open_repair_mailbox_member_typescript.py", "tests/test_open_mailbox_client_typescript.py", "tests/test_open_mailbox_receipts_typescript.py", "tests/test_open_repair_mailbox_root.py", "tests/test_open_repair_mailbox_status.py", "tests/test_open_repair_mailbox_source.py", "tests/test_open_repair_mailbox_snapshot.py", "tests/test_open_repair_mailbox_copy.py", "tests/test_open_repair_mailbox_copy_authority.py", "tests/test_open_repair_mailbox_copy_upload.py", "tests/test_open_repair_mailbox_reservation.py", "tests/test_open_repair_mailbox_consent.py", "tests/test_open_mailbox_replica_receive.py", "tests/test_open_mailbox_receipt_jobs.py", "tests/test_open_ack_replica_send.py", "tests/test_open_mailbox_copy_jobs.py", "tests/test_open_repair_mailbox_copy_admin.py", "tests/test_open_repair_mailbox_feed_copy.py", "tests/test_open_repair_mailbox_message_copy.py", + "tests/test_open_repair_copy_prepare.py", "tests/test_open_repair_copy_state.py", "tests/test_open_repair_copy_service.py", "tests/test_open_repair_copy_upload.py", "tests/test_open_repair_copy_empty.py", "tests/test_open_repair_mailbox_activation.py", "tests/test_open_repair_mailbox_range.py", "tests/test_open_repair_mailbox_authority_typescript.py", "tests/test_open_repair_mailbox_member_typescript.py", "tests/test_open_mailbox_client_typescript.py", "tests/test_open_mailbox_receipts_typescript.py", "tests/test_open_mailbox_destination_typescript.py", "tests/test_open_mailbox_send_typescript.py", "tests/test_open_repair_mailbox_root.py", "tests/test_open_repair_mailbox_status.py", "tests/test_open_repair_mailbox_source.py", "tests/test_open_repair_mailbox_snapshot.py", "tests/test_open_repair_mailbox_copy.py", "tests/test_open_repair_mailbox_copy_authority.py", "tests/test_open_repair_mailbox_copy_upload.py", "tests/test_open_repair_mailbox_reservation.py", "tests/test_open_repair_mailbox_consent.py", "tests/test_open_mailbox_replica_receive.py", "tests/test_open_mailbox_receipt_jobs.py", "tests/test_open_ack_replica_send.py", "tests/test_open_mailbox_copy_jobs.py", "tests/test_open_repair_mailbox_copy_admin.py", "tests/test_open_repair_mailbox_feed_copy.py", "tests/test_open_repair_mailbox_message_copy.py", "tests/test_open_repair_probe.py", "tests/test_open_repair_probe_typescript.py", "tests/test_open_repair_offer_client_typescript.py", "tests/test_open_repair_put_client_typescript.py", "tests/test_open_repair_receipt_typescript.py", @@ -188,8 +188,8 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): "clients/typescript/network/open-repair-wire.ts", "clients/typescript/network/open-repair-history.ts", "clients/typescript/network/open-repair-original.ts", "clients/typescript/network/open-repair-resource.ts", "clients/typescript/network/open-repair-bootstrap.ts", "clients/typescript/network/open-repair-status.ts", - "clients/typescript/network/open-repair-mailbox-range.ts", "clients/typescript/network/open-repair-mailbox-authority.ts", "clients/typescript/network/open-repair-mailbox-member.ts", "clients/typescript/network/open-repair-mailbox-feed.ts", "clients/typescript/network/open-repair-mailbox-read.ts", "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", - "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-capacity.ts", + "clients/typescript/network/open-repair-mailbox-range.ts", "clients/typescript/network/open-repair-mailbox-authority.ts", "clients/typescript/network/open-repair-mailbox-member.ts", "clients/typescript/network/open-repair-mailbox-feed.ts", "clients/typescript/network/open-repair-mailbox-read.ts", "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", "clients/typescript/network/open-mailbox-destination.ts", "clients/typescript/network/open-mailbox-send.ts", + "clients/typescript/network/open-repair-mailbox-replica.ts", "clients/typescript/network/open-mailbox-replica-client.ts", "clients/typescript/network/open-repair-mailbox-replica-inbox.ts", "clients/typescript/network/open-mailbox-replicas.ts", "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-capacity.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", "clients/typescript/network/open-repair-client.ts", "clients/typescript/network/open-ack-client.ts", "clients/typescript/network/open-ack-status.ts", "clients/typescript/network/open-ack-discovery.ts", "clients/typescript/network/open-repair-admin.ts", @@ -340,7 +340,10 @@ def test_isolated_source_runtime_launcher_and_strict_inventory(self): data = (ROOT / name).read_bytes() (runtime / name).write_bytes(data) hashes[name] = hashlib.sha256(data).hexdigest() - (runtime / "MANIFEST.json").write_text(json.dumps({"schema_version": "memory-vault-client-runtime/v1", "modules": hashes})) + manifest = runtime / "MANIFEST.json" + manifest.write_text(json.dumps({"schema_version": "memory-vault-client-runtime/v1", "modules": hashes}, indent=2)) + inventory_bytes = manifest.read_bytes() + self.assertGreater(len(inventory_bytes), 16 * 1024) config = root / "must-not-be-created.json" command = [sys.executable, "-I", "-S", "-B", str(launcher), "--config", str(config), "agent", "request"] def launch(): @@ -353,6 +356,11 @@ def launch(): self.assertTrue(answer["ok"], answer) self.assertFalse(answer["result"]["network_accessed"]) self.assertFalse(config.exists()) + # The full ordinary builder inventory must launch, while an + # oversized metadata document still fails before application code. + manifest.write_bytes(inventory_bytes + b" " * (32 * 1024 + 1 - len(inventory_bytes))) + self.assertEqual(launch().returncode, 1) + manifest.write_bytes(inventory_bytes) # The packaged client's new recovery command must be reachable # without reading a config or installing network/server extras. help_result = subprocess.run(command[:-2] + ["network-recovery", "--help"], diff --git a/tests/test_open_delivery_distribution.py b/tests/test_open_delivery_distribution.py new file mode 100644 index 0000000..ee22ba0 --- /dev/null +++ b/tests/test_open_delivery_distribution.py @@ -0,0 +1,214 @@ +"""Explicit independent grants distribute real encrypted delivery and receipts.""" +import asyncio +from concurrent.futures import ThreadPoolExecutor +import json +from pathlib import Path +import sqlite3 +import tempfile +import time +from types import SimpleNamespace +import unittest + +from memory_vault import canonical_bytes +from memory_vault_client import ClientConfig +from memory_vault_open_contact_client import CONNECT_SCHEMA, OpenContactClient +from memory_vault_open_delivery_client import OpenDeliveryClient +from memory_vault_open_routing import LookupBudget +from memory_vault_storage import atomic_write +from memory_vault_trust import TrustStore +from tests.test_open_agent import configured_agent +from tests.test_open_node import HTTPNodes + + +class DeliveryDistributionFixture: + """A and B retain their own keys/Vaults; R0/R1 get distinct owner grants.""" + def __init__(self, directory, *, host_class=HTTPNodes, late_node=False): + self.root = Path(directory).resolve() + self.host = host_class(self.root, 2) + for i in range(2): + self.host.stop(i) + config=json.loads(self.host.configs[i].read_bytes()) + config.update(contact_policy={"enabled":True},delivery_policy={"enabled":True}) + atomic_write(self.host.configs[i],canonical_bytes(config),replace=True) + if not late_node or i == 0: + self.host.start(i) + self.a,self.ai,*_=configured_agent(SimpleNamespace(root=self.root/"a",nodes=self.host.nodes[:1])) + self.b,self.bi,*_=configured_agent(SimpleNamespace(root=self.root/"b",nodes=self.host.nodes[:1])) + TrustStore(ClientConfig.load(self.b.client_config).trust_path).add(self.ai.public_descriptor()) + self.approvals=[] + self.setup_attempts=[] + + def close(self): + self.host.close() + + @staticmethod + def call(agent, **request): + value=agent.handle(request) + if not value["ok"]: raise AssertionError(value) + return value["result"] + + def contact(self, agent, action, **values): + request_id = values.pop("request_id") if action == "request" else None + return self.call(agent,op="connect",**({"request_id":request_id} if request_id else {}), + invitation={"schema_version":CONNECT_SCHEMA,"action":action,**values}) + + def approve(self, index, *, items=4, bytes_limit=2*1024*1024): + enabled=self.contact(self.b,"enable",node=self.host.nodes[index], + allocation_id=f"req_synthetic_distribution_knock_{index}",max_pending=2,lease_seconds=3600,revision=index+1) + request_id=f"req_synthetic_distribution_contact_{index}" + # A newly started R may not have been challenged by its first neighbor + # yet. These explicit fixture setup attempts are finite and retained. + for attempt in range(6): + value=self.a.handle({"op":"connect","request_id":request_id,"invitation":{ + "schema_version":CONNECT_SCHEMA,"action":"request","recipient_key_id":self.bi.key_id}}) + self.setup_attempts.append({"node":index,"attempt":attempt+1,"ok":value["ok"], + "code":value.get("error",{}).get("code"),"time":time.monotonic()}) + if value["ok"]: break + if not value["error"]["retryable"] or attempt==5: raise AssertionError(value) + time.sleep(.5) + pending=self.contact(self.b,"poll",lease_id=enabled["lease_id"]) + reference=next(row["request_ref"] for row in pending["requests"] if row["request_id"]==request_id) + self.contact(self.b,"decide",request_ref=reference,decision="approved",max_items=items,max_bytes=bytes_limit) + self.contact(self.a,"result",request_id=request_id) + self.approvals.append(enabled) + return enabled + + def send(self, i, *, memory_ids=None): + return self.a.handle({"op":"send","request_id":f"req_synthetic_distribution_send_{i}", + "recipients":[self.bi.key_id],"text":f"Synthetic distributed payload {i}","memory_ids":memory_ids or []}) + + def stored(self): + result=[] + for i in range(2): + with sqlite3.connect(self.root/f"node_{i}"/"transport/network.sqlite3") as db: + result.append(db.execute("SELECT count(*) FROM open_delivery_messages").fetchone()[0]) + return result + + +class DeliveryDistributionTests(unittest.TestCase): + def setUp(self): + temporary=tempfile.TemporaryDirectory(prefix="synthetic-delivery-distribution-") + self.addCleanup(temporary.cleanup) + self.fixture=DeliveryDistributionFixture(temporary.name) + self.addCleanup(self.fixture.close) + + def test_late_independent_approval_carries_new_messages_and_saved_memory_receipts(self): + f=self.fixture;f.approve(0,items=2) + memory=f.call(f.a,op="remember",request_id="req_synthetic_distribution_memory",kind="observation", + text="Synthetic shared observation retained through approved delivery.") + first=f.send(0,memory_ids=[memory["memory_id"]]);self.assertTrue(first["ok"],first) + f.approve(1,items=2) + messages=[first]+[f.send(i,memory_ids=[memory["memory_id"]]) for i in range(1,4)] + self.assertTrue(all(x["ok"] for x in messages),messages) + self.assertEqual(f.stored(),[2,2]) + received=[] + for _ in range(4): + value=f.call(f.b,op="receive",limit=1) + self.assertEqual(value["errors"],[]) + received+=value["messages"] + self.assertEqual(len(received),4) + self.assertEqual({x["state"] for x in received},{"validated_saved"}) + location={} + for index in range(2): + with sqlite3.connect(f.root/f"node_{index}"/"transport/network.sqlite3") as db: + location.update((row[0],index) for row in db.execute("SELECT message_id FROM open_delivery_messages")) + self.assertNotEqual(location[received[0]["message_id"]],location[received[1]["message_id"]]) + for i,result in enumerate(messages): + ack=f.send(i,memory_ids=[memory["memory_id"]]) + self.assertTrue(ack["ok"],ack) + self.assertTrue(ack["result"]["endpoint_validated"]) + read=f.call(f.b,op="receive",message_id=result["result"]["message_id"],offset=0) + self.assertIn(f"payload {i}",read["text"]) + recalled=f.call(f.b,op="recall",memory_id=memory["memory_id"]) + self.assertIn("Synthetic shared observation",recalled["hits"][0]["text"]) + self.assertEqual(f.stored(),[2,2]) + extra=f.send(4) + self.assertFalse(extra["ok"]) + self.assertEqual(extra["error"]["code"],"open_delivery_approval_capacity") + self.assertEqual(f.stored(),[2,2]) + + def test_atomic_new_send_placement_preserves_each_grant_and_frozen_retry(self): + f=self.fixture;f.approve(0,items=2);f.approve(1,items=2) + with f.a._network() as network: + delivery=OpenDeliveryClient(network.participant,network.encryption,network.client_config) + sessions=delivery._contact_sessions(outgoing=True,recipient=f.bi.key_id) + rows=[] + for i in range(4): + request_id,text=f"synthetic_freeze_{i}",f"Synthetic frozen {i}" + input_sha,_=delivery._send_input(request_id,[f.bi.key_id],text,[],None) + rows.append(delivery._prepare_outbox(request_id,f.bi.key_id,input_sha,text,[])) + with ThreadPoolExecutor(max_workers=4) as pool: + frozen=list(pool.map(lambda row:delivery._freeze_dispatch(row,sessions),rows)) + counts={} + for row in frozen: + session=json.loads(row["session"]) + scope=delivery._session_scope(session);counts[scope]=counts.get(scope,0)+1 + retry=delivery._freeze_dispatch(row,list(reversed(sessions))) + self.assertEqual(retry["session"],row["session"]) + self.assertEqual(retry["envelope"],row["envelope"]) + self.assertEqual(sorted(counts.values()),[2,2]) + + def test_revoked_exact_policy_does_not_store_on_another_node_without_approval(self): + f=self.fixture;enabled=f.approve(0,items=2) + with f.b._network() as network: + contact=OpenContactClient(network.participant,network.encryption) + session=contact._load("policy",enabled["lease_id"]) + payload={**session["policy"]["payload"],"revision":2,"status":"revoked"} + revoked={"payload":payload,"proof":f.bi.sign_message(payload)} + asyncio.run(contact.call(f.host.nodes[0],"policy.put",{"lease":session["lease"],"policy":revoked},LookupBudget())) + denied=f.send(0) + self.assertFalse(denied["ok"]) + self.assertEqual(denied["error"]["code"],"open_delivery_revoked") + self.assertEqual(f.stored(),[0,0]) + + def test_duplicate_session_aliases_do_not_multiply_one_signed_resource(self): + f=self.fixture;f.approve(0,items=1) + with f.a._network() as network: + delivery=OpenDeliveryClient(network.participant,network.encryption,network.client_config) + session=delivery._contact_sessions(outgoing=True,recipient=f.bi.key_id)[0] + rows=[] + for i in range(2): + request_id,text=f"synthetic_alias_{i}",f"Synthetic alias {i}" + sha,_=delivery._send_input(request_id,[f.bi.key_id],text,[],None) + rows.append(delivery._prepare_outbox(request_id,f.bi.key_id,sha,text,[])) + delivery._freeze_dispatch(rows[0],[session,dict(session)]) + from memory_vault import MemoryError + with self.assertRaises(MemoryError) as caught: + delivery._freeze_dispatch(rows[1],[session,dict(session)]) + self.assertEqual(caught.exception.code,"open_delivery_approval_capacity") + + def test_known_old_node_revocation_preserves_frozen_send_and_independent_new_grant(self): + from memory_vault import MemoryError + from memory_vault_open_control import issue_node + f=self.fixture;f.approve(0,items=2) + first=f.send(0);self.assertTrue(first["ok"],first) + f.approve(1,items=2) + raw=f.host.nodes[0]["payload"];now=int(time.time()) + revoked=issue_node(f.host.identities[0],base_url=raw["base_url"],storage_epoch=raw["storage_epoch"], + roles=raw["roles"],revision=2,status="revoked",issued_at=now,expires_at=now+600) + with f.a._network() as network: + with self.assertRaises(MemoryError):network.participant._accept(revoked) + independent=f.send(1);self.assertTrue(independent["ok"],independent) + self.assertEqual(f.stored(),[1,1]) + retry=f.send(0);self.assertTrue(retry["ok"],retry) + self.assertEqual(retry["result"]["message_id"],first["result"]["message_id"]) + self.assertFalse(retry["result"]["endpoint_validated"]) + self.assertEqual(f.stored(),[1,1]) + + def test_one_nodes_approved_grant_cannot_be_retargeted_to_another_node(self): + from memory_vault_open_delivery import sign_rpc, verify_response + f=self.fixture;f.approve(0,items=1) + self.assertTrue(f.send(0)["ok"]) + with f.a._network() as network: + with network.participant.state.db() as db: + row=db.execute("SELECT intent FROM open_delivery_outbox WHERE request_id='req_synthetic_distribution_send_0'").fetchone() + request=sign_rpc(f.ai,node=f.host.nodes[0],action="prepare",body={"intent":json.loads(row[0])}) + request["payload"]["node_key_id"]=f.host.nodes[1]["payload"]["signing_key"]["key_id"] + request["payload"]["storage_epoch"]=f.host.nodes[1]["payload"]["storage_epoch"] + request["proof"]=f.ai.sign_message(request["payload"]) + reply=network.participant.transport.request(f.host.nodes[1]["payload"]["base_url"],request,deadline=time.monotonic()+4) + body=verify_response(reply.response,request=request,node=f.host.nodes[1])["body"] + self.assertIn("error",body) + self.assertEqual(body["error"]["code"],"contact_wrong_node") + self.assertNotIn("challenge",body) + self.assertEqual(f.stored(),[1,0]) diff --git a/tests/test_open_delivery_distribution_typescript.py b/tests/test_open_delivery_distribution_typescript.py new file mode 100644 index 0000000..94b53a0 --- /dev/null +++ b/tests/test_open_delivery_distribution_typescript.py @@ -0,0 +1,115 @@ +"""Native Agent and Python share finite grants, placement and receive cursors.""" +import asyncio +import json +import sqlite3 +import tempfile +import unittest + +from memory_vault_open_contact_client import OpenContactClient +from memory_vault_open_routing import LookupBudget +from tests.test_open_delivery_distribution import DeliveryDistributionFixture +from tests import test_open_ack_typescript_agent as native_ack +from tests import test_network_typescript_agent_network as native_runtime + + +class NativeDistributionTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + native_runtime.TypeScriptAgentNetworkTests.setUpClass.__func__(cls) + (cls.fixture/'driver.mjs').write_text(native_ack.DRIVER) + + def setUp(self): + temporary=tempfile.TemporaryDirectory(prefix='synthetic-native-distribution-') + self.addCleanup(temporary.cleanup) + self.f=DeliveryDistributionFixture(temporary.name) + self.addCleanup(self.f.close) + + def native(self, agent, requests, **options): + return native_ack.NativeAckAgentTests.native(self,agent,requests,**options) + + def send(self, i, memory_ids=()): + return dict(op='send',request_id=f'req_synthetic_distribution_send_{i}', + recipients=[self.f.bi.key_id],text=f'Synthetic distributed payload {i}',memory_ids=list(memory_ids)) + + def test_python_native_alternate_sender_receiver_and_reopen_saved_receipts(self): + f=self.f;f.approve(0,items=2) + memory=f.call(f.a,op='remember',request_id='req_native_distributed_memory',kind='observation', + text='Synthetic cross-language resource reuse observation.')['memory_id'] + first=self.native(f.a,[self.send(0,[memory])])['results'][0];self.assertTrue(first['ok'],first) + f.approve(1,items=2) + sent=[first,f.send(1,memory_ids=[memory]),self.native(f.a,[self.send(2,[memory])])['results'][0],f.send(3,memory_ids=[memory])] + self.assertTrue(all(row['ok'] for row in sent),sent);self.assertEqual(f.stored(),[2,2]) + received=[] + for index in range(4): + value=(self.native(f.b,[dict(op='receive',limit=1)])['results'][0]['result'] + if index%2==0 else f.call(f.b,op='receive',limit=1)) + self.assertEqual(value['errors'],[]);self.assertEqual(len(value['messages']),1,value) + received+=value['messages'] + locations={} + for index in range(2): + with sqlite3.connect(f.root/f'node_{index}'/'transport/network.sqlite3') as db: + locations.update((row[0],index) for row in db.execute('SELECT message_id FROM open_delivery_messages')) + self.assertNotEqual(locations[received[0]['message_id']],locations[received[1]['message_id']]) + for index,row in enumerate(sent): + result=self.native(f.a,[self.send(index,[memory])])['results'][0] + self.assertTrue(result['ok'],result);self.assertTrue(result['result']['endpoint_validated']) + read=self.native(f.b,[dict(op='receive',message_id=row['result']['message_id']), + dict(op='recall',memory_id=memory)],no_network=True)['results'] + self.assertTrue(all(value['ok'] for value in read),read) + self.assertIn('Synthetic cross-language',read[1]['result']['hits'][0]['text']) + full=self.native(f.a,[self.send(4)])['results'][0] + self.assertFalse(full['ok']);self.assertEqual(full['error']['code'],'open_delivery_approval_capacity') + self.assertEqual(f.stored(),[2,2]) + + def test_native_obeys_live_policy_revocation_without_unapproved_retarget(self): + f=self.f;enabled=f.approve(0,items=2) + with f.b._network() as network: + contact=OpenContactClient(network.participant,network.encryption) + session=contact._load('policy',enabled['lease_id']) + payload={**session['policy']['payload'],'revision':2,'status':'revoked'} + revoked={'payload':payload,'proof':f.bi.sign_message(payload)} + asyncio.run(contact.call(f.host.nodes[0],'policy.put',{'lease':session['lease'],'policy':revoked},LookupBudget())) + refused=self.native(f.a,[self.send(0)])['results'][0] + self.assertFalse(refused['ok']);self.assertEqual(refused['error']['code'],'open_delivery_revoked') + self.assertEqual(f.stored(),[0,0]) + + def test_native_freeze_survives_source_failure_and_restarts_without_duplicate_store(self): + f=self.f;f.approve(0,items=2) + first=self.native(f.a,[self.send(0)])['results'][0];self.assertTrue(first['ok'],first) + with f.a._network() as network: + with network.participant.state.db() as db: + frozen=tuple(bytes(x) for x in db.execute('SELECT envelope,session,intent FROM open_delivery_outbox WHERE request_id=?',(self.send(0)['request_id'],)).fetchone()) + f.approve(1,items=2);f.host.stop(0) + retried=self.native(f.a,[self.send(0)])['results'][0] + self.assertTrue(retried['ok'],retried);self.assertFalse(retried['result']['endpoint_validated']) + self.assertEqual(retried['result']['message_id'],first['result']['message_id']) + f.host.start(0) + received=f.call(f.b,op='receive',limit=1);self.assertEqual(len(received['messages']),1,received) + confirmed=self.native(f.a,[self.send(0)])['results'][0] + self.assertTrue(confirmed['ok'],confirmed);self.assertTrue(confirmed['result']['endpoint_validated']) + with f.a._network() as network: + with network.participant.state.db() as db: + again=tuple(bytes(x) for x in db.execute('SELECT envelope,session,intent FROM open_delivery_outbox WHERE request_id=?',(self.send(0)['request_id'],)).fetchone()) + self.assertEqual(again,frozen);self.assertEqual(f.stored(),[1,0]) + + def test_native_new_send_uses_independent_grant_after_known_old_node_revocation(self): + import time + from memory_vault import MemoryError + from memory_vault_open_control import issue_node + f=self.f;f.approve(0,items=2) + first=self.native(f.a,[self.send(0)])['results'][0];self.assertTrue(first['ok'],first) + f.approve(1,items=2) + p=f.host.nodes[0]['payload'];now=int(time.time()) + revoked=issue_node(f.host.identities[0],base_url=p['base_url'],storage_epoch=p['storage_epoch'], + roles=p['roles'],revision=2,status='revoked',issued_at=now,expires_at=now+600) + with f.a._network() as network: + with self.assertRaises(MemoryError):network.participant._accept(revoked) + next_send=self.native(f.a,[self.send(1)])['results'][0] + self.assertTrue(next_send['ok'],next_send);self.assertEqual(f.stored(),[1,1]) + old=self.native(f.a,[self.send(0)])['results'][0] + self.assertTrue(old['ok'],old);self.assertFalse(old['result']['endpoint_validated']) + self.assertEqual(old['result']['message_id'],first['result']['message_id']) + self.assertEqual(f.stored(),[1,1]) + + +if __name__=='__main__':unittest.main() diff --git a/tests/test_open_mailbox_destination_typescript.py b/tests/test_open_mailbox_destination_typescript.py new file mode 100644 index 0000000..cf898f3 --- /dev/null +++ b/tests/test_open_mailbox_destination_typescript.py @@ -0,0 +1,106 @@ +"""Native recipient authority drives real sender mailbox admission over HTTP.""" +import base64 +import json +import sqlite3 +import subprocess +import unittest +import time +from pathlib import Path +from tests import test_open_delivery_http as fixture +from tests import test_open_ack_typescript_agent as runtime + +class NativeMailboxDestinationTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + runtime.NativeAckAgentTests.setUpClass.__func__(cls) + + def native(self,agent,requests): + run=subprocess.run([self.node,'--experimental-strip-types',str(self.fixture/'driver.mjs')],cwd=self.fixture, + input=json.dumps(dict(client_config=str(agent.client_config),network_config=str(agent.network_config),requests=requests,no_network=True)).encode(), + stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=45) + self.assertEqual(run.returncode,0,run.stderr.decode(errors='replace')[-2500:]) + result=json.loads(run.stdout);self.assertEqual(result['calls'],[]);self.assertEqual(result['subprocessCalls'],0) + state=Path(json.loads(Path(agent.network_config).read_bytes())['state_directory'])/'network.sqlite3' + db=sqlite3.connect(state) + try:self.assertEqual(db.execute('PRAGMA integrity_check').fetchall(),[('ok',)]) + finally:db.close() + return result['results'] + + def test_native_authorizes_real_admission_and_python_reopens_exact_pages(self): + self.journey(False) + + def test_native_authorization_rejects_conflicts_expiry_and_changed_receiver(self): + self.journey(True) + + def test_expired_retained_revocation_blocks_cached_and_new_authorization(self): + self.journey(False,True) + + def journey(self,negative,revoked=False): + h=fixture.MailboxStagingHTTPTests('test_sender_admits_message_over_http');h.setUp();self.addCleanup(h.doCleanups) + original=h.call;visited=[];pages=[];first=[] + def dispatch(agent,**request): + invitation=request.get('invitation',{}) + if agent is not h.b or invitation.get('schema_version')!='memory-vault-open-mailbox-connect/v1' or invitation.get('action') not in ('register','authorize'): + return original(agent,**request) + rows=self.native(agent,[request]);self.assertTrue(rows[0]['ok'],rows[0]);result=rows[0]['result'] + if invitation['action']=='register': + self.assertEqual(result,original(agent,**request)) + inspected=self.native(agent,[dict(op='connect',invitation=dict(schema_version=invitation['schema_version'],action='inspect',receiver_id=result['receiver_id']))])[0] + self.assertTrue(inspected['ok'],inspected);self.assertEqual(inspected['result'],original(agent,op='connect',invitation=dict(schema_version=invitation['schema_version'],action='inspect',receiver_id=result['receiver_id']))) + return result + # Each page reopens in a new native process, then Python reads the same exact originals. + self.assertEqual(result,original(agent,**request));pages.append(base64.b64decode(result['authorization_chunk'],validate=True)) + if 'cursor' not in invitation: + visited.append(True);first.append(dict(invitation)) + if negative: + cases=[(dict(invitation,expires_at=invitation['expires_at']-1),'repair_destination_conflict'), + (dict(invitation,status_revision=1),'repair_destination_revision_rollback'), + (dict(invitation,status_revision=4,expires_at=1),'repair_resource_expired'), + (dict(invitation,status_revision=4,status_until=1),'repair_resource_expired'), + (dict(invitation,cursor=dict(sha256='0'*64,offset=3072)),'open_invalid_mailbox_cursor'), + (dict(invitation,receiver_id='mailbox_'+'0'*64),'open_mailbox_receiver_missing')] + for changed,code in cases: + denied=self.native(agent,[dict(op='connect',invitation=changed)])[0] + self.assertFalse(denied['ok'],denied);self.assertEqual(denied['error']['code'],code,denied) + state=Path(json.loads(Path(agent.network_config).read_bytes())['state_directory'])/'network.sqlite3' + db=sqlite3.connect(state) + try: + self.assertEqual(db.execute('SELECT revision FROM open_mailbox_destinations ORDER BY revision').fetchall(),[(2,),(3,)]) + row=db.execute('SELECT body FROM open_mailbox_receivers WHERE receiver_id=?',(invitation['receiver_id'],)).fetchone() + config=json.loads(row[0]);config['expected_sender']=config['expected_target'] + db.execute('UPDATE open_mailbox_receivers SET body=? WHERE receiver_id=?',(json.dumps(config).encode(),invitation['receiver_id']));db.commit() + finally:db.close() + wrong=self.native(agent,[request])[0];self.assertFalse(wrong['ok']);self.assertEqual(wrong['error']['code'],'open_invalid_mailbox_receiver') + db=sqlite3.connect(state) + try: + db.execute('UPDATE open_mailbox_receivers SET body=? WHERE receiver_id=?',(row[0],invitation['receiver_id']));db.commit() + finally:db.close() + self.assertEqual(self.native(agent,[request])[0]['result'],result) + return result + h.call=dispatch;h.test_sender_admits_message_over_http();self.assertEqual(visited,[True]);self.assertGreater(len(pages),1) + authorization=json.loads(b''.join(pages));self.assertEqual(authorization,h.sender_authorization) + # The fixture performs actual admission, feed/body recovery and authority denial checks. + self.assertIn('destination_entry',authorization);self.assertIn('owner_status_entry',authorization) + if revoked: + from memory_vault_open_provider import issue_status + from memory_vault_open_repair_status import status_scope,authenticate_status_original + from memory_vault_open_repair_state import DEFAULT_POLICY + from memory_vault_open_repair_wire import RepairBudget + from memory_vault_open_repair_client import MailboxSetupJournal + from tests.test_open_repair_status import status_entry + slot=json.loads(authorization['slot_entries']['slot']['raw'])['payload'];root=slot['slot_key']['root_key'];now=int(time.time()) + scope=dict(scope_kind='mailbox_slot',scope_id=status_scope(root,'mailbox_slot',slot['slot_key'],DEFAULT_POLICY,RepairBudget(DEFAULT_POLICY))) + entry=status_entry(issue_status(h.bi,root=root,revision=99,entries=[dict(scope,minimum_document_revision=slot['revision'],status='revoked',operation_mask=127)],issued_at=now-10,valid_until=now-1)) + observed=authenticate_status_original(entry,expected_root=root,expected_signing_key=h.bi.public_descriptor(),at=now-10,allowed_scopes=[scope],policy=DEFAULT_POLICY,budget=RepairBudget(DEFAULT_POLICY)) + with h.b._network() as network: + with network.participant.state.db() as db: + jobs=db.execute('SELECT job_key FROM open_mailbox_setup_jobs').fetchall();self.assertEqual(len(jobs),1) + MailboxSetupJournal(db).observe(jobs[0][0],observed) + for revision in (3,100): + denied=self.native(h.b,[dict(op='connect',invitation=dict(first[0],status_revision=revision))])[0] + self.assertFalse(denied['ok']);self.assertEqual(denied['error']['code'],'repair_authority_revoked',denied) + reopened=h.b.handle(dict(op='connect',invitation=dict(first[0],status_revision=revision))) + self.assertFalse(reopened['ok']);self.assertEqual(reopened['error']['code'],'repair_authority_revoked',reopened) + + +if __name__=='__main__':unittest.main() diff --git a/tests/test_open_mailbox_replica_typescript.py b/tests/test_open_mailbox_replica_typescript.py new file mode 100644 index 0000000..ad176e4 --- /dev/null +++ b/tests/test_open_mailbox_replica_typescript.py @@ -0,0 +1,54 @@ +"""Native reconstruction of exact historical COPY, without Python delegation.""" +import json +import subprocess +import unittest +from tests import test_network_typescript_agent_network as runtime +from tests import test_open_repair_mailbox_message_copy as fixtures + +DRIVER=runtime.DRIVER.split('const {Agent}',1)[0]+r''' +import {RepairBudget,LocalRawResolver} from './open-repair-wire.ts'; +import {DEFAULT_REPAIR_CLIENT_POLICY} from './open-repair-client.ts'; +import {verifyMailboxMessageReplica} from './open-repair-mailbox-replica.ts'; +const parts=[];let size=0;for await(const p of process.stdin){size+=p.length;if(size>1048576)throw Error('finite fixture input');parts.push(p);} +const v=JSON.parse(Buffer.concat(parts)),decode=e=>({ref:e.ref,raw:Buffer.from(e.raw_base64url,'base64url')}),results=[]; +for(const change of v.contexts){try{const policy={...DEFAULT_REPAIR_CLIENT_POLICY,max_signature_checks:512},budget=new RepairBudget(policy),resolver=new LocalRawResolver(policy,budget); + for(const item of v.originals){const e=decode(item);resolver.put(e.ref.namespace,e.ref.key,e.raw);} + const result=verifyMailboxMessageReplica(decode(v.manifest),resolver,decode(v.custody),{...change,policy,budget}); + results.push({ok:true,envelope:result.source.message_scope.envelope_ref,originals:result.entries.size,read_until:result.authority.read_until}); +}catch(error){results.push({ok:false,code:error.code??error.name});}} +process.stdout.write(JSON.stringify({results,subprocessCalls})); +''' + +class NativeMessageReplicaProofTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + runtime.TypeScriptAgentNetworkTests.setUpClass.__func__(cls) + (cls.fixture/'driver.mjs').write_text(DRIVER) + + def run_fixture(self,inspect): + return fixtures.MailboxMessageCopyTests.run_fixture(self,inspect) + + def test_full_original_closure_and_independent_expected_bindings(self): + def inspect(h): + custody=h.commit();replica=h.store.restore_message(h.rid,**h.context()) + manifest_ref=json.loads(custody['raw'])['payload']['replica_manifest_ref'] + manifest=h.wrap(h.store.read_local_original(h.rid,manifest_ref)) + originals={json.dumps(e['ref'],sort_keys=True):e for values in replica['entries'].values() for e in values} + context=dict(expectedSlot=h.slot,expectedOwner=h.owner,expectedSender=h.sender,expectedSource=h.source.target, + sourceStorageEpoch=h.slot['writer_storage_epoch'],expectedMaintainer=h.source.target,expectedTarget=h.target, + targetStorageEpoch=h.node['payload']['storage_epoch'],expectedEnvelopeRef=h.envelope_ref,limitPolicy=h.limits) + changes=[context] + for name,value in (('sourceStorageEpoch','synthetic_replaced_source'),('targetStorageEpoch','synthetic_replaced_target'), + ('expectedSource',h.target),('expectedSender',h.owner),('expectedMaintainer',h.owner)): + changes.append(dict(context,**{name:value})) + bad=json.loads(json.dumps(context));bad['expectedEnvelopeRef']['raw_sha256']='f'*64;changes.append(bad) + result=subprocess.run([self.node,'--experimental-strip-types',str(self.fixture/'driver.mjs')],cwd=self.fixture, + input=json.dumps(dict(manifest=h.encode(manifest),custody=h.encode(custody),originals=[h.encode(e) for e in originals.values()],contexts=changes)).encode(), + stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=30) + self.assertEqual(result.returncode,0,result.stderr.decode(errors='replace')[-2000:]);out=json.loads(result.stdout) + self.assertEqual(out['subprocessCalls'],0);self.assertTrue(out['results'][0]['ok'],out) + self.assertEqual(out['results'][0]['envelope'],h.envelope_ref) + self.assertTrue(all(not row['ok'] for row in out['results'][1:]),out) + self.run_fixture(inspect) + +if __name__=='__main__':unittest.main() diff --git a/tests/test_open_mailbox_send_typescript.py b/tests/test_open_mailbox_send_typescript.py new file mode 100644 index 0000000..64f8292 --- /dev/null +++ b/tests/test_open_mailbox_send_typescript.py @@ -0,0 +1,78 @@ +"""Native sender retains exact drafts, submits actual HTTP and reopens in Python.""" +import json +import sqlite3 +import subprocess +import unittest +from pathlib import Path +from tests import test_open_delivery_http as fixtures +from tests import test_open_ack_typescript_agent as runtime + +class NativeMailboxSenderTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + runtime.NativeAckAgentTests.setUpClass.__func__(cls) + driver=runtime.DRIVER.replace("const {Agent}", "const {NetworkError}=await import('./io.ts');\nconst {Agent}",1) + driver=driver.replace("const result=await original.apply(this,args);", "const result=await original.apply(this,args);\n if(name==='requestRepair'&&input.lose_reply)throw new NetworkError('open_network_unavailable',true);") + (cls.fixture/'driver.mjs').write_text(driver) + + def native(self,agent,request,lose=False,no_network=False): + run=subprocess.run([self.node,'--experimental-strip-types',str(self.fixture/'driver.mjs')],cwd=self.fixture, + input=json.dumps(dict(client_config=str(agent.client_config),network_config=str(agent.network_config),requests=[request],lose_reply=lose,no_network=no_network)).encode(), + stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=45) + self.assertEqual(run.returncode,0,run.stderr.decode(errors='replace')[-2500:]) + result=json.loads(run.stdout);self.assertEqual(result['subprocessCalls'],0) + if no_network:self.assertEqual(result['calls'],[]) + return result['results'][0] + + def test_native_prepare_admit_restart_and_python_reuse(self): + self.journey(False) + + def test_native_sender_preserves_independent_receipt_permission(self): + self.journey(True) + + def test_native_compressed_receipt_draft_admitted_over_http(self): + self.journey(True,True) + + def journey(self,ack,remote=False): + name='test_cold_mailbox_returns_independent_receipt' if ack else 'test_sender_admits_message_over_http' + if remote:name='test_ack_configuration_admitted_over_http' + h=fixtures.MailboxStagingHTTPTests(name);h.setUp();self.addCleanup(h.doCleanups) + original=h.a.handle;actions=[];lost=[];snapshots={} + def state():return Path(json.loads(Path(h.a.network_config).read_bytes())['state_directory'])/'network.sqlite3' + def saved(message): + db=sqlite3.connect(state()) + try: + self.assertEqual(db.execute('PRAGMA integrity_check').fetchall(),[('ok',)]) + draft=db.execute('SELECT originals,envelope FROM open_mailbox_message_drafts WHERE message_id=?',(message,)).fetchone() + if db.execute("SELECT 1 FROM sqlite_master WHERE name='open_mailbox_message_requests'").fetchone(): + packet=db.execute('SELECT raw FROM open_mailbox_message_requests WHERE message_id=?',(message,)).fetchone() + else:packet=None + return draft,packet + finally:db.close() + def dispatch(request): + value=request.get('invitation',{}) + if value.get('schema_version')!='memory-vault-open-mailbox-connect/v1' or value.get('action') not in ('prepare','admit','retain'): + return original(request) + action=value['action'];lose=action=='retain' and not lost + result=self.native(h.a,request,lose,action=='prepare');actions.append(action) + if lose: + lost.append(True);self.assertFalse(result['ok'],result);snapshots[value['message_id']]=saved(value['message_id']);return result + self.assertTrue(result['ok'],result) + before=saved(value['message_id']) + if value['message_id'] in snapshots:self.assertEqual(before,snapshots[value['message_id']]) + # Reopen the same originals through Python and compare response and persisted bytes. + reopened=original(request);self.assertTrue(reopened['ok'],reopened);self.assertEqual(reopened['result'],result['result']);self.assertEqual(saved(value['message_id']),before) + if action=='prepare': + changed=dict(value,consent_until=value['consent_until']-1) + denied=self.native(h.a,dict(request,invitation=changed),no_network=True);self.assertFalse(denied['ok']);self.assertEqual(denied['error']['code'],'repair_message_conflict') + self.assertEqual(saved(value['message_id']),before) + if action=='admit': + changed=dict(value,enum_until=value['enum_until']+1) + denied=self.native(h.a,dict(request,invitation=changed),no_network=True);self.assertFalse(denied['ok']);self.assertEqual(denied['error']['code'],'repair_message_conflict') + return result + h.a.handle=dispatch + getattr(h,name)() + self.assertIn('prepare',actions) + if not ack or remote:self.assertIn('admit',actions);self.assertEqual(lost,[True]) + +if __name__=='__main__':unittest.main() diff --git a/tests/test_open_neighbors.py b/tests/test_open_neighbors.py new file mode 100644 index 0000000..bf82170 --- /dev/null +++ b/tests/test_open_neighbors.py @@ -0,0 +1,142 @@ +"""Bounded synthetic routing hints; exact signatures and checkpoints stay live.""" +import asyncio +import copy +import tempfile +import time +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from memory_vault import MemoryError, canonical_bytes +from memory_vault_open_control import coordinate, issue_node, sign_response, verify_request, verify_response +from memory_vault_open_node import OpenParticipant +from memory_vault_open_routing import LookupBudget +from memory_vault_trust import Identity + + +class NeighborTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.identities = [Identity.generate(Path(self.tmp.name).resolve() / str(i) / "identity.json") for i in range(4)] + now = int(time.time()) + self.nodes = [issue_node(identity, base_url=f"https://synthetic-{i}.invalid", + storage_epoch=f"synthetic_epoch_{i}", roles=["directory", "router"], + revision=1, issued_at=now, expires_at=now+600) for i, identity in enumerate(self.identities[:3])] + self.client = OpenParticipant(self.identities[3], Path(self.tmp.name).resolve()/"client", seeds=[self.nodes[0]]) + self.addCleanup(self.client.close) + self.client.transport.close() + self.offline = set() + self.bad_proofs = set() + self.calls = [] + self.reply_nodes = None + self.client.transport = SimpleNamespace(request=self.request, close=lambda: None) + self.target = coordinate(self.identities[3].key_id) + + def request(self, url, request, *, deadline): + index = next(i for i, node in enumerate(self.nodes) if node["payload"]["base_url"] == url) + self.calls.append((index, request["payload"]["action"])) + if index in self.offline: + raise OSError("synthetic offline") + checked = verify_request(request, node=self.nodes[index]) + if checked["action"] == "hello": + body = {"node": self.nodes[index]} + else: + body = {"nodes": self.reply_nodes if self.reply_nodes is not None else [self.nodes[(index+1) % 3]]} + now = int(time.time()) + response = sign_response(self.identities[index], request=request, node=self.nodes[index], body=body, + issued_at=now, expires_at=min(now+60, checked["expires_at"])) + if index in self.bad_proofs: + response = copy.deepcopy(response) + response["payload"]["expires_at"] -= 1 + return SimpleNamespace(response=response, observed_address=f"10.{index}.0.1", wire_bytes=len(canonical_bytes(response))) + + def step(self, **kwargs): + return asyncio.run(self.client.find_neighbors(self.target, **kwargs)) + + def test_response_after_second_boundary_retains_original_request_deadline(self): + request = self.client._request(self.nodes[0], "find", {"target": self.target, "view": "general"}) + now = request["payload"]["issued_at"] + 1 + # Only the synthetic responder clock moves; protocol verification stays live. + with patch("tests.test_open_neighbors.time", SimpleNamespace(time=lambda: now)): + reply = self.request(self.nodes[0]["payload"]["base_url"], request, deadline=float("inf")) + response = verify_response(reply.response, request=request, node=self.nodes[0], now=now) + self.assertEqual(response["issued_at"], now) + self.assertEqual(response["expires_at"], request["payload"]["expires_at"]) + + def test_new_neighbor_is_challenged_then_used_without_injecting_members(self): + first = self.step(view="directory") + self.assertEqual(first["state"], "introduced") + self.assertFalse(first["introductions_verified"]) + self.assertFalse(first["discovery_grants_access"]) + self.assertEqual(self.calls, [(0, "find"), (1, "hello")]) + self.assertTrue(self.client.table.has_verified(self.nodes[1])) + before = len(self.calls) + for _ in range(6): + self.step() + reads = [index for index, action in self.calls[before:] if action == "find"] + self.assertEqual(reads.count(0), 3) + self.assertEqual(reads.count(1), 3) + self.assertNotIn((2, "find"), self.calls) + self.client._neighbor_probe_after = 0 # advance the bounded probe gate + for _ in range(2): + self.step() + self.assertIn((2, "hello"), self.calls) + for _ in range(3): + self.step() + self.assertIn((2, "find"), self.calls) + + def test_bad_introduction_proof_never_enters_dispatch_pool(self): + self.bad_proofs.add(1) + result = self.step() + self.assertEqual(result["state"], "introduced") + self.assertTrue(result["errors"]) + self.assertFalse(self.client.table.has_verified(self.nodes[1])) + for _ in range(4): + self.step() + self.assertNotIn((1, "find"), self.calls) + + def test_offline_peer_fails_over_but_observed_revocation_is_not_usable(self): + self.step() + self.offline.add(0) + for _ in range(3): + self.assertEqual(self.step()["state"], "introduced") + now = int(time.time()) + revoked = issue_node(self.identities[1], base_url=self.nodes[1]["payload"]["base_url"], + storage_epoch="synthetic_epoch_1", roles=["directory", "router"], revision=2, + status="revoked", issued_at=now, expires_at=now+600) + with self.assertRaises(MemoryError): + self.client._accept(revoked) + self.assertEqual(self.step()["state"], "unreachable") + + def test_shared_budget_bounds_probe_and_invalid_target_sends_nothing(self): + result = self.step(budget=LookupBudget(maximum_requests=1)) + self.assertEqual(result["state"], "introduced") + self.assertEqual(result["metrics"]["requests"], 1) + self.assertEqual(self.calls, [(0, "find")]) + self.assertFalse(self.client.table.has_verified(self.nodes[1])) + before = len(self.calls) + with self.assertRaises(MemoryError): + asyncio.run(self.client.find_neighbors("not-a-digest")) + with self.assertRaises(MemoryError): + self.step(view="private") + self.assertEqual(len(self.calls), before) + + def test_failed_old_peer_does_not_starve_a_new_introduction(self): + self.step() + self.client.table.mark_failed(self.nodes[1]["payload"]["signing_key"]["key_id"]) + self.reply_nodes = [self.nodes[1], self.nodes[2]] + self.client._neighbor_probe_after = 0 + self.step() + self.assertIn((2, "hello"), self.calls) + self.assertTrue(self.client.table.has_verified(self.nodes[2])) + + def test_empty_reply_does_not_delay_the_first_actual_new_peer_probe(self): + self.reply_nodes=[] + self.step() + self.assertEqual(self.client._neighbor_probe_after,0) + self.reply_nodes=[self.nodes[1]] + self.step() + self.assertIn((1,"hello"),self.calls) + self.assertGreater(self.client._neighbor_probe_after,0) diff --git a/tests/test_open_read_concurrency.py b/tests/test_open_read_concurrency.py new file mode 100644 index 0000000..6581551 --- /dev/null +++ b/tests/test_open_read_concurrency.py @@ -0,0 +1,144 @@ +"""Read admission under real WAL writers and bounded real socket overload.""" +import socket +import sqlite3 +import tempfile +import threading +import time +import unittest +from pathlib import Path + +from memory_vault import MemoryError +from memory_vault_open_node import OpenHTTPServer, _TransportState +from memory_vault_trust import Identity +from tests import test_open_index as index_fixture + + +class OpenReadConcurrencyTests(unittest.TestCase): + def fixture(self): + fixture = index_fixture.OpenIndexTests() + fixture.setUp() + self.addCleanup(fixture.doCleanups) + fixture.db.execute("PRAGMA journal_mode=WAL") + fixture.db.execute("PRAGMA busy_timeout=10") + fixture.put() + writer = sqlite3.connect(fixture.db_path, timeout=.01) + self.addCleanup(writer.close) + return fixture, writer + + def test_reader_observes_committed_revocation_without_waiting_for_uncommitted_writer(self): + fixture, writer = self.fixture() + writer.execute("BEGIN IMMEDIATE") + writer.execute("UPDATE open_contact_floors SET status='revoked'") + self.assertEqual(fixture.get()["state"], "found") + writer.commit() + self.assertEqual(fixture.get(), {"state": "revoked"}) + + def test_expired_rows_are_invisible_without_mutation_and_reclaimed_on_next_write(self): + fixture, writer = self.fixture() + writer.execute("UPDATE open_contacts SET expires_at=?", (fixture.now,)) + writer.execute("UPDATE open_contact_floors SET retain_until=?", (fixture.now,)) + writer.execute("UPDATE open_index_replay SET retain_until=?", (fixture.now,)) + writer.commit() + changes = fixture.db.total_changes + self.assertEqual(fixture.get(), {"state": "not_found"}) + self.assertEqual(fixture.db.total_changes, changes) + self.assertEqual(fixture.db.execute("SELECT count(*) FROM open_contact_floors").fetchone()[0], 1) + fixture.put(request_id="synthetic_after_expiry") + self.assertEqual(fixture.get()["state"], "found") + self.assertEqual(fixture.db.execute("SELECT count(*) FROM open_index_replay").fetchone()[0], 1) + + def test_transport_open_remains_readable_under_writer_and_checks_binding_every_time(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + identity = Identity.generate(root/"synthetic_identity.json") + state = _TransportState(root/"transport", identity, "synthetic_epoch") + with state.db(): + pass + writer = sqlite3.connect(root/"transport"/"network.sqlite3") + try: + writer.execute("BEGIN IMMEDIATE") + writer.execute("INSERT INTO state VALUES('synthetic_writer','uncommitted')") + with state.db() as reader: + self.assertIsNone(reader.execute("SELECT value FROM state WHERE key='synthetic_writer'").fetchone()) + writer.rollback() + writer.execute("UPDATE state SET value='{}' WHERE key='configuration_binding'") + writer.commit() + with self.assertRaisesRegex(MemoryError, "network_state_configuration_mismatch"): + with state.db(): + pass + finally: + writer.close() + + +class OpenHTTPAdmissionTests(unittest.TestCase): + def test_occupied_listener_preserves_bind_error_without_starting_workers(self): + with socket.socket() as reservation: + reservation.bind(("127.0.0.1",0)) + reservation.listen() + with self.assertRaises(OSError): + OpenHTTPServer(reservation.getsockname(),None) + + def start(self, participant): + server = OpenHTTPServer(("127.0.0.1", 0), participant) + worker = threading.Thread(target=server.serve_forever, kwargs={"poll_interval":.01}, daemon=True) + worker.start() + def cleanup(): + server.shutdown();server.server_close();worker.join(1) + self.addCleanup(cleanup) + return server + + def connect(self, server, request=b"GET /open/v1/node HTTP/1.0\r\n\r\n"): + connection=socket.create_connection(server.server_address,timeout=4) + self.addCleanup(connection.close) + connection.sendall(request) + return connection + + def wait(self, check): + deadline=time.monotonic()+2 + while time.monotonic()