diff --git a/MAINTAINER.md b/MAINTAINER.md index 3cda5a8f..2aa33d39 100644 --- a/MAINTAINER.md +++ b/MAINTAINER.md @@ -24,7 +24,7 @@ release this kit. User-facing docs live in [README.md](README.md); this file is ### CLI shape (`bin/agentic-kit.mjs`) -- **Porcelain** (daily): `setup`, `status`, `sync`, `dashboard`, `admin`, `usage`, `models`, `system`, +- **Porcelain** (daily): `setup`, `status`, `sync`, `dashboard`, `admin`, `usage`, `telemetry`, `models`, `system`, `about`, `audit`, `heal`, `run`, `host`, `maintain`, `uninstall`. Bare `ak` → `status --hint`. (`dashboard`, `admin`, and `host` are also reachable under `ak x`.) - **Plumbing** (power users): `ak x admin | codex-context | daemon-gc | dashboard | harvest | host | mcp | @@ -96,7 +96,7 @@ docs/ **Published tarball** = the `files` whitelist in `package.json`: `bin/agentic-kit.mjs`, `src/`, `claude/`, `docs/DEJA-VU.md`, `docs/DASHBOARD.md`, `docs/HOST-SUPPORT.md`, `docs/HOOKS.md`, -`docs/INSTALLATION.md`, `docs/MODELS.md`, `docs/PROVIDERS.md`, `docs/SETUP.md`, +`docs/INSTALLATION.md`, `docs/MODELS.md`, `docs/TELEMETRY.md`, `docs/PROVIDERS.md`, `docs/SETUP.md`, `docs/MAINTENANCE.md`, `docs/TROUBLESHOOTING.md`, `docs/UPGRADING.md`, `docs/CODEX-STATUSLINE.md`, `docs/evidence/codex-context-0.153.4.md`, `docs/adr/0015-managed-codex-native-statusline.md`, @@ -105,6 +105,7 @@ docs/ `docs/adr/0043-managed-ruflo-browser-executor.md`, `docs/adr/0044-receipt-aware-maintenance-control-plane.md`, `docs/adr/0051-supported-peer-delegation-and-host-realignment.md`, +`docs/adr/0054-fleet-evidence-export.md`, `tests/live/aqe-external-provider-transport.test.mjs`, `tests/live/qe-court-participant-transport.test.mjs`, `tests/live/codex-context-contract.test.mjs`, and diff --git a/README.md b/README.md index ee6bc2b7..006a49a6 100644 --- a/README.md +++ b/README.md @@ -355,3 +355,10 @@ current release): > v4 (npm, cross-platform). The shell-based v3 kit is archived in > [docs/archive/](https://github.com/pacphi/agentic-kit/tree/main/docs/archive) — `ak setup` migrates an existing shell-kit > install automatically. A thin, reversible layer — not a fork. PRs welcome. + +## Fleet evidence + +Use `ak telemetry export --output ./snapshot.json` to produce a private, versioned JSON snapshot +of usage and retained maintenance evidence. `ak telemetry aggregate FILE...` combines the latest +snapshot per installation without counting repeated exports twice. No vendor dashboard or network +service is required. See [Fleet evidence export](docs/TELEMETRY.md) for schema, coverage and limits. diff --git a/bin/agentic-kit.mjs b/bin/agentic-kit.mjs index fdc1f602..c336b690 100755 --- a/bin/agentic-kit.mjs +++ b/bin/agentic-kit.mjs @@ -29,6 +29,7 @@ const PORCELAIN = Object.assign(Object.create(null), { dashboard: () => import('../src/commands/x/dashboard.mjs'), admin: () => import('../src/commands/x/admin.mjs'), usage: () => import('../src/commands/usage.mjs'), + telemetry: () => import('../src/commands/telemetry.mjs'), models: () => import('../src/commands/models.mjs'), system: () => import('../src/commands/system.mjs'), maintain: () => import('../src/commands/maintain.mjs'), @@ -66,6 +67,7 @@ Usage (ak = alias of agentic-kit): ak dashboard open the local web dashboard (localhost; auto-opens browser) [--port N] [--no-open] ak admin maintainer-only telemetry admin (localhost; GitHub/npm egress) [--port N] [--no-open] ak usage offline scorecard, prompt patterns, provider cache [status|score|prompts|refresh openrouter] + ak telemetry export, validate and aggregate fleet evidence [export|validate|aggregate|schema|metrics] ak models inspect/refresh model lifecycle evidence [status|refresh|diff|explain|plan] ak system what this stack occupies on your machine [--deep] [--json] ak maintain inventory, guidance, discovery, guarded one-action plans [inventory|guidance|plan|apply|...] [--json] @@ -172,6 +174,10 @@ async function main() { }); } catch (err) { if (!String(err?.code ?? '').startsWith('ERR_PARSE_ARGS_')) throw err; + if (cmd === 'telemetry') { + console.error('Telemetry failed: invalid command options.'); + return 2; + } fail(`ak ${cmd}: ${err.message}`); console.log(mod.help ?? `ak ${cmd} — flags: ${ Object.keys(mod.options ?? {}).map((o) => `--${o}`).join(' ') || '(none)'}`); @@ -185,7 +191,7 @@ async function main() { // zero behavior change: no dynamic import, no config read, nothing. // Refusals are warnings on stderr, never fatal — a bad external adapter // must never block a command that doesn't use it. - if (process.env.AK_EXPERIMENTAL_HOST_ADAPTERS === '1') { + if (cmd !== 'telemetry' && process.env.AK_EXPERIMENTAL_HOST_ADAPTERS === '1') { try { const { loadKitConfig } = await import('../src/lib/config.mjs'); const { bootstrapHostAdapters } = await import('../src/lib/adapters/admission.mjs'); @@ -210,7 +216,7 @@ async function main() { // setup and host own complete mutation/reporting flows. Running the generic // nudge after a declined trust preflight could write version-cache state and // violate their "before any changes" boundary. - if (!values.json && !values['dry-run'] && !['sync', 'usage', 'models', 'setup', 'host', 'audit', 'heal', 'maintain', 'ruflo-mcp', 'aqe-provider'].includes(cmd)) { + if (!values.json && !values['dry-run'] && !['sync', 'usage', 'telemetry', 'models', 'setup', 'host', 'audit', 'heal', 'maintain', 'ruflo-mcp', 'aqe-provider'].includes(cmd)) { try { const { driftReport } = await import('../src/lib/versions.mjs'); for (const r of await driftReport()) { diff --git a/docs/OBSERVABILITY.md b/docs/OBSERVABILITY.md index d39f994b..098ca8ad 100644 --- a/docs/OBSERVABILITY.md +++ b/docs/OBSERVABILITY.md @@ -484,3 +484,10 @@ harder, while also cautioning that stability must serve the user's task [eyes-have-it]: https://www.cs.umd.edu/~ben/papers/Shneiderman1996eyes.pdf [mental-map-review]: https://doi.org/10.1016/j.ijhcs.2013.08.004 [stable-graphs]: https://doi.org/10.1177/1473871620972339 + +## Fleet data export + +For vendor-neutral, offline export and aggregation independent of the dashboard, see +[Fleet evidence export](TELEMETRY.md). `ak telemetry` exports a strict subset of local usage +and retained maintenance evidence. Live SSE and transcript endpoints remain local dashboard +interfaces; they are not the fleet export contract. diff --git a/docs/TELEMETRY.md b/docs/TELEMETRY.md new file mode 100644 index 00000000..2e746d0b --- /dev/null +++ b/docs/TELEMETRY.md @@ -0,0 +1,171 @@ +# Fleet evidence export and aggregation + +`ak telemetry` exports a consistent, vendor-neutral subset of local usage and retained maintenance +evidence. It works without the dashboard, a service account, a collector, or a network connection. +The versioned contract is intended for your own ingestion pipeline, reporting system, or archive. +It does not export all local databases or raw transcripts. + +## Export and aggregate + +On each installation: + +```bash +ak telemetry export --days 30 --output ./machine-a-2026-09-20.json +ak telemetry validate ./machine-a-2026-09-20.json +``` + +Transfer the files using your organization's approved mechanism. On the aggregation machine: + +```bash +ak telemetry aggregate ./machine-a-2026-09-20.json ./machine-b-2026-09-20.json \ + --as-of 2026-09-20T23:59:59.000Z --output ./fleet-2026-09-20.json +``` + +Without `--output`, commands print JSON to stdout. With it, they create a new file with POSIX mode +0600 and refuse an existing destination, including a symlink. Windows permissions follow the +account/directory ACL; POSIX modes do not establish a Windows ACL guarantee. Parent directories must +already exist. Use unique filenames when scheduling exports; this command does not install a job, +retain an export history, send data, or retry uploads. `--output` avoids shell redirection's default +permissions and truncation behavior. + +All successful machine output is JSON. `validate` prints `{valid:true,schemaVersion,snapshotId}`. +Failure returns exit code 2 and a generic stderr message without repeating private paths or input. +A successfully exported snapshot can contain unavailable source sections; success means the +contract was produced, not that every source was healthy. Inspect its coverage fields. + +## Discover the contract + +```bash +ak telemetry schema # JSON Schema, draft 2020-12, for snapshot admission +ak telemetry metrics # metric units, evidence basis and aggregation rules +``` + +The executable schema is [schema.mjs](../src/lib/telemetry/schema.mjs). The reference admission and +reducer are [contract.mjs](../src/lib/telemetry/contract.mjs) and +[aggregate.mjs](../src/lib/telemetry/aggregate.mjs). Third-party importers must check the schema, +content digest and the semantic invariants enforced by `validateSnapshot`; JSON Schema alone does +not establish digest validity, unique record identities or cross-field evidence consistency. + +Version 1 requires its exact field set: unknown fields and versions are rejected. Breaking or +additive wire changes require an explicitly supported new contract version; internal Usage cache +schema changes do not change this contract automatically. Canonical timestamps are UTC with +milliseconds: `YYYY-MM-DDTHH:mm:ss.sssZ`. Files and combined aggregate input are bounded to 64 MiB; +an aggregate accepts at most 256 snapshots. A snapshot admits at most 100,000 sessions and 100,000 +receipts. Exceeding bounds fails instead of silently truncating evidence. + +### Snapshot envelope + +| Field | Meaning | +| --- | --- | +| `schemaVersion`, `kind` | `1`, `agentic-kit.telemetry.snapshot` | +| `snapshotId` | SHA-256 over canonical JSON of the envelope excluding `snapshotId` | +| `installationId` | Random UUID for one user/environment installation | +| `generatedAt`, `producerVersion` | Collection reference time and agentic-kit package version | +| `selection.days` | Integer 1–365; defaults to 30 | +| `selection.scope` | `whole-retained-sessions-selected-by-end` | +| `usage` | Session observations, source health, acquisition completeness and pricing date | +| `inventory` | Available retained scan counts and its original capture time, or nulls | +| `maintenance` | Retained receipt references and controlled states, or unavailable | + +Canonical JSON recursively sorts object keys lexically, preserves array order and uses JSON string +and number encoding without whitespace. Export sorts session/receipt arrays by their opaque IDs. +The digest detects changes, **not authenticity**. It is not a signature or compliance attestation. +A receiver must authenticate senders and map installations to authorized organizations separately. + +### Data and metric semantics + +Every session reference is qualified by execution host (`claude`, `codex`, `opencode`) before +pseudonymization. Hosts are not inference providers. Provider/model strings, repository labels and +prompt fingerprints are deliberately absent from this initial contract. + +| Fields | Unit and interpretation | +| --- | --- | +| `input`, `output`, `cacheRead`, `cacheWrite` | Token counts from existing normalized host parsers; reasoning output is not added again | +| `prompts`, `responses` | Normalized transcript activity counts; not task success or productivity | +| `exceptions`, `aborts` | Observed parser counters; not a complete count of tool failures or security events | +| `observedCostUsd` | Source-reported USD, not reconciled invoice spend | +| `estimatedCostUsd` | API-equivalent USD using local pricing; never presented as billing | +| `observedCostMessages`, `estimatedCostMessages`, `unpricedMessages` | Coverage for the distinct cost bases | +| `latencyBuckets` | Six non-cumulative response-latency counts with inclusive upper bounds 2, 5, 10, 30, 60 seconds, then overflow | +| `inventory.resources`, `inventory.placements` | Last retained inventory counts, not a fresh machine scan | +| `maintenance.receipts` | Retained action receipt references, status codes and last update timestamps | + +Missing values remain `null`; measured zero remains zero. When the source carries no usage-message +evidence (including a replay-excluded record), token counts are null. Unknown future receipt status +codes become `unknown`. Unreadable/integrity-failed journals remain `unknown-recovery-required`. +The absent or unverifiable inventory store is `unavailable`, not an inventory containing zero items. + +`usage.sourceHealth` reports each transcript source and the Codex ledger independently. These are +collection facts, not integration runtime health. `acquisitionComplete` reflects the existing +index's bounded-acquisition signal only; it does not prove every possible session was discovered. +An unavailable source can coexist with retained session observations. Pricing dates can differ +between installations and are retained per installation in the aggregate. + +## Correct fleet aggregation + +The reference reducer selects the newest **whole snapshot per installation** before computing any +metric. It accepts identical retries. Different content at the same installation timestamp is a +conflict; supply a corrected later snapshot, never arbitrarily pick an input ordering. Different +lookback selections are rejected, including among superseded inputs. Future snapshots relative to +`--as-of` are rejected. Fix the clocks or use an appropriate explicit as-of time. + +The aggregate's `kind` is `agentic-kit.telemetry.aggregate`, with schema version 1. It contains: + +- `selection`, `asOf`, `staleAfterSeconds` and `installationCount`; +- `installations`: selected snapshot identities, collection age/staleness, source coverage, + pricing date, original inventory capture timestamp/age and maintenance availability; +- `usage.metrics`: each metric's `{value, measured, missing}` across selected session observations; +- `usage.sessionObservations`, available/unavailable/incomplete installation counts; +- `usage.cacheReadShare` and its numerator, denominator and measured/missing session counts; +- `usage.latency`: bucket counts, second-based upper bounds and measured/missing session counts; +- `inventory`: resource and placement `{value, measured, missing}` across installations; +- `maintenance`: available/unavailable installations, retained receipt count and counts by status. + +All-missing metric contributions yield `value:null`, even when no session was retained. Empty source +arrays alone cannot distinguish no activity from unavailable evidence, so retain the coverage fields. +Counts of sessions and receipts are installation-scoped observations, not global unique tasks. + +Cache-read share is `sum(cacheRead) / sum(input + cacheRead + cacheWrite)`, using only sessions with +all three known counts; a zero denominator yields null. Percentages and machine percentiles are +never averaged. Latency buckets are summed by position, with null when no histogram is available. +Token and count sums preserve safe integers; arithmetic overflow fails. USD totals are rounded to +six decimal places. No fleet operator-hours metric is supplied because concurrent sessions across +machines cannot establish unique human engagement. + +Repeated 30-day exports do **not** provide disjoint 30-day increments. The source retains whole +sessions selected by their end time, so a selected session can include older tokens. Files can be +collected at different times; `asOf` does not reconstruct an atomic fleet instant. A newer snapshot +replaces all older sections, even when a source has become unavailable or a session disappeared. +Consumers needing historical trends should store these dated snapshots as separate as-of views. +Do not sum successive views or carry forward old rows as if they were current. + +`--stale-after` defaults to 86,400 seconds and accepts 1–31,536,000. Stale installations remain in +totals and are flagged; inventory has its own capture age. There is no expected-machine roster, so +an installation that never reports is absent rather than silently counted as healthy. + +## Identity, privacy and operations + +First explicit export creates `telemetry/identity.json` under agentic-kit's platform config +directory, alongside `kit.json`. It contains a UUID and a random 256-bit HMAC key. The directory +and file are owner-private on POSIX. Existing unsafe, linked or corrupt state is rejected; it is +never silently regenerated. Keep it across upgrades, back it up privately, and restore the original +if damaged. Do not include it in telemetry or clone it to other machines. Losing it creates a new +identity on the next export and requires the receiver to retire the old installation explicitly. + +Session/receipt references use HMAC-SHA256 with separate domains. These pseudonyms permit linking +exports from the same installation; they are not anonymous evidence. Copies of the same transcript +on separate installations cannot be deduplicated automatically. An ingestion system must account +for clones or choose an authoritative source when fleet-wide unique activity matters. + +The allowlist excludes raw prompts, responses, paths, filenames, hostnames, usernames, commands, +configuration, environment values, free-form errors and credentials. Export refreshes the existing +local Usage cache and reads retained Maintenance stores. It does not inspect arbitrary repository +files, export databases, start scans, execute host probes or bootstrap experimental adapters. + +Detailed authorization, policy decisions and verification remain in the existing maintenance +receipts; this telemetry summary is not their replacement. Use `ak maintain receipt --receipt ID +--export --json` locally when an authorized audit requires a detailed sanitized receipt. Enforce +organization mapping, transport encryption, access controls and retention at your receiver. No +network destination, scheduler, upload queue or vendor dashboard is bundled. + +Architecture and implementation evidence: [ADR-0054](adr/0054-fleet-evidence-export.md). diff --git a/docs/adr/0009-usage-scorecard-local-transcript-analytics.md b/docs/adr/0009-usage-scorecard-local-transcript-analytics.md index 602f3430..3a435f26 100644 --- a/docs/adr/0009-usage-scorecard-local-transcript-analytics.md +++ b/docs/adr/0009-usage-scorecard-local-transcript-analytics.md @@ -2,7 +2,9 @@ - **Status:** Implemented - **Date:** 2026-07-25 -- **Updated:** 2026-09-09 — reconciled against repository source and tests for issue #211 +- **Updated:** 2026-09-20 — ADR-0054 adds an explicit, offline, allowlisted fleet export boundary; + local analytics and dashboard collection semantics remain unchanged. +- **Earlier update:** 2026-09-09 — reconciled against repository source and tests for issue #211 - **Earlier update:** 2026-09-08 - **Pricing update:** Added GPT-6 Astra standard rates and per-entry verification dates; verified Claude 5.1 cache rates, removed Sonnet 5’s canceled September increase, and documented @@ -501,3 +503,10 @@ two evidence streams requires a new decision and fixtures from both sides. [Stop Overthinking (arXiv 2503.16419)](https://arxiv.org/pdf/2503.16419). - ruflo swarm surface grounded via `search_ruvnet`: `ruflo/plugins/ruflo-swarm/commands/swarm.md` (topology init, then native Task-tool fan-out). + +## Fleet export boundary + +[ADR-0054](0054-fleet-evidence-export.md) adds `ak telemetry` as an explicit export consumer. +Its versioned snapshots preserve source coverage and whole-session selection semantics, omit +transcript content, and aggregate by replacing each installation's prior snapshot. It neither +turns the dashboard into a fleet service nor makes telemetry collection continuous. diff --git a/docs/adr/0012-observability.md b/docs/adr/0012-observability.md index 7fc7e149..336b61d5 100644 --- a/docs/adr/0012-observability.md +++ b/docs/adr/0012-observability.md @@ -2,7 +2,9 @@ - **Status:** Implemented - **Date:** 2026-07-27 -- **Updated:** 2026-09-09 — reconciled against repository source and tests for issue #211 +- **Updated:** 2026-09-20 — ADR-0054 adds an explicit, offline, allowlisted fleet export boundary; + local analytics and dashboard collection semantics remain unchanged. +- **Earlier update:** 2026-09-09 — reconciled against repository source and tests for issue #211 - **Earlier update:** 2026-08-04 - **GA surface:** Canonical naming and retired vocabulary follow [ADR-0020](0020-ga-stable-surfaces.md). @@ -618,3 +620,10 @@ Acceptance does not imply automatic knowledge of every upstream store: [stable-graphs]: https://doi.org/10.1177/1473871620972339 [wcag]: https://www.w3.org/TR/WCAG22/ [websocket]: https://datatracker.ietf.org/doc/html/rfc6455 + +## Fleet export boundary + +[ADR-0054](0054-fleet-evidence-export.md) adds `ak telemetry` as an explicit export consumer. +Its versioned snapshots preserve source coverage and whole-session selection semantics, omit +transcript content, and aggregate by replacing each installation's prior snapshot. It neither +turns the dashboard into a fleet service nor makes telemetry collection continuous. diff --git a/docs/adr/0048-inventory-led-maintenance-resource-management.md b/docs/adr/0048-inventory-led-maintenance-resource-management.md index 82e39de7..11ddcab9 100644 --- a/docs/adr/0048-inventory-led-maintenance-resource-management.md +++ b/docs/adr/0048-inventory-led-maintenance-resource-management.md @@ -2,7 +2,9 @@ - **Status:** Accepted — implementation delivered 2026-09-05; Implemented withheld pending human-evaluation and cross-platform gates -- **Updated:** 2026-09-08 — project language badges use locally bundled SVGs with +- **Updated:** 2026-09-20 — ADR-0054 consumer hardens retained inventory reads with compressed, + uncompressed and decompression limits; existing acceptance gates remain outstanding. +- **Earlier update:** 2026-09-08 — project language badges use locally bundled SVGs with name tooltips and accessible labels; ADR-0050 subsequently replaces the three-icon limit with a wrapping row of all detected language icons. - **Earlier update:** 2026-09-08 — installed tools preserve measured executable locations diff --git a/docs/adr/0054-fleet-evidence-export.md b/docs/adr/0054-fleet-evidence-export.md new file mode 100644 index 00000000..9f656a2d --- /dev/null +++ b/docs/adr/0054-fleet-evidence-export.md @@ -0,0 +1,107 @@ +# ADR-0054 — Vendor-neutral fleet evidence export + +- **Status:** Implemented +- **Date:** 2026-09-20 +- **Updated:** 2026-09-20 — snapshot export, admission, aggregation, CLI and documentation delivered; + local regression/static/package gates passed on `feat/fleet-evidence-export`. + +## Context + +Operators need consistent data across agentic-kit installations without a vendor dashboard. +Existing CLI JSON and dashboard projections are local presentation contracts, not additive fleet +records. ADR-0009 and ADR-0012 are Implemented; ADR-0048 remains Accepted with its existing +human-evaluation and cross-platform gates. This decision does not resolve those gates. + +Usage selects whole retained sessions by end time. Tokens may precede the lookback boundary. +Repeated rolling-window totals must never be summed. Receipts and inventory have different +retention and freshness. Missing evidence is not a measured zero. + +## Decision + +Introduce a Telemetry bounded context under `src/lib/telemetry/`, with a public version-1 JSON +snapshot contract, strict admission, an explicit metric catalogue, and a reference reducer. +`ak telemetry export` collects local evidence independently of the dashboard; `validate`, +`schema`, and `aggregate` operate offline. Export itself makes no network request and never +executes integration probes or starts maintenance scans. + +Each installation gets a random UUID and a private HMAC key on its first explicit export. +The identity belongs to a user/environment installation, not physical hardware or a person. +Session and receipt references are domain-separated HMACs. No hostname, username, paths, +repository names, prompt text, model/provider free text, commands, errors, configuration, or +credentials enter the contract. No raw data dump is provided. The installation identity file +must not be copied to another installation; retain it across upgrades. Deleting it starts a new +identity. The aggregate cannot automatically recognize copied transcripts on different machines. + +Snapshots contain: + +- Producer version, collection timestamp, installation identity, content digest, selection scope. +- Usage: host-qualified session references, nullable token totals, prompt/response/error/abort + observations, separate observed and API-equivalent estimated USD, cost coverage counters, + mergeable response latency buckets, source health, acquisition coverage and pricing date. +- Inventory: last retained capture time and resource/placement counts; unavailable stays null. +- Maintenance: retained receipt references and controlled status codes. Integrity failures remain + recovery evidence; the export is not a signed compliance attestation or complete audit archive. + +These are observations, not proof of user identity, authorization, runtime success or billing. +Unknown source fields are discarded by allowlist projection. Unknown wire fields and schema +versions are rejected on admission. Content hashes detect alteration but do not authenticate +senders; a receiving system supplies organization membership, access controls and authentication. + +## Aggregation rules + +Select the newest whole snapshot per installation, never merge old and new partial sections. +Identical re-delivery is idempotent. Conflicting content at the same installation timestamp is +an error, independent of input order. Require identical schema and lookback selection. Sort +outputs deterministically. Missing sessions in a newer snapshot do not resurrect from an older +snapshot. Aggregation is an as-of view of supplied installations, not a longitudinal ledger. + +Sum session counters only after snapshot replacement. Emit measured contribution sums together +with measured/missing counts; all-missing measures remain null. Preserve observed and estimated +cost separately. Compute cache-read share from summed numerator and denominator, never averaged +percentages. Sum fixed non-cumulative latency bucket counts; do not average percentiles. Omit +operator engaged time because concurrent work across installations cannot establish unique human +hours. Report source health, incomplete acquisition, collection age and inventory capture age. +Without an external expected-installation roster, absent machines cannot be called healthy or +counted as reporting. Reject future snapshots relative to the aggregator's explicit as-of time. + +## Alternatives + +1. Scrape dashboard JSON: low effort, but ties collection to browser lifecycle and unstable private + projections, with no fleet identity, deduplication or export privacy boundary. +2. Ship a network daemon and OTLP transport now: adds credentials, retry queues and service lifecycle + before a stable domain contract exists. Defer transport; scheduled exports and external ingestion + can consume the same contract without a built-in remote destination. +3. Selected: self-contained snapshots and deterministic local aggregation. Simple recovery and + replay semantics; larger payloads and no automatic delivery or cross-machine transcript dedup. + +## Validation and limits + +Contract tests exercise allowlist privacy, null/zero, immutable input, digest validation, strict +versions/fields, bounded file reads, hostile paths/symlinks, duplicate/reordered delivery, +replacement, conflicting ties, selection mismatch, stale/future input, and histogram arithmetic. +Adapter tests use existing parsers with synthetic local stores. CLI tests isolate home/config roots. +Static, build and repository regression gates must pass before marking this ADR Implemented. +No vendor integration, autonomous upload, scheduler, machine enrollment, remote control, raw +transcript export, financial billing reconciliation or complete historical audit claim is included. + +## Implementation evidence + +Implemented by the Telemetry modules, CLI dispatch and shared retained-inventory read bounds. +Local verification on Node v26.4.0, macOS arm64: + +- 56 focused telemetry/source-bound tests passed; 99.81% telemetry line coverage, + 93.81% branch coverage and 97.47% function coverage. Contract and private-store modules + reached 100% line coverage; this is not a claim of exhaustive security verification. +- Full coverage-enforced suite: 4,230 passed, 6 existing skips, 0 failed, followed by all legacy + suites passing. Overall measured coverage: 92.16% lines, 81.44% branches, 91.57% functions. +- Type checking, lint, complexity gate, markdown lint and build/package checks passed. + Lint retains repository warnings (70 general and 3 complexity-gate warnings), with zero errors. +- Independent read-only review found counter rounding, timestamp-schema mismatch, CLI parser + error echo and unbounded retained inventory reads; regression tests and fixes address each. +- AQE execution returned internally inconsistent results without actionable diagnostics; + it supplies no passing assurance here. Direct Node test results are the validation authority. +- Synthetic 10-installation / 10,000-session aggregation: approximately 52 ms per measured run, + 3,546,741 input JSON bytes. This is one local workload, not a fleet service guarantee. + +Windows/Linux execution, remote transport and authenticated fleet enrollment are not claimed. +The original Maintenance acceptance gates remain unchanged. No commit or publication was performed. diff --git a/docs/adr/README.md b/docs/adr/README.md index fef5e994..ed55de98 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -370,3 +370,5 @@ summed and each delta books on its own day and model, oversized rollouts are rea streaming reader, and rollouts that still cannot be parsed are reported. Cache schema 23. - [ADR-0053 — Qualified host health and separate usage diagnostics](0053-host-setup-evidence-and-usage-diagnostics.md) — scoped local and connected health; usage acquisition remains separate. + +- [ADR-0054 — Vendor-neutral fleet evidence export](0054-fleet-evidence-export.md) — Implemented — versioned local snapshots, privacy allowlists and deterministic aggregation. diff --git a/docs/implementation/fleet-evidence-export-verification.json b/docs/implementation/fleet-evidence-export-verification.json new file mode 100644 index 00000000..cc61a25d --- /dev/null +++ b/docs/implementation/fleet-evidence-export-verification.json @@ -0,0 +1,102 @@ +{ + "schemaVersion": 1, + "kind": "local-development-verification", + "branch": "feat/fleet-evidence-export", + "baseCommit": "261af3269bf0d266f68d1889b75a1e53afd65115", + "recordedAt": "2026-09-20T14:34:08.194153+00:00", + "environment": { + "node": "v26.4.0", + "platform": "Darwin", + "architecture": "arm64" + }, + "authority": { + "user": "local implementation and new branch authorized; no commit, push or publication", + "rufloPolicyReceipt": "sha256:e21b09352327083b5d4b154805efb12db33692905cff727746fbf9da3115e070", + "policyMode": "legacy-default-allow" + }, + "checks": { + "focused": { + "passed": 56, + "failed": 0, + "lineCoverage": 99.81, + "branchCoverage": 93.81, + "functionCoverage": 97.47 + }, + "fullNodeSuite": { + "passed": 4230, + "skipped": 6, + "failed": 0, + "lineCoverage": 92.16, + "branchCoverage": 81.44, + "functionCoverage": 91.57 + }, + "legacySuites": "passed via npm test", + "typecheck": "passed", + "lint": { + "errors": 0, + "warnings": 70 + }, + "complexityGate": { + "errors": 0, + "warnings": 3 + }, + "markdown": "passed", + "build": "passed; 433 shipped files syntax-checked, package dry run 482 files", + "aqe": "inconsistent executor response; no passing claim" + }, + "benchmark": { + "sourceDigest": "4fc698f419a48b77418546abc4602f06f6ab63a7f76522e06ec87ca903619f5e", + "node": "v26.4.0", + "platform": "darwin", + "arch": "arm64", + "workload": { + "installations": 10, + "sessionsEach": 1000, + "totalSessions": 10000 + }, + "exportMs": 168.304625, + "aggregateMs": [ + 53.474541999999985, + 51.82387499999999, + 51.767375000000015, + 52.09941700000002, + 51.998875 + ], + "jsonBytes": 3546741, + "note": "Synthetic workload; one warmup and five samples. No pre-feature equivalent or performance improvement claim." + }, + "limits": [ + "macOS arm64 execution only; no Windows/Linux live verification", + "No network exporter, dashboard, scheduler or identity authentication", + "Installation-scoped observations; copied transcripts across installations not deduplicated", + "Whole retained sessions selected by end time; no disjoint time-window billing claim" + ], + "files": { + "MAINTAINER.md": "769d17dc19e91ad5ea3ef9a3402715b77b10ba01f3544f28c81d5630a99e5448", + "README.md": "56460bda8cfe48bd924de47ece046b0de976030baba6a8520c0793c806da58fa", + "bin/agentic-kit.mjs": "e60f10d711000e1bcb1b5172f706904009c6b3b6a8562616dd667632d439dcd0", + "docs/OBSERVABILITY.md": "b29abd41bb39910364be92c951d2825baf50c381567456de7787bd939557dc43", + "docs/TELEMETRY.md": "f5e181982b6b691f0325edc0040d1f4ef6ff37b3a68f4757f8e44c17dc8ad3d6", + "docs/adr/0009-usage-scorecard-local-transcript-analytics.md": "139256c2d3b5ba87010277f405327ebac31911950ceab386e926db48cdbe642d", + "docs/adr/0012-observability.md": "960387190f1198e3ccb76db9484b246626bf886166d4f88d8d41be16758b1f27", + "docs/adr/0048-inventory-led-maintenance-resource-management.md": "ffebb6898ff68508674a4b159cd25787800383782250f7dcd6e9a8ce5de5c0ca", + "docs/adr/0054-fleet-evidence-export.md": "f68dee4840a938664a3146da7f95def9e6bb07ebe0e6fc4d0283aaed1c2b8343", + "docs/adr/README.md": "1d7fca77ac663393af013576a31c459152eba8cd644ae7e2af33f796ac0551eb", + "docs/implementation/fleet-evidence-export.md": "3e83c72220ee7666897ef438dbe6a91fb843230b40d76e66f9a7f80eac55bc7a", + "package.json": "9f515b0ec7378b3f076042ef51a58da0e52ab3b79c9df4b57f37071012343fad", + "src/commands/telemetry.mjs": "05cb2e4e54dc9288c21fa72c1e68ee1b97571f82c92e37fa7a870f9b49c6a22c", + "src/lib/maintenance/management/service-store.mjs": "71db4936d1a5ec008a9234a018d39886e59546e8f986744afcf467441cb6677a", + "src/lib/paths.mjs": "9aad3185d23a7d6cb721af380c612e4094c24a47415d86f0ca4747bb9725229e", + "src/lib/telemetry/aggregate.mjs": "56360425fa14c3ae07e1d8d601b1dd24fdb8121b99fafa9af257e04feba71ec7", + "src/lib/telemetry/collect.mjs": "856a1da5a47080a2f5650dce16065cb2a82976a8a70d6013b1085e70166d5428", + "src/lib/telemetry/contract.mjs": "31641f485da1aa9501bac1ce1a591820c917f179a1ed2ec91a1b4988a05ce3da", + "src/lib/telemetry/projection.mjs": "9ca8330f9db49685d31aaecfb9f2654d829a3879b21125c9f6974775d77c24cb", + "src/lib/telemetry/schema.mjs": "f377f2dd08114a9642666915c491db3e20a5e15556b8b5a5a911e1bf3756c6ec", + "src/lib/telemetry/store.mjs": "f7805889ba1e656d2cca002cd601f9c239519b025073b7dc7680376ab062a91c", + "tests/kit/helpers/telemetry.mjs": "e4dfb06551ebbe4d619f1acd7448f94446597a7d767c4b36c62090624b364aad", + "tests/kit/telemetry-aggregate.test.mjs": "6943c996afe31241e87d9d90d7c226aff8dfe07864eeeb3df43f1428fffa6dbe", + "tests/kit/telemetry-cli.test.mjs": "b1c5da609bf81bc9b90aed1719e0d184a45849450d8a7fda79b685c0d779ea86", + "tests/kit/telemetry-contract.test.mjs": "a337910f016a3ae4e17a7bfdb95280088b7b0e73a9442eefb5ace09f61b34eb8", + "tests/kit/telemetry-source-bounds.test.mjs": "859fc7eef072024df1515c3064f4a7b8953babdc2960f6c7c27ba7dd6aad64a5" + } +} diff --git a/docs/implementation/fleet-evidence-export.md b/docs/implementation/fleet-evidence-export.md new file mode 100644 index 00000000..eec0c89a --- /dev/null +++ b/docs/implementation/fleet-evidence-export.md @@ -0,0 +1,45 @@ +# Fleet evidence export implementation plan + +**Goal:** A documented, private, versioned fleet data contract with working export, validation and aggregation. + +**Architecture:** Telemetry is an application read model over Usage and retained Maintenance evidence. +Pure projection/admission/reduction modules are separate from local file access and CLI dispatch. +No runtime dependency or remote service is added. + +**Spec:** [ADR-0054](../adr/0054-fleet-evidence-export.md). + +## Ownership and authority + +Root is the only writer in the isolated `feat/fleet-evidence-export` worktree. Research/review +workers are read-only. User authorization covers local branch, architecture, documentation, +implementation and tests. No commit, push, merge or publication is authorized. +Ruflo policy receipt: `sha256:e21b09352327083b5d4b154805efb12db33692905cff727746fbf9da3115e070` +(legacy-default-allow; not an independent security assurance). + +## Execution gates + +- [x] Contract: write failing `tests/kit/telemetry-contract.test.mjs`; implement `schema.mjs`, + `contract.mjs` and `projection.mjs`. Verify hostile field exclusion, null evidence and digest checks. +- [x] Reducer: write failing `tests/kit/telemetry-aggregate.test.mjs`; implement `aggregate.mjs`. + Verify replay, permutation, replacement, conflict and weighted arithmetic against fixed fixtures. +- [x] I/O and CLI: write failing `tests/kit/telemetry-cli.test.mjs`; implement `store.mjs`, + `collect.mjs`, `src/commands/telemetry.mjs`, path helper and dispatch entry. Verify bounded reads, + private stable identity, no-clobber file writes and hermetic end-to-end commands. +- [x] Document: publish `docs/TELEMETRY.md`, metric semantics, schema discovery, examples, + operational/privacy limits; amend relevant ADR cross-references and package documentation list. +- [x] Validate: focused tests, independent review, typecheck, lint, markdown, build and full tests. + Bind evidence to source digest, report failures/limitations, update ADR status and this checklist. + +## Acceptance + +`ak telemetry export` yields a validated snapshot; repeated exports preserve installation identity. +`ak telemetry schema` provides JSON Schema; `validate FILE` rejects malformed/unsupported records. +`aggregate FILE...` gives deterministic fleet totals without duplicate exports inflating counters. +Exported objects contain no copied arbitrary source strings. Missing sections remain unavailable. +No command starts a server, submits remote telemetry, executes a maintenance change or exposes keys. + +## Verification receipt + +See [source-bound local results](fleet-evidence-export-verification.json). Native Node tests are +authoritative for this run; the AQE executor returned inconsistent evidence and is not counted as +a successful quality gate. No overall repository quality score is claimed. diff --git a/package.json b/package.json index 274f7d22..08e49460 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "docs/INSTALLATION.md", "docs/MAINTENANCE.md", "docs/MODELS.md", + "docs/TELEMETRY.md", "docs/PROVIDERS.md", "docs/SETUP.md", "docs/TROUBLESHOOTING.md", @@ -39,6 +40,7 @@ "docs/adr/0043-managed-ruflo-browser-executor.md", "docs/adr/0044-receipt-aware-maintenance-control-plane.md", "docs/adr/0051-supported-peer-delegation-and-host-realignment.md", + "docs/adr/0054-fleet-evidence-export.md", "tests/live/aqe-external-provider-transport.test.mjs", "tests/live/qe-court-participant-transport.test.mjs", "tests/live/codex-context-contract.test.mjs", diff --git a/src/commands/telemetry.mjs b/src/commands/telemetry.mjs new file mode 100644 index 00000000..77d3c503 --- /dev/null +++ b/src/commands/telemetry.mjs @@ -0,0 +1,104 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { telemetryDir } from '../lib/paths.mjs'; +import { SNAPSHOT_SCHEMA, METRIC_CATALOG, DERIVED_METRICS, MAX_FILE_BYTES } from '../lib/telemetry/schema.mjs'; +import { validateSnapshot } from '../lib/telemetry/contract.mjs'; +import { aggregateSnapshots, MAX_INPUT_FILES } from '../lib/telemetry/aggregate.mjs'; +import { collectSnapshot } from '../lib/telemetry/collect.mjs'; +import { readJsonDocument, readJsonFile, readOrCreateIdentity, writeNewJson } from '../lib/telemetry/store.mjs'; + +export const options = { + days: { type: 'string' }, output: { type: 'string' }, + 'as-of': { type: 'string' }, 'stale-after': { type: 'string' }, +}; +export const help = `ak telemetry — vendor-neutral, offline fleet evidence + +Usage: + ak telemetry export [--days 1..365] [--output NEW_FILE] + ak telemetry validate FILE + ak telemetry aggregate FILE... [--as-of UTC_ISO] [--stale-after SECONDS] [--output NEW_FILE] + ak telemetry schema + ak telemetry metrics + +All successful output is JSON. --output creates a new owner-private file; never overwrites. +Export creates a private installation identity on first use, refreshes the local usage index, +and reads retained maintenance evidence. No upload, integration probe or maintenance scan. +--days defaults to 30. Whole retained sessions are selected by end time; their cumulative +usage can precede that lookback. Exports are snapshots, never additive time-window increments. +Aggregate selects the latest snapshot per installation and requires matching lookbacks. +--as-of defaults to now; use canonical UTC ISO (for example 2026-09-20T12:00:00.000Z). +--stale-after defaults to 86400 seconds. Stale data remains included and explicitly flagged. +No raw transcripts, private paths, hostnames, commands, secrets or configuration are exported. +Copied transcripts across installations are distinct observations; identity is installation-scoped. + +Examples: + ak telemetry export --days 30 --output ./machine-a.json + ak telemetry validate ./machine-a.json + ak telemetry aggregate ./machine-a.json ./machine-b.json --output ./fleet.json + ak telemetry schema + ak telemetry metrics`; + +function integerOption(value, fallback, max) { + if (value === undefined) return fallback; + if (!/^[1-9][0-9]*$/.test(value)) throw new TypeError('Invalid telemetry numeric option'); + const n = Number(value); + if (!Number.isSafeInteger(n) || n > max) throw new TypeError('Invalid telemetry numeric option'); + return n; +} +function checkOptions(command, flags, files) { + const allowed = { export: ['days', 'output'], validate: [], aggregate: ['as-of', 'stale-after', 'output'], schema: [], metrics: [] }; + if (!Object.hasOwn(allowed, command)) throw new TypeError('Unknown telemetry command'); + for (const [key, value] of Object.entries(flags)) { + if (value !== undefined && !allowed[command].includes(key)) throw new TypeError('Unsupported telemetry option'); + } + if (command === 'validate' && files.length !== 1) throw new TypeError('Telemetry validate requires one file'); + if (command === 'aggregate' && (!files.length || files.length > MAX_INPUT_FILES)) throw new TypeError('Telemetry aggregate requires 1..256 files'); + if (!['validate', 'aggregate'].includes(command) && files.length) throw new TypeError('Unexpected telemetry arguments'); +} + +/** CLI owns transport to files/stdout; pure domain functions never print. + * @param {{ flags: Record, positionals: string[], pkgRoot: string, + * deps?: { identityDir?: string, collect?: typeof collectSnapshot } }} input + */ +export async function run({ flags, positionals, pkgRoot, deps = {} }) { + try { + const [command, ...files] = positionals; + checkOptions(command, flags, files); + let result; + if (command === 'schema') result = SNAPSHOT_SCHEMA; + if (command === 'metrics') result = { schemaVersion: 1, metrics: METRIC_CATALOG, derived: DERIVED_METRICS }; + if (command === 'validate') { + const snapshot = validateSnapshot(readJsonFile(files[0])); + result = { valid: true, schemaVersion: snapshot.schemaVersion, snapshotId: snapshot.snapshotId }; + } + if (command === 'aggregate') { + const staleAfterSeconds = integerOption(flags['stale-after'], 86400, 31_536_000); + // Bound the combined input as well as each individual file. + let bytes = 0; + const snapshots = files.map(file => { + const document = readJsonDocument(file, MAX_FILE_BYTES - bytes); + bytes += document.bytes; + return document.value; + }); + result = aggregateSnapshots(snapshots, { asOf: flags['as-of'], staleAfterSeconds }); + } + if (command === 'export') { + const days = integerOption(flags.days, 30, 365); + const identity = readOrCreateIdentity(deps.identityDir ?? telemetryDir()); + const producerVersion = JSON.parse(fs.readFileSync(path.join(pkgRoot, 'package.json'), 'utf8')).version; + result = await (deps.collect ?? collectSnapshot)({ identity, producerVersion, days }); + validateSnapshot(result); + } + if (flags.output) writeNewJson(flags.output, result); + else { + const text = JSON.stringify(result, null, 2); + if (Buffer.byteLength(text) > MAX_FILE_BYTES) throw new Error('Telemetry output exceeds safe bounds'); + console.log(text); + } + return 0; + } catch { + // Source failures and hostile input must not echo filenames or parser details. + console.error('Telemetry failed: check command options, schema/digest, compatible snapshots, private identity, and readable input/new output files.'); + return 2; + } +} diff --git a/src/lib/maintenance/management/service-store.mjs b/src/lib/maintenance/management/service-store.mjs index 153a963c..64d7d4ec 100644 --- a/src/lib/maintenance/management/service-store.mjs +++ b/src/lib/maintenance/management/service-store.mjs @@ -5,6 +5,7 @@ // `scan-store.mjs`; a failed build never replaces the last-good snapshot // (MNT-PERF-007). import { createHash } from 'node:crypto'; +import fs from 'node:fs'; import path from 'node:path'; import zlib from 'node:zlib'; @@ -46,6 +47,26 @@ function sealedEnvelope(schema, payloadKey, payload) { return { ...base, integrity: { algorithm: 'sha256', digest: sha256(base) } }; } +/** Bound retained input before allocation and also reject growth/replacement while opening. */ +function readInventoryBytes(file, maxBytes, fsImpl) { + const before = fsImpl.lstatSync(file); + if (!before.isFile() || before.isSymbolicLink() || before.size > maxBytes) throw new Error('inventory input exceeds bounds'); + const fd = fsImpl.openSync(file, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0)); + try { + const stat = fsImpl.fstatSync(fd); + if (!stat.isFile() || stat.ino !== before.ino || stat.dev !== before.dev || stat.size > maxBytes) throw new Error('inventory input changed'); + const buffer = Buffer.alloc(stat.size + 1); + let size = 0; + while (size < buffer.length) { + const bytes = fsImpl.readSync(fd, buffer, size, buffer.length - size, null); + if (!bytes) break; + size += bytes; + } + if (size > stat.size) throw new Error('inventory input grew'); + return buffer.subarray(0, size); + } finally { fsImpl.closeSync(fd); } +} + /** `/inventory-latest.json.gz` — the last successfully built, * privacy-projected ManagementInventory plus its admitted guidance, gzip- * compressed on disk (D6). Read by every path-free facade method so a read @@ -59,15 +80,15 @@ export function createInventorySnapshotStore(root, { fsImpl }) { const compressedFile = path.join(root, 'inventory-latest.json.gz'); const legacyFile = path.join(root, 'inventory-latest.json'); - function readFrom(file, decode) { + function readFrom(file, maxBytes, decode) { let raw; - try { raw = decode(fsImpl.readFileSync(file)); } catch { return null; } + try { raw = decode(readInventoryBytes(file, maxBytes, fsImpl)); } catch { return null; } try { return readVerifiedEnvelope(raw, INVENTORY_SNAPSHOT_SCHEMA, 'snapshot'); } catch { return null; } } function read() { - return readFrom(compressedFile, (buffer) => zlib.gunzipSync(buffer).toString('utf8')) - ?? readFrom(legacyFile, (buffer) => buffer.toString('utf8')); + return readFrom(compressedFile, MAX_COMPRESSED_BYTES, (buffer) => zlib.gunzipSync(buffer, { maxOutputLength: MAX_UNCOMPRESSED_BYTES }).toString('utf8')) + ?? readFrom(legacyFile, MAX_UNCOMPRESSED_BYTES, (buffer) => buffer.toString('utf8')); } /** @returns {{ uncompressedBytes: number, compressedBytes: number }} the diff --git a/src/lib/paths.mjs b/src/lib/paths.mjs index db5f0457..08cc05b7 100644 --- a/src/lib/paths.mjs +++ b/src/lib/paths.mjs @@ -18,6 +18,7 @@ function stateBase() { return process.env.XDG_STATE_HOME || path.join(home, '.local', 'state'); } export const configDir = () => path.join(configBase(), 'agentic-kit'); +export const telemetryDir = () => path.join(configDir(), 'telemetry'); export const hookHealingTransactionsDir = () => path.join(stateBase(), 'agentic-kit', 'hook-healing'); export const maintenanceControlDir = () => path.join(stateBase(), 'agentic-kit', 'maintenance'); /** The ruflo-era config dir — read-fallback for kit.json migration and the diff --git a/src/lib/telemetry/aggregate.mjs b/src/lib/telemetry/aggregate.mjs new file mode 100644 index 00000000..1c282085 --- /dev/null +++ b/src/lib/telemetry/aggregate.mjs @@ -0,0 +1,89 @@ +import { canonicalJson, validateSnapshot } from './contract.mjs'; +import { COST_FIELDS, LATENCY_BOUNDS_SECONDS, METRIC_FIELDS, RECEIPT_STATES } from './schema.mjs'; + +export const MAX_INPUT_FILES = 256; +function sum(a, b) { + const n = a + b; + if (!Number.isFinite(n) || n > Number.MAX_SAFE_INTEGER) throw new RangeError('Telemetry aggregate exceeds numeric bounds'); + return n; +} +function measure(values, monetary = false) { + const known = values.filter(value => value !== null); + const total = known.length ? known.reduce(sum, 0) : null; + return { value: total === null ? null : monetary ? Number(total.toFixed(6)) : total, + measured: known.length, missing: values.length - known.length }; +} +function latestSnapshots(snapshots, asOf) { + if (!Array.isArray(snapshots) || !snapshots.length || snapshots.length > MAX_INPUT_FILES) { + throw new TypeError('Telemetry aggregate requires 1..256 snapshots'); + } + const latest = new Map(); + const revisions = new Map(); + let selection; + for (const raw of snapshots) { + const snapshot = validateSnapshot(raw); + if (snapshot.generatedAt > asOf) throw new TypeError('Telemetry snapshot is in the future'); + const thisSelection = canonicalJson(snapshot.selection); + if (selection && selection !== thisSelection) throw new TypeError('Telemetry selection mismatch'); + selection = thisSelection; + const revision = `${snapshot.installationId}:${snapshot.generatedAt}`; + if (revisions.has(revision) && revisions.get(revision) !== snapshot.snapshotId) { + throw new TypeError('Conflicting telemetry snapshots at one installation timestamp'); + } + revisions.set(revision, snapshot.snapshotId); + const previous = latest.get(snapshot.installationId); + if (!previous || previous.generatedAt < snapshot.generatedAt) latest.set(snapshot.installationId, snapshot); + } + return [...latest.values()].sort((a, b) => a.installationId.localeCompare(b.installationId)); +} +function usageTotals(selected) { + const sessions = selected.flatMap(s => s.usage.sessions); + const metrics = Object.fromEntries(METRIC_FIELDS.map(key => [key, measure(sessions.map(s => s[key]), COST_FIELDS.includes(key))])); + const cacheSessions = sessions.filter(s => [s.input, s.cacheRead, s.cacheWrite].every(x => x !== null)); + const numerator = cacheSessions.reduce((n, s) => sum(n, s.cacheRead), 0); + const denominator = cacheSessions.reduce((n, s) => sum(n, sum(sum(s.input, s.cacheRead), s.cacheWrite)), 0); + const histograms = sessions.map(s => s.latencyBuckets).filter(h => h !== null); + return { + sessionObservations: sessions.length, + availableInstallations: selected.filter(s => s.usage.state === 'available').length, + unavailableInstallations: selected.filter(s => s.usage.state === 'unavailable').length, + incompleteInstallations: selected.filter(s => s.usage.acquisitionComplete !== true).length, + metrics, + cacheReadShare: denominator > 0 ? numerator / denominator : null, + cacheReadShareEvidence: { numerator, denominator, measuredSessions: cacheSessions.length, + missingSessions: sessions.length - cacheSessions.length }, + latency: { unit: 's', upperBounds: [...LATENCY_BOUNDS_SECONDS], + bucketCounts: histograms.length ? LATENCY_BOUNDS_SECONDS.concat(Infinity).map((_, i) => histograms.reduce((n, h) => sum(n, h[i]), 0)) : null, + measuredSessions: histograms.length, missingSessions: sessions.length - histograms.length }, + }; +} + +/** Latest-per-installation as-of reduction, not an event ledger or time-series rollup. */ +export function aggregateSnapshots(snapshots, { asOf = new Date().toISOString(), staleAfterSeconds = 86400 } = {}) { + if (!Number.isFinite(Date.parse(asOf)) || new Date(asOf).toISOString() !== asOf) throw new TypeError('Invalid telemetry as-of time'); + if (!Number.isSafeInteger(staleAfterSeconds) || staleAfterSeconds < 1 || staleAfterSeconds > 31_536_000) { + throw new TypeError('Invalid telemetry freshness threshold'); + } + const selected = latestSnapshots(snapshots, asOf); + const receipts = selected.flatMap(s => s.maintenance.receipts); + return { + schemaVersion: 1, kind: 'agentic-kit.telemetry.aggregate', asOf, staleAfterSeconds, + selection: { ...selected[0].selection }, + installationCount: selected.length, + installations: selected.map(s => ({ installationId: s.installationId, snapshotId: s.snapshotId, + generatedAt: s.generatedAt, producerVersion: s.producerVersion, + ageSeconds: (Date.parse(asOf) - Date.parse(s.generatedAt)) / 1000, + stale: Date.parse(asOf) - Date.parse(s.generatedAt) > staleAfterSeconds * 1000, + usageState: s.usage.state, sourceHealth: { ...s.usage.sourceHealth }, + acquisitionComplete: s.usage.acquisitionComplete, pricesAsOf: s.usage.pricesAsOf, + inventoryState: s.inventory.state, inventoryCapturedAt: s.inventory.capturedAt, + inventoryAgeSeconds: s.inventory.capturedAt ? (Date.parse(asOf) - Date.parse(s.inventory.capturedAt)) / 1000 : null, + maintenanceState: s.maintenance.state })), + usage: usageTotals(selected), + inventory: { resources: measure(selected.map(s => s.inventory.resources)), placements: measure(selected.map(s => s.inventory.placements)) }, + maintenance: { availableInstallations: selected.filter(s => s.maintenance.state === 'available').length, + unavailableInstallations: selected.filter(s => s.maintenance.state === 'unavailable').length, + retainedReceipts: receipts.length, + byStatus: Object.fromEntries(RECEIPT_STATES.map(status => [status, receipts.filter(r => r.status === status).length])) }, + }; +} diff --git a/src/lib/telemetry/collect.mjs b/src/lib/telemetry/collect.mjs new file mode 100644 index 00000000..901f2b81 --- /dev/null +++ b/src/lib/telemetry/collect.mjs @@ -0,0 +1,31 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { readIndex } from '../usage-index.mjs'; +import { maintenanceControlDir } from '../paths.mjs'; +import { createInventorySnapshotStore } from '../maintenance/management/service-store.mjs'; +import { listMaintenanceReceiptsReadOnly } from '../maintenance/transaction-store.mjs'; +import { createSnapshot } from './projection.mjs'; + +function retainedInventory() { + return createInventorySnapshotStore(path.join(maintenanceControlDir(), 'management'), { fsImpl: fs }).read(); +} +function retainedReceipts() { + const root = path.join(maintenanceControlDir(), 'transactions'); + // No store is unavailable evidence; it is not proof that no action ever occurred. + if (!fs.existsSync(root)) return null; + return listMaintenanceReceiptsReadOnly(root); +} +async function evidence(read) { + try { return await read(); } catch { return null; } +} + +/** Sources are independent; one failed reader cannot manufacture successful empty evidence. */ +export async function collectSnapshot({ identity, producerVersion, days = 30, + generatedAt = new Date().toISOString(), readUsage = readIndex, + readInventory = retainedInventory, readReceipts = retainedReceipts }) { + const [usage, inventory, receipts] = await Promise.all([ + evidence(() => readUsage({ days, now: Date.parse(generatedAt), maxAgeMs: 0 })), + evidence(readInventory), evidence(readReceipts), + ]); + return createSnapshot({ identity, producerVersion, days, generatedAt, usage, inventory, receipts }); +} diff --git a/src/lib/telemetry/contract.mjs b/src/lib/telemetry/contract.mjs new file mode 100644 index 00000000..c83235d3 --- /dev/null +++ b/src/lib/telemetry/contract.mjs @@ -0,0 +1,77 @@ +import { createHash } from 'node:crypto'; +import { SNAPSHOT_SCHEMA, TOKEN_FIELDS } from './schema.mjs'; + +/** Deterministic JSON; object order never changes the content identity. */ +export function canonicalJson(value) { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; + if (value !== null && typeof value === 'object') { + return `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(',')}}`; + } + return JSON.stringify(value); +} + +export function snapshotDigest(snapshot) { + const { snapshotId: _id, ...body } = snapshot; + return createHash('sha256').update(canonicalJson(body)).digest('hex'); +} + +function invalid() { throw new TypeError('Invalid telemetry contract'); } +function hasType(value, type) { + if (type === 'null') return value === null; + if (type === 'array') return Array.isArray(value); + if (type === 'integer') return Number.isSafeInteger(value); + if (type === 'number') return typeof value === 'number' && Number.isFinite(value); + if (type === 'object') return value !== null && typeof value === 'object' && !Array.isArray(value); + return typeof value === type; +} + +function checkString(value, schema) { + if (value.length > schema.maxLength || (schema.pattern && !new RegExp(schema.pattern).test(value))) invalid(); + if (schema.format === 'date-time' && (!Number.isFinite(Date.parse(value)) || new Date(value).toISOString() !== value)) invalid(); +} + +// Deliberately limited to the keywords used by the published schema, not a general schema engine. +function check(value, schema) { + if (Object.hasOwn(schema, 'const') && value !== schema.const) invalid(); + if (schema.enum && !schema.enum.includes(value)) invalid(); + if (schema.type && ![schema.type].flat().some(type => hasType(value, type))) invalid(); + if (value === null) return; + if (typeof value === 'number' && (value < schema.minimum || value > schema.maximum)) invalid(); + if (typeof value === 'string') checkString(value, schema); + if (Array.isArray(value)) { + if (value.length < (schema.minItems ?? 0) || value.length > schema.maxItems) invalid(); + value.forEach(item => check(item, schema.items)); + } else if (schema.properties) { + if (Object.keys(value).length !== schema.required.length || schema.required.some(key => !Object.hasOwn(value, key))) invalid(); + for (const key of schema.required) check(value[key], schema.properties[key]); + } +} + +function unique(rows, key) { + if (new Set(rows.map(row => row[key])).size !== rows.length) invalid(); +} + +/** Admit an untrusted snapshot. Error messages never repeat supplied data. */ +export function validateSnapshot(snapshot) { + check(snapshot, SNAPSHOT_SCHEMA); + unique(snapshot.usage.sessions, 'sessionId'); + unique(snapshot.maintenance.receipts, 'receiptId'); + if (snapshot.usage.state === 'unavailable' && (snapshot.usage.sessions.length || snapshot.usage.acquisitionComplete !== null)) invalid(); + if (snapshot.maintenance.state === 'unavailable' && snapshot.maintenance.receipts.length) invalid(); + const inv = snapshot.inventory; + if (inv.state === 'unavailable' && [inv.capturedAt, inv.resources, inv.placements].some(x => x !== null)) invalid(); + if (inv.state === 'available' && [inv.capturedAt, inv.resources, inv.placements].some(x => x === null)) invalid(); + if (inv.capturedAt && inv.capturedAt > snapshot.generatedAt) invalid(); + for (const s of snapshot.usage.sessions) { + const messages = s.observedCostMessages + s.estimatedCostMessages + s.unpricedMessages; + if (!messages && TOKEN_FIELDS.some(key => s[key] !== null)) invalid(); + if (s.observedCostMessages === 0 && s.observedCostUsd !== null) invalid(); + if (s.estimatedCostMessages === 0 && s.estimatedCostUsd !== null) invalid(); + } + if (snapshotDigest(snapshot) !== snapshot.snapshotId) throw new TypeError('Telemetry digest mismatch'); + return snapshot; +} + +export function sealSnapshot(body) { + return validateSnapshot({ ...body, snapshotId: snapshotDigest(body) }); +} diff --git a/src/lib/telemetry/projection.mjs b/src/lib/telemetry/projection.mjs new file mode 100644 index 00000000..f185f9ae --- /dev/null +++ b/src/lib/telemetry/projection.mjs @@ -0,0 +1,69 @@ +import { createHmac } from 'node:crypto'; +import { sealSnapshot } from './contract.mjs'; +import { HOSTS, RECEIPT_STATES, TOKEN_FIELDS } from './schema.mjs'; + +const count = value => Number.isSafeInteger(value) && value >= 0 ? value : null; +const money = value => typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : null; +function iso(value) { + if (typeof value !== 'string' || !Number.isFinite(Date.parse(value))) return null; + return new Date(value).toISOString(); +} +function reference(identity, domain, parts) { + return createHmac('sha256', Buffer.from(identity.key, 'hex')).update(JSON.stringify([domain, ...parts])).digest('hex'); +} +function health(value) { + return ['ok', 'absent', 'degraded', 'unavailable'].includes(value) ? value : 'unknown'; +} +function sessionProjection(s, identity) { + if (!HOSTS.includes(s.host) || typeof s.id !== 'string' || !s.id || s.id.length > 4096) { + throw new TypeError('Unsupported telemetry session identity'); + } + const cost = s.costEvidence ?? {}; + const observed = count(cost.observedMessages); + const estimated = count(cost.estimatedMessages); + const unpriced = count(cost.unpricedMessages); + const hasUsage = (observed ?? 0) + (estimated ?? 0) + (unpriced ?? 0) > 0; + const buckets = Array.isArray(s.latHist) && s.latHist.length === 6 + && s.latHist.every(x => count(x) !== null) ? [...s.latHist] : null; + return { + sessionId: reference(identity, 'session', [s.host, s.id]), host: s.host, + ...Object.fromEntries(TOKEN_FIELDS.map(key => [key, hasUsage ? count(s[key]) : null])), + prompts: count(s.prompts), responses: count(s.responses), exceptions: count(s.exceptions), aborts: count(s.aborts), + observedCostUsd: observed > 0 ? money(cost.observedUsd) : null, + estimatedCostUsd: estimated > 0 ? money(cost.estimatedUsd) : null, + observedCostMessages: observed, estimatedCostMessages: estimated, unpricedMessages: unpriced, + latencyBuckets: buckets, + }; +} + +/** Pure anti-corruption layer: construct each field, never spread source records. */ +export function createSnapshot({ identity, generatedAt, producerVersion, days, usage, inventory, receipts }) { + if (!/^[a-f0-9]{64}$/.test(identity?.key ?? '')) throw new TypeError('Invalid telemetry identity'); + const sessions = (usage?.sessions ?? []).map(s => sessionProjection(s, identity)) + .sort((a, b) => a.sessionId.localeCompare(b.sessionId)); + const capturedAt = iso(inventory?.capturedAt); + const inventoryAvailable = capturedAt !== null && Array.isArray(inventory?.resources) && Array.isArray(inventory?.placements); + return sealSnapshot({ + schemaVersion: 1, kind: 'agentic-kit.telemetry.snapshot', installationId: identity.installationId, + generatedAt, producerVersion, + selection: { days, scope: 'whole-retained-sessions-selected-by-end' }, + usage: { + state: usage ? 'available' : 'unavailable', + acquisitionComplete: typeof usage?.acquisitionCoverage?.complete === 'boolean' ? usage.acquisitionCoverage.complete : null, + pricesAsOf: /^\d{4}-\d{2}-\d{2}$/.test(usage?.pricesAsOf ?? '') ? usage.pricesAsOf : null, + sourceHealth: Object.fromEntries([...HOSTS, 'codexLedger'].map(host => [host, health(usage?.sourceHealth?.[host]?.status)])), + sessions, + }, + inventory: { state: inventoryAvailable ? 'available' : 'unavailable', + capturedAt: inventoryAvailable ? capturedAt : null, + resources: inventoryAvailable ? inventory.resources.length : null, + placements: inventoryAvailable ? inventory.placements.length : null }, + maintenance: { state: receipts ? 'available' : 'unavailable', + receipts: (receipts ?? []).map(r => { + if (typeof r.id !== 'string' || !r.id || r.id.length > 4096) throw new TypeError('Invalid telemetry receipt identity'); + return { receiptId: reference(identity, 'receipt', [r.id]), + status: RECEIPT_STATES.includes(r.status) ? r.status : 'unknown', updatedAt: iso(r.updatedAt) }; + }).sort((a, b) => a.receiptId.localeCompare(b.receiptId)), + }, + }); +} diff --git a/src/lib/telemetry/schema.mjs b/src/lib/telemetry/schema.mjs new file mode 100644 index 00000000..601bf4a8 --- /dev/null +++ b/src/lib/telemetry/schema.mjs @@ -0,0 +1,74 @@ +// ADR-0054: the public contract is independent of internal cache schemas. +export const MAX_SESSIONS = 100_000; +export const MAX_RECEIPTS = 100_000; +export const MAX_FILE_BYTES = 64 * 1024 * 1024; +export const HOSTS = ['claude', 'codex', 'opencode']; +export const LATENCY_BOUNDS_SECONDS = [2, 5, 10, 30, 60]; +export const RECEIPT_STATES = ['prepared', 'applying', 'verifying', 'refreshing-catalog', 'undoing', + 'failed', 'partial', 'partial-recovery-required', 'outcome-unknown', 'committed', 'rolled-back', + 'undone', 'aborted-no-change', 'unknown-recovery-required', 'unknown']; +export const TOKEN_FIELDS = ['input', 'output', 'cacheRead', 'cacheWrite']; +export const COUNT_FIELDS = ['prompts', 'responses', 'exceptions', 'aborts', + 'observedCostMessages', 'estimatedCostMessages', 'unpricedMessages']; +export const COST_FIELDS = ['observedCostUsd', 'estimatedCostUsd']; +export const METRIC_FIELDS = [...TOKEN_FIELDS, ...COUNT_FIELDS, ...COST_FIELDS]; + +const object = (properties) => ({ type: 'object', properties, required: Object.keys(properties), additionalProperties: false }); +const enumeration = (values) => ({ enum: values }); +const integer = { type: 'integer', minimum: 0, maximum: Number.MAX_SAFE_INTEGER }; +const nullableInteger = { ...integer, type: ['integer', 'null'] }; +const money = { type: ['number', 'null'], minimum: 0, maximum: Number.MAX_SAFE_INTEGER }; +const digest = { type: 'string', pattern: '^[a-f0-9]{64}$', maxLength: 64 }; +const timestamp = { type: 'string', format: 'date-time', pattern: '^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$', maxLength: 24 }; +const nullableTimestamp = { ...timestamp, type: ['string', 'null'] }; +const available = enumeration(['available', 'unavailable']); +const hostHealth = enumeration(['ok', 'absent', 'degraded', 'unavailable', 'unknown']); +const session = object({ + sessionId: digest, host: enumeration(HOSTS), + ...Object.fromEntries([...TOKEN_FIELDS, ...COUNT_FIELDS].map(key => [key, nullableInteger])), + ...Object.fromEntries(COST_FIELDS.map(key => [key, money])), + latencyBuckets: { type: ['array', 'null'], items: integer, minItems: 6, maxItems: 6 }, +}); + +export const SNAPSHOT_SCHEMA = { + $schema: 'https://json-schema.org/draft/2020-12/schema', + title: 'Agentic-kit telemetry snapshot v1', + ...object({ + schemaVersion: { const: 1 }, kind: { const: 'agentic-kit.telemetry.snapshot' }, snapshotId: digest, + installationId: { type: 'string', pattern: '^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$', maxLength: 36 }, + generatedAt: timestamp, + producerVersion: { type: 'string', pattern: '^[0-9]+\\.[0-9]+\\.[0-9]+(?:-[A-Za-z0-9.-]+)?$', maxLength: 80 }, + selection: object({ days: { type: 'integer', minimum: 1, maximum: 365 }, scope: { const: 'whole-retained-sessions-selected-by-end' } }), + usage: object({ state: available, acquisitionComplete: { type: ['boolean', 'null'] }, + pricesAsOf: { type: ['string', 'null'], pattern: '^\\d{4}-\\d{2}-\\d{2}$', maxLength: 10 }, + sourceHealth: object(Object.fromEntries([...HOSTS, 'codexLedger'].map(host => [host, hostHealth]))), + sessions: { type: 'array', items: session, maxItems: MAX_SESSIONS }, + }), + inventory: object({ state: available, capturedAt: nullableTimestamp, resources: nullableInteger, placements: nullableInteger }), + maintenance: object({ state: available, receipts: { type: 'array', maxItems: MAX_RECEIPTS, + items: object({ receiptId: digest, status: enumeration(RECEIPT_STATES), updatedAt: nullableTimestamp }), + } }), + }), +}; + +/** Machine-readable metric semantics; every measure uses snapshot replacement before folding. */ +export const METRIC_CATALOG = Object.fromEntries(METRIC_FIELDS.map(name => [name, { + unit: TOKEN_FIELDS.includes(name) ? 'token' : COST_FIELDS.includes(name) ? 'USD' : 'count', + aggregation: 'sum-known-values-after-latest-installation-snapshot', + missing: 'null; measured and missing contribution counts accompany every aggregate', + basis: name === 'estimatedCostUsd' ? 'API-equivalent estimate, not billing' + : name === 'observedCostUsd' ? 'source-reported cost, not reconciled billing' : 'normalized retained session observation', +}])); + +export const DERIVED_METRICS = { + cacheReadShare: { unit: 'ratio', numerator: 'sum(cacheRead)', + denominator: 'sum(input + cacheRead + cacheWrite)', + eligibility: 'only sessions with all three known counts', zeroDenominator: null }, + latency: { unit: 's', upperBounds: LATENCY_BOUNDS_SECONDS, + buckets: 'six non-cumulative counts; upper bounds inclusive; final bucket above 60s', + aggregation: 'sum corresponding buckets; never average percentiles', missing: null }, + inventoryResources: { unit: 'count', aggregation: 'sum known resource counts from latest installation snapshots' }, + inventoryPlacements: { unit: 'count', aggregation: 'sum known placement counts from latest installation snapshots' }, + retainedReceipts: { unit: 'count', aggregation: 'count retained receipt observations by controlled status' }, + sessionObservations: { unit: 'count', aggregation: 'count retained sessions across installation snapshots; not unique people or tasks' }, +}; diff --git a/src/lib/telemetry/store.mjs b/src/lib/telemetry/store.mjs new file mode 100644 index 00000000..e042200d --- /dev/null +++ b/src/lib/telemetry/store.mjs @@ -0,0 +1,73 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { randomBytes, randomUUID } from 'node:crypto'; +import { MAX_FILE_BYTES } from './schema.mjs'; + +/** Read only bounded regular files; refuse final-component links and detect replacement races. */ +export function readJsonDocument(file, maxBytes = MAX_FILE_BYTES) { + const absolute = path.resolve(file); + const before = fs.lstatSync(absolute); + if (!before.isFile() || before.isSymbolicLink() || before.size > maxBytes) throw new Error('Telemetry file exceeds safe bounds'); + const fd = fs.openSync(absolute, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0)); + try { + const stat = fs.fstatSync(fd); + if (!stat.isFile() || stat.ino !== before.ino || stat.dev !== before.dev || stat.size > maxBytes) { + throw new Error('Telemetry file exceeds safe bounds'); + } + const buffer = Buffer.alloc(Math.min(stat.size + 1, maxBytes + 1)); + let size = 0; + while (size < buffer.length) { + const n = fs.readSync(fd, buffer, size, buffer.length - size, null); + if (!n) break; + size += n; + } + if (size > maxBytes || size > stat.size) throw new Error('Telemetry file exceeds safe bounds'); + try { return { value: JSON.parse(buffer.subarray(0, size).toString('utf8')), bytes: size }; } + catch { throw new Error('Invalid telemetry JSON'); } + } finally { fs.closeSync(fd); } +} + +/** Convenience reader for one document. */ +export function readJsonFile(file, maxBytes = MAX_FILE_BYTES) { + return readJsonDocument(file, maxBytes).value; +} + +/** Publish complete JSON with no-clobber semantics, including against destination symlinks. */ +export function writeNewJson(file, value) { + const absolute = path.resolve(file); + const text = `${JSON.stringify(value, null, 2)}\n`; + if (Buffer.byteLength(text) > MAX_FILE_BYTES) throw new Error('Telemetry output exceeds safe bounds'); + const temp = path.join(path.dirname(absolute), `.ak-telemetry-${randomBytes(16).toString('hex')}.tmp`); + let fd; + try { + fd = fs.openSync(temp, 'wx', 0o600); + fs.writeFileSync(fd, text); fs.fsyncSync(fd); fs.closeSync(fd); fd = undefined; + // A hard link publishes already-complete bytes atomically and fails if the destination exists. + fs.linkSync(temp, absolute); + } finally { + if (fd !== undefined) fs.closeSync(fd); + fs.rmSync(temp, { force: true }); + } +} + +/** Explicit export enrolls a local installation; never silently replace a lost/corrupt identity. */ +export function readOrCreateIdentity(directory) { + const root = path.resolve(directory); + fs.mkdirSync(root, { recursive: true, mode: 0o700 }); + const stat = fs.lstatSync(root); + if (!stat.isDirectory() || stat.isSymbolicLink() || (process.platform !== 'win32' && (stat.mode & 0o077))) { + throw new Error('Telemetry identity directory must be private'); + } + const file = path.join(root, 'identity.json'); + if (!fs.existsSync(file)) { + try { writeNewJson(file, { installationId: randomUUID(), key: randomBytes(32).toString('hex') }); } + catch (error) { if (error.code !== 'EEXIST') throw error; } + } + const fileStat = fs.lstatSync(file); + if (process.platform !== 'win32' && (fileStat.mode & 0o077)) throw new Error('Telemetry identity must be private'); + const value = readJsonFile(file, 4096); + if (!value || Object.keys(value).sort().join(',') !== 'installationId,key' + || !/^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/.test(value.installationId) + || !/^[a-f0-9]{64}$/.test(value.key)) throw new Error('Invalid telemetry identity; restore its original private file'); + return value; +} diff --git a/tests/kit/helpers/telemetry.mjs b/tests/kit/helpers/telemetry.mjs new file mode 100644 index 00000000..c8a8c805 --- /dev/null +++ b/tests/kit/helpers/telemetry.mjs @@ -0,0 +1,16 @@ +export const identity = { installationId: '11111111-1111-4111-8111-111111111111', key: 'ab'.repeat(32) }; +export const now = '2026-09-20T12:00:00.000Z'; +export function source(overrides = {}) { + return { id: 'native-private-session', host: 'codex', title: 'SECRET', project: '/private/SECRET', + start: '2026-09-19T12:00:00.000Z', input: 100, output: 20, cacheRead: 80, cacheWrite: 0, + prompts: 2, responses: 3, exceptions: 1, aborts: 0, latHist: [1, 1, 0, 1, 0, 0], latCount: 3, + costEvidence: { observedUsd: 0, estimatedUsd: 0.02, observedMessages: 0, estimatedMessages: 3, unpricedMessages: 0 }, + ...overrides }; +} +export async function fixture({ sessions = [source()], generatedAt = now, ...rest } = {}) { + const { createSnapshot } = await import('../../../src/lib/telemetry/projection.mjs'); + return createSnapshot({ identity, generatedAt, producerVersion: '4.0.0-alpha.51', days: 30, + usage: { sessions, sourceHealth: { codex: { status: 'ok' } }, acquisitionCoverage: { complete: true }, pricesAsOf: '2026-09-09' }, + inventory: null, receipts: [], ...rest }); +} + diff --git a/tests/kit/telemetry-aggregate.test.mjs b/tests/kit/telemetry-aggregate.test.mjs new file mode 100644 index 00000000..9f06f064 --- /dev/null +++ b/tests/kit/telemetry-aggregate.test.mjs @@ -0,0 +1,75 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { fixture, source, identity, now } from './helpers/telemetry.mjs'; +const modulePath = '../../src/lib/telemetry/aggregate.mjs'; +const secondIdentity = { ...identity, installationId: '22222222-2222-4222-8222-222222222222' }; +async function fold(snapshots, options = {}) { + const { aggregateSnapshots } = await import(modulePath); + return aggregateSnapshots(snapshots, { asOf: now, ...options }); +} +test('should_exposeReducer_when_telemetryIsImplemented', async () => { + const module = await import(modulePath).catch(() => null); + assert.equal(typeof module?.aggregateSnapshots, 'function'); +}); +test('should_beIdempotent_when_replayingAnExport', async () => { + const a = await fixture(); + assert.deepEqual(await fold([a, a]), await fold([a])); +}); +test('should_replaceWholeSnapshot_when_newerExportOmitsOldSessions', async () => { + const old = await fixture({ generatedAt: '2026-09-19T12:00:00.000Z' }); + const recent = await fixture({ sessions: [] }); + assert.equal((await fold([old, recent])).usage.sessionObservations, 0); +}); +test('should_beOrderIndependent_when_installationsAndRevisionsAreReordered', async () => { + const a = await fixture(); const b = await fixture({ identity: secondIdentity }); + assert.deepEqual(await fold([a, b]), await fold([b, a])); +}); +test('should_rejectConflictingTies_when_timestampMatchesButContentDiffers', async () => { + await assert.rejects(fold([await fixture(), await fixture({ sessions: [source({ input: 20 })] })]), /conflict/i); +}); +test('should_rejectMixedLookbacks_when_selectionCannotBeCompared', async () => { + await assert.rejects(fold([await fixture(), await fixture({ days: 7, identity: secondIdentity })]), /selection/i); +}); +test('should_computeWeightedCacheShare_when_installationsDiffer', async () => { + const result = await fold([await fixture(), await fixture({ identity: secondIdentity, sessions: [source({ input: 20, cacheRead: 0 })] })]); + assert.equal(result.usage.cacheReadShare, 0.4); +}); +test('should_mergeBuckets_when_latencyEvidenceIsPresent', async () => { + const result = await fold([await fixture(), await fixture({ identity: secondIdentity })]); + assert.deepEqual(result.usage.latency.bucketCounts, [2, 2, 0, 2, 0, 0]); +}); +test('should_reportMissingContributions_when_onlySomeSessionsHaveUsage', async () => { + const result = await fold([await fixture({ sessions: [source(), source({ id: 'other', costEvidence: null })] })]); + assert.deepEqual(result.usage.metrics.input, { value: 100, measured: 1, missing: 1 }); +}); +test('should_keepAllMissingNull_when_usageCollectionFailed', async () => { + const result = await fold([await fixture({ usage: null })]); + assert.equal(result.usage.metrics.input.value, null); +}); +test('should_reportStaleness_when_collectionIsOld', async () => { + const result = await fold([await fixture({ generatedAt: '2026-09-18T12:00:00.000Z' })]); + assert.equal(result.installations[0].stale, true); +}); +test('should_rejectFutureRecords_when_asOfPrecedesExport', async () => { + await assert.rejects(fold([await fixture()], { asOf: '2026-09-19T12:00:00.000Z' }), /future/i); +}); +test('should_notMutateInputs_when_aggregating', async () => { + const a = await fixture(); const before = structuredClone(a); await fold([a]); + assert.deepEqual(a, before); +}); +test('should_preserveLargeIntegers_when_summingCounters', async () => { + const result = await fold([await fixture({ sessions: [source({ prompts: 1689509126518595 })] })]); + assert.equal(result.usage.metrics.prompts.value, 1689509126518595); +}); +test('should_rejectNumericOverflow_when_combiningInstallations', async () => { + const sessions = [source({ prompts: Number.MAX_SAFE_INTEGER })]; + await assert.rejects(fold([await fixture({ sessions }), await fixture({ sessions, identity: secondIdentity })]), /numeric bounds/); +}); +test('should_rejectInvalidReducerOptions_when_boundsAreViolated', async () => { + const snapshot = await fixture(); + for (const options of [{ asOf: 'bad' }, { staleAfterSeconds: 0 }, { staleAfterSeconds: 31_536_001 }]) { + await assert.rejects(fold([snapshot], options), /Invalid/); + } + await assert.rejects(fold([]), /requires/); + await assert.rejects(fold(Array(257).fill(snapshot)), /requires/); +}); diff --git a/tests/kit/telemetry-cli.test.mjs b/tests/kit/telemetry-cli.test.mjs new file mode 100644 index 00000000..4abd5d03 --- /dev/null +++ b/tests/kit/telemetry-cli.test.mjs @@ -0,0 +1,164 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { fixture, identity, now } from './helpers/telemetry.mjs'; +const storePath = '../../src/lib/telemetry/store.mjs'; +const bin = new URL('../../bin/agentic-kit.mjs', import.meta.url); +function temporary(t) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ak-telemetry-')); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); return dir; +} +function cli(args) { return spawnSync(process.execPath, [fileURLToPath(bin), 'telemetry', ...args], { encoding: 'utf8' }); } + +test('should_advertiseTelemetry_when_requestingCommandHelp', () => { + assert.match(cli(['--help']).stdout, /ak telemetry export/); +}); +test('should_preserveIdentity_when_reopeningInstallationStore', async t => { + const { readOrCreateIdentity } = await import(storePath); const dir = temporary(t); + assert.deepEqual(readOrCreateIdentity(dir), readOrCreateIdentity(dir)); +}); +test('should_rejectCorruptIdentity_when_existingStoreIsInvalid', async t => { + const { readOrCreateIdentity } = await import(storePath); const dir = temporary(t); + fs.writeFileSync(path.join(dir, 'identity.json'), '{}', { mode: 0o600 }); + assert.throws(() => readOrCreateIdentity(dir), /identity/i); +}); +test('should_refuseSymlinkIdentity_when_loadingState', async t => { + const { readOrCreateIdentity } = await import(storePath); const dir = temporary(t); + fs.writeFileSync(path.join(dir, 'target'), JSON.stringify(identity)); + fs.symlinkSync(path.join(dir, 'target'), path.join(dir, 'identity.json')); + assert.throws(() => readOrCreateIdentity(dir)); +}); +test('should_notOverwriteExistingOutput_when_exportDestinationExists', async t => { + const { writeNewJson } = await import(storePath); const file = path.join(temporary(t), 'export.json'); + fs.writeFileSync(file, 'original'); + assert.throws(() => writeNewJson(file, {})); + assert.equal(fs.readFileSync(file, 'utf8'), 'original'); +}); +test('should_boundInputReads_when_fileExceedsLimit', async t => { + const { readJsonFile } = await import(storePath); const file = path.join(temporary(t), 'large.json'); + fs.writeFileSync(file, ' '.repeat(100)); + assert.throws(() => readJsonFile(file, 20), /bound/i); +}); +test('should_validateAndAggregateFiles_when_usingRealCli', async t => { + const dir = temporary(t); const file = path.join(dir, 'snapshot.json'); + fs.writeFileSync(file, JSON.stringify(await fixture())); + assert.equal(cli(['validate', file]).status, 0); + const result = cli(['aggregate', file, file, '--as-of', now]); + assert.equal(result.status, 0, result.stderr); + assert.equal(JSON.parse(result.stdout).usage.metrics.input.value, 100); +}); +test('should_exposeJsonSchema_when_requested', () => { + const result = cli(['schema']); + assert.equal(result.status, 0, result.stderr); + assert.equal(JSON.parse(result.stdout).properties.schemaVersion.const, 1); +}); +test('should_keepErrorsGeneric_when_malformedFilesContainSecrets', t => { + const file = path.join(temporary(t), 'SECRET.json'); fs.writeFileSync(file, 'SECRET'); + const result = cli(['validate', file]); + assert.equal(result.status, 2); + assert.doesNotMatch(result.stdout + result.stderr, /SECRET/); +}); +test('should_degradeSourcesIndependently_when_usageReaderFails', async () => { + const { collectSnapshot } = await import('../../src/lib/telemetry/collect.mjs'); + const snapshot = await collectSnapshot({ identity, days: 30, generatedAt: now, producerVersion: '1.0.0', + readUsage: async () => { throw new Error('SECRET'); }, readInventory: () => null, readReceipts: () => [] }); + assert.equal(snapshot.usage.state, 'unavailable'); + assert.equal(snapshot.maintenance.state, 'available'); +}); +test('should_exportHermeticLocalEvidence_when_invokingRealCli', t => { + const dir = temporary(t); + const env = { ...process.env, HOME: dir, USERPROFILE: dir, XDG_CONFIG_HOME: path.join(dir, 'config'), + XDG_STATE_HOME: path.join(dir, 'state'), APPDATA: path.join(dir, 'config'), LOCALAPPDATA: path.join(dir, 'state'), + CLAUDE_CONFIG_DIR: path.join(dir, '.claude') }; + const invoke = args => spawnSync(process.execPath, [fileURLToPath(bin), 'telemetry', ...args], { encoding: 'utf8', env }); + const first = invoke(['export']); + assert.equal(first.status, 0, first.stderr); + const a = JSON.parse(first.stdout); + const second = invoke(['export']); + assert.equal(second.status, 0, second.stderr); + assert.equal(JSON.parse(second.stdout).installationId, a.installationId); + assert.equal(a.inventory.state, 'unavailable'); + assert.equal(a.maintenance.state, 'unavailable'); + assert.equal(a.usage.sessions.length, 0); +}); +test('should_projectActualParserEvidence_when_localFixtureCorpusIsRead', async t => { + const { collectSnapshot } = await import('../../src/lib/telemetry/collect.mjs'); + const { buildIndex } = await import('../../src/lib/usage-index.mjs'); + const dir = temporary(t); + fs.cpSync(new URL('../fixtures/usage', import.meta.url), path.join(dir, 'corpus'), { recursive: true }); + const roots = { claude: path.join(dir, 'corpus/claude'), codex: path.join(dir, 'corpus/codex') }; + const readUsage = options => buildIndex({ ...options, roots, cachePath: path.join(dir, 'cache.json'), + deps: { costOf: ({ input, output, cacheRead, cacheWrite }) => (input + output + cacheRead + cacheWrite) / 1000, + pricesAsOf: '2026-07-01', classify: () => ({}), detectInsights: () => [] } }); + const snapshot = await collectSnapshot({ identity, producerVersion: '1.0.0', days: 30, + generatedAt: '2026-07-25T12:00:00.000Z', readUsage, readInventory: () => null, readReceipts: () => null }); + assert.ok(snapshot.usage.sessions.length > 0); + assert.ok(snapshot.usage.sessions.some(s => s.input > 0)); + assert.doesNotMatch(JSON.stringify(snapshot), /Users-me|aaaa1111|rollout|corpus/); +}); +test('should_rejectInvalidOptions_before_creatingIdentity', async t => { + const { run } = await import('../../src/commands/telemetry.mjs'); + const dir = path.join(temporary(t), 'identity'); + const error = t.mock.method(console, 'error', () => {}); + const code = await run({ flags: { days: '0' }, positionals: ['export'], pkgRoot: fileURLToPath(new URL('../..', import.meta.url)), deps: { identityDir: dir } }); + assert.equal(code, 2); assert.equal(fs.existsSync(dir), false); assert.equal(error.mock.callCount(), 1); +}); +test('should_writePrivateCompleteJson_when_newOutputIsRequested', async t => { + const { writeNewJson, readJsonFile } = await import(storePath); + const file = path.join(temporary(t), 'new.json'); writeNewJson(file, { valid: true }); + assert.deepEqual(readJsonFile(file), { valid: true }); + if (process.platform !== 'win32') assert.equal(fs.statSync(file).mode & 0o777, 0o600); +}); +test('should_rejectReplacedFiles_when_openedInodeDiffers', async t => { + const { readJsonFile } = await import(storePath); const file = path.join(temporary(t), 'data.json'); + fs.writeFileSync(file, '{}'); + const original = fs.fstatSync; + t.mock.method(fs, 'fstatSync', (...args) => { const stat = original(...args); stat.ino++; return stat; }); + assert.throws(() => readJsonFile(file), /bounds/); +}); +test('should_rejectPublicIdentityDirectory_when_permissionsAreLoose', { skip: process.platform === 'win32' }, async t => { + const { readOrCreateIdentity } = await import(storePath); const dir = temporary(t); fs.chmodSync(dir, 0o755); + assert.throws(() => readOrCreateIdentity(dir), /private/); +}); +test('should_rejectPublicIdentityFile_when_permissionsAreLoose', { skip: process.platform === 'win32' }, async t => { + const { readOrCreateIdentity } = await import(storePath); const dir = temporary(t); readOrCreateIdentity(dir); + fs.chmodSync(path.join(dir, 'identity.json'), 0o644); + assert.throws(() => readOrCreateIdentity(dir), /private/); +}); +test('should_failSafely_when_publishingIdentityCannotLink', async t => { + const { readOrCreateIdentity } = await import(storePath); const dir = temporary(t); + t.mock.method(fs, 'linkSync', () => { throw Object.assign(new Error('denied'), { code: 'EACCES' }); }); + assert.throws(() => readOrCreateIdentity(dir), /denied/); + assert.deepEqual(fs.readdirSync(dir), []); +}); +test('should_rejectBadCommandForms_when_cliArgumentsConflict', () => { + for (const args of [['missing'], ['validate'], ['aggregate'], ['export', '--days', '366'], + ['export', '--days', '1.5'], ['schema', '--days', '30'], ['schema', 'extra']]) { + assert.equal(cli(args).status, 2, args.join(' ')); + } +}); +test('should_readRetainedMaintenance_when_exportingOffline', t => { + const dir = temporary(t); const state = path.join(dir, 'state'); + const root = path.join(state, 'agentic-kit', 'maintenance', 'transactions'); + fs.mkdirSync(path.join(root, 'mnt-corrupt'), { recursive: true, mode: 0o700 }); + const result = spawnSync(process.execPath, [fileURLToPath(bin), 'telemetry', 'export'], { encoding: 'utf8', + env: { ...process.env, HOME: dir, USERPROFILE: dir, XDG_CONFIG_HOME: path.join(dir, 'config'), + XDG_STATE_HOME: state, LOCALAPPDATA: state, APPDATA: path.join(dir, 'config'), CLAUDE_CONFIG_DIR: path.join(dir, '.claude') } }); + assert.equal(result.status, 0, result.stderr); + assert.equal(JSON.parse(result.stdout).maintenance.receipts[0].status, 'unknown-recovery-required'); +}); +test('should_countRawBytes_when_inputContainsWhitespace', async t => { + const module = await import(storePath); + assert.equal(typeof module.readJsonDocument, 'function'); + const file = path.join(temporary(t), 'padded.json'); fs.writeFileSync(file, ' {} '); + assert.deepEqual(module.readJsonDocument(file), { value: {}, bytes: 6 }); +}); +test('should_notEchoUnknownOptionValues_when_cliParsingFails', () => { + const result = cli(['export', '--secret-token=PRIVATE-MARKER']); + assert.equal(result.status, 2); + assert.doesNotMatch(result.stdout + result.stderr, /PRIVATE-MARKER|secret-token/); +}); diff --git a/tests/kit/telemetry-contract.test.mjs b/tests/kit/telemetry-contract.test.mjs new file mode 100644 index 00000000..47b387d7 --- /dev/null +++ b/tests/kit/telemetry-contract.test.mjs @@ -0,0 +1,96 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +const projectionPath = '../../src/lib/telemetry/projection.mjs'; +const contractPath = '../../src/lib/telemetry/contract.mjs'; +import { fixture, source, now } from './helpers/telemetry.mjs'; + +test('should_exposeSnapshotProjection_when_telemetryIsImplemented', async () => { + const module = await import(projectionPath).catch(() => null); + assert.equal(typeof module?.createSnapshot, 'function'); +}); +test('should_excludePrivateSourceFields_when_exporting', async () => { + const snapshot = await fixture(); + assert.doesNotMatch(JSON.stringify(snapshot), /SECRET|native-private|abababab|title|project/); +}); +test('should_validateOwnExport_when_projectingKnownEvidence', async () => { + const { validateSnapshot } = await import(contractPath); + assert.equal(validateSnapshot(await fixture()).kind, 'agentic-kit.telemetry.snapshot'); +}); +test('should_preserveUnknownTokens_when_noUsageEvidenceExists', async () => { + const snapshot = await fixture({ sessions: [source({ costEvidence: { observedUsd: 0, estimatedUsd: 0, observedMessages: 0, estimatedMessages: 0, unpricedMessages: 0 } })] }); + assert.equal(snapshot.usage.sessions[0].input, null); +}); +test('should_preserveMeasuredZero_when_usageWasReported', async () => { + const snapshot = await fixture({ sessions: [source({ input: 0 })] }); + assert.equal(snapshot.usage.sessions[0].input, 0); +}); +test('should_distinguishHostIdentities_when_nativeIdsMatch', async () => { + const snapshot = await fixture({ sessions: [source(), source({ host: 'claude' })] }); + assert.equal(new Set(snapshot.usage.sessions.map(x => x.sessionId)).size, 2); +}); +test('should_rejectUnsupportedVersion_when_validating', async () => { + const { validateSnapshot } = await import(contractPath); + assert.throws(() => validateSnapshot({ schemaVersion: 2 }), /schema|contract/i); +}); +test('should_rejectUnknownFields_when_validating', async () => { + const { validateSnapshot } = await import(contractPath); + const snapshot = await fixture(); + assert.throws(() => validateSnapshot({ ...snapshot, secret: 'do not echo' }), /contract/i); +}); +test('should_rejectTampering_when_digestNoLongerMatches', async () => { + const { validateSnapshot } = await import(contractPath); + const snapshot = await fixture(); snapshot.usage.sessions[0].input++; + assert.throws(() => validateSnapshot(snapshot), /digest/i); +}); +test('should_omitArbitraryReceiptFields_when_projecting', async () => { + const snapshot = await fixture({ receipts: [{ id: 'private-receipt', status: 'committed', updatedAt: now, command: 'SECRET', actions: [{ token: 'SECRET' }] }] }); + assert.doesNotMatch(JSON.stringify(snapshot), /private-receipt|SECRET|command/); +}); +test('should_preserveMissingInventory_when_noRetainedScanExists', async () => { + assert.equal((await fixture()).inventory.resources, null); +}); +test('should_publishCanonicalTimestampPattern_when_exposingSchema', async () => { + const { SNAPSHOT_SCHEMA } = await import('../../src/lib/telemetry/schema.mjs'); + assert.equal(typeof SNAPSHOT_SCHEMA.properties.generatedAt.pattern, 'string'); + assert.equal(new RegExp(SNAPSHOT_SCHEMA.properties.generatedAt.pattern).test('2026-09-20T12:00:00Z'), false); +}); +test('should_rejectMalformedAdmission_when_knownFieldsAreHostile', async () => { + const { validateSnapshot, snapshotDigest } = await import(contractPath); + const mutations = [ + s => { s.usage.sessions[0].input = -1; }, + s => { s.usage.sessions[0].responses = 1.5; }, + s => { s.usage.sessions[0].input = Infinity; }, + s => { s.usage.sessions[0].latencyBuckets = [0]; }, + s => { s.usage.sessions[0].latencyBuckets[0] = -1; }, + s => { s.usage.sessions[0].host = 'SECRET'; }, + s => { s.usage.sessions.push(s.usage.sessions[0]); }, + s => { s.usage.state = 'unavailable'; }, + s => { s.inventory.resources = 0; }, + s => { s.inventory.state = 'available'; }, + s => { s.generatedAt = '2026-02-30T12:00:00.000Z'; }, + s => { delete s.selection; }, + s => { s.selection.days = 366; }, + s => { s.usage.sourceHealth.codex = 'SECRET'; }, + s => { s.usage.sessions[0].input = 'SECRET'; }, + s => { s.usage.sessions[0].secret = 'SECRET'; }, + ]; + for (const mutate of mutations) { + const snapshot = await fixture(); mutate(snapshot); snapshot.snapshotId = snapshotDigest(snapshot); + assert.throws(() => validateSnapshot(snapshot), /contract/i); + } +}); +test('should_normalizeUnknownReceiptState_when_sourceUsesFutureStatus', async () => { + const snapshot = await fixture({ receipts: [{ id: 'receipt', status: 'SECRET', updatedAt: 'bad' }] }); + assert.equal(snapshot.maintenance.receipts[0].status, 'unknown'); + assert.equal(snapshot.maintenance.receipts[0].updatedAt, null); +}); +test('should_preserveInventoryCapture_when_retainedSnapshotIsAvailable', async () => { + const snapshot = await fixture({ inventory: { capturedAt: now, resources: [{ secret: 'SECRET' }], placements: [{}, {}] } }); + assert.deepEqual(snapshot.inventory, { state: 'available', capturedAt: now, resources: 1, placements: 2 }); +}); +test('should_rejectInvalidNativeIdentity_when_projectingSources', async () => { + await assert.rejects(fixture({ sessions: [source({ host: 'SECRET' })] }), /identity/); + await assert.rejects(fixture({ sessions: [source({ id: '' })] }), /identity/); + await assert.rejects(fixture({ receipts: [{ id: '', status: 'committed' }] }), /identity/); +}); diff --git a/tests/kit/telemetry-source-bounds.test.mjs b/tests/kit/telemetry-source-bounds.test.mjs new file mode 100644 index 00000000..60cdef82 --- /dev/null +++ b/tests/kit/telemetry-source-bounds.test.mjs @@ -0,0 +1,27 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import zlib from 'node:zlib'; +import { createInventorySnapshotStore } from '../../src/lib/maintenance/management/service-store.mjs'; + +test('should_refuseOversizedInventory_before_readingRetainedBytes', () => { + let reads = 0; + const fsImpl = { ...fs, + lstatSync: () => ({ isFile: () => true, isSymbolicLink: () => false, size: 70 * 1024 * 1024 }), + readFileSync: () => { reads++; throw new Error('unexpected read'); }, + }; + createInventorySnapshotStore(path.join(os.tmpdir(), 'unused'), { fsImpl }).read(); + assert.equal(reads, 0); +}); +test('should_boundDecompression_when_readingCompressedInventory', t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ak-telemetry-bound-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + fs.writeFileSync(path.join(root, 'inventory-latest.json.gz'), zlib.gzipSync('{}')); + const original = zlib.gunzipSync; + let options; + t.mock.method(zlib, 'gunzipSync', (buffer, value) => { options = value; return original(buffer, value); }); + createInventorySnapshotStore(root, { fsImpl: fs }).read(); + assert.equal(options?.maxOutputLength, 64 * 1024 * 1024); +});