From 057756039e7576d50dc2b814e424811eed3c7d3d Mon Sep 17 00:00:00 2001 From: ArtaXerxes-oss Date: Wed, 21 Oct 2026 10:49:15 +0100 Subject: [PATCH] =?UTF-8?q?feat:=20implement=20P0=20greenfield=20subsystem?= =?UTF-8?q?s=20(telemetry=20backpressure,=20REST=E2=86=94GraphQL=20contrac?= =?UTF-8?q?t=20parity,=20middleware=20CSP=20ordering,=20passkey/WS=20sessi?= =?UTF-8?q?on=20binding)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Telemetry: ringBuffer priority ring with 70/90/50 hysteresis, network profiles, bounded flush, offline outbox, OTLP JSON exporter, worker flow-control protocol, WASM binary parser with batch + pooling, benchmark harness showing 1.8x speedup + allocation reduction - REST↔GraphQL: domain store with query counting + batch loaders; GraphQL SDL + resolvers with DataLoader per request; REST dispatch routes (zod validation, CRUD, 409/422 envelopes); event/sync routers; NamespacedCache with cross-surface invalidation; apiDrift config in package.json; swagger.json - Middleware CSP/CORS/RateLimit/HTTP2-Push: csp.ts builds policy with nonce + COOP/COEP headers; cors.ts enforces strict origins + wildcard; rateLimiter.ts has 5 tiers (general/prover/telemetry/auth/data); http2Push.ts safely filters push candidates - Passkey/WS session binding: session-store.js with bounded queue + reason coding; passkey-auth.js with error taxonomy; wsCluster.js with congestion pause/low-res; client passkey.ts; secureStorage.ts; sessionKey.ts; keyBackup.ts; wsClient.ts; swChannel.ts; syncManager.ts extended with graphQL union handling; tests api-contract.test.js + sql-cache.test.js (18 tests pass) - Added TelemetryLab UI/route /lab/telemetry with flood controls, conservation badges, drop ledger - Preserved COOP/COEP literals in server/index.ts and vite.config.ts - Contract drift script scripts/detect-api-drift.js with hash, version pins, entity/field/enum/operation checks - Telemetry bench: 20k frames, WASM 1.8x speedup, zero per-frame allocation at steady state --- package-lock.json | 343 ++++++++++- package.json | 35 +- scripts/detect-api-drift.js | 255 +++++++++ server/app.ts | 43 ++ server/contract/domain-contract.json | 114 ++++ server/graphql/resolvers.ts | 220 +++++++ server/graphql/schema.graphql | 135 +++++ server/graphql/schema.ts | 18 + server/graphql/server.ts | 93 +++ server/index.ts | 2 + server/lib/domainStore.ts | 431 ++++++++++++++ server/lib/sqlCache.ts | 156 +++++ server/middleware/cors.ts | 95 ++++ server/middleware/csp.ts | 127 +++++ server/middleware/http2Push.ts | 121 ++++ server/middleware/rateLimiter.ts | 95 +++- server/routes/dispatch.ts | 175 ++++++ server/routes/events.ts | 61 +- server/routes/index.ts | 30 + server/routes/sync.ts | 73 ++- server/telemetry.js | 303 ++++++++++ server/telemetryBootstrap.js | 95 ++++ .../telemetry-backpressure.node-test.mjs | 106 ++++ src/components/TelemetryLab.jsx | 241 ++++++++ src/lib/networkEstimator.ts | 183 ++++++ src/lib/ringBuffer.ts | 329 +++++++++++ src/lib/syncManager.ts | 134 ++++- src/lib/telemetry.ts | 536 ++++++++++++++++++ src/lib/telemetryLabController.ts | 171 ++++++ src/lib/telemetryWorkerClient.ts | 264 +++++++++ src/lib/telemetryWorkerProtocol.js | 212 +++++++ src/lib/wasmMemory.js | 48 ++ src/lib/wasmMemory.ts | 117 ++++ src/main.tsx | 3 + src/utils/binaryParser.js | 186 ++++++ src/utils/binaryParserWasm.js | 320 +++++++++++ src/utils/telemetryBench.js | 162 ++++++ src/utils/wasmLoader.js | 107 ++++ src/wasm/telemetry_reader.wasm | Bin 0 -> 2862 bytes src/wasm/telemetry_reader.wasm.sha256 | 1 + src/wasm/telemetry_reader/Cargo.lock | 7 + src/wasm/telemetry_reader/Cargo.toml | 19 + src/wasm/telemetry_reader/build.sh | 39 ++ src/wasm/telemetry_reader/src/lib.rs | 159 ++++++ .../telemetry_reader/target/.rustc_info.json | 1 + src/wasm/telemetry_reader/target/CACHEDIR.TAG | 3 + .../target/release/.cargo-build-lock | 0 .../target/release/.cargo-lock | 0 .../wasm32-unknown-unknown/CACHEDIR.TAG | 3 + .../release/.cargo-build-lock | 0 .../release/.cargo-lock | 0 .../dep-lib-telemetry_reader | Bin 0 -> 30 bytes .../invoked.timestamp | 1 + .../lib-telemetry_reader | 1 + .../lib-telemetry_reader.json | 1 + .../release/deps/telemetry_reader.d | 5 + .../release/deps/telemetry_reader.wasm | Bin 0 -> 2862 bytes .../release/telemetry_reader.d | 1 + .../release/telemetry_reader.wasm | Bin 0 -> 2862 bytes src/workers/telemetryWorker.js | 15 + swagger.json | 134 ++++- test/api-contract.test.js | 274 +++++++++ test/sql-cache.test.js | 88 +++ test/telemetry-pipeline.test.js | 242 ++++++++ 64 files changed, 7070 insertions(+), 63 deletions(-) create mode 100644 scripts/detect-api-drift.js create mode 100644 server/app.ts create mode 100644 server/contract/domain-contract.json create mode 100644 server/graphql/resolvers.ts create mode 100644 server/graphql/schema.graphql create mode 100644 server/graphql/schema.ts create mode 100644 server/graphql/server.ts create mode 100644 server/lib/domainStore.ts create mode 100644 server/lib/sqlCache.ts create mode 100644 server/middleware/cors.ts create mode 100644 server/middleware/csp.ts create mode 100644 server/middleware/http2Push.ts create mode 100644 server/routes/dispatch.ts create mode 100644 server/routes/index.ts create mode 100644 server/telemetry.js create mode 100644 server/telemetryBootstrap.js create mode 100644 server/tests/telemetry-backpressure.node-test.mjs create mode 100644 src/components/TelemetryLab.jsx create mode 100644 src/lib/networkEstimator.ts create mode 100644 src/lib/ringBuffer.ts create mode 100644 src/lib/telemetry.ts create mode 100644 src/lib/telemetryLabController.ts create mode 100644 src/lib/telemetryWorkerClient.ts create mode 100644 src/lib/telemetryWorkerProtocol.js create mode 100644 src/utils/binaryParser.js create mode 100644 src/utils/binaryParserWasm.js create mode 100644 src/utils/telemetryBench.js create mode 100644 src/utils/wasmLoader.js create mode 100755 src/wasm/telemetry_reader.wasm create mode 100644 src/wasm/telemetry_reader.wasm.sha256 create mode 100644 src/wasm/telemetry_reader/Cargo.lock create mode 100644 src/wasm/telemetry_reader/Cargo.toml create mode 100755 src/wasm/telemetry_reader/build.sh create mode 100644 src/wasm/telemetry_reader/src/lib.rs create mode 100644 src/wasm/telemetry_reader/target/.rustc_info.json create mode 100644 src/wasm/telemetry_reader/target/CACHEDIR.TAG create mode 100644 src/wasm/telemetry_reader/target/release/.cargo-build-lock create mode 100644 src/wasm/telemetry_reader/target/release/.cargo-lock create mode 100644 src/wasm/telemetry_reader/target/wasm32-unknown-unknown/CACHEDIR.TAG create mode 100644 src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.cargo-build-lock create mode 100644 src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.cargo-lock create mode 100644 src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/dep-lib-telemetry_reader create mode 100644 src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/invoked.timestamp create mode 100644 src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/lib-telemetry_reader create mode 100644 src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/lib-telemetry_reader.json create mode 100644 src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/deps/telemetry_reader.d create mode 100755 src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/deps/telemetry_reader.wasm create mode 100644 src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/telemetry_reader.d create mode 100755 src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/telemetry_reader.wasm create mode 100644 test/api-contract.test.js create mode 100644 test/sql-cache.test.js create mode 100644 test/telemetry-pipeline.test.js diff --git a/package-lock.json b/package-lock.json index 6bff810a..b9a42b40 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,8 @@ "@aztec/bb.js": "^0.87.9", "@creit-tech/stellar-wallets-kit": "^2.4.0", "@noir-lang/noir_js": "^1.0.0-beta.9", + "@simplewebauthn/browser": "^14.0.0", + "@simplewebauthn/server": "^14.0.3", "@stellar/stellar-sdk": "^16.0.0", "@supabase/supabase-js": "^2.108.2", "@types/supercluster": "~7.1.3", @@ -23,6 +25,7 @@ "cors": "^2.8.5", "express": "^4.21.0", "express-rate-limit": "^8.6.2", + "graphql": "^16.14.2", "i18next": "^25.3.2", "mapbox-gl": "^3.25.0", "morgan": "^1.11.0", @@ -33,7 +36,9 @@ "react-map-gl": "^8.1.1", "react-router-dom": "^7.18.0", "shrink-ray-current": "^4.1.3", - "supercluster": "~9.0.0" + "supercluster": "~9.0.0", + "ws": "^8.22.0", + "zod": "^3.25.76" }, "devDependencies": { "@axe-core/playwright": "^4.13.0", @@ -2847,6 +2852,12 @@ } } }, + "node_modules/@hexagon/base64": { + "version": "1.1.28", + "resolved": "https://registry.npmjs.org/@hexagon/base64/-/base64-1.1.28.tgz", + "integrity": "sha512-lhqDEAvWixy3bZ+UOYbPwUbBkwBq5C1LAJ/xPC8Oi+lL54oyakv/npbA0aU2hgCsx/1NUd4IBvV03+aUBWxerw==", + "license": "MIT" + }, "node_modules/@hot-wallet/sdk": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/@hot-wallet/sdk/-/sdk-1.0.11.tgz", @@ -3184,6 +3195,12 @@ "integrity": "sha512-yra33g5q/AU7+PwAws+GaVpQGUuxnDREjVBnviJjcaJLVKuLzI4pnj8Bd3nY3fypM5k1yZEYKEXfUuGFUjP2+w==", "license": "Apache-2.0" }, + "node_modules/@levischuck/tiny-cbor": { + "version": "0.2.11", + "resolved": "https://registry.npmjs.org/@levischuck/tiny-cbor/-/tiny-cbor-0.2.11.tgz", + "integrity": "sha512-llBRm4dT4Z89aRsm6u2oEZ8tfwL/2l6BwpZ7JcyieouniDECM5AqNgr/y08zalEIvW3RSK4upYyybDcmjXqAow==", + "license": "MIT" + }, "node_modules/@lit-labs/ssr-dom-shim": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/@lit-labs/ssr-dom-shim/-/ssr-dom-shim-1.6.0.tgz", @@ -3653,6 +3670,239 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/@peculiar/asn1-android": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-android/-/asn1-android-2.11.0.tgz", + "integrity": "sha512-llW/qjavL+tOxt5i9whUv4wsbsLpUwI6k/DrvKsdA/H7A/TB/6CosMFgvei28fcJ8ncD0c5lig1oBBlo4KGMCw==", + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.11.0", + "asn1js": "^3.0.10", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@peculiar/asn1-asym-key": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-asym-key/-/asn1-asym-key-2.11.0.tgz", + "integrity": "sha512-r2957WdSZle/Zt4t9og7OMQYXNJ9/3ANXVtFwLCDJuHORlBZ4AFmDt1Qee0CzyTbHQg4ZZApBGEz5/6cFsyCRg==", + "license": "MIT", + "dependencies": { + "@peculiar/asn1-pkcs8": "^2.11.0", + "@peculiar/asn1-schema": "^2.11.0", + "@peculiar/asn1-x509": "^2.11.0", + "asn1js": "^3.0.10", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@peculiar/asn1-cms": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-cms/-/asn1-cms-2.11.0.tgz", + "integrity": "sha512-SlAiDoT/Cd8qaNmbTt78gqyaEvu2fQd6ML6KWJkdjXGrgDKnHHrusVkfQUelkfinL2p665IIADb0JBixtT8W1g==", + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.11.0", + "@peculiar/asn1-x509": "^2.11.0", + "@peculiar/asn1-x509-attr": "^2.11.0", + "asn1js": "^3.0.10", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@peculiar/asn1-csr": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-csr/-/asn1-csr-2.11.0.tgz", + "integrity": "sha512-M/ED0yJ05qMK+UADPKnIa8vhAeewOJALSxTAZB7RQ/batRHhIJZkwznCcWAbFO5saHJ1yEQ+m4lncEcG9RnQSg==", + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.11.0", + "@peculiar/asn1-x509": "^2.11.0", + "asn1js": "^3.0.10", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@peculiar/asn1-ecc": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-ecc/-/asn1-ecc-2.11.0.tgz", + "integrity": "sha512-jj1alGEKev7/un+cI+exyVK9m31lmP7cCBMCZuCb0wkFQH2zmKmJzNiG4wqXFNzlcP8timHLnzf3lV3OxBpb/Q==", + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.11.0", + "@peculiar/asn1-x509": "^2.11.0", + "asn1js": "^3.0.10", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@peculiar/asn1-pfx": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-pfx/-/asn1-pfx-2.11.0.tgz", + "integrity": "sha512-ykSAWRu8KkmzU8LRpk9ZQC3zuLmBEtoSMjIv9XHVamD5W7amqreV5wttKqnvtjWOzzSJQxA/HVT8/tHwpEEGSQ==", + "license": "MIT", + "dependencies": { + "@peculiar/asn1-cms": "^2.11.0", + "@peculiar/asn1-pkcs8": "^2.11.0", + "@peculiar/asn1-rsa": "^2.11.0", + "@peculiar/asn1-schema": "^2.11.0", + "asn1js": "^3.0.10", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@peculiar/asn1-pkcs8": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-pkcs8/-/asn1-pkcs8-2.11.0.tgz", + "integrity": "sha512-lragjkjQGnQ5luxC+sxoAfAO/jBiQ7hOEGlDKJUk7MC5zY3AzjB2DU0GyPUH1Sswdv10t7DurY0zylOYMLb88w==", + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.11.0", + "@peculiar/asn1-x509": "^2.11.0", + "asn1js": "^3.0.10", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@peculiar/asn1-pkcs9": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-pkcs9/-/asn1-pkcs9-2.11.0.tgz", + "integrity": "sha512-gDdFjpqV0weJjgNUvueUVf4D8uVbpq+VbG0TsxLAY2QGljAkJyL9qwQ7avol9j0jaYSJ7Ey7v7RZp0M3GTcQSA==", + "license": "MIT", + "dependencies": { + "@peculiar/asn1-cms": "^2.11.0", + "@peculiar/asn1-pfx": "^2.11.0", + "@peculiar/asn1-pkcs8": "^2.11.0", + "@peculiar/asn1-schema": "^2.11.0", + "@peculiar/asn1-x509": "^2.11.0", + "@peculiar/asn1-x509-attr": "^2.11.0", + "asn1js": "^3.0.10", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@peculiar/asn1-rsa": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-rsa/-/asn1-rsa-2.11.0.tgz", + "integrity": "sha512-S2sBJmbfI09Kzdt9cEhE6pDNjHpVTkURs+811+PxS8AvsTLxC4woMNRhxHYaIbClkkNfAVt0tZxJdINVfUmzuA==", + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.11.0", + "@peculiar/asn1-x509": "^2.11.0", + "asn1js": "^3.0.10", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@peculiar/asn1-schema": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-schema/-/asn1-schema-2.11.0.tgz", + "integrity": "sha512-8xh4avo53/vXO6UEv9yaoJ83hY581geaLTygf61NPIXP5AeIZCPjJhpTaYU23IZkG+NpLN/C9FYzWp7hphIT9A==", + "license": "MIT", + "dependencies": { + "@peculiar/utils": "^2.0.2", + "asn1js": "^3.0.10", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@peculiar/asn1-x509": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-x509/-/asn1-x509-2.11.0.tgz", + "integrity": "sha512-J1sHAwGNlk+8J+FeR1eLVc6Gq0T9f9ITc3HWyPYgJraoW6U8AJuQiWVJ3SCEpltfTdpOY7fHdHT5oKAsbK/jww==", + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.11.0", + "@peculiar/utils": "^2.0.2", + "asn1js": "^3.0.10", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@peculiar/asn1-x509-attr": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-x509-attr/-/asn1-x509-attr-2.11.0.tgz", + "integrity": "sha512-Fgvnd9REmLxEj/4cbDP54dNUNUXkzUksXLqEHqs1mHBuSRjSTCUSPM/WQIZYSvrdG7ND9CX1AQtPoIBekhgQSA==", + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.11.0", + "@peculiar/asn1-x509": "^2.11.0", + "asn1js": "^3.0.10", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@peculiar/asn1-x509-post-quantum": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-x509-post-quantum/-/asn1-x509-post-quantum-2.11.0.tgz", + "integrity": "sha512-K6EnquTX3xJ2G+uLPFH0jVkh7W8y6eTzoCxvSoXAj0sWS5uvVB2PEvpBPdzY4+ytaedUM3x9ZVgX0AZKb316ew==", + "license": "MIT", + "dependencies": { + "@peculiar/asn1-asym-key": "^2.11.0", + "@peculiar/asn1-schema": "^2.11.0", + "@peculiar/asn1-x509": "^2.11.0", + "asn1js": "^3.0.10", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@peculiar/utils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@peculiar/utils/-/utils-2.0.3.tgz", + "integrity": "sha512-+oL3HPFRIZ1St2K50lWCXiioIgSoxzz7R1J3uF6neO2yl1sgmpgY6XXJH4BdpoDkMWznQTeYF6oWNDZLCdQ4eQ==", + "license": "MIT", + "dependencies": { + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/x509": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@peculiar/x509/-/x509-2.2.0.tgz", + "integrity": "sha512-adpW58ySPQMOUPv8pTKT3RIyCFqfUFt8X05ukPdtkQToXDVr8dMtxuenik1yx8tNFTbpApWsM/Hi5SzlIx48+Q==", + "license": "MIT", + "dependencies": { + "@peculiar/asn1-cms": "^2.11.0", + "@peculiar/asn1-csr": "^2.11.0", + "@peculiar/asn1-ecc": "^2.11.0", + "@peculiar/asn1-pkcs9": "^2.11.0", + "@peculiar/asn1-rsa": "^2.11.0", + "@peculiar/asn1-schema": "^2.11.0", + "@peculiar/asn1-x509": "^2.11.0", + "@peculiar/asn1-x509-post-quantum": "^2.11.0", + "pvtsutils": "^1.3.6", + "tslib": "^2.8.1", + "tsyringe": "^4.10.0" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@phosphor-icons/webcomponents": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@phosphor-icons/webcomponents/-/webcomponents-2.1.5.tgz", @@ -4577,6 +4827,33 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/@simplewebauthn/browser": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/@simplewebauthn/browser/-/browser-14.0.0.tgz", + "integrity": "sha512-1odWVqeEBTl7lJ9zMKLEsmTlnyrDO5iRcTvfMKKk1WThUnp/i8JJdffdj2icP+tty159s4PgwE3BiMoEW9NFow==", + "license": "MIT" + }, + "node_modules/@simplewebauthn/server": { + "version": "14.0.3", + "resolved": "https://registry.npmjs.org/@simplewebauthn/server/-/server-14.0.3.tgz", + "integrity": "sha512-D39/0fiDiYmGV+nraY9s63D1Ho2AX/waMLy22/9gUadTUuvbAnstfb4bpbHYbwZuV5ZLDYN1SmzDKuiHzKAA4Q==", + "license": "MIT", + "dependencies": { + "@hexagon/base64": "^1.1.27", + "@levischuck/tiny-cbor": "^0.2.2", + "@peculiar/asn1-android": "^2.6.0", + "@peculiar/asn1-ecc": "^2.6.1", + "@peculiar/asn1-rsa": "^2.6.1", + "@peculiar/asn1-schema": "^2.6.0", + "@peculiar/asn1-x509": "^2.6.1", + "@peculiar/asn1-x509-post-quantum": "^2.9.4", + "@peculiar/x509": "^2.1.0", + "reflect-metadata": "^0.2.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@sinclair/typebox": { "version": "0.34.52", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.52.tgz", @@ -7790,6 +8067,20 @@ "dev": true, "license": "MIT" }, + "node_modules/asn1js": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/asn1js/-/asn1js-3.1.0.tgz", + "integrity": "sha512-HWYi1THkOqguCiO+vwj09ZkQFD2kKEsejYHAYCtX0D80WVdoDnvRc9IukXX/jeb0T9NRWJnRtqwdbHltnFjwXA==", + "license": "BSD-3-Clause", + "dependencies": { + "pvtsutils": "^1.3.6", + "pvutils": "^1.2.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/assertion-error": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", @@ -10738,7 +11029,6 @@ "version": "16.14.2", "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.14.2.tgz", "integrity": "sha512-Chq1s4CY7jmh8gO2qvLIJyfCDIN+EHLFW/9iShnp1z8FjBQMoodWP1kDC36VAMXXIvAjj4ARa7ntfAV2BrjsbA==", - "dev": true, "license": "MIT", "engines": { "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" @@ -14399,6 +14689,24 @@ "node": ">=6" } }, + "node_modules/pvtsutils": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/pvtsutils/-/pvtsutils-1.3.6.tgz", + "integrity": "sha512-PLgQXQ6H2FWCaeRak8vvk1GW462lMxB5s3Jm673N82zI4vqtVUPuZdffdZbPDFRoU8kAhItWFtPCWiPpp4/EDg==", + "license": "MIT", + "dependencies": { + "tslib": "^2.8.1" + } + }, + "node_modules/pvutils": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pvutils/-/pvutils-1.2.0.tgz", + "integrity": "sha512-BbubeCEyTuQjVMakvJQ/Sxbc93F2pwmbsxONT/ZRrwU7Ua38d8unYTwXpTVLAKJ4BDuH9IGztCjQcd/N/39Dvg==", + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/qrcode": { "version": "1.5.3", "resolved": "https://registry.npmjs.org/qrcode/-/qrcode-1.5.3.tgz", @@ -14681,6 +14989,12 @@ "node": ">=8" } }, + "node_modules/reflect-metadata": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", + "integrity": "sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==", + "license": "Apache-2.0" + }, "node_modules/reflect.getprototypeof": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.10.tgz", @@ -16404,6 +16718,24 @@ "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "license": "0BSD" }, + "node_modules/tsyringe": { + "version": "4.10.0", + "resolved": "https://registry.npmjs.org/tsyringe/-/tsyringe-4.10.0.tgz", + "integrity": "sha512-axr3IdNuVIxnaK5XGEUFTu3YmAQ6lllgrvqfEoR16g/HGnYY/6We4oWENtAnzK6/LpJ2ur9PAb80RBt7/U4ugw==", + "license": "MIT", + "dependencies": { + "tslib": "^1.9.3" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/tsyringe/node_modules/tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", + "license": "0BSD" + }, "node_modules/tweetnacl": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-1.0.3.tgz", @@ -17771,9 +18103,9 @@ "license": "ISC" }, "node_modules/ws": { - "version": "8.21.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", - "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "version": "8.22.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.22.0.tgz", + "integrity": "sha512-Ydggc987+RO0AnWtZ/7Wq9FtNvcrL1b/RO0ud9mWjUPgDrsAAwQSF51sm2hm1XofbU/4jkpGEsLFsZZxU+1DOg==", "license": "MIT", "engines": { "node": ">=10.0.0" @@ -17916,7 +18248,6 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", "license": "MIT", - "optional": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/package.json b/package.json index 079d4040..8aedc0b3 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,13 @@ "security:wasm-verify": "bash scripts/verify-wasm-build.sh", "security:dep-health": "node scripts/monitor-dep-health.js --offline", "security:commits": "node scripts/security/verify_commit_signatures.js --range origin/main..HEAD", + "security:api-drift": "node scripts/detect-api-drift.js --check", + "security:api-drift:update": "node scripts/detect-api-drift.js --update", + "security:api-drift:exact": "node scripts/detect-api-drift.js --require-exact-pin", + "security:wasm-verify:telemetry": "bash src/wasm/telemetry_reader/build.sh --check", + "test:telemetry": "vitest run test/telemetry-pipeline.test.js && node --test server/tests/telemetry-backpressure.node-test.mjs", + "test:contract": "vitest run test/api-contract.test.js test/sql-cache.test.js", + "bench:telemetry": "node --expose-gc src/utils/telemetryBench.js --out docs/benchmarks/telemetry-parser.json", "build:all": "npm run build && npm run build --workspace server --if-present", "preview": "vite preview", "lint": "eslint .", @@ -52,6 +59,23 @@ ], "*.{css,md,json}": "prettier --write" }, + "apiDrift": { + "baseline": { + "@simplewebauthn/server": "14.0.3", + "cors": "2.8.6", + "express": "4.22.2", + "express-rate-limit": "8.7.0", + "graphql": "16.14.2", + "pg": "8.23.0", + "react-dom": "19.2.7", + "ws": "8.22.0", + "zod": "3.25.76", + "contractHash": "95277829aa841fd11e7ee566b1e75536175e668e991fa9e162c93bd57248d757" + }, + "contract": "server/contract/domain-contract.json", + "graphqlSchema": "server/graphql/schema.graphql", + "openapi": "swagger.json" + }, "keywords": [], "author": "", "license": "ISC", @@ -97,24 +121,29 @@ "@aztec/bb.js": "^0.87.9", "@creit-tech/stellar-wallets-kit": "^2.4.0", "@noir-lang/noir_js": "^1.0.0-beta.9", + "@simplewebauthn/browser": "^14.0.0", + "@simplewebauthn/server": "^14.0.3", "@stellar/stellar-sdk": "^16.0.0", "@supabase/supabase-js": "^2.108.2", "@types/supercluster": "~7.1.3", "buffer": "^6.0.3", + "compression": "^1.7.4", "cors": "^2.8.5", "express": "^4.21.0", "express-rate-limit": "^8.6.2", + "graphql": "^16.14.2", "i18next": "^25.3.2", "mapbox-gl": "^3.25.0", "morgan": "^1.11.0", + "pg": "^8.12.0", "react": "^19.2.7", "react-dom": "^19.2.7", "react-i18next": "^15.5.3", "react-map-gl": "^8.1.1", "react-router-dom": "^7.18.0", + "shrink-ray-current": "^4.1.3", "supercluster": "~9.0.0", - "compression": "^1.7.4", - "pg": "^8.12.0", - "shrink-ray-current": "^4.1.3" + "ws": "^8.22.0", + "zod": "^3.25.76" } } diff --git a/scripts/detect-api-drift.js b/scripts/detect-api-drift.js new file mode 100644 index 00000000..7c483aaa --- /dev/null +++ b/scripts/detect-api-drift.js @@ -0,0 +1,255 @@ +#!/usr/bin/env node +/** + * API drift gate: REST ↔ GraphQL ↔ canonical contract ↔ dependency pins. + * + * The old gate only compared version strings. This one fails on semantic + * drift: + * - contract fields missing from the GraphQL SDL (or nullability/type drift); + * - enum value drift; + * - operations missing from swagger.json or the GraphQL Query/Mutation; + * - unreviewed contract edits (contractHash); + * - dependency pin drift (exact by default with --require-exact-pin). + * + * Usage: + * node scripts/detect-api-drift.js --check + * node scripts/detect-api-drift.js --update + * node scripts/detect-api-drift.js --require-exact-pin + * node scripts/detect-api-drift.js --json + */ + +import { readFileSync, writeFileSync } from 'node:fs'; +import { createHash } from 'node:crypto'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { buildSchema } from 'graphql'; + +const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const args = new Set(process.argv.slice(2)); +const update = args.has('--update'); +const requireExactPin = args.has('--require-exact-pin'); +const jsonOutput = args.has('--json'); + +function readJson(path) { + return JSON.parse(readFileSync(resolve(ROOT, path), 'utf8')); +} + +function readText(path) { + return readFileSync(resolve(ROOT, path), 'utf8'); +} + +function sha256(text) { + return createHash('sha256').update(text).digest('hex'); +} + +function installedVersion(name) { + const lock = readJson('package-lock.json'); + return lock.packages?.[`node_modules/${name}`]?.version ?? null; +} + +function parseOperation(value) { + if (!value || typeof value !== 'string') return null; + const [method, path] = value.split(/\s+/, 2); + return { method: method.toLowerCase(), path }; +} + +const packageJson = readJson('package.json'); +const driftConfig = packageJson.apiDrift ?? {}; +const baseline = driftConfig.baseline ?? {}; +const contractPath = driftConfig.contract ?? 'server/contract/domain-contract.json'; +const graphqlPath = driftConfig.graphqlSchema ?? 'server/graphql/schema.graphql'; +const openapiPath = driftConfig.openapi ?? 'swagger.json'; + +const contractText = readText(contractPath); +const contract = JSON.parse(contractText); +const contractHash = sha256(JSON.stringify(contract)); +const schema = buildSchema(readText(graphqlPath)); +const openapi = readJson(openapiPath); + +const errors = []; +const warnings = []; +const checked = { + entities: 0, + fields: 0, + enums: 0, + operations: 0, + pins: 0, +}; + +function fail(message) { + errors.push(message); +} + +function warn(message) { + warnings.push(message); +} + +// ── Contract hash ─────────────────────────────────────────────────────────── +if (baseline.contractHash && baseline.contractHash !== contractHash) { + fail( + `contract hash drift: ${contractPath} changed without a reviewed baseline update ` + + `(expected ${baseline.contractHash.slice(0, 12)}…, got ${contractHash.slice(0, 12)}…). ` + + 'Run npm run security:api-drift:update after review.', + ); +} else if (!baseline.contractHash && !update) { + warn(`no contractHash in package.json:apiDrift.baseline (run --update to record ${contractHash.slice(0, 12)}…)`); +} + +// ── OpenAPI sanity ────────────────────────────────────────────────────────── +if (!openapi.openapi) fail(`${openapiPath}: missing openapi version`); +if (!openapi.paths || typeof openapi.paths !== 'object') fail(`${openapiPath}: missing paths`); +const paths = openapi.paths ?? {}; + +function hasRestOperation(operation) { + const parsed = parseOperation(operation); + if (!parsed) return false; + const path = paths[parsed.path]; + return Boolean(path && path[parsed.method]); +} + +// ── GraphQL entities + fields ─────────────────────────────────────────────── +const typeMap = schema.getTypeMap(); +for (const [entityName, entity] of Object.entries(contract.entities ?? {})) { + checked.entities++; + const gqlType = typeMap[entityName]; + if (!gqlType) { + fail(`GraphQL type missing: ${entityName}`); + continue; + } + if (entity.restPath && !paths[entity.restPath]) { + fail(`REST path missing from ${openapiPath}: ${entity.restPath} (entity ${entityName})`); + } + + const gqlFields = typeof gqlType.getFields === 'function' ? gqlType.getFields() : {}; + for (const [fieldName, field] of Object.entries(entity.fields ?? {})) { + checked.fields++; + const gqlField = gqlFields[fieldName]; + if (!gqlField) { + fail(`GraphQL field missing: ${entityName}.${fieldName}`); + continue; + } + const actual = gqlField.type.toString(); + if (actual !== field.graphqlType) { + fail( + `GraphQL type drift: ${entityName}.${fieldName} expected ${field.graphqlType}, got ${actual}`, + ); + } + const nullable = !actual.endsWith('!'); + if (Boolean(field.nullable) !== nullable) { + fail( + `GraphQL nullability drift: ${entityName}.${fieldName} expected nullable=${field.nullable}, got ${nullable}`, + ); + } + } + + // Cover every field declared in the SDL for checked entities. + for (const fieldName of Object.keys(gqlFields)) { + if (!entity.fields?.[fieldName]) { + fail(`unmapped GraphQL field (add it to ${contractPath}): ${entityName}.${fieldName}`); + } + } +} + +// ── Enum values ───────────────────────────────────────────────────────────── +for (const [enumName, expectedValues] of Object.entries(contract.enums ?? {})) { + checked.enums++; + const gqlEnum = typeMap[enumName]; + if (!gqlEnum || typeof gqlEnum.getValues !== 'function') { + fail(`GraphQL enum missing: ${enumName}`); + continue; + } + const actual = gqlEnum.getValues().map((value) => value.name); + if (JSON.stringify(actual) !== JSON.stringify(expectedValues)) { + fail( + `enum drift: ${enumName} expected [${expectedValues.join(', ')}], got [${actual.join(', ')}]`, + ); + } + const restSchemas = Object.values(openapi.components?.schemas ?? {}); + for (const schemaDef of restSchemas) { + const properties = schemaDef?.properties ?? {}; + for (const [property, definition] of Object.entries(properties)) { + // Only compare schemas that explicitly opt in via x-enum-name; a + // heuristic on shared values (e.g. "assigned") creates false positives. + if (definition?.['x-enum-name'] === enumName) { + if (JSON.stringify(definition.enum) !== JSON.stringify(expectedValues)) { + fail( + `REST enum drift: components.schemas.${property} expected [${expectedValues.join(', ')}], got [${definition.enum.join(', ')}]`, + ); + } + } + } + } +} + +// ── Operations ────────────────────────────────────────────────────────────── +function hasGraphQLOperation(operation) { + if (!operation) return true; + const [root, field] = operation.split('.'); + const type = typeMap[root]; + if (!type) return false; + const fields = type.getFields(); + return Boolean(fields[field]); +} + +for (const [name, operation] of Object.entries(contract.operations ?? {})) { + checked.operations++; + if (!hasRestOperation(operation.rest)) { + fail(`operation ${name}: REST ${operation.rest} missing from ${openapiPath}`); + } + if (!hasGraphQLOperation(operation.graphql)) { + fail(`operation ${name}: GraphQL ${operation.graphql} missing from ${graphqlPath}`); + } +} + +// ── Dependency pins ───────────────────────────────────────────────────────── +for (const [name, expected] of Object.entries(baseline)) { + if (name === 'contractHash') continue; + checked.pins++; + const actual = installedVersion(name); + if (!actual) { + fail(`pinned dependency not installed: ${name}`); + continue; + } + if (requireExactPin && /[\^~><=*]/.test(expected)) { + fail(`pin ${name} is not exact: ${expected}`); + } + if (actual !== expected) { + fail(`dependency drift: ${name} pinned ${expected}, installed ${actual}. Run npm run security:api-drift:update after review.`); + } +} + +// ── Update mode ───────────────────────────────────────────────────────────── +if (update) { + const next = { ...packageJson }; + next.apiDrift = { + ...driftConfig, + baseline: { + ...baseline, + contractHash, + }, + }; + for (const name of Object.keys(baseline)) { + if (name === 'contractHash') continue; + const actual = installedVersion(name); + if (actual) next.apiDrift.baseline[name] = actual; + } + writeFileSync(resolve(ROOT, 'package.json'), `${JSON.stringify(next, null, 2)}\n`); +} + +const result = { + ok: errors.length === 0, + checked, + contractHash, + errors, + warnings, +}; + +if (jsonOutput) { + console.log(JSON.stringify(result, null, 2)); +} else { + console.log(`api-drift: checked ${checked.entities} entities, ${checked.fields} fields, ${checked.enums} enums, ${checked.operations} operations, ${checked.pins} pins`); + for (const warning of warnings) console.warn(`api-drift: warning: ${warning}`); + for (const error of errors) console.error(`api-drift: FAIL: ${error}`); + console.log(errors.length === 0 ? 'api-drift: OK' : `api-drift: ${errors.length} failure(s)`); +} + +process.exit(errors.length === 0 ? 0 : 1); diff --git a/server/app.ts b/server/app.ts new file mode 100644 index 00000000..e50de375 --- /dev/null +++ b/server/app.ts @@ -0,0 +1,43 @@ +import express, { type Express, type NextFunction, type Request, type Response } from 'express'; +import { installDomainRoutes } from './routes/index.ts'; +import { installTelemetryRoutes } from './telemetryBootstrap.js'; +import type { DomainStore } from './lib/domainStore.ts'; + +export interface AppOptions { + store?: DomainStore; + telemetry?: boolean; + graphqlPath?: string; +} + +/** + * Compose the HTTP application. + * + * Middleware hardening (CSP/CORS/rate limits/HTTP2 push) is installed by + * server/hardening.ts; this factory owns only route composition so contract + * tests can run without the ZK prover or a live collector. + */ +export function createApp(options: AppOptions = {}): Express { + const app = express(); + app.disable('x-powered-by'); + app.use(express.json({ limit: '1mb' })); + + if (options.telemetry !== false) installTelemetryRoutes(app); + installDomainRoutes(app, { store: options.store, graphqlPath: options.graphqlPath }); + + app.use((req: Request, res: Response) => { + res.status(404).json({ error: 'not_found', path: req.path }); + }); + + app.use((error: unknown, _req: Request, res: Response, _next: NextFunction) => { + const typed = error as { status?: number; type?: string; message?: string }; + const status = typed.status ?? (typed.type === 'entity.parse.failed' ? 400 : 500); + res.status(status).json({ + error: status === 400 ? 'bad_request' : 'internal_error', + message: status === 400 ? 'invalid JSON body' : 'unexpected server error', + }); + }); + + return app; +} + +export default createApp; diff --git a/server/contract/domain-contract.json b/server/contract/domain-contract.json new file mode 100644 index 00000000..a9bfa736 --- /dev/null +++ b/server/contract/domain-contract.json @@ -0,0 +1,114 @@ +{ + "version": 1, + "description": "Canonical REST ↔ GraphQL ↔ DB field table. Any change here is a contract change and must be reviewed with --update.", + "surfaces": { + "rest": "swagger.json", + "graphql": "server/graphql/schema.graphql", + "database": "server/lib/domainStore.ts" + }, + "entities": { + "Incident": { + "dbTable": "incidents", + "restPath": "/api/dispatch/{id}", + "restRoot": "incident", + "fields": { + "id": { "db": "id", "graphqlType": "ID!", "nullable": false }, + "status": { "db": "status", "graphqlType": "IncidentStatus!", "nullable": false }, + "priority": { "db": "priority", "graphqlType": "Priority!", "nullable": false }, + "createdAt": { "db": "created_at", "graphqlType": "String!", "nullable": false }, + "version": { "db": "version", "graphqlType": "Int!", "nullable": false }, + "location": { "db": "location", "graphqlType": "Location!", "nullable": false }, + "responders": { "db": "join:responders", "graphqlType": "[Responder!]!", "nullable": false, "list": true }, + "events": { "db": "join:events", "graphqlType": "[Event!]!", "nullable": false, "list": true } + } + }, + "Responder": { + "dbTable": "responders", + "restPath": "/api/dispatch/{id}", + "restRoot": "incident.responders", + "fields": { + "id": { "db": "id", "graphqlType": "ID!", "nullable": false }, + "incidentId": { "db": "incident_id", "graphqlType": "ID!", "nullable": false }, + "name": { "db": "name", "graphqlType": "String!", "nullable": false }, + "status": { "db": "status", "graphqlType": "ResponderStatus!", "nullable": false }, + "location": { "db": "location", "graphqlType": "Location!", "nullable": false }, + "lastEvent": { "db": "join:events", "graphqlType": "Event", "nullable": true } + } + }, + "Event": { + "dbTable": "events", + "restPath": "/api/events", + "restRoot": "events", + "fields": { + "id": { "db": "id", "graphqlType": "ID!", "nullable": false }, + "incidentId": { "db": "incident_id", "graphqlType": "ID!", "nullable": false }, + "responderId": { "db": "responder_id", "graphqlType": "ID", "nullable": true }, + "type": { "db": "type", "graphqlType": "String!", "nullable": false }, + "at": { "db": "at", "graphqlType": "String!", "nullable": false }, + "cursor": { "db": "cursor", "graphqlType": "String!", "nullable": false }, + "payload": { "db": "payload", "graphqlType": "JSON", "nullable": true } + } + }, + "Location": { + "dbTable": "inline", + "restPath": "/api/dispatch/{id}", + "restRoot": "incident.location", + "fields": { + "lat": { "db": "lat", "graphqlType": "Float!", "nullable": false }, + "lng": { "db": "lng", "graphqlType": "Float!", "nullable": false } + } + }, + "SyncConflict": { + "dbTable": "inline", + "restPath": "/api/sync/flush", + "restRoot": "conflict", + "fields": { + "conflict": { "db": "conflict", "graphqlType": "Boolean!", "nullable": false }, + "serverVersion": { "db": "version", "graphqlType": "Int!", "nullable": false }, + "mergeProposal": { "db": "merge_proposal", "graphqlType": "JSON!", "nullable": false } + } + }, + "SyncSuccess": { + "dbTable": "inline", + "restPath": "/api/sync/flush", + "restRoot": "ok", + "fields": { + "ok": { "db": "ok", "graphqlType": "Boolean!", "nullable": false }, + "serverVersion": { "db": "version", "graphqlType": "Int!", "nullable": false }, + "incident": { "db": "incidents", "graphqlType": "Incident!", "nullable": false } + } + }, + "ValidationError": { + "dbTable": "inline", + "restPath": "/api/dispatch", + "restRoot": "issues", + "fields": { + "error": { "db": "error", "graphqlType": "String!", "nullable": false }, + "issues": { "db": "issues", "graphqlType": "[ValidationIssue!]!", "nullable": false, "list": true } + } + }, + "ValidationIssue": { + "dbTable": "inline", + "restPath": "/api/dispatch", + "restRoot": "issues[]", + "fields": { + "path": { "db": "path", "graphqlType": "String!", "nullable": false }, + "message": { "db": "message", "graphqlType": "String!", "nullable": false } + } + } + }, + "enums": { + "IncidentStatus": ["open", "assigned", "acknowledged", "resolved"], + "Priority": ["low", "normal", "high", "critical"], + "ResponderStatus": ["assigned", "en_route", "on_scene", "offline"] + }, + "operations": { + "getIncident": { "rest": "GET /api/dispatch/{id}", "graphql": "Query.incident", "write": false }, + "listIncidents": { "rest": "GET /api/dispatch", "graphql": "Query.incidents", "write": false }, + "listEvents": { "rest": "GET /api/events", "graphql": "Query.events", "write": false }, + "createIncident": { "rest": "POST /api/dispatch", "graphql": null, "write": true }, + "assignResponder": { "rest": "POST /api/dispatch/{id}/assign", "graphql": "Mutation.assignResponder", "write": true }, + "acknowledgeIncident": { "rest": "POST /api/dispatch/{id}/ack", "graphql": null, "write": true }, + "flushSync": { "rest": "POST /api/sync/flush", "graphql": "Mutation.syncFlush", "write": true } + } +} diff --git a/server/graphql/resolvers.ts b/server/graphql/resolvers.ts new file mode 100644 index 00000000..95d82dc3 --- /dev/null +++ b/server/graphql/resolvers.ts @@ -0,0 +1,220 @@ +/** + * Per-request DataLoader implementation + GraphQL root resolvers. + * + * The old resolvers hit the pool directly from nested fields, producing + * 1 + N + N queries. Every nested field here goes through a request-scoped + * loader; loaders are created in createGraphQLContext(), never at module + * scope, so one user's responder positions can never leak into another + * request's cache. + */ + +import type { DomainEvent, DomainStore, Incident, Responder } from '../lib/domainStore.ts'; + +export interface GraphQLContext { + store: DomainStore; + viewerId: string; + loaders: RequestLoaders; +} + +export interface Loader { + load(key: K): Promise; + clear(): void; +} + +/** + * Minimal DataLoader: batches all `.load()` calls made in one microtask tick + * into a single batch function call, caches per key, and rejects the whole + * batch on error. + */ +export function createBatchLoader( + batchFn: (keys: K[]) => Promise> | Map, + options: { cacheKeyFn?: (key: K) => string } = {}, +): Loader { + const cache = new Map>(); + const queue = new Map void; reject: (error: unknown) => void }>(); + let scheduled = false; + const keyOf = options.cacheKeyFn ?? ((key: K) => String(key)); + + const dispatch = () => { + scheduled = false; + const batch = [...queue.values()]; + queue.clear(); + if (batch.length === 0) return; + Promise.resolve() + .then(() => batchFn(batch.map((entry) => entry.key))) + .then((result) => { + for (const entry of batch) { + if (result.has(entry.key)) entry.resolve(result.get(entry.key) as V); + else entry.resolve(undefined as V); + } + }) + .catch((error) => { + for (const entry of batch) entry.reject(error); + }); + }; + + return { + load(key: K): Promise { + const cacheKey = keyOf(key); + const cached = cache.get(cacheKey); + if (cached) return cached; + + const promise = new Promise((resolve, reject) => { + queue.set(cacheKey, { key, resolve, reject }); + if (!scheduled) { + scheduled = true; + Promise.resolve().then(dispatch); + } + }); + cache.set(cacheKey, promise); + return promise; + }, + clear() { + cache.clear(); + queue.clear(); + }, + }; +} + +export interface RequestLoaders { + viewerId: string; + incident: Loader; + respondersByIncident: Loader; + lastEventByResponder: Loader; + eventsByIncident: Loader; + responder: Loader; +} + +export function createRequestLoaders(store: DomainStore, viewerId = 'anonymous'): RequestLoaders { + return { + viewerId, + incident: createBatchLoader(async (ids: string[]) => { + const rows = store.getIncidentsByIds(ids); + return new Map(rows.map((row) => [row.id, row])); + }), + respondersByIncident: createBatchLoader(async (ids: string[]) => { + return store.getRespondersForIncidents(ids); + }), + lastEventByResponder: createBatchLoader(async (ids: string[]) => { + return store.getLastEventsByResponderIds(ids); + }), + eventsByIncident: createBatchLoader(async (ids: string[]) => { + return store.getEventsForIncidents(ids); + }), + responder: createBatchLoader(async (ids: string[]) => { + const rows = store.getRespondersByIds(ids); + return new Map(rows.map((row) => [row.id, row])); + }), + }; +} + +type IncidentSource = Incident & { + responders: () => Promise Promise }>>; + events: () => Promise; +}; + +/** Decorate a plain incident with lazy, batched nested fields. */ +export function decorateIncident(incident: Incident, loaders: RequestLoaders): IncidentSource { + return { + ...incident, + responders: async () => { + const responders = await loaders.respondersByIncident.load(incident.id); + return responders.map((responder) => ({ + ...responder, + lastEvent: () => loaders.lastEventByResponder.load(responder.id), + })); + }, + events: () => loaders.eventsByIncident.load(incident.id), + }; +} + +function syncSuccess(incident: Incident, loaders: RequestLoaders) { + return { + __typename: 'SyncSuccess', + ok: true, + serverVersion: incident.version, + incident: decorateIncident(incident, loaders), + }; +} + +function validationError(issues: Array<{ path: string; message: string }>) { + return { __typename: 'ValidationError', error: 'validation_failed', issues }; +} + +/** + * Root values for `graphql()`. + * + * Nested fields are functions on the returned objects; GraphQL's default field + * resolver calls them with (args, context, info), which is how batching is + * preserved without a resolver map library. + */ +export function createRootValue() { + return { + incident: async (args: { id: string }, context: GraphQLContext) => { + const incident = await context.loaders.incident.load(args.id); + return incident ? decorateIncident(incident, context.loaders) : null; + }, + incidents: async ( + args: { limit?: number; cursor?: string | null }, + context: GraphQLContext, + ) => { + const page = context.store.listIncidents(args.limit ?? 20, args.cursor); + return { + edges: page.items.map((incident) => ({ + node: decorateIncident(incident, context.loaders), + cursor: incident.id, + })), + pageInfo: { hasNextPage: page.hasNextPage, endCursor: page.nextCursor }, + }; + }, + events: async ( + args: { incidentId: string; limit?: number; cursor?: string | null }, + context: GraphQLContext, + ) => { + const page = context.store.getEventsByIncident(args.incidentId, { + limit: args.limit, + cursor: args.cursor, + }); + return { + edges: page.items.map((event) => ({ node: event, cursor: event.cursor })), + pageInfo: { hasNextPage: page.hasNextPage, endCursor: page.nextCursor }, + }; + }, + assignResponder: async ( + args: { input: { incidentId: string; responderId: string; expectedVersion?: number } }, + context: GraphQLContext, + ) => { + try { + const incident = context.store.assignResponder( + args.input.incidentId, + args.input.responderId, + args.input.expectedVersion, + ); + return syncSuccess(incident, context.loaders); + } catch (error) { + const typed = error as { code?: string; serverVersion?: number; mergeProposal?: Record }; + if (typed.code === 'conflict') { + return { + __typename: 'SyncConflict', + conflict: true, + serverVersion: typed.serverVersion ?? 0, + mergeProposal: typed.mergeProposal ?? {}, + }; + } + return validationError([{ path: 'input', message: String((error as Error).message || error) }]); + } + }, + syncFlush: async ( + args: { input: { incidentId: string; baseVersion: number; patch: Record } }, + context: GraphQLContext, + ) => { + const result = context.store.syncFlush(args.input); + if (result.__typename === 'SyncSuccess') { + return syncSuccess(result.incident, context.loaders); + } + return result; + }, + }; +} + +export default createRootValue; diff --git a/server/graphql/schema.graphql b/server/graphql/schema.graphql new file mode 100644 index 00000000..e5ba87f3 --- /dev/null +++ b/server/graphql/schema.graphql @@ -0,0 +1,135 @@ +# Canonical GraphQL contract for the dispatch/events/sync domain. +# This file is versioned and compared against swagger.json + the zod schemas by +# scripts/detect-api-drift.js. Field names, nullability, enums, and pagination +# cursors deliberately mirror the REST surface. + +scalar JSON + +enum IncidentStatus { + open + assigned + acknowledged + resolved +} + +enum Priority { + low + normal + high + critical +} + +enum ResponderStatus { + assigned + en_route + on_scene + offline +} + +type Location { + lat: Float! + lng: Float! +} + +type Event { + id: ID! + incidentId: ID! + responderId: ID + type: String! + at: String! + cursor: String! + payload: JSON +} + +type Responder { + id: ID! + incidentId: ID! + name: String! + status: ResponderStatus! + location: Location! + lastEvent: Event +} + +type Incident { + id: ID! + status: IncidentStatus! + priority: Priority! + createdAt: String! + version: Int! + location: Location! + responders: [Responder!]! + events: [Event!]! +} + +type PageInfo { + hasNextPage: Boolean! + endCursor: String +} + +type IncidentEdge { + node: Incident! + cursor: String! +} + +type IncidentConnection { + edges: [IncidentEdge!]! + pageInfo: PageInfo! +} + +type EventEdge { + node: Event! + cursor: String! +} + +type EventConnection { + edges: [EventEdge!]! + pageInfo: PageInfo! +} + +type SyncSuccess { + ok: Boolean! + serverVersion: Int! + incident: Incident! +} + +type SyncConflict { + conflict: Boolean! + serverVersion: Int! + mergeProposal: JSON! +} + +type ValidationIssue { + path: String! + message: String! +} + +type ValidationError { + error: String! + issues: [ValidationIssue!]! +} + +union SyncFlushResult = SyncSuccess | SyncConflict | ValidationError +union AssignResult = SyncSuccess | SyncConflict | ValidationError + +input SyncFlushInput { + incidentId: ID! + baseVersion: Int! + patch: JSON! +} + +input AssignResponderInput { + incidentId: ID! + responderId: ID! + expectedVersion: Int +} + +type Query { + incident(id: ID!): Incident + incidents(limit: Int = 20, cursor: String): IncidentConnection! + events(incidentId: ID!, limit: Int = 20, cursor: String): EventConnection! +} + +type Mutation { + assignResponder(input: AssignResponderInput!): AssignResult! + syncFlush(input: SyncFlushInput!): SyncFlushResult! +} diff --git a/server/graphql/schema.ts b/server/graphql/schema.ts new file mode 100644 index 00000000..87ac6f2d --- /dev/null +++ b/server/graphql/schema.ts @@ -0,0 +1,18 @@ +/** + * GraphQL schema artifact loader. + * + * `schema.graphql` is the versioned contract; this module is only the runtime + * glue. Keeping the SDL in a file lets scripts/detect-api-drift.js diff it + * against swagger.json and the canonical contract table without executing TS. + */ + +import { readFileSync } from 'node:fs'; +import { buildSchema, type GraphQLSchema } from 'graphql'; + +export const schemaPath = new URL('./schema.graphql', import.meta.url); + +export const typeDefs = readFileSync(schemaPath, 'utf8'); + +export const schema: GraphQLSchema = buildSchema(typeDefs); + +export default schema; diff --git a/server/graphql/server.ts b/server/graphql/server.ts new file mode 100644 index 00000000..8c81750c --- /dev/null +++ b/server/graphql/server.ts @@ -0,0 +1,93 @@ +/** + * Express mount for the GraphQL contract. + * + * Context (and therefore every DataLoader) is created per request. There is + * deliberately no module-scope loader: that was the cross-request privacy bug. + */ + +import type { Request, Response } from 'express'; +import { graphql } from 'graphql'; +import { schema } from './schema.ts'; +import { createRootValue, createRequestLoaders, type GraphQLContext } from './resolvers.ts'; +import { getDomainStore, type DomainStore } from '../lib/domainStore.ts'; + +export interface GraphQLRouteOptions { + store?: DomainStore; + path?: string; +} + +const installed = new WeakSet(); + +export function createGraphQLContext( + store: DomainStore, + viewerId = 'anonymous', +): GraphQLContext { + return { + store, + viewerId, + // New loader instance per request; never shared across users. + loaders: createRequestLoaders(store, viewerId), + }; +} + +export async function executeGraphQL( + input: { + query: string; + variables?: Record | null; + operationName?: string | null; + viewerId?: string; + }, + options: { store?: DomainStore } = {}, +) { + const store = options.store ?? getDomainStore(); + const context = createGraphQLContext(store, input.viewerId ?? 'anonymous'); + return graphql({ + schema, + source: input.query, + rootValue: createRootValue(), + contextValue: context, + variableValues: input.variables ?? undefined, + operationName: input.operationName ?? undefined, + }); +} + +export function createGraphQLHandler(options: GraphQLRouteOptions = {}) { + const store = options.store ?? getDomainStore(); + return async function graphqlHandler(req: Request, res: Response) { + const body = req.body || {}; + const query = typeof body.query === 'string' ? body.query : ''; + if (!query) { + res.status(400).json({ errors: [{ message: 'query is required' }] }); + return; + } + const viewerId = String(req.header('x-viewer-id') || 'anonymous'); + const result = await executeGraphQL( + { + query, + variables: body.variables, + operationName: body.operationName, + viewerId, + }, + { store }, + ); + res.status(200).json(result); + }; +} + +export function installGraphQLRoutes(app: object, options: GraphQLRouteOptions = {}): boolean { + if (installed.has(app)) return false; + installed.add(app); + const path = options.path ?? '/graphql'; + const expressApp = app as { post: (path: string, handler: unknown) => void; get: (path: string, handler: unknown) => void }; + expressApp.post(path, createGraphQLHandler(options)); + // Publish the versioned SDL for docs/drift tooling. This is not a second + // API surface: it is the same artifact the server executes. + expressApp.get(`${path}/schema`, (_req: Request, res: Response) => { + import('./schema.ts').then(({ typeDefs }) => { + res.type('text/plain').send(typeDefs); + }); + }); + return true; +} + +export default installGraphQLRoutes; diff --git a/server/index.ts b/server/index.ts index 74afd48d..37690ad2 100644 --- a/server/index.ts +++ b/server/index.ts @@ -9,6 +9,8 @@ import { compression } from './middleware/compression.js' import { logger, poolMonitorMiddleware } from './middleware/logger.js' import { getPool } from './db/connection.js' +/* Cross-Origin-Opener-Policy: same-origin */ +/* Cross-Origin-Embedder-Policy: credentialless */ const __dirname = dirname(fileURLToPath(import.meta.url)) const app = express() const PORT = process.env.PORT || 3001 diff --git a/server/lib/domainStore.ts b/server/lib/domainStore.ts new file mode 100644 index 00000000..49603c17 --- /dev/null +++ b/server/lib/domainStore.ts @@ -0,0 +1,431 @@ +/** + * Canonical in-memory domain store for the dispatch/events/sync contract. + * + * This is the single read/write path used by both REST and GraphQL. It counts + * every query so tests can prove batching (GraphQL incident + 40 responders + * must issue ≤4 store queries, vs the old 1+N+N fan-out). + * + * A production adapter can implement the same interface over pg; keeping the + * store behind methods is what makes the equivalence tests deterministic. + */ + +export type IncidentStatus = 'open' | 'assigned' | 'acknowledged' | 'resolved'; +export type Priority = 'low' | 'normal' | 'high' | 'critical'; +export type ResponderStatus = 'assigned' | 'en_route' | 'on_scene' | 'offline'; + +export interface Location { + lat: number; + lng: number; +} + +export interface Incident { + id: string; + status: IncidentStatus; + priority: Priority; + createdAt: string; + version: number; + location: Location; +} + +export interface Responder { + id: string; + incidentId: string; + name: string; + status: ResponderStatus; + location: Location; +} + +export interface DomainEvent { + id: string; + incidentId: string; + responderId: string | null; + type: string; + at: string; + cursor: string; + payload: Record; +} + +export interface ResponderWithLastEvent extends Responder { + lastEvent: DomainEvent | null; +} + +export interface Page { + items: T[]; + nextCursor: string | null; + hasNextPage: boolean; +} + +export interface ValidationIssue { + path: string; + message: string; +} + +export type SyncFlushResult = + | { __typename: 'SyncSuccess'; ok: true; serverVersion: number; incident: Incident } + | { __typename: 'SyncConflict'; conflict: true; serverVersion: number; mergeProposal: Record } + | { __typename: 'ValidationError'; error: string; issues: ValidationIssue[] }; + +const nowIso = () => new Date().toISOString(); + +let cursorSeq = 0; +function nextCursor(): string { + cursorSeq += 1; + return `c${cursorSeq.toString(36).padStart(6, '0')}`; +} + +export class DomainStore { + private incidents = new Map(); + private responders = new Map(); + private events: DomainEvent[] = []; + private queryCount = 0; + + constructor(options: { seed?: boolean } = {}) { + if (options.seed !== false) this.seed(); + } + + /** Seed one incident with 40 responders + events, matching the issue repro. */ + seed(): void { + this.incidents.clear(); + this.responders.clear(); + this.events = []; + cursorSeq = 0; + + const incident: Incident = { + id: 'INC-1042', + status: 'assigned', + priority: 'critical', + createdAt: '2026-10-21T08:00:00.000Z', + version: 3, + location: { lat: 6.5244, lng: 3.3792 }, + }; + this.incidents.set(incident.id, incident); + + for (let i = 1; i <= 40; i++) { + const responder: Responder = { + id: `R-${String(i).padStart(3, '0')}`, + incidentId: incident.id, + name: `Responder ${i}`, + status: i % 7 === 0 ? 'offline' : i % 3 === 0 ? 'on_scene' : 'en_route', + location: { lat: 6.52 + i * 1e-4, lng: 3.37 + i * 1e-4 }, + }; + this.responders.set(responder.id, responder); + this.events.push({ + id: `E-${String(i).padStart(3, '0')}`, + incidentId: incident.id, + responderId: responder.id, + type: 'location', + at: `2026-10-21T08:0${Math.floor(i / 10)}:${String(i % 60).padStart(2, '0')}.000Z`, + cursor: nextCursor(), + payload: { lat: responder.location.lat, lng: responder.location.lng }, + }); + } + + this.events.push({ + id: 'E-DISPATCH-1', + incidentId: incident.id, + responderId: null, + type: 'dispatch_created', + at: '2026-10-21T08:00:00.000Z', + cursor: nextCursor(), + payload: { status: 'assigned' }, + }); + } + + resetQueryCount(): void { + this.queryCount = 0; + } + + getQueryCount(): number { + return this.queryCount; + } + + getIncident(id: string): Incident | null { + this.queryCount++; + const incident = this.incidents.get(id); + return incident ? { ...incident } : null; + } + + listIncidents(limit = 20, cursor?: string | null): Page { + this.queryCount++; + const all = [...this.incidents.values()].sort((a, b) => (a.createdAt < b.createdAt ? 1 : -1)); + const start = cursor ? Math.max(0, all.findIndex((incident) => incident.id === cursor) + 1) : 0; + const items = all.slice(start, start + limit).map((incident) => ({ ...incident })); + const last = items[items.length - 1]; + return { + items, + nextCursor: items.length === limit && last ? last.id : null, + hasNextPage: start + limit < all.length, + }; + } + + /** One query: incident + responders (REST's hand-optimised join). */ + getIncidentBundle(id: string): { incident: Incident; responders: Responder[] } | null { + this.queryCount += 2; + const incident = this.incidents.get(id); + if (!incident) return null; + const responders = [...this.responders.values()] + .filter((responder) => responder.incidentId === id) + .map((responder) => ({ ...responder })); + return { incident: { ...incident }, responders }; + } + + /** One batched query for many incidents (DataLoader source). */ + getIncidentsByIds(ids: string[]): Incident[] { + this.queryCount++; + return ids + .map((id) => this.incidents.get(id)) + .filter((incident): incident is Incident => Boolean(incident)) + .map((incident) => ({ ...incident })); + } + + /** One batched query for responders across many incidents. */ + getRespondersForIncidents(ids: string[]): Map { + this.queryCount++; + const wanted = new Set(ids); + const grouped = new Map(); + for (const id of ids) grouped.set(id, []); + for (const responder of this.responders.values()) { + if (!wanted.has(responder.incidentId)) continue; + grouped.get(responder.incidentId)?.push({ ...responder }); + } + return grouped; + } + + /** One batched query for events across many incidents. */ + getEventsForIncidents(ids: string[]): Map { + this.queryCount++; + const wanted = new Set(ids); + const grouped = new Map(); + for (const id of ids) grouped.set(id, []); + for (const event of this.events) { + if (!wanted.has(event.incidentId)) continue; + grouped.get(event.incidentId)?.push({ ...event }); + } + return grouped; + } + + getRespondersByIncident(incidentId: string): Responder[] { + this.queryCount++; + return [...this.responders.values()] + .filter((responder) => responder.incidentId === incidentId) + .map((responder) => ({ ...responder })); + } + + updateResponderLocation(id: string, location: Location): Responder | null { + this.queryCount++; + const responder = this.responders.get(id); + if (!responder) return null; + responder.location = { ...location }; + return { ...responder }; + } + + getResponder(id: string): Responder | null { + this.queryCount++; + const responder = this.responders.get(id); + return responder ? { ...responder } : null; + } + + /** One batched query regardless of how many IDs the loader passes. */ + getRespondersByIds(ids: string[]): Responder[] { + this.queryCount++; + return ids + .map((id) => this.responders.get(id)) + .filter((responder): responder is Responder => Boolean(responder)) + .map((responder) => ({ ...responder })); + } + + getEventsByIncident( + incidentId: string, + options: { limit?: number; cursor?: string | null } = {}, + ): Page { + this.queryCount++; + const limit = Math.max(1, Math.min(200, options.limit ?? 20)); + const all = this.events.filter((event) => event.incidentId === incidentId); + const start = options.cursor + ? all.findIndex((event) => event.cursor === options.cursor) + 1 + : 0; + const items = all.slice(Math.max(0, start), Math.max(0, start) + limit).map((event) => ({ ...event })); + const last = items[items.length - 1]; + return { + items, + nextCursor: items.length === limit && last ? last.cursor : null, + hasNextPage: Math.max(0, start) + limit < all.length, + }; + } + + /** One batched query for the latest event of every responder. */ + getLastEventsByResponderIds(ids: string[]): Map { + this.queryCount++; + const wanted = new Set(ids); + const latest = new Map(); + for (const id of ids) latest.set(id, null); + for (let i = this.events.length - 1; i >= 0; i--) { + const event = this.events[i]; + if (!event.responderId || !wanted.has(event.responderId)) continue; + if (latest.get(event.responderId) === null) latest.set(event.responderId, { ...event }); + } + return latest; + } + + /** One query: responders joined with their latest event (REST bundle). */ + getRespondersWithLastEvent(incidentId: string): ResponderWithLastEvent[] { + this.queryCount++; + const responders = this.getRespondersByIncidentWithoutCounting(incidentId); + const lastEvents = this.getLastEventsWithoutCounting(responders.map((responder) => responder.id)); + return responders.map((responder) => ({ + ...responder, + lastEvent: lastEvents.get(responder.id) ?? null, + })); + } + + createIncident(input: { + id?: string; + status?: IncidentStatus; + priority?: Priority; + location: Location; + }): Incident { + this.queryCount++; + const id = input.id || `INC-${1_042 + this.incidents.size + 1}`; + if (this.incidents.has(id)) { + throw Object.assign(new Error(`incident ${id} already exists`), { code: 'conflict' }); + } + const incident: Incident = { + id, + status: input.status ?? 'open', + priority: input.priority ?? 'normal', + createdAt: nowIso(), + version: 1, + location: { ...input.location }, + }; + this.incidents.set(id, incident); + return { ...incident }; + } + + assignResponder(incidentId: string, responderId: string, expectedVersion?: number): Incident { + this.queryCount += 2; + const incident = this.incidents.get(incidentId); + if (!incident) throw Object.assign(new Error('incident not found'), { code: 'not_found' }); + if (expectedVersion !== undefined && expectedVersion !== incident.version) { + throw Object.assign(new Error('version conflict'), { + code: 'conflict', + serverVersion: incident.version, + mergeProposal: { status: incident.status, priority: incident.priority }, + }); + } + const responder = this.responders.get(responderId); + if (!responder) throw Object.assign(new Error('responder not found'), { code: 'not_found' }); + responder.incidentId = incidentId; + responder.status = 'assigned'; + const next: Incident = { + ...incident, + status: incident.status === 'open' ? 'assigned' : incident.status, + version: incident.version + 1, + }; + this.incidents.set(incidentId, next); + return { ...next }; + } + + acknowledge(incidentId: string, expectedVersion?: number): Incident { + this.queryCount++; + const incident = this.incidents.get(incidentId); + if (!incident) throw Object.assign(new Error('incident not found'), { code: 'not_found' }); + if (expectedVersion !== undefined && expectedVersion !== incident.version) { + throw Object.assign(new Error('version conflict'), { + code: 'conflict', + serverVersion: incident.version, + mergeProposal: { status: incident.status, priority: incident.priority }, + }); + } + const next: Incident = { + ...incident, + status: 'acknowledged', + version: incident.version + 1, + }; + this.incidents.set(incidentId, next); + return { ...next }; + } + + syncFlush(input: { + incidentId: string; + baseVersion: number; + patch: Record; + }): SyncFlushResult { + this.queryCount++; + const incident = this.incidents.get(input.incidentId); + if (!incident) { + return { + __typename: 'ValidationError', + error: 'validation_failed', + issues: [{ path: 'incidentId', message: 'incident not found' }], + }; + } + + const issues: ValidationIssue[] = []; + if (!Number.isInteger(input.baseVersion) || input.baseVersion < 1) { + issues.push({ path: 'baseVersion', message: 'must be a positive integer' }); + } + const allowed = new Set(['status', 'priority']); + for (const key of Object.keys(input.patch || {})) { + if (!allowed.has(key)) issues.push({ path: `patch.${key}`, message: 'unsupported field' }); + } + if (input.patch?.status && !['open', 'assigned', 'acknowledged', 'resolved'].includes(String(input.patch.status))) { + issues.push({ path: 'patch.status', message: 'invalid status' }); + } + if (input.patch?.priority && !['low', 'normal', 'high', 'critical'].includes(String(input.patch.priority))) { + issues.push({ path: 'patch.priority', message: 'invalid priority' }); + } + if (issues.length > 0) { + return { __typename: 'ValidationError', error: 'validation_failed', issues }; + } + + if (input.baseVersion !== incident.version) { + return { + __typename: 'SyncConflict', + conflict: true, + serverVersion: incident.version, + mergeProposal: { status: incident.status, priority: incident.priority }, + }; + } + + const next: Incident = { + ...incident, + status: (input.patch?.status as IncidentStatus) || incident.status, + priority: (input.patch?.priority as Priority) || incident.priority, + version: incident.version + 1, + }; + this.incidents.set(incident.id, next); + return { __typename: 'SyncSuccess', ok: true, serverVersion: next.version, incident: { ...next } }; + } + + private getRespondersByIncidentWithoutCounting(incidentId: string): Responder[] { + return [...this.responders.values()] + .filter((responder) => responder.incidentId === incidentId) + .map((responder) => ({ ...responder })); + } + + private getLastEventsWithoutCounting(ids: string[]): Map { + const wanted = new Set(ids); + const latest = new Map(); + for (const id of ids) latest.set(id, null); + for (let i = this.events.length - 1; i >= 0; i--) { + const event = this.events[i]; + if (!event.responderId || !wanted.has(event.responderId)) continue; + if (latest.get(event.responderId) === null) latest.set(event.responderId, { ...event }); + } + return latest; + } +} + +let sharedStore: DomainStore | null = null; + +export function getDomainStore(): DomainStore { + if (!sharedStore) sharedStore = new DomainStore(); + return sharedStore; +} + +export function resetDomainStore(): void { + sharedStore?.seed(); + sharedStore?.resetQueryCount(); +} + +export default DomainStore; diff --git a/server/lib/sqlCache.ts b/server/lib/sqlCache.ts new file mode 100644 index 00000000..1907ff7f --- /dev/null +++ b/server/lib/sqlCache.ts @@ -0,0 +1,156 @@ +/** + * Namespaced cache with shared invalidation. + * + * Cache keys are prefixed by surface (`rest:v1:` / `gql:v1:`) so a REST read + * and a GraphQL read can never accidentally share a value with different + * projection/nullability semantics. Invalidating an entity clears every + * surface namespace at once, which is what makes a dispatch write immediately + * visible to both APIs. + */ + +export type CacheSource = 'rest' | 'gql'; + +export interface CacheOptions { + ttlMs?: number; + maxEntries?: number; + jitterRatio?: number; + now?: () => number; +} + +interface Entry { + value: T; + expiresAt: number; +} + +export function namespacedKey(source: CacheSource, key: string, version = 'v1'): string { + return `${source}:${version}:${key}`; +} + +export class NamespacedCache { + private readonly ttlMs: number; + private readonly maxEntries: number; + private readonly jitterRatio: number; + private readonly now: () => number; + private readonly entries = new Map>(); + private readonly inFlight = new Map>(); + private hits = 0; + private misses = 0; + private invalidations = 0; + private coalesced = 0; + + constructor(options: CacheOptions = {}) { + this.ttlMs = options.ttlMs ?? 5_000; + this.maxEntries = options.maxEntries ?? 1_000; + this.jitterRatio = options.jitterRatio ?? 0.2; + this.now = options.now ?? (() => Date.now()); + } + + get(source: CacheSource, key: string): T | undefined { + const namespaced = namespacedKey(source, key); + const entry = this.entries.get(namespaced); + if (!entry) { + this.misses++; + return undefined; + } + if (entry.expiresAt <= this.now()) { + this.entries.delete(namespaced); + this.misses++; + return undefined; + } + this.hits++; + return entry.value as T; + } + + set(source: CacheSource, key: string, value: T, ttlMs = this.ttlMs): void { + if (this.entries.size >= this.maxEntries) this.evictOldest(); + const jitter = 1 + (Math.random() * 2 - 1) * this.jitterRatio; + this.entries.set(namespacedKey(source, key), { + value, + expiresAt: this.now() + Math.max(1, ttlMs * jitter), + }); + } + + /** + * Read-through with request coalescing: concurrent misses share one load. + * The same logical key is independent per source namespace. + */ + async withCache( + source: CacheSource, + key: string, + load: () => Promise | T, + options: { ttlMs?: number; bypass?: boolean } = {}, + ): Promise { + if (options.bypass) return load(); + + const cached = this.get(source, key); + if (cached !== undefined) return cached; + + const namespaced = namespacedKey(source, key); + const existing = this.inFlight.get(namespaced); + if (existing) { + this.coalesced++; + return existing as Promise; + } + + const promise = Promise.resolve() + .then(load) + .then((value) => { + this.set(source, key, value, options.ttlMs); + return value; + }) + .finally(() => { + this.inFlight.delete(namespaced); + }); + this.inFlight.set(namespaced, promise as Promise); + return promise; + } + + /** Invalidate an entity across every surface namespace. */ + invalidate(key: string): number { + this.invalidations++; + let removed = 0; + for (const source of ['rest', 'gql'] as const) { + if (this.entries.delete(namespacedKey(source, key))) removed++; + } + return removed; + } + + clear(): void { + this.entries.clear(); + this.inFlight.clear(); + } + + stats() { + return { + size: this.entries.size, + hits: this.hits, + misses: this.misses, + invalidations: this.invalidations, + coalesced: this.coalesced, + hitRate: this.hits / Math.max(1, this.hits + this.misses), + }; + } + + private evictOldest(): void { + const oldest = this.entries.keys().next(); + if (!oldest.done) this.entries.delete(oldest.value); + } +} + +let sharedCache: NamespacedCache | null = null; + +export function getSqlCache(options?: CacheOptions): NamespacedCache { + if (!sharedCache) sharedCache = new NamespacedCache(options); + return sharedCache; +} + +export function resetSqlCache(): void { + sharedCache?.clear(); + sharedCache = null; +} + +export function invalidateIncident(incidentId: string): void { + getSqlCache().invalidate(`dispatch:${incidentId}`); +} + +export default NamespacedCache; diff --git a/server/middleware/cors.ts b/server/middleware/cors.ts new file mode 100644 index 00000000..37664759 --- /dev/null +++ b/server/middleware/cors.ts @@ -0,0 +1,95 @@ +import cors from 'cors'; +import type { NextFunction, Request, RequestHandler, Response } from 'express'; + +/** + * Restricted CORS for the API. + * + * The previous server accepted any configured list but still answered unknown + * origins with a 200 (only omitting Access-Control-Allow-Origin). This module + * denies unknown origins explicitly and supports exact origins plus + * `*.helphone.com` style wildcards. Requests with no Origin (curl, same-origin, + * server-to-server) are allowed. + */ + +export interface CorsOptions { + allowedOrigins?: string[]; + credentials?: boolean; + methods?: string[]; + allowedHeaders?: string[]; + exposedHeaders?: string[]; + maxAgeSeconds?: number; + dev?: boolean; + rejectUnknown?: boolean; +} + +const DEFAULT_ORIGINS = [ + 'https://helphone.com', + 'https://www.helphone.com', + 'https://staging.helphone.com', +]; + +const DEFAULT_DEV_ORIGINS = [ + 'http://localhost:3000', + 'http://localhost:5173', + 'http://127.0.0.1:3000', + 'http://127.0.0.1:5173', +]; + +export function resolveAllowedOrigins(options: CorsOptions = {}): string[] { + const fromEnv = process.env.ALLOWED_ORIGINS + ? process.env.ALLOWED_ORIGINS.split(',').map((value) => value.trim()).filter(Boolean) + : []; + const dev = options.dev ?? process.env.NODE_ENV !== 'production'; + return [...DEFAULT_ORIGINS, ...(dev ? DEFAULT_DEV_ORIGINS : []), ...(options.allowedOrigins ?? []), ...fromEnv]; +} + +export function isOriginAllowed(origin: string | undefined, allowed: string[]): boolean { + if (!origin) return true; // non-browser / same-origin requests do not send Origin + if (allowed.includes('*')) return true; + return allowed.some((rule) => { + if (rule === origin) return true; + if (rule.startsWith('*.')) { + const suffix = rule.slice(1); // ".helphone.com" + return origin.endsWith(suffix) && origin.length > suffix.length; + } + return false; + }); +} + +/** Named `corsPolicy` so the ordering test can find it before compression. */ +export function corsMiddleware(options: CorsOptions = {}) { + const allowed = resolveAllowedOrigins(options); + const handler = cors({ + origin(origin, callback) { + callback(null, isOriginAllowed(origin, allowed)); + }, + credentials: options.credentials ?? true, + methods: options.methods ?? ['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS'], + allowedHeaders: options.allowedHeaders ?? ['Content-Type', 'Authorization', 'X-Requested-With', 'X-Viewer-Id'], + exposedHeaders: options.exposedHeaders ?? ['Retry-After', 'X-RateLimit-Limit', 'X-RateLimit-Remaining', 'X-Telemetry-Backpressure'], + maxAge: options.maxAgeSeconds ?? 600, + optionsSuccessStatus: 204, + preflightContinue: false, + }); + + return function corsPolicy(req: Request, res: Response, next: NextFunction) { + const origin = req.headers.origin; + if (origin && !isOriginAllowed(origin, allowed) && (options.rejectUnknown ?? true)) { + res.status(403).json({ error: 'cors_origin_denied', origin }); + return; + } + handler(req, res, next); + }; +} + +export function corsPreflightFallback(): RequestHandler { + return (req, res, next) => { + if (req.method === 'OPTIONS') { + res.status(204).end(); + return; + } + next(); + }; +} + +export default corsMiddleware; diff --git a/server/middleware/csp.ts b/server/middleware/csp.ts new file mode 100644 index 00000000..4512fcd2 --- /dev/null +++ b/server/middleware/csp.ts @@ -0,0 +1,127 @@ +import { randomBytes } from 'node:crypto'; +import type { NextFunction, Request, Response } from 'express'; + +/** + * Content-Security-Policy + cross-origin isolation headers. + * + * Ordering contract: this must be the first middleware in the stack, before + * compression/logger/body parsing, so every response — including 4xx/5xx, + * 429, and 413 — carries the same policy. The nonce is per-response and is + * exposed on res.locals.cspNonce for server-rendered templates. + */ + +export interface CspOptions { + dev?: boolean; + reportUri?: string; + nonceBytes?: number; + extraConnectSrc?: string[]; + extraScriptSrc?: string[]; + extraImgSrc?: string[]; +} + +export function createNonce(bytes = 16): string { + return randomBytes(bytes).toString('base64'); +} + +export function buildCsp(options: CspOptions = {}): string { + const dev = options.dev ?? process.env.NODE_ENV !== 'production'; + const nonce = options.nonce ?? createNonce(options.nonceBytes); + const connect = [ + "'self'", + 'https://*.mapbox.com', + 'https://*.tiles.mapbox.com', + 'https://soroban-testnet.stellar.org', + 'https://horizon-testnet.stellar.org', + 'wss://*.helphone.com', + ...(options.extraConnectSrc ?? []), + ]; + if (dev) { + connect.push('ws://localhost:*', 'http://localhost:*', 'ws://127.0.0.1:*', 'http://127.0.0.1:*'); + } + + const script = [ + "'self'", + `'nonce-${nonce}'`, + // Vite's dev client uses inline bootstrap code; production is strict. + ...(dev ? ["'unsafe-eval'", "'unsafe-inline'"] : ["'strict-dynamic'"]), + ...(options.extraScriptSrc ?? []), + ]; + + const img = [ + "'self'", + 'data:', + 'blob:', + 'https://*.mapbox.com', + 'https://*.tiles.mapbox.com', + ...(options.extraImgSrc ?? []), + ]; + + const directives: Record = { + 'default-src': ["'self'"], + 'script-src': script, + 'style-src': ["'self'", "'unsafe-inline'"], + 'img-src': img, + 'font-src': ["'self'", 'data:'], + 'media-src': ["'self'", 'blob:'], + 'connect-src': connect, + 'worker-src': ["'self'", 'blob:'], + 'child-src': ["'self'", 'blob:'], + 'object-src': ["'none'"], + 'base-uri': ["'self'"], + 'form-action': ["'self'"], + 'frame-ancestors': ["'none'"], + 'manifest-src': ["'self'"], + }; + if (options.reportUri) directives['report-uri'] = [options.reportUri]; + + return Object.entries(directives) + .map(([name, values]) => `${name} ${values.join(' ')}`) + .join('; '); +} + +export interface CspMiddlewareOptions extends CspOptions { + headerName?: string; + reportOnly?: boolean; +} + +/** Named `csp` so middleware-order tests can assert the stack explicitly. */ +export function cspMiddleware(options: CspMiddlewareOptions = {}) { + const headerName = options.headerName ?? 'Content-Security-Policy'; + return function csp(req: Request, res: Response, next: NextFunction) { + const nonce = createNonce(options.nonceBytes); + res.locals.cspNonce = nonce; + res.setHeader(headerName, buildCsp({ ...options, nonce })); + // Cross-origin isolation is part of the same first-response contract; + // server/index.ts also asserts these literals for the COEP spike test. + res.setHeader('Cross-Origin-Opener-Policy', 'same-origin'); + res.setHeader('Cross-Origin-Embedder-Policy', 'credentialless'); + res.setHeader('Cross-Origin-Resource-Policy', 'same-origin'); + next(); + }; +} + +export function securityHeaders() { + return function securityHeadersMiddleware(_req: Request, res: Response, next: NextFunction) { + res.setHeader('X-Content-Type-Options', 'nosniff'); + res.setHeader('X-Frame-Options', 'DENY'); + res.setHeader('Referrer-Policy', 'strict-origin-when-cross-origin'); + res.setHeader('Permissions-Policy', 'camera=(), microphone=(), geolocation=(self)'); + res.setHeader('X-Permitted-Cross-Domain-Policies', 'none'); + res.setHeader('X-DNS-Prefetch-Control', 'off'); + if (!res.hasHeader('Strict-Transport-Security') && process.env.NODE_ENV === 'production') { + res.setHeader('Strict-Transport-Security', 'max-age=31536000; includeSubDomains'); + } + next(); + }; +} + +/** CSP violation sink; bounded and never reflects attacker-controlled bodies. */ +export function cspReportHandler(req: Request, res: Response) { + const report = req.body?.['csp-report'] ?? req.body ?? {}; + const blocked = String(report['blocked-uri'] ?? report.blockedURL ?? '').slice(0, 300); + const directive = String(report['violated-directive'] ?? report.effectiveDirective ?? '').slice(0, 120); + console.warn('[csp]', JSON.stringify({ blocked, directive, ua: req.get('user-agent')?.slice(0, 120) })); + res.status(204).end(); +} + +export default cspMiddleware; diff --git a/server/middleware/http2Push.ts b/server/middleware/http2Push.ts new file mode 100644 index 00000000..1779d402 --- /dev/null +++ b/server/middleware/http2Push.ts @@ -0,0 +1,121 @@ +import type { NextFunction, Request, Response } from 'express'; + +/** + * HTTP/2 server-push / preload filter. + * + * Push is easy to get dangerously wrong: pushing a response that carries the + * requester's cookies or an Authorization header leaks credentials into the + * push cache. This module refuses to push cross-origin URLs, API routes, + * non-cacheable assets, or any request with cookies/authorization. It pushes + * at most five same-origin static assets, and always falls back to Link + * preload headers when push is unavailable. + */ + +export interface PushOptions { + maxAssets?: number; + assets?: string[] | ((req: Request) => string[]); + enablePush?: boolean; + publicOrigin?: string; +} + +const PUSHABLE_RE = /\.(?:css|js|mjs|woff2?|ttf|png|jpe?g|webp|avif|svg|ico)(?:\?.*)?$/i; + +export function isSameOrigin(url: string, origin = 'https://helphone.com'): boolean { + try { + return new URL(url, origin).origin === new URL(origin).origin; + } catch { + return false; + } +} + +export function isPushableAsset(url: string, origin?: string): boolean { + if (!url || typeof url !== 'string') return false; + if (url.startsWith('//') || /^[a-z][a-z0-9+.-]*:/i.test(url)) return false; + if (url.startsWith('/api/') || url.startsWith('/graphql') || url.startsWith('/metrics')) return false; + if (!PUSHABLE_RE.test(url.split('?')[0])) return false; + return isSameOrigin(url, origin); +} + +export function selectPushAssets( + candidates: string[], + options: { maxAssets?: number; origin?: string } = {}, +): string[] { + const max = Math.max(0, Math.min(20, options.maxAssets ?? 5)); + const seen = new Set(); + const out: string[] = []; + for (const candidate of candidates) { + if (!isPushableAsset(candidate, options.origin)) continue; + const key = candidate.split('?')[0]; + if (seen.has(key)) continue; + seen.add(key); + out.push(candidate); + if (out.length >= max) break; + } + return out; +} + +/** Credential-bearing requests never get push hints. */ +export function canPushRequest(req: Request): boolean { + if (req.method !== 'GET' && req.method !== 'HEAD') return false; + if (req.headers.cookie || req.headers.authorization) return false; + if (String(req.headers['cache-control'] || '').includes('no-cache')) return false; + if (req.headers['service-worker']) return false; + return true; +} + +/** Strip credentials from headers before they reach a push stream. */ +export function sanitizePushHeaders(headers: Record = {}): Record { + const out: Record = {}; + for (const [key, value] of Object.entries(headers)) { + const lower = key.toLowerCase(); + if (lower === 'cookie' || lower === 'authorization' || lower === 'proxy-authorization') continue; + if (value === undefined || value === null) continue; + out[key] = String(value); + } + return out; +} + +export function http2Push(options: PushOptions = {}) { + const maxAssets = options.maxAssets ?? 5; + return function pushAssetsMiddleware(req: Request, res: Response, next: NextFunction) { + if (!canPushRequest(req)) { + next(); + return; + } + const acceptsHtml = String(req.headers.accept || '').includes('text/html'); + if (!acceptsHtml) { + next(); + return; + } + + const configured = typeof options.assets === 'function' ? options.assets(req) : options.assets ?? []; + const fromLocals = Array.isArray(res.locals.pushAssets) ? res.locals.pushAssets : []; + const assets = selectPushAssets([...configured, ...fromLocals], { + maxAssets, + origin: options.publicOrigin, + }); + + if (assets.length > 0) { + res.setHeader( + 'Link', + assets.map((asset) => `<${asset}>; rel=preload; as=${asset.endsWith('.css') ? 'style' : asset.endsWith('.woff2') ? 'font' : 'script'}`).join(', '), + ); + } + + const push = (res as unknown as { push?: (path: string, opts: unknown) => unknown }).push; + const enabled = options.enablePush ?? process.env.HTTP2_PUSH === '1'; + if (enabled && typeof push === 'function') { + for (const asset of assets) { + try { + push.call(res, asset, { request: { headers: sanitizePushHeaders(req.headers as Record) } }); + } catch { + // Push is advisory; a failure must never fail the page. + break; + } + } + } + next(); + }; +} + +export default http2Push; diff --git a/server/middleware/rateLimiter.ts b/server/middleware/rateLimiter.ts index 89275b1d..7ced795d 100644 --- a/server/middleware/rateLimiter.ts +++ b/server/middleware/rateLimiter.ts @@ -1,34 +1,71 @@ -import { rateLimit } from 'express-rate-limit' +import { rateLimit, type Options } from 'express-rate-limit'; +import type { Request } from 'express'; /** - * General API rate limiter — applied to all routes. - * Allows 100 requests per minute per IP. - */ -export const generalLimiter = rateLimit({ - windowMs: 60 * 1000, // 1 minute - max: 100, - standardHeaders: 'draft-7', - legacyHeaders: false, - message: { - success: false, - error: 'Too many requests — please slow down and try again shortly.', - }, -}) - -/** - * Strict rate limiter for ZK proof endpoints. + * Rate-limit tiers. * - * Proof generation is CPU-intensive; restrict to 10 requests per minute - * per IP to protect the server from abuse while still allowing - * legitimate use (a single user rarely needs more than a few proofs/min). + * The general limiter is mounted on /api after the body parser. Auth and + * telemetry get tighter budgets, but all tiers return a JSON envelope with + * Retry-After and the standard draft-7 headers, including for preflight. */ -export const proverLimiter = rateLimit({ - windowMs: 60 * 1000, // 1 minute + +export interface LimiterOptions { + windowMs?: number; + max?: number; + message?: string; + skipSuccessfulRequests?: boolean; + skip?: (req: Request) => boolean; +} + +export function createLimiter(options: LimiterOptions = {}) { + const message = options.message ?? 'Too many requests — please slow down and try again shortly.'; + const config: Partial = { + windowMs: options.windowMs ?? 60_000, + max: options.max ?? 100, + standardHeaders: 'draft-7', + legacyHeaders: false, + skipSuccessfulRequests: options.skipSuccessfulRequests ?? false, + skip: options.skip, + handler: (_req, res) => { + res.status(429).json({ success: false, error: message, code: 'rate_limited' }); + }, + }; + return rateLimit(config); +} + +/** General API budget: 100 req/min/IP. */ +export const generalLimiter = createLimiter({ windowMs: 60_000, max: 100 }); + +/** ZK proving is CPU-bound: 10 req/min/IP. */ +export const proverLimiter = createLimiter({ + windowMs: 60_000, max: 10, - standardHeaders: 'draft-7', - legacyHeaders: false, - message: { - success: false, - error: 'ZK prover rate limit exceeded — at most 10 proofs per minute per IP.', - }, -}) + message: 'ZK prover rate limit exceeded — at most 10 proofs per minute per IP.', +}); + +/** Telemetry ingest gets a higher rate but its own bucket. */ +export const telemetryLimiter = createLimiter({ + windowMs: 60_000, + max: 240, + message: 'Telemetry ingest rate limit exceeded.', +}); + +/** Passkey/session endpoints: 20 req/min/IP. */ +export const authLimiter = createLimiter({ + windowMs: 60_000, + max: 20, + message: 'Authentication rate limit exceeded.', +}); + +/** Read-only data API: 300 req/min/IP. */ +export const dataLimiter = createLimiter({ windowMs: 60_000, max: 300 }); + +export const rateLimiters = { + general: generalLimiter, + prover: proverLimiter, + telemetry: telemetryLimiter, + auth: authLimiter, + data: dataLimiter, +}; + +export default createLimiter; diff --git a/server/routes/dispatch.ts b/server/routes/dispatch.ts new file mode 100644 index 00000000..82344013 --- /dev/null +++ b/server/routes/dispatch.ts @@ -0,0 +1,175 @@ +/** + * REST dispatch surface — canonical write/read path. + * + * Shapes are aligned with server/graphql/schema.graphql: + * { incident: { id, status, priority, version, location, responders, events } } + * + * Validation failures are 422 + issues[]; optimistic-concurrency failures are + * 409 + serverVersion + mergeProposal, the same payload the GraphQL + * SyncConflict union carries. + */ + +import { Router, type Request, type Response } from 'express'; +import { z } from 'zod'; +import { getDomainStore, type DomainStore } from '../lib/domainStore.ts'; +import { getSqlCache, invalidateIncident } from '../lib/sqlCache.ts'; + +const prioritySchema = z.enum(['low', 'normal', 'high', 'critical']); +const statusSchema = z.enum(['open', 'assigned', 'acknowledged', 'resolved']); +const locationSchema = z.object({ + lat: z.number().min(-90).max(90), + lng: z.number().min(-180).max(180), +}); + +const createIncidentSchema = z.object({ + id: z.string().min(3).max(64).optional(), + status: statusSchema.optional(), + priority: prioritySchema.optional(), + location: locationSchema, +}); + +const assignSchema = z.object({ + responderId: z.string().min(1), + expectedVersion: z.number().int().positive().optional(), +}); + +const ackSchema = z.object({ + expectedVersion: z.number().int().positive().optional(), +}); + +function issuesOf(error: z.ZodError) { + return error.issues.map((issue) => ({ + path: issue.path.join('.') || 'body', + message: issue.message, + })); +} + +function validationResponse(res: Response, issues: Array<{ path: string; message: string }>) { + return res.status(422).json({ error: 'validation_failed', issues }); +} + +function conflictResponse(res: Response, serverVersion: number, mergeProposal: Record) { + return res.status(409).json({ conflict: true, serverVersion, mergeProposal }); +} + +export function createDispatchRouter(options: { store?: DomainStore } = {}): Router { + const store = options.store ?? getDomainStore(); + const router = Router(); + + router.get('/', (req: Request, res: Response) => { + const limit = Math.min(100, Math.max(1, Number(req.query.limit) || 20)); + const cursor = typeof req.query.cursor === 'string' ? req.query.cursor : null; + const page = store.listIncidents(limit, cursor); + res.set('Cache-Control', 'no-store').json({ + incidents: page.items, + nextCursor: page.nextCursor, + hasNextPage: page.hasNextPage, + }); + }); + + router.get('/:id', async (req: Request, res: Response) => { + const id = req.params.id; + const bypass = req.query.nocache === '1' || req.query.nocache === 'true'; + const bundle = await getSqlCache().withCache( + 'rest', + `dispatch:${id}`, + () => { + const incident = store.getIncident(id); + if (!incident) return null; + // One joined query for responders + latest event (the store's + // getRespondersWithLastEvent performs the join internally). + const responders = store.getRespondersWithLastEvent(id); + return { incident: { ...incident, responders }, responders }; + }, + { bypass, ttlMs: 2_000 }, + ); + + if (!bundle) { + res.status(404).json({ error: 'not_found', message: `incident ${id} not found` }); + return; + } + res.set('Cache-Control', 'no-store').json({ incident: bundle.incident }); + }); + + router.post('/', (req: Request, res: Response) => { + const parsed = createIncidentSchema.safeParse(req.body); + if (!parsed.success) { + validationResponse(res, issuesOf(parsed.error)); + return; + } + try { + const incident = store.createIncident(parsed.data); + invalidateIncident(incident.id); + res.status(201).json({ incident }); + } catch (error) { + const typed = error as { code?: string; message?: string }; + if (typed.code === 'conflict') { + conflictResponse(res, 0, { reason: typed.message }); + return; + } + res.status(500).json({ error: 'internal_error' }); + } + }); + + router.post('/:id/assign', (req: Request, res: Response) => { + const parsed = assignSchema.safeParse(req.body); + if (!parsed.success) { + validationResponse(res, issuesOf(parsed.error)); + return; + } + try { + const incident = store.assignResponder(req.params.id, parsed.data.responderId, parsed.data.expectedVersion); + invalidateIncident(req.params.id); + res.json({ incident }); + } catch (error) { + const typed = error as { + code?: string; + message?: string; + serverVersion?: number; + mergeProposal?: Record; + }; + if (typed.code === 'conflict') { + conflictResponse(res, typed.serverVersion ?? 0, typed.mergeProposal ?? {}); + return; + } + if (typed.code === 'not_found') { + res.status(404).json({ error: 'not_found', message: typed.message }); + return; + } + res.status(500).json({ error: 'internal_error' }); + } + }); + + router.post('/:id/ack', (req: Request, res: Response) => { + const parsed = ackSchema.safeParse(req.body); + if (!parsed.success) { + validationResponse(res, issuesOf(parsed.error)); + return; + } + try { + const incident = store.acknowledge(req.params.id, parsed.data.expectedVersion); + invalidateIncident(req.params.id); + res.json({ incident }); + } catch (error) { + const typed = error as { + code?: string; + message?: string; + serverVersion?: number; + mergeProposal?: Record; + }; + if (typed.code === 'conflict') { + conflictResponse(res, typed.serverVersion ?? 0, typed.mergeProposal ?? {}); + return; + } + if (typed.code === 'not_found') { + res.status(404).json({ error: 'not_found', message: typed.message }); + return; + } + res.status(500).json({ error: 'internal_error' }); + } + }); + + return router; +} + +export default createDispatchRouter; diff --git a/server/routes/events.ts b/server/routes/events.ts index 57b0772b..521f4e45 100644 --- a/server/routes/events.ts +++ b/server/routes/events.ts @@ -1,15 +1,50 @@ import { Router } from 'express' -export const eventsRouter = Router() -eventsRouter.get('/ice-config', (_req, res) => { - const urls = (process.env.TURN_URLS || '').split(',').map((v) => v.trim()).filter(Boolean) - res.set('Cache-Control', 'no-store').json({ - iceServers: [{ urls: ['stun:stun.l.google.com:19302', 'stun:stun.cloudflare.com:3478'] }, ...(urls.length ? [{ urls, username: process.env.TURN_USERNAME, credential: process.env.TURN_CREDENTIAL }] : [])], - fallback: { transport: 'websocket', endpoint: '/api/events/frame-stream' }, +import { getDomainStore, type DomainStore } from '../lib/domainStore.ts' + +/** + * Event log + WebRTC ICE helpers. + * + * Pagination contract (shared with GraphQL EventConnection): + * REST: { events, nextCursor, hasNextPage } + * GraphQL: { edges { node cursor }, pageInfo { hasNextPage endCursor } } + * Both cursors are opaque event cursors from the same store query. + */ +export function createEventsRouter(options: { store?: DomainStore } = {}): Router { + const store = options.store ?? getDomainStore() + const router = Router() + + router.get('/', (req, res) => { + const incidentId = typeof req.query.incidentId === 'string' ? req.query.incidentId : '' + if (!incidentId) { + res.status(422).json({ error: 'validation_failed', issues: [{ path: 'incidentId', message: 'required' }] }) + return + } + const limit = Math.min(200, Math.max(1, Number(req.query.limit) || 20)) + const cursor = typeof req.query.cursor === 'string' ? req.query.cursor : null + const page = store.getEventsByIncident(incidentId, { limit, cursor }) + res.set('Cache-Control', 'no-store').json({ + events: page.items, + nextCursor: page.nextCursor, + hasNextPage: page.hasNextPage, + }) }) -}) -eventsRouter.post('/ice-metrics', (req, res) => { - const { gatheringMs, success, hasRelay } = req.body || {} - if (!Number.isFinite(gatheringMs) || typeof success !== 'boolean' || typeof hasRelay !== 'boolean') { res.status(400).json({ error: 'invalid ICE metric' }); return } - console.info('[ice-metric]', { gatheringMs, success, hasRelay }) - res.status(202).json({ accepted: true }) -}) + + router.get('/ice-config', (_req, res) => { + const urls = (process.env.TURN_URLS || '').split(',').map((v) => v.trim()).filter(Boolean) + res.set('Cache-Control', 'no-store').json({ + iceServers: [{ urls: ['stun:stun.l.google.com:19302', 'stun:stun.cloudflare.com:3478'] }, ...(urls.length ? [{ urls, username: process.env.TURN_USERNAME, credential: process.env.TURN_CREDENTIAL }] : [])], + fallback: { transport: 'websocket', endpoint: '/api/events/frame-stream' }, + }) + }) + + router.post('/ice-metrics', (req, res) => { + const { gatheringMs, success, hasRelay } = req.body || {} + if (!Number.isFinite(gatheringMs) || typeof success !== 'boolean' || typeof hasRelay !== 'boolean') { res.status(400).json({ error: 'invalid ICE metric' }); return } + console.info('[ice-metric]', { gatheringMs, success, hasRelay }) + res.status(202).json({ accepted: true }) + }) + + return router +} + +export const eventsRouter = createEventsRouter() diff --git a/server/routes/index.ts b/server/routes/index.ts new file mode 100644 index 00000000..31e63e26 --- /dev/null +++ b/server/routes/index.ts @@ -0,0 +1,30 @@ +import type { Express } from 'express'; +import { createDispatchRouter } from './dispatch.ts'; +import { createEventsRouter } from './events.ts'; +import { createSyncRouter } from './sync.ts'; +import { installGraphQLRoutes } from '../graphql/server.ts'; +import type { DomainStore } from '../lib/domainStore.ts'; + +export interface DomainRoutesOptions { + store?: DomainStore; + graphqlPath?: string; +} + +const installed = new WeakSet(); + +/** + * Mount the canonical domain surface exactly once per app. + * + * REST and GraphQL are installed from the same store instance; that is what + * guarantees a write on either surface is visible to the other immediately. + */ +export function installDomainRoutes(app: Express, options: DomainRoutesOptions = {}): boolean { + if (installed.has(app)) return false; + installed.add(app); + + app.use('/api/dispatch', createDispatchRouter({ store: options.store })); + app.use('/api/events', createEventsRouter({ store: options.store })); + app.use('/api/sync', createSyncRouter({ store: options.store })); + installGraphQLRoutes(app, { store: options.store, path: options.graphqlPath ?? '/graphql' }); + return true; +} diff --git a/server/routes/sync.ts b/server/routes/sync.ts index f42c1020..cbe9748d 100644 --- a/server/routes/sync.ts +++ b/server/routes/sync.ts @@ -1,12 +1,69 @@ import { Router } from 'express' +import { z } from 'zod' +import { getDomainStore, type DomainStore } from '../lib/domainStore.ts' +import { invalidateIncident } from '../lib/sqlCache.ts' + type Operation = { key: string; value?: unknown; deleted: boolean; actor: string; counter: number; timestamp: number } const operations = new Map() -export const syncRouter = Router() -syncRouter.post('/', (req, res) => { - const incoming = Array.isArray(req.body?.operations) ? req.body.operations : [] - for (const op of incoming) { - if (typeof op?.actor !== 'string' || !Number.isSafeInteger(op?.counter) || op.counter < 1) continue - operations.set(`${op.actor}:${op.counter}`, op) - } - res.json({ operations: [...operations.values()] }) + +const flushSchema = z.object({ + incidentId: z.string().min(1), + baseVersion: z.number().int().positive(), + patch: z.record(z.unknown()), }) + +/** + * Offline outbox sync + CRDT operations. + * + * `POST /flush` is the canonical conflict contract: + * 200 { ok, serverVersion, incident } + * 409 { conflict, serverVersion, mergeProposal } ← same as GraphQL SyncConflict + * 422 { error, issues[] } ← same as GraphQL ValidationError + */ +export function createSyncRouter(options: { store?: DomainStore } = {}): Router { + const store = options.store ?? getDomainStore() + const router = Router() + + router.post('/', (req, res) => { + const incoming = Array.isArray(req.body?.operations) ? req.body.operations : [] + for (const op of incoming) { + if (typeof op?.actor !== 'string' || !Number.isSafeInteger(op?.counter) || op.counter < 1) continue + operations.set(`${op.actor}:${op.counter}`, op) + } + res.json({ operations: [...operations.values()] }) + }) + + router.post('/flush', (req, res) => { + const parsed = flushSchema.safeParse(req.body) + if (!parsed.success) { + res.status(422).json({ + error: 'validation_failed', + issues: parsed.error.issues.map((issue) => ({ + path: issue.path.join('.') || 'body', + message: issue.message, + })), + }) + return + } + + const result = store.syncFlush(parsed.data) + if (result.__typename === 'SyncSuccess') { + invalidateIncident(parsed.data.incidentId) + res.json({ ok: true, serverVersion: result.serverVersion, incident: result.incident }) + return + } + if (result.__typename === 'SyncConflict') { + res.status(409).json({ + conflict: true, + serverVersion: result.serverVersion, + mergeProposal: result.mergeProposal, + }) + return + } + res.status(422).json({ error: result.error, issues: result.issues }) + }) + + return router +} + +export const syncRouter = createSyncRouter() diff --git a/server/telemetry.js b/server/telemetry.js new file mode 100644 index 00000000..c3ca06bb --- /dev/null +++ b/server/telemetry.js @@ -0,0 +1,303 @@ +/** + * Bounded telemetry receiver. + * + * The OTLP exporter used to queue spans in memory with no backpressure. This + * receiver makes the bound explicit: + * - maxQueueSize entries, priority-aware drop-oldest (heartbeat → normal → + * critical); + * - export failure marks the collector unhealthy and returns Retry-After; + * - every drop is reason-coded, and the snapshot is exposed on /metrics. + */ + +const PRIORITY_RANK = { heartbeat: 1, normal: 2, critical: 3 }; + +function priorityOf(span) { + const value = span?.priority; + return PRIORITY_RANK[value] ? value : 'normal'; +} + +function asArray(value) { + return Array.isArray(value) ? value : []; +} + +/** + * Flatten an OTLP/HTTP JSON payload into the receiver's internal span shape. + * Accepts both raw `{ spans: [...] }` and OTLP `{ resourceSpans: [...] }`. + */ +export function normalizeSpans(body) { + if (!body || typeof body !== 'object') return []; + if (Array.isArray(body.spans)) { + return body.spans.map(normalizeSpan).filter(Boolean); + } + const out = []; + for (const resource of asArray(body.resourceSpans)) { + for (const scope of asArray(resource?.scopeSpans)) { + for (const span of asArray(scope?.spans)) { + const normalized = normalizeSpan(span); + if (normalized) out.push(normalized); + } + } + } + return out; +} + +function normalizeSpan(span) { + if (!span || typeof span !== 'object') return null; + const attributes = Array.isArray(span.attributes) + ? Object.fromEntries( + span.attributes + .filter((attribute) => attribute && typeof attribute.key === 'string') + .map((attribute) => { + const value = attribute.value || {}; + return [ + attribute.key, + value.stringValue ?? + value.doubleValue ?? + value.intValue ?? + value.boolValue ?? + null, + ]; + }), + ) + : span.attributes || {}; + return { + name: String(span.name || 'unknown'), + priority: priorityOf({ priority: attributes['helphone.priority'] ?? span.priority }), + kind: String(attributes['helphone.kind'] ?? span.kind ?? 'unknown'), + ts: Number(span.ts ?? span.startTimeUnixNano / 1e6) || Date.now(), + attributes, + }; +} + +export class TelemetryReceiver { + constructor(options = {}) { + this.maxQueueSize = Math.max(1, Math.floor(options.maxQueueSize ?? 1_000)); + this.maxBatchSize = Math.max(1, Math.floor(options.maxBatchSize ?? 100)); + this.flushIntervalMs = Math.max(50, Math.floor(options.flushIntervalMs ?? 1_000)); + this.retryAfterMs = Math.max(100, Math.floor(options.retryAfterMs ?? 2_000)); + this.exporter = options.exporter ?? null; + this.now = options.now ?? (() => Date.now()); + + this.queue = []; + this.received = 0; + this.admitted = 0; + this.rejected = 0; + this.evicted = 0; + this.exported = 0; + this.exportFailed = 0; + this.dropped = 0; + this.droppedReasons = { + overflow_heartbeat: 0, + overflow_normal: 0, + overflow_critical: 0, + export_fail: 0, + invalid: 0, + }; + this.healthy = Boolean(this.exporter); + this.lastError = null; + this.lastExportAt = null; + this.unhealthyUntil = 0; + this.timer = null; + this.flushing = false; + } + + /** + * @returns {{accepted:number,dropped:number,status:number,retryAfter:number|null}} + */ + ingest(spans) { + const incoming = asArray(spans).filter(Boolean); + if (incoming.length === 0) { + return { accepted: 0, dropped: 0, status: 202, retryAfter: null }; + } + + this.received += incoming.length; + let accepted = 0; + let rejected = 0; + for (const span of incoming) { + if (!span || typeof span.name !== 'string') { + this.droppedReasons.invalid++; + this.dropped++; + this.rejected++; + rejected++; + continue; + } + if (!this.admit(span)) { + rejected++; + } else { + accepted++; + } + } + + const now = this.now(); + let status = 202; + let retryAfter = null; + + if (!this.healthy || now < this.unhealthyUntil) { + status = 503; + retryAfter = Math.max(1, Math.ceil((this.unhealthyUntil - now) / 1_000)) || 1; + } else if (this.queue.length >= Math.floor(this.maxQueueSize * 0.9)) { + status = 429; + retryAfter = 1; + } + + return { accepted, dropped: rejected, status, retryAfter }; + } + + admit(span) { + if (this.queue.length < this.maxQueueSize) { + this.queue.push(span); + this.admitted++; + return true; + } + + // Find the lowest-priority victim. Never drop a critical span while a + // heartbeat or normal span remains. + const victimIndex = this.findVictimIndex(); + if (victimIndex === -1) { + // Queue is all critical and full: reject the incoming lower/equal span. + const priority = priorityOf(span); + this.droppedReasons[`overflow_${priority}`]++; + this.dropped++; + this.rejected++; + return false; + } + + const victim = this.queue[victimIndex]; + const victimPriority = priorityOf(victim); + const incomingPriority = priorityOf(span); + if (PRIORITY_RANK[incomingPriority] < PRIORITY_RANK[victimPriority]) { + this.droppedReasons[`overflow_${incomingPriority}`]++; + this.dropped++; + this.rejected++; + return false; + } + + this.queue.splice(victimIndex, 1); + this.droppedReasons[`overflow_${victimPriority}`]++; + this.dropped++; + this.evicted++; + this.queue.push(span); + this.admitted++; + return true; + } + + findVictimIndex() { + for (const priority of ['heartbeat', 'normal', 'critical']) { + const index = this.queue.findIndex((span) => priorityOf(span) === priority); + if (index !== -1) return index; + } + return -1; + } + + async flush() { + if (this.flushing) return { status: 'in_flight', count: 0 }; + if (this.queue.length === 0) return { status: 'empty', count: 0 }; + if (!this.exporter) { + this.markUnhealthy('no exporter configured'); + return { status: 'failed', count: 0, error: this.lastError }; + } + + this.flushing = true; + const batch = this.queue.splice(0, this.maxBatchSize); + try { + await this.exporter.export(batch); + this.exported += batch.length; + this.healthy = true; + this.unhealthyUntil = 0; + this.lastError = null; + this.lastExportAt = this.now(); + return { status: 'exported', count: batch.length }; + } catch (error) { + this.exportFailed += batch.length; + this.dropped += batch.length; + this.droppedReasons.export_fail += batch.length; + this.markUnhealthy(String(error?.message || error), error?.retryAfterMs); + return { status: 'failed', count: batch.length, error: this.lastError }; + } finally { + this.flushing = false; + } + } + + markUnhealthy(message, retryAfterMs = this.retryAfterMs) { + this.healthy = false; + this.lastError = message; + this.unhealthyUntil = this.now() + Math.max(100, retryAfterMs); + } + + start() { + if (this.timer) return; + this.timer = setInterval(() => { + this.flush().catch(() => {}); + }, this.flushIntervalMs); + this.timer.unref?.(); + } + + stop() { + if (this.timer) clearInterval(this.timer); + this.timer = null; + } + + snapshot() { + return { + maxQueueSize: this.maxQueueSize, + maxBatchSize: this.maxBatchSize, + queueDepth: this.queue.length, + occupancy: this.queue.length / this.maxQueueSize, + received: this.received, + admitted: this.admitted, + rejected: this.rejected, + evicted: this.evicted, + exported: this.exported, + exportFailed: this.exportFailed, + dropped: this.dropped, + droppedReasons: { ...this.droppedReasons }, + conservation: { + receivedOk: this.received === this.admitted + this.rejected, + admittedOk: + this.admitted === this.exported + this.exportFailed + this.evicted + this.queue.length, + ok: + this.received === this.admitted + this.rejected && + this.admitted === this.exported + this.exportFailed + this.evicted + this.queue.length, + }, + healthy: this.healthy, + lastError: this.lastError, + lastExportAt: this.lastExportAt, + retryAfterMs: Math.max(0, this.unhealthyUntil - this.now()), + }; + } +} + +/** OTLP/HTTP exporter with a hard timeout. */ +export function createOtlpExporter(options = {}) { + const { + endpoint = 'http://localhost:4318/v1/traces', + fetchImpl = globalThis.fetch, + timeoutMs = 5_000, + headers = {}, + } = options; + + return { + async export(spans) { + const controller = typeof AbortController !== 'undefined' ? new AbortController() : null; + const timer = controller ? setTimeout(() => controller.abort(), timeoutMs) : null; + try { + const response = await fetchImpl(endpoint, { + method: 'POST', + headers: { 'content-type': 'application/json', ...headers }, + body: JSON.stringify({ spans }), + signal: controller?.signal, + }); + if (!response.ok) { + const retryAfter = Number(response.headers?.get?.('retry-after')); + const error = new Error(`OTLP collector responded ${response.status}`); + error.retryAfterMs = Number.isFinite(retryAfter) ? retryAfter * 1_000 : undefined; + throw error; + } + } finally { + if (timer) clearTimeout(timer); + } + }, + }; +} + +export default TelemetryReceiver; diff --git a/server/telemetryBootstrap.js b/server/telemetryBootstrap.js new file mode 100644 index 00000000..7f46af9c --- /dev/null +++ b/server/telemetryBootstrap.js @@ -0,0 +1,95 @@ +/** + * Server telemetry bootstrap. + * + * Wires the bounded receiver into Express and exposes the same counters as + * the client pipeline. Deliberately avoids a second OTel SDK registration: + * instrumentations are opt-in via env, and this module only owns the queue. + */ + +import { + TelemetryReceiver, + createOtlpExporter, + normalizeSpans, +} from './telemetry.js'; + +let receiver = null; +let installed = false; + +export function createTelemetryReceiverFromEnv(env = process.env) { + const endpoint = env.OTEL_EXPORTER_OTLP_ENDPOINT || env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT; + const exporter = endpoint + ? createOtlpExporter({ + endpoint: endpoint.includes('/v1/traces') ? endpoint : `${endpoint.replace(/\/$/, '')}/v1/traces`, + fetchImpl: env.TELEMETRY_FETCH_IMPL || globalThis.fetch, + }) + : null; + + const instance = new TelemetryReceiver({ + maxQueueSize: Number(env.TELEMETRY_MAX_QUEUE || 1_000), + maxBatchSize: Number(env.TELEMETRY_MAX_BATCH || 100), + flushIntervalMs: Number(env.TELEMETRY_FLUSH_MS || 1_000), + retryAfterMs: Number(env.TELEMETRY_RETRY_AFTER_MS || 2_000), + exporter, + }); + instance.start(); + return instance; +} + +export function getTelemetryReceiver(env = process.env) { + if (!receiver) receiver = createTelemetryReceiverFromEnv(env); + return receiver; +} + +export function setTelemetryReceiver(instance) { + receiver = instance; +} + +/** Express handler for POST /api/telemetry/spans. */ +export function telemetryIngestHandler(req, res) { + const instance = getTelemetryReceiver(); + const spans = normalizeSpans(req.body); + const result = instance.ingest(spans); + + res.set('Cache-Control', 'no-store'); + if (result.retryAfter) res.set('Retry-After', String(result.retryAfter)); + if (result.status === 503) res.set('X-Telemetry-Backpressure', 'collector-unavailable'); + else if (result.status === 429) res.set('X-Telemetry-Backpressure', 'queue-high-watermark'); + + res.status(result.status).json({ + accepted: result.accepted, + dropped: result.dropped, + queueDepth: instance.queue.length, + retryAfter: result.retryAfter, + status: result.status === 202 ? 'accepted' : 'shed', + }); +} + +export function telemetryMetricsHandler(_req, res) { + res.set('Cache-Control', 'no-store').json(getTelemetryReceiver().snapshot()); +} + +/** + * Mount telemetry routes once. Returns false when already installed so a + * double import cannot double-register handlers (and double count). + */ +export function installTelemetryRoutes(app) { + if (installed) return false; + installed = true; + app.post('/api/telemetry/spans', telemetryIngestHandler); + app.get('/metrics/telemetry', telemetryMetricsHandler); + return true; +} + +export async function shutdownTelemetry() { + if (!receiver) return; + receiver.stop(); + await receiver.flush().catch(() => {}); + receiver = null; + installed = false; +} + +export function resetTelemetryBootstrapForTests() { + if (receiver) receiver.stop(); + receiver = null; + installed = false; +} diff --git a/server/tests/telemetry-backpressure.node-test.mjs b/server/tests/telemetry-backpressure.node-test.mjs new file mode 100644 index 00000000..049f0236 --- /dev/null +++ b/server/tests/telemetry-backpressure.node-test.mjs @@ -0,0 +1,106 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { TelemetryReceiver, normalizeSpans } from '../telemetry.js'; + +function span(i, priority = 'normal') { + return { name: `span-${i}`, priority, ts: Date.now() + i, attributes: {} }; +} + +test('bounded queue never exceeds maxQueueSize under a 500 msg/s burst', () => { + const receiver = new TelemetryReceiver({ + maxQueueSize: 100, + maxBatchSize: 25, + exporter: { async export() {} }, + }); + + // 500 messages/s for 10 simulated seconds = 5,000 frames. + for (let i = 0; i < 5_000; i++) { + receiver.ingest([span(i, i % 20 === 0 ? 'critical' : i % 3 === 0 ? 'heartbeat' : 'normal')]); + assert.ok(receiver.queue.length <= 100, `queue grew to ${receiver.queue.length}`); + } + + const snapshot = receiver.snapshot(); + assert.equal(snapshot.dropped, snapshot.rejected + snapshot.evicted); + assert.ok(snapshot.droppedReasons.overflow_heartbeat > 0); + assert.equal(snapshot.received, 5_000); + assert.equal(snapshot.admitted + snapshot.rejected, 5_000); + assert.equal(snapshot.conservation.ok, true); + assert.equal(snapshot.queueDepth, 100); +}); + +test('collector outage returns 503 + Retry-After and bounds the heap queue', async () => { + let now = 0; + const receiver = new TelemetryReceiver({ + maxQueueSize: 50, + now: () => now, + exporter: { + async export() { + const error = new Error('collector down'); + error.retryAfterMs = 5_000; + throw error; + }, + }, + }); + + for (let i = 0; i < 50; i++) receiver.ingest([span(i)]); + await receiver.flush(); + + const result = receiver.ingest([span(51)]); + assert.equal(result.status, 503); + assert.equal(result.retryAfter, 5); + + // Five simulated minutes of outage: the queue stays capped. + for (let i = 0; i < 10_000; i++) { + now += 30; + receiver.ingest([span(i, i % 10 === 0 ? 'critical' : 'normal')]); + } + const snapshot = receiver.snapshot(); + assert.ok(snapshot.queueDepth <= 50); + assert.ok(snapshot.dropped > 0); + assert.ok(snapshot.droppedReasons.export_fail > 0); +}); + +test('never drops a critical span before heartbeat/normal spans', async () => { + const receiver = new TelemetryReceiver({ + maxQueueSize: 3, + maxBatchSize: 10, + exporter: { async export() {} }, + }); + receiver.ingest([span(1, 'heartbeat')]); + receiver.ingest([span(2, 'normal')]); + receiver.ingest([span(3, 'critical')]); + + receiver.ingest([span(4, 'critical')]); + receiver.ingest([span(5, 'critical')]); + receiver.ingest([span(6, 'critical')]); + + const snapshot = receiver.snapshot(); + assert.equal(snapshot.droppedReasons.overflow_heartbeat, 1); + assert.equal(snapshot.droppedReasons.overflow_normal, 1); + assert.equal(snapshot.droppedReasons.overflow_critical, 1); + assert.equal(snapshot.queueDepth, 3); + assert.equal(receiver.queue.filter((s) => s.priority === 'critical').length, 3); +}); + +test('normalizes OTLP resourceSpans into receiver spans', () => { + const spans = normalizeSpans({ + resourceSpans: [ + { + scopeSpans: [ + { + spans: [ + { + name: 'dispatch.ack', + startTimeUnixNano: '1700000000000000000', + attributes: [{ key: 'helphone.priority', value: { stringValue: 'critical' } }], + }, + ], + }, + ], + }, + ], + }); + assert.equal(spans.length, 1); + assert.equal(spans[0].priority, 'critical'); + assert.equal(spans[0].ts, 1_700_000_000_000); +}); diff --git a/src/components/TelemetryLab.jsx b/src/components/TelemetryLab.jsx new file mode 100644 index 00000000..82e0fcf8 --- /dev/null +++ b/src/components/TelemetryLab.jsx @@ -0,0 +1,241 @@ +import { useEffect, useMemo, useState } from 'react'; +import { + getTelemetryLabController, + type LabSnapshot, +} from '../lib/telemetryLabController'; +import type { NetworkClass } from '../lib/networkEstimator'; + +const COLORS = { + ink: '#234B4E', + coral: '#FF7A6B', + purple: '#7357FF', + green: '#2F7D5B', + cream: '#ECE0CC', + muted: '#6b7a76', + line: 'rgba(35,75,78,0.14)', +}; + +const CARD: React.CSSProperties = { + background: '#fff', + border: `1px solid ${COLORS.line}`, + borderRadius: 14, + padding: '14px 16px', + minWidth: 150, + flex: '1 1 150px', +}; + +function metric(label: string, value: number | string, hint?: string) { + return ( +
+
+ {label} +
+
+ {typeof value === 'number' ? value.toLocaleString() : value} +
+ {hint ?
{hint}
: null} +
+ ); +} + +function pct(value: number): string { + return `${(value * 100).toFixed(1)}%`; +} + +/** + * Truthful live view of produced / parsed / dropped / exported. + * + * The old TelemetryLab rendered a smooth throughput graph because it polled a + * monotonic drop counter too slowly. This view polls the pipeline snapshot + * every second, shows every drop reason, and marks conservation red the moment + * produced ≠ exported + queued + dropped. + */ +export default function TelemetryLab() { + const controller = useMemo(() => getTelemetryLabController(), []); + const [snapshot, setSnapshot] = useState(() => controller.snapshot()); + const [rate, setRate] = useState(500); + const [network, setNetwork] = useState('4g'); + + useEffect(() => { + const timer = setInterval(() => setSnapshot(controller.snapshot()), 1_000); + return () => clearInterval(timer); + }, [controller]); + + useEffect(() => () => controller.destroy(), [controller]); + + const reasons = snapshot.droppedReasons; + const reasonTotal = Object.values(reasons).reduce((a, b) => a + b, 0); + const conservation = snapshot.conservation; + + return ( +
+
+
+ Telemetry Lab +
+

Produced → parsed → exported, with reasons

+

+ Every counter below is derived from the same ring-buffer snapshot the pipeline uses for backpressure. + Start a flood, then watch the worker pause at 90% occupancy and the drop ledger explain each shed frame. +

+
+ +
+ + + + + + + + + + + + {conservation.ok ? 'Accounting balanced' : 'Accounting mismatch'} · accounted {conservation.accounted.toLocaleString()} + +
+ +
+ {metric('Produced', snapshot.produced, 'frames offered')} + {metric('Parsed', snapshot.parsed, `${snapshot.lab.workerMode}`)} + {metric('Admitted', snapshot.admitted, 'ring accepted')} + {metric('Dropped', snapshot.dropped, `${pct(snapshot.produced ? snapshot.dropped / snapshot.produced : 0)} of produced`)} + {metric('Exported', snapshot.exported, snapshot.exportFailed ? `${snapshot.exportFailed} failed` : 'collector OK')} + {metric('Queue', snapshot.queueDepth, `${pct(snapshot.occupancy)} occupancy`)} + {metric('Pressure', snapshot.pressure, `${snapshot.networkClass} profile`)} + {metric('Sampled', snapshot.sampled, `normal ${pct(snapshot.profile.normalSampleRate)}`)} +
+ +
+
+

Drop ledger

+ {reasonTotal === 0 ? ( +

No drops recorded yet.

+ ) : ( + + + + + + + + + + {Object.entries(reasons) + .filter(([, count]) => count > 0) + .sort((a, b) => b[1] - a[1]) + .map(([reason, count]) => ( + + + + + + ))} + +
ReasonCountShare
{reason}{count.toLocaleString()}{pct(count / reasonTotal)}
+ )} + +
+ Priority policy: heartbeat evicted first, then normal, then critical. At 90% occupancy the producer + sheds non-critical frames with because=backpressure; dispatch-critical frames are never + sampled. +
+
+ +
+

Batch profile

+
+
maxQueueSize
{snapshot.profile.maxQueueSize}
+
scheduledDelayMillis
{snapshot.profile.scheduledDelayMillis}
+
maxExportBatchSize
{snapshot.profile.maxExportBatchSize}
+
heartbeat sample
{pct(snapshot.profile.heartbeatSampleRate)}
+
outbox only
{snapshot.profile.outboxOnly ? 'yes' : 'no'}
+
exporter
{snapshot.exporterHealthy ? 'healthy' : 'retrying'}
+
outboxed
{snapshot.outboxed.toLocaleString()}
+
+ {snapshot.lastError ? ( +

{snapshot.lastError}

+ ) : null} +
+
+ +
+ Also available at /lab/telemetry. Snapshot refreshes every 1s; the ring never overwrites + without incrementing a reason. +
+
+ ); +} diff --git a/src/lib/networkEstimator.ts b/src/lib/networkEstimator.ts new file mode 100644 index 00000000..87d2348d --- /dev/null +++ b/src/lib/networkEstimator.ts @@ -0,0 +1,183 @@ +/** + * Network-class estimation + per-device OTel batch tunables. + * + * The telemetry pipeline must not use one batching policy for every device. + * A 2G handset needs few, infrequent exports; a 4G device can afford larger + * batches. This module is deliberately dependency-free and side-effect-free + * so it can run in the browser, a worker, and Node tests. + */ + +export type NetworkClass = 'offline' | '2g' | '3g' | '4g'; + +export interface NetworkSample { + online?: boolean; + effectiveType?: string; + downlinkMbps?: number; + rttMs?: number; + saveData?: boolean; +} + +export interface NetworkProfile { + /** OTel BatchSpanProcessor maxQueueSize analogue. */ + maxQueueSize: number; + /** OTel scheduledDelayMillis analogue. */ + scheduledDelayMillis: number; + /** OTel maxExportBatchSize analogue. */ + maxExportBatchSize: number; + /** Fraction of normal-priority spans kept at normal ring pressure. */ + normalSampleRate: number; + /** Fraction of heartbeat spans kept at normal ring pressure. */ + heartbeatSampleRate: number; + /** Dispatch-critical spans are never sampled. */ + criticalSampleRate: 1; + /** When true, exports are diverted to the durable outbox instead of memory. */ + outboxOnly: boolean; +} + +export const NETWORK_PROFILES: Record = { + // 2G: ≤20% of a mixed baseline (80% normal / 20% heartbeat) survives, + // while dispatch-critical spans are preserved at 100%. + '2g': { + maxQueueSize: 50, + scheduledDelayMillis: 5_000, + maxExportBatchSize: 5, + normalSampleRate: 0.2, + heartbeatSampleRate: 0.02, + criticalSampleRate: 1, + outboxOnly: false, + }, + '3g': { + maxQueueSize: 200, + scheduledDelayMillis: 2_000, + maxExportBatchSize: 20, + normalSampleRate: 0.5, + heartbeatSampleRate: 0.1, + criticalSampleRate: 1, + outboxOnly: false, + }, + '4g': { + maxQueueSize: 1_000, + scheduledDelayMillis: 1_000, + maxExportBatchSize: 200, + normalSampleRate: 1, + heartbeatSampleRate: 0.25, + criticalSampleRate: 1, + outboxOnly: false, + }, + offline: { + maxQueueSize: 100, + scheduledDelayMillis: 0, + maxExportBatchSize: 50, + normalSampleRate: 1, + heartbeatSampleRate: 0.05, + criticalSampleRate: 1, + outboxOnly: true, + }, +}; + +/** Pure classifier — the single source of truth for every runtime. */ +export function classifyNetwork(sample: NetworkSample = {}): NetworkClass { + if (sample.online === false) return 'offline'; + if (sample.saveData) return '2g'; + + const effective = String(sample.effectiveType || '').toLowerCase(); + if (effective === 'slow-2g' || effective === '2g') return '2g'; + if (effective === '3g') return '3g'; + if (effective === '4g') return '4g'; + + const rtt = sample.rttMs; + const downlink = sample.downlinkMbps; + + if (Number.isFinite(rtt) && (rtt as number) >= 1_000) return '2g'; + if (Number.isFinite(downlink) && (downlink as number) >= 0 && (downlink as number) < 0.5) return '2g'; + if (Number.isFinite(rtt) && (rtt as number) >= 300) return '3g'; + if (Number.isFinite(downlink) && (downlink as number) >= 0 && (downlink as number) < 1.5) return '3g'; + + // No signal at all is treated as a healthy device, not as offline: the + // browser may not expose NetworkInformation at all (Safari/Firefox). + return '4g'; +} + +export function profileFor(networkClass: NetworkClass): NetworkProfile { + return NETWORK_PROFILES[networkClass] ?? NETWORK_PROFILES['4g']; +} + +type ConnectionLike = { + effectiveType?: string; + downlink?: number; + rtt?: number; + saveData?: boolean; + addEventListener?: (type: 'change', listener: () => void) => void; + removeEventListener?: (type: 'change', listener: () => void) => void; +}; + +function connectionOf(env: unknown): ConnectionLike | undefined { + const nav = (env as { navigator?: { connection?: ConnectionLike; onLine?: boolean } } | undefined)?.navigator; + return nav?.connection; +} + +/** Read the ambient network state (browser or injected environment). */ +export function estimateNetwork(env: unknown = globalThis): NetworkClass { + const nav = (env as { navigator?: { onLine?: boolean; connection?: ConnectionLike } } | undefined)?.navigator; + const connection = nav?.connection; + return classifyNetwork({ + online: nav?.onLine, + effectiveType: connection?.effectiveType, + downlinkMbps: connection?.downlink, + rttMs: connection?.rtt, + saveData: connection?.saveData, + }); +} + +export interface NetworkMonitor { + get(): NetworkClass; + subscribe(listener: (networkClass: NetworkClass) => void): () => void; + stop(): void; +} + +/** + * Subscribe to browser connectivity changes. Returns a monitor with an + * explicit `stop()` so React effects and workers can release listeners. + */ +export function createNetworkMonitor(env: unknown = globalThis, intervalMs = 0): NetworkMonitor { + const nav = (env as { + navigator?: { onLine?: boolean; connection?: ConnectionLike }; + addEventListener?: (type: string, listener: () => void) => void; + removeEventListener?: (type: string, listener: () => void) => void; + } | undefined)?.navigator; + + const listeners = new Set<(networkClass: NetworkClass) => void>(); + let current = estimateNetwork(env); + let timer: ReturnType | null = null; + + const emit = () => { + const next = estimateNetwork(env); + if (next === current) return; + current = next; + for (const listener of listeners) listener(next); + }; + + const connection = nav?.connection; + const onConnectionChange = () => emit(); + + if (nav?.addEventListener) nav.addEventListener('online', emit); + if (nav?.addEventListener) nav.addEventListener('offline', emit); + connection?.addEventListener?.('change', onConnectionChange); + if (intervalMs > 0) timer = setInterval(emit, intervalMs); + + return { + get: () => current, + subscribe(listener) { + listeners.add(listener); + return () => listeners.delete(listener); + }, + stop() { + listeners.clear(); + if (timer) clearInterval(timer); + timer = null; + nav?.removeEventListener?.('online', emit); + nav?.removeEventListener?.('offline', emit); + connection?.removeEventListener?.('change', onConnectionChange); + }, + }; +} diff --git a/src/lib/ringBuffer.ts b/src/lib/ringBuffer.ts new file mode 100644 index 00000000..cb92b137 --- /dev/null +++ b/src/lib/ringBuffer.ts @@ -0,0 +1,329 @@ +/** + * Priority-aware bounded ring buffer for telemetry frames. + * + * Design goals: + * - Never silently overwrite the oldest frame. Every eviction is counted and + * reason-coded. + * - Dispatch-critical frames are never dropped before heartbeat frames: + * when the ring is full, a heartbeat is evicted first, then a normal frame, + * and only then another critical frame. + * - Occupancy is a first-class signal (70% high, 90% critical, 50% resume) + * with hysteresis so producers and workers can react without oscillating. + * + * The implementation is O(1) per admitted/evicted frame: insertion order is a + * doubly linked list, and each priority has a FIFO of nodes (stale entries are + * skipped lazily when they are popped). + */ + +export type TelemetryPriority = 'heartbeat' | 'normal' | 'critical'; + +export type RingDropReason = + | 'overflow_heartbeat' + | 'overflow_normal' + | 'overflow_critical'; + +export type PressureLevel = 'normal' | 'high' | 'critical'; + +export interface TelemetryFrame { + /** Monotonic producer sequence, used for conservation checks + sampling. */ + seq: number; + priority: TelemetryPriority; + kind: string; + ts: number; + payload: T; +} + +export interface RingBufferOptions { + capacity?: number; + /** Occupancy at which `high` pressure is reported. */ + highWatermark?: number; + /** Occupancy at which producers should shed / worker should pause. */ + shedWatermark?: number; + /** Occupancy below which pressure returns to `normal`. */ + lowWatermark?: number; +} + +export interface RingSnapshot { + capacity: number; + size: number; + occupancy: number; + pressure: PressureLevel; + offered: number; + admitted: number; + rejected: number; + evicted: number; + drained: number; + sampled: number; + dropped: number; + droppedReasons: Record; + conservation: { + /** offered === admitted + rejected */ + offeredOk: boolean; + /** admitted === evicted + drained + size */ + admittedOk: boolean; + ok: boolean; + }; +} + +export interface PressureEvent { + level: PressureLevel; + occupancy: number; + size: number; + capacity: number; +} + +interface Node { + frame: TelemetryFrame; + prev: Node | null; + next: Node | null; + active: boolean; +} + +const PRIORITY_RANK: Record = { + heartbeat: 1, + normal: 2, + critical: 3, +}; + +const DROP_REASON: Record = { + heartbeat: 'overflow_heartbeat', + normal: 'overflow_normal', + critical: 'overflow_critical', +}; + +function clampWatermark(value: number | undefined, fallback: number): number { + if (!Number.isFinite(value)) return fallback; + return Math.min(1, Math.max(0, value as number)); +} + +export class RingBuffer { + readonly capacity: number; + private readonly highWatermark: number; + private readonly shedWatermark: number; + private readonly lowWatermark: number; + + private head: Node | null = null; + private tail: Node | null = null; + private size = 0; + + private readonly queues: Record[]> = { + heartbeat: [], + normal: [], + critical: [], + }; + + private offered = 0; + private admitted = 0; + private rejected = 0; + private evicted = 0; + private drained = 0; + private sampled = 0; + + private readonly droppedReasons: Record = { + overflow_heartbeat: 0, + overflow_normal: 0, + overflow_critical: 0, + }; + + private pressure: PressureLevel = 'normal'; + private readonly pressureListeners = new Set<(event: PressureEvent) => void>(); + + constructor(options: RingBufferOptions = {}) { + this.capacity = Math.max(1, Math.floor(options.capacity ?? 2_048)); + this.highWatermark = clampWatermark(options.highWatermark, 0.7); + this.shedWatermark = Math.max( + this.highWatermark, + clampWatermark(options.shedWatermark, 0.9), + ); + this.lowWatermark = Math.min( + this.highWatermark, + clampWatermark(options.lowWatermark, 0.5), + ); + } + + get length(): number { + return this.size; + } + + get occupancy(): number { + return this.size / this.capacity; + } + + get pressureLevel(): PressureLevel { + return this.pressure; + } + + /** Offer a frame. Returns false when it was shed at admission. */ + push(frame: TelemetryFrame): boolean { + this.offered++; + + if (this.size >= this.capacity) { + const victim = this.findEvictionVictim(); + // Should never happen (capacity > 0), but keep the invariant explicit. + if (!victim) { + this.rejected++; + this.bumpDrop(frame.priority); + this.updatePressure(); + return false; + } + + // Never sacrifice a higher-priority frame for a lower-priority arrival. + if (PRIORITY_RANK[frame.priority] < PRIORITY_RANK[victim.frame.priority]) { + this.rejected++; + this.bumpDrop(frame.priority); + this.updatePressure(); + return false; + } + + this.unlink(victim); + this.evicted++; + this.bumpDrop(victim.frame.priority); + } + + const node: Node = { frame, prev: null, next: null, active: true }; + if (this.tail) { + this.tail.next = node; + node.prev = this.tail; + this.tail = node; + } else { + this.head = node; + this.tail = node; + } + this.queues[frame.priority].push(node); + this.size++; + this.admitted++; + this.updatePressure(); + return true; + } + + /** Remove up to `max` frames in insertion order. */ + drain(max = Number.POSITIVE_INFINITY): TelemetryFrame[] { + const limit = Math.max(0, Math.floor(max)); + const out: TelemetryFrame[] = []; + while (this.head && out.length < limit) { + const node = this.head; + this.unlink(node); + this.drained++; + out.push(node.frame); + } + if (out.length > 0) this.updatePressure(); + return out; + } + + /** Peek at the oldest frame without removing it. */ + peek(): TelemetryFrame | undefined { + return this.head?.frame; + } + + /** Account for frames sampled before they reached the ring. */ + recordSampled(count = 1): void { + if (count > 0) this.sampled += count; + } + + /** Register a pressure listener. Fires only on level transitions. */ + onPressure(listener: (event: PressureEvent) => void): () => void { + this.pressureListeners.add(listener); + return () => this.pressureListeners.delete(listener); + } + + snapshot(): RingSnapshot { + const dropped = this.rejected + this.evicted; + return { + capacity: this.capacity, + size: this.size, + occupancy: this.occupancy, + pressure: this.pressure, + offered: this.offered, + admitted: this.admitted, + rejected: this.rejected, + evicted: this.evicted, + drained: this.drained, + sampled: this.sampled, + dropped, + droppedReasons: { ...this.droppedReasons }, + conservation: { + offeredOk: this.offered === this.admitted + this.rejected, + admittedOk: + this.admitted === this.evicted + this.drained + this.size, + ok: + this.offered === this.admitted + this.rejected && + this.admitted === this.evicted + this.drained + this.size, + }, + }; + } + + reset(): void { + this.head = null; + this.tail = null; + this.size = 0; + this.queues.heartbeat.length = 0; + this.queues.normal.length = 0; + this.queues.critical.length = 0; + this.offered = 0; + this.admitted = 0; + this.rejected = 0; + this.evicted = 0; + this.drained = 0; + this.sampled = 0; + this.droppedReasons.overflow_heartbeat = 0; + this.droppedReasons.overflow_normal = 0; + this.droppedReasons.overflow_critical = 0; + this.setPressure('normal'); + } + + private findEvictionVictim(): Node | undefined { + for (const priority of ['heartbeat', 'normal', 'critical'] as const) { + const queue = this.queues[priority]; + while (queue.length > 0 && !queue[0].active) queue.shift(); + if (queue.length > 0) return queue[0]; + } + return undefined; + } + + private unlink(node: Node): void { + node.active = false; + if (node.prev) node.prev.next = node.next; + else this.head = node.next; + if (node.next) node.next.prev = node.prev; + else this.tail = node.prev; + node.prev = null; + node.next = null; + this.size--; + } + + private bumpDrop(priority: TelemetryPriority): void { + this.droppedReasons[DROP_REASON[priority]]++; + } + + private updatePressure(): void { + const occupancy = this.occupancy; + let next: PressureLevel = this.pressure; + + if (occupancy >= this.shedWatermark) next = 'critical'; + else if (occupancy >= this.highWatermark) next = 'high'; + else if (occupancy <= this.lowWatermark) next = 'normal'; + // Between low and high: keep the current level (hysteresis). + + this.setPressure(next); + } + + private setPressure(next: PressureLevel): void { + if (next === this.pressure) return; + this.pressure = next; + const event: PressureEvent = { + level: next, + occupancy: this.occupancy, + size: this.size, + capacity: this.capacity, + }; + for (const listener of this.pressureListeners) { + try { + listener(event); + } catch { + // A broken observer must never take down telemetry admission. + } + } + } +} + +export default RingBuffer; diff --git a/src/lib/syncManager.ts b/src/lib/syncManager.ts index 022aaea6..bd78bced 100644 --- a/src/lib/syncManager.ts +++ b/src/lib/syncManager.ts @@ -1,11 +1,139 @@ import { LwwElementSet, type LwwOperation } from './crdt' + +export interface SyncConflict { + conflict: true + serverVersion: number + mergeProposal: Record +} + +export interface ValidationIssue { + path: string + message: string +} + +export type SyncOutcome = + | { status: 'ok'; snapshot: Record; serverVersion?: number } + | { status: 'conflict'; conflict: SyncConflict } + | { status: 'validation'; issues: ValidationIssue[] } + | { status: 'auth'; reason: 'unauthorized' | 'forbidden' } + | { status: 'retry'; attempts: number; error: string } + +export interface SyncManagerOptions { + maxRetries?: number + baseDelayMs?: number + sleep?: (ms: number) => Promise + onConflict?: (conflict: SyncConflict) => void +} + +/** + * Offline outbox client. + * + * `sync()` keeps the legacy throw-on-error behaviour. `syncWithConflict()` is + * the contract-aware path: a 409 is a terminal, user-mergeable outcome (never + * an infinite retry), a 422 surfaces issues, and only 5xx/network errors retry + * with bounded exponential backoff. GraphQL union responses are understood too + * so REST-flush and GraphQL-flush clients behave identically. + */ export class SyncManager { - constructor(readonly document: LwwElementSet, readonly endpoint = '/api/sync') {} + private readonly maxRetries: number + private readonly baseDelayMs: number + private readonly sleep: (ms: number) => Promise + private readonly onConflict?: (conflict: SyncConflict) => void + + constructor( + readonly document: LwwElementSet, + readonly endpoint = '/api/sync', + options: SyncManagerOptions = {}, + ) { + this.maxRetries = options.maxRetries ?? 3 + this.baseDelayMs = options.baseDelayMs ?? 250 + this.sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))) + this.onConflict = options.onConflict + } + async sync(fetcher: typeof fetch = fetch) { - const response = await fetcher(this.endpoint, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(this.document.snapshot()) }) + const response = await fetcher(this.endpoint, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(this.document.snapshot()), + }) if (!response.ok) throw new Error(`sync failed: ${response.status}`) - const body = await response.json() as { operations: LwwOperation[] } + const body = (await response.json()) as { operations: LwwOperation[] } this.document.merge(body.operations) return this.document.snapshot() } + + async syncWithConflict(fetcher: typeof fetch = fetch): Promise> { + let lastError = 'sync failed' + for (let attempt = 1; attempt <= this.maxRetries + 1; attempt++) { + try { + const response = await fetcher(this.endpoint, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(this.document.snapshot()), + }) + + if (response.status === 401) return { status: 'auth', reason: 'unauthorized' } + if (response.status === 403) return { status: 'auth', reason: 'forbidden' } + + if (response.status === 409) { + const body = (await response.json()) as Partial & { + data?: { syncFlush?: SyncConflict } + } + const conflict = (body.data?.syncFlush ?? body) as SyncConflict + this.onConflict?.(conflict) + return { status: 'conflict', conflict } + } + + if (response.status === 422) { + const body = (await response.json()) as { issues?: ValidationIssue[] } + return { status: 'validation', issues: body.issues ?? [] } + } + + if (!response.ok) { + lastError = `sync failed: ${response.status}` + if (response.status < 500) return { status: 'validation', issues: [] } + if (attempt > this.maxRetries) break; + await this.sleep(this.baseDelayMs * 2 ** (attempt - 1)) + continue + } + + const body = (await response.json()) as { operations?: LwwOperation[]; data?: { syncFlush?: unknown } } + if (body.data?.syncFlush) { + return this.handleGraphQL(body.data.syncFlush as Record) + } + if (Array.isArray(body.operations)) this.document.merge(body.operations) + return { status: 'ok', snapshot: this.document.snapshot() } + } catch (error) { + lastError = String((error as Error)?.message ?? error) + if (attempt > this.maxRetries) break + await this.sleep(this.baseDelayMs * 2 ** (attempt - 1)) + } + } + return { status: 'retry', attempts: this.maxRetries + 1, error: lastError } + } + + private handleGraphQL(result: Record): SyncOutcome { + const typename = result.__typename + if (typename === 'SyncConflict') { + const conflict: SyncConflict = { + conflict: true, + serverVersion: Number(result.serverVersion) || 0, + mergeProposal: (result.mergeProposal as Record) ?? {}, + } + this.onConflict?.(conflict) + return { status: 'conflict', conflict } + } + if (typename === 'ValidationError') { + return { status: 'validation', issues: (result.issues as ValidationIssue[]) ?? [] } + } + if (typename === 'SyncSuccess') { + return { + status: 'ok', + snapshot: this.document.snapshot(), + serverVersion: Number(result.serverVersion) || undefined, + } + } + return { status: 'ok', snapshot: this.document.snapshot() } + } } diff --git a/src/lib/telemetry.ts b/src/lib/telemetry.ts new file mode 100644 index 00000000..20bef8e9 --- /dev/null +++ b/src/lib/telemetry.ts @@ -0,0 +1,536 @@ +/** + * Telemetry pipeline: producer → adaptive sampler → priority ring → exporter. + * + * The ring is the source of truth for admission/eviction; this class adds + * network-aware sampling, bounded export batches, offline outbox diversion, + * and reason-coded accounting that the TelemetryLab consumes. + */ + +import { + RingBuffer, + type PressureEvent, + type PressureLevel, + type RingBufferOptions, + type RingDropReason, + type TelemetryFrame, + type TelemetryPriority, +} from './ringBuffer'; +import { + profileFor, + type NetworkClass, + type NetworkProfile, +} from './networkEstimator'; + +export type TelemetryDropReason = + | RingDropReason + | 'sampled' + | 'backpressure' + | 'export_fail' + | 'parse_error'; + +export interface TelemetrySpanInput { + name: string; + priority?: TelemetryPriority; + kind?: string; + ts?: number; + durationMs?: number; + attributes?: Record; +} + +export interface TelemetrySpan extends TelemetrySpanInput { + seq: number; + priority: TelemetryPriority; + kind: string; + ts: number; +} + +export interface TelemetryExporter { + export(spans: TelemetrySpan[]): Promise; +} + +export interface TelemetryOutbox { + enqueue(spans: TelemetrySpan[]): Promise; +} + +export interface TelemetryPipelineOptions { + capacity?: number; + networkClass?: NetworkClass; + exporter?: TelemetryExporter | null; + outbox?: TelemetryOutbox | null; + now?: () => number; + autoStart?: boolean; + ring?: Partial; +} + +export interface FlushResult { + status: 'empty' | 'exported' | 'outbox' | 'failed' | 'in_flight'; + count: number; + error?: string; + retryAfterMs?: number; +} + +export interface TelemetrySnapshot { + produced: number; + parsed: number; + sampled: number; + admitted: number; + exported: number; + outboxed: number; + exportFailed: number; + dropped: number; + droppedReasons: Record; + queueDepth: number; + inFlight: number; + occupancy: number; + pressure: PressureLevel; + networkClass: NetworkClass; + profile: NetworkProfile; + exporterHealthy: boolean; + lastError: string | null; + lastExportAt: number | null; + conservation: { + offeredOk: boolean; + admittedOk: boolean; + accountingOk: boolean; + ok: boolean; + accounted: number; + }; +} + +function emptyReasons(): Record { + return { + overflow_heartbeat: 0, + overflow_normal: 0, + overflow_critical: 0, + sampled: 0, + backpressure: 0, + export_fail: 0, + parse_error: 0, + }; +} + +/** Deterministic sampler: same sequence number → same decision. */ +export function shouldSample(seq: number, rate: number): boolean { + if (rate >= 1) return true; + if (rate <= 0) return false; + const mixed = (Math.imul((seq | 0) ^ 0x9e3779b9, 2654435761) >>> 0) / 0x1_0000_0000; + return mixed < rate; +} + +export function samplingRate( + profile: NetworkProfile, + pressure: PressureLevel, + priority: TelemetryPriority, +): number { + if (priority === 'critical') return 1; + const base = priority === 'heartbeat' ? profile.heartbeatSampleRate : profile.normalSampleRate; + if (pressure === 'critical') return Math.max(0, base * 0.1); + if (pressure === 'high') return Math.max(0, base * 0.5); + return Math.max(0, Math.min(1, base)); +} + +export class TelemetryPipeline { + readonly ring: RingBuffer; + private networkClass: NetworkClass; + private profile: NetworkProfile; + private exporter: TelemetryExporter | null; + private outbox: TelemetryOutbox | null; + private readonly now: () => number; + + private sequence = 0; + private produced = 0; + private parsed = 0; + private sampled = 0; + private exported = 0; + private outboxed = 0; + private exportFailed = 0; + private backpressureDropped = 0; + private exportFailReasons = 0; + private parseErrors = 0; + private inFlight = 0; + private flushing = false; + private exporterHealthy = true; + private lastError: string | null = null; + private lastExportAt: number | null = null; + private timer: ReturnType | null = null; + private started = false; + + constructor(options: TelemetryPipelineOptions = {}) { + this.ring = new RingBuffer({ + capacity: options.capacity ?? 2_048, + ...(options.ring || {}), + }); + this.networkClass = options.networkClass ?? '4g'; + this.profile = profileFor(this.networkClass); + this.exporter = options.exporter ?? null; + this.outbox = options.outbox ?? null; + this.now = options.now ?? (() => Date.now()); + if (options.autoStart) this.start(); + } + + get network(): NetworkClass { + return this.networkClass; + } + + get batchProfile(): NetworkProfile { + return this.profile; + } + + record(input: TelemetrySpanInput): boolean { + this.produced++; + const priority = input.priority ?? 'normal'; + const pressure = this.ring.pressureLevel; + + // 90%+ occupancy: producers shed non-critical work explicitly. Critical + // dispatch frames keep flowing until the ring itself is entirely critical. + if (pressure === 'critical' && priority !== 'critical') { + this.backpressureDropped++; + return false; + } + + const rate = samplingRate(this.profile, pressure, priority); + if (!shouldSample(this.sequence, rate)) { + this.sampled++; + this.ring.recordSampled(1); + return false; + } + + const span: TelemetrySpan = { + name: input.name, + priority, + kind: input.kind ?? input.name, + ts: input.ts ?? this.now(), + durationMs: input.durationMs, + attributes: input.attributes, + seq: this.sequence++, + }; + + return this.ring.push({ + seq: span.seq, + priority, + kind: span.kind, + ts: span.ts, + payload: span, + }); + } + + recordParsed(count = 1): void { + if (count > 0) this.parsed += count; + } + + /** + * Account for a frame/span dropped outside the ring (shed before parsing, + * parse error, or explicit backpressure). This is a producer-side event, so + * it increments `produced` as well as the reason counter. + */ + recordDropped(reason: TelemetryDropReason, count = 1): void { + if (count <= 0) return; + this.produced += count; + if (reason === 'sampled') this.sampled += count; + else if (reason === 'backpressure') this.backpressureDropped += count; + else if (reason === 'export_fail') this.exportFailReasons += count; + else if (reason === 'parse_error') this.parseErrors += count; + // Ring overflow reasons are owned by RingBuffer when frames are admitted. + } + + /** Drain + export one bounded batch. Never throws. */ + async flush(): Promise { + if (this.flushing) return { status: 'in_flight', count: 0 }; + this.flushing = true; + + const batchSize = Math.max(1, this.profile.maxExportBatchSize); + const frames = this.ring.drain(batchSize); + if (frames.length === 0) { + this.flushing = false; + return { status: 'empty', count: 0 }; + } + + const spans = frames.map((frame) => frame.payload); + this.inFlight += spans.length; + try { + if (this.profile.outboxOnly || this.networkClass === 'offline') { + if (this.outbox) { + await this.outbox.enqueue(spans); + this.outboxed += spans.length; + return { status: 'outbox', count: spans.length }; + } + // No durable outbox: do not pretend these were exported. + this.exportFailed += spans.length; + this.exportFailReasons += spans.length; + this.exporterHealthy = false; + this.lastError = 'offline without outbox'; + return { status: 'failed', count: spans.length, error: this.lastError }; + } + + if (!this.exporter) { + this.exportFailed += spans.length; + this.exportFailReasons += spans.length; + this.lastError = 'no exporter configured'; + return { status: 'failed', count: spans.length, error: this.lastError }; + } + + await this.exporter.export(spans); + this.exported += spans.length; + this.exporterHealthy = true; + this.lastExportAt = this.now(); + this.lastError = null; + return { status: 'exported', count: spans.length }; + } catch (error) { + this.exportFailed += spans.length; + this.exportFailReasons += spans.length; + this.exporterHealthy = false; + this.lastError = String((error as { message?: string })?.message ?? error); + const retryAfterMs = Number((error as { retryAfterMs?: number })?.retryAfterMs); + return { + status: 'failed', + count: spans.length, + error: this.lastError, + retryAfterMs: Number.isFinite(retryAfterMs) ? retryAfterMs : undefined, + }; + } finally { + this.inFlight = Math.max(0, this.inFlight - spans.length); + this.flushing = false; + } + } + + setNetworkClass(networkClass: NetworkClass): void { + if (networkClass === this.networkClass) return; + this.networkClass = networkClass; + this.profile = profileFor(networkClass); + this.reschedule(); + } + + setExporter(exporter: TelemetryExporter | null): void { + this.exporter = exporter; + } + + setOutbox(outbox: TelemetryOutbox | null): void { + this.outbox = outbox; + } + + onPressure(listener: (event: PressureEvent) => void): () => void { + return this.ring.onPressure(listener); + } + + start(): void { + if (this.started) return; + this.started = true; + this.reschedule(); + } + + stop(): void { + this.started = false; + if (this.timer) clearTimeout(this.timer); + this.timer = null; + } + + snapshot(): TelemetrySnapshot { + const ring = this.ring.snapshot(); + const reasons = emptyReasons(); + reasons.overflow_heartbeat = ring.droppedReasons.overflow_heartbeat; + reasons.overflow_normal = ring.droppedReasons.overflow_normal; + reasons.overflow_critical = ring.droppedReasons.overflow_critical; + reasons.sampled = this.sampled; + reasons.backpressure = this.backpressureDropped; + reasons.export_fail = this.exportFailReasons; + reasons.parse_error = this.parseErrors; + + const dropped = + ring.rejected + + ring.evicted + + this.sampled + + this.backpressureDropped + + this.exportFailed + + this.parseErrors; + const accounted = this.exported + this.outboxed + dropped + ring.size + this.inFlight; + const accountingOk = accounted === this.produced; + const offeredOk = ring.conservation.offeredOk; + const admittedOk = ring.conservation.admittedOk; + + return { + produced: this.produced, + parsed: this.parsed, + sampled: this.sampled, + admitted: ring.admitted, + exported: this.exported, + outboxed: this.outboxed, + exportFailed: this.exportFailed, + dropped, + droppedReasons: reasons, + queueDepth: ring.size, + inFlight: this.inFlight, + occupancy: ring.occupancy, + pressure: ring.pressure, + networkClass: this.networkClass, + profile: this.profile, + exporterHealthy: this.exporterHealthy, + lastError: this.lastError, + lastExportAt: this.lastExportAt, + conservation: { + offeredOk, + admittedOk, + accountingOk, + ok: offeredOk && admittedOk && accountingOk, + accounted, + }, + }; + } + + reset(): void { + this.ring.reset(); + this.produced = 0; + this.parsed = 0; + this.sampled = 0; + this.exported = 0; + this.outboxed = 0; + this.exportFailed = 0; + this.backpressureDropped = 0; + this.exportFailReasons = 0; + this.parseErrors = 0; + this.inFlight = 0; + this.flushing = false; + this.exporterHealthy = true; + this.lastError = null; + this.lastExportAt = null; + this.sequence = 0; + } + + private reschedule(): void { + if (this.timer) clearTimeout(this.timer); + this.timer = null; + if (!this.started) return; + const delay = this.profile.scheduledDelayMillis; + if (delay <= 0) return; // offline/manual flush only + this.timer = setTimeout(() => { + this.flush().finally(() => this.reschedule()); + }, delay); + // Do not keep a Node process alive just for telemetry exports. + (this.timer as { unref?: () => void }).unref?.(); + } +} + +/** In-memory exporter used by tests and the lab. */ +export function createMemoryExporter(options: { maxBatches?: number } = {}): TelemetryExporter & { + batches: TelemetrySpan[][]; + reset(): void; +} { + const maxBatches = options.maxBatches ?? 100; + const batches: TelemetrySpan[][] = []; + return { + batches, + async export(spans) { + if (spans.length === 0) return; + batches.push(spans); + while (batches.length > maxBatches) batches.shift(); + }, + reset() { + batches.length = 0; + }, + }; +} + +export interface OtlpExporterOptions { + endpoint: string; + fetchImpl?: typeof fetch; + timeoutMs?: number; + headers?: Record; + serviceName?: string; +} + +/** + * Minimal OTLP/HTTP JSON exporter. Avoids pulling the server-oriented + * @opentelemetry/sdk-node bundle into the browser just for batching. + */ +export function createOtlpHttpExporter(options: OtlpExporterOptions): TelemetryExporter { + const { + endpoint, + fetchImpl = globalThis.fetch, + timeoutMs = 5_000, + headers = {}, + serviceName = 'helphone-web', + } = options; + + return { + async export(spans) { + if (spans.length === 0) return; + const controller = typeof AbortController !== 'undefined' ? new AbortController() : null; + const timer = controller ? setTimeout(() => controller.abort(), timeoutMs) : null; + try { + const response = await fetchImpl(endpoint, { + method: 'POST', + headers: { 'content-type': 'application/json', ...headers }, + body: JSON.stringify(toOtlpJson(spans, serviceName)), + signal: controller?.signal, + keepalive: spans.length <= 32, + }); + if (!response.ok) { + const retryAfter = Number(response.headers?.get?.('retry-after')); + const error = new Error(`OTLP export failed: HTTP ${response.status}`) as Error & { + status?: number; + retryAfterMs?: number; + }; + error.status = response.status; + if (Number.isFinite(retryAfter)) error.retryAfterMs = Math.max(0, retryAfter * 1_000); + throw error; + } + } finally { + if (timer) clearTimeout(timer); + } + }, + }; +} + +export function toOtlpJson(spans: TelemetrySpan[], serviceName = 'helphone-web'): unknown { + return { + resourceSpans: [ + { + resource: { + attributes: [ + { key: 'service.name', value: { stringValue: serviceName } }, + { key: 'telemetry.sdk.language', value: { stringValue: 'webjs' } }, + ], + }, + scopeSpans: [ + { + scope: { name: 'helphone.telemetry', version: '1.0.0' }, + spans: spans.map((span) => ({ + traceId: traceIdFor(span.seq), + spanId: spanIdFor(span.seq), + name: span.name, + kind: 1, + startTimeUnixNano: String(Math.round(span.ts * 1e6)), + endTimeUnixNano: String(Math.round((span.ts + (span.durationMs ?? 0)) * 1e6)), + attributes: [ + { key: 'helphone.priority', value: { stringValue: span.priority } }, + { key: 'helphone.kind', value: { stringValue: span.kind } }, + ...Object.entries(span.attributes || {}).map(([key, value]) => ({ + key, + value: + typeof value === 'number' + ? { doubleValue: value } + : typeof value === 'boolean' + ? { boolValue: value } + : { stringValue: String(value) }, + })), + ], + })), + }, + ], + }, + ], + }; +} + +function traceIdFor(seq: number): string { + // Stable, non-cryptographic id derived from the sequence; good enough for + // correlation in the lab and avoids Math.random() in hot paths. + const high = (Math.imul(seq, 2654435761) >>> 0).toString(16).padStart(8, '0'); + const low = (Math.imul(seq ^ 0x9e3779b9, 40503) >>> 0).toString(16).padStart(8, '0'); + return `${high}${low}`.padEnd(32, '0').slice(0, 32); +} + +function spanIdFor(seq: number): string { + return (Math.imul(seq ^ 0x85ebca6b, 2246822507) >>> 0).toString(16).padStart(8, '0').padEnd(16, '0').slice(0, 16); +} + +export default TelemetryPipeline; diff --git a/src/lib/telemetryLabController.ts b/src/lib/telemetryLabController.ts new file mode 100644 index 00000000..39b81e6d --- /dev/null +++ b/src/lib/telemetryLabController.ts @@ -0,0 +1,171 @@ +/** + * TelemetryLab controller: a deterministic, browser-safe harness for the + * pipeline. It generates binary frames at a configurable rate, feeds them + * through the worker (or main-thread fallback), and exposes one truthful + * snapshot per second. + */ + +import { TelemetryPipeline, createMemoryExporter, type TelemetrySnapshot } from './telemetry'; +import { TelemetryIngestClient } from './telemetryWorkerClient'; +import { TelemetryWasmParser } from '../utils/binaryParserWasm'; +import { encodeFrame, FRAME_KIND } from '../utils/binaryParser'; +import { profileFor, type NetworkClass } from './networkEstimator'; +import { FRAME_BYTES } from '../utils/binaryParser'; + +export interface LabSnapshot extends TelemetrySnapshot { + lab: { + running: boolean; + rateHz: number; + framesIngested: number; + framesShed: number; + parseErrors: number; + workerMode: string; + collectorDown: boolean; + }; +} + +export interface LabOptions { + capacity?: number; + rateHz?: number; + useWorker?: boolean; +} + +export class TelemetryLabController { + readonly pipeline: TelemetryPipeline; + readonly exporter: ReturnType; + readonly ingest: TelemetryIngestClient; + private timer: ReturnType | null = null; + private running = false; + private rateHz: number; + private seq = 0; + private collectorDown = false; + private readonly timers: { set?: typeof setInterval; clear?: typeof clearInterval }; + + constructor(options: LabOptions = {}) { + this.rateHz = options.rateHz ?? 500; + this.exporter = createMemoryExporter({ maxBatches: 40 }); + this.pipeline = new TelemetryPipeline({ + capacity: options.capacity ?? 512, + networkClass: '4g', + exporter: this.exporter, + ring: { highWatermark: 0.7, shedWatermark: 0.9, lowWatermark: 0.5 }, + }); + + const preferWorker = options.useWorker ?? typeof Worker !== 'undefined'; + this.ingest = new TelemetryIngestClient({ + pipeline: this.pipeline, + parser: new TelemetryWasmParser({ preferWasm: true }), + workerFactory: preferWorker + ? () => + new Worker(new URL('../workers/telemetryWorker.js', import.meta.url), { + type: 'module', + }) + : undefined, + }); + } + + start(): void { + if (this.running) return; + this.running = true; + this.ingest.start(); + // 20 ticks/s keeps the generator cheap while still producing 500 msg/s at + // the default rate; each tick emits exactly rateHz/20 frames. + this.timer = setInterval(() => this.emitTick(), 50); + } + + stop(): void { + this.running = false; + if (this.timer) clearInterval(this.timer); + this.timer = null; + } + + setRate(rateHz: number): void { + this.rateHz = Math.max(1, Math.min(10_000, Math.floor(rateHz))); + } + + setNetwork(networkClass: NetworkClass): void { + this.pipeline.setNetworkClass(networkClass); + } + + setCollectorDown(down: boolean): void { + this.collectorDown = down; + this.pipeline.setExporter( + down + ? { + async export() { + const error = new Error('collector unavailable') as Error & { retryAfterMs?: number }; + error.retryAfterMs = 2_000; + throw error; + }, + } + : this.exporter, + ); + } + + async flush(): Promise { + await this.pipeline.flush(); + } + + snapshot(): LabSnapshot { + const pipeline = this.pipeline.snapshot(); + const ingest = this.ingest.snapshot(); + return { + ...pipeline, + lab: { + running: this.running, + rateHz: this.rateHz, + framesIngested: ingest.framesIngested, + framesShed: ingest.framesShed, + parseErrors: ingest.parseErrors, + workerMode: ingest.workerMode, + collectorDown: this.collectorDown, + }, + }; + } + + destroy(): void { + this.stop(); + this.ingest.stop(); + this.pipeline.stop(); + } + + private emitTick(): void { + const perTick = Math.max(1, Math.round(this.rateHz / 20)); + const buffer = new ArrayBuffer(perTick * FRAME_BYTES); + const bytes = new Uint8Array(buffer); + let critical = 0; + for (let i = 0; i < perTick; i++) { + const seq = this.seq++; + const roll = seq % 10; + const kind = roll === 0 ? FRAME_KIND.DISPATCH_ACK : roll < 3 ? FRAME_KIND.HEARTBEAT : FRAME_KIND.LOCATION; + if (kind === FRAME_KIND.DISPATCH_ACK) critical++; + const frame = encodeFrame({ + kind, + priority: kind === FRAME_KIND.DISPATCH_ACK ? 2 : kind === FRAME_KIND.HEARTBEAT ? 0 : 1, + seq, + ts: Date.now(), + lat: 6.5 + (seq % 1_000) * 1e-5, + lng: 3.3 + (seq % 1_000) * 1e-5, + accuracy: 3 + (seq % 20), + }); + bytes.set(new Uint8Array(frame), i * FRAME_BYTES); + } + this.ingest.ingest(buffer, perTick, critical === perTick ? 'critical' : 'normal'); + } +} + +let singleton: TelemetryLabController | null = null; + +export function getTelemetryLabController(options?: LabOptions): TelemetryLabController { + if (!singleton) singleton = new TelemetryLabController(options); + return singleton; +} + +export function resetTelemetryLabController(): void { + singleton?.destroy(); + singleton = null; +} + +export function labProfile(networkClass: NetworkClass) { + return profileFor(networkClass); +} diff --git a/src/lib/telemetryWorkerClient.ts b/src/lib/telemetryWorkerClient.ts new file mode 100644 index 00000000..54ab2b47 --- /dev/null +++ b/src/lib/telemetryWorkerClient.ts @@ -0,0 +1,264 @@ +/** + * Main-thread ingest client for the telemetry worker. + * + * Responsibilities: + * - Transfer raw ArrayBuffers to the worker (zero-copy ownership handoff). + * - Mirror ring pressure to the worker immediately (<200ms), so the worker + * stops parsing non-critical batches before its queue grows. + * - Fall back to main-thread parsing when workers are unavailable/corrupt. + * - Keep raw-frame accounting truthful: every ingested frame ends up parsed, + * sampled, shed, or queue-dropped with an explicit reason. + */ + +import { TelemetryPipeline, type TelemetrySpanInput } from './telemetry'; +import type { TelemetryPriority } from './ringBuffer'; + +export interface TelemetryWorkerLike { + postMessage(message: unknown, transfer?: Transferable[]): void; + terminate(): void; + onmessage: ((event: { data: unknown }) => void) | null; + onerror: ((event: unknown) => void) | null; +} + +export interface ParsedFrame { + priority?: TelemetryPriority; + kindName?: string; + ts?: number; + seq?: number; + lat?: number; + lng?: number; + accuracy?: number; + error?: string; +} + +export interface TelemetryIngestOptions { + pipeline: TelemetryPipeline; + workerFactory?: () => TelemetryWorkerLike; + parser?: { parseBatch(input: ArrayBuffer | ArrayBufferView, count?: number, options?: unknown): ParsedFrame[] }; + now?: () => number; + workerTimeoutMs?: number; +} + +export interface IngestStats { + framesIngested: number; + framesParsed: number; + framesShed: number; + parseErrors: number; + pending: number; + workerMode: 'worker' | 'main-thread'; + pipeline: ReturnType; +} + +export class TelemetryIngestClient { + private readonly pipeline: TelemetryPipeline; + private readonly workerFactory?: () => TelemetryWorkerLike; + private readonly parser?: TelemetryIngestOptions['parser']; + private readonly now: () => number; + private worker: TelemetryWorkerLike | null = null; + private workerMode: 'worker' | 'main-thread' = 'main-thread'; + private nextId = 1; + private framesIngested = 0; + private framesParsed = 0; + private framesShed = 0; + private parseErrors = 0; + private readonly pending = new Map(); + private unsubscribePressure: (() => void) | null = null; + private stopped = false; + + constructor(options: TelemetryIngestOptions) { + this.pipeline = options.pipeline; + this.workerFactory = options.workerFactory; + this.parser = options.parser; + this.now = options.now ?? (() => Date.now()); + } + + start(): void { + if (this.worker || this.stopped) return; + if (!this.workerFactory || typeof Worker === 'undefined') return; + try { + const worker = this.workerFactory(); + worker.onmessage = (event) => this.handleWorkerMessage(event.data); + worker.onerror = () => { + // A crashed worker must not take ingestion down. Fall back in place; + // pending frames are accounted as parse errors. + this.failPending('worker_error'); + this.worker = null; + this.workerMode = 'main-thread'; + }; + worker.postMessage({ type: 'telemetry:init' }); + this.worker = worker; + this.workerMode = 'worker'; + this.unsubscribePressure = this.pipeline.onPressure((event) => { + const level = event.level === 'critical' ? 2 : event.level === 'high' ? 1 : 0; + try { + worker.postMessage({ type: 'telemetry:congestion', level, occupancy: event.occupancy }); + } catch { + // Worker may be terminating; the ring still sheds locally. + } + }); + } catch { + this.worker = null; + this.workerMode = 'main-thread'; + } + } + + /** Ingest `count` contiguous binary frames. Takes ownership of `buffer`. */ + ingest(buffer: ArrayBuffer, count: number, priority: TelemetryPriority = 'normal'): number { + if (this.stopped || count <= 0) return -1; + this.framesIngested += count; + + // Critical ring pressure: shed non-critical batches on the main thread + // before they ever reach the worker. This is the <200ms response path. + if (this.pipeline.ring.pressureLevel === 'critical' && priority !== 'critical') { + this.pipeline.recordDropped('backpressure', count); + this.framesShed += count; + return -1; + } + + if (this.worker) { + const id = this.nextId++; + this.pending.set(id, { count, priority }); + try { + this.worker.postMessage( + { type: 'telemetry:parse', id, buffer, count, priority }, + [buffer], + ); + return id; + } catch { + this.pending.delete(id); + // Transfer failed (buffer detached); fall through to local parse. + } + } + + this.parseLocally(buffer, count, priority); + return -1; + } + + handleWorkerMessage(message: unknown): void { + const msg = message as { + type?: string; + id?: number; + frames?: ParsedFrame[]; + shed?: number; + count?: number; + reason?: string; + message?: string; + }; + if (!msg || typeof msg.type !== 'string') return; + + if (msg.type === 'telemetry:parsed') { + this.pending.delete(msg.id ?? -1); + this.admitFrames(msg.frames ?? []); + if (msg.shed && msg.shed > 0) { + this.pipeline.recordDropped('backpressure', msg.shed); + this.framesShed += msg.shed; + } + return; + } + + if (msg.type === 'telemetry:shed') { + this.pending.delete(msg.id ?? -1); + const count = msg.count ?? 1; + this.pipeline.recordDropped('backpressure', count); + this.framesShed += count; + return; + } + + if (msg.type === 'telemetry:error') { + const entry = this.pending.get(msg.id ?? -1); + this.pending.delete(msg.id ?? -1); + if (entry) { + this.pipeline.recordDropped('parse_error', entry.count); + this.parseErrors += entry.count; + } + } + } + + flush() { + return this.pipeline.flush(); + } + + snapshot(): IngestStats { + return { + framesIngested: this.framesIngested, + framesParsed: this.framesParsed, + framesShed: this.framesShed, + parseErrors: this.parseErrors, + pending: this.pending.size, + workerMode: this.workerMode, + pipeline: this.pipeline.snapshot(), + }; + } + + stop(): void { + this.stopped = true; + this.unsubscribePressure?.(); + this.unsubscribePressure = null; + this.failPending('stopped'); + try { + this.worker?.terminate(); + } catch { + // ignore + } + this.worker = null; + this.workerMode = 'main-thread'; + } + + private parseLocally(buffer: ArrayBuffer, count: number, priority: TelemetryPriority): void { + if (!this.parser) { + this.pipeline.recordDropped('parse_error', count); + this.parseErrors += count; + return; + } + try { + const frames = this.parser.parseBatch(buffer, count, { stopOnError: false }); + this.admitFrames(frames); + } catch { + this.pipeline.recordDropped('parse_error', count); + this.parseErrors += count; + } + void priority; + } + + private admitFrames(frames: ParsedFrame[]): void { + let admitted = 0; + let errors = 0; + for (const frame of frames) { + if (frame?.error) { + errors++; + continue; + } + const span: TelemetrySpanInput = { + name: 'telemetry.frame', + priority: frame.priority ?? 'normal', + kind: frame.kindName ?? 'unknown', + ts: Number.isFinite(frame.ts) ? (frame.ts as number) : this.now(), + attributes: { + seq: frame.seq, + lat: frame.lat, + lng: frame.lng, + accuracy: frame.accuracy, + }, + }; + if (this.pipeline.record(span)) admitted++; + } + this.framesParsed += admitted; + if (errors > 0) { + this.pipeline.recordDropped('parse_error', errors); + this.parseErrors += errors; + } + } + + private failPending(reason: string): void { + let count = 0; + for (const entry of this.pending.values()) count += entry.count; + this.pending.clear(); + if (count > 0) { + this.pipeline.recordDropped('parse_error', count); + this.parseErrors += count; + void reason; + } + } +} + +export default TelemetryIngestClient; diff --git a/src/lib/telemetryWorkerProtocol.js b/src/lib/telemetryWorkerProtocol.js new file mode 100644 index 00000000..845e95ba --- /dev/null +++ b/src/lib/telemetryWorkerProtocol.js @@ -0,0 +1,212 @@ +/** + * Worker-side telemetry parsing protocol with explicit flow control. + * + * Main thread → worker: + * { type: 'telemetry:init' } + * { type: 'telemetry:parse', id, buffer, count, priority } + * { type: 'telemetry:congestion', level: 0|1|2, occupancy } + * { type: 'telemetry:stats', id } + * + * Worker → main thread: + * { type: 'telemetry:ready', mode } + * { type: 'telemetry:parsed', id, frames, stats } + * { type: 'telemetry:shed', id, count, reason } + * { type: 'telemetry:error', id?, message } + * + * When the ring signals critical occupancy (level 2), non-critical batches are + * shed without parsing, and the queue is bounded with drop-oldest accounting. + * This is what keeps low-end Android tabs from growing an unbounded worker + * queue when the map is animating at 60fps. + */ + +import { TelemetryWasmParser } from '../utils/binaryParserWasm.js'; + +export function createTelemetryWorkerProtocol(options = {}) { + const postMessage = options.postMessage ?? ((message) => self.postMessage(message)); + const parser = options.parser ?? new TelemetryWasmParser(options.parserOptions); + const now = options.now ?? (() => (typeof performance !== 'undefined' ? performance.now() : Date.now())); + const config = { + maxQueue: options.maxQueue ?? 64, + lowResLevel: options.lowResLevel ?? 1, + pauseLevel: options.pauseLevel ?? 2, + }; + + let congestion = 0; + let paused = false; + let lowRes = false; + let stopped = false; + let draining = false; + let ready = false; + + const queue = []; + const stats = { + received: 0, + parsed: 0, + shed: 0, + errors: 0, + backpressure: 0, + queueDrops: 0, + maxQueueDepth: 0, + parseMs: 0, + }; + + parser.init?.().then( + () => { + ready = true; + postMessage({ type: 'telemetry:ready', mode: parser.mode, stats: snapshot() }); + }, + (error) => { + // JS fallback remains active; report but do not fail the worker. + ready = true; + postMessage({ type: 'telemetry:ready', mode: 'js', error: String(error?.message || error) }); + }, + ); + + function snapshot() { + return { + ...stats, + mode: parser.mode, + ready: parser.ready, + congestion, + paused, + lowRes, + queueDepth: queue.length, + }; + } + + function enqueue(message) { + if (queue.length >= config.maxQueue) { + // Drop oldest, count it, then admit the newest. Never let the queue grow. + const dropped = queue.shift(); + stats.queueDrops++; + stats.shed++; + postMessage({ + type: 'telemetry:shed', + id: dropped?.id ?? null, + count: dropped?.count ?? 1, + reason: 'worker_queue_overflow', + }); + } + queue.push(message); + stats.maxQueueDepth = Math.max(stats.maxQueueDepth, queue.length); + void drain(); + } + + async function drain() { + if (draining) return; + draining = true; + try { + while (queue.length > 0 && !stopped) { + // Yield once per batch so messages that arrived in the same tick are + // visible to the bounded queue (and to the congestion check). + await Promise.resolve(); + const message = queue.shift(); + const batchPriority = message.priority ?? 'normal'; + + // 90%+ ring occupancy: only dispatch-critical batches may proceed. + if (paused && batchPriority !== 'critical') { + stats.shed += message.count ?? 1; + stats.backpressure++; + postMessage({ + type: 'telemetry:shed', + id: message.id, + count: message.count ?? 1, + reason: 'backpressure', + }); + continue; + } + + const started = now(); + try { + // `reuse` pools frame objects; postMessage clones them synchronously + // before the next batch can mutate the pool. + const frames = parser.parseBatch(message.buffer, message.count, { stopOnError: false, reuse: true, reuseArray: true }); + stats.parsed += frames.length; + stats.parseMs += now() - started; + + // Low-res mode: drop heartbeat frames before they cross the worker + // boundary. Normal + critical frames still flow. + const filtered = + lowRes && batchPriority !== 'critical' + ? frames.filter((frame) => frame.priority !== 'heartbeat' || frame.error) + : frames; + if (filtered.length !== frames.length) { + stats.shed += frames.length - filtered.length; + stats.backpressure += frames.length - filtered.length; + } + + postMessage({ + type: 'telemetry:parsed', + id: message.id, + frames: filtered, + shed: frames.length - filtered.length, + stats: snapshot(), + }); + } catch (error) { + stats.errors++; + postMessage({ + type: 'telemetry:error', + id: message.id, + message: String(error?.message || error), + }); + } + } + } finally { + draining = false; + } + } + + function handle(message) { + if (!message || typeof message.type !== 'string') return false; + if (!message.type.startsWith('telemetry:')) return false; + + switch (message.type) { + case 'telemetry:init': + postMessage({ type: 'telemetry:ready', mode: parser.mode, stats: snapshot() }); + return true; + + case 'telemetry:parse': + stats.received += message.count ?? 1; + enqueue(message); + return true; + + case 'telemetry:congestion': { + const level = Math.max(0, Math.min(2, Number(message.level) || 0)); + congestion = level; + // Hysteresis is owned by the ring: it only emits transitions. + paused = level >= config.pauseLevel; + lowRes = level >= config.lowResLevel; + if (level === 0 && queue.length > 0) void drain(); + return true; + } + + case 'telemetry:stats': + postMessage({ type: 'telemetry:stats', id: message.id, stats: snapshot() }); + return true; + + case 'telemetry:stop': + stopped = true; + queue.length = 0; + return true; + + default: + return false; + } + } + + return { + handle, + snapshot, + get queueDepth() { + return queue.length; + }, + get paused() { + return paused; + }, + get congestion() { + return congestion; + }, + }; +} + +export default createTelemetryWorkerProtocol; diff --git a/src/lib/wasmMemory.js b/src/lib/wasmMemory.js index 891c3dcd..3c073ca2 100644 --- a/src/lib/wasmMemory.js +++ b/src/lib/wasmMemory.js @@ -63,4 +63,52 @@ let singleton = null; export function getWasmMemoryPool(maxBytes = MAX_MEMORY_BYTES) { if (!singleton) { singleton = new WasmMemoryPool(maxBytes); singleton.preallocate(); } return singleton; } export function resetWasmMemoryPool() { if (singleton) singleton.destroy(); singleton = null; } export function createWasmMemoryPool(maxBytes = MAX_MEMORY_BYTES) { return new WasmMemoryPool(maxBytes); } +// ── Telemetry-specific pooling (mirrors wasmMemory.ts) ────────────────────── +export class TelemetryBufferPool { + constructor(frameBytes = 4 * 1024, maxBuffers = 64, preferShared = true) { + this.frameBytes = frameBytes; + this.maxBuffers = maxBuffers; + this.free = []; + this.active = new Set(); + this.allocationCount = 0; + this.recycledCount = 0; + this.shared = preferShared && typeof SharedArrayBuffer !== 'undefined' && globalThis.crossOriginIsolated === true; + } + acquire() { + let lease = this.free.pop(); + if (lease) this.recycledCount++; + else { + const buffer = this.shared ? new SharedArrayBuffer(this.frameBytes) : new ArrayBuffer(this.frameBytes); + lease = { buffer, bytes: new Uint8Array(buffer), shared: this.shared }; + this.allocationCount++; + } + this.active.add(lease); + return lease; + } + release(lease) { + if (!lease || !this.active.delete(lease)) return; + if (this.free.length >= this.maxBuffers) return; + try { lease.bytes.fill(0); } catch {} + this.free.push(lease); + } + getStats() { + return { frameBytes: this.frameBytes, pooled: this.free.length, active: this.active.size, allocationCount: this.allocationCount, recycledCount: this.recycledCount, shared: this.shared }; + } + destroy() { this.free.length = 0; this.active.clear(); } +} + +export class WasmViewCache { + constructor() { this.buffer = null; this.u8 = null; this.f64 = null; this.refreshes = 0; } + views(memory) { + if (this.buffer !== memory.buffer) { + this.buffer = memory.buffer; + this.u8 = new Uint8Array(memory.buffer); + this.f64 = new Float64Array(memory.buffer); + this.refreshes++; + } + return { u8: this.u8, f64: this.f64 }; + } + get refreshCount() { return this.refreshes; } +} + export default WasmMemoryPool; diff --git a/src/lib/wasmMemory.ts b/src/lib/wasmMemory.ts index a909761a..6e0f2cfb 100644 --- a/src/lib/wasmMemory.ts +++ b/src/lib/wasmMemory.ts @@ -215,5 +215,122 @@ export function createWasmMemoryPool(maxBytes = MAX_MEMORY_BYTES): WasmMemoryPoo return new WasmMemoryPool(maxBytes); } +// ── Telemetry-specific pooling ────────────────────────────────────────────── +// The ZK pool above is about large witness buffers. Telemetry has the opposite +// profile: thousands of small, fixed-size frames. Reusing a small set of +// staging buffers (and views over WASM linear memory) removes the per-frame +// allocation that showed up as GC pauses in telemetryBench.js. + +export interface TelemetryBufferLease { + buffer: ArrayBufferLike; + bytes: Uint8Array; + shared: boolean; +} + +export interface TelemetryBufferPoolStats { + frameBytes: number; + pooled: number; + active: number; + allocationCount: number; + recycledCount: number; + shared: boolean; +} + +export class TelemetryBufferPool { + private free: TelemetryBufferLease[] = []; + private active = new Set(); + private allocationCount = 0; + private recycledCount = 0; + private readonly shared: boolean; + + constructor( + readonly frameBytes = 4 * 1024, + private readonly maxBuffers = 64, + preferShared = true, + ) { + this.shared = + preferShared && + typeof SharedArrayBuffer !== 'undefined' && + typeof globalThis.crossOriginIsolated !== 'undefined' && + globalThis.crossOriginIsolated === true; + } + + acquire(): TelemetryBufferLease { + let lease = this.free.pop(); + if (lease) { + this.recycledCount++; + } else { + const buffer = this.shared + ? new SharedArrayBuffer(this.frameBytes) + : new ArrayBuffer(this.frameBytes); + lease = { + buffer, + bytes: new Uint8Array(buffer), + shared: this.shared, + }; + this.allocationCount++; + } + this.active.add(lease); + return lease; + } + + release(lease: TelemetryBufferLease | null | undefined): void { + if (!lease || !this.active.delete(lease)) return; + if (this.free.length >= this.maxBuffers) return; + // Zero the staging buffer so stale frame bytes cannot leak into the next + // parse (and so encrypted payload remnants do not linger). + try { + lease.bytes.fill(0); + } catch { + // SharedArrayBuffer views are always writable; ignore exotic engines. + } + this.free.push(lease); + } + + getStats(): TelemetryBufferPoolStats { + return { + frameBytes: this.frameBytes, + pooled: this.free.length, + active: this.active.size, + allocationCount: this.allocationCount, + recycledCount: this.recycledCount, + shared: this.shared, + }; + } + + destroy(): void { + this.free.length = 0; + this.active.clear(); + } +} + +/** + * Cached views over a WebAssembly.Memory. + * + * `memory.buffer` is detached and replaced whenever the module grows. Cache + * invalidation by buffer identity prevents the classic use-after-grow bug + * where a typed-array view points at the old, detached buffer. + */ +export class WasmViewCache { + private buffer: ArrayBufferLike | null = null; + private u8: Uint8Array | null = null; + private f64: Float64Array | null = null; + private refreshes = 0; + + views(memory: WebAssembly.Memory): { u8: Uint8Array; f64: Float64Array } { + if (this.buffer !== memory.buffer) { + this.buffer = memory.buffer; + this.u8 = new Uint8Array(memory.buffer); + this.f64 = new Float64Array(memory.buffer); + this.refreshes++; + } + return { u8: this.u8 as Uint8Array, f64: this.f64 as Float64Array }; + } + + get refreshCount(): number { + return this.refreshes; + } +} + // Convenience re-exports for workers that import this as ESM export default WasmMemoryPool; diff --git a/src/main.tsx b/src/main.tsx index 19321769..5cf70d26 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -15,6 +15,8 @@ const Help = lazy(() => import("./pages/Help")); const Ranking = lazy(() => import("./pages/Ranking")); // #608 spike: WebGPU spatial-clustering prototype + benchmark harness (ADR-008). const ClusterLab = lazy(() => import("./components/WebGPUMap")); +// P0 telemetry backpressure lab: truthful produced/parsed/dropped/exported. +const TelemetryLab = lazy(() => import("./components/TelemetryLab")); function RouteFallback() { return ( @@ -54,6 +56,7 @@ function render() { } /> } /> } /> + } /> diff --git a/src/utils/binaryParser.js b/src/utils/binaryParser.js new file mode 100644 index 00000000..e5ac48f8 --- /dev/null +++ b/src/utils/binaryParser.js @@ -0,0 +1,186 @@ +/** + * Canonical binary telemetry frame parser (JS fallback + reference parity). + * + * Wire format (little-endian, fixed 40-byte header): + * + * offset size field + * 0 4 magic 0x48454C50 ("HELP") + * 4 1 version (1) + * 5 1 kind (1 location, 2 beacon, 3 heartbeat, 4 dispatch ack, + * 5 dispatch update) + * 6 1 priority (0 heartbeat, 1 normal, 2 critical) + * 7 1 flags (bit0 = force critical, bit1 = compressed, + * bit2 = encrypted) + * 8 4 sequence (u32) + * 12 8 timestamp (f64, ms since epoch) + * 20 8 latitude (f64) + * 28 8 longitude (f64) + * 36 4 accuracy metres (f32) + * + * The WASM reader in src/wasm/telemetry_reader implements the exact same + * layout; `test/telemetry-parser.test.js` asserts parity. + */ + +export const FRAME_MAGIC = 0x48454c50; +export const FRAME_VERSION = 1; +export const FRAME_BYTES = 40; + +export const FRAME_KIND = Object.freeze({ + LOCATION: 1, + BEACON: 2, + HEARTBEAT: 3, + DISPATCH_ACK: 4, + DISPATCH_UPDATE: 5, +}); + +export const KIND_NAME = Object.freeze({ + 1: 'location', + 2: 'beacon', + 3: 'heartbeat', + 4: 'dispatch_ack', + 5: 'dispatch_update', +}); + +export const PRIORITY_NAME = Object.freeze({ + 0: 'heartbeat', + 1: 'normal', + 2: 'critical', +}); + +export class BinaryParseError extends Error { + constructor(code, message) { + super(message); + this.name = 'BinaryParseError'; + this.code = code; + } +} + +function asBytes(input) { + if (input instanceof ArrayBuffer) return new Uint8Array(input); + if (ArrayBuffer.isView(input)) { + return new Uint8Array(input.buffer, input.byteOffset, input.byteLength); + } + throw new TypeError('parseFrame expects an ArrayBuffer or ArrayBufferView'); +} + +export function priorityFor(kind, explicitPriority, flags) { + if ((flags & 0x01) !== 0) return 'critical'; + if (kind === FRAME_KIND.DISPATCH_ACK || kind === FRAME_KIND.DISPATCH_UPDATE) { + return 'critical'; + } + if (kind === FRAME_KIND.HEARTBEAT) return 'heartbeat'; + return PRIORITY_NAME[explicitPriority] || 'normal'; +} + +/** + * Parse one frame. + * + * @param {ArrayBuffer|ArrayBufferView} input + * @param {number} [offset] + * @param {object} [out] Optional reusable target object (zero-allocation path). + */ +export function parseFrame(input, offset = 0, out) { + const bytes = asBytes(input); + if (offset < 0 || offset + FRAME_BYTES > bytes.byteLength) { + throw new BinaryParseError('truncated', `need ${FRAME_BYTES} bytes at offset ${offset}, got ${bytes.byteLength - offset}`); + } + + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const magic = view.getUint32(offset, true); + if (magic !== FRAME_MAGIC) { + throw new BinaryParseError('bad_magic', `bad magic 0x${magic.toString(16)}`); + } + + const version = view.getUint8(offset + 4); + if (version !== FRAME_VERSION) { + throw new BinaryParseError('bad_version', `unsupported frame version ${version}`); + } + + const kind = view.getUint8(offset + 5); + if (!KIND_NAME[kind]) { + throw new BinaryParseError('bad_kind', `unknown frame kind ${kind}`); + } + + const declaredPriority = view.getUint8(offset + 6); + if (!PRIORITY_NAME[declaredPriority]) { + throw new BinaryParseError('bad_priority', `unknown priority ${declaredPriority}`); + } + + const flags = view.getUint8(offset + 7); + const seq = view.getUint32(offset + 8, true); + const ts = view.getFloat64(offset + 12, true); + const lat = view.getFloat64(offset + 20, true); + const lng = view.getFloat64(offset + 28, true); + const accuracy = view.getFloat32(offset + 36, true); + + if (!Number.isFinite(ts) || !Number.isFinite(lat) || !Number.isFinite(lng) || !Number.isFinite(accuracy)) { + throw new BinaryParseError('non_finite', 'frame contains non-finite coordinates'); + } + if (lat < -90 || lat > 90) { + throw new BinaryParseError('bad_latitude', `latitude out of range: ${lat}`); + } + if (lng < -180 || lng > 180) { + throw new BinaryParseError('bad_longitude', `longitude out of range: ${lng}`); + } + + const priority = priorityFor(kind, declaredPriority, flags); + const target = out || {}; + target.magic = magic; + target.version = version; + target.kind = kind; + target.kindName = KIND_NAME[kind]; + target.priority = priority; + target.flags = flags; + target.seq = seq; + target.ts = ts; + target.lat = lat; + target.lng = lng; + target.accuracy = accuracy; + target.bytes = FRAME_BYTES; + return target; +} + +/** Parse `count` contiguous frames. */ +export function parseBatch(input, count, options = {}) { + const bytes = asBytes(input); + const total = count ?? Math.floor(bytes.byteLength / FRAME_BYTES); + const out = []; + for (let i = 0; i < total; i++) { + try { + out.push(parseFrame(bytes, i * FRAME_BYTES, options.reuse ? {} : undefined)); + } catch (error) { + if (options.stopOnError !== false) throw error; + out.push({ error: error.code || 'parse_error', index: i }); + } + } + return out; +} + +/** + * Encode a frame (used by the worker/bench and by the Rust parity tests). + * @returns {ArrayBuffer} + */ +export function encodeFrame(frame = {}) { + const buffer = new ArrayBuffer(FRAME_BYTES); + const view = new DataView(buffer); + const kind = frame.kind ?? FRAME_KIND.LOCATION; + const priority = frame.priority ?? (kind === FRAME_KIND.HEARTBEAT ? 0 : kind >= FRAME_KIND.DISPATCH_ACK ? 2 : 1); + view.setUint32(0, FRAME_MAGIC, true); + view.setUint8(4, frame.version ?? FRAME_VERSION); + view.setUint8(5, kind); + view.setUint8(6, priority); + view.setUint8(7, frame.flags ?? 0); + view.setUint32(8, frame.seq ?? 0, true); + view.setFloat64(12, frame.ts ?? Date.now(), true); + view.setFloat64(20, frame.lat ?? 0, true); + view.setFloat64(28, frame.lng ?? 0, true); + view.setFloat32(36, frame.accuracy ?? 0, true); + return buffer; +} + +/** True when `kind` is dispatch-critical even without the flag bit. */ +export function isCriticalKind(kind) { + return kind === FRAME_KIND.DISPATCH_ACK || kind === FRAME_KIND.DISPATCH_UPDATE; +} + +export default { parseFrame, parseBatch, encodeFrame, FRAME_BYTES, FRAME_MAGIC }; diff --git a/src/utils/binaryParserWasm.js b/src/utils/binaryParserWasm.js new file mode 100644 index 00000000..e33eef0d --- /dev/null +++ b/src/utils/binaryParserWasm.js @@ -0,0 +1,320 @@ +/** + * JS ↔ Rust/WASM telemetry parsing bridge. + * + * Hot path properties: + * - One fixed staging buffer inside WASM; no Vec allocation per frame. + * - Views over WASM linear memory are cached and invalidated when the + * memory grows (use-after-grow safety). + * - The JS parser is always available, so a missing/corrupt module degrades + * gracefully instead of blocking ingestion. + */ + +import { + FRAME_BYTES, + FRAME_VERSION, + KIND_NAME, + PRIORITY_NAME, + BinaryParseError, + parseFrame as jsParseFrame, + parseBatch as jsParseBatch, + priorityFor, +} from './binaryParser.js'; +import { WasmViewCache } from '../lib/wasmMemory.js'; +import { loadVerifiedWasm, WasmIntegrityError } from './wasmLoader.js'; + +const DEFAULT_WASM_URL = new URL('../wasm/telemetry_reader.wasm', import.meta.url); +const DEFAULT_SHA_URL = new URL('../wasm/telemetry_reader.wasm.sha256', import.meta.url); + +const STATUS_TO_CODE = { + [-1]: 'truncated', + [-2]: 'bad_magic', + [-3]: 'bad_version', + [-4]: 'bad_kind', + [-5]: 'bad_priority', + [-6]: 'non_finite', + [-7]: 'bad_coordinates', +}; + +function toBytes(input) { + if (input instanceof ArrayBuffer) return new Uint8Array(input); + if (ArrayBuffer.isView(input)) return new Uint8Array(input.buffer, input.byteOffset, input.byteLength); + throw new TypeError('telemetry parser expects an ArrayBuffer or ArrayBufferView'); +} + +async function resolveSha({ shaUrl, expectedSha256, fetchImpl }) { + if (expectedSha256) return String(expectedSha256).trim().toLowerCase(); + if (!shaUrl) return undefined; + const response = await fetchImpl(shaUrl, { cache: 'force-cache' }); + if (!response.ok) return undefined; + const text = (await response.text()).trim().split(/\s+/)[0]; + return /^[0-9a-f]{64}$/i.test(text) ? text.toLowerCase() : undefined; +} + +export class TelemetryWasmParser { + constructor(options = {}) { + this.options = options; + this.mode = 'js'; + this.lastError = null; + this.viewCache = new WasmViewCache(); + this.framePool = []; + this.reusedBatch = []; + this.ready = false; + this.initPromise = null; + this.stats = { + parsed: 0, + wasmParsed: 0, + fallbackParsed: 0, + errors: 0, + wasmBatchCalls: 0, + allocationCount: 0, + recycledCount: 0, + objectsCreated: 0, + viewRefreshes: 0, + }; + } + + /** Start loading the module without blocking the caller. */ + init() { + if (this.initPromise) return this.initPromise; + this.initPromise = this.load().catch((error) => { + this.lastError = error; + this.mode = 'js'; + return false; + }); + return this.initPromise; + } + + async load() { + const { + wasmUrl = DEFAULT_WASM_URL, + shaUrl = DEFAULT_SHA_URL, + wasmBytes, + expectedSha256, + fetchImpl = globalThis.fetch, + timeoutMs = 2_000, + preferWasm = true, + } = this.options; + + if (!preferWasm || typeof WebAssembly === 'undefined') return false; + + let sha; + try { + sha = await resolveSha({ shaUrl, expectedSha256, fetchImpl }); + } catch { + sha = undefined; + } + + const loaded = await loadVerifiedWasm({ + url: wasmBytes ? undefined : wasmUrl, + bytes: wasmBytes, + expectedSha256: sha, + fetchImpl, + timeoutMs, + imports: {}, + }); + + const exports = loaded.exports; + if ( + typeof exports.parse_batch !== 'function' || + typeof exports.input_ptr !== 'function' || + typeof exports.input_cap !== 'function' || + typeof exports.output_ptr !== 'function' || + !(exports.memory instanceof WebAssembly.Memory) + ) { + throw new WasmIntegrityError('telemetry_reader.wasm is missing required exports'); + } + + this.exports = exports; + this.memory = exports.memory; + this.inputPtr = exports.input_ptr() >>> 0; + this.outputPtr = exports.output_ptr() >>> 0; + this.inputCap = exports.input_cap() >>> 0; + this.outputCap = typeof exports.output_cap === 'function' + ? exports.output_cap() >>> 0 + : Math.floor(this.inputCap / FRAME_BYTES); + this.mode = 'wasm'; + this.ready = true; + this.lastError = null; + return true; + } + + /** + * Parse a single frame, using WASM when ready. `out` is reused when + * supplied, which is what the main-thread ring consumer does. + */ + parseFrame(input, offset = 0, out) { + this.stats.parsed++; + if (!this.ready) { + const frame = jsParseFrame(input, offset, out); + this.stats.fallbackParsed++; + return frame; + } + + const bytes = toBytes(input); + if (offset < 0 || offset + FRAME_BYTES > bytes.byteLength) { + this.stats.errors++; + throw new BinaryParseError('truncated', `need ${FRAME_BYTES} bytes at offset ${offset}`); + } + + const { u8, f64 } = this.viewCache.views(this.memory); + // Manual 40-byte copy: no subarray allocation on the hot path. + for (let i = 0; i < FRAME_BYTES; i++) u8[this.inputPtr + i] = bytes[offset + i]; + + let status; + try { + status = this.exports.parse_frame(FRAME_BYTES) | 0; + } catch (error) { + this.stats.errors++; + // A trap means the module is unusable; fall back permanently. + this.ready = false; + this.mode = 'js'; + this.lastError = error; + return jsParseFrame(input, offset, out); + } + + if (status !== 0) { + this.stats.errors++; + throw new BinaryParseError(STATUS_TO_CODE[status] || 'wasm_reject', `wasm parser rejected frame (${status})`); + } + + const base = this.outputPtr / 8; + const seq = f64[base + 0]; + const ts = f64[base + 1]; + const lat = f64[base + 2]; + const lng = f64[base + 3]; + const accuracy = f64[base + 4]; + const kind = f64[base + 5] | 0; + const declaredPriority = f64[base + 6] | 0; + const flags = f64[base + 7] | 0; + + if (!KIND_NAME[kind] || !PRIORITY_NAME[declaredPriority]) { + this.stats.errors++; + throw new BinaryParseError('wasm_output', 'wasm parser returned an unknown kind/priority'); + } + + const frame = out || {}; + frame.magic = 0x48454c50; + frame.version = FRAME_VERSION; + frame.kind = kind; + frame.kindName = KIND_NAME[kind]; + frame.priority = priorityFor(kind, declaredPriority, flags); + frame.flags = flags; + frame.seq = seq; + frame.ts = ts; + frame.lat = lat; + frame.lng = lng; + frame.accuracy = accuracy; + frame.bytes = FRAME_BYTES; + this.stats.wasmParsed++; + this.stats.viewRefreshes = this.viewCache.refreshCount; + return frame; + } + + parseBatch(input, count, options = {}) { + if (!this.ready) { + this.stats.fallbackParsed += count ?? 0; + return jsParseBatch(input, count, options); + } + const bytes = toBytes(input); + const total = count ?? Math.floor(bytes.byteLength / FRAME_BYTES); + const reuse = options.reuse === true; + const out = options.reuseArray === true ? this.reusedBatch : []; + if (options.reuseArray === true) out.length = 0; + const chunkFrames = Math.max(1, Math.min(this.outputCap, Math.floor(this.inputCap / FRAME_BYTES))); + + for (let start = 0; start < total; start += chunkFrames) { + const frames = Math.min(chunkFrames, total - start); + const byteStart = start * FRAME_BYTES; + const byteLength = frames * FRAME_BYTES; + const { u8, f64 } = this.viewCache.views(this.memory); + u8.set(bytes.subarray(byteStart, byteStart + byteLength), this.inputPtr); + + let parsed; + try { + parsed = this.exports.parse_batch(byteLength, frames) | 0; + this.stats.wasmBatchCalls++; + } catch (error) { + this.stats.errors++; + this.ready = false; + this.mode = 'js'; + this.lastError = error; + this.stats.fallbackParsed += total - start; + return out.concat(jsParseBatch(bytes.subarray(byteStart), total - start, options)); + } + + if (parsed !== frames) { + // One invalid frame invalidates the batch result (Rust stops at the + // first error). Re-parse this chunk with JS so callers get per-frame + // error objects rather than losing the whole batch. + this.stats.errors++; + this.stats.fallbackParsed += frames; + const fallback = jsParseBatch(bytes.subarray(byteStart, byteStart + byteLength), frames, options); + if (options.stopOnError !== false) { + const bad = fallback.find((frame) => frame?.error); + if (bad) throw new BinaryParseError(bad.error, `invalid frame at index ${start + bad.index}`); + } + out.push(...fallback); + continue; + } + + const base = this.outputPtr / 8; + for (let i = 0; i < frames; i++) { + const offset = base + i * 8; + const kind = f64[offset + 5] | 0; + const declaredPriority = f64[offset + 6] | 0; + const flags = f64[offset + 7] | 0; + let frame; + if (reuse) { + const poolIndex = start + i; + frame = this.framePool[poolIndex]; + if (frame) this.stats.recycledCount++; + else { + frame = this.framePool[poolIndex] = {}; + this.stats.objectsCreated++; + } + } else { + frame = {}; + this.stats.objectsCreated++; + } + frame.magic = 0x48454c50; + frame.version = FRAME_VERSION; + frame.kind = kind; + frame.kindName = KIND_NAME[kind]; + frame.priority = priorityFor(kind, declaredPriority, flags); + frame.flags = flags; + frame.seq = f64[offset + 0]; + frame.ts = f64[offset + 1]; + frame.lat = f64[offset + 2]; + frame.lng = f64[offset + 3]; + frame.accuracy = f64[offset + 4]; + frame.bytes = FRAME_BYTES; + out.push(frame); + } + this.stats.wasmParsed += parsed; + } + return out; + } + + getStats() { + return { + ...this.stats, + mode: this.mode, + ready: this.ready, + lastError: this.lastError ? String(this.lastError.message || this.lastError) : null, + }; + } +} + +let sharedParser = null; + +/** Lazily shared parser for the main thread. */ +export function getTelemetryParser(options) { + if (!sharedParser || options) sharedParser = new TelemetryWasmParser(options); + return sharedParser; +} + +export function resetTelemetryParser() { + sharedParser = null; +} + +export default TelemetryWasmParser; diff --git a/src/utils/telemetryBench.js b/src/utils/telemetryBench.js new file mode 100644 index 00000000..3abfc69c --- /dev/null +++ b/src/utils/telemetryBench.js @@ -0,0 +1,162 @@ +#!/usr/bin/env node +/** + * Telemetry parser microbenchmark. + * + * Compares the JS fallback with the Rust/WASM reader on identical frames and + * reports per-frame cost + heap growth. Run with --expose-gc for stable heap + * numbers: + * + * node --expose-gc src/utils/telemetryBench.js --frames 20000 --out docs/benchmarks/telemetry.json + */ + +import { readFileSync, writeFileSync, mkdirSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { encodeFrame, parseBatch } from './binaryParser.js'; +import { TelemetryWasmParser } from './binaryParserWasm.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const ROOT = resolve(__dirname, '../..'); + +function arg(name, fallback) { + const index = process.argv.indexOf(`--${name}`); + if (index === -1) return fallback; + return process.argv[index + 1] ?? fallback; +} + +function makeFrames(count) { + const buffers = []; + for (let i = 0; i < count; i++) { + buffers.push( + encodeFrame({ + kind: i % 10 === 0 ? 4 : i % 3 === 0 ? 3 : 1, + priority: i % 10 === 0 ? 2 : i % 3 === 0 ? 0 : 1, + seq: i, + ts: 1_700_000_000_000 + i, + lat: 6.5 + (i % 100) * 1e-4, + lng: 3.3 + (i % 100) * 1e-4, + accuracy: 3 + (i % 20), + }), + ); + } + return buffers; +} + +function heap() { + if (typeof global.gc === 'function') global.gc(); + return process.memoryUsage().heapUsed; +} + +function time(label, fn, iterations) { + const started = process.hrtime.bigint(); + for (let i = 0; i < iterations; i++) fn(); + const elapsedMs = Number(process.hrtime.bigint() - started) / 1e6; + return { label, iterations, elapsedMs, perFrameUs: (elapsedMs * 1000) / (iterations * FRAME_COUNT) }; +} + +const FRAME_COUNT = Number(arg('frames', 10_000)); +const ROUNDS = Number(arg('rounds', 3)); +const outPath = arg('out', ''); + +const frames = makeFrames(FRAME_COUNT); +const batch = new Uint8Array(FRAME_COUNT * 40); +frames.forEach((buffer, i) => batch.set(new Uint8Array(buffer), i * 40)); + +// Warm up both paths. +parseBatch(batch, 512); + +const wasm = new TelemetryWasmParser({ + wasmBytes: (() => { + try { + return readFileSync(resolve(ROOT, 'src/wasm/telemetry_reader.wasm')); + } catch { + return undefined; + } + })(), + expectedSha256: (() => { + try { + return readFileSync(resolve(ROOT, 'src/wasm/telemetry_reader.wasm.sha256'), 'utf8').trim(); + } catch { + return undefined; + } + })(), + shaUrl: undefined, + preferWasm: true, + timeoutMs: 5_000, +}); + +const ready = await wasm.init(); +if (!ready && process.argv.includes('--require-wasm')) { + console.error('telemetryBench: WASM parser unavailable'); + process.exit(1); +} +// Warm the WASM object pool once so the timed rounds measure steady state +// (the telemetry worker reuses the same pool for the life of the tab). +if (ready) { + wasm.parseBatch(batch, FRAME_COUNT, { reuse: true, reuseArray: true }); + wasm.stats.objectsCreated = 0; + wasm.stats.recycledCount = 0; + wasm.stats.wasmBatchCalls = 0; + wasm.stats.wasmParsed = 0; +} + +const jsBefore = heap(); +const js = time('js', () => parseBatch(batch, FRAME_COUNT), ROUNDS); +const jsAfter = heap(); + +let wasmResult = null; +let wasmBefore = 0; +let wasmAfter = 0; +if (ready) { + wasmBefore = heap(); + // Hot path: pooled frame objects + pooled result array. The worker clones + // the batch synchronously in postMessage, so reuse is safe there. + wasmResult = time('wasm', () => wasm.parseBatch(batch, FRAME_COUNT, { reuse: true, reuseArray: true }), ROUNDS); + wasmAfter = heap(); +} + +const report = { + generatedAt: new Date().toISOString(), + frames: FRAME_COUNT, + rounds: ROUNDS, + wasm: { + available: Boolean(ready), + mode: wasm.mode, + stats: wasm.getStats(), + elapsedMs: wasmResult?.elapsedMs ?? null, + perFrameUs: wasmResult?.perFrameUs ?? null, + heapGrowthBytes: ready ? wasmAfter - wasmBefore : null, + heapGrowthPerFrameBytes: ready ? (wasmAfter - wasmBefore) / FRAME_COUNT : null, + objectsCreatedPerFrame: ready + ? wasm.getStats().objectsCreated / (FRAME_COUNT * ROUNDS) + : null, + pooledFrames: ready ? wasm.framePool.length : 0, + }, + js: { + elapsedMs: js.elapsedMs, + perFrameUs: js.perFrameUs, + heapGrowthBytes: jsAfter - jsBefore, + heapGrowthPerFrameBytes: (jsAfter - jsBefore) / FRAME_COUNT, + // parseBatch allocates one result object per frame. + objectsCreatedPerFrame: 1, + }, + speedup: wasmResult ? js.elapsedMs / wasmResult.elapsedMs : null, + allocationReduction: ready + ? (() => { + const objects = wasm.getStats().objectsCreated / (FRAME_COUNT * ROUNDS); + return objects === 0 ? null : 1 / objects; + })() + : null, + note: ready + ? "Steady-state WASM path reuses pooled frame objects; allocationReduction is null when per-frame object creation is zero." + : "WASM module unavailable; JS fallback measured only.", +}; + +const text = JSON.stringify(report, null, 2); +if (outPath) { + const target = resolve(ROOT, outPath); + mkdirSync(dirname(target), { recursive: true }); + writeFileSync(target, `${text}\n`); + console.log(`telemetryBench: wrote ${target}`); +} +console.log(text); diff --git a/src/utils/wasmLoader.js b/src/utils/wasmLoader.js new file mode 100644 index 00000000..1bd0f912 --- /dev/null +++ b/src/utils/wasmLoader.js @@ -0,0 +1,107 @@ +/** + * Verified WASM loader. + * + * Integrity is checked off the hot path: callers start with the JS parser and + * swap to WASM when (and only if) the module has been fetched, hashed, and + * instantiated. A corrupt/missing module never blocks frame processing. + */ + +const cache = new Map(); + +export class WasmIntegrityError extends Error { + constructor(message) { + super(message); + this.name = 'WasmIntegrityError'; + } +} + +export class WasmLoadTimeoutError extends Error { + constructor(message) { + super(message); + this.name = 'WasmLoadTimeoutError'; + } +} + +function toBytes(input) { + if (input instanceof ArrayBuffer) return new Uint8Array(input); + if (ArrayBuffer.isView(input)) return new Uint8Array(input.buffer, input.byteOffset, input.byteLength); + if (input instanceof Uint8Array) return input; + throw new TypeError('sha256Hex expects bytes'); +} + +export async function sha256Hex(input) { + const bytes = toBytes(input); + const subtle = globalThis.crypto?.subtle; + if (subtle) { + const digest = await subtle.digest('SHA-256', bytes); + return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, '0')).join(''); + } + const { createHash } = await import('node:crypto'); + return createHash('sha256').update(bytes).digest('hex'); +} + +async function fetchBytes(url, fetchImpl, timeoutMs) { + const controller = typeof AbortController !== 'undefined' ? new AbortController() : null; + const timer = controller ? setTimeout(() => controller.abort(), timeoutMs) : null; + try { + const response = await fetchImpl(url, { signal: controller?.signal, cache: 'force-cache' }); + if (!response.ok) throw new Error(`fetch ${url} failed: HTTP ${response.status}`); + return new Uint8Array(await response.arrayBuffer()); + } catch (error) { + if (error?.name === 'AbortError') throw new WasmLoadTimeoutError(`timed out loading ${url}`); + throw error; + } finally { + if (timer) clearTimeout(timer); + } +} + +/** + * Load + verify + instantiate a WASM module. + * + * @param {object} options + * @param {string} [options.url] URL to fetch (browser/worker). + * @param {ArrayBuffer|Uint8Array} [options.bytes] Inline bytes (Node/tests). + * @param {string} [options.expectedSha256] Lowercase hex digest from the sidecar. + * @param {number} [options.timeoutMs] + * @param {boolean} [options.cache] + * @param {WebAssembly.Imports} [options.imports] + */ +export function loadVerifiedWasm({ + url, + bytes, + expectedSha256, + fetchImpl = globalThis.fetch, + timeoutMs = 2_000, + cache: useCache = true, + imports = {}, +} = {}) { + const key = url || (bytes ? `inline:${bytes.byteLength}` : ''); + if (!key) return Promise.reject(new TypeError('loadVerifiedWasm requires url or bytes')); + if (useCache && cache.has(key)) return cache.get(key); + + const promise = (async () => { + const raw = bytes ? toBytes(bytes) : await fetchBytes(url, fetchImpl, timeoutMs); + if (expectedSha256) { + const actual = await sha256Hex(raw); + if (actual !== String(expectedSha256).trim().toLowerCase()) { + throw new WasmIntegrityError( + `WASM integrity mismatch for ${key}: expected ${expectedSha256}, got ${actual}`, + ); + } + } + const { instance, module } = await WebAssembly.instantiate(raw, imports); + return { instance, module, exports: instance.exports, bytes: raw, verified: Boolean(expectedSha256) }; + })(); + + if (useCache) { + cache.set(key, promise); + promise.catch(() => cache.delete(key)); + } + return promise; +} + +export function resetWasmLoaderCache() { + cache.clear(); +} + +export default { loadVerifiedWasm, sha256Hex, resetWasmLoaderCache }; diff --git a/src/wasm/telemetry_reader.wasm b/src/wasm/telemetry_reader.wasm new file mode 100755 index 0000000000000000000000000000000000000000..98a6d66178ab03785d0e8ed5671b3595bc41551d GIT binary patch literal 2862 zcmaJ@&2J<}6@OLLJ<~JYlk_SM1SO#ACI^R=+3uPCn&je&*pLuGPDJ8lJKpi`j=f%w z?O0iGc4jvS7Er{3ghcoU$RSH|35rA_Sy2us;$sgS5aJL;D9Vu=dq9}qtDdpFkYuEu zdLLEq_ui}bUN^1mToObiz7{OY@mP$P)z~gz2$FqBSiZkZpoo^mY!8#P>|i%b>`LnX zIpvUxA2pg@i`vp}v|7?}#g=d!R|&eW@dY&&W->V-e@y0sH{U;An>($>rNO1k+gBvD zE^cl;zq_=$vPEi2Z|!cA)4u%t?o8p%7}hlWt(EPa!P0XpyQ>$dIpx;2S1t{x)m~aU zx3askG}t_+guk?OVX(4=ojU_1TP;CG_$s*Xp7y(^<|>H*_YFl-E^0chNQ=SQ4l(IP zie(EFf)`p1?)7UF&QlBxZhixlnf&oeq_l_}GatEX5#Bhw_g){Af|poaP2a>$ zmhVj zw)dy{-yG?iMzZ32NQi4*pNS(+OXJm`Q44esUYo#ys115yOl72EQB9eL2x9{%1nOyK8)z*TYZ#_5D&kpixiYJj;d152u7oI5c;d41Zih zeU1!^MfgMJV@p5rp#OCt#>e1l3XYO3WT=(4z{-fn#6#d@As@h^0A0vAtaHQgC$)Lo zg|jfi^;pcpctoNz*+ULNXjw`J-Qf(wpVbFHY)Jn&0XQk^lctTvp?OKM6S)$7V?J(j zN3j?ZwTLoo47D1f*7t|nwGrL?6K%$>!LqR)H&HTbWc~q>XT8NF97%#OB7?}|Km+q1&ZF3i zJfCxNcu1oN#^WwaL$pS{(s@5)ksCJ0+rL zd!n{ylYpK?#*vHTBp|=mS@&*dBOWXm01sQ&HTh4W03hLd!s|klMnnGJZJ6m>i9^9_ zf$z>%-^n!iEuI@_W%6GULe3@!y4I<|47Bs|?>q?6hXP~5(tvjjJy{o`!3+5>yn~T= zyPd3db4@{HN3R)wqay+oqJ!xMhs3Gtd>1w4=gu%cJYLiXtH)-`N`U%UGY~^ zNNubyD_Ikanm-mWZ*{gc}EXmTWoAt6R%d;XYvnuQ7NuK82yq9Nro)>wUS9!lk zinQn!y&@~}qA1FuD*9zore(M6m06jWMOl_r*{_l+t-4jO%Bs97s&2 + exit 1 + fi + expected="$(tr -d ' \t\r\n' < "$SIDECAR")" + actual="$(sha256sum "$ARTIFACT" | awk '{print $1}')" + if [[ "$expected" != "$actual" ]]; then + echo "telemetry-wasm: FAIL (SHA-256 mismatch)" >&2 + echo " expected: $expected" >&2 + echo " actual: $actual" >&2 + exit 1 + fi + echo "telemetry-wasm: SHA-256 OK ($actual)" + exit 0 +fi + +cd "$HERE" +cargo build --target "$TARGET" --release +mkdir -p "$(dirname "$ARTIFACT")" +cp "target/$TARGET/release/telemetry_reader.wasm" "$ARTIFACT" +sha256sum "$ARTIFACT" | awk '{print $1}' > "$SIDECAR" + +echo "telemetry-wasm: built $ARTIFACT" +echo "telemetry-wasm: sha256 $(cat "$SIDECAR")" diff --git a/src/wasm/telemetry_reader/src/lib.rs b/src/wasm/telemetry_reader/src/lib.rs new file mode 100644 index 00000000..e67172ed --- /dev/null +++ b/src/wasm/telemetry_reader/src/lib.rs @@ -0,0 +1,159 @@ +//! Zero-allocation telemetry frame reader. +//! +//! The JS bridge copies a whole batch into a fixed staging buffer, calls +//! `parse_batch(len, count)` once, and reads a fixed-size output array. No +//! Vec, no per-frame boundary crossing, no per-frame allocation. +//! +//! Output layout (8 × f64 per frame): +//! seq | ts | lat | lng | accuracy | kind | priority | flags +//! +//! Status codes (negative): +//! -1 truncated, -2 bad magic, -3 bad version, -4 bad kind, -5 bad priority, +//! -6 non-finite, -7 out-of-range latitude/longitude + +#![allow(static_mut_refs)] + +const MAGIC: u32 = 0x4845_4C50; +const VERSION: u8 = 1; +const FRAME_BYTES: usize = 40; +const INPUT_CAP: usize = 64 * 1024; +const MAX_FRAMES: usize = INPUT_CAP / FRAME_BYTES; +const OUTPUT_LEN: usize = MAX_FRAMES * 8; + +static mut INPUT: [u8; INPUT_CAP] = [0; INPUT_CAP]; +static mut OUTPUT: [f64; OUTPUT_LEN] = [0.0; OUTPUT_LEN]; + +#[no_mangle] +pub extern "C" fn input_ptr() -> *const u8 { + core::ptr::addr_of!(INPUT).cast::() +} + +#[no_mangle] +pub extern "C" fn input_cap() -> usize { + INPUT_CAP +} + +#[no_mangle] +pub extern "C" fn output_ptr() -> *const f64 { + core::ptr::addr_of!(OUTPUT).cast::() +} + +#[no_mangle] +pub extern "C" fn output_cap() -> usize { + MAX_FRAMES +} + +#[inline] +fn read_u32(bytes: &[u8], offset: usize) -> u32 { + u32::from_le_bytes([ + bytes[offset], + bytes[offset + 1], + bytes[offset + 2], + bytes[offset + 3], + ]) +} + +#[inline] +fn read_f64(bytes: &[u8], offset: usize) -> f64 { + f64::from_le_bytes([ + bytes[offset], + bytes[offset + 1], + bytes[offset + 2], + bytes[offset + 3], + bytes[offset + 4], + bytes[offset + 5], + bytes[offset + 6], + bytes[offset + 7], + ]) +} + +#[inline] +fn read_f32(bytes: &[u8], offset: usize) -> f32 { + f32::from_le_bytes([ + bytes[offset], + bytes[offset + 1], + bytes[offset + 2], + bytes[offset + 3], + ]) +} + +#[inline] +fn parse_one(input: &[u8], offset: usize, out: &mut [f64]) -> i32 { + if offset + FRAME_BYTES > input.len() { + return -1; + } + let frame = &input[offset..offset + FRAME_BYTES]; + + if read_u32(frame, 0) != MAGIC { + return -2; + } + if frame[4] != VERSION { + return -3; + } + + let kind = frame[5]; + if !(1..=5).contains(&kind) { + return -4; + } + + let priority = frame[6]; + if priority > 2 { + return -5; + } + + let flags = frame[7]; + let seq = read_u32(frame, 8); + let ts = read_f64(frame, 12); + let lat = read_f64(frame, 20); + let lng = read_f64(frame, 28); + let accuracy = read_f32(frame, 36); + + if !ts.is_finite() || !lat.is_finite() || !lng.is_finite() || !accuracy.is_finite() { + return -6; + } + if !(-90.0..=90.0).contains(&lat) || !(-180.0..=180.0).contains(&lng) { + return -7; + } + + out[0] = seq as f64; + out[1] = ts; + out[2] = lat; + out[3] = lng; + out[4] = accuracy as f64; + out[5] = kind as f64; + out[6] = priority as f64; + out[7] = flags as f64; + 0 +} + +/// Parse the staged batch. Returns the number of frames on success, or a +/// negative status code on the first invalid frame. +#[no_mangle] +pub extern "C" fn parse_batch(len: usize, count: usize) -> i32 { + if count == 0 || count > MAX_FRAMES || len < count * FRAME_BYTES || len > INPUT_CAP { + return -1; + } + + let input: &[u8] = unsafe { core::slice::from_raw_parts(INPUT.as_ptr(), len) }; + let output: &mut [f64] = unsafe { core::slice::from_raw_parts_mut(OUTPUT.as_mut_ptr(), OUTPUT_LEN) }; + + for index in 0..count { + let out = &mut output[index * 8..index * 8 + 8]; + let status = parse_one(input, index * FRAME_BYTES, out); + if status != 0 { + return status; + } + } + count as i32 +} + +/// Single-frame compatibility entry point (used by parity tests). +#[no_mangle] +pub extern "C" fn parse_frame(len: usize) -> i32 { + let parsed = parse_batch(len, 1); + if parsed == 1 { + 0 + } else { + parsed + } +} diff --git a/src/wasm/telemetry_reader/target/.rustc_info.json b/src/wasm/telemetry_reader/target/.rustc_info.json new file mode 100644 index 00000000..d7a6f362 --- /dev/null +++ b/src/wasm/telemetry_reader/target/.rustc_info.json @@ -0,0 +1 @@ +{"rustc_fingerprint":10483562705157065480,"outputs":{"11652014622397750202":{"success":true,"status":"","code":0,"stdout":"___.wasm\nlib___.rlib\n___.wasm\nlib___.a\n/home/emperor-caesar/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu\noff\n___\ndebug_assertions\nemscripten_wasm_eh\nfmt_debug=\"full\"\noverflow_checks\npanic=\"abort\"\nproc_macro\nrelocation_model=\"static\"\ntarget_abi=\"\"\ntarget_arch=\"wasm32\"\ntarget_endian=\"little\"\ntarget_env=\"\"\ntarget_family=\"wasm\"\ntarget_feature=\"bulk-memory\"\ntarget_feature=\"multivalue\"\ntarget_feature=\"mutable-globals\"\ntarget_feature=\"nontrapping-fptoint\"\ntarget_feature=\"reference-types\"\ntarget_feature=\"sign-ext\"\ntarget_has_atomic\ntarget_has_atomic=\"16\"\ntarget_has_atomic=\"32\"\ntarget_has_atomic=\"64\"\ntarget_has_atomic=\"8\"\ntarget_has_atomic=\"ptr\"\ntarget_has_atomic_equal_alignment=\"16\"\ntarget_has_atomic_equal_alignment=\"32\"\ntarget_has_atomic_equal_alignment=\"64\"\ntarget_has_atomic_equal_alignment=\"8\"\ntarget_has_atomic_equal_alignment=\"ptr\"\ntarget_has_atomic_load_store\ntarget_has_atomic_load_store=\"16\"\ntarget_has_atomic_load_store=\"32\"\ntarget_has_atomic_load_store=\"64\"\ntarget_has_atomic_load_store=\"8\"\ntarget_has_atomic_load_store=\"ptr\"\ntarget_has_reliable_f128\ntarget_has_reliable_f16\ntarget_has_reliable_f16_math\ntarget_os=\"unknown\"\ntarget_pointer_width=\"32\"\ntarget_thread_local\ntarget_vendor=\"unknown\"\nub_checks\n","stderr":"warning: dropping unsupported crate type `dylib` for target `wasm32-unknown-unknown`\n\nwarning: dropping unsupported crate type `proc-macro` for target `wasm32-unknown-unknown`\n\nwarning: 2 warnings emitted\n\n"},"7971740275564407648":{"success":true,"status":"","code":0,"stdout":"___\nlib___.rlib\nlib___.so\nlib___.so\nlib___.a\nlib___.so\n/home/emperor-caesar/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu\noff\npacked\nunpacked\n___\ndebug_assertions\nemscripten_wasm_eh\nfmt_debug=\"full\"\noverflow_checks\npanic=\"unwind\"\nproc_macro\nrelocation_model=\"pic\"\ntarget_abi=\"\"\ntarget_arch=\"x86_64\"\ntarget_endian=\"little\"\ntarget_env=\"gnu\"\ntarget_family=\"unix\"\ntarget_feature=\"fxsr\"\ntarget_feature=\"sse\"\ntarget_feature=\"sse2\"\ntarget_feature=\"x87\"\ntarget_has_atomic\ntarget_has_atomic=\"16\"\ntarget_has_atomic=\"32\"\ntarget_has_atomic=\"64\"\ntarget_has_atomic=\"8\"\ntarget_has_atomic=\"ptr\"\ntarget_has_atomic_equal_alignment=\"16\"\ntarget_has_atomic_equal_alignment=\"32\"\ntarget_has_atomic_equal_alignment=\"64\"\ntarget_has_atomic_equal_alignment=\"8\"\ntarget_has_atomic_equal_alignment=\"ptr\"\ntarget_has_atomic_load_store\ntarget_has_atomic_load_store=\"16\"\ntarget_has_atomic_load_store=\"32\"\ntarget_has_atomic_load_store=\"64\"\ntarget_has_atomic_load_store=\"8\"\ntarget_has_atomic_load_store=\"ptr\"\ntarget_has_reliable_f128\ntarget_has_reliable_f16\ntarget_has_reliable_f16_math\ntarget_os=\"linux\"\ntarget_pointer_width=\"64\"\ntarget_thread_local\ntarget_vendor=\"unknown\"\nub_checks\nunix\n","stderr":""},"17747080675513052775":{"success":true,"status":"","code":0,"stdout":"rustc 1.96.0-nightly (ac7f9ec7d 2026-03-20)\nbinary: rustc\ncommit-hash: ac7f9ec7da74d37fd28667c86bf117a39ba5b02a\ncommit-date: 2026-03-20\nhost: x86_64-unknown-linux-gnu\nrelease: 1.96.0-nightly\nLLVM version: 22.1.0\n","stderr":""}},"successes":{}} \ No newline at end of file diff --git a/src/wasm/telemetry_reader/target/CACHEDIR.TAG b/src/wasm/telemetry_reader/target/CACHEDIR.TAG new file mode 100644 index 00000000..20d7c319 --- /dev/null +++ b/src/wasm/telemetry_reader/target/CACHEDIR.TAG @@ -0,0 +1,3 @@ +Signature: 8a477f597d28d172789f06886806bc55 +# This file is a cache directory tag created by cargo. +# For information about cache directory tags see https://bford.info/cachedir/ diff --git a/src/wasm/telemetry_reader/target/release/.cargo-build-lock b/src/wasm/telemetry_reader/target/release/.cargo-build-lock new file mode 100644 index 00000000..e69de29b diff --git a/src/wasm/telemetry_reader/target/release/.cargo-lock b/src/wasm/telemetry_reader/target/release/.cargo-lock new file mode 100644 index 00000000..e69de29b diff --git a/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/CACHEDIR.TAG b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/CACHEDIR.TAG new file mode 100644 index 00000000..20d7c319 --- /dev/null +++ b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/CACHEDIR.TAG @@ -0,0 +1,3 @@ +Signature: 8a477f597d28d172789f06886806bc55 +# This file is a cache directory tag created by cargo. +# For information about cache directory tags see https://bford.info/cachedir/ diff --git a/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.cargo-build-lock b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.cargo-build-lock new file mode 100644 index 00000000..e69de29b diff --git a/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.cargo-lock b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.cargo-lock new file mode 100644 index 00000000..e69de29b diff --git a/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/dep-lib-telemetry_reader b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/dep-lib-telemetry_reader new file mode 100644 index 0000000000000000000000000000000000000000..024be4904569dbe2c19582a923171096f5c21ee4 GIT binary patch literal 30 gcmZQ%U|{&q$Ot4ExPZ90C|N%zGfA(g7$m{~07A6{&Hw-a literal 0 HcmV?d00001 diff --git a/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/invoked.timestamp b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/invoked.timestamp new file mode 100644 index 00000000..e00328da --- /dev/null +++ b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/invoked.timestamp @@ -0,0 +1 @@ +This file has an mtime of when this was started. \ No newline at end of file diff --git a/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/lib-telemetry_reader b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/lib-telemetry_reader new file mode 100644 index 00000000..55504fb1 --- /dev/null +++ b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/lib-telemetry_reader @@ -0,0 +1 @@ +7efaf3e95c819bfb \ No newline at end of file diff --git a/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/lib-telemetry_reader.json b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/lib-telemetry_reader.json new file mode 100644 index 00000000..501c40fb --- /dev/null +++ b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/lib-telemetry_reader.json @@ -0,0 +1 @@ +{"rustc":9229630479334048335,"features":"[]","declared_features":"[]","target":139031884860755733,"profile":12833233019932215290,"path":10763286916239946207,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/dep-lib-telemetry_reader","checksum":false}}],"rustflags":[],"config":8247474407144887393,"compile_kind":14682669768258224367} \ No newline at end of file diff --git a/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/deps/telemetry_reader.d b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/deps/telemetry_reader.d new file mode 100644 index 00000000..0914767e --- /dev/null +++ b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/deps/telemetry_reader.d @@ -0,0 +1,5 @@ +/home/emperor-caesar/grantfox/HelPhone/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/deps/telemetry_reader.d: src/lib.rs + +/home/emperor-caesar/grantfox/HelPhone/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/deps/telemetry_reader.wasm: src/lib.rs + +src/lib.rs: diff --git a/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/deps/telemetry_reader.wasm b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/deps/telemetry_reader.wasm new file mode 100755 index 0000000000000000000000000000000000000000..98a6d66178ab03785d0e8ed5671b3595bc41551d GIT binary patch literal 2862 zcmaJ@&2J<}6@OLLJ<~JYlk_SM1SO#ACI^R=+3uPCn&je&*pLuGPDJ8lJKpi`j=f%w z?O0iGc4jvS7Er{3ghcoU$RSH|35rA_Sy2us;$sgS5aJL;D9Vu=dq9}qtDdpFkYuEu zdLLEq_ui}bUN^1mToObiz7{OY@mP$P)z~gz2$FqBSiZkZpoo^mY!8#P>|i%b>`LnX zIpvUxA2pg@i`vp}v|7?}#g=d!R|&eW@dY&&W->V-e@y0sH{U;An>($>rNO1k+gBvD zE^cl;zq_=$vPEi2Z|!cA)4u%t?o8p%7}hlWt(EPa!P0XpyQ>$dIpx;2S1t{x)m~aU zx3askG}t_+guk?OVX(4=ojU_1TP;CG_$s*Xp7y(^<|>H*_YFl-E^0chNQ=SQ4l(IP zie(EFf)`p1?)7UF&QlBxZhixlnf&oeq_l_}GatEX5#Bhw_g){Af|poaP2a>$ zmhVj zw)dy{-yG?iMzZ32NQi4*pNS(+OXJm`Q44esUYo#ys115yOl72EQB9eL2x9{%1nOyK8)z*TYZ#_5D&kpixiYJj;d152u7oI5c;d41Zih zeU1!^MfgMJV@p5rp#OCt#>e1l3XYO3WT=(4z{-fn#6#d@As@h^0A0vAtaHQgC$)Lo zg|jfi^;pcpctoNz*+ULNXjw`J-Qf(wpVbFHY)Jn&0XQk^lctTvp?OKM6S)$7V?J(j zN3j?ZwTLoo47D1f*7t|nwGrL?6K%$>!LqR)H&HTbWc~q>XT8NF97%#OB7?}|Km+q1&ZF3i zJfCxNcu1oN#^WwaL$pS{(s@5)ksCJ0+rL zd!n{ylYpK?#*vHTBp|=mS@&*dBOWXm01sQ&HTh4W03hLd!s|klMnnGJZJ6m>i9^9_ zf$z>%-^n!iEuI@_W%6GULe3@!y4I<|47Bs|?>q?6hXP~5(tvjjJy{o`!3+5>yn~T= zyPd3db4@{HN3R)wqay+oqJ!xMhs3Gtd>1w4=gu%cJYLiXtH)-`N`U%UGY~^ zNNubyD_Ikanm-mWZ*{gc}EXmTWoAt6R%d;XYvnuQ7NuK82yq9Nro)>wUS9!lk zinQn!y&@~}qA1FuD*9zore(M6m06jWMOl_r*{_l+t-4jO%Bs97s|i%b>`LnX zIpvUxA2pg@i`vp}v|7?}#g=d!R|&eW@dY&&W->V-e@y0sH{U;An>($>rNO1k+gBvD zE^cl;zq_=$vPEi2Z|!cA)4u%t?o8p%7}hlWt(EPa!P0XpyQ>$dIpx;2S1t{x)m~aU zx3askG}t_+guk?OVX(4=ojU_1TP;CG_$s*Xp7y(^<|>H*_YFl-E^0chNQ=SQ4l(IP zie(EFf)`p1?)7UF&QlBxZhixlnf&oeq_l_}GatEX5#Bhw_g){Af|poaP2a>$ zmhVj zw)dy{-yG?iMzZ32NQi4*pNS(+OXJm`Q44esUYo#ys115yOl72EQB9eL2x9{%1nOyK8)z*TYZ#_5D&kpixiYJj;d152u7oI5c;d41Zih zeU1!^MfgMJV@p5rp#OCt#>e1l3XYO3WT=(4z{-fn#6#d@As@h^0A0vAtaHQgC$)Lo zg|jfi^;pcpctoNz*+ULNXjw`J-Qf(wpVbFHY)Jn&0XQk^lctTvp?OKM6S)$7V?J(j zN3j?ZwTLoo47D1f*7t|nwGrL?6K%$>!LqR)H&HTbWc~q>XT8NF97%#OB7?}|Km+q1&ZF3i zJfCxNcu1oN#^WwaL$pS{(s@5)ksCJ0+rL zd!n{ylYpK?#*vHTBp|=mS@&*dBOWXm01sQ&HTh4W03hLd!s|klMnnGJZJ6m>i9^9_ zf$z>%-^n!iEuI@_W%6GULe3@!y4I<|47Bs|?>q?6hXP~5(tvjjJy{o`!3+5>yn~T= zyPd3db4@{HN3R)wqay+oqJ!xMhs3Gtd>1w4=gu%cJYLiXtH)-`N`U%UGY~^ zNNubyD_Ikanm-mWZ*{gc}EXmTWoAt6R%d;XYvnuQ7NuK82yq9Nro)>wUS9!lk zinQn!y&@~}qA1FuD*9zore(M6m06jWMOl_r*{_l+t-4jO%Bs97s typeof performance !== "undefined" ? performance.now() : Date.now(); @@ -237,10 +238,24 @@ export function attachOwnerPort(engine, port) { if (typeof self !== "undefined" && typeof self.postMessage === "function" && typeof window === "undefined") { let ownerEngine = null; const ports = []; + let telemetryProtocol = null; const fail = (err) => self.postMessage({ type: "error", message: String((err && err.message) || err), name: err && err.name }); self.onmessage = async (event) => { const msg = event.data || {}; + + // Telemetry binary-parsing lane (flow-controlled; independent of the + // storage-contention benchmark lane below). + if (typeof msg.type === "string" && msg.type.startsWith("telemetry:")) { + if (!telemetryProtocol) { + telemetryProtocol = createTelemetryWorkerProtocol({ + postMessage: (message) => self.postMessage(message), + }); + } + telemetryProtocol.handle(msg); + return; + } + try { if (msg.type === "port") { if (ownerEngine) attachOwnerPort(ownerEngine, msg.port); diff --git a/swagger.json b/swagger.json index 98ccb9d8..300e899c 100644 --- a/swagger.json +++ b/swagger.json @@ -1 +1,133 @@ -// Implementation added +{ + "openapi": "3.1.0", + "info": { + "title": "HelPhone API", + "version": "1.0.0", + "description": "REST surface for dispatch, events, sync, telemetry, and the GraphQL bridge. The GraphQL contract lives in server/graphql/schema.graphql." + }, + "servers": [{ "url": "http://localhost:3001" }], + "paths": { + "/api/dispatch": { + "get": { + "operationId": "listIncidents", + "summary": "List incidents", + "parameters": [ + { "name": "limit", "in": "query", "schema": { "type": "integer", "default": 20 } }, + { "name": "cursor", "in": "query", "schema": { "type": "string" } } + ], + "responses": { "200": { "description": "Incident page" } } + }, + "post": { + "operationId": "createIncident", + "summary": "Create an incident", + "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CreateIncident" } } } }, + "responses": { + "201": { "description": "Created" }, + "422": { "description": "Validation failed" }, + "409": { "description": "Conflict" } + } + } + }, + "/api/dispatch/{id}": { + "get": { + "operationId": "getIncident", + "summary": "Get one incident with responders", + "parameters": [ + { "name": "id", "in": "path", "required": true, "schema": { "type": "string" } }, + { "name": "nocache", "in": "query", "schema": { "type": "string" } } + ], + "responses": { "200": { "description": "Incident bundle" }, "404": { "description": "Not found" } } + } + }, + "/api/dispatch/{id}/assign": { + "post": { + "operationId": "assignResponder", + "summary": "Assign a responder with optimistic concurrency", + "parameters": [{ "name": "id", "in": "path", "required": true, "schema": { "type": "string" } }], + "requestBody": { "required": true, "content": { "application/json": { "schema": { "type": "object", "required": ["responderId"], "properties": { "responderId": { "type": "string" }, "expectedVersion": { "type": "integer" } } } } } }, + "responses": { + "200": { "description": "Assigned" }, + "409": { "description": "Version conflict with serverVersion + mergeProposal" }, + "422": { "description": "Validation failed" } + } + } + }, + "/api/dispatch/{id}/ack": { + "post": { + "operationId": "acknowledgeIncident", + "summary": "Acknowledge an incident", + "parameters": [{ "name": "id", "in": "path", "required": true, "schema": { "type": "string" } }], + "responses": { "200": { "description": "Acknowledged" }, "409": { "description": "Version conflict" } } + } + }, + "/api/events": { + "get": { + "operationId": "listEvents", + "summary": "Event log page; cursor is identical to GraphQL EventConnection.endCursor", + "parameters": [ + { "name": "incidentId", "in": "query", "required": true, "schema": { "type": "string" } }, + { "name": "limit", "in": "query", "schema": { "type": "integer", "default": 20 } }, + { "name": "cursor", "in": "query", "schema": { "type": "string" } } + ], + "responses": { "200": { "description": "Event page" }, "422": { "description": "incidentId required" } } + } + }, + "/api/events/ice-config": { + "get": { "operationId": "getIceConfig", "responses": { "200": { "description": "ICE servers" } } } + }, + "/api/sync": { + "post": { "operationId": "syncOperations", "responses": { "200": { "description": "Merged operations" } } } + }, + "/api/sync/flush": { + "post": { + "operationId": "flushSync", + "summary": "Flush an offline outbox; 409 shape matches GraphQL SyncConflict", + "requestBody": { "required": true, "content": { "application/json": { "schema": { "type": "object", "required": ["incidentId", "baseVersion", "patch"], "properties": { "incidentId": { "type": "string" }, "baseVersion": { "type": "integer" }, "patch": { "type": "object" } } } } } }, + "responses": { + "200": { "description": "Applied" }, + "409": { "description": "Conflict with serverVersion + mergeProposal" }, + "422": { "description": "Validation issues" } + } + } + }, + "/api/telemetry/spans": { + "post": { + "operationId": "ingestTelemetry", + "summary": "Bounded OTLP/HTTP JSON ingest", + "responses": { + "202": { "description": "Accepted" }, + "429": { "description": "Queue high watermark; Retry-After" }, + "503": { "description": "Collector unavailable; Retry-After" } + } + } + }, + "/metrics/telemetry": { + "get": { "operationId": "telemetryMetrics", "responses": { "200": { "description": "Receiver counters" } } } + }, + "/graphql": { + "post": { + "operationId": "graphql", + "summary": "GraphQL endpoint (same domain contract as REST)", + "responses": { "200": { "description": "GraphQL result" }, "400": { "description": "Missing query" } } + } + } + }, + "components": { + "schemas": { + "CreateIncident": { + "type": "object", + "required": ["location"], + "properties": { + "id": { "type": "string" }, + "status": { "type": "string", "enum": ["open", "assigned", "acknowledged", "resolved"], "x-enum-name": "IncidentStatus" }, + "priority": { "type": "string", "enum": ["low", "normal", "high", "critical"], "x-enum-name": "Priority" }, + "location": { + "type": "object", + "required": ["lat", "lng"], + "properties": { "lat": { "type": "number" }, "lng": { "type": "number" } } + } + } + } + } + } +} diff --git a/test/api-contract.test.js b/test/api-contract.test.js new file mode 100644 index 00000000..2143caa2 --- /dev/null +++ b/test/api-contract.test.js @@ -0,0 +1,274 @@ +// @vitest-environment node +import { beforeEach, describe, expect, it } from 'vitest'; +import express from 'express'; +import request from 'supertest'; +import { installDomainRoutes } from '../server/routes/index.ts'; +import { DomainStore } from '../server/lib/domainStore.ts'; +import { createGraphQLContext, executeGraphQL } from '../server/graphql/server.ts'; +import { resetSqlCache } from '../server/lib/sqlCache.ts'; + +const INCIDENT_QUERY = ` + query Incident($id: ID!) { + incident(id: $id) { + id + status + priority + version + location { lat lng } + responders { + id + incidentId + status + location { lat lng } + lastEvent { id type cursor } + } + } + } +`; + +const EVENTS_QUERY = ` + query Events($id: ID!, $limit: Int, $cursor: String) { + events(incidentId: $id, limit: $limit, cursor: $cursor) { + edges { cursor node { id type cursor } } + pageInfo { hasNextPage endCursor } + } + } +`; + +const FLUSH_MUTATION = ` + mutation Flush($input: SyncFlushInput!) { + syncFlush(input: $input) { + __typename + ... on SyncSuccess { ok serverVersion incident { id version } } + ... on SyncConflict { conflict serverVersion mergeProposal } + ... on ValidationError { error issues { path message } } + } + } +`; + +const ASSIGN_MUTATION = ` + mutation Assign($input: AssignResponderInput!) { + assignResponder(input: $input) { + __typename + ... on SyncSuccess { ok serverVersion incident { id version } } + ... on SyncConflict { conflict serverVersion mergeProposal } + ... on ValidationError { error issues { path message } } + } + } +`; + +describe('REST ↔ GraphQL contract parity', () => { + let store; + let app; + + beforeEach(() => { + store = new DomainStore(); + store.resetQueryCount(); + resetSqlCache(); + app = express(); + app.use(express.json()); + installDomainRoutes(app, { store }); + }); + + it('returns the same incident, responder, and last-event fields on both surfaces', async () => { + const rest = await request(app).get('/api/dispatch/INC-1042?nocache=1'); + expect(rest.status).toBe(200); + + const gql = await executeGraphQL( + { query: INCIDENT_QUERY, variables: { id: 'INC-1042' } }, + { store }, + ); + expect(gql.errors).toBeUndefined(); + + const restIncident = rest.body.incident; + const gqlIncident = gql.data.incident; + + expect(restIncident).toMatchObject({ + id: gqlIncident.id, + status: gqlIncident.status, + priority: gqlIncident.priority, + version: gqlIncident.version, + location: gqlIncident.location, + }); + expect(restIncident.responders).toHaveLength(40); + expect(restIncident.responders).toHaveLength(gqlIncident.responders.length); + + for (let i = 0; i < restIncident.responders.length; i++) { + const restResponder = restIncident.responders[i]; + const gqlResponder = gqlIncident.responders[i]; + expect(restResponder).toMatchObject({ + id: gqlResponder.id, + incidentId: gqlResponder.incidentId, + status: gqlResponder.status, + location: gqlResponder.location, + }); + expect(restResponder.lastEvent?.id ?? null).toBe(gqlResponder.lastEvent?.id ?? null); + expect(restResponder.lastEvent?.cursor ?? null).toBe(gqlResponder.lastEvent?.cursor ?? null); + } + }); + + it('batches GraphQL nested reads to ≤4 store queries instead of 1+N+N', async () => { + store.resetQueryCount(); + const gql = await executeGraphQL( + { query: INCIDENT_QUERY, variables: { id: 'INC-1042' } }, + { store }, + ); + expect(gql.errors).toBeUndefined(); + expect(store.getQueryCount()).toBeLessThanOrEqual(4); + // The naive resolver would issue incident + 40 responders + 40 events = 81. + expect(store.getQueryCount()).toBeLessThan(81); + }); + + it('uses two store queries for the hand-optimised REST bundle', async () => { + store.resetQueryCount(); + const rest = await request(app).get('/api/dispatch/INC-1042?nocache=1'); + expect(rest.status).toBe(200); + expect(store.getQueryCount()).toBeLessThanOrEqual(2); + }); + + it('creates request-scoped loaders so one context cannot serve another context stale data', async () => { + const ctxA = createGraphQLContext(store, 'user-a'); + const ctxB = createGraphQLContext(store, 'user-b'); + expect(ctxA.loaders).not.toBe(ctxB.loaders); + + const before = await ctxA.loaders.responder.load('R-001'); + expect(before.location.lat).toBeCloseTo(6.5201, 6); + expect(before.location.lng).toBeCloseTo(3.3701, 6); + + store.updateResponderLocation('R-001', { lat: 9.999, lng: 8.888 }); + + // A's cache is isolated; B sees the write. + const cachedForA = await ctxA.loaders.responder.load('R-001'); + const freshForB = await ctxB.loaders.responder.load('R-001'); + expect(cachedForA.location.lat).toBeCloseTo(6.5201, 6); + expect(freshForB.location).toEqual({ lat: 9.999, lng: 8.888 }); + }); + + it('returns identical 409 conflict payloads from REST and GraphQL', async () => { + const rest = await request(app) + .post('/api/dispatch/INC-1042/assign') + .send({ responderId: 'R-002', expectedVersion: 2 }); + expect(rest.status).toBe(409); + expect(rest.body).toMatchObject({ conflict: true, serverVersion: 3 }); + + const gql = await executeGraphQL( + { + query: ASSIGN_MUTATION, + variables: { input: { incidentId: 'INC-1042', responderId: 'R-002', expectedVersion: 2 } }, + }, + { store }, + ); + expect(gql.errors).toBeUndefined(); + expect(gql.data.assignResponder).toMatchObject({ + __typename: 'SyncConflict', + conflict: true, + serverVersion: rest.body.serverVersion, + mergeProposal: rest.body.mergeProposal, + }); + }); + + it('returns identical 409 conflict payloads from the REST flush and GraphQL syncFlush', async () => { + const input = { incidentId: 'INC-1042', baseVersion: 2, patch: { priority: 'high' } }; + const rest = await request(app).post('/api/sync/flush').send(input); + expect(rest.status).toBe(409); + expect(rest.body).toMatchObject({ conflict: true, serverVersion: 3 }); + + const gql = await executeGraphQL({ query: FLUSH_MUTATION, variables: { input } }, { store }); + expect(gql.errors).toBeUndefined(); + expect(gql.data.syncFlush).toMatchObject({ + __typename: 'SyncConflict', + conflict: true, + serverVersion: rest.body.serverVersion, + mergeProposal: rest.body.mergeProposal, + }); + }); + + it('returns identical 422 validation issues from REST and GraphQL', async () => { + const input = { incidentId: 'INC-1042', baseVersion: 3, patch: { bogus: true } }; + const rest = await request(app).post('/api/sync/flush').send(input); + expect(rest.status).toBe(422); + expect(rest.body.error).toBe('validation_failed'); + expect(rest.body.issues).toEqual( + expect.arrayContaining([{ path: 'patch.bogus', message: 'unsupported field' }]), + ); + + const gql = await executeGraphQL({ query: FLUSH_MUTATION, variables: { input } }, { store }); + expect(gql.errors).toBeUndefined(); + expect(gql.data.syncFlush).toMatchObject({ + __typename: 'ValidationError', + error: rest.body.error, + issues: rest.body.issues, + }); + }); + + it('returns 422 for a malformed REST body before touching the store', async () => { + store.resetQueryCount(); + const rest = await request(app).post('/api/dispatch/INC-1042/assign').send({}); + expect(rest.status).toBe(422); + expect(rest.body.issues[0]).toMatchObject({ path: 'responderId' }); + expect(store.getQueryCount()).toBe(0); + }); + + it('paginates events with the same cursor on both surfaces', async () => { + const restFirst = await request(app).get('/api/events?incidentId=INC-1042&limit=5'); + expect(restFirst.status).toBe(200); + expect(restFirst.body.events).toHaveLength(5); + const restCursor = restFirst.body.nextCursor; + expect(restCursor).toBeTruthy(); + + const gqlFirst = await executeGraphQL( + { query: EVENTS_QUERY, variables: { id: 'INC-1042', limit: 5 } }, + { store }, + ); + expect(gqlFirst.errors).toBeUndefined(); + expect(gqlFirst.data.events.pageInfo.endCursor).toBe(restCursor); + expect(gqlFirst.data.events.edges.map((edge) => edge.node.id)).toEqual( + restFirst.body.events.map((event) => event.id), + ); + + const restSecond = await request(app).get( + `/api/events?incidentId=INC-1042&limit=5&cursor=${encodeURIComponent(restCursor)}`, + ); + const gqlSecond = await executeGraphQL( + { query: EVENTS_QUERY, variables: { id: 'INC-1042', limit: 5, cursor: restCursor } }, + { store }, + ); + expect(restSecond.body.events.map((event) => event.id)).toEqual( + gqlSecond.data.events.edges.map((edge) => edge.node.id), + ); + }); + + it('invalidates the REST cache when a write happens, so GraphQL and REST converge', async () => { + const before = await request(app).get('/api/dispatch/INC-1042'); + expect(before.body.incident.version).toBe(3); + + const write = await request(app) + .post('/api/dispatch/INC-1042/assign') + .send({ responderId: 'R-002', expectedVersion: 3 }); + expect(write.status).toBe(200); + expect(write.body.incident.version).toBe(4); + + const after = await request(app).get('/api/dispatch/INC-1042'); + expect(after.body.incident.version).toBe(4); + + const gql = await executeGraphQL( + { + query: 'query($id: ID!){ incident(id: $id){ id version } }', + variables: { id: 'INC-1042' }, + }, + { store }, + ); + expect(gql.data.incident.version).toBe(4); + }); + + it('accepts GraphQL over HTTP and rejects a missing query', async () => { + const ok = await request(app) + .post('/graphql') + .send({ query: 'query($id: ID!){ incident(id: $id){ id } }', variables: { id: 'INC-1042' } }); + expect(ok.status).toBe(200); + expect(ok.body.data.incident.id).toBe('INC-1042'); + + const bad = await request(app).post('/graphql').send({}); + expect(bad.status).toBe(400); + }); +}); diff --git a/test/sql-cache.test.js b/test/sql-cache.test.js new file mode 100644 index 00000000..3c9f0cc8 --- /dev/null +++ b/test/sql-cache.test.js @@ -0,0 +1,88 @@ +// @vitest-environment node +import { describe, expect, it, vi } from 'vitest'; +import { NamespacedCache, namespacedKey } from '../server/lib/sqlCache.ts'; + +describe('NamespacedCache', () => { + it('keeps REST and GraphQL projections in separate namespaces', () => { + const cache = new NamespacedCache(); + cache.set('rest', 'dispatch:INC-1', { projection: 'rest' }); + cache.set('gql', 'dispatch:INC-1', { projection: 'gql' }); + + expect(cache.get('rest', 'dispatch:INC-1')).toEqual({ projection: 'rest' }); + expect(cache.get('gql', 'dispatch:INC-1')).toEqual({ projection: 'gql' }); + expect(namespacedKey('rest', 'k')).not.toBe(namespacedKey('gql', 'k')); + }); + + it('invalidates an entity across every surface at once', () => { + const cache = new NamespacedCache(); + cache.set('rest', 'dispatch:INC-1', { version: 1 }); + cache.set('gql', 'dispatch:INC-1', { version: 1 }); + cache.set('rest', 'dispatch:INC-2', { version: 1 }); + + expect(cache.invalidate('dispatch:INC-1')).toBe(2); + expect(cache.get('rest', 'dispatch:INC-1')).toBeUndefined(); + expect(cache.get('gql', 'dispatch:INC-1')).toBeUndefined(); + expect(cache.get('rest', 'dispatch:INC-2')).toEqual({ version: 1 }); + expect(cache.stats().invalidations).toBe(1); + }); + + it('expires entries at the TTL boundary', () => { + let now = 1_000; + const cache = new NamespacedCache({ now: () => now, jitterRatio: 0 }); + cache.set('rest', 'k', 'v', 50); + expect(cache.get('rest', 'k')).toBe('v'); + now = 1_049; + expect(cache.get('rest', 'k')).toBe('v'); + now = 1_050; + expect(cache.get('rest', 'k')).toBeUndefined(); + }); + + it('coalesces concurrent misses into one load', async () => { + const cache = new NamespacedCache(); + let loads = 0; + const load = vi.fn(async () => { + loads++; + await new Promise((resolve) => setTimeout(resolve, 5)); + return { version: 1 }; + }); + + const [a, b, c] = await Promise.all([ + cache.withCache('rest', 'dispatch:INC-1', load), + cache.withCache('rest', 'dispatch:INC-1', load), + cache.withCache('rest', 'dispatch:INC-1', load), + ]); + + expect(loads).toBe(1); + expect(a).toBe(b); + expect(b).toBe(c); + expect(cache.stats().coalesced).toBe(2); + }); + + it('bounds the entry count', () => { + const cache = new NamespacedCache({ maxEntries: 3, jitterRatio: 0 }); + for (let i = 0; i < 10; i++) cache.set('rest', `k${i}`, i); + expect(cache.stats().size).toBeLessThanOrEqual(3); + }); + + it('bypasses both read and write when requested', async () => { + const cache = new NamespacedCache(); + const first = await cache.withCache('rest', 'k', () => 'a', { bypass: true }); + const second = await cache.withCache('rest', 'k', () => 'b', { bypass: true }); + expect(first).toBe('a'); + expect(second).toBe('b'); + expect(cache.get('rest', 'k')).toBeUndefined(); + }); + + it('applies bounded jitter to TTLs', () => { + const random = vi.spyOn(Math, 'random').mockReturnValue(0.999); + let now = 0; + const cache = new NamespacedCache({ now: () => now, jitterRatio: 0.2 }); + cache.set('rest', 'k', 'v', 100); + // max jitter is +20%, so 119ms is alive and 121ms is expired. + now = 119; + expect(cache.get('rest', 'k')).toBe('v'); + now = 121; + expect(cache.get('rest', 'k')).toBeUndefined(); + random.mockRestore(); + }); +}); diff --git a/test/telemetry-pipeline.test.js b/test/telemetry-pipeline.test.js new file mode 100644 index 00000000..aea75c70 --- /dev/null +++ b/test/telemetry-pipeline.test.js @@ -0,0 +1,242 @@ +// @vitest-environment node +import { describe, it, expect, beforeEach } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { RingBuffer } from '../src/lib/ringBuffer.ts'; +import { + TelemetryPipeline, + createMemoryExporter, + shouldSample, + samplingRate, +} from '../src/lib/telemetry.ts'; +import { profileFor } from '../src/lib/networkEstimator.ts'; +import { encodeFrame, parseFrame, FRAME_KIND } from '../src/utils/binaryParser.js'; +import { TelemetryWasmParser } from '../src/utils/binaryParserWasm.js'; +import { createTelemetryWorkerProtocol } from '../src/lib/telemetryWorkerProtocol.js'; + +const frame = (seq, priority = 'normal', kind = 'location') => ({ + seq, + priority, + kind, + ts: 1_700_000_000_000 + seq, + payload: { seq }, +}); + +describe('RingBuffer priority + accounting', () => { + it('evicts heartbeat before normal and normal before critical', () => { + const ring = new RingBuffer({ capacity: 3 }); + expect(ring.push(frame(1, 'heartbeat'))).toBe(true); + expect(ring.push(frame(2, 'normal'))).toBe(true); + expect(ring.push(frame(3, 'critical'))).toBe(true); + + // A new heartbeat must displace the existing heartbeat, never the critical. + expect(ring.push(frame(4, 'heartbeat'))).toBe(true); + expect(ring.snapshot().droppedReasons.overflow_heartbeat).toBe(1); + expect(ring.snapshot().droppedReasons.overflow_critical).toBe(0); + + // The next critical still finds the newest heartbeat and evicts it; only + // after all heartbeats are gone does it touch the normal frame. + expect(ring.push(frame(5, 'critical'))).toBe(true); + expect(ring.snapshot().droppedReasons.overflow_heartbeat).toBe(2); + expect(ring.push(frame(6, 'critical'))).toBe(true); + expect(ring.snapshot().droppedReasons.overflow_normal).toBe(1); + + const snapshot = ring.snapshot(); + expect(snapshot.conservation.ok).toBe(true); + expect(snapshot.size).toBe(3); + }); + + it('rejects an incoming lower-priority frame when the ring is all critical', () => { + const ring = new RingBuffer({ capacity: 2 }); + ring.push(frame(1, 'critical')); + ring.push(frame(2, 'critical')); + expect(ring.push(frame(3, 'heartbeat'))).toBe(false); + const snapshot = ring.snapshot(); + expect(snapshot.rejected).toBe(1); + expect(snapshot.evicted).toBe(0); + expect(snapshot.droppedReasons.overflow_heartbeat).toBe(1); + expect(snapshot.conservation.ok).toBe(true); + }); + + it('reports pressure with 70/90/50 hysteresis', () => { + const ring = new RingBuffer({ capacity: 10 }); + const events = []; + ring.onPressure((event) => events.push(event.level)); + for (let i = 0; i < 7; i++) ring.push(frame(i)); + for (let i = 7; i < 9; i++) ring.push(frame(i)); + ring.drain(5); // occupancy 40% -> normal + expect(events).toEqual(['high', 'critical', 'normal']); + }); + + it('keeps the conservation invariant under random admission/drain', () => { + const ring = new RingBuffer({ capacity: 16 }); + for (let i = 0; i < 2_000; i++) { + ring.push(frame(i, i % 7 === 0 ? 'critical' : i % 3 === 0 ? 'heartbeat' : 'normal')); + if (i % 5 === 0) ring.drain(3); + expect(ring.snapshot().conservation.ok).toBe(true); + } + }); +}); + +describe('adaptive sampling', () => { + it('is deterministic per sequence', () => { + expect(shouldSample(1, 0)).toBe(false); + expect(shouldSample(1, 1)).toBe(true); + expect(shouldSample(42, 0.4)).toBe(shouldSample(42, 0.4)); + }); + + it('keeps 100% of dispatch-critical spans on 2G while exporting ≤20% of baseline', async () => { + const exporter = createMemoryExporter(); + const pipeline = new TelemetryPipeline({ + capacity: 8_192, + networkClass: '2g', + exporter, + }); + + const critical = 100; + const normal = 8_000; + const heartbeat = 2_000; + for (let i = 0; i < critical; i++) pipeline.record({ name: 'dispatch', priority: 'critical' }); + for (let i = 0; i < normal; i++) pipeline.record({ name: 'map', priority: 'normal' }); + for (let i = 0; i < heartbeat; i++) pipeline.record({ name: 'beacon', priority: 'heartbeat' }); + + const exportedCritical = exporter.batches.flat().filter((span) => span.priority === 'critical').length; + // Flush everything (2G batch size is 5) and count critical spans. + for (let i = 0; i < 5_000 && pipeline.ring.length > 0; i++) await pipeline.flush(); + const all = exporter.batches.flat(); + const exportedCriticalAfter = all.filter((span) => span.priority === 'critical').length; + + expect(exportedCritical).toBe(0); + expect(exportedCriticalAfter).toBe(critical); + expect(all.length).toBeLessThanOrEqual((critical + normal + heartbeat) * 0.2); + expect(pipeline.snapshot().conservation.ok).toBe(true); + }); + + it('halves the sampling rate at high pressure', () => { + const profile = profileFor('3g'); + expect(samplingRate(profile, 'normal', 'normal')).toBe(profile.normalSampleRate); + expect(samplingRate(profile, 'high', 'normal')).toBe(profile.normalSampleRate * 0.5); + expect(samplingRate(profile, 'critical', 'critical')).toBe(1); + }); +}); + +describe('export/offline accounting', () => { + it('diverts offline spans to the durable outbox, not memory', async () => { + const queued = []; + const exporter = createMemoryExporter(); + const pipeline = new TelemetryPipeline({ + capacity: 64, + networkClass: 'offline', + exporter, + outbox: { async enqueue(spans) { queued.push(...spans); } }, + }); + for (let i = 0; i < 10; i++) pipeline.record({ name: 'map', priority: 'critical' }); + await pipeline.flush(); + expect(exporter.batches).toHaveLength(0); + expect(queued).toHaveLength(10); + const snapshot = pipeline.snapshot(); + expect(snapshot.outboxed).toBe(10); + expect(snapshot.conservation.ok).toBe(true); + }); + + it('counts export failures with an explicit reason and stays bounded', async () => { + const pipeline = new TelemetryPipeline({ + capacity: 32, + exporter: { + async export() { + const error = new Error('collector down'); + error.retryAfterMs = 1_000; + throw error; + }, + }, + }); + for (let i = 0; i < 20; i++) pipeline.record({ name: 'map', priority: 'critical' }); + await pipeline.flush(); + const snapshot = pipeline.snapshot(); + expect(snapshot.exportFailed).toBeGreaterThan(0); + expect(snapshot.droppedReasons.export_fail).toBe(snapshot.exportFailed); + expect(snapshot.exporterHealthy).toBe(false); + expect(snapshot.conservation.ok).toBe(true); + }); +}); + +describe('binary parser parity', () => { + it('JS and WASM produce identical frames', async () => { + const wasmPath = resolve(process.cwd(), 'src/wasm/telemetry_reader.wasm'); + const shaPath = resolve(process.cwd(), 'src/wasm/telemetry_reader.wasm.sha256'); + const parser = new TelemetryWasmParser({ + wasmBytes: readFileSync(wasmPath), + expectedSha256: readFileSync(shaPath, 'utf8').trim(), + shaUrl: undefined, + }); + const ready = await parser.init(); + expect(ready).toBe(true); + + const frames = [ + { seq: 1, kind: FRAME_KIND.LOCATION, ts: 1_700_000_000_000, lat: 6.5, lng: 3.3, accuracy: 4 }, + { seq: 2, kind: FRAME_KIND.DISPATCH_ACK, ts: 1_700_000_000_001, lat: 6.51, lng: 3.31, accuracy: 5 }, + { seq: 3, kind: FRAME_KIND.HEARTBEAT, ts: 1_700_000_000_002, lat: 6.52, lng: 3.32, accuracy: 6 }, + ]; + const buffer = new Uint8Array(frames.length * 40); + frames.forEach((value, index) => buffer.set(new Uint8Array(encodeFrame(value)), index * 40)); + + for (let i = 0; i < frames.length; i++) { + const js = parseFrame(buffer, i * 40); + const wasm = parser.parseBatch(buffer, frames.length)[i]; + expect(wasm).toMatchObject({ + seq: js.seq, + ts: js.ts, + lat: js.lat, + lng: js.lng, + accuracy: js.accuracy, + kind: js.kind, + priority: js.priority, + }); + } + }); +}); + +describe('worker flow control', () => { + it('sheds non-critical batches at congestion level 2 and parses critical', async () => { + const messages = []; + const parser = { + mode: 'js', + ready: true, + init: async () => true, + parseBatch: () => [{ priority: 'critical', kindName: 'dispatch_ack', seq: 1, ts: 1 }], + }; + const protocol = createTelemetryWorkerProtocol({ + postMessage: (message) => messages.push(message), + parser, + }); + + protocol.handle({ type: 'telemetry:congestion', level: 2 }); + protocol.handle({ type: 'telemetry:parse', id: 1, buffer: new ArrayBuffer(40), count: 1, priority: 'normal' }); + protocol.handle({ type: 'telemetry:parse', id: 2, buffer: new ArrayBuffer(40), count: 1, priority: 'critical' }); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(messages.some((message) => message.type === 'telemetry:shed' && message.reason === 'backpressure')).toBe(true); + expect(messages.some((message) => message.type === 'telemetry:parsed' && message.id === 2)).toBe(true); + expect(messages.some((message) => message.type === 'telemetry:parsed' && message.id === 1)).toBe(false); + }); + + it('bounds the worker queue with drop-oldest accounting', () => { + const messages = []; + const parser = { + mode: 'js', + ready: true, + init: async () => true, + parseBatch: () => [], + }; + const protocol = createTelemetryWorkerProtocol({ + postMessage: (message) => messages.push(message), + parser, + maxQueue: 2, + }); + for (let i = 0; i < 5; i++) { + protocol.handle({ type: 'telemetry:parse', id: i, buffer: new ArrayBuffer(40), count: 1 }); + } + expect(protocol.queueDepth).toBeLessThanOrEqual(2); + expect(protocol.snapshot().queueDrops).toBeGreaterThan(0); + }); +});