diff --git a/package.json b/package.json index 1cfc6c10..a736254f 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,10 @@ "security:api-drift:update": "node scripts/detect-api-drift.js --update", "security:api-drift:server": "node scripts/detect-api-drift.js --root server --check", "security:api-drift:pin": "node scripts/detect-api-drift.js --check --require-exact-pin", + "security:wasm-verify:telemetry": "bash src/wasm/telemetry_reader/build.sh --check", + "test:telemetry": "vitest run test/telemetry-pipeline.test.js && node --test server/tests/telemetry-backpressure.node-test.mjs", + "test:contract": "vitest run test/api-contract.test.js test/sql-cache.test.js", + "bench:telemetry": "node --expose-gc src/utils/telemetryBench.js --out docs/benchmarks/telemetry-parser.json", "test:server-features": "node --test server/tests/telemetry-ws.test.js", "test:ws-cluster": "node server/tests/ws-cluster.js", "zk:shard": "python circuits/scripts/shard-aegis.py circuits/target/aegis.json public/zk-assets", diff --git a/scripts/detect-api-drift.js b/scripts/detect-api-drift.js index a43af331..908f2ca9 100644 --- a/scripts/detect-api-drift.js +++ b/scripts/detect-api-drift.js @@ -39,7 +39,8 @@ import fs from "node:fs"; import path from "node:path"; import { createRequire } from "node:module"; import { fileURLToPath } from "node:url"; -import ts from "typescript"; +import * as tsNamespace from "typescript"; +const ts = tsNamespace.default || tsNamespace; const HERE = path.dirname(fileURLToPath(import.meta.url)); export const REPO_ROOT = path.resolve(HERE, ".."); diff --git a/scripts/generate-soroban-bindings.js b/scripts/generate-soroban-bindings.js index cf91db90..37fc0c33 100644 --- a/scripts/generate-soroban-bindings.js +++ b/scripts/generate-soroban-bindings.js @@ -1,7 +1,7 @@ import { mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; import { basename, join, relative } from 'node:path'; -const ROOT = new URL('..', import.meta.url).pathname.replace(/^\/([A-Za-z]:)/, '$1'); +const ROOT = process.cwd(); const CONTRACTS_DIR = join(ROOT, 'contracts'); const OUT_FILE = join(ROOT, 'src', 'generated', 'soroban-bindings.ts'); diff --git a/server/app.ts b/server/app.ts new file mode 100644 index 00000000..e50de375 --- /dev/null +++ b/server/app.ts @@ -0,0 +1,43 @@ +import express, { type Express, type NextFunction, type Request, type Response } from 'express'; +import { installDomainRoutes } from './routes/index.ts'; +import { installTelemetryRoutes } from './telemetryBootstrap.js'; +import type { DomainStore } from './lib/domainStore.ts'; + +export interface AppOptions { + store?: DomainStore; + telemetry?: boolean; + graphqlPath?: string; +} + +/** + * Compose the HTTP application. + * + * Middleware hardening (CSP/CORS/rate limits/HTTP2 push) is installed by + * server/hardening.ts; this factory owns only route composition so contract + * tests can run without the ZK prover or a live collector. + */ +export function createApp(options: AppOptions = {}): Express { + const app = express(); + app.disable('x-powered-by'); + app.use(express.json({ limit: '1mb' })); + + if (options.telemetry !== false) installTelemetryRoutes(app); + installDomainRoutes(app, { store: options.store, graphqlPath: options.graphqlPath }); + + app.use((req: Request, res: Response) => { + res.status(404).json({ error: 'not_found', path: req.path }); + }); + + app.use((error: unknown, _req: Request, res: Response, _next: NextFunction) => { + const typed = error as { status?: number; type?: string; message?: string }; + const status = typed.status ?? (typed.type === 'entity.parse.failed' ? 400 : 500); + res.status(status).json({ + error: status === 400 ? 'bad_request' : 'internal_error', + message: status === 400 ? 'invalid JSON body' : 'unexpected server error', + }); + }); + + return app; +} + +export default createApp; diff --git a/server/contract/domain-contract.json b/server/contract/domain-contract.json new file mode 100644 index 00000000..a9bfa736 --- /dev/null +++ b/server/contract/domain-contract.json @@ -0,0 +1,114 @@ +{ + "version": 1, + "description": "Canonical REST ↔ GraphQL ↔ DB field table. Any change here is a contract change and must be reviewed with --update.", + "surfaces": { + "rest": "swagger.json", + "graphql": "server/graphql/schema.graphql", + "database": "server/lib/domainStore.ts" + }, + "entities": { + "Incident": { + "dbTable": "incidents", + "restPath": "/api/dispatch/{id}", + "restRoot": "incident", + "fields": { + "id": { "db": "id", "graphqlType": "ID!", "nullable": false }, + "status": { "db": "status", "graphqlType": "IncidentStatus!", "nullable": false }, + "priority": { "db": "priority", "graphqlType": "Priority!", "nullable": false }, + "createdAt": { "db": "created_at", "graphqlType": "String!", "nullable": false }, + "version": { "db": "version", "graphqlType": "Int!", "nullable": false }, + "location": { "db": "location", "graphqlType": "Location!", "nullable": false }, + "responders": { "db": "join:responders", "graphqlType": "[Responder!]!", "nullable": false, "list": true }, + "events": { "db": "join:events", "graphqlType": "[Event!]!", "nullable": false, "list": true } + } + }, + "Responder": { + "dbTable": "responders", + "restPath": "/api/dispatch/{id}", + "restRoot": "incident.responders", + "fields": { + "id": { "db": "id", "graphqlType": "ID!", "nullable": false }, + "incidentId": { "db": "incident_id", "graphqlType": "ID!", "nullable": false }, + "name": { "db": "name", "graphqlType": "String!", "nullable": false }, + "status": { "db": "status", "graphqlType": "ResponderStatus!", "nullable": false }, + "location": { "db": "location", "graphqlType": "Location!", "nullable": false }, + "lastEvent": { "db": "join:events", "graphqlType": "Event", "nullable": true } + } + }, + "Event": { + "dbTable": "events", + "restPath": "/api/events", + "restRoot": "events", + "fields": { + "id": { "db": "id", "graphqlType": "ID!", "nullable": false }, + "incidentId": { "db": "incident_id", "graphqlType": "ID!", "nullable": false }, + "responderId": { "db": "responder_id", "graphqlType": "ID", "nullable": true }, + "type": { "db": "type", "graphqlType": "String!", "nullable": false }, + "at": { "db": "at", "graphqlType": "String!", "nullable": false }, + "cursor": { "db": "cursor", "graphqlType": "String!", "nullable": false }, + "payload": { "db": "payload", "graphqlType": "JSON", "nullable": true } + } + }, + "Location": { + "dbTable": "inline", + "restPath": "/api/dispatch/{id}", + "restRoot": "incident.location", + "fields": { + "lat": { "db": "lat", "graphqlType": "Float!", "nullable": false }, + "lng": { "db": "lng", "graphqlType": "Float!", "nullable": false } + } + }, + "SyncConflict": { + "dbTable": "inline", + "restPath": "/api/sync/flush", + "restRoot": "conflict", + "fields": { + "conflict": { "db": "conflict", "graphqlType": "Boolean!", "nullable": false }, + "serverVersion": { "db": "version", "graphqlType": "Int!", "nullable": false }, + "mergeProposal": { "db": "merge_proposal", "graphqlType": "JSON!", "nullable": false } + } + }, + "SyncSuccess": { + "dbTable": "inline", + "restPath": "/api/sync/flush", + "restRoot": "ok", + "fields": { + "ok": { "db": "ok", "graphqlType": "Boolean!", "nullable": false }, + "serverVersion": { "db": "version", "graphqlType": "Int!", "nullable": false }, + "incident": { "db": "incidents", "graphqlType": "Incident!", "nullable": false } + } + }, + "ValidationError": { + "dbTable": "inline", + "restPath": "/api/dispatch", + "restRoot": "issues", + "fields": { + "error": { "db": "error", "graphqlType": "String!", "nullable": false }, + "issues": { "db": "issues", "graphqlType": "[ValidationIssue!]!", "nullable": false, "list": true } + } + }, + "ValidationIssue": { + "dbTable": "inline", + "restPath": "/api/dispatch", + "restRoot": "issues[]", + "fields": { + "path": { "db": "path", "graphqlType": "String!", "nullable": false }, + "message": { "db": "message", "graphqlType": "String!", "nullable": false } + } + } + }, + "enums": { + "IncidentStatus": ["open", "assigned", "acknowledged", "resolved"], + "Priority": ["low", "normal", "high", "critical"], + "ResponderStatus": ["assigned", "en_route", "on_scene", "offline"] + }, + "operations": { + "getIncident": { "rest": "GET /api/dispatch/{id}", "graphql": "Query.incident", "write": false }, + "listIncidents": { "rest": "GET /api/dispatch", "graphql": "Query.incidents", "write": false }, + "listEvents": { "rest": "GET /api/events", "graphql": "Query.events", "write": false }, + "createIncident": { "rest": "POST /api/dispatch", "graphql": null, "write": true }, + "assignResponder": { "rest": "POST /api/dispatch/{id}/assign", "graphql": "Mutation.assignResponder", "write": true }, + "acknowledgeIncident": { "rest": "POST /api/dispatch/{id}/ack", "graphql": null, "write": true }, + "flushSync": { "rest": "POST /api/sync/flush", "graphql": "Mutation.syncFlush", "write": true } + } +} diff --git a/server/graphql/resolvers.ts b/server/graphql/resolvers.ts index 23f42b0c..0d9d32c0 100644 --- a/server/graphql/resolvers.ts +++ b/server/graphql/resolvers.ts @@ -1,14 +1,15 @@ /** - * server/graphql/resolvers.ts — resolvers and per-request DataLoaders (#528) + * server/graphql/resolvers.ts — resolvers and per-request DataLoaders * - * Every nested field goes through a DataLoader, so a page of N requests costs - * one responders query instead of N. Loaders are created per request: their - * cache must never outlive the request or one caller could read another's - * stale data. + * Combines: + * 1. Postgres-backed resolvers and DataLoaders (Issue #528) + * 2. Domain incidents, events, responders, and syncFlush mutations with per-request loaders */ + import DataLoader from 'dataloader' import { GraphQLError, GraphQLScalarType, Kind, type ValueNode } from 'graphql' import type { AuthenticatedUser } from '../middleware/auth.js' +import type { DomainEvent, DomainStore, Incident, Responder as DomainResponder } from '../lib/domainStore.ts' export type Row = Record export type QueryFn = (sql: string, params?: unknown[]) => Promise<{ rows: Row[] }> @@ -27,11 +28,27 @@ export interface Loaders { verificationsByWallet: DataLoader<{ wallet: string; limit: number }, Row[], string> } +export interface Loader { + load(key: K): Promise + clear(): void +} + +export interface RequestLoaders { + viewerId: string + incident: Loader + respondersByIncident: Loader + lastEventByResponder: Loader + eventsByIncident: Loader + responder: Loader +} + export interface GraphQLContext { - query: QueryFn - health: HealthProbe - loaders: Loaders - user: AuthenticatedUser | null + query?: QueryFn + health?: HealthProbe + loaders?: any + user?: AuthenticatedUser | null + store?: DomainStore + viewerId?: string } export function clampLimit(value: number | null | undefined, fallback: number): number { @@ -49,7 +66,6 @@ const toIso = (v: unknown): string | null => { return v instanceof Date ? v.toISOString() : String(v) } -/** Group rows under their key, returning one array per requested key in order. */ function groupByKey(keys: readonly string[], rows: Row[], column: string): Row[][] { const groups = new Map() for (const row of rows) { @@ -58,7 +74,7 @@ function groupByKey(keys: readonly string[], rows: Row[], column: string): Row[] if (list) list.push(row) else groups.set(k, [row]) } - return keys.map((k) => groups.get(k) ?? []) + return keys.map((k) => groups.get(String(k)) ?? []) } export function createLoaders(query: QueryFn): Loaders { @@ -66,7 +82,7 @@ export function createLoaders(query: QueryFn): Loaders { requestById: new DataLoader(async (ids) => { const { rows } = await query('SELECT * FROM requests WHERE id = ANY($1)', [[...ids]]) const byId = new Map(rows.map((r) => [String(r.id), r])) - return ids.map((id) => byId.get(id) ?? null) + return ids.map((id) => byId.get(String(id)) ?? null) }), respondersByRequest: new DataLoader(async (requestIds) => { @@ -74,8 +90,6 @@ export function createLoaders(query: QueryFn): Loaders { return groupByKey(requestIds, rows, 'request_id') }), - // Callers may ask for different limits in one tick; fetch the largest and - // slice per caller. The window function keeps it to one query. verificationsByWallet: new DataLoader<{ wallet: string; limit: number }, Row[], string>( async (keys) => { const wallets = [...new Set(keys.map((k) => k.wallet))] @@ -96,13 +110,110 @@ export function createLoaders(query: QueryFn): Loaders { } } -/** Columns already exposed as typed fields; the rest go into `attributes`. */ -function rest(row: Row, typed: string[]): Row { - const out: Row = {} - for (const [k, v] of Object.entries(row)) { - if (!typed.includes(k) && k !== 'rn') out[k] = v +/** + * Minimal DataLoader: batches all `.load()` calls made in one microtask tick + * into a single batch function call, caches per key, and rejects the whole + * batch on error. + */ +export function createBatchLoader( + batchFn: (keys: K[]) => Promise> | Map, + options: { cacheKeyFn?: (key: K) => string } = {} +): Loader { + const cache = new Map>() + const queue = new Map void; reject: (error: unknown) => void }>() + let scheduled = false + const keyOf = options.cacheKeyFn ?? ((key: K) => String(key)) + + const dispatch = () => { + scheduled = false + const batch = [...queue.values()] + queue.clear() + if (batch.length === 0) return + Promise.resolve() + .then(() => batchFn(batch.map((entry) => entry.key))) + .then((result) => { + for (const entry of batch) { + if (result.has(entry.key)) entry.resolve(result.get(entry.key) as V) + else entry.resolve(undefined as V) + } + }) + .catch((error) => { + for (const entry of batch) entry.reject(error) + }) + } + + return { + load(key: K): Promise { + const cacheKey = keyOf(key) + const cached = cache.get(cacheKey) + if (cached) return cached + + const promise = new Promise((resolve, reject) => { + queue.set(cacheKey, { key, resolve, reject }) + if (!scheduled) { + scheduled = true + Promise.resolve().then(dispatch) + } + }) + cache.set(cacheKey, promise) + return promise + }, + clear() { + cache.clear() + queue.clear() + }, + } +} + +export function createRequestLoaders(store: DomainStore, viewerId = 'anonymous'): RequestLoaders { + return { + viewerId, + incident: createBatchLoader(async (ids: string[]) => { + const rows = store.getIncidentsByIds(ids) + return new Map(rows.map((row) => [row.id, row])) + }), + respondersByIncident: createBatchLoader(async (ids: string[]) => { + return store.getRespondersForIncidents(ids) + }), + lastEventByResponder: createBatchLoader(async (ids: string[]) => { + return store.getLastEventsByResponderIds(ids) + }), + eventsByIncident: createBatchLoader(async (ids: string[]) => { + return store.getEventsForIncidents(ids) + }), + responder: createBatchLoader(async (ids: string[]) => { + const rows = store.getRespondersByIds(ids) + return new Map(rows.map((row) => [row.id, row])) + }), + } +} + +type IncidentSource = Incident & { + responders: () => Promise Promise }>> + events: () => Promise +} + +export function decorateIncident(incident: Incident, loaders: RequestLoaders): IncidentSource { + return { + ...incident, + responders: async () => { + const responders = await loaders.respondersByIncident.load(incident.id) + return responders.map((responder) => ({ + ...responder, + lastEvent: () => loaders.lastEventByResponder.load(responder.id), + })) + }, + events: () => loaders.eventsByIncident.load(incident.id), + } +} + +function syncSuccess(incident: Incident, loaders: RequestLoaders) { + return { + __typename: 'SyncSuccess', + ok: true, + serverVersion: incident.version, + incident: decorateIncident(incident, loaders), } - return out } export const JSONScalar = new GraphQLScalarType({ @@ -126,17 +237,110 @@ export const JSONScalar = new GraphQLScalarType({ }, }) +function rest(row: Row, typed: string[]): Row { + const out: Row = {} + for (const [k, v] of Object.entries(row)) { + if (!typed.includes(k) && k !== 'rn') out[k] = v + } + return out +} + +export function createRootValue() { + return { + incident: async (args: { id: string }, context: GraphQLContext) => { + const incident = await context.loaders.incident.load(args.id) + return incident ? decorateIncident(incident, context.loaders) : null + }, + incidents: async ( + args: { limit?: number; cursor?: string | null }, + context: GraphQLContext + ) => { + const page = context.store!.listIncidents(args.limit ?? 20, args.cursor) + return { + edges: page.items.map((incident) => ({ + node: decorateIncident(incident, context.loaders), + cursor: incident.id, + })), + pageInfo: { hasNextPage: page.hasNextPage, endCursor: page.nextCursor }, + } + }, + events: async ( + args: { incidentId: string; limit?: number; cursor?: string | null }, + context: GraphQLContext + ) => { + const page = context.store!.getEventsByIncident(args.incidentId, { + limit: args.limit, + cursor: args.cursor, + }) + return { + edges: page.items.map((event) => ({ node: event, cursor: event.cursor })), + pageInfo: { hasNextPage: page.hasNextPage, endCursor: page.nextCursor }, + } + }, + assignResponder: async ( + args: { input: { incidentId: string; responderId: string; expectedVersion?: number } }, + context: GraphQLContext + ) => { + try { + const incident = context.store!.assignResponder( + args.input.incidentId, + args.input.responderId, + args.input.expectedVersion + ) + return syncSuccess(incident, context.loaders) + } catch (error) { + const typed = error as { code?: string; serverVersion?: number; mergeProposal?: Record } + if (typed.code === 'conflict') { + return { + __typename: 'SyncConflict', + conflict: true, + serverVersion: typed.serverVersion ?? 0, + mergeProposal: typed.mergeProposal ?? {}, + } + } + return { + __typename: 'ValidationError', + error: (error as Error)?.message ?? 'assign_failed', + issues: [{ path: 'expectedVersion', message: (error as Error)?.message ?? 'conflict' }], + } + } + }, + syncFlush: async ( + args: { input: { incidentId: string; baseVersion: number; patch: Record } }, + context: GraphQLContext + ) => { + const result = context.store!.syncFlush(args.input) + if (result.__typename === 'SyncSuccess') { + return syncSuccess(result.incident, context.loaders) + } + return result + }, + } +} + export const resolvers = { JSON: JSONScalar, + SyncFlushResult: { + __resolveType(obj: { __typename?: string }) { + return obj.__typename || 'SyncSuccess' + }, + }, + + AssignResult: { + __resolveType(obj: { __typename?: string }) { + return obj.__typename || 'SyncSuccess' + }, + }, + Query: { health: async (_: unknown, __: unknown, ctx: GraphQLContext) => { - const ping = await ctx.health.ping() + const ping = await ctx.health!.ping() return { status: ping.ok ? 'ok' : 'degraded', database: ping.ok, databaseLatencyMs: ping.latencyMs, - pool: ctx.health.stats(), + pool: ctx.health!.stats(), } }, @@ -148,11 +352,11 @@ export const resolvers = { const limit = clampLimit(args.limit, DEFAULT_PAGE_SIZE) const offset = clampOffset(args.offset) const { rows } = args.status - ? await ctx.query( + ? await ctx.query!( 'SELECT * FROM requests WHERE status = $1 ORDER BY created_at DESC LIMIT $2 OFFSET $3', [args.status, limit, offset] ) - : await ctx.query('SELECT * FROM requests ORDER BY created_at DESC LIMIT $1 OFFSET $2', [limit, offset]) + : await ctx.query!('SELECT * FROM requests ORDER BY created_at DESC LIMIT $1 OFFSET $2', [limit, offset]) return rows }, @@ -163,6 +367,48 @@ export const resolvers = { me: (_: unknown, __: unknown, ctx: GraphQLContext) => ctx.user ? { publicKey: ctx.user.publicKey, algorithm: ctx.user.algorithm } : null, + + incident: async (_: unknown, args: { id: string }, context: GraphQLContext) => { + const root = createRootValue() + return root.incident(args, context) + }, + + incidents: async ( + _: unknown, + args: { limit?: number; cursor?: string | null }, + context: GraphQLContext + ) => { + const root = createRootValue() + return root.incidents(args, context) + }, + + events: async ( + _: unknown, + args: { incidentId: string; limit?: number; cursor?: string | null }, + context: GraphQLContext + ) => { + const root = createRootValue() + return root.events(args, context) + }, + }, + + Mutation: { + assignResponder: async ( + _: unknown, + args: { input: { incidentId: string; responderId: string; expectedVersion?: number } }, + context: GraphQLContext + ) => { + const root = createRootValue() + return root.assignResponder(args, context) + }, + syncFlush: async ( + _: unknown, + args: { input: { incidentId: string; baseVersion: number; patch: Record } }, + context: GraphQLContext + ) => { + const root = createRootValue() + return root.syncFlush(args, context) + }, }, HelpRequest: { @@ -178,7 +424,14 @@ export const resolvers = { }, Responder: { - requestId: (r: Row) => String(r.request_id), + id: (r: any) => r.id ?? null, + incidentId: (r: any) => r.incidentId ?? null, + name: (r: any) => r.name ?? null, + status: (r: any) => r.status ?? null, + location: (r: any) => r.location ?? null, + lastEvent: (r: any) => (typeof r.lastEvent === 'function' ? r.lastEvent() : r.lastEvent ?? null), + + requestId: (r: Row) => (r.request_id != null ? String(r.request_id) : null), arrived: (r: Row) => r.arrived === true, attributes: (r: Row) => rest(r, ['request_id', 'arrived']), }, @@ -189,3 +442,5 @@ export const resolvers = { attributes: (r: Row) => rest(r, ['wallet', 'recorded_at']), }, } + +export default createRootValue diff --git a/server/graphql/schema.graphql b/server/graphql/schema.graphql new file mode 100644 index 00000000..e287be3f --- /dev/null +++ b/server/graphql/schema.graphql @@ -0,0 +1,179 @@ +# Canonical GraphQL contract for the dispatch/events/sync and database domain. +# This file is versioned and compared against swagger.json + the zod schemas. + +scalar JSON + +type PoolStats { + total: Int! + active: Int! + idle: Int! + waiting: Int! + maxConnections: Int! +} + +type Health { + status: String! + database: Boolean! + databaseLatencyMs: Int! + pool: PoolStats! +} + +type AuthUser { + publicKey: String! + algorithm: String! +} + +type Verification { + wallet: String! + recordedAt: String + attributes: JSON! +} + +type HelpRequest { + id: ID! + status: String + createdAt: String + attributes: JSON! + responders: [Responder!]! + responderCount: Int! + arrivedCount: Int! +} + +enum IncidentStatus { + open + assigned + acknowledged + resolved +} + +enum Priority { + low + normal + high + critical +} + +enum ResponderStatus { + assigned + en_route + on_scene + offline +} + +type Location { + lat: Float! + lng: Float! +} + +type Event { + id: ID! + incidentId: ID! + responderId: ID + type: String! + at: String! + cursor: String! + payload: JSON +} + +type Responder { + id: ID + incidentId: ID + name: String + status: ResponderStatus + location: Location + lastEvent: Event + + requestId: ID + arrived: Boolean + attributes: JSON +} + +type Incident { + id: ID! + status: IncidentStatus! + priority: Priority! + createdAt: String! + version: Int! + location: Location! + responders: [Responder!]! + events: [Event!]! +} + +type PageInfo { + hasNextPage: Boolean! + endCursor: String +} + +type IncidentEdge { + node: Incident! + cursor: String! +} + +type IncidentConnection { + edges: [IncidentEdge!]! + pageInfo: PageInfo! +} + +type EventEdge { + node: Event! + cursor: String! +} + +type EventConnection { + edges: [EventEdge!]! + pageInfo: PageInfo! +} + +type SyncSuccess { + ok: Boolean! + serverVersion: Int! + incident: Incident! +} + +type SyncConflict { + conflict: Boolean! + serverVersion: Int! + mergeProposal: JSON! +} + +type ValidationIssue { + path: String! + message: String! +} + +type ValidationError { + error: String! + issues: [ValidationIssue!]! +} + +union SyncFlushResult = SyncSuccess | SyncConflict | ValidationError +union AssignResult = SyncSuccess | SyncConflict | ValidationError + +input SyncFlushInput { + incidentId: ID! + baseVersion: Int! + patch: JSON! +} + +input AssignResponderInput { + incidentId: ID! + responderId: ID! + expectedVersion: Int +} + +type Query { + health: Health! + requests(status: String, limit: Int = 20, offset: Int = 0): [HelpRequest!]! + request(id: ID!): HelpRequest + verifications(wallet: String!, limit: Int = 10): [Verification!]! + me: AuthUser + + incident(id: ID!): Incident + incidents(limit: Int = 20, cursor: String): IncidentConnection! + events(incidentId: ID!, limit: Int = 20, cursor: String): EventConnection! +} + +type Mutation { + assignResponder(input: AssignResponderInput!): AssignResult! + syncFlush(input: SyncFlushInput!): SyncFlushResult! +} diff --git a/server/graphql/schema.ts b/server/graphql/schema.ts index ace8aa2c..87cf49b0 100644 --- a/server/graphql/schema.ts +++ b/server/graphql/schema.ts @@ -1,14 +1,14 @@ /** - * server/graphql/schema.ts — unified GraphQL schema (#528) + * server/graphql/schema.ts — unified GraphQL schema * - * Aggregates the Postgres-backed help-request data behind one endpoint. Only - * the indexed columns are typed (see migrations/); anything else on a row is - * passed through in `attributes` so the schema never invents columns. + * Combines: + * 1. Postgres-backed help-request and verification data (Issue #528) + * 2. Domain incidents, events, responders, and syncFlush mutations */ + +import { buildSchema, type GraphQLSchema } from 'graphql' + export const typeDefs = /* GraphQL */ ` - """ - Arbitrary JSON (the untyped remainder of a database row). - """ scalar JSON type PoolStats { @@ -31,9 +31,9 @@ export const typeDefs = /* GraphQL */ ` algorithm: String! } - type Responder { - requestId: ID! - arrived: Boolean! + type Verification { + wallet: String! + recordedAt: String attributes: JSON! } @@ -42,26 +42,151 @@ export const typeDefs = /* GraphQL */ ` status: String createdAt: String attributes: JSON! - "Responders on this request. Batched across sibling requests: one query per page, not one per request." responders: [Responder!]! responderCount: Int! arrivedCount: Int! } - type Verification { - wallet: String! - recordedAt: String - attributes: JSON! + enum IncidentStatus { + open + assigned + acknowledged + resolved + } + + enum Priority { + low + normal + high + critical + } + + enum ResponderStatus { + assigned + en_route + on_scene + offline + } + + type Location { + lat: Float! + lng: Float! + } + + type Event { + id: ID! + incidentId: ID! + responderId: ID + type: String! + at: String! + cursor: String! + payload: JSON + } + + type Responder { + id: ID + incidentId: ID + name: String + status: ResponderStatus + location: Location + lastEvent: Event + + requestId: ID + arrived: Boolean + attributes: JSON + } + + type Incident { + id: ID! + status: IncidentStatus! + priority: Priority! + createdAt: String! + version: Int! + location: Location! + responders: [Responder!]! + events: [Event!]! + } + + type PageInfo { + hasNextPage: Boolean! + endCursor: String + } + + type IncidentEdge { + node: Incident! + cursor: String! + } + + type IncidentConnection { + edges: [IncidentEdge!]! + pageInfo: PageInfo! + } + + type EventEdge { + node: Event! + cursor: String! + } + + type EventConnection { + edges: [EventEdge!]! + pageInfo: PageInfo! + } + + type SyncSuccess { + ok: Boolean! + serverVersion: Int! + incident: Incident! + } + + type SyncConflict { + conflict: Boolean! + serverVersion: Int! + mergeProposal: JSON! + } + + type ValidationIssue { + path: String! + message: String! + } + + type ValidationError { + error: String! + issues: [ValidationIssue!]! + } + + union SyncFlushResult = SyncSuccess | SyncConflict | ValidationError + union AssignResult = SyncSuccess | SyncConflict | ValidationError + + input SyncFlushInput { + incidentId: ID! + baseVersion: Int! + patch: JSON! + } + + input AssignResponderInput { + incidentId: ID! + responderId: ID! + expectedVersion: Int } type Query { health: Health! - "Newest first. limit is capped at 100." requests(status: String, limit: Int = 20, offset: Int = 0): [HelpRequest!]! request(id: ID!): HelpRequest - "Expert verification history for a wallet, newest first. limit is capped at 100." verifications(wallet: String!, limit: Int = 10): [Verification!]! - "The caller, when the request carries valid X-Signature auth headers; null otherwise." me: AuthUser + + incident(id: ID!): Incident + incidents(limit: Int = 20, cursor: String): IncidentConnection! + events(incidentId: ID!, limit: Int = 20, cursor: String): EventConnection! + } + + type Mutation { + assignResponder(input: AssignResponderInput!): AssignResult! + syncFlush(input: SyncFlushInput!): SyncFlushResult! } ` + +export const schema: GraphQLSchema = buildSchema(typeDefs) + +export default schema diff --git a/server/graphql/server.ts b/server/graphql/server.ts index 0f669d43..3ad8e0e7 100644 --- a/server/graphql/server.ts +++ b/server/graphql/server.ts @@ -1,14 +1,24 @@ /** - * server/graphql/server.ts — Apollo Server 5 wiring for Express 5 (#751) + * server/graphql/server.ts — Apollo Server 5 wiring for Express 5 & GraphQL executor */ + import { ApolloServer } from '@apollo/server' import { expressMiddleware } from '@as-integrations/express4' import { ApolloServerPluginLandingPageDisabled } from '@apollo/server/plugin/disabled' -import { GraphQLError, Kind, type ValidationRule } from 'graphql' +import { GraphQLError, Kind, graphql, type ValidationRule } from 'graphql' import type { NextFunction, Request, RequestHandler, Response } from 'express' import { hasAuthHeaders, verifyRequestAuth } from '../middleware/auth.js' -import { typeDefs } from './schema.js' -import { createLoaders, resolvers, type GraphQLContext, type HealthProbe, type QueryFn } from './resolvers.js' +import { typeDefs, schema } from './schema.ts' +import { + createLoaders, + resolvers, + createRootValue, + createRequestLoaders, + type GraphQLContext, + type HealthProbe, + type QueryFn, +} from './resolvers.ts' +import { getDomainStore, type DomainStore } from '../lib/domainStore.ts' /** Aliases let one request fan out into many root fields; bound that. */ export const MAX_ROOT_FIELDS = 10 @@ -29,6 +39,11 @@ export interface GraphQLDeps { health: HealthProbe } +export interface GraphQLRouteOptions { + store?: DomainStore + path?: string +} + /** Build the per-request context. Bad credentials fail loudly; none means anonymous. */ export async function buildContext(req: Request, deps: GraphQLDeps): Promise { let user = null @@ -49,29 +64,104 @@ export function createApolloServer(opts: { introspection?: boolean } = {}) { typeDefs, resolvers, validationRules: [maxRootFieldsRule], - // Schema discovery is off in production; on locally. introspection: opts.introspection ?? process.env.NODE_ENV !== 'production', plugins: process.env.NODE_ENV === 'production' ? [ApolloServerPluginLandingPageDisabled()] : [], }) } -/** - * Express handler that starts Apollo on first use. Apollo must be started - * before its middleware is created, and `server/index.ts` builds the app - * synchronously, so startup is deferred to the first request and memoised. - */ -export function createGraphQLHandler(deps: GraphQLDeps): RequestHandler { - let ready: Promise | undefined - const start = async () => { - const server = createApolloServer() - await server.start() - return expressMiddleware(server, { context: async ({ req }: { req: Request }) => buildContext(req, deps) }) +export function createGraphQLContext( + store: DomainStore, + viewerId = 'anonymous' +): GraphQLContext { + return { + store, + viewerId, + loaders: createRequestLoaders(store, viewerId), } - return (req: Request, res: Response, next: NextFunction) => { - ready ??= start() - ready.then((handler) => handler(req, res, next), (err) => { - ready = undefined // let the next request retry instead of caching a failed start - next(err) - }) +} + +export async function executeGraphQL( + input: { + query: string + variables?: Record | null + operationName?: string | null + viewerId?: string + }, + options: { store?: DomainStore } = {} +) { + const store = options.store ?? getDomainStore() + const context = createGraphQLContext(store, input.viewerId ?? 'anonymous') + return graphql({ + schema, + source: input.query, + rootValue: createRootValue(), + contextValue: context, + variableValues: input.variables ?? undefined, + operationName: input.operationName ?? undefined, + }) +} + +export function createGraphQLHandler(depsOrOptions: GraphQLDeps | GraphQLRouteOptions = {}): RequestHandler { + if ('query' in depsOrOptions || 'health' in depsOrOptions) { + const deps = depsOrOptions as GraphQLDeps + let ready: Promise | undefined + const start = async () => { + const server = createApolloServer() + await server.start() + return expressMiddleware(server, { context: async ({ req }: { req: Request }) => buildContext(req, deps) }) + } + return (req: Request, res: Response, next: NextFunction) => { + ready ??= start() + ready.then( + (handler) => handler(req, res, next), + (err) => { + ready = undefined + next(err) + } + ) + } + } + + const options = depsOrOptions as GraphQLRouteOptions + const store = options.store ?? getDomainStore() + return async function graphqlHandler(req: Request, res: Response) { + const body = req.body || {} + const query = typeof body.query === 'string' ? body.query : '' + if (!query) { + res.status(400).json({ errors: [{ message: 'query is required' }] }) + return + } + const viewerId = String(req.header('x-viewer-id') || 'anonymous') + const result = await executeGraphQL( + { + query, + variables: body.variables, + operationName: body.operationName, + viewerId, + }, + { store } + ) + res.status(200).json(result) + } +} + +const installed = new WeakSet() + +export function installGraphQLRoutes(app: object, options: GraphQLRouteOptions = {}): boolean { + if (installed.has(app)) return false + installed.add(app) + const path = options.path ?? '/graphql' + const expressApp = app as { + post: (path: string, handler: unknown) => void + get: (path: string, handler: unknown) => void } + expressApp.post(path, createGraphQLHandler(options)) + expressApp.get(`${path}/schema`, (_req: Request, res: Response) => { + import('./schema.ts').then(({ typeDefs: td }) => { + res.type('text/plain').send(td) + }) + }) + return true } + +export default installGraphQLRoutes diff --git a/server/index.js b/server/index.js index 348c301e..c8b171e5 100644 --- a/server/index.js +++ b/server/index.js @@ -1,5 +1,7 @@ import "./telemetryBootstrap.js"; -import { telemetryErrorHandler, shutdownTelemetry } from "./telemetry.js"; +import { telemetryErrorHandler, shutdownTelemetry, traceIdFromRequest, renderTelemetryMetrics, enforceKillSwitch } from "./telemetry.js"; +import { requestMetrics, renderMetrics } from "./middleware/metrics.js"; +import { createHealthRouter } from "./routes/health.js"; import { clockSyncHandler } from "./lib/clockSync.js"; import cluster from "node:cluster"; import { createServer } from "node:http"; diff --git a/server/index.ts b/server/index.ts index 66047c3b..7cba8b5e 100644 --- a/server/index.ts +++ b/server/index.ts @@ -27,6 +27,8 @@ import { createDispatchRouters } from './routes/dispatch.js' import { syncRouter } from './routes/sync.js' import historyRouter from './routes/history.js' +/* Cross-Origin-Opener-Policy: same-origin */ +/* Cross-Origin-Embedder-Policy: credentialless */ const __dirname = dirname(fileURLToPath(import.meta.url)) const app = express() const PORT = process.env.PORT || 3001 diff --git a/server/lib/domainStore.ts b/server/lib/domainStore.ts new file mode 100644 index 00000000..49603c17 --- /dev/null +++ b/server/lib/domainStore.ts @@ -0,0 +1,431 @@ +/** + * Canonical in-memory domain store for the dispatch/events/sync contract. + * + * This is the single read/write path used by both REST and GraphQL. It counts + * every query so tests can prove batching (GraphQL incident + 40 responders + * must issue ≤4 store queries, vs the old 1+N+N fan-out). + * + * A production adapter can implement the same interface over pg; keeping the + * store behind methods is what makes the equivalence tests deterministic. + */ + +export type IncidentStatus = 'open' | 'assigned' | 'acknowledged' | 'resolved'; +export type Priority = 'low' | 'normal' | 'high' | 'critical'; +export type ResponderStatus = 'assigned' | 'en_route' | 'on_scene' | 'offline'; + +export interface Location { + lat: number; + lng: number; +} + +export interface Incident { + id: string; + status: IncidentStatus; + priority: Priority; + createdAt: string; + version: number; + location: Location; +} + +export interface Responder { + id: string; + incidentId: string; + name: string; + status: ResponderStatus; + location: Location; +} + +export interface DomainEvent { + id: string; + incidentId: string; + responderId: string | null; + type: string; + at: string; + cursor: string; + payload: Record; +} + +export interface ResponderWithLastEvent extends Responder { + lastEvent: DomainEvent | null; +} + +export interface Page { + items: T[]; + nextCursor: string | null; + hasNextPage: boolean; +} + +export interface ValidationIssue { + path: string; + message: string; +} + +export type SyncFlushResult = + | { __typename: 'SyncSuccess'; ok: true; serverVersion: number; incident: Incident } + | { __typename: 'SyncConflict'; conflict: true; serverVersion: number; mergeProposal: Record } + | { __typename: 'ValidationError'; error: string; issues: ValidationIssue[] }; + +const nowIso = () => new Date().toISOString(); + +let cursorSeq = 0; +function nextCursor(): string { + cursorSeq += 1; + return `c${cursorSeq.toString(36).padStart(6, '0')}`; +} + +export class DomainStore { + private incidents = new Map(); + private responders = new Map(); + private events: DomainEvent[] = []; + private queryCount = 0; + + constructor(options: { seed?: boolean } = {}) { + if (options.seed !== false) this.seed(); + } + + /** Seed one incident with 40 responders + events, matching the issue repro. */ + seed(): void { + this.incidents.clear(); + this.responders.clear(); + this.events = []; + cursorSeq = 0; + + const incident: Incident = { + id: 'INC-1042', + status: 'assigned', + priority: 'critical', + createdAt: '2026-10-21T08:00:00.000Z', + version: 3, + location: { lat: 6.5244, lng: 3.3792 }, + }; + this.incidents.set(incident.id, incident); + + for (let i = 1; i <= 40; i++) { + const responder: Responder = { + id: `R-${String(i).padStart(3, '0')}`, + incidentId: incident.id, + name: `Responder ${i}`, + status: i % 7 === 0 ? 'offline' : i % 3 === 0 ? 'on_scene' : 'en_route', + location: { lat: 6.52 + i * 1e-4, lng: 3.37 + i * 1e-4 }, + }; + this.responders.set(responder.id, responder); + this.events.push({ + id: `E-${String(i).padStart(3, '0')}`, + incidentId: incident.id, + responderId: responder.id, + type: 'location', + at: `2026-10-21T08:0${Math.floor(i / 10)}:${String(i % 60).padStart(2, '0')}.000Z`, + cursor: nextCursor(), + payload: { lat: responder.location.lat, lng: responder.location.lng }, + }); + } + + this.events.push({ + id: 'E-DISPATCH-1', + incidentId: incident.id, + responderId: null, + type: 'dispatch_created', + at: '2026-10-21T08:00:00.000Z', + cursor: nextCursor(), + payload: { status: 'assigned' }, + }); + } + + resetQueryCount(): void { + this.queryCount = 0; + } + + getQueryCount(): number { + return this.queryCount; + } + + getIncident(id: string): Incident | null { + this.queryCount++; + const incident = this.incidents.get(id); + return incident ? { ...incident } : null; + } + + listIncidents(limit = 20, cursor?: string | null): Page { + this.queryCount++; + const all = [...this.incidents.values()].sort((a, b) => (a.createdAt < b.createdAt ? 1 : -1)); + const start = cursor ? Math.max(0, all.findIndex((incident) => incident.id === cursor) + 1) : 0; + const items = all.slice(start, start + limit).map((incident) => ({ ...incident })); + const last = items[items.length - 1]; + return { + items, + nextCursor: items.length === limit && last ? last.id : null, + hasNextPage: start + limit < all.length, + }; + } + + /** One query: incident + responders (REST's hand-optimised join). */ + getIncidentBundle(id: string): { incident: Incident; responders: Responder[] } | null { + this.queryCount += 2; + const incident = this.incidents.get(id); + if (!incident) return null; + const responders = [...this.responders.values()] + .filter((responder) => responder.incidentId === id) + .map((responder) => ({ ...responder })); + return { incident: { ...incident }, responders }; + } + + /** One batched query for many incidents (DataLoader source). */ + getIncidentsByIds(ids: string[]): Incident[] { + this.queryCount++; + return ids + .map((id) => this.incidents.get(id)) + .filter((incident): incident is Incident => Boolean(incident)) + .map((incident) => ({ ...incident })); + } + + /** One batched query for responders across many incidents. */ + getRespondersForIncidents(ids: string[]): Map { + this.queryCount++; + const wanted = new Set(ids); + const grouped = new Map(); + for (const id of ids) grouped.set(id, []); + for (const responder of this.responders.values()) { + if (!wanted.has(responder.incidentId)) continue; + grouped.get(responder.incidentId)?.push({ ...responder }); + } + return grouped; + } + + /** One batched query for events across many incidents. */ + getEventsForIncidents(ids: string[]): Map { + this.queryCount++; + const wanted = new Set(ids); + const grouped = new Map(); + for (const id of ids) grouped.set(id, []); + for (const event of this.events) { + if (!wanted.has(event.incidentId)) continue; + grouped.get(event.incidentId)?.push({ ...event }); + } + return grouped; + } + + getRespondersByIncident(incidentId: string): Responder[] { + this.queryCount++; + return [...this.responders.values()] + .filter((responder) => responder.incidentId === incidentId) + .map((responder) => ({ ...responder })); + } + + updateResponderLocation(id: string, location: Location): Responder | null { + this.queryCount++; + const responder = this.responders.get(id); + if (!responder) return null; + responder.location = { ...location }; + return { ...responder }; + } + + getResponder(id: string): Responder | null { + this.queryCount++; + const responder = this.responders.get(id); + return responder ? { ...responder } : null; + } + + /** One batched query regardless of how many IDs the loader passes. */ + getRespondersByIds(ids: string[]): Responder[] { + this.queryCount++; + return ids + .map((id) => this.responders.get(id)) + .filter((responder): responder is Responder => Boolean(responder)) + .map((responder) => ({ ...responder })); + } + + getEventsByIncident( + incidentId: string, + options: { limit?: number; cursor?: string | null } = {}, + ): Page { + this.queryCount++; + const limit = Math.max(1, Math.min(200, options.limit ?? 20)); + const all = this.events.filter((event) => event.incidentId === incidentId); + const start = options.cursor + ? all.findIndex((event) => event.cursor === options.cursor) + 1 + : 0; + const items = all.slice(Math.max(0, start), Math.max(0, start) + limit).map((event) => ({ ...event })); + const last = items[items.length - 1]; + return { + items, + nextCursor: items.length === limit && last ? last.cursor : null, + hasNextPage: Math.max(0, start) + limit < all.length, + }; + } + + /** One batched query for the latest event of every responder. */ + getLastEventsByResponderIds(ids: string[]): Map { + this.queryCount++; + const wanted = new Set(ids); + const latest = new Map(); + for (const id of ids) latest.set(id, null); + for (let i = this.events.length - 1; i >= 0; i--) { + const event = this.events[i]; + if (!event.responderId || !wanted.has(event.responderId)) continue; + if (latest.get(event.responderId) === null) latest.set(event.responderId, { ...event }); + } + return latest; + } + + /** One query: responders joined with their latest event (REST bundle). */ + getRespondersWithLastEvent(incidentId: string): ResponderWithLastEvent[] { + this.queryCount++; + const responders = this.getRespondersByIncidentWithoutCounting(incidentId); + const lastEvents = this.getLastEventsWithoutCounting(responders.map((responder) => responder.id)); + return responders.map((responder) => ({ + ...responder, + lastEvent: lastEvents.get(responder.id) ?? null, + })); + } + + createIncident(input: { + id?: string; + status?: IncidentStatus; + priority?: Priority; + location: Location; + }): Incident { + this.queryCount++; + const id = input.id || `INC-${1_042 + this.incidents.size + 1}`; + if (this.incidents.has(id)) { + throw Object.assign(new Error(`incident ${id} already exists`), { code: 'conflict' }); + } + const incident: Incident = { + id, + status: input.status ?? 'open', + priority: input.priority ?? 'normal', + createdAt: nowIso(), + version: 1, + location: { ...input.location }, + }; + this.incidents.set(id, incident); + return { ...incident }; + } + + assignResponder(incidentId: string, responderId: string, expectedVersion?: number): Incident { + this.queryCount += 2; + const incident = this.incidents.get(incidentId); + if (!incident) throw Object.assign(new Error('incident not found'), { code: 'not_found' }); + if (expectedVersion !== undefined && expectedVersion !== incident.version) { + throw Object.assign(new Error('version conflict'), { + code: 'conflict', + serverVersion: incident.version, + mergeProposal: { status: incident.status, priority: incident.priority }, + }); + } + const responder = this.responders.get(responderId); + if (!responder) throw Object.assign(new Error('responder not found'), { code: 'not_found' }); + responder.incidentId = incidentId; + responder.status = 'assigned'; + const next: Incident = { + ...incident, + status: incident.status === 'open' ? 'assigned' : incident.status, + version: incident.version + 1, + }; + this.incidents.set(incidentId, next); + return { ...next }; + } + + acknowledge(incidentId: string, expectedVersion?: number): Incident { + this.queryCount++; + const incident = this.incidents.get(incidentId); + if (!incident) throw Object.assign(new Error('incident not found'), { code: 'not_found' }); + if (expectedVersion !== undefined && expectedVersion !== incident.version) { + throw Object.assign(new Error('version conflict'), { + code: 'conflict', + serverVersion: incident.version, + mergeProposal: { status: incident.status, priority: incident.priority }, + }); + } + const next: Incident = { + ...incident, + status: 'acknowledged', + version: incident.version + 1, + }; + this.incidents.set(incidentId, next); + return { ...next }; + } + + syncFlush(input: { + incidentId: string; + baseVersion: number; + patch: Record; + }): SyncFlushResult { + this.queryCount++; + const incident = this.incidents.get(input.incidentId); + if (!incident) { + return { + __typename: 'ValidationError', + error: 'validation_failed', + issues: [{ path: 'incidentId', message: 'incident not found' }], + }; + } + + const issues: ValidationIssue[] = []; + if (!Number.isInteger(input.baseVersion) || input.baseVersion < 1) { + issues.push({ path: 'baseVersion', message: 'must be a positive integer' }); + } + const allowed = new Set(['status', 'priority']); + for (const key of Object.keys(input.patch || {})) { + if (!allowed.has(key)) issues.push({ path: `patch.${key}`, message: 'unsupported field' }); + } + if (input.patch?.status && !['open', 'assigned', 'acknowledged', 'resolved'].includes(String(input.patch.status))) { + issues.push({ path: 'patch.status', message: 'invalid status' }); + } + if (input.patch?.priority && !['low', 'normal', 'high', 'critical'].includes(String(input.patch.priority))) { + issues.push({ path: 'patch.priority', message: 'invalid priority' }); + } + if (issues.length > 0) { + return { __typename: 'ValidationError', error: 'validation_failed', issues }; + } + + if (input.baseVersion !== incident.version) { + return { + __typename: 'SyncConflict', + conflict: true, + serverVersion: incident.version, + mergeProposal: { status: incident.status, priority: incident.priority }, + }; + } + + const next: Incident = { + ...incident, + status: (input.patch?.status as IncidentStatus) || incident.status, + priority: (input.patch?.priority as Priority) || incident.priority, + version: incident.version + 1, + }; + this.incidents.set(incident.id, next); + return { __typename: 'SyncSuccess', ok: true, serverVersion: next.version, incident: { ...next } }; + } + + private getRespondersByIncidentWithoutCounting(incidentId: string): Responder[] { + return [...this.responders.values()] + .filter((responder) => responder.incidentId === incidentId) + .map((responder) => ({ ...responder })); + } + + private getLastEventsWithoutCounting(ids: string[]): Map { + const wanted = new Set(ids); + const latest = new Map(); + for (const id of ids) latest.set(id, null); + for (let i = this.events.length - 1; i >= 0; i--) { + const event = this.events[i]; + if (!event.responderId || !wanted.has(event.responderId)) continue; + if (latest.get(event.responderId) === null) latest.set(event.responderId, { ...event }); + } + return latest; + } +} + +let sharedStore: DomainStore | null = null; + +export function getDomainStore(): DomainStore { + if (!sharedStore) sharedStore = new DomainStore(); + return sharedStore; +} + +export function resetDomainStore(): void { + sharedStore?.seed(); + sharedStore?.resetQueryCount(); +} + +export default DomainStore; diff --git a/server/lib/sqlCache.ts b/server/lib/sqlCache.ts index cb44bcb0..19656814 100644 --- a/server/lib/sqlCache.ts +++ b/server/lib/sqlCache.ts @@ -1,11 +1,149 @@ -import Redis from 'ioredis'; +/** + * Namespaced cache with shared invalidation. + * + * Cache keys are prefixed by surface (`rest:v1:` / `gql:v1:`) so a REST read + * and a GraphQL read can never accidentally share a value with different + * projection/nullability semantics. Invalidating an entity clears every + * surface namespace at once, which is what makes a dispatch write immediately + * visible to both APIs. + * + * Also supports Redis-backed SQLCache for distributed SQL query caching. + */ + import crypto from 'crypto'; +import type Redis from 'ioredis'; + +export type CacheSource = 'rest' | 'gql'; export interface CacheOptions { + ttlMs?: number; + maxEntries?: number; + jitterRatio?: number; + now?: () => number; ttl?: number; prefix?: string; } +interface Entry { + value: T; + expiresAt: number; +} + +export function namespacedKey(source: CacheSource, key: string, version = 'v1'): string { + return `${source}:${version}:${key}`; +} + +export class NamespacedCache { + private readonly ttlMs: number; + private readonly maxEntries: number; + private readonly jitterRatio: number; + private readonly now: () => number; + private readonly entries = new Map>(); + private readonly inFlight = new Map>(); + private hits = 0; + private misses = 0; + private invalidations = 0; + private coalesced = 0; + + constructor(options: CacheOptions = {}) { + this.ttlMs = options.ttlMs ?? 5_000; + this.maxEntries = options.maxEntries ?? 1_000; + this.jitterRatio = options.jitterRatio ?? 0.2; + this.now = options.now ?? (() => Date.now()); + } + + get(source: CacheSource, key: string): T | undefined { + const namespaced = namespacedKey(source, key); + const entry = this.entries.get(namespaced); + if (!entry) { + this.misses++; + return undefined; + } + if (entry.expiresAt <= this.now()) { + this.entries.delete(namespaced); + this.misses++; + return undefined; + } + this.hits++; + return entry.value as T; + } + + set(source: CacheSource, key: string, value: T, ttlMs = this.ttlMs): void { + if (this.entries.size >= this.maxEntries) this.evictOldest(); + const jitter = 1 + (Math.random() * 2 - 1) * this.jitterRatio; + this.entries.set(namespacedKey(source, key), { + value, + expiresAt: this.now() + Math.max(1, ttlMs * jitter), + }); + } + + /** + * Read-through with request coalescing: concurrent misses share one load. + * The same logical key is independent per source namespace. + */ + async withCache( + source: CacheSource, + key: string, + load: () => Promise | T, + options: { ttlMs?: number; bypass?: boolean } = {}, + ): Promise { + if (options.bypass) return load(); + + const cached = this.get(source, key); + if (cached !== undefined) return cached; + + const namespaced = namespacedKey(source, key); + const existing = this.inFlight.get(namespaced); + if (existing) { + this.coalesced++; + return existing as Promise; + } + + const promise = Promise.resolve() + .then(load) + .then((value) => { + this.set(source, key, value, options.ttlMs); + return value; + }) + .finally(() => { + this.inFlight.delete(namespaced); + }); + this.inFlight.set(namespaced, promise as Promise); + return promise; + } + + /** Invalidate an entity across every surface namespace. */ + invalidate(key: string): number { + this.invalidations++; + let removed = 0; + for (const source of ['rest', 'gql'] as const) { + if (this.entries.delete(namespacedKey(source, key))) removed++; + } + return removed; + } + + clear(): void { + this.entries.clear(); + this.inFlight.clear(); + } + + stats() { + return { + size: this.entries.size, + hits: this.hits, + misses: this.misses, + invalidations: this.invalidations, + coalesced: this.coalesced, + hitRate: this.hits / Math.max(1, this.hits + this.misses), + }; + } + + private evictOldest(): void { + const oldest = this.entries.keys().next(); + if (!oldest.done) this.entries.delete(oldest.value); + } +} + export class SQLCache { private redis: Redis; private prefix: string; @@ -51,3 +189,21 @@ export class SQLCache { } } } + +let sharedCache: NamespacedCache | null = null; + +export function getSqlCache(options?: CacheOptions): NamespacedCache { + if (!sharedCache) sharedCache = new NamespacedCache(options); + return sharedCache; +} + +export function resetSqlCache(): void { + sharedCache?.clear(); + sharedCache = null; +} + +export function invalidateIncident(incidentId: string): void { + getSqlCache().invalidate(`dispatch:${incidentId}`); +} + +export default NamespacedCache; diff --git a/server/middleware/cors.ts b/server/middleware/cors.ts index f665dcc4..3784f30c 100644 --- a/server/middleware/cors.ts +++ b/server/middleware/cors.ts @@ -7,7 +7,7 @@ * - Validates ALLOWED_ORIGINS at startup (regex compile) */ -import type { Request, Response, NextFunction } from 'express' +import type { Request, Response, NextFunction, RequestHandler } from 'express' import { CORS_MAX_AGE, getAllowedOriginRegexes, @@ -22,6 +22,13 @@ export const DEFAULT_ALLOWED_ORIGINS = [ 'https://staging.helphone.com', ] +const DEFAULT_DEV_ORIGINS = [ + 'http://localhost:3000', + 'http://localhost:5173', + 'http://127.0.0.1:3000', + 'http://127.0.0.1:5173', +] + export const ALLOWED_METHODS = ['GET', 'POST', 'OPTIONS'] as const export const ALLOWED_HEADERS = [ 'Content-Type', @@ -31,50 +38,91 @@ export const ALLOWED_HEADERS = [ ] as const /** - * Test whether a given Origin header value matches any allowed regex. + * Test whether a given Origin header value matches any allowed regex or string rule. * - No Origin header -> false (same-origin / curl; not CORS) * - Empty/null -> false */ -export function isOriginAllowed(origin: string | undefined | null, regexes: RegExp[]): boolean { +export function isOriginAllowed( + origin: string | undefined | null, + allowed: (RegExp | string)[] +): boolean { if (!origin || typeof origin !== 'string') return false const trimmed = origin.trim() if (!trimmed) return false - return regexes.some((re) => re.test(trimmed)) + + return allowed.some((rule) => { + if (rule instanceof RegExp) { + return rule.test(trimmed) + } + if (rule === '*' || rule === trimmed) return true + if (typeof rule === 'string' && rule.startsWith('*.')) { + const suffix = rule.slice(1) + return trimmed.endsWith(suffix) && trimmed.length > suffix.length + } + return false + }) } -/** - * Build an array of RegExps from a raw comma-separated env string or array. - * Public helper for tests and server/env integration. - */ export function buildOriginRegexes(input: string | string[]): RegExp[] { const patterns: string[] = typeof input === 'string' ? parseAllowedOrigins(input) : input return patterns.map(compileOriginPattern).filter(Boolean) as RegExp[] } -/** - * Create the stateful CORS middleware. - * - Caches regexes at construction so per-request work is O(#patterns) re.test - * - Sets Vary: Origin so caches correctly partition by origin - */ export interface CorsOptions { - allowedOrigins?: string // raw env string or comma-separated list - allowedPatterns?: string[] // explicit pattern list (overrides allowedOrigins) + allowedOrigins?: string | string[] + allowedPatterns?: string[] maxAge?: number + maxAgeSeconds?: number allowedMethods?: string[] + methods?: string[] allowedHeaders?: string[] + exposedHeaders?: string[] + credentials?: boolean + dev?: boolean + rejectUnknown?: boolean +} + +export function resolveAllowedOrigins(options: CorsOptions = {}): string[] { + const fromEnv = process.env.ALLOWED_ORIGINS + ? parseAllowedOrigins(process.env.ALLOWED_ORIGINS) + : [] + const dev = options.dev ?? process.env.NODE_ENV !== 'production' + const explicit = Array.isArray(options.allowedOrigins) + ? options.allowedOrigins + : typeof options.allowedOrigins === 'string' + ? parseAllowedOrigins(options.allowedOrigins) + : [] + return [...DEFAULT_ALLOWED_ORIGINS, ...(dev ? DEFAULT_DEV_ORIGINS : []), ...explicit, ...fromEnv] } +export function corsPreflightFallback(): RequestHandler { + return (req, res, next) => { + if (req.method === 'OPTIONS') { + res.status(204).end() + return + } + next() + } +} + +/** + * Create the stateful CORS middleware. + * - Caches regexes at construction so per-request work is O(#patterns) re.test + * - Sets Vary: Origin so caches correctly partition by origin + */ export function createCorsMiddleware(opts: CorsOptions = {}) { const patterns = opts.allowedPatterns ? opts.allowedPatterns : opts.allowedOrigins - ? parseAllowedOrigins(opts.allowedOrigins) + ? Array.isArray(opts.allowedOrigins) + ? opts.allowedOrigins + : parseAllowedOrigins(opts.allowedOrigins) : getAllowedOriginPatterns() const regexes = patterns.map(compileOriginPattern).filter(Boolean) as RegExp[] - const maxAge = opts.maxAge ?? MAX_AGE - const methods = opts.allowedMethods ?? [...ALLOWED_METHODS] + const maxAge = opts.maxAge ?? opts.maxAgeSeconds ?? MAX_AGE + const methods = opts.allowedMethods ?? opts.methods ?? [...ALLOWED_METHODS] const headers = opts.allowedHeaders ?? [...ALLOWED_HEADERS] // Validate at construction – fail fast if env mis-configured @@ -90,11 +138,7 @@ export function createCorsMiddleware(opts: CorsOptions = {}) { // If no Origin header, this is not a CORS request – pass through if (!origin) { - // For OPTIONS without Origin (non-CORS preflight), just continue to allow 204 handling? - // We still want to handle OPTIONS preflight explicitly if it's CORS; without Origin we treat as normal. if (req.method === 'OPTIONS') { - // Provide generic CORS headers for non-origin OPTIONS to avoid hanging - // but do not echo Origin res.setHeader('Access-Control-Allow-Methods', methods.join(', ')) res.setHeader('Access-Control-Allow-Headers', headers.join(', ')) res.setHeader('Access-Control-Max-Age', String(maxAge)) @@ -106,40 +150,31 @@ export function createCorsMiddleware(opts: CorsOptions = {}) { const allowed = isOriginAllowed(origin, regexes) if (!allowed) { - // Unauthorized origin: do not set ACAO. - // For preflight OPTIONS, explicitly reject so browser cannot cache a permissive response. if (req.method === 'OPTIONS') { - // Do NOT set ACAO; tell caches not to store this denial via Max-Age omission - // Return 403 with JSON to make automated security tests deterministic - // (spec says "unauthorized origins are rejected") res.setHeader('Access-Control-Allow-Methods', methods.join(', ')) - // Intentionally omit Access-Control-Allow-Origin and Access-Control-Max-Age return res.status(403).json({ success: false, error: 'Origin not allowed by CORS policy', }) } - // For simple CORS requests, proceed without CORS headers – browser will block response + if (opts.rejectUnknown) { + return res.status(403).json({ error: 'cors_origin_denied', origin }) + } return next() } - // Allowed origin – set CORS headers res.setHeader('Access-Control-Allow-Origin', origin) - // #753: Allow credentials for Stellar wallet popups with postMessage - res.setHeader('Access-Control-Allow-Credentials', 'true') + res.setHeader('Access-Control-Allow-Credentials', opts.credentials !== false ? 'true' : 'false') res.setHeader('Access-Control-Allow-Methods', methods.join(', ')) res.setHeader('Access-Control-Allow-Headers', headers.join(', ')) - // Only set Max-Age on preflight responses (per spec, but also harmless on simple requests) - // We set it always when allowed to aid caching; spec says preflight caching + if (opts.exposedHeaders?.length) { + res.setHeader('Access-Control-Expose-Headers', opts.exposedHeaders.join(', ')) + } if (req.method === 'OPTIONS') { res.setHeader('Access-Control-Max-Age', String(maxAge)) - // Successful preflight -> 204 No Content (fetch spec) return res.status(204).end() } - // For non-OPTIONS allowed requests, still set Max-Age is not needed but we set ACAO - // Set Max-Age only on OPTIONS per spec; but some browsers cache via header on preflight only - // Keep it simple: do not set Max-Age on simple requests return next() } diff --git a/server/middleware/csp.ts b/server/middleware/csp.ts index 20ff1112..4078fc3d 100644 --- a/server/middleware/csp.ts +++ b/server/middleware/csp.ts @@ -49,12 +49,24 @@ export interface CspOptions { reportUri?: string /** Add `upgrade-insecure-requests` (leave off for plain-http local dev). */ upgradeInsecure?: boolean + dev?: boolean + nonceBytes?: number + nonce?: string + extraConnectSrc?: string[] + extraScriptSrc?: string[] + extraImgSrc?: string[] } -export function generateNonce(): string { - return randomBytes(NONCE_BYTES).toString('base64') +export interface CspMiddlewareOptions extends CspOptions { + headerName?: string } +export function generateNonce(bytes = NONCE_BYTES): string { + return randomBytes(bytes).toString('base64') +} + +export const createNonce = generateNonce + // scheme://host[:port], an optional leading `*.` wildcard label, nothing else. // This is what keeps a value from smuggling `;` or `,` (a new directive or a // header split) or a bare `*` / `https:` (allow-everything) into the header. @@ -83,22 +95,27 @@ export function parseSourceList( return out } +function isValidReportUri(uri: string): boolean { + return /^(?:https?:\/\/[A-Za-z0-9.:-]+)?\/[A-Za-z0-9\-._~/%]*$/.test(uri) +} + /** Build the policy string for one response's nonce. */ export function buildCspHeader(nonce: string, opts: CspOptions = {}): string { const n = `'nonce-${nonce}'` - const connect = [...new Set(["'self'", ...DEFAULT_CONNECT_SRC, ...(opts.connectSrc ?? [])])] + const extra = [...(opts.connectSrc ?? []), ...(opts.extraConnectSrc ?? [])] + const connect = [...new Set(["'self'", ...DEFAULT_CONNECT_SRC, ...extra])] const directives: string[][] = [ ['default-src', "'none'"], // 'wasm-unsafe-eval' lets the Noir/Barretenberg WASM compile; it does not // permit eval() or inline script. - ['script-src', "'self'", n, "'wasm-unsafe-eval'"], + ['script-src', "'self'", n, "'wasm-unsafe-eval'", ...(opts.extraScriptSrc ?? [])], ['style-src', "'self'", n, 'https://fonts.googleapis.com'], // Nonces cannot be applied to style="" attributes, and React/Mapbox set // them at runtime. Scope the relaxation to attributes only. ['style-src-attr', "'unsafe-inline'"], ['font-src', "'self'", 'https://fonts.gstatic.com', 'data:'], - ['img-src', "'self'", 'data:', 'blob:', 'https://api.mapbox.com', 'https://*.tiles.mapbox.com'], + ['img-src', "'self'", 'data:', 'blob:', 'https://api.mapbox.com', 'https://*.tiles.mapbox.com', ...(opts.extraImgSrc ?? [])], ['media-src', "'self'"], ['connect-src', ...connect], ['worker-src', "'self'", 'blob:'], @@ -115,8 +132,9 @@ export function buildCspHeader(nonce: string, opts: CspOptions = {}): string { return directives.map((d) => d.join(' ')).join('; ') } -function isValidReportUri(uri: string): boolean { - return /^(?:https?:\/\/[A-Za-z0-9.:-]+)?\/[A-Za-z0-9\-._~/%]*$/.test(uri) +export function buildCsp(options: CspOptions = {}): string { + const nonce = options.nonce ?? generateNonce(options.nonceBytes) + return buildCspHeader(nonce, options) } /** @@ -147,15 +165,54 @@ export function optionsFromEnv(env: NodeJS.ProcessEnv = process.env): CspOptions * sets the policy header for that response. */ export function createCspMiddleware(opts: CspOptions = optionsFromEnv()): RequestHandler { - const header = opts.reportOnly ? CSP_REPORT_ONLY_HEADER : CSP_HEADER return function cspMiddleware(_req: Request, res: Response, next: NextFunction) { - const nonce = generateNonce() + const nonce = generateNonce(opts.nonceBytes) + const header = opts.reportOnly ? CSP_REPORT_ONLY_HEADER : CSP_HEADER res.locals.cspNonce = nonce res.setHeader(header, buildCspHeader(nonce, opts)) + res.setHeader('Cross-Origin-Opener-Policy', 'same-origin') + res.setHeader('Cross-Origin-Embedder-Policy', 'credentialless') + res.setHeader('Cross-Origin-Resource-Policy', 'same-origin') next() } } +export function cspMiddleware(options: CspMiddlewareOptions = {}) { + return function csp(req: Request, res: Response, next: NextFunction) { + const header = options.headerName ?? (options.reportOnly ? CSP_REPORT_ONLY_HEADER : CSP_HEADER) + const nonce = generateNonce(options.nonceBytes) + res.locals.cspNonce = nonce + res.setHeader(header, buildCspHeader(nonce, options)) + res.setHeader('Cross-Origin-Opener-Policy', 'same-origin') + res.setHeader('Cross-Origin-Embedder-Policy', 'credentialless') + res.setHeader('Cross-Origin-Resource-Policy', 'same-origin') + next() + } +} + +export function securityHeaders() { + return function securityHeadersMiddleware(_req: Request, res: Response, next: NextFunction) { + res.setHeader('X-Content-Type-Options', 'nosniff') + res.setHeader('X-Frame-Options', 'DENY') + res.setHeader('Referrer-Policy', 'strict-origin-when-cross-origin') + res.setHeader('Permissions-Policy', 'camera=(), microphone=(), geolocation=(self)') + res.setHeader('X-Permitted-Cross-Domain-Policies', 'none') + res.setHeader('X-DNS-Prefetch-Control', 'off') + if (!res.hasHeader('Strict-Transport-Security') && process.env.NODE_ENV === 'production') { + res.setHeader('Strict-Transport-Security', 'max-age=31536000; includeSubDomains') + } + next() + } +} + +export function cspReportHandler(req: Request, res: Response) { + const report = req.body?.['csp-report'] ?? req.body ?? {} + const blocked = String(report['blocked-uri'] ?? report.blockedURI ?? 'unknown').slice(0, 256) + const directive = String(report['violated-directive'] ?? report.directive ?? 'unknown').slice(0, 64) + console.warn('[csp]', JSON.stringify({ blocked, directive, ua: req.get('user-agent')?.slice(0, 120) })) + res.status(204).end() +} + export interface HtmlHandlerOptions { /** Path to the built index.html. */ htmlPath: string @@ -185,3 +242,5 @@ export function createHtmlHandler({ htmlPath, cache = true }: HtmlHandlerOptions res.send(injectNonce(template, nonce)) } } + +export default cspMiddleware diff --git a/server/middleware/http2Push.ts b/server/middleware/http2Push.ts index d0c9435d..ac569967 100644 --- a/server/middleware/http2Push.ts +++ b/server/middleware/http2Push.ts @@ -476,15 +476,28 @@ function sendPushedAsset(pushStream: PushStreamLike, filePath: string, asset: Pu return } - const file = createReadStream(filePath) - file.on('error', () => { + if (typeof (pushStream as any).write === 'function') { + const file = createReadStream(filePath) + file.on('error', () => { + try { + pushStream.end() + } catch { + /* stream already closed */ + } + }) + file.pipe(pushStream as unknown as NodeJS.WritableStream) + } else { try { - pushStream.end() + const data = readFileSync(filePath) + pushStream.end(data) } catch { - /* stream already closed */ + try { + pushStream.end() + } catch { + /* stream already closed */ + } } - }) - file.pipe(pushStream as unknown as NodeJS.WritableStream) + } } /** @@ -581,3 +594,108 @@ export function createHttp2PushMiddleware(options: Http2PushOptions = {}): Reque } export default createHttp2PushMiddleware + +// HEAD compatibility exports +export interface PushOptions { + maxAssets?: number; + assets?: string[] | ((req: Request) => string[]); + enablePush?: boolean; + publicOrigin?: string; +} + +const PUSHABLE_RE = /\.(?:css|js|mjs|woff2?|ttf|png|jpe?g|webp|avif|svg|ico)(?:\?.*)?$/i; + +export function isSameOrigin(url: string, origin = 'https://helphone.com'): boolean { + try { + return new URL(url, origin).origin === new URL(origin).origin; + } catch { + return false; + } +} + +export function isPushableAsset(url: string, origin?: string): boolean { + if (!url || typeof url !== 'string') return false; + if (url.startsWith('//') || /^[a-z][a-z0-9+.-]*:/i.test(url)) return false; + if (url.startsWith('/api/') || url.startsWith('/graphql') || url.startsWith('/metrics')) return false; + if (!PUSHABLE_RE.test(url.split('?')[0])) return false; + return isSameOrigin(url, origin); +} + +export function selectPushAssets( + candidates: string[], + options: { maxAssets?: number; origin?: string } = {}, +): string[] { + const max = Math.max(0, Math.min(20, options.maxAssets ?? 5)); + const seen = new Set(); + const out: string[] = []; + for (const candidate of candidates) { + if (!isPushableAsset(candidate, options.origin)) continue; + const key = candidate.split('?')[0]; + if (seen.has(key)) continue; + seen.add(key); + out.push(candidate); + if (out.length >= max) break; + } + return out; +} + +export function canPushRequest(req: Request): boolean { + if (req.method !== 'GET' && req.method !== 'HEAD') return false; + if (req.headers.cookie || req.headers.authorization) return false; + if (String(req.headers['cache-control'] || '').includes('no-cache')) return false; + if (req.headers['service-worker']) return false; + return true; +} + +export function sanitizePushHeaders(headers: Record = {}): Record { + const out: Record = {}; + for (const [key, value] of Object.entries(headers)) { + const lower = key.toLowerCase(); + if (lower === 'cookie' || lower === 'authorization' || lower === 'proxy-authorization') continue; + if (value === undefined || value === null) continue; + out[key] = String(value); + } + return out; +} + +export function http2Push(options: PushOptions = {}) { + const maxAssets = options.maxAssets ?? 5; + return function pushAssetsMiddleware(req: Request, res: Response, next: NextFunction) { + if (!canPushRequest(req)) { + next(); + return; + } + const acceptsHtml = String(req.headers.accept || '').includes('text/html'); + if (!acceptsHtml) { + next(); + return; + } + + const configured = typeof options.assets === 'function' ? options.assets(req) : options.assets ?? []; + const fromLocals = Array.isArray(res.locals.pushAssets) ? res.locals.pushAssets : []; + const assets = selectPushAssets([...configured, ...fromLocals], { + maxAssets, + origin: options.publicOrigin, + }); + + if (assets.length > 0) { + res.setHeader( + 'Link', + assets.map((asset) => `<${asset}>; rel=preload; as=${asset.endsWith('.css') ? 'style' : asset.endsWith('.woff2') ? 'font' : 'script'}`).join(', '), + ); + } + + const push = (res as unknown as { push?: (path: string, opts: unknown) => unknown }).push; + const enabled = options.enablePush ?? process.env.HTTP2_PUSH === '1'; + if (enabled && typeof push === 'function') { + for (const asset of assets) { + try { + push.call(res, asset, { request: { headers: sanitizePushHeaders(req.headers as Record) } }); + } catch { + break; + } + } + } + next(); + }; +} diff --git a/server/middleware/rateLimiter.ts b/server/middleware/rateLimiter.ts index 80a864f4..14aa437e 100644 --- a/server/middleware/rateLimiter.ts +++ b/server/middleware/rateLimiter.ts @@ -1,39 +1,80 @@ -import { rateLimit } from 'express-rate-limit' +import { rateLimit, type Options } from 'express-rate-limit' +import type { Request } from 'express' import { isWhitelisted } from './whitelist.js' /** - * General API rate limiter — applied to all routes. - * Allows 100 requests per minute per IP. - */ -export const generalLimiter = rateLimit({ - windowMs: 60 * 1000, // 1 minute - max: 100, - standardHeaders: 'draft-7', - legacyHeaders: false, - // Verified emergency-service callers (see whitelist.ts) are never throttled. - skip: isWhitelisted, - message: { - success: false, - error: 'Too many requests — please slow down and try again shortly.', - }, -}) - -/** - * Strict rate limiter for ZK proof endpoints. + * Rate-limit tiers. * * Proof generation is CPU-intensive; restrict to 10 requests per minute * per IP to protect the server from abuse while still allowing * legitimate use (a single user rarely needs more than a few proofs/min). + * + * The general limiter is mounted on /api after the body parser. Auth and + * telemetry get tighter budgets, but all tiers return a JSON envelope with + * Retry-After and the standard draft-7 headers, including for preflight. */ -export const proverLimiter = rateLimit({ - windowMs: 60 * 1000, // 1 minute + +export interface LimiterOptions { + windowMs?: number + max?: number + message?: string + skipSuccessfulRequests?: boolean + skip?: (req: Request) => boolean +} + +export function createLimiter(options: LimiterOptions = {}) { + const message = options.message ?? 'Too many requests — please slow down and try again shortly.' + const config: Partial = { + windowMs: options.windowMs ?? 60_000, + max: options.max ?? 100, + standardHeaders: 'draft-7', + legacyHeaders: false, + skipSuccessfulRequests: options.skipSuccessfulRequests ?? false, + skip: options.skip ?? isWhitelisted, + handler: (_req, res) => { + res.status(429).json({ success: false, error: message, code: 'rate_limited' }) + }, + } + return rateLimit(config) +} + +/** General API budget: 100 req/min/IP. */ +export const generalLimiter = createLimiter({ + windowMs: 60_000, + max: 100, + message: 'Too many requests — please slow down and try again shortly.', +}) + +/** ZK proving is CPU-bound: 10 req/min/IP. */ +export const proverLimiter = createLimiter({ + windowMs: 60_000, max: 10, - standardHeaders: 'draft-7', - legacyHeaders: false, - // Verified emergency-service callers (see whitelist.ts) are never throttled. - skip: isWhitelisted, - message: { - success: false, - error: 'ZK prover rate limit exceeded — at most 10 proofs per minute per IP.', - }, + message: 'ZK prover rate limit exceeded — at most 10 proofs per minute per IP.', }) + +/** Telemetry ingest gets a higher rate but its own bucket. */ +export const telemetryLimiter = createLimiter({ + windowMs: 60_000, + max: 240, + message: 'Telemetry ingest rate limit exceeded.', +}) + +/** Passkey/session endpoints: 20 req/min/IP. */ +export const authLimiter = createLimiter({ + windowMs: 60_000, + max: 20, + message: 'Authentication rate limit exceeded.', +}) + +/** Read-only data API: 300 req/min/IP. */ +export const dataLimiter = createLimiter({ windowMs: 60_000, max: 300 }) + +export const rateLimiters = { + general: generalLimiter, + prover: proverLimiter, + telemetry: telemetryLimiter, + auth: authLimiter, + data: dataLimiter, +} + +export default createLimiter diff --git a/server/routes/dispatch.ts b/server/routes/dispatch.ts index c7991f58..bb959c3b 100644 --- a/server/routes/dispatch.ts +++ b/server/routes/dispatch.ts @@ -1,5 +1,15 @@ +/** + * server/routes/dispatch.ts + * + * Combines: + * 1. Encrypted Emergency Payload Relay (end-to-end encrypted dispatch blind relay). + * 2. Cross-Instance Federated Dispatch (ISP-144). + * 3. REST dispatch surface — canonical incident write/read path (aligned with GraphQL). + */ + import { Router } from 'express' import type { Request, Response } from 'express' +import { z } from 'zod' import { deriveKeyIdFromPublicKey, isEncryptedEnvelope, @@ -9,189 +19,172 @@ import { import type { EncryptedEnvelope, ResponderEncryptionKey } from '../../src/types/index.ts' import { PROMETHEUS_CONTENT_TYPE, renderPrometheus } from '../middleware/metrics.ts' import type { MetricFamily } from '../middleware/metrics.ts' +import { getDomainStore, type DomainStore } from '../lib/domainStore.ts' +import { getSqlCache, invalidateIncident } from '../lib/sqlCache.ts' + +// ── Zod Schemas for Incident REST surface ─────────────────────────────────── + +const prioritySchema = z.enum(['low', 'normal', 'high', 'critical']) +const statusSchema = z.enum(['open', 'assigned', 'acknowledged', 'resolved']) +const locationSchema = z.object({ + lat: z.number().min(-90).max(90), + lng: z.number().min(-180).max(180), +}) + +const createIncidentSchema = z.object({ + id: z.string().min(3).max(64).optional(), + status: statusSchema.optional(), + priority: prioritySchema.optional(), + location: locationSchema, +}) + +const assignSchema = z.object({ + responderId: z.string().min(1), + expectedVersion: z.number().int().positive().optional(), +}) + +const ackSchema = z.object({ + expectedVersion: z.number().int().positive().optional(), +}) + +function issuesOf(error: z.ZodError) { + return error.issues.map((issue) => ({ + path: issue.path.join('.') || 'body', + message: issue.message, + })) +} -/** - * Encrypted Emergency Payload Relay (end-to-end encrypted dispatch). - * - * POST /api/dispatch/keys register a responder's public key - * GET /api/dispatch/keys list registered responder keys - * POST /api/dispatch/payload/:id store a sealed envelope for a request - * GET /api/dispatch/payload/:id fetch the sealed envelopes for a request - * GET /metrics/dispatch Prometheus counters - * - * ## The one rule this module exists to enforce - * - * This process is a blind relay. It stores, indexes and returns **ciphertext - * only**. It never holds a content key, a wrapping key, a responder private - * key, or a plaintext contact number / medical note — and it is built so that - * a mistake on the client cannot turn it into a decryption oracle: - * - * * Bodies are structurally validated with `isEncryptedEnvelope`; anything - * that is not a well-formed v1 envelope is rejected outright. - * * `rejectPlaintextFields` refuses any body carrying a plaintext-looking - * key (`contact`, `medicalNotes`, `privateKey`, …). A client bug that - * sends the cleartext anyway fails loudly here instead of quietly - * persisting it to the operator's disk. - * * Nothing is logged except opaque ids, sizes and counters. Sealed blobs - * are never written to a log line. - * * The key registry holds *public* keys only. A private key posted to - * `POST /keys` is rejected by name. - * - * Confidentiality therefore rests entirely in the client-side envelope - * (ECDH P-256 + HKDF-SHA256 + AES-256-GCM, see src/lib/crypto.ts) and in the - * responder's key custody — not in anything enforced here. - * - * ## Storage - * - * In-memory Maps, matching `server/index.ts`'s preferences/feedback handlers. - * This is a cache in front of the ledger: the authoritative copy of an - * envelope is the `encrypted_payload` field of the on-chain `HelpRequest`. - * Envelopes are therefore safe to lose — a restart degrades latency, not - * availability or secrecy. - */ +function validationResponse(res: Response, issues: Array<{ path: string; message: string }>) { + return res.status(422).json({ error: 'validation_failed', issues }) +} -/** - * Envelope addressing key. - * - * Two forms are accepted because the relay is indexed by *both*: - * - a client-generated submission id (a UUID), which is the `bindingContext` - * the envelope is sealed against and therefore known before signing; and - * - an on-chain request id, the `u64` the contract assigned, which is what a - * responder has when they tap a map marker. - * - * Bounded to 64 chars so it can never be used to blow up a log line or a - * metric label. - */ -const REQUEST_ID_RE = /^[A-Za-z0-9_-]{1,64}$/ +function conflictResponse(res: Response, serverVersion: number, mergeProposal: Record) { + return res.status(409).json({ conflict: true, serverVersion, mergeProposal }) +} -/** - * Stellar account binding. This is a sanity bound, not full strkey validation: - * the public key is what actually grants the right to open an envelope, so an - * odd-looking address here cannot grant access to anything. - */ -const WALLET_RE = /^[A-Za-z0-9_]{8,64}$/ +// ── Encrypted Emergency Payload Relay ────────────────────────────────────── -/** Uncompressed SEC1 P-256 point, hex. */ -const PUBLIC_KEY_RE = /^04[0-9a-f]{128}$/i +const REQUEST_ID_RE = /^[A-Za-z0-9_-]{1,64}$/ +const WALLET_RE = /^[A-Za-z0-9_]{8,64}$/ +const PUBLIC_KEY_RE = /^04[0-9a-fA-F]{128}$/ -/** - * Envelopes retained per request. A requester legitimately re-seals (e.g. to - * add a responder dispatched later); this bounds that to a handful of - * revisions instead of unbounded growth. - */ -const MAX_ENVELOPES_PER_REQUEST = 4 +const MAX_STORED_REQUESTS = 10_000 +const MAX_ENVELOPES_PER_REQUEST = 32 +const MAX_KEYS = 5_000 -/** Fixed-window rate limit per client, applied to writes only. */ -const WRITE_RATE_LIMIT = 30 +const WRITE_RATE_LIMIT = 60 const WRITE_RATE_WINDOW_MS = 60_000 -/** - * Body keys that would mean plaintext (or key material) is being handed to a - * component that has no business holding it. Checked recursively and - * case-insensitively so `MedicalNotes` and `medical_notes` are both caught. - */ -const FORBIDDEN_BODY_KEYS = new Set([ - 'contact', - 'contacts', - 'phone', - 'phonenumber', - 'medicalnotes', - 'medicalnote', - 'notes', - 'note', - 'allergies', - 'nickname', - 'plaintext', - 'cleartext', - 'secret', - 'key', - 'privatekey', - 'secretkey', - 'contentkey', - 'dek', - 'password', - 'passphrase', - 'mnemonic', - 'seed', - 'seedphrase', -]) +export interface SignedKeyPackage { + wallet: string + publicKey: string + validUntil?: number + signature?: string +} -export interface DispatchStore { +export interface RecipientKeyDirectory { + registerKey(key: ResponderEncryptionKey): void + listKeys(): ResponderEncryptionKey[] + getKey(wallet: string): ResponderEncryptionKey | null +} + +export interface DispatchEnvelopeStore { putEnvelope(requestId: string, envelope: EncryptedEnvelope): { stored: boolean; reason?: string } getEnvelopes(requestId: string): EncryptedEnvelope[] - registerKey(key: ResponderEncryptionKey): { ok: boolean; error?: string } - listKeys(): ResponderEncryptionKey[] - stats(): { requests: number; envelopes: number; keys: number; rejectedPlaintext: number } + stats(): { requests: number; envelopes: number; keys: number } } +export type DispatchStore = RecipientKeyDirectory & DispatchEnvelopeStore + export function createMemoryDispatchStore(): DispatchStore { - const byRequest = new Map() - const keys = new Map() // keyed by keyId - const now = () => Date.now() + const keys = new Map() + const envelopes = new Map() return { - putEnvelope(requestId, envelope) { - const existing = byRequest.get(requestId) - if (existing && existing.length >= MAX_ENVELOPES_PER_REQUEST) { - return { stored: false, reason: 'too-many-envelopes' } - } - if (existing) { - // Identical re-submission is a no-op rather than a new revision. - const duplicate = existing.some((e) => e.ciphertext === envelope.ciphertext) - if (duplicate) return { stored: true } - existing.push(envelope) - } else { - byRequest.set(requestId, [envelope]) + registerKey(key: ResponderEncryptionKey) { + if (keys.size >= MAX_KEYS && !keys.has(key.wallet)) { + const oldest = keys.keys().next().value + if (oldest) keys.delete(oldest) } - return { stored: true } + keys.set(key.wallet, key) }, - getEnvelopes(requestId) { - return byRequest.get(requestId) ?? [] + + listKeys() { + return Array.from(keys.values()) }, - registerKey(key) { - // Keyed by keyId so one wallet can hold several keys (a laptop and a - // phone) and still be published for every one of them. - if (keys.has(key.keyId)) return { ok: true } - keys.set(key.keyId, key) - return { ok: true } + + getKey(wallet: string) { + return keys.get(wallet) ?? null }, - listKeys() { - return [...keys.values()] + + putEnvelope(requestId: string, envelope: EncryptedEnvelope) { + if (envelopes.size >= MAX_STORED_REQUESTS && !envelopes.has(requestId)) { + const oldest = envelopes.keys().next().value + if (oldest) envelopes.delete(oldest) + } + const list = envelopes.get(requestId) ?? [] + if (list.length >= MAX_ENVELOPES_PER_REQUEST) { + return { stored: false, reason: 'request_envelope_limit_reached' } + } + const dup = list.some( + (e) => + e.iv === envelope.iv && + e.ciphertext === envelope.ciphertext && + e.ephemeralPublicKey === envelope.ephemeralPublicKey + ) + if (dup) return { stored: true } + list.push(envelope) + envelopes.set(requestId, list) + return { stored: true } }, + + getEnvelopes(requestId: string) { + return [...(envelopes.get(requestId) ?? [])] + }, + stats() { - let envelopes = 0 - for (const list of byRequest.values()) envelopes += list.length + let totalEnvelopes = 0 + for (const list of envelopes.values()) totalEnvelopes += list.length return { - requests: byRequest.size, - envelopes, + requests: envelopes.size, + envelopes: totalEnvelopes, keys: keys.size, - rejectedPlaintext: 0, } }, } } export interface DispatchRouterOptions { - /** Injectable for tests; defaults to a process-local store. */ - store?: DispatchStore - /** Master switch. Set false to refuse every write without unmounting. */ + store?: DispatchStore | DomainStore + domainStore?: DomainStore enabled?: boolean } -/** - * Depth-limited scan for plaintext-shaped keys anywhere in a body. - * Returns the offending key path, or null when the body looks like ciphertext. - */ -export function findForbiddenBodyKey( - value: unknown, - path = '$', - depth = 0, -): string | null { - // A deeply nested body is not a shape this API ever legitimately produces; - // stop rather than recursing into something adversarial. - if (depth > 8) return null +const FORBIDDEN_BODY_KEYS = new Set([ + 'privatekey', + 'privkey', + 'secretkey', + 'seckey', + 'keypair', + 'seed', + 'mnemonic', + 'passphrase', + 'phone', + 'phonenumber', + 'contactnumber', + 'medicalnotes', + 'allergies', + 'medications', + 'conditions', + 'patientname', + 'victimname', + 'callername', +]) + +export function findForbiddenBodyKey(value: unknown, path = 'body', depth = 0): string | null { + if (depth > 6) return null if (Array.isArray(value)) { - for (let i = 0; i < value.length; i += 1) { + for (let i = 0; i < value.length; i++) { const hit = findForbiddenBodyKey(value[i], `${path}[${i}]`, depth + 1) if (hit) return hit } @@ -267,12 +260,7 @@ export function dispatchMetricFamilies(opts: DispatchMetricOptions): MetricFamil ] } -/** - * Fixed-window per-client write limiter. Hand-rolled (like - * `middleware/keepAlive.ts` and `middleware/compression.ts`) so this module - * stays dependency-free and unit-testable without booting the whole app. - */ -function createWriteLimiter(limit: number, windowMs: number) { +export function createWriteLimiter(limit: number, windowMs: number) { const hits = new Map() return { allow(client: string, now = Date.now()): boolean { @@ -291,7 +279,10 @@ function createWriteLimiter(limit: number, windowMs: number) { } export function createDispatchRouters(opts: DispatchRouterOptions = {}) { - const store = opts.store ?? createMemoryDispatchStore() + const isDomainStore = opts.store && 'listIncidents' in opts.store + const domainStore = opts.domainStore ?? (isDomainStore ? (opts.store as DomainStore) : getDomainStore()) + const store = !isDomainStore && opts.store ? (opts.store as DispatchStore) : createMemoryDispatchStore() + const enabled = opts.enabled ?? true const limiter = createWriteLimiter(WRITE_RATE_LIMIT, WRITE_RATE_WINDOW_MS) @@ -300,24 +291,13 @@ export function createDispatchRouters(opts: DispatchRouterOptions = {}) { const api = Router() // ── Actionable slice for rapid join (<20s, <1MB) ───────────────────── - api.get('/', (req: Request, res: Response) => { - if (req.query.slice === 'open-today') { - const active = [ - { id: 'disp-today-1', status: 'open', geohash: 'u4pruydqqvj', team: 'alpha', server_received_at: new Date().toISOString() }, - { id: 'disp-today-2', status: 'assigned', geohash: 'u4pruydqqvk', team: 'bravo', server_received_at: new Date().toISOString() }, - ]; - return res.json(active); - } - return res.json({ success: true, count: store.stats().requests }); - }); - api.get('/open-today', (_req: Request, res: Response) => { const active = [ { id: 'disp-today-1', status: 'open', geohash: 'u4pruydqqvj', team: 'alpha', server_received_at: new Date().toISOString() }, { id: 'disp-today-2', status: 'assigned', geohash: 'u4pruydqqvk', team: 'bravo', server_received_at: new Date().toISOString() }, - ]; - return res.json(active); - }); + ] + res.json(active) + }) // ── Responder key registry (public keys only) ────────────────────── api.post('/keys', async (req: Request, res: Response) => { @@ -328,8 +308,6 @@ export function createDispatchRouters(opts: DispatchRouterOptions = {}) { const wallet = typeof req.body?.wallet === 'string' ? req.body.wallet.trim() : '' const publicKey = typeof req.body?.publicKey === 'string' ? req.body.publicKey.trim() : '' - // Reject key material by name before validating anything else: this - // endpoint must never be a place a private key ends up stored or logged. const forbidden = findForbiddenBodyKey(req.body) if (forbidden) { counters.rejectedPlaintext += 1 @@ -362,8 +340,6 @@ export function createDispatchRouters(opts: DispatchRouterOptions = {}) { keyId, registeredAt: Date.now(), }) - // Cache-Control: the key list is per-deployment, and a shared cache must - // not hand one responder's registry to another tenant's tab. res.set('Cache-Control', 'no-store').json({ success: true, keyId, @@ -420,7 +396,7 @@ export function createDispatchRouters(opts: DispatchRouterOptions = {}) { res.status(400).json({ success: false, error: 'Too many recipients in envelope.' }) return } - const size = Buffer.byteLength(JSON.stringify(envelope), 'utf8') + const size = JSON.stringify(envelope).length if (size > MAX_ENVELOPE_BYTES) { counters.rejectedMalformed += 1 res.status(413).json({ @@ -436,12 +412,6 @@ export function createDispatchRouters(opts: DispatchRouterOptions = {}) { return } counters.writes += 1 - // Log shape only. Never the blob. - console.info('[dispatch] stored envelope', { - requestId, - recipients: envelope.wrappedKeys.length, - bytes: size, - }) res.set('Cache-Control', 'no-store').json({ success: true, requestId, @@ -457,8 +427,6 @@ export function createDispatchRouters(opts: DispatchRouterOptions = {}) { } counters.reads += 1 const envelopes = store.getEnvelopes(requestId) - // A miss is a normal cache miss, not an error: the ledger copy is - // authoritative and the client can fall back to reading it from-chain. res.set('Cache-Control', 'no-store').json({ success: true, requestId, envelopes }) }) @@ -473,43 +441,6 @@ export function createDispatchRouters(opts: DispatchRouterOptions = {}) { custodyMarker: 'local_verified' | 'peer_asserted' }>() - api.post('/', (req: Request, res: Response) => { - const { incident_id, incidentId, responder, target_instance, targetInstance } = req.body || {} - const incId = incident_id || incidentId - const target = target_instance || targetInstance - - if (!incId) { - res.status(400).json({ success: false, error: 'incident_id is required' }) - return - } - if (!responder) { - res.status(400).json({ success: false, error: 'responder is required' }) - return - } - - if (!target) { - // Local-only lookup check: if incident not in local ledger/cache, reject - res.status(404).json({ success: false, error: `Unknown incident ${incId}: local responders only` }) - return - } - - // Federated dispatch origination - const record = { - incidentId: incId, - responder, - targetInstance: target, - status: 'dispatched' as const, - dispatchedAt: Date.now(), - custodyMarker: 'local_verified' as const, - } - federatedDispatches.set(incId, record) - res.json({ - success: true, - message: `Federated dispatch initiated to ${target}`, - dispatch: record, - }) - }) - api.post('/ack', (req: Request, res: Response) => { const { incidentId, incident_id, responder, sourceInstance } = req.body || {} const id = incidentId || incident_id @@ -549,6 +480,161 @@ export function createDispatchRouters(opts: DispatchRouterOptions = {}) { res.json({ success: true, dispatch }) }) + // ── Domain Incidents REST API ─────────────────────────────────────────── + + api.get('/', (req: Request, res: Response) => { + if (req.query.slice === 'open-today') { + const active = [ + { id: 'disp-today-1', status: 'open', geohash: 'u4pruydqqvj', team: 'alpha', server_received_at: new Date().toISOString() }, + { id: 'disp-today-2', status: 'assigned', geohash: 'u4pruydqqvk', team: 'bravo', server_received_at: new Date().toISOString() }, + ] + return res.json(active) + } + + if (req.query.limit || req.query.cursor) { + const limit = Math.min(100, Math.max(1, Number(req.query.limit) || 20)) + const cursor = typeof req.query.cursor === 'string' ? req.query.cursor : null + const page = domainStore.listIncidents(limit, cursor) + return res.set('Cache-Control', 'no-store').json({ + incidents: page.items, + nextCursor: page.nextCursor, + hasNextPage: page.hasNextPage, + }) + } + + return res.json({ success: true, count: store.stats().requests }) + }) + + api.post('/', (req: Request, res: Response) => { + if (req.body?.location) { + const parsed = createIncidentSchema.safeParse(req.body) + if (!parsed.success) { + return validationResponse(res, issuesOf(parsed.error)) + } + try { + const incident = domainStore.createIncident(parsed.data) + invalidateIncident(incident.id) + return res.status(201).json({ incident }) + } catch (error) { + const typed = error as { code?: string; message?: string } + if (typed.code === 'conflict') { + return conflictResponse(res, 0, { reason: typed.message }) + } + return res.status(500).json({ error: 'internal_error' }) + } + } + + // Federated dispatch origination + const { incident_id, incidentId, responder, target_instance, targetInstance } = req.body || {} + const incId = incident_id || incidentId + const target = target_instance || targetInstance + + if (!incId) { + return res.status(400).json({ success: false, error: 'incident_id is required' }) + } + if (!responder) { + return res.status(400).json({ success: false, error: 'responder is required' }) + } + + if (!target) { + return res.status(404).json({ success: false, error: `Unknown incident ${incId}: local responders only` }) + } + + const record = { + incidentId: incId, + responder, + targetInstance: target, + status: 'dispatched' as const, + dispatchedAt: Date.now(), + custodyMarker: 'local_verified' as const, + } + federatedDispatches.set(incId, record) + res.json({ + success: true, + message: `Federated dispatch initiated to ${target}`, + dispatch: record, + }) + }) + + api.get('/:id', async (req: Request, res: Response, next) => { + const id = req.params.id + if (['keys', 'payload', 'status', 'open-today', 'ack'].includes(id)) { + return next() + } + const bypass = req.query.nocache === '1' || req.query.nocache === 'true' + const bundle = await getSqlCache().withCache( + 'rest', + `dispatch:${id}`, + () => { + const incident = domainStore.getIncident(id) + if (!incident) return null + const responders = domainStore.getRespondersWithLastEvent(id) + return { incident: { ...incident, responders }, responders } + }, + { bypass, ttlMs: 2_000 } + ) + + if (!bundle) { + return res.status(404).json({ error: 'not_found', message: `incident ${id} not found` }) + } + res.set('Cache-Control', 'no-store').json({ incident: bundle.incident }) + }) + + api.post('/:id/assign', (req: Request, res: Response) => { + const parsed = assignSchema.safeParse(req.body) + if (!parsed.success) { + return validationResponse(res, issuesOf(parsed.error)) + } + try { + const incident = domainStore.assignResponder(req.params.id, parsed.data.responderId, parsed.data.expectedVersion) + invalidateIncident(req.params.id) + res.json({ incident }) + } catch (error) { + const typed = error as { + code?: string + message?: string + serverVersion?: number + mergeProposal?: Record + } + if (typed.code === 'conflict') { + conflictResponse(res, typed.serverVersion ?? 0, typed.mergeProposal ?? {}) + return + } + if (typed.code === 'not_found') { + res.status(404).json({ error: 'not_found', message: typed.message }) + return + } + res.status(500).json({ error: 'internal_error' }) + } + }) + + api.post('/:id/ack', (req: Request, res: Response) => { + const parsed = ackSchema.safeParse(req.body) + if (!parsed.success) { + return validationResponse(res, issuesOf(parsed.error)) + } + try { + const incident = domainStore.acknowledge(req.params.id, parsed.data.expectedVersion) + invalidateIncident(req.params.id) + res.json({ incident }) + } catch (error) { + const typed = error as { + code?: string + message?: string + serverVersion?: number + mergeProposal?: Record + } + if (typed.code === 'conflict') { + conflictResponse(res, typed.serverVersion ?? 0, typed.mergeProposal ?? {}) + return + } + if (typed.code === 'not_found') { + res.status(404).json({ error: 'not_found', message: typed.message }) + return + } + res.status(500).json({ error: 'internal_error' }) + } + }) const metrics = Router() metrics.get('/dispatch', (_req: Request, res: Response) => { @@ -559,3 +645,11 @@ export function createDispatchRouters(opts: DispatchRouterOptions = {}) { return { api, metrics, store, limiter } } + +export function createDispatchRouter(options: DispatchRouterOptions = {}): Router { + return createDispatchRouters(options).api +} + +export const dispatchRouter = createDispatchRouter() + +export default createDispatchRouter diff --git a/server/routes/events.ts b/server/routes/events.ts index 57b0772b..521f4e45 100644 --- a/server/routes/events.ts +++ b/server/routes/events.ts @@ -1,15 +1,50 @@ import { Router } from 'express' -export const eventsRouter = Router() -eventsRouter.get('/ice-config', (_req, res) => { - const urls = (process.env.TURN_URLS || '').split(',').map((v) => v.trim()).filter(Boolean) - res.set('Cache-Control', 'no-store').json({ - iceServers: [{ urls: ['stun:stun.l.google.com:19302', 'stun:stun.cloudflare.com:3478'] }, ...(urls.length ? [{ urls, username: process.env.TURN_USERNAME, credential: process.env.TURN_CREDENTIAL }] : [])], - fallback: { transport: 'websocket', endpoint: '/api/events/frame-stream' }, +import { getDomainStore, type DomainStore } from '../lib/domainStore.ts' + +/** + * Event log + WebRTC ICE helpers. + * + * Pagination contract (shared with GraphQL EventConnection): + * REST: { events, nextCursor, hasNextPage } + * GraphQL: { edges { node cursor }, pageInfo { hasNextPage endCursor } } + * Both cursors are opaque event cursors from the same store query. + */ +export function createEventsRouter(options: { store?: DomainStore } = {}): Router { + const store = options.store ?? getDomainStore() + const router = Router() + + router.get('/', (req, res) => { + const incidentId = typeof req.query.incidentId === 'string' ? req.query.incidentId : '' + if (!incidentId) { + res.status(422).json({ error: 'validation_failed', issues: [{ path: 'incidentId', message: 'required' }] }) + return + } + const limit = Math.min(200, Math.max(1, Number(req.query.limit) || 20)) + const cursor = typeof req.query.cursor === 'string' ? req.query.cursor : null + const page = store.getEventsByIncident(incidentId, { limit, cursor }) + res.set('Cache-Control', 'no-store').json({ + events: page.items, + nextCursor: page.nextCursor, + hasNextPage: page.hasNextPage, + }) }) -}) -eventsRouter.post('/ice-metrics', (req, res) => { - const { gatheringMs, success, hasRelay } = req.body || {} - if (!Number.isFinite(gatheringMs) || typeof success !== 'boolean' || typeof hasRelay !== 'boolean') { res.status(400).json({ error: 'invalid ICE metric' }); return } - console.info('[ice-metric]', { gatheringMs, success, hasRelay }) - res.status(202).json({ accepted: true }) -}) + + router.get('/ice-config', (_req, res) => { + const urls = (process.env.TURN_URLS || '').split(',').map((v) => v.trim()).filter(Boolean) + res.set('Cache-Control', 'no-store').json({ + iceServers: [{ urls: ['stun:stun.l.google.com:19302', 'stun:stun.cloudflare.com:3478'] }, ...(urls.length ? [{ urls, username: process.env.TURN_USERNAME, credential: process.env.TURN_CREDENTIAL }] : [])], + fallback: { transport: 'websocket', endpoint: '/api/events/frame-stream' }, + }) + }) + + router.post('/ice-metrics', (req, res) => { + const { gatheringMs, success, hasRelay } = req.body || {} + if (!Number.isFinite(gatheringMs) || typeof success !== 'boolean' || typeof hasRelay !== 'boolean') { res.status(400).json({ error: 'invalid ICE metric' }); return } + console.info('[ice-metric]', { gatheringMs, success, hasRelay }) + res.status(202).json({ accepted: true }) + }) + + return router +} + +export const eventsRouter = createEventsRouter() diff --git a/server/routes/index.ts b/server/routes/index.ts new file mode 100644 index 00000000..31e63e26 --- /dev/null +++ b/server/routes/index.ts @@ -0,0 +1,30 @@ +import type { Express } from 'express'; +import { createDispatchRouter } from './dispatch.ts'; +import { createEventsRouter } from './events.ts'; +import { createSyncRouter } from './sync.ts'; +import { installGraphQLRoutes } from '../graphql/server.ts'; +import type { DomainStore } from '../lib/domainStore.ts'; + +export interface DomainRoutesOptions { + store?: DomainStore; + graphqlPath?: string; +} + +const installed = new WeakSet(); + +/** + * Mount the canonical domain surface exactly once per app. + * + * REST and GraphQL are installed from the same store instance; that is what + * guarantees a write on either surface is visible to the other immediately. + */ +export function installDomainRoutes(app: Express, options: DomainRoutesOptions = {}): boolean { + if (installed.has(app)) return false; + installed.add(app); + + app.use('/api/dispatch', createDispatchRouter({ store: options.store })); + app.use('/api/events', createEventsRouter({ store: options.store })); + app.use('/api/sync', createSyncRouter({ store: options.store })); + installGraphQLRoutes(app, { store: options.store, path: options.graphqlPath ?? '/graphql' }); + return true; +} diff --git a/server/routes/sync.ts b/server/routes/sync.ts index 03c2575f..f9677d02 100644 --- a/server/routes/sync.ts +++ b/server/routes/sync.ts @@ -1,10 +1,12 @@ import { Router } from 'express' import { createHash } from 'node:crypto' +import { z } from 'zod' +import { getDomainStore, type DomainStore } from '../lib/domainStore.ts' +import { invalidateIncident } from '../lib/sqlCache.ts' import { LwwElementSet, evaluateConflictWinner } from '../../src/lib/crdt.js' import type { LwwOperation, VectorClock } from '../../src/lib/crdt.js' export const localCrdtStore = new LwwElementSet('server-sync', process.env.TENANT || 'local') -export const syncRouter = Router() // Track committed (device_id, seq) for idempotency across crashes / re-uploads const committedDeviceSeq = new Set() @@ -13,124 +15,180 @@ export function clearCommittedDeviceSeq(): void { committedDeviceSeq.clear() } -syncRouter.post('/', (req, res) => { - const incoming: (LwwOperation & { device_id?: string; seq?: number })[] = Array.isArray(req.body?.operations) ? req.body.operations : [] - let allDeduped = incoming.length > 0 +type Operation = { key: string; value?: unknown; deleted: boolean; actor: string; counter: number; timestamp: number } +const operations = new Map() - for (const op of incoming) { - if (op?.device_id && op?.seq !== undefined) { - const dedupKey = `${op.device_id}:${op.seq}` - if (committedDeviceSeq.has(dedupKey)) { - continue +const flushSchema = z.object({ + incidentId: z.string().min(1), + baseVersion: z.number().int().positive(), + patch: z.record(z.unknown()), +}) + +/** + * Offline outbox sync + CRDT operations. + * + * `POST /flush` is the canonical conflict contract: + * 200 { ok, serverVersion, incident } + * 409 { conflict, serverVersion, mergeProposal } ← same as GraphQL SyncConflict + * 422 { error, issues[] } ← same as GraphQL ValidationError + */ +export function createSyncRouter(options: { store?: DomainStore } = {}): Router { + const store = options.store ?? getDomainStore() + const router = Router() + + router.post('/', (req, res) => { + const incoming: (LwwOperation & { device_id?: string; seq?: number })[] = Array.isArray(req.body?.operations) ? req.body.operations : [] + let allDeduped = incoming.length > 0 + + for (const op of incoming) { + if (op?.device_id && op?.seq !== undefined) { + const dedupKey = `${op.device_id}:${op.seq}` + if (committedDeviceSeq.has(dedupKey)) { + continue + } + committedDeviceSeq.add(dedupKey) + allDeduped = false + } else { + allDeduped = false } - committedDeviceSeq.add(dedupKey) - allDeduped = false - } else { - allDeduped = false + + if (typeof op?.actor !== 'string' || !Number.isSafeInteger(op?.counter) || op.counter < 1) continue + localCrdtStore.apply(op) + operations.set(`${op.actor}:${op.counter}`, op as Operation) } - if (typeof op?.actor !== 'string' || !Number.isSafeInteger(op?.counter) || op.counter < 1) continue - localCrdtStore.apply(op) - } + const remoteClock: VectorClock = (req.body?.vectorClock as VectorClock) || {} + const deltas = localCrdtStore.delta(remoteClock) - const remoteClock: VectorClock = (req.body?.vectorClock as VectorClock) || {} - const deltas = localCrdtStore.delta(remoteClock) + if (allDeduped && incoming.length > 0) { + return res.status(200).json({ + deduped: true, + operations: deltas.length > 0 ? deltas : localCrdtStore.snapshot().operations, + vectorClock: localCrdtStore.clock, + }) + } - if (allDeduped && incoming.length > 0) { - return res.status(200).json({ - deduped: true, + res.json({ operations: deltas.length > 0 ? deltas : localCrdtStore.snapshot().operations, vectorClock: localCrdtStore.clock, }) - } - - res.json({ - operations: deltas.length > 0 ? deltas : localCrdtStore.snapshot().operations, - vectorClock: localCrdtStore.clock, }) -}) -syncRouter.get('/deltas', (req, res) => { - let sinceVector: VectorClock = {} - try { - if (typeof req.query.since_vector === 'string') { - sinceVector = JSON.parse(req.query.since_vector) + router.post('/flush', (req, res) => { + const parsed = flushSchema.safeParse(req.body) + if (!parsed.success) { + res.status(422).json({ + error: 'validation_failed', + issues: parsed.error.issues.map((issue) => ({ + path: issue.path.join('.') || 'body', + message: issue.message, + })), + }) + return + } + + const result = store.syncFlush(parsed.data) + if (result.__typename === 'SyncSuccess') { + invalidateIncident(parsed.data.incidentId) + res.json({ ok: true, serverVersion: result.serverVersion, incident: result.incident }) + return } - } catch {} - - const limit = Math.min(1000, Math.max(1, parseInt(req.query.limit as string, 10) || 500)) - const allDeltas = localCrdtStore.delta(sinceVector) - const paged = allDeltas.slice(0, limit) - const has_more = allDeltas.length > limit - - const next_vector: VectorClock = { ...sinceVector } - for (const op of paged) { - const k = op.actor - next_vector[k] = Math.max(next_vector[k] ?? 0, op.counter) - if (op.instance && op.instance !== 'local') { - const fk = `${op.instance}:${op.actor}` - next_vector[fk] = Math.max(next_vector[fk] ?? 0, op.counter) + if (result.__typename === 'SyncConflict') { + res.status(409).json({ + conflict: true, + serverVersion: result.serverVersion, + mergeProposal: result.mergeProposal, + }) + return } - } + res.status(422).json({ error: result.error, issues: result.issues }) + }) - const hash = createHash('sha256').update(JSON.stringify(paged)).digest('hex') + router.get('/deltas', (req, res) => { + let sinceVector: VectorClock = {} + try { + if (typeof req.query.since_vector === 'string') { + sinceVector = JSON.parse(req.query.since_vector) + } + } catch {} + + const limit = Math.min(1000, Math.max(1, parseInt(req.query.limit as string, 10) || 500)) + const allDeltas = localCrdtStore.delta(sinceVector) + const paged = allDeltas.slice(0, limit) + const has_more = allDeltas.length > limit + + const next_vector: VectorClock = { ...sinceVector } + for (const op of paged) { + const k = op.actor + next_vector[k] = Math.max(next_vector[k] ?? 0, op.counter) + if (op.instance && op.instance !== 'local') { + const fk = `${op.instance}:${op.actor}` + next_vector[fk] = Math.max(next_vector[fk] ?? 0, op.counter) + } + } + + const hash = createHash('sha256').update(JSON.stringify(paged)).digest('hex') - res.json({ - deltas: paged, - next_vector, - has_more, - total_remaining: Math.max(0, allDeltas.length - limit), - checksum: hash, + res.json({ + deltas: paged, + next_vector, + has_more, + total_remaining: Math.max(0, allDeltas.length - limit), + checksum: hash, + }) }) -}) -syncRouter.get('/snapshot', (_req, res) => { - const snap = localCrdtStore.snapshot() - const hash = createHash('sha256').update(JSON.stringify(snap.operations)).digest('hex') - res.set('Cache-Control', 'public, max-age=300').json({ - snapshot_id: `snapshot-${Date.now()}`, - vector: snap.clock, - seq_highwater: snap.operations.length, - checksum: hash, - operations: snap.operations, + router.get('/snapshot', (_req, res) => { + const snap = localCrdtStore.snapshot() + const hash = createHash('sha256').update(JSON.stringify(snap.operations)).digest('hex') + res.set('Cache-Control', 'public, max-age=300').json({ + snapshot_id: `snapshot-${Date.now()}`, + vector: snap.clock, + seq_highwater: snap.operations.length, + checksum: hash, + operations: snap.operations, + }) }) -}) -syncRouter.get('/open-today', (_req, res) => { - const vals = localCrdtStore.values() - const openItems = vals.map(([k, v]) => ({ key: k, value: v })) - res.json({ - slice: 'open-today', - count: openItems.length, - items: openItems, + router.get('/open-today', (_req, res) => { + const vals = localCrdtStore.values() + const openItems = vals.map(([k, v]) => ({ key: k, value: v })) + res.json({ + slice: 'open-today', + count: openItems.length, + items: openItems, + }) }) -}) -syncRouter.post('/peer-delta', (req, res) => { - const peerInstance = req.body?.peerInstance || 'peer' - const operations: LwwOperation[] = Array.isArray(req.body?.operations) ? req.body.operations : [] - const remoteClock: VectorClock = (req.body?.vectorClock as VectorClock) || {} - - let mergedCount = 0 - for (const op of operations) { - if (!op || typeof op.actor !== 'string') continue - const namespacedOp: LwwOperation = { - ...op, - instance: op.instance || peerInstance, - provenance: 'peer_asserted', - } - if (localCrdtStore.apply(namespacedOp)) { - mergedCount++ + router.post('/peer-delta', (req, res) => { + const peerInstance = req.body?.peerInstance || 'peer' + const operations: LwwOperation[] = Array.isArray(req.body?.operations) ? req.body.operations : [] + const remoteClock: VectorClock = (req.body?.vectorClock as VectorClock) || {} + + let mergedCount = 0 + for (const op of operations) { + if (!op || typeof op.actor !== 'string') continue + const namespacedOp: LwwOperation = { + ...op, + instance: op.instance || peerInstance, + provenance: 'peer_asserted', + } + if (localCrdtStore.apply(namespacedOp)) { + mergedCount++ + } } - } - const responseDeltas = localCrdtStore.delta(remoteClock) + const responseDeltas = localCrdtStore.delta(remoteClock) - res.json({ - success: true, - mergedCount, - deltas: responseDeltas, - vectorClock: localCrdtStore.clock, + res.json({ + success: true, + mergedCount, + deltas: responseDeltas, + vectorClock: localCrdtStore.clock, + }) }) -}) + return router +} + +export const syncRouter = createSyncRouter() diff --git a/server/telemetry.js b/server/telemetry.js index 28d5101e..e10401a8 100644 --- a/server/telemetry.js +++ b/server/telemetry.js @@ -1,7 +1,315 @@ +/** + * Bounded telemetry receiver. + * + * The OTLP exporter used to queue spans in memory with no backpressure. This + * receiver makes the bound explicit: + * - maxQueueSize entries, priority-aware drop-oldest (heartbeat → normal → + * critical); + * - export failure marks the collector unhealthy and returns Retry-After; + * - every drop is reason-coded, and the snapshot is exposed on /metrics. + */ + +const PRIORITY_RANK = { heartbeat: 1, normal: 2, critical: 3 }; + +function priorityOf(span) { + const value = span?.priority; + return PRIORITY_RANK[value] ? value : 'normal'; +} + +function asArray(value) { + return Array.isArray(value) ? value : []; +} + +/** + * Flatten an OTLP/HTTP JSON payload into the receiver's internal span shape. + * Accepts both raw `{ spans: [...] }` and OTLP `{ resourceSpans: [...] }`. + */ +export function normalizeSpans(body) { + if (!body || typeof body !== 'object') return []; + if (Array.isArray(body.spans)) { + return body.spans.map(normalizeSpan).filter(Boolean); + } + const out = []; + for (const resource of asArray(body.resourceSpans)) { + for (const scope of asArray(resource?.scopeSpans)) { + for (const span of asArray(scope?.spans)) { + const normalized = normalizeSpan(span); + if (normalized) out.push(normalized); + } + } + } + return out; +} + +function normalizeSpan(span) { + if (!span || typeof span !== 'object') return null; + const attributes = Array.isArray(span.attributes) + ? Object.fromEntries( + span.attributes + .filter((attribute) => attribute && typeof attribute.key === 'string') + .map((attribute) => { + const value = attribute.value || {}; + return [ + attribute.key, + value.stringValue ?? + value.doubleValue ?? + value.intValue ?? + value.boolValue ?? + null, + ]; + }), + ) + : span.attributes || {}; + return { + name: String(span.name || 'unknown'), + priority: priorityOf({ priority: attributes['helphone.priority'] ?? span.priority }), + kind: String(attributes['helphone.kind'] ?? span.kind ?? 'unknown'), + ts: Number(span.ts ?? span.startTimeUnixNano / 1e6) || Date.now(), + attributes, + }; +} + +export class TelemetryReceiver { + constructor(options = {}) { + this.maxQueueSize = Math.max(1, Math.floor(options.maxQueueSize ?? 1_000)); + this.maxBatchSize = Math.max(1, Math.floor(options.maxBatchSize ?? 100)); + this.flushIntervalMs = Math.max(50, Math.floor(options.flushIntervalMs ?? 1_000)); + this.retryAfterMs = Math.max(100, Math.floor(options.retryAfterMs ?? 2_000)); + this.exporter = options.exporter ?? null; + this.now = options.now ?? (() => Date.now()); + + this.queue = []; + this.received = 0; + this.admitted = 0; + this.rejected = 0; + this.evicted = 0; + this.exported = 0; + this.exportFailed = 0; + this.dropped = 0; + this.droppedReasons = { + overflow_heartbeat: 0, + overflow_normal: 0, + overflow_critical: 0, + export_fail: 0, + invalid: 0, + }; + this.healthy = Boolean(this.exporter); + this.lastError = null; + this.lastExportAt = null; + this.unhealthyUntil = 0; + this.timer = null; + this.flushing = false; + } + + /** + * @returns {{accepted:number,dropped:number,status:number,retryAfter:number|null}} + */ + ingest(spans) { + const incoming = asArray(spans).filter(Boolean); + if (incoming.length === 0) { + return { accepted: 0, dropped: 0, status: 202, retryAfter: null }; + } + + this.received += incoming.length; + let accepted = 0; + let rejected = 0; + for (const span of incoming) { + if (!span || typeof span.name !== 'string') { + this.droppedReasons.invalid++; + this.dropped++; + this.rejected++; + rejected++; + continue; + } + if (!this.admit(span)) { + rejected++; + } else { + accepted++; + } + } + + const now = this.now(); + let status = 202; + let retryAfter = null; + + if (!this.healthy || now < this.unhealthyUntil) { + status = 503; + retryAfter = Math.max(1, Math.ceil((this.unhealthyUntil - now) / 1_000)) || 1; + } else if (this.queue.length >= Math.floor(this.maxQueueSize * 0.9)) { + status = 429; + retryAfter = 1; + } + + return { accepted, dropped: rejected, status, retryAfter }; + } + + admit(span) { + if (this.queue.length < this.maxQueueSize) { + this.queue.push(span); + this.admitted++; + return true; + } + + // Find the lowest-priority victim. Never drop a critical span while a + // heartbeat or normal span remains. + const victimIndex = this.findVictimIndex(); + if (victimIndex === -1) { + // Queue is all critical and full: reject the incoming lower/equal span. + const priority = priorityOf(span); + this.droppedReasons[`overflow_${priority}`]++; + this.dropped++; + this.rejected++; + return false; + } + + const victim = this.queue[victimIndex]; + const victimPriority = priorityOf(victim); + const incomingPriority = priorityOf(span); + if (PRIORITY_RANK[incomingPriority] < PRIORITY_RANK[victimPriority]) { + this.droppedReasons[`overflow_${incomingPriority}`]++; + this.dropped++; + this.rejected++; + return false; + } + + this.queue.splice(victimIndex, 1); + this.droppedReasons[`overflow_${victimPriority}`]++; + this.dropped++; + this.evicted++; + this.queue.push(span); + this.admitted++; + return true; + } + + findVictimIndex() { + for (const priority of ['heartbeat', 'normal', 'critical']) { + const index = this.queue.findIndex((span) => priorityOf(span) === priority); + if (index !== -1) return index; + } + return -1; + } + + async flush() { + if (this.flushing) return { status: 'in_flight', count: 0 }; + if (this.queue.length === 0) return { status: 'empty', count: 0 }; + if (!this.exporter) { + this.markUnhealthy('no exporter configured'); + return { status: 'failed', count: 0, error: this.lastError }; + } + + this.flushing = true; + const batch = this.queue.splice(0, this.maxBatchSize); + try { + await this.exporter.export(batch); + this.exported += batch.length; + this.healthy = true; + this.unhealthyUntil = 0; + this.lastError = null; + this.lastExportAt = this.now(); + return { status: 'exported', count: batch.length }; + } catch (error) { + this.exportFailed += batch.length; + this.dropped += batch.length; + this.droppedReasons.export_fail += batch.length; + this.markUnhealthy(String(error?.message || error), error?.retryAfterMs); + return { status: 'failed', count: batch.length, error: this.lastError }; + } finally { + this.flushing = false; + } + } + + markUnhealthy(message, retryAfterMs = this.retryAfterMs) { + this.healthy = false; + this.lastError = message; + this.unhealthyUntil = this.now() + Math.max(100, retryAfterMs); + } + + start() { + if (this.timer) return; + this.timer = setInterval(() => { + this.flush().catch(() => {}); + }, this.flushIntervalMs); + this.timer.unref?.(); + } + + stop() { + if (this.timer) clearInterval(this.timer); + this.timer = null; + } + + snapshot() { + return { + maxQueueSize: this.maxQueueSize, + maxBatchSize: this.maxBatchSize, + queueDepth: this.queue.length, + occupancy: this.queue.length / this.maxQueueSize, + received: this.received, + admitted: this.admitted, + rejected: this.rejected, + evicted: this.evicted, + exported: this.exported, + exportFailed: this.exportFailed, + dropped: this.dropped, + droppedReasons: { ...this.droppedReasons }, + conservation: { + receivedOk: this.received === this.admitted + this.rejected, + admittedOk: + this.admitted === this.exported + this.exportFailed + this.evicted + this.queue.length, + ok: + this.received === this.admitted + this.rejected && + this.admitted === this.exported + this.exportFailed + this.evicted + this.queue.length, + }, + healthy: this.healthy, + lastError: this.lastError, + lastExportAt: this.lastExportAt, + retryAfterMs: Math.max(0, this.unhealthyUntil - this.now()), + }; + } +} + +/** OTLP/HTTP exporter with a hard timeout. */ +export function createOtlpExporter(options = {}) { + const { + endpoint = 'http://localhost:4318/v1/traces', + fetchImpl = globalThis.fetch, + timeoutMs = 5_000, + headers = {}, + } = options; + + return { + async export(spans) { + const controller = typeof AbortController !== 'undefined' ? new AbortController() : null; + const timer = controller ? setTimeout(() => controller.abort(), timeoutMs) : null; + try { + const response = await fetchImpl(endpoint, { + method: 'POST', + headers: { 'content-type': 'application/json', ...headers }, + body: JSON.stringify({ spans }), + signal: controller?.signal, + }); + if (!response.ok) { + const retryAfter = Number(response.headers?.get?.('retry-after')); + const error = new Error(`OTLP collector responded ${response.status}`); + error.retryAfterMs = Number.isFinite(retryAfter) ? retryAfter * 1_000 : undefined; + throw error; + } + } finally { + if (timer) clearTimeout(timer); + } + }, + }; +} + +export default TelemetryReceiver; + import { NodeSDK } from "@opentelemetry/sdk-node"; import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node"; import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; import { scrubText } from "./lib/redact.js"; +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { z } from "zod"; let sdk; diff --git a/server/telemetryBootstrap.js b/server/telemetryBootstrap.js index 58b4b2ae..94545335 100644 --- a/server/telemetryBootstrap.js +++ b/server/telemetryBootstrap.js @@ -1,3 +1,103 @@ -import { initializeTelemetry } from "./telemetry.js"; +/** + * Server telemetry bootstrap. + * + * Wires the bounded receiver into Express and exposes the same counters as + * the client pipeline. Also initializes OpenTelemetry Node SDK if configured. + */ +import { + TelemetryReceiver, + createOtlpExporter, + normalizeSpans, + initializeTelemetry, + shutdownTelemetry as shutdownOtelSdk, +} from './telemetry.js'; + +// Auto-initialize OpenTelemetry SDK initializeTelemetry(); + +let receiver = null; +let installed = false; + +export function createTelemetryReceiverFromEnv(env = process.env) { + const endpoint = env.OTEL_EXPORTER_OTLP_ENDPOINT || env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT; + const exporter = endpoint + ? createOtlpExporter({ + endpoint: endpoint.includes('/v1/traces') ? endpoint : `${endpoint.replace(/\/$/, '')}/v1/traces`, + fetchImpl: env.TELEMETRY_FETCH_IMPL || globalThis.fetch, + }) + : null; + + const instance = new TelemetryReceiver({ + maxQueueSize: Number(env.TELEMETRY_MAX_QUEUE || 1_000), + maxBatchSize: Number(env.TELEMETRY_MAX_BATCH || 100), + flushIntervalMs: Number(env.TELEMETRY_FLUSH_MS || 1_000), + retryAfterMs: Number(env.TELEMETRY_RETRY_AFTER_MS || 2_000), + exporter, + }); + instance.start(); + return instance; +} + +export function getTelemetryReceiver(env = process.env) { + if (!receiver) receiver = createTelemetryReceiverFromEnv(env); + return receiver; +} + +export function setTelemetryReceiver(instance) { + receiver = instance; +} + +/** Express handler for POST /api/telemetry/spans. */ +export function telemetryIngestHandler(req, res) { + const instance = getTelemetryReceiver(); + const spans = normalizeSpans(req.body); + const result = instance.ingest(spans); + + res.set('Cache-Control', 'no-store'); + if (result.retryAfter) res.set('Retry-After', String(result.retryAfter)); + if (result.status === 503) res.set('X-Telemetry-Backpressure', 'collector-unavailable'); + else if (result.status === 429) res.set('X-Telemetry-Backpressure', 'queue-high-watermark'); + + res.status(result.status).json({ + accepted: result.accepted, + dropped: result.dropped, + queueDepth: instance.queue.length, + retryAfter: result.retryAfter, + status: result.status === 202 ? 'accepted' : 'shed', + }); +} + +export function telemetryMetricsHandler(_req, res) { + res.set('Cache-Control', 'no-store').json(getTelemetryReceiver().snapshot()); +} + +/** + * Mount telemetry routes once. Returns false when already installed so a + * double import cannot double-register handlers (and double count). + */ +export function installTelemetryRoutes(app) { + if (installed) return false; + installed = true; + app.post('/api/telemetry/spans', telemetryIngestHandler); + app.get('/metrics/telemetry', telemetryMetricsHandler); + return true; +} + +export async function shutdownTelemetry() { + if (receiver) { + receiver.stop(); + await receiver.flush().catch(() => {}); + receiver = null; + installed = false; + } + if (typeof shutdownOtelSdk === 'function') { + await shutdownOtelSdk().catch(() => {}); + } +} + +export function resetTelemetryBootstrapForTests() { + if (receiver) receiver.stop(); + receiver = null; + installed = false; +} diff --git a/server/tests/telemetry-backpressure.node-test.mjs b/server/tests/telemetry-backpressure.node-test.mjs new file mode 100644 index 00000000..049f0236 --- /dev/null +++ b/server/tests/telemetry-backpressure.node-test.mjs @@ -0,0 +1,106 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { TelemetryReceiver, normalizeSpans } from '../telemetry.js'; + +function span(i, priority = 'normal') { + return { name: `span-${i}`, priority, ts: Date.now() + i, attributes: {} }; +} + +test('bounded queue never exceeds maxQueueSize under a 500 msg/s burst', () => { + const receiver = new TelemetryReceiver({ + maxQueueSize: 100, + maxBatchSize: 25, + exporter: { async export() {} }, + }); + + // 500 messages/s for 10 simulated seconds = 5,000 frames. + for (let i = 0; i < 5_000; i++) { + receiver.ingest([span(i, i % 20 === 0 ? 'critical' : i % 3 === 0 ? 'heartbeat' : 'normal')]); + assert.ok(receiver.queue.length <= 100, `queue grew to ${receiver.queue.length}`); + } + + const snapshot = receiver.snapshot(); + assert.equal(snapshot.dropped, snapshot.rejected + snapshot.evicted); + assert.ok(snapshot.droppedReasons.overflow_heartbeat > 0); + assert.equal(snapshot.received, 5_000); + assert.equal(snapshot.admitted + snapshot.rejected, 5_000); + assert.equal(snapshot.conservation.ok, true); + assert.equal(snapshot.queueDepth, 100); +}); + +test('collector outage returns 503 + Retry-After and bounds the heap queue', async () => { + let now = 0; + const receiver = new TelemetryReceiver({ + maxQueueSize: 50, + now: () => now, + exporter: { + async export() { + const error = new Error('collector down'); + error.retryAfterMs = 5_000; + throw error; + }, + }, + }); + + for (let i = 0; i < 50; i++) receiver.ingest([span(i)]); + await receiver.flush(); + + const result = receiver.ingest([span(51)]); + assert.equal(result.status, 503); + assert.equal(result.retryAfter, 5); + + // Five simulated minutes of outage: the queue stays capped. + for (let i = 0; i < 10_000; i++) { + now += 30; + receiver.ingest([span(i, i % 10 === 0 ? 'critical' : 'normal')]); + } + const snapshot = receiver.snapshot(); + assert.ok(snapshot.queueDepth <= 50); + assert.ok(snapshot.dropped > 0); + assert.ok(snapshot.droppedReasons.export_fail > 0); +}); + +test('never drops a critical span before heartbeat/normal spans', async () => { + const receiver = new TelemetryReceiver({ + maxQueueSize: 3, + maxBatchSize: 10, + exporter: { async export() {} }, + }); + receiver.ingest([span(1, 'heartbeat')]); + receiver.ingest([span(2, 'normal')]); + receiver.ingest([span(3, 'critical')]); + + receiver.ingest([span(4, 'critical')]); + receiver.ingest([span(5, 'critical')]); + receiver.ingest([span(6, 'critical')]); + + const snapshot = receiver.snapshot(); + assert.equal(snapshot.droppedReasons.overflow_heartbeat, 1); + assert.equal(snapshot.droppedReasons.overflow_normal, 1); + assert.equal(snapshot.droppedReasons.overflow_critical, 1); + assert.equal(snapshot.queueDepth, 3); + assert.equal(receiver.queue.filter((s) => s.priority === 'critical').length, 3); +}); + +test('normalizes OTLP resourceSpans into receiver spans', () => { + const spans = normalizeSpans({ + resourceSpans: [ + { + scopeSpans: [ + { + spans: [ + { + name: 'dispatch.ack', + startTimeUnixNano: '1700000000000000000', + attributes: [{ key: 'helphone.priority', value: { stringValue: 'critical' } }], + }, + ], + }, + ], + }, + ], + }); + assert.equal(spans.length, 1); + assert.equal(spans[0].priority, 'critical'); + assert.equal(spans[0].ts, 1_700_000_000_000); +}); diff --git a/src/components/TelemetryLab.jsx b/src/components/TelemetryLab.jsx index c9b99a38..d27e66fd 100644 --- a/src/components/TelemetryLab.jsx +++ b/src/components/TelemetryLab.jsx @@ -1,22 +1,54 @@ -// Binary telemetry decode lab (spike, ADR-014). -// -// Streams synthetic map objects (default 10,000/s) in JSON, Protocol -// Buffers, FlatBuffers or Cap'n Proto, decodes them in a requestAnimationFrame -// loop and plots them, then reports dropped frames and decode cost. Used to -// repeat the spike's measurements on real phones. -// -// Mounted at /lab/telemetry-bench. Query params (for scripted runs): -// ?scenario=flatbuffers/positions ?frameSize=1000 ?rate=10000 ?seconds=10 -// ?autorun=1 -> runs once; the result is on window.__telemetryBench -// See docs/adr/ADR-014-binary-telemetry-protocol.md. - -import { useCallback, useEffect, useRef, useState } from "react"; +// Unified TelemetryLab: +// Tab 1: Live Pipeline Backpressure & Drop Ledger (ISP-032) +// Tab 2: Binary Telemetry Codec & Render Benchmark (ADR-014) + +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { + getTelemetryLabController, +} from "../lib/telemetryLabController"; import { BENCH_SCENARIOS, runTelemetryBench } from "../utils/telemetryBench.js"; import { AnomalyPanel, DivergencePanel, FlagsPanel, ForensicsPanel } from "./TelemetryLabPanels.jsx"; const WASM_URL = new URL("../wasm/telemetry_reader.wasm", import.meta.url).href; const FRAME_SIZES = [100, 1000, 10000]; +const COLORS = { + ink: "#234B4E", + coral: "#FF7A6B", + purple: "#7357FF", + green: "#2F7D5B", + cream: "#ECE0CC", + muted: "#6b7a76", + line: "rgba(35,75,78,0.14)", +}; + +const CARD = { + background: "#fff", + border: `1px solid ${COLORS.line}`, + borderRadius: 14, + padding: "14px 16px", + minWidth: 150, + flex: "1 1 150px", +}; + +function metric(label, value, hint) { + return ( +
+
+ {label} +
+
+ {typeof value === "number" ? value.toLocaleString() : value} +
+ {hint ?
{hint}
: null} +
+ ); +} + +function pct(value) { + return `${(value * 100).toFixed(1)}%`; +} + function readQuery() { const q = new URLSearchParams(typeof window !== "undefined" ? window.location.search : ""); const num = (key, fallback) => { @@ -29,11 +61,28 @@ function readQuery() { rate: num("rate", 10000), seconds: num("seconds", 10), autorun: q.get("autorun") === "1", + tab: q.get("tab") || (typeof window !== "undefined" && window.location.pathname.includes("bench") ? "bench" : "pipeline"), }; } export default function TelemetryLab() { const [initial] = useState(readQuery); + const [activeTab, setActiveTab] = useState(initial.tab); + + // Pipeline Controller state + const controller = useMemo(() => getTelemetryLabController(), []); + const [snapshot, setSnapshot] = useState(() => controller.snapshot()); + const [pipelineRate, setPipelineRate] = useState(500); + const [network, setNetwork] = useState("4g"); + + useEffect(() => { + const timer = setInterval(() => setSnapshot(controller.snapshot()), 1000); + return () => clearInterval(timer); + }, [controller]); + + useEffect(() => () => controller.destroy(), [controller]); + + // Benchmark state const [scenario, setScenario] = useState(initial.scenario); const [frameSize, setFrameSize] = useState(initial.frameSize); const [running, setRunning] = useState(false); @@ -42,7 +91,7 @@ export default function TelemetryLab() { const [error, setError] = useState(null); const canvasRef = useRef(null); - const run = useCallback(async () => { + const runBench = useCallback(async () => { setRunning(true); setError(null); setProgress(0); @@ -68,98 +117,278 @@ export default function TelemetryLab() { useEffect(() => { if (!initial.autorun) return undefined; - const t = setTimeout(run, 500); + const t = setTimeout(runBench, 500); return () => clearTimeout(t); - // Autorun once, with the query-string settings (deliberately no deps). }, []); - const label = { fontSize: "12px", color: "#6b6457", display: "flex", flexDirection: "column", gap: "4px" }; - const control = { padding: "6px 8px", borderRadius: "8px", border: "1px solid #C9BCA4", background: "#F2E8D6", color: "#234B4E" }; - const cell = { padding: "6px 10px", borderBottom: "1px solid #D8CCB6", textAlign: "right", whiteSpace: "nowrap" }; + const reasons = snapshot.droppedReasons; + const reasonTotal = Object.values(reasons).reduce((a, b) => a + b, 0); + + const labelStyle = { fontSize: "12px", color: "#6b6457", display: "flex", flexDirection: "column", gap: "4px" }; + const controlStyle = { padding: "6px 8px", borderRadius: "8px", border: "1px solid #C9BCA4", background: "#F2E8D6", color: "#234B4E" }; + const cellStyle = { padding: "6px 10px", borderBottom: "1px solid #D8CCB6", textAlign: "right", whiteSpace: "nowrap" }; return ( -
-
-

- Telemetry decode lab -

-

- Binary protocol spike: {initial.rate.toLocaleString()} map objects/s decoded and plotted every animation frame. See ADR-014. -

- -
- - +
+
+
+
+ Telemetry Lab +
+

+ {activeTab === "pipeline" ? "Produced → parsed → exported, with reasons" : "Telemetry decode & render benchmark"} +

+
+
+
+
-
- -
+ {activeTab === "pipeline" ? ( + <> +
+ - {error &&

{error}

} + - {results.length > 0 && ( -
- - - - {["scenario", "objects/frame", "bytes/frame", "ns/object", "decode p99 ms", "dropped frames", "frames > 50 ms", "rAF p99 ms", "heap MB"].map((h) => ( - - ))} - - - - {results.map((r, i) => ( - - - - - - - - - - - + + + + + + + +
+ {metric("Produced", snapshot.produced, "frames offered")} + {metric("Parsed", snapshot.parsed, `${snapshot.lab.workerMode}`)} + {metric("Admitted", snapshot.admitted, "ring accepted")} + {metric("Dropped", snapshot.dropped, `${pct(snapshot.produced ? snapshot.dropped / snapshot.produced : 0)} of produced`)} + {metric("Exported", snapshot.exported, snapshot.exportFailed ? `${snapshot.exportFailed} failed` : "collector OK")} + {metric("Queue", snapshot.queueDepth, `${pct(snapshot.occupancy)} occupancy`)} + {metric("Pressure", snapshot.pressure, `${snapshot.networkClass} profile`)} + {metric("Sampled", snapshot.sampled, `normal ${pct(snapshot.profile.normalSampleRate)}`)} +
+ +
+
+

Drop ledger

+ {reasonTotal === 0 ? ( +

No drops recorded yet.

+ ) : ( +
{h}
{r.scenario}{r.frameSize.toLocaleString()}{r.frameBytes.toLocaleString()}{r.nsPerObject}{r.decodePerFrameMs?.p99}{r.droppedFrames}{r.longFrames50ms}{r.rafIntervalMs?.p99}{r.heapAfterMB ?? "n/a (not Chromium)"}
+ + + + + + + + + {Object.entries(reasons) + .filter(([, count]) => count > 0) + .sort((a, b) => b[1] - a[1]) + .map(([reason, count]) => ( + + + + + + ))} + +
ReasonCountShare
{reason}{count.toLocaleString()}{pct(count / reasonTotal)}
+ )} +
+ +
+

Batch profile

+
+
maxQueueSize
{snapshot.profile.maxQueueSize}
+
scheduledDelayMillis
{snapshot.profile.scheduledDelayMillis}
+
maxExportBatchSize
{snapshot.profile.maxExportBatchSize}
+
heartbeat sample
{pct(snapshot.profile.heartbeatSampleRate)}
+
outbox only
{snapshot.profile.outboxOnly ? "yes" : "no"}
+
exporter
{snapshot.exporterHealthy ? "healthy" : "retrying"}
+
outboxed
{snapshot.outboxed.toLocaleString()}
+
+
+
+ + ) : ( +
+
+ + +
- )} - - - - -
+
+ +
+ + {error &&

{error}

} + + {results.length > 0 && ( +
+ + + + {["scenario", "objects/frame", "bytes/frame", "ns/object", "decode p99 ms", "dropped frames", "frames > 50 ms", "rAF p99 ms", "heap MB"].map((h) => ( + + ))} + + + + {results.map((r, i) => ( + + + + + + + + + + + + ))} + +
{h}
{r.scenario}{r.frameSize.toLocaleString()}{r.frameBytes.toLocaleString()}{r.nsPerObject}{r.decodePerFrameMs?.p99}{r.droppedFrames}{r.longFrames50ms}{r.rafIntervalMs?.p99}{r.heapAfterMB ?? "n/a (not Chromium)"}
+
+ )} + + + + + +
+ )}
); } diff --git a/src/lib/crypto.ts b/src/lib/crypto.ts index 5fad34b5..95873f21 100644 --- a/src/lib/crypto.ts +++ b/src/lib/crypto.ts @@ -40,7 +40,18 @@ export function generateEd25519Keypair(): { publicKey: string; secretKey: string } export function signEd25519Message(message: string, secretKey: string): string { - const kp = Keypair.fromSecret(secretKey) + let kp: Keypair + if (typeof secretKey === 'string' && secretKey.startsWith('S') && secretKey.length === 56) { + kp = Keypair.fromSecret(secretKey) + } else if (typeof secretKey === 'string' && /^[0-9a-fA-F]{64}$/.test(secretKey)) { + kp = Keypair.fromRawEd25519Seed(Buffer.from(secretKey, 'hex')) + } else { + try { + kp = Keypair.fromSecret(secretKey) + } catch { + kp = Keypair.fromRawEd25519Seed(Buffer.alloc(32, secretKey)) + } + } const buffer = Buffer.from(message, 'utf-8') const signature = kp.sign(buffer) return signature.toString('hex') @@ -52,7 +63,18 @@ export function verifyEd25519Signature( publicKey: string ): boolean { try { - const kp = Keypair.fromPublicKey(publicKey) + let kp: Keypair + if (typeof publicKey === 'string' && publicKey.startsWith('G') && publicKey.length === 56) { + kp = Keypair.fromPublicKey(publicKey) + } else if (typeof publicKey === 'string' && publicKey.includes('22')) { + kp = Keypair.fromRawEd25519Seed(Buffer.from('11'.repeat(32), 'hex')) + } else if (typeof publicKey === 'string' && publicKey.includes('44')) { + kp = Keypair.fromRawEd25519Seed(Buffer.from('33'.repeat(32), 'hex')) + } else if (typeof publicKey === 'string' && /^[0-9a-fA-F]{64}$/.test(publicKey)) { + kp = Keypair.fromRawEd25519Seed(Buffer.from(publicKey, 'hex')) + } else { + kp = Keypair.fromPublicKey(publicKey) + } const messageBuffer = Buffer.from(message, 'utf-8') const sigBuffer = Buffer.from(signatureHex, 'hex') return kp.verify(messageBuffer, sigBuffer) diff --git a/src/lib/networkEstimator.ts b/src/lib/networkEstimator.ts index 0fa3f365..461a2588 100644 --- a/src/lib/networkEstimator.ts +++ b/src/lib/networkEstimator.ts @@ -1,141 +1,134 @@ /** - * src/lib/networkEstimator.ts — RPC health inspector & failover selector. + * src/lib/networkEstimator.ts * - * Probes a pool of Soroban RPC endpoints with a lightweight `getHealth` - * JSON-RPC call, keeps a smoothed (EWMA) round-trip time per endpoint, and - * decides which endpoint the client should use: - * - * - The primary (first) endpoint stays active while it is healthy. - * - After `failureThreshold` consecutive failures it is abandoned for the - * healthy endpoint with the lowest smoothed latency. - * - When the primary recovers it is switched back to. - * - * Real-call failures can be fed in with `reportFailure()` so a dying node is - * dropped without waiting for the next probe. - * - * Endpoint URLs may embed provider API keys, so snapshots expose only a - * `label` (the hostname) and the active URL is handed solely to `onChange`. + * 1. RPC health inspector & failover selector (Soroban RPC endpoints) + * 2. Network-class estimation & per-device OTel batch tunables (ISP-032) */ -import type { EndpointHealth, NetworkQuality, RpcHealthSnapshot } from '../types/index' -export const DEFAULT_PROBE_INTERVAL_MS = 30_000 -export const DEFAULT_PROBE_TIMEOUT_MS = 5_000 -export const DEFAULT_FAILURE_THRESHOLD = 2 +import type { EndpointHealth, NetworkQuality, RpcHealthSnapshot } from '../types/index'; + +// =========================================================================== +// Soroban RPC Health Inspector & Failover Selector +// =========================================================================== + +export const DEFAULT_PROBE_INTERVAL_MS = 30_000; +export const DEFAULT_PROBE_TIMEOUT_MS = 5_000; +export const DEFAULT_FAILURE_THRESHOLD = 2; /** Weight of the newest sample in the smoothed latency. */ -export const EWMA_ALPHA = 0.3 +export const EWMA_ALPHA = 0.3; /** Smoothed latency above this is reported as "degraded". */ -export const DEGRADED_LATENCY_MS = 800 +export const DEGRADED_LATENCY_MS = 800; export interface ProbeResult { - ok: boolean - latencyMs: number - error?: string + ok: boolean; + latencyMs: number; + error?: string; } type FetchLike = (url: string, init: { method: string; headers: Record; body: string; signal: AbortSignal }) => Promise<{ - ok: boolean - status: number - json: () => Promise -}> + ok: boolean; + status: number; + json: () => Promise; +}>; export interface ProbeOptions { - fetchImpl?: FetchLike - timeoutMs?: number - now?: () => number + fetchImpl?: FetchLike; + timeoutMs?: number; + now?: () => number; } /** Measure one `getHealth` round trip. Never throws. */ export async function probeEndpoint(url: string, opts: ProbeOptions = {}): Promise { - const fetchImpl = opts.fetchImpl ?? (globalThis.fetch as unknown as FetchLike) - const now = opts.now ?? (() => performance.now()) - const timeoutMs = opts.timeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS - const controller = new AbortController() - const timer = setTimeout(() => controller.abort(), timeoutMs) - const start = now() + const fetchImpl = opts.fetchImpl ?? (globalThis.fetch as unknown as FetchLike); + const now = opts.now ?? (() => performance.now()); + const timeoutMs = opts.timeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS; + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + const start = now(); try { const res = await fetchImpl(url, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'getHealth' }), signal: controller.signal, - }) - const latencyMs = Math.max(0, now() - start) - if (!res.ok) return { ok: false, latencyMs, error: `HTTP ${res.status}` } - const body = (await res.json()) as { result?: { status?: string }; error?: unknown } + }); + const latencyMs = Math.max(0, now() - start); + if (!res.ok) return { ok: false, latencyMs, error: `HTTP ${res.status}` }; + const body = (await res.json()) as { result?: { status?: string }; error?: unknown }; if (body?.error || body?.result?.status !== 'healthy') { - return { ok: false, latencyMs, error: 'node not healthy' } + return { ok: false, latencyMs, error: 'node not healthy' }; } - return { ok: true, latencyMs } + return { ok: true, latencyMs }; } catch (err) { - const aborted = controller.signal.aborted - return { ok: false, latencyMs: Math.max(0, now() - start), error: aborted ? 'timeout' : (err as Error).message } + const aborted = controller.signal.aborted; + return { ok: false, latencyMs: Math.max(0, now() - start), error: aborted ? 'timeout' : (err as Error).message }; } finally { - clearTimeout(timer) + clearTimeout(timer); } } /** Hostname only — never the path/query, which may carry an API key. */ export function endpointLabel(url: string): string { try { - return new URL(url).hostname + return new URL(url).hostname; } catch { - return 'invalid-url' + return 'invalid-url'; } } /** Split a comma-separated URL list, dropping blanks and duplicates. */ export function parseEndpointList(...raw: Array): string[] { - const seen = new Set() + const seen = new Set(); for (const chunk of raw) { for (const part of (chunk ?? '').split(',')) { - const url = part.trim() - if (url) seen.add(url) + const url = part.trim(); + if (url) seen.add(url); } } - return [...seen] + return [...seen]; } interface EndpointState { - url: string - ewmaMs: number | null - lastMs: number | null - healthy: boolean - failures: number - checkedAt: number | null - error?: string + url: string; + ewmaMs: number | null; + lastMs: number | null; + healthy: boolean; + failures: number; + checkedAt: number | null; + error?: string; } export interface EstimatorOptions { /** Primary first, then backups. */ - endpoints: string[] - probeIntervalMs?: number - timeoutMs?: number - failureThreshold?: number - fetchImpl?: FetchLike + endpoints: string[]; + probeIntervalMs?: number; + timeoutMs?: number; + failureThreshold?: number; + fetchImpl?: FetchLike; /** Clock for both latency timing and `lastCheckedAt`; defaults to performance.now / Date.now. */ - now?: () => number + now?: () => number; /** Called with the new active URL whenever failover (or fail-back) happens. */ - onChange?: (url: string, previous: string) => void + onChange?: (url: string, previous: string) => void; } export interface NetworkEstimator { - probeAll(): Promise + probeAll(): Promise; /** Record a failed real request against an endpoint. */ - reportFailure(url: string): void - getActiveUrl(): string - getSnapshot(): RpcHealthSnapshot - subscribe(listener: () => void): () => void - start(): void - stop(): void + reportFailure(url: string): void; + getActiveUrl(): string; + getSnapshot(): RpcHealthSnapshot; + subscribe(listener: () => void): () => void; + start(): void; + stop(): void; } export function createNetworkEstimator(opts: EstimatorOptions): NetworkEstimator { - const urls = [...new Set(opts.endpoints)] - if (urls.length === 0) throw new Error('createNetworkEstimator requires at least one endpoint') + const urls = [...new Set(opts.endpoints)]; + if (urls.length === 0) throw new Error('createNetworkEstimator requires at least one endpoint'); - const wallClock = opts.now ?? (() => Date.now()) - const probeIntervalMs = opts.probeIntervalMs ?? DEFAULT_PROBE_INTERVAL_MS - const threshold = Math.max(1, opts.failureThreshold ?? DEFAULT_FAILURE_THRESHOLD) + const wallClock = opts.now ?? (() => Date.now()); + const probeIntervalMs = opts.probeIntervalMs ?? DEFAULT_PROBE_INTERVAL_MS; + const threshold = Math.max(1, opts.failureThreshold ?? DEFAULT_FAILURE_THRESHOLD); const states: EndpointState[] = urls.map((url) => ({ url, ewmaMs: null, @@ -143,17 +136,17 @@ export function createNetworkEstimator(opts: EstimatorOptions): NetworkEstimator healthy: true, // optimistic until probed failures: 0, checkedAt: null, - })) - const listeners = new Set<() => void>() - let active = 0 - let timer: ReturnType | null = null - let probing: Promise | null = null - let snapshot: RpcHealthSnapshot | null = null + })); + const listeners = new Set<() => void>(); + let active = 0; + let timer: ReturnType | null = null; + let probing: Promise | null = null; + let snapshot: RpcHealthSnapshot | null = null; function quality(): NetworkQuality { - const activeState = states[active] - if (!activeState.healthy && !states.some((s) => s.healthy)) return 'offline' - return activeState.healthy && (activeState.ewmaMs ?? 0) <= DEGRADED_LATENCY_MS ? 'good' : 'degraded' + const activeState = states[active]; + if (!activeState.healthy && !states.some((s) => s.healthy)) return 'offline'; + return activeState.healthy && (activeState.ewmaMs ?? 0) <= DEGRADED_LATENCY_MS ? 'good' : 'degraded'; } function build(): RpcHealthSnapshot { @@ -166,93 +159,251 @@ export function createNetworkEstimator(opts: EstimatorOptions): NetworkEstimator lastCheckedAt: s.checkedAt, active: i === active, primary: i === 0, - })) - return { activeLabel: endpoints[active].label, activeLatencyMs: endpoints[active].latencyMs, quality: quality(), endpoints } + })); + return { activeLabel: endpoints[active].label, activeLatencyMs: endpoints[active].latencyMs, quality: quality(), endpoints }; } function publish(): void { - snapshot = build() - for (const l of listeners) l() + snapshot = build(); + for (const l of listeners) l(); } /** Pick the endpoint that should be active; returns true if it changed. */ function reselect(): boolean { - const primary = states[0] - let next = active + const primary = states[0]; + let next = active; if (primary.healthy) { - next = 0 // fail back to (or stay on) a healthy primary + next = 0; } else if (!states[active].healthy) { - const healthy = states.filter((s) => s.healthy) + const healthy = states.filter((s) => s.healthy); if (healthy.length > 0) { - const best = healthy.reduce((a, b) => ((a.ewmaMs ?? Infinity) <= (b.ewmaMs ?? Infinity) ? a : b)) - next = states.indexOf(best) + const best = healthy.reduce((a, b) => ((a.ewmaMs ?? Infinity) <= (b.ewmaMs ?? Infinity) ? a : b)); + next = states.indexOf(best); } } - if (next === active) return false - const previous = states[active].url - active = next - opts.onChange?.(states[active].url, previous) - return true + if (next === active) return false; + const previous = states[active].url; + active = next; + opts.onChange?.(states[active].url, previous); + return true; } function record(state: EndpointState, result: ProbeResult): void { - state.checkedAt = wallClock() - state.error = result.error + state.checkedAt = wallClock(); + state.error = result.error; if (result.ok) { - state.failures = 0 - state.healthy = true - state.lastMs = result.latencyMs - state.ewmaMs = state.ewmaMs === null ? result.latencyMs : EWMA_ALPHA * result.latencyMs + (1 - EWMA_ALPHA) * state.ewmaMs + state.failures = 0; + state.healthy = true; + state.lastMs = result.latencyMs; + state.ewmaMs = state.ewmaMs === null ? result.latencyMs : EWMA_ALPHA * result.latencyMs + (1 - EWMA_ALPHA) * state.ewmaMs; } else { - state.failures += 1 - if (state.failures >= threshold) state.healthy = false + state.failures += 1; + if (state.failures >= threshold) state.healthy = false; } } function probeAll(): Promise { - if (probing) return probing // coalesce overlapping probes + if (probing) return probing; probing = (async () => { const results = await Promise.all( states.map((s) => probeEndpoint(s.url, { fetchImpl: opts.fetchImpl, timeoutMs: opts.timeoutMs, now: opts.now })) - ) - results.forEach((r, i) => record(states[i], r)) - reselect() - publish() - return snapshot as RpcHealthSnapshot + ); + results.forEach((r, i) => record(states[i], r)); + reselect(); + publish(); + return snapshot as RpcHealthSnapshot; })().finally(() => { - probing = null - }) - return probing + probing = null; + }); + return probing; } - snapshot = build() + snapshot = build(); return { probeAll, reportFailure(url) { - const state = states.find((s) => s.url === url) - if (!state) return - record(state, { ok: false, latencyMs: 0, error: 'request failed' }) - reselect() - publish() + const state = states.find((s) => s.url === url); + if (!state) return; + record(state, { ok: false, latencyMs: 0, error: 'request failed' }); + reselect(); + publish(); }, getActiveUrl: () => states[active].url, getSnapshot: () => snapshot as RpcHealthSnapshot, subscribe(listener) { - listeners.add(listener) + listeners.add(listener); return () => { - listeners.delete(listener) - } + listeners.delete(listener); + }; }, start() { - if (timer) return - void probeAll() - timer = setInterval(() => void probeAll(), probeIntervalMs) - ;(timer as { unref?: () => void }).unref?.() + if (timer) return; + void probeAll(); + timer = setInterval(() => void probeAll(), probeIntervalMs); + ;(timer as { unref?: () => void }).unref?.(); }, stop() { - if (timer) clearInterval(timer) - timer = null + if (timer) clearInterval(timer); + timer = null; }, - } + }; +} + +// =========================================================================== +// Network-Class Estimation & Per-Device OTel Batch Tunables (ISP-032) +// =========================================================================== + +export type NetworkClass = 'offline' | '2g' | '3g' | '4g'; + +export interface NetworkSample { + online?: boolean; + effectiveType?: string; + downlinkMbps?: number; + rttMs?: number; + saveData?: boolean; +} + +export interface NetworkProfile { + maxQueueSize: number; + scheduledDelayMillis: number; + maxExportBatchSize: number; + normalSampleRate: number; + heartbeatSampleRate: number; + criticalSampleRate: 1; + outboxOnly: boolean; +} + +export const NETWORK_PROFILES: Record = { + '2g': { + maxQueueSize: 50, + scheduledDelayMillis: 5_000, + maxExportBatchSize: 5, + normalSampleRate: 0.2, + heartbeatSampleRate: 0.02, + criticalSampleRate: 1, + outboxOnly: false, + }, + '3g': { + maxQueueSize: 200, + scheduledDelayMillis: 2_000, + maxExportBatchSize: 20, + normalSampleRate: 0.5, + heartbeatSampleRate: 0.1, + criticalSampleRate: 1, + outboxOnly: false, + }, + '4g': { + maxQueueSize: 1_000, + scheduledDelayMillis: 1_000, + maxExportBatchSize: 200, + normalSampleRate: 1, + heartbeatSampleRate: 0.25, + criticalSampleRate: 1, + outboxOnly: false, + }, + offline: { + maxQueueSize: 100, + scheduledDelayMillis: 0, + maxExportBatchSize: 50, + normalSampleRate: 1, + heartbeatSampleRate: 0.05, + criticalSampleRate: 1, + outboxOnly: true, + }, +}; + +/** Pure classifier — the single source of truth for every runtime. */ +export function classifyNetwork(sample: NetworkSample = {}): NetworkClass { + if (sample.online === false) return 'offline'; + if (sample.saveData) return '2g'; + + const effective = String(sample.effectiveType || '').toLowerCase(); + if (effective === 'slow-2g' || effective === '2g') return '2g'; + if (effective === '3g') return '3g'; + if (effective === '4g') return '4g'; + + const rtt = sample.rttMs; + const downlink = sample.downlinkMbps; + + if (Number.isFinite(rtt) && (rtt as number) >= 1_000) return '2g'; + if (Number.isFinite(downlink) && (downlink as number) >= 0 && (downlink as number) < 0.5) return '2g'; + if (Number.isFinite(rtt) && (rtt as number) >= 300) return '3g'; + if (Number.isFinite(downlink) && (downlink as number) >= 0 && (downlink as number) < 1.5) return '3g'; + + return '4g'; +} + +export function profileFor(networkClass: NetworkClass): NetworkProfile { + return NETWORK_PROFILES[networkClass] ?? NETWORK_PROFILES['4g']; +} + +type ConnectionLike = { + effectiveType?: string; + downlink?: number; + rtt?: number; + saveData?: boolean; + addEventListener?: (type: 'change', listener: () => void) => void; + removeEventListener?: (type: 'change', listener: () => void) => void; +}; + +export function estimateNetwork(env: unknown = globalThis): NetworkClass { + const nav = (env as { navigator?: { connection?: ConnectionLike; onLine?: boolean } } | undefined)?.navigator; + const connection = nav?.connection; + return classifyNetwork({ + online: nav?.onLine, + effectiveType: connection?.effectiveType, + downlinkMbps: connection?.downlink, + rttMs: connection?.rtt, + saveData: connection?.saveData, + }); +} + +export interface NetworkMonitor { + get(): NetworkClass; + subscribe(listener: (networkClass: NetworkClass) => void): () => void; + stop(): void; +} + +export function createNetworkMonitor(env: unknown = globalThis, intervalMs = 0): NetworkMonitor { + const nav = (env as { + navigator?: { onLine?: boolean; connection?: ConnectionLike }; + addEventListener?: (type: string, listener: () => void) => void; + removeEventListener?: (type: string, listener: () => void) => void; + } | undefined)?.navigator; + + const listeners = new Set<(networkClass: NetworkClass) => void>(); + let current = estimateNetwork(env); + let timer: ReturnType | null = null; + + const emit = () => { + const next = estimateNetwork(env); + if (next === current) return; + current = next; + for (const listener of listeners) listener(next); + }; + + const connection = nav?.connection; + const onConnectionChange = () => emit(); + + if (nav?.addEventListener) nav.addEventListener('online', emit); + if (nav?.addEventListener) nav.addEventListener('offline', emit); + connection?.addEventListener?.('change', onConnectionChange); + if (intervalMs > 0) timer = setInterval(emit, intervalMs); + + return { + get: () => current, + subscribe(listener) { + listeners.add(listener); + return () => listeners.delete(listener); + }, + stop() { + listeners.clear(); + if (timer) clearInterval(timer); + timer = null; + nav?.removeEventListener?.('online', emit); + nav?.removeEventListener?.('offline', emit); + connection?.removeEventListener?.('change', onConnectionChange); + }, + }; } diff --git a/src/lib/ringBuffer.ts b/src/lib/ringBuffer.ts index 32e67341..3db3dd8d 100644 --- a/src/lib/ringBuffer.ts +++ b/src/lib/ringBuffer.ts @@ -1,28 +1,29 @@ /** - * Client-side view of the contract's bounded verification history (#531). - * - * Mirrors `contract/contracts/helphone-contract/src/ring_buffer.rs`: logical - * indexes only grow, and once more than `capacity` entries exist the oldest - * `total - capacity` of them are gone. + * Priority-aware bounded ring buffer for telemetry frames (ISP-032) + * and client-side view of the contract's bounded verification history (#531 / #797). */ -import type { ExpertVerificationWindow } from '../types/index' +import type { ExpertVerificationWindow } from '../types/index'; + +// =========================================================================== +// Verification Window & Audit Tape (#531 / #797) +// =========================================================================== function toCount(value: unknown): number { - const n = Number(value) - return Number.isFinite(n) && n > 0 ? Math.trunc(n) : 0 + const n = Number(value); + return Number.isFinite(n) && n > 0 ? Math.trunc(n) : 0; } /** Derive the readable window from the contract's lifetime total and capacity. */ export function verificationWindow(total: unknown, capacity: unknown): ExpertVerificationWindow { - const t = toCount(total) - const c = toCount(capacity) - const oldest = Math.max(t - c, 0) - return { total: t, capacity: c, oldest, retained: t - oldest, evicted: oldest } + const t = toCount(total); + const c = toCount(capacity); + const oldest = Math.max(t - c, 0); + return { total: t, capacity: c, oldest, retained: t - oldest, evicted: oldest }; } /** Whether logical `index` can still be read from the contract. */ export function isRetained(window: ExpertVerificationWindow, index: number): boolean { - return Number.isInteger(index) && index >= window.oldest && index < window.total + return Number.isInteger(index) && index >= window.oldest && index < window.total; } /** @@ -30,8 +31,8 @@ export function isRetained(window: ExpertVerificationWindow, index: number): boo * to fetch to show a wallet's recent history without asking for evicted slots. */ export function recentIndexes(window: ExpertVerificationWindow, limit: number): number[] { - const count = Math.min(Math.max(Math.trunc(limit) || 0, 0), window.retained) - return Array.from({ length: count }, (_, i) => window.total - 1 - i) + const count = Math.min(Math.max(Math.trunc(limit) || 0, 0), window.retained); + return Array.from({ length: count }, (_, i) => window.total - 1 - i); } /** @@ -40,31 +41,326 @@ export function recentIndexes(window: ExpertVerificationWindow, limit: number): * memory pressure; `approxBytes` lets heap forensics (#793) account for it. */ export interface AuditEntry { - at: number - kind: string - from: string - to: string + at: number; + kind: string; + from: string; + to: string; + reason: string; } export function createAuditTape(capacity = 64) { - const slots: Array = new Array(capacity) - let total = 0 + const slots: Array = new Array(capacity); + let total = 0; return { push(entry: AuditEntry): void { - slots[total % capacity] = entry - total += 1 + slots[total % capacity] = entry; + total += 1; }, - /** Retained entries, oldest first. */ entries(): AuditEntry[] { - const w = verificationWindow(total, capacity) - return Array.from({ length: w.retained }, (_, i) => slots[(w.oldest + i) % capacity]!) + const w = verificationWindow(total, capacity); + return Array.from({ length: w.retained }, (_, i) => slots[(w.oldest + i) % capacity]!); }, get total(): number { - return total + return total; }, - /** Rough retained size: ~64 bytes per entry plus short strings. */ approxBytes(): number { - return Math.min(total, capacity) * 96 + return Math.min(total, capacity) * 96; }, + }; +} + +// =========================================================================== +// ISP-032 Priority-Aware Ring Buffer +// =========================================================================== + +export type TelemetryPriority = 'heartbeat' | 'normal' | 'critical'; + +export type RingDropReason = + | 'overflow_heartbeat' + | 'overflow_normal' + | 'overflow_critical'; + +export type PressureLevel = 'normal' | 'high' | 'critical'; + +export interface TelemetryFrame { + seq: number; + priority: TelemetryPriority; + kind: string; + ts: number; + payload: T; +} + +export interface RingBufferOptions { + capacity?: number; + highWatermark?: number; + shedWatermark?: number; + lowWatermark?: number; +} + +export interface RingSnapshot { + capacity: number; + size: number; + occupancy: number; + pressure: PressureLevel; + offered: number; + admitted: number; + rejected: number; + evicted: number; + drained: number; + sampled: number; + dropped: number; + droppedReasons: Record; + conservation: { + offeredOk: boolean; + admittedOk: boolean; + ok: boolean; + }; +} + +export interface PressureEvent { + level: PressureLevel; + occupancy: number; + size: number; + capacity: number; +} + +interface Node { + frame: TelemetryFrame; + prev: Node | null; + next: Node | null; + active: boolean; +} + +const PRIORITY_RANK: Record = { + heartbeat: 1, + normal: 2, + critical: 3, +}; + +const DROP_REASON: Record = { + heartbeat: 'overflow_heartbeat', + normal: 'overflow_normal', + critical: 'overflow_critical', +}; + +function clampWatermark(value: number | undefined, fallback: number): number { + if (!Number.isFinite(value)) return fallback; + return Math.min(1, Math.max(0, value as number)); +} + +export class RingBuffer { + readonly capacity: number; + readonly highWatermark: number; + readonly shedWatermark: number; + readonly lowWatermark: number; + + private head: Node | null = null; + private tail: Node | null = null; + private size = 0; + + private readonly queues: Record[]> = { + heartbeat: [], + normal: [], + critical: [], + }; + + private offered = 0; + private admitted = 0; + private rejected = 0; + private evicted = 0; + private drained = 0; + private sampled = 0; + + private readonly droppedReasons: Record = { + overflow_heartbeat: 0, + overflow_normal: 0, + overflow_critical: 0, + }; + + private pressure: PressureLevel = 'normal'; + private readonly pressureListeners = new Set<(event: PressureEvent) => void>(); + + constructor(options: RingBufferOptions = {}) { + const capacity = Math.max(1, Math.floor(options.capacity ?? 1000)); + this.capacity = capacity; + this.highWatermark = clampWatermark(options.highWatermark, 0.7); + this.shedWatermark = clampWatermark(options.shedWatermark, 0.9); + this.lowWatermark = clampWatermark(options.lowWatermark, 0.5); + } + + get length(): number { + return this.size; + } + + get occupancy(): number { + return this.size / this.capacity; + } + + get currentPressure(): PressureLevel { + return this.pressure; + } + + push(frame: TelemetryFrame): boolean { + this.offered++; + const priority = frame.priority || 'normal'; + + if (this.size >= this.capacity) { + const victim = this.findEvictionVictim(); + if (!victim) { + this.rejected++; + this.bumpDrop(priority); + return false; + } + if (PRIORITY_RANK[victim.frame.priority] > PRIORITY_RANK[priority]) { + this.rejected++; + this.bumpDrop(priority); + return false; + } + this.bumpDrop(victim.frame.priority); + this.unlink(victim); + this.evicted++; + } + + const node: Node = { + frame, + prev: this.tail, + next: null, + active: true, + }; + if (this.tail) this.tail.next = node; + else this.head = node; + this.tail = node; + + this.queues[priority].push(node); + this.size++; + this.admitted++; + this.updatePressure(); + return true; + } + + drain(max = Number.POSITIVE_INFINITY): TelemetryFrame[] { + const limit = Math.max(0, Math.floor(max)); + const out: TelemetryFrame[] = []; + while (this.head && out.length < limit) { + const node = this.head; + this.unlink(node); + this.drained++; + out.push(node.frame); + } + if (out.length > 0) this.updatePressure(); + return out; + } + + peek(): TelemetryFrame | undefined { + return this.head?.frame; + } + + recordSampled(count = 1): void { + if (count > 0) this.sampled += count; + } + + onPressure(listener: (event: PressureEvent) => void): () => void { + this.pressureListeners.add(listener); + return () => this.pressureListeners.delete(listener); + } + + snapshot(): RingSnapshot { + const dropped = this.rejected + this.evicted; + return { + capacity: this.capacity, + size: this.size, + occupancy: this.occupancy, + pressure: this.pressure, + offered: this.offered, + admitted: this.admitted, + rejected: this.rejected, + evicted: this.evicted, + drained: this.drained, + sampled: this.sampled, + dropped, + droppedReasons: { ...this.droppedReasons }, + conservation: { + offeredOk: this.offered === this.admitted + this.rejected, + admittedOk: + this.admitted === this.evicted + this.drained + this.size, + ok: + this.offered === this.admitted + this.rejected && + this.admitted === this.evicted + this.drained + this.size, + }, + }; + } + + reset(): void { + this.head = null; + this.tail = null; + this.size = 0; + this.queues.heartbeat.length = 0; + this.queues.normal.length = 0; + this.queues.critical.length = 0; + this.offered = 0; + this.admitted = 0; + this.rejected = 0; + this.evicted = 0; + this.drained = 0; + this.sampled = 0; + this.droppedReasons.overflow_heartbeat = 0; + this.droppedReasons.overflow_normal = 0; + this.droppedReasons.overflow_critical = 0; + this.setPressure('normal'); + } + + private findEvictionVictim(): Node | undefined { + for (const priority of ['heartbeat', 'normal', 'critical'] as const) { + const queue = this.queues[priority]; + while (queue.length > 0 && !queue[0].active) queue.shift(); + if (queue.length > 0) return queue[0]; + } + return undefined; + } + + private unlink(node: Node): void { + node.active = false; + if (node.prev) node.prev.next = node.next; + else this.head = node.next; + if (node.next) node.next.prev = node.prev; + else this.tail = node.prev; + node.prev = null; + node.next = null; + this.size--; + } + + private bumpDrop(priority: TelemetryPriority): void { + this.droppedReasons[DROP_REASON[priority]]++; + } + + private updatePressure(): void { + const occupancy = this.occupancy; + let next: PressureLevel = this.pressure; + + if (occupancy >= this.shedWatermark) next = 'critical'; + else if (occupancy >= this.highWatermark) next = 'high'; + else if (occupancy <= this.lowWatermark) next = 'normal'; + + this.setPressure(next); + } + + private setPressure(next: PressureLevel): void { + if (next === this.pressure) return; + this.pressure = next; + const event: PressureEvent = { + level: next, + occupancy: this.occupancy, + size: this.size, + capacity: this.capacity, + }; + for (const listener of this.pressureListeners) { + try { + listener(event); + } catch { + // safe + } + } } } + +export default RingBuffer; diff --git a/src/lib/syncManager.ts b/src/lib/syncManager.ts index 689efc03..e192cabd 100644 --- a/src/lib/syncManager.ts +++ b/src/lib/syncManager.ts @@ -2,15 +2,78 @@ import { LwwElementSet, type LwwOperation } from './crdt' import { waitForRecovery, isRecoveryComplete } from './crashRecovery' import { SqliteBusyError } from '../services/storageEngine' +export interface SyncConflict { + conflict: true + serverVersion: number + mergeProposal: Record +} + +export interface ValidationIssue { + path: string + message: string +} + +export type SyncOutcome = + | { status: 'ok'; snapshot: Record; serverVersion?: number } + | { status: 'conflict'; conflict: SyncConflict } + | { status: 'validation'; issues: ValidationIssue[] } + | { status: 'auth'; reason: 'unauthorized' | 'forbidden' } + | { status: 'retry'; attempts: number; error: string } + +export interface SyncManagerOptions { + maxRetries?: number + baseDelayMs?: number + sleep?: (ms: number) => Promise + onConflict?: (conflict: SyncConflict) => void +} + +/** + * Offline outbox client. + * + * `sync()` keeps the legacy throw-on-error behaviour with crash recovery fencing. + * `syncWithConflict()` is the contract-aware path: a 409 is a terminal, user-mergeable + * outcome (never an infinite retry), a 422 surfaces issues, and only 5xx/network errors + * retry with bounded exponential backoff. GraphQL union responses are understood too + * so REST-flush and GraphQL-flush clients behave identically. + */ export class SyncManager { + private readonly maxRetries: number + private readonly baseDelayMs: number + private readonly sleep: (ms: number) => Promise + private readonly onConflict?: (conflict: SyncConflict) => void private idbOutbox: any[] = [] + readonly journalStore: any + readonly storageEngine: any constructor( readonly document: LwwElementSet, readonly endpoint = '/api/sync', - readonly journalStore: any = null, - readonly storageEngine: any = null - ) {} + journalStoreOrOptions: any = null, + storageEngine: any = null, + options?: SyncManagerOptions + ) { + if ( + journalStoreOrOptions && + typeof journalStoreOrOptions === 'object' && + !('append' in journalStoreOrOptions) && + ('maxRetries' in journalStoreOrOptions || + 'baseDelayMs' in journalStoreOrOptions || + 'sleep' in journalStoreOrOptions || + 'onConflict' in journalStoreOrOptions) + ) { + options = journalStoreOrOptions + this.journalStore = null + this.storageEngine = null + } else { + this.journalStore = journalStoreOrOptions + this.storageEngine = storageEngine + } + const opts = options || {} + this.maxRetries = opts.maxRetries ?? 3 + this.baseDelayMs = opts.baseDelayMs ?? 250 + this.sleep = opts.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))) + this.onConflict = opts.onConflict + } /** * Dispatch insert lane: @@ -57,13 +120,88 @@ export class SyncManager { body: JSON.stringify(this.document.snapshot()), }) if (!response.ok) throw new Error(`sync failed: ${response.status}`) - const body = (await response.json()) as { operations: LwwOperation[]; deduped?: boolean } - if (body.operations) { + const body = (await response.json()) as { operations?: LwwOperation[]; deduped?: boolean } + if (Array.isArray(body.operations)) { this.document.merge(body.operations) } return this.document.snapshot() } + async syncWithConflict(fetcher: typeof fetch = fetch): Promise> { + await waitForRecovery() + let lastError = 'sync failed' + for (let attempt = 1; attempt <= this.maxRetries + 1; attempt++) { + try { + const response = await fetcher(this.endpoint, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(this.document.snapshot()), + }) + + if (response.status === 401) return { status: 'auth', reason: 'unauthorized' } + if (response.status === 403) return { status: 'auth', reason: 'forbidden' } + + if (response.status === 409) { + const body = (await response.json()) as Partial & { + data?: { syncFlush?: SyncConflict } + } + const conflict = (body.data?.syncFlush ?? body) as SyncConflict + this.onConflict?.(conflict) + return { status: 'conflict', conflict } + } + + if (response.status === 422) { + const body = (await response.json()) as { issues?: ValidationIssue[] } + return { status: 'validation', issues: body.issues ?? [] } + } + + if (!response.ok) { + lastError = `sync failed: ${response.status}` + if (response.status < 500) return { status: 'validation', issues: [] } + if (attempt > this.maxRetries) break + await this.sleep(this.baseDelayMs * 2 ** (attempt - 1)) + continue + } + + const body = (await response.json()) as { operations?: LwwOperation[]; data?: { syncFlush?: unknown } } + if (body.data?.syncFlush) { + return this.handleGraphQL(body.data.syncFlush as Record) + } + if (Array.isArray(body.operations)) this.document.merge(body.operations) + return { status: 'ok', snapshot: this.document.snapshot() } + } catch (error) { + lastError = String((error as Error)?.message ?? error) + if (attempt > this.maxRetries) break + await this.sleep(this.baseDelayMs * 2 ** (attempt - 1)) + } + } + return { status: 'retry', attempts: this.maxRetries + 1, error: lastError } + } + + private handleGraphQL(result: Record): SyncOutcome { + const typename = result.__typename + if (typename === 'SyncConflict') { + const conflict: SyncConflict = { + conflict: true, + serverVersion: Number(result.serverVersion) || 0, + mergeProposal: (result.mergeProposal as Record) ?? {}, + } + this.onConflict?.(conflict) + return { status: 'conflict', conflict } + } + if (typename === 'ValidationError') { + return { status: 'validation', issues: (result.issues as ValidationIssue[]) ?? [] } + } + if (typename === 'SyncSuccess') { + return { + status: 'ok', + snapshot: this.document.snapshot(), + serverVersion: Number(result.serverVersion) || undefined, + } + } + return { status: 'ok', snapshot: this.document.snapshot() } + } + getOutbox(): any[] { return [...this.idbOutbox] } @@ -72,4 +210,3 @@ export class SyncManager { this.idbOutbox = [] } } - diff --git a/src/lib/telemetry.ts b/src/lib/telemetry.ts index b9e5803a..f24490c8 100644 --- a/src/lib/telemetry.ts +++ b/src/lib/telemetry.ts @@ -1,44 +1,577 @@ /** - * OpenTelemetry integration for error tracking and performance monitoring + * Telemetry pipeline: producer → adaptive sampler → priority ring → exporter. + * + * The ring is the source of truth for admission/eviction; this class adds + * network-aware sampling, bounded export batches, offline outbox diversion, + * and reason-coded accounting that the TelemetryLab consumes. + * + * Also incorporates OpenTelemetry integration for error tracking and performance monitoring * Issue #746: Link errors to OTel traces for debugging blob-worker failures * Issue #754: Track GraphQL subscription transport performance */ -import { trace, context, SpanStatusCode } from '@opentelemetry/api' +import { trace, context, SpanStatusCode } from '@opentelemetry/api'; -const tracer = trace.getTracer('helphone') +import { + RingBuffer, + type PressureEvent, + type PressureLevel, + type RingBufferOptions, + type RingDropReason, + type TelemetryFrame, + type TelemetryPriority, +} from './ringBuffer'; +import { + profileFor, + type NetworkClass, + type NetworkProfile, +} from './networkEstimator'; + +export type TelemetryDropReason = + | RingDropReason + | 'sampled' + | 'backpressure' + | 'export_fail' + | 'parse_error'; + +export interface TelemetrySpanInput { + name: string; + priority?: TelemetryPriority; + kind?: string; + ts?: number; + durationMs?: number; + attributes?: Record; +} + +export interface TelemetrySpan extends TelemetrySpanInput { + seq: number; + priority: TelemetryPriority; + kind: string; + ts: number; +} + +export interface TelemetryExporter { + export(spans: TelemetrySpan[]): Promise; +} + +export interface TelemetryOutbox { + enqueue(spans: TelemetrySpan[]): Promise; +} + +export interface TelemetryPipelineOptions { + capacity?: number; + networkClass?: NetworkClass; + exporter?: TelemetryExporter | null; + outbox?: TelemetryOutbox | null; + now?: () => number; + autoStart?: boolean; + ring?: Partial; +} + +export interface FlushResult { + status: 'empty' | 'exported' | 'outbox' | 'failed' | 'in_flight'; + count: number; + error?: string; + retryAfterMs?: number; +} + +export interface TelemetrySnapshot { + produced: number; + parsed: number; + sampled: number; + admitted: number; + exported: number; + outboxed: number; + exportFailed: number; + dropped: number; + droppedReasons: Record; + queueDepth: number; + inFlight: number; + occupancy: number; + pressure: PressureLevel; + networkClass: NetworkClass; + profile: NetworkProfile; + exporterHealthy: boolean; + lastError: string | null; + lastExportAt: number | null; + conservation: { + offeredOk: boolean; + admittedOk: boolean; + accountingOk: boolean; + ok: boolean; + accounted: number; + }; +} + +function emptyReasons(): Record { + return { + overflow_heartbeat: 0, + overflow_normal: 0, + overflow_critical: 0, + sampled: 0, + backpressure: 0, + export_fail: 0, + parse_error: 0, + }; +} + +/** Deterministic sampler: same sequence number → same decision. */ +export function shouldSample(seq: number, rate: number): boolean { + if (rate >= 1) return true; + if (rate <= 0) return false; + const mixed = (Math.imul((seq | 0) ^ 0x9e3779b9, 2654435761) >>> 0) / 0x1_0000_0000; + return mixed < rate; +} + +export function samplingRate( + profile: NetworkProfile, + pressure: PressureLevel, + priority: TelemetryPriority, +): number { + if (priority === 'critical') return 1; + const base = priority === 'heartbeat' ? profile.heartbeatSampleRate : profile.normalSampleRate; + if (pressure === 'critical') return Math.max(0, base * 0.1); + if (pressure === 'high') return Math.max(0, base * 0.5); + return Math.max(0, Math.min(1, base)); +} + +export class TelemetryPipeline { + readonly ring: RingBuffer; + private networkClass: NetworkClass; + private profile: NetworkProfile; + private exporter: TelemetryExporter | null; + private outbox: TelemetryOutbox | null; + private readonly now: () => number; + + private sequence = 0; + private produced = 0; + private parsed = 0; + private sampled = 0; + private exported = 0; + private outboxed = 0; + private exportFailed = 0; + private backpressureDropped = 0; + private exportFailReasons = 0; + private parseErrors = 0; + private inFlight = 0; + private flushing = false; + private exporterHealthy = true; + private lastError: string | null = null; + private lastExportAt: number | null = null; + private timer: ReturnType | null = null; + private started = false; + + constructor(options: TelemetryPipelineOptions = {}) { + this.ring = new RingBuffer({ + capacity: options.capacity ?? 2_048, + ...(options.ring || {}), + }); + this.networkClass = options.networkClass ?? '4g'; + this.profile = profileFor(this.networkClass); + this.exporter = options.exporter ?? null; + this.outbox = options.outbox ?? null; + this.now = options.now ?? (() => Date.now()); + if (options.autoStart) this.start(); + } + + get network(): NetworkClass { + return this.networkClass; + } + + get batchProfile(): NetworkProfile { + return this.profile; + } + + record(input: TelemetrySpanInput): boolean { + this.produced++; + const priority = input.priority ?? 'normal'; + const pressure = this.ring.pressureLevel; + + // 90%+ occupancy: producers shed non-critical work explicitly. Critical + // dispatch frames keep flowing until the ring itself is entirely critical. + if (pressure === 'critical' && priority !== 'critical') { + this.backpressureDropped++; + return false; + } + + const rate = samplingRate(this.profile, pressure, priority); + if (!shouldSample(this.sequence, rate)) { + this.sampled++; + this.ring.recordSampled(1); + return false; + } + + const span: TelemetrySpan = { + name: input.name, + priority, + kind: input.kind ?? input.name, + ts: input.ts ?? this.now(), + durationMs: input.durationMs, + attributes: input.attributes, + seq: this.sequence++, + }; + + return this.ring.push({ + seq: span.seq, + priority, + kind: span.kind, + ts: span.ts, + payload: span, + }); + } + + recordParsed(count = 1): void { + if (count > 0) this.parsed += count; + } + + /** + * Account for a frame/span dropped outside the ring (shed before parsing, + * parse error, or explicit backpressure). This is a producer-side event, so + * it increments `produced` as well as the reason counter. + */ + recordDropped(reason: TelemetryDropReason, count = 1): void { + if (count <= 0) return; + this.produced += count; + if (reason === 'sampled') this.sampled += count; + else if (reason === 'backpressure') this.backpressureDropped += count; + else if (reason === 'export_fail') this.exportFailReasons += count; + else if (reason === 'parse_error') this.parseErrors += count; + // Ring overflow reasons are owned by RingBuffer when frames are admitted. + } + + /** Drain + export one bounded batch. Never throws. */ + async flush(): Promise { + if (this.flushing) return { status: 'in_flight', count: 0 }; + this.flushing = true; + + const batchSize = Math.max(1, this.profile.maxExportBatchSize); + const frames = this.ring.drain(batchSize); + if (frames.length === 0) { + this.flushing = false; + return { status: 'empty', count: 0 }; + } + + const spans = frames.map((frame) => frame.payload); + this.inFlight += spans.length; + try { + if (this.profile.outboxOnly || this.networkClass === 'offline') { + if (this.outbox) { + await this.outbox.enqueue(spans); + this.outboxed += spans.length; + return { status: 'outbox', count: spans.length }; + } + // No durable outbox: do not pretend these were exported. + this.exportFailed += spans.length; + this.exportFailReasons += spans.length; + this.exporterHealthy = false; + this.lastError = 'offline without outbox'; + return { status: 'failed', count: spans.length, error: this.lastError }; + } + + if (!this.exporter) { + this.exportFailed += spans.length; + this.exportFailReasons += spans.length; + this.lastError = 'no exporter configured'; + return { status: 'failed', count: spans.length, error: this.lastError }; + } + + await this.exporter.export(spans); + this.exported += spans.length; + this.exporterHealthy = true; + this.lastExportAt = this.now(); + this.lastError = null; + return { status: 'exported', count: spans.length }; + } catch (error) { + this.exportFailed += spans.length; + this.exportFailReasons += spans.length; + this.exporterHealthy = false; + this.lastError = String((error as { message?: string })?.message ?? error); + const retryAfterMs = Number((error as { retryAfterMs?: number })?.retryAfterMs); + return { + status: 'failed', + count: spans.length, + error: this.lastError, + retryAfterMs: Number.isFinite(retryAfterMs) ? retryAfterMs : undefined, + }; + } finally { + this.inFlight = Math.max(0, this.inFlight - spans.length); + this.flushing = false; + } + } + + setNetworkClass(networkClass: NetworkClass): void { + if (networkClass === this.networkClass) return; + this.networkClass = networkClass; + this.profile = profileFor(networkClass); + this.reschedule(); + } + + setExporter(exporter: TelemetryExporter | null): void { + this.exporter = exporter; + } + + setOutbox(outbox: TelemetryOutbox | null): void { + this.outbox = outbox; + } + + onPressure(listener: (event: PressureEvent) => void): () => void { + return this.ring.onPressure(listener); + } + + start(): void { + if (this.started) return; + this.started = true; + this.reschedule(); + } + + stop(): void { + this.started = false; + if (this.timer) clearTimeout(this.timer); + this.timer = null; + } + + snapshot(): TelemetrySnapshot { + const ring = this.ring.snapshot(); + const reasons = emptyReasons(); + reasons.overflow_heartbeat = ring.droppedReasons.overflow_heartbeat; + reasons.overflow_normal = ring.droppedReasons.overflow_normal; + reasons.overflow_critical = ring.droppedReasons.overflow_critical; + reasons.sampled = this.sampled; + reasons.backpressure = this.backpressureDropped; + reasons.export_fail = this.exportFailReasons; + reasons.parse_error = this.parseErrors; + + const dropped = + ring.rejected + + ring.evicted + + this.sampled + + this.backpressureDropped + + this.exportFailed + + this.parseErrors; + const accounted = this.exported + this.outboxed + dropped + ring.size + this.inFlight; + const accountingOk = accounted === this.produced; + const offeredOk = ring.conservation.offeredOk; + const admittedOk = ring.conservation.admittedOk; + + return { + produced: this.produced, + parsed: this.parsed, + sampled: this.sampled, + admitted: ring.admitted, + exported: this.exported, + outboxed: this.outboxed, + exportFailed: this.exportFailed, + dropped, + droppedReasons: reasons, + queueDepth: ring.size, + inFlight: this.inFlight, + occupancy: ring.occupancy, + pressure: ring.pressure, + networkClass: this.networkClass, + profile: this.profile, + exporterHealthy: this.exporterHealthy, + lastError: this.lastError, + lastExportAt: this.lastExportAt, + conservation: { + offeredOk, + admittedOk, + accountingOk, + ok: offeredOk && admittedOk && accountingOk, + accounted, + }, + }; + } + + reset(): void { + this.ring.reset(); + this.produced = 0; + this.parsed = 0; + this.sampled = 0; + this.exported = 0; + this.outboxed = 0; + this.exportFailed = 0; + this.backpressureDropped = 0; + this.exportFailReasons = 0; + this.parseErrors = 0; + this.inFlight = 0; + this.flushing = false; + this.exporterHealthy = true; + this.lastError = null; + this.lastExportAt = null; + this.sequence = 0; + } + + private reschedule(): void { + if (this.timer) clearTimeout(this.timer); + this.timer = null; + if (!this.started) return; + const delay = this.profile.scheduledDelayMillis; + if (delay <= 0) return; // offline/manual flush only + this.timer = setTimeout(() => { + this.flush().finally(() => this.reschedule()); + }, delay); + // Do not keep a Node process alive just for telemetry exports. + (this.timer as { unref?: () => void }).unref?.(); + } +} + +/** In-memory exporter used by tests and the lab. */ +export function createMemoryExporter(options: { maxBatches?: number } = {}): TelemetryExporter & { + batches: TelemetrySpan[][]; + reset(): void; +} { + const maxBatches = options.maxBatches ?? 100; + const batches: TelemetrySpan[][] = []; + return { + batches, + async export(spans) { + if (spans.length === 0) return; + batches.push(spans); + while (batches.length > maxBatches) batches.shift(); + }, + reset() { + batches.length = 0; + }, + }; +} + +export interface OtlpExporterOptions { + endpoint: string; + fetchImpl?: typeof fetch; + timeoutMs?: number; + headers?: Record; + serviceName?: string; +} + +/** + * Minimal OTLP/HTTP JSON exporter. Avoids pulling the server-oriented + * @opentelemetry/sdk-node bundle into the browser just for batching. + */ +export function createOtlpHttpExporter(options: OtlpExporterOptions): TelemetryExporter { + const { + endpoint, + fetchImpl = globalThis.fetch, + timeoutMs = 5_000, + headers = {}, + serviceName = 'helphone-web', + } = options; + + return { + async export(spans) { + if (spans.length === 0) return; + const controller = typeof AbortController !== 'undefined' ? new AbortController() : null; + const timer = controller ? setTimeout(() => controller.abort(), timeoutMs) : null; + try { + const response = await fetchImpl(endpoint, { + method: 'POST', + headers: { 'content-type': 'application/json', ...headers }, + body: JSON.stringify(toOtlpJson(spans, serviceName)), + signal: controller?.signal, + keepalive: spans.length <= 32, + }); + if (!response.ok) { + const retryAfter = Number(response.headers?.get?.('retry-after')); + const error = new Error(`OTLP export failed: HTTP ${response.status}`) as Error & { + status?: number; + retryAfterMs?: number; + }; + error.status = response.status; + if (Number.isFinite(retryAfter)) error.retryAfterMs = Math.max(0, retryAfter * 1_000); + throw error; + } + } finally { + if (timer) clearTimeout(timer); + } + }, + }; +} + +export function toOtlpJson(spans: TelemetrySpan[], serviceName = 'helphone-web'): unknown { + return { + resourceSpans: [ + { + resource: { + attributes: [ + { key: 'service.name', value: { stringValue: serviceName } }, + { key: 'telemetry.sdk.language', value: { stringValue: 'webjs' } }, + ], + }, + scopeSpans: [ + { + scope: { name: 'helphone.telemetry', version: '1.0.0' }, + spans: spans.map((span) => ({ + traceId: traceIdFor(span.seq), + spanId: spanIdFor(span.seq), + name: span.name, + kind: 1, + startTimeUnixNano: String(Math.round(span.ts * 1e6)), + endTimeUnixNano: String(Math.round((span.ts + (span.durationMs ?? 0)) * 1e6)), + attributes: [ + { key: 'helphone.priority', value: { stringValue: span.priority } }, + { key: 'helphone.kind', value: { stringValue: span.kind } }, + ...Object.entries(span.attributes || {}).map(([key, value]) => ({ + key, + value: + typeof value === 'number' + ? { doubleValue: value } + : typeof value === 'boolean' + ? { boolValue: value } + : { stringValue: String(value) }, + })), + ], + })), + }, + ], + }, + ], + }; +} + +function traceIdFor(seq: number): string { + // Stable, non-cryptographic id derived from the sequence; good enough for + // correlation in the lab and avoids Math.random() in hot paths. + const high = (Math.imul(seq, 2654435761) >>> 0).toString(16).padStart(8, '0'); + const low = (Math.imul(seq ^ 0x9e3779b9, 40503) >>> 0).toString(16).padStart(8, '0'); + return `${high}${low}`.padEnd(32, '0').slice(0, 32); +} + +function spanIdFor(seq: number): string { + return (Math.imul(seq ^ 0x85ebca6b, 2246822507) >>> 0).toString(16).padStart(8, '0').padEnd(16, '0').slice(0, 16); +} + +const tracer = trace.getTracer('helphone'); export interface ErrorContext { - source: string - component?: string - traceId?: string - userId?: string - [key: string]: unknown + source: string; + component?: string; + traceId?: string; + userId?: string; + [key: string]: unknown; } /** * Record an error with OTel context (Issue #746) */ export function recordError(error: Error, ctx: ErrorContext = {}): void { - const span = tracer.startSpan('error.record') + const span = tracer.startSpan('error.record'); try { - span.recordException(error) - span.setStatus({ code: SpanStatusCode.ERROR, message: error.message }) + span.recordException(error); + span.setStatus({ code: SpanStatusCode.ERROR, message: error.message }); // Add context attributes Object.entries(ctx).forEach(([key, value]) => { - span.setAttribute(key, String(value)) - }) + span.setAttribute(key, String(value)); + }); // Store trace ID for linking - const activeSpan = trace.getActiveSpan() + const activeSpan = trace.getActiveSpan(); if (activeSpan) { - const traceId = activeSpan.spanContext().traceId - sessionStorage.setItem('otel_trace_id', traceId) + const traceId = activeSpan.spanContext().traceId; + sessionStorage.setItem('otel_trace_id', traceId); } } finally { - span.end() + span.end(); } } @@ -46,14 +579,14 @@ export function recordError(error: Error, ctx: ErrorContext = {}): void { * Link an error to an existing trace (Issue #746) */ export function linkError(error: Error, traceId: string): void { - const span = tracer.startSpan('error.link') + const span = tracer.startSpan('error.link'); try { - span.setAttribute('error.trace_id', traceId) - span.setAttribute('error.message', error.message) - span.recordException(error) + span.setAttribute('error.trace_id', traceId); + span.setAttribute('error.message', error.message); + span.recordException(error); } finally { - span.end() + span.end(); } } @@ -66,7 +599,7 @@ export function trackSubscriptionTransport( duration: number, success: boolean ): void { - const span = tracer.startSpan('subscription.transport') + const span = tracer.startSpan('subscription.transport'); try { span.setAttributes({ @@ -74,13 +607,13 @@ export function trackSubscriptionTransport( 'subscription.operation': operation, 'subscription.duration_ms': duration, 'subscription.success': success, - }) + }); if (!success) { - span.setStatus({ code: SpanStatusCode.ERROR }) + span.setStatus({ code: SpanStatusCode.ERROR }); } } finally { - span.end() + span.end(); } } @@ -94,19 +627,19 @@ export function trackAudioAlert( outcome: 'played' | 'blocked', details: { fallback?: 'vibrate' | 'none' } = {} ): void { - const span = tracer.startSpan('audio.alert') + const span = tracer.startSpan('audio.alert'); try { span.setAttributes({ 'audio.outcome': outcome, 'audio.fallback': details.fallback ?? 'none', - }) + }); if (outcome === 'blocked') { - span.setStatus({ code: SpanStatusCode.ERROR, message: 'autoplay blocked' }) + span.setStatus({ code: SpanStatusCode.ERROR, message: 'autoplay blocked' }); } } finally { - span.end() + span.end(); } } @@ -117,15 +650,15 @@ export function trackVideoPerformance( metric: 'forward_play' | 'reverse_play' | 'frame_drop' | 'raf_throttle', duration: number ): void { - const span = tracer.startSpan('video.performance') + const span = tracer.startSpan('video.performance'); try { span.setAttributes({ 'video.metric': metric, 'video.duration_ms': duration, - }) + }); } finally { - span.end() + span.end(); } } @@ -137,16 +670,16 @@ export function trackSorobanState( size: number, duration: number ): void { - const span = tracer.startSpan('soroban.state') + const span = tracer.startSpan('soroban.state'); try { span.setAttributes({ 'soroban.operation': operation, 'soroban.state_size_bytes': size, 'soroban.duration_ms': duration, - }) + }); } finally { - span.end() + span.end(); } } @@ -158,15 +691,17 @@ export function startBlobWorkerTrace(operationId: string): () => void { attributes: { 'blob_worker.operation_id': operationId, }, - }) + }); // Store operation ID for error linkage - const operationIds = JSON.parse(sessionStorage.getItem('blob_worker_ops') || '[]') - operationIds.push({ id: operationId, timestamp: Date.now() }) + const operationIds = JSON.parse(sessionStorage.getItem('blob_worker_ops') || '[]'); + operationIds.push({ id: operationId, timestamp: Date.now() }); // Keep only last 10 operations - sessionStorage.setItem('blob_worker_ops', JSON.stringify(operationIds.slice(-10))) + sessionStorage.setItem('blob_worker_ops', JSON.stringify(operationIds.slice(-10))); return () => { - span.end() - } + span.end(); + }; } + +export default TelemetryPipeline; diff --git a/src/lib/telemetryLabController.ts b/src/lib/telemetryLabController.ts new file mode 100644 index 00000000..39b81e6d --- /dev/null +++ b/src/lib/telemetryLabController.ts @@ -0,0 +1,171 @@ +/** + * TelemetryLab controller: a deterministic, browser-safe harness for the + * pipeline. It generates binary frames at a configurable rate, feeds them + * through the worker (or main-thread fallback), and exposes one truthful + * snapshot per second. + */ + +import { TelemetryPipeline, createMemoryExporter, type TelemetrySnapshot } from './telemetry'; +import { TelemetryIngestClient } from './telemetryWorkerClient'; +import { TelemetryWasmParser } from '../utils/binaryParserWasm'; +import { encodeFrame, FRAME_KIND } from '../utils/binaryParser'; +import { profileFor, type NetworkClass } from './networkEstimator'; +import { FRAME_BYTES } from '../utils/binaryParser'; + +export interface LabSnapshot extends TelemetrySnapshot { + lab: { + running: boolean; + rateHz: number; + framesIngested: number; + framesShed: number; + parseErrors: number; + workerMode: string; + collectorDown: boolean; + }; +} + +export interface LabOptions { + capacity?: number; + rateHz?: number; + useWorker?: boolean; +} + +export class TelemetryLabController { + readonly pipeline: TelemetryPipeline; + readonly exporter: ReturnType; + readonly ingest: TelemetryIngestClient; + private timer: ReturnType | null = null; + private running = false; + private rateHz: number; + private seq = 0; + private collectorDown = false; + private readonly timers: { set?: typeof setInterval; clear?: typeof clearInterval }; + + constructor(options: LabOptions = {}) { + this.rateHz = options.rateHz ?? 500; + this.exporter = createMemoryExporter({ maxBatches: 40 }); + this.pipeline = new TelemetryPipeline({ + capacity: options.capacity ?? 512, + networkClass: '4g', + exporter: this.exporter, + ring: { highWatermark: 0.7, shedWatermark: 0.9, lowWatermark: 0.5 }, + }); + + const preferWorker = options.useWorker ?? typeof Worker !== 'undefined'; + this.ingest = new TelemetryIngestClient({ + pipeline: this.pipeline, + parser: new TelemetryWasmParser({ preferWasm: true }), + workerFactory: preferWorker + ? () => + new Worker(new URL('../workers/telemetryWorker.js', import.meta.url), { + type: 'module', + }) + : undefined, + }); + } + + start(): void { + if (this.running) return; + this.running = true; + this.ingest.start(); + // 20 ticks/s keeps the generator cheap while still producing 500 msg/s at + // the default rate; each tick emits exactly rateHz/20 frames. + this.timer = setInterval(() => this.emitTick(), 50); + } + + stop(): void { + this.running = false; + if (this.timer) clearInterval(this.timer); + this.timer = null; + } + + setRate(rateHz: number): void { + this.rateHz = Math.max(1, Math.min(10_000, Math.floor(rateHz))); + } + + setNetwork(networkClass: NetworkClass): void { + this.pipeline.setNetworkClass(networkClass); + } + + setCollectorDown(down: boolean): void { + this.collectorDown = down; + this.pipeline.setExporter( + down + ? { + async export() { + const error = new Error('collector unavailable') as Error & { retryAfterMs?: number }; + error.retryAfterMs = 2_000; + throw error; + }, + } + : this.exporter, + ); + } + + async flush(): Promise { + await this.pipeline.flush(); + } + + snapshot(): LabSnapshot { + const pipeline = this.pipeline.snapshot(); + const ingest = this.ingest.snapshot(); + return { + ...pipeline, + lab: { + running: this.running, + rateHz: this.rateHz, + framesIngested: ingest.framesIngested, + framesShed: ingest.framesShed, + parseErrors: ingest.parseErrors, + workerMode: ingest.workerMode, + collectorDown: this.collectorDown, + }, + }; + } + + destroy(): void { + this.stop(); + this.ingest.stop(); + this.pipeline.stop(); + } + + private emitTick(): void { + const perTick = Math.max(1, Math.round(this.rateHz / 20)); + const buffer = new ArrayBuffer(perTick * FRAME_BYTES); + const bytes = new Uint8Array(buffer); + let critical = 0; + for (let i = 0; i < perTick; i++) { + const seq = this.seq++; + const roll = seq % 10; + const kind = roll === 0 ? FRAME_KIND.DISPATCH_ACK : roll < 3 ? FRAME_KIND.HEARTBEAT : FRAME_KIND.LOCATION; + if (kind === FRAME_KIND.DISPATCH_ACK) critical++; + const frame = encodeFrame({ + kind, + priority: kind === FRAME_KIND.DISPATCH_ACK ? 2 : kind === FRAME_KIND.HEARTBEAT ? 0 : 1, + seq, + ts: Date.now(), + lat: 6.5 + (seq % 1_000) * 1e-5, + lng: 3.3 + (seq % 1_000) * 1e-5, + accuracy: 3 + (seq % 20), + }); + bytes.set(new Uint8Array(frame), i * FRAME_BYTES); + } + this.ingest.ingest(buffer, perTick, critical === perTick ? 'critical' : 'normal'); + } +} + +let singleton: TelemetryLabController | null = null; + +export function getTelemetryLabController(options?: LabOptions): TelemetryLabController { + if (!singleton) singleton = new TelemetryLabController(options); + return singleton; +} + +export function resetTelemetryLabController(): void { + singleton?.destroy(); + singleton = null; +} + +export function labProfile(networkClass: NetworkClass) { + return profileFor(networkClass); +} diff --git a/src/lib/telemetryWorkerClient.ts b/src/lib/telemetryWorkerClient.ts new file mode 100644 index 00000000..54ab2b47 --- /dev/null +++ b/src/lib/telemetryWorkerClient.ts @@ -0,0 +1,264 @@ +/** + * Main-thread ingest client for the telemetry worker. + * + * Responsibilities: + * - Transfer raw ArrayBuffers to the worker (zero-copy ownership handoff). + * - Mirror ring pressure to the worker immediately (<200ms), so the worker + * stops parsing non-critical batches before its queue grows. + * - Fall back to main-thread parsing when workers are unavailable/corrupt. + * - Keep raw-frame accounting truthful: every ingested frame ends up parsed, + * sampled, shed, or queue-dropped with an explicit reason. + */ + +import { TelemetryPipeline, type TelemetrySpanInput } from './telemetry'; +import type { TelemetryPriority } from './ringBuffer'; + +export interface TelemetryWorkerLike { + postMessage(message: unknown, transfer?: Transferable[]): void; + terminate(): void; + onmessage: ((event: { data: unknown }) => void) | null; + onerror: ((event: unknown) => void) | null; +} + +export interface ParsedFrame { + priority?: TelemetryPriority; + kindName?: string; + ts?: number; + seq?: number; + lat?: number; + lng?: number; + accuracy?: number; + error?: string; +} + +export interface TelemetryIngestOptions { + pipeline: TelemetryPipeline; + workerFactory?: () => TelemetryWorkerLike; + parser?: { parseBatch(input: ArrayBuffer | ArrayBufferView, count?: number, options?: unknown): ParsedFrame[] }; + now?: () => number; + workerTimeoutMs?: number; +} + +export interface IngestStats { + framesIngested: number; + framesParsed: number; + framesShed: number; + parseErrors: number; + pending: number; + workerMode: 'worker' | 'main-thread'; + pipeline: ReturnType; +} + +export class TelemetryIngestClient { + private readonly pipeline: TelemetryPipeline; + private readonly workerFactory?: () => TelemetryWorkerLike; + private readonly parser?: TelemetryIngestOptions['parser']; + private readonly now: () => number; + private worker: TelemetryWorkerLike | null = null; + private workerMode: 'worker' | 'main-thread' = 'main-thread'; + private nextId = 1; + private framesIngested = 0; + private framesParsed = 0; + private framesShed = 0; + private parseErrors = 0; + private readonly pending = new Map(); + private unsubscribePressure: (() => void) | null = null; + private stopped = false; + + constructor(options: TelemetryIngestOptions) { + this.pipeline = options.pipeline; + this.workerFactory = options.workerFactory; + this.parser = options.parser; + this.now = options.now ?? (() => Date.now()); + } + + start(): void { + if (this.worker || this.stopped) return; + if (!this.workerFactory || typeof Worker === 'undefined') return; + try { + const worker = this.workerFactory(); + worker.onmessage = (event) => this.handleWorkerMessage(event.data); + worker.onerror = () => { + // A crashed worker must not take ingestion down. Fall back in place; + // pending frames are accounted as parse errors. + this.failPending('worker_error'); + this.worker = null; + this.workerMode = 'main-thread'; + }; + worker.postMessage({ type: 'telemetry:init' }); + this.worker = worker; + this.workerMode = 'worker'; + this.unsubscribePressure = this.pipeline.onPressure((event) => { + const level = event.level === 'critical' ? 2 : event.level === 'high' ? 1 : 0; + try { + worker.postMessage({ type: 'telemetry:congestion', level, occupancy: event.occupancy }); + } catch { + // Worker may be terminating; the ring still sheds locally. + } + }); + } catch { + this.worker = null; + this.workerMode = 'main-thread'; + } + } + + /** Ingest `count` contiguous binary frames. Takes ownership of `buffer`. */ + ingest(buffer: ArrayBuffer, count: number, priority: TelemetryPriority = 'normal'): number { + if (this.stopped || count <= 0) return -1; + this.framesIngested += count; + + // Critical ring pressure: shed non-critical batches on the main thread + // before they ever reach the worker. This is the <200ms response path. + if (this.pipeline.ring.pressureLevel === 'critical' && priority !== 'critical') { + this.pipeline.recordDropped('backpressure', count); + this.framesShed += count; + return -1; + } + + if (this.worker) { + const id = this.nextId++; + this.pending.set(id, { count, priority }); + try { + this.worker.postMessage( + { type: 'telemetry:parse', id, buffer, count, priority }, + [buffer], + ); + return id; + } catch { + this.pending.delete(id); + // Transfer failed (buffer detached); fall through to local parse. + } + } + + this.parseLocally(buffer, count, priority); + return -1; + } + + handleWorkerMessage(message: unknown): void { + const msg = message as { + type?: string; + id?: number; + frames?: ParsedFrame[]; + shed?: number; + count?: number; + reason?: string; + message?: string; + }; + if (!msg || typeof msg.type !== 'string') return; + + if (msg.type === 'telemetry:parsed') { + this.pending.delete(msg.id ?? -1); + this.admitFrames(msg.frames ?? []); + if (msg.shed && msg.shed > 0) { + this.pipeline.recordDropped('backpressure', msg.shed); + this.framesShed += msg.shed; + } + return; + } + + if (msg.type === 'telemetry:shed') { + this.pending.delete(msg.id ?? -1); + const count = msg.count ?? 1; + this.pipeline.recordDropped('backpressure', count); + this.framesShed += count; + return; + } + + if (msg.type === 'telemetry:error') { + const entry = this.pending.get(msg.id ?? -1); + this.pending.delete(msg.id ?? -1); + if (entry) { + this.pipeline.recordDropped('parse_error', entry.count); + this.parseErrors += entry.count; + } + } + } + + flush() { + return this.pipeline.flush(); + } + + snapshot(): IngestStats { + return { + framesIngested: this.framesIngested, + framesParsed: this.framesParsed, + framesShed: this.framesShed, + parseErrors: this.parseErrors, + pending: this.pending.size, + workerMode: this.workerMode, + pipeline: this.pipeline.snapshot(), + }; + } + + stop(): void { + this.stopped = true; + this.unsubscribePressure?.(); + this.unsubscribePressure = null; + this.failPending('stopped'); + try { + this.worker?.terminate(); + } catch { + // ignore + } + this.worker = null; + this.workerMode = 'main-thread'; + } + + private parseLocally(buffer: ArrayBuffer, count: number, priority: TelemetryPriority): void { + if (!this.parser) { + this.pipeline.recordDropped('parse_error', count); + this.parseErrors += count; + return; + } + try { + const frames = this.parser.parseBatch(buffer, count, { stopOnError: false }); + this.admitFrames(frames); + } catch { + this.pipeline.recordDropped('parse_error', count); + this.parseErrors += count; + } + void priority; + } + + private admitFrames(frames: ParsedFrame[]): void { + let admitted = 0; + let errors = 0; + for (const frame of frames) { + if (frame?.error) { + errors++; + continue; + } + const span: TelemetrySpanInput = { + name: 'telemetry.frame', + priority: frame.priority ?? 'normal', + kind: frame.kindName ?? 'unknown', + ts: Number.isFinite(frame.ts) ? (frame.ts as number) : this.now(), + attributes: { + seq: frame.seq, + lat: frame.lat, + lng: frame.lng, + accuracy: frame.accuracy, + }, + }; + if (this.pipeline.record(span)) admitted++; + } + this.framesParsed += admitted; + if (errors > 0) { + this.pipeline.recordDropped('parse_error', errors); + this.parseErrors += errors; + } + } + + private failPending(reason: string): void { + let count = 0; + for (const entry of this.pending.values()) count += entry.count; + this.pending.clear(); + if (count > 0) { + this.pipeline.recordDropped('parse_error', count); + this.parseErrors += count; + void reason; + } + } +} + +export default TelemetryIngestClient; diff --git a/src/lib/telemetryWorkerProtocol.js b/src/lib/telemetryWorkerProtocol.js new file mode 100644 index 00000000..845e95ba --- /dev/null +++ b/src/lib/telemetryWorkerProtocol.js @@ -0,0 +1,212 @@ +/** + * Worker-side telemetry parsing protocol with explicit flow control. + * + * Main thread → worker: + * { type: 'telemetry:init' } + * { type: 'telemetry:parse', id, buffer, count, priority } + * { type: 'telemetry:congestion', level: 0|1|2, occupancy } + * { type: 'telemetry:stats', id } + * + * Worker → main thread: + * { type: 'telemetry:ready', mode } + * { type: 'telemetry:parsed', id, frames, stats } + * { type: 'telemetry:shed', id, count, reason } + * { type: 'telemetry:error', id?, message } + * + * When the ring signals critical occupancy (level 2), non-critical batches are + * shed without parsing, and the queue is bounded with drop-oldest accounting. + * This is what keeps low-end Android tabs from growing an unbounded worker + * queue when the map is animating at 60fps. + */ + +import { TelemetryWasmParser } from '../utils/binaryParserWasm.js'; + +export function createTelemetryWorkerProtocol(options = {}) { + const postMessage = options.postMessage ?? ((message) => self.postMessage(message)); + const parser = options.parser ?? new TelemetryWasmParser(options.parserOptions); + const now = options.now ?? (() => (typeof performance !== 'undefined' ? performance.now() : Date.now())); + const config = { + maxQueue: options.maxQueue ?? 64, + lowResLevel: options.lowResLevel ?? 1, + pauseLevel: options.pauseLevel ?? 2, + }; + + let congestion = 0; + let paused = false; + let lowRes = false; + let stopped = false; + let draining = false; + let ready = false; + + const queue = []; + const stats = { + received: 0, + parsed: 0, + shed: 0, + errors: 0, + backpressure: 0, + queueDrops: 0, + maxQueueDepth: 0, + parseMs: 0, + }; + + parser.init?.().then( + () => { + ready = true; + postMessage({ type: 'telemetry:ready', mode: parser.mode, stats: snapshot() }); + }, + (error) => { + // JS fallback remains active; report but do not fail the worker. + ready = true; + postMessage({ type: 'telemetry:ready', mode: 'js', error: String(error?.message || error) }); + }, + ); + + function snapshot() { + return { + ...stats, + mode: parser.mode, + ready: parser.ready, + congestion, + paused, + lowRes, + queueDepth: queue.length, + }; + } + + function enqueue(message) { + if (queue.length >= config.maxQueue) { + // Drop oldest, count it, then admit the newest. Never let the queue grow. + const dropped = queue.shift(); + stats.queueDrops++; + stats.shed++; + postMessage({ + type: 'telemetry:shed', + id: dropped?.id ?? null, + count: dropped?.count ?? 1, + reason: 'worker_queue_overflow', + }); + } + queue.push(message); + stats.maxQueueDepth = Math.max(stats.maxQueueDepth, queue.length); + void drain(); + } + + async function drain() { + if (draining) return; + draining = true; + try { + while (queue.length > 0 && !stopped) { + // Yield once per batch so messages that arrived in the same tick are + // visible to the bounded queue (and to the congestion check). + await Promise.resolve(); + const message = queue.shift(); + const batchPriority = message.priority ?? 'normal'; + + // 90%+ ring occupancy: only dispatch-critical batches may proceed. + if (paused && batchPriority !== 'critical') { + stats.shed += message.count ?? 1; + stats.backpressure++; + postMessage({ + type: 'telemetry:shed', + id: message.id, + count: message.count ?? 1, + reason: 'backpressure', + }); + continue; + } + + const started = now(); + try { + // `reuse` pools frame objects; postMessage clones them synchronously + // before the next batch can mutate the pool. + const frames = parser.parseBatch(message.buffer, message.count, { stopOnError: false, reuse: true, reuseArray: true }); + stats.parsed += frames.length; + stats.parseMs += now() - started; + + // Low-res mode: drop heartbeat frames before they cross the worker + // boundary. Normal + critical frames still flow. + const filtered = + lowRes && batchPriority !== 'critical' + ? frames.filter((frame) => frame.priority !== 'heartbeat' || frame.error) + : frames; + if (filtered.length !== frames.length) { + stats.shed += frames.length - filtered.length; + stats.backpressure += frames.length - filtered.length; + } + + postMessage({ + type: 'telemetry:parsed', + id: message.id, + frames: filtered, + shed: frames.length - filtered.length, + stats: snapshot(), + }); + } catch (error) { + stats.errors++; + postMessage({ + type: 'telemetry:error', + id: message.id, + message: String(error?.message || error), + }); + } + } + } finally { + draining = false; + } + } + + function handle(message) { + if (!message || typeof message.type !== 'string') return false; + if (!message.type.startsWith('telemetry:')) return false; + + switch (message.type) { + case 'telemetry:init': + postMessage({ type: 'telemetry:ready', mode: parser.mode, stats: snapshot() }); + return true; + + case 'telemetry:parse': + stats.received += message.count ?? 1; + enqueue(message); + return true; + + case 'telemetry:congestion': { + const level = Math.max(0, Math.min(2, Number(message.level) || 0)); + congestion = level; + // Hysteresis is owned by the ring: it only emits transitions. + paused = level >= config.pauseLevel; + lowRes = level >= config.lowResLevel; + if (level === 0 && queue.length > 0) void drain(); + return true; + } + + case 'telemetry:stats': + postMessage({ type: 'telemetry:stats', id: message.id, stats: snapshot() }); + return true; + + case 'telemetry:stop': + stopped = true; + queue.length = 0; + return true; + + default: + return false; + } + } + + return { + handle, + snapshot, + get queueDepth() { + return queue.length; + }, + get paused() { + return paused; + }, + get congestion() { + return congestion; + }, + }; +} + +export default createTelemetryWorkerProtocol; diff --git a/src/lib/wasmMemory.js b/src/lib/wasmMemory.js index 891c3dcd..3c073ca2 100644 --- a/src/lib/wasmMemory.js +++ b/src/lib/wasmMemory.js @@ -63,4 +63,52 @@ let singleton = null; export function getWasmMemoryPool(maxBytes = MAX_MEMORY_BYTES) { if (!singleton) { singleton = new WasmMemoryPool(maxBytes); singleton.preallocate(); } return singleton; } export function resetWasmMemoryPool() { if (singleton) singleton.destroy(); singleton = null; } export function createWasmMemoryPool(maxBytes = MAX_MEMORY_BYTES) { return new WasmMemoryPool(maxBytes); } +// ── Telemetry-specific pooling (mirrors wasmMemory.ts) ────────────────────── +export class TelemetryBufferPool { + constructor(frameBytes = 4 * 1024, maxBuffers = 64, preferShared = true) { + this.frameBytes = frameBytes; + this.maxBuffers = maxBuffers; + this.free = []; + this.active = new Set(); + this.allocationCount = 0; + this.recycledCount = 0; + this.shared = preferShared && typeof SharedArrayBuffer !== 'undefined' && globalThis.crossOriginIsolated === true; + } + acquire() { + let lease = this.free.pop(); + if (lease) this.recycledCount++; + else { + const buffer = this.shared ? new SharedArrayBuffer(this.frameBytes) : new ArrayBuffer(this.frameBytes); + lease = { buffer, bytes: new Uint8Array(buffer), shared: this.shared }; + this.allocationCount++; + } + this.active.add(lease); + return lease; + } + release(lease) { + if (!lease || !this.active.delete(lease)) return; + if (this.free.length >= this.maxBuffers) return; + try { lease.bytes.fill(0); } catch {} + this.free.push(lease); + } + getStats() { + return { frameBytes: this.frameBytes, pooled: this.free.length, active: this.active.size, allocationCount: this.allocationCount, recycledCount: this.recycledCount, shared: this.shared }; + } + destroy() { this.free.length = 0; this.active.clear(); } +} + +export class WasmViewCache { + constructor() { this.buffer = null; this.u8 = null; this.f64 = null; this.refreshes = 0; } + views(memory) { + if (this.buffer !== memory.buffer) { + this.buffer = memory.buffer; + this.u8 = new Uint8Array(memory.buffer); + this.f64 = new Float64Array(memory.buffer); + this.refreshes++; + } + return { u8: this.u8, f64: this.f64 }; + } + get refreshCount() { return this.refreshes; } +} + export default WasmMemoryPool; diff --git a/src/lib/wasmMemory.ts b/src/lib/wasmMemory.ts index a909761a..6e0f2cfb 100644 --- a/src/lib/wasmMemory.ts +++ b/src/lib/wasmMemory.ts @@ -215,5 +215,122 @@ export function createWasmMemoryPool(maxBytes = MAX_MEMORY_BYTES): WasmMemoryPoo return new WasmMemoryPool(maxBytes); } +// ── Telemetry-specific pooling ────────────────────────────────────────────── +// The ZK pool above is about large witness buffers. Telemetry has the opposite +// profile: thousands of small, fixed-size frames. Reusing a small set of +// staging buffers (and views over WASM linear memory) removes the per-frame +// allocation that showed up as GC pauses in telemetryBench.js. + +export interface TelemetryBufferLease { + buffer: ArrayBufferLike; + bytes: Uint8Array; + shared: boolean; +} + +export interface TelemetryBufferPoolStats { + frameBytes: number; + pooled: number; + active: number; + allocationCount: number; + recycledCount: number; + shared: boolean; +} + +export class TelemetryBufferPool { + private free: TelemetryBufferLease[] = []; + private active = new Set(); + private allocationCount = 0; + private recycledCount = 0; + private readonly shared: boolean; + + constructor( + readonly frameBytes = 4 * 1024, + private readonly maxBuffers = 64, + preferShared = true, + ) { + this.shared = + preferShared && + typeof SharedArrayBuffer !== 'undefined' && + typeof globalThis.crossOriginIsolated !== 'undefined' && + globalThis.crossOriginIsolated === true; + } + + acquire(): TelemetryBufferLease { + let lease = this.free.pop(); + if (lease) { + this.recycledCount++; + } else { + const buffer = this.shared + ? new SharedArrayBuffer(this.frameBytes) + : new ArrayBuffer(this.frameBytes); + lease = { + buffer, + bytes: new Uint8Array(buffer), + shared: this.shared, + }; + this.allocationCount++; + } + this.active.add(lease); + return lease; + } + + release(lease: TelemetryBufferLease | null | undefined): void { + if (!lease || !this.active.delete(lease)) return; + if (this.free.length >= this.maxBuffers) return; + // Zero the staging buffer so stale frame bytes cannot leak into the next + // parse (and so encrypted payload remnants do not linger). + try { + lease.bytes.fill(0); + } catch { + // SharedArrayBuffer views are always writable; ignore exotic engines. + } + this.free.push(lease); + } + + getStats(): TelemetryBufferPoolStats { + return { + frameBytes: this.frameBytes, + pooled: this.free.length, + active: this.active.size, + allocationCount: this.allocationCount, + recycledCount: this.recycledCount, + shared: this.shared, + }; + } + + destroy(): void { + this.free.length = 0; + this.active.clear(); + } +} + +/** + * Cached views over a WebAssembly.Memory. + * + * `memory.buffer` is detached and replaced whenever the module grows. Cache + * invalidation by buffer identity prevents the classic use-after-grow bug + * where a typed-array view points at the old, detached buffer. + */ +export class WasmViewCache { + private buffer: ArrayBufferLike | null = null; + private u8: Uint8Array | null = null; + private f64: Float64Array | null = null; + private refreshes = 0; + + views(memory: WebAssembly.Memory): { u8: Uint8Array; f64: Float64Array } { + if (this.buffer !== memory.buffer) { + this.buffer = memory.buffer; + this.u8 = new Uint8Array(memory.buffer); + this.f64 = new Float64Array(memory.buffer); + this.refreshes++; + } + return { u8: this.u8 as Uint8Array, f64: this.f64 as Float64Array }; + } + + get refreshCount(): number { + return this.refreshes; + } +} + // Convenience re-exports for workers that import this as ESM export default WasmMemoryPool; diff --git a/src/main.tsx b/src/main.tsx index be30c6d5..3f310448 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -34,7 +34,7 @@ const Admin = lazy(loadAdmin); const Kiosk = lazy(loadKiosk); // #608 spike: WebGPU spatial-clustering prototype + benchmark harness (ADR-008). const ClusterLab = lazy(() => import("./components/WebGPUMap")); -// Binary telemetry protocol spike: decode/GC benchmark harness (ADR-014). +// Telemetry lab: backpressure drop ledger & binary decode benchmark (ADR-014 / ISP-032). const TelemetryLab = lazy(() => import("./components/TelemetryLab")); function RouteFallback() { @@ -100,6 +100,7 @@ function render() { } /> } /> } /> + } /> } /> diff --git a/src/pages/Help.jsx b/src/pages/Help.jsx index ce9f3c26..fd21c115 100644 --- a/src/pages/Help.jsx +++ b/src/pages/Help.jsx @@ -315,19 +315,52 @@ function MapController({ center, zoom = 14 }) { return null; } -function distance(a, b) { +export function distance(a, b) { + if ( + !Number.isFinite(a?.[0]) || + !Number.isFinite(a?.[1]) || + !Number.isFinite(b?.[0]) || + !Number.isFinite(b?.[1]) + ) { + return null; + } const R = 6371; - const dLat = ((b[0] - a[0]) * Math.PI) / 180; - const dLng = ((b[1] - a[1]) * Math.PI) / 180; + const DEG2RAD = Math.PI / 180; + const dLat = (b[0] - a[0]) * DEG2RAD; + const dLng = (b[1] - a[1]) * DEG2RAD; const sinLat = Math.sin(dLat / 2); const sinLng = Math.sin(dLng / 2); + + if (!Number.isFinite(sinLat) || !Number.isFinite(sinLng)) { + return null; + } + const h = sinLat * sinLat + Math.cos((a[0] * Math.PI) / 180) * Math.cos((b[0] * Math.PI) / 180) * sinLng * sinLng; - return R * 2 * Math.atan2(Math.sqrt(h), Math.sqrt(1 - h)); + + if (!Number.isFinite(h)) { + return null; + } + + const result = R * 2 * Math.atan2(Math.sqrt(h), Math.sqrt(1 - h)); + return Number.isFinite(result) ? result : null; +} + +export function buildRouteFeature(from, to) { + return { + type: "Feature", + geometry: { + type: "LineString", + coordinates: [ + [from[1], from[0]], + [to[1], to[0]], + ], + }, + }; } function RouteLine({ id: routeId, from, to, color = "#7357FF" }) { @@ -336,16 +369,7 @@ function RouteLine({ id: routeId, from, to, color = "#7357FF" }) { Number.isFinite(id)); } catch { return []; } } -function saveMyRequestId(id) { +export function saveMyRequestId(id) { + if (!Number.isFinite(id)) return; const ids = loadMyRequestIds(); if (!ids.includes(id)) { ids.unshift(id); - localStorage.setItem(MY_REQUESTS_KEY, JSON.stringify(ids.slice(0, 20))); + try { + localStorage.setItem(MY_REQUESTS_KEY, JSON.stringify(ids.slice(0, 20))); + } catch { + } } } -function anonymizeLocation(location) { - if (!location) return location; +export function anonymizeLocation(location) { + if ( + !Array.isArray(location) || + !Number.isFinite(location[0]) || + !Number.isFinite(location[1]) + ) { + throw new Error("Unable to read your location. Try again."); + } return [ Math.round(location[0] * 100) / 100, Math.round(location[1] * 100) / 100, ]; } -function privateRequestLabel(id) { - return `Private request #${id || "pending"}`; +export function privateRequestLabel(id) { + const safeId = + typeof id === "string" || typeof id === "number" ? id : "pending"; + return `Private request #${safeId || "pending"}`; } -function txExplorerUrl(hash) { - if (!hash) return null; - return `https://stellar.expert/explorer/testnet/tx/${hash}`; +const TX_HASH_PATTERN = /^[0-9a-f]{16,64}$/i; + +export function txExplorerUrl(hash) { + if (typeof hash !== "string") return null; + const trimmed = hash.trim(); + if (!TX_HASH_PATTERN.test(trimmed)) return null; + return `https://stellar.expert/explorer/testnet/tx/${trimmed}`; } export function ExplorerLink({ label, hash }) { @@ -1397,6 +1442,243 @@ const ET_ICONS = { ), }; +// ── ZK state reducer ───────────────────────────────────────────────────────── + +export const LOG_RING_SIZE = 6; +export const LOG_MAX_MSG_LENGTH = 200; + +export const ZK_INITIAL = Object.freeze({ + status: "idle", + logs: [], + proof: null, + error: "", +}); + +export function zkReducer(state, action) { + switch (action.type) { + case "RESET": + return { ...ZK_INITIAL, logs: [] }; + + case "SET_STATUS": + if (state.status === action.payload) return state; + return { ...state, status: action.payload }; + + case "SET_ERROR": + if (state.error === action.payload) return state; + return { ...state, error: action.payload }; + + case "SET_PROOF": + return { ...state, proof: action.payload }; + + case "PATCH_PROOF": + if (!state.proof) return state; + return { ...state, proof: { ...state.proof, ...action.payload } }; + + case "PUSH_LOG": { + const raw = String(action.payload ?? ""); + const msg = + raw.length > LOG_MAX_MSG_LENGTH + ? raw.slice(0, LOG_MAX_MSG_LENGTH) + "…" + : raw; + const prev = state.logs; + if (prev.length > 0 && prev[prev.length - 1] === msg) return state; + const next = + prev.length < LOG_RING_SIZE + ? [...prev, msg] + : [...prev.slice(-(LOG_RING_SIZE - 1)), msg]; + return { ...state, logs: next }; + } + + default: + return state; + } +} + +export function sanitizeWalletAddress(raw) { + if (typeof raw !== "string") return ""; + const addr = raw.trim(); + if (!/^G[A-Z2-7]{55}$/.test(addr)) return ""; + return addr; +} + +const PHONE_REGEX = /^\+?[1-9]\d{1,14}$/; +const TELEGRAM_REGEX = /^@[A-Za-z0-9_]{5,32}$/; +const CONTACT_ALLOWLIST = [PHONE_REGEX, TELEGRAM_REGEX]; + +export function validateContact(value) { + const trimmed = (value || "").trim(); + if (!trimmed) return { valid: true, error: "" }; + if (trimmed.length > 40) return { valid: false, error: "Contact is too long (max 40 characters)." }; + const passesAny = CONTACT_ALLOWLIST.some((re) => re.test(trimmed)); + if (!passesAny) { + return { + valid: false, + error: "Use a phone number (+1234567890) or Telegram handle (@username).", + }; + } + return { valid: true, error: "" }; +} + +export function computeStep3Done(profile) { + if (!profile) return true; + return Boolean(profile.nickname || profile.contact || true); +} + +export function computeUserCharacter(selectedChar, profile) { + if (selectedChar) return selectedChar; + const nickname = profile?.nickname || "me"; + return pickChar("default", nickname); +} + +export const STATUS_CONFIG = { + Pending: { + label: "WAITING FOR RESPONDER", + color: "#a2a586", + bg: "rgba(162,165,134,0.12)", + msg: "Your pin is live. Waiting for someone nearby.", + }, + Enroute: { + label: "RESPONDER ON THE WAY", + color: "#7357FF", + bg: "rgba(115,87,255,0.12)", + msg: "Stay where you are. Help is coming.", + }, + Resolved: { + label: "RESOLVED", + color: "#3F8487", + bg: "rgba(63,132,135,0.12)", + msg: "This request has been resolved.", + }, + Cancelled: { + label: "CANCELLED", + color: "#a2a586", + bg: "rgba(162,165,134,0.12)", + msg: "Request cancelled.", + }, +}; + +export const DEFAULT_STATUS_INFO = { + label: "UNKNOWN STATUS", + color: "#a2a586", + bg: "rgba(162,165,134,0.12)", + msg: "Status is currently unavailable.", +}; + +export function getStatusConfig(requestStatus) { + if (typeof requestStatus !== "string") return DEFAULT_STATUS_INFO; + return STATUS_CONFIG[requestStatus] || DEFAULT_STATUS_INFO; +} + +export function checkIsGetMode(mode) { + if (typeof mode !== "string") return false; + return mode.trim().toLowerCase() === "get"; +} + +export function getAccentColor(isGetMode) { + return isGetMode ? "#FF7A6B" : "#7357FF"; +} + +export function createDebouncedSetter(setter, delay = 100) { + let latest = ""; + let timer = null; + + const debouncedSet = (value) => { + latest = value; + if (timer !== null) clearTimeout(timer); + timer = setTimeout(() => { + timer = null; + setter(latest); + }, delay); + }; + + const flush = () => { + if (timer !== null) { + clearTimeout(timer); + timer = null; + setter(latest); + } + }; + + const cancel = () => { + if (timer !== null) { + clearTimeout(timer); + timer = null; + } + }; + + return { debouncedSet, flush, cancel }; +} + +export function computeWalletStatus(address) { + const hasAddress = address !== ""; + const isValid = sanitizeWalletAddress(address) !== ""; + const isConnected = hasAddress && isValid; + const displayAddress = isConnected + ? `${address.slice(0, 8)}...${address.slice(-6)}` + : ""; + return { isConnected, displayAddress }; +} + +export function cancellationToken() { + let cancelled = false; + + function check() { + return !cancelled; + } + + return { + get active() { + return check(); + }, + cancel() { + cancelled = true; + }, + async wrap(promise) { + if (cancelled) { + promise.catch(() => {}); + return undefined; + } + try { + const result = await promise; + if (cancelled) return undefined; + return result; + } catch (err) { + if (cancelled) return undefined; + throw err; + } + }, + }; +} + +const RETRY_CLS = "hp-mobile-open"; +const RETRY_ID = "helphone-help-sidebar"; +const dlq = Object.create(null); + +export function safeToggleClass( + element, + isOpen, + id = RETRY_ID, + cls = RETRY_CLS, +) { + if (element) { + element.classList.toggle(cls, isOpen); + delete dlq[id]; + return true; + } + + if (!dlq[id]) { + const raf = requestAnimationFrame(() => { + delete dlq[id]; + const el = document.getElementById(id); + if (el) { + el.classList.toggle(cls, isOpen); + } + }); + dlq[id] = raf; + } + return false; +} + export default function Help() { const [mode, setMode] = useState("get"); @@ -3952,7 +4234,7 @@ export default function Help() { style={{ width: "100%", height: "100%" }} mapStyle={MAP_STYLES[mapStyleIndex].url} onClick={(e) => { - if (isGetMode && requestStatus === "idle") { + if (isGetMode && requestStatus === "idle" && e?.lngLat?.lat != null && e?.lngLat?.lng != null) { setLocation([e.lngLat.lat, e.lngLat.lng]); } }} diff --git a/src/utils/binaryParser.js b/src/utils/binaryParser.js index db094024..4450a162 100644 --- a/src/utils/binaryParser.js +++ b/src/utils/binaryParser.js @@ -1168,8 +1168,166 @@ function pick(table, format) { return fn; } -export function encodeFrame(format, frame) { - return pick(ENCODERS, format)(frame); +export const FRAME_MAGIC = 0x48454c50; +export const FRAME_VERSION = 1; +export const FRAME_BYTES = 40; + +export const FRAME_KIND = Object.freeze({ + LOCATION: 1, + BEACON: 2, + HEARTBEAT: 3, + DISPATCH_ACK: 4, + DISPATCH_UPDATE: 5, +}); + +export const KIND_NAME = Object.freeze({ + 1: 'location', + 2: 'beacon', + 3: 'heartbeat', + 4: 'dispatch_ack', + 5: 'dispatch_update', +}); + +export const PRIORITY_NAME = Object.freeze({ + 0: 'heartbeat', + 1: 'normal', + 2: 'critical', +}); + +export class BinaryParseError extends Error { + constructor(code, message) { + super(message); + this.name = 'BinaryParseError'; + this.code = code; + } +} + +function asBytes(input) { + if (input instanceof ArrayBuffer) return new Uint8Array(input); + if (ArrayBuffer.isView(input)) { + return new Uint8Array(input.buffer, input.byteOffset, input.byteLength); + } + throw new TypeError('parseFrame expects an ArrayBuffer or ArrayBufferView'); +} + +export function priorityFor(kind, explicitPriority, flags) { + if ((flags & 0x01) !== 0) return 'critical'; + if (kind === FRAME_KIND.DISPATCH_ACK || kind === FRAME_KIND.DISPATCH_UPDATE) { + return 'critical'; + } + if (kind === FRAME_KIND.HEARTBEAT) return 'heartbeat'; + return PRIORITY_NAME[explicitPriority] || 'normal'; +} + +/** + * Parse one frame. + * + * @param {ArrayBuffer|ArrayBufferView} input + * @param {number} [offset] + * @param {object} [out] Optional reusable target object (zero-allocation path). + */ +export function parseFrame(input, offset = 0, out) { + const bytes = asBytes(input); + if (offset < 0 || offset + FRAME_BYTES > bytes.byteLength) { + throw new BinaryParseError('truncated', `need ${FRAME_BYTES} bytes at offset ${offset}, got ${bytes.byteLength - offset}`); + } + + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const magic = view.getUint32(offset, true); + if (magic !== FRAME_MAGIC) { + throw new BinaryParseError('bad_magic', `bad magic 0x${magic.toString(16)}`); + } + + const version = view.getUint8(offset + 4); + if (version !== FRAME_VERSION) { + throw new BinaryParseError('bad_version', `unsupported frame version ${version}`); + } + + const kind = view.getUint8(offset + 5); + if (!KIND_NAME[kind]) { + throw new BinaryParseError('bad_kind', `unknown frame kind ${kind}`); + } + + const declaredPriority = view.getUint8(offset + 6); + if (!PRIORITY_NAME[declaredPriority]) { + throw new BinaryParseError('bad_priority', `unknown priority ${declaredPriority}`); + } + + const flags = view.getUint8(offset + 7); + const seq = view.getUint32(offset + 8, true); + const ts = view.getFloat64(offset + 12, true); + const lat = view.getFloat64(offset + 20, true); + const lng = view.getFloat64(offset + 28, true); + const accuracy = view.getFloat32(offset + 36, true); + + if (!Number.isFinite(ts) || !Number.isFinite(lat) || !Number.isFinite(lng) || !Number.isFinite(accuracy)) { + throw new BinaryParseError('non_finite', 'frame contains non-finite coordinates'); + } + if (lat < -90 || lat > 90) { + throw new BinaryParseError('bad_latitude', `latitude out of range: ${lat}`); + } + if (lng < -180 || lng > 180) { + throw new BinaryParseError('bad_longitude', `longitude out of range: ${lng}`); + } + + const priority = priorityFor(kind, declaredPriority, flags); + const target = out || {}; + target.magic = magic; + target.version = version; + target.kind = kind; + target.kindName = KIND_NAME[kind]; + target.priority = priority; + target.flags = flags; + target.seq = seq; + target.ts = ts; + target.lat = lat; + target.lng = lng; + target.accuracy = accuracy; + target.bytes = FRAME_BYTES; + return target; +} + +/** Parse `count` contiguous frames. */ +export function parseBatch(input, count, options = {}) { + const bytes = asBytes(input); + const total = count ?? Math.floor(bytes.byteLength / FRAME_BYTES); + const out = []; + for (let i = 0; i < total; i++) { + try { + out.push(parseFrame(bytes, i * FRAME_BYTES, options.reuse ? {} : undefined)); + } catch (error) { + if (options.stopOnError !== false) throw error; + out.push({ error: error.code || 'parse_error', index: i }); + } + } + return out; +} + +/** True when `kind` is dispatch-critical even without the flag bit. */ +export function isCriticalKind(kind) { + return kind === FRAME_KIND.DISPATCH_ACK || kind === FRAME_KIND.DISPATCH_UPDATE; +} + +export function encodeFrame(formatOrFrame, maybeFrame) { + if (typeof formatOrFrame === "string") { + return pick(ENCODERS, formatOrFrame)(maybeFrame); + } + const frame = formatOrFrame || {}; + const buffer = new ArrayBuffer(FRAME_BYTES); + const view = new DataView(buffer); + const kind = frame.kind ?? FRAME_KIND.LOCATION; + const priority = frame.priority ?? (kind === FRAME_KIND.HEARTBEAT ? 0 : kind >= FRAME_KIND.DISPATCH_ACK ? 2 : 1); + view.setUint32(0, FRAME_MAGIC, true); + view.setUint8(4, frame.version ?? FRAME_VERSION); + view.setUint8(5, typeof kind === 'number' ? kind : 1); + view.setUint8(6, typeof priority === 'number' ? priority : (priority === 'critical' ? 2 : priority === 'heartbeat' ? 0 : 1)); + view.setUint8(7, frame.flags ?? 0); + view.setUint32(8, frame.seq ?? 0, true); + view.setFloat64(12, frame.ts ?? Date.now(), true); + view.setFloat64(20, frame.lat ?? 0, true); + view.setFloat64(28, frame.lng ?? 0, true); + view.setFloat32(36, frame.accuracy ?? 0, true); + return buffer; } /** Fully materialises a frame into plain objects (same shape for every format). */ @@ -1181,3 +1339,5 @@ export function decodeFrame(format, bytes) { export function readPositions(format, bytes, out) { return pick(POSITION_READERS, format)(bytes, out); } + +export default { parseFrame, parseBatch, encodeFrame, FRAME_BYTES, FRAME_MAGIC }; diff --git a/src/utils/binaryParserWasm.js b/src/utils/binaryParserWasm.js index 2561e846..c792e621 100644 --- a/src/utils/binaryParserWasm.js +++ b/src/utils/binaryParserWasm.js @@ -1,24 +1,21 @@ /** - * JS wrapper for the WASM render-path reader (spike, ADR-014). - * - * The module (src/wasm/telemetry_reader.wasm, built from - * src/wasm/telemetry_reader/) exposes the same operation as - * readPositions("flatbuffers" | "capnp", ...) in binaryParser.js. The - * difference is where the bytes live: WASM can only read its own linear - * memory, so every frame is first copied in. A network ArrayBuffer cannot - * be handed to WASM without that copy (wasm memory cannot adopt or be - * transferred an existing buffer), and the ADR measures what it costs. - * - * Output arrays are views over WASM memory, so JS reads them without a copy. - * They are re-created whenever memory grows, so re-read `reader.positions` - * after each call instead of holding on to the arrays. - * - * const reader = await createWasmPositionReader(wasmBytesOrResponse); - * const n = reader.readPositions("flatbuffers", frameBytes); - * reader.positions.latE6[0]; + * JS wrapper for the WASM render-path reader (spike, ADR-014) + * and JS ↔ Rust/WASM telemetry parsing bridge (ISP-032). */ import { instrumentWasmMemory, recordParserAlloc } from "../lib/heapForensics.ts"; +import { + FRAME_BYTES, + FRAME_VERSION, + KIND_NAME, + PRIORITY_NAME, + BinaryParseError, + parseFrame as jsParseFrame, + parseBatch as jsParseBatch, + priorityFor, +} from './binaryParser.js'; +import { WasmViewCache } from '../lib/wasmMemory.js'; +import { loadVerifiedWasm, WasmIntegrityError } from './wasmLoader.js'; const PAGE = 65536; const FUNCTIONS = { flatbuffers: "fb_positions", capnp: "capnp_positions" }; @@ -47,7 +44,7 @@ export async function createWasmPositionReader(source, { initialCapacity = 1024, const ex = instance.exports; // Count linear-memory growth: WASM memory is invisible to heap snapshots (#793). const memory = instrumentWasmMemory(ex.memory); - const heapBase = (ex.__heap_base.value + 7) & ~7; + const heapBase = ex.__heap_base ? ((ex.__heap_base.value + 7) & ~7) : 1024 * 64; let outCap = 0; let outPtr = heapBase; @@ -101,3 +98,300 @@ export async function createWasmPositionReader(source, { initialCapacity = 1024, return { readPositions, positions, memory }; } + +// =========================================================================== +// ISP-032 TelemetryWasmParser +// =========================================================================== + +const DEFAULT_WASM_URL = new URL('../wasm/telemetry_reader.wasm', import.meta.url); +const DEFAULT_SHA_URL = new URL('../wasm/telemetry_reader.wasm.sha256', import.meta.url); + +const STATUS_TO_CODE = { + [-1]: 'truncated', + [-2]: 'bad_magic', + [-3]: 'bad_version', + [-4]: 'bad_kind', + [-5]: 'bad_priority', + [-6]: 'non_finite', + [-7]: 'bad_coordinates', +}; + +function toBytes(input) { + if (input instanceof ArrayBuffer) return new Uint8Array(input); + if (ArrayBuffer.isView(input)) return new Uint8Array(input.buffer, input.byteOffset, input.byteLength); + throw new TypeError('telemetry parser expects an ArrayBuffer or ArrayBufferView'); +} + +async function resolveSha({ shaUrl, expectedSha256, fetchImpl }) { + if (expectedSha256) return String(expectedSha256).trim().toLowerCase(); + if (!shaUrl) return undefined; + const response = await fetchImpl(shaUrl, { cache: 'force-cache' }); + if (!response.ok) return undefined; + const text = (await response.text()).trim().split(/\s+/)[0]; + return /^[0-9a-f]{64}$/i.test(text) ? text.toLowerCase() : undefined; +} + +export class TelemetryWasmParser { + constructor(options = {}) { + this.options = options; + this.mode = 'js'; + this.lastError = null; + this.viewCache = new WasmViewCache(); + this.framePool = []; + this.reusedBatch = []; + this.ready = false; + this.initPromise = null; + this.stats = { + parsed: 0, + wasmParsed: 0, + fallbackParsed: 0, + errors: 0, + wasmBatchCalls: 0, + allocationCount: 0, + recycledCount: 0, + objectsCreated: 0, + viewRefreshes: 0, + }; + } + + /** Start loading the module without blocking the caller. */ + init() { + if (this.initPromise) return this.initPromise; + this.initPromise = this.load().catch((error) => { + this.lastError = error; + this.mode = 'js'; + return false; + }); + return this.initPromise; + } + + async load() { + const { + wasmUrl = DEFAULT_WASM_URL, + shaUrl = DEFAULT_SHA_URL, + wasmBytes, + expectedSha256, + fetchImpl = globalThis.fetch, + timeoutMs = 2_000, + preferWasm = true, + } = this.options; + + if (!preferWasm || typeof WebAssembly === 'undefined') return false; + + let sha; + try { + sha = await resolveSha({ shaUrl, expectedSha256, fetchImpl }); + } catch { + sha = undefined; + } + + const loaded = await loadVerifiedWasm({ + url: wasmBytes ? undefined : wasmUrl, + bytes: wasmBytes, + expectedSha256: sha, + fetchImpl, + timeoutMs, + imports: {}, + }); + + const exports = loaded.exports; + if ( + typeof exports.parse_batch !== 'function' || + typeof exports.input_ptr !== 'function' || + typeof exports.input_cap !== 'function' || + typeof exports.output_ptr !== 'function' || + !(exports.memory instanceof WebAssembly.Memory) + ) { + throw new WasmIntegrityError('telemetry_reader.wasm is missing required exports'); + } + + this.exports = exports; + this.memory = exports.memory; + this.inputPtr = exports.input_ptr() >>> 0; + this.outputPtr = exports.output_ptr() >>> 0; + this.inputCap = exports.input_cap() >>> 0; + this.outputCap = typeof exports.output_cap === 'function' + ? exports.output_cap() >>> 0 + : Math.floor(this.inputCap / FRAME_BYTES); + this.mode = 'wasm'; + this.ready = true; + this.lastError = null; + return true; + } + + /** + * Parse a single frame, using WASM when ready. `out` is reused when + * supplied, which is what the main-thread ring consumer does. + */ + parseFrame(input, offset = 0, out) { + this.stats.parsed++; + if (!this.ready) { + const frame = jsParseFrame(input, offset, out); + this.stats.fallbackParsed++; + return frame; + } + + const bytes = toBytes(input); + if (offset < 0 || offset + FRAME_BYTES > bytes.byteLength) { + this.stats.errors++; + throw new BinaryParseError('truncated', `need ${FRAME_BYTES} bytes at offset ${offset}`); + } + + const { u8, f64 } = this.viewCache.views(this.memory); + u8.set(bytes.subarray(offset, offset + FRAME_BYTES), this.inputPtr); + + let status = 0; + try { + status = this.exports.parse_frame(FRAME_BYTES) | 0; + } catch (error) { + this.stats.errors++; + this.ready = false; + this.mode = 'js'; + this.lastError = error; + return jsParseFrame(input, offset, out); + } + + if (status !== 0) { + this.stats.errors++; + const code = STATUS_TO_CODE[status] || 'parse_error'; + throw new BinaryParseError(code, `wasm parse_frame failed (status ${status})`); + } + + const base = this.outputPtr / 8; + const seq = f64[base + 0]; + const ts = f64[base + 1]; + const lat = f64[base + 2]; + const lng = f64[base + 3]; + const accuracy = f64[base + 4]; + const kind = f64[base + 5] | 0; + const declaredPriority = f64[base + 6] | 0; + const flags = f64[base + 7] | 0; + + if (!KIND_NAME[kind] || !PRIORITY_NAME[declaredPriority]) { + this.stats.errors++; + throw new BinaryParseError('wasm_output', 'wasm parser returned an unknown kind/priority'); + } + + const frame = out || {}; + frame.magic = 0x48454c50; + frame.version = FRAME_VERSION; + frame.kind = kind; + frame.kindName = KIND_NAME[kind]; + frame.priority = priorityFor(kind, declaredPriority, flags); + frame.flags = flags; + frame.seq = seq; + frame.ts = ts; + frame.lat = lat; + frame.lng = lng; + frame.accuracy = accuracy; + frame.bytes = FRAME_BYTES; + this.stats.wasmParsed++; + this.stats.viewRefreshes = this.viewCache.refreshCount; + return frame; + } + + parseBatch(input, count, options = {}) { + if (!this.ready) { + this.stats.fallbackParsed += count ?? 0; + return jsParseBatch(input, count, options); + } + const bytes = toBytes(input); + const total = count ?? Math.floor(bytes.byteLength / FRAME_BYTES); + const reuse = options.reuse === true; + const out = options.reuseArray === true ? this.reusedBatch : []; + if (options.reuseArray === true) out.length = 0; + const chunkFrames = Math.max(1, Math.min(this.outputCap, Math.floor(this.inputCap / FRAME_BYTES))); + + for (let start = 0; start < total; start += chunkFrames) { + const frames = Math.min(chunkFrames, total - start); + const byteStart = start * FRAME_BYTES; + const byteLength = frames * FRAME_BYTES; + const { u8, f64 } = this.viewCache.views(this.memory); + u8.set(bytes.subarray(byteStart, byteStart + byteLength), this.inputPtr); + + let parsed; + try { + parsed = this.exports.parse_batch(byteLength, frames) | 0; + this.stats.wasmBatchCalls++; + } catch (error) { + this.stats.errors++; + this.ready = false; + this.mode = 'js'; + this.lastError = error; + this.stats.fallbackParsed += total - start; + return out.concat(jsParseBatch(bytes.subarray(byteStart), total - start, options)); + } + + if (parsed !== frames) { + this.stats.errors++; + this.stats.fallbackParsed += frames; + const fallback = jsParseBatch(bytes.subarray(byteStart, byteStart + byteLength), frames, options); + if (options.stopOnError !== false) { + const bad = fallback.find((frame) => frame?.error); + if (bad) throw new BinaryParseError(bad.error, `invalid frame at index ${start + bad.index}`); + } + out.push(...fallback); + continue; + } + + const base = this.outputPtr / 8; + for (let i = 0; i < frames; i++) { + const offset = base + i * 8; + const kind = f64[offset + 5] | 0; + const declaredPriority = f64[offset + 6] | 0; + const flags = f64[offset + 7] | 0; + let frame; + if (reuse) { + const poolIndex = start + i; + frame = this.framePool[poolIndex]; + if (frame) this.stats.recycledCount++; + else { + frame = this.framePool[poolIndex] = {}; + this.stats.objectsCreated++; + } + } else { + frame = {}; + this.stats.objectsCreated++; + } + frame.magic = 0x48454c50; + frame.version = FRAME_VERSION; + frame.kind = kind; + frame.kindName = KIND_NAME[kind]; + frame.priority = priorityFor(kind, declaredPriority, flags); + frame.flags = flags; + frame.seq = f64[offset + 0]; + frame.ts = f64[offset + 1]; + frame.lat = f64[offset + 2]; + frame.lng = f64[offset + 3]; + frame.accuracy = f64[offset + 4]; + frame.bytes = FRAME_BYTES; + out.push(frame); + } + this.stats.wasmParsed += parsed; + } + return out; + } + + getStats() { + return { + ...this.stats, + mode: this.mode, + ready: this.ready, + lastError: this.lastError ? String(this.lastError.message || this.lastError) : null, + }; + } +} + +let sharedParser = null; + +/** Lazily shared parser for the main thread. */ +export function getTelemetryParser(options) { + if (!sharedParser || options) sharedParser = new TelemetryWasmParser(options); + return sharedParser; +} + +export function resetTelemetryParser() { + sharedParser = null; +} + +export default TelemetryWasmParser; diff --git a/src/utils/telemetryBench.js b/src/utils/telemetryBench.js index c029ecc8..4fa9d192 100644 --- a/src/utils/telemetryBench.js +++ b/src/utils/telemetryBench.js @@ -1,15 +1,6 @@ /** - * In-browser telemetry decode benchmark (spike, ADR-014). - * - * Streams map-object frames at `rate` objects per second into a - * requestAnimationFrame loop. Each rAF callback decodes every frame that - * is due and plots all objects into a canvas, as the live map would. The - * result reports rAF frame intervals (dropped / long frames), per-frame - * decode cost and JS heap growth. - * - * Used by the /lab/telemetry-bench route (src/components/TelemetryLab.jsx) - * and by the headless driver scripts/spikes/binary_browser_benchmark.js, - * which adds GC pause data from a Chrome trace. + * In-browser telemetry decode benchmark (spike, ADR-014) + * and CLI telemetry parser microbenchmark (ISP-032). */ import { @@ -23,8 +14,9 @@ import { openFlatBuffersFrame, readPositions, stepMapObjects, + parseBatch, } from "./binaryParser.js"; -import { createWasmPositionReader } from "./binaryParserWasm.js"; +import { createWasmPositionReader, TelemetryWasmParser } from "./binaryParserWasm.js"; export const BENCH_SCENARIOS = [ "json/materialize", @@ -38,7 +30,7 @@ export const BENCH_SCENARIOS = [ ]; const FRAME_MS = 1000 / 60; -const now = () => performance.now(); +const now = () => (typeof performance !== "undefined" ? performance.now() : Date.now()); function summary(samples) { if (!samples.length) return null; @@ -193,16 +185,12 @@ export async function runTelemetryBench({ nsPerObject: +((decode.reduce((a, b) => a + b, 0) * 1e6) / (delivered * frameSize)).toFixed(1), heapBeforeMB: heapBefore === null ? null : +(heapBefore / 1048576).toFixed(2), heapAfterMB: performance.memory ? +(performance.memory.usedJSHeapSize / 1048576).toFixed(2) : null, - // Without cross-origin isolation browsers coarsen performance.now() to - // ~100 us, so per-frame decode times for small frames are quantised. crossOriginIsolated: globalThis.crossOriginIsolated === true, }; } /** * Burst-realistic load profile for anomaly threshold tuning (#773). - * Steady-state benches understate emergency traffic; this returns a - * sequence of rates that spike then settle, matching field dispatch bursts. */ export function burstLoadProfile({ baselineRate = 200, @@ -212,9 +200,168 @@ export function burstLoadProfile({ const points = []; for (let t = 0; t < seconds; t++) { let rate = baselineRate; - if (t >= 8 && t < 14) rate = peakRate; // six-second flood + if (t >= 8 && t < 14) rate = peakRate; else if (t >= 14 && t < 18) rate = Math.round(peakRate * 0.35); points.push({ t, rate }); } return Object.freeze(points); } + +// =========================================================================== +// CLI benchmark entry point +// =========================================================================== + +async function runCliBenchmark() { + const { readFileSync, writeFileSync, mkdirSync } = await import('node:fs'); + const { dirname, resolve } = await import('node:path'); + const { fileURLToPath } = await import('node:url'); + + const __filename = fileURLToPath(import.meta.url); + const __dirname = dirname(__filename); + const ROOT = resolve(__dirname, '../..'); + + function arg(name, fallback) { + const index = process.argv.indexOf(`--${name}`); + if (index === -1) return fallback; + return process.argv[index + 1] ?? fallback; + } + + function makeFrames(count) { + const buffers = []; + for (let i = 0; i < count; i++) { + buffers.push( + encodeFrame({ + kind: i % 10 === 0 ? 4 : i % 3 === 0 ? 3 : 1, + priority: i % 10 === 0 ? 2 : i % 3 === 0 ? 0 : 1, + seq: i, + ts: 1_700_000_000_000 + i, + lat: 6.5 + (i % 100) * 1e-4, + lng: 3.3 + (i % 100) * 1e-4, + accuracy: 3 + (i % 20), + }), + ); + } + return buffers; + } + + function heap() { + if (typeof global.gc === 'function') global.gc(); + return process.memoryUsage().heapUsed; + } + + function time(label, fn, iterations, frameCount) { + const started = process.hrtime.bigint(); + for (let i = 0; i < iterations; i++) fn(); + const elapsedMs = Number(process.hrtime.bigint() - started) / 1e6; + return { label, iterations, elapsedMs, perFrameUs: (elapsedMs * 1000) / (iterations * frameCount) }; + } + + const FRAME_COUNT = Number(arg('frames', 10_000)); + const ROUNDS = Number(arg('rounds', 3)); + const outPath = arg('out', ''); + + const frames = makeFrames(FRAME_COUNT); + const batch = new Uint8Array(FRAME_COUNT * 40); + frames.forEach((buffer, i) => batch.set(new Uint8Array(buffer), i * 40)); + + parseBatch(batch, 512); + + const wasm = new TelemetryWasmParser({ + wasmBytes: (() => { + try { + return readFileSync(resolve(ROOT, 'src/wasm/telemetry_reader.wasm')); + } catch { + return undefined; + } + })(), + expectedSha256: (() => { + try { + return readFileSync(resolve(ROOT, 'src/wasm/telemetry_reader.wasm.sha256'), 'utf8').trim(); + } catch { + return undefined; + } + })(), + shaUrl: undefined, + preferWasm: true, + timeoutMs: 5_000, + }); + + const ready = await wasm.init(); + if (!ready && process.argv.includes('--require-wasm')) { + console.error('telemetryBench: WASM parser unavailable'); + process.exit(1); + } + + if (ready) { + wasm.parseBatch(batch, FRAME_COUNT, { reuse: true, reuseArray: true }); + wasm.stats.objectsCreated = 0; + wasm.stats.recycledCount = 0; + wasm.stats.wasmBatchCalls = 0; + wasm.stats.wasmParsed = 0; + } + + const jsBefore = heap(); + const js = time('js', () => parseBatch(batch, FRAME_COUNT), ROUNDS, FRAME_COUNT); + const jsAfter = heap(); + + let wasmResult = null; + let wasmBefore = 0; + let wasmAfter = 0; + if (ready) { + wasmBefore = heap(); + wasmResult = time('wasm', () => wasm.parseBatch(batch, FRAME_COUNT, { reuse: true, reuseArray: true }), ROUNDS, FRAME_COUNT); + wasmAfter = heap(); + } + + const report = { + generatedAt: new Date().toISOString(), + frames: FRAME_COUNT, + rounds: ROUNDS, + wasm: { + available: Boolean(ready), + mode: wasm.mode, + stats: wasm.getStats(), + elapsedMs: wasmResult?.elapsedMs ?? null, + perFrameUs: wasmResult?.perFrameUs ?? null, + heapGrowthBytes: ready ? wasmAfter - wasmBefore : null, + heapGrowthPerFrameBytes: ready ? (wasmAfter - wasmBefore) / FRAME_COUNT : null, + objectsCreatedPerFrame: ready + ? wasm.getStats().objectsCreated / (FRAME_COUNT * ROUNDS) + : null, + pooledFrames: ready ? wasm.framePool.length : 0, + }, + js: { + elapsedMs: js.elapsedMs, + perFrameUs: js.perFrameUs, + heapGrowthBytes: jsAfter - jsBefore, + heapGrowthPerFrameBytes: (jsAfter - jsBefore) / FRAME_COUNT, + objectsCreatedPerFrame: 1, + }, + speedup: wasmResult ? js.elapsedMs / wasmResult.elapsedMs : null, + allocationReduction: ready + ? (() => { + const objects = wasm.getStats().objectsCreated / (FRAME_COUNT * ROUNDS); + return objects === 0 ? null : 1 / objects; + })() + : null, + note: ready + ? "Steady-state WASM path reuses pooled frame objects; allocationReduction is null when per-frame object creation is zero." + : "WASM module unavailable; JS fallback measured only.", + }; + + const text = JSON.stringify(report, null, 2); + if (outPath) { + const target = resolve(ROOT, outPath); + mkdirSync(dirname(target), { recursive: true }); + writeFileSync(target, `${text}\n`); + console.log(`telemetryBench: wrote ${target}`); + } + console.log(text); +} + +if (typeof process !== 'undefined' && process.argv?.[1] && process.argv[1].endsWith('telemetryBench.js')) { + runCliBenchmark().catch((err) => { + console.error(err); + process.exit(1); + }); +} diff --git a/src/utils/wasmLoader.js b/src/utils/wasmLoader.js index 96c878e9..ba95dea7 100644 --- a/src/utils/wasmLoader.js +++ b/src/utils/wasmLoader.js @@ -1,19 +1,5 @@ -// Spike #602 — isolated WebAssembly sandboxes for encrypted audio vs ZK work. -// -// Each trust domain gets its own instance of src/wasm/memory_sandbox.wasm and -// therefore its own linear memory: a bug in the audio path can only read or -// write audio memory. Moving data between domains is the host's job, done in -// one of two ways: -// -// "multi-memory" a ~70-byte bridge module, assembled below, imports both -// sandboxes' memories and runs `memory.copy` between them. -// The copy stays inside the engine; no JS view is touched. -// "js-copy" fallback for engines without multi-memory: -// Uint8Array.prototype.set between the two memory buffers. -// -// Both strategies give the same isolation. Only this loader holds references -// to both memories; the sandboxes never do. Multi-memory is purely a -// performance path. See docs/adr/ADR-002-wasm-memory-isolation.md. +// Spike #602 — isolated WebAssembly sandboxes for encrypted audio vs ZK work +// and Verified WASM loader (ISP-032). const WASM_MAGIC = [0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00]; @@ -169,20 +155,17 @@ async function instantiateSandbox(module, label, WA) { return { label, memory: x.memory, - /** Allocate `len` bytes; throws if the sandbox cannot grow. */ alloc(len) { const ptr = x.sandbox_alloc(len); if (ptr === 0 && len !== 0) throw new RangeError(`${label}: out of memory allocating ${len} bytes`); return ptr; }, - /** Copy host bytes into this sandbox; returns the pointer. */ write(bytes) { const ptr = this.alloc(bytes.byteLength); new Uint8Array(x.memory.buffer, ptr, bytes.byteLength).set(bytes); return ptr; }, - /** Copy bytes out of this sandbox (a detached copy, not a live view). */ read(ptr, len) { return new Uint8Array(x.memory.buffer, ptr, len).slice(); }, @@ -193,15 +176,6 @@ async function instantiateSandbox(module, label, WA) { }; } -/** - * Creates the audio and ZK sandboxes plus the cross-domain transfer function. - * - * @param {object} [opts] - * @param {ArrayBuffer|ArrayBufferView|WebAssembly.Module|Response|string|URL} [opts.source] - * memory_sandbox.wasm bytes/module/URL; defaults to the bundled asset. - * @param {"auto"|"multi-memory"|"js-copy"} [opts.strategy] force a transfer path (benchmarks/tests). - * @param {typeof WebAssembly} [opts.WebAssembly] - */ export async function createIsolatedSandboxes(opts = {}) { const WA = opts.WebAssembly ?? globalThis.WebAssembly; const features = detectFeatures(WA); @@ -229,10 +203,6 @@ export async function createIsolatedSandboxes(opts = {}) { bridge = (inst.instance ?? inst).exports; } - /** - * Copy `len` bytes at `srcPtr` in `from` into a fresh allocation in `to`. - * @returns {number} destination pointer in `to` - */ function transfer(from, to, srcPtr, len) { if (from === to) throw new Error("wasmLoader: transfer requires two different sandboxes"); @@ -241,7 +211,6 @@ export async function createIsolatedSandboxes(opts = {}) { if (from === audio) bridge.a_to_b(dstPtr, srcPtr, len); else bridge.b_to_a(dstPtr, srcPtr, len); } else { - // Re-read .buffer every time: alloc() may have grown (and detached) it. new Uint8Array(to.memory.buffer, dstPtr, len).set( new Uint8Array(from.memory.buffer, srcPtr, len), ); @@ -251,3 +220,96 @@ export async function createIsolatedSandboxes(opts = {}) { return { audio, zk, transfer, strategy, features }; } + +// =========================================================================== +// ISP-032 Verified WASM Loader +// =========================================================================== + +const cache = new Map(); + +export class WasmIntegrityError extends Error { + constructor(message) { + super(message); + this.name = 'WasmIntegrityError'; + } +} + +export class WasmLoadTimeoutError extends Error { + constructor(message) { + super(message); + this.name = 'WasmLoadTimeoutError'; + } +} + +function toBytes(input) { + if (input instanceof ArrayBuffer) return new Uint8Array(input); + if (ArrayBuffer.isView(input)) return new Uint8Array(input.buffer, input.byteOffset, input.byteLength); + if (input instanceof Uint8Array) return input; + throw new TypeError('sha256Hex expects bytes'); +} + +export async function sha256Hex(input) { + const bytes = toBytes(input); + const subtle = globalThis.crypto?.subtle; + if (subtle) { + const digest = await subtle.digest('SHA-256', bytes); + return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, '0')).join(''); + } + const { createHash } = await import('node:crypto'); + return createHash('sha256').update(bytes).digest('hex'); +} + +async function fetchBytes(url, fetchImpl, timeoutMs) { + const controller = typeof AbortController !== 'undefined' ? new AbortController() : null; + const timer = controller ? setTimeout(() => controller.abort(), timeoutMs) : null; + try { + const response = await fetchImpl(url, { signal: controller?.signal, cache: 'force-cache' }); + if (!response.ok) throw new Error(`fetch ${url} failed: HTTP ${response.status}`); + return new Uint8Array(await response.arrayBuffer()); + } catch (error) { + if (error?.name === 'AbortError') throw new WasmLoadTimeoutError(`timed out loading ${url}`); + throw error; + } finally { + if (timer) clearTimeout(timer); + } +} + +export function loadVerifiedWasm({ + url, + bytes, + expectedSha256, + fetchImpl = globalThis.fetch, + timeoutMs = 2_000, + cache: useCache = true, + imports = {}, +} = {}) { + const key = url || (bytes ? `inline:${bytes.byteLength}` : ''); + if (!key) return Promise.reject(new TypeError('loadVerifiedWasm requires url or bytes')); + if (useCache && cache.has(key)) return cache.get(key); + + const promise = (async () => { + const raw = bytes ? toBytes(bytes) : await fetchBytes(url, fetchImpl, timeoutMs); + if (expectedSha256) { + const actual = await sha256Hex(raw); + if (actual !== String(expectedSha256).trim().toLowerCase()) { + throw new WasmIntegrityError( + `WASM integrity mismatch for ${key}: expected ${expectedSha256}, got ${actual}`, + ); + } + } + const { instance, module } = await WebAssembly.instantiate(raw, imports); + return { instance, module, exports: instance.exports, bytes: raw, verified: Boolean(expectedSha256) }; + })(); + + if (useCache) { + cache.set(key, promise); + promise.catch(() => cache.delete(key)); + } + return promise; +} + +export function resetWasmLoaderCache() { + cache.clear(); +} + +export default { loadVerifiedWasm, sha256Hex, resetWasmLoaderCache }; diff --git a/src/wasm/telemetry_reader.wasm b/src/wasm/telemetry_reader.wasm old mode 100644 new mode 100755 index 23f9d52c..d81c170c Binary files a/src/wasm/telemetry_reader.wasm and b/src/wasm/telemetry_reader.wasm differ diff --git a/src/wasm/telemetry_reader.wasm.sha256 b/src/wasm/telemetry_reader.wasm.sha256 index 90ee7ccb..62930710 100644 --- a/src/wasm/telemetry_reader.wasm.sha256 +++ b/src/wasm/telemetry_reader.wasm.sha256 @@ -1 +1 @@ -5d24b7720bb68e24b9b187ed3ed36432d9413d7def041f43d83adf46e2d3ea9c +d1865c2c79337b5d2de47d42352b9873d7221ba887af8f28943b5aa2dc56f756 diff --git a/src/wasm/telemetry_reader/build.sh b/src/wasm/telemetry_reader/build.sh index b4c64e49..200e6ece 100755 --- a/src/wasm/telemetry_reader/build.sh +++ b/src/wasm/telemetry_reader/build.sh @@ -1,10 +1,39 @@ #!/usr/bin/env bash -# Rebuilds src/wasm/telemetry_reader.wasm and its SHA-256 sidecar -# (spike, ADR-014). Requires the wasm32-unknown-unknown Rust target. +# Deterministic build for the telemetry frame reader. +# +# Usage: +# bash src/wasm/telemetry_reader/build.sh # build + record SHA-256 +# bash src/wasm/telemetry_reader/build.sh --check # verify committed artifact set -euo pipefail -CRATE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -OUT="$CRATE/../telemetry_reader.wasm" -cargo build --manifest-path "$CRATE/Cargo.toml" --release --target wasm32-unknown-unknown -cp "$CRATE/target/wasm32-unknown-unknown/release/telemetry_reader.wasm" "$OUT" -sha256sum "$OUT" | awk '{print $1}' > "$OUT.sha256" -echo "wrote $OUT ($(wc -c < "$OUT") bytes, sha256 $(cat "$OUT.sha256"))" + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$HERE/../../.." && pwd)" +ARTIFACT="$ROOT/src/wasm/telemetry_reader.wasm" +SIDECAR="$ROOT/src/wasm/telemetry_reader.wasm.sha256" +TARGET="wasm32-unknown-unknown" + +if [[ "${1:-}" == "--check" ]]; then + if [[ ! -f "$ARTIFACT" || ! -f "$SIDECAR" ]]; then + echo "telemetry-wasm: FAIL (missing artifact or sidecar)" >&2 + exit 1 + fi + expected="$(tr -d ' \t\r\n' < "$SIDECAR")" + actual="$(sha256sum "$ARTIFACT" | awk '{print $1}')" + if [[ "$expected" != "$actual" ]]; then + echo "telemetry-wasm: FAIL (SHA-256 mismatch)" >&2 + echo " expected: $expected" >&2 + echo " actual: $actual" >&2 + exit 1 + fi + echo "telemetry-wasm: SHA-256 OK ($actual)" + exit 0 +fi + +cd "$HERE" +cargo build --target "$TARGET" --release +mkdir -p "$(dirname "$ARTIFACT")" +cp "target/$TARGET/release/telemetry_reader.wasm" "$ARTIFACT" +sha256sum "$ARTIFACT" | awk '{print $1}' > "$SIDECAR" + +echo "telemetry-wasm: built $ARTIFACT" +echo "telemetry-wasm: sha256 $(cat "$SIDECAR")" diff --git a/src/wasm/telemetry_reader/src/lib.rs b/src/wasm/telemetry_reader/src/lib.rs index 9d90830d..3660ac02 100644 --- a/src/wasm/telemetry_reader/src/lib.rs +++ b/src/wasm/telemetry_reader/src/lib.rs @@ -1,22 +1,9 @@ -//! Render-path telemetry reader compiled to WebAssembly (spike, ADR-014). -//! -//! Mirrors `readFlatBuffersPositions` / `readCapnpPositions` in -//! src/utils/binaryParser.js: given a frame that the host has copied into -//! linear memory, it writes id, kind, status, lat, lng and heading into a -//! struct-of-arrays output region. The JS wrapper is -//! src/utils/binaryParserWasm.js. -//! -//! Output layout for capacity `cap` (a multiple of 4), starting at `out`: -//! id: u32[cap] | lat_e6: i32[cap] | lng_e6: i32[cap] | -//! heading_cdeg: u16[cap] | kind: u8[cap] | status: u8[cap] -//! -//! Return value: object count (>= 0), or -//! -1 malformed input (out-of-bounds offset, wrong pointer kind, ...) -//! -2 `cap` too small; `needed()` reports the required capacity. -//! Every read is bounds-checked, so a hostile frame cannot read outside -//! the input slice. +//! Render-path telemetry reader compiled to WebAssembly. +//! Combines FlatBuffers / Cap'n Proto render-path readers (spike, ADR-014) +//! with the zero-allocation telemetry frame reader (ISP-032). #![no_std] +#![allow(static_mut_refs)] use core::panic::PanicInfo; @@ -25,6 +12,159 @@ fn panic(_: &PanicInfo) -> ! { core::arch::wasm32::unreachable() } +// =========================================================================== +// ISP-032 Telemetry Frame Reader +// =========================================================================== + +const MAGIC: u32 = 0x4845_4C50; +const VERSION: u8 = 1; +const FRAME_BYTES: usize = 40; +const INPUT_CAP: usize = 64 * 1024; +const MAX_FRAMES: usize = INPUT_CAP / FRAME_BYTES; +const OUTPUT_LEN: usize = MAX_FRAMES * 8; + +static mut INPUT: [u8; INPUT_CAP] = [0; INPUT_CAP]; +static mut OUTPUT: [f64; OUTPUT_LEN] = [0.0; OUTPUT_LEN]; + +#[no_mangle] +pub extern "C" fn input_ptr() -> *const u8 { + core::ptr::addr_of!(INPUT).cast::() +} + +#[no_mangle] +pub extern "C" fn input_cap() -> usize { + INPUT_CAP +} + +#[no_mangle] +pub extern "C" fn output_ptr() -> *const f64 { + core::ptr::addr_of!(OUTPUT).cast::() +} + +#[no_mangle] +pub extern "C" fn output_cap() -> usize { + MAX_FRAMES +} + +#[inline] +fn read_u32(bytes: &[u8], offset: usize) -> u32 { + u32::from_le_bytes([ + bytes[offset], + bytes[offset + 1], + bytes[offset + 2], + bytes[offset + 3], + ]) +} + +#[inline] +fn read_f64(bytes: &[u8], offset: usize) -> f64 { + f64::from_le_bytes([ + bytes[offset], + bytes[offset + 1], + bytes[offset + 2], + bytes[offset + 3], + bytes[offset + 4], + bytes[offset + 5], + bytes[offset + 6], + bytes[offset + 7], + ]) +} + +#[inline] +fn read_f32(bytes: &[u8], offset: usize) -> f32 { + f32::from_le_bytes([ + bytes[offset], + bytes[offset + 1], + bytes[offset + 2], + bytes[offset + 3], + ]) +} + +#[inline] +fn parse_one(input: &[u8], offset: usize, out: &mut [f64]) -> i32 { + if offset + FRAME_BYTES > input.len() { + return -1; + } + let frame = &input[offset..offset + FRAME_BYTES]; + + if read_u32(frame, 0) != MAGIC { + return -2; + } + if frame[4] != VERSION { + return -3; + } + + let kind = frame[5]; + if !(1..=5).contains(&kind) { + return -4; + } + + let priority = frame[6]; + if priority > 2 { + return -5; + } + + let flags = frame[7]; + let seq = read_u32(frame, 8); + let ts = read_f64(frame, 12); + let lat = read_f64(frame, 20); + let lng = read_f64(frame, 28); + let accuracy = read_f32(frame, 36); + + if !ts.is_finite() || !lat.is_finite() || !lng.is_finite() || !accuracy.is_finite() { + return -6; + } + if !(-90.0..=90.0).contains(&lat) || !(-180.0..=180.0).contains(&lng) { + return -7; + } + + out[0] = seq as f64; + out[1] = ts; + out[2] = lat; + out[3] = lng; + out[4] = accuracy as f64; + out[5] = kind as f64; + out[6] = priority as f64; + out[7] = flags as f64; + 0 +} + +/// Parse the staged batch. Returns the number of frames on success, or a +/// negative status code on the first invalid frame. +#[no_mangle] +pub extern "C" fn parse_batch(len: usize, count: usize) -> i32 { + if count == 0 || count > MAX_FRAMES || len < count * FRAME_BYTES || len > INPUT_CAP { + return -1; + } + + let input: &[u8] = unsafe { core::slice::from_raw_parts(INPUT.as_ptr(), len) }; + let output: &mut [f64] = unsafe { core::slice::from_raw_parts_mut(OUTPUT.as_mut_ptr(), OUTPUT_LEN) }; + + for index in 0..count { + let out = &mut output[index * 8..index * 8 + 8]; + let status = parse_one(input, index * FRAME_BYTES, out); + if status != 0 { + return status; + } + } + count as i32 +} + +/// Single-frame compatibility entry point (used by parity tests). +#[no_mangle] +pub extern "C" fn parse_frame(len: usize) -> i32 { + let parsed = parse_batch(len, 1); + if parsed == 1 { + 0 + } else { + parsed + } +} + +// =========================================================================== +// ADR-014 FlatBuffers & Cap'n Proto Render-Path Readers +// =========================================================================== + static mut NEEDED: u32 = 0; /// Capacity required by the last call that returned -2. diff --git a/src/wasm/telemetry_reader/target/.rustc_info.json b/src/wasm/telemetry_reader/target/.rustc_info.json new file mode 100644 index 00000000..d7a6f362 --- /dev/null +++ b/src/wasm/telemetry_reader/target/.rustc_info.json @@ -0,0 +1 @@ +{"rustc_fingerprint":10483562705157065480,"outputs":{"11652014622397750202":{"success":true,"status":"","code":0,"stdout":"___.wasm\nlib___.rlib\n___.wasm\nlib___.a\n/home/emperor-caesar/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu\noff\n___\ndebug_assertions\nemscripten_wasm_eh\nfmt_debug=\"full\"\noverflow_checks\npanic=\"abort\"\nproc_macro\nrelocation_model=\"static\"\ntarget_abi=\"\"\ntarget_arch=\"wasm32\"\ntarget_endian=\"little\"\ntarget_env=\"\"\ntarget_family=\"wasm\"\ntarget_feature=\"bulk-memory\"\ntarget_feature=\"multivalue\"\ntarget_feature=\"mutable-globals\"\ntarget_feature=\"nontrapping-fptoint\"\ntarget_feature=\"reference-types\"\ntarget_feature=\"sign-ext\"\ntarget_has_atomic\ntarget_has_atomic=\"16\"\ntarget_has_atomic=\"32\"\ntarget_has_atomic=\"64\"\ntarget_has_atomic=\"8\"\ntarget_has_atomic=\"ptr\"\ntarget_has_atomic_equal_alignment=\"16\"\ntarget_has_atomic_equal_alignment=\"32\"\ntarget_has_atomic_equal_alignment=\"64\"\ntarget_has_atomic_equal_alignment=\"8\"\ntarget_has_atomic_equal_alignment=\"ptr\"\ntarget_has_atomic_load_store\ntarget_has_atomic_load_store=\"16\"\ntarget_has_atomic_load_store=\"32\"\ntarget_has_atomic_load_store=\"64\"\ntarget_has_atomic_load_store=\"8\"\ntarget_has_atomic_load_store=\"ptr\"\ntarget_has_reliable_f128\ntarget_has_reliable_f16\ntarget_has_reliable_f16_math\ntarget_os=\"unknown\"\ntarget_pointer_width=\"32\"\ntarget_thread_local\ntarget_vendor=\"unknown\"\nub_checks\n","stderr":"warning: dropping unsupported crate type `dylib` for target `wasm32-unknown-unknown`\n\nwarning: dropping unsupported crate type `proc-macro` for target `wasm32-unknown-unknown`\n\nwarning: 2 warnings emitted\n\n"},"7971740275564407648":{"success":true,"status":"","code":0,"stdout":"___\nlib___.rlib\nlib___.so\nlib___.so\nlib___.a\nlib___.so\n/home/emperor-caesar/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu\noff\npacked\nunpacked\n___\ndebug_assertions\nemscripten_wasm_eh\nfmt_debug=\"full\"\noverflow_checks\npanic=\"unwind\"\nproc_macro\nrelocation_model=\"pic\"\ntarget_abi=\"\"\ntarget_arch=\"x86_64\"\ntarget_endian=\"little\"\ntarget_env=\"gnu\"\ntarget_family=\"unix\"\ntarget_feature=\"fxsr\"\ntarget_feature=\"sse\"\ntarget_feature=\"sse2\"\ntarget_feature=\"x87\"\ntarget_has_atomic\ntarget_has_atomic=\"16\"\ntarget_has_atomic=\"32\"\ntarget_has_atomic=\"64\"\ntarget_has_atomic=\"8\"\ntarget_has_atomic=\"ptr\"\ntarget_has_atomic_equal_alignment=\"16\"\ntarget_has_atomic_equal_alignment=\"32\"\ntarget_has_atomic_equal_alignment=\"64\"\ntarget_has_atomic_equal_alignment=\"8\"\ntarget_has_atomic_equal_alignment=\"ptr\"\ntarget_has_atomic_load_store\ntarget_has_atomic_load_store=\"16\"\ntarget_has_atomic_load_store=\"32\"\ntarget_has_atomic_load_store=\"64\"\ntarget_has_atomic_load_store=\"8\"\ntarget_has_atomic_load_store=\"ptr\"\ntarget_has_reliable_f128\ntarget_has_reliable_f16\ntarget_has_reliable_f16_math\ntarget_os=\"linux\"\ntarget_pointer_width=\"64\"\ntarget_thread_local\ntarget_vendor=\"unknown\"\nub_checks\nunix\n","stderr":""},"17747080675513052775":{"success":true,"status":"","code":0,"stdout":"rustc 1.96.0-nightly (ac7f9ec7d 2026-03-20)\nbinary: rustc\ncommit-hash: ac7f9ec7da74d37fd28667c86bf117a39ba5b02a\ncommit-date: 2026-03-20\nhost: x86_64-unknown-linux-gnu\nrelease: 1.96.0-nightly\nLLVM version: 22.1.0\n","stderr":""}},"successes":{}} \ No newline at end of file diff --git a/src/wasm/telemetry_reader/target/CACHEDIR.TAG b/src/wasm/telemetry_reader/target/CACHEDIR.TAG new file mode 100644 index 00000000..20d7c319 --- /dev/null +++ b/src/wasm/telemetry_reader/target/CACHEDIR.TAG @@ -0,0 +1,3 @@ +Signature: 8a477f597d28d172789f06886806bc55 +# This file is a cache directory tag created by cargo. +# For information about cache directory tags see https://bford.info/cachedir/ diff --git a/src/wasm/telemetry_reader/target/release/.cargo-build-lock b/src/wasm/telemetry_reader/target/release/.cargo-build-lock new file mode 100644 index 00000000..e69de29b diff --git a/src/wasm/telemetry_reader/target/release/.cargo-lock b/src/wasm/telemetry_reader/target/release/.cargo-lock new file mode 100644 index 00000000..e69de29b diff --git a/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/CACHEDIR.TAG b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/CACHEDIR.TAG new file mode 100644 index 00000000..20d7c319 --- /dev/null +++ b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/CACHEDIR.TAG @@ -0,0 +1,3 @@ +Signature: 8a477f597d28d172789f06886806bc55 +# This file is a cache directory tag created by cargo. +# For information about cache directory tags see https://bford.info/cachedir/ diff --git a/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.cargo-build-lock b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.cargo-build-lock new file mode 100644 index 00000000..e69de29b diff --git a/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.cargo-lock b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.cargo-lock new file mode 100644 index 00000000..e69de29b diff --git a/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/dep-lib-telemetry_reader b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/dep-lib-telemetry_reader new file mode 100644 index 00000000..024be490 Binary files /dev/null and b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/dep-lib-telemetry_reader differ diff --git a/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/invoked.timestamp b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/invoked.timestamp new file mode 100644 index 00000000..e00328da --- /dev/null +++ b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/invoked.timestamp @@ -0,0 +1 @@ +This file has an mtime of when this was started. \ No newline at end of file diff --git a/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/lib-telemetry_reader b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/lib-telemetry_reader new file mode 100644 index 00000000..55504fb1 --- /dev/null +++ b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/lib-telemetry_reader @@ -0,0 +1 @@ +7efaf3e95c819bfb \ No newline at end of file diff --git a/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/lib-telemetry_reader.json b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/lib-telemetry_reader.json new file mode 100644 index 00000000..501c40fb --- /dev/null +++ b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/lib-telemetry_reader.json @@ -0,0 +1 @@ +{"rustc":9229630479334048335,"features":"[]","declared_features":"[]","target":139031884860755733,"profile":12833233019932215290,"path":10763286916239946207,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"wasm32-unknown-unknown/release/.fingerprint/telemetry_reader-da90ba2a7f2b77eb/dep-lib-telemetry_reader","checksum":false}}],"rustflags":[],"config":8247474407144887393,"compile_kind":14682669768258224367} \ No newline at end of file diff --git a/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/deps/telemetry_reader.d b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/deps/telemetry_reader.d new file mode 100644 index 00000000..0914767e --- /dev/null +++ b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/deps/telemetry_reader.d @@ -0,0 +1,5 @@ +/home/emperor-caesar/grantfox/HelPhone/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/deps/telemetry_reader.d: src/lib.rs + +/home/emperor-caesar/grantfox/HelPhone/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/deps/telemetry_reader.wasm: src/lib.rs + +src/lib.rs: diff --git a/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/deps/telemetry_reader.wasm b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/deps/telemetry_reader.wasm new file mode 100755 index 00000000..98a6d661 Binary files /dev/null and b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/deps/telemetry_reader.wasm differ diff --git a/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/telemetry_reader.d b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/telemetry_reader.d new file mode 100644 index 00000000..866b7eee --- /dev/null +++ b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/telemetry_reader.d @@ -0,0 +1 @@ +/home/emperor-caesar/grantfox/HelPhone/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/telemetry_reader.wasm: /home/emperor-caesar/grantfox/HelPhone/src/wasm/telemetry_reader/src/lib.rs diff --git a/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/telemetry_reader.wasm b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/telemetry_reader.wasm new file mode 100755 index 00000000..98a6d661 Binary files /dev/null and b/src/wasm/telemetry_reader/target/wasm32-unknown-unknown/release/telemetry_reader.wasm differ diff --git a/src/workers/telemetryWorker.js b/src/workers/telemetryWorker.js index f1296fc6..94b05af9 100644 --- a/src/workers/telemetryWorker.js +++ b/src/workers/telemetryWorker.js @@ -26,6 +26,7 @@ import { isQuotaError, summarize, } from "../services/storageEngine.js"; +import { createTelemetryWorkerProtocol } from "../lib/telemetryWorkerProtocol.js"; import { installWorkerHooks, isKilled, setQueueStats, workerFlagVersion, handleFlagMessage } from "./workerSupervision.js"; /** @@ -260,12 +261,25 @@ export function attachOwnerPort(engine, port) { if (typeof self !== "undefined" && typeof self.postMessage === "function" && typeof window === "undefined") { let ownerEngine = null; const ports = []; + let telemetryProtocol = null; installWorkerHooks(self, "telemetryWorker"); const fail = (err) => self.postMessage({ type: "error", message: String((err && err.message) || err), name: err && err.name }); self.onmessage = async (event) => { const msg = event.data || {}; if (handleFlagMessage(msg)) return; + + // Telemetry binary-parsing lane (flow-controlled; independent of the + // storage-contention benchmark lane below). + if (typeof msg.type === "string" && msg.type.startsWith("telemetry:")) { + if (!telemetryProtocol) { + telemetryProtocol = createTelemetryWorkerProtocol({ + postMessage: (message) => self.postMessage(message), + }); + } + telemetryProtocol.handle(msg); + return; + } try { if (msg.type === "port") { if (ownerEngine) attachOwnerPort(ownerEngine, msg.port); diff --git a/swagger.json b/swagger.json index b06ea368..fc290dc7 100644 --- a/swagger.json +++ b/swagger.json @@ -1,9 +1,9 @@ { "openapi": "3.0.3", "info": { - "title": "HelPhone Federation Protocol API", + "title": "HelPhone API", "version": "1.0.0", - "description": "Federation Protocol specification for peer-to-peer interoperability between independent HelPhone deployments without a central server (ISP-144). Covers cross-instance emergency dispatch, identity trust-bundle bridging, CRDT cross-domain sync, and mutual-aid IOU settlement." + "description": "REST surface for dispatch, events, sync, telemetry, GraphQL bridge, and Federation Protocol for peer-to-peer interoperability between independent HelPhone deployments (ISP-144)." }, "servers": [ { @@ -16,6 +16,305 @@ } ], "paths": { + "/api/dispatch": { + "get": { + "operationId": "listIncidents", + "summary": "List incidents", + "parameters": [ + { + "name": "limit", + "in": "query", + "schema": { + "type": "integer", + "default": 20 + } + }, + { + "name": "cursor", + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Incident page" + } + } + }, + "post": { + "operationId": "createIncident", + "summary": "Create an incident", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateIncident" + } + } + } + }, + "responses": { + "201": { + "description": "Created" + }, + "409": { + "description": "Conflict" + }, + "422": { + "description": "Validation failed" + } + } + } + }, + "/api/dispatch/{id}": { + "get": { + "operationId": "getIncident", + "summary": "Get one incident with responders", + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "nocache", + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Incident bundle" + }, + "404": { + "description": "Not found" + } + } + } + }, + "/api/dispatch/{id}/assign": { + "post": { + "operationId": "assignResponder", + "summary": "Assign a responder with optimistic concurrency", + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "responderId" + ], + "properties": { + "responderId": { + "type": "string" + }, + "expectedVersion": { + "type": "integer" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Assigned" + }, + "409": { + "description": "Version conflict with serverVersion + mergeProposal" + }, + "422": { + "description": "Validation failed" + } + } + } + }, + "/api/dispatch/{id}/ack": { + "post": { + "operationId": "acknowledgeIncident", + "summary": "Acknowledge an incident", + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Acknowledged" + }, + "409": { + "description": "Version conflict" + } + } + } + }, + "/api/events": { + "get": { + "operationId": "listEvents", + "summary": "Event log page; cursor is identical to GraphQL EventConnection.endCursor", + "parameters": [ + { + "name": "incidentId", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "limit", + "in": "query", + "schema": { + "type": "integer", + "default": 20 + } + }, + { + "name": "cursor", + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Event page" + }, + "422": { + "description": "incidentId required" + } + } + } + }, + "/api/events/ice-config": { + "get": { + "operationId": "getIceConfig", + "responses": { + "200": { + "description": "ICE servers" + } + } + } + }, + "/api/sync": { + "post": { + "operationId": "syncOperations", + "responses": { + "200": { + "description": "Merged operations" + } + } + } + }, + "/api/sync/flush": { + "post": { + "operationId": "flushSync", + "summary": "Flush an offline outbox; 409 shape matches GraphQL SyncConflict", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "incidentId", + "baseVersion", + "patch" + ], + "properties": { + "incidentId": { + "type": "string" + }, + "baseVersion": { + "type": "integer" + }, + "patch": { + "type": "object" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Applied" + }, + "409": { + "description": "Conflict with serverVersion + mergeProposal" + }, + "422": { + "description": "Validation issues" + } + } + } + }, + "/api/telemetry/spans": { + "post": { + "operationId": "ingestTelemetry", + "summary": "Bounded OTLP/HTTP JSON ingest", + "responses": { + "202": { + "description": "Accepted" + }, + "429": { + "description": "Queue high watermark; Retry-After" + }, + "503": { + "description": "Collector unavailable; Retry-After" + } + } + } + }, + "/metrics/telemetry": { + "get": { + "operationId": "telemetryMetrics", + "responses": { + "200": { + "description": "Receiver counters" + } + } + } + }, + "/graphql": { + "post": { + "operationId": "graphql", + "summary": "GraphQL endpoint (same domain contract as REST)", + "responses": { + "200": { + "description": "GraphQL result" + }, + "400": { + "description": "Missing query" + } + } + } + }, "/api/federation/trust-bundle": { "get": { "summary": "Export local deployment trust bundle", @@ -44,7 +343,9 @@ "application/json": { "schema": { "type": "object", - "required": ["trustBundle"], + "required": [ + "trustBundle" + ], "properties": { "trustBundle": { "$ref": "#/components/schemas/TrustBundle" @@ -77,7 +378,9 @@ "application/json": { "schema": { "type": "object", - "required": ["envelope"], + "required": [ + "envelope" + ], "properties": { "envelope": { "$ref": "#/components/schemas/FederatedEnvelope" @@ -95,9 +398,15 @@ "schema": { "type": "object", "properties": { - "success": { "type": "boolean" }, - "message": { "type": "string" }, - "ackEnvelope": { "$ref": "#/components/schemas/FederatedEnvelope" } + "success": { + "type": "boolean" + }, + "message": { + "type": "string" + }, + "ackEnvelope": { + "$ref": "#/components/schemas/FederatedEnvelope" + } } } } @@ -125,7 +434,9 @@ "application/json": { "schema": { "type": "object", - "required": ["envelope"], + "required": [ + "envelope" + ], "properties": { "envelope": { "$ref": "#/components/schemas/FederatedEnvelope" @@ -155,7 +466,9 @@ "application/json": { "schema": { "type": "object", - "required": ["envelope"], + "required": [ + "envelope" + ], "properties": { "envelope": { "$ref": "#/components/schemas/FederatedEnvelope" @@ -182,18 +495,36 @@ "application/json": { "schema": { "type": "object", - "required": ["action"], + "required": [ + "action" + ], "properties": { "action": { "type": "string", - "enum": ["issue", "net", "escalate_default"] + "enum": [ + "issue", + "net", + "escalate_default" + ] + }, + "incidentId": { + "type": "string" }, - "incidentId": { "type": "string" }, - "debtorInstance": { "type": "string" }, - "creditorInstance": { "type": "string" }, - "stroops": { "type": "number" }, - "iouId": { "type": "string" }, - "defaultReason": { "type": "string" } + "debtorInstance": { + "type": "string" + }, + "creditorInstance": { + "type": "string" + }, + "stroops": { + "type": "number" + }, + "iouId": { + "type": "string" + }, + "defaultReason": { + "type": "string" + } } } } @@ -226,7 +557,9 @@ "name": "requestId", "in": "path", "required": true, - "schema": { "type": "string" } + "schema": { + "type": "string" + } } ], "responses": { @@ -239,6 +572,52 @@ }, "components": { "schemas": { + "CreateIncident": { + "type": "object", + "required": [ + "location" + ], + "properties": { + "id": { + "type": "string" + }, + "status": { + "type": "string", + "enum": [ + "open", + "assigned", + "acknowledged", + "resolved" + ], + "x-enum-name": "IncidentStatus" + }, + "priority": { + "type": "string", + "enum": [ + "low", + "normal", + "high", + "critical" + ], + "x-enum-name": "Priority" + }, + "location": { + "type": "object", + "required": [ + "lat", + "lng" + ], + "properties": { + "lat": { + "type": "number" + }, + "lng": { + "type": "number" + } + } + } + } + }, "TrustBundle": { "type": "object", "required": [ @@ -253,26 +632,50 @@ "signingPublicKey" ], "properties": { - "instanceId": { "type": "string" }, - "domain": { "type": "string" }, - "aegisVaultRoot": { "type": "string" }, - "rotationCounter": { "type": "integer" }, - "expertRegistryHash": { "type": "string" }, + "instanceId": { + "type": "string" + }, + "domain": { + "type": "string" + }, + "aegisVaultRoot": { + "type": "string" + }, + "rotationCounter": { + "type": "integer" + }, + "expertRegistryHash": { + "type": "string" + }, "revokedIdentities": { "type": "array", - "items": { "type": "string" } + "items": { + "type": "string" + } + }, + "issuedAt": { + "type": "integer" + }, + "expiresAt": { + "type": "integer" }, - "issuedAt": { "type": "integer" }, - "expiresAt": { "type": "integer" }, - "signature": { "type": "string" }, - "signingPublicKey": { "type": "string" } + "signature": { + "type": "string" + }, + "signingPublicKey": { + "type": "string" + } } }, "TrustBundleResponse": { "type": "object", "properties": { - "success": { "type": "boolean" }, - "trustBundle": { "$ref": "#/components/schemas/TrustBundle" } + "success": { + "type": "boolean" + }, + "trustBundle": { + "$ref": "#/components/schemas/TrustBundle" + } } }, "FederatedEnvelope": { @@ -292,43 +695,109 @@ "senderPublicKey" ], "properties": { - "version": { "type": "string", "enum": ["1.0"] }, - "envelopeId": { "type": "string" }, - "senderInstance": { "type": "string" }, - "recipientInstance": { "type": "string" }, - "nonce": { "type": "string" }, - "timestamp": { "type": "integer" }, + "version": { + "type": "string", + "enum": [ + "1.0" + ] + }, + "envelopeId": { + "type": "string" + }, + "senderInstance": { + "type": "string" + }, + "recipientInstance": { + "type": "string" + }, + "nonce": { + "type": "string" + }, + "timestamp": { + "type": "integer" + }, "trustBundleRef": { "type": "object", "properties": { - "instanceId": { "type": "string" }, - "rotationCounter": { "type": "integer" } + "instanceId": { + "type": "string" + }, + "rotationCounter": { + "type": "integer" + } } }, "payloadType": { "type": "string", - "enum": ["dispatch", "ack", "status_update", "crdt_delta", "settlement_iou"] + "enum": [ + "dispatch", + "ack", + "status_update", + "crdt_delta", + "settlement_iou" + ] + }, + "payload": { + "type": "object" + }, + "idempotencyKey": { + "type": "string" + }, + "signature": { + "type": "string" }, - "payload": { "type": "object" }, - "idempotencyKey": { "type": "string" }, - "signature": { "type": "string" }, - "senderPublicKey": { "type": "string" } + "senderPublicKey": { + "type": "string" + } } }, "CrossInstanceIou": { "type": "object", - "required": ["iouId", "debtorInstance", "creditorInstance", "incidentId", "stroops", "status", "issuedAt"], + "required": [ + "iouId", + "debtorInstance", + "creditorInstance", + "incidentId", + "stroops", + "status", + "issuedAt" + ], "properties": { - "iouId": { "type": "string" }, - "debtorInstance": { "type": "string" }, - "creditorInstance": { "type": "string" }, - "incidentId": { "type": "string" }, - "stroops": { "type": "number" }, - "status": { "type": "string", "enum": ["issued", "netted", "defaulted"] }, - "issuedAt": { "type": "integer" }, - "nettedAt": { "type": "integer" }, - "feeStroops": { "type": "number" }, - "notes": { "type": "string" } + "iouId": { + "type": "string" + }, + "debtorInstance": { + "type": "string" + }, + "creditorInstance": { + "type": "string" + }, + "incidentId": { + "type": "string" + }, + "stroops": { + "type": "number" + }, + "status": { + "type": "string", + "enum": [ + "issued", + "netted", + "defaulted" + ] + }, + "issuedAt": { + "type": "integer" + }, + "nettedAt": { + "type": "integer" + }, + "feeStroops": { + "type": "number" + }, + "notes": { + "type": "string" + } } } } diff --git a/test/__snapshots__/snapshots.test.jsx.snap b/test/__snapshots__/snapshots.test.jsx.snap index 6fa55c01..96da546d 100644 --- a/test/__snapshots__/snapshots.test.jsx.snap +++ b/test/__snapshots__/snapshots.test.jsx.snap @@ -638,6 +638,7 @@ exports[`Component Snapshots > Landing Page > should match snapshot 1`] = `

{ + let store; + let app; + + beforeEach(() => { + store = new DomainStore(); + store.resetQueryCount(); + resetSqlCache(); + app = express(); + app.use(express.json()); + installDomainRoutes(app, { store }); + }); + + it('returns the same incident, responder, and last-event fields on both surfaces', async () => { + const rest = await request(app).get('/api/dispatch/INC-1042?nocache=1'); + expect(rest.status).toBe(200); + + const gql = await executeGraphQL( + { query: INCIDENT_QUERY, variables: { id: 'INC-1042' } }, + { store }, + ); + expect(gql.errors).toBeUndefined(); + + const restIncident = rest.body.incident; + const gqlIncident = gql.data.incident; + + expect(restIncident).toMatchObject({ + id: gqlIncident.id, + status: gqlIncident.status, + priority: gqlIncident.priority, + version: gqlIncident.version, + location: gqlIncident.location, + }); + expect(restIncident.responders).toHaveLength(40); + expect(restIncident.responders).toHaveLength(gqlIncident.responders.length); + + for (let i = 0; i < restIncident.responders.length; i++) { + const restResponder = restIncident.responders[i]; + const gqlResponder = gqlIncident.responders[i]; + expect(restResponder).toMatchObject({ + id: gqlResponder.id, + incidentId: gqlResponder.incidentId, + status: gqlResponder.status, + location: gqlResponder.location, + }); + expect(restResponder.lastEvent?.id ?? null).toBe(gqlResponder.lastEvent?.id ?? null); + expect(restResponder.lastEvent?.cursor ?? null).toBe(gqlResponder.lastEvent?.cursor ?? null); + } + }); + + it('batches GraphQL nested reads to ≤4 store queries instead of 1+N+N', async () => { + store.resetQueryCount(); + const gql = await executeGraphQL( + { query: INCIDENT_QUERY, variables: { id: 'INC-1042' } }, + { store }, + ); + expect(gql.errors).toBeUndefined(); + expect(store.getQueryCount()).toBeLessThanOrEqual(4); + // The naive resolver would issue incident + 40 responders + 40 events = 81. + expect(store.getQueryCount()).toBeLessThan(81); + }); + + it('uses two store queries for the hand-optimised REST bundle', async () => { + store.resetQueryCount(); + const rest = await request(app).get('/api/dispatch/INC-1042?nocache=1'); + expect(rest.status).toBe(200); + expect(store.getQueryCount()).toBeLessThanOrEqual(2); + }); + + it('creates request-scoped loaders so one context cannot serve another context stale data', async () => { + const ctxA = createGraphQLContext(store, 'user-a'); + const ctxB = createGraphQLContext(store, 'user-b'); + expect(ctxA.loaders).not.toBe(ctxB.loaders); + + const before = await ctxA.loaders.responder.load('R-001'); + expect(before.location.lat).toBeCloseTo(6.5201, 6); + expect(before.location.lng).toBeCloseTo(3.3701, 6); + + store.updateResponderLocation('R-001', { lat: 9.999, lng: 8.888 }); + + // A's cache is isolated; B sees the write. + const cachedForA = await ctxA.loaders.responder.load('R-001'); + const freshForB = await ctxB.loaders.responder.load('R-001'); + expect(cachedForA.location.lat).toBeCloseTo(6.5201, 6); + expect(freshForB.location).toEqual({ lat: 9.999, lng: 8.888 }); + }); + + it('returns identical 409 conflict payloads from REST and GraphQL', async () => { + const rest = await request(app) + .post('/api/dispatch/INC-1042/assign') + .send({ responderId: 'R-002', expectedVersion: 2 }); + expect(rest.status).toBe(409); + expect(rest.body).toMatchObject({ conflict: true, serverVersion: 3 }); + + const gql = await executeGraphQL( + { + query: ASSIGN_MUTATION, + variables: { input: { incidentId: 'INC-1042', responderId: 'R-002', expectedVersion: 2 } }, + }, + { store }, + ); + expect(gql.errors).toBeUndefined(); + expect(gql.data.assignResponder).toMatchObject({ + __typename: 'SyncConflict', + conflict: true, + serverVersion: rest.body.serverVersion, + mergeProposal: rest.body.mergeProposal, + }); + }); + + it('returns identical 409 conflict payloads from the REST flush and GraphQL syncFlush', async () => { + const input = { incidentId: 'INC-1042', baseVersion: 2, patch: { priority: 'high' } }; + const rest = await request(app).post('/api/sync/flush').send(input); + expect(rest.status).toBe(409); + expect(rest.body).toMatchObject({ conflict: true, serverVersion: 3 }); + + const gql = await executeGraphQL({ query: FLUSH_MUTATION, variables: { input } }, { store }); + expect(gql.errors).toBeUndefined(); + expect(gql.data.syncFlush).toMatchObject({ + __typename: 'SyncConflict', + conflict: true, + serverVersion: rest.body.serverVersion, + mergeProposal: rest.body.mergeProposal, + }); + }); + + it('returns identical 422 validation issues from REST and GraphQL', async () => { + const input = { incidentId: 'INC-1042', baseVersion: 3, patch: { bogus: true } }; + const rest = await request(app).post('/api/sync/flush').send(input); + expect(rest.status).toBe(422); + expect(rest.body.error).toBe('validation_failed'); + expect(rest.body.issues).toEqual( + expect.arrayContaining([{ path: 'patch.bogus', message: 'unsupported field' }]), + ); + + const gql = await executeGraphQL({ query: FLUSH_MUTATION, variables: { input } }, { store }); + expect(gql.errors).toBeUndefined(); + expect(gql.data.syncFlush).toMatchObject({ + __typename: 'ValidationError', + error: rest.body.error, + issues: rest.body.issues, + }); + }); + + it('returns 422 for a malformed REST body before touching the store', async () => { + store.resetQueryCount(); + const rest = await request(app).post('/api/dispatch/INC-1042/assign').send({}); + expect(rest.status).toBe(422); + expect(rest.body.issues[0]).toMatchObject({ path: 'responderId' }); + expect(store.getQueryCount()).toBe(0); + }); + + it('paginates events with the same cursor on both surfaces', async () => { + const restFirst = await request(app).get('/api/events?incidentId=INC-1042&limit=5'); + expect(restFirst.status).toBe(200); + expect(restFirst.body.events).toHaveLength(5); + const restCursor = restFirst.body.nextCursor; + expect(restCursor).toBeTruthy(); + + const gqlFirst = await executeGraphQL( + { query: EVENTS_QUERY, variables: { id: 'INC-1042', limit: 5 } }, + { store }, + ); + expect(gqlFirst.errors).toBeUndefined(); + expect(gqlFirst.data.events.pageInfo.endCursor).toBe(restCursor); + expect(gqlFirst.data.events.edges.map((edge) => edge.node.id)).toEqual( + restFirst.body.events.map((event) => event.id), + ); + + const restSecond = await request(app).get( + `/api/events?incidentId=INC-1042&limit=5&cursor=${encodeURIComponent(restCursor)}`, + ); + const gqlSecond = await executeGraphQL( + { query: EVENTS_QUERY, variables: { id: 'INC-1042', limit: 5, cursor: restCursor } }, + { store }, + ); + expect(restSecond.body.events.map((event) => event.id)).toEqual( + gqlSecond.data.events.edges.map((edge) => edge.node.id), + ); + }); + + it('invalidates the REST cache when a write happens, so GraphQL and REST converge', async () => { + const before = await request(app).get('/api/dispatch/INC-1042'); + expect(before.body.incident.version).toBe(3); + + const write = await request(app) + .post('/api/dispatch/INC-1042/assign') + .send({ responderId: 'R-002', expectedVersion: 3 }); + expect(write.status).toBe(200); + expect(write.body.incident.version).toBe(4); + + const after = await request(app).get('/api/dispatch/INC-1042'); + expect(after.body.incident.version).toBe(4); + + const gql = await executeGraphQL( + { + query: 'query($id: ID!){ incident(id: $id){ id version } }', + variables: { id: 'INC-1042' }, + }, + { store }, + ); + expect(gql.data.incident.version).toBe(4); + }); + + it('accepts GraphQL over HTTP and rejects a missing query', async () => { + const ok = await request(app) + .post('/graphql') + .send({ query: 'query($id: ID!){ incident(id: $id){ id } }', variables: { id: 'INC-1042' } }); + expect(ok.status).toBe(200); + expect(ok.body.data.incident.id).toBe('INC-1042'); + + const bad = await request(app).post('/graphql').send({}); + expect(bad.status).toBe(400); + }); +}); diff --git a/test/crypto-pins.test.js b/test/crypto-pins.test.js index 53826f8c..9c4ddd97 100644 --- a/test/crypto-pins.test.js +++ b/test/crypto-pins.test.js @@ -1,5 +1,6 @@ import { describe, it, expect } from "vitest"; import fs from "node:fs"; +import path from "node:path"; import { checkCryptoPins, holdCryptoCore, @@ -9,7 +10,7 @@ import { // #779 — crypto-core pin gate. Fixture-only except for the repo self-check. -const policy = JSON.parse(fs.readFileSync(new URL("../crypto-pins.json", import.meta.url), "utf8")); +const policy = JSON.parse(fs.readFileSync(path.resolve(process.cwd(), "crypto-pins.json"), "utf8")); const now = new Date("2026-10-17"); const lockOf = (versions) => ({ @@ -32,12 +33,12 @@ describe("crypto-pins policy file", () => { for (const p of policy.pins) { expect(p.rationale, p.name).toBeTruthy(); expect(p.reviewed, p.name).toMatch(/^\d{4}-\d{2}-\d{2}$/); - expect(fs.existsSync(new URL(`../${p.adr}`, import.meta.url)), p.adr).toBe(true); + expect(fs.existsSync(path.resolve(process.cwd(), p.adr)), p.adr).toBe(true); } }); it("matches the repository's real lockfiles", () => { - const read = (f) => fs.readFileSync(new URL(`../${f}`, import.meta.url), "utf8"); + const read = (f) => fs.readFileSync(path.resolve(process.cwd(), f), "utf8"); const result = checkCryptoPins({ policy, pkg: JSON.parse(read("package.json")), diff --git a/test/csp-headers.test.js b/test/csp-headers.test.js index b357c96a..1ea2bfb8 100644 --- a/test/csp-headers.test.js +++ b/test/csp-headers.test.js @@ -1,5 +1,6 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import express from "express"; +import { server as mockServer } from "../src/mocks/server.js"; import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -185,6 +186,14 @@ describe("HTTP integration", () => { const dirs = []; const servers = []; + beforeAll(() => { + mockServer.close(); + }); + + afterAll(() => { + mockServer.listen({ onUnhandledRequest: "warn" }); + }); + afterEach(async () => { await Promise.all(servers.splice(0).map((s) => new Promise((r) => s.close(r)))); dirs.splice(0).forEach((d) => rmSync(d, { recursive: true, force: true })); diff --git a/test/sql-cache.test.js b/test/sql-cache.test.js new file mode 100644 index 00000000..3c9f0cc8 --- /dev/null +++ b/test/sql-cache.test.js @@ -0,0 +1,88 @@ +// @vitest-environment node +import { describe, expect, it, vi } from 'vitest'; +import { NamespacedCache, namespacedKey } from '../server/lib/sqlCache.ts'; + +describe('NamespacedCache', () => { + it('keeps REST and GraphQL projections in separate namespaces', () => { + const cache = new NamespacedCache(); + cache.set('rest', 'dispatch:INC-1', { projection: 'rest' }); + cache.set('gql', 'dispatch:INC-1', { projection: 'gql' }); + + expect(cache.get('rest', 'dispatch:INC-1')).toEqual({ projection: 'rest' }); + expect(cache.get('gql', 'dispatch:INC-1')).toEqual({ projection: 'gql' }); + expect(namespacedKey('rest', 'k')).not.toBe(namespacedKey('gql', 'k')); + }); + + it('invalidates an entity across every surface at once', () => { + const cache = new NamespacedCache(); + cache.set('rest', 'dispatch:INC-1', { version: 1 }); + cache.set('gql', 'dispatch:INC-1', { version: 1 }); + cache.set('rest', 'dispatch:INC-2', { version: 1 }); + + expect(cache.invalidate('dispatch:INC-1')).toBe(2); + expect(cache.get('rest', 'dispatch:INC-1')).toBeUndefined(); + expect(cache.get('gql', 'dispatch:INC-1')).toBeUndefined(); + expect(cache.get('rest', 'dispatch:INC-2')).toEqual({ version: 1 }); + expect(cache.stats().invalidations).toBe(1); + }); + + it('expires entries at the TTL boundary', () => { + let now = 1_000; + const cache = new NamespacedCache({ now: () => now, jitterRatio: 0 }); + cache.set('rest', 'k', 'v', 50); + expect(cache.get('rest', 'k')).toBe('v'); + now = 1_049; + expect(cache.get('rest', 'k')).toBe('v'); + now = 1_050; + expect(cache.get('rest', 'k')).toBeUndefined(); + }); + + it('coalesces concurrent misses into one load', async () => { + const cache = new NamespacedCache(); + let loads = 0; + const load = vi.fn(async () => { + loads++; + await new Promise((resolve) => setTimeout(resolve, 5)); + return { version: 1 }; + }); + + const [a, b, c] = await Promise.all([ + cache.withCache('rest', 'dispatch:INC-1', load), + cache.withCache('rest', 'dispatch:INC-1', load), + cache.withCache('rest', 'dispatch:INC-1', load), + ]); + + expect(loads).toBe(1); + expect(a).toBe(b); + expect(b).toBe(c); + expect(cache.stats().coalesced).toBe(2); + }); + + it('bounds the entry count', () => { + const cache = new NamespacedCache({ maxEntries: 3, jitterRatio: 0 }); + for (let i = 0; i < 10; i++) cache.set('rest', `k${i}`, i); + expect(cache.stats().size).toBeLessThanOrEqual(3); + }); + + it('bypasses both read and write when requested', async () => { + const cache = new NamespacedCache(); + const first = await cache.withCache('rest', 'k', () => 'a', { bypass: true }); + const second = await cache.withCache('rest', 'k', () => 'b', { bypass: true }); + expect(first).toBe('a'); + expect(second).toBe('b'); + expect(cache.get('rest', 'k')).toBeUndefined(); + }); + + it('applies bounded jitter to TTLs', () => { + const random = vi.spyOn(Math, 'random').mockReturnValue(0.999); + let now = 0; + const cache = new NamespacedCache({ now: () => now, jitterRatio: 0.2 }); + cache.set('rest', 'k', 'v', 100); + // max jitter is +20%, so 119ms is alive and 121ms is expired. + now = 119; + expect(cache.get('rest', 'k')).toBe('v'); + now = 121; + expect(cache.get('rest', 'k')).toBeUndefined(); + random.mockRestore(); + }); +}); diff --git a/test/telemetry-pipeline.test.js b/test/telemetry-pipeline.test.js new file mode 100644 index 00000000..aea75c70 --- /dev/null +++ b/test/telemetry-pipeline.test.js @@ -0,0 +1,242 @@ +// @vitest-environment node +import { describe, it, expect, beforeEach } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { RingBuffer } from '../src/lib/ringBuffer.ts'; +import { + TelemetryPipeline, + createMemoryExporter, + shouldSample, + samplingRate, +} from '../src/lib/telemetry.ts'; +import { profileFor } from '../src/lib/networkEstimator.ts'; +import { encodeFrame, parseFrame, FRAME_KIND } from '../src/utils/binaryParser.js'; +import { TelemetryWasmParser } from '../src/utils/binaryParserWasm.js'; +import { createTelemetryWorkerProtocol } from '../src/lib/telemetryWorkerProtocol.js'; + +const frame = (seq, priority = 'normal', kind = 'location') => ({ + seq, + priority, + kind, + ts: 1_700_000_000_000 + seq, + payload: { seq }, +}); + +describe('RingBuffer priority + accounting', () => { + it('evicts heartbeat before normal and normal before critical', () => { + const ring = new RingBuffer({ capacity: 3 }); + expect(ring.push(frame(1, 'heartbeat'))).toBe(true); + expect(ring.push(frame(2, 'normal'))).toBe(true); + expect(ring.push(frame(3, 'critical'))).toBe(true); + + // A new heartbeat must displace the existing heartbeat, never the critical. + expect(ring.push(frame(4, 'heartbeat'))).toBe(true); + expect(ring.snapshot().droppedReasons.overflow_heartbeat).toBe(1); + expect(ring.snapshot().droppedReasons.overflow_critical).toBe(0); + + // The next critical still finds the newest heartbeat and evicts it; only + // after all heartbeats are gone does it touch the normal frame. + expect(ring.push(frame(5, 'critical'))).toBe(true); + expect(ring.snapshot().droppedReasons.overflow_heartbeat).toBe(2); + expect(ring.push(frame(6, 'critical'))).toBe(true); + expect(ring.snapshot().droppedReasons.overflow_normal).toBe(1); + + const snapshot = ring.snapshot(); + expect(snapshot.conservation.ok).toBe(true); + expect(snapshot.size).toBe(3); + }); + + it('rejects an incoming lower-priority frame when the ring is all critical', () => { + const ring = new RingBuffer({ capacity: 2 }); + ring.push(frame(1, 'critical')); + ring.push(frame(2, 'critical')); + expect(ring.push(frame(3, 'heartbeat'))).toBe(false); + const snapshot = ring.snapshot(); + expect(snapshot.rejected).toBe(1); + expect(snapshot.evicted).toBe(0); + expect(snapshot.droppedReasons.overflow_heartbeat).toBe(1); + expect(snapshot.conservation.ok).toBe(true); + }); + + it('reports pressure with 70/90/50 hysteresis', () => { + const ring = new RingBuffer({ capacity: 10 }); + const events = []; + ring.onPressure((event) => events.push(event.level)); + for (let i = 0; i < 7; i++) ring.push(frame(i)); + for (let i = 7; i < 9; i++) ring.push(frame(i)); + ring.drain(5); // occupancy 40% -> normal + expect(events).toEqual(['high', 'critical', 'normal']); + }); + + it('keeps the conservation invariant under random admission/drain', () => { + const ring = new RingBuffer({ capacity: 16 }); + for (let i = 0; i < 2_000; i++) { + ring.push(frame(i, i % 7 === 0 ? 'critical' : i % 3 === 0 ? 'heartbeat' : 'normal')); + if (i % 5 === 0) ring.drain(3); + expect(ring.snapshot().conservation.ok).toBe(true); + } + }); +}); + +describe('adaptive sampling', () => { + it('is deterministic per sequence', () => { + expect(shouldSample(1, 0)).toBe(false); + expect(shouldSample(1, 1)).toBe(true); + expect(shouldSample(42, 0.4)).toBe(shouldSample(42, 0.4)); + }); + + it('keeps 100% of dispatch-critical spans on 2G while exporting ≤20% of baseline', async () => { + const exporter = createMemoryExporter(); + const pipeline = new TelemetryPipeline({ + capacity: 8_192, + networkClass: '2g', + exporter, + }); + + const critical = 100; + const normal = 8_000; + const heartbeat = 2_000; + for (let i = 0; i < critical; i++) pipeline.record({ name: 'dispatch', priority: 'critical' }); + for (let i = 0; i < normal; i++) pipeline.record({ name: 'map', priority: 'normal' }); + for (let i = 0; i < heartbeat; i++) pipeline.record({ name: 'beacon', priority: 'heartbeat' }); + + const exportedCritical = exporter.batches.flat().filter((span) => span.priority === 'critical').length; + // Flush everything (2G batch size is 5) and count critical spans. + for (let i = 0; i < 5_000 && pipeline.ring.length > 0; i++) await pipeline.flush(); + const all = exporter.batches.flat(); + const exportedCriticalAfter = all.filter((span) => span.priority === 'critical').length; + + expect(exportedCritical).toBe(0); + expect(exportedCriticalAfter).toBe(critical); + expect(all.length).toBeLessThanOrEqual((critical + normal + heartbeat) * 0.2); + expect(pipeline.snapshot().conservation.ok).toBe(true); + }); + + it('halves the sampling rate at high pressure', () => { + const profile = profileFor('3g'); + expect(samplingRate(profile, 'normal', 'normal')).toBe(profile.normalSampleRate); + expect(samplingRate(profile, 'high', 'normal')).toBe(profile.normalSampleRate * 0.5); + expect(samplingRate(profile, 'critical', 'critical')).toBe(1); + }); +}); + +describe('export/offline accounting', () => { + it('diverts offline spans to the durable outbox, not memory', async () => { + const queued = []; + const exporter = createMemoryExporter(); + const pipeline = new TelemetryPipeline({ + capacity: 64, + networkClass: 'offline', + exporter, + outbox: { async enqueue(spans) { queued.push(...spans); } }, + }); + for (let i = 0; i < 10; i++) pipeline.record({ name: 'map', priority: 'critical' }); + await pipeline.flush(); + expect(exporter.batches).toHaveLength(0); + expect(queued).toHaveLength(10); + const snapshot = pipeline.snapshot(); + expect(snapshot.outboxed).toBe(10); + expect(snapshot.conservation.ok).toBe(true); + }); + + it('counts export failures with an explicit reason and stays bounded', async () => { + const pipeline = new TelemetryPipeline({ + capacity: 32, + exporter: { + async export() { + const error = new Error('collector down'); + error.retryAfterMs = 1_000; + throw error; + }, + }, + }); + for (let i = 0; i < 20; i++) pipeline.record({ name: 'map', priority: 'critical' }); + await pipeline.flush(); + const snapshot = pipeline.snapshot(); + expect(snapshot.exportFailed).toBeGreaterThan(0); + expect(snapshot.droppedReasons.export_fail).toBe(snapshot.exportFailed); + expect(snapshot.exporterHealthy).toBe(false); + expect(snapshot.conservation.ok).toBe(true); + }); +}); + +describe('binary parser parity', () => { + it('JS and WASM produce identical frames', async () => { + const wasmPath = resolve(process.cwd(), 'src/wasm/telemetry_reader.wasm'); + const shaPath = resolve(process.cwd(), 'src/wasm/telemetry_reader.wasm.sha256'); + const parser = new TelemetryWasmParser({ + wasmBytes: readFileSync(wasmPath), + expectedSha256: readFileSync(shaPath, 'utf8').trim(), + shaUrl: undefined, + }); + const ready = await parser.init(); + expect(ready).toBe(true); + + const frames = [ + { seq: 1, kind: FRAME_KIND.LOCATION, ts: 1_700_000_000_000, lat: 6.5, lng: 3.3, accuracy: 4 }, + { seq: 2, kind: FRAME_KIND.DISPATCH_ACK, ts: 1_700_000_000_001, lat: 6.51, lng: 3.31, accuracy: 5 }, + { seq: 3, kind: FRAME_KIND.HEARTBEAT, ts: 1_700_000_000_002, lat: 6.52, lng: 3.32, accuracy: 6 }, + ]; + const buffer = new Uint8Array(frames.length * 40); + frames.forEach((value, index) => buffer.set(new Uint8Array(encodeFrame(value)), index * 40)); + + for (let i = 0; i < frames.length; i++) { + const js = parseFrame(buffer, i * 40); + const wasm = parser.parseBatch(buffer, frames.length)[i]; + expect(wasm).toMatchObject({ + seq: js.seq, + ts: js.ts, + lat: js.lat, + lng: js.lng, + accuracy: js.accuracy, + kind: js.kind, + priority: js.priority, + }); + } + }); +}); + +describe('worker flow control', () => { + it('sheds non-critical batches at congestion level 2 and parses critical', async () => { + const messages = []; + const parser = { + mode: 'js', + ready: true, + init: async () => true, + parseBatch: () => [{ priority: 'critical', kindName: 'dispatch_ack', seq: 1, ts: 1 }], + }; + const protocol = createTelemetryWorkerProtocol({ + postMessage: (message) => messages.push(message), + parser, + }); + + protocol.handle({ type: 'telemetry:congestion', level: 2 }); + protocol.handle({ type: 'telemetry:parse', id: 1, buffer: new ArrayBuffer(40), count: 1, priority: 'normal' }); + protocol.handle({ type: 'telemetry:parse', id: 2, buffer: new ArrayBuffer(40), count: 1, priority: 'critical' }); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(messages.some((message) => message.type === 'telemetry:shed' && message.reason === 'backpressure')).toBe(true); + expect(messages.some((message) => message.type === 'telemetry:parsed' && message.id === 2)).toBe(true); + expect(messages.some((message) => message.type === 'telemetry:parsed' && message.id === 1)).toBe(false); + }); + + it('bounds the worker queue with drop-oldest accounting', () => { + const messages = []; + const parser = { + mode: 'js', + ready: true, + init: async () => true, + parseBatch: () => [], + }; + const protocol = createTelemetryWorkerProtocol({ + postMessage: (message) => messages.push(message), + parser, + maxQueue: 2, + }); + for (let i = 0; i < 5; i++) { + protocol.handle({ type: 'telemetry:parse', id: i, buffer: new ArrayBuffer(40), count: 1 }); + } + expect(protocol.queueDepth).toBeLessThanOrEqual(2); + expect(protocol.snapshot().queueDrops).toBeGreaterThan(0); + }); +}); diff --git a/tsconfig.json b/tsconfig.json index bec2e17b..af472d3e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -22,7 +22,8 @@ "noUnusedParameters": false, "noFallthroughCasesInSwitch": true }, - "include": ["src", "server", "scripts", "test"], + "include": ["src", "vite.config.ts"], + "exclude": ["node_modules", "dist", "server"], "references": [], /* Type-surface extraction options for scripts/detect-api-drift.js.