fix(observability): record every documented metric, including notifications

.changeset/otel-documented-metrics.md

@@ -0,0 +1,11 @@
+---
+"browserhive": patch
+---
+
+OpenTelemetry now exports the notification, attention, session-launch, WebSocket, write-queue and browser-memory metrics the docs describe.
+
+- **Metrics that were documented but never sent** now reach your collector: `browserhive.attention.wait`, `browserhive.session.launch.duration`, `browserhive.ws.connections`, `browserhive.ws.buffered_bytes`, `browserhive.ws.frames_dropped`, `browserhive.db.dropped_writes`, `browserhive.browser.rss_bytes` (each session's browser with all its processes, every 10 seconds; Linux and macOS) and `browserhive.process.event_loop_lag`.
+- **Attributes the docs promised** are now set: `closed_reason` on `browserhive.session.lifetime`, `kind` on `browserhive.attention.open` (vault confirmations count too), `table` on `browserhive.retention.pruned_rows`.
+- **The notification metrics** `browserhive.notifications.deliveries`, `.actions` and `.reports` are now in the [telemetry guide](https://browserhive.ai/docs/guide/telemetry). `.reports` now counts on-demand digests as `manual`, as documented, and no longer counts a silent revision of an open in-app anomaly alert.
+- **Gauges only report what exists**: a closed session's browser memory or a closed WebSocket's buffered bytes disappears from the next export instead of repeating its last value. `browserhive.db.dropped_writes` reports every recorder table from the start, at 0.
+- **Every metric has a unit** (`ms`, `By`, or a count such as `{call}`), and the guide's table lists each one with its type, unit and attributes. With telemetry off nothing is measured, as before.

docs/guide/telemetry.md

@@ -40,30 +40,35 @@ If the MCP client sends a W3C `traceparent` in the tool call's `_meta`, the tool
 
 ### Metrics
 
-| Instrument | Type | Attributes |
-|---|---|---|
-| `browserhive.tool_calls` | counter | `tool`, `ok`, `error_code`, `harness` |
-| `browserhive.tool_call.duration` | histogram (ms) | `tool` |
-| `browserhive.sessions.active` | up-down counter | `harness` |
-| `browserhive.session.launch.duration` | histogram (ms) | `channel`, `stealth` |
-| `browserhive.session.lifetime` | histogram (ms) | `closed_reason` |
-| `browserhive.ws.connections` | up-down counter | |
-| `browserhive.ws.buffered_bytes` | gauge | `connection_id` |
-| `browserhive.ws.frames_dropped` | counter | `channel` |
-| `browserhive.db.write_queue.depth` | gauge | |
-| `browserhive.db.dropped_writes` | counter | `table` |
-| `browserhive.db.size_bytes` | gauge | |
-| `browserhive.browser.rss_bytes` | gauge | `session_id` |
-| `browserhive.attention.open` | up-down counter | `kind` |
-| `browserhive.attention.wait` | histogram (ms) | `status` |
-| `browserhive.vault.fills` | counter | `result` |
-| `browserhive.blocklist.hits` | counter | `source` |
-| `browserhive.retention.pruned_rows` | counter | `table` |
-| `browserhive.process.*` | gauges | rss, heap, event-loop lag |
+| Instrument | Type | Unit | Attributes | What it measures |
+|---|---|---|---|---|
+| `browserhive.tool_calls` | counter | `{call}` | `tool`, `ok`, `error_code` (failed calls only), `harness` | Tool calls. |
+| `browserhive.tool_call.duration` | histogram | `ms` | `tool` | How long each tool call took. |
+| `browserhive.sessions.active` | up-down counter | `{session}` | `harness` | Live sessions. |
+| `browserhive.session.launch.duration` | histogram | `ms` | `channel`, `stealth` | From the create request until the browser is up, per successful launch. |
+| `browserhive.session.lifetime` | histogram | `ms` | `closed_reason` | How long each closed session lived. |
+| `browserhive.ws.connections` | up-down counter | `{connection}` | | Open dashboard WebSocket connections. |
+| `browserhive.ws.buffered_bytes` | gauge | `By` | `connection_id` | Bytes waiting to be sent, for the 50 most backed-up connections. |
+| `browserhive.ws.frames_dropped` | counter | `{frame}` | `channel` (`screencast`, `logs`, `feed`) | Frames not delivered because a connection was congested. |
+| `browserhive.db.write_queue.depth` | gauge | `{write}` | | Writes waiting to be saved. |
+| `browserhive.db.dropped_writes` | counter | `{write}` | `table` | Writes lost because the queue was full or the write failed. |
+| `browserhive.db.size_bytes` | gauge | `By` | | Database size. |
+| `browserhive.browser.rss_bytes` | gauge | `By` | `session_id` | Memory of each session's browser, all its processes added up (memory they share counts once per process), sampled every 10 seconds. Linux and macOS only. |
+| `browserhive.attention.open` | up-down counter | `{request}` | `kind` (`attention`, `vault_confirm`) | Operator requests waiting for an answer. |
+| `browserhive.attention.wait` | histogram | `ms` | `status` | How long each attention request waited until it was answered, timed out or cancelled. |
+| `browserhive.vault.fills` | counter | `{fill}` | `result` | Vault fills. |
+| `browserhive.blocklist.hits` | counter | `{hit}` | `source` | Blocked requests. |
+| `browserhive.retention.pruned_rows` | counter | `{row}` | `table` | Rows removed by retention. |
+| `browserhive.notifications.deliveries` | counter | `{delivery}` | `channel_kind`, `status` | Notification deliveries by outcome: `sent`, `retrying`, `dead`, `suppressed`, `superseded`. |
+| `browserhive.notifications.reports` | counter | `{report}` | `kind`, `outcome` | Digest and anomaly-alert decisions: `sent`, `late`, `empty`, `skipped`, `manual`, `resolved`, `in_app`. |
+| `browserhive.notifications.actions` | counter | `{press}` | `channel_kind`, `outcome` | Presses of act buttons in Telegram, Discord and ntfy, by outcome (`unknown` for buttons BrowserHive did not create). |
+| `browserhive.process.rss_bytes` | gauge | `By` | | Memory of the BrowserHive process. |
+| `browserhive.process.heap_bytes` | gauge | `By` | | JavaScript heap in use. |
+| `browserhive.process.event_loop_lag` | gauge | `ms` | | Event-loop delay (99th percentile since the previous export). |
 
 `harness` on metrics is one of the known harness names, `unknown` or `other` (every name BrowserHive doesn't know is folded into `other`), so it adds a bounded number of series. The model, workspace and extra labels are never metric attributes.
 
-Metrics are exported every 30 seconds.
+Metrics are exported every 30 seconds. Gauges and the WebSocket and write-queue totals are read at export time, and a gauge only reports what exists at that moment: a closed session or connection disappears from the next export. With telemetry off nothing is measured.
 
 ### Logs
 

packages/browserhive/src/composition/adapters/browser-memory.test.ts

@@ -0,0 +1,118 @@
+/** @module composition/adapters/browser-memory.test — the 10 s browser-memory sampler behind `browserhive.browser.rss_bytes` (spec 10 §7): one tree per live session keyed by session id, sessions without a pid skipped, overlapping samples coalesced, no timer without a reader, stop. */
+
+import { describe, expect, it } from 'bun:test';
+import type { ProcessTreeReader } from '@browserhive/core/runtime';
+import { startBrowserMemorySampler } from './browser-memory.ts';
+
+function manualTimer() {
+  const timers: { fn: () => void; ms: number; cancelled: boolean }[] = [];
+  return {
+    timers,
+    repeat: (fn: () => void, ms: number) => {
+      const t = { fn, ms, cancelled: false };
+      timers.push(t);
+      return () => {
+        t.cancelled = true;
+      };
+    },
+  };
+}
+
+describe('startBrowserMemorySampler', () => {
+  it('samples at once and every 10 s, one process tree per session', async () => {
+    const reads: number[][] = [];
+    const reader: ProcessTreeReader = {
+      rssOfTrees: async (roots) => {
+        reads.push([...roots]);
+        return new Map(roots.filter((pid) => pid !== 30).map((pid) => [pid, pid * 1000]));
+      },
+    };
+    let sessions = [
+      { id: 's-a', browserPid: async () => 10 },
+      { id: 's-b', browserPid: async () => 20 },
+      { id: 's-launching' },
+      { id: 's-unreadable', browserPid: async () => null },
+      { id: 's-gone', browserPid: async () => 30 },
+    ];
+    const timer = manualTimer();
+    const sampler = startBrowserMemorySampler({
+      sessions: () => sessions,
+      reader,
+      repeat: timer.repeat,
+    });
+    await sampler.sample();
+    expect(timer.timers.map((t) => t.ms)).toEqual([10_000]);
+    expect(reads[0]).toEqual([10, 20, 30]);
+    expect([...sampler.latest()]).toEqual([
+      ['s-a', 10_000],
+      ['s-b', 20_000],
+    ]);
+    // A closed session disappears at the next sample.
+    sessions = sessions.slice(0, 1);
+    timer.timers[0]?.fn();
+    await sampler.sample();
+    expect([...sampler.latest()]).toEqual([['s-a', 10_000]]);
+    sampler.stop();
+    expect(timer.timers[0]?.cancelled).toBe(true);
+    expect(sampler.latest().size).toBe(0);
+  });
+
+  it('coalesces a sample requested while one is running', async () => {
+    let release: () => void = () => undefined;
+    let calls = 0;
+    const reader: ProcessTreeReader = {
+      rssOfTrees: () => {
+        calls++;
+        return new Promise((resolve) => {
+          release = () => resolve(new Map([[10, 1]]));
+        });
+      },
+    };
+    const sampler = startBrowserMemorySampler({
+      sessions: () => [{ id: 's-a', browserPid: async () => 10 }],
+      reader,
+      repeat: manualTimer().repeat,
+    });
+    const second = sampler.sample();
+    await new Promise((resolve) => setTimeout(resolve, 5));
+    release();
+    await second;
+    expect(calls).toBe(1);
+    sampler.stop();
+  });
+
+  it('never starts a timer without a reader (Windows)', async () => {
+    const timer = manualTimer();
+    const sampler = startBrowserMemorySampler({
+      sessions: () => [{ id: 's-a', browserPid: async () => 10 }],
+      reader: null,
+      repeat: timer.repeat,
+    });
+    await sampler.sample();
+    expect(timer.timers).toEqual([]);
+    expect(sampler.latest().size).toBe(0);
+    sampler.stop();
+  });
+
+  it('reports a failing read and keeps the previous sample', async () => {
+    const errors: unknown[] = [];
+    let fail = false;
+    const sampler = startBrowserMemorySampler({
+      sessions: () => [{ id: 's-a', browserPid: async () => 10 }],
+      reader: {
+        rssOfTrees: async () => {
+          if (fail) throw new Error('boom');
+          return new Map([[10, 5]]);
+        },
+      },
+      repeat: manualTimer().repeat,
+      onError: (err) => errors.push(err),
+    });
+    await sampler.sample();
+    fail = true;
+    await sampler.sample();
+    expect(errors).toHaveLength(1);
+    expect([...sampler.latest()]).toEqual([['s-a', 5]]);
+    sampler.stop();
+  });
+});

packages/browserhive/src/composition/adapters/browser-memory.ts

@@ -0,0 +1,77 @@
+/** @module composition/adapters/browser-memory — samples the resident memory of each live session's browser process tree every 10 s for `browserhive.browser.rss_bytes` (spec 10 §7). Only started when metrics are exported. */
+
+import type { ProcessTreeReader } from '@browserhive/core/runtime';
+import { BROWSER_RSS_SAMPLE_INTERVAL_MS } from '@browserhive/core/runtime';
+import type { EveryFn } from './timers.ts';
+
+/** A live session as the sampler sees it. */
+export interface SampledSession {
+  readonly id: string;
+  /** The browser's main pid (cached by the handle); absent before launch and in fakes. */
+  readonly browserPid?: () => Promise<number | null>;
+}
+
+/** Dependencies of {@link startBrowserMemorySampler}. */
+export interface BrowserMemorySamplerDeps {
+  /** The sessions to sample now. */
+  readonly sessions: () => Iterable<SampledSession>;
+  /** `null` where the platform has no reader (Windows): the sampler then never runs. */
+  readonly reader: ProcessTreeReader | null;
+  /** Repeating timer (`createTimers().every`). */
+  readonly repeat: EveryFn;
+  readonly intervalMs?: number;
+  readonly onError?: (error: unknown) => void;
+}
+
+/** A running sampler. */
+export interface BrowserMemorySampler {
+  /** RSS bytes per session id from the latest sample. */
+  latest(): ReadonlyMap<string, number>;
+  /** Takes one sample now (the timer calls this). */
+  sample(): Promise<void>;
+  stop(): void;
+}
+
+/** Starts sampling; the first sample is taken at once so the first export has data. */
+export function startBrowserMemorySampler(deps: BrowserMemorySamplerDeps): BrowserMemorySampler {
+  let latest: ReadonlyMap<string, number> = new Map();
+  let running: Promise<void> | undefined;
+  const reader = deps.reader;
+  const take = async (): Promise<void> => {
+    if (reader === null) return;
+    const roots = new Map<number, string>();
+    for (const session of deps.sessions()) {
+      const pid = await session.browserPid?.().catch(() => null);
+      if (pid !== null && pid !== undefined) roots.set(pid, session.id);
+    }
+    const sums = await reader.rssOfTrees([...roots.keys()]);
+    const next = new Map<string, number>();
+    for (const [pid, bytes] of sums) {
+      const id = roots.get(pid);
+      if (id !== undefined) next.set(id, bytes);
+    }
+    latest = next;
+  };
+  const sample = (): Promise<void> => {
+    // Coalesce: a slow table read never stacks samples.
+    running ??= take()
+      .catch((err: unknown) => deps.onError?.(err))
+      .finally(() => {
+        running = undefined;
+      });
+    return running;
+  };
+  const cancel =
+    reader === null
+      ? () => undefined
+      : deps.repeat(() => void sample(), deps.intervalMs ?? BROWSER_RSS_SAMPLE_INTERVAL_MS);
+  void sample();
+  return {
+    latest: () => latest,
+    sample,
+    stop() {
+      cancel();
+      latest = new Map();
+    },
+  };
+}