feat: a window says what changed (ADR 0089)

cmd/agent/main.go

@@ -303,6 +303,9 @@ func run(ctx context.Context, logger *slog.Logger, clientset kubernetes.Interfac
 	// that ended leaves no field behind in a payload that supersedes, so the
 	// only place it can be reported is a window (ADR 0088).
 	stateJournal := journal.NewStates(collector.UsageWindowLength)
+	// And its counters as change rather than as totals, on the same windows and
+	// for the same reason: a total says how much and never when (ADR 0089).
+	counterJournal := journal.NewCounters(collector.UsageWindowLength)
 	podWatcher.OnJobFinished(func(r model.JobRun) {
 		logger.Info("job run finished",
 			"namespace", r.Namespace,
@@ -431,6 +434,7 @@ func run(ctx context.Context, logger *slog.Logger, clientset kubernetes.Interfac
 			jobs:        jobJournal,
 			goroutines:  goroutineJournal,
 			states:      stateJournal,
+			counters:    counterJournal,
 		}, time.Now())
 	}
 
@@ -880,6 +884,17 @@ func run(ctx context.Context, logger *slog.Logger, clientset kubernetes.Interfac
 				logger.Error("spooling agent states", "error", err)
 			}
 		}
+		// The same readings the snapshot just carried, as the change since the
+		// previous pass. `startedAt` is the base they were counted from, and a
+		// base that moved is a restart the subtraction must not cross.
+		counterJournal.Observe(now, startedAt,
+			agentCounters(c, usagePoller.Observation(), pullCoverage, rejections))
+		counts := append(counterJournal.CloseBefore(now), counterJournal.Snapshots()...)
+		if len(counts) > 0 {
+			if err := spool.WriteAgentCounters(now, counts); err != nil {
+				logger.Error("spooling agent counters", "error", err)
+			}
+		}
 	}
 
 	// Watchers run until ctx is canceled; a failing watcher must bring the

cmd/agent/recover.go

@@ -9,12 +9,13 @@ import (
 	"github.com/RebuildStackCo/runtime-agent/internal/sink"
 )
 
-// journals is the six accumulators a restart resumes beside the usage poller.
+// journals is the seven accumulators a restart resumes beside the usage poller.
 type journals struct {
 	restarts    *journal.Restarts
 	disruptions *journal.Disruptions
 	nodes       *journal.NodeEvents
 	states      *journal.States
+	counters    *journal.Counters
 	jobs        *journal.JobRuns
 	goroutines  *journal.Goroutines
 }
@@ -48,7 +49,8 @@ func seedOpenWindows(logger *slog.Logger, spool *sink.Spool, poller *collector.U
 		j.nodes.Resume(recovery.NodeEvents) +
 		j.jobs.Resume(recovery.JobRuns) +
 		j.goroutines.Resume(recovery.GoroutineCounts) +
-		j.states.Resume(recovery.AgentStates)
+		j.states.Resume(recovery.AgentStates) +
+		j.counters.Resume(recovery.AgentCounters)
 	if recovery.JournalWindows > 0 {
 		logger.Info("resumed open journal windows from the spool",
 			"windows", recovery.JournalWindows, "records", seeded)

cmd/agent/states.go

@@ -1,8 +1,11 @@
 package main
 
 import (
+	"math"
+
 	"github.com/RebuildStackCo/runtime-agent/internal/journal"
 	"github.com/RebuildStackCo/runtime-agent/internal/model"
+	"github.com/RebuildStackCo/runtime-agent/internal/pprofpull"
 )
 
 // agentStates is the pass's own states in the shape the window accumulates,
@@ -32,3 +35,55 @@ func agentStates(sources []model.SourceHealth, shipping *model.Shipping) []journ
 	}
 	return out
 }
+
+// agentCounters is what this pass read of the agent's own counters, keyed by
+// the path each number has where it is reported cumulatively (ADR 0089 §2).
+//
+// Only monotonic counters appear, and ADR 0089 §2 names what each exclusion
+// was excluded for.
+func agentCounters(filter model.Coverage, obs model.Observation,
+	pull *pprofpull.Coverage, intake *model.IntakeRejections,
+) map[string]int64 {
+	out := map[string]int64{
+		"filter.pods_observed":                      filter.PodsObserved,
+		"filter.excluded_namespace_filter":          filter.ExcludedNamespaceFilter,
+		"filter.excluded_namespace_annotation":      filter.ExcludedNamespaceAnnotation,
+		"filter.excluded_workload_annotation":       filter.ExcludedWorkloadAnnotation,
+		"filter.excluded_pod_annotation":            filter.ExcludedPodAnnotation,
+		"filter.excluded_profiling_annotation":      filter.ExcludedProfilingAnnotation,
+		"filter.workload_unknown_kind":              filter.WorkloadUnknownKind,
+		"filter.workload_not_cached":                filter.WorkloadNotCached,
+		"filter.jobs_observed":                      filter.JobsObserved,
+		"filter.jobs_excluded_namespace_filter":     filter.JobsExcludedNamespaceFilter,
+		"filter.jobs_excluded_namespace_annotation": filter.JobsExcludedNamespaceAnnotation,
+		"filter.jobs_excluded_workload_annotation":  filter.JobsExcludedWorkloadAnnotation,
+		"filter.jobs_excluded_annotation":           filter.JobsExcludedAnnotation,
+		"observation.polls_attempted":               obs.PollsAttempted,
+		"observation.polls_failed":                  obs.PollsFailed,
+	}
+	// Present only when the component that counts them is deployed. A counter
+	// absent from the map writes no record, which is the same claim its block
+	// makes by being absent from the coverage payload.
+	if pull != nil {
+		out["pprof_pull.shipped"] = int64(pull.Shipped)
+		out["pprof_pull.refused"] = int64(pull.Refused)
+		out["pprof_pull.unreachable"] = int64(pull.Unreachable)
+		out["pprof_pull.invalid"] = int64(pull.Invalid)
+	}
+	if intake != nil {
+		out["intake_rejections.unauthorized"] = asCount(intake.Unauthorized)
+		out["intake_rejections.too_large"] = asCount(intake.TooLarge)
+		out["intake_rejections.malformed"] = asCount(intake.Malformed)
+	}
+	return out
+}
+
+// asCount narrows a counter the model keeps unsigned. Saturating rather than
+// wrapping: a wrapped value would read as a counter that fell and be dropped
+// in silence, where a pinned one stays visibly wrong (ADR 0089 §4).
+func asCount(n uint64) int64 {
+	if n > math.MaxInt64 {
+		return math.MaxInt64
+	}
+	return int64(n)
+}

docs/adr/0054-coverage-says-how-much-was-hidden-never-what.md

@@ -4,7 +4,7 @@ Date: 2026-08-30
 
 Status: Accepted
 
-Amended by: 0057, 0058, 0060, 0087, 0088
+Amended by: 0057, 0058, 0060, 0087, 0088, 0089
 
 Amends: 0012 §2, 0033
 

docs/adr/0088-a-window-says-how-long-a-state-held.md

@@ -4,6 +4,8 @@ Date: 2026-09-18
 
 Status: Accepted
 
+Amended by: 0089
+
 Amends: 0054, 0087
 
 Adds the payload kind `agent_states` and a sixth journal accumulator, resumed at

docs/adr/0089-a-window-says-what-changed.md

@@ -0,0 +1,122 @@
+# 0089. A window says what changed
+
+Date: 2026-09-19
+
+Status: Accepted
+
+Amends: 0054, 0088
+
+Adds the payload kind `agent_counters` and a seventh journal accumulator,
+resumed at startup on ADR 0077's terms. No new collection, no new privilege, no
+new read: its input is the counters `collection_coverage` already carries, taken
+from the same pass.
+
+## Context
+
+ADR 0088 gave the agent's two states a window and named what it left: the
+counters have the same blind spot for the same reason, and subtracting a
+previous reading is a different mechanism. This is that mechanism.
+
+Every number in `collection_coverage` is cumulative from `since`. That answers
+how much and never when:
+
+```
+filter.excluded_namespace_annotation: 12
+```
+
+A namespace annotated an hour ago and one annotated three weeks ago produce the
+same twelve. The customer's data stops arriving for a workload and the counter
+that explains it cannot say whether the explanation is today's.
+
+Differencing successive payloads on the other side does not rescue it. The kind
+supersedes — the contract says upsert, last write wins — so keeping a history is
+something the backend was never asked for; a restart moves `since` and starts
+the counters over, so any interval spanning one is unrecoverable; and the
+resolution would be whatever the backend happened to retain.
+
+Where a subtraction belongs was settled once already. ADR 0062 moved the
+`/proc` counter subtraction onto the node because the node holds both readings
+and the key that makes them comparable, and set the rule that matters more than
+the arithmetic: three cases produce **no delta rather than a wrong one**. The
+same shape applies here, with `since` as the key instead of a process start
+time.
+
+## Decision
+
+**1. A window kind for the agent's own counters.** `agent_counters`, one record
+per `(window, counter)`, hour-aligned, superseding within its window, written
+every pass — the shape `agent_states` already has, so it resumes through the
+same machinery and the backend implements one thing twice rather than two
+things.
+
+**2. The counter's name is its path where it is reported cumulatively.**
+`filter.excluded_pod_annotation`, `pprof_pull.refused`,
+`intake_rejections.malformed`. The name is the join, so this payload says when
+and never restates what: the meaning of each number stays in the one place that
+already documents it, which is the rule ADR 0024 set for prose and holds for
+schemas.
+
+**3. What is counted, and what each exclusion was excluded for.** The filter's
+pod and Job counters, the puller's four outcomes, the receiver's three
+rejections, and the kubelet poll attempts and failures. Left out:
+
+- the placement and node drops — they describe the shape of the data, not
+  behaviour over time;
+- the endpoint and goroutine coverages — they count targets by their latest
+  answer, which is a snapshot and not a total, and subtracting two snapshots
+  produces a number with no referent;
+- the inventory, scan and eBPF blocks — fleet aggregates over per-node
+  counters, which nothing may subtract across a changing set of nodes;
+- the shipping counters — the one class whose reader can already date them. The
+  backend issued those statuses itself and saw the gaps in `captured_at`
+  itself, so a window here would tell it what it wrote down.
+
+**4. The subtraction is keyed by `since`, and refuses rather than guesses.** A
+pass whose base differs from the previous pass's accrues nothing: the readings
+belong to different processes and are not subtractable. A counter that fell
+under an unchanged base accrues nothing either — whatever produced it, the
+reading is not of the series the last one described. Both are ADR 0062 §1, with
+`since` in the role of the process start time.
+
+**5. The interval splits at a window boundary; the rise does not.** Observation
+is a duration and is attributed to both windows it spans. A rise is not: it
+happened at an instant no pass can see, so it lands whole in the window the pass
+observed it in. Splitting it by elapsed time would be inventing the instant, and
+the error is bounded by one pass, which is one minute.
+
+**6. Observation travels with every counter, including one that did not move.**
+A record with a zero delta says the agent was running and nothing happened; no
+record at all says nobody was counting. That distinction is the reason a window
+with nothing in it is still written, and it is ADR 0054's founding one.
+
+## What this still cannot say
+
+- **A restart loses the rises inside its own gap.** The new process's counters
+  start at zero, so there is no reading anywhere that spans the outage: what
+  happened while the agent was down is counted by nobody. `observed_nanos` is
+  what makes the hole visible rather than silent.
+- **A rise within one pass of a window boundary may land in the later window.**
+  Bounded by the pass interval and not correctable without an instant the agent
+  does not have.
+- **A delta is not a count of objects.** The counters count decisions, not pods
+  (ADR 0054), and a window of them inherits that exactly.
+
+## Consequences
+
+**Easier.** "This workload's data stopped on Tuesday — why?" gets an answer in
+the same hour as the question: the exclusion counter that explains it moved in
+that window or it did not. The same for profiles that stopped arriving and for
+a node whose reports began being refused.
+
+**Harder / given up.** An eighth kind in the startup read and a second payload
+per hour whose subject is the agent. Both are records of a few dozen bytes; what
+they cost is a reader's attention, which is why the counter list is closed and
+argued above rather than "every number we have".
+
+**Not changed.** What is collected about the cluster, what is filtered, what is
+read, any other kind's cadence or key, or the one-way protocol. Nothing here
+names an object: a counter name is a counter name, and its value is a number of
+decisions.
+
+This ADR records decisions implemented in the same pull request, per the process
+in [`README.md`](README.md).

docs/adr/README.md

@@ -146,3 +146,4 @@ line to paste.
 - [0086. A toolchain move is a review of what the toolchain reports](0086-a-toolchain-move-reviews-what-it-reports.md)
 - [0087. A state says when it began](0087-a-state-says-when-it-began.md)
 - [0088. A window says how long a state held](0088-a-window-says-how-long-a-state-held.md)
+- [0089. A window says what changed](0089-a-window-says-what-changed.md)

docs/backend-requirements.md

@@ -500,6 +500,27 @@ same rule applies — one is a reading, the other a window.
   `shipping`. No object in the customer's cluster appears here at any window
   length, and the backend MUST NOT ask for one.
 
+**`agent_counters` is the same counters as change, and MUST NOT be added to
+them** ([ADR 0089](adr/0089-a-window-says-what-changed.md)). One record per
+(window, counter), superseding by (window start, window length) like the kind
+above. `counter` is the dotted path that number has in the payload reporting it
+cumulatively — `filter.excluded_pod_annotation`, `pprof_pull.refused` — so the
+name is the join and nothing here restates what the number means.
+
+- `delta` is the rise within the window. A rise within one pass of a boundary
+  may land in the later window, so the backend MUST NOT align a delta to a
+  sub-window instant.
+- **`observed_nanos` bounds it, and a window the agent was absent for part of
+  has a smaller delta for reasons about the agent.** What rose while the agent
+  was down is counted by nobody: its counters restart at zero, so no reading
+  spans the outage, and the backend MUST NOT treat a gap as quiet.
+- A counter present in one window and absent from the next was not being
+  counted, which is not the same as counting zero. The two are distinguished by
+  the record existing at all.
+- Deltas count decisions, not objects — the rule `collection_coverage`'s filter
+  counters already carry — so the backend MUST NOT recover an object count by
+  summing them.
+
 **`network_window` says how much moved, never where it went**
 ([ADR 0053](adr/0053-network-counters-are-the-pods.md)). The counters are read at
 the pod's own interfaces, so the payload carries no destination, no peer and no

docs/prose_test.go

@@ -36,22 +36,22 @@ var ceilings = []struct {
 }{
 	// Each raise names the obligation that bought it: payloads leave the cluster
 	// (0075); a window states when it was written (0078); an endpoint is read
-	// once a minute (0080); GODEBUG names come out of the build (0081); a
-	// request names its agent (0083); a listener names what the filters excluded
-	// (0084); the image states what it is made of (0085); a failing read says
-	// since when (0087); and a window says how long a state held (0088), whose
-	// row is the section's raise and whose diff moves the startup-read list to
-	// the seven kinds it reads.
-	{"security.md", 1139, 367},
+	// once a minute (0080); GODEBUG names leave the build (0081); a request
+	// names its agent (0083); a listener names what the filters excluded (0084);
+	// the image states what it is made of (0085); a failing read says since when
+	// (0087); and two windows say how long a state held and what changed (0088,
+	// 0089) — their rows raise the section, and their diffs move the
+	// startup-read list to the eight kinds it reads.
+	{"security.md", 1140, 369},
 	// Each raise names its obligation here too. A payload's size is no longer
 	// its request's (0076); a window states whether it is finished (0078); gauge
-	// samples arrive, where adding them up is the wrong ingest (0080); `godebug`
-	// carries names where neither value is safe (0081); the filter counters
-	// count decisions (0054); a port's `pprof` is absent when nothing asked
-	// (0082); a request may name an agent with no version (0083); an absent
-	// instant is told apart from an absent state (0087); and a duration arrives
-	// with the observation bounding it (0088). The last two moved the section.
-	{"backend-requirements.md", 851, 673},
+	// samples are not summed (0080); `godebug` carries names where neither value
+	// is safe (0081); the filter counters count decisions (0054); a port's
+	// `pprof` is absent when nothing asked (0082); an agent may state no version
+	// (0083); an absent instant differs from an absent state (0087); a duration
+	// arrives with the observation bounding it (0088); and a delta says which
+	// hour it belongs to (0089). The last three moved the section too.
+	{"backend-requirements.md", 872, 694},
 }
 
 func TestNoCommentRunIsLongerThanAPointerToAnADR(t *testing.T) {

docs/security.md

@@ -272,9 +272,9 @@ and 20000 files, dropping the oldest first; and the `emptyDir` declares a 1 GiB
 - **The agent reads it once, at startup**, so a restart resumes its open windows
   rather than starting them over ([ADR 0072](adr/0072-a-restart-resumes-the-window-the-spool-holds.md),
   [ADR 0077](adr/0077-a-restart-resumes-every-open-window.md)). It is a pure read
-  of seven kinds and no others — `usage_snapshot`, `container_restarts`,
+  of eight kinds and no others — `usage_snapshot`, `container_restarts`,
   `pod_disruptions`, `node_lifecycle`, `job_runs`, `goroutine_counts`,
-  `agent_states` — and it deletes nothing.
+  `agent_states`, `agent_counters` — and it deletes nothing.
 - **Configuration is never cached on it.** Filters are read from the ConfigMap at
   every start and never reread while running; a stale filter set cannot resurrect
   from disk, and a filter change takes a restart.
@@ -557,6 +557,7 @@ nothing to what leaves your cluster.
 | `ebpf_profile` | One capture: allow-list-filtered symbolized pprof bytes, keyed by workload, image digest and capture window | no |
 | `pprof_profile` | One ten-second CPU profile fetched from the workload's own `/debug/pprof` endpoint, allow-list-filtered, under the same key ([ADR 0058](adr/0058-the-pull-starts-a-profiler-and-the-binary-says-whose-code-it-is.md)). The mappings — the executable's path on disk and its build ID — the sample labels, and the full source paths are removed with the redacted frames, so what remains is function names your allow-list admits, base file names and line numbers. Beside it travel the counts of what was redacted, never what | no |
 | `agent_states` | Per window, for each of the agent's own states: how much of the hour it held, how many episodes that was, and how much of the hour the agent was running to watch at all ([ADR 0088](adr/0088-a-window-says-how-long-a-state-held.md)). Its subjects are this agent and the resource classes it watches — `services`, `endpoint_slices` — never an object in your cluster. Nothing here is about your workloads: it is how the agent answers for its own outages | no |
+| `agent_counters` | Per window, for each of the agent's own counters: how much it rose within the hour, and how much of the hour the agent was running to count ([ADR 0089](adr/0089-a-window-says-what-changed.md)). The same decisions the coverage report totals, placed in time — how many exclusions your annotations caused this hour, not since the agent started. Counts only, and no object is named | no |
 | `goroutine_counts` | Per sampled process and window: how many goroutines it was running at each reading, and the pod, container and build it was read from ([ADR 0080](adr/0080-the-index-page-already-counts-the-goroutines.md)). A count and a timestamp. No stack, no function name and nothing about what those goroutines were doing | no |
 
 Each declares its provenance in a `source` field — `structural` (read from a