feat: a window says how long a state held (ADR 0088)

cmd/agent/main.go

@@ -298,6 +298,11 @@ func run(ctx context.Context, logger *slog.Logger, clientset kubernetes.Interfac
 	// supply for a workload that ran for part of a window (ADR 0029).
 	jobJournal := journal.NewJobRuns(collector.UsageWindowLength)
 	nodeJournal := journal.NewNodeEvents(collector.UsageWindowLength)
+	// The agent's own states over time. Its subject is this agent rather than
+	// the cluster, and it is a window for the reason the others are: a state
+	// that ended leaves no field behind in a payload that supersedes, so the
+	// only place it can be reported is a window (ADR 0088).
+	stateJournal := journal.NewStates(collector.UsageWindowLength)
 	podWatcher.OnJobFinished(func(r model.JobRun) {
 		logger.Info("job run finished",
 			"namespace", r.Namespace,
@@ -425,6 +430,7 @@ func run(ctx context.Context, logger *slog.Logger, clientset kubernetes.Interfac
 			nodes:       nodeJournal,
 			jobs:        jobJournal,
 			goroutines:  goroutineJournal,
+			states:      stateJournal,
 		}, time.Now())
 	}
 
@@ -855,11 +861,25 @@ func run(ctx context.Context, logger *slog.Logger, clientset kubernetes.Interfac
 			sc := ship.Coverage()
 			shipping = &sc
 		}
-		if err := spool.WriteCollectionCoverage(time.Now(), startedAt, agentInfo,
-			podWatcher.SourceHealths(), c, podWatcher.PlacementDrops(), nodeWatcher.Drops(), inv, scan,
+		// One reading of the clock stamps the snapshot and bounds the window's
+		// accrual, so the two cannot describe different instants (ADR 0078 §2).
+		now := time.Now()
+		sources := podWatcher.SourceHealths()
+		if err := spool.WriteCollectionCoverage(now, startedAt, agentInfo,
+			sources, c, podWatcher.PlacementDrops(), nodeWatcher.Drops(), inv, scan,
 			ebpf, probeCoverage, pullCoverage, countCoverage, rejections, shipping); err != nil {
 			logger.Error("spooling collection coverage", "error", err)
 		}
+		// The same two states the snapshot just carried, folded into the window
+		// that will outlive them: what holds now is above, how long it held and
+		// how many episodes it was is here (ADR 0088).
+		stateJournal.Observe(now, agentStates(sources, shipping))
+		records := append(stateJournal.CloseBefore(now), stateJournal.Snapshots()...)
+		if len(records) > 0 {
+			if err := spool.WriteAgentStates(now, records); err != nil {
+				logger.Error("spooling agent states", "error", err)
+			}
+		}
 	}
 
 	// Watchers run until ctx is canceled; a failing watcher must bring the

cmd/agent/recover.go

@@ -9,11 +9,12 @@ import (
 	"github.com/RebuildStackCo/runtime-agent/internal/sink"
 )
 
-// journals is the five accumulators a restart resumes beside the usage poller.
+// journals is the six accumulators a restart resumes beside the usage poller.
 type journals struct {
 	restarts    *journal.Restarts
 	disruptions *journal.Disruptions
 	nodes       *journal.NodeEvents
+	states      *journal.States
 	jobs        *journal.JobRuns
 	goroutines  *journal.Goroutines
 }
@@ -46,7 +47,8 @@ func seedOpenWindows(logger *slog.Logger, spool *sink.Spool, poller *collector.U
 		j.disruptions.Resume(recovery.Disruptions) +
 		j.nodes.Resume(recovery.NodeEvents) +
 		j.jobs.Resume(recovery.JobRuns) +
-		j.goroutines.Resume(recovery.GoroutineCounts)
+		j.goroutines.Resume(recovery.GoroutineCounts) +
+		j.states.Resume(recovery.AgentStates)
 	if recovery.JournalWindows > 0 {
 		logger.Info("resumed open journal windows from the spool",
 			"windows", recovery.JournalWindows, "records", seeded)

cmd/agent/states.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"github.com/RebuildStackCo/runtime-agent/internal/journal"
+	"github.com/RebuildStackCo/runtime-agent/internal/model"
+)
+
+// agentStates is the pass's own states in the shape the window accumulates,
+// built from the values the coverage snapshot was just written from so the two
+// cannot disagree (ADR 0088 §2).
+//
+// Every watched source contributes, failing or not: a source with no record is
+// one nobody watched, and that is the distinction the window exists to keep.
+func agentStates(sources []model.SourceHealth, shipping *model.Shipping) []journal.AgentState {
+	out := make([]journal.AgentState, 0, len(sources)+1)
+	for _, src := range sources {
+		out = append(out, journal.AgentState{
+			State:   journal.StateFailing,
+			Subject: src.Name,
+			Since:   src.FailingSince,
+		})
+	}
+	// Absent when the agent has no backend: an agent told to ship nowhere has
+	// not stopped shipping, and an hour of "not halted" would claim it had a
+	// backend to be halted from.
+	if shipping != nil {
+		out = append(out, journal.AgentState{
+			State:   journal.StateHalted,
+			Subject: journal.SubjectShip,
+			Since:   shipping.HaltedSince,
+		})
+	}
+	return out
+}

docs/adr/0054-coverage-says-how-much-was-hidden-never-what.md

@@ -4,7 +4,7 @@ Date: 2026-08-30
 
 Status: Accepted
 
-Amended by: 0057, 0058, 0060, 0087
+Amended by: 0057, 0058, 0060, 0087, 0088
 
 Amends: 0012 §2, 0033
 

docs/adr/0087-a-state-says-when-it-began.md

@@ -4,6 +4,8 @@ Date: 2026-09-18
 
 Status: Accepted
 
+Amended by: 0088
+
 Amends: 0054, 0067
 
 `sources[].failing` and `shipping.halted` gain a start instant beside them, and

docs/adr/0088-a-window-says-how-long-a-state-held.md

@@ -0,0 +1,131 @@
+# 0088. A window says how long a state held
+
+Date: 2026-09-18
+
+Status: Accepted
+
+Amends: 0054, 0087
+
+Adds the payload kind `agent_states` and a sixth journal accumulator, resumed at
+startup on ADR 0077's terms. No new collection, no new privilege, no new read:
+its input is the two states `collection_coverage` already carries, taken from
+the same call on the same pass.
+
+## Context
+
+ADR 0087 gave `sources[].failing` and `shipping.halted` the instant they began,
+and said plainly what it did not do: a state that ended is absent from the next
+payload by design. That is not an oversight in the field — it is what a state
+field is. It describes the present, and this kind supersedes, so the backend is
+entitled to keep one row.
+
+So the defect ADR 0087 closed was half of one. "Is it broken, and since when" is
+answerable. "Was it broken" is not:
+
+```
+13:47  failing_since: 13:47     the payload says so
+14:06  (no field)               recovered — and the row is clean
+                                nineteen minutes are now unrecoverable
+```
+
+Two different questions hide under one word. A state that **holds** is a fact
+about now, and a boolean with a start is the right shape for it. A state that
+**held** is a fact about an interval, and no field describing the present can
+carry it.
+
+This repository has already answered a question of exactly this shape.
+`container_restarts` is a window and `restart_counters` is a reading, and the
+contract carries the sentence that keeps them apart: *"`restart_counters` is a
+reading, and MUST NOT be added to the windows"* (ADR 0034). One fact in two
+shapes, with the reader told which is which. The six window kinds beside it
+share one key — `(window start, window length)`, hour-aligned — and ADR 0078
+made an open window declare itself open. The agent's own behaviour was the last
+subject still reported only in the pre-window shape.
+
+What is deliberately **not** in scope is the counters. `filter.excluded_*`,
+`shipping.deferred`, the puller's outcomes and the intake rejections are
+cumulative from `since` and have the same blind spot for the same reason, but
+subtracting a previous reading is a different mechanism with its own failure
+modes (ADR 0062 priced them), and it is not decided here.
+
+## Decision
+
+**1. A window kind for the agent's own states.** `agent_states`, one record per
+`(window, state, subject)`, hour-aligned, superseding within its window, written
+every pass as a slice of the open window on ADR 0078's terms. Subjects are the
+resource classes the agent watches and the agent's own shipping; no object in
+the customer's cluster is nameable here, at any resolution.
+
+**2. Occupancy and episodes, not events.** Each record carries how many
+nanoseconds of the window the subject spent in the state and how many episodes
+began in it. Two numbers rather than a record per transition, because the cost
+is then bounded by construction at whatever flap rate — and because one
+nineteen-minute outage and three six-minute ones are different diagnoses that a
+single duration would merge.
+
+**3. Accrual moves forward, and never re-measures.** A pass attributes the
+interval since the previous pass, split at any window boundary it crosses.
+`accrued_to` is how far a record is true, and is where a later process resumes
+from; `open_since` is the marker that tells that process the episode it finds
+open is the one already counted rather than a new one. Occupancy is never
+recomputed as `now - open_since` — that expression is correct exactly once and
+double-counts every span before a restart, which is the whole reason the marker
+exists rather than a start instant alone.
+
+**4. Sampling at the pass loses nothing here, and that is measured.** A pass
+runs every minute (`everyPass`), and a run of watch failures becomes a new
+episode only after two minutes of quiet (`watchStreakGap`). Two episodes
+therefore cannot both begin and end between two passes, and a single one that
+does is still seen, because the state is defined with a three-minute tail. The
+halt cannot flap at all: it latches until the process ends.
+
+**5. Observation travels beside occupancy.** Zero occupancy against a full
+window is a healthy hour; zero against zero observation is an hour nobody
+watched. Without the second number they are the same bytes, which is ADR 0054's
+founding defect arriving in a new shape. A restart's gap is in neither: the
+first pass of a process accrues nothing, so time the agent was not running is
+never claimed as time it was watching.
+
+**6. Every watched subject writes a record, including a clear one.** A subject
+with no record is one nobody watched. This is why a window with nothing wrong in
+it is still written, where the journals beside this one write nothing.
+
+**7. An episode is counted where it was seen, not where it began.** Back-dating
+an entry to the instant the episode started would reopen a window that had
+already ended and been written final. The instant is not lost — it is in
+`open_since` and in the occupancy split across the boundary.
+
+## What this still cannot say
+
+Stated here rather than discovered later:
+
+- **A restart across a window boundary adds one entry.** The earlier window has
+  ended and is not reopened (ADR 0077 §1), so the continuing episode is seen as
+  new. It is detectable: `since` in `collection_coverage` moves at the same
+  moment, so an entry adjacent to a moved `since` is suspect.
+- **An episode that closed between two passes undercounts by up to one pass.**
+  The instant it closed is not knowable from two readings, and the alternative
+  is to invent it.
+- **The marker survives a process restart, not a pod replacement.** The spool is
+  an `emptyDir` by decision (ADR 0007, ADR 0026); a rescheduled agent resumes
+  nothing, as ADR 0077 §4 already priced.
+
+## Consequences
+
+**Easier.** An outage that ended is reportable for the first time. "No finding
+from EndpointSlices this week" acquires an answer that is neither a guess nor a
+trip into the customer's logs, and `observed_nanos` makes every finding that
+rests on a window quotable with the coverage it actually had.
+
+**Harder / given up.** A seventh kind in the startup read, so `security.md`'s
+closed list moves again — the mechanism ADR 0077 declared and ADR 0080 then
+failed to use. One file per hour of a handful of records: negligible in bytes,
+and one more payload the backend implements.
+
+**Not changed.** What is collected about the cluster, what is filtered, what is
+read, any other kind's cadence or key, or the one-way protocol. The two states
+this measures are the two ADR 0087 named, read from the same call that writes
+them to `collection_coverage`, so the window and the snapshot cannot disagree.
+
+This ADR records decisions implemented in the same pull request, per the process
+in [`README.md`](README.md).

docs/adr/README.md

@@ -145,3 +145,4 @@ line to paste.
 - [0085. The image is built and scanned where the code is, and pinned to what it was built from](0085-the-image-is-built-and-scanned-by-ci.md)
 - [0086. A toolchain move is a review of what the toolchain reports](0086-a-toolchain-move-reviews-what-it-reports.md)
 - [0087. A state says when it began](0087-a-state-says-when-it-began.md)
+- [0088. A window says how long a state held](0088-a-window-says-how-long-a-state-held.md)

docs/backend-requirements.md

@@ -478,6 +478,28 @@ agent failed on.
 No field of this payload names an object the customer excluded, and the backend
 MUST NOT ask for one.
 
+**`agent_states` is the same two states as time, and MUST NOT be added to them**
+([ADR 0088](adr/0088-a-window-says-how-long-a-state-held.md)). One record per
+(window, state, subject), superseding by (window start, window length): the
+backend MUST upsert and MUST NOT append. `collection_coverage` says which states
+hold now; this says how much of each hour they held and how many episodes that
+was. The relationship is `container_restarts` to `restart_counters`, and the
+same rule applies — one is a reading, the other a window.
+
+- `occupied_nanos` and `entries` are independent and neither derives the other:
+  one long episode and several short ones can share a duration. Summing
+  `entries` across windows counts episodes, because an episode continuing from
+  the previous window contributes occupancy here and no entry.
+- **`observed_nanos` bounds both, and the backend MUST NOT read occupancy
+  without it.** Zero occupancy over a full window is a healthy hour; zero over
+  zero observation is an hour nobody watched. A restart's gap is in neither
+  number, which is what makes the second one the honest denominator.
+- A restart that crosses a window boundary may add one entry to a continuing
+  episode. It is detectable: `collection_coverage.since` moves at the same time.
+- `subject` is a resource class the agent watches or the agent's own
+  `shipping`. No object in the customer's cluster appears here at any window
+  length, and the backend MUST NOT ask for one.
+
 **`network_window` says how much moved, never where it went**
 ([ADR 0053](adr/0053-network-counters-are-the-pods.md)). The counters are read at
 the pod's own interfaces, so the payload carries no destination, no peer and no

docs/prose_test.go

@@ -35,23 +35,23 @@ var ceilings = []struct {
 	section int
 }{
 	// Each raise names the obligation that bought it: payloads leave the cluster
-	// at all (0075); a journal window states when it was written (0078); a
-	// confirmed endpoint is read once a minute per process (0080); twelve more
-	// GODEBUG names come out of the build (0081); a request names the agent that
-	// made it (0083); a listener names inside the cluster what the filters
-	// excluded (0084); the image states what it is made of and what does not yet
-	// back it (0085); a failing read says since when (0087), whose diff also
-	// corrects the startup-read list from five kinds to the six it reads.
-	{"security.md", 1138, 366},
-	// Each raise names its obligation here too. The body is encoded, so a
-	// payload's size is no longer the request's (0076); a window states whether
-	// it is finished (0078); gauge samples arrive, where adding them up is the
-	// wrong ingest (0080); `godebug` carries names where neither value is safe
-	// (0081); the filter counters count decisions (0054); a port's `pprof` is
-	// absent when nothing asked, not `absent` (0082); a request may name an
-	// agent that states no version (0083); and an absent instant is told apart
-	// from an absent state (0087) — the raise that also moved the section.
-	{"backend-requirements.md", 829, 651},
+	// (0075); a window states when it was written (0078); an endpoint is read
+	// once a minute (0080); GODEBUG names come out of the build (0081); a
+	// request names its agent (0083); a listener names what the filters excluded
+	// (0084); the image states what it is made of (0085); a failing read says
+	// since when (0087); and a window says how long a state held (0088), whose
+	// row is the section's raise and whose diff moves the startup-read list to
+	// the seven kinds it reads.
+	{"security.md", 1139, 367},
+	// Each raise names its obligation here too. A payload's size is no longer
+	// its request's (0076); a window states whether it is finished (0078); gauge
+	// samples arrive, where adding them up is the wrong ingest (0080); `godebug`
+	// carries names where neither value is safe (0081); the filter counters
+	// count decisions (0054); a port's `pprof` is absent when nothing asked
+	// (0082); a request may name an agent with no version (0083); an absent
+	// instant is told apart from an absent state (0087); and a duration arrives
+	// with the observation bounding it (0088). The last two moved the section.
+	{"backend-requirements.md", 851, 673},
 }
 
 func TestNoCommentRunIsLongerThanAPointerToAnADR(t *testing.T) {

docs/security.md

@@ -272,9 +272,9 @@ and 20000 files, dropping the oldest first; and the `emptyDir` declares a 1 GiB
 - **The agent reads it once, at startup**, so a restart resumes its open windows
   rather than starting them over ([ADR 0072](adr/0072-a-restart-resumes-the-window-the-spool-holds.md),
   [ADR 0077](adr/0077-a-restart-resumes-every-open-window.md)). It is a pure read
-  of six kinds and no others — `usage_snapshot`, `container_restarts`,
-  `pod_disruptions`, `node_lifecycle`, `job_runs`, `goroutine_counts` — and it
-  deletes nothing.
+  of seven kinds and no others — `usage_snapshot`, `container_restarts`,
+  `pod_disruptions`, `node_lifecycle`, `job_runs`, `goroutine_counts`,
+  `agent_states` — and it deletes nothing.
 - **Configuration is never cached on it.** Filters are read from the ConfigMap at
   every start and never reread while running; a stale filter set cannot resurrect
   from disk, and a filter change takes a restart.
@@ -556,6 +556,7 @@ nothing to what leaves your cluster.
 | `listening_ports` | Per collected workload container and build: the TCP ports its Go processes accept connections on, each with whether it is bound to loopback, and — for a port the agent asked — whether the Go pprof index answered there ([ADR 0082](adr/0082-a-confirmed-endpoint-is-named-on-its-port.md)). Read from the processes' own sockets, so a port nothing declared is here and a declared port nothing binds is not ([ADR 0056](adr/0056-a-pprof-endpoint-is-proved-not-probed.md)). No address, and no connection the workload has open | no |
 | `ebpf_profile` | One capture: allow-list-filtered symbolized pprof bytes, keyed by workload, image digest and capture window | no |
 | `pprof_profile` | One ten-second CPU profile fetched from the workload's own `/debug/pprof` endpoint, allow-list-filtered, under the same key ([ADR 0058](adr/0058-the-pull-starts-a-profiler-and-the-binary-says-whose-code-it-is.md)). The mappings — the executable's path on disk and its build ID — the sample labels, and the full source paths are removed with the redacted frames, so what remains is function names your allow-list admits, base file names and line numbers. Beside it travel the counts of what was redacted, never what | no |
+| `agent_states` | Per window, for each of the agent's own states: how much of the hour it held, how many episodes that was, and how much of the hour the agent was running to watch at all ([ADR 0088](adr/0088-a-window-says-how-long-a-state-held.md)). Its subjects are this agent and the resource classes it watches — `services`, `endpoint_slices` — never an object in your cluster. Nothing here is about your workloads: it is how the agent answers for its own outages | no |
 | `goroutine_counts` | Per sampled process and window: how many goroutines it was running at each reading, and the pod, container and build it was read from ([ADR 0080](adr/0080-the-index-page-already-counts-the-goroutines.md)). A count and a timestamp. No stack, no function name and nothing about what those goroutines were doing | no |
 
 Each declares its provenance in a `source` field — `structural` (read from a