feat: the node is described, and its comings and goings recorded (ADR 0064)

cmd/agent/main.go

@@ -247,6 +247,7 @@ func run(ctx context.Context, logger *slog.Logger, clientset kubernetes.Interfac
 	// the run's own instants and outcome, which is what a usage rollup cannot
 	// supply for a workload that ran for part of a window (ADR 0029).
 	jobJournal := journal.NewJobRuns(collector.UsageWindowLength)
+	nodeJournal := journal.NewNodeEvents(collector.UsageWindowLength)
 	podWatcher.OnJobFinished(func(r collector.JobRun) {
 		logger.Info("job run finished",
 			"namespace", r.Namespace,
@@ -273,8 +274,29 @@ func run(ctx context.Context, logger *slog.Logger, clientset kubernetes.Interfac
 			"allocatable_memory_bytes", n.AllocatableMemoryBytes,
 			"capacity_cpu_milli", n.CapacityCPUMilli,
 			"capacity_memory_bytes", n.CapacityMemoryBytes,
+			"kubelet_version", n.KubeletVersion,
+			"os_image", n.OSImage,
+			"container_runtime", n.ContainerRuntime,
+			"devices", len(n.Devices),
+			"taints", len(n.Taints),
 		)
 	})
+	nodeWatcher.OnNodeLifecycle(func(e collector.NodeLifecycle) {
+		event := "node left"
+		if e.Joined {
+			event = "node joined"
+		}
+		logger.Info(event,
+			"node", e.Node.Name,
+			"at", e.At,
+			"at_observed", e.Observed,
+			"instance_type", e.Node.InstanceType,
+			"capacity_type", e.Node.CapacityType,
+			"capacity_cpu_milli", e.Node.CapacityCPUMilli,
+			"capacity_memory_bytes", e.Node.CapacityMemoryBytes,
+		)
+		nodeJournal.Observe(e)
+	})
 
 	// logRecords writes rollup records one line each; the JSON body is the
 	// exact record shape (kind marks snapshots vs. final closed windows).
@@ -547,6 +569,21 @@ func run(ctx context.Context, logger *slog.Logger, clientset kubernetes.Interfac
 		logger.Info("pod disruptions flushed", "records", len(records))
 	}
 
+	flushNodeLifecycle := func() {
+		if spool == nil {
+			return
+		}
+		records := append(nodeJournal.CloseBefore(time.Now()), nodeJournal.Snapshots()...)
+		if len(records) == 0 {
+			return // a fleet that did not change writes nothing
+		}
+		if err := spool.WriteNodeLifecycle(records); err != nil {
+			logger.Error("spooling node lifecycle", "error", err)
+			return
+		}
+		logger.Info("node lifecycle flushed", "records", len(records))
+	}
+
 	flushJobRuns := func() {
 		if spool == nil {
 			return
@@ -658,7 +695,7 @@ func run(ctx context.Context, logger *slog.Logger, clientset kubernetes.Interfac
 			pullCoverage = &pc
 		}
 		if err := spool.WriteCollectionCoverage(time.Now(), startedAt, agentInfo,
-			podWatcher.SourceHealths(), c, podWatcher.PlacementDrops(), inv, scan,
+			podWatcher.SourceHealths(), c, podWatcher.PlacementDrops(), nodeWatcher.Drops(), inv, scan,
 			ebpf, probeCoverage, pullCoverage); err != nil {
 			logger.Error("spooling collection coverage", "error", err)
 		}
@@ -695,6 +732,7 @@ func run(ctx context.Context, logger *slog.Logger, clientset kubernetes.Interfac
 				flushRestarts()
 				flushDisruptions()
 				flushJobRuns()
+				flushNodeLifecycle()
 				// Unconditionally, on the agent's own cadence rather than the
 				// usage poller's: the spool's bounds must hold on a cluster
 				// where no usage record is ever written, which is exactly the
@@ -817,6 +855,7 @@ func run(ctx context.Context, logger *slog.Logger, clientset kubernetes.Interfac
 	flushRestarts()
 	flushDisruptions()
 	flushJobRuns()
+	flushNodeLifecycle()
 	return errors.Join(errs...)
 }
 

docs/adr/0012-payload-registry-and-provenance.md

@@ -2,7 +2,7 @@
 
 Date: 2026-08-20
 Status: Accepted
-Amended by: 0013, 0014, 0017, 0020, 0021, 0022, 0027, 0054
+Amended by: 0013, 0014, 0017, 0020, 0021, 0022, 0027, 0054, 0064
 
 ## Context
 

docs/adr/0064-the-node-is-described-and-its-comings-recorded.md

@@ -0,0 +1,167 @@
+# 0064. The node is described, and its comings and goings recorded
+
+Date: 2026-09-01
+
+Status: Accepted
+
+Amends: 0012
+
+Widens `node_metadata` from ten fields to a description of the machine, and adds
+`node_lifecycle`, the first payload that says what the fleet *was* rather than
+what it is. No new RBAC — the node informer already lists and watches whole node
+objects — and no new privilege.
+
+## Context
+
+`node_metadata` was drawn for one job: turning a node name into a zone and an
+instance type so a usage record could be attributed. It carried the two sizes,
+four labels and two fields of `status.nodeInfo`, and the type's comment said
+"nothing else is read from node objects" as though that were a promise rather
+than a consequence of nobody having needed more.
+
+Four questions arrive at that payload and leave unanswered.
+
+- **Why a pod will not schedule.** CPU and memory are two of at least five
+  scheduling dimensions. A node can be idle in both and still refuse every pod
+  because it is at its kubelet pod ceiling, out of ephemeral storage, tainted,
+  or `Ready=False`. Every one of those is on the object the agent already holds
+  in its informer cache.
+- **What the expensive machines are.** An accelerator node costs an order of
+  magnitude more than the general-purpose node beside it, and to this payload the
+  two are the same row with different labels. A fleet's GPUs are invisible.
+- **What software the fleet runs.** `kernelVersion` is collected because it
+  decides whether the eBPF profiler can attach. The kubelet version, OS image and
+  container runtime sit in the same struct, are read in the same call, and are
+  what an upgrade plan is written against.
+- **What the fleet used to be.** Every payload the agent ships about nodes is a
+  snapshot that replaces its predecessor. A node that leaves is simply absent
+  from the next one. Fleet churn — the central fact about a cluster running spot
+  capacity or an aggressive autoscaler — is reconstructible only by diffing
+  consecutive snapshots, and only at flush granularity, and only if none was
+  missed.
+
+The last of these is different in kind from the first three. The others are
+fields on an existing payload; this one is a fact no snapshot can hold.
+
+## Decision
+
+### 1. `node_metadata` describes the machine
+
+Age (`metadata.creationTimestamp`), the three remaining `status.nodeInfo`
+strings, and four more resource dimensions: ephemeral storage and pods, both
+capacity and allocatable.
+
+Pods and ephemeral storage are core resources and were left out for no reason
+better than that the first finding did not need them. They join as named fields
+like CPU and memory.
+
+### 2. Three lists, each bounded by an allow-list
+
+The parts of a node object with unbounded shape do not enter as they are. Each
+follows the discipline ADR 0019 set for build settings and ADR 0031 for
+placement: an allow-list rather than a deny-list, a length bound, a drop counted
+rather than a value truncated.
+
+**Conditions** are allow-listed **by type** — the four the kubelet maintains plus
+`NetworkUnavailable`. Kept: status, reason, last transition. Never kept:
+`message`. It is free text, and the two things that write it — the kubelet and
+node-problem-detector — put paths, device names and command output there. The
+type allow-list is what makes `reason` safe to carry: a custom condition's reason
+is written by whoever installed the agent that sets it, and no custom type
+survives the list.
+
+The transition time is kept and the heartbeat is not. The heartbeat moves every
+few seconds on every node in the cluster; carrying it would make every node
+differ from its previous view on every flush, which is both noise in the payload
+and a defeat of the change-detection the informer callback rests on.
+
+All five allow-listed conditions are reported whenever the node reports them,
+healthy or not. A list holding only problems would make "this node is fine" and
+"the agent did not read this" the same bytes, which is exactly what ADR 0048 §2
+forbade for the update strategy.
+
+**Taints** are kept field for field — key, value, effect — with the same 128-byte
+bound as placement values. This is the one place operator-written strings enter
+from the node side, and it is justified by symmetry: tolerations are *already*
+collected from every pod spec, and a toleration without the taint it answers is
+half a fact. It says what a pod would put up with, not what the fleet fences off.
+
+Unlike tolerations, no taint is filtered as a cluster default. The two the node
+controller adds — `not-ready` and `unreachable` — appear only on a node that is
+actually broken, so there they are signal rather than the noise the matching
+tolerations are on every pod.
+
+**Extended resources** are allow-listed by **hardware-vendor prefix**, not by
+exact name and not wholesale. Prefix because a vendor's line is open — a GPU and
+a MIG slice of it are the same fact under different names — and an allow-list
+because the rest of that namespace belongs to whoever runs the cluster, where a
+resource named for an internal team, project or licence is an identity that must
+not travel. A vendor absent from the list is a one-line change; an operator's
+private resource name leaking is not recoverable.
+
+### 3. `node_lifecycle`: a journal, because a snapshot cannot hold this
+
+A windowed journal keyed by (window start, window length), superseding, exactly
+the shape ADR 0021 settled for disruptions. The window is a delivery boundary
+rather than an aggregation, and it bounds the spool's file count by time rather
+than letting a churning autoscaler control it.
+
+**A record carries the node's size, kind and accelerators.** This duplicates
+`node_metadata` for a live node and is the entire point for a dead one: once the
+object is deleted the snapshot has no row to join against, so without the size on
+the event, "twelve nodes left this hour" can never become "and they were 96 cores
+of spot capacity, four of them with GPUs".
+
+**An arrival is only reported when it can be proved.** The test is the node
+object's creation timestamp against the moment this process started watching, not
+the fact that the informer had not seen the node before: at cache sync the add
+handler fires for every node in the cluster, and counting those would report an
+agent restart as the entire fleet arriving at once. This is ADR 0034's rule —
+what predates observation is not an event — applied to nodes. The cost is a node
+created in the seconds before the agent started, which is reported as no arrival
+at all rather than as a false one.
+
+**A departure carries the agent's observation instant, and says so.** A deleted
+object is simply absent; there is no deletion time to read. The record's `at`
+therefore means two different things depending on the event, and rather than let
+one field silently mix provenances, `at_observed` marks the case where it is the
+agent's clock. The precedent is `restarts_before_observation` and
+`reasons_unobserved`: state the limit in the payload rather than in a document
+the reader does not have.
+
+The journal is in memory and is lost on restart, like the other three. A
+departure that happens while the agent is down is not reported — it appears only
+as a node missing from the next snapshot, which is exactly where the fleet stood
+before this ADR.
+
+### 4. What the reductions refuse is counted
+
+A `nodes` block in `collection_coverage`: conditions refused for their type,
+devices for their vendor, taints for an oversized string, and version strings
+that did not fit. Aggregate counts, never a name — CLAUDE.md invariant 6, and the
+same treatment `placement` already gets. A cluster whose nodes do not fit these
+bounds is visible rather than quietly under-described, and the device counter in
+particular is how the vendor allow-list learns it is short a vendor.
+
+## Consequences
+
+The node payload roughly triples in size: five conditions of about sixty bytes
+each are the bulk of it, and on a 500-node fleet that is on the order of 150 KB
+per flush. It is a superseding snapshot, so it does not accumulate.
+
+`NodeInfo` is no longer comparable, and the informer's "report only when the view
+changed" test moved from `==` to a reflective deep equality. That is affordable
+because node objects change on the order of minutes — since Kubernetes 1.17 the
+kubelet heartbeats to a Lease, not to the node — and the one sub-second field it
+does write is the condition heartbeat, which §2 deliberately does not collect.
+
+Taints are the first customer-written strings the agent reads from a node.
+`team=payments:NoSchedule` names an internal team, and that string now leaves the
+cluster. The justification is symmetry with tolerations rather than a claim that
+the string is harmless: the same value was already leaving in every pod spec that
+tolerates it, and refusing it here while carrying it there would have been a
+promise the payloads do not keep.
+
+What is still not collected: node labels beyond the four read by name,
+annotations, addresses, `status.images`, `status.volumesInUse`, and condition
+messages. `nodes/proxy` is unchanged; nothing here reaches a kubelet.

docs/adr/README.md

@@ -121,3 +121,4 @@ line to paste.
 - [0061. What the process holds, beside the peak it once reached](0061-what-the-process-holds-beside-its-peak.md)
 - [0062. The counters are subtracted on the node, and never leave without their interval](0062-the-counters-and-the-window-they-cover.md)
 - [0063. A profile the collector dominates is still a profile](0063-a-profile-of-the-collector-is-still-a-profile.md)
+- [0064. The node is described, and its comings and goings recorded](0064-the-node-is-described-and-its-comings-recorded.md)

docs/backend-requirements.md

@@ -272,6 +272,34 @@ was taken away, which is the normal state.
   `EvictionByEvictionAPI`, `Evicted`) plus `other`. The backend MUST tolerate
   `other` and MUST NOT assume the set never grows.
 
+**`node_lifecycle` is the only payload that describes a node that no longer
+exists** ([ADR 0064](adr/0064-the-node-is-described-and-its-comings-recorded.md)).
+One record per (node, event) per hour-aligned window, `event` being `joined` or
+`left`. It supersedes by (window start, window length), so the backend MUST
+upsert and MUST NOT append.
+
+- Each record repeats the node's instance type, capacity type, zone, capacity and
+  devices. **The backend MUST NOT resolve a `left` record against
+  `node_metadata`** — the node is gone from that snapshot by definition, which is
+  why the size travels on the event.
+- `at` carries two provenances and `at_observed` says which. Absent, `at` is the
+  node object's creation timestamp; `true`, it is the instant the agent noticed
+  the node was gone, because a deleted object carries no deletion time. The
+  backend MUST NOT treat a departure's `at` as exact.
+- Arrivals are reported only when the agent can prove them. A node already
+  present when the agent started produces no `joined` record, so **the backend
+  MUST NOT compute fleet size by summing arrivals and departures** — that is
+  `node_metadata`'s job. This payload is churn, not inventory.
+- A window in which the fleet did not change produces no file. A node that cycles
+  twice within one window produces one record per event type, not two.
+
+**`node_metadata.nodes[].conditions` is present in full or not at all** (ADR
+0064). All five allow-listed condition types are reported when the node reports
+them, healthy or not, so **the backend MUST NOT read a missing condition as
+healthy** — it means the node never reported one. `devices` is allow-listed by
+hardware vendor: an absent `devices` list means no *recognized* accelerator, not
+no extended resource. Both, and `taints`, are omitted when empty.
+
 **`workload_metadata.workload` and `.pod` are two different scopes, and neither
 is per record** (ADR 0014). A pod's containers each produce a record repeating
 both blocks, so **the backend MUST NOT sum either across one workload's

docs/prose_test.go

@@ -34,8 +34,8 @@ var ceilings = []struct {
 	lines   int
 	section int
 }{
-	{"security.md", 1038, 360},
-	{"backend-requirements.md", 633, 490},
+	{"security.md", 1043, 360},
+	{"backend-requirements.md", 661, 518},
 }
 
 func TestNoCommentRunIsLongerThanAPointerToAnADR(t *testing.T) {