diff --git a/cmd/agent/main.go b/cmd/agent/main.go index a1e72f8..ff846e0 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -247,6 +247,7 @@ func run(ctx context.Context, logger *slog.Logger, clientset kubernetes.Interfac // the run's own instants and outcome, which is what a usage rollup cannot // supply for a workload that ran for part of a window (ADR 0029). jobJournal := journal.NewJobRuns(collector.UsageWindowLength) + nodeJournal := journal.NewNodeEvents(collector.UsageWindowLength) podWatcher.OnJobFinished(func(r collector.JobRun) { logger.Info("job run finished", "namespace", r.Namespace, @@ -273,8 +274,29 @@ func run(ctx context.Context, logger *slog.Logger, clientset kubernetes.Interfac "allocatable_memory_bytes", n.AllocatableMemoryBytes, "capacity_cpu_milli", n.CapacityCPUMilli, "capacity_memory_bytes", n.CapacityMemoryBytes, + "kubelet_version", n.KubeletVersion, + "os_image", n.OSImage, + "container_runtime", n.ContainerRuntime, + "devices", len(n.Devices), + "taints", len(n.Taints), ) }) + nodeWatcher.OnNodeLifecycle(func(e collector.NodeLifecycle) { + event := "node left" + if e.Joined { + event = "node joined" + } + logger.Info(event, + "node", e.Node.Name, + "at", e.At, + "at_observed", e.Observed, + "instance_type", e.Node.InstanceType, + "capacity_type", e.Node.CapacityType, + "capacity_cpu_milli", e.Node.CapacityCPUMilli, + "capacity_memory_bytes", e.Node.CapacityMemoryBytes, + ) + nodeJournal.Observe(e) + }) // logRecords writes rollup records one line each; the JSON body is the // exact record shape (kind marks snapshots vs. final closed windows). @@ -547,6 +569,21 @@ func run(ctx context.Context, logger *slog.Logger, clientset kubernetes.Interfac logger.Info("pod disruptions flushed", "records", len(records)) } + flushNodeLifecycle := func() { + if spool == nil { + return + } + records := append(nodeJournal.CloseBefore(time.Now()), nodeJournal.Snapshots()...) + if len(records) == 0 { + return // a fleet that did not change writes nothing + } + if err := spool.WriteNodeLifecycle(records); err != nil { + logger.Error("spooling node lifecycle", "error", err) + return + } + logger.Info("node lifecycle flushed", "records", len(records)) + } + flushJobRuns := func() { if spool == nil { return @@ -658,7 +695,7 @@ func run(ctx context.Context, logger *slog.Logger, clientset kubernetes.Interfac pullCoverage = &pc } if err := spool.WriteCollectionCoverage(time.Now(), startedAt, agentInfo, - podWatcher.SourceHealths(), c, podWatcher.PlacementDrops(), inv, scan, + podWatcher.SourceHealths(), c, podWatcher.PlacementDrops(), nodeWatcher.Drops(), inv, scan, ebpf, probeCoverage, pullCoverage); err != nil { logger.Error("spooling collection coverage", "error", err) } @@ -695,6 +732,7 @@ func run(ctx context.Context, logger *slog.Logger, clientset kubernetes.Interfac flushRestarts() flushDisruptions() flushJobRuns() + flushNodeLifecycle() // Unconditionally, on the agent's own cadence rather than the // usage poller's: the spool's bounds must hold on a cluster // where no usage record is ever written, which is exactly the @@ -817,6 +855,7 @@ func run(ctx context.Context, logger *slog.Logger, clientset kubernetes.Interfac flushRestarts() flushDisruptions() flushJobRuns() + flushNodeLifecycle() return errors.Join(errs...) } diff --git a/docs/adr/0012-payload-registry-and-provenance.md b/docs/adr/0012-payload-registry-and-provenance.md index b65bf4e..b46d646 100644 --- a/docs/adr/0012-payload-registry-and-provenance.md +++ b/docs/adr/0012-payload-registry-and-provenance.md @@ -2,7 +2,7 @@ Date: 2026-08-20 Status: Accepted -Amended by: 0013, 0014, 0017, 0020, 0021, 0022, 0027, 0054 +Amended by: 0013, 0014, 0017, 0020, 0021, 0022, 0027, 0054, 0064 ## Context diff --git a/docs/adr/0064-the-node-is-described-and-its-comings-recorded.md b/docs/adr/0064-the-node-is-described-and-its-comings-recorded.md new file mode 100644 index 0000000..4bcf5fb --- /dev/null +++ b/docs/adr/0064-the-node-is-described-and-its-comings-recorded.md @@ -0,0 +1,167 @@ +# 0064. The node is described, and its comings and goings recorded + +Date: 2026-09-01 + +Status: Accepted + +Amends: 0012 + +Widens `node_metadata` from ten fields to a description of the machine, and adds +`node_lifecycle`, the first payload that says what the fleet *was* rather than +what it is. No new RBAC — the node informer already lists and watches whole node +objects — and no new privilege. + +## Context + +`node_metadata` was drawn for one job: turning a node name into a zone and an +instance type so a usage record could be attributed. It carried the two sizes, +four labels and two fields of `status.nodeInfo`, and the type's comment said +"nothing else is read from node objects" as though that were a promise rather +than a consequence of nobody having needed more. + +Four questions arrive at that payload and leave unanswered. + +- **Why a pod will not schedule.** CPU and memory are two of at least five + scheduling dimensions. A node can be idle in both and still refuse every pod + because it is at its kubelet pod ceiling, out of ephemeral storage, tainted, + or `Ready=False`. Every one of those is on the object the agent already holds + in its informer cache. +- **What the expensive machines are.** An accelerator node costs an order of + magnitude more than the general-purpose node beside it, and to this payload the + two are the same row with different labels. A fleet's GPUs are invisible. +- **What software the fleet runs.** `kernelVersion` is collected because it + decides whether the eBPF profiler can attach. The kubelet version, OS image and + container runtime sit in the same struct, are read in the same call, and are + what an upgrade plan is written against. +- **What the fleet used to be.** Every payload the agent ships about nodes is a + snapshot that replaces its predecessor. A node that leaves is simply absent + from the next one. Fleet churn — the central fact about a cluster running spot + capacity or an aggressive autoscaler — is reconstructible only by diffing + consecutive snapshots, and only at flush granularity, and only if none was + missed. + +The last of these is different in kind from the first three. The others are +fields on an existing payload; this one is a fact no snapshot can hold. + +## Decision + +### 1. `node_metadata` describes the machine + +Age (`metadata.creationTimestamp`), the three remaining `status.nodeInfo` +strings, and four more resource dimensions: ephemeral storage and pods, both +capacity and allocatable. + +Pods and ephemeral storage are core resources and were left out for no reason +better than that the first finding did not need them. They join as named fields +like CPU and memory. + +### 2. Three lists, each bounded by an allow-list + +The parts of a node object with unbounded shape do not enter as they are. Each +follows the discipline ADR 0019 set for build settings and ADR 0031 for +placement: an allow-list rather than a deny-list, a length bound, a drop counted +rather than a value truncated. + +**Conditions** are allow-listed **by type** — the four the kubelet maintains plus +`NetworkUnavailable`. Kept: status, reason, last transition. Never kept: +`message`. It is free text, and the two things that write it — the kubelet and +node-problem-detector — put paths, device names and command output there. The +type allow-list is what makes `reason` safe to carry: a custom condition's reason +is written by whoever installed the agent that sets it, and no custom type +survives the list. + +The transition time is kept and the heartbeat is not. The heartbeat moves every +few seconds on every node in the cluster; carrying it would make every node +differ from its previous view on every flush, which is both noise in the payload +and a defeat of the change-detection the informer callback rests on. + +All five allow-listed conditions are reported whenever the node reports them, +healthy or not. A list holding only problems would make "this node is fine" and +"the agent did not read this" the same bytes, which is exactly what ADR 0048 §2 +forbade for the update strategy. + +**Taints** are kept field for field — key, value, effect — with the same 128-byte +bound as placement values. This is the one place operator-written strings enter +from the node side, and it is justified by symmetry: tolerations are *already* +collected from every pod spec, and a toleration without the taint it answers is +half a fact. It says what a pod would put up with, not what the fleet fences off. + +Unlike tolerations, no taint is filtered as a cluster default. The two the node +controller adds — `not-ready` and `unreachable` — appear only on a node that is +actually broken, so there they are signal rather than the noise the matching +tolerations are on every pod. + +**Extended resources** are allow-listed by **hardware-vendor prefix**, not by +exact name and not wholesale. Prefix because a vendor's line is open — a GPU and +a MIG slice of it are the same fact under different names — and an allow-list +because the rest of that namespace belongs to whoever runs the cluster, where a +resource named for an internal team, project or licence is an identity that must +not travel. A vendor absent from the list is a one-line change; an operator's +private resource name leaking is not recoverable. + +### 3. `node_lifecycle`: a journal, because a snapshot cannot hold this + +A windowed journal keyed by (window start, window length), superseding, exactly +the shape ADR 0021 settled for disruptions. The window is a delivery boundary +rather than an aggregation, and it bounds the spool's file count by time rather +than letting a churning autoscaler control it. + +**A record carries the node's size, kind and accelerators.** This duplicates +`node_metadata` for a live node and is the entire point for a dead one: once the +object is deleted the snapshot has no row to join against, so without the size on +the event, "twelve nodes left this hour" can never become "and they were 96 cores +of spot capacity, four of them with GPUs". + +**An arrival is only reported when it can be proved.** The test is the node +object's creation timestamp against the moment this process started watching, not +the fact that the informer had not seen the node before: at cache sync the add +handler fires for every node in the cluster, and counting those would report an +agent restart as the entire fleet arriving at once. This is ADR 0034's rule — +what predates observation is not an event — applied to nodes. The cost is a node +created in the seconds before the agent started, which is reported as no arrival +at all rather than as a false one. + +**A departure carries the agent's observation instant, and says so.** A deleted +object is simply absent; there is no deletion time to read. The record's `at` +therefore means two different things depending on the event, and rather than let +one field silently mix provenances, `at_observed` marks the case where it is the +agent's clock. The precedent is `restarts_before_observation` and +`reasons_unobserved`: state the limit in the payload rather than in a document +the reader does not have. + +The journal is in memory and is lost on restart, like the other three. A +departure that happens while the agent is down is not reported — it appears only +as a node missing from the next snapshot, which is exactly where the fleet stood +before this ADR. + +### 4. What the reductions refuse is counted + +A `nodes` block in `collection_coverage`: conditions refused for their type, +devices for their vendor, taints for an oversized string, and version strings +that did not fit. Aggregate counts, never a name — CLAUDE.md invariant 6, and the +same treatment `placement` already gets. A cluster whose nodes do not fit these +bounds is visible rather than quietly under-described, and the device counter in +particular is how the vendor allow-list learns it is short a vendor. + +## Consequences + +The node payload roughly triples in size: five conditions of about sixty bytes +each are the bulk of it, and on a 500-node fleet that is on the order of 150 KB +per flush. It is a superseding snapshot, so it does not accumulate. + +`NodeInfo` is no longer comparable, and the informer's "report only when the view +changed" test moved from `==` to a reflective deep equality. That is affordable +because node objects change on the order of minutes — since Kubernetes 1.17 the +kubelet heartbeats to a Lease, not to the node — and the one sub-second field it +does write is the condition heartbeat, which §2 deliberately does not collect. + +Taints are the first customer-written strings the agent reads from a node. +`team=payments:NoSchedule` names an internal team, and that string now leaves the +cluster. The justification is symmetry with tolerations rather than a claim that +the string is harmless: the same value was already leaving in every pod spec that +tolerates it, and refusing it here while carrying it there would have been a +promise the payloads do not keep. + +What is still not collected: node labels beyond the four read by name, +annotations, addresses, `status.images`, `status.volumesInUse`, and condition +messages. `nodes/proxy` is unchanged; nothing here reaches a kubelet. diff --git a/docs/adr/README.md b/docs/adr/README.md index 8f1e070..d6a91a9 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -121,3 +121,4 @@ line to paste. - [0061. What the process holds, beside the peak it once reached](0061-what-the-process-holds-beside-its-peak.md) - [0062. The counters are subtracted on the node, and never leave without their interval](0062-the-counters-and-the-window-they-cover.md) - [0063. A profile the collector dominates is still a profile](0063-a-profile-of-the-collector-is-still-a-profile.md) +- [0064. The node is described, and its comings and goings recorded](0064-the-node-is-described-and-its-comings-recorded.md) diff --git a/docs/backend-requirements.md b/docs/backend-requirements.md index 248b439..0001e13 100644 --- a/docs/backend-requirements.md +++ b/docs/backend-requirements.md @@ -272,6 +272,34 @@ was taken away, which is the normal state. `EvictionByEvictionAPI`, `Evicted`) plus `other`. The backend MUST tolerate `other` and MUST NOT assume the set never grows. +**`node_lifecycle` is the only payload that describes a node that no longer +exists** ([ADR 0064](adr/0064-the-node-is-described-and-its-comings-recorded.md)). +One record per (node, event) per hour-aligned window, `event` being `joined` or +`left`. It supersedes by (window start, window length), so the backend MUST +upsert and MUST NOT append. + +- Each record repeats the node's instance type, capacity type, zone, capacity and + devices. **The backend MUST NOT resolve a `left` record against + `node_metadata`** — the node is gone from that snapshot by definition, which is + why the size travels on the event. +- `at` carries two provenances and `at_observed` says which. Absent, `at` is the + node object's creation timestamp; `true`, it is the instant the agent noticed + the node was gone, because a deleted object carries no deletion time. The + backend MUST NOT treat a departure's `at` as exact. +- Arrivals are reported only when the agent can prove them. A node already + present when the agent started produces no `joined` record, so **the backend + MUST NOT compute fleet size by summing arrivals and departures** — that is + `node_metadata`'s job. This payload is churn, not inventory. +- A window in which the fleet did not change produces no file. A node that cycles + twice within one window produces one record per event type, not two. + +**`node_metadata.nodes[].conditions` is present in full or not at all** (ADR +0064). All five allow-listed condition types are reported when the node reports +them, healthy or not, so **the backend MUST NOT read a missing condition as +healthy** — it means the node never reported one. `devices` is allow-listed by +hardware vendor: an absent `devices` list means no *recognized* accelerator, not +no extended resource. Both, and `taints`, are omitted when empty. + **`workload_metadata.workload` and `.pod` are two different scopes, and neither is per record** (ADR 0014). A pod's containers each produce a record repeating both blocks, so **the backend MUST NOT sum either across one workload's diff --git a/docs/prose_test.go b/docs/prose_test.go index 8747ae5..603de11 100644 --- a/docs/prose_test.go +++ b/docs/prose_test.go @@ -34,8 +34,8 @@ var ceilings = []struct { lines int section int }{ - {"security.md", 1038, 360}, - {"backend-requirements.md", 633, 490}, + {"security.md", 1043, 360}, + {"backend-requirements.md", 661, 518}, } func TestNoCommentRunIsLongerThanAPointerToAnADR(t *testing.T) { diff --git a/docs/security.md b/docs/security.md index 4dca782..b27bffa 100644 --- a/docs/security.md +++ b/docs/security.md @@ -515,12 +515,13 @@ test fails if a kind ships without appearing here (ADR 0022). | `container_restarts` | Per (namespace, pod, container) and hour: restart count, breakdown by termination reason, how many restarts had no reason visible, last exit code | **yes** | | `restart_counters` | Per flush: for each collected container that has ever restarted, the kubelet's restart counter as the agent last observed it, how much of it the agent has watched and since when, when the pod was created, how long the current incarnation has been running, and the reason, exit code and instant of the most recent termination. The counter and the watched-since figure are read from one observation, so their difference is never negative (ADR 0043) | **yes** | | `pod_disruptions` | Per hour: the pods the *cluster* removed — preempted, evicted under node pressure, drained, or removed via the eviction API — with the node and the instant | **yes** | +| `node_lifecycle` | Per hour: each node that joined or left the cluster, with its instance and capacity type, zone, capacity and accelerators. The size travels with the event because a node that left is gone from `node_metadata` by definition, so nothing could be joined to afterwards ([ADR 0064](adr/0064-the-node-is-described-and-its-comings-recorded.md)). An arrival is timestamped by the node object; a departure carries the instant the agent *noticed*, flagged as such. Only arrivals the agent can prove are reported — a node already present when the agent started is not an arrival | no | | `job_runs` | Per hour: each Job that finished — when it started and finished, whether it succeeded, the failure reason, its pod success/failure counts, and its declared `parallelism`/`completions`/`backoffLimit` | **yes** | | `workload_revisions` | Per flush: each ReplicaSet of a collected workload that manages ReplicaSets — a Deployment, or a custom resource such as an Argo Rollout ([ADR 0049](adr/0049-revisions-are-not-only-deployments.md)) — with when it was created, its desired/current/ready replica counts, each container's image reference, and a revision number for a Deployment only | **yes** | | `workload_metadata` | Declared shape per (namespace, workload, container, image digest): image, requests, limits, ports, probe schedules without what any probe checks, the five runtime knobs of the field-minimization list below, QoS, replica counts by phase, by node, and by unscheduled reason, the pod's reduced placement constraints (ADR 0031), and — for a Deployment, StatefulSet or DaemonSet — the update strategy by which the workload replaces its own replicas ([ADR 0048](adr/0048-findings-name-the-fields-they-need.md)). A workload of any other kind carries no strategy and nothing standing in for one | no | | `workload_policy` | Per collected workload, what bounds it from outside its own spec: disruption budgets covering it, autoscalers driving it, the volume claims its pods mount (ADR 0032), and the Services routing to it (ADR 0048) with, per Service, whether topology-aware routing was asked for and how many of its ready endpoints the cluster actually gave a zone hint to ([ADR 0051](adr/0051-topology-routing-is-asked-and-not-granted.md)); plus `unavailable_sources`, naming any of those the agent could not read — never granted, or no longer being fed (ADR 0033, ADR 0035) | not directly — but a claim is named, and a StatefulSet's claim is `