feat: finished Job runs ship as a windowed journal (ADR 0029)

cmd/agent/main.go

@@ -220,6 +220,26 @@ func run(ctx context.Context, logger *slog.Logger, clientset kubernetes.Interfac
 		)
 		disruptionJournal.Observe(d)
 	})
+
+	// Finished Job runs. Same window shape as the journals above; the facts are
+	// the run's own instants and outcome, which is what a usage rollup cannot
+	// supply for a workload that ran for part of a window (ADR 0029).
+	jobJournal := journal.NewJobRuns(collector.UsageWindowLength)
+	podWatcher.OnJobFinished(func(r collector.JobRun) {
+		logger.Info("job run finished",
+			"namespace", r.Namespace,
+			"workload_kind", r.Workload.Kind,
+			"workload_name", r.Workload.Name,
+			"job", r.Name,
+			"result", r.Result,
+			"fail_reason", r.FailReason,
+			"started_at", r.StartedAt,
+			"finished_at", r.FinishedAt,
+			"succeeded", r.Succeeded,
+			"failed", r.Failed,
+		)
+		jobJournal.Observe(r)
+	})
 	podWatcher.SetFilter(filter)
 	nodeWatcher := collector.NewNodeWatcher(clientset, func(n collector.NodeInfo) {
 		logger.Info("node observed",
@@ -392,6 +412,21 @@ func run(ctx context.Context, logger *slog.Logger, clientset kubernetes.Interfac
 		logger.Info("pod disruptions flushed", "records", len(records))
 	}
 
+	flushJobRuns := func() {
+		if spool == nil {
+			return
+		}
+		records := append(jobJournal.CloseBefore(time.Now()), jobJournal.Snapshots()...)
+		if len(records) == 0 {
+			return // a cluster with no batch workloads writes nothing
+		}
+		if err := spool.WriteJobRuns(records); err != nil {
+			logger.Error("spooling job runs", "error", err)
+			return
+		}
+		logger.Info("job runs flushed", "records", len(records))
+	}
+
 	// Excluded pods are reported as aggregate counts only, never by name
 	// (docs/security.md §8). Inventory counters are aggregate too: no identity
 	// of an unjoined fact appears, only a count (CLAUDE.md invariant 6).
@@ -403,6 +438,13 @@ func run(ctx context.Context, logger *slog.Logger, clientset kubernetes.Interfac
 			"excluded_namespace_annotation", c.ExcludedNamespaceAnnotation,
 			"excluded_workload_annotation", c.ExcludedWorkloadAnnotation,
 			"excluded_pod_annotation", c.ExcludedPodAnnotation,
+			// Jobs are counted apart from pods: one Job makes many pods, so a
+			// shared number would answer neither question (ADR 0029).
+			"jobs_observed", c.JobsObserved,
+			"jobs_excluded_namespace_filter", c.JobsExcludedNamespaceFilter,
+			"jobs_excluded_namespace_annotation", c.JobsExcludedNamespaceAnnotation,
+			"jobs_excluded_workload_annotation", c.JobsExcludedWorkloadAnnotation,
+			"jobs_excluded_annotation", c.JobsExcludedAnnotation,
 			// The blind spot, not an exclusion: pods collected without their
 			// workload-level opt-out being checked (ADR 0028). The first is a
 			// standing property of a cluster running operators the agent does
@@ -446,6 +488,7 @@ func run(ctx context.Context, logger *slog.Logger, clientset kubernetes.Interfac
 				flushInventory()
 				flushRestarts()
 				flushDisruptions()
+				flushJobRuns()
 			}
 		}
 	}()
@@ -556,6 +599,7 @@ func run(ctx context.Context, logger *slog.Logger, clientset kubernetes.Interfac
 	flushInventory()
 	flushRestarts()
 	flushDisruptions()
+	flushJobRuns()
 	return errors.Join(errs...)
 }
 

docs/adr/0029-job-runs.md

@@ -0,0 +1,121 @@
+# 0029. Finished Job runs ship as a windowed journal, and are not baselined
+
+Date: 2026-08-23
+Status: Accepted
+
+Adds the payload kind `job_runs`. Nothing existing changes shape; the filter
+gains a Job path built on ADR 0028's workload step.
+
+## Context
+
+Usage rollups systematically misrepresent short-lived workloads. A Job that runs
+for ninety seconds inside an hour-long window reports a tiny average, because the
+average is over the window and the workload existed for a fortieth of it. ADR
+0013's `covered_nanoseconds` says the coverage was low; it does not say the
+workload was a batch run that started, did its work, and finished.
+
+That gap lands where it costs most. Requests for CronJobs are routinely set from
+the peak run and then left, so a batch fleet is where overprovisioning
+accumulates quietly — and it is the one workload class the agent measures but
+cannot explain. Every fact needed to explain it is already on the Job object,
+already watched by an informer the controller runs for owner resolution, and
+already covered by RBAC the product has.
+
+## Decision
+
+**1. `job_runs` is a windowed journal, one record per finished run.** Kind
+`job_runs`, source `journal`, natural key (window start, window length),
+superseding — the same shape as `container_restarts` and `pod_disruptions`.
+
+The window is a delivery boundary, not an aggregation: a run finishes once and
+carries its own instants. It exists for ADR 0021's reason, which applies harder
+here. A fleet of CronJobs can finish hundreds of runs an hour, and one file per
+run would put the spool's file count under the cluster's control.
+
+The record carries the run's namespace, its workload (the scheduling CronJob, or
+the Job itself when it has none), the run's own object name, `started_at` and
+`finished_at`, the result, the failure reason, the run's pod success and failure
+counts, and the declared `parallelism`, `completions` and `backoffLimit`.
+
+The pod counts and the declared shape are there because without them the outcome
+is unreadable. A run that succeeded with `failed: 2` retried twice, and those
+retries consumed resources nothing else in the protocol reports. "Three
+succeeded" is a complete run against `completions: 3` and a third of one against
+`completions: 9`.
+
+**2. Failure reasons yes, messages no.** The terminal condition's `reason` is
+Kubernetes' own vocabulary, and `BackoffLimitExceeded` and `DeadlineExceeded` are
+different facts for a capacity analysis: one is a workload that cannot succeed,
+the other one that cannot succeed *in time*, which is often a resource
+statement. The adjacent free-text `message` is never read (ADR 0020 §6).
+
+**3. Runs are not baselined on first observation, and this is the opposite of
+ADR 0020 §5.** A Job already finished when the informer syncs is reported, and it
+lands in the window where it actually finished.
+
+ADR 0020 baselines restart counters because a restart counter carries no time:
+reporting a pre-existing count would file history in whichever window happens to
+be open at startup, dating events by when the agent started. A Job carries
+`status.startTime`, `status.completionTime` and its condition's transition time,
+so nothing has to be invented — this is ADR 0021's distinction, made there for
+disruptions, and it applies here for the same reason.
+
+The startup burst is bounded by the cluster's own retention: a CronJob keeps
+three successful and one failed Job by default, and `ttlSecondsAfterFinished`
+removes them sooner where it is set. What the agent gains is the recent history
+of a cluster it just started watching, for free and correctly dated.
+
+**4. A run's Job is filtered on four objects, one more than a pod.** Namespace
+allow/deny, the namespace's annotation, the owning CronJob's annotation (ADR
+0028's workload step), then the Job's own annotations **or its pod template's**.
+
+The last of those is the one that matters. Before ADR 0028 the documented way to
+opt a workload out was an annotation in its pod template, so a customer who did
+that has excluded pods — and if `job_runs` read only the Job object, the agent
+would ship facts about a workload whose pods it refuses to measure. Collected and
+sent is worse than collected and dropped, and worst of all when the customer
+explicitly asked for neither.
+
+Reading `spec.template.metadata.annotations` is metadata. The `env`, `args` and
+`command` beneath it are never read, on a Job no less than on a pod (CLAUDE.md
+invariant 4).
+
+**5. Job counters are kept apart from pod counters.** One Job produces many
+pods, so a shared `observed` number would answer neither question. The coverage
+report gains `jobs_observed` and four `jobs_excluded_*` counters. Aggregates
+only; no Job is named (CLAUDE.md invariant 6).
+
+**6. No resource envelope in this payload.** Requests and limits belong to
+`workload_metadata`, and repeating them here would give one question two answers
+that can disagree.
+
+## Consequences
+
+**Easier.** A batch workload's cost becomes readable: the run took this long,
+succeeded on the third try, and was declared with this parallelism, joined to the
+usage window it sat in. `covered_nanoseconds` stops being the only hint that
+something short-lived happened. And a cluster that starts the agent gets its
+recent batch history immediately rather than after an hour.
+
+**Harder / given up.** A run whose Job is reaped before the agent sees it is
+lost — `ttlSecondsAfterFinished` set aggressively, and an agent restart across
+it. This is the property `oom_kill`, `container_restarts` and `pod_disruptions`
+already have, and it is why none of them is a billing record.
+
+A run reported with no resource envelope is possible: if the Job's pods were
+reaped before the agent observed them, `workload_metadata` has nothing to join
+to. The run's timings and outcome are still true; only the "how much did it ask
+for" half is missing, and it is missing because the cluster deleted it.
+
+Reporting a Job that finished before startup means the first flush after a
+restart can write a window that has already closed. The spool's age sweep bounds
+how far back that can reach, and the payload is idempotent under its key, so a
+rewrite is a rewrite rather than a duplicate.
+
+**Not changed.** No existing payload's shape, no RBAC, and no new informer — the
+Job handler rides the watch already running for owner resolution. A cluster with
+no batch workloads writes nothing at all: a window in which no Job finished
+produces no file.
+
+This ADR records decisions implemented in the same pull request, per the process
+in [`README.md`](README.md).

docs/adr/README.md

@@ -86,3 +86,4 @@ line to paste.
 - [0026. The controller needs no persistent volume, and therefore no StatefulSet](0026-no-persistent-volume.md)
 - [0027. No payload carries an ordering field: the spool holds one version of a key](0027-no-payload-ordering-field.md)
 - [0028. The opt-out annotation works on the workload, and the workload step fails open](0028-workload-level-opt-out.md)
+- [0029. Finished Job runs ship as a windowed journal, and are not baselined](0029-job-runs.md)

docs/plans/job-and-rollout-facts.md

@@ -65,7 +65,7 @@ everything; C1b and C2 are independent of each other.
 
 ---
 
-## Slice C1a — the opt-out annotation works on the workload
+## Slice C1a — the opt-out annotation works on the workload  ✅ MERGED (#56)
 
 Not originally planned. It surfaced while designing C1b's filter: a Job's pods
 inherit `spec.template.metadata.annotations`, so the documented way to opt a
@@ -84,7 +84,7 @@ and `workload_not_cached`. `PodFilter` becomes `Filter`.
 
 ---
 
-## Slice C1b — `job_runs`
+## Slice C1b — `job_runs`  ✅
 
 **Kind:** `job_runs` · **source:** `journal` · **key:** (window start, window
 length) · **delivery:** supersedes
@@ -114,11 +114,15 @@ duplicating them here would give two answers to one question. Consequence to
 state in the ADR: when a Job's pods are reaped before the agent sees them, the
 run is reported with no resource envelope to join to.
 
-**Reporting rule — baseline on first observation.** Jobs already in a terminal
-state when the informer syncs are recorded as seen and never reported. ADR 0020
-§5 set this: reporting pre-existing history would place it in whichever window
-happens to be open at startup, dating events by when the agent started rather
-than when they happened.
+**Reporting rule — corrected while building: no baseline.** The plan said to
+baseline already-terminal Jobs on the ADR 0020 §5 precedent. That precedent does
+not transfer. Restarts are baselined because a restart counter carries no time;
+a Job carries `startTime`, `completionTime` and its condition's transition time,
+so an already-finished run files itself in the window where it actually
+finished. This is ADR 0021's distinction, and the agent gains the cluster's
+recent batch history at startup instead of discarding it. The startup burst is
+bounded by the cluster's own retention — three successful and one failed Job per
+CronJob by default.
 
 **Honest limitation:** with `ttlSecondsAfterFinished` set, a Job disappears
 shortly after finishing, so a run completed while the agent was down is lost.

docs/security.md

@@ -115,7 +115,7 @@ bound to nothing and its token is not mounted — so it appears in no row here.
 | Resource | Verbs | Why |
 |---|---|---|
 | `pods` | get/list/watch | Map containers to workloads; read `requests`/`limits` from spec; detect OOM kills and count restarts from `status.containerStatuses[]` (`restartCount`, and the reason and exit code of `lastState.terminated`); read the `PodScheduled` and `DisruptionTarget` conditions to report why a replica is not running and when the cluster took one away; read `containerPorts` to locate pprof endpoints |
-| `replicasets`, `deployments`, `statefulsets`, `daemonsets`, `jobs`, `cronjobs` | get/list/watch | Resolve the `ownerReferences` chain (Pod → ReplicaSet → Deployment) so findings are aggregated per workload, not per pod |
+| `replicasets`, `deployments`, `statefulsets`, `daemonsets`, `jobs`, `cronjobs` | get/list/watch | Resolve the `ownerReferences` chain (Pod → ReplicaSet → Deployment) so findings are aggregated per workload, not per pod; read each workload's own annotations to honor an opt-out written there ([ADR 0028](adr/0028-workload-level-opt-out.md)); and read finished Jobs' timings and outcomes for `job_runs` ([ADR 0029](adr/0029-job-runs.md)) |
 | `nodes` | get/list/watch | `allocatable`/`capacity` for node idle computation; labels `node.kubernetes.io/instance-type` and capacity-type (spot/on-demand) for the cost model; `status.nodeInfo.kernelVersion` to report whether nodes meet the kernel floor for the eBPF profile (`CAP_BPF` requires kernel 5.8+, see [§7](#7-node-privileges)) |
 | `namespaces` | get/list/watch | Evaluate namespace allow/deny filters and the opt-out annotation |
 | `nodes/proxy` | get | Poll each kubelet's `/stats/summary` and `/metrics/cadvisor` through the API server for usage counters: CPU, memory working set, CFS throttling, PSI where exposed (ADR 0006). **Honest disclosure:** this verb technically permits any kubelet GET endpoint through the API server, including node logs. The agent calls exactly the two stats paths above — auditable, since all kubelet access lives in a single poll loop |
@@ -379,6 +379,7 @@ appearing here (ADR 0022).
 | `oom_kill` | One out-of-memory kill: namespace, pod, container, workload, when it finished, exit code, restart count, and the memory limit in force | **yes** |
 | `container_restarts` | Per (namespace, pod, container) and hour: restart count, breakdown by termination reason, how many restarts had no reason visible, last exit code | **yes** |
 | `pod_disruptions` | Per hour: the pods the *cluster* removed — preempted, evicted under node pressure, drained, or removed via the eviction API — with the node and the instant | **yes** |
+| `job_runs` | Per hour: each Job that finished — when it started and finished, whether it succeeded, the failure reason, its pod success/failure counts, and its declared `parallelism`/`completions`/`backoffLimit` | **yes** |
 | `workload_metadata` | Declared shape per (namespace, workload, container, image digest): image, requests, limits, ports, QoS, replica counts by phase, by node, and by unscheduled reason | no |
 | `node_metadata` | Per node: name, size, instance and capacity type, zone, region, kernel version, CPU architecture | no |
 | `go_inventory` | Per (namespace, workload, container): Go version, main module, image digest, PGO flag, plus a fleet-coverage block | no |
@@ -395,6 +396,15 @@ section describes each in detail.
 ### Field minimization
 
 - The agent **never reads Secrets or ConfigMaps** (no RBAC for them exists).
+  Its own configuration is a different thing and arrives as a mounted file, not
+  through the API.
+- From Jobs, the agent keeps `metadata`, the terminal condition's *type*,
+  *reason* and *transition time*, `status.startTime`, `status.completionTime`,
+  `status.succeeded`, `status.failed`, and three declared fields:
+  `spec.parallelism`, `spec.completions`, `spec.backoffLimit`. From
+  `spec.template` it reads **annotations only**, to honor an opt-out written
+  there — never the `env`, `args` or `command` beneath it, on a Job no less than
+  on a pod.
 - From pod specs, the agent keeps only `metadata`, `spec.containers[].resources`,
   `spec.containers[].ports`, `ownerReferences`, and `status`. It explicitly
   discards `env`, `args`, and `command` before anything is stored or
@@ -448,6 +458,13 @@ cluster, so that a stale snapshot can be recognized rather than assumed current
 
 ### Object history, and the one place pods are named (ADR 0020)
 
+`job_runs` names the Job object, which for a scheduled job is a generated
+per-run name such as `rollup-29123456`, and the CronJob that scheduled it. The
+name is what tells two runs of one schedule apart, and it is a name your cluster
+generated, never workload content — the same line `container_restarts` draws
+([ADR 0029](adr/0029-job-runs.md)). A run whose Job, CronJob, namespace or pod
+template carries the opt-out annotation produces no record at all.
+
 `oom_kill`, `container_restarts` and `pod_disruptions` report what the cluster's
 own object status records about a container's history — see the table above for
 what each carries. All three are `journal` provenance, and all three name the