From 8e48f0b97c0fc632c16250ff5ab80853a2f7515b Mon Sep 17 00:00:00 2001 From: Konstantins Feofantovs Date: Sun, 30 Aug 2026 10:36:26 +0300 Subject: [PATCH] feat: coverage says how much was hidden, never what (ADR 0054) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An empty report and a broken agent produce identical bytes today. Everything needed to tell them apart was already computed — twelve filter counters, two placement drops, five node-scan counters, nine inventory counters, the watch health of every non-gating source — and all of it went to a log line the backend never sees. The node scanner's counters were worse off: they crossed the channel and the join dropped them on the floor. New kind `collection_coverage`, and a fifth provenance class `agent`: the other four answer "where in the cluster did this come from", and this payload's subject is the agent itself. No hash of the configuration. A digest of a handful of short namespace names is reversible by trying the plausible ones, and what it would expose is the deny list — precisely the set the customer chose to hide. The payload carries the configuration's shape (entry counts, switches) and the file's mtime instead: nothing to reverse, and a time answers "have the filters changed" better than an opaque token. Effective access is measured, not declared: per source, did the cache fill and is it failing. A grant the ClusterRole holds but a webhook defeats reads as granted in any review of the rules and as failing here. That also removes the reason to call SelfSubjectRulesReview, which is a `create` from an agent whose customer document says it only gets, lists and watches. The one inference the counters do not close — a workload disappearing as an opt-out counter moves — is stated in the ADR and in security.md rather than left for a reviewer, per ADR 0039. The disappearance is observable without the counter; the counter only explains it. go test -race ./... && make lint && make chart-lint # green make inventory-e2e # coverage: 12 pods observed, 41 processes scanned on 1 node(s) # (2 Go, 26 out of scope), signals [cpu memory network psi throttling] Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011YRKuL97KRndaopfpixrhp --- cmd/agent/main.go | 36 ++++- cmd/agent/main_test.go | 4 +- .../0012-payload-registry-and-provenance.md | 2 +- docs/adr/0033-policy-sources-degrade.md | 2 +- ...age-says-how-much-was-hidden-never-what.md | 125 ++++++++++++++++++ docs/adr/README.md | 1 + docs/backend-requirements.md | 21 +++ docs/prose_test.go | 4 +- docs/security.md | 31 +++-- internal/collector/filter.go | 24 ++-- internal/collector/pods.go | 41 +++++- internal/config/shape.go | 65 +++++++++ internal/inventory/store.go | 66 +++++++-- internal/sink/registry.go | 10 ++ internal/sink/registry_test.go | 5 +- internal/sink/spool.go | 68 ++++++++++ internal/sink/spool_test.go | 95 +++++++++++++ .../testdata/collection-coverage.golden.json | 71 ++++++++++ test/e2e/inventory_test.go | 71 ++++++++++ 19 files changed, 701 insertions(+), 41 deletions(-) create mode 100644 docs/adr/0054-coverage-says-how-much-was-hidden-never-what.md create mode 100644 internal/config/shape.go create mode 100644 internal/sink/testdata/collection-coverage.golden.json diff --git a/cmd/agent/main.go b/cmd/agent/main.go index bc4fc40..ef403f6 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -95,10 +95,15 @@ func runController(ctx context.Context, logger *slog.Logger, args []string) erro return err } + startedAt := time.Now() cfg, err := config.Load(*configPath) if err != nil { return fmt.Errorf("loading configuration: %w", err) } + // Counts and switches, never a name from the file: a digest of a handful of + // short namespace names is reversible, and the list it would expose is the + // one the customer chose to hide (ADR 0054 §2). + configShape := cfg.Describe(*configPath, startedAt) clientset, restConfig, err := connect() if err != nil { @@ -106,7 +111,7 @@ func runController(ctx context.Context, logger *slog.Logger, args []string) erro } logger.Info("connected to cluster", "host", restConfig.Host) - return run(ctx, logger, clientset, restConfig, cfg) + return run(ctx, logger, clientset, restConfig, cfg, configShape, startedAt) } // connect builds a clientset from the in-cluster service account when @@ -133,12 +138,14 @@ func connect() (kubernetes.Interface, *rest.Config, error) { return clientset, config, nil } -// coverageInterval is how often the aggregate coverage counters are logged. +// coverageInterval is how often the aggregate coverage counters are logged and +// written as a payload. Unlike every other flush it runs whether or not there is +// anything to report: staleness here is a fact about the agent (ADR 0054 §5). const coverageInterval = time.Minute // run is the agent's lifecycle: it starts, works until ctx is canceled, and // returns. -func run(ctx context.Context, logger *slog.Logger, clientset kubernetes.Interface, restConfig *rest.Config, cfg config.Config) error { +func run(ctx context.Context, logger *slog.Logger, clientset kubernetes.Interface, restConfig *rest.Config, cfg config.Config, configShape config.Shape, startedAt time.Time) error { logger.Info("agent starting", "version", version) defer logger.Info("agent stopping") @@ -531,6 +538,29 @@ func run(ctx context.Context, logger *slog.Logger, clientset kubernetes.Interfac ) } logger.Info("coverage", attrs...) + + // The same counters as a payload. The log serves whoever is reading the + // agent's own output; this serves the report, which otherwise cannot + // tell an empty cluster from a blind agent (ADR 0054). + if spool == nil { + return + } + var inv *inventory.Counters + var scan *inventory.ScanCoverage + if goStore != nil { + ic := goStore.Counters() + sc := goStore.ScanCoverage() + inv, scan = &ic, &sc + } + agentInfo := sink.AgentInfo{ + Version: version, + Config: configShape, + UsageSignals: usagePoller.Signals(), + } + if err := spool.WriteCollectionCoverage(time.Now(), startedAt, agentInfo, + podWatcher.SourceHealths(), c, podWatcher.PlacementDrops(), inv, scan); err != nil { + logger.Error("spooling collection coverage", "error", err) + } } // Watchers run until ctx is canceled; a failing watcher must bring the diff --git a/cmd/agent/main_test.go b/cmd/agent/main_test.go index be4f388..643feb8 100644 --- a/cmd/agent/main_test.go +++ b/cmd/agent/main_test.go @@ -20,7 +20,9 @@ func TestRunStopsOnContextCancel(t *testing.T) { done := make(chan error, 1) // A minimal REST config; node intake is disabled in this config, so it is // never used to build a JWKS client. - go func() { done <- run(ctx, logger, fake.NewClientset(), &rest.Config{}, config.Config{}) }() + go func() { + done <- run(ctx, logger, fake.NewClientset(), &rest.Config{}, config.Config{}, config.Shape{}, time.Now()) + }() cancel() select { diff --git a/docs/adr/0012-payload-registry-and-provenance.md b/docs/adr/0012-payload-registry-and-provenance.md index 1de9fbc..b65bf4e 100644 --- a/docs/adr/0012-payload-registry-and-provenance.md +++ b/docs/adr/0012-payload-registry-and-provenance.md @@ -2,7 +2,7 @@ Date: 2026-08-20 Status: Accepted -Amended by: 0013, 0014, 0017, 0020, 0021, 0022, 0027 +Amended by: 0013, 0014, 0017, 0020, 0021, 0022, 0027, 0054 ## Context diff --git a/docs/adr/0033-policy-sources-degrade.md b/docs/adr/0033-policy-sources-degrade.md index 7226996..e89a365 100644 --- a/docs/adr/0033-policy-sources-degrade.md +++ b/docs/adr/0033-policy-sources-degrade.md @@ -4,7 +4,7 @@ Date: 2026-08-24 Status: Accepted Amends: 0032 §1 -Amended by: 0035 +Amended by: 0035, 0054 Changes how the agent behaves when it cannot read a policy source, and adds an `unavailable_sources` line to `workload_policy` and `cluster_policy`. No new diff --git a/docs/adr/0054-coverage-says-how-much-was-hidden-never-what.md b/docs/adr/0054-coverage-says-how-much-was-hidden-never-what.md new file mode 100644 index 0000000..30aa680 --- /dev/null +++ b/docs/adr/0054-coverage-says-how-much-was-hidden-never-what.md @@ -0,0 +1,125 @@ +# 0054. Coverage says how much was hidden, never what + +Date: 2026-08-30 + +Status: Accepted + +Amends: 0012 §2, 0033 + +Adds the payload kind `collection_coverage` and a fifth provenance class, +`agent`. The counters it carries already existed; none of them left the cluster. + +## Context + +"Nothing found" is a result, and it is the result a customer most needs to trust. +An empty report and a broken agent produce identical bytes today: the backend +receives some payloads and has no way to say "we looked at 412 pods, 37 of them +were excluded by your own namespace filter, and one cache we hold a grant for +never synced". + +Everything needed to say that is already computed. The filter keeps twelve +counters, the placement reduction keeps two, the node scanner keeps five, the +inventory store keeps nine, and the watch health of every non-gating source is +tracked continuously (ADR 0033, ADR 0035). All of it goes to a structured log +line and stops there. `security.md` §11 promises that "filtering behavior is +verifiable from output rather than only from configuration" — and the output is +the agent's own log, which travels the customer's log pipeline and reaches +nobody who renders a report. + +The node scanner's counters are worse off still: they cross the channel inside +`nodescan.Report`, arrive at the controller, and are dropped on the floor by a +join that reads only the binaries beside them. + +## Decision + +**1. A fifth provenance class, `agent`.** The four classes of ADR 0012 — +`structural`, `measured`, `journal`, `sampled` — all answer "where in the +cluster did this come from". This payload's subject is the agent itself. Calling +it `measured` would be the discriminator lying in the one payload whose entire +purpose is to be trusted about its own limits, so the vocabulary widens by one +and `registry_test.go` records the widening. + +**2. No hash of the configuration. Counts and a time instead.** The backend +needs to tell "the data changed" from "the filters changed". The obvious +mechanism — a digest of the effective configuration — is wrong here, for a +reason specific to what that file holds. + +The configuration is a handful of short namespace names. A digest of a +low-entropy value is reversible by trying the plausible ones, and the list it +would expose is the **deny list** — precisely the set the customer chose to hide +from us. The allow list is no secret (those namespaces are named in every other +payload); the denied one is the whole of the privacy question, and hashing it +would ship it in a form that merely looks safe. + +So the payload carries the configuration's *shape*: how many entries each list +holds, which switches are on. Nothing to reverse, because nothing is concealed — +these are numbers we are willing to state plainly. And it carries `config.since`, +the modification time of the file, which answers "have the filters changed since +the last upload" better than an opaque token does. A time is not content. + +The file cannot change under a running agent — the chart stamps a checksum of the +ConfigMap onto the Deployment, so an edit replaces the pod — which is why this +needs no watch and no reload path. + +**3. Effective access is measured, not declared.** The payload reports, per +non-gating source, whether its cache filled and whether it is failing. This is +better evidence than the `SelfSubjectRulesReview` the earlier plan called for: a +grant the ClusterRole holds but an admission webhook or a network policy defeats +reads as *granted* in any review of the rules, and as *failing* here. A report +resting on a source needs to know which of those is true. + +It also avoids a question that deserved its own decision rather than a quiet +slip: `SelfSubjectRulesReview` is a `create` call, made by an agent whose +customer document says it only ever gets, lists and watches. That call is not +made, and if it is ever wanted it arrives with its own ADR. + +The gating caches — pods, namespaces, nodes, the owner chain — are absent from +the list on purpose. Their failure stops the agent (ADR 0035), so a payload +arriving at all is the evidence about them. + +**4. Node scan counters are summed, and the sum is of latest passes.** Aggregate +rather than per node: "how many processes did the fleet skip" is a cluster +question, and a per-node breakdown would grow the payload with the fleet to +answer it in a shape nobody asks. Latest pass per node rather than a running +total, because the counters describe one walk of a process table — adding +successive walks would count one long-lived process once per scan interval. + +**5. It ships on every flush, including one with nothing to say.** This is the +only kind for which "nothing changed" is not a reason to stay silent: its +staleness is a fact about the agent where silence in every other kind is a fact +about the cluster, or about nothing. An old `captured_at` says the agent +stopped. + +## Consequences + +**Easier.** A report can state what it rests on and what it did not see. "Nothing +found" becomes a sentence with evidence behind it rather than an absence +indistinguishable from a failure. Two `[planned]` markers leave `security.md`, +and `backend-requirements.md` — which has described this payload as arriving +since it was written — stops disagreeing with the code. + +**Harder / given up.** One inference channel is not closed by any of this, and +it is honest to name it rather than let a reviewer find it (ADR 0039). + +The backend sees every collected workload. If one disappears from the collected +set at the moment `excluded_workload_annotation` moves from 0 to 1, which +workload was hidden follows by subtraction. The counter does not create that +channel — the disappearance is observable without it — but it does explain it. +The alternative is silence, under which the backend concludes the workload was +deleted and tells the customer something false. Explaining an observable change +is the better of the two, and the customer's protection against the inference is +that the object was already collected before they opted it out, not that the +count is secret. + +Nothing here closes it: a count fine-grained enough to be useful is +fine-grained enough to subtract. What the design does guarantee is that no name +of an excluded namespace, workload or pod appears in this payload or any other, +and `TestCoverageNamesNothingItExcluded` asserts it against a configuration +whose deny list holds real names. + +**Not changed.** No RBAC, no new read, no new API call — every counter in this +payload was already being computed and thrown away. The `create` verb still +appears nowhere. + +This ADR records decisions implemented in the same pull request, per the process +in [`README.md`](README.md). diff --git a/docs/adr/README.md b/docs/adr/README.md index 4ae8502..77dc0b4 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -111,3 +111,4 @@ line to paste. - [0051. Topology-aware routing is a request the cluster can silently decline](0051-topology-routing-is-asked-and-not-granted.md) - [0052. The peak the kernel remembers, and the second file the scanner opens](0052-the-peak-the-kernel-remembers.md) - [0053. The network counters are the pod's, and they say nothing about where the bytes went](0053-network-counters-are-the-pods.md) +- [0054. Coverage says how much was hidden, never what](0054-coverage-says-how-much-was-hidden-never-what.md) diff --git a/docs/backend-requirements.md b/docs/backend-requirements.md index c8eea73..56ffc88 100644 --- a/docs/backend-requirements.md +++ b/docs/backend-requirements.md @@ -360,6 +360,27 @@ in nothing. A `GODEBUG` variable in `workload_metadata.runtime_env` overrides both at startup; it is a fact about a workload, this is a fact about a build, and the backend MUST NOT merge them into one value. +**`collection_coverage` is how the backend knows an empty report is empty rather +than broken** ([ADR 0054](adr/0054-coverage-says-how-much-was-hidden-never-what.md)). +It arrives on every flush, including one where nothing else did, and supersedes +its predecessor. A stale or absent `captured_at` is therefore a statement about +the **agent**, not about the cluster, and the backend SHOULD treat it as one: no +other kind's silence means anything. + +Its counters are cumulative from `since`, which is when that process started +counting; a restart moves the base, and a drop across a change in `since` MUST +NOT be read as a decline. `agent.config` carries the *shape* of the customer's +configuration — entry counts and switches — and never a name from it, and never +a hash of it; `agent.config.since` is when that configuration took effect, and it +is what separates "the data changed" from "the filters changed". `sources[]` +reports what the agent's reads actually did, so a source that is `synced: false` +or `failing: true` bounds every finding resting on it — the backend MUST NOT +present such a finding as complete. The scan block is a fleet aggregate over the +latest pass of each reporting node and MUST NOT be summed across flushes. + +No field of this payload names an object the customer excluded, and the backend +MUST NOT ask for one. + **`network_window` says how much moved, never where it went** ([ADR 0053](adr/0053-network-counters-are-the-pods.md)). The counters are read at the pod's own interfaces, so the payload carries no destination, no peer and no diff --git a/docs/prose_test.go b/docs/prose_test.go index 1e336a8..b9f3864 100644 --- a/docs/prose_test.go +++ b/docs/prose_test.go @@ -34,8 +34,8 @@ var ceilings = []struct { lines int section int }{ - {"security.md", 960, 310}, - {"backend-requirements.md", 530, 385}, + {"security.md", 975, 325}, + {"backend-requirements.md", 555, 405}, } func TestNoCommentRunIsLongerThanAPointerToAnADR(t *testing.T) { diff --git a/docs/security.md b/docs/security.md index 5e994df..d9b1992 100644 --- a/docs/security.md +++ b/docs/security.md @@ -494,6 +494,7 @@ test fails if a kind ships without appearing here (ADR 0022). | Payload | What it carries | Names pods? | |---|---|---| +| `collection_coverage` | What the agent did rather than what it found: how many pods and Jobs it observed, how many each of your four controls excluded, how many placement terms the reduction dropped, what the node scanners walked and skipped, which of the agent's reads worked, its version, and the shape of your configuration. **Aggregate counts only — no name of anything you excluded appears here or anywhere else** ([ADR 0054](adr/0054-coverage-says-how-much-was-hidden-never-what.md)) | no | | `usage_snapshot` | The still-open hour's resource rollup: per (namespace, workload, container) histograms of CPU and memory, throttling and PSI counters, plus how much of the window was observed. Shipped once a minute, each replacing the last | no | | `usage_window` | The same shape, final, when the hour closes | no | | `network_window` | Per collected workload and closed hour: bytes and errors received and transmitted, summed over its pods and their interfaces. Interface **names are not read** — only how many there were. Counters are the *pod's*, so nothing here is attributed to a container, and a `host_network` workload's counters are the **node's** and describe the whole machine ([ADR 0053](adr/0053-network-counters-are-the-pods.md)). The kubelet counts bytes at the interface and never says where they went: there is no destination, no peer and no address in this payload | no | @@ -761,9 +762,12 @@ Node-level totals (allocatable, aggregate node usage) are collected regardless o workload filters. They attribute to no workload and are what reconciles cluster cost against your invoice. -**[planned]** A payload carrying the per-filter exclusion counts and a -fingerprint of the effective configuration with the time it last changed. The -counters exist in the agent; no payload carries them out yet. +The per-filter exclusion counts leave in the `collection_coverage` payload, +beside the *shape* of the configuration in force — how many entries each filter +list holds and when the file last changed, never a name from it. There is +deliberately **no hash of your configuration**: a digest of a few short +namespace names is reversible by trying the plausible ones, and what it would +expose is your deny list ([ADR 0054](adr/0054-coverage-says-how-much-was-hidden-never-what.md)). ### Transport **[planned]** @@ -916,12 +920,21 @@ one; it was never decided in an ADR and never existed in the configuration ### What the agent says about itself -The agent tallies pods observed and pods excluded by each filter, so filtering -behavior is verifiable from output rather than only from configuration. -Delivering those counts out of the cluster is **[planned]**, as is a startup -self-audit that would enumerate the agent's effective permissions via -`SelfSubjectRulesReview`. What holds today: a 403 degrades with a log line, never -a crash-loop or retry storm. +The agent tallies pods observed and pods excluded by each filter, and those +counts leave in `collection_coverage`, so filtering behavior is verifiable from +what we receive and not only from what you configured. The same payload reports +which of the agent's own reads worked — measured from its caches, not read back +from the rules, so a grant defeated by a webhook reads as failing rather than as +granted. No startup self-audit via `SelfSubjectRulesReview` is performed: it is a +`create` call, and this agent makes none (ADR 0054 §3). A 403 still degrades with +a log line, never a crash-loop or retry storm. + +**What those counts can and cannot tell us.** They say how much was excluded and +under which of your four controls, never which object. One inference remains and +is stated rather than hidden ([ADR 0039](adr/0039-stated-limits-are-measured-limits.md)): +a workload that was collected and then opted out disappears from the data, and +that disappearance is visible whether or not a counter explains it. The counter +explains it; without it the conclusion would be that you deleted the workload. ### Opt-out diff --git a/internal/collector/filter.go b/internal/collector/filter.go index e1e18d5..4ee713d 100644 --- a/internal/collector/filter.go +++ b/internal/collector/filter.go @@ -208,26 +208,26 @@ func (f *Filter) countUnresolvedWorkload(reason UnresolvedWorkload) { // report (docs/security.md §11): full information about what is collected, // only counts about what is not. type Coverage struct { - PodsObserved int64 - ExcludedNamespaceFilter int64 - ExcludedNamespaceAnnotation int64 - ExcludedWorkloadAnnotation int64 - ExcludedPodAnnotation int64 + PodsObserved int64 `json:"pods_observed"` + ExcludedNamespaceFilter int64 `json:"excluded_namespace_filter"` + ExcludedNamespaceAnnotation int64 `json:"excluded_namespace_annotation"` + ExcludedWorkloadAnnotation int64 `json:"excluded_workload_annotation"` + ExcludedPodAnnotation int64 `json:"excluded_pod_annotation"` // WorkloadUnknownKind and WorkloadNotCached are the blind spot: pods // admitted without their workload-level opt-out being checked. They are // kept apart because they mean different things — the first is a standing // property of a cluster running operators the agent does not read, the // second should sit at zero. - WorkloadUnknownKind int64 - WorkloadNotCached int64 + WorkloadUnknownKind int64 `json:"workload_unknown_kind"` + WorkloadNotCached int64 `json:"workload_not_cached"` // Job counters, kept apart from the pod ones on purpose: one Job produces // many pods, so a shared counter would answer neither question. - JobsObserved int64 - JobsExcludedNamespaceFilter int64 - JobsExcludedNamespaceAnnotation int64 - JobsExcludedWorkloadAnnotation int64 - JobsExcludedAnnotation int64 + JobsObserved int64 `json:"jobs_observed"` + JobsExcludedNamespaceFilter int64 `json:"jobs_excluded_namespace_filter"` + JobsExcludedNamespaceAnnotation int64 `json:"jobs_excluded_namespace_annotation"` + JobsExcludedWorkloadAnnotation int64 `json:"jobs_excluded_workload_annotation"` + JobsExcludedAnnotation int64 `json:"jobs_excluded_annotation"` } // Snapshot returns the current coverage counters. diff --git a/internal/collector/pods.go b/internal/collector/pods.go index 8c867b5..a0bc644 100644 --- a/internal/collector/pods.go +++ b/internal/collector/pods.go @@ -239,8 +239,8 @@ type PodWatcher struct { // PlacementDrops is what the placement reduction refused to carry, for the // coverage report. type PlacementDrops struct { - Values int64 - Terms int64 + Values int64 `json:"values_dropped"` + Terms int64 `json:"terms_dropped"` } // PlacementDrops returns the running totals. @@ -391,6 +391,43 @@ func (w *PodWatcher) clock() time.Time { return w.now() } +// SourceHealth is one watched source as the agent actually found it: the +// agent's effective read access, measured rather than declared. A grant the +// ClusterRole holds but a webhook defeats reads as granted in any review of the +// rules, and as failing here (ADR 0054 §3). +type SourceHealth struct { + // Name is the resource class, not a customer object: "services", + // "endpoint_slices", and so on. + Name string `json:"name"` + // Synced is whether the cache ever filled. + Synced bool `json:"synced"` + // Failing is whether its watch has been erroring recently enough to treat + // the cache as no longer fed (ADR 0035). + Failing bool `json:"failing,omitempty"` +} + +// SourceHealths reports every non-gating source the agent watches, sorted by +// name. The gating caches — pods, namespaces, nodes and the owner chain — are +// absent on purpose: their failure stops the agent, so a payload arriving at all +// is the evidence about them (ADR 0035). +func (w *PodWatcher) SourceHealths() []SourceHealth { + now := w.clock() + window := w.limits.unavailableFor + if window == 0 { + window = watchUnavailableFor + } + out := make([]SourceHealth, 0, len(w.policySources)) + for _, source := range w.policySources { + out = append(out, SourceHealth{ + Name: source.name, + Synced: source.synced != nil && source.synced(), + Failing: source.health != nil && source.health.failedWithin(now, window), + }) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out +} + // unavailablePolicySources returns the names of the caches this payload cannot // be assembled from, sorted, for the payload to declare. // diff --git a/internal/config/shape.go b/internal/config/shape.go new file mode 100644 index 0000000..6430d19 --- /dev/null +++ b/internal/config/shape.go @@ -0,0 +1,65 @@ +package config + +import ( + "os" + "time" +) + +// Shape is what the coverage payload says about the configuration in force: +// how many entries each filter list holds and which switches are on, and since +// when. It carries no name from the file (ADR 0054 §2). +// +// Deliberately not a hash: a digest of a handful of short namespace names is +// reversible by trying the plausible ones, and what it would expose is the deny +// list — the set the customer chose to hide (ADR 0054 §2). +type Shape struct { + // Since is when the configuration in force took effect: the file's + // modification time, or the process start when there is no file. It answers + // "have the filters changed since the last upload" with a time rather than + // with an opaque token, and a time is not content. + Since time.Time `json:"since"` + // NamespacesAllowed and NamespacesDenied are entry counts. An empty allow + // list admits every namespace, so zero and three are different postures + // rather than more or less of one. + NamespacesAllowed int `json:"namespaces_allowed"` + NamespacesDenied int `json:"namespaces_denied"` + // ProfilingEnabled is the controller's half of the profiling switch; TopN is + // how many workloads an answer may name. + ProfilingEnabled bool `json:"profiling_enabled"` + ProfilingTopN int `json:"profiling_top_n,omitempty"` + // NodeIntakeEnabled is whether the receiver for node reports is open. + NodeIntakeEnabled bool `json:"node_intake_enabled"` + // SpoolMaxAgeHours is 0 when the agent's own default applies. + SpoolMaxAgeHours int `json:"spool_max_age_hours,omitempty"` +} + +// Describe reduces a loaded configuration to its shape. path is the file it came +// from; started is the fallback for Since when there is no file to date. +func (c Config) Describe(path string, started time.Time) Shape { + return Shape{ + Since: configSince(path, started), + NamespacesAllowed: len(c.Filters.Namespaces.Allow), + NamespacesDenied: len(c.Filters.Namespaces.Deny), + ProfilingEnabled: c.Profiling.Enabled, + ProfilingTopN: c.Profiling.TopN, + NodeIntakeEnabled: c.NodeIntake.Enabled, + SpoolMaxAgeHours: c.Spool.MaxAgeHours, + } +} + +// configSince is the file's modification time, falling back to started. +// +// The fallback is not a degradation in practice: the chart stamps a checksum of +// the ConfigMap onto the controller Deployment, so editing the configuration +// replaces the pod. The file cannot change under a running agent, which is why +// this needs no watch. +func configSince(path string, started time.Time) time.Time { + if path == "" { + return started.UTC() + } + info, err := os.Stat(path) + if err != nil { + return started.UTC() + } + return info.ModTime().UTC() +} diff --git a/internal/inventory/store.go b/internal/inventory/store.go index 1f29968..6752219 100644 --- a/internal/inventory/store.go +++ b/internal/inventory/store.go @@ -152,6 +152,12 @@ type Store struct { // it (ADR 0052 §3). peaks map[PeakKey]map[string]nodePeaks + // scans holds the latest scan counters each reporting node sent. Latest + // rather than summed: they describe one pass over a node's process table, + // and adding successive passes would count the same long-lived process once + // per scan interval (ADR 0054 §4). + scans map[string]nodescan.Counters + // nodesReported is the set of nodes still in the cluster that have delivered // at least one report. Compared against the node count in the node-metadata // payload it is what makes a DaemonSet that never started visible @@ -182,6 +188,7 @@ func NewStore(since time.Time) *Store { written: make(map[string]struct{}), peaks: make(map[PeakKey]map[string]nodePeaks), nodesReported: make(map[string]struct{}), + scans: make(map[string]nodescan.Counters), since: since, } } @@ -198,6 +205,7 @@ func (s *Store) Ingest(report nodescan.Report, resolver ContainerResolver) { defer s.mu.Unlock() if report.Node != "" { s.nodesReported[report.Node] = struct{}{} + s.scans[report.Node] = report.Counters } seen := make(map[PeakKey]bool, len(report.Binaries)) for _, b := range report.Binaries { @@ -428,6 +436,7 @@ func (s *Store) RetainNodes(live []string) { for n := range s.nodesReported { if _, ok := set[n]; !ok { delete(s.nodesReported, n) + delete(s.scans, n) } } // A departed node's peaks go with it. Its pods are gone, so the number it @@ -500,25 +509,64 @@ func (s *Store) PeakSnapshot() []PeakRecord { return out } +// ScanCoverage is what the node scanners did, summed over the nodes that +// reported: how many processes they walked and how many they dropped, by +// reason. +// +// Aggregate, never per node. The question "how many processes did the fleet +// skip" is a cluster question, and a per-node breakdown would grow the payload +// with the fleet for an answer nobody asks that way (ADR 0054 §4). +type ScanCoverage struct { + // Nodes is how many nodes contributed the counters below. + Nodes int `json:"nodes"` + // ProcessesScanned is every PID the passes attempted; GoFound is what they + // kept. The three drops in between are the whole of what the node did not + // report, and none of them names anything (CLAUDE.md invariant 6). + ProcessesScanned int `json:"processes_scanned"` + GoFound int `json:"go_found"` + // FilteredScope is processes whose pod the controller's filters excluded — + // the customer's own choice, counted on the node before an executable was + // opened. FilteredInfra is cluster infrastructure and this agent itself. + // Unreadable is a real executable carrying no Go build information. + FilteredScope int `json:"filtered_scope"` + FilteredInfra int `json:"filtered_infra"` + Unreadable int `json:"unreadable"` +} + +// ScanCoverage sums the latest scan counters of every reporting node. +func (s *Store) ScanCoverage() ScanCoverage { + s.mu.Lock() + defer s.mu.Unlock() + out := ScanCoverage{Nodes: len(s.scans)} + for _, c := range s.scans { + out.ProcessesScanned += c.ProcessesScanned + out.GoFound += c.GoFound + out.FilteredScope += c.FilteredScope + out.FilteredInfra += c.FilteredInfra + out.Unreadable += c.Unreadable + } + return out +} + // Counters is the aggregate inventory state for the coverage report. type Counters struct { // Records is the number of distinct (namespace, workload, container) // records currently held. - Records int + Records int `json:"records"` // GoVersions is the number of distinct Go versions across those records. - GoVersions int + GoVersions int `json:"go_versions"` // PGOBuilds is how many records were built with PGO. - PGOBuilds int + PGOBuilds int `json:"pgo_builds"` // Builds is the number of distinct image digests whose facts are held. - Builds int + Builds int `json:"builds"` // FactsReceived / FactsJoined / FactsUnjoined / FactsUndigested are // cumulative since start. - FactsReceived int64 - FactsJoined int64 - FactsUnjoined int64 - FactsUndigested int64 + FactsReceived int64 `json:"facts_received"` + FactsJoined int64 `json:"facts_joined"` + FactsUnjoined int64 `json:"facts_unjoined"` + FactsUndigested int64 `json:"facts_undigested"` // NodesReported is how many distinct nodes have delivered a report. - NodesReported int + NodesReported int `json:"nodes_reported"` } // Counters returns a snapshot of the aggregate inventory state. diff --git a/internal/sink/registry.go b/internal/sink/registry.go index ad3301b..92c1737 100644 --- a/internal/sink/registry.go +++ b/internal/sink/registry.go @@ -52,6 +52,16 @@ type PayloadKind struct { // registry is the fixed list. Order here is irrelevant — Registry sorts. var registry = []PayloadKind{ + { + // Written on every flush, including one with nothing to report: it is + // the only kind whose absence or staleness is a fact about the agent + // rather than about the cluster (ADR 0054). + Kind: "collection_coverage", + Source: SourceAgent, + NaturalKey: "the kind itself (one per cluster)", + Delivery: DeliverySupersedes, + ADR: "0054", + }, { Kind: "usage_snapshot", Source: SourceMeasured, diff --git a/internal/sink/registry_test.go b/internal/sink/registry_test.go index f531c75..429b4fb 100644 --- a/internal/sink/registry_test.go +++ b/internal/sink/registry_test.go @@ -88,13 +88,16 @@ func TestEveryRegisteredKindShips(t *testing.T) { } // Provenance is a closed vocabulary (ADR 0012 §2): the backend switches on it, -// so a fifth class invented at a call site would be silently unhandled there. +// so a class invented at a call site would be silently unhandled there. The list +// below is the whole vocabulary, and widening it is a decision that shows up in +// this diff — which is what ADR 0054 did to add `agent`. func TestRegistrySourcesAreKnownProvenanceClasses(t *testing.T) { known := map[string]bool{ SourceStructural: true, SourceMeasured: true, SourceJournal: true, SourceSampled: true, + SourceAgent: true, } for _, entry := range Registry() { if !known[entry.Source] { diff --git a/internal/sink/spool.go b/internal/sink/spool.go index 3e41c9b..953b280 100644 --- a/internal/sink/spool.go +++ b/internal/sink/spool.go @@ -17,6 +17,7 @@ import ( "time" "github.com/RebuildStackCo/runtime-agent/internal/collector" + "github.com/RebuildStackCo/runtime-agent/internal/config" "github.com/RebuildStackCo/runtime-agent/internal/inventory" "github.com/RebuildStackCo/runtime-agent/internal/journal" "github.com/RebuildStackCo/runtime-agent/internal/metadata" @@ -76,6 +77,13 @@ const ( // (`ebpf_profile`), and a profile pulled from a /debug/pprof endpoint would // be a different kind of the same class (ADR 0023). SourceSampled = "sampled" + // SourceAgent is a fact about the agent's own collection rather than about + // the cluster: what it looked at, what it was configured to skip, and which + // of its reads worked. The other four classes all answer "where in the + // cluster did this come from", and stretching one of them over a fact whose + // subject is the agent would make the discriminator lie in the one payload + // whose whole purpose is to be trusted about itself (ADR 0054 §1). + SourceAgent = "agent" ) // Spool writes payload files into one directory. Methods may be called from @@ -134,6 +142,47 @@ type networkPayload struct { Records []*rollup.NetworkRecord `json:"records"` } +// collectionCoveragePayload is what the agent did, not what it found: what it +// saw, what it was told to skip and under which control, which of its reads +// worked, and the shape of the configuration in force. +// +// It ships on every flush whether or not anything changed, so an old CapturedAt +// says the agent stopped where silence in every other kind says nothing +// (ADR 0054 §5). Every number here is an aggregate; no name of an excluded +// object appears (CLAUDE.md invariant 6). +type collectionCoveragePayload struct { + Kind string `json:"kind"` + Source string `json:"source"` + CapturedAt time.Time `json:"captured_at"` + // Since is the base of every cumulative counter below: the moment this + // process started counting. It travels in the bytes so a restart reads as a + // new base rather than as counters that fell. + Since time.Time `json:"since"` + Agent AgentInfo `json:"agent"` + Sources []collector.SourceHealth `json:"sources"` + Filter collector.Coverage `json:"filter"` + // Placement counts the placement terms and values the reduction refused to + // carry (ADR 0031). Not an exclusion — the workload is collected — but it is + // something the customer's manifests hold and the payload does not. + Placement collector.PlacementDrops `json:"placement"` + // Inventory and Scan are present only when the node role is deployed. + Inventory *inventory.Counters `json:"inventory,omitempty"` + Scan *inventory.ScanCoverage `json:"scan,omitempty"` +} + +// AgentInfo is what the agent is and how it is set up, for a report that needs +// to state what its findings rest on. +type AgentInfo struct { + // Version is the build. "dev" means an unstamped binary. + Version string `json:"version"` + // Config is the shape of the configuration in force — counts and switches, + // never a name from it (ADR 0054 §2). + Config config.Shape `json:"config"` + // UsageSignals is which kubelet signals this cluster actually exposes. A + // finding resting on PSI cannot be made on a cluster that reports none. + UsageSignals []string `json:"usage_signals,omitempty"` +} + // oomPayload is one OOM kill event; events bypass windows and ship // immediately (ADR 0006). type oomPayload struct { @@ -453,6 +502,25 @@ func (s *Spool) WriteNetworkWindows(records []*rollup.NetworkRecord, obs collect return nil } +// WriteCollectionCoverage writes the coverage payload, superseding its +// predecessor. It is written on every flush, including one that found nothing: +// an empty report and a broken agent are the same bytes without it (ADR 0054). +func (s *Spool) WriteCollectionCoverage(capturedAt, since time.Time, agent AgentInfo, sources []collector.SourceHealth, filter collector.Coverage, placement collector.PlacementDrops, inv *inventory.Counters, scan *inventory.ScanCoverage) error { + payload := collectionCoveragePayload{ + Kind: "collection_coverage", + Source: SourceAgent, + CapturedAt: capturedAt.UTC(), + Since: since.UTC(), + Agent: agent, + Sources: sources, + Filter: filter, + Placement: placement, + Inventory: inv, + Scan: scan, + } + return s.write("collection-coverage.json", payload) +} + // WriteContainerRestarts writes the restart records of each window they belong // to, one file per window, atomically replacing that window's previous file. // records must already be in a deterministic order (the accumulator sorts them) diff --git a/internal/sink/spool_test.go b/internal/sink/spool_test.go index dd3fc5b..ffac3ac 100644 --- a/internal/sink/spool_test.go +++ b/internal/sink/spool_test.go @@ -14,6 +14,7 @@ import ( "k8s.io/utils/ptr" "github.com/RebuildStackCo/runtime-agent/internal/collector" + "github.com/RebuildStackCo/runtime-agent/internal/config" "github.com/RebuildStackCo/runtime-agent/internal/inventory" "github.com/RebuildStackCo/runtime-agent/internal/journal" "github.com/RebuildStackCo/runtime-agent/internal/metadata" @@ -102,6 +103,50 @@ func TestGoldenNetworkWindowPayload(t *testing.T) { checkGolden(t, filepath.Join(dir, name), "network-window.golden.json") } +// The coverage fixture is a cluster where the customer hid things and one read +// is refused: three namespaces excluded by the filter, two workloads opted out, +// and the EndpointSlice cache never synced. Nothing here names any of them. +func TestGoldenCollectionCoveragePayload(t *testing.T) { + s, dir := newTestSpool(t) + agent := AgentInfo{ + Version: "1.4.2", + Config: config.Shape{ + Since: capturedAt.Add(-72 * time.Hour), + NamespacesAllowed: 0, + NamespacesDenied: 3, + ProfilingEnabled: true, + ProfilingTopN: 5, + NodeIntakeEnabled: true, + }, + UsageSignals: []string{"cpu", "memory", "network", "throttling"}, + } + sources := []collector.SourceHealth{ + {Name: "endpoint_slices", Synced: false, Failing: true}, + {Name: "services", Synced: true}, + } + filter := collector.Coverage{ + PodsObserved: 412, + ExcludedNamespaceFilter: 37, + ExcludedWorkloadAnnotation: 2, + WorkloadUnknownKind: 6, + JobsObserved: 18, + JobsExcludedNamespaceFilter: 4, + } + inv := inventory.Counters{ + Records: 24, GoVersions: 3, Builds: 11, + FactsReceived: 240, FactsJoined: 231, FactsUnjoined: 9, NodesReported: 4, + } + scan := inventory.ScanCoverage{ + Nodes: 4, ProcessesScanned: 1840, GoFound: 96, + FilteredScope: 1204, FilteredInfra: 512, Unreadable: 28, + } + if err := s.WriteCollectionCoverage(capturedAt, capturedAt.Add(-6*time.Hour), agent, + sources, filter, collector.PlacementDrops{Values: 3, Terms: 1}, &inv, &scan); err != nil { + t.Fatal(err) + } + checkGolden(t, filepath.Join(dir, "collection-coverage.json"), "collection-coverage.golden.json") +} + func fixedObservation() collector.Observation { return collector.Observation{ PollIntervalSeconds: 30, @@ -1655,3 +1700,53 @@ func TestTheSpoolIsBoundedOnAClusterThatWritesNoUsage(t *testing.T) { t.Errorf("%d expired payloads survived on a cluster with no usage records", len(entries)) } } + +// The payload has to say that something was hidden without saying what. This +// builds it from a configuration whose deny list names real namespaces and +// asserts not one of those strings reaches the bytes — the property the counts +// exist to preserve, and the reason there is no hash of the configuration in +// here to reverse (ADR 0054 §2, CLAUDE.md invariant 6). +func TestCoverageNamesNothingItExcluded(t *testing.T) { + secret := []string{"acme-payments", "customer-pii", "hr-internal"} + cfg := config.Config{} + cfg.Filters.Namespaces.Deny = secret + cfg.Filters.Namespaces.Allow = []string{"shop"} + + s, dir := newTestSpool(t) + agent := AgentInfo{Version: "1.4.2", Config: cfg.Describe("", capturedAt)} + err := s.WriteCollectionCoverage(capturedAt, capturedAt, agent, + []collector.SourceHealth{{Name: "services", Synced: true}}, + collector.Coverage{PodsObserved: 10, ExcludedNamespaceFilter: 3}, + collector.PlacementDrops{}, nil, nil) + if err != nil { + t.Fatal(err) + } + + raw, err := os.ReadFile(filepath.Join(dir, "collection-coverage.json")) // #nosec G304 -- test-controlled path + if err != nil { + t.Fatal(err) + } + for _, name := range append(secret, "shop") { + if strings.Contains(string(raw), name) { + t.Errorf("the coverage payload carries the namespace name %q", name) + } + } + + // And it still says the filters exist and how big they are, or it would be + // preserving privacy by saying nothing at all. + var payload struct { + Agent struct { + Config config.Shape `json:"config"` + } `json:"agent"` + Filter collector.Coverage `json:"filter"` + } + if err := json.Unmarshal(raw, &payload); err != nil { + t.Fatal(err) + } + if payload.Agent.Config.NamespacesDenied != 3 || payload.Agent.Config.NamespacesAllowed != 1 { + t.Errorf("config shape = %+v, want 1 allowed and 3 denied", payload.Agent.Config) + } + if payload.Filter.ExcludedNamespaceFilter != 3 { + t.Errorf("excluded count = %d, want 3", payload.Filter.ExcludedNamespaceFilter) + } +} diff --git a/internal/sink/testdata/collection-coverage.golden.json b/internal/sink/testdata/collection-coverage.golden.json new file mode 100644 index 0000000..7c705c1 --- /dev/null +++ b/internal/sink/testdata/collection-coverage.golden.json @@ -0,0 +1,71 @@ +{ + "kind": "collection_coverage", + "source": "agent", + "captured_at": "2026-08-06T10:12:00Z", + "since": "2026-08-06T04:12:00Z", + "agent": { + "version": "1.4.2", + "config": { + "since": "2026-08-03T10:12:00Z", + "namespaces_allowed": 0, + "namespaces_denied": 3, + "profiling_enabled": true, + "profiling_top_n": 5, + "node_intake_enabled": true + }, + "usage_signals": [ + "cpu", + "memory", + "network", + "throttling" + ] + }, + "sources": [ + { + "name": "endpoint_slices", + "synced": false, + "failing": true + }, + { + "name": "services", + "synced": true + } + ], + "filter": { + "pods_observed": 412, + "excluded_namespace_filter": 37, + "excluded_namespace_annotation": 0, + "excluded_workload_annotation": 2, + "excluded_pod_annotation": 0, + "workload_unknown_kind": 6, + "workload_not_cached": 0, + "jobs_observed": 18, + "jobs_excluded_namespace_filter": 4, + "jobs_excluded_namespace_annotation": 0, + "jobs_excluded_workload_annotation": 0, + "jobs_excluded_annotation": 0 + }, + "placement": { + "values_dropped": 3, + "terms_dropped": 1 + }, + "inventory": { + "records": 24, + "go_versions": 3, + "pgo_builds": 0, + "builds": 11, + "facts_received": 240, + "facts_joined": 231, + "facts_unjoined": 9, + "facts_undigested": 0, + "nodes_reported": 4 + }, + "scan": { + "nodes": 4, + "processes_scanned": 1840, + "go_found": 96, + "filtered_scope": 1204, + "filtered_infra": 512, + "unreadable": 28 + } +} diff --git a/test/e2e/inventory_test.go b/test/e2e/inventory_test.go index 6d1215c..e12de7d 100644 --- a/test/e2e/inventory_test.go +++ b/test/e2e/inventory_test.go @@ -43,6 +43,8 @@ const nodeMetadataSpoolPath = spoolDir + "/node-metadata.json" const processPeaksSpoolPath = spoolDir + "/process-peaks.json" +const coverageSpoolPath = spoolDir + "/collection-coverage.json" + // buildSpoolPath mirrors the sink's filename rule for a build payload: one file // per image digest, with the digest's colon replaced so the name is filesystem- // and tooling-safe (internal/sink.digestFileToken). @@ -156,6 +158,7 @@ func TestGoInventoryEndToEnd(t *testing.T) { checkSampleBuild(ctx, t, config, clientset, ns, controllerPod, rec.ImageDigest) checkSamplePeak(ctx, t, config, clientset, ns, controllerPod, rec.ImageDigest) checkNodeArchitecture(ctx, t, config, clientset, ns, controllerPod) + checkCoverageReported(ctx, t, config, clientset, ns, controllerPod) checkOptOutRemovesTheRecord(ctx, t, config, clientset, ns, controllerPod) return } @@ -249,6 +252,74 @@ func findSampleRecord(ctx context.Context, t *testing.T, config *rest.Config, cs return goInventoryRecord{}, inventoryCoverage{}, false } +// checkCoverageReported asserts that the agent says what it did, from a real +// cluster: what it observed, which of its reads worked, and the shape of the +// configuration — without a name of anything excluded (ADR 0054). +func checkCoverageReported(ctx context.Context, t *testing.T, config *rest.Config, cs kubernetes.Interface, ns, pod string) { + t.Helper() + raw, ok := readSpoolFile(ctx, t, config, cs, ns, pod, coverageSpoolPath) + if !ok { + t.Errorf("collection_coverage payload never appeared at %s", coverageSpoolPath) + return + } + var payload struct { + Kind string `json:"kind"` + Source string `json:"source"` + Agent struct { + Version string `json:"version"` + Config struct { + Since string `json:"since"` + NamespacesAllowed int `json:"namespaces_allowed"` + NamespacesDenied int `json:"namespaces_denied"` + } `json:"config"` + UsageSignals []string `json:"usage_signals"` + } `json:"agent"` + Sources []struct { + Name string `json:"name"` + Synced bool `json:"synced"` + Failing bool `json:"failing"` + } `json:"sources"` + Filter struct { + PodsObserved int64 `json:"pods_observed"` + } `json:"filter"` + Scan struct { + Nodes int `json:"nodes"` + ProcessesScanned int `json:"processes_scanned"` + GoFound int `json:"go_found"` + FilteredScope int `json:"filtered_scope"` + } `json:"scan"` + } + if err := json.Unmarshal([]byte(raw), &payload); err != nil { + t.Fatalf("coverage payload not valid JSON: %v", err) + } + if payload.Kind != "collection_coverage" || payload.Source != "agent" { + t.Errorf("kind/source = %q/%q", payload.Kind, payload.Source) + } + if payload.Filter.PodsObserved == 0 { + t.Error("pods_observed = 0; the agent claims to have looked at nothing") + } + if payload.Agent.Config.Since == "" { + t.Error("config.since is empty; nothing dates the configuration in force") + } + // Measured access, not declared: every source the chart granted should have + // filled its cache on a healthy cluster. + if len(payload.Sources) == 0 { + t.Error("sources is empty; the payload says nothing about what the agent could read") + } + for _, s := range payload.Sources { + if !s.Synced || s.Failing { + t.Errorf("source %s: synced=%v failing=%v, want a healthy cache", s.Name, s.Synced, s.Failing) + } + } + // The node half arrived over the channel and was not dropped at the join. + if payload.Scan.Nodes == 0 || payload.Scan.ProcessesScanned == 0 || payload.Scan.GoFound == 0 { + t.Errorf("scan = %+v; the node scanners' own counters did not reach the payload", payload.Scan) + } + t.Logf("coverage: %d pods observed, %d processes scanned on %d node(s) (%d Go, %d out of scope), signals %v", + payload.Filter.PodsObserved, payload.Scan.ProcessesScanned, payload.Scan.Nodes, + payload.Scan.GoFound, payload.Scan.FilteredScope, payload.Agent.UsageSignals) +} + // checkSamplePeak asserts that the measured half of the same node reports // arrived as its own payload: a real high-water mark, read from // `/proc//status` on the node, for the sample's own build (ADR 0052).