petruanica · Nov 9, 2026
@@ -7,6 +7,7 @@ package standard
 
 import (
 	"context"
+	"errors"
 	"sort"
 	"testing"
 	"time"
@@ -22,10 +23,20 @@ import (
 // Reverse label reconciliation. TestNodeLabelsMatchKubernetesAPI checks that
 // every k8s.node.label.* on a metric exists on the node; it cannot notice a
 // label that never arrived. These tests start from the Kubernetes API instead:
-// every node and pod label the running denylist does not match must arrive with
-// the same value, and every label it matches must not. The expectation comes
-// from the cluster and the live configuration; the one maintained list is the
-// never-denied attributes, which stops an over-broad denylist entry passing.
+// every node and pod label that the running k8sattributes rules extract and the
+// running denylist does not match must arrive with the same value, and every
+// other label must not. The expectation comes from the cluster and the live
+// configuration; the one maintained list is the never-denied attributes, which
+// stops an over-broad denylist entry passing.
+//
+// Non-default filter values change the expectation. A label include list
+// narrows the extraction rules, so a label it leaves out must not arrive. An
+// include entry that names a label inside a built-in prefix makes the chart
+// render a label exception transform, which removes the rest of that prefix,
+// so the denylist includes it. A namespace filter drops the series of every
+// pod in a namespace it removes, so those pods need no series. Node series
+// carry no k8s.namespace.name, so the namespace filter keeps them. A filter the
+// model cannot read skips the test with the reason.
 //
 // Labels reach metrics on two paths, each with its own k8sattributes extraction
 // and denylist: the cluster scraper's kube-state-metrics pipeline, which reports
@@ -52,25 +63,87 @@ const (
 	recheckDelay = 90 * time.Second
 )
 
-// agentDenylist reads the attribute-limit configuration an agent is running.
-func agentDenylist(t *testing.T, configMap string) otelmetrics.AttributeDenylist {
+// agentConfig reads the collector configuration an agent is running.
+func agentConfig(t *testing.T, configMap string) string {
 	t.Helper()
 	cm, err := k8sClientset(t).CoreV1().ConfigMaps(agentNamespace).Get(
 		context.Background(), configMap, metav1.GetOptions{})
 	require.NoError(t, err, "reading ConfigMap %s/%s", agentNamespace, configMap)
 	raw, ok := cm.Data[otelConfigKey]
 	require.True(t, ok, "ConfigMap %s has no %s", configMap, otelConfigKey)
+	return raw
+}
+
+// agentDenylist reads the attribute-limit configuration and the label exception
+// transforms an agent is running. It skips the test for a label filter the model
+// cannot read.
+func agentDenylist(t *testing.T, configMap string) otelmetrics.AttributeDenylist {
+	t.Helper()
+	return parseDenylist(t, configMap, agentConfig(t, configMap))
+}
+
+func parseDenylist(t *testing.T, configMap, raw string) otelmetrics.AttributeDenylist {
+	t.Helper()
 	d, err := otelmetrics.ParseAttributeDenylist(raw)
-	require.NoError(t, err, "ConfigMap %s", configMap)
+	skipIfNotModeled(t, configMap, err)
+	for _, x := range d.Exceptions {
+		t.Logf("ConfigMap %s: %s removes %v except the keys that %s matches", configMap, x.Processor, x.Prefixes, x.Keep)
+	}
 	return d
 }
 
+// labelFilters is what an agent's configuration does on one label path.
+type labelFilters struct {
+	extraction otelmetrics.LabelExtraction
+	denylist   otelmetrics.AttributeDenylist
+	namespaces otelmetrics.NamespaceFilter // read for pods only
+}
+
+// agentLabelFilters reads the filters an agent runs for labels from "node" or
+// "pod". It skips the test for a filter the model cannot read.
+func agentLabelFilters(t *testing.T, configMap, from string) labelFilters {
+	t.Helper()
+	raw := agentConfig(t, configMap)
+	f := labelFilters{denylist: parseDenylist(t, configMap, raw)}
+	var err error
+	f.extraction, err = otelmetrics.ParseLabelExtraction(raw, from)
+	skipIfNotModeled(t, configMap, err)
+	t.Logf("ConfigMap %s: %s labels extracted by %s, every label: %v", configMap, from, f.extraction.Processor, f.extraction.ExtractsAll())
+	if from == "pod" {
+		f.namespaces, err = otelmetrics.ParseNamespaceFilter(raw)
+		skipIfNotModeled(t, configMap, err)
+		t.Logf("ConfigMap %s: namespace filter processors %v", configMap, f.namespaces.Processors)
+	}
+	return f
+}
+
+func skipIfNotModeled(t *testing.T, configMap string, err error) {
+	t.Helper()
+	if errors.Is(err, otelmetrics.ErrFilterNotModeled) {
+		t.Skipf("ConfigMap %s: %v. The test cannot tell which labels or objects this filter removes", configMap, err)
+	}
+	require.NoError(t, err, "ConfigMap %s", configMap)
+}
+
+// TestDenylistSparesNeverDeniedAttributes fails for a denylist entry or a label
+// exception transform that removes a never-denied attribute. A customer may
+// remove a protected label with otelFilters.nodeLabels.exclude or
+// otelFilters.podLabels.exclude, so the entries that the declared
+// clusterLabelExcludes render to are logged instead.
 func TestDenylistSparesNeverDeniedAttributes(t *testing.T) {
 	for _, configMap := range []string{nodeAgentConfigMap, clusterScraperConfigMap} {
 		t.Run(configMap, func(t *testing.T) {
 			d := agentDenylist(t, configMap)
-			for _, v := range d.NeverDeniedViolations() {
-				t.Errorf("%s: %s", d.Processor, v)
+			chart, customer := d.SplitCustomerExcludes(clusterLabelExcludes)
+			if entries := customer.Entries(); len(entries) > 0 {
+				t.Logf("%s: %d entries match a declared label exclude: %v", d.Processor, len(entries), entries)
+			}
+			for _, v := range customer.NeverDeniedViolations() {
+				t.Logf("%s: declared label exclude, allowed: %s", d.Processor, v)
+			}
+			for _, v := range chart.NeverDeniedViolations() {
+				t.Errorf("%s: %s (if a label exclude of the deployment adds it, declare the exclude in %s or %s)",
+					d.Processor, v, otelmetrics.PodLabelExcludesEnvVar, otelmetrics.NodeLabelExcludesEnvVar)
 			}
 		})
 	}
@@ -94,14 +167,18 @@ func failedObjects(r otelmetrics.ObjectReconciliation) map[string]bool {
 
 // reconcileTwice runs pass and, if any object failed, runs it again after
 // recheckDelay. Only objects that fail both passes are reported.
-func reconcileTwice(t *testing.T, kind string, pass reconcilePass) {
+func reconcileTwice(t *testing.T, kind string, e otelmetrics.LabelExtraction, pass reconcilePass) {
 	t.Helper()
 	first := pass(t)
 	t.Logf("reconciled %d %ss, %d labels, %d series skipped at the attribute cap, %d failed, %d of %d required with no series",
 		first.Objects, kind, first.Labels, first.AtCap, len(first.Failed), len(first.NoSeries), first.Required)
 	require.NotZero(t, first.Required, "the Kubernetes API listed no %s that must have a series", kind)
 	require.NotZero(t, first.Objects, "no series matched a %s from the Kubernetes API", kind)
-	require.NotZero(t, first.Labels, "%ss matched but carried no checkable label", kind)
+	if first.Labels == 0 && !e.ExtractsAll() {
+		t.Logf("the label include list in %s extracts no label that these %ss carry", e.Processor, kind)
+	} else {
+		require.NotZero(t, first.Labels, "%ss matched but carried no checkable label", kind)
+	}
 	failedBefore := failedObjects(first)
 	if len(failedBefore) == 0 {
 		return
@@ -122,7 +199,7 @@ func reconcileTwice(t *testing.T, kind string, pass reconcilePass) {
 			t.Errorf("%s %s: no series", kind, key)
 			continue
 		}
-		t.Errorf("%s %s: missing %v, value mismatch %v, denied but present %v",
+		t.Errorf("%s %s: missing %v, value mismatch %v, removed but present %v",
 			kind, key, diff.Missing, diff.Mismatched, diff.Leaked)
 	}
 }
@@ -195,6 +272,17 @@ func runningPodsWithAgent(gt *k8sGroundTruth) []string {
 	return keys
 }
 
+// withoutFilteredNamespaces returns the pod keys whose namespace the filter keeps.
+func withoutFilteredNamespaces(gt *k8sGroundTruth, keys []string, f otelmetrics.NamespaceFilter) []string {
+	var kept []string
+	for _, key := range keys {
+		if !f.Drops(gt.pods[key].Namespace) {
+			kept = append(kept, key)
+		}
+	}
+	return kept
+}
+
 // freshGroundTruth lists the nodes and pods now, unlike the session-wide getGroundTruth.
 func freshGroundTruth(t *testing.T) *k8sGroundTruth {
 	t.Helper()
@@ -229,15 +317,15 @@ func TestNodeLabelsReachMetrics(t *testing.T) {
 	} {
 		p := p
 		t.Run(p.agent, func(t *testing.T) {
-			d := agentDenylist(t, p.configMap)
-			reconcileTwice(t, "node", func(t *testing.T) otelmetrics.ObjectReconciliation {
+			f := agentLabelFilters(t, p.configMap, "node")
+			reconcileTwice(t, "node", f.extraction, func(t *testing.T) otelmetrics.ObjectReconciliation {
 				gt := freshGroundTruth(t)
 				return otelmetrics.ReconcileObjects(queryAllSeries(t, p.metric),
 					func(l otelmetrics.MetricLabels) (string, map[string]string, bool) {
 						name := l.Resource["k8s.node.name"]
 						node, ok := gt.nodes[name] // absent if it joined or left between the two reads
 						return name, node.Labels, ok
-					}, p.required(gt), "k8s.node.label.", d)
+					}, p.required(gt), f.extraction, f.denylist)
 			})
 		})
 	}
@@ -250,15 +338,20 @@ func TestPodLabelsReachMetrics(t *testing.T) {
 	} {
 		p := p
 		t.Run(p.agent, func(t *testing.T) {
-			d := agentDenylist(t, p.configMap)
-			reconcileTwice(t, "pod", func(t *testing.T) otelmetrics.ObjectReconciliation {
+			f := agentLabelFilters(t, p.configMap, "pod")
+			reconcileTwice(t, "pod", f.extraction, func(t *testing.T) otelmetrics.ObjectReconciliation {
 				gt := freshGroundTruth(t)
+				reported := p.required(gt)
+				required := withoutFilteredNamespaces(gt, reported, f.namespaces)
+				if len(required) == 0 && len(reported) > 0 {
+					t.Skipf("the namespace filter %v removes all %d pods this path reports", f.namespaces.Processors, len(reported))
+				}
 				return otelmetrics.ReconcileObjects(queryAllSeries(t, p.metric),
 					func(l otelmetrics.MetricLabels) (string, map[string]string, bool) {
 						key := l.Resource["k8s.namespace.name"] + "/" + l.Resource["k8s.pod.name"]
 						pod, ok := gt.pods[key] // absent if it was created or deleted between the two reads
 						return key, pod.Labels, ok
-					}, p.required(gt), "k8s.pod.label.", d)
+					}, required, f.extraction, f.denylist)
 			})
 		})
 	}

@@ -12,6 +12,9 @@ var clusterHostTypes = otelmetrics.ResolveHostTypes([]string{"t3.medium"})
 
 var clusterNodeGroups = otelmetrics.NodeGroupsFor(clusterHostTypes, map[string]string{"t3.medium": "standard"})
 
+// Label exclude patterns the cluster was deployed with; OTEL_POD_LABEL_EXCLUDES and OTEL_NODE_LABEL_EXCLUDES set them.
+var clusterLabelExcludes = otelmetrics.ResolveLabelExcludes()
+
 // --- Metric definitions (standard cluster only) ---
 
 var nodeExporterMetrics = []otelmetrics.MetricDefinition{

@@ -82,6 +82,8 @@ func TestMain(m *testing.M) {
 	}
 
 	fmt.Fprintf(os.Stderr, "Host types: %v (set %s to override)\n", clusterHostTypes, otelmetrics.HostTypesEnvVar)
+	fmt.Fprintf(os.Stderr, "Label excludes: pod %v, node %v (set %s and %s to the cluster's otelFilters.podLabels.exclude and otelFilters.nodeLabels.exclude)\n",
+		clusterLabelExcludes.Pod, clusterLabelExcludes.Node, otelmetrics.PodLabelExcludesEnvVar, otelmetrics.NodeLabelExcludesEnvVar)
 
 	hostMappings := []otelmetrics.SourceHostMapping{
 		{Source: otelmetrics.SourceNodeExporter, HostTypes: clusterHostTypes},

@@ -18,6 +18,24 @@ func ResolveHostTypes(defaults []string) []string {
 	return parseHostTypes(os.Getenv(HostTypesEnvVar), defaults)
 }
 
+// PodLabelExcludesEnvVar and NodeLabelExcludesEnvVar hold the comma-separated
+// otelFilters.podLabels.exclude and otelFilters.nodeLabels.exclude patterns the
+// cluster was deployed with. The denylist test logs the entries they render to,
+// and fails for any other entry that removes a never-denied attribute.
+const (
+	PodLabelExcludesEnvVar  = "OTEL_POD_LABEL_EXCLUDES"
+	NodeLabelExcludesEnvVar = "OTEL_NODE_LABEL_EXCLUDES"
+)
+
+// ResolveLabelExcludes returns the patterns from PodLabelExcludesEnvVar and
+// NodeLabelExcludesEnvVar. Unset means the cluster has no label excludes.
+func ResolveLabelExcludes() LabelExcludes {
+	return LabelExcludes{
+		Pod:  splitList(os.Getenv(PodLabelExcludesEnvVar)),
+		Node: splitList(os.Getenv(NodeLabelExcludesEnvVar)),
+	}
+}
+
 func parseHostTypes(raw string, defaults []string) []string {
 	if hostTypes := splitList(raw); len(hostTypes) > 0 {
 		return hostTypes

@@ -43,6 +43,23 @@ func TestResolveHostTypes(t *testing.T) {
 	}
 }
 
+func TestResolveLabelExcludes(t *testing.T) {
+	for _, name := range []string{PodLabelExcludesEnvVar, NodeLabelExcludesEnvVar} {
+		t.Setenv(name, "")
+		os.Unsetenv(name)
+	}
+	if got := ResolveLabelExcludes(); len(got.Pod) != 0 || len(got.Node) != 0 {
+		t.Fatalf("unset: got %+v, want no excludes", got)
+	}
+
+	t.Setenv(PodLabelExcludesEnvVar, " app.kubernetes.io/* ,, team")
+	t.Setenv(NodeLabelExcludesEnvVar, "karpenter.sh/nodepool")
+	want := LabelExcludes{Pod: []string{"app.kubernetes.io/*", "team"}, Node: []string{"karpenter.sh/nodepool"}}
+	if got := ResolveLabelExcludes(); !reflect.DeepEqual(got, want) {
+		t.Fatalf("got %+v, want %+v", got, want)
+	}
+}
+
 func TestNodeGroupsFor(t *testing.T) {
 	descriptions := map[string]string{"t3.medium": "standard"}
 	got := NodeGroupsFor([]string{"t3.medium", "g5.xlarge"}, descriptions)