From 475fdd75e8afab4d563d9aaed3e8ad3a16b6aad2 Mon Sep 17 00:00:00 2001 From: Petru Anica-Popa Date: Wed, 14 Oct 2026 10:54:57 +0000 Subject: [PATCH] test(otel): cover Karpenter node-label and pod-label counterpart removal --- test/otel/standard/labels_common_test.go | 235 +++++++++++++++++++++++ 1 file changed, 235 insertions(+) diff --git a/test/otel/standard/labels_common_test.go b/test/otel/standard/labels_common_test.go index a6ab05541..102008ca6 100644 --- a/test/otel/standard/labels_common_test.go +++ b/test/otel/standard/labels_common_test.go @@ -13,6 +13,8 @@ import ( "testing" "github.com/stretchr/testify/require" + + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" ) // --------------------------------------------------------------------------- @@ -672,3 +674,236 @@ func TestScrapeMetadataFiltered(t *testing.T) { }) } } + +// --------------------------------------------------------------------------- +// Karpenter node labels. The instance labels are denied by karpenter.k8s.aws/instance- +// prefixes and keys, all but instance-tenancy, a launch property that is +// retained. The EKS Auto Mode copies are denied by eks.amazonaws.com/instance-; +// no retained key matches it. The three karpenter.sh/ keys are enumerated, +// because nodepool and capacity-type are retained in that domain. +// TestKarpenterInstanceLabelsRemoved covers the removal, +// TestKarpenterGroupingLabelsRetained covers the retention — a regression that +// widened either to a whole domain prefix fails the second, even where it passes the first. +// +// Both agents run the denylist, so the tests read the node agent's DaemonSet +// metrics and the cluster scraper's kube-state-metrics node metrics. They read the +// series of every Karpenter-managed node rather than the clusterHostTypes pin, +// because Karpenter picks the instance type of the node it launches. +// --------------------------------------------------------------------------- + +// deniedNodeLabelPrefixes are the instance-label prefixes the denylist covers, except instanceTenancyLabel. +var deniedNodeLabelPrefixes = []string{ + "k8s.node.label.karpenter.k8s.aws/instance-", + "k8s.node.label.eks.amazonaws.com/instance-", +} + +// instanceTenancyLabel is the one retained instance label: tenancy is not derived from the instance type. +const instanceTenancyLabel = "k8s.node.label.karpenter.k8s.aws/instance-tenancy" + +// deniedKarpenterNodeLabels are the karpenter.sh/ entries in unconditional_removal_keys. +var deniedKarpenterNodeLabels = []string{ + "k8s.node.label.karpenter.sh/do-not-sync-taints", + "k8s.node.label.karpenter.sh/initialized", + "k8s.node.label.karpenter.sh/registered", +} + +// karpenterNodePoolLabel is set by Karpenter on every node it launches. +const karpenterNodePoolLabel = "k8s.node.label.karpenter.sh/nodepool" + +// retainedKarpenterNodeLabels are set by Karpenter on every node it manages and +// must survive. capacity-reservation-* and placement-group-* are retained too, +// but Karpenter only sets those on nodes launched into a capacity reservation or +// a placement group, so they are not asserted here. +var retainedKarpenterNodeLabels = []string{ + karpenterNodePoolLabel, + "k8s.node.label.karpenter.sh/capacity-type", + "k8s.node.label.karpenter.k8s.aws/ec2nodeclass", + instanceTenancyLabel, +} + +// autoModeComputeTypeLabel is "auto" on the nodes EKS Auto Mode manages. +const autoModeComputeTypeLabel = "k8s.node.label.eks.amazonaws.com/compute-type" + +// retainedAutoModeNodeLabels are the retained labels of an EKS Auto Mode node, which names +// its node class eks.amazonaws.com/nodeclass and sets no karpenter.k8s.aws/ labels. +var retainedAutoModeNodeLabels = []string{ + karpenterNodePoolLabel, + "k8s.node.label.karpenter.sh/capacity-type", + "k8s.node.label.eks.amazonaws.com/nodeclass", + autoModeComputeTypeLabel, +} + +// karpenterLabelMetricNames are the metrics of both agents that carry node labels. +func karpenterLabelMetricNames() []string { + return append(append([]string{}, nodeLabelEnrichedNames()...), ksmNodeBucket...) +} + +// podLabelMetricNames are the metrics of both agents that carry pod labels. Node-scoped +// metrics have no pod, so they never carry k8s.pod.label.* — see TestNodeExporterNoPodLabels. +func podLabelMetricNames() []string { + names := append(append([]string{}, podMetricNames()...), ksmPodBucket...) + return append(names, ksmContainerBucket...) +} + +// allNodeSeries returns every series of a metric in this cluster, on every node. +func allNodeSeries(t *testing.T, metricName string) []otelmetrics.MetricResult { + t.Helper() + results, err := queryCache.GetWithFilter(context.Background(), metricName, nil) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + return results +} + +// karpenterNodeSeries returns the series of the nodes that carry karpenterNodePoolLabel, +// on every instance type. The filter also keeps the result under the endpoint's +// per-query series limit on a large cluster. +// +// This suite targets a cluster that runs Karpenter, so both tests assert rather +// than skip when they find none: a silent skip is how a regression that stripped +// every karpenter.* attribute, or a read that missed the Karpenter node, would +// hide, and a loud failure on a cluster without Karpenter is the safer of the two +// mistakes. +func karpenterNodeSeries(t *testing.T, metricName string) []otelmetrics.MetricResult { + t.Helper() + results, err := queryCache.GetWithFilter(context.Background(), metricName, + map[string]string{"~@resource." + karpenterNodePoolLabel: ".+"}) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, + "%s has no series of a node with @resource.%s. Either the cluster does not run "+ + "Karpenter, or every karpenter.* attribute was stripped — which is what happens "+ + "if the denylist widens to the karpenter.k8s.aws/ and karpenter.sh/ domain "+ + "prefixes", metricName, karpenterNodePoolLabel) + return results +} + +func deniedKarpenterNodeLabel(key string) bool { + if key == instanceTenancyLabel { + return false + } + for _, k := range deniedKarpenterNodeLabels { + if key == k { + return true + } + } + for _, p := range deniedNodeLabelPrefixes { + if strings.HasPrefix(key, p) { + return true + } + } + return false +} + +func TestKarpenterInstanceLabelsRemoved(t *testing.T) { + for _, metricName := range karpenterLabelMetricNames() { + t.Run(metricName, func(t *testing.T) { + for _, r := range karpenterNodeSeries(t, metricName) { + for key := range r.Labels.Resource { + require.False(t, deniedKarpenterNodeLabel(key), + "%s still carries denied Karpenter label @resource.%s on node %s", + metricName, key, r.Labels.Resource["k8s.node.name"]) + } + } + }) + } +} + +func TestKarpenterGroupingLabelsRetained(t *testing.T) { + for _, metricName := range karpenterLabelMetricNames() { + t.Run(metricName, func(t *testing.T) { + // Karpenter and EKS Auto Mode nodes carry different labels, so each kind + // is checked against its own list, and a kind with no nodes is skipped. + var karpenter, autoMode []otelmetrics.MetricResult + for _, r := range karpenterNodeSeries(t, metricName) { + if r.Labels.Resource[autoModeComputeTypeLabel] == "auto" { + autoMode = append(autoMode, r) + } else { + karpenter = append(karpenter, r) + } + } + + // Only managed nodes carry these. A cluster can mix managed nodes with + // a static node group, so require at least one node with each key + // rather than every node. + for _, kind := range []struct { + name string + results []otelmetrics.MetricResult + keys []string + }{ + {"Karpenter", karpenter, retainedKarpenterNodeLabels}, + {"EKS Auto Mode", autoMode, retainedAutoModeNodeLabels}, + } { + if len(kind.results) == 0 { + continue + } + for _, key := range kind.keys { + found := false + for _, r := range kind.results { + if _, present := r.Labels.Resource[key]; present { + found = true + break + } + } + require.True(t, found, + "%s has no @resource.%s on any %s node — a retained grouping label was dropped, "+ + "which happens if the denylist widens to the karpenter.k8s.aws/, karpenter.sh/ "+ + "or eks.amazonaws.com/ domain prefix", metricName, key, kind.name) + } + } + }) + } +} + +// deniedCounterpartNodeAndPodLabels are the two pod-side topology counterparts of node +// labels the denylist already removed, plus pod-template-generation from the +// controller-revision family. All are resource-level. +var deniedCounterpartNodeAndPodLabels = []string{ + "k8s.pod.label.topology.kubernetes.io/region", + "k8s.pod.label.topology.kubernetes.io/zone", + "k8s.pod.label.pod-template-generation", +} + +// TestPodLabelCounterpartsRemoved asserts none of the three counterpart keys reach resource +// scope on the pod metrics of either agent. The node forms of four of them were already +// denied, so a datapoint carrying the pod form is the duplicate this change removes. +func TestPodLabelCounterpartsRemoved(t *testing.T) { + for _, metricName := range podLabelMetricNames() { + t.Run(metricName, func(t *testing.T) { + results := allNodeSeries(t, metricName) + for _, r := range results { + for _, key := range deniedCounterpartNodeAndPodLabels { + _, present := r.Labels.Resource[key] + require.False(t, present, + "%s still carries denied label @resource.%s on node %s", + metricName, key, r.Labels.Resource["k8s.node.name"]) + } + } + }) + } +} + +// TestPodLabelsStillEnriched guards against the denylist being widened to the +// k8s.pod.label. prefix, which would strip every pod label rather than these three. It +// requires at least one pod label to survive somewhere in the result set, so a prefix +// regression fails here instead of passing both this file's removal assertions. It +// reads the same metrics as TestPodLabelCounterpartsRemoved. +func TestPodLabelsStillEnriched(t *testing.T) { + for _, metricName := range podLabelMetricNames() { + t.Run(metricName, func(t *testing.T) { + results := allNodeSeries(t, metricName) + + denied := map[string]bool{} + for _, k := range deniedCounterpartNodeAndPodLabels { + denied[k] = true + } + for _, r := range results { + for k := range r.Labels.Resource { + if strings.HasPrefix(k, "k8s.pod.label.") && !denied[k] { + return // a pod label other than the denied ones survived + } + } + } + t.Fatalf("%s carries no k8s.pod.label.* attribute other than the denied keys; "+ + "the denylist may have been widened to the k8s.pod.label. prefix", metricName) + }) + } +}