diff --git a/test/otel/ebs_csi/attribute_scope_test.go b/test/otel/ebs_csi/attribute_scope_test.go new file mode 100644 index 000000000..67d18837a --- /dev/null +++ b/test/otel/ebs_csi/attribute_scope_test.go @@ -0,0 +1,37 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package ebs_csi + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +// TestEBSCSIDeprecatedSemconvAttributesRemoved asserts the ebs-csi pipeline deletes +// the deprecated semantic-convention attributes the Prometheus receiver injects. +// +// url.scheme is not asserted present here, because the pipeline runs on every node +// and a node near max_total_attributes loses attributes to the tier cascade. +// +// The ebs-csi pipeline needs no resource-scope raw-label check. It runs no +// groupbyattrs processor, so no raw Prometheus name reaches resource scope. +func TestEBSCSIDeprecatedSemconvAttributesRemoved(t *testing.T) { + t.Parallel() + for _, metricName := range ebsCsiMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available (no EBS CSI driver?)", metricName) + require.NoError(t, otelmetrics.CheckDeprecatedSemconvRemoved(metricName, results)) + }) + } +} diff --git a/test/otel/gpu/attribute_scope_test.go b/test/otel/gpu/attribute_scope_test.go new file mode 100644 index 000000000..162473ce0 --- /dev/null +++ b/test/otel/gpu/attribute_scope_test.go @@ -0,0 +1,52 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package gpu + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +// TestDCGMDeprecatedSemconvAttributesRemoved asserts the dcgm pipeline deletes the +// deprecated semantic-convention attributes the Prometheus receiver injects. +// +// url.scheme is not asserted present here. GPU nodes carry enough labels to reach +// max_total_attributes, and the attribute-limit processor then drops by tier. +func TestDCGMDeprecatedSemconvAttributesRemoved(t *testing.T) { + t.Parallel() + for _, metricName := range dcgmMetricNamesList { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available (no GPU nodes?)", metricName) + require.NoError(t, otelmetrics.CheckDeprecatedSemconvRemoved(metricName, results)) + }) + } +} + +// TestDCGMRawLabelsOnlyAtDatapointScope asserts the raw Prometheus labels the dcgm +// groupbyattrs processor lifts to resource scope are deleted there after the +// pipeline copies them back down. TestDCGMHasRawPromotedKeys covers the datapoint side. +func TestDCGMRawLabelsOnlyAtDatapointScope(t *testing.T) { + t.Parallel() + rawKeys := []string{"pod", "namespace", "container"} + for _, metricName := range dcgmMetricNamesList { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available (no GPU nodes?)", metricName) + require.NoError(t, otelmetrics.CheckRawLabelsNotAtResourceScope(metricName, results, rawKeys)) + }) + } +} diff --git a/test/otel/lis_csi/attribute_scope_test.go b/test/otel/lis_csi/attribute_scope_test.go new file mode 100644 index 000000000..6574f9fd9 --- /dev/null +++ b/test/otel/lis_csi/attribute_scope_test.go @@ -0,0 +1,38 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package lis_csi + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +// TestLISCSIDeprecatedSemconvAttributesRemoved asserts the lis-csi pipeline deletes +// the deprecated semantic-convention attributes the Prometheus receiver injects. +// +// url.scheme is not asserted present here, because the pipeline runs on every +// instance-store node and a node near max_total_attributes loses attributes to the +// tier cascade. +// +// The lis-csi pipeline needs no resource-scope raw-label check. It runs no +// groupbyattrs processor, so no raw Prometheus name reaches resource scope. +func TestLISCSIDeprecatedSemconvAttributesRemoved(t *testing.T) { + t.Parallel() + for _, metricName := range lisCsiMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + require.NoError(t, otelmetrics.CheckDeprecatedSemconvRemoved(metricName, results)) + }) + } +} diff --git a/test/otel/neuron/attribute_scope_test.go b/test/otel/neuron/attribute_scope_test.go new file mode 100644 index 000000000..418f24d90 --- /dev/null +++ b/test/otel/neuron/attribute_scope_test.go @@ -0,0 +1,38 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package neuron + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +// TestNeuronDeprecatedSemconvAttributesRemoved asserts the neuron pipeline deletes +// the deprecated semantic-convention attributes the Prometheus receiver injects. +// +// url.scheme is not asserted present here. Trainium nodes carry enough labels to +// reach max_total_attributes, and the attribute-limit processor then drops by tier. +// +// The neuron pipeline needs no resource-scope raw-label check. Its groupbyattrs +// processor lifts the semconv names and runtime_tag rather than raw Prometheus +// names, and neuron_promote renames then deletes runtime_tag. +func TestNeuronDeprecatedSemconvAttributesRemoved(t *testing.T) { + t.Parallel() + for _, metricName := range neuronMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available (no Neuron nodes?)", metricName) + require.NoError(t, otelmetrics.CheckDeprecatedSemconvRemoved(metricName, results)) + }) + } +} diff --git a/test/otel/solutions/karpenter/attribute_scope_test.go b/test/otel/solutions/karpenter/attribute_scope_test.go new file mode 100644 index 000000000..c2741c4b4 --- /dev/null +++ b/test/otel/solutions/karpenter/attribute_scope_test.go @@ -0,0 +1,55 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package karpenter + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +// TestKarpenterDeprecatedSemconvAttributesRemoved asserts the karpenter pipeline +// deletes the deprecated semantic-convention attributes and keeps url.scheme. +// +// The url.scheme assertion is the valuable half. karpenter_promote used to delete +// url.scheme, the current spelling, while keeping http.scheme, the deprecated one. +// These metrics are cluster-scoped and carry a small attribute set, so the tier +// cascade cannot explain a missing url.scheme here. +func TestKarpenterDeprecatedSemconvAttributesRemoved(t *testing.T) { + t.Parallel() + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + require.NoError(t, otelmetrics.CheckDeprecatedSemconvRemoved(metricName, results)) + require.NoError(t, otelmetrics.CheckCurrentSemconvPresent(metricName, results)) + }) + } +} + +// TestKarpenterRawLabelsOnlyAtDatapointScope asserts the raw Prometheus labels the +// karpenter groupbyattrs processor lifts to resource scope are deleted there after +// the pipeline copies them back down to datapoint scope. +func TestKarpenterRawLabelsOnlyAtDatapointScope(t *testing.T) { + t.Parallel() + rawKeys := []string{"pod", "namespace", "node"} + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + require.NoError(t, otelmetrics.CheckRawLabelsNotAtResourceScope(metricName, results, rawKeys)) + }) + } +} diff --git a/test/otel/solutions/keda/attribute_scope_test.go b/test/otel/solutions/keda/attribute_scope_test.go new file mode 100644 index 000000000..37c079233 --- /dev/null +++ b/test/otel/solutions/keda/attribute_scope_test.go @@ -0,0 +1,55 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package keda + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +// TestKEDADeprecatedSemconvAttributesRemoved asserts the keda pipeline deletes the +// deprecated semantic-convention attributes and keeps url.scheme. +// +// The url.scheme assertion is the valuable half. keda_promote used to delete +// url.scheme, the current spelling, while keeping http.scheme, the deprecated one. +// These metrics are cluster-scoped and carry a small attribute set, so the tier +// cascade cannot explain a missing url.scheme here. +func TestKEDADeprecatedSemconvAttributesRemoved(t *testing.T) { + t.Parallel() + for _, metricName := range kedaMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + require.NoError(t, otelmetrics.CheckDeprecatedSemconvRemoved(metricName, results)) + require.NoError(t, otelmetrics.CheckCurrentSemconvPresent(metricName, results)) + }) + } +} + +// TestKEDARawLabelsOnlyAtDatapointScope asserts the raw Prometheus labels the keda +// groupbyattrs processor lifts to resource scope are deleted there after the +// pipeline copies them back down to datapoint scope. +func TestKEDARawLabelsOnlyAtDatapointScope(t *testing.T) { + t.Parallel() + rawKeys := []string{"pod", "namespace", "node"} + for _, metricName := range kedaMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + require.NoError(t, otelmetrics.CheckRawLabelsNotAtResourceScope(metricName, results, rawKeys)) + }) + } +} diff --git a/test/otel/standard/labels_common_test.go b/test/otel/standard/labels_common_test.go index 102008ca6..8834acf3c 100644 --- a/test/otel/standard/labels_common_test.go +++ b/test/otel/standard/labels_common_test.go @@ -907,3 +907,67 @@ func TestPodLabelsStillEnriched(t *testing.T) { }) } } + +// --------------------------------------------------------------------------- +// TestDeprecatedSemconvAttributesRemoved — the deprecated semconv attributes go, +// their current-spelling twins stay. url.scheme is asserted present on purpose: +// karpenter/keda used to delete it and keep http.scheme, which was backwards. +// The checks live in util/otelmetrics so the gpu, neuron, ebs_csi, lis_csi, +// karpenter and keda suites assert the same rule on their own pipelines. +// --------------------------------------------------------------------------- + +func TestDeprecatedSemconvAttributesRemoved(t *testing.T) { + names := prometheusScrapedNames() + names = append(names, metricNames(allKSMMetricDefs)...) + + for _, metricName := range names { + t.Run(metricName, func(t *testing.T) { + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + require.NoError(t, otelmetrics.CheckDeprecatedSemconvRemoved(metricName, results)) + require.NoError(t, otelmetrics.CheckCurrentSemconvPresent(metricName, results)) + }) + } +} + +// --------------------------------------------------------------------------- +// TestRawLabelsOnlyAtDatapointScope — the raw Prometheus labels must exist only at +// datapoint scope, not at resource scope too. The datapoint side is covered by +// TestCadvisorHasRawPromotedKeys and the TestKSM_*_HasRaw*Label family. +// --------------------------------------------------------------------------- + +func TestRawLabelsOnlyAtDatapointScope(t *testing.T) { + cases := []struct { + pipeline string + names []string + rawKeys []string + }{ + { + pipeline: "cadvisor", + names: cadvisorMetricNamesList, + rawKeys: []string{"pod", "namespace", "container"}, + }, + { + pipeline: "kube-state-metrics", + names: metricNames(allKSMMetricDefs), + rawKeys: []string{ + "pod", "namespace", "node", "uid", "container", + "owner_name", "owner_kind", "deployment", "daemonset", + "statefulset", "replicaset", "job_name", "cronjob", + }, + }, + } + + for _, c := range cases { + c := c + for _, metricName := range c.names { + t.Run(c.pipeline+"/"+metricName, func(t *testing.T) { + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + require.NoError(t, otelmetrics.CheckRawLabelsNotAtResourceScope(metricName, results, c.rawKeys)) + }) + } + } +} diff --git a/util/otelmetrics/attribute_scope.go b/util/otelmetrics/attribute_scope.go new file mode 100644 index 000000000..0806cb026 --- /dev/null +++ b/util/otelmetrics/attribute_scope.go @@ -0,0 +1,93 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package otelmetrics + +import ( + "fmt" + "sort" + "strings" +) + +// DeprecatedSemconvResourceKeys are the deprecated semantic-convention attributes +// the Prometheus receiver injects. Every Container Insights pipeline deletes them; +// their current-spelling twins (server.address, server.port, url.scheme) stay. +var DeprecatedSemconvResourceKeys = []string{"net.host.name", "net.host.port", "http.scheme"} + +// CheckDeprecatedSemconvRemoved returns an error if any result carries a +// deprecated semantic-convention attribute at resource scope. Pass results from a +// pipeline that uses the Prometheus receiver. +// +// Call CheckCurrentSemconvPresent alongside this on pipelines where the inverted +// deletion is a live risk. Deleting both spellings also satisfies this check. +func CheckDeprecatedSemconvRemoved(metricName string, results []MetricResult) error { + var problems []string + for _, r := range results { + for _, key := range DeprecatedSemconvResourceKeys { + if _, present := r.Labels.Resource[key]; present { + problems = append(problems, fmt.Sprintf("@resource.%s on node %s", + key, r.Labels.Resource["k8s.node.name"])) + } + } + } + if len(problems) == 0 { + return nil + } + return fmt.Errorf("%s still carries deprecated semconv attributes: %s", + metricName, strings.Join(dedupeSorted(problems), ", ")) +} + +// CheckCurrentSemconvPresent returns an error if no result carries url.scheme at +// resource scope. This is the guard against an inverted deletion that removes the +// current spelling and keeps the deprecated one, which the karpenter, keda and +// kserve promote transforms each did at some point. +// +// Do not call this on a pipeline whose nodes can exceed max_total_attributes, +// because the attribute-limit processor then drops attributes by tier and +// url.scheme is not on the protected list. +func CheckCurrentSemconvPresent(metricName string, results []MetricResult) error { + for _, r := range results { + if r.Labels.Resource["url.scheme"] != "" { + return nil + } + } + return fmt.Errorf("%s has no url.scheme on any series: the current-spelling "+ + "attribute must survive when the deprecated http.scheme is deleted", metricName) +} + +// CheckRawLabelsNotAtResourceScope returns an error if any result carries one of +// rawKeys at resource scope. Pipelines that run groupbyattrs lift these raw +// Prometheus label names to resource scope, copy them back down to datapoint +// scope for dashboard compatibility, then delete the resource-scope copies. +// +// This checks the resource side only. Assert the datapoint copies separately, +// because a pipeline that deletes both would otherwise pass. +func CheckRawLabelsNotAtResourceScope(metricName string, results []MetricResult, rawKeys []string) error { + var problems []string + for _, r := range results { + for _, key := range rawKeys { + if _, present := r.Labels.Resource[key]; present { + problems = append(problems, fmt.Sprintf("@resource.%s on node %s", + key, r.Labels.Resource["k8s.node.name"])) + } + } + } + if len(problems) == 0 { + return nil + } + return fmt.Errorf("%s carries raw labels at resource scope as well as datapoint scope: %s", + metricName, strings.Join(dedupeSorted(problems), ", ")) +} + +func dedupeSorted(in []string) []string { + seen := make(map[string]bool, len(in)) + out := make([]string, 0, len(in)) + for _, s := range in { + if !seen[s] { + seen[s] = true + out = append(out, s) + } + } + sort.Strings(out) + return out +} diff --git a/util/otelmetrics/attribute_scope_test.go b/util/otelmetrics/attribute_scope_test.go new file mode 100644 index 000000000..eee35d33a --- /dev/null +++ b/util/otelmetrics/attribute_scope_test.go @@ -0,0 +1,115 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package otelmetrics + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func resultWithResource(attrs map[string]string) MetricResult { + labels := NewMetricLabels() + for k, v := range attrs { + labels.Resource[k] = v + } + return MetricResult{Labels: labels} +} + +func TestCheckDeprecatedSemconvRemoved(t *testing.T) { + t.Run("clean", func(t *testing.T) { + results := []MetricResult{resultWithResource(map[string]string{ + "server.address": "10.0.0.1", "server.port": "9100", "url.scheme": "http", + })} + require.NoError(t, CheckDeprecatedSemconvRemoved("node_load1", results)) + }) + + t.Run("each deprecated key is caught", func(t *testing.T) { + for _, key := range DeprecatedSemconvResourceKeys { + results := []MetricResult{resultWithResource(map[string]string{ + key: "x", "k8s.node.name": "ip-10-0-0-1", + })} + err := CheckDeprecatedSemconvRemoved("node_load1", results) + require.Error(t, err, "key %s went undetected", key) + require.Contains(t, err.Error(), key) + require.Contains(t, err.Error(), "ip-10-0-0-1") + } + }) + + t.Run("empty results pass", func(t *testing.T) { + require.NoError(t, CheckDeprecatedSemconvRemoved("node_load1", nil)) + }) + + t.Run("one offending series among many is caught", func(t *testing.T) { + results := []MetricResult{ + resultWithResource(map[string]string{"url.scheme": "http"}), + resultWithResource(map[string]string{"http.scheme": "http"}), + } + require.Error(t, CheckDeprecatedSemconvRemoved("node_load1", results)) + }) +} + +func TestCheckCurrentSemconvPresent(t *testing.T) { + t.Run("present on one series is enough", func(t *testing.T) { + results := []MetricResult{ + resultWithResource(map[string]string{}), + resultWithResource(map[string]string{"url.scheme": "http"}), + } + require.NoError(t, CheckCurrentSemconvPresent("node_load1", results)) + }) + + t.Run("absent everywhere fails", func(t *testing.T) { + results := []MetricResult{resultWithResource(map[string]string{"http.scheme": "http"})} + err := CheckCurrentSemconvPresent("node_load1", results) + require.Error(t, err) + require.Contains(t, err.Error(), "url.scheme") + }) + + t.Run("empty value counts as absent", func(t *testing.T) { + results := []MetricResult{resultWithResource(map[string]string{"url.scheme": ""})} + require.Error(t, CheckCurrentSemconvPresent("node_load1", results)) + }) + + t.Run("empty results fail", func(t *testing.T) { + require.Error(t, CheckCurrentSemconvPresent("node_load1", nil)) + }) +} + +func TestCheckRawLabelsNotAtResourceScope(t *testing.T) { + rawKeys := []string{"pod", "namespace", "container"} + + t.Run("semconv names at resource scope are fine", func(t *testing.T) { + results := []MetricResult{resultWithResource(map[string]string{ + "k8s.pod.name": "p", "k8s.namespace.name": "n", "k8s.container.name": "c", + })} + require.NoError(t, CheckRawLabelsNotAtResourceScope("container_cpu_usage_seconds_total", results, rawKeys)) + }) + + t.Run("each raw key is caught", func(t *testing.T) { + for _, key := range rawKeys { + results := []MetricResult{resultWithResource(map[string]string{ + key: "x", "k8s.node.name": "ip-10-0-0-1", + })} + err := CheckRawLabelsNotAtResourceScope("container_cpu_usage_seconds_total", results, rawKeys) + require.Error(t, err, "key %s went undetected", key) + require.Contains(t, err.Error(), key) + } + }) + + t.Run("empty value still counts as present", func(t *testing.T) { + results := []MetricResult{resultWithResource(map[string]string{"pod": ""})} + require.Error(t, CheckRawLabelsNotAtResourceScope("container_cpu_usage_seconds_total", results, rawKeys)) + }) + + t.Run("duplicate problems are reported once", func(t *testing.T) { + results := []MetricResult{ + resultWithResource(map[string]string{"pod": "a", "k8s.node.name": "ip-10-0-0-1"}), + resultWithResource(map[string]string{"pod": "b", "k8s.node.name": "ip-10-0-0-1"}), + } + err := CheckRawLabelsNotAtResourceScope("container_cpu_usage_seconds_total", results, rawKeys) + require.Error(t, err) + require.Equal(t, 1, strings.Count(err.Error(), "@resource.pod")) + }) +}