diff --git a/test/otel/attr_limit/setup_test.go b/test/otel/attr_limit/setup_test.go index bf25761e0..85f4d7d70 100644 --- a/test/otel/attr_limit/setup_test.go +++ b/test/otel/attr_limit/setup_test.go @@ -26,7 +26,8 @@ var ( queryCache *otelmetrics.QueryCache ) -var clusterHostTypes = []string{"t3.medium"} +// Instance types in the attribute-limit cluster; OTEL_HOST_TYPES overrides them. +var clusterHostTypes = otelmetrics.ResolveHostTypes([]string{"t3.medium"}) func TestMain(m *testing.M) { environment.RegisterEnvironmentMetaDataFlags() @@ -80,6 +81,8 @@ func TestMain(m *testing.M) { os.Exit(1) } + fmt.Fprintf(os.Stderr, "Host types: %v (set %s to override)\n", clusterHostTypes, otelmetrics.HostTypesEnvVar) + hostMappings := []otelmetrics.SourceHostMapping{ {Source: otelmetrics.SourceNodeExporter, HostTypes: clusterHostTypes}, {Source: otelmetrics.SourceCadvisor, HostTypes: clusterHostTypes}, diff --git a/test/otel/ebs_csi/setup_test.go b/test/otel/ebs_csi/setup_test.go index 95a95a94a..6226e3ca4 100644 --- a/test/otel/ebs_csi/setup_test.go +++ b/test/otel/ebs_csi/setup_test.go @@ -29,8 +29,8 @@ var ( // Instrumentation scope name constant. const scopePrometheus = "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver" -// Instance types in the EBS CSI cluster. -var clusterHostTypes = []string{"t3.medium"} +// Instance types in the EBS CSI cluster; OTEL_HOST_TYPES overrides them. +var clusterHostTypes = otelmetrics.ResolveHostTypes([]string{"t3.medium"}) func TestMain(m *testing.M) { environment.RegisterEnvironmentMetaDataFlags() @@ -84,6 +84,8 @@ func TestMain(m *testing.M) { os.Exit(1) } + fmt.Fprintf(os.Stderr, "Host types: %v (set %s to override)\n", clusterHostTypes, otelmetrics.HostTypesEnvVar) + hostMappings := []otelmetrics.SourceHostMapping{ {Source: otelmetrics.SourceEBSCSI, HostTypes: nil}, } diff --git a/test/otel/efa/attribute_schema_test.go b/test/otel/efa/attribute_schema_test.go new file mode 100644 index 000000000..9ec6544d3 --- /dev/null +++ b/test/otel/efa/attribute_schema_test.go @@ -0,0 +1,54 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package efa + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +// --------------------------------------------------------------------------- +// TestAttributeSchema — the closed set of resource and instrumentation-scope +// attributes on EFA series. Every series must have every required key and +// nothing undeclared, expected keys must be on at least one series, and keys set +// together must arrive together. Node and pod labels are open by construction +// (k8sattributes extracts "(.*)"). Datapoint scope is not covered. +// --------------------------------------------------------------------------- + +// awsdevicepodcorrelation then efa_promote; only on devices a pod holds. +var efaPodKeys = []string{"k8s.pod.name", "k8s.namespace.name", "k8s.container.name"} + +// awsefareceiver sets no resource attributes, so unlike the scraped pipelines +// there are no server.* or service.* keys. This suite runs no workload that holds +// a device, so the pod keys are optional rather than expected. +var efaResourceSchema = otelmetrics.KeySchema{ + Required: otelmetrics.ConcatKeys(otelmetrics.CloudKeys, otelmetrics.HostKeys, []string{ + "k8s.cluster.name", // set_cluster_name; protected + "k8s.node.name", // set_node_name, promote_node_name; protected + "k8s.node.uid", // k8sattributes/cw_k8s_ci_v0_node + }), + Optional: otelmetrics.ConcatKeys(efaPodKeys, []string{"k8s.pod.uid"}, otelmetrics.OwnerKindKeys, otelmetrics.WorkloadKeys), + OpenPrefixes: []string{"k8s.node.label.", "k8s.pod.label."}, + Implies: otelmetrics.MergeImplies(otelmetrics.WorkloadImplies(), otelmetrics.Together(efaPodKeys...)), +} + +func TestAttributeSchema(t *testing.T) { + t.Parallel() + tally, err := otelmetrics.TallySchema(context.Background(), allNodesCache, efaMetricNamesList, otelmetrics.GroupSchema{ + Resource: efaResourceSchema, + Instrumentation: otelmetrics.KeySchema{Required: otelmetrics.ScopeKeys}, // set_scope_efa + }, nil) + require.NoError(t, err, "no EFA nodes?") + t.Logf("%d series across %d metrics", tally.Series, len(efaMetricNamesList)) + require.NotZero(t, tally.Series, "no series checked") + for _, p := range tally.Problems() { + t.Errorf("%s: change the schema if intended", p) + } +} diff --git a/test/otel/efa/dedup_test.go b/test/otel/efa/dedup_test.go index 4d8b7e73c..51b968bd4 100644 --- a/test/otel/efa/dedup_test.go +++ b/test/otel/efa/dedup_test.go @@ -48,7 +48,7 @@ func TestEFANoDuplicateSeries(t *testing.T) { `efa_rx_bytes{"@resource.k8s.cluster.name"="%s","@resource.k8s.node.name"="%s","aws.efa.device"="%s"}`, escaped, pinNode, pinDevice, ) - pinned, err := client.Query(ctx, promql) + pinned, err := client.QueryComplete(ctx, promql) require.NoError(t, err, "querying pinned efa_rx_bytes") require.NotEmpty(t, pinned, "pinned efa_rx_bytes returned 0 results") require.Equal(t, 1, len(pinned), diff --git a/test/otel/efa/efa_test.go b/test/otel/efa/efa_test.go index 2d80ff1ef..0f2993192 100644 --- a/test/otel/efa/efa_test.go +++ b/test/otel/efa/efa_test.go @@ -406,7 +406,7 @@ func TestEFANodeGroupCoverage(t *testing.T) { promql := fmt.Sprintf( `efa_rx_bytes{"@resource.k8s.cluster.name"="%s","@resource.host.type"="%s"}`, escaped, "c5n.9xlarge") - results, err := client.Query(context.Background(), promql) + results, err := client.QueryComplete(context.Background(), promql) require.NoError(t, err, "querying efa_rx_bytes on c5n.9xlarge") require.True(t, len(results) > 0, "EFA metrics missing from c5n.9xlarge nodes — awsefareceiver not collecting?") diff --git a/test/otel/efa/node_color_test.go b/test/otel/efa/node_color_test.go index ab80aca2d..29dd6bf4c 100644 --- a/test/otel/efa/node_color_test.go +++ b/test/otel/efa/node_color_test.go @@ -31,7 +31,7 @@ func TestEFAHasYellowNodeColor(t *testing.T) { promql := fmt.Sprintf( `%s{"@resource.k8s.cluster.name"="%s","@resource.%s"="yellow"}`, metricName, escaped, nodeColorLabel) - results, err := client.Query(context.Background(), promql) + results, err := client.QueryComplete(context.Background(), promql) require.NoError(t, err, "querying %s with node-color=yellow filter", metricName) require.True(t, len(results) > 0, "%s: no results from c5n.9xlarge nodes (expected node-color=yellow)", metricName) diff --git a/test/otel/efa/setup_test.go b/test/otel/efa/setup_test.go index 1d218f48c..1ec843e96 100644 --- a/test/otel/efa/setup_test.go +++ b/test/otel/efa/setup_test.go @@ -24,10 +24,12 @@ var ( cfg otelmetrics.TestConfig client *otelmetrics.OtelMetricsClient queryCache *otelmetrics.QueryCache + // allNodesCache reads every series on every node, for the attribute schema. + allNodesCache *otelmetrics.QueryCache ) -// Instance types in the EFA cluster. -var clusterHostTypes = []string{"c5n.9xlarge"} +// Instance types in the EFA cluster; OTEL_HOST_TYPES overrides them. +var clusterHostTypes = otelmetrics.ResolveHostTypes([]string{"c5n.9xlarge"}) func TestMain(m *testing.M) { environment.RegisterEnvironmentMetaDataFlags() @@ -81,6 +83,8 @@ func TestMain(m *testing.M) { os.Exit(1) } + fmt.Fprintf(os.Stderr, "Host types: %v (set %s to override)\n", clusterHostTypes, otelmetrics.HostTypesEnvVar) + hostMappings := []otelmetrics.SourceHostMapping{ {Source: otelmetrics.SourceNodeExporter, HostTypes: clusterHostTypes}, {Source: otelmetrics.SourceCadvisor, HostTypes: clusterHostTypes}, @@ -88,17 +92,23 @@ func TestMain(m *testing.M) { {Source: otelmetrics.SourceEFA, HostTypes: clusterHostTypes}, } - registry := otelmetrics.NewSourceRegistry(clusterHostTypes, hostMappings, - otelmetrics.SourceMapping{Source: otelmetrics.SourceNodeExporter, Metrics: nodeExporterMetrics}, - otelmetrics.SourceMapping{Source: otelmetrics.SourceCadvisor, Metrics: cadvisorMetrics}, - otelmetrics.SourceMapping{Source: otelmetrics.SourceKubeletstats, Metrics: kubeletstatsMetrics}, - otelmetrics.SourceMapping{Source: otelmetrics.SourceEFA, Metrics: efaMetrics}, - ) + sourceMappings := []otelmetrics.SourceMapping{ + {Source: otelmetrics.SourceNodeExporter, Metrics: nodeExporterMetrics}, + {Source: otelmetrics.SourceCadvisor, Metrics: cadvisorMetrics}, + {Source: otelmetrics.SourceKubeletstats, Metrics: kubeletstatsMetrics}, + {Source: otelmetrics.SourceEFA, Metrics: efaMetrics}, + } + registry := otelmetrics.NewSourceRegistry(clusterHostTypes, hostMappings, sourceMappings...) queryCache = otelmetrics.NewQueryCache(client, cfg.ClusterName, otelmetrics.WithHostTypes(clusterHostTypes), otelmetrics.WithSourceRegistry(registry), ) + // Same routing with no host types, sharing queryCache's concurrency limit. + allNodesCache = otelmetrics.NewQueryCache(client, cfg.ClusterName, + otelmetrics.WithSourceRegistry(otelmetrics.NewSourceRegistry(nil, nil, sourceMappings...)), + otelmetrics.WithSharedConcurrency(queryCache), + ) os.Exit(m.Run()) } diff --git a/test/otel/gpu/attribute_schema_test.go b/test/otel/gpu/attribute_schema_test.go new file mode 100644 index 000000000..d501088d5 --- /dev/null +++ b/test/otel/gpu/attribute_schema_test.go @@ -0,0 +1,54 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package gpu + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +// --------------------------------------------------------------------------- +// TestAttributeSchema — the closed set of resource and instrumentation-scope +// attributes on DCGM series. Every series must have every required key and +// nothing undeclared, expected keys must be on at least one series, and keys set +// together must arrive together. Node and pod labels are open by construction +// (k8sattributes extracts "(.*)"). Datapoint scope is not covered. +// --------------------------------------------------------------------------- + +// dcgm_promote; dcgm-exporter maps a GPU to a pod only while the pod holds it. +var dcgmPodKeys = []string{"k8s.pod.name", "k8s.namespace.name", "k8s.container.name"} + +var dcgmResourceSchema = otelmetrics.KeySchema{ + Required: otelmetrics.ConcatKeys(otelmetrics.PrometheusReceiverKeys, otelmetrics.CloudKeys, otelmetrics.HostKeys, []string{ + "k8s.cluster.name", // set_cluster_name; protected + "k8s.node.name", // set_node_name, promote_node_name; protected + "k8s.node.uid", // k8sattributes/cw_k8s_ci_v0_node + }), + // The multi-gpu-burn Deployment holds GPUs; TestDCGMPodName requires it. + Expected: otelmetrics.ConcatKeys(dcgmPodKeys, []string{"k8s.deployment.name", "k8s.replicaset.name"}, otelmetrics.WorkloadKeys), + Optional: otelmetrics.ConcatKeys([]string{"k8s.pod.uid"}, // k8sattributes/cw_k8s_ci_v0_pod + otelmetrics.Without(otelmetrics.OwnerKindKeys, "k8s.deployment.name", "k8s.replicaset.name")), + OpenPrefixes: []string{"k8s.node.label.", "k8s.pod.label."}, + Implies: otelmetrics.MergeImplies(otelmetrics.WorkloadImplies(), otelmetrics.Together(dcgmPodKeys...)), +} + +func TestAttributeSchema(t *testing.T) { + t.Parallel() + tally, err := otelmetrics.TallySchema(context.Background(), allNodesCache, dcgmMetricNamesList, otelmetrics.GroupSchema{ + Resource: dcgmResourceSchema, + Instrumentation: otelmetrics.KeySchema{Required: otelmetrics.ScopeKeys}, // set_scope_dcgm + }, nil) + require.NoError(t, err, "no GPU nodes?") + t.Logf("%d series across %d metrics", tally.Series, len(dcgmMetricNamesList)) + require.NotZero(t, tally.Series, "no series checked") + for _, p := range tally.Problems() { + t.Errorf("%s: change the schema if intended", p) + } +} diff --git a/test/otel/gpu/dcgm_test.go b/test/otel/gpu/dcgm_test.go index 4a096ab5b..3ecbbaae4 100644 --- a/test/otel/gpu/dcgm_test.go +++ b/test/otel/gpu/dcgm_test.go @@ -87,7 +87,7 @@ func TestDCGMNamespace(t *testing.T) { } // TestDCGMMultiNodeCoverage validates that DCGM_FI_DEV_GPU_UTIL is reported -// from at least 2 GPU nodes (g4dn.xlarge + g4dn.12xlarge). +// from at least 2 GPU nodes (the idle node and the busy node). func TestDCGMMultiNodeCoverage(t *testing.T) { t.Parallel() results, err := queryCache.Get(context.Background(), "DCGM_FI_DEV_GPU_UTIL") @@ -104,8 +104,22 @@ func TestDCGMMultiNodeCoverage(t *testing.T) { "Expected DCGM metrics from at least 2 GPU nodes, got %d", len(nodes)) } +// TestDCGMGPUCountMatchesAllocatable validates that every GPU node reports +// one distinct gpu in DCGM_FI_DEV_GPU_UTIL per device in its nvidia.com/gpu +// allocatable. +func TestDCGMGPUCountMatchesAllocatable(t *testing.T) { + t.Parallel() + for _, node := range gpuRoles(t).Nodes { + want := allocatableGPUs(t, node.Name) + gpus := node.GPUs() + t.Logf("%s: nvidia.com/gpu allocatable %d, DCGM gpu %v", node.Name, want, gpus) + require.Equal(t, want, len(gpus), "Expected %d distinct gpu on %s (nvidia.com/gpu allocatable), got %d: %v", + want, node.Name, len(gpus), gpus) + } +} + // TestDCGMActiveNodeNonzero validates that at least one GPU node has -// non-zero utilization (multi-gpu-burn should be running on g4dn.12xlarge). +// non-zero utilization (multi-gpu-burn should be running on the busy node). func TestDCGMActiveNodeNonzero(t *testing.T) { t.Parallel() results, err := queryCache.Get(context.Background(), "DCGM_FI_DEV_GPU_UTIL") @@ -211,7 +225,7 @@ func TestDCGMExpectedLabels(t *testing.T) { // TestDCGMGpuburnContainerName validates that multi-gpu-burn pod results have // k8s.container.name=gpu-burn for all DCGM metrics. -// In this ephemeral cluster, multi-gpu-burn on g4dn.12xlarge is the active workload. +// In this ephemeral cluster, multi-gpu-burn on the busy node is the active workload. func TestDCGMGpuburnContainerName(t *testing.T) { t.Parallel() for _, metricName := range dcgmMetricNamesList { @@ -238,9 +252,10 @@ func TestDCGMGpuburnContainerName(t *testing.T) { } // TestDCGMIdleNodeNoContainerName validates that uncorrelated DCGM results -// (from idle g4dn.xlarge) do NOT have k8s.container.name. +// (from the idle node) do NOT have k8s.container.name. func TestDCGMIdleNodeNoContainerName(t *testing.T) { t.Parallel() + nodes := idleNodes(t) for _, metricName := range dcgmMetricNamesList { metricName := metricName t.Run(metricName, func(t *testing.T) { @@ -248,8 +263,8 @@ func TestDCGMIdleNodeNoContainerName(t *testing.T) { results, err := queryCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available (no GPU nodes?)", metricName) - idle := filterByHostType(results, "g4dn.xlarge") - require.NotEmpty(t, idle, "No %s results from idle g4dn.xlarge node", metricName) + idle := otelmetrics.SeriesOnNodes(results, nodes) + require.NotEmpty(t, idle, "No %s results from idle nodes %v", metricName, nodeNames(nodes)) for _, r := range idle { _, has := r.Labels.Resource["k8s.container.name"] require.False(t, has, @@ -289,9 +304,10 @@ func TestDCGMGpuburnWorkloadLabels(t *testing.T) { } // TestDCGMIdleNodeNoWorkloadLabels validates that DCGM results from the idle -// g4dn.xlarge node do NOT have k8s.workload.name. +// node do NOT have k8s.workload.name. func TestDCGMIdleNodeNoWorkloadLabels(t *testing.T) { t.Parallel() + nodes := idleNodes(t) for _, metricName := range dcgmMetricNamesList { metricName := metricName t.Run(metricName, func(t *testing.T) { @@ -299,8 +315,8 @@ func TestDCGMIdleNodeNoWorkloadLabels(t *testing.T) { results, err := queryCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available (no GPU nodes?)", metricName) - idle := filterByHostType(results, "g4dn.xlarge") - require.NotEmpty(t, idle, "No results from idle g4dn.xlarge node for %s", metricName) + idle := otelmetrics.SeriesOnNodes(results, nodes) + require.NotEmpty(t, idle, "No results from idle nodes %v for %s", nodeNames(nodes), metricName) for _, r := range idle { _, hasName := r.Labels.Resource["k8s.workload.name"] require.False(t, hasName, @@ -389,9 +405,10 @@ func TestDCGMHasRawPromotedKeys(t *testing.T) { } // TestDCGMIdleNodeNoPodColor validates that DCGM results from the idle -// g4dn.xlarge node do NOT have the pod color label. +// node do NOT have the pod color label. func TestDCGMIdleNodeNoPodColor(t *testing.T) { t.Parallel() + nodes := idleNodes(t) for _, metricName := range dcgmMetricNamesList { metricName := metricName t.Run(metricName, func(t *testing.T) { @@ -399,8 +416,8 @@ func TestDCGMIdleNodeNoPodColor(t *testing.T) { results, err := queryCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available (no GPU nodes?)", metricName) - idle := filterByHostType(results, "g4dn.xlarge") - require.NotEmpty(t, idle, "No results from idle g4dn.xlarge node for %s", metricName) + idle := otelmetrics.SeriesOnNodes(results, nodes) + require.NotEmpty(t, idle, "No results from idle nodes %v for %s", nodeNames(nodes), metricName) for _, r := range idle { _, has := r.Labels.Resource[podColorLabel] require.False(t, has, @@ -422,7 +439,7 @@ func TestDCGMNodeGroupCoverage(t *testing.T) { promql := fmt.Sprintf( `DCGM_FI_DEV_GPU_UTIL{"@resource.k8s.cluster.name"="%s","@resource.host.type"="%s"}`, escaped, ng.InstanceType) - results, err := client.Query(context.Background(), promql) + results, err := client.QueryComplete(context.Background(), promql) require.NoError(t, err, "querying DCGM_FI_DEV_GPU_UTIL on %s", ng.Description) require.True(t, len(results) > 0, "DCGM missing from %s nodes (%s) — dcgm-exporter not running?", diff --git a/test/otel/gpu/dedup_test.go b/test/otel/gpu/dedup_test.go index abc5d860e..7a796bf73 100644 --- a/test/otel/gpu/dedup_test.go +++ b/test/otel/gpu/dedup_test.go @@ -50,7 +50,7 @@ func TestDCGMNoDuplicateSeries(t *testing.T) { `DCGM_FI_DEV_GPU_UTIL{"@resource.k8s.cluster.name"="%s","@resource.k8s.node.name"="%s",UUID="%s"}`, escaped, pinNode, pinUUID, ) - pinned, err := client.Query(ctx, promql) + pinned, err := client.QueryComplete(ctx, promql) require.NoError(t, err, "querying pinned DCGM_FI_DEV_GPU_UTIL") require.NotEmpty(t, pinned, "pinned DCGM_FI_DEV_GPU_UTIL returned 0 results") require.Equal(t, 1, len(pinned), diff --git a/test/otel/gpu/helpers_test.go b/test/otel/gpu/helpers_test.go index 5d07e96ba..a26c77f2d 100644 --- a/test/otel/gpu/helpers_test.go +++ b/test/otel/gpu/helpers_test.go @@ -6,21 +6,115 @@ package gpu import ( + "context" "sort" + "sync" + "testing" + + "github.com/stretchr/testify/require" "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" ) -// filterByHostType returns only results whose resource `host.type` matches the -// given instance type. -func filterByHostType(results []otelmetrics.MetricResult, instanceType string) []otelmetrics.MetricResult { - var out []otelmetrics.MetricResult - for _, r := range results { - if r.Labels.Resource["host.type"] == instanceType { - out = append(out, r) +// gpuBurnPodPrefix names the pods of the multi-gpu-burn Deployment, which holds one GPU on the busy node. +const gpuBurnPodPrefix = "multi-gpu-burn" + +// gpuRoles reads the node roles from DCGM_FI_DEV_GPU_UTIL, so every test sees the same nodes. +func gpuRoles(t *testing.T) otelmetrics.GPURoles { + t.Helper() + results, err := queryCache.Get(context.Background(), "DCGM_FI_DEV_GPU_UTIL") + require.NoError(t, err, "querying DCGM_FI_DEV_GPU_UTIL") + require.NotEmpty(t, results, "DCGM_FI_DEV_GPU_UTIL not available (no GPU nodes?)") + return otelmetrics.ReadGPURoles(results, gpuBurnPodPrefix) +} + +// idleNodes returns the GPU nodes where no pod holds a GPU. +func idleNodes(t *testing.T) []otelmetrics.GPUNode { + t.Helper() + nodes := gpuRoles(t).Idle + require.NotEmpty(t, nodes, "No idle GPU node: every node of host types %v has DCGM series with k8s.pod.name", clusterHostTypes) + return nodes +} + +// busyNodes returns the GPU nodes where multi-gpu-burn holds a GPU. +func busyNodes(t *testing.T) []otelmetrics.GPUNode { + t.Helper() + nodes := gpuRoles(t).Busy + require.NotEmpty(t, nodes, "No GPU node of host types %v runs %s", clusterHostTypes, gpuBurnPodPrefix) + return nodes +} + +// multiGPUNodes returns the GPU nodes whose nvidia.com/gpu allocatable is more than 1. +// With none, it skips the test when OTEL_HOST_TYPES overrides the defaults and fails it otherwise. +func multiGPUNodes(t *testing.T) []otelmetrics.GPUNode { + t.Helper() + nodes, err := otelmetrics.MultiGPUNodes(gpuRoles(t).Nodes, gpuAllocatable(t)) + require.NoError(t, err) + requireMultiGPU(t, len(nodes) > 0, "No GPU node of host types %v has an nvidia.com/gpu allocatable above 1", clusterHostTypes) + return nodes +} + +// busyMultiGPUNodes returns the multi-GPU nodes where multi-gpu-burn holds a GPU, and skips or +// fails like multiGPUNodes when there are none. +func busyMultiGPUNodes(t *testing.T) []otelmetrics.GPUNode { + t.Helper() + var nodes []otelmetrics.GPUNode + for _, n := range multiGPUNodes(t) { + if n.Runs(gpuBurnPodPrefix) { + nodes = append(nodes, n) } } - return out + requireMultiGPU(t, len(nodes) > 0, "%s runs on no multi-GPU node of host types %v", gpuBurnPodPrefix, clusterHostTypes) + return nodes +} + +// requireMultiGPU skips the test when ok is false and OTEL_HOST_TYPES overrides the defaults, +// because such a cluster can have no multi-GPU node. Otherwise it requires ok. +func requireMultiGPU(t *testing.T, ok bool, format string, args ...interface{}) { + t.Helper() + if !ok && hostTypesOverridden { + t.Skipf(format+" (%s overrides the default host types %v)", append(args, otelmetrics.HostTypesEnvVar, defaultHostTypes)...) + } + require.Truef(t, ok, format, args...) +} + +// nodeNames returns the names of the nodes, for messages. +func nodeNames(nodes []otelmetrics.GPUNode) []string { + names := make([]string, len(nodes)) + for i, n := range nodes { + names[i] = n.Name + } + return names +} + +var ( + gpuAllocatableByNode map[string]int + gpuAllocatableOnce sync.Once + gpuAllocatableErr error +) + +// gpuAllocatable maps node names to their nvidia.com/gpu allocatable from kube_node_status_allocatable. +// It reads through allNodesCache, which pins no host types, because kube-state-metrics series are cluster-scoped. +func gpuAllocatable(t *testing.T) map[string]int { + t.Helper() + gpuAllocatableOnce.Do(func() { + var results []otelmetrics.MetricResult + results, gpuAllocatableErr = allNodesCache.GetWithFilter(context.Background(), "kube_node_status_allocatable", + map[string]string{"resource": otelmetrics.GPUResource}) + if gpuAllocatableErr == nil { + gpuAllocatableByNode, gpuAllocatableErr = otelmetrics.GPUAllocatable(results) + } + }) + require.NoError(t, gpuAllocatableErr, "reading kube_node_status_allocatable{resource=%q}", otelmetrics.GPUResource) + return gpuAllocatableByNode +} + +// allocatableGPUs returns the node's nvidia.com/gpu allocatable. +func allocatableGPUs(t *testing.T, node string) int { + t.Helper() + n, ok := gpuAllocatable(t)[node] + require.True(t, ok, "No kube_node_status_allocatable{resource=%q} series for node %s", otelmetrics.GPUResource, node) + return n } // uniqueDatapointValues returns the sorted unique non-empty values of a diff --git a/test/otel/gpu/metrics_test.go b/test/otel/gpu/metrics_test.go index 21ecd1d3c..79c00e1ef 100644 --- a/test/otel/gpu/metrics_test.go +++ b/test/otel/gpu/metrics_test.go @@ -5,10 +5,21 @@ package gpu -import "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +import ( + "reflect" -// Instance types in the GPU cluster. -var clusterHostTypes = []string{"g4dn.xlarge", "g4dn.12xlarge"} + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +// defaultHostTypes are the instance types of the CI GPU cluster. +var defaultHostTypes = []string{"g4dn.xlarge", "g4dn.12xlarge"} + +// Instance types in the GPU cluster; OTEL_HOST_TYPES overrides them. +var clusterHostTypes = otelmetrics.ResolveHostTypes(defaultHostTypes) + +// hostTypesOverridden is true when OTEL_HOST_TYPES replaces the defaults. The +// multi-GPU tests then skip on a cluster with no multi-GPU node. The CI cluster has one. +var hostTypesOverridden = !reflect.DeepEqual(clusterHostTypes, defaultHostTypes) // --- Metric definitions --- @@ -76,11 +87,9 @@ func metricNames(defs []otelmetrics.MetricDefinition) []string { // Custom pod label constant. const podColorLabel = "k8s.pod.label.ci-test.example.com/pod-color" -// gpuInstanceTypes lists instance types with NVIDIA GPUs in this cluster. -var gpuInstanceTypes = []struct { - InstanceType string - Description string -}{ - {"g4dn.xlarge", "GPU-single"}, - {"g4dn.12xlarge", "GPU-multi"}, -} +// gpuInstanceTypes lists instance types with NVIDIA GPUs in this cluster. The +// descriptions only name the CI node groups. The tests read node roles from the data. +var gpuInstanceTypes = otelmetrics.NodeGroupsFor(clusterHostTypes, map[string]string{ + "g4dn.xlarge": "GPU-single", + "g4dn.12xlarge": "GPU-multi", +}) diff --git a/test/otel/gpu/multi_gpu_test.go b/test/otel/gpu/multi_gpu_test.go index a806d5ff7..126186003 100644 --- a/test/otel/gpu/multi_gpu_test.go +++ b/test/otel/gpu/multi_gpu_test.go @@ -3,8 +3,9 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -// Multi-GPU tests validate that the g4dn.12xlarge node with 4 T4 GPUs -// produces distinct per-device metric series with correct device attributes. +// Multi-GPU tests validate that a node whose nvidia.com/gpu allocatable is more +// than 1 produces one series per device, for every device in the allocatable, +// with correct device attributes. package gpu @@ -15,165 +16,145 @@ import ( "testing" "github.com/stretchr/testify/require" -) -const ( - multiGpuInstanceType = "g4dn.12xlarge" - expectedMultiGPUCount = 4 + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" ) func TestMultiGPUDeviceCount(t *testing.T) { t.Parallel() - results, err := queryCache.Get(context.Background(), "DCGM_FI_DEV_GPU_UTIL") - require.NoError(t, err, "querying DCGM_FI_DEV_GPU_UTIL") - require.NotEmpty(t, results, "DCGM_FI_DEV_GPU_UTIL not available") - - multiGPU := filterByHostType(results, multiGpuInstanceType) - require.True(t, len(multiGPU) > 0, - "No DCGM_FI_DEV_GPU_UTIL results from %s node", multiGpuInstanceType) - - gpus := uniqueDatapointValues(multiGPU, "gpu") - require.Equal(t, expectedMultiGPUCount, len(gpus), "Expected %d distinct gpu on %s, got %d: %v", - expectedMultiGPUCount, multiGpuInstanceType, len(gpus), gpus) + for _, node := range multiGPUNodes(t) { + want := allocatableGPUs(t, node.Name) + gpus := node.GPUs() + require.Equal(t, want, len(gpus), "Expected %d distinct gpu on %s (nvidia.com/gpu allocatable), got %d: %v", + want, node.Name, len(gpus), gpus) + } } func TestMultiGPUUniqueUUIDs(t *testing.T) { t.Parallel() - results, err := queryCache.Get(context.Background(), "DCGM_FI_DEV_GPU_UTIL") - require.NoError(t, err, "querying DCGM_FI_DEV_GPU_UTIL") - require.NotEmpty(t, results, "DCGM_FI_DEV_GPU_UTIL not available") - - multiGPU := filterByHostType(results, multiGpuInstanceType) - require.True(t, len(multiGPU) > 0, "No results from %s node", multiGpuInstanceType) - - uuids := uniqueDatapointValues(multiGPU, "UUID") - require.Equal(t, expectedMultiGPUCount, len(uuids), "Expected %d unique UUIDs, got %d", expectedMultiGPUCount, len(uuids)) - for _, uuid := range uuids { - require.True(t, strings.HasPrefix(uuid, "GPU-"), - "UUID should start with 'GPU-', got '%s'", uuid) + for _, node := range multiGPUNodes(t) { + want := allocatableGPUs(t, node.Name) + uuids := uniqueDatapointValues(node.Series, "UUID") + require.Equal(t, want, len(uuids), "Expected %d unique UUIDs on %s, got %d", want, node.Name, len(uuids)) + for _, uuid := range uuids { + require.True(t, strings.HasPrefix(uuid, "GPU-"), + "UUID should start with 'GPU-', got '%s'", uuid) + } } } func TestMultiGPUConsecutiveIndices(t *testing.T) { t.Parallel() - results, err := queryCache.Get(context.Background(), "DCGM_FI_DEV_GPU_UTIL") - require.NoError(t, err, "querying DCGM_FI_DEV_GPU_UTIL") - require.NotEmpty(t, results, "DCGM_FI_DEV_GPU_UTIL not available") - - multiGPU := filterByHostType(results, multiGpuInstanceType) - require.True(t, len(multiGPU) > 0, "No results from %s node", multiGpuInstanceType) - - gpus := uniqueDatapointValues(multiGPU, "gpu") - require.Equal(t, expectedMultiGPUCount, len(gpus), "Expected %d GPU indices, got %d: %v", expectedMultiGPUCount, len(gpus), gpus) - for i := 0; i < expectedMultiGPUCount; i++ { - expected := strconv.Itoa(i) - found := false - for _, g := range gpus { - if g == expected { - found = true - break + for _, node := range multiGPUNodes(t) { + want := allocatableGPUs(t, node.Name) + gpus := node.GPUs() + require.Equal(t, want, len(gpus), "Expected %d GPU indices on %s, got %d: %v", want, node.Name, len(gpus), gpus) + for i := 0; i < want; i++ { + expected := strconv.Itoa(i) + found := false + for _, g := range gpus { + if g == expected { + found = true + break + } } + require.True(t, found, "Expected GPU index %d on %s, got indices: %v", i, node.Name, gpus) } - require.True(t, found, "Expected GPU index %d, got indices: %v", i, gpus) } } func TestMultiGPUAllMetricsPerDevice(t *testing.T) { t.Parallel() + nodes := multiGPUNodes(t) + want := make(map[string]int, len(nodes)) + for _, node := range nodes { + want[node.Name] = allocatableGPUs(t, node.Name) + } for _, metricName := range dcgmMetricNamesList { metricName := metricName t.Run(metricName, func(t *testing.T) { t.Parallel() results, err := queryCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) - multiGPU := filterByHostType(results, multiGpuInstanceType) - require.True(t, len(multiGPU) > 0, "No %s results from %s node", metricName, multiGpuInstanceType) + for _, node := range nodes { + multiGPU := otelmetrics.SeriesOnNodes(results, []otelmetrics.GPUNode{node}) + require.True(t, len(multiGPU) > 0, "No %s results from %s", metricName, node.Name) - gpus := uniqueDatapointValues(multiGPU, "gpu") - require.Equal(t, expectedMultiGPUCount, len(gpus), "%s: expected %d GPUs, got %d: %v", - metricName, expectedMultiGPUCount, len(gpus), gpus) + gpus := uniqueDatapointValues(multiGPU, "gpu") + require.Equal(t, want[node.Name], len(gpus), "%s on %s: expected %d GPUs, got %d: %v", + metricName, node.Name, want[node.Name], len(gpus), gpus) + } }) } } func TestMultiGPUDeviceModelConsistency(t *testing.T) { t.Parallel() - results, err := queryCache.Get(context.Background(), "DCGM_FI_DEV_GPU_UTIL") - require.NoError(t, err, "querying DCGM_FI_DEV_GPU_UTIL") - multiGPU := filterByHostType(results, multiGpuInstanceType) - require.True(t, len(multiGPU) > 0, "No results from %s node", multiGpuInstanceType) - - models := uniqueDatapointValues(multiGPU, "modelName") - require.Equal(t, 1, len(models), "Expected 1 GPU model on %s, got %d: %v", multiGpuInstanceType, len(models), models) + for _, node := range multiGPUNodes(t) { + models := uniqueDatapointValues(node.Series, "modelName") + require.Equal(t, 1, len(models), "Expected 1 GPU model on %s, got %d: %v", node.Name, len(models), models) + } } func TestMultiGPUDriverVersionConsistency(t *testing.T) { t.Parallel() - results, err := queryCache.Get(context.Background(), "DCGM_FI_DEV_GPU_UTIL") - require.NoError(t, err, "querying DCGM_FI_DEV_GPU_UTIL") - multiGPU := filterByHostType(results, multiGpuInstanceType) - require.True(t, len(multiGPU) > 0, "No results from %s node", multiGpuInstanceType) - - versions := uniqueDatapointValues(multiGPU, "DCGM_FI_DRIVER_VERSION") - require.Equal(t, 1, len(versions), "Expected 1 driver version on %s, got %d: %v", multiGpuInstanceType, len(versions), versions) + for _, node := range multiGPUNodes(t) { + versions := uniqueDatapointValues(node.Series, "DCGM_FI_DRIVER_VERSION") + require.Equal(t, 1, len(versions), "Expected 1 driver version on %s, got %d: %v", node.Name, len(versions), versions) + } } // TestMultiGPUCorrelatedCount validates that exactly 1 GPU on the multi-GPU -// node is correlated to the multi-gpu-burn pod, and the rest are uncorrelated. +// node that runs multi-gpu-burn is correlated to the pod, which requests one +// GPU, and the rest are uncorrelated. func TestMultiGPUCorrelatedCount(t *testing.T) { t.Parallel() - results, err := queryCache.Get(context.Background(), "DCGM_FI_DEV_GPU_UTIL") - require.NoError(t, err, "querying DCGM_FI_DEV_GPU_UTIL") - multi := filterByHostType(results, multiGpuInstanceType) - require.True(t, len(multi) > 0, "No results from %s node", multiGpuInstanceType) - - var correlated, uncorrelated int - for _, r := range multi { - if strings.HasPrefix(r.Labels.Resource["k8s.pod.name"], "multi-gpu-burn") { - correlated++ - } else if _, hasPod := r.Labels.Resource["k8s.pod.name"]; !hasPod { - uncorrelated++ + for _, node := range busyMultiGPUNodes(t) { + var correlated, uncorrelated int + for _, r := range node.Series { + if strings.HasPrefix(r.Labels.Resource["k8s.pod.name"], gpuBurnPodPrefix) { + correlated++ + } else if _, hasPod := r.Labels.Resource["k8s.pod.name"]; !hasPod { + uncorrelated++ + } } + want := allocatableGPUs(t, node.Name) - 1 + require.Equal(t, 1, correlated, "Expected 1 GPU on %s correlated to multi-gpu-burn, got %d", node.Name, correlated) + require.Equal(t, want, uncorrelated, "Expected %d uncorrelated GPUs on %s, got %d", want, node.Name, uncorrelated) } - require.Equal(t, 1, correlated, "Expected 1 GPU correlated to multi-gpu-burn, got %d", correlated) - require.Equal(t, expectedMultiGPUCount-1, uncorrelated, "Expected %d uncorrelated GPUs, got %d", expectedMultiGPUCount-1, uncorrelated) } // TestMultiGPUCorrelatedPodLabels validates that the correlated GPU result // has the expected pod labels. func TestMultiGPUCorrelatedPodLabels(t *testing.T) { t.Parallel() - results, err := queryCache.Get(context.Background(), "DCGM_FI_DEV_GPU_UTIL") - require.NoError(t, err, "querying DCGM_FI_DEV_GPU_UTIL") - multi := filterByHostType(results, multiGpuInstanceType) - - for _, r := range multi { - if strings.HasPrefix(r.Labels.Resource["k8s.pod.name"], "multi-gpu-burn") { - require.Equal(t, "gpu-burn", r.Labels.Resource["k8s.container.name"], "multi-gpu-burn container name") - require.Equal(t, "default", r.Labels.Resource["k8s.namespace.name"], "multi-gpu-burn namespace") - require.Equal(t, "multi-gpu-burn", r.Labels.Resource["k8s.workload.name"], "multi-gpu-burn workload name") - require.Equal(t, "Deployment", r.Labels.Resource["k8s.workload.type"], "multi-gpu-burn workload type") - return + for _, node := range busyMultiGPUNodes(t) { + for _, r := range node.Series { + if strings.HasPrefix(r.Labels.Resource["k8s.pod.name"], gpuBurnPodPrefix) { + require.Equal(t, "gpu-burn", r.Labels.Resource["k8s.container.name"], "multi-gpu-burn container name") + require.Equal(t, "default", r.Labels.Resource["k8s.namespace.name"], "multi-gpu-burn namespace") + require.Equal(t, "multi-gpu-burn", r.Labels.Resource["k8s.workload.name"], "multi-gpu-burn workload name") + require.Equal(t, "Deployment", r.Labels.Resource["k8s.workload.type"], "multi-gpu-burn workload type") + return + } } } t.Fatal("No DCGM_FI_DEV_GPU_UTIL result correlated to multi-gpu-burn pod") } -// TestMultiGPUHostType validates the g4dn.12xlarge node reports host.type=g4dn.12xlarge -// and a valid EC2 instance ID. +// TestMultiGPUHostType validates that every series of the multi-GPU node reports +// one host.type of the configured host types and a valid EC2 instance ID. func TestMultiGPUHostType(t *testing.T) { t.Parallel() - results, err := queryCache.Get(context.Background(), "DCGM_FI_DEV_GPU_UTIL") - require.NoError(t, err, "querying DCGM_FI_DEV_GPU_UTIL") - require.NotEmpty(t, results, "DCGM_FI_DEV_GPU_UTIL not available") - - multi := filterByHostType(results, multiGpuInstanceType) - require.NotEmpty(t, multi, "No DCGM results from %s node", multiGpuInstanceType) - for _, r := range multi { - require.Equal(t, multiGpuInstanceType, r.Labels.Resource["host.type"], - "multi-GPU node host.type") - hostID := r.Labels.Resource["host.id"] - require.True(t, strings.HasPrefix(hostID, "i-"), - "multi-GPU node host.id should start with 'i-', got %q", hostID) + for _, node := range multiGPUNodes(t) { + hostType := node.Series[0].Labels.Resource["host.type"] + require.Contains(t, clusterHostTypes, hostType, "multi-GPU node %s host.type", node.Name) + for _, r := range node.Series { + require.Equal(t, hostType, r.Labels.Resource["host.type"], + "multi-GPU node %s host.type", node.Name) + hostID := r.Labels.Resource["host.id"] + require.True(t, strings.HasPrefix(hostID, "i-"), + "multi-GPU node %s host.id should start with 'i-', got %q", node.Name, hostID) + } } } diff --git a/test/otel/gpu/node_color_test.go b/test/otel/gpu/node_color_test.go index c9d1be06e..6d2826ee2 100644 --- a/test/otel/gpu/node_color_test.go +++ b/test/otel/gpu/node_color_test.go @@ -4,7 +4,7 @@ // SPDX-License-Identifier: MIT // Node-color tests validate that the custom node-color label applied by the -// GPU cluster's terraform (green on g4dn.xlarge + g4dn.12xlarge) propagates +// GPU cluster's terraform (green on every GPU node group) propagates // to DaemonSet-enriched metrics. // // Ports the monorepo TestCustomNodeLabels/has_green subtest. @@ -35,7 +35,7 @@ func TestDCGMHasGreenNodeColor(t *testing.T) { promql := fmt.Sprintf( `%s{"@resource.k8s.cluster.name"="%s","@resource.%s"="green"}`, metricName, escaped, nodeColorLabel) - results, err := client.Query(context.Background(), promql) + results, err := client.QueryComplete(context.Background(), promql) require.NoError(t, err, "querying %s with node-color=green filter", metricName) require.True(t, len(results) > 0, "%s: no results from GPU nodes (expected node-color=green)", metricName) diff --git a/test/otel/gpu/resolution_test.go b/test/otel/gpu/resolution_test.go index 4ae1eaca6..22951b169 100644 --- a/test/otel/gpu/resolution_test.go +++ b/test/otel/gpu/resolution_test.go @@ -22,9 +22,10 @@ import ( const expectedDCGMResolution = 30 * time.Second // TestDCGMResolution validates DCGM_FI_DEV_GPU_UTIL is scraped at ~30s intervals -// by querying a 5-minute range and checking sample counts on the multi-GPU node. +// by querying a 5-minute range and checking sample counts on the busy node. func TestDCGMResolution(t *testing.T) { t.Parallel() + busy := busyNodes(t)[0].Name end := time.Now() start := end.Add(-5 * time.Minute) step := expectedDCGMResolution @@ -34,12 +35,12 @@ func TestDCGMResolution(t *testing.T) { escaped := otelmetrics.EscapePromQLValue(cfg.ClusterName) promql := fmt.Sprintf( - `DCGM_FI_DEV_GPU_UTIL{"@resource.k8s.cluster.name"="%s","@resource.host.type"="%s"}`, - escaped, multiGpuInstanceType) + `DCGM_FI_DEV_GPU_UTIL{"@resource.k8s.cluster.name"="%s","@resource.k8s.node.name"="%s"}`, + escaped, otelmetrics.EscapePromQLValue(busy)) results, err := client.QueryRange(context.Background(), promql, start, end, step) require.NoError(t, err, "range querying DCGM_FI_DEV_GPU_UTIL") - require.True(t, len(results) > 0, "No DCGM_FI_DEV_GPU_UTIL range results from %s", multiGpuInstanceType) + require.True(t, len(results) > 0, "No DCGM_FI_DEV_GPU_UTIL range results from %s", busy) // Pick the series with the most samples (best coverage). var bestSeries *otelmetrics.RangeResult diff --git a/test/otel/gpu/setup_test.go b/test/otel/gpu/setup_test.go index 282db1936..1cbf5cfc2 100644 --- a/test/otel/gpu/setup_test.go +++ b/test/otel/gpu/setup_test.go @@ -24,6 +24,8 @@ var ( cfg otelmetrics.TestConfig client *otelmetrics.OtelMetricsClient queryCache *otelmetrics.QueryCache + // allNodesCache reads every series on every node, for the attribute schema. + allNodesCache *otelmetrics.QueryCache ) func TestMain(m *testing.M) { @@ -78,6 +80,8 @@ func TestMain(m *testing.M) { os.Exit(1) } + fmt.Fprintf(os.Stderr, "Host types: %v (set %s to override)\n", clusterHostTypes, otelmetrics.HostTypesEnvVar) + hostMappings := []otelmetrics.SourceHostMapping{ {Source: otelmetrics.SourceNodeExporter, HostTypes: clusterHostTypes}, {Source: otelmetrics.SourceCadvisor, HostTypes: clusterHostTypes}, @@ -85,17 +89,23 @@ func TestMain(m *testing.M) { {Source: otelmetrics.SourceDCGM, HostTypes: clusterHostTypes}, } - registry := otelmetrics.NewSourceRegistry(clusterHostTypes, hostMappings, - otelmetrics.SourceMapping{Source: otelmetrics.SourceNodeExporter, Metrics: nodeExporterMetrics}, - otelmetrics.SourceMapping{Source: otelmetrics.SourceCadvisor, Metrics: cadvisorMetrics}, - otelmetrics.SourceMapping{Source: otelmetrics.SourceKubeletstats, Metrics: kubeletstatsMetrics}, - otelmetrics.SourceMapping{Source: otelmetrics.SourceDCGM, Metrics: dcgmMetrics}, - ) + sourceMappings := []otelmetrics.SourceMapping{ + {Source: otelmetrics.SourceNodeExporter, Metrics: nodeExporterMetrics}, + {Source: otelmetrics.SourceCadvisor, Metrics: cadvisorMetrics}, + {Source: otelmetrics.SourceKubeletstats, Metrics: kubeletstatsMetrics}, + {Source: otelmetrics.SourceDCGM, Metrics: dcgmMetrics}, + } + registry := otelmetrics.NewSourceRegistry(clusterHostTypes, hostMappings, sourceMappings...) queryCache = otelmetrics.NewQueryCache(client, cfg.ClusterName, otelmetrics.WithHostTypes(clusterHostTypes), otelmetrics.WithSourceRegistry(registry), ) + // Same routing with no host types, sharing queryCache's concurrency limit. + allNodesCache = otelmetrics.NewQueryCache(client, cfg.ClusterName, + otelmetrics.WithSourceRegistry(otelmetrics.NewSourceRegistry(nil, nil, sourceMappings...)), + otelmetrics.WithSharedConcurrency(queryCache), + ) os.Exit(m.Run()) } diff --git a/test/otel/lis_csi/setup_test.go b/test/otel/lis_csi/setup_test.go index 3493936cb..3fca3f1a5 100644 --- a/test/otel/lis_csi/setup_test.go +++ b/test/otel/lis_csi/setup_test.go @@ -28,7 +28,8 @@ var ( const scopePrometheus = "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver" -var clusterHostTypes = []string{"i7i.xlarge"} +// Instance types in the LIS CSI cluster; OTEL_HOST_TYPES overrides them. +var clusterHostTypes = otelmetrics.ResolveHostTypes([]string{"i7i.xlarge"}) func TestMain(m *testing.M) { environment.RegisterEnvironmentMetaDataFlags() @@ -83,6 +84,8 @@ func TestMain(m *testing.M) { os.Exit(1) } + fmt.Fprintf(os.Stderr, "Host types: %v (set %s to override)\n", clusterHostTypes, otelmetrics.HostTypesEnvVar) + hostMappings := []otelmetrics.SourceHostMapping{ {Source: otelmetrics.SourceNodeExporter, HostTypes: clusterHostTypes}, {Source: otelmetrics.SourceCadvisor, HostTypes: clusterHostTypes}, diff --git a/test/otel/multi_efa/setup_test.go b/test/otel/multi_efa/setup_test.go index 5ee83cbf0..78e29d65c 100644 --- a/test/otel/multi_efa/setup_test.go +++ b/test/otel/multi_efa/setup_test.go @@ -26,8 +26,8 @@ var ( queryCache *otelmetrics.QueryCache ) -// Instance types in this cluster. -var clusterHostTypes = []string{"c6in.32xlarge"} +// Instance types in this cluster; OTEL_HOST_TYPES overrides them. +var clusterHostTypes = otelmetrics.ResolveHostTypes([]string{"c6in.32xlarge"}) func TestMain(m *testing.M) { environment.RegisterEnvironmentMetaDataFlags() @@ -81,6 +81,8 @@ func TestMain(m *testing.M) { os.Exit(1) } + fmt.Fprintf(os.Stderr, "Host types: %v (set %s to override)\n", clusterHostTypes, otelmetrics.HostTypesEnvVar) + hostMappings := []otelmetrics.SourceHostMapping{ {Source: otelmetrics.SourceEFA, HostTypes: clusterHostTypes}, } diff --git a/test/otel/neuron/attribute_schema_test.go b/test/otel/neuron/attribute_schema_test.go new file mode 100644 index 000000000..58cc3fe07 --- /dev/null +++ b/test/otel/neuron/attribute_schema_test.go @@ -0,0 +1,57 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package neuron + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +// --------------------------------------------------------------------------- +// TestAttributeSchema — the closed set of resource and instrumentation-scope +// attributes on Neuron series. Every series must have every required key and +// nothing undeclared, expected keys must be on at least one series, and keys set +// together must arrive together. Node and pod labels are open by construction +// (k8sattributes extracts "(.*)"). Datapoint scope is not covered. +// --------------------------------------------------------------------------- + +// awsdevicepodcorrelation then groupbyattrs; only on cores or devices a pod holds. +var neuronPodKeys = []string{"k8s.pod.name", "k8s.namespace.name", "k8s.container.name"} + +var neuronResourceSchema = otelmetrics.KeySchema{ + Required: otelmetrics.ConcatKeys(otelmetrics.PrometheusReceiverKeys, otelmetrics.CloudKeys, otelmetrics.HostKeys, []string{ + "k8s.cluster.name", // set_cluster_name; protected + "k8s.node.name", // set_node_name, promote_node_name; protected + "k8s.node.uid", // k8sattributes/cw_k8s_ci_v0_node + }), + // The neuron-burn Deployments hold cores and devices and run a Neuron runtime. + Expected: otelmetrics.ConcatKeys(neuronPodKeys, []string{ + "aws.neuron.runtime.tag", // neuron_promote; zero-filled series of an idle device have no runtime_tag + "k8s.deployment.name", "k8s.replicaset.name", + }, otelmetrics.WorkloadKeys), + Optional: otelmetrics.ConcatKeys([]string{"k8s.pod.uid"}, // k8sattributes/cw_k8s_ci_v0_pod + otelmetrics.Without(otelmetrics.OwnerKindKeys, "k8s.deployment.name", "k8s.replicaset.name")), + OpenPrefixes: []string{"k8s.node.label.", "k8s.pod.label."}, + Implies: otelmetrics.MergeImplies(otelmetrics.WorkloadImplies(), otelmetrics.Together(neuronPodKeys...)), +} + +func TestAttributeSchema(t *testing.T) { + t.Parallel() + tally, err := otelmetrics.TallySchema(context.Background(), allNodesCache, neuronMetricNamesList, otelmetrics.GroupSchema{ + Resource: neuronResourceSchema, + Instrumentation: otelmetrics.KeySchema{Required: otelmetrics.ScopeKeys}, // set_scope_neuron_monitor + }, nil) + require.NoError(t, err, "no Neuron nodes?") + t.Logf("%d series across %d metrics", tally.Series, len(neuronMetricNamesList)) + require.NotZero(t, tally.Series, "no series checked") + for _, p := range tally.Problems() { + t.Errorf("%s: change the schema if intended", p) + } +} diff --git a/test/otel/neuron/multi_device_test.go b/test/otel/neuron/multi_device_test.go index 9b2e96dc2..c82698b32 100644 --- a/test/otel/neuron/multi_device_test.go +++ b/test/otel/neuron/multi_device_test.go @@ -273,7 +273,7 @@ func TestMultiNeuronCorrelatedCoreCount(t *testing.T) { promql := fmt.Sprintf( `neuroncore_utilization_ratio{"@resource.k8s.cluster.name"="%s","@resource.k8s.pod.name"=~"neuron-burn-multi.*"}`, escaped) - burnResults, err := client.Query(ctx, promql) + burnResults, err := client.QueryComplete(ctx, promql) require.NoError(t, err, "querying neuroncore_utilization_ratio for neuron-burn-multi") require.Equal(t, 2, len(burnResults), fmt.Sprintf("Expected 2 cores correlated to neuron-burn-multi (1 whole device × 2 cores), got %d", len(burnResults))) @@ -289,7 +289,7 @@ func TestMultiNeuronBurnDevicePodLabels(t *testing.T) { promql := fmt.Sprintf( `neuroncore_utilization_ratio{"@resource.k8s.cluster.name"="%s","@resource.k8s.pod.name"=~"neuron-burn-multi.*"}`, escaped) - burnResults, err := client.Query(ctx, promql) + burnResults, err := client.QueryComplete(ctx, promql) require.NoError(t, err, "querying neuroncore_utilization_ratio for neuron-burn-multi") require.Equal(t, 2, len(burnResults), fmt.Sprintf("Expected 2 cores correlated to neuron-burn-multi, got %d", len(burnResults))) diff --git a/test/otel/neuron/neuron_test.go b/test/otel/neuron/neuron_test.go index abe2e578e..7582f9bbf 100644 --- a/test/otel/neuron/neuron_test.go +++ b/test/otel/neuron/neuron_test.go @@ -19,13 +19,10 @@ import ( var neuronMetricNamesList = metricNames(neuronMetrics) // neuronInstanceTypes lists instance types with Neuron devices in this cluster. -var neuronInstanceTypes = []struct { - InstanceType string - Description string -}{ - {"inf2.xlarge", "Neuron"}, - {"inf2.24xlarge", "Neuron multi-device"}, -} +var neuronInstanceTypes = otelmetrics.NodeGroupsFor(clusterHostTypes, map[string]string{ + "inf2.xlarge": "Neuron", + "inf2.24xlarge": "Neuron multi-device", +}) // TestNeuronInstrumentationSource validates that all Neuron metrics // have @instrumentation.@name == "awsneuron". @@ -528,7 +525,7 @@ func TestNeuronNodeGroupCoverage(t *testing.T) { promql := fmt.Sprintf( `neuroncore_utilization_ratio{"@resource.k8s.cluster.name"="%s","@resource.host.type"="%s"}`, escaped, ng.InstanceType) - results, err := client.Query(context.Background(), promql) + results, err := client.QueryComplete(context.Background(), promql) require.NoError(t, err, "querying neuroncore_utilization_ratio on %s", ng.Description) require.True(t, len(results) > 0, "Neuron metrics missing from %s nodes (%s) — neuron-monitor not running?", diff --git a/test/otel/neuron/node_color_test.go b/test/otel/neuron/node_color_test.go index e033946e4..7d24db650 100644 --- a/test/otel/neuron/node_color_test.go +++ b/test/otel/neuron/node_color_test.go @@ -33,7 +33,7 @@ func TestNeuronHasRedNodeColor(t *testing.T) { promql := fmt.Sprintf( `%s{"@resource.k8s.cluster.name"="%s","@resource.%s"="red"}`, metricName, escaped, nodeColorLabel) - results, err := client.Query(context.Background(), promql) + results, err := client.QueryComplete(context.Background(), promql) require.NoError(t, err, "querying %s with node-color=red filter", metricName) require.True(t, len(results) > 0, "%s: no results from inf2.xlarge nodes (expected node-color=red)", metricName) @@ -55,7 +55,7 @@ func TestNeuronHasPurpleNodeColor(t *testing.T) { promql := fmt.Sprintf( `%s{"@resource.k8s.cluster.name"="%s","@resource.%s"="purple"}`, md.Name, escaped, nodeColorLabel) - results, err := client.Query(context.Background(), promql) + results, err := client.QueryComplete(context.Background(), promql) require.NoError(t, err, "querying %s with node-color=purple filter", md.Name) require.True(t, len(results) > 0, "%s: no results from inf2.24xlarge node (expected node-color=purple)", md.Name) diff --git a/test/otel/neuron/setup_test.go b/test/otel/neuron/setup_test.go index 00d074e82..3fd024522 100644 --- a/test/otel/neuron/setup_test.go +++ b/test/otel/neuron/setup_test.go @@ -24,10 +24,12 @@ var ( cfg otelmetrics.TestConfig client *otelmetrics.OtelMetricsClient queryCache *otelmetrics.QueryCache + // allNodesCache reads every series on every node, for the attribute schema. + allNodesCache *otelmetrics.QueryCache ) -// clusterHostTypes lists the accelerator instance types in this cluster. -var clusterHostTypes = []string{"inf2.xlarge", "inf2.24xlarge"} +// clusterHostTypes lists the accelerator instance types in this cluster; OTEL_HOST_TYPES overrides them. +var clusterHostTypes = otelmetrics.ResolveHostTypes([]string{"inf2.xlarge", "inf2.24xlarge"}) func TestMain(m *testing.M) { environment.RegisterEnvironmentMetaDataFlags() @@ -81,6 +83,8 @@ func TestMain(m *testing.M) { os.Exit(1) } + fmt.Fprintf(os.Stderr, "Host types: %v (set %s to override)\n", clusterHostTypes, otelmetrics.HostTypesEnvVar) + hostMappings := []otelmetrics.SourceHostMapping{ {Source: otelmetrics.SourceNodeExporter, HostTypes: clusterHostTypes}, {Source: otelmetrics.SourceCadvisor, HostTypes: clusterHostTypes}, @@ -88,17 +92,23 @@ func TestMain(m *testing.M) { {Source: otelmetrics.SourceNeuron, HostTypes: clusterHostTypes}, } - registry := otelmetrics.NewSourceRegistry(clusterHostTypes, hostMappings, - otelmetrics.SourceMapping{Source: otelmetrics.SourceNodeExporter, Metrics: nodeExporterMetrics}, - otelmetrics.SourceMapping{Source: otelmetrics.SourceCadvisor, Metrics: cadvisorMetrics}, - otelmetrics.SourceMapping{Source: otelmetrics.SourceKubeletstats, Metrics: kubeletstatsMetrics}, - otelmetrics.SourceMapping{Source: otelmetrics.SourceNeuron, Metrics: neuronMetrics}, - ) + sourceMappings := []otelmetrics.SourceMapping{ + {Source: otelmetrics.SourceNodeExporter, Metrics: nodeExporterMetrics}, + {Source: otelmetrics.SourceCadvisor, Metrics: cadvisorMetrics}, + {Source: otelmetrics.SourceKubeletstats, Metrics: kubeletstatsMetrics}, + {Source: otelmetrics.SourceNeuron, Metrics: neuronMetrics}, + } + registry := otelmetrics.NewSourceRegistry(clusterHostTypes, hostMappings, sourceMappings...) queryCache = otelmetrics.NewQueryCache(client, cfg.ClusterName, otelmetrics.WithHostTypes(clusterHostTypes), otelmetrics.WithSourceRegistry(registry), ) + // Same routing with no host types, sharing queryCache's concurrency limit. + allNodesCache = otelmetrics.NewQueryCache(client, cfg.ClusterName, + otelmetrics.WithSourceRegistry(otelmetrics.NewSourceRegistry(nil, nil, sourceMappings...)), + otelmetrics.WithSharedConcurrency(queryCache), + ) os.Exit(m.Run()) } diff --git a/test/otel/performance/performance_test.go b/test/otel/performance/performance_test.go index 3c1fa7be8..bd8798a3c 100644 --- a/test/otel/performance/performance_test.go +++ b/test/otel/performance/performance_test.go @@ -62,7 +62,7 @@ func getNodeAllocatable(t *testing.T, metricName string, resource string) (float // and skew the percent-of-node denominator the thresholds are calibrated to. query := fmt.Sprintf(`%s{"@resource.k8s.cluster.name"="%s", resource="%s"}`, metricName, otelmetrics.EscapePromQLValue(cfg.ClusterName), resource) - results, err := client.Query(context.Background(), query) + results, err := client.QueryComplete(context.Background(), query) require.NoError(t, err, "failed to query node allocatable for resource=%s", resource) require.NotEmpty(t, results, "no kube_node_status_allocatable data for resource=%s — are KSM metrics being collected?", resource) var sum float64 diff --git a/test/otel/solutions/karpenter/setup_test.go b/test/otel/solutions/karpenter/setup_test.go index c3f437853..f15a1bcc3 100644 --- a/test/otel/solutions/karpenter/setup_test.go +++ b/test/otel/solutions/karpenter/setup_test.go @@ -29,8 +29,8 @@ var ( // Instrumentation scope name constant. const scopeKarpenter = "github.com/aws/karpenter" -// Instance types in the Karpenter cluster. -var clusterHostTypes = []string{"t3.medium"} +// Instance types in the Karpenter cluster; OTEL_HOST_TYPES overrides them. +var clusterHostTypes = otelmetrics.ResolveHostTypes([]string{"t3.medium"}) func TestMain(m *testing.M) { environment.RegisterEnvironmentMetaDataFlags() @@ -84,6 +84,8 @@ func TestMain(m *testing.M) { os.Exit(1) } + fmt.Fprintf(os.Stderr, "Host types: %v (set %s to override)\n", clusterHostTypes, otelmetrics.HostTypesEnvVar) + hostMappings := []otelmetrics.SourceHostMapping{ {Source: otelmetrics.SourceKarpenter, HostTypes: nil}, } diff --git a/test/otel/solutions/keda/setup_test.go b/test/otel/solutions/keda/setup_test.go index e9337c6d6..20a47e0e6 100644 --- a/test/otel/solutions/keda/setup_test.go +++ b/test/otel/solutions/keda/setup_test.go @@ -29,8 +29,8 @@ var ( // Instrumentation scope name constant. const scopeKEDA = "github.com/kedacore/keda" -// Instance types in the cluster. -var clusterHostTypes = []string{"t3.medium"} +// Instance types in the cluster; OTEL_HOST_TYPES overrides them. +var clusterHostTypes = otelmetrics.ResolveHostTypes([]string{"t3.medium"}) func TestMain(m *testing.M) { environment.RegisterEnvironmentMetaDataFlags() @@ -84,6 +84,8 @@ func TestMain(m *testing.M) { os.Exit(1) } + fmt.Fprintf(os.Stderr, "Host types: %v (set %s to override)\n", clusterHostTypes, otelmetrics.HostTypesEnvVar) + hostMappings := []otelmetrics.SourceHostMapping{ {Source: otelmetrics.SourceKEDA, HostTypes: nil}, } diff --git a/test/otel/standard/cadvisor_test.go b/test/otel/standard/cadvisor_test.go index 8e1a8ccdb..9c67b7b2f 100644 --- a/test/otel/standard/cadvisor_test.go +++ b/test/otel/standard/cadvisor_test.go @@ -21,7 +21,7 @@ var cadvisorMetricNamesList = metricNames(cadvisorMetrics) func TestCadvisorInstrumentationSource(t *testing.T) { for _, name := range cadvisorMetricNamesList { t.Run(name, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), name) + results, err := allNodesCache.Get(context.Background(), name) require.NoError(t, err, "querying %s", name) require.NotEmpty(t, results, "%s not available", name) for _, r := range results { @@ -36,7 +36,7 @@ func TestCadvisorInstrumentationSource(t *testing.T) { func TestCadvisorInstrumentationConsistent(t *testing.T) { for _, name := range cadvisorMetricNamesList { t.Run(name, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), name) + results, err := allNodesCache.Get(context.Background(), name) require.NoError(t, err, "querying %s", name) require.NotEmpty(t, results, "%s not available", name) names := make(map[string]struct{}) @@ -53,7 +53,7 @@ func TestCadvisorInstrumentationConsistent(t *testing.T) { func TestCadvisorPodName(t *testing.T) { for _, name := range cadvisorMetricNamesList { t.Run(name, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), name) + results, err := allNodesCache.Get(context.Background(), name) require.NoError(t, err, "querying %s", name) require.NotEmpty(t, results, "%s not available", name) for _, r := range results { @@ -67,7 +67,7 @@ func TestCadvisorPodName(t *testing.T) { func TestCadvisorNamespace(t *testing.T) { for _, name := range cadvisorMetricNamesList { t.Run(name, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), name) + results, err := allNodesCache.Get(context.Background(), name) require.NoError(t, err, "querying %s", name) require.NotEmpty(t, results, "%s not available", name) for _, r := range results { @@ -84,7 +84,7 @@ func TestCadvisorContainerName(t *testing.T) { continue } t.Run(md.Name, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), md.Name) + results, err := allNodesCache.Get(context.Background(), md.Name) require.NoError(t, err, "querying %s", md.Name) require.NotEmpty(t, results, "%s not available", md.Name) hasContainer := 0 @@ -105,7 +105,7 @@ func TestCadvisorExpectedLabels(t *testing.T) { continue } t.Run(md.Name, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), md.Name) + results, err := allNodesCache.Get(context.Background(), md.Name) require.NoError(t, err, "querying %s", md.Name) require.NotEmpty(t, results, "%s not available", md.Name) for _, r := range results { @@ -121,7 +121,7 @@ func TestCadvisorExpectedLabels(t *testing.T) { func TestCadvisorPodLabelsStrict(t *testing.T) { for _, name := range cadvisorMetricNamesList { t.Run(name, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), name) + results, err := allNodesCache.Get(context.Background(), name) require.NoError(t, err, "querying %s", name) require.NotEmpty(t, results, "%s not available", name) for _, r := range results { @@ -138,7 +138,7 @@ func TestCadvisorNginxWorkloadLabels(t *testing.T) { t.Run(name, func(t *testing.T) { promql := fmt.Sprintf(`%s{"@resource.k8s.cluster.name"="%s","@resource.k8s.pod.name"=~"nginx-test.*"}`, name, cfg.ClusterName) - nginx, err := client.Query(context.Background(), promql) + nginx, err := client.QueryComplete(context.Background(), promql) require.NoError(t, err, "querying %s for nginx-test", name) require.True(t, len(nginx) > 0, "No %s from nginx-test pods", name) for _, r := range nginx { @@ -176,7 +176,7 @@ func TestCadvisorHasRawPromotedKeys(t *testing.T) { func TestCadvisorNoPodSandboxMetrics(t *testing.T) { for _, name := range cadvisorMetricNamesList { t.Run(name, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), name) + results, err := allNodesCache.Get(context.Background(), name) require.NoError(t, err, "querying %s", name) require.NotEmpty(t, results, "%s not available", name) for _, r := range results { @@ -195,7 +195,7 @@ func TestCadvisorNoPodScopeRollup(t *testing.T) { continue } t.Run(md.Name, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), md.Name) + results, err := allNodesCache.Get(context.Background(), md.Name) require.NoError(t, err, "querying %s", md.Name) require.NotEmpty(t, results, "%s not available", md.Name) for _, r := range results { @@ -218,7 +218,7 @@ func TestCadvisorNetworkKeepsPodScope(t *testing.T) { continue } t.Run(md.Name, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), md.Name) + results, err := allNodesCache.Get(context.Background(), md.Name) require.NoError(t, err, "querying %s", md.Name) require.NotEmpty(t, results, "%s not available — the container_network_* exemption in filter/cw_k8s_ci_v0_cadvisor_rollup may have been narrowed", @@ -237,7 +237,7 @@ func TestCadvisorNodeGroupCoverage(t *testing.T) { promql := fmt.Sprintf( `container_memory_working_set_bytes{"@resource.k8s.cluster.name"="%s","@resource.host.type"="%s"}`, otelmetrics.EscapePromQLValue(cfg.ClusterName), ng.InstanceType) - results, err := client.Query(context.Background(), promql) + results, err := client.QueryComplete(context.Background(), promql) require.NoError(t, err, "querying cadvisor on %s", ng.Description) require.True(t, len(results) > 0, "cadvisor missing from %s (%s)", ng.Description, ng.InstanceType) }) diff --git a/test/otel/standard/dedup_test.go b/test/otel/standard/dedup_test.go index e5c95006d..c63df529e 100644 --- a/test/otel/standard/dedup_test.go +++ b/test/otel/standard/dedup_test.go @@ -29,7 +29,7 @@ func TestNodeExporterNoDuplicateSeries(t *testing.T) { `node_cpu_seconds_total{"@resource.k8s.cluster.name"="%s","@resource.host.type"="%s"}`, escaped, clusterHostTypes[0], ) - results, err := client.Query(ctx, promql) + results, err := client.QueryComplete(ctx, promql) require.NoError(t, err, "querying node_cpu_seconds_total") require.NotEmpty(t, results, "node_cpu_seconds_total not available") @@ -44,7 +44,7 @@ func TestNodeExporterNoDuplicateSeries(t *testing.T) { `node_cpu_seconds_total{"@resource.k8s.cluster.name"="%s","@resource.k8s.node.name"="%s",cpu="%s",mode="%s"}`, escaped, pinNode, pinCPU, pinMode, ) - pinned, err := client.Query(ctx, promql) + pinned, err := client.QueryComplete(ctx, promql) require.NoError(t, err, "querying pinned node_cpu_seconds_total") require.NotEmpty(t, pinned, "pinned node_cpu_seconds_total returned 0 results") require.Equal(t, 1, len(pinned), @@ -62,7 +62,7 @@ func TestCadvisorNoDuplicateSeries(t *testing.T) { `container_memory_working_set_bytes{"@resource.k8s.cluster.name"="%s","@resource.k8s.pod.name"=~"nginx-test.*"}`, escaped, ) - results, err := client.Query(ctx, promql) + results, err := client.QueryComplete(ctx, promql) require.NoError(t, err, "querying container_memory_working_set_bytes for nginx-test") require.NotEmpty(t, results, "container_memory_working_set_bytes not available for nginx-test pods") @@ -84,7 +84,7 @@ func TestCadvisorNoDuplicateSeries(t *testing.T) { `container_memory_working_set_bytes{"@resource.k8s.cluster.name"="%s","@resource.k8s.node.name"="%s","@resource.k8s.pod.name"="%s","@resource.k8s.container.name"="%s"}`, escaped, pinNode, pinPod, pinContainer, ) - pinned, err := client.Query(ctx, promql) + pinned, err := client.QueryComplete(ctx, promql) require.NoError(t, err, "querying pinned container_memory_working_set_bytes") require.NotEmpty(t, pinned, "pinned container_memory_working_set_bytes returned 0 results") require.Equal(t, 1, len(pinned), @@ -97,7 +97,7 @@ func TestCadvisorNoDuplicateSeries(t *testing.T) { func TestNodeExporterNoSchemaUrlLabel(t *testing.T) { for _, metricName := range nodeExporterMetricNames { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -128,7 +128,7 @@ func TestKubeletstatsNoDuplicateSeries(t *testing.T) { `{"__name__"="k8s.node.cpu.usage","@resource.k8s.cluster.name"="%s","@resource.k8s.node.name"="%s"}`, escaped, pinNode, ) - pinned, err := client.Query(ctx, promql) + pinned, err := client.QueryComplete(ctx, promql) require.NoError(t, err, "querying pinned k8s.node.cpu.usage") require.NotEmpty(t, pinned, "pinned k8s.node.cpu.usage returned 0 results") require.Equal(t, 1, len(pinned), diff --git a/test/otel/standard/host_validation_test.go b/test/otel/standard/host_validation_test.go index b4f8d06d0..394389d8f 100644 --- a/test/otel/standard/host_validation_test.go +++ b/test/otel/standard/host_validation_test.go @@ -32,6 +32,7 @@ func TestHostTypeMatchesNodeGroup(t *testing.T) { } for _, metricName := range hostEnrichedMetricNames() { t.Run(metricName, func(t *testing.T) { + // Pinned read: Karpenter picks the extra node's type at runtime, so it is in no node group. results, err := queryCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) @@ -57,7 +58,7 @@ func TestHostTypeMatchesNodeGroup(t *testing.T) { func TestHostNameMatchesNodeName(t *testing.T) { for _, metricName := range hostEnrichedMetricNames() { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) matchCount := 0 @@ -83,7 +84,7 @@ func TestHostNameMatchesNodeName(t *testing.T) { // --------------------------------------------------------------------------- func TestUniqueHostIdPerNode(t *testing.T) { - results, err := queryCache.Get(context.Background(), "node_cpu_seconds_total") + results, err := allNodesCache.Get(context.Background(), "node_cpu_seconds_total") require.NoError(t, err, "querying node_cpu_seconds_total") require.NotEmpty(t, results, "node_cpu_seconds_total not available") @@ -118,7 +119,7 @@ func TestUniqueHostIdPerNode(t *testing.T) { func TestCloudRegionMatchesConfig(t *testing.T) { for _, metricName := range hostEnrichedMetricNames() { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -140,7 +141,7 @@ func TestCloudRegionMatchesConfig(t *testing.T) { func TestCloudAccountMatchesConfig(t *testing.T) { for _, metricName := range hostEnrichedMetricNames() { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -163,7 +164,7 @@ func TestCloudAccountMatchesConfig(t *testing.T) { func TestCloudAvailabilityZonePresent(t *testing.T) { for _, metricName := range hostEnrichedMetricNames() { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -184,7 +185,7 @@ func TestCloudAvailabilityZonePresent(t *testing.T) { func TestCloudProviderIsAWS(t *testing.T) { for _, metricName := range hostEnrichedMetricNames() { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -202,7 +203,7 @@ func TestCloudProviderIsAWS(t *testing.T) { func TestCloudPlatformIsEKS(t *testing.T) { for _, metricName := range hostEnrichedMetricNames() { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -223,7 +224,7 @@ func TestCloudResourceIdIsEKSArn(t *testing.T) { expectedSuffix := fmt.Sprintf(":cluster/%s", cfg.ClusterName) for _, metricName := range hostEnrichedMetricNames() { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -251,7 +252,7 @@ func TestPodScheduledOnCorrectNode(t *testing.T) { gt := getGroundTruth(t) for _, metricName := range podScopedMetricNames() { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -283,7 +284,7 @@ func TestHostIdMatchesProviderID(t *testing.T) { gt := getGroundTruth(t) for _, metricName := range hostEnrichedMetricNames() { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -319,7 +320,7 @@ func TestNodeUidMatchesKubernetesAPI(t *testing.T) { gt := getGroundTruth(t) for _, metricName := range hostEnrichedMetricNames() { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -350,7 +351,7 @@ func TestContainerExistsInPodSpec(t *testing.T) { gt := getGroundTruth(t) for _, metricName := range podScopedMetricNames() { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -397,7 +398,7 @@ func TestPodNamespaceMatchesKubernetesAPI(t *testing.T) { gt := getGroundTruth(t) for _, metricName := range daemonsetMetricNames() { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -429,7 +430,7 @@ func TestNodeLabelsMatchKubernetesAPI(t *testing.T) { const labelPrefix = "k8s.node.label." for _, metricName := range hostEnrichedMetricNames() { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -471,7 +472,7 @@ func TestDeploymentOwnerMatchesKubernetesAPI(t *testing.T) { gt := getGroundTruth(t) for _, metricName := range daemonsetMetricNames() { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -517,7 +518,7 @@ func TestStatefulSetOwnerMatchesKubernetesAPI(t *testing.T) { gt := getGroundTruth(t) for _, metricName := range daemonsetMetricNames() { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -558,7 +559,7 @@ func TestDaemonSetOwnerMatchesKubernetesAPI(t *testing.T) { gt := getGroundTruth(t) for _, metricName := range daemonsetMetricNames() { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -599,7 +600,7 @@ func TestJobOwnerMatchesKubernetesAPI(t *testing.T) { gt := getGroundTruth(t) for _, metricName := range daemonsetMetricNames() { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { diff --git a/test/otel/standard/k8s_helpers_test.go b/test/otel/standard/k8s_helpers_test.go index 3269ea071..805ba15a5 100644 --- a/test/otel/standard/k8s_helpers_test.go +++ b/test/otel/standard/k8s_helpers_test.go @@ -8,16 +8,16 @@ package standard import ( "context" "fmt" - "os" - "path/filepath" "strings" "sync" "testing" "time" + "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) @@ -45,22 +45,46 @@ func getGroundTruth(t *testing.T) *k8sGroundTruth { return groundTruth } -func buildGroundTruth() (*k8sGroundTruth, error) { - kubeconfig := os.Getenv("KUBECONFIG") - if kubeconfig == "" { - kubeconfig = filepath.Join(os.Getenv("HOME"), ".kube", "config") - } +var ( + k8sClient *kubernetes.Clientset + k8sClientOnce sync.Once + k8sClientErr error +) - restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfig) - if err != nil { - return nil, fmt.Errorf("building kubeconfig: %w", err) - } +// sharedK8sClientset builds one clientset from the default kubeconfig loading rules +// ($KUBECONFIG, then ~/.kube/config), falling back to the in-cluster configuration. +func sharedK8sClientset() (*kubernetes.Clientset, error) { + k8sClientOnce.Do(func() { + restConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + clientcmd.NewDefaultClientConfigLoadingRules(), nil).ClientConfig() + if err != nil { + if restConfig, err = rest.InClusterConfig(); err != nil { + k8sClientErr = fmt.Errorf("no kubeconfig or in-cluster config: %w", err) + return + } + } + k8sClient, k8sClientErr = kubernetes.NewForConfig(restConfig) + }) + return k8sClient, k8sClientErr +} - clientset, err := kubernetes.NewForConfig(restConfig) +func k8sClientset(t *testing.T) *kubernetes.Clientset { + t.Helper() + clientset, err := sharedK8sClientset() + require.NoError(t, err, "creating K8s clientset") + return clientset +} + +func buildGroundTruth() (*k8sGroundTruth, error) { + clientset, err := sharedK8sClientset() if err != nil { - return nil, fmt.Errorf("creating K8s clientset: %w", err) + return nil, err } + return snapshotGroundTruth(clientset) +} +// snapshotGroundTruth lists every node and pod now. +func snapshotGroundTruth(clientset kubernetes.Interface) (*k8sGroundTruth, error) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -116,19 +140,7 @@ func imageTagFromPod(t *testing.T, gt *k8sGroundTruth, namespace, labelKey, labe // k8sServerVersion returns the Kubernetes API server version string. func k8sServerVersion(t *testing.T) string { t.Helper() - kubeconfig := os.Getenv("KUBECONFIG") - if kubeconfig == "" { - kubeconfig = filepath.Join(os.Getenv("HOME"), ".kube", "config") - } - restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfig) - if err != nil { - t.Fatalf("building kubeconfig: %v", err) - } - clientset, err := kubernetes.NewForConfig(restConfig) - if err != nil { - t.Fatalf("creating K8s clientset: %v", err) - } - sv, err := clientset.Discovery().ServerVersion() + sv, err := k8sClientset(t).Discovery().ServerVersion() if err != nil { t.Fatalf("getting K8s server version: %v", err) } diff --git a/test/otel/standard/ksm_availability_zone_test.go b/test/otel/standard/ksm_availability_zone_test.go new file mode 100644 index 000000000..c84eddf4e --- /dev/null +++ b/test/otel/standard/ksm_availability_zone_test.go @@ -0,0 +1,95 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package standard + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +// --------------------------------------------------------------------------- +// TestKSMAvailabilityZoneIsTheObjectsZone — the cluster scraper runs on one +// node, so its own AZ is wrong for most kube-state-metrics series. On every KSM +// series, cloud.availability_zone must be the zone of the node the series names: +// nodemetadataenricher sets it, or else the node's topology.kubernetes.io/zone label +// does, and a pod's own zone label replaces it. A series whose node k8sattributes +// knows must carry it. A series that names no node, such as a Deployment's, must not. +// The pod zone label itself is denied, so the API is the reference for it. +// Series come from allNodesCache, which reads KSM metrics with only the cluster +// matcher. A cached read can predate the API read, but a node that left in between +// is skipped and a node does not change zone, so the two still agree. +// --------------------------------------------------------------------------- + +const zoneLabel = "topology.kubernetes.io/zone" + +func TestKSMAvailabilityZoneIsTheObjectsZone(t *testing.T) { + gt := freshGroundTruth(t) + buckets := map[string][]string{ + "node": ksmNodeBucket, "pod": ksmPodBucket, "container": ksmContainerBucket, + "object": otelmetrics.ConcatKeys(ksmWorkloadBucket.Deployment, ksmWorkloadBucket.DaemonSet, + ksmWorkloadBucket.StatefulSet, ksmWorkloadBucket.Job, ksmWorkloadBucket.CronJob, ksmClusterBucket), + } + for _, bucket := range []string{"node", "pod", "container", "object"} { + bucket := bucket + t.Run(bucket, func(t *testing.T) { + var series, withZone, fromPodLabel int + wrong, missing, absent, unexpected := map[string]int{}, map[string]int{}, map[string]int{}, map[string]int{} + for _, metricName := range buckets[bucket] { + for _, r := range allNodeSeries(t, metricName) { + series++ + res := r.Labels.Resource + az, hasZone := res[ksmZoneKey] + if hasZone { + withZone++ + } + nodeName := res["k8s.node.name"] + if nodeName == "" { + if hasZone { + unexpected[metricName]++ + } + continue + } + node, known := gt.nodes[nodeName] + if !known { + continue // the node left between the API read and the scrape + } + if hasZone && az != node.Labels[zoneLabel] { + wrong[metricName+": "+az+" on a node in "+node.Labels[zoneLabel]]++ + } + pod, ok := gt.pods[res["k8s.namespace.name"]+"/"+res["k8s.pod.name"]] + if want := pod.Labels[zoneLabel]; ok && want != "" && pod.Spec.NodeName == nodeName { + fromPodLabel++ + if az != want { + missing[metricName]++ + } + } else if !hasZone && node.Labels[zoneLabel] != "" && res["k8s.node.uid"] != "" { + absent[metricName]++ // transform/cw_k8s_ci_v0_ksm_zone takes the node's zone label + } + } + } + t.Logf("%d series, %d with an AZ, %d of a pod with a zone label", series, withZone, fromPodLabel) + require.NotZero(t, series, "no %s series", bucket) + for _, k := range otelmetrics.SortedKeys(wrong) { + t.Errorf("%s: wrong AZ on %d series", k, wrong[k]) + } + for _, k := range otelmetrics.SortedKeys(missing) { + t.Errorf("%s: %d series of a pod with a zone label lack that AZ", k, missing[k]) + } + for _, k := range otelmetrics.SortedKeys(absent) { + t.Errorf("%s: %d series of a known node with a zone label carry no AZ", k, absent[k]) + } + for _, k := range otelmetrics.SortedKeys(unexpected) { + t.Errorf("%s: %d series name no node but carry an AZ, the cluster scraper's", k, unexpected[k]) + } + if bucket == "node" || bucket == "pod" { + require.NotZero(t, withZone, "no %s series carries an AZ", bucket) + } + }) + } +} diff --git a/test/otel/standard/ksm_test.go b/test/otel/standard/ksm_test.go index d2ad15d88..598ce8ac1 100644 --- a/test/otel/standard/ksm_test.go +++ b/test/otel/standard/ksm_test.go @@ -13,9 +13,6 @@ import ( "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" ) @@ -544,7 +541,7 @@ func TestKSM_PodBucket_NginxDeploymentName(t *testing.T) { ctx := context.Background() promql := fmt.Sprintf(`%s{"@resource.k8s.cluster.name"="%s","@resource.k8s.pod.name"=~"nginx-test.*"}`, metricName, cfg.ClusterName) - results, err := client.Query(ctx, promql) + results, err := client.QueryComplete(ctx, promql) require.NoError(t, err, "querying %s for nginx-test", metricName) require.True(t, len(results) > 0, "No %s results from nginx-test pods", metricName) for _, r := range results { @@ -643,7 +640,7 @@ func TestKSM_PodBucket_StatefulSetPodName(t *testing.T) { t.Parallel() promql := fmt.Sprintf(`%s{"@resource.k8s.cluster.name"="%s","@resource.k8s.pod.name"=~"ksm-test-statefulset.*"}`, metricName, cfg.ClusterName) - results, err := client.Query(context.Background(), promql) + results, err := client.QueryComplete(context.Background(), promql) require.NoError(t, err, "querying %s for ksm-test-statefulset", metricName) require.True(t, len(results) > 0, "No %s results from ksm-test-statefulset pods", metricName) for _, r := range results { @@ -665,7 +662,7 @@ func TestKSM_PodBucket_JobPodName(t *testing.T) { t.Parallel() promql := fmt.Sprintf(`%s{"@resource.k8s.cluster.name"="%s","@resource.k8s.pod.name"=~"ksm-test-job.*"}`, metricName, cfg.ClusterName) - results, err := client.Query(context.Background(), promql) + results, err := client.QueryComplete(context.Background(), promql) require.NoError(t, err, "querying %s for ksm-test-job", metricName) require.True(t, len(results) > 0, "No %s results from ksm-test-job pods", metricName) for _, r := range results { @@ -687,7 +684,7 @@ func TestKSM_PodBucket_CronJobPodName(t *testing.T) { t.Parallel() promql := fmt.Sprintf(`%s{"@resource.k8s.cluster.name"="%s","@resource.k8s.pod.name"=~"ksm-test-cronjob.*"}`, metricName, cfg.ClusterName) - results, err := client.Query(context.Background(), promql) + results, err := client.QueryComplete(context.Background(), promql) require.NoError(t, err, "querying %s for ksm-test-cronjob", metricName) if len(results) == 0 { t.Skipf("No %s results from ksm-test-cronjob pods (may have been cleaned up)", metricName) @@ -714,7 +711,7 @@ func TestKSM_PodBucket_ReplicaSetPodName(t *testing.T) { t.Parallel() promql := fmt.Sprintf(`%s{"@resource.k8s.cluster.name"="%s","@resource.k8s.pod.name"=~"ksm-test-replicaset.*"}`, metricName, cfg.ClusterName) - results, err := client.Query(context.Background(), promql) + results, err := client.QueryComplete(context.Background(), promql) require.NoError(t, err, "querying %s for ksm-test-replicaset", metricName) require.True(t, len(results) > 0, "No %s results from ksm-test-replicaset pods", metricName) for _, r := range results { @@ -1008,7 +1005,7 @@ func TestKSM_ContainerBucket_NginxDeploymentName(t *testing.T) { ctx := context.Background() promql := fmt.Sprintf(`%s{"@resource.k8s.cluster.name"="%s","@resource.k8s.pod.name"=~"nginx-test.*"}`, metricName, cfg.ClusterName) - results, err := client.Query(ctx, promql) + results, err := client.QueryComplete(ctx, promql) require.NoError(t, err, "querying %s for nginx-test", metricName) require.True(t, len(results) > 0, "No %s results from nginx-test pods", metricName) for _, r := range results { @@ -1571,18 +1568,8 @@ func TestKSM_LeaseExistence(t *testing.T) { t.Parallel() gt := getGroundTruth(t) - loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() - kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, nil) - restConfig, err := kubeConfig.ClientConfig() - if err != nil { - restConfig, err = rest.InClusterConfig() - require.NoError(t, err, "no kubeconfig or in-cluster config") - } - clientset, err := kubernetes.NewForConfig(restConfig) - require.NoError(t, err, "creating K8s clientset") - ctx := context.Background() - leases, err := clientset.CoordinationV1().Leases("amazon-cloudwatch").List(ctx, metav1.ListOptions{}) + leases, err := k8sClientset(t).CoordinationV1().Leases("amazon-cloudwatch").List(ctx, metav1.ListOptions{}) require.NoError(t, err, "listing Leases") leaseByNode := make(map[string]bool) diff --git a/test/otel/standard/kubeletstats_test.go b/test/otel/standard/kubeletstats_test.go index 27a40860b..9b3a2dea8 100644 --- a/test/otel/standard/kubeletstats_test.go +++ b/test/otel/standard/kubeletstats_test.go @@ -33,7 +33,7 @@ func TestKubeletstatsMetricExistence(t *testing.T) { func TestKubeletstatsInstrumentationSource(t *testing.T) { for _, metricName := range kubeletstatsMetricNamesList { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -49,7 +49,7 @@ func TestKubeletstatsInstrumentationSource(t *testing.T) { func TestKubeletstatsClusterIdentity(t *testing.T) { for _, metricName := range kubeletstatsMetricNamesList { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -63,7 +63,7 @@ func TestKubeletstatsClusterIdentity(t *testing.T) { func TestKubeletstatsNodeName(t *testing.T) { for _, metricName := range kubeletstatsMetricNamesList { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -81,7 +81,7 @@ func TestKubeletstatsPodName(t *testing.T) { podAndContainerMetrics = append(podAndContainerMetrics, metricNames(kubeletstatsContainerMetrics)...) for _, metricName := range podAndContainerMetrics { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -99,7 +99,7 @@ func TestKubeletstatsNamespace(t *testing.T) { podAndContainerMetrics = append(podAndContainerMetrics, metricNames(kubeletstatsContainerMetrics)...) for _, metricName := range podAndContainerMetrics { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -114,7 +114,7 @@ func TestKubeletstatsNamespace(t *testing.T) { func TestKubeletstatsContainerName(t *testing.T) { for _, md := range kubeletstatsContainerMetrics { t.Run(md.Name, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), md.Name) + results, err := allNodesCache.Get(context.Background(), md.Name) require.NoError(t, err, "querying %s", md.Name) require.NotEmpty(t, results, "%s not available", md.Name) hasContainer := 0 @@ -133,7 +133,7 @@ func TestKubeletstatsContainerName(t *testing.T) { func TestKubeletstatsCloudDetection(t *testing.T) { for _, metricName := range kubeletstatsMetricNamesList { t.Run(metricName+"/provider", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -142,7 +142,7 @@ func TestKubeletstatsCloudDetection(t *testing.T) { } }) t.Run(metricName+"/region", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -152,7 +152,7 @@ func TestKubeletstatsCloudDetection(t *testing.T) { } }) t.Run(metricName+"/account_id", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -167,7 +167,7 @@ func TestKubeletstatsCloudDetection(t *testing.T) { func TestKubeletstatsHostDetection(t *testing.T) { for _, metricName := range kubeletstatsMetricNamesList { t.Run(metricName+"/host_id", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -178,7 +178,7 @@ func TestKubeletstatsHostDetection(t *testing.T) { } }) t.Run(metricName+"/host_name", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -188,7 +188,7 @@ func TestKubeletstatsHostDetection(t *testing.T) { } }) t.Run(metricName+"/host_type", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -206,7 +206,7 @@ func TestKubeletstatsExpectedLabels(t *testing.T) { continue } t.Run(md.Name, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), md.Name) + results, err := allNodesCache.Get(context.Background(), md.Name) require.NoError(t, err, "querying %s", md.Name) require.NotEmpty(t, results, "%s not available", md.Name) for _, r := range results { @@ -229,7 +229,7 @@ func TestKubeletstatsUnitValidation(t *testing.T) { continue } t.Run(md.Name, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), md.Name) + results, err := allNodesCache.Get(context.Background(), md.Name) require.NoError(t, err, "querying %s", md.Name) require.NotEmpty(t, results, "%s not available", md.Name) for _, r := range results { @@ -247,7 +247,7 @@ func TestKubeletstatsNodeGroupCoverage(t *testing.T) { promql := fmt.Sprintf( `{"__name__"="k8s.node.cpu.usage","@resource.k8s.cluster.name"="%s","@resource.host.type"="%s"}`, otelmetrics.EscapePromQLValue(cfg.ClusterName), ng.InstanceType) - results, err := client.Query(context.Background(), promql) + results, err := client.QueryComplete(context.Background(), promql) require.NoError(t, err, "querying k8s.node.cpu.usage on %s", ng.Description) require.True(t, len(results) > 0, "kubeletstats missing from %s nodes (%s) — kubeletstatsreceiver not scraping?", diff --git a/test/otel/standard/labels_common_test.go b/test/otel/standard/labels_common_test.go index 8834acf3c..301a213d7 100644 --- a/test/otel/standard/labels_common_test.go +++ b/test/otel/standard/labels_common_test.go @@ -24,7 +24,7 @@ import ( func TestClusterIdentity(t *testing.T) { for _, metricName := range allMetricNames() { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.True(t, len(results) > 0, "%s returned 0 results when filtering by @resource.k8s.cluster.name=%s", @@ -44,7 +44,7 @@ func TestClusterIdentity(t *testing.T) { func TestAWSLabels(t *testing.T) { for _, metricName := range allMetricNames() { t.Run(metricName+"/account", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -54,7 +54,7 @@ func TestAWSLabels(t *testing.T) { } }) t.Run(metricName+"/region", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -73,7 +73,7 @@ func TestAWSLabels(t *testing.T) { func TestInstrumentationLabels(t *testing.T) { for _, metricName := range allMetricNames() { t.Run(metricName+"/name", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -83,7 +83,7 @@ func TestInstrumentationLabels(t *testing.T) { } }) t.Run(metricName+"/version", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -92,7 +92,7 @@ func TestInstrumentationLabels(t *testing.T) { } }) t.Run(metricName+"/cloudwatch_source", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -102,7 +102,7 @@ func TestInstrumentationLabels(t *testing.T) { } }) t.Run(metricName+"/cloudwatch_solution", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -112,7 +112,7 @@ func TestInstrumentationLabels(t *testing.T) { } }) t.Run(metricName+"/cloudwatch_pipeline", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -185,7 +185,7 @@ func TestScopeNameAndVersionPerSource(t *testing.T) { func TestCloudResourceDetection(t *testing.T) { for _, metricName := range hostEnrichedMetricNames() { t.Run(metricName+"/provider", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -193,7 +193,7 @@ func TestCloudResourceDetection(t *testing.T) { } }) t.Run(metricName+"/platform", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -203,7 +203,7 @@ func TestCloudResourceDetection(t *testing.T) { } }) t.Run(metricName+"/region", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -213,7 +213,7 @@ func TestCloudResourceDetection(t *testing.T) { } }) t.Run(metricName+"/availability_zone", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -223,7 +223,7 @@ func TestCloudResourceDetection(t *testing.T) { } }) t.Run(metricName+"/account_id", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -242,7 +242,7 @@ func TestCloudResourceDetection(t *testing.T) { func TestHostResourceDetection(t *testing.T) { for _, metricName := range hostEnrichedMetricNames() { t.Run(metricName+"/host_id", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -253,7 +253,7 @@ func TestHostResourceDetection(t *testing.T) { } }) t.Run(metricName+"/host_type", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -263,7 +263,7 @@ func TestHostResourceDetection(t *testing.T) { } }) t.Run(metricName+"/host_name", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -273,7 +273,7 @@ func TestHostResourceDetection(t *testing.T) { } }) t.Run(metricName+"/host_image_id", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -293,7 +293,7 @@ func TestHostResourceDetection(t *testing.T) { func TestNodeMetricLabels(t *testing.T) { for _, metricName := range daemonsetMetricNames() { t.Run(metricName+"/node_name", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -307,7 +307,7 @@ func TestNodeMetricLabels(t *testing.T) { for _, metricName := range nodeLabelEnrichedNames() { t.Run(metricName+"/node_uid", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -328,7 +328,7 @@ func TestNodeMetricLabels(t *testing.T) { func TestPodMetricLabels(t *testing.T) { for _, metricName := range podScopedCadvisorNames() { t.Run(metricName+"/cadvisor_strict", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -352,7 +352,7 @@ func TestContainerMetricLabels(t *testing.T) { require.True(t, len(names) > 0, "No container-scoped metrics defined in test suite") for _, metricName := range names { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) hasContainer := 0 @@ -378,7 +378,7 @@ func TestContainerMetricLabels(t *testing.T) { func TestLabelPreservation(t *testing.T) { for _, metricName := range allMetricNames() { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -395,7 +395,7 @@ func TestLabelPreservation(t *testing.T) { func TestMetricNamePreservation(t *testing.T) { for _, metricName := range allMetricNames() { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -412,7 +412,7 @@ func TestMetricNamePreservation(t *testing.T) { func TestServiceLabels(t *testing.T) { for _, metricName := range prometheusScrapedNames() { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -456,7 +456,7 @@ func TestNodeLabelEnrichment(t *testing.T) { }) t.Run(metricName+"/nfd_labels_dropped", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -482,7 +482,7 @@ func TestWorkloadLabels(t *testing.T) { // Node-level: node_exporter metrics must NOT have workload labels. for _, metricName := range metricNames(nodeExporterMetrics) { t.Run(metricName+"/no_workload", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -503,7 +503,7 @@ func TestWorkloadLabels(t *testing.T) { t.Run(metricName+"/cadvisor_nginx_workload", func(t *testing.T) { promql := fmt.Sprintf(`%s{"@resource.k8s.cluster.name"="%s","@resource.k8s.pod.name"=~"nginx-test.*"}`, metricName, cfg.ClusterName) - results, err := client.Query(context.Background(), promql) + results, err := client.QueryComplete(context.Background(), promql) require.NoError(t, err, "querying %s for nginx-test", metricName) require.True(t, len(results) > 0, "No %s results from nginx-test pods", metricName) for _, r := range results { @@ -526,7 +526,7 @@ func TestMetricTypeLabels(t *testing.T) { continue } t.Run(md.Name+"/counter_type", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), md.Name) + results, err := allNodesCache.Get(context.Background(), md.Name) require.NoError(t, err, "querying %s", md.Name) require.NotEmpty(t, results, "%s not available", md.Name) for _, r := range results { @@ -542,7 +542,7 @@ func TestMetricTypeLabels(t *testing.T) { continue } t.Run(md.Name+"/gauge_type", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), md.Name) + results, err := allNodesCache.Get(context.Background(), md.Name) require.NoError(t, err, "querying %s", md.Name) require.NotEmpty(t, results, "%s not available", md.Name) for _, r := range results { @@ -562,7 +562,7 @@ func TestMetricUnitLabels(t *testing.T) { continue } t.Run(md.Name, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), md.Name) + results, err := allNodesCache.Get(context.Background(), md.Name) require.NoError(t, err, "querying %s", md.Name) require.NotEmpty(t, results, "%s not available", md.Name) for _, r := range results { @@ -586,7 +586,7 @@ func TestCustomNodeLabels(t *testing.T) { for _, metricName := range nodeLabelEnrichedNames() { t.Run(metricName+"/color_present", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -607,7 +607,7 @@ func TestCustomNodeLabels(t *testing.T) { t.Run(metricName+"/has_blue", func(t *testing.T) { promql := fmt.Sprintf(`%s{"@resource.k8s.cluster.name"="%s","@resource.%s"="blue"}`, metricName, cfg.ClusterName, nodeColorLabel) - results, err := client.Query(context.Background(), promql) + results, err := client.QueryComplete(context.Background(), promql) require.NoError(t, err, "querying %s with blue filter", metricName) require.True(t, len(results) > 0, "%s no results from standard nodes (color=blue)", metricName) @@ -625,7 +625,7 @@ func TestCloudResourceId(t *testing.T) { for _, metricName := range allMetricNames() { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -745,10 +745,11 @@ func podLabelMetricNames() []string { return append(names, ksmContainerBucket...) } -// allNodeSeries returns every series of a metric in this cluster, on every node. +// allNodeSeries returns every series of a metric in this cluster, on every node, from the +// read that the suite's other every-series checks share. func allNodeSeries(t *testing.T, metricName string) []otelmetrics.MetricResult { t.Helper() - results, err := queryCache.GetWithFilter(context.Background(), metricName, nil) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) return results @@ -922,7 +923,7 @@ func TestDeprecatedSemconvAttributesRemoved(t *testing.T) { for _, metricName := range names { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) require.NoError(t, otelmetrics.CheckDeprecatedSemconvRemoved(metricName, results)) @@ -963,7 +964,7 @@ func TestRawLabelsOnlyAtDatapointScope(t *testing.T) { c := c for _, metricName := range c.names { t.Run(c.pipeline+"/"+metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) require.NoError(t, otelmetrics.CheckRawLabelsNotAtResourceScope(metricName, results, c.rawKeys)) diff --git a/test/otel/standard/labels_reverse_test.go b/test/otel/standard/labels_reverse_test.go new file mode 100644 index 000000000..baac9c456 --- /dev/null +++ b/test/otel/standard/labels_reverse_test.go @@ -0,0 +1,347 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package standard + +import ( + "context" + "errors" + "sort" + "testing" + "time" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +// --------------------------------------------------------------------------- +// Reverse label reconciliation. TestNodeLabelsMatchKubernetesAPI checks that +// every k8s.node.label.* on a metric exists on the node; it cannot notice a +// label that never arrived. These tests start from the Kubernetes API instead: +// every node and pod label that the running k8sattributes rules extract and the +// running denylist does not match must arrive with the same value, and every +// other label must not. The expectation comes from the cluster and the live +// configuration; the one maintained list is the never-denied attributes, which +// stops an over-broad denylist entry passing. +// +// Non-default filter values change the expectation. A label include list +// narrows the extraction rules, so a label it leaves out must not arrive. A +// namespace filter drops the series of every pod in a namespace it removes, so +// those pods need no series. Node series carry no k8s.namespace.name, so the +// namespace filter keeps them. A filter the model cannot read skips the test +// with the reason. +// +// Labels reach metrics on two paths, each with its own k8sattributes extraction +// and denylist: the cluster scraper's kube-state-metrics pipeline, which reports +// every node and pod, and the node agent's kubeletstats pipeline, which reports +// the nodes and pods where the agent runs. Both are queried cluster-wide, so the +// clusterHostTypes pin does not limit them. Every object a path reports must +// have a series, or an object that lost all of its series would pass: every +// node and pod on the cluster scraper path, and on the node agent path every +// node with a ready agent pod and every running pod on such a node. Each pass +// reads the API, then every series at one evaluation time, so the two sides are +// seconds apart. An object fails only if it fails two passes recheckDelay apart, +// because a label change or a new object reaches the API before the next +// collection is ingested. +// --------------------------------------------------------------------------- + +const ( + agentNamespace = "amazon-cloudwatch" + nodeAgentConfigMap = "cloudwatch-agent" + nodeAgentDaemonSet = "cloudwatch-agent" + clusterScraperConfigMap = "cloudwatch-agent-cluster-scraper" + otelConfigKey = "cwagentotelconfig.yaml" + + // recheckDelay covers one collection interval of either agent plus ingestion. + recheckDelay = 90 * time.Second +) + +// agentConfig reads the collector configuration an agent is running. +func agentConfig(t *testing.T, configMap string) string { + t.Helper() + cm, err := k8sClientset(t).CoreV1().ConfigMaps(agentNamespace).Get( + context.Background(), configMap, metav1.GetOptions{}) + require.NoError(t, err, "reading ConfigMap %s/%s", agentNamespace, configMap) + raw, ok := cm.Data[otelConfigKey] + require.True(t, ok, "ConfigMap %s has no %s", configMap, otelConfigKey) + return raw +} + +// agentDenylist reads the attribute-limit configuration an agent is running. +func agentDenylist(t *testing.T, configMap string) otelmetrics.AttributeDenylist { + t.Helper() + d, err := otelmetrics.ParseAttributeDenylist(agentConfig(t, configMap)) + require.NoError(t, err, "ConfigMap %s", configMap) + return d +} + +// labelFilters is what an agent's configuration does on one label path. +type labelFilters struct { + extraction otelmetrics.LabelExtraction + denylist otelmetrics.AttributeDenylist + namespaces otelmetrics.NamespaceFilter // read for pods only +} + +// agentLabelFilters reads the filters an agent runs for labels from "node" or +// "pod". It skips the test for a filter the model cannot read. +func agentLabelFilters(t *testing.T, configMap, from string) labelFilters { + t.Helper() + raw := agentConfig(t, configMap) + var f labelFilters + var err error + f.denylist, err = otelmetrics.ParseAttributeDenylist(raw) + require.NoError(t, err, "ConfigMap %s", configMap) + f.extraction, err = otelmetrics.ParseLabelExtraction(raw, from) + skipIfNotModeled(t, configMap, err) + t.Logf("ConfigMap %s: %s labels extracted by %s, every label: %v", configMap, from, f.extraction.Processor, f.extraction.ExtractsAll()) + if from == "pod" { + f.namespaces, err = otelmetrics.ParseNamespaceFilter(raw) + skipIfNotModeled(t, configMap, err) + t.Logf("ConfigMap %s: namespace filter processors %v", configMap, f.namespaces.Processors) + } + return f +} + +func skipIfNotModeled(t *testing.T, configMap string, err error) { + t.Helper() + if errors.Is(err, otelmetrics.ErrFilterNotModeled) { + t.Skipf("ConfigMap %s: %v. The test cannot tell which labels or objects this filter removes", configMap, err) + } + require.NoError(t, err, "ConfigMap %s", configMap) +} + +// TestDenylistSparesNeverDeniedAttributes fails for a denylist entry that removes +// a never-denied attribute. A customer may remove a protected label with +// nodeLabels.exclude or podLabels.exclude, so the entries that the declared +// clusterLabelExcludes render to are logged instead. +func TestDenylistSparesNeverDeniedAttributes(t *testing.T) { + for _, configMap := range []string{nodeAgentConfigMap, clusterScraperConfigMap} { + t.Run(configMap, func(t *testing.T) { + d := agentDenylist(t, configMap) + chart, customer := d.SplitCustomerExcludes(clusterLabelExcludes) + if entries := customer.Entries(); len(entries) > 0 { + t.Logf("%s: %d entries match a declared label exclude: %v", d.Processor, len(entries), entries) + } + for _, v := range customer.NeverDeniedViolations() { + t.Logf("%s: declared label exclude, allowed: %s", d.Processor, v) + } + for _, v := range chart.NeverDeniedViolations() { + t.Errorf("%s: %s (if a label exclude of the deployment adds it, declare the exclude in %s or %s)", + d.Processor, v, otelmetrics.PodLabelExcludesEnvVar, otelmetrics.NodeLabelExcludesEnvVar) + } + }) + } +} + +// reconcilePass reads both sides once and reconciles them. +type reconcilePass func(t *testing.T) otelmetrics.ObjectReconciliation + +// failedObjects returns the objects that failed a pass: a series that did not +// reconcile, or a required object with no series. +func failedObjects(r otelmetrics.ObjectReconciliation) map[string]bool { + failed := make(map[string]bool, len(r.Failed)+len(r.NoSeries)) + for key := range r.Failed { + failed[key] = true + } + for _, key := range r.NoSeries { + failed[key] = true + } + return failed +} + +// reconcileTwice runs pass and, if any object failed, runs it again after +// recheckDelay. Only objects that fail both passes are reported. +func reconcileTwice(t *testing.T, kind string, e otelmetrics.LabelExtraction, pass reconcilePass) { + t.Helper() + first := pass(t) + t.Logf("reconciled %d %ss, %d labels, %d series skipped at the attribute cap, %d failed, %d of %d required with no series", + first.Objects, kind, first.Labels, first.AtCap, len(first.Failed), len(first.NoSeries), first.Required) + require.NotZero(t, first.Required, "the Kubernetes API listed no %s that must have a series", kind) + require.NotZero(t, first.Objects, "no series matched a %s from the Kubernetes API", kind) + if first.Labels == 0 && !e.ExtractsAll() { + t.Logf("the label include list in %s extracts no label that these %ss carry", e.Processor, kind) + } else { + require.NotZero(t, first.Labels, "%ss matched but carried no checkable label", kind) + } + failedBefore := failedObjects(first) + if len(failedBefore) == 0 { + return + } + t.Logf("re-reading the API and the metrics in %s to separate label churn and new objects from a real drop", recheckDelay) + time.Sleep(recheckDelay) + second := pass(t) + keys := make([]string, 0, len(failedBefore)) + for key := range failedObjects(second) { + if failedBefore[key] { + keys = append(keys, key) + } + } + sort.Strings(keys) + for _, key := range keys { + diff, hasSeries := second.Failed[key] + if !hasSeries { + t.Errorf("%s %s: no series", kind, key) + continue + } + t.Errorf("%s %s: missing %v, value mismatch %v, removed but present %v", + kind, key, diff.Missing, diff.Mismatched, diff.Leaked) + } +} + +// agentNodes returns the listed nodes where a node agent pod is ready. +func agentNodes(gt *k8sGroundTruth) map[string]bool { + nodes := map[string]bool{} + for _, p := range gt.pods { + if p.Namespace != agentNamespace || p.DeletionTimestamp != nil || !ownedByDaemonSet(p, nodeAgentDaemonSet) { + continue + } + if _, listed := gt.nodes[p.Spec.NodeName]; !listed { + continue // it left between the two reads + } + for _, c := range p.Status.Conditions { + if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue { + nodes[p.Spec.NodeName] = true + } + } + } + return nodes +} + +func ownedByDaemonSet(p corev1.Pod, name string) bool { + for _, o := range p.OwnerReferences { + if o.Kind == "DaemonSet" && o.Name == name { + return true + } + } + return false +} + +// everyNode and everyPod are the objects that must have a series on the cluster +// scraper path, because kube-state-metrics reports every object the API lists. +func everyNode(gt *k8sGroundTruth) []string { + keys := make([]string, 0, len(gt.nodes)) + for name := range gt.nodes { + keys = append(keys, name) + } + return keys +} + +func everyPod(gt *k8sGroundTruth) []string { + keys := make([]string, 0, len(gt.pods)) + for key := range gt.pods { + keys = append(keys, key) + } + return keys +} + +// nodesWithAgent and runningPodsWithAgent are the objects that must have a series +// on the node agent path, which reports its own node and the pods running there. +func nodesWithAgent(gt *k8sGroundTruth) []string { + var keys []string + for name := range agentNodes(gt) { + keys = append(keys, name) + } + return keys +} + +func runningPodsWithAgent(gt *k8sGroundTruth) []string { + agents := agentNodes(gt) + var keys []string + for key, p := range gt.pods { + // A terminating pod can stop reporting before the API drops it. + if agents[p.Spec.NodeName] && p.Status.Phase == corev1.PodRunning && p.DeletionTimestamp == nil { + keys = append(keys, key) + } + } + return keys +} + +// withoutFilteredNamespaces returns the pod keys whose namespace the filter keeps. +func withoutFilteredNamespaces(gt *k8sGroundTruth, keys []string, f otelmetrics.NamespaceFilter) []string { + var kept []string + for _, key := range keys { + if !f.Drops(gt.pods[key].Namespace) { + kept = append(kept, key) + } + } + return kept +} + +// freshGroundTruth lists the nodes and pods now, unlike the session-wide getGroundTruth. +func freshGroundTruth(t *testing.T) *k8sGroundTruth { + t.Helper() + gt, err := snapshotGroundTruth(k8sClientset(t)) + require.NoError(t, err, "listing nodes and pods") + return gt +} + +// queryAllSeries returns every series of a metric in this cluster, on every node, +// evaluated at one time. Unlike allNodeSeries it reads now, not from the cache, so a +// pass compares the API with metrics seconds apart. +func queryAllSeries(t *testing.T, metric string) []otelmetrics.MetricResult { + t.Helper() + results, err := allNodesCache.GetWithFilter(context.Background(), metric, nil) + require.NoError(t, err, "querying %s", metric) + require.NotEmpty(t, results, "%s not available", metric) + return results +} + +// labelPath is one metric that carries an object's labels, one series per object, +// the agent whose denylist the series went through, and the objects that must +// have a series. +type labelPath struct { + agent, metric, configMap string + required func(*k8sGroundTruth) []string +} + +func TestNodeLabelsReachMetrics(t *testing.T) { + for _, p := range []labelPath{ + {"cluster-scraper", "kube_node_info", clusterScraperConfigMap, everyNode}, + {"node-agent", "k8s.node.cpu.usage", nodeAgentConfigMap, nodesWithAgent}, + } { + p := p + t.Run(p.agent, func(t *testing.T) { + f := agentLabelFilters(t, p.configMap, "node") + reconcileTwice(t, "node", f.extraction, func(t *testing.T) otelmetrics.ObjectReconciliation { + gt := freshGroundTruth(t) + return otelmetrics.ReconcileObjects(queryAllSeries(t, p.metric), + func(l otelmetrics.MetricLabels) (string, map[string]string, bool) { + name := l.Resource["k8s.node.name"] + node, ok := gt.nodes[name] // absent if it joined or left between the two reads + return name, node.Labels, ok + }, p.required(gt), f.extraction, f.denylist) + }) + }) + } +} + +func TestPodLabelsReachMetrics(t *testing.T) { + for _, p := range []labelPath{ + {"cluster-scraper", "kube_pod_info", clusterScraperConfigMap, everyPod}, + {"node-agent", "k8s.pod.cpu.usage", nodeAgentConfigMap, runningPodsWithAgent}, + } { + p := p + t.Run(p.agent, func(t *testing.T) { + f := agentLabelFilters(t, p.configMap, "pod") + reconcileTwice(t, "pod", f.extraction, func(t *testing.T) otelmetrics.ObjectReconciliation { + gt := freshGroundTruth(t) + reported := p.required(gt) + required := withoutFilteredNamespaces(gt, reported, f.namespaces) + if len(required) == 0 && len(reported) > 0 { + t.Skipf("the namespace filter %v removes all %d pods this path reports", f.namespaces.Processors, len(reported)) + } + return otelmetrics.ReconcileObjects(queryAllSeries(t, p.metric), + func(l otelmetrics.MetricLabels) (string, map[string]string, bool) { + key := l.Resource["k8s.namespace.name"] + "/" + l.Resource["k8s.pod.name"] + pod, ok := gt.pods[key] // absent if it was created or deleted between the two reads + return key, pod.Labels, ok + }, required, f.extraction, f.denylist) + }) + }) + } +} diff --git a/test/otel/standard/metrics_test.go b/test/otel/standard/metrics_test.go index 6c3168dcd..3f43d5ba9 100644 --- a/test/otel/standard/metrics_test.go +++ b/test/otel/standard/metrics_test.go @@ -7,15 +7,13 @@ package standard import "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" -// Instance types in the standard cluster. -var clusterHostTypes = []string{"t3.medium"} +// Instance types in the standard cluster; OTEL_HOST_TYPES overrides them. +var clusterHostTypes = otelmetrics.ResolveHostTypes([]string{"t3.medium"}) -var clusterNodeGroups = []struct { - InstanceType string - Description string -}{ - {"t3.medium", "standard"}, -} +var clusterNodeGroups = otelmetrics.NodeGroupsFor(clusterHostTypes, map[string]string{"t3.medium": "standard"}) + +// Label exclude patterns the cluster was deployed with; OTEL_POD_LABEL_EXCLUDES and OTEL_NODE_LABEL_EXCLUDES set them. +var clusterLabelExcludes = otelmetrics.ResolveLabelExcludes() // --- Metric definitions (standard cluster only) --- diff --git a/test/otel/standard/node_exporter_test.go b/test/otel/standard/node_exporter_test.go index 69db04781..3b7cc4ba2 100644 --- a/test/otel/standard/node_exporter_test.go +++ b/test/otel/standard/node_exporter_test.go @@ -20,7 +20,7 @@ var nodeExporterMetricNames = metricNames(nodeExporterMetrics) func TestNodeExporterInstrumentationSource(t *testing.T) { for _, metricName := range nodeExporterMetricNames { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -36,7 +36,7 @@ func TestNodeExporterInstrumentationSource(t *testing.T) { func TestNodeExporterInstrumentationConsistent(t *testing.T) { for _, metricName := range nodeExporterMetricNames { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) names := make(map[string]struct{}) @@ -57,7 +57,7 @@ func TestNodeExporterExpectedLabels(t *testing.T) { continue } t.Run(md.Name, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), md.Name) + results, err := allNodesCache.Get(context.Background(), md.Name) require.NoError(t, err, "querying %s", md.Name) require.NotEmpty(t, results, "%s not available", md.Name) for _, r := range results { @@ -76,7 +76,7 @@ func TestNodeExporterExpectedLabels(t *testing.T) { func TestNodeExporterNoPodLabels(t *testing.T) { for _, metricName := range nodeExporterMetricNames { t.Run(metricName+"/no_container_name", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -87,7 +87,7 @@ func TestNodeExporterNoPodLabels(t *testing.T) { } }) t.Run(metricName+"/no_pod_name", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -98,7 +98,7 @@ func TestNodeExporterNoPodLabels(t *testing.T) { } }) t.Run(metricName+"/no_namespace", func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -114,7 +114,7 @@ func TestNodeExporterNoPodLabels(t *testing.T) { func TestNodeExporterNoWorkloadLabels(t *testing.T) { for _, metricName := range nodeExporterMetricNames { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { @@ -137,7 +137,7 @@ func TestNodeExporterNodeGroupCoverage(t *testing.T) { promql := fmt.Sprintf( `node_load1{"@resource.k8s.cluster.name"="%s","@resource.host.type"="%s"}`, otelmetrics.EscapePromQLValue(cfg.ClusterName), ng.InstanceType) - results, err := client.Query(context.Background(), promql) + results, err := client.QueryComplete(context.Background(), promql) require.NoError(t, err, "querying node_load1 on %s", ng.Description) require.True(t, len(results) > 0, "node_exporter missing from %s nodes (%s) — DaemonSet not scheduling?", @@ -154,7 +154,7 @@ func TestNodeExporterScrapeSelfTelemetryDropped(t *testing.T) { "node_scrape_collector_success", } { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.Empty(t, results, "%s should be dropped by filter/cw_k8s_ci_v0_scrape_metadata", metricName) @@ -165,7 +165,7 @@ func TestNodeExporterScrapeSelfTelemetryDropped(t *testing.T) { func TestNodeExporterHasRawNodeName(t *testing.T) { for _, metricName := range nodeExporterMetricNames { t.Run(metricName, func(t *testing.T) { - results, err := queryCache.Get(context.Background(), metricName) + results, err := allNodesCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { diff --git a/test/otel/standard/schema_ksm_test.go b/test/otel/standard/schema_ksm_test.go new file mode 100644 index 000000000..14297708b --- /dev/null +++ b/test/otel/standard/schema_ksm_test.go @@ -0,0 +1,140 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package standard + +import "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" + +// --------------------------------------------------------------------------- +// kube-state-metrics rows of TestAttributeSchema, one per ksm_test.go bucket and +// workload kind. Every key comes from the cluster scraper's kube_state_metrics +// pipeline. The raw labels groupbyattrs lifts (pod, namespace, node, uid, ...) +// are deleted from the resource after ksm_promote copies them back, so no row +// allows them. EKS only: this suite does not run on AKS. +// --------------------------------------------------------------------------- + +// On every KSM series. +var ksmCommonKeys = otelmetrics.ConcatKeys(otelmetrics.PrometheusReceiverKeys, + otelmetrics.Without(otelmetrics.CloudKeys, ksmZoneKey), []string{ + "k8s.cluster.name", // set_cluster_name; protected + }) + +// The zone of the node a series names: nodemetadataenricher sets it with host.id, and +// transform/cw_k8s_ci_v0_ksm_zone takes it from the node's zone label otherwise. A pod's +// zone label replaces it (transform/cw_k8s_ci_v0_pod_topology). A series that names no +// node has none. TestKSMAvailabilityZoneIsTheObjectsZone checks the value. +const ksmZoneKey = "cloud.availability_zone" + +// A series whose node k8sattributes knows has the node's labels, so it carries the AZ. +var ksmZoneImplies = map[string][]string{"k8s.node.uid": {ksmZoneKey}} + +// ksm_promote from namespace, pod and uid; protected. +var ksmPodKeys = []string{"k8s.namespace.name", "k8s.pod.name", "k8s.pod.uid"} + +// nodemetadataenricher sets the host keys and the AZ together, only while the node's +// Lease is fresh, that is while its DaemonSet agent runs, so some series carry them. +// TestKSM_NodeBucket_HasHostAttributes requires them. +var ksmHostImplies = otelmetrics.MergeImplies(otelmetrics.Together(otelmetrics.HostKeys...), + map[string][]string{"host.id": {ksmZoneKey}}) + +// The chart installs a DaemonSet (the agent) and Deployments (the operator and the +// cluster scraper), so these owner keys are on some pod series of any cluster. +// Jobs, CronJobs and StatefulSets depend on the cluster's workloads. +var ksmOwnerExpected = otelmetrics.ConcatKeys( + []string{"k8s.daemonset.name", "k8s.deployment.name", "k8s.replicaset.name"}, otelmetrics.WorkloadKeys) +var ksmOwnerOptional = []string{"k8s.cronjob.name", "k8s.job.name", "k8s.statefulset.name"} + +// One row per workload kind rather than one with the object-name keys optional, +// so a series that loses its own k8s..name or gains another kind's fails. +func ksmWorkloadSchema(name string, metrics []string, objectKey string) groupSchema { + return groupSchema{ + name: name, + metrics: func() []string { return metrics }, + schema: otelmetrics.GroupSchema{ + Resource: otelmetrics.KeySchema{ + Required: otelmetrics.ConcatKeys(ksmCommonKeys, []string{ + "k8s.namespace.name", // ksm_promote; protected + objectKey, // ksm_promote; protected + "k8s.workload.name", // set_workload; protected + "k8s.workload.type", // set_workload; protected + }), + }, + Instrumentation: scopeInstrumentation, + }, + } +} + +func init() { + attributeSchemas = append(attributeSchemas, + groupSchema{ + name: "ksm-node", + metrics: func() []string { return ksmNodeBucket }, + schema: otelmetrics.GroupSchema{ + Resource: otelmetrics.KeySchema{ + Required: otelmetrics.ConcatKeys(ksmCommonKeys, []string{ + "k8s.node.name", // ksm_promote; protected + "k8s.node.uid", // k8sattributes/cw_k8s_ci_v0_node, set by its node-label watch + }), + Expected: append([]string{ksmZoneKey}, otelmetrics.HostKeys...), + OpenPrefixes: []string{"k8s.node.label."}, + Implies: otelmetrics.MergeImplies(ksmHostImplies, ksmZoneImplies), + }, + Instrumentation: scopeInstrumentation, + }, + }, + groupSchema{ + name: "ksm-pod", + metrics: func() []string { return ksmPodBucket }, + schema: otelmetrics.GroupSchema{ + Resource: otelmetrics.KeySchema{ + Required: otelmetrics.ConcatKeys(ksmCommonKeys, ksmPodKeys), + // k8s.node.name from ksm_promote or k8sattributes/cw_k8s_ci_v0_pod, then k8s.node.uid + // from k8sattributes/cw_k8s_ci_v0_node; both only on scheduled pods. + Expected: otelmetrics.ConcatKeys([]string{"k8s.node.name", "k8s.node.uid", ksmZoneKey}, otelmetrics.HostKeys, ksmOwnerExpected), + Optional: ksmOwnerOptional, + OpenPrefixes: []string{"k8s.node.label.", "k8s.pod.label."}, + Implies: otelmetrics.MergeImplies(ksmHostImplies, ksmZoneImplies, otelmetrics.WorkloadImplies(), + otelmetrics.Together("k8s.node.name", "k8s.node.uid")), + }, + Instrumentation: scopeInstrumentation, + }, + }, + groupSchema{ + name: "ksm-container", + metrics: func() []string { return ksmContainerBucket }, + schema: otelmetrics.GroupSchema{ + Resource: otelmetrics.KeySchema{ + Required: otelmetrics.ConcatKeys(ksmCommonKeys, ksmPodKeys, []string{ + "k8s.container.name", // ksm_promote; protected + "k8s.node.name", // k8sattributes/cw_k8s_ci_v0_pod; container status implies a bound pod + "k8s.node.uid", // k8sattributes/cw_k8s_ci_v0_node + }), + Expected: otelmetrics.ConcatKeys([]string{ksmZoneKey}, otelmetrics.HostKeys, ksmOwnerExpected), + Optional: ksmOwnerOptional, + OpenPrefixes: []string{"k8s.node.label.", "k8s.pod.label."}, + Implies: otelmetrics.MergeImplies(ksmHostImplies, ksmZoneImplies, otelmetrics.WorkloadImplies()), + }, + Instrumentation: scopeInstrumentation, + }, + }, + ksmWorkloadSchema("ksm-deployment", ksmWorkloadBucket.Deployment, "k8s.deployment.name"), + ksmWorkloadSchema("ksm-daemonset", ksmWorkloadBucket.DaemonSet, "k8s.daemonset.name"), + ksmWorkloadSchema("ksm-statefulset", ksmWorkloadBucket.StatefulSet, "k8s.statefulset.name"), + ksmWorkloadSchema("ksm-job", ksmWorkloadBucket.Job, "k8s.job.name"), + ksmWorkloadSchema("ksm-cronjob", ksmWorkloadBucket.CronJob, "k8s.cronjob.name"), + groupSchema{ + name: "ksm-cluster", + metrics: func() []string { return ksmClusterBucket }, + schema: otelmetrics.GroupSchema{ + Resource: otelmetrics.KeySchema{ + Required: otelmetrics.ConcatKeys(ksmCommonKeys, []string{ + "k8s.namespace.name", // ksm_promote; protected + }), + }, + Instrumentation: scopeInstrumentation, + }, + }, + ) +} diff --git a/test/otel/standard/schema_node_test.go b/test/otel/standard/schema_node_test.go new file mode 100644 index 000000000..2fa8d954f --- /dev/null +++ b/test/otel/standard/schema_node_test.go @@ -0,0 +1,94 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package standard + +import "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" + +// Rows for the node agent's cadvisor and kubeletstats pipelines, one per series +// scope because each scope carries a different set of resource keys. +func init() { + labels := []string{"k8s.node.label.", "k8s.pod.label."} + // The agent DaemonSet runs a pod on every node, so its owner keys are on some series of + // every pod-level row. The other kinds depend on the cluster's workloads. + ownerExpected := otelmetrics.ConcatKeys([]string{"k8s.daemonset.name"}, otelmetrics.WorkloadKeys) + ownerOptional := otelmetrics.Without(otelmetrics.OwnerKindKeys, "k8s.daemonset.name") + podSchema := func(required []string) otelmetrics.KeySchema { + return otelmetrics.KeySchema{ + Required: required, + Expected: ownerExpected, + Optional: ownerOptional, + OpenPrefixes: labels, + Implies: otelmetrics.WorkloadImplies(), + } + } + + cadvisorKeys := otelmetrics.ConcatKeys(otelmetrics.PrometheusReceiverKeys, otelmetrics.CloudKeys, otelmetrics.HostKeys, []string{ + "k8s.cluster.name", // set_cluster_name; protected + "k8s.namespace.name", // cadvisor_promote; protected + "k8s.node.name", // set_node_name, promote_node_name; protected + "k8s.node.uid", // k8sattributes/cw_k8s_ci_v0_node + "k8s.pod.name", // cadvisor_promote; protected + "k8s.pod.uid", // k8sattributes/cw_k8s_ci_v0_pod; protected + }) + kubeletstatsPodKeys := otelmetrics.ConcatKeys(otelmetrics.CloudKeys, otelmetrics.HostKeys, []string{ + "k8s.cluster.name", // set_cluster_name; protected + "k8s.namespace.name", // kubeletstats/cw_k8s_ci_v0; protected + "k8s.node.name", // k8sattributes/cw_k8s_ci_v0_pod; protected + "k8s.node.uid", // k8sattributes/cw_k8s_ci_v0_node + "k8s.pod.name", // kubeletstats/cw_k8s_ci_v0; protected + "k8s.pod.uid", // kubeletstats/cw_k8s_ci_v0, k8sattributes/cw_k8s_ci_v0_pod; protected + }) + + attributeSchemas = append(attributeSchemas, + groupSchema{ + name: "cadvisor-container", + metrics: func() []string { return metricNamesByScope(cadvisorMetrics, otelmetrics.ScopeContainer) }, + schema: otelmetrics.GroupSchema{ + Resource: podSchema(otelmetrics.ConcatKeys(cadvisorKeys, []string{ + "k8s.container.name", // cadvisor_promote; cadvisor_rollup drops series without it; protected + })), + Instrumentation: scopeInstrumentation, + }, + }, + groupSchema{ + // container_network_* is sandbox-only and arrives without `container`, so + // cadvisor_promote never sets k8s.container.name. + name: "cadvisor-pod", + metrics: func() []string { return metricNamesByScope(cadvisorMetrics, otelmetrics.ScopePod) }, + schema: otelmetrics.GroupSchema{Resource: podSchema(cadvisorKeys), Instrumentation: scopeInstrumentation}, + }, + groupSchema{ + name: "kubeletstats-node", + metrics: func() []string { return metricNames(kubeletstatsNodeMetrics) }, + schema: otelmetrics.GroupSchema{ + Resource: otelmetrics.KeySchema{ + Required: otelmetrics.ConcatKeys(otelmetrics.CloudKeys, otelmetrics.HostKeys, []string{ + "k8s.cluster.name", // set_cluster_name; protected + "k8s.node.name", // kubeletstats/cw_k8s_ci_v0; protected + "k8s.node.uid", // k8sattributes/cw_k8s_ci_v0_node + }), + OpenPrefixes: []string{"k8s.node.label."}, + }, + Instrumentation: scopeInstrumentation, + }, + }, + groupSchema{ + name: "kubeletstats-pod", + metrics: func() []string { return metricNames(kubeletstatsPodMetrics) }, + schema: otelmetrics.GroupSchema{Resource: podSchema(kubeletstatsPodKeys), Instrumentation: scopeInstrumentation}, + }, + groupSchema{ + name: "kubeletstats-container", + metrics: func() []string { return metricNames(kubeletstatsContainerMetrics) }, + schema: otelmetrics.GroupSchema{ + Resource: podSchema(otelmetrics.ConcatKeys(kubeletstatsPodKeys, []string{ + "k8s.container.name", // kubeletstats/cw_k8s_ci_v0; protected + })), + Instrumentation: scopeInstrumentation, + }, + }, + ) +} diff --git a/test/otel/standard/schema_test.go b/test/otel/standard/schema_test.go new file mode 100644 index 000000000..bf806c28b --- /dev/null +++ b/test/otel/standard/schema_test.go @@ -0,0 +1,132 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package standard + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +// --------------------------------------------------------------------------- +// TestAttributeSchema — the closed set of resource and instrumentation-scope +// attributes each metric group carries. Every series must have every required +// key and nothing undeclared, so an attribute removed by mistake fails here +// even though no test names it, and so does one that appears unnoticed. Node and +// pod labels are open by construction (k8sattributes extracts "(.*)") and are +// covered by TestNodeLabelsReachMetrics and TestPodLabelsReachMetrics instead. +// +// Every key carries the chart component that sets it. Required keys are on every +// series, expected keys on at least one series of the row, and an implication +// (otelmetrics.KeySchema.Implies) ties keys that are set together, so removing a +// key family from all series fails even where each series may lack it. +// Datapoint scope is not covered: its keys come from the scraped exporter and +// move with its version. Every series in the cluster is read through +// allNodesCache, so the clusterHostTypes pin does not limit the check. +// +// An intentional attribute change edits the row in the same change. To add a +// key, list it as optional, merge the chart change, then make it required. Keys +// marked "protected" are on the attribute-limit processor's never-drop list. +// --------------------------------------------------------------------------- + +type groupSchema struct { + name string + metrics func() []string + schema otelmetrics.GroupSchema +} + +var scopeInstrumentation = otelmetrics.KeySchema{Required: otelmetrics.ScopeKeys} + +var attributeSchemas = []groupSchema{ + { + name: "apiserver", + metrics: func() []string { return metricNames(controlPlaneMetrics) }, + schema: otelmetrics.GroupSchema{ + Resource: otelmetrics.KeySchema{Required: otelmetrics.ConcatKeys(otelmetrics.PrometheusReceiverKeys, otelmetrics.CloudKeys, []string{ + "k8s.cluster.name", // set_cluster_name; protected + "k8s.component.name", // set_component, promote_component; protected + "k8s.namespace.name", // Prometheus receiver, from Kubernetes service discovery; protected + })}, + Instrumentation: scopeInstrumentation, + }, + }, + { + name: "node-exporter", + metrics: func() []string { return metricNames(nodeExporterMetrics) }, + schema: otelmetrics.GroupSchema{ + Resource: otelmetrics.KeySchema{ + Required: otelmetrics.ConcatKeys(otelmetrics.PrometheusReceiverKeys, otelmetrics.CloudKeys, otelmetrics.HostKeys, []string{ + "k8s.cluster.name", // set_cluster_name; protected + "k8s.node.name", // set_node_name, promote_node_name; protected + "k8s.node.uid", // k8sattributes/cw_k8s_ci_v0_node + }), + OpenPrefixes: []string{"k8s.node.label."}, + }, + Instrumentation: scopeInstrumentation, + }, + }, +} + +func TestAttributeSchema(t *testing.T) { + for _, g := range attributeSchemas { + g := g + t.Run(g.name, func(t *testing.T) { + deleted := deletedNodes(t) + tally, err := otelmetrics.TallySchema(context.Background(), allNodesCache, g.metrics(), g.schema, deleted.forgive) + require.NoError(t, err) + t.Logf("%d series across %d metrics; %d series of deleted nodes forgiven a missing k8s.node.uid", + tally.Series, len(g.metrics()), deleted.forgiven) + require.NotZero(t, tally.Series, "no series checked") + for _, p := range tally.Problems() { + t.Errorf("%s: add or change the %q row if intended", p, g.name) + } + }) + } +} + +// deletedNodeForgiveness forgives the one problem a deleted node causes: +// k8sattributes forgets the node, so k8s.node.uid is no longer set, while the +// node's last series stay inside the query lookback window. The API is read only +// when a series lacks k8s.node.uid. +type deletedNodeForgiveness struct { + t *testing.T + live map[string]bool + forgiven int +} + +func deletedNodes(t *testing.T) *deletedNodeForgiveness { + return &deletedNodeForgiveness{t: t} +} + +func (d *deletedNodeForgiveness) forgive(l otelmetrics.MetricLabels, c otelmetrics.SeriesCheck) otelmetrics.SeriesCheck { + node := l.Resource["k8s.node.name"] + if _, hasUID := l.Resource["k8s.node.uid"]; hasUID || node == "" { + return c + } + if d.live == nil { + d.live = map[string]bool{} + for name := range freshGroundTruth(d.t).nodes { + d.live[name] = true + } + } + if d.live[node] { + return c + } + d.forgiven++ + c.Missing = otelmetrics.Without(c.Missing, "k8s.node.uid") + var broken []string + for _, b := range c.Broken { + if !strings.HasSuffix(b, " without k8s.node.uid") { + broken = append(broken, b) + } + } + c.Broken = broken + return c +} diff --git a/test/otel/standard/setup_test.go b/test/otel/standard/setup_test.go index d2ce66762..b751201f9 100644 --- a/test/otel/standard/setup_test.go +++ b/test/otel/standard/setup_test.go @@ -24,6 +24,8 @@ var ( cfg otelmetrics.TestConfig client *otelmetrics.OtelMetricsClient queryCache *otelmetrics.QueryCache + // allNodesCache reads node-agent metrics from every node, for checks that must hold on all of them. + allNodesCache *otelmetrics.QueryCache ) func TestMain(m *testing.M) { @@ -79,6 +81,10 @@ func TestMain(m *testing.M) { os.Exit(1) } + fmt.Fprintf(os.Stderr, "Host types: %v (set %s to override)\n", clusterHostTypes, otelmetrics.HostTypesEnvVar) + fmt.Fprintf(os.Stderr, "Label excludes: pod %v, node %v (set %s and %s to the cluster's filters.metrics.podLabels.exclude and nodeLabels.exclude)\n", + clusterLabelExcludes.Pod, clusterLabelExcludes.Node, otelmetrics.PodLabelExcludesEnvVar, otelmetrics.NodeLabelExcludesEnvVar) + hostMappings := []otelmetrics.SourceHostMapping{ {Source: otelmetrics.SourceNodeExporter, HostTypes: clusterHostTypes}, {Source: otelmetrics.SourceCadvisor, HostTypes: clusterHostTypes}, @@ -88,19 +94,26 @@ func TestMain(m *testing.M) { {Source: otelmetrics.SourceKSMNodeScoped, HostTypes: nil}, } - registry := otelmetrics.NewSourceRegistry(clusterHostTypes, hostMappings, - otelmetrics.SourceMapping{Source: otelmetrics.SourceNodeExporter, Metrics: nodeExporterMetrics}, - otelmetrics.SourceMapping{Source: otelmetrics.SourceCadvisor, Metrics: cadvisorMetrics}, - otelmetrics.SourceMapping{Source: otelmetrics.SourceKubeletstats, Metrics: kubeletstatsMetrics}, - otelmetrics.SourceMapping{Source: otelmetrics.SourceControlPlane, Metrics: controlPlaneMetrics}, - otelmetrics.SourceMapping{Source: otelmetrics.SourceKubeStateMetrics, Metrics: ksmClusterScopedMetrics}, - otelmetrics.SourceMapping{Source: otelmetrics.SourceKSMNodeScoped, Metrics: ksmNodeScopedMetrics}, - ) + sourceMappings := []otelmetrics.SourceMapping{ + {Source: otelmetrics.SourceNodeExporter, Metrics: nodeExporterMetrics}, + {Source: otelmetrics.SourceCadvisor, Metrics: cadvisorMetrics}, + {Source: otelmetrics.SourceKubeletstats, Metrics: kubeletstatsMetrics}, + {Source: otelmetrics.SourceControlPlane, Metrics: controlPlaneMetrics}, + {Source: otelmetrics.SourceKubeStateMetrics, Metrics: ksmClusterScopedMetrics}, + {Source: otelmetrics.SourceKSMNodeScoped, Metrics: ksmNodeScopedMetrics}, + } + registry := otelmetrics.NewSourceRegistry(clusterHostTypes, hostMappings, sourceMappings...) queryCache = otelmetrics.NewQueryCache(client, cfg.ClusterName, otelmetrics.WithHostTypes(clusterHostTypes), otelmetrics.WithSourceRegistry(registry), ) + // Same routing with no host types: node-agent metrics span every node, cluster-scoped ones are unchanged. + // It shares queryCache's concurrency limit, so the two caches do not double the load on the endpoint. + allNodesCache = otelmetrics.NewQueryCache(client, cfg.ClusterName, + otelmetrics.WithSourceRegistry(otelmetrics.NewSourceRegistry(nil, nil, sourceMappings...)), + otelmetrics.WithSharedConcurrency(queryCache), + ) // Gate the suite on the agent's Lease-based node-metadata enrichment being // warm before any cached KSM node query is issued. See waitForKSMNodeEnrichment. diff --git a/util/otelmetrics/attribute_schema.go b/util/otelmetrics/attribute_schema.go new file mode 100644 index 000000000..ceb1ae442 --- /dev/null +++ b/util/otelmetrics/attribute_schema.go @@ -0,0 +1,196 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package otelmetrics + +import ( + "fmt" + "sort" + "strings" +) + +// KeySchema is the closed set of attribute keys one scope of a metric group +// carries. Required keys must be on every series. Expected keys may be absent +// from a series but must be on at least one series of the group, so a change +// that removes them everywhere fails. Optional keys may be absent everywhere. +// Keys under an open prefix (for example "k8s.node.label.") are allowed without +// being listed, because their names come from the cluster rather than the pipeline. +type KeySchema struct { + Required []string + Expected []string + Optional []string + OpenPrefixes []string + // Implies maps a key to keys that every series carrying it must also carry, + // so losing part of a family that is set together fails. + Implies map[string][]string +} + +// SeriesCheck is the result of checking one scope of one series. +type SeriesCheck struct { + Missing []string // required keys absent + Unexpected []string // keys the schema does not allow + Broken []string // "a without b" for each implication that does not hold +} + +// OK reports whether the series matched the schema. +func (c SeriesCheck) OK() bool { + return len(c.Missing) == 0 && len(c.Unexpected) == 0 && len(c.Broken) == 0 +} + +// Check checks one series. Keys starting with "@" are structural (the scope name +// and version) rather than attributes and are ignored. +func (s KeySchema) Check(observed map[string]string) SeriesCheck { + var c SeriesCheck + allowed := make(map[string]struct{}, len(s.Required)+len(s.Expected)+len(s.Optional)) + for _, k := range s.Required { + allowed[k] = struct{}{} + if _, ok := observed[k]; !ok { + c.Missing = append(c.Missing, k) + } + } + for _, k := range concatStrings(s.Expected, s.Optional) { + allowed[k] = struct{}{} + } + for k := range observed { + if strings.HasPrefix(k, "@") { + continue + } + if _, ok := allowed[k]; ok || hasAnyPrefix(k, s.OpenPrefixes) { + continue + } + c.Unexpected = append(c.Unexpected, k) + } + for k, implied := range s.Implies { + if _, ok := observed[k]; !ok { + continue + } + for _, want := range implied { + if _, ok := observed[want]; !ok { + c.Broken = append(c.Broken, k+" without "+want) + } + } + } + sort.Strings(c.Missing) + sort.Strings(c.Unexpected) + sort.Strings(c.Broken) + return c +} + +// Together returns implications under which the keys are all present or all absent. +func Together(keys ...string) map[string][]string { + out := make(map[string][]string, len(keys)) + for _, k := range keys { + for _, other := range keys { + if other != k { + out[k] = append(out[k], other) + } + } + } + return out +} + +// MergeImplies combines implication maps. +func MergeImplies(maps ...map[string][]string) map[string][]string { + out := map[string][]string{} + for _, m := range maps { + for k, v := range m { + out[k] = append(out[k], v...) + } + } + return out +} + +// GroupSchema is the attribute schema of one metric group. Datapoint scope is not +// covered: its keys come from the scraped exporter and move with its version. +type GroupSchema struct { + Resource KeySchema + Instrumentation KeySchema +} + +// SchemaTally counts schema problems over the series of one metric group. +type SchemaTally struct { + schema GroupSchema + Series int + counts map[string]int // series count per problem + seen map[string]bool // "@resource." and "@instrumentation." seen on a recorded series +} + +func NewSchemaTally(schema GroupSchema) *SchemaTally { + return &SchemaTally{schema: schema, counts: map[string]int{}, seen: map[string]bool{}} +} + +// Check checks one series without recording it. +func (t *SchemaTally) Check(l MetricLabels) (resource, instrumentation SeriesCheck) { + return t.schema.Resource.Check(l.Resource), t.schema.Instrumentation.Check(l.Instrumentation) +} + +// Record adds one series and the result of Check for it. +func (t *SchemaTally) Record(l MetricLabels, resource, instrumentation SeriesCheck) { + t.Series++ + for scope, c := range map[string]SeriesCheck{"@resource.": resource, "@instrumentation.": instrumentation} { + for _, k := range c.Missing { + t.counts["missing required "+scope+k]++ + } + for _, k := range c.Unexpected { + t.counts["undeclared "+scope+k]++ + } + for _, b := range c.Broken { + t.counts[scope+strings.Replace(b, " without ", " without "+scope, 1)]++ + } + } + for k := range l.Resource { + t.seen["@resource."+k] = true + } + for k := range l.Instrumentation { + t.seen["@instrumentation."+k] = true + } +} + +// Problems describes every problem with the number of series it affects, then +// every Expected key that no recorded series carried. It is empty when the group passed. +func (t *SchemaTally) Problems() []string { + var out []string + for _, p := range SortedKeys(t.counts) { + out = append(out, fmt.Sprintf("%s on %d of %d series", p, t.counts[p], t.Series)) + } + var unseen []string + for scope, s := range map[string]KeySchema{"@resource.": t.schema.Resource, "@instrumentation.": t.schema.Instrumentation} { + for _, k := range s.Expected { + if !t.seen[scope+k] { + unseen = append(unseen, scope+k) + } + } + } + sort.Strings(unseen) + for _, k := range unseen { + out = append(out, fmt.Sprintf("expected %s on at least one of %d series, found on none", k, t.Series)) + } + return out +} + +// SortedKeys returns the keys of m in order. +func SortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func concatStrings(groups ...[]string) []string { + var out []string + for _, g := range groups { + out = append(out, g...) + } + return out +} + +func hasAnyPrefix(s string, prefixes []string) bool { + for _, p := range prefixes { + if strings.HasPrefix(s, p) { + return true + } + } + return false +} diff --git a/util/otelmetrics/attribute_schema_test.go b/util/otelmetrics/attribute_schema_test.go new file mode 100644 index 000000000..967a53554 --- /dev/null +++ b/util/otelmetrics/attribute_schema_test.go @@ -0,0 +1,133 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package otelmetrics + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestKeySchemaCheck(t *testing.T) { + s := KeySchema{ + Required: []string{"k8s.cluster.name", "server.port"}, + Optional: []string{"k8s.node.uid"}, + OpenPrefixes: []string{"k8s.node.label."}, + } + + t.Run("exact set passes", func(t *testing.T) { + require.True(t, s.Check(map[string]string{"k8s.cluster.name": "c", "server.port": "9100"}).OK()) + }) + + t.Run("optional and open-prefix keys are allowed", func(t *testing.T) { + require.True(t, s.Check(map[string]string{ + "k8s.cluster.name": "c", "server.port": "9100", + "k8s.node.uid": "u", "k8s.node.label.kubernetes.io/os": "linux", + }).OK()) + }) + + t.Run("a required key deleted by mistake is missing", func(t *testing.T) { + c := s.Check(map[string]string{"k8s.cluster.name": "c", "net.host.port": "9100"}) + require.Equal(t, []string{"server.port"}, c.Missing) + }) + + t.Run("an undeclared key is unexpected", func(t *testing.T) { + c := s.Check(map[string]string{"k8s.cluster.name": "c", "server.port": "9100", "net.host.port": "9100"}) + require.Equal(t, []string{"net.host.port"}, c.Unexpected) + }) + + t.Run("a key under another prefix is not open", func(t *testing.T) { + c := s.Check(map[string]string{"k8s.cluster.name": "c", "server.port": "9100", "k8s.pod.label.app": "x"}) + require.Equal(t, []string{"k8s.pod.label.app"}, c.Unexpected) + }) + + t.Run("scope name and version are not attributes", func(t *testing.T) { + require.True(t, s.Check(map[string]string{ + "k8s.cluster.name": "c", "server.port": "9100", "@name": "scope", "@version": "1", + }).OK()) + }) +} + +func TestKeySchemaImplies(t *testing.T) { + s := KeySchema{ + Optional: []string{"k8s.deployment.name", "k8s.workload.name", "k8s.workload.type"}, + Implies: MergeImplies( + map[string][]string{"k8s.deployment.name": {"k8s.workload.name", "k8s.workload.type"}}, + Together("k8s.workload.name", "k8s.workload.type"), + ), + } + require.True(t, s.Check(map[string]string{}).OK(), "a bare pod has none of them") + require.True(t, s.Check(map[string]string{ + "k8s.deployment.name": "d", "k8s.workload.name": "d", "k8s.workload.type": "Deployment", + }).OK()) + require.Equal(t, []string{"k8s.deployment.name without k8s.workload.name", "k8s.deployment.name without k8s.workload.type"}, + s.Check(map[string]string{"k8s.deployment.name": "d"}).Broken, "set_workload removed") + require.Equal(t, []string{"k8s.workload.name without k8s.workload.type"}, + s.Check(map[string]string{"k8s.workload.name": "d"}).Broken) +} + +func TestSchemaTally(t *testing.T) { + schema := GroupSchema{ + Resource: KeySchema{ + Required: []string{"k8s.pod.name"}, + Expected: []string{"k8s.node.name"}, + Implies: Together("host.id", "host.name"), + Optional: []string{"host.id", "host.name"}, + }, + Instrumentation: KeySchema{Required: []string{"cloudwatch.source"}}, + } + record := func(tally *SchemaTally, resource map[string]string) { + l := MetricLabels{Resource: resource, Instrumentation: map[string]string{"cloudwatch.source": "s"}} + r, i := tally.Check(l) + tally.Record(l, r, i) + } + + ok := NewSchemaTally(schema) + record(ok, map[string]string{"k8s.pod.name": "p1"}) // unscheduled + record(ok, map[string]string{"k8s.pod.name": "p2", "k8s.node.name": "n"}) + require.Empty(t, ok.Problems()) + + bad := NewSchemaTally(schema) + record(bad, map[string]string{"k8s.pod.name": "p1", "host.id": "i"}) + record(bad, map[string]string{"extra": "x"}) + require.Equal(t, []string{ + "@resource.host.id without @resource.host.name on 1 of 2 series", + "missing required @resource.k8s.pod.name on 1 of 2 series", + "undeclared @resource.extra on 1 of 2 series", + "expected @resource.k8s.node.name on at least one of 2 series, found on none", + }, bad.Problems(), "node keys removed from every series fail even though each series may lack them") +} + +func TestTallySchemaChecksEveryHostType(t *testing.T) { + series := func(hostType string, extra ...string) fakeSeries { + l := map[string]string{ + "__name__": "m", "@resource.k8s.cluster.name": "c1", + "@resource.host.type": hostType, "@resource.k8s.pod.name": "p-" + hostType, + } + for _, k := range extra { + l["@resource."+k] = "x" + } + return fakeSeries{labels: l} + } + f, c := newFake(t, 100, []fakeSeries{series("a"), series("b", "leaked")}) + schema := GroupSchema{ + Resource: KeySchema{Required: []string{"k8s.cluster.name", "k8s.pod.name"}, Optional: []string{"host.type"}}, + } + + pinned := NewQueryCache(c, "c1", WithHostTypes([]string{"a"})) + _, err := TallySchema(context.Background(), pinned, []string{"m"}, schema, nil) + require.ErrorContains(t, err, "pass a cache without host types", "a pinned cache would hide the series of host type b") + + allNodes := NewQueryCache(c, "c1") + tally, err := TallySchema(context.Background(), allNodes, []string{"m"}, schema, nil) + require.NoError(t, err) + require.Equal(t, []string{"undeclared @resource.leaked on 1 of 2 series"}, tally.Problems()) + + requests := len(f.queries()) + results, err := allNodes.Get(context.Background(), "m") + require.NoError(t, err) + require.Len(t, results, 2) + require.Len(t, f.queries(), requests, "other every-series checks reuse the read of the schema row") +} diff --git a/util/otelmetrics/client.go b/util/otelmetrics/client.go index c9315830b..6d308bdbe 100644 --- a/util/otelmetrics/client.go +++ b/util/otelmetrics/client.go @@ -12,6 +12,7 @@ import ( "net/http" "net/url" "strconv" + "strings" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -39,11 +40,13 @@ type OtelMetricsClient struct { region string signingService string maxRetries int + seriesLimit int // endpoint's per-query series cap; 0 means defaultSeriesLimit } type promqlResponse struct { - Status string `json:"status"` - Data promqlData `json:"data"` + Status string `json:"status"` + Data promqlData `json:"data"` + Warnings []string `json:"warnings"` } type promqlData struct { @@ -74,18 +77,43 @@ func NewClient(ctx context.Context, config TestConfig) (*OtelMetricsClient, erro }, nil } -// Query executes a PromQL instant query and returns parsed results. +// Query executes a single PromQL instant query and returns parsed results. The endpoint caps +// results at 500 series, often without saying so; use QueryComplete when every series matters. func (c *OtelMetricsClient) Query(ctx context.Context, promql string) ([]MetricResult, error) { + results, truncated, err := c.queryAt(ctx, promql, "") + if truncated { + log.Printf("WARNING: result truncated by the endpoint series limit query=%s", promql) + } + return results, err +} + +// queryAt runs one instant query, evaluated at evalTime when non-empty. It also returns +// whether the endpoint warned of truncation. +func (c *OtelMetricsClient) queryAt(ctx context.Context, promql, evalTime string) ([]MetricResult, bool, error) { params := url.Values{"query": {promql}} + if evalTime != "" { + params.Set("time", evalTime) + } log.Printf("querying promql=%s", promql) raw, err := c.requestWithRetry(ctx, c.queryURL, params) if err != nil { - return nil, err + return nil, false, err } results := c.parseResponse(raw) log.Printf("query returned count=%d", len(results)) - return results, nil + return results, hasTruncationWarning(raw.Warnings), nil +} + +// hasTruncationWarning reports the endpoint's "results truncated due to series limit" warning. +// Aggregations carry it reliably; plain selectors only sometimes, so it is never proof of completeness. +func hasTruncationWarning(warnings []string) bool { + for _, w := range warnings { + if strings.Contains(strings.ToLower(w), "truncated") { + return true + } + } + return false } // MetricNames returns the distinct metric names present for the given PromQL matcher over [start, end]. @@ -143,8 +171,13 @@ func (c *OtelMetricsClient) requestRawWithRetry(ctx context.Context, baseURL str return nil, fmt.Errorf("signing request: %w", err) } + release, err := acquireRequestSlot(ctx) + if err != nil { + return nil, err + } resp, err := c.httpClient.Do(req) if err != nil { + release() lastErr = err log.Printf("request failed attempt=%d error=%v", attempt+1, err) if attempt < c.maxRetries-1 { @@ -155,6 +188,7 @@ func (c *OtelMetricsClient) requestRawWithRetry(ctx context.Context, baseURL str body, readErr := io.ReadAll(resp.Body) resp.Body.Close() + release() if readErr != nil { return nil, fmt.Errorf("reading response: %w", readErr) } @@ -164,7 +198,7 @@ func (c *OtelMetricsClient) requestRawWithRetry(ctx context.Context, baseURL str lastErr = fmt.Errorf("server error %d: %s", resp.StatusCode, truncate(string(body), 200)) log.Printf("server error, retrying status=%d attempt=%d", resp.StatusCode, attempt+1) if attempt < c.maxRetries-1 { - time.Sleep(time.Duration(1<= 0 { + d = time.Duration(secs) * time.Second + } else if at, err := http.ParseTime(retryAfter); err == nil { + d = at.Sub(now) + } + if d < 0 { + d = 0 + } + if d > maxRetryAfter { + d = maxRetryAfter + } + return d +} + +type requestSlotsKey struct{} + +// withRequestSlots makes every request issued under ctx hold one slot of sem while it is in flight. +func withRequestSlots(ctx context.Context, sem chan struct{}) context.Context { + return context.WithValue(ctx, requestSlotsKey{}, sem) +} + +func acquireRequestSlot(ctx context.Context) (release func(), err error) { + sem, _ := ctx.Value(requestSlotsKey{}).(chan struct{}) + if sem == nil { + return func() {}, nil + } + select { + case sem <- struct{}{}: + return func() { <-sem }, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + func (c *OtelMetricsClient) parseResponse(response *promqlResponse) []MetricResult { var results []MetricResult for _, series := range response.Data.Result { @@ -242,8 +316,9 @@ type RangeResult struct { } type promqlRangeResponse struct { - Status string `json:"status"` - Data promqlRangeData `json:"data"` + Status string `json:"status"` + Data promqlRangeData `json:"data"` + Warnings []string `json:"warnings"` } type promqlRangeData struct { @@ -275,6 +350,9 @@ func (c *OtelMetricsClient) QueryRange(ctx context.Context, promql string, start if err := json.Unmarshal(rawBytes, &rangeResp); err != nil { return nil, fmt.Errorf("parsing range response: %w", err) } + if hasTruncationWarning(rangeResp.Warnings) { + log.Printf("WARNING: range result truncated by the endpoint series limit query=%s", promql) + } var results []RangeResult for _, series := range rangeResp.Data.Result { diff --git a/util/otelmetrics/client_test.go b/util/otelmetrics/client_test.go new file mode 100644 index 000000000..811141381 --- /dev/null +++ b/util/otelmetrics/client_test.go @@ -0,0 +1,58 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package otelmetrics + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRetryDelay(t *testing.T) { + now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC) + for _, tc := range []struct { + header string + attempt int + want time.Duration + }{ + {"", 0, time.Second}, + {"", 2, 4 * time.Second}, + {"not-a-delay", 1, 2 * time.Second}, + {"0", 2, 0}, + {"5", 0, 5 * time.Second}, + {"120", 0, maxRetryAfter}, + {now.Add(3 * time.Second).Format(http.TimeFormat), 0, 3 * time.Second}, + {now.Add(-time.Minute).Format(http.TimeFormat), 0, 0}, + } { + assert.Equal(t, tc.want, retryDelay(tc.header, tc.attempt, now), "Retry-After %q attempt %d", tc.header, tc.attempt) + } +} + +func TestQuery_ThrottledRequestWaitsForRetryAfter(t *testing.T) { + var calls int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if atomic.AddInt32(&calls, 1) == 1 { + w.Header().Set("Retry-After", "0") + http.Error(w, "throttled", http.StatusTooManyRequests) + return + } + fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`) + })) + defer srv.Close() + c := newTestClient(srv.URL) + c.maxRetries = 2 + + start := time.Now() + _, err := c.Query(context.Background(), "up") + require.NoError(t, err) + assert.EqualValues(t, 2, atomic.LoadInt32(&calls)) + assert.Less(t, time.Since(start), 500*time.Millisecond, "Retry-After: 0 replaces the 1 s backoff") +} diff --git a/util/otelmetrics/complete.go b/util/otelmetrics/complete.go new file mode 100644 index 000000000..1fae4ccec --- /dev/null +++ b/util/otelmetrics/complete.go @@ -0,0 +1,370 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package otelmetrics + +import ( + "context" + "errors" + "fmt" + "log" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +// ErrTruncated reports that the endpoint dropped series to stay under its per-query +// series limit and the complete result could not be recovered. +var ErrTruncated = errors.New("query result truncated by the endpoint series limit") + +const ( + // defaultSeriesLimit is the endpoint's per-query cap ("series limit (500)" in its warnings). + defaultSeriesLimit = 500 + // maxMatcherValueLen is the endpoint's limit on one matcher value, which bounds a shard's regex. + maxMatcherValueLen = 1000 + sampleSize = 50 + maxShardDepth = 4 + maxSplitCandidates = 3 + shardConcurrency = 4 + nodeNameLabel = "@resource.k8s.node.name" + // evalTimeLag keeps the evaluation time that the client picks behind the endpoint's clock + // and its newest samples. + evalTimeLag = 5 * time.Second +) + +var metricNameRE = regexp.MustCompile(`^[a-zA-Z_:][a-zA-Z0-9_:]*$`) + +// QueryComplete returns every series matched by an instant vector selector. The endpoint caps +// selector results and does not reliably say so, so the result is compared with count() at the +// same evaluation time and, only when short, re-fetched as disjoint shards (by node first). +// The client picks that time, so the selector and its count run at once. +// Anything that is not a plain vector selector is passed to Query unchanged. +func (c *OtelMetricsClient) QueryComplete(ctx context.Context, selector string) ([]MetricResult, error) { + if !isVectorSelector(selector) { + return c.Query(ctx, selector) + } + evalTime := evalTimeAt(time.Now()) + var before int + var countErr error + counted := make(chan struct{}) + go func() { + defer close(counted) + before, countErr = c.countAt(ctx, selector, evalTime) + }() + results, _, err := c.queryAt(ctx, selector, evalTime) + <-counted + if err != nil { + return nil, err + } + if countErr != nil { + return nil, countErr + } + if len(results) >= before { + return results, nil + } + log.Printf("result truncated: got %d of %d series, re-fetching in shards query=%s", len(results), before, selector) + + all, err := c.split(ctx, selector, evalTime, before, results, nil, 1) + if err != nil { + return nil, err + } + after, err := c.countAt(ctx, selector, evalTime) + if err != nil { + return nil, err + } + lo, hi := before, after + if lo > hi { + lo, hi = hi, lo + } + if len(all) < lo || len(all) > hi { + return nil, fmt.Errorf("%w: shards returned %d series but count() was %d before and %d after (query %s)", + ErrTruncated, len(all), before, after, selector) + } + log.Printf("sharded fetch complete: %d series query=%s", len(all), selector) + return all, nil +} + +type shardGroup struct { + value string // "" when the label is absent + count int +} + +type shard struct { + matcher string + want int + exact bool // the shard pins the label to one value (or to absent) +} + +// evalTimeAt is the evaluation time, in Unix seconds, that QueryComplete pins its queries to. +func evalTimeAt(now time.Time) string { + return strconv.FormatInt(now.Add(-evalTimeLag).Unix(), 10) +} + +// countAt returns count(selector) at evalTime. +func (c *OtelMetricsClient) countAt(ctx context.Context, selector, evalTime string) (int, error) { + groups, err := c.countBy(ctx, selector, "", evalTime) + if err != nil { + return 0, err + } + return groupTotal(groups), nil +} + +func groupTotal(groups []shardGroup) int { + total := 0 + for _, g := range groups { + total += g.count + } + return total +} + +// countBy returns the series count per value of label (or the total when label is ""). +// It fails with ErrTruncated when the label has too many values to list in one response. +func (c *OtelMetricsClient) countBy(ctx context.Context, selector, label, evalTime string) ([]shardGroup, error) { + promql := "count(" + selector + ")" + if label != "" { + promql = fmt.Sprintf("count by (%s)(%s)", quoteLabel(label), selector) + } + results, truncated, err := c.queryAt(ctx, promql, evalTime) + if err != nil { + return nil, err + } + if truncated { + return nil, fmt.Errorf("%w: too many groups (query %s)", ErrTruncated, promql) + } + groups := make([]shardGroup, 0, len(results)) + for _, r := range results { + g := shardGroup{count: int(r.Value)} + if label != "" { + g.value = r.Labels.AllLabels()[label] + } + groups = append(groups, g) + } + return groups, nil +} + +// split re-fetches a truncated selector of want series as disjoint shards over the first label that divides it. +func (c *OtelMetricsClient) split(ctx context.Context, selector, evalTime string, want int, sample []MetricResult, pinned map[string]bool, depth int) ([]MetricResult, error) { + if depth > maxShardDepth { + return nil, fmt.Errorf("%w: still truncated after %d shard levels (query %s)", ErrTruncated, maxShardDepth, selector) + } + if len(sample) == 0 { + // Stale series filled the cap; topk evaluates fully, so it yields live series to inspect. + sample, _, _ = c.queryAt(ctx, fmt.Sprintf("topk(%d,%s)", sampleSize, selector), evalTime) + } + for _, label := range splitCandidates(sample, pinned) { + groups, err := c.countBy(ctx, selector, label, evalTime) + if errors.Is(err, ErrTruncated) { + continue + } + if err != nil { + return nil, err + } + if len(groups) < 2 || groupTotal(groups) < want { + continue // label does not divide the set, or its groups were cut short + } + shards := packShards(label, groups, c.shardTarget(depth)) + log.Printf("fetching %d shards by %s depth=%d query=%s", len(shards), label, depth, selector) + return c.fetchShards(ctx, selector, evalTime, label, shards, pinned, depth) + } + return nil, fmt.Errorf("%w: no label divides the result further (query %s)", ErrTruncated, selector) +} + +func (c *OtelMetricsClient) fetchShards(ctx context.Context, selector, evalTime, label string, shards []shard, pinned map[string]bool, depth int) ([]MetricResult, error) { + type outcome struct { + results []MetricResult + err error + } + outcomes := make([]outcome, len(shards)) + sem := make(chan struct{}, shardConcurrency) + var wg sync.WaitGroup + for i := range shards { + wg.Add(1) + go func(i int) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + sh := shards[i] + sub := withMatcher(selector, sh.matcher) + results, _, err := c.queryAt(ctx, sub, evalTime) + if err == nil && len(results) < sh.want { + subPinned := pinned + if sh.exact { + subPinned = withPinned(pinned, label) + } + results, err = c.split(ctx, sub, evalTime, sh.want, results, subPinned, depth+1) + } + outcomes[i] = outcome{results: results, err: err} + }(i) + } + wg.Wait() + + var all []MetricResult + for _, o := range outcomes { + if o.err != nil { + return nil, o.err + } + all = append(all, o.results...) + } + return all, nil +} + +// shardTarget is the series budget per shard: 80% of the cap, since stale series also count +// against it, halved at each level so a re-split always yields smaller shards. +func (c *OtelMetricsClient) shardTarget(depth int) int { + limit := c.seriesLimit + if limit <= 0 { + limit = defaultSeriesLimit + } + return (limit * 4 / 5) >> (depth - 1) +} + +// splitCandidates orders the labels worth splitting on: node name first, then the sample's +// labels by number of distinct values. Labels already pinned to one value are skipped. +func splitCandidates(sample []MetricResult, pinned map[string]bool) []string { + distinct := make(map[string]map[string]struct{}) + for _, r := range sample { + for k, v := range r.Labels.AllLabels() { + if distinct[k] == nil { + distinct[k] = make(map[string]struct{}) + } + distinct[k][v] = struct{}{} + } + } + var out []string + if !pinned[nodeNameLabel] && (len(sample) == 0 || distinct[nodeNameLabel] != nil) { + out = append(out, nodeNameLabel) + } + var keys []string + for k, values := range distinct { + if k != nodeNameLabel && !pinned[k] && len(values) > 1 { + keys = append(keys, k) + } + } + sort.Slice(keys, func(i, j int) bool { + if ni, nj := len(distinct[keys[i]]), len(distinct[keys[j]]); ni != nj { + return ni > nj + } + return keys[i] < keys[j] + }) + out = append(out, keys...) + if len(out) > maxSplitCandidates { + out = out[:maxSplitCandidates] + } + return out +} + +// packShards bins label values into shards of at most target series. The absent value gets +// its own !~".+" shard because ="" does not match a missing label on this endpoint. +func packShards(label string, groups []shardGroup, target int) []shard { + if target < 1 { + target = 1 + } + sorted := append([]shardGroup(nil), groups...) + sort.Slice(sorted, func(i, j int) bool { + if sorted[i].count != sorted[j].count { + return sorted[i].count > sorted[j].count + } + return sorted[i].value < sorted[j].value + }) + + var out []shard + type bin struct { + values []string + want, chars int + } + var bins []bin + for _, g := range sorted { + if g.value == "" { + out = append(out, shard{matcher: quoteLabel(label) + `!~".+"`, want: g.count, exact: true}) + continue + } + chars := len(regexp.QuoteMeta(g.value)) + placed := false + for i := range bins { + if bins[i].want+g.count <= target && bins[i].chars+1+chars <= maxMatcherValueLen { + bins[i].values = append(bins[i].values, g.value) + bins[i].want += g.count + bins[i].chars += 1 + chars + placed = true + break + } + } + if !placed { + bins = append(bins, bin{values: []string{g.value}, want: g.count, chars: chars}) + } + } + for _, b := range bins { + out = append(out, shard{matcher: valueMatcher(label, b.values), want: b.want, exact: len(b.values) == 1}) + } + return out +} + +func valueMatcher(label string, values []string) string { + if len(values) == 1 { + return fmt.Sprintf(`%s="%s"`, quoteLabel(label), promqlEscaper.Replace(values[0])) + } + alts := make([]string, len(values)) + for i, v := range values { + alts[i] = regexp.QuoteMeta(v) + } + sort.Strings(alts) + return fmt.Sprintf(`%s=~"%s"`, quoteLabel(label), promqlEscaper.Replace(strings.Join(alts, "|"))) +} + +func quoteLabel(label string) string { + return `"` + promqlEscaper.Replace(label) + `"` +} + +func withPinned(pinned map[string]bool, label string) map[string]bool { + out := make(map[string]bool, len(pinned)+1) + for k := range pinned { + out[k] = true + } + out[label] = true + return out +} + +// withMatcher adds one label matcher to a vector selector. +func withMatcher(selector, matcher string) string { + s := strings.TrimSpace(selector) + if !strings.HasSuffix(s, "}") { + return s + "{" + matcher + "}" + } + body := strings.TrimSpace(s[:len(s)-1]) + if strings.HasSuffix(body, "{") || strings.HasSuffix(body, ",") { + return body + matcher + "}" + } + return body + "," + matcher + "}" +} + +// isVectorSelector reports whether s is a single instant vector selector (name, name{...} +// or {...}) with no range, offset, function or operator around it. +func isVectorSelector(s string) bool { + s = strings.TrimSpace(s) + open := strings.IndexByte(s, '{') + if open < 0 { + return metricNameRE.MatchString(s) + } + if name := strings.TrimSpace(s[:open]); name != "" && !metricNameRE.MatchString(name) { + return false + } + var quote byte + for i := open + 1; i < len(s); i++ { + ch := s[i] + switch { + case quote != 0 && ch == '\\' && quote != '`': + i++ + case quote != 0 && ch == quote: + quote = 0 + case quote != 0: + case ch == '"' || ch == '\'' || ch == '`': + quote = ch + case ch == '}': + return i == len(s)-1 + } + } + return false +} diff --git a/util/otelmetrics/complete_test.go b/util/otelmetrics/complete_test.go new file mode 100644 index 000000000..16a94e479 --- /dev/null +++ b/util/otelmetrics/complete_test.go @@ -0,0 +1,671 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package otelmetrics + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const fakeEvalTime = "1700000000.123" + +type fakeSeries struct { + labels map[string]string + stale bool // occupies the cap like a recently ended series, but has no sample +} + +type fakeRequest struct { + query, time string +} + +// fakePromQL mimics the endpoint: plain selectors are capped at limit series (stale ones +// included), aggregations warn when capped, and `=""` matches an absent label only for emptyMatchesAbsent. +type fakePromQL struct { + t *testing.T + series []fakeSeries + limit int + countOffset int // added to count() without by, to simulate an inconsistent endpoint + brokenNotRegex bool // make !~ match everything, so shards overlap + emptyMatchesAbsent string // label whose `=""` matches absent, as @resource.host.type does on the endpoint + mu sync.Mutex + requests []fakeRequest +} + +func (f *fakePromQL) handler(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + query, ts := q.Get("query"), q.Get("time") + f.mu.Lock() + f.requests = append(f.requests, fakeRequest{query: query, time: ts}) + f.mu.Unlock() + if ts == "" { + ts = fakeEvalTime + } + + type result struct { + Metric map[string]string `json:"metric"` + Value []json.RawMessage `json:"value"` + } + resp := struct { + Status string `json:"status"` + Warnings []string `json:"warnings,omitempty"` + Data struct { + ResultType string `json:"resultType"` + Result []result `json:"result"` + } `json:"data"` + }{Status: "success"} + resp.Data.ResultType = "vector" + sample := func(metric map[string]string, v int) result { + return result{Metric: metric, Value: []json.RawMessage{json.RawMessage(ts), json.RawMessage(strconv.Quote(strconv.Itoa(v)))}} + } + + switch { + case strings.HasPrefix(query, "topk("): + m := regexp.MustCompile(`^topk\((\d+),(.*)\)$`).FindStringSubmatch(query) + if m == nil { + f.badRequest(w, "unexpected topk query %q", query) + return + } + k, _ := strconv.Atoi(m[1]) + for _, s := range f.match(m[2]) { + if !s.stale && len(resp.Data.Result) < k { + resp.Data.Result = append(resp.Data.Result, sample(s.labels, 1)) + } + } + case strings.HasPrefix(query, "count by ("): + label, inner, ok := parseCountBy(query) + if !ok { + f.badRequest(w, "unexpected count by query %q", query) + return + } + groups := map[string]int{} + for _, s := range f.match(inner) { + if !s.stale { + groups[s.labels[label]]++ + } + } + keys := make([]string, 0, len(groups)) + for k := range groups { + keys = append(keys, k) + } + sort.Strings(keys) + if len(keys) > f.limit { + keys = keys[:f.limit] + resp.Warnings = []string{fmt.Sprintf("results truncated due to series limit (%d).", f.limit)} + } + for _, k := range keys { + m := map[string]string{} + if k != "" { + m[label] = k + } + resp.Data.Result = append(resp.Data.Result, sample(m, groups[k])) + } + case strings.HasPrefix(query, "count("): + n := 0 + for _, s := range f.match(query[len("count(") : len(query)-1]) { + if !s.stale { + n++ + } + } + if n > 0 { + resp.Data.Result = append(resp.Data.Result, sample(map[string]string{}, n+f.countOffset)) + } + default: + matched := f.match(query) + capped := len(matched) > f.limit + if capped { + matched = matched[:f.limit] + } + for _, s := range matched { + if !s.stale { + resp.Data.Result = append(resp.Data.Result, sample(s.labels, 1)) + } + } + // The endpoint warns only when the capped output is full, so stale series hide it. + if capped && len(resp.Data.Result) == f.limit { + resp.Warnings = []string{fmt.Sprintf("results truncated due to series limit (%d).", f.limit)} + } + } + if err := json.NewEncoder(w).Encode(resp); err != nil { + f.t.Errorf("encoding response: %v", err) + } +} + +// badRequest reports a query the fake cannot parse; the handler runs off the test goroutine. +func (f *fakePromQL) badRequest(w http.ResponseWriter, format string, args ...interface{}) { + f.t.Errorf(format, args...) + http.Error(w, fmt.Sprintf(format, args...), http.StatusBadRequest) +} + +func (f *fakePromQL) match(selector string) []fakeSeries { + matchers, ok := parseFakeSelector(selector) + if !ok { + f.t.Errorf("unparseable selector %q", selector) + return nil + } + var out []fakeSeries + for _, s := range f.series { + ok := true + for _, m := range matchers { + v, present := s.labels[m.key] + switch m.op { + case "=": + ok = present && v == m.val || !present && m.val == "" && m.key == f.emptyMatchesAbsent + case "=~": + ok = regexp.MustCompile("^(?:" + m.val + ")$").MatchString(v) + case "!~": + ok = f.brokenNotRegex || !regexp.MustCompile("^(?:"+m.val+")$").MatchString(v) + default: + f.t.Errorf("unsupported matcher op %q", m.op) + ok = false + } + if !ok { + break + } + } + if ok { + out = append(out, s) + } + } + return out +} + +func (f *fakePromQL) queries() []fakeRequest { + f.mu.Lock() + defer f.mu.Unlock() + return append([]fakeRequest(nil), f.requests...) +} + +type fakeMatcher struct{ key, op, val string } + +func parseCountBy(q string) (string, string, bool) { + m := regexp.MustCompile(`^count by \("((?:[^"\\]|\\.)*)"\)\((.*)\)$`).FindStringSubmatch(q) + if m == nil { + return "", "", false + } + return unescape(m[1]), m[2], true +} + +func parseFakeSelector(s string) ([]fakeMatcher, bool) { + open := strings.IndexByte(s, '{') + if open < 0 { + return []fakeMatcher{{"__name__", "=", s}}, true + } + var out []fakeMatcher + if open > 0 { + out = append(out, fakeMatcher{"__name__", "=", s[:open]}) + } + re := regexp.MustCompile(`^\s*(?:"((?:[^"\\]|\\.)*)"|([a-zA-Z_][a-zA-Z0-9_]*))\s*(=~|!~|=)\s*"((?:[^"\\]|\\.)*)"\s*,?`) + body := s[open+1 : len(s)-1] + for strings.TrimSpace(body) != "" { + m := re.FindStringSubmatch(body) + if m == nil { + return nil, false + } + key := m[2] + if m[1] != "" { + key = unescape(m[1]) + } + out = append(out, fakeMatcher{key, m[3], unescape(m[4])}) + body = body[len(m[0]):] + } + return out, true +} + +func unescape(s string) string { + return strings.NewReplacer(`\\`, `\`, `\"`, `"`).Replace(s) +} + +func newFake(t *testing.T, limit int, series []fakeSeries) (*fakePromQL, *OtelMetricsClient) { + f := &fakePromQL{t: t, series: series, limit: limit} + srv := httptest.NewServer(http.HandlerFunc(f.handler)) + t.Cleanup(srv.Close) + c := newTestClient(srv.URL) + c.seriesLimit = limit + return f, c +} + +// podSeries builds kube_pod_info-like series: perNode pods on each node, plus extra. +func podSeries(nodes, perNode int) []fakeSeries { + var out []fakeSeries + for n := 0; n < nodes; n++ { + for p := 0; p < perNode; p++ { + out = append(out, fakeSeries{labels: map[string]string{ + "__name__": "kube_pod_info", + "@resource.k8s.cluster.name": "c1", + "@resource.k8s.node.name": fmt.Sprintf("ip-10-0-%d-1.ec2.internal", n), + "@resource.k8s.pod.name": fmt.Sprintf("pod-%d-%d", n, p), + "@resource.k8s.namespace.name": "default", + "@instrumentation.@name": "ksm", + "@aws.account": "123456789012", + "__type__": "Gauge", + }}) + } + } + return out +} + +const podSelector = `kube_pod_info{"@resource.k8s.cluster.name"="c1"}` + +func podNames(results []MetricResult) map[string]bool { + out := map[string]bool{} + for _, r := range results { + out[r.Labels.Resource["k8s.pod.name"]] = true + } + return out +} + +func countQueries(reqs []fakeRequest, pred func(string) bool) int { + n := 0 + for _, r := range reqs { + if pred(r.query) { + n++ + } + } + return n +} + +func isPlain(q string) bool { return !strings.HasPrefix(q, "count") } + +// assertOneEvalTime checks that the client pinned every request to the same evaluation time. +func assertOneEvalTime(t *testing.T, reqs []fakeRequest) { + t.Helper() + require.NotEmpty(t, reqs) + require.NotEmpty(t, reqs[0].time, "the client must pick the evaluation time: %s", reqs[0].query) + for _, r := range reqs { + assert.Equal(t, reqs[0].time, r.time, "every query is pinned to one evaluation time: %s", r.query) + } +} + +func TestQueryComplete_NotTruncatedIsOneDataQuery(t *testing.T) { + f, c := newFake(t, 10, podSeries(2, 3)) + + results, err := c.QueryComplete(context.Background(), podSelector) + require.NoError(t, err) + assert.Len(t, results, 6) + + reqs := f.queries() + require.Len(t, reqs, 2, "expected the selector plus one count(): %+v", reqs) + assert.ElementsMatch(t, []string{podSelector, "count(" + podSelector + ")"}, []string{reqs[0].query, reqs[1].query}) + assertOneEvalTime(t, reqs) +} + +func TestQueryComplete_SelectorAndCountRunTogether(t *testing.T) { + f := &fakePromQL{t: t, series: podSeries(2, 3), limit: 10} + var mu sync.Mutex + arrived := 0 + both := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + if arrived++; arrived == 2 { + close(both) + } + mu.Unlock() + // Answer only once both requests are in flight, which a client that waits for one cannot do. + select { + case <-both: + case <-time.After(2 * time.Second): + t.Errorf("request %q was not sent together with the other one", r.URL.Query().Get("query")) + } + f.handler(w, r) + })) + t.Cleanup(srv.Close) + c := newTestClient(srv.URL) + c.seriesLimit = 10 + + results, err := c.QueryComplete(context.Background(), podSelector) + require.NoError(t, err) + assert.Len(t, results, 6) + assertOneEvalTime(t, f.queries()) +} + +func TestEvalTimeAt(t *testing.T) { + assert.Equal(t, "1700000005", evalTimeAt(time.Unix(1700000010, 900e6)), "Unix seconds, evalTimeLag in the past") +} + +func TestQueryComplete_TruncatedShardsByNode(t *testing.T) { + series := podSeries(6, 5) + series[0].stale, series[7].stale = true, true // stale series still use up the cap + f, c := newFake(t, 10, series) + + results, err := c.QueryComplete(context.Background(), podSelector) + require.NoError(t, err) + assert.Len(t, results, 28) + assert.Len(t, podNames(results), 28, "every live pod exactly once") + + reqs := f.queries() + assert.Equal(t, 1, countQueries(reqs, func(q string) bool { + return q == `count by ("@resource.k8s.node.name")(`+podSelector+")" + })) + shards := countQueries(reqs, func(q string) bool { return isPlain(q) && strings.Contains(q, `"@resource.k8s.node.name"`) }) + assert.GreaterOrEqual(t, shards, 4, "28 series under a cap of 10 need at least 4 shards") + assert.Equal(t, 2, countQueries(reqs, func(q string) bool { return q == "count("+podSelector+")" }), "count() before and after") + assertOneEvalTime(t, reqs) +} + +func TestQueryComplete_AbsentNodeShardUsesNotRegex(t *testing.T) { + series := podSeries(3, 4) + for i := 0; i < 3; i++ { + series = append(series, fakeSeries{labels: map[string]string{ + "__name__": "kube_pod_info", + "@resource.k8s.cluster.name": "c1", + "@resource.k8s.pod.name": fmt.Sprintf("pending-%d", i), + }}) + } + f, c := newFake(t, 5, series) + + results, err := c.QueryComplete(context.Background(), podSelector) + require.NoError(t, err) + assert.Len(t, results, 15) + names := podNames(results) + for i := 0; i < 3; i++ { + assert.True(t, names[fmt.Sprintf("pending-%d", i)], "series without a node must be returned") + } + + reqs := f.queries() + assert.Equal(t, 1, countQueries(reqs, func(q string) bool { + return strings.Contains(q, `"@resource.k8s.node.name"!~".+"`) + })) + assert.Zero(t, countQueries(reqs, func(q string) bool { + return strings.Contains(q, `"@resource.k8s.node.name"=""`) + }), `="" does not match an absent label on this endpoint`) +} + +func TestQueryComplete_SingleNodeOverLimitSplitsFurther(t *testing.T) { + var series []fakeSeries + for cpu := 0; cpu < 8; cpu++ { + for _, mode := range []string{"idle", "user", "system"} { + series = append(series, fakeSeries{labels: map[string]string{ + "__name__": "node_cpu_seconds_total", + "@resource.k8s.cluster.name": "c1", + "@resource.k8s.node.name": "big-node", + "cpu": strconv.Itoa(cpu), + "mode": mode, + }}) + } + } + f, c := newFake(t, 10, series) + + results, err := c.QueryComplete(context.Background(), `node_cpu_seconds_total{"@resource.k8s.cluster.name"="c1"}`) + require.NoError(t, err) + assert.Len(t, results, 24) + assert.Positive(t, countQueries(f.queries(), func(q string) bool { return strings.HasPrefix(q, `count by ("cpu")`) }), + "one node holds every series, so the split must fall through to the cpu label") +} + +func TestQueryComplete_SkipsLabelWithTooManyValues(t *testing.T) { + var series []fakeSeries + for i := 0; i < 20; i++ { + series = append(series, fakeSeries{labels: map[string]string{ + "__name__": "m", + "id": strconv.Itoa(i), + "zone": fmt.Sprintf("z%d", i%3), + }}) + } + f, c := newFake(t, 10, series) + + results, err := c.QueryComplete(context.Background(), `m{}`) + require.NoError(t, err) + assert.Len(t, results, 20) + reqs := f.queries() + assert.Equal(t, 1, countQueries(reqs, func(q string) bool { return strings.HasPrefix(q, `count by ("id")`) })) + assert.Equal(t, 3, countQueries(reqs, func(q string) bool { return isPlain(q) && strings.Contains(q, `"zone"="z`) })) +} + +func TestQueryComplete_InconsistentCountIsError(t *testing.T) { + f, c := newFake(t, 10, podSeries(6, 5)) + f.countOffset = 3 // count() claims 33, but every split only accounts for 30 + + _, err := c.QueryComplete(context.Background(), podSelector) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrTruncated), "got %v", err) +} + +func TestQueryComplete_ShardTotalMismatchIsError(t *testing.T) { + series := podSeries(3, 4) + for i := 0; i < 3; i++ { + series = append(series, fakeSeries{labels: map[string]string{ + "__name__": "kube_pod_info", + "@resource.k8s.cluster.name": "c1", + "@resource.k8s.pod.name": fmt.Sprintf("pending-%d", i), + }}) + } + f, c := newFake(t, 5, series) + f.brokenNotRegex = true // the absent-node shard overlaps the node shards + + _, err := c.QueryComplete(context.Background(), podSelector) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrTruncated), "got %v", err) + assert.Contains(t, err.Error(), "shards returned 17 series but count() was 15 before and 15 after") +} + +func TestQueryComplete_UnsplittableIsError(t *testing.T) { + var series []fakeSeries + for i := 0; i < 20; i++ { + series = append(series, fakeSeries{labels: map[string]string{"__name__": "m", "id": strconv.Itoa(i)}}) + } + _, c := newFake(t, 10, series) + // id has 20 values, so its count by is capped too; nothing else varies. + _, err := c.QueryComplete(context.Background(), `m{}`) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrTruncated), "got %v", err) +} + +func TestQueryComplete_EmptyResultWithLiveSeries(t *testing.T) { + series := podSeries(2, 3) + stale := make([]fakeSeries, 0, 5) + for i := 0; i < 5; i++ { + s := podSeries(1, 1)[0] + s.labels["@resource.k8s.pod.name"] = fmt.Sprintf("done-%d", i) + s.stale = true + stale = append(stale, s) + } + f, c := newFake(t, 5, append(stale, series...)) + + results, err := c.QueryComplete(context.Background(), podSelector) + require.NoError(t, err) + assert.Len(t, results, 6, "the first query sees only stale series, yet six are live") + assertOneEvalTime(t, f.queries()) +} + +func TestQueryComplete_NonSelectorPassesThrough(t *testing.T) { + f, c := newFake(t, 10, podSeries(1, 2)) + _, err := c.QueryComplete(context.Background(), "count("+podSelector+")") + require.NoError(t, err) + assert.Len(t, f.queries(), 1) +} + +func TestQueryComplete_WarnedSelectorStillShards(t *testing.T) { + f, c := newFake(t, 10, podSeries(4, 5)) + + results, err := c.QueryComplete(context.Background(), podSelector) + require.NoError(t, err) + assert.Len(t, results, 20) + assert.Positive(t, countQueries(f.queries(), func(q string) bool { return strings.HasPrefix(q, "count by (") })) +} + +func TestQuery_TruncationWarningKeepsResults(t *testing.T) { + _, c := newFake(t, 3, podSeries(2, 3)) + results, err := c.Query(context.Background(), podSelector) + require.NoError(t, err, "Query stays a single best-effort request") + assert.Len(t, results, 3) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{"status":"success","warnings":["results truncated due to series limit (500)."],"data":{"resultType":"matrix","result":[{"metric":{"a":"b"},"values":[[1,"1"]]}]}}`) + })) + defer srv.Close() + rr, err := newTestClient(srv.URL).QueryRange(context.Background(), "up", time.Unix(0, 0), time.Unix(60, 0), time.Minute) + require.NoError(t, err) + assert.Len(t, rr, 1) +} + +func TestQueryCache_GetReturnsEverySeriesWhenTruncated(t *testing.T) { + f, c := newFake(t, 10, podSeries(5, 7)) + qc := NewQueryCache(c, "c1") + + results, err := qc.Get(context.Background(), "kube_pod_info") + require.NoError(t, err) + assert.Len(t, results, 35) + + n := len(f.queries()) + again, err := qc.Get(context.Background(), "kube_pod_info") + require.NoError(t, err) + assert.Len(t, again, 35) + assert.Len(t, f.queries(), n, "a cached metric must not be re-queried") +} + +func TestQueryCache_TruncatedHostTypeFailsFetch(t *testing.T) { + series := podSeries(1, 3) + for i := range series { + series[i].labels["@resource.host.type"] = "a" + } + for i := 0; i < 20; i++ { // host type b cannot be split, so its shard fetch fails + series = append(series, fakeSeries{labels: map[string]string{ + "__name__": "kube_pod_info", "@resource.k8s.cluster.name": "c1", + "@resource.host.type": "b", "id": strconv.Itoa(i), + }}) + } + _, c := newFake(t, 10, series) + qc := NewQueryCache(c, "c1", WithHostTypes([]string{"a", "b"})) + + results, err := qc.Get(context.Background(), "kube_pod_info") + require.Error(t, err, "the 3 series of host type a are not the whole result") + assert.True(t, errors.Is(err, ErrTruncated), "got %v", err) + assert.Empty(t, results) +} + +func TestQueryCache_ShardRequestsStayWithinConcurrencyLimit(t *testing.T) { + var series []fakeSeries + for _, ht := range []string{"a", "b"} { + for _, s := range podSeries(6, 5) { + s.labels["@resource.host.type"] = ht + s.labels["@resource.k8s.node.name"] += "-" + ht + series = append(series, s) + } + } + f := &fakePromQL{t: t, series: series, limit: 10} + var inflight, peak int + var mu sync.Mutex + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + inflight++ + if inflight > peak { + peak = inflight + } + mu.Unlock() + time.Sleep(5 * time.Millisecond) + f.handler(w, r) + mu.Lock() + inflight-- + mu.Unlock() + })) + t.Cleanup(srv.Close) + c := newTestClient(srv.URL) + c.seriesLimit = 10 + qc := NewQueryCache(c, "c1", WithHostTypes([]string{"a", "b"}), WithMaxConcurrency(2)) + + results, err := qc.Get(context.Background(), "kube_pod_info") + require.NoError(t, err) + assert.Len(t, results, 60) + assert.LessOrEqual(t, peak, 2, "shard re-fetches must share the cache's concurrency limit") +} + +func TestIsVectorSelector(t *testing.T) { + for s, want := range map[string]bool{ + `up`: true, + `kube_pod_info{"a"="b"}`: true, + `{"__name__"="k8s.pod.cpu.usage","a"="}"}`: true, + ` m{a=~"x|y", b!~".+"} `: true, + `k8s.pod.cpu.usage`: false, + `count(m{a="b"})`: false, + `m{a="b"}[5m]`: false, + `m{a="b"} or n{c="d"}`: false, + `rate(m[5m])`: false, + `m{a="b"} offset 5m`: false, + } { + assert.Equal(t, want, isVectorSelector(s), s) + } +} + +func TestWithMatcher(t *testing.T) { + assert.Equal(t, `m{x="1"}`, withMatcher("m", `x="1"`)) + assert.Equal(t, `m{x="1"}`, withMatcher("m{}", `x="1"`)) + assert.Equal(t, `m{a="b",x="1"}`, withMatcher(`m{a="b"}`, `x="1"`)) + assert.Equal(t, `{"__name__"="a.b",x="1"}`, withMatcher(`{"__name__"="a.b",}`, `x="1"`)) +} + +func TestPackShards(t *testing.T) { + groups := []shardGroup{{"a.b", 30}, {"c", 25}, {"", 4}, {"d", 10}, {"e", 50}} + shards := packShards("@resource.k8s.node.name", groups, 40) + + total := 0 + for _, s := range shards { + total += s.want + assert.LessOrEqual(t, s.want, 50) + } + assert.Equal(t, 119, total) + assert.Contains(t, shards, shard{matcher: `"@resource.k8s.node.name"!~".+"`, want: 4, exact: true}) + assert.Contains(t, shards, shard{matcher: `"@resource.k8s.node.name"="e"`, want: 50, exact: true}) + assert.Contains(t, shards, shard{matcher: `"@resource.k8s.node.name"=~"a\\.b|d"`, want: 40, exact: false}) + + var many []shardGroup + for i := 0; i < 200; i++ { + many = append(many, shardGroup{fmt.Sprintf("ip-10-0-%d-%d.ec2.internal", i/50, i), 1}) + } + for _, s := range packShards("@resource.k8s.node.name", many, 400) { + value := s.matcher[strings.Index(s.matcher, `"=~"`)+4 : len(s.matcher)-1] + assert.LessOrEqual(t, len(unescape(value)), maxMatcherValueLen, "the endpoint rejects longer matcher values") + } +} + +func TestQueryCache_ParallelGetWithFilterStaysWithinConcurrencyLimit(t *testing.T) { + f := &fakePromQL{t: t, series: podSeries(6, 5), limit: 10} + var inflight, peak int + var mu sync.Mutex + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + inflight++ + if inflight > peak { + peak = inflight + } + mu.Unlock() + time.Sleep(5 * time.Millisecond) + f.handler(w, r) + mu.Lock() + inflight-- + mu.Unlock() + })) + t.Cleanup(srv.Close) + c := newTestClient(srv.URL) + c.seriesLimit = 10 + qc := NewQueryCache(c, "c1", WithMaxConcurrency(2)) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + var wg sync.WaitGroup + for i := 0; i < 4; i++ { // more callers than slots, each needing several shard requests + wg.Add(1) + go func() { + defer wg.Done() + results, err := qc.GetWithFilter(ctx, "kube_pod_info", map[string]string{"@resource.k8s.namespace.name": "default"}) + assert.NoError(t, err, "a call that held a slot while its shards waited for one would deadlock here") + assert.Len(t, results, 30) + }() + } + wg.Wait() + assert.LessOrEqual(t, peak, 2, "shard re-fetches must share the cache's concurrency limit") +} diff --git a/util/otelmetrics/gpu_roles.go b/util/otelmetrics/gpu_roles.go new file mode 100644 index 000000000..bd476bb36 --- /dev/null +++ b/util/otelmetrics/gpu_roles.go @@ -0,0 +1,155 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package otelmetrics + +import ( + "fmt" + "math" + "sort" + "strings" +) + +// GPU node roles read from DCGM and kube-state-metrics series, so a GPU suite finds +// its idle, busy and multi-GPU nodes on any cluster instead of by instance type. + +// GPUResource is the kube-state-metrics resource label value for nvidia.com/gpu. +const GPUResource = "nvidia_com_gpu" + +// GPUNode is one node and its DCGM series. +type GPUNode struct { + Name string + Series []MetricResult +} + +// GPUs returns the sorted distinct non-empty values of the gpu datapoint attribute. +func (n GPUNode) GPUs() []string { + seen := make(map[string]struct{}) + for _, r := range n.Series { + if v := r.Labels.Datapoint["gpu"]; v != "" { + seen[v] = struct{}{} + } + } + out := make([]string, 0, len(seen)) + for v := range seen { + out = append(out, v) + } + sort.Strings(out) + return out +} + +// Idle reports whether no series of the node carries k8s.pod.name, so no pod holds a GPU. +func (n GPUNode) Idle() bool { + for _, r := range n.Series { + if _, ok := r.Labels.Resource["k8s.pod.name"]; ok { + return false + } + } + return true +} + +// Runs reports whether a pod whose name starts with podPrefix holds one of the node's GPUs. +func (n GPUNode) Runs(podPrefix string) bool { + for _, r := range n.Series { + if strings.HasPrefix(r.Labels.Resource["k8s.pod.name"], podPrefix) { + return true + } + } + return false +} + +// GPURoles sorts the nodes of DCGM series by role. A node can have more than one role. +type GPURoles struct { + Nodes []GPUNode // every node, sorted by name + Idle []GPUNode // no pod holds a GPU + Busy []GPUNode // a pod with the busy prefix holds a GPU +} + +// ReadGPURoles groups DCGM series by resource k8s.node.name and sorts the nodes by role. +// Series without k8s.node.name are left out. The GPU suite's attribute schema requires it. +func ReadGPURoles(results []MetricResult, busyPodPrefix string) GPURoles { + byNode := make(map[string][]MetricResult) + for _, r := range results { + if node := r.Labels.Resource["k8s.node.name"]; node != "" { + byNode[node] = append(byNode[node], r) + } + } + var roles GPURoles + for name, series := range byNode { + roles.Nodes = append(roles.Nodes, GPUNode{Name: name, Series: series}) + } + sort.Slice(roles.Nodes, func(i, j int) bool { return roles.Nodes[i].Name < roles.Nodes[j].Name }) + for _, n := range roles.Nodes { + if n.Idle() { + roles.Idle = append(roles.Idle, n) + } + if n.Runs(busyPodPrefix) { + roles.Busy = append(roles.Busy, n) + } + } + return roles +} + +// MultiGPUNodes returns the nodes whose allocatable, from GPUAllocatable, is more than +// one GPU. The allocatable decides, so a node whose DCGM series report too few gpu +// values is still a multi-GPU node. It returns an error for a node with no allocatable. +func MultiGPUNodes(nodes []GPUNode, allocatable map[string]int) ([]GPUNode, error) { + var out []GPUNode + var missing []string + for _, n := range nodes { + gpus, ok := allocatable[n.Name] + if !ok { + missing = append(missing, n.Name) + continue + } + if gpus > 1 { + out = append(out, n) + } + } + if len(missing) > 0 { + return nil, fmt.Errorf("no %s allocatable for nodes %v", GPUResource, missing) + } + return out, nil +} + +// SeriesOnNodes returns the results whose resource k8s.node.name is one of the nodes, +// so a role read from one metric selects the same nodes in another. +func SeriesOnNodes(results []MetricResult, nodes []GPUNode) []MetricResult { + names := make(map[string]struct{}, len(nodes)) + for _, n := range nodes { + names[n.Name] = struct{}{} + } + var out []MetricResult + for _, r := range results { + if _, ok := names[r.Labels.Resource["k8s.node.name"]]; ok { + out = append(out, r) + } + } + return out +} + +// GPUAllocatable maps resource k8s.node.name to the GPUResource value of +// kube_node_status_allocatable series. Series of other resources are ignored. It +// returns an error for a series without a node name, a value that is not a GPU +// count, or two series of one node with different values. +func GPUAllocatable(results []MetricResult) (map[string]int, error) { + out := make(map[string]int) + for _, r := range results { + if r.Labels.Datapoint["resource"] != GPUResource { + continue + } + node := r.Labels.Resource["k8s.node.name"] + if node == "" { + return nil, fmt.Errorf("%s series for %s has no k8s.node.name", r.MetricName, GPUResource) + } + if r.Value < 0 || r.Value != math.Trunc(r.Value) { + return nil, fmt.Errorf("node %s: %s allocatable %v is not a GPU count", node, GPUResource, r.Value) + } + n := int(r.Value) + if prev, ok := out[node]; ok && prev != n { + return nil, fmt.Errorf("node %s: %s allocatable is both %d and %d", node, GPUResource, prev, n) + } + out[node] = n + } + return out, nil +} diff --git a/util/otelmetrics/gpu_roles_test.go b/util/otelmetrics/gpu_roles_test.go new file mode 100644 index 000000000..e07d016a0 --- /dev/null +++ b/util/otelmetrics/gpu_roles_test.go @@ -0,0 +1,136 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package otelmetrics + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// dcgmSeries is one fake DCGM series of a node's gpu, held by pod when pod is not empty. +func dcgmSeries(node, gpu, pod string) MetricResult { + labels := NewMetricLabels() + labels.Resource["k8s.node.name"] = node + labels.Datapoint["gpu"] = gpu + if pod != "" { + labels.Resource["k8s.pod.name"] = pod + } + return MetricResult{MetricName: "DCGM_FI_DEV_GPU_UTIL", Labels: labels} +} + +func nodeNames(nodes []GPUNode) []string { + var out []string + for _, n := range nodes { + out = append(out, n.Name) + } + return out +} + +func TestReadGPURoles(t *testing.T) { + t.Run("idle_node_and_busy_multi_gpu_node", func(t *testing.T) { + roles := ReadGPURoles([]MetricResult{ + dcgmSeries("multi", "0", "multi-gpu-burn-abc"), + dcgmSeries("multi", "1", ""), + dcgmSeries("multi", "2", ""), + dcgmSeries("multi", "3", ""), + dcgmSeries("idle", "0", ""), + }, "multi-gpu-burn") + require.Equal(t, []string{"idle", "multi"}, nodeNames(roles.Nodes)) + require.Equal(t, []string{"idle"}, nodeNames(roles.Idle)) + require.Equal(t, []string{"multi"}, nodeNames(roles.Busy)) + require.Equal(t, []string{"0", "1", "2", "3"}, roles.Nodes[1].GPUs()) + }) + + t.Run("single_gpu_nodes", func(t *testing.T) { + roles := ReadGPURoles([]MetricResult{ + dcgmSeries("busy", "0", "multi-gpu-burn-abc"), + dcgmSeries("idle", "0", ""), + }, "multi-gpu-burn") + require.Equal(t, []string{"idle"}, nodeNames(roles.Idle)) + require.Equal(t, []string{"busy"}, nodeNames(roles.Busy)) + }) + + t.Run("other_pod_is_neither_idle_nor_busy", func(t *testing.T) { + roles := ReadGPURoles([]MetricResult{dcgmSeries("other", "0", "training-xyz")}, "multi-gpu-burn") + require.Equal(t, []string{"other"}, nodeNames(roles.Nodes)) + require.Empty(t, roles.Idle) + require.Empty(t, roles.Busy) + }) + + t.Run("series_of_one_gpu_count_once", func(t *testing.T) { + roles := ReadGPURoles([]MetricResult{ + dcgmSeries("node", "0", ""), + dcgmSeries("node", "0", ""), + dcgmSeries("node", "", ""), + }, "multi-gpu-burn") + require.Equal(t, []string{"0"}, roles.Nodes[0].GPUs(), "a repeated or empty gpu value is not a second GPU") + }) + + t.Run("series_without_node_name_are_left_out", func(t *testing.T) { + roles := ReadGPURoles([]MetricResult{dcgmSeries("", "0", "")}, "multi-gpu-burn") + require.Empty(t, roles.Nodes) + }) +} + +func TestMultiGPUNodes(t *testing.T) { + multi := GPUNode{Name: "multi", Series: []MetricResult{dcgmSeries("multi", "0", "")}} + single := GPUNode{Name: "single", Series: []MetricResult{dcgmSeries("single", "0", "")}} + nodes := []GPUNode{multi, single} + + got, err := MultiGPUNodes(nodes, map[string]int{"multi": 4, "single": 1, "other": 8}) + require.NoError(t, err) + require.Equal(t, []string{"multi"}, nodeNames(got), + "the allocatable decides, even when DCGM reports one gpu, and a node without DCGM series is not a GPU node") + + got, err = MultiGPUNodes(nodes, map[string]int{"single": 1}) + require.Error(t, err, "a GPU node with no allocatable must be an error, not a single-GPU node") + require.Contains(t, err.Error(), "multi") + require.Nil(t, got) + + got, err = MultiGPUNodes(nodes, map[string]int{"multi": 1, "single": 1}) + require.NoError(t, err) + require.Empty(t, got) +} + +func TestSeriesOnNodes(t *testing.T) { + a0, a1, b0 := dcgmSeries("a", "0", ""), dcgmSeries("a", "1", ""), dcgmSeries("b", "0", "") + results := []MetricResult{a0, b0, a1} + require.Equal(t, []MetricResult{a0, a1}, SeriesOnNodes(results, []GPUNode{{Name: "a"}})) + require.Equal(t, results, SeriesOnNodes(results, []GPUNode{{Name: "a"}, {Name: "b"}})) + require.Empty(t, SeriesOnNodes(results, nil)) +} + +// allocatableSeries is one fake kube_node_status_allocatable series. +func allocatableSeries(node, resource string, value float64) MetricResult { + labels := NewMetricLabels() + if node != "" { + labels.Resource["k8s.node.name"] = node + } + labels.Datapoint["resource"] = resource + return MetricResult{MetricName: "kube_node_status_allocatable", Labels: labels, Value: value} +} + +func TestGPUAllocatable(t *testing.T) { + got, err := GPUAllocatable([]MetricResult{ + allocatableSeries("multi", GPUResource, 4), + allocatableSeries("multi", GPUResource, 4), + allocatableSeries("multi", "cpu", 48), + allocatableSeries("single", GPUResource, 1), + }) + require.NoError(t, err) + require.Equal(t, map[string]int{"multi": 4, "single": 1}, got, "only the GPU resource counts, and an equal duplicate is fine") + + for name, results := range map[string][]MetricResult{ + "no_node_name": {allocatableSeries("", GPUResource, 1)}, + "fractional": {allocatableSeries("node", GPUResource, 1.5)}, + "negative": {allocatableSeries("node", GPUResource, -1)}, + "conflicting_node": {allocatableSeries("node", GPUResource, 1), allocatableSeries("node", GPUResource, 4)}, + } { + t.Run(name, func(t *testing.T) { + _, err := GPUAllocatable(results) + require.Error(t, err) + }) + } +} diff --git a/util/otelmetrics/host_types.go b/util/otelmetrics/host_types.go new file mode 100644 index 000000000..1cfbc33ec --- /dev/null +++ b/util/otelmetrics/host_types.go @@ -0,0 +1,75 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package otelmetrics + +import ( + "os" + "strings" +) + +// HostTypesEnvVar holds a comma-separated list of instance types that replaces +// a suite's default host types, so a suite can run against a cluster with other nodes. +const HostTypesEnvVar = "OTEL_HOST_TYPES" + +// ResolveHostTypes returns the host types from HostTypesEnvVar, or defaults when +// it is unset or lists no non-blank entries. +func ResolveHostTypes(defaults []string) []string { + return parseHostTypes(os.Getenv(HostTypesEnvVar), defaults) +} + +// PodLabelExcludesEnvVar and NodeLabelExcludesEnvVar hold the comma-separated +// filters.metrics.podLabels.exclude and nodeLabels.exclude patterns the cluster +// was deployed with. The denylist test logs the entries they render to, and fails +// for any other entry that removes a never-denied attribute. +const ( + PodLabelExcludesEnvVar = "OTEL_POD_LABEL_EXCLUDES" + NodeLabelExcludesEnvVar = "OTEL_NODE_LABEL_EXCLUDES" +) + +// ResolveLabelExcludes returns the patterns from PodLabelExcludesEnvVar and +// NodeLabelExcludesEnvVar. Unset means the cluster has no label excludes. +func ResolveLabelExcludes() LabelExcludes { + return LabelExcludes{ + Pod: splitList(os.Getenv(PodLabelExcludesEnvVar)), + Node: splitList(os.Getenv(NodeLabelExcludesEnvVar)), + } +} + +func parseHostTypes(raw string, defaults []string) []string { + if hostTypes := splitList(raw); len(hostTypes) > 0 { + return hostTypes + } + return defaults +} + +// splitList returns the non-blank entries of a comma-separated list, trimmed. +func splitList(raw string) []string { + var entries []string + for _, entry := range strings.Split(raw, ",") { + if entry = strings.TrimSpace(entry); entry != "" { + entries = append(entries, entry) + } + } + return entries +} + +// NodeGroup is one instance type a suite expects nodes of. +type NodeGroup struct { + InstanceType string + Description string +} + +// NodeGroupsFor returns one group per host type, so OTEL_HOST_TYPES also moves the per-group +// tests. Types missing from descriptions are described as "override". +func NodeGroupsFor(hostTypes []string, descriptions map[string]string) []NodeGroup { + groups := make([]NodeGroup, 0, len(hostTypes)) + for _, hostType := range hostTypes { + description, ok := descriptions[hostType] + if !ok { + description = "override" + } + groups = append(groups, NodeGroup{InstanceType: hostType, Description: description}) + } + return groups +} diff --git a/util/otelmetrics/host_types_test.go b/util/otelmetrics/host_types_test.go new file mode 100644 index 000000000..c565b6e0d --- /dev/null +++ b/util/otelmetrics/host_types_test.go @@ -0,0 +1,70 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package otelmetrics + +import ( + "os" + "reflect" + "testing" +) + +func TestResolveHostTypes(t *testing.T) { + defaults := []string{"g4dn.xlarge", "g4dn.12xlarge"} + + t.Run("unset", func(t *testing.T) { + t.Setenv(HostTypesEnvVar, "") + os.Unsetenv(HostTypesEnvVar) + if got := ResolveHostTypes(defaults); !reflect.DeepEqual(got, defaults) { + t.Fatalf("got %v, want %v", got, defaults) + } + }) + + cases := []struct { + name string + raw string + want []string + }{ + {"empty", "", defaults}, + {"whitespace_only", " \t ", defaults}, + {"separators_only", " , ,, ", defaults}, + {"single", "g5.xlarge", []string{"g5.xlarge"}}, + {"multiple", "g5.xlarge,trn1.32xlarge", []string{"g5.xlarge", "trn1.32xlarge"}}, + {"trims_whitespace", " g5.xlarge , trn1.32xlarge\t", []string{"g5.xlarge", "trn1.32xlarge"}}, + {"drops_empty_entries", ",g5.xlarge,, ,trn1.32xlarge,", []string{"g5.xlarge", "trn1.32xlarge"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(HostTypesEnvVar, tc.raw) + if got := ResolveHostTypes(defaults); !reflect.DeepEqual(got, tc.want) { + t.Fatalf("ResolveHostTypes with %q: got %v, want %v", tc.raw, got, tc.want) + } + }) + } +} + +func TestResolveLabelExcludes(t *testing.T) { + for _, name := range []string{PodLabelExcludesEnvVar, NodeLabelExcludesEnvVar} { + t.Setenv(name, "") + os.Unsetenv(name) + } + if got := ResolveLabelExcludes(); len(got.Pod) != 0 || len(got.Node) != 0 { + t.Fatalf("unset: got %+v, want no excludes", got) + } + + t.Setenv(PodLabelExcludesEnvVar, " app.kubernetes.io/* ,, team") + t.Setenv(NodeLabelExcludesEnvVar, "karpenter.sh/nodepool") + want := LabelExcludes{Pod: []string{"app.kubernetes.io/*", "team"}, Node: []string{"karpenter.sh/nodepool"}} + if got := ResolveLabelExcludes(); !reflect.DeepEqual(got, want) { + t.Fatalf("got %+v, want %+v", got, want) + } +} + +func TestNodeGroupsFor(t *testing.T) { + descriptions := map[string]string{"t3.medium": "standard"} + got := NodeGroupsFor([]string{"t3.medium", "g5.xlarge"}, descriptions) + want := []NodeGroup{{"t3.medium", "standard"}, {"g5.xlarge", "override"}} + if !reflect.DeepEqual(got, want) { + t.Fatalf("got %v, want %v", got, want) + } +} diff --git a/util/otelmetrics/label_extraction.go b/util/otelmetrics/label_extraction.go new file mode 100644 index 000000000..b50bd4807 --- /dev/null +++ b/util/otelmetrics/label_extraction.go @@ -0,0 +1,168 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package otelmetrics + +import ( + "errors" + "fmt" + "regexp" + "regexp/syntax" + "sort" + "strings" + + "gopkg.in/yaml.v3" +) + +// ErrFilterNotModeled marks a filter in a collector configuration that the +// reconciliation cannot model, so a test that depends on it cannot tell which +// labels or objects the filter removes. +var ErrFilterNotModeled = errors.New("filter not modeled") + +// LabelExtraction is the label extraction of the k8sattributes processor for one +// source, "node" or "pod": the labels it copies onto a series, each as Prefix +// plus the label key. Build it with ParseLabelExtraction; the zero value +// extracts nothing. +type LabelExtraction struct { + From string + Prefix string // "k8s..label." + Processor string + keys map[string]struct{} + regexes []*regexp.Regexp + all bool +} + +// labelRuleFields are the fields of an extraction rule the model covers. Any +// other field, such as a value regex, changes the attribute or its value. +var labelRuleFields = map[string]bool{"tag_name": true, "key": true, "key_regex": true, "from": true} + +// keyRegexTagSuffix is a tag name back reference to the first group, with the +// configuration's $ escapes. +var keyRegexTagSuffix = regexp.MustCompile(`^\$+(1|\{1\})$`) + +// ParseLabelExtraction reads the k8sattributes label rules for from ("node" or +// "pod") from a rendered collector configuration. A rule must write the label +// key unchanged under the prefix k8s..label., as the chart renders it: a +// key with that tag name, or a key_regex whose single group spans the key, with +// a back reference to it. Any other rule, or rules for from in more than one +// processor, returns an error that wraps ErrFilterNotModeled. A configuration +// without rules for from is an error. +func ParseLabelExtraction(otelConfigYAML, from string) (LabelExtraction, error) { + var cfg struct { + Processors map[string]yaml.Node `yaml:"processors"` + } + if err := yaml.Unmarshal([]byte(otelConfigYAML), &cfg); err != nil { + return LabelExtraction{}, fmt.Errorf("parsing collector configuration: %w", err) + } + e := LabelExtraction{From: from, Prefix: "k8s." + from + ".label.", keys: map[string]struct{}{}} + var processors []string + for _, name := range sortedProcessors(cfg.Processors, "k8sattributes") { + node := cfg.Processors[name] + var p struct { + Extract struct { + Labels []map[string]string `yaml:"labels"` + } `yaml:"extract"` + } + if err := node.Decode(&p); err != nil { + return LabelExtraction{}, fmt.Errorf("parsing processor %s: %w", name, err) + } + found := false + for _, rule := range p.Extract.Labels { + if src := rule["from"]; src != from && !(src == "" && from == "pod") { + continue // k8sattributes reads a rule without from from the pod + } + found = true + if err := e.addRule(rule); err != nil { + return LabelExtraction{}, fmt.Errorf("processor %s: %w", name, err) + } + } + if found { + processors = append(processors, name) + } + } + switch len(processors) { + case 0: + return LabelExtraction{}, fmt.Errorf("no k8sattributes processor extracts %s labels", from) + case 1: + e.Processor = processors[0] + return e, nil + default: + return LabelExtraction{}, fmt.Errorf("%w: %s labels are extracted by more than one k8sattributes processor %v, and which pipeline runs which is not modeled", + ErrFilterNotModeled, from, processors) + } +} + +func (e *LabelExtraction) addRule(rule map[string]string) error { + var unknown []string + for field := range rule { + if !labelRuleFields[field] { + unknown = append(unknown, field) + } + } + if len(unknown) > 0 { + sort.Strings(unknown) + return fmt.Errorf("%w: label rule %v sets %v", ErrFilterNotModeled, rule, unknown) + } + key, keyRegex, tag := rule["key"], rule["key_regex"], rule["tag_name"] + switch { + case key != "" && keyRegex == "": + if tag != e.Prefix+key { + return fmt.Errorf("%w: label rule for key %q has tag_name %q, not %q", ErrFilterNotModeled, key, tag, e.Prefix+key) + } + e.keys[key] = struct{}{} + case keyRegex != "" && key == "": + re, err := regexp.Compile("^(?:" + keyRegex + ")$") // as k8sattributes compiles it + if err != nil { + return fmt.Errorf("label rule key_regex %q: %w", keyRegex, err) + } + tree, err := syntax.Parse(keyRegex, syntax.Perl) + if err != nil { + return fmt.Errorf("label rule key_regex %q: %w", keyRegex, err) + } + if tree.Op != syntax.OpCapture || tree.Cap != 1 { + return fmt.Errorf("%w: label rule key_regex %q is not one group around the whole key", ErrFilterNotModeled, keyRegex) + } + if !strings.HasPrefix(tag, e.Prefix) || !keyRegexTagSuffix.MatchString(strings.TrimPrefix(tag, e.Prefix)) { + return fmt.Errorf("%w: label rule key_regex %q has tag_name %q, not %q followed by a reference to the group", + ErrFilterNotModeled, keyRegex, tag, e.Prefix) + } + e.regexes = append(e.regexes, re) + if sub := tree.Sub[0]; sub.Op == syntax.OpStar && (sub.Sub[0].Op == syntax.OpAnyChar || sub.Sub[0].Op == syntax.OpAnyCharNotNL) { + e.all = true + } + default: + return fmt.Errorf("label rule %v must set exactly one of key and key_regex", rule) + } + return nil +} + +// Extracts reports whether a rule copies the label key onto a series. +func (e LabelExtraction) Extracts(key string) bool { + if _, ok := e.keys[key]; ok { + return true + } + for _, re := range e.regexes { + if re.MatchString(key) { + return true + } + } + return false +} + +// ExtractsAll reports whether a rule copies every label, as the chart does when +// no include list narrows the extraction. +func (e LabelExtraction) ExtractsAll() bool { + return e.all +} + +// sortedProcessors returns the names of the processors of type typ, sorted. +func sortedProcessors(processors map[string]yaml.Node, typ string) []string { + var names []string + for name := range processors { + if name == typ || strings.HasPrefix(name, typ+"/") { + names = append(names, name) + } + } + sort.Strings(names) + return names +} diff --git a/util/otelmetrics/label_extraction_test.go b/util/otelmetrics/label_extraction_test.go new file mode 100644 index 000000000..ceb8af6ab --- /dev/null +++ b/util/otelmetrics/label_extraction_test.go @@ -0,0 +1,128 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package otelmetrics + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" +) + +// includeListConfig has the rules the chart renders for the include lists +// nodeLabels [kubernetes.io/*, eks.amazonaws.com/nodegroup] and podLabels +// [app.kubernetes.io/*, app], plus the labels the chart reads itself. +const includeListConfig = ` +processors: + k8sattributes/cw_k8s_ci_v0_node: + extract: + labels: + - {from: node, key_regex: "(kubernetes\\.io/.*)", tag_name: "k8s.node.label.$$$1"} + - {from: node, key: eks.amazonaws.com/nodegroup, tag_name: k8s.node.label.eks.amazonaws.com/nodegroup} + - {from: node, key: topology.kubernetes.io/zone, tag_name: k8s.node.label.topology.kubernetes.io/zone} + metadata: [k8s.node.name] + k8sattributes/cw_k8s_ci_v0_pod: + extract: + labels: + - {from: pod, key_regex: "(app\\.kubernetes\\.io/.*)", tag_name: "k8s.pod.label.$$$1"} + - {from: pod, key: app, tag_name: k8s.pod.label.app} + - {from: pod, key: component, tag_name: k8s.pod.label.component} +` + +func TestParseLabelExtraction(t *testing.T) { + t.Run("extract all", func(t *testing.T) { + e := mustExtraction(t, extractAllConfig, "node") + require.Equal(t, "k8s.node.label.", e.Prefix) + require.Equal(t, "k8sattributes/cw_k8s_ci_v0_node", e.Processor) + require.True(t, e.ExtractsAll()) + require.True(t, e.Extracts("anything/at-all")) + }) + + t.Run("include lists", func(t *testing.T) { + node := mustExtraction(t, includeListConfig, "node") + require.Equal(t, "k8sattributes/cw_k8s_ci_v0_node", node.Processor) + require.False(t, node.ExtractsAll()) + require.True(t, node.Extracts("kubernetes.io/os")) + require.True(t, node.Extracts("eks.amazonaws.com/nodegroup")) + require.True(t, node.Extracts("topology.kubernetes.io/zone"), "the chart extracts the labels it reads") + require.False(t, node.Extracts("node.kubernetes.io/instance-type"), "the regex is anchored at the start") + require.False(t, node.Extracts("kubernetesXio/os"), "the dot is literal") + require.False(t, node.Extracts("eks.amazonaws.com/nodegroup-image"), "a key rule is exact") + + pod := mustExtraction(t, includeListConfig, "pod") + require.Equal(t, "k8s.pod.label.", pod.Prefix) + require.Equal(t, "k8sattributes/cw_k8s_ci_v0_pod", pod.Processor) + require.True(t, pod.Extracts("app.kubernetes.io/name")) + require.True(t, pod.Extracts("app")) + require.False(t, pod.Extracts("kubernetes.io/os"), "node rules do not apply to pods") + require.False(t, pod.Extracts("app.kubernetes.io"), "the prefix needs its slash") + }) + + t.Run("a rule without from reads the pod", func(t *testing.T) { + e := mustExtraction(t, ` +processors: + k8sattributes: + extract: + labels: + - {key: app, tag_name: k8s.pod.label.app} +`, "pod") + require.True(t, e.Extracts("app")) + }) + + for name, tc := range map[string]struct { + cfg, from string + notModeled bool + }{ + "no rules for the source": {cfg: includeListConfig, from: "namespace"}, + "no processor": {cfg: "processors:\n batch/x: {}\n", from: "node"}, + "rules in two processors": {cfg: ` +processors: + k8sattributes/a: + extract: {labels: [{from: node, key: a, tag_name: k8s.node.label.a}]} + k8sattributes/b: + extract: {labels: [{from: node, key: b, tag_name: k8s.node.label.b}]} +`, from: "node", notModeled: true}, + "value regex": {cfg: ` +processors: + k8sattributes: + extract: {labels: [{from: node, key: a, tag_name: k8s.node.label.a, regex: "v(.*)"}]} +`, from: "node", notModeled: true}, + "renamed key": {cfg: ` +processors: + k8sattributes: + extract: {labels: [{from: node, key: a, tag_name: k8s.node.label.b}]} +`, from: "node", notModeled: true}, + "default tag name": {cfg: ` +processors: + k8sattributes: + extract: {labels: [{from: node, key: a}]} +`, from: "node", notModeled: true}, + "group inside the key": {cfg: ` +processors: + k8sattributes: + extract: {labels: [{from: node, key_regex: "kubernetes\\.io/(.*)", tag_name: "k8s.node.label.$$1"}]} +`, from: "node", notModeled: true}, + "tag name without the group": {cfg: ` +processors: + k8sattributes: + extract: {labels: [{from: node, key_regex: "(.*)", tag_name: "k8s.node.label.x"}]} +`, from: "node", notModeled: true}, + "key and key_regex": {cfg: ` +processors: + k8sattributes: + extract: {labels: [{from: node, key: a, key_regex: "(a)", tag_name: k8s.node.label.a}]} +`, from: "node"}, + "invalid key_regex": {cfg: ` +processors: + k8sattributes: + extract: {labels: [{from: node, key_regex: "(a", tag_name: "k8s.node.label.$$$1"}]} +`, from: "node"}, + } { + t.Run(name, func(t *testing.T) { + _, err := ParseLabelExtraction(tc.cfg, tc.from) + require.Error(t, err) + require.Equal(t, tc.notModeled, errors.Is(err, ErrFilterNotModeled), "%v", err) + }) + } +} diff --git a/util/otelmetrics/label_reconcile.go b/util/otelmetrics/label_reconcile.go new file mode 100644 index 000000000..4592dc882 --- /dev/null +++ b/util/otelmetrics/label_reconcile.go @@ -0,0 +1,319 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package otelmetrics + +import ( + "fmt" + "sort" + "strings" + + "gopkg.in/yaml.v3" +) + +// AttributeDenylist is the unconditional removal configuration of an +// awsattributelimit processor, plus its attribute cap. +type AttributeDenylist struct { + Prefixes []string + Keys map[string]struct{} + MaxTotal int + Processor string +} + +// ParseAttributeDenylist reads the awsattributelimit processor from a rendered +// collector configuration. It returns an error if the configuration has no such +// processor or more than one, because the expectation would then be ambiguous. +func ParseAttributeDenylist(otelConfigYAML string) (AttributeDenylist, error) { + var cfg struct { + Processors map[string]struct { + Prefixes []string `yaml:"unconditional_removal_prefixes"` + Keys []string `yaml:"unconditional_removal_keys"` + MaxTotal int `yaml:"max_total_attributes"` + } `yaml:"processors"` + } + if err := yaml.Unmarshal([]byte(otelConfigYAML), &cfg); err != nil { + return AttributeDenylist{}, fmt.Errorf("parsing collector configuration: %w", err) + } + var names []string + for name := range cfg.Processors { + if name == "awsattributelimit" || strings.HasPrefix(name, "awsattributelimit/") { + names = append(names, name) + } + } + if len(names) != 1 { + return AttributeDenylist{}, fmt.Errorf("want exactly one awsattributelimit processor, found %d: %v", len(names), names) + } + p := cfg.Processors[names[0]] + d := AttributeDenylist{ + Prefixes: p.Prefixes, + Keys: make(map[string]struct{}, len(p.Keys)), + MaxTotal: p.MaxTotal, + Processor: names[0], + } + for _, k := range p.Keys { + d.Keys[k] = struct{}{} + } + return d, nil +} + +// Denies reports whether the processor removes the resource attribute key, +// using the same exact-key-then-prefix rule as the processor itself. +func (d AttributeDenylist) Denies(key string) bool { + if _, ok := d.Keys[key]; ok { + return true + } + for _, p := range d.Prefixes { + if strings.HasPrefix(key, p) { + return true + } + } + return false +} + +// AtCap reports whether a series is at the processor's attribute cap, where +// tier-based removal can explain a missing attribute. The count matches the +// processor's: resource, scope and datapoint attributes. Keys the backend +// synthesises (the @-prefixed scope name and version, and __x__ datapoint keys) +// are not attributes and are excluded. +func (d AttributeDenylist) AtCap(labels MetricLabels) bool { + if d.MaxTotal <= 0 { + return false + } + n := len(labels.Resource) + for k := range labels.Instrumentation { + if !strings.HasPrefix(k, "@") { + n++ + } + } + for k := range labels.Datapoint { + if !(strings.HasPrefix(k, "__") && strings.HasSuffix(k, "__")) { + n++ + } + } + return n >= d.MaxTotal +} + +// LabelDiff is the result of reconciling an object's Kubernetes labels with the +// resource attributes on one series. +type LabelDiff struct { + Checked int // labels expected on the series + Missing []string // expected but absent + Mismatched []string // present with a different value, as "key: got, want" + Leaked []string // removed by the extraction rules or the denylist, but present +} + +// OK reports whether the series matched the object exactly. +func (l LabelDiff) OK() bool { + return len(l.Missing) == 0 && len(l.Mismatched) == 0 && len(l.Leaked) == 0 +} + +// ReconcileLabels compares an object's labels with the attributes under e.Prefix +// (for example "k8s.node.label.") in resource. Every label that the extraction +// rules copy and the denylist does not match must arrive with the same value, +// and every other label must not. Empty-valued labels are skipped, because +// PromQL drops empty label values. +func ReconcileLabels(objectLabels, resource map[string]string, e LabelExtraction, d AttributeDenylist) LabelDiff { + var diff LabelDiff + for k, want := range objectLabels { + attr := e.Prefix + k + got, present := resource[attr] + if !e.Extracts(k) || d.Denies(attr) { + if present { + diff.Leaked = append(diff.Leaked, k) + } + continue + } + if want == "" { + continue + } + diff.Checked++ + switch { + case !present: + diff.Missing = append(diff.Missing, k) + case got != want: + diff.Mismatched = append(diff.Mismatched, fmt.Sprintf("%s: got %q, want %q", k, got, want)) + } + } + sort.Strings(diff.Missing) + sort.Strings(diff.Mismatched) + sort.Strings(diff.Leaked) + return diff +} + +// Attributes no denylist entry the chart adds may remove; a customer label exclude +// may remove the k8s.node.label.* and k8s.pod.label.* ones. The helm-charts render +// test prometheus_attribute_shape.sh holds the same list; keep the two in step. +var ( + // protectedResourceKeys and protectedPrefixes of the awsattributelimit processor; + // unconditional removal runs before that protection, so a denylist entry would override it. + NeverDeniedKeys = []string{ + "k8s.cluster.name", "k8s.node.name", "k8s.pod.name", "k8s.pod.uid", "k8s.namespace.name", + "k8s.container.name", "k8s.deployment.name", "k8s.statefulset.name", "k8s.daemonset.name", + "k8s.replicaset.name", "k8s.job.name", "k8s.cronjob.name", "k8s.workload.name", + "k8s.workload.type", "neurondevice", "neuroncore", "efa.device", "aws.efa.eni.id", + "volume_id", "instance_id", "k8s.pod.label.app.kubernetes.io/name", + "k8s.pod.label.app.kubernetes.io/instance", "k8s.pod.label.app.kubernetes.io/component", + "k8s.component.name", + // Kept on purpose inside a label domain the denylist partly removes, so widening an entry to the domain fails. + "k8s.node.label.kubernetes.io/os", "k8s.node.label.kubernetes.io/arch", + "k8s.node.label.eks.amazonaws.com/nodegroup", "k8s.node.label.eks.amazonaws.com/capacityType", + "k8s.node.label.karpenter.sh/nodepool", "k8s.node.label.karpenter.sh/capacity-type", + "k8s.node.label.karpenter.k8s.aws/ec2nodeclass", + } + NeverDeniedPrefixes = []string{"cloud.", "host.", "hw."} +) + +// NeverDeniedViolations lists each denylist entry that would remove a never-denied attribute. +func (d AttributeDenylist) NeverDeniedViolations() []string { + var out []string + for _, key := range NeverDeniedKeys { + if _, ok := d.Keys[key]; ok { + out = append(out, fmt.Sprintf("key %q", key)) + } + for _, p := range d.Prefixes { + if strings.HasPrefix(key, p) { + out = append(out, fmt.Sprintf("prefix %q removes %q", p, key)) + } + } + } + for _, protected := range NeverDeniedPrefixes { + for key := range d.Keys { + if strings.HasPrefix(key, protected) { + out = append(out, fmt.Sprintf("key %q is under %q", key, protected)) + } + } + for _, p := range d.Prefixes { + if strings.HasPrefix(p, protected) || strings.HasPrefix(protected, p) { + out = append(out, fmt.Sprintf("prefix %q overlaps %q", p, protected)) + } + } + } + sort.Strings(out) + return out +} + +// LabelExcludes are the filters.metrics.podLabels.exclude and nodeLabels.exclude +// patterns a cluster was deployed with: a label key, a prefix followed by "*", or +// "*" alone. +type LabelExcludes struct { + Pod, Node []string +} + +// Denylist renders the patterns as the chart does: the key k8s.pod.label. +// or k8s.node.label., or a prefix without the "*" when the pattern ends in "*". +func (e LabelExcludes) Denylist() AttributeDenylist { + d := AttributeDenylist{Keys: map[string]struct{}{}} + for _, scope := range []struct { + prefix string + patterns []string + }{{"k8s.pod.label.", e.Pod}, {"k8s.node.label.", e.Node}} { + for _, pattern := range scope.patterns { + if strings.HasSuffix(pattern, "*") { + d.Prefixes = append(d.Prefixes, scope.prefix+strings.TrimSuffix(pattern, "*")) + } else { + d.Keys[scope.prefix+pattern] = struct{}{} + } + } + } + return d +} + +// SplitCustomerExcludes splits d into the entries that the declared customer +// excludes render to and every other entry, which the chart added. +func (d AttributeDenylist) SplitCustomerExcludes(e LabelExcludes) (chart, customer AttributeDenylist) { + declared := e.Denylist() + declaredPrefixes := make(map[string]struct{}, len(declared.Prefixes)) + for _, p := range declared.Prefixes { + declaredPrefixes[p] = struct{}{} + } + chart = AttributeDenylist{Keys: map[string]struct{}{}, MaxTotal: d.MaxTotal, Processor: d.Processor} + customer = AttributeDenylist{Keys: map[string]struct{}{}, MaxTotal: d.MaxTotal, Processor: d.Processor} + for _, p := range d.Prefixes { + if _, ok := declaredPrefixes[p]; ok { + customer.Prefixes = append(customer.Prefixes, p) + } else { + chart.Prefixes = append(chart.Prefixes, p) + } + } + for k := range d.Keys { + if _, ok := declared.Keys[k]; ok { + customer.Keys[k] = struct{}{} + } else { + chart.Keys[k] = struct{}{} + } + } + return chart, customer +} + +// Entries lists the denylist's keys and prefixes, sorted. +func (d AttributeDenylist) Entries() []string { + out := make([]string, 0, len(d.Keys)+len(d.Prefixes)) + for k := range d.Keys { + out = append(out, fmt.Sprintf("key %q", k)) + } + for _, p := range d.Prefixes { + out = append(out, fmt.Sprintf("prefix %q", p)) + } + sort.Strings(out) + return out +} + +// ObjectReconciliation is the result of reconciling every series of a set of +// Kubernetes objects. An object passes if any one of its series reconciles, +// because a label change or a collector restart leaves an older series inside +// the query lookback window next to the current one. +type ObjectReconciliation struct { + Objects int // objects with at least one series below the cap + Labels int // labels checked, counted once per passing object + AtCap int // series skipped at the attribute cap + Failed map[string]LabelDiff // objects no series reconciled, with their closest series + Required int // objects that must have a series + NoSeries []string // required objects with no series at all, sorted +} + +// ReconcileObjects groups series by object and reconciles each. objectOf returns the +// object key and its Kubernetes labels, or false for a series of an unknown object. +// Every key in required must have at least one series, at the cap or below it, so +// leave out an object that a filter removes. +func ReconcileObjects(results []MetricResult, objectOf func(MetricLabels) (string, map[string]string, bool), + required []string, e LabelExtraction, d AttributeDenylist) ObjectReconciliation { + out := ObjectReconciliation{Failed: map[string]LabelDiff{}, Required: len(required)} + passed := map[string]bool{} + hasSeries := map[string]bool{} + for _, r := range results { + key, labels, ok := objectOf(r.Labels) + if !ok { + continue + } + hasSeries[key] = true + if d.AtCap(r.Labels) { + out.AtCap++ + continue + } + if passed[key] { + continue + } + diff := ReconcileLabels(labels, r.Labels.Resource, e, d) + if diff.OK() { + passed[key] = true + out.Labels += diff.Checked + delete(out.Failed, key) + continue + } + if cur, seen := out.Failed[key]; !seen || diff.problems() < cur.problems() { + out.Failed[key] = diff + } + } + out.Objects = len(passed) + len(out.Failed) + for _, key := range required { + if !hasSeries[key] { + out.NoSeries = append(out.NoSeries, key) + } + } + sort.Strings(out.NoSeries) + return out +} + +func (l LabelDiff) problems() int { + return len(l.Missing) + len(l.Mismatched) + len(l.Leaked) +} diff --git a/util/otelmetrics/label_reconcile_test.go b/util/otelmetrics/label_reconcile_test.go new file mode 100644 index 000000000..dfb370097 --- /dev/null +++ b/util/otelmetrics/label_reconcile_test.go @@ -0,0 +1,306 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package otelmetrics + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +const denylistConfig = ` +processors: + batch/x: {} + awsattributelimit/cw_k8s_ci_v0: + max_total_attributes: 150 + unconditional_removal_prefixes: + - "k8s.node.label.karpenter.k8s.aws/instance-" + unconditional_removal_keys: + - "k8s.node.label.topology.kubernetes.io/zone" + - "k8s.pod.label.pod-template-hash" +` + +const extractAllConfig = ` +processors: + k8sattributes/cw_k8s_ci_v0_node: + extract: + labels: + - {from: node, key_regex: "(.*)", tag_name: "k8s.node.label.$$$1"} +` + +func mustExtraction(t *testing.T, cfg, from string) LabelExtraction { + t.Helper() + e, err := ParseLabelExtraction(cfg, from) + require.NoError(t, err) + return e +} + +func mustDenylist(t *testing.T) AttributeDenylist { + t.Helper() + d, err := ParseAttributeDenylist(denylistConfig) + require.NoError(t, err) + return d +} + +func TestParseAttributeDenylist(t *testing.T) { + d := mustDenylist(t) + require.Equal(t, "awsattributelimit/cw_k8s_ci_v0", d.Processor) + require.Equal(t, 150, d.MaxTotal) + require.Equal(t, []string{"k8s.node.label.karpenter.k8s.aws/instance-"}, d.Prefixes) + require.Len(t, d.Keys, 2) + + _, err := ParseAttributeDenylist("processors:\n batch/x: {}\n") + require.Error(t, err, "no awsattributelimit processor must be an error, not an empty denylist") + + _, err = ParseAttributeDenylist("processors:\n awsattributelimit/a: {}\n awsattributelimit/b: {}\n") + require.Error(t, err, "two processors make the expectation ambiguous") +} + +func TestDenies(t *testing.T) { + d := mustDenylist(t) + require.True(t, d.Denies("k8s.node.label.topology.kubernetes.io/zone")) + require.True(t, d.Denies("k8s.node.label.karpenter.k8s.aws/instance-family")) + require.False(t, d.Denies("k8s.node.label.karpenter.k8s.aws/ec2nodeclass"), "the prefix must not widen to the domain") + require.False(t, d.Denies("k8s.pod.label.topology.kubernetes.io/zone"), "exact keys are scope-specific") +} + +func TestReconcileLabels(t *testing.T) { + d := mustDenylist(t) + all := mustExtraction(t, extractAllConfig, "node") + node := map[string]string{ + "kubernetes.io/os": "linux", + "karpenter.k8s.aws/ec2nodeclass": "default", + "karpenter.k8s.aws/instance-family": "m5", + "topology.kubernetes.io/zone": "us-east-1a", + "node-role.kubernetes.io/control-plane": "", + } + clean := map[string]string{ + "k8s.node.label.kubernetes.io/os": "linux", + "k8s.node.label.karpenter.k8s.aws/ec2nodeclass": "default", + } + + t.Run("exact match", func(t *testing.T) { + diff := ReconcileLabels(node, clean, all, d) + require.True(t, diff.OK(), "%+v", diff) + require.Equal(t, 2, diff.Checked, "denied and empty-valued labels are not checked") + }) + + t.Run("over-broad denylist drops a retained label", func(t *testing.T) { + got := map[string]string{"k8s.node.label.kubernetes.io/os": "linux"} + diff := ReconcileLabels(node, got, all, d) + require.Equal(t, []string{"karpenter.k8s.aws/ec2nodeclass"}, diff.Missing) + }) + + t.Run("denied label still present", func(t *testing.T) { + got := map[string]string{"k8s.node.label.karpenter.k8s.aws/instance-family": "m5"} + for k, v := range clean { + got[k] = v + } + diff := ReconcileLabels(node, got, all, d) + require.Equal(t, []string{"karpenter.k8s.aws/instance-family"}, diff.Leaked) + }) + + t.Run("value changed in transit", func(t *testing.T) { + got := map[string]string{"k8s.node.label.kubernetes.io/os": "windows", + "k8s.node.label.karpenter.k8s.aws/ec2nodeclass": "default"} + diff := ReconcileLabels(node, got, all, d) + require.Len(t, diff.Mismatched, 1) + require.Contains(t, diff.Mismatched[0], "kubernetes.io/os") + }) + + t.Run("labels under another prefix are ignored", func(t *testing.T) { + got := map[string]string{"k8s.pod.label.kubernetes.io/os": "linux"} + diff := ReconcileLabels(node, got, all, d) + require.Len(t, diff.Missing, 2, "a pod label must not satisfy a node label") + }) + + include := mustExtraction(t, ` +processors: + k8sattributes/n: + extract: + labels: + - {from: node, key_regex: "(kubernetes\\.io/.*)", tag_name: "k8s.node.label.$$$1"} + - {from: node, key: "topology.kubernetes.io/zone", tag_name: "k8s.node.label.topology.kubernetes.io/zone"} +`, "node") + + t.Run("label the include list leaves out is not expected", func(t *testing.T) { + got := map[string]string{"k8s.node.label.kubernetes.io/os": "linux"} + diff := ReconcileLabels(node, got, include, d) + require.True(t, diff.OK(), "%+v", diff) + require.Equal(t, 1, diff.Checked, "only kubernetes.io/os is extracted and not denied") + }) + + t.Run("label the include list leaves out still present", func(t *testing.T) { + got := map[string]string{"k8s.node.label.kubernetes.io/os": "linux"} + for k, v := range clean { + got[k] = v + } + diff := ReconcileLabels(node, got, include, d) + require.Equal(t, []string{"karpenter.k8s.aws/ec2nodeclass"}, diff.Leaked) + }) +} + +func TestAtCap(t *testing.T) { + d := AttributeDenylist{MaxTotal: 3} + l := NewMetricLabels() + l.Resource["a"], l.Resource["b"] = "1", "2" + l.Instrumentation["@name"] = "scope" + l.Datapoint["__type__"] = "gauge" + require.False(t, d.AtCap(l), "scope name and backend keys are not attributes") + l.Datapoint["c"] = "3" + require.True(t, d.AtCap(l)) + require.False(t, AttributeDenylist{}.AtCap(l), "no cap configured") +} + +func TestNeverDeniedViolations(t *testing.T) { + require.Empty(t, mustDenylist(t).NeverDeniedViolations()) + + for name, tc := range map[string]struct { + keys, prefixes []string + want []string + }{ + "protected key": {keys: []string{"k8s.cluster.name"}, want: []string{`key "k8s.cluster.name"`}}, + "widened label domain": {prefixes: []string{"k8s.node.label.kubernetes.io/"}, want: []string{`prefix "k8s.node.label.kubernetes.io/" removes "k8s.node.label.kubernetes.io/arch"`, `prefix "k8s.node.label.kubernetes.io/" removes "k8s.node.label.kubernetes.io/os"`}}, + "key under protected prefix": {keys: []string{"cloud.availability_zone"}, want: []string{`key "cloud.availability_zone" is under "cloud."`}}, + "prefix covering protected prefix": {prefixes: []string{"ho"}, want: []string{`prefix "ho" overlaps "host."`}}, + } { + t.Run(name, func(t *testing.T) { + d := AttributeDenylist{Prefixes: tc.prefixes, Keys: map[string]struct{}{}} + for _, k := range tc.keys { + d.Keys[k] = struct{}{} + } + require.Equal(t, tc.want, d.NeverDeniedViolations()) + }) + } + + d := AttributeDenylist{Prefixes: []string{"k8s.node.label.karpenter.k8s.aws/instance-"}, Keys: map[string]struct{}{}} + require.Empty(t, d.NeverDeniedViolations(), "the instance- prefix spares karpenter.k8s.aws/ec2nodeclass") +} + +func TestLabelExcludesDenylist(t *testing.T) { + d := LabelExcludes{Pod: []string{"app.kubernetes.io/*", "team", "*"}, Node: []string{"karpenter.sh/nodepool"}}.Denylist() + require.Equal(t, []string{ + `key "k8s.node.label.karpenter.sh/nodepool"`, + `key "k8s.pod.label.team"`, + `prefix "k8s.pod.label."`, + `prefix "k8s.pod.label.app.kubernetes.io/"`, + }, d.Entries()) + require.Empty(t, LabelExcludes{}.Denylist().Entries()) +} + +func TestSplitCustomerExcludes(t *testing.T) { + // The chart's render of podLabels.exclude: ["app.kubernetes.io/*"] and nodeLabels.exclude: ["karpenter.sh/nodepool"]. + excluded := mustDenylist(t) + excluded.Prefixes = append(excluded.Prefixes, "k8s.pod.label.app.kubernetes.io/") + excluded.Keys["k8s.node.label.karpenter.sh/nodepool"] = struct{}{} + chartEntries := mustDenylist(t).Entries() + + t.Run("no declared excludes", func(t *testing.T) { + t.Setenv(PodLabelExcludesEnvVar, "") + t.Setenv(NodeLabelExcludesEnvVar, "") + chart, customer := excluded.SplitCustomerExcludes(ResolveLabelExcludes()) + require.Empty(t, customer.Entries()) + require.Equal(t, excluded.Entries(), chart.Entries()) + require.Len(t, chart.NeverDeniedViolations(), 4, "the protected label entries fail") + require.Equal(t, excluded.Processor, chart.Processor) + }) + + t.Run("matching declared excludes", func(t *testing.T) { + t.Setenv(PodLabelExcludesEnvVar, "app.kubernetes.io/*") + t.Setenv(NodeLabelExcludesEnvVar, "karpenter.sh/nodepool") + chart, customer := excluded.SplitCustomerExcludes(ResolveLabelExcludes()) + require.Equal(t, chartEntries, chart.Entries()) + require.Empty(t, chart.NeverDeniedViolations()) + require.Equal(t, []string{`key "k8s.node.label.karpenter.sh/nodepool"`, `prefix "k8s.pod.label.app.kubernetes.io/"`}, customer.Entries()) + require.Len(t, customer.NeverDeniedViolations(), 4, "the excludes remove three protected app.kubernetes.io/ pod labels and the node pool label") + }) + + for name, e := range map[string]LabelExcludes{ + "key pattern for a prefix entry": {Pod: []string{"app.kubernetes.io/name"}}, + "other prefix": {Pod: []string{"app.kubernetes.io/n*"}}, + "pod pattern for a node entry": {Pod: []string{"karpenter.sh/nodepool"}}, + "node pattern for a pod entry": {Node: []string{"app.kubernetes.io/*"}}, + "pattern with the attribute scope": {Pod: []string{"k8s.pod.label.app.kubernetes.io/*"}}, + } { + t.Run("non-matching "+name, func(t *testing.T) { + chart, customer := excluded.SplitCustomerExcludes(e) + require.Empty(t, customer.Entries()) + require.Equal(t, excluded.Entries(), chart.Entries()) + require.NotEmpty(t, chart.NeverDeniedViolations()) + }) + } + + t.Run("an undeclared wider chart entry fails", func(t *testing.T) { + d := AttributeDenylist{Prefixes: []string{"k8s.node.label.kubernetes.io/"}, Keys: map[string]struct{}{}} + chart, _ := d.SplitCustomerExcludes(LabelExcludes{Pod: []string{"app.kubernetes.io/*"}}) + require.Len(t, chart.NeverDeniedViolations(), 2, "the prefix removes kubernetes.io/os and kubernetes.io/arch") + }) +} + +func TestReconcileObjects(t *testing.T) { + d := mustDenylist(t) + nodes := map[string]map[string]string{ + "n1": {"kubernetes.io/os": "linux", "role": "new"}, + "n2": {"kubernetes.io/os": "linux", "topology.kubernetes.io/zone": "a"}, + } + series := func(node string, attrs map[string]string) MetricResult { + res := map[string]string{"k8s.node.name": node} + for k, v := range attrs { + res["k8s.node.label."+k] = v + } + return MetricResult{Labels: MetricLabels{Resource: res}} + } + objectOf := func(l MetricLabels) (string, map[string]string, bool) { + name := l.Resource["k8s.node.name"] + labels, ok := nodes[name] + return name, labels, ok + } + + got := ReconcileObjects([]MetricResult{ + series("n1", map[string]string{"kubernetes.io/os": "linux", "role": "old"}), // before a relabel + series("n1", map[string]string{"kubernetes.io/os": "linux", "role": "new"}), + series("n2", map[string]string{"topology.kubernetes.io/zone": "a"}), // missing and leaked + series("n2", map[string]string{"kubernetes.io/os": "windows"}), // mismatched only + series("gone", map[string]string{}), + }, objectOf, []string{"n1", "n2"}, mustExtraction(t, extractAllConfig, "node"), d) + + require.Equal(t, 2, got.Objects) + require.Equal(t, 2, got.Required) + require.Empty(t, got.NoSeries) + require.Equal(t, 2, got.Labels, "only the passing n1 series counts") + require.Equal(t, map[string]LabelDiff{"n2": {Checked: 1, Mismatched: []string{`kubernetes.io/os: got "windows", want "linux"`}}}, + got.Failed, "n2 keeps its closest series; the older n1 series no longer counts against it") +} + +func TestReconcileObjectsRequiresASeries(t *testing.T) { + all := mustExtraction(t, extractAllConfig, "node") + nodes := map[string]map[string]string{"n1": {}, "n2": {}, "n3": {}, "capped": {}} + series := func(node string, extra ...string) MetricResult { + res := map[string]string{"k8s.node.name": node} + for _, k := range extra { + res[k] = "x" + } + return MetricResult{Labels: MetricLabels{Resource: res}} + } + objectOf := func(l MetricLabels) (string, map[string]string, bool) { + name := l.Resource["k8s.node.name"] + labels, ok := nodes[name] + return name, labels, ok + } + + got := ReconcileObjects([]MetricResult{ + series("n1"), + series("capped", "a", "b"), // at the cap of 3, so not reconciled, but still a series + series("gone"), // an object the API no longer lists + }, objectOf, []string{"n3", "n1", "capped", "n2"}, all, AttributeDenylist{MaxTotal: 3}) + + require.Equal(t, 1, got.Objects) + require.Equal(t, 1, got.AtCap) + require.Equal(t, 4, got.Required) + require.Equal(t, []string{"n2", "n3"}, got.NoSeries, "sorted, and a series at the cap counts as a series") + require.Empty(t, got.Failed) + + got = ReconcileObjects([]MetricResult{series("n1")}, objectOf, nil, all, AttributeDenylist{}) + require.Empty(t, got.NoSeries, "with nothing required, an object without a series passes") +} diff --git a/util/otelmetrics/namespace_filter.go b/util/otelmetrics/namespace_filter.go new file mode 100644 index 000000000..9a1d7b147 --- /dev/null +++ b/util/otelmetrics/namespace_filter.go @@ -0,0 +1,343 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package otelmetrics + +import ( + "fmt" + "regexp" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +const namespaceAttribute = "k8s.namespace.name" + +// NamespaceFilter is the metric conditions of the filter processors in a +// collector configuration that read k8s.namespace.name. A series whose +// namespace makes any condition true is dropped. +type NamespaceFilter struct { + Processors []string + conditions []ottlNode +} + +// ParseNamespaceFilter reads every metric and datapoint condition that names +// k8s.namespace.name from the filter processors of a rendered collector +// configuration. A condition outside the subset of OTTL that the chart renders +// (and, or, not, ==, != and IsMatch over resource.attributes["k8s.namespace.name"], +// string literals, true, false and nil) returns an error that wraps +// ErrFilterNotModeled. A configuration without such conditions has an empty +// filter that drops nothing. +func ParseNamespaceFilter(otelConfigYAML string) (NamespaceFilter, error) { + var cfg struct { + Processors map[string]yaml.Node `yaml:"processors"` + } + if err := yaml.Unmarshal([]byte(otelConfigYAML), &cfg); err != nil { + return NamespaceFilter{}, fmt.Errorf("parsing collector configuration: %w", err) + } + var f NamespaceFilter + for _, name := range sortedProcessors(cfg.Processors, "filter") { + node := cfg.Processors[name] + var p struct { + Metrics struct { + Metric []string `yaml:"metric"` + Datapoint []string `yaml:"datapoint"` + } `yaml:"metrics"` + } + if err := node.Decode(&p); err != nil { + return NamespaceFilter{}, fmt.Errorf("parsing processor %s: %w", name, err) + } + found := false + for _, condition := range append(p.Metrics.Metric, p.Metrics.Datapoint...) { + if !strings.Contains(condition, namespaceAttribute) { + continue + } + n, err := parseOTTLCondition(condition) + if err != nil { + return NamespaceFilter{}, fmt.Errorf("%w: processor %s condition %q: %v", ErrFilterNotModeled, name, condition, err) + } + f.conditions = append(f.conditions, n) + found = true + } + if found { + f.Processors = append(f.Processors, name) + } + } + return f, nil +} + +// Drops reports whether the filter drops the series of a namespace. +func (f NamespaceFilter) Drops(namespace string) bool { + for _, c := range f.conditions { + if c.eval(namespace).b { + return true + } + } + return false +} + +// Active reports whether the filter has any condition. +func (f NamespaceFilter) Active() bool { + return len(f.conditions) > 0 +} + +// --------------------------------------------------------------------------- +// The OTTL subset. Each node evaluates for a series whose namespace is set, so +// the namespace path is always a string and never nil. +// --------------------------------------------------------------------------- + +type ottlKind int + +const ( + ottlBool ottlKind = iota + ottlString + ottlNil +) + +type ottlValue struct { + kind ottlKind + b bool + s string +} + +type ottlNode struct { + kind ottlKind + eval func(namespace string) ottlValue +} + +type ottlToken struct { + text string // an operator, a parenthesis, a comma or a name + str string // the value of a string literal + isStr bool + index string // the key of a name["key"] +} + +func tokenizeOTTL(s string) ([]ottlToken, error) { + var tokens []ottlToken + for i := 0; i < len(s); { + c := s[i] + switch { + case c == ' ' || c == '\t' || c == '\n': + i++ + case c == '(' || c == ')' || c == ',': + tokens = append(tokens, ottlToken{text: string(c)}) + i++ + case strings.HasPrefix(s[i:], "==") || strings.HasPrefix(s[i:], "!="): + tokens = append(tokens, ottlToken{text: s[i : i+2]}) + i += 2 + case c == '"': + lit, n, err := scanOTTLString(s[i:]) + if err != nil { + return nil, err + } + tokens = append(tokens, ottlToken{str: lit, isStr: true}) + i += n + case c == '_' || c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z': + j := i + for j < len(s) && (s[j] == '_' || s[j] == '.' || s[j] >= '0' && s[j] <= '9' || s[j] >= 'A' && s[j] <= 'Z' || s[j] >= 'a' && s[j] <= 'z') { + j++ + } + tok := ottlToken{text: s[i:j]} + if j < len(s) && s[j] == '[' { + lit, n, err := scanOTTLString(s[j+1:]) + if err != nil || j+1+n >= len(s) || s[j+1+n] != ']' { + return nil, fmt.Errorf("unsupported index after %s", tok.text) + } + tok.index = lit + j += n + 2 + } + tokens = append(tokens, tok) + i = j + default: + return nil, fmt.Errorf("unsupported character %q", c) + } + } + return tokens, nil +} + +// scanOTTLString reads the string literal at the start of s and returns its +// value and its length in s. +func scanOTTLString(s string) (string, int, error) { + if !strings.HasPrefix(s, `"`) { + return "", 0, fmt.Errorf("want a string literal") + } + for i := 1; i < len(s); i++ { + switch s[i] { + case '\\': + i++ + case '"': + v, err := strconv.Unquote(s[:i+1]) + return v, i + 1, err + } + } + return "", 0, fmt.Errorf("unterminated string literal") +} + +type ottlParser struct { + tokens []ottlToken + pos int +} + +func parseOTTLCondition(condition string) (ottlNode, error) { + tokens, err := tokenizeOTTL(condition) + if err != nil { + return ottlNode{}, err + } + p := &ottlParser{tokens: tokens} + n, err := p.or() + if err != nil { + return ottlNode{}, err + } + if p.pos != len(p.tokens) { + return ottlNode{}, fmt.Errorf("unexpected %q", p.tokens[p.pos].text) + } + if n.kind != ottlBool { + return ottlNode{}, fmt.Errorf("the condition is not a boolean") + } + return n, nil +} + +func (p *ottlParser) peek(text string) bool { + return p.pos < len(p.tokens) && !p.tokens[p.pos].isStr && p.tokens[p.pos].index == "" && p.tokens[p.pos].text == text +} + +func (p *ottlParser) expect(text string) error { + if !p.peek(text) { + return fmt.Errorf("want %q", text) + } + p.pos++ + return nil +} + +func (p *ottlParser) or() (ottlNode, error) { + return p.binary("or", p.and, func(a, b bool) bool { return a || b }) +} + +func (p *ottlParser) and() (ottlNode, error) { + return p.binary("and", p.not, func(a, b bool) bool { return a && b }) +} + +func (p *ottlParser) binary(op string, next func() (ottlNode, error), combine func(a, b bool) bool) (ottlNode, error) { + left, err := next() + if err != nil { + return ottlNode{}, err + } + for p.peek(op) { + p.pos++ + right, err := next() + if err != nil { + return ottlNode{}, err + } + if left.kind != ottlBool || right.kind != ottlBool { + return ottlNode{}, fmt.Errorf("%q needs boolean operands", op) + } + l, r := left.eval, right.eval + left = ottlNode{kind: ottlBool, eval: func(ns string) ottlValue { + return ottlValue{kind: ottlBool, b: combine(l(ns).b, r(ns).b)} + }} + } + return left, nil +} + +func (p *ottlParser) not() (ottlNode, error) { + if !p.peek("not") { + return p.comparison() + } + p.pos++ + n, err := p.not() + if err != nil { + return ottlNode{}, err + } + if n.kind != ottlBool { + return ottlNode{}, fmt.Errorf(`"not" needs a boolean operand`) + } + return ottlNode{kind: ottlBool, eval: func(ns string) ottlValue { + return ottlValue{kind: ottlBool, b: !n.eval(ns).b} + }}, nil +} + +func (p *ottlParser) comparison() (ottlNode, error) { + left, err := p.operand() + if err != nil { + return ottlNode{}, err + } + if !p.peek("==") && !p.peek("!=") { + return left, nil + } + equal := p.tokens[p.pos].text == "==" + p.pos++ + right, err := p.operand() + if err != nil { + return ottlNode{}, err + } + return ottlNode{kind: ottlBool, eval: func(ns string) ottlValue { + return ottlValue{kind: ottlBool, b: (left.eval(ns) == right.eval(ns)) == equal} + }}, nil +} + +func (p *ottlParser) operand() (ottlNode, error) { + if p.pos >= len(p.tokens) { + return ottlNode{}, fmt.Errorf("unexpected end of condition") + } + tok := p.tokens[p.pos] + p.pos++ + constant := func(v ottlValue) ottlNode { + return ottlNode{kind: v.kind, eval: func(string) ottlValue { return v }} + } + switch { + case tok.isStr: + return constant(ottlValue{kind: ottlString, s: tok.str}), nil + case tok.index != "": + if tok.text != "resource.attributes" || tok.index != namespaceAttribute { + return ottlNode{}, fmt.Errorf("unsupported path %s[%q]", tok.text, tok.index) + } + return ottlNode{kind: ottlString, eval: func(ns string) ottlValue { return ottlValue{kind: ottlString, s: ns} }}, nil + case tok.text == "(": + n, err := p.or() + if err != nil { + return ottlNode{}, err + } + return n, p.expect(")") + case tok.text == "true" || tok.text == "false": + return constant(ottlValue{kind: ottlBool, b: tok.text == "true"}), nil + case tok.text == "nil": + return constant(ottlValue{kind: ottlNil}), nil + case tok.text == "IsMatch": + return p.isMatch() + default: + return ottlNode{}, fmt.Errorf("unsupported %q", tok.text) + } +} + +// isMatch parses the arguments of IsMatch(target, "pattern"). Like OTTL, it +// matches anywhere in the target unless the pattern is anchored. +func (p *ottlParser) isMatch() (ottlNode, error) { + if err := p.expect("("); err != nil { + return ottlNode{}, err + } + target, err := p.operand() + if err != nil { + return ottlNode{}, err + } + if err := p.expect(","); err != nil { + return ottlNode{}, err + } + if p.pos >= len(p.tokens) || !p.tokens[p.pos].isStr { + return ottlNode{}, fmt.Errorf("IsMatch needs a string literal pattern") + } + re, err := regexp.Compile(p.tokens[p.pos].str) + if err != nil { + return ottlNode{}, fmt.Errorf("IsMatch pattern: %w", err) + } + p.pos++ + if err := p.expect(")"); err != nil { + return ottlNode{}, err + } + if target.kind != ottlString { + return ottlNode{}, fmt.Errorf("IsMatch needs a string target") + } + return ottlNode{kind: ottlBool, eval: func(ns string) ottlValue { + return ottlValue{kind: ottlBool, b: re.MatchString(target.eval(ns).s)} + }}, nil +} diff --git a/util/otelmetrics/namespace_filter_test.go b/util/otelmetrics/namespace_filter_test.go new file mode 100644 index 000000000..a717d6761 --- /dev/null +++ b/util/otelmetrics/namespace_filter_test.go @@ -0,0 +1,97 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package otelmetrics + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" +) + +// namespaceFilterConfig has the conditions the chart renders for the metrics +// namespaces include [kube-*, amazon-cloudwatch, default] and exclude +// [kube-public], next to filters that do not read the namespace. +const namespaceFilterConfig = ` +processors: + filter/cw_k8s_ci_v0_cadvisor_pod: + metrics: + datapoint: + - attributes["container"] == "POD" + filter/cw_k8s_ci_v0_scrape_metadata: + metrics: + metric: + - IsMatch(name, "^(up|scrape_duration_seconds)$") + filter/cw_k8s_ci_v0_metrics_namespaces: + error_mode: ignore + metrics: + datapoint: + - 'resource.attributes["k8s.namespace.name"] != nil and not (IsMatch(resource.attributes["k8s.namespace.name"], "^kube-") or resource.attributes["k8s.namespace.name"] == "amazon-cloudwatch" or resource.attributes["k8s.namespace.name"] == "default")' + - 'resource.attributes["k8s.namespace.name"] != nil and (resource.attributes["k8s.namespace.name"] == "kube-public")' + k8sattributes/cw_k8s_ci_v0_pod: + pod_association: + - sources: [{from: resource_attribute, name: k8s.namespace.name}] +` + +func TestParseNamespaceFilter(t *testing.T) { + f, err := ParseNamespaceFilter(namespaceFilterConfig) + require.NoError(t, err) + require.True(t, f.Active()) + require.Equal(t, []string{"filter/cw_k8s_ci_v0_metrics_namespaces"}, f.Processors, "only filters that read the namespace") + for ns, dropped := range map[string]bool{ + "kube-system": false, + "amazon-cloudwatch": false, + "default": false, + "kube-public": true, // excluded, and exclude wins + "monitoring": true, // not included + "my-kube-app": true, // the prefix is anchored + "default-2": true, // an exact entry is exact + } { + require.Equal(t, dropped, f.Drops(ns), ns) + } + + f, err = ParseNamespaceFilter("processors:\n batch/x: {}\n") + require.NoError(t, err) + require.False(t, f.Active()) + require.False(t, f.Drops("default"), "without a namespace filter nothing is dropped") +} + +func TestNamespaceFilterConditions(t *testing.T) { + const ns = `resource.attributes["k8s.namespace.name"]` + for cond, want := range map[string]map[string]bool{ + ns + ` != nil and (true)`: {"a": true, "b": true}, // exclude: ["*"] + ns + ` == "a"`: {"a": true, "b": false}, + ns + ` != "a"`: {"a": false, "b": true}, + ns + ` == nil`: {"a": false}, + `not ` + ns + ` == "a"`: {"a": false, "b": true}, + `IsMatch(` + ns + `, "^a\\.b") != true`: {"a.b": false, "axb": true}, + ns + ` == "a" or ` + ns + ` == "b" and false`: {"a": true, "b": false}, // and binds tighter + `(` + ns + ` == "a" or ` + ns + ` == "b") and true`: {"a": true, "b": true, "c": false}, + } { + n, err := parseOTTLCondition(cond) + require.NoError(t, err, cond) + for namespace, dropped := range want { + require.Equal(t, dropped, n.eval(namespace).b, "%s for %s", cond, namespace) + } + } +} + +func TestNamespaceFilterNotModeled(t *testing.T) { + for name, cond := range map[string]string{ + "another attribute": `resource.attributes["k8s.namespace.name"] == "a" and resource.attributes["k8s.pod.name"] == "p"`, + "datapoint attribute": `attributes["k8s.namespace.name"] == "a"`, + "metric name": `resource.attributes["k8s.namespace.name"] == "a" and name == "m"`, + "other converter": `Concat([resource.attributes["k8s.namespace.name"]], "") == "a"`, + "ordering": `resource.attributes["k8s.namespace.name"] > "a"`, + "not a boolean": `resource.attributes["k8s.namespace.name"]`, + "unbalanced": `(resource.attributes["k8s.namespace.name"] == "a"`, + "pattern not literal": `IsMatch(resource.attributes["k8s.namespace.name"], resource.attributes["k8s.namespace.name"])`, + } { + t.Run(name, func(t *testing.T) { + _, err := ParseNamespaceFilter("processors:\n filter/x:\n metrics:\n metric:\n - '" + cond + "'\n") + require.Error(t, err) + require.True(t, errors.Is(err, ErrFilterNotModeled), "%v", err) + }) + } +} diff --git a/util/otelmetrics/query_cache.go b/util/otelmetrics/query_cache.go index 51aef0f4f..f6e2e2152 100644 --- a/util/otelmetrics/query_cache.go +++ b/util/otelmetrics/query_cache.go @@ -32,6 +32,8 @@ type cacheEntry struct { // QueryCache provides session-scoped caching of PromQL queries. // Each unique metric name is queried exactly once; subsequent calls return cached data. // Concurrent requests for the same metric are deduplicated via singleflight. +// Every fetch goes through QueryComplete, so results are never silently truncated, and the +// concurrency limit bounds HTTP requests, so shard re-fetches stay inside it. type QueryCache struct { mu sync.RWMutex filtered map[string]cacheEntry @@ -59,6 +61,12 @@ func WithMaxConcurrency(n int) QueryCacheOption { return func(qc *QueryCache) { qc.sem = make(chan struct{}, n) } } +// WithSharedConcurrency makes the requests of this cache and of other count against other's +// concurrency limit, so two caches over one endpoint do not double the parallel load on it. +func WithSharedConcurrency(other *QueryCache) QueryCacheOption { + return func(qc *QueryCache) { qc.sem = other.sem } +} + func NewQueryCache(client *OtelMetricsClient, clusterName string, opts ...QueryCacheOption) *QueryCache { qc := &QueryCache{ filtered: make(map[string]cacheEntry), @@ -140,36 +148,18 @@ func (qc *QueryCache) Get(ctx context.Context, metricName string) ([]MetricResul return entry.results, entry.err } -// fetchFiltered performs the actual HTTP query for a filtered metric. +// fetchFiltered performs the actual HTTP query for a filtered metric, split by host type only +// when the registry (or WithHostTypes) gives the metric host types. func (qc *QueryCache) fetchFiltered(ctx context.Context, metricName string) cacheEntry { + ctx = withRequestSlots(ctx, qc.sem) escaped := promqlEscaper.Replace(qc.cluster) sel := promqlMetricSelector(metricName) var results []MetricResult var firstErr error - var targetHosts []string - clusterScopedOnly := false - - if qc.registry != nil { - if qc.registry.IsClusterScoped(metricName) { - clusterScopedOnly = true - } else { - targetHosts = qc.registry.HostTypesFor(metricName) - } - } else if len(qc.hostTypes) > 0 { - targetHosts = qc.hostTypes - } - - if clusterScopedOnly { - promql := fmt.Sprintf(`%s"@resource.k8s.cluster.name"="%s","@resource.host.type"=""}`, sel, escaped) - r, err := qc.client.Query(ctx, promql) - if err != nil { - firstErr = err - } else { - results = append(results, r...) - } - } else if len(targetHosts) > 0 { + targetHosts := qc.hostTypesFor(metricName) + if len(targetHosts) > 0 { type queryResult struct { results []MetricResult err error @@ -181,15 +171,8 @@ func (qc *QueryCache) fetchFiltered(ctx context.Context, metricName string) cach wg.Add(1) go func(hostType string) { defer wg.Done() - select { - case qc.sem <- struct{}{}: - defer func() { <-qc.sem }() - case <-ctx.Done(): - ch <- queryResult{err: ctx.Err()} - return - } promql := fmt.Sprintf(`%s"@resource.k8s.cluster.name"="%s","@resource.host.type"="%s"}`, sel, escaped, hostType) - r, err := qc.client.Query(ctx, promql) + r, err := qc.client.QueryComplete(ctx, promql) ch <- queryResult{results: r, err: err} }(ht) } @@ -206,30 +189,35 @@ func (qc *QueryCache) fetchFiltered(ctx context.Context, metricName string) cach results = append(results, qr.results...) } } else { + // Cluster-scoped metrics are read here without a host.type matcher, so leaked host keys stay visible. promql := fmt.Sprintf(`%s"@resource.k8s.cluster.name"="%s"}`, sel, escaped) - r, err := qc.client.Query(ctx, promql) + r, err := qc.client.QueryComplete(ctx, promql) results = r firstErr = err } - if len(results) == 0 && firstErr != nil { + // One failed host type makes the union incomplete, so it fails the whole fetch. + if firstErr != nil { slog.Debug("query failed", "metric", metricName, "error", firstErr) return cacheEntry{err: firstErr} } return cacheEntry{results: results} } +// hostTypesFor returns the host types Get splits the metric's query by; Get sees no series of +// other host types. Empty means one query with only the cluster matcher. +func (qc *QueryCache) hostTypesFor(metricName string) []string { + if qc.registry != nil { + return qc.registry.HostTypesFor(metricName) + } + return qc.hostTypes +} + // GetWithFilter returns results with additional PromQL label filters. Not cached. -// Shares the cache's concurrency limit, so a suite of parallel filtered queries -// doesn't burst past the endpoint's rate limit. +// Each HTTP request holds one slot of the cache's concurrency limit, so a suite of +// parallel filtered queries, and the shard re-fetches of a truncated one, stay inside it. func (qc *QueryCache) GetWithFilter(ctx context.Context, metricName string, extraFilters map[string]string) ([]MetricResult, error) { - select { - case qc.sem <- struct{}{}: - defer func() { <-qc.sem }() - case <-ctx.Done(): - return nil, ctx.Err() - } - return qc.client.Query(ctx, promqlMetricSelector(metricName)+qc.filterMatchers(extraFilters)+"}") + return qc.client.QueryComplete(withRequestSlots(ctx, qc.sem), promqlMetricSelector(metricName)+qc.filterMatchers(extraFilters)+"}") } // filterMatchers renders the cluster matcher plus extraFilters as PromQL label @@ -267,7 +255,7 @@ func (qc *QueryCache) GetUnfiltered(ctx context.Context, metricName string) ([]M } qc.mu.Unlock() - results, queryErr := qc.client.Query(ctx, metricName) + results, queryErr := qc.client.QueryComplete(withRequestSlots(ctx, qc.sem), metricName) entry := cacheEntry{results: results, err: queryErr} // Do not cache empty successful results; a transient miss would poison diff --git a/util/otelmetrics/query_cache_test.go b/util/otelmetrics/query_cache_test.go index 3baf46fd3..309c784d1 100644 --- a/util/otelmetrics/query_cache_test.go +++ b/util/otelmetrics/query_cache_test.go @@ -11,9 +11,12 @@ import ( "net/http/httptest" "sync" "testing" + "time" "github.com/aws/aws-sdk-go-v2/aws" v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // newTestClient creates an OtelMetricsClient pointing at a test HTTP server. @@ -278,3 +281,81 @@ func TestFilterMatchers(t *testing.T) { } } } + +// A cluster-scoped series that wrongly gained host keys must reach the absence tests, not be filtered out. +func TestQueryCache_ClusterScopedReadsHostStampedSeries(t *testing.T) { + hostTypes := []string{"t3.medium"} + for _, tc := range []struct { + source MetricSource + metric string + }{ + {SourceKubeStateMetrics, "kube_deployment_status_replicas"}, + {SourceControlPlane, "apiserver_request_total"}, + } { + t.Run(tc.metric, func(t *testing.T) { + f, c := newFake(t, 10, []fakeSeries{ + {labels: map[string]string{"__name__": tc.metric, "@resource.k8s.cluster.name": "c1", "id": "clean"}}, + {labels: map[string]string{"__name__": tc.metric, "@resource.k8s.cluster.name": "c1", "id": "leaked", + "@resource.host.type": "t3.medium", "@resource.host.id": "i-0abc"}}, + {labels: map[string]string{"__name__": tc.metric, "@resource.k8s.cluster.name": "c2", "id": "other-cluster"}}, + }) + f.emptyMatchesAbsent = "@resource.host.type" + registry := NewSourceRegistry(hostTypes, + []SourceHostMapping{{Source: SourceCadvisor, HostTypes: hostTypes}, {Source: tc.source, HostTypes: nil}}, + SourceMapping{Source: tc.source, Metrics: []MetricDefinition{{Name: tc.metric}}}, + ) + qc := NewQueryCache(c, "c1", WithHostTypes(hostTypes), WithSourceRegistry(registry)) + + results, err := qc.Get(context.Background(), tc.metric) + require.NoError(t, err) + hostTypeByID := map[string]string{} + for _, r := range results { + hostTypeByID[r.Labels.Datapoint["id"]] = r.Labels.Resource["host.type"] + } + assert.Equal(t, map[string]string{"clean": "", "leaked": "t3.medium"}, hostTypeByID, + "the host-stamped series must be returned so the absence tests can fail on it") + for _, r := range f.queries() { + assert.NotContains(t, r.query, "host.type", "cluster-scoped metrics are read with only the cluster filter") + } + }) + } +} + +func TestQueryCache_SharedConcurrencyBoundsBothCaches(t *testing.T) { + f := &fakePromQL{t: t, series: podSeries(6, 5), limit: 10} + var inflight, peak int + var mu sync.Mutex + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + inflight++ + if inflight > peak { + peak = inflight + } + mu.Unlock() + time.Sleep(5 * time.Millisecond) + f.handler(w, r) + mu.Lock() + inflight-- + mu.Unlock() + })) + t.Cleanup(srv.Close) + c := newTestClient(srv.URL) + c.seriesLimit = 10 + pinned := NewQueryCache(c, "c1", WithMaxConcurrency(2)) + allNodes := NewQueryCache(c, "c1", WithSharedConcurrency(pinned)) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + var wg sync.WaitGroup + for _, qc := range []*QueryCache{pinned, pinned, allNodes, allNodes} { // each call needs several shard requests + wg.Add(1) + go func(qc *QueryCache) { + defer wg.Done() + results, err := qc.GetWithFilter(ctx, "kube_pod_info", nil) + assert.NoError(t, err) + assert.Len(t, results, 30) + }(qc) + } + wg.Wait() + assert.LessOrEqual(t, peak, 2, "the requests of both caches must count against one limit") +} diff --git a/util/otelmetrics/schema_keys.go b/util/otelmetrics/schema_keys.go new file mode 100644 index 000000000..7bc432217 --- /dev/null +++ b/util/otelmetrics/schema_keys.go @@ -0,0 +1,94 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package otelmetrics + +import ( + "context" + "fmt" +) + +// Attribute keys the OTel Container Insights configuration sets on many metric +// groups, shared by the attribute schema rows of every suite. +var ( + // Set by the Prometheus receiver on every scrape target. + PrometheusReceiverKeys = []string{"server.address", "server.port", "service.instance.id", "service.name", "url.scheme"} + // Set by resourcedetection/cw_k8s_ci_v0 (eks, ec2) and set_cloud_resource_id. + CloudKeys = []string{ + "cloud.account.id", "cloud.availability_zone", "cloud.platform", "cloud.provider", + "cloud.region", "cloud.resource_id", + } + // Set by resourcedetection/cw_k8s_ci_v0 (ec2) on the node agent, and by nodemetadataenricher + // on the cluster scraper while the node's Lease is fresh. + HostKeys = []string{"host.id", "host.image.id", "host.name", "host.type"} + // Set by the set_scope_* transform of every pipeline. + ScopeKeys = []string{"cloudwatch.pipeline", "cloudwatch.solution", "cloudwatch.source"} + // Set by k8sattributes/cw_k8s_ci_v0_pod from the pod's owner chain, so a bare pod has none; protected. + OwnerKindKeys = []string{ + "k8s.cronjob.name", "k8s.daemonset.name", "k8s.deployment.name", "k8s.job.name", + "k8s.replicaset.name", "k8s.statefulset.name", + } + // Set by set_workload from the owner kind keys; protected. + WorkloadKeys = []string{"k8s.workload.name", "k8s.workload.type"} +) + +// WorkloadImplies is set_workload's rule: any owner kind key sets both workload keys. +func WorkloadImplies() map[string][]string { + out := Together(WorkloadKeys...) + for _, k := range OwnerKindKeys { + out[k] = append(out[k], WorkloadKeys...) + } + return out +} + +// ConcatKeys joins key lists. +func ConcatKeys(groups ...[]string) []string { + return concatStrings(groups...) +} + +// Without returns keys minus the removed ones. +func Without(keys []string, removed ...string) []string { + drop := make(map[string]struct{}, len(removed)) + for _, k := range removed { + drop[k] = struct{}{} + } + var out []string + for _, k := range keys { + if _, ok := drop[k]; !ok { + out = append(out, k) + } + } + return out +} + +// Forgive may clear the problems of one series that a known transient state explains. +type Forgive func(l MetricLabels, resource SeriesCheck) SeriesCheck + +// TallySchema checks every series of metrics in the cluster against schema. It reads through +// qc.Get, so it shares one cached read per metric with the suite's other every-series checks. +// qc must split no metric by host type, because that would hide the series of other instance +// types, so TallySchema fails for such a cache. It also fails if a query fails or a metric has +// no series, because an empty group would pass vacuously. +func TallySchema(ctx context.Context, qc *QueryCache, metrics []string, schema GroupSchema, forgive Forgive) (*SchemaTally, error) { + tally := NewSchemaTally(schema) + for _, metricName := range metrics { + if hostTypes := qc.hostTypesFor(metricName); len(hostTypes) > 0 { + return nil, fmt.Errorf("%s: the cache reads only host types %v, so pass a cache without host types", metricName, hostTypes) + } + results, err := qc.Get(ctx, metricName) + if err != nil { + return nil, fmt.Errorf("querying %s: %w", metricName, err) + } + if len(results) == 0 { + return nil, fmt.Errorf("%s not available", metricName) + } + for _, r := range results { + resource, instrumentation := tally.Check(r.Labels) + if forgive != nil && !resource.OK() { + resource = forgive(r.Labels, resource) + } + tally.Record(r.Labels, resource, instrumentation) + } + } + return tally, nil +} diff --git a/util/otelmetrics/source_registry.go b/util/otelmetrics/source_registry.go index d70f0a569..08abe55c7 100644 --- a/util/otelmetrics/source_registry.go +++ b/util/otelmetrics/source_registry.go @@ -31,10 +31,10 @@ type SourceMapping struct { } // SourceHostMapping maps a MetricSource to the host types that produce it. -// Nil means cluster-scoped (query with host.type=""). +// Nil means one query with only the cluster filter, which returns series with or without host keys. type SourceHostMapping struct { Source MetricSource - HostTypes []string // nil = cluster-scoped + HostTypes []string // nil = cluster filter only } // SourceRegistry maps metric names to their MetricSource and resolves @@ -68,21 +68,27 @@ func NewSourceRegistry(allHostTypes []string, hostMappings []SourceHostMapping, } } -// HostTypesFor returns the host types that can produce the given metric. -// Returns nil for cluster-scoped sources. Returns allHostTypes for unknown metrics. +// HostTypesFor returns the host types to split the metric's query by; nil means one query with +// only the cluster filter. Returns nil for cluster-scoped sources whatever their host mapping, +// and allHostTypes for unknown metrics. func (sr *SourceRegistry) HostTypesFor(metricName string) []string { source, ok := sr.metricToSource[metricName] if !ok { return sr.allHostTypes } + if clusterScoped(source) { + return nil + } return sr.sourceToHosts[source] } -// IsClusterScoped returns true if the metric comes from a cluster-scoped source. +// IsClusterScoped returns true if the metric comes from a cluster-scoped source. Such a metric is +// read with only the cluster filter, so a series that wrongly gained host keys is still returned. func (sr *SourceRegistry) IsClusterScoped(metricName string) bool { source, ok := sr.metricToSource[metricName] - if !ok { - return false - } + return ok && clusterScoped(source) +} + +func clusterScoped(source MetricSource) bool { return source == SourceControlPlane || source == SourceKubeStateMetrics }