From 9b077ca11ed798c7e0ff2b8ea68e93809fa2b0df Mon Sep 17 00:00:00 2001 From: Petru Anica-Popa Date: Mon, 5 Oct 2026 13:22:08 +0000 Subject: [PATCH] feat(container-insights): drop duplicate cadvisor series and two collector self-telemetry metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent-side counterpart to the equivalent change in the amazon-cloudwatch-observability Helm chart. The chart and this translator each carry their own copy of the CI pipeline config and must stay in sync. cAdvisor reports most container_* families three times per pod: once per application container, once for the pod cgroup slice, and once for the pause/sandbox container. The latter two carry no `container` with `pod` set, and the slice value is the sum over the pod's containers, so both are duplicates of series already being sent. Add filter/cw_k8s_ci_v0_cadvisor_rollup to cadvisor.yaml, placed ahead of groupbyattrs and k8sattributes so a dropped datapoint never acquires the node and pod label sets. container_network_* is exempt: the sandbox owns the pod's network namespace, so for those eight families the sandbox series is the only series that exists. `container` is tested against both nil and "" because the Prometheus receiver strips empty-valued labels before they reach the pipeline. This was measured on a test cluster: of 19,186 rollup and sandbox datapoints in one sample, all 19,186 arrived with the `container` key absent from the map and none with it present and empty. The nil arm is the one that fires; the "" arm is kept for symmetry with filter/cw_k8s_ci_v0_cadvisor_empty. Also add node_scrape_collector_duration_seconds and node_scrape_collector_success to filter/cw_k8s_ci_v0_scrape_metadata. These are node-exporter's own per-collector scrape timing and status, the same category as the five names that filter already drops. node_textfile_scrape_error is deliberately left in place because the kubernetes-mixin alerting rules reference it. Note that every CI pipeline defines filter/cw_k8s_ci_v0_scrape_metadata under the same name and they are merged into one collector config, so the names have to be added to all ten copies. Updating only node_exporter.yaml left the merged result unchanged, because another pipeline's definition won. A comment to that effect is added to each copy. Golden sample configs regenerated. default_otel_config_aks.yaml, default_otel_config_gke.yaml and combined_v1_v2_eks_config.yaml are hand-edited because verifyToYamlTranslation applies token substitution to those goldens, so writing generated YAML back over them would destroy the placeholders. Verified with AWS_EC2_METADATA_DISABLED=true go test ./translator/... — the whole tocwconfig suite passes. Without that flag several unrelated tests fail on an IMDS lookup and TestOtlpMetricsConfigKubernetes panics, which aborts the test binary and silently skips everything after it; that behaviour is present on main. --- .../opentelemetry/combined_v1_v2_eks_config.yaml | 8 +++++++- .../container_insights_cluster_config.yaml | 2 +- .../container_insights_cluster_config_aks.yaml | 2 +- .../container_insights_cluster_config_gke.yaml | 2 +- .../container_insights_node_config.yaml | 8 +++++++- .../opentelemetry/default_otel_config_aks.yaml | 8 +++++++- .../opentelemetry/default_otel_config_gke.yaml | 8 +++++++- .../opentelemetry/containerinsights/apiserver.yaml | 4 +++- .../opentelemetry/containerinsights/cadvisor.yaml | 12 +++++++++++- .../opentelemetry/containerinsights/dcgm.yaml | 4 +++- .../opentelemetry/containerinsights/ebs_csi.yaml | 4 +++- .../opentelemetry/containerinsights/karpenter.yaml | 4 +++- .../opentelemetry/containerinsights/keda.yaml | 4 +++- .../containerinsights/kube_state_metrics.yaml | 4 +++- .../opentelemetry/containerinsights/lis_csi.yaml | 4 +++- .../opentelemetry/containerinsights/neuron.yaml | 4 +++- .../containerinsights/node_exporter.yaml | 4 +++- 17 files changed, 69 insertions(+), 17 deletions(-) diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml index 6aa73a976ea..357769b11f4 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml @@ -619,6 +619,11 @@ processors: metrics: datapoint: - attributes["container"] == "POD" + filter/cw_k8s_ci_v0_cadvisor_rollup: + error_mode: ignore + metrics: + datapoint: + - (attributes["container"] == nil or attributes["container"] == "") and attributes["pod"] != nil and attributes["pod"] != "" and IsMatch(metric.name, "^container_network_") != true filter/cw_k8s_ci_v0_neuron: error_mode: ignore metrics: @@ -628,7 +633,7 @@ processors: error_mode: ignore metrics: metric: - - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added)$") + - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added|node_scrape_collector_duration_seconds|node_scrape_collector_success)$") filter/dbi_exclude_monitor_0: error_mode: propagate logs: @@ -2878,6 +2883,7 @@ service: - transform/cw_k8s_ci_v0_set_cluster_name - filter/cw_k8s_ci_v0_cadvisor_empty - filter/cw_k8s_ci_v0_cadvisor_pod + - filter/cw_k8s_ci_v0_cadvisor_rollup - groupbyattrs/cw_k8s_ci_v0_cadvisor - transform/cw_k8s_ci_v0_cadvisor_promote - transform/cw_k8s_ci_v0_set_node_name diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/container_insights_cluster_config.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/container_insights_cluster_config.yaml index 00e352ae86d..2ab31ab874b 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/container_insights_cluster_config.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/container_insights_cluster_config.yaml @@ -90,7 +90,7 @@ processors: error_mode: ignore metrics: metric: - - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added)$") + - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added|node_scrape_collector_duration_seconds|node_scrape_collector_success)$") groupbyattrs/cw_k8s_ci_v0_ksm: keys: - pod diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/container_insights_cluster_config_aks.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/container_insights_cluster_config_aks.yaml index 1ce205360f9..7d9d6f85546 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/container_insights_cluster_config_aks.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/container_insights_cluster_config_aks.yaml @@ -90,7 +90,7 @@ processors: error_mode: ignore metrics: metric: - - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added)$") + - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added|node_scrape_collector_duration_seconds|node_scrape_collector_success)$") groupbyattrs/cw_k8s_ci_v0_ksm: keys: - pod diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/container_insights_cluster_config_gke.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/container_insights_cluster_config_gke.yaml index 22ed10b38d3..d7b6fb0b631 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/container_insights_cluster_config_gke.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/container_insights_cluster_config_gke.yaml @@ -90,7 +90,7 @@ processors: error_mode: ignore metrics: metric: - - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added)$") + - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added|node_scrape_collector_duration_seconds|node_scrape_collector_success)$") groupbyattrs/cw_k8s_ci_v0_ksm: keys: - pod diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/container_insights_node_config.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/container_insights_node_config.yaml index c0d5f70bbee..be15f482645 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/container_insights_node_config.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/container_insights_node_config.yaml @@ -172,6 +172,11 @@ processors: metrics: datapoint: - attributes["container"] == "POD" + filter/cw_k8s_ci_v0_cadvisor_rollup: + error_mode: ignore + metrics: + datapoint: + - (attributes["container"] == nil or attributes["container"] == "") and attributes["pod"] != nil and attributes["pod"] != "" and IsMatch(metric.name, "^container_network_") != true filter/cw_k8s_ci_v0_neuron: error_mode: ignore metrics: @@ -181,7 +186,7 @@ processors: error_mode: ignore metrics: metric: - - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added)$") + - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added|node_scrape_collector_duration_seconds|node_scrape_collector_success)$") groupbyattrs/cw_k8s_ci_v0_cadvisor: keys: - container @@ -1636,6 +1641,7 @@ service: - transform/cw_k8s_ci_v0_set_cluster_name - filter/cw_k8s_ci_v0_cadvisor_empty - filter/cw_k8s_ci_v0_cadvisor_pod + - filter/cw_k8s_ci_v0_cadvisor_rollup - groupbyattrs/cw_k8s_ci_v0_cadvisor - transform/cw_k8s_ci_v0_cadvisor_promote - transform/cw_k8s_ci_v0_set_node_name diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/default_otel_config_aks.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/default_otel_config_aks.yaml index 7967c4c8b9b..3d90f6e991f 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/default_otel_config_aks.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/default_otel_config_aks.yaml @@ -242,6 +242,11 @@ processors: metrics: datapoint: - attributes["container"] == "POD" + filter/cw_k8s_ci_v0_cadvisor_rollup: + error_mode: ignore + metrics: + datapoint: + - (attributes["container"] == nil or attributes["container"] == "") and attributes["pod"] != nil and attributes["pod"] != "" and IsMatch(metric.name, "^container_network_") != true filter/cw_k8s_ci_v0_neuron: error_mode: ignore metrics: @@ -251,7 +256,7 @@ processors: error_mode: ignore metrics: metric: - - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added)$") + - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added|node_scrape_collector_duration_seconds|node_scrape_collector_success)$") groupbyattrs/cw_k8s_ci_v0_cadvisor: keys: - container @@ -1628,6 +1633,7 @@ service: - transform/cw_k8s_ci_v0_set_cluster_name - filter/cw_k8s_ci_v0_cadvisor_empty - filter/cw_k8s_ci_v0_cadvisor_pod + - filter/cw_k8s_ci_v0_cadvisor_rollup - groupbyattrs/cw_k8s_ci_v0_cadvisor - transform/cw_k8s_ci_v0_cadvisor_promote - transform/cw_k8s_ci_v0_set_node_name diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/default_otel_config_gke.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/default_otel_config_gke.yaml index 6119bdff763..878ddf9ba04 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/default_otel_config_gke.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/default_otel_config_gke.yaml @@ -242,6 +242,11 @@ processors: metrics: datapoint: - attributes["container"] == "POD" + filter/cw_k8s_ci_v0_cadvisor_rollup: + error_mode: ignore + metrics: + datapoint: + - (attributes["container"] == nil or attributes["container"] == "") and attributes["pod"] != nil and attributes["pod"] != "" and IsMatch(metric.name, "^container_network_") != true filter/cw_k8s_ci_v0_neuron: error_mode: ignore metrics: @@ -251,7 +256,7 @@ processors: error_mode: ignore metrics: metric: - - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added)$") + - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added|node_scrape_collector_duration_seconds|node_scrape_collector_success)$") groupbyattrs/cw_k8s_ci_v0_cadvisor: keys: - container @@ -1627,6 +1632,7 @@ service: - transform/cw_k8s_ci_v0_set_cluster_name - filter/cw_k8s_ci_v0_cadvisor_empty - filter/cw_k8s_ci_v0_cadvisor_pod + - filter/cw_k8s_ci_v0_cadvisor_rollup - groupbyattrs/cw_k8s_ci_v0_cadvisor - transform/cw_k8s_ci_v0_cadvisor_promote - transform/cw_k8s_ci_v0_set_node_name diff --git a/translator/translate/otel/pipeline/opentelemetry/containerinsights/apiserver.yaml b/translator/translate/otel/pipeline/opentelemetry/containerinsights/apiserver.yaml index 53bc5162cf1..25d111d1f52 100644 --- a/translator/translate/otel/pipeline/opentelemetry/containerinsights/apiserver.yaml +++ b/translator/translate/otel/pipeline/opentelemetry/containerinsights/apiserver.yaml @@ -33,11 +33,13 @@ receivers: server_name: {{ .ApiserverTLSServerName }} {{- end }} processors: + # All CI pipelines declare this processor under one name and it is merged, so every + # copy must list the same names. node_scrape_collector_* come from node-exporter. filter/cw_k8s_ci_v0_scrape_metadata: error_mode: ignore metrics: metric: - - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added)$") + - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added|node_scrape_collector_duration_seconds|node_scrape_collector_success)$") transform/cw_k8s_ci_v0_set_unit: error_mode: ignore metric_statements: diff --git a/translator/translate/otel/pipeline/opentelemetry/containerinsights/cadvisor.yaml b/translator/translate/otel/pipeline/opentelemetry/containerinsights/cadvisor.yaml index cb45e020389..7b140ab594a 100644 --- a/translator/translate/otel/pipeline/opentelemetry/containerinsights/cadvisor.yaml +++ b/translator/translate/otel/pipeline/opentelemetry/containerinsights/cadvisor.yaml @@ -17,11 +17,13 @@ receivers: tls_config: insecure_skip_verify: true processors: + # All CI pipelines declare this processor under one name and it is merged, so every + # copy must list the same names. node_scrape_collector_* come from node-exporter. filter/cw_k8s_ci_v0_scrape_metadata: error_mode: ignore metrics: metric: - - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added)$") + - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added|node_scrape_collector_duration_seconds|node_scrape_collector_success)$") transform/cw_k8s_ci_v0_set_unit: error_mode: ignore metric_statements: @@ -78,6 +80,13 @@ processors: metrics: datapoint: - attributes["container"] == "POD" + # Drop the pod cgroup slice and pause/sandbox duplicates of each container_* family. + # container_network_* is sandbox-only so must stay; `container` arrives absent, not "". + filter/cw_k8s_ci_v0_cadvisor_rollup: + error_mode: ignore + metrics: + datapoint: + - (attributes["container"] == nil or attributes["container"] == "") and attributes["pod"] != nil and attributes["pod"] != "" and IsMatch(metric.name, "^container_network_") != true groupbyattrs/cw_k8s_ci_v0_cadvisor: keys: - container @@ -225,6 +234,7 @@ service: - transform/cw_k8s_ci_v0_set_cluster_name - filter/cw_k8s_ci_v0_cadvisor_empty - filter/cw_k8s_ci_v0_cadvisor_pod + - filter/cw_k8s_ci_v0_cadvisor_rollup - groupbyattrs/cw_k8s_ci_v0_cadvisor - transform/cw_k8s_ci_v0_cadvisor_promote - transform/cw_k8s_ci_v0_set_node_name diff --git a/translator/translate/otel/pipeline/opentelemetry/containerinsights/dcgm.yaml b/translator/translate/otel/pipeline/opentelemetry/containerinsights/dcgm.yaml index 389fdc3c4ba..c82acf09303 100644 --- a/translator/translate/otel/pipeline/opentelemetry/containerinsights/dcgm.yaml +++ b/translator/translate/otel/pipeline/opentelemetry/containerinsights/dcgm.yaml @@ -15,11 +15,13 @@ receivers: tls_config: ca_file: /etc/amazon-cloudwatch-observability-agent-client-cert/tls-ca.crt processors: + # All CI pipelines declare this processor under one name and it is merged, so every + # copy must list the same names. node_scrape_collector_* come from node-exporter. filter/cw_k8s_ci_v0_scrape_metadata: error_mode: ignore metrics: metric: - - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added)$") + - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added|node_scrape_collector_duration_seconds|node_scrape_collector_success)$") transform/cw_k8s_ci_v0_set_unit: error_mode: ignore metric_statements: diff --git a/translator/translate/otel/pipeline/opentelemetry/containerinsights/ebs_csi.yaml b/translator/translate/otel/pipeline/opentelemetry/containerinsights/ebs_csi.yaml index 4a464a724cd..6399e0a8112 100644 --- a/translator/translate/otel/pipeline/opentelemetry/containerinsights/ebs_csi.yaml +++ b/translator/translate/otel/pipeline/opentelemetry/containerinsights/ebs_csi.yaml @@ -27,11 +27,13 @@ receivers: scrape_interval: {{.CollectionInterval}} scrape_timeout: {{.ScrapeTimeout}} processors: + # All CI pipelines declare this processor under one name and it is merged, so every + # copy must list the same names. node_scrape_collector_* come from node-exporter. filter/cw_k8s_ci_v0_scrape_metadata: error_mode: ignore metrics: metric: - - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added)$") + - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added|node_scrape_collector_duration_seconds|node_scrape_collector_success)$") transform/cw_k8s_ci_v0_set_unit: error_mode: ignore metric_statements: diff --git a/translator/translate/otel/pipeline/opentelemetry/containerinsights/karpenter.yaml b/translator/translate/otel/pipeline/opentelemetry/containerinsights/karpenter.yaml index fcc4804c69f..caaafc9e078 100644 --- a/translator/translate/otel/pipeline/opentelemetry/containerinsights/karpenter.yaml +++ b/translator/translate/otel/pipeline/opentelemetry/containerinsights/karpenter.yaml @@ -28,11 +28,13 @@ receivers: - source_labels: [__meta_kubernetes_pod_node_name] target_label: node processors: + # All CI pipelines declare this processor under one name and it is merged, so every + # copy must list the same names. node_scrape_collector_* come from node-exporter. filter/cw_k8s_ci_v0_scrape_metadata: error_mode: ignore metrics: metric: - - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added)$") + - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added|node_scrape_collector_duration_seconds|node_scrape_collector_success)$") transform/cw_k8s_ci_v0_set_unit: error_mode: ignore metric_statements: diff --git a/translator/translate/otel/pipeline/opentelemetry/containerinsights/keda.yaml b/translator/translate/otel/pipeline/opentelemetry/containerinsights/keda.yaml index 3dfc977e35b..9ac1477090a 100644 --- a/translator/translate/otel/pipeline/opentelemetry/containerinsights/keda.yaml +++ b/translator/translate/otel/pipeline/opentelemetry/containerinsights/keda.yaml @@ -28,11 +28,13 @@ receivers: - source_labels: [__meta_kubernetes_pod_node_name] target_label: node processors: + # All CI pipelines declare this processor under one name and it is merged, so every + # copy must list the same names. node_scrape_collector_* come from node-exporter. filter/cw_k8s_ci_v0_scrape_metadata: error_mode: ignore metrics: metric: - - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added)$") + - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added|node_scrape_collector_duration_seconds|node_scrape_collector_success)$") # Drop generic client-go / control-plane / Go-runtime metrics KEDA exposes; they # collide with the apiserver pipeline which owns them under a different scope. filter/cw_k8s_ci_v0_keda_drop_non_keda: diff --git a/translator/translate/otel/pipeline/opentelemetry/containerinsights/kube_state_metrics.yaml b/translator/translate/otel/pipeline/opentelemetry/containerinsights/kube_state_metrics.yaml index 91a6a28495a..ac0ce6685d1 100644 --- a/translator/translate/otel/pipeline/opentelemetry/containerinsights/kube_state_metrics.yaml +++ b/translator/translate/otel/pipeline/opentelemetry/containerinsights/kube_state_metrics.yaml @@ -15,11 +15,13 @@ receivers: tls_config: ca_file: /etc/amazon-cloudwatch-observability-agent-cert/tls-ca.crt processors: + # All CI pipelines declare this processor under one name and it is merged, so every + # copy must list the same names. node_scrape_collector_* come from node-exporter. filter/cw_k8s_ci_v0_scrape_metadata: error_mode: ignore metrics: metric: - - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added)$") + - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added|node_scrape_collector_duration_seconds|node_scrape_collector_success)$") transform/cw_k8s_ci_v0_set_unit: error_mode: ignore metric_statements: diff --git a/translator/translate/otel/pipeline/opentelemetry/containerinsights/lis_csi.yaml b/translator/translate/otel/pipeline/opentelemetry/containerinsights/lis_csi.yaml index a718751efca..cd61d9dee24 100644 --- a/translator/translate/otel/pipeline/opentelemetry/containerinsights/lis_csi.yaml +++ b/translator/translate/otel/pipeline/opentelemetry/containerinsights/lis_csi.yaml @@ -27,11 +27,13 @@ receivers: scrape_interval: {{.CollectionInterval}} scrape_timeout: {{.ScrapeTimeout}} processors: + # All CI pipelines declare this processor under one name and it is merged, so every + # copy must list the same names. node_scrape_collector_* come from node-exporter. filter/cw_k8s_ci_v0_scrape_metadata: error_mode: ignore metrics: metric: - - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added)$") + - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added|node_scrape_collector_duration_seconds|node_scrape_collector_success)$") transform/cw_k8s_ci_v0_set_unit: error_mode: ignore metric_statements: diff --git a/translator/translate/otel/pipeline/opentelemetry/containerinsights/neuron.yaml b/translator/translate/otel/pipeline/opentelemetry/containerinsights/neuron.yaml index beeed7e846d..c2844d81c75 100644 --- a/translator/translate/otel/pipeline/opentelemetry/containerinsights/neuron.yaml +++ b/translator/translate/otel/pipeline/opentelemetry/containerinsights/neuron.yaml @@ -15,11 +15,13 @@ receivers: tls_config: ca_file: /etc/amazon-cloudwatch-observability-agent-client-cert/tls-ca.crt processors: + # All CI pipelines declare this processor under one name and it is merged, so every + # copy must list the same names. node_scrape_collector_* come from node-exporter. filter/cw_k8s_ci_v0_scrape_metadata: error_mode: ignore metrics: metric: - - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added)$") + - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added|node_scrape_collector_duration_seconds|node_scrape_collector_success)$") metricstarttime/cw_k8s_ci_v0: {} transform/cw_k8s_ci_v0_set_cluster_name: error_mode: ignore diff --git a/translator/translate/otel/pipeline/opentelemetry/containerinsights/node_exporter.yaml b/translator/translate/otel/pipeline/opentelemetry/containerinsights/node_exporter.yaml index d01e4535871..516a15a817e 100644 --- a/translator/translate/otel/pipeline/opentelemetry/containerinsights/node_exporter.yaml +++ b/translator/translate/otel/pipeline/opentelemetry/containerinsights/node_exporter.yaml @@ -15,11 +15,13 @@ receivers: tls_config: ca_file: /etc/amazon-cloudwatch-observability-agent-client-cert/tls-ca.crt processors: + # node-exporter's own per-collector scrape timing/status, same category as the five + # scrape_* names. node_textfile_scrape_error stays: kubernetes-mixin alerts on it. filter/cw_k8s_ci_v0_scrape_metadata: error_mode: ignore metrics: metric: - - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added)$") + - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added|node_scrape_collector_duration_seconds|node_scrape_collector_success)$") transform/cw_k8s_ci_v0_set_unit: error_mode: ignore metric_statements: