From 13764510b87ac3a5894ed534c8d28d5bd982dc6b Mon Sep 17 00:00:00 2001 From: Musa Asad Date: Tue, 16 Jun 2026 09:21:01 +0000 Subject: [PATCH 01/10] Register enable-prometheus-cr-watcher flag and fix PrometheusCR watcher startup The target-allocator declared the enable-prometheus-cr-watcher flag name as a constant but never registered it on the flag set, while the operator passes --enable-prometheus-cr-watcher whenever PrometheusCR.enabled is true. Because args are parsed with pflag.ExitOnError, the unregistered flag caused the binary to print 'unknown flag' and exit(2), putting the target-allocator pod into CrashLoopBackOff. This change registers the flag and ORs it with the YAML prometheus_cr.enabled setting, then fixes three latent defects that were previously unreachable because the binary crashed first: - promOperator: set a non-empty Namespace on the synthetic Prometheus object so the prometheus-operator config generator no longer panics with 'namespace can't be empty' in store.ForNamespace. - promOperator: set EvaluationInterval so the generated config does not render an empty global.evaluation_interval, which the prometheus config parser rejects with 'empty duration string'. - main: create and register service-discovery metrics and pass them to discovery.NewManager; passing a nil sdMetrics map makes every SD provider fail to register, yielding zero discovered targets. RELEASE_NOTES updated. --- RELEASE_NOTES | 6 ++++++ .../config/config.go | 9 +++++++++ .../config/flags.go | 5 +++++ .../config/flags_test.go | 13 ++++++++++++ .../main.go | 13 +++++++++++- .../watcher/promOperator.go | 20 +++++++++++++++++++ 6 files changed, 65 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 6128a2210..ff73e4d26 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,3 +1,9 @@ +======================================================================== +Amazon CloudWatch Agent Operator (Unreleased) +======================================================================== +Bug Fixes: +* [TargetAllocator] Register the `--enable-prometheus-cr-watcher` CLI flag so the target-allocator no longer exits with "unknown flag" (CrashLoopBackOff) when the operator passes it for PrometheusCR.enabled. The flag is OR'd with the YAML `prometheus_cr.enabled` setting. + ======================================================================== Amazon CloudWatch Agent Operator v3.5.0 (2026-06-05) ======================================================================== diff --git a/cmd/amazon-cloudwatch-agent-target-allocator/config/config.go b/cmd/amazon-cloudwatch-agent-target-allocator/config/config.go index 2272eedbe..6b2a3d81d 100644 --- a/cmd/amazon-cloudwatch-agent-target-allocator/config/config.go +++ b/cmd/amazon-cloudwatch-agent-target-allocator/config/config.go @@ -119,6 +119,15 @@ func LoadFromCLI(target *Config, flagSet *pflag.FlagSet) error { return err } + // Enable the Prometheus CR watcher when requested via the CLI flag. The YAML + // `prometheus_cr.enabled` value is loaded before this point, so OR the flag in + // rather than overwriting it: the watcher is enabled if either source sets it. + prometheusCREnabled, err := getPrometheusCREnabled(flagSet) + if err != nil { + return err + } + target.PrometheusCR.Enabled = target.PrometheusCR.Enabled || prometheusCREnabled + target.HTTPS.Enabled, err = getHttpsEnabled(flagSet) if err != nil { return err diff --git a/cmd/amazon-cloudwatch-agent-target-allocator/config/flags.go b/cmd/amazon-cloudwatch-agent-target-allocator/config/flags.go index caf639ed9..294f9bfe6 100644 --- a/cmd/amazon-cloudwatch-agent-target-allocator/config/flags.go +++ b/cmd/amazon-cloudwatch-agent-target-allocator/config/flags.go @@ -34,6 +34,7 @@ func getFlagSet(errorHandling pflag.ErrorHandling) *pflag.FlagSet { flagSet := pflag.NewFlagSet(targetAllocatorName, errorHandling) flagSet.String(configFilePathFlagName, DefaultConfigFilePath, "The path to the config file.") flagSet.String(kubeConfigPathFlagName, filepath.Join(homedir.HomeDir(), ".kube", "config"), "absolute path to the KubeconfigPath file") + flagSet.Bool(prometheusCREnabledFlagName, false, "Enable watching of Prometheus Operator custom resources (ServiceMonitor/PodMonitor) to dynamically generate scrape configs.") flagSet.Bool(reloadConfigFlagName, false, "Enable automatic configuration reloading. This functionality is deprecated and will be removed in a future release.") flagSet.Bool(httpsEnabledFlagName, true, "Enable HTTPS additional server") flagSet.String(listenAddrHttpsFlagName, ":8443", "The address where this service serves over HTTPS.") @@ -58,6 +59,10 @@ func getConfigReloadEnabled(flagSet *pflag.FlagSet) (bool, error) { return flagSet.GetBool(reloadConfigFlagName) } +func getPrometheusCREnabled(flagSet *pflag.FlagSet) (bool, error) { + return flagSet.GetBool(prometheusCREnabledFlagName) +} + func getHttpsListenAddr(flagSet *pflag.FlagSet) (string, error) { return flagSet.GetString(listenAddrHttpsFlagName) } diff --git a/cmd/amazon-cloudwatch-agent-target-allocator/config/flags_test.go b/cmd/amazon-cloudwatch-agent-target-allocator/config/flags_test.go index 8beed29d0..66c71a268 100644 --- a/cmd/amazon-cloudwatch-agent-target-allocator/config/flags_test.go +++ b/cmd/amazon-cloudwatch-agent-target-allocator/config/flags_test.go @@ -17,6 +17,7 @@ func TestGetFlagSet(t *testing.T) { // Check if each flag exists assert.NotNil(t, fs.Lookup(configFilePathFlagName), "Flag %s not found", configFilePathFlagName) assert.NotNil(t, fs.Lookup(kubeConfigPathFlagName), "Flag %s not found", kubeConfigPathFlagName) + assert.NotNil(t, fs.Lookup(prometheusCREnabledFlagName), "Flag %s not found", prometheusCREnabledFlagName) } func TestFlagGetters(t *testing.T) { @@ -45,6 +46,18 @@ func TestFlagGetters(t *testing.T) { expectedValue: true, getterFunc: func(fs *pflag.FlagSet) (interface{}, error) { return getConfigReloadEnabled(fs) }, }, + { + name: "GetPrometheusCREnabled", + flagArgs: []string{"--" + prometheusCREnabledFlagName}, + expectedValue: true, + getterFunc: func(fs *pflag.FlagSet) (interface{}, error) { return getPrometheusCREnabled(fs) }, + }, + { + name: "GetPrometheusCREnabledDefault", + flagArgs: []string{}, + expectedValue: false, + getterFunc: func(fs *pflag.FlagSet) (interface{}, error) { return getPrometheusCREnabled(fs) }, + }, { name: "InvalidFlag", flagArgs: []string{"--invalid-flag", "value"}, diff --git a/cmd/amazon-cloudwatch-agent-target-allocator/main.go b/cmd/amazon-cloudwatch-agent-target-allocator/main.go index dbc956ebf..57185184d 100644 --- a/cmd/amazon-cloudwatch-agent-target-allocator/main.go +++ b/cmd/amazon-cloudwatch-agent-target-allocator/main.go @@ -88,7 +88,18 @@ func main() { srv := server.NewServer(log, allocator, cfg.ListenAddr, httpOptions...) discoveryCtx, discoveryCancel := context.WithCancel(ctx) - discoveryManager = discovery.NewManager(discoveryCtx, nil, prometheus.NewRegistry(), nil) + // Service Discovery metrics MUST be created and passed to the discovery + // manager. Each SD provider's NewDiscoverer fails when its DiscovererMetrics + // is nil, so passing a nil sdMetrics map makes every provider (including + // kubernetes_sd) fail to register, yielding zero discovered targets. Mirror + // the upstream opentelemetry-operator target-allocator. + sdRegistry := prometheus.NewRegistry() + sdMetrics, sdMetricsErr := discovery.CreateAndRegisterSDMetrics(sdRegistry) + if sdMetricsErr != nil { + setupLog.Error(sdMetricsErr, "Unable to register service discovery metrics") + os.Exit(1) + } + discoveryManager = discovery.NewManager(discoveryCtx, nil, sdRegistry, sdMetrics) targetDiscoverer = target.NewDiscoverer(log, discoveryManager, allocatorPrehook, srv) collectorWatcher, collectorWatcherErr := collector.NewClient(log, cfg.ClusterConfig) diff --git a/cmd/amazon-cloudwatch-agent-target-allocator/watcher/promOperator.go b/cmd/amazon-cloudwatch-agent-target-allocator/watcher/promOperator.go index 2651e8d29..a15db9d78 100644 --- a/cmd/amazon-cloudwatch-agent-target-allocator/watcher/promOperator.go +++ b/cmd/amazon-cloudwatch-agent-target-allocator/watcher/promOperator.go @@ -21,6 +21,7 @@ import ( kubeDiscovery "github.com/prometheus/prometheus/discovery/kubernetes" "gopkg.in/yaml.v2" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" @@ -49,11 +50,30 @@ func NewPrometheusCRWatcher(logger logr.Logger, cfg allocatorconfig.Config) (*Pr } // TODO: We should make these durations configurable + // The synthetic Prometheus object must carry a non-empty namespace: the + // prometheus-operator config generator calls store.ForNamespace(prom.Namespace) + // when generating the server configuration, which panics on an empty namespace + // ("namespace can't be empty"). Mirror the upstream opentelemetry-operator + // target-allocator by setting it to the collector/TA namespace, derived from the + // operator-injected OTELCOL_NAMESPACE downward-API env (fallback: amazon-cloudwatch). + collectorNamespace := os.Getenv("OTELCOL_NAMESPACE") + if collectorNamespace == "" { + collectorNamespace = "amazon-cloudwatch" + } prom := &monitoringv1.Prometheus{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: collectorNamespace, + }, Spec: monitoringv1.PrometheusSpec{ CommonPrometheusFields: monitoringv1.CommonPrometheusFields{ ScrapeInterval: monitoringv1.Duration(cfg.PrometheusCR.ScrapeInterval.String()), }, + // EvaluationInterval must be non-empty: the prometheus-operator config + // generator renders it verbatim as global.evaluation_interval, and the + // downstream prometheus config parser rejects an empty value with + // "empty duration string". The CWA TA does not evaluate rules, so mirror + // the scrape interval as a sane non-empty default. + EvaluationInterval: monitoringv1.Duration(cfg.PrometheusCR.ScrapeInterval.String()), }, } From 0405f4decb019dd2ee173333a3985ea517b4ffb5 Mon Sep 17 00:00:00 2001 From: Musa Asad Date: Tue, 16 Jun 2026 15:57:02 +0000 Subject: [PATCH 02/10] test(target-allocator): guard scrape_protocols defaulting on config load Add a regression test asserting that loading a Target Allocator config whose static scrape job omits scrape_protocols still yields a non-empty ScrapeProtocols on every loaded scrape config. This is defaulted by the pinned Prometheus library during yaml.UnmarshalStrict into the prometheus Config type, so the distributed /scrape_configs payload is never empty and the agent's prometheus-receiver validation passes. The test fails fast if a future dependency or load-path change drops this defaulting. --- .../scrape_protocols_regression_test.go | 36 +++++++++++++++++++ .../scrape_protocols_omitted_test.yaml | 14 ++++++++ 2 files changed, 50 insertions(+) create mode 100644 cmd/amazon-cloudwatch-agent-target-allocator/config/scrape_protocols_regression_test.go create mode 100644 cmd/amazon-cloudwatch-agent-target-allocator/config/testdata/scrape_protocols_omitted_test.yaml diff --git a/cmd/amazon-cloudwatch-agent-target-allocator/config/scrape_protocols_regression_test.go b/cmd/amazon-cloudwatch-agent-target-allocator/config/scrape_protocols_regression_test.go new file mode 100644 index 000000000..9c432d7ee --- /dev/null +++ b/cmd/amazon-cloudwatch-agent-target-allocator/config/scrape_protocols_regression_test.go @@ -0,0 +1,36 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestScrapeProtocolsDefaultedOnLoad guards against "scrape_protocols cannot be empty" regression. +func TestScrapeProtocolsDefaultedOnLoad(t *testing.T) { + got := CreateDefaultConfig() + err := LoadFromFile("./testdata/scrape_protocols_omitted_test.yaml", &got) + require.NoError(t, err) + + require.NotNil(t, got.PromConfig) + require.NotEmpty(t, got.PromConfig.ScrapeConfigs) + + for _, sc := range got.PromConfig.ScrapeConfigs { + assert.NotEmpty(t, sc.ScrapeProtocols, + "scrape_protocols must be defaulted for job %q", sc.JobName) + } + + // Spot-check the specific static job from the reproduction by name. + found := false + for _, sc := range got.PromConfig.ScrapeConfigs { + if sc.JobName == "prometheus-sample-app" { + found = true + assert.Greater(t, len(sc.ScrapeProtocols), 0) + } + } + require.True(t, found, "expected the 'prometheus-sample-app' job to be present") +} diff --git a/cmd/amazon-cloudwatch-agent-target-allocator/config/testdata/scrape_protocols_omitted_test.yaml b/cmd/amazon-cloudwatch-agent-target-allocator/config/testdata/scrape_protocols_omitted_test.yaml new file mode 100644 index 000000000..9405f61e8 --- /dev/null +++ b/cmd/amazon-cloudwatch-agent-target-allocator/config/testdata/scrape_protocols_omitted_test.yaml @@ -0,0 +1,14 @@ +# Regression fixture: a scrape job that omits scrape_protocols. +# The TA config-load path must default scrape_protocols so the agent does not +# reject the config with "scrape_protocols cannot be empty". +config: + scrape_configs: + - job_name: prometheus-sample-app + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_label_app] + regex: prometheus-sample-app + action: keep + - target_label: label1 + replacement: value1 From d61d693fa6f0536786951e8e4d08f378a5e77c82 Mon Sep 17 00:00:00 2001 From: Musa Asad Date: Tue, 16 Jun 2026 16:01:54 +0000 Subject: [PATCH 03/10] fix(collector): roll pods when Prometheus config changes The pod-template restart-trigger sha256 was computed from Spec.Config only, so a change to Spec.Prometheus (rendered into a separate ConfigMap) left the pod template byte-identical and the workload controller did not roll the pods. Fold the serialized Spec.Prometheus (PrometheusConfig.Yaml()) into the hash input when it is non-empty, so a Prometheus-only change bumps the pod-template annotation and triggers a rolling restart, matching agent-config behavior. When no Prometheus config is set the hash input is byte-identical to the agent config alone, leaving non-Prometheus agents unaffected. --- RELEASE_NOTES | 6 -- .../config/config.go | 4 +- .../main.go | 6 +- .../watcher/promOperator.go | 22 +++---- internal/manifests/collector/annotations.go | 21 ++++++- .../manifests/collector/annotations_test.go | 61 +++++++++++++++++++ 6 files changed, 92 insertions(+), 28 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index ff73e4d26..6128a2210 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,9 +1,3 @@ -======================================================================== -Amazon CloudWatch Agent Operator (Unreleased) -======================================================================== -Bug Fixes: -* [TargetAllocator] Register the `--enable-prometheus-cr-watcher` CLI flag so the target-allocator no longer exits with "unknown flag" (CrashLoopBackOff) when the operator passes it for PrometheusCR.enabled. The flag is OR'd with the YAML `prometheus_cr.enabled` setting. - ======================================================================== Amazon CloudWatch Agent Operator v3.5.0 (2026-06-05) ======================================================================== diff --git a/cmd/amazon-cloudwatch-agent-target-allocator/config/config.go b/cmd/amazon-cloudwatch-agent-target-allocator/config/config.go index 6b2a3d81d..081e646e9 100644 --- a/cmd/amazon-cloudwatch-agent-target-allocator/config/config.go +++ b/cmd/amazon-cloudwatch-agent-target-allocator/config/config.go @@ -119,9 +119,7 @@ func LoadFromCLI(target *Config, flagSet *pflag.FlagSet) error { return err } - // Enable the Prometheus CR watcher when requested via the CLI flag. The YAML - // `prometheus_cr.enabled` value is loaded before this point, so OR the flag in - // rather than overwriting it: the watcher is enabled if either source sets it. + // OR the CLI flag into the YAML value so either source can enable the watcher. prometheusCREnabled, err := getPrometheusCREnabled(flagSet) if err != nil { return err diff --git a/cmd/amazon-cloudwatch-agent-target-allocator/main.go b/cmd/amazon-cloudwatch-agent-target-allocator/main.go index 57185184d..2b553dce6 100644 --- a/cmd/amazon-cloudwatch-agent-target-allocator/main.go +++ b/cmd/amazon-cloudwatch-agent-target-allocator/main.go @@ -88,11 +88,7 @@ func main() { srv := server.NewServer(log, allocator, cfg.ListenAddr, httpOptions...) discoveryCtx, discoveryCancel := context.WithCancel(ctx) - // Service Discovery metrics MUST be created and passed to the discovery - // manager. Each SD provider's NewDiscoverer fails when its DiscovererMetrics - // is nil, so passing a nil sdMetrics map makes every provider (including - // kubernetes_sd) fail to register, yielding zero discovered targets. Mirror - // the upstream opentelemetry-operator target-allocator. + // SD metrics must be non-nil; providers fail to register without them. sdRegistry := prometheus.NewRegistry() sdMetrics, sdMetricsErr := discovery.CreateAndRegisterSDMetrics(sdRegistry) if sdMetricsErr != nil { diff --git a/cmd/amazon-cloudwatch-agent-target-allocator/watcher/promOperator.go b/cmd/amazon-cloudwatch-agent-target-allocator/watcher/promOperator.go index a15db9d78..5dfc314dd 100644 --- a/cmd/amazon-cloudwatch-agent-target-allocator/watcher/promOperator.go +++ b/cmd/amazon-cloudwatch-agent-target-allocator/watcher/promOperator.go @@ -29,6 +29,8 @@ import ( allocatorconfig "github.com/aws/amazon-cloudwatch-agent-operator/cmd/amazon-cloudwatch-agent-target-allocator/config" ) +const defaultCollectorNamespace = "amazon-cloudwatch" + const minEventInterval = time.Second * 5 func NewPrometheusCRWatcher(logger logr.Logger, cfg allocatorconfig.Config) (*PrometheusCRWatcher, error) { @@ -50,15 +52,15 @@ func NewPrometheusCRWatcher(logger logr.Logger, cfg allocatorconfig.Config) (*Pr } // TODO: We should make these durations configurable - // The synthetic Prometheus object must carry a non-empty namespace: the - // prometheus-operator config generator calls store.ForNamespace(prom.Namespace) - // when generating the server configuration, which panics on an empty namespace - // ("namespace can't be empty"). Mirror the upstream opentelemetry-operator - // target-allocator by setting it to the collector/TA namespace, derived from the - // operator-injected OTELCOL_NAMESPACE downward-API env (fallback: amazon-cloudwatch). + // Namespace must be non-empty; the config generator panics otherwise. collectorNamespace := os.Getenv("OTELCOL_NAMESPACE") if collectorNamespace == "" { - collectorNamespace = "amazon-cloudwatch" + if ns, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil && len(ns) > 0 { + collectorNamespace = string(ns) + } else { + collectorNamespace = defaultCollectorNamespace + } + logger.Info("OTELCOL_NAMESPACE not set, resolved namespace", "namespace", collectorNamespace) } prom := &monitoringv1.Prometheus{ ObjectMeta: metav1.ObjectMeta{ @@ -68,11 +70,7 @@ func NewPrometheusCRWatcher(logger logr.Logger, cfg allocatorconfig.Config) (*Pr CommonPrometheusFields: monitoringv1.CommonPrometheusFields{ ScrapeInterval: monitoringv1.Duration(cfg.PrometheusCR.ScrapeInterval.String()), }, - // EvaluationInterval must be non-empty: the prometheus-operator config - // generator renders it verbatim as global.evaluation_interval, and the - // downstream prometheus config parser rejects an empty value with - // "empty duration string". The CWA TA does not evaluate rules, so mirror - // the scrape interval as a sane non-empty default. + // Must be non-empty; default to scrape interval. EvaluationInterval: monitoringv1.Duration(cfg.PrometheusCR.ScrapeInterval.String()), }, } diff --git a/internal/manifests/collector/annotations.go b/internal/manifests/collector/annotations.go index a665e465e..364072330 100644 --- a/internal/manifests/collector/annotations.go +++ b/internal/manifests/collector/annotations.go @@ -6,6 +6,7 @@ package collector import ( "crypto/sha256" "fmt" + "log/slog" "github.com/aws/amazon-cloudwatch-agent-operator/apis/v1alpha1" ) @@ -28,7 +29,7 @@ func Annotations(instance v1alpha1.AmazonCloudWatchAgent) map[string]string { } // make sure sha256 for configMap is always calculated - annotations["amazon-cloudwatch-agent-operator-config/sha256"] = getConfigMapSHA(instance.Spec.Config) + annotations["amazon-cloudwatch-agent-operator-config/sha256"] = getConfigMapSHA(configHashInput(instance)) return annotations } @@ -51,11 +52,27 @@ func PodAnnotations(instance v1alpha1.AmazonCloudWatchAgent) map[string]string { } // make sure sha256 for configMap is always calculated - podAnnotations["amazon-cloudwatch-agent-operator-config/sha256"] = getConfigMapSHA(instance.Spec.Config) + podAnnotations["amazon-cloudwatch-agent-operator-config/sha256"] = getConfigMapSHA(configHashInput(instance)) return podAnnotations } +// configHashInput returns the combined config string used for the pod-template restart hash. +func configHashInput(instance v1alpha1.AmazonCloudWatchAgent) string { + config := instance.Spec.Config + if !instance.Spec.Prometheus.IsEmpty() { + promYaml, err := instance.Spec.Prometheus.Yaml() + if err != nil { + // Static sentinel; Yaml() over map[string]interface{} rarely fails in practice. + slog.Warn("failed to serialize Spec.Prometheus for config hash", "error", err) + config += "\x00prometheus-serialize-error" + } else { + config += "\x00" + promYaml // null byte prevents collision between config suffix and promYAML prefix + } + } + return config +} + func getConfigMapSHA(config string) string { h := sha256.Sum256([]byte(config)) return fmt.Sprintf("%x", h) diff --git a/internal/manifests/collector/annotations_test.go b/internal/manifests/collector/annotations_test.go index e02ef1678..1ab8e3bd7 100644 --- a/internal/manifests/collector/annotations_test.go +++ b/internal/manifests/collector/annotations_test.go @@ -79,3 +79,64 @@ func TestAnnotationsPropagateDown(t *testing.T) { assert.Equal(t, "mycomponent", podAnnotations["myapp"]) assert.Equal(t, "pod_annotation_value", podAnnotations["pod_annotation"]) } + +func promConfig(t *testing.T, replacement string) v1alpha1.PrometheusConfig { + t.Helper() + cfg := map[string]interface{}{ + "scrape_configs": []interface{}{ + map[string]interface{}{ + "job_name": "kubernetes-pods-annotated", + "relabel_configs": []interface{}{ + map[string]interface{}{ + "target_label": "bug2probe", + "replacement": replacement, + }, + }, + }, + }, + } + return v1alpha1.PrometheusConfig{ + Config: &v1alpha1.AnyConfig{Object: cfg}, + } +} + +// TestPrometheusConfigChangeBumpsHash verifies a Prometheus-only change bumps the pod-template hash. +func TestPrometheusConfigChangeBumpsHash(t *testing.T) { + base := v1alpha1.AmazonCloudWatchAgent{ + ObjectMeta: metav1.ObjectMeta{Name: "my-instance", Namespace: "my-ns"}, + Spec: v1alpha1.AmazonCloudWatchAgentSpec{ + Config: "agent-config", + Prometheus: promConfig(t, "value2"), + }, + } + + // same spec twice -> stable hash + h1 := PodAnnotations(base)["amazon-cloudwatch-agent-operator-config/sha256"] + h2 := PodAnnotations(base)["amazon-cloudwatch-agent-operator-config/sha256"] + assert.Equal(t, h1, h2, "hash must be stable when nothing changes") + + // change ONLY the prometheus config -> hash must change + changed := base + changed.Spec.Prometheus = promConfig(t, "value3") + h3 := PodAnnotations(changed)["amazon-cloudwatch-agent-operator-config/sha256"] + assert.NotEqual(t, h1, h3, "pod annotation hash must change when only Spec.Prometheus changes") + + // metadata annotations hash must also reflect the prometheus change + a1 := Annotations(base)["amazon-cloudwatch-agent-operator-config/sha256"] + a3 := Annotations(changed)["amazon-cloudwatch-agent-operator-config/sha256"] + assert.NotEqual(t, a1, a3, "metadata annotation hash must change when only Spec.Prometheus changes") +} + +// TestEmptyPrometheusHashUnchanged verifies non-Prometheus agents keep a stable hash. +func TestEmptyPrometheusHashUnchanged(t *testing.T) { + otelcol := v1alpha1.AmazonCloudWatchAgent{ + ObjectMeta: metav1.ObjectMeta{Name: "my-instance", Namespace: "my-ns"}, + Spec: v1alpha1.AmazonCloudWatchAgentSpec{Config: "test"}, + } + + // sha256("test") == 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08 + assert.Equal(t, "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", + Annotations(otelcol)["amazon-cloudwatch-agent-operator-config/sha256"]) + assert.Equal(t, "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", + PodAnnotations(otelcol)["amazon-cloudwatch-agent-operator-config/sha256"]) +} From ae7a53c1796180cbb66d1fc0094c5f072105daae Mon Sep 17 00:00:00 2001 From: Musa Asad Date: Tue, 28 Jul 2026 18:29:18 +0000 Subject: [PATCH 04/10] Document frozen Prometheus config-hash sentinel, drop unreachable log configHashInput's error branch is unreachable from the CRD path: Spec.Prometheus.Config is an *AnyConfig whose Object is a map[string]interface{} populated by JSON decoding, and every type that produces is encodable by gopkg.in/yaml.v3. Logging a warning there implied a runtime condition operators should watch for and act on, which is misleading. Replace the slog.Warn with a comment that records three things a reader cannot recover from the code: that the branch is unreachable from the CRD path, why the sentinel is deliberately a constant rather than derived from the error (two different unserializable specs hash equal, so a Prometheus-only change would not roll pods while the error persists), and that the hash's stability across operator restarts rests on gopkg.in/yaml.v3 v3.0.1 sorting map keys -- a yaml library preserving insertion order here would roll every agent pod on restart. Removing the call also removes the log/slog import, which was the package's only use of it. The non-error path, the IsEmpty() gate and the \x00 separator are untouched, so the pinned config-hash values asserted in annotations_test.go are unchanged. --- internal/manifests/collector/annotations.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/manifests/collector/annotations.go b/internal/manifests/collector/annotations.go index 364072330..1e739c83f 100644 --- a/internal/manifests/collector/annotations.go +++ b/internal/manifests/collector/annotations.go @@ -6,7 +6,6 @@ package collector import ( "crypto/sha256" "fmt" - "log/slog" "github.com/aws/amazon-cloudwatch-agent-operator/apis/v1alpha1" ) @@ -63,8 +62,12 @@ func configHashInput(instance v1alpha1.AmazonCloudWatchAgent) string { if !instance.Spec.Prometheus.IsEmpty() { promYaml, err := instance.Spec.Prometheus.Yaml() if err != nil { - // Static sentinel; Yaml() over map[string]interface{} rarely fails in practice. - slog.Warn("failed to serialize Spec.Prometheus for config hash", "error", err) + // Unreachable from the CRD path: Config.Object only ever holds JSON-decoded + // types, all of which gopkg.in/yaml.v3 encodes. Deliberately a constant: two + // different unserializable specs hash equal, so a Prometheus-only change would + // not roll pods while the error persists. Hash stability across operator + // restarts depends on gopkg.in/yaml.v3 (v3.0.1) sorting map keys + // (encode.go:189, and :242 for the inline-map branch). config += "\x00prometheus-serialize-error" } else { config += "\x00" + promYaml // null byte prevents collision between config suffix and promYAML prefix From c5dee74c630c8c687fc4001aba7f9b31b9d622f6 Mon Sep 17 00:00:00 2001 From: Musa Asad Date: Tue, 28 Jul 2026 18:32:33 +0000 Subject: [PATCH 05/10] Trim and validate target allocator collector namespace resolution The namespace fed to the Prometheus config generator was taken verbatim from OTELCOL_NAMESPACE, or from the service account namespace file when that env var was empty. Neither source was trimmed, so a trailing newline or surrounding whitespace yielded a string that is not a valid Kubernetes namespace. The file branch also gated on len(ns) > 0, which treats a whitespace-only file as a real value and so skips the defaultCollectorNamespace fallback. Both sources are now whitespace-trimmed, and a value that is blank after trimming falls through to the next source: env var, then the service account file, then defaultCollectorNamespace. The resolution moves out of NewPrometheusCRWatcher into resolveCollectorNamespace, with the service account path held in a package variable so a test can redirect the read. The log line still fires exactly once when the env var is unset, and still reports the resolved value. --- .../watcher/promOperator.go | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/cmd/amazon-cloudwatch-agent-target-allocator/watcher/promOperator.go b/cmd/amazon-cloudwatch-agent-target-allocator/watcher/promOperator.go index 5dfc314dd..b9321407b 100644 --- a/cmd/amazon-cloudwatch-agent-target-allocator/watcher/promOperator.go +++ b/cmd/amazon-cloudwatch-agent-target-allocator/watcher/promOperator.go @@ -8,6 +8,7 @@ import ( "fmt" "log/slog" "os" + "strings" "time" "github.com/go-logr/logr" @@ -33,6 +34,23 @@ const defaultCollectorNamespace = "amazon-cloudwatch" const minEventInterval = time.Second * 5 +// serviceAccountNamespacePath is a variable so tests can redirect the read. +var serviceAccountNamespacePath = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" + +func resolveCollectorNamespace(logger logr.Logger) string { + if ns := strings.TrimSpace(os.Getenv("OTELCOL_NAMESPACE")); ns != "" { + return ns + } + namespace := defaultCollectorNamespace + if data, err := os.ReadFile(serviceAccountNamespacePath); err == nil { + if ns := strings.TrimSpace(string(data)); ns != "" { + namespace = ns + } + } + logger.Info("OTELCOL_NAMESPACE not set, resolved namespace", "namespace", namespace) + return namespace +} + func NewPrometheusCRWatcher(logger logr.Logger, cfg allocatorconfig.Config) (*PrometheusCRWatcher, error) { mClient, err := monitoringclient.NewForConfig(cfg.ClusterConfig) if err != nil { @@ -53,15 +71,7 @@ func NewPrometheusCRWatcher(logger logr.Logger, cfg allocatorconfig.Config) (*Pr // TODO: We should make these durations configurable // Namespace must be non-empty; the config generator panics otherwise. - collectorNamespace := os.Getenv("OTELCOL_NAMESPACE") - if collectorNamespace == "" { - if ns, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil && len(ns) > 0 { - collectorNamespace = string(ns) - } else { - collectorNamespace = defaultCollectorNamespace - } - logger.Info("OTELCOL_NAMESPACE not set, resolved namespace", "namespace", collectorNamespace) - } + collectorNamespace := resolveCollectorNamespace(logger) prom := &monitoringv1.Prometheus{ ObjectMeta: metav1.ObjectMeta{ Namespace: collectorNamespace, From 479e2511ff649e3e271287a3cfac4f655fea446c Mon Sep 17 00:00:00 2001 From: Musa Asad Date: Tue, 28 Jul 2026 18:35:07 +0000 Subject: [PATCH 06/10] Test target allocator namespace resolution and evaluation interval default Adds coverage for the two behaviors NewPrometheusCRWatcher depends on and that nothing else in the watcher package exercises. TestResolveCollectorNamespace pins the resolution chain across seven cases: OTELCOL_NAMESPACE wins when set, both the env var and the service account namespace file are whitespace-trimmed, a whitespace-only value in either source falls through to the next source rather than becoming the namespace, and an absent or blank file falls back to the default. This guards against a future edit dropping a TrimSpace, reverting to a length-only emptiness check, or reordering the fallback chain -- any of which yields an empty or blank namespace, and the config generator panics on a non-empty namespace requirement. TestNewPrometheusCRWatcherDefaultsEvaluationInterval pins EvaluationInterval defaulting to ScrapeInterval. Without that assignment the generated config carries an empty evaluation_interval, which fails to parse. The tests live in a new file because promOperator_test.go imports neither k8s.io/client-go/rest, the allocator config package, nor logr. The test redirects serviceAccountNamespacePath and restores it via t.Cleanup, so the package's later test files do not inherit a path into a removed temp dir. The constructor is pointed at an unroutable host to confirm it does no dialing. --- .../watcher/namespace_resolution_test.go | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 cmd/amazon-cloudwatch-agent-target-allocator/watcher/namespace_resolution_test.go diff --git a/cmd/amazon-cloudwatch-agent-target-allocator/watcher/namespace_resolution_test.go b/cmd/amazon-cloudwatch-agent-target-allocator/watcher/namespace_resolution_test.go new file mode 100644 index 000000000..23aa311d9 --- /dev/null +++ b/cmd/amazon-cloudwatch-agent-target-allocator/watcher/namespace_resolution_test.go @@ -0,0 +1,69 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package watcher + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/go-logr/logr" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/client-go/rest" + + allocatorconfig "github.com/aws/amazon-cloudwatch-agent-operator/cmd/amazon-cloudwatch-agent-target-allocator/config" +) + +func TestResolveCollectorNamespace(t *testing.T) { + for _, tc := range []struct { + name string + env string + fileBody *string + want string + }{ + {"env var set", "my-env-ns", nil, "my-env-ns"}, + {"env var set with whitespace", " my-env-ns\n", nil, "my-env-ns"}, + {"env whitespace only falls through to file", " \n", ptrStr("file-ns"), "file-ns"}, + {"env empty, file set", "", ptrStr("file-ns"), "file-ns"}, + {"env empty, file trailing newline", "", ptrStr("file-ns\n"), "file-ns"}, + {"env empty, file whitespace only", "", ptrStr(" \n\t"), defaultCollectorNamespace}, + {"both missing falls back to default", "", nil, defaultCollectorNamespace}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("OTELCOL_NAMESPACE", tc.env) + orig := serviceAccountNamespacePath + t.Cleanup(func() { serviceAccountNamespacePath = orig }) + if tc.fileBody == nil { + serviceAccountNamespacePath = filepath.Join(t.TempDir(), "does-not-exist") + } else { + p := filepath.Join(t.TempDir(), "namespace") + require.NoError(t, os.WriteFile(p, []byte(*tc.fileBody), 0o600)) + serviceAccountNamespacePath = p + } + assert.Equal(t, tc.want, resolveCollectorNamespace(logr.Discard())) + }) + } +} + +func ptrStr(s string) *string { return &s } + +func TestNewPrometheusCRWatcherDefaultsEvaluationInterval(t *testing.T) { + t.Setenv("OTELCOL_NAMESPACE", "eval-ns") + cfg := allocatorconfig.Config{ + ClusterConfig: &rest.Config{Host: "https://127.0.0.1:65535"}, + PrometheusCR: allocatorconfig.PrometheusCRConfig{ + Enabled: true, + ScrapeInterval: model.Duration(45 * time.Second), + }, + } + w, err := NewPrometheusCRWatcher(logr.Discard(), cfg) + require.NoError(t, err) + require.NotNil(t, w.prom) + assert.Equal(t, "eval-ns", w.prom.Namespace) + assert.Equal(t, w.prom.Spec.ScrapeInterval, w.prom.Spec.EvaluationInterval) + assert.Equal(t, "45s", string(w.prom.Spec.EvaluationInterval)) +} From 02a4bb5d5bb30c3e751bd4f66fb6d4f9ed5d1e16 Mon Sep 17 00:00:00 2001 From: Musa Asad Date: Tue, 28 Jul 2026 18:36:57 +0000 Subject: [PATCH 07/10] Trim OTELCOL_NAMESPACE in target allocator collector client The collector client resolves its watch namespace once at process init from OTELCOL_NAMESPACE and passes the result to Pods(ns).List and Pods(ns).Watch. A value carrying surrounding whitespace is not a valid namespace name, so those calls matched nothing, no collectors were discovered, and allocation stayed empty with no error surfaced. Trim the value so padded input resolves to the intended namespace, matching how the namespace is now resolved on the watcher path. --- .../collector/collector.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/amazon-cloudwatch-agent-target-allocator/collector/collector.go b/cmd/amazon-cloudwatch-agent-target-allocator/collector/collector.go index 60bda6bca..bd2cd2ef4 100644 --- a/cmd/amazon-cloudwatch-agent-target-allocator/collector/collector.go +++ b/cmd/amazon-cloudwatch-agent-target-allocator/collector/collector.go @@ -6,6 +6,7 @@ package collector import ( "context" "os" + "strings" "time" "github.com/go-logr/logr" @@ -26,7 +27,7 @@ const ( ) var ( - ns = os.Getenv("OTELCOL_NAMESPACE") + ns = strings.TrimSpace(os.Getenv("OTELCOL_NAMESPACE")) collectorsDiscovered = promauto.NewGauge(prometheus.GaugeOpts{ Name: "amazon_cloudwatch_agent_allocator_collectors_discovered", Help: "Number of collectors discovered.", From ebd16294725e3a4042f75683fbca589f8508b9fc Mon Sep 17 00:00:00 2001 From: Musa Asad Date: Tue, 28 Jul 2026 18:38:40 +0000 Subject: [PATCH 08/10] Document enable-only semantics of --enable-prometheus-cr-watcher LoadFromCLI ORs the CLI flag with prometheus_cr.enabled from the config file, so the flag can only turn the PrometheusCR watcher on. Passing =false is silently ineffective when the config file enables it, which the previous help text did not convey. State the OR behavior in the flag's usage string so the asymmetry is discoverable from --help. No behavior change: only the usage string passed to flagSet.Bool changes. --- cmd/amazon-cloudwatch-agent-target-allocator/config/flags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/amazon-cloudwatch-agent-target-allocator/config/flags.go b/cmd/amazon-cloudwatch-agent-target-allocator/config/flags.go index 294f9bfe6..d2385be8f 100644 --- a/cmd/amazon-cloudwatch-agent-target-allocator/config/flags.go +++ b/cmd/amazon-cloudwatch-agent-target-allocator/config/flags.go @@ -34,7 +34,7 @@ func getFlagSet(errorHandling pflag.ErrorHandling) *pflag.FlagSet { flagSet := pflag.NewFlagSet(targetAllocatorName, errorHandling) flagSet.String(configFilePathFlagName, DefaultConfigFilePath, "The path to the config file.") flagSet.String(kubeConfigPathFlagName, filepath.Join(homedir.HomeDir(), ".kube", "config"), "absolute path to the KubeconfigPath file") - flagSet.Bool(prometheusCREnabledFlagName, false, "Enable watching of Prometheus Operator custom resources (ServiceMonitor/PodMonitor) to dynamically generate scrape configs.") + flagSet.Bool(prometheusCREnabledFlagName, false, "Enable watching of Prometheus Operator custom resources (ServiceMonitor/PodMonitor) to dynamically generate scrape configs. Enable-only: this flag is OR-ed with prometheus_cr.enabled from the config file, so passing =false cannot disable a watcher the config file enables.") flagSet.Bool(reloadConfigFlagName, false, "Enable automatic configuration reloading. This functionality is deprecated and will be removed in a future release.") flagSet.Bool(httpsEnabledFlagName, true, "Enable HTTPS additional server") flagSet.String(listenAddrHttpsFlagName, ":8443", "The address where this service serves over HTTPS.") From 95adef28be22e2d7b13926a74c781eabcb908a6a Mon Sep 17 00:00:00 2001 From: Musa Asad Date: Tue, 28 Jul 2026 18:42:12 +0000 Subject: [PATCH 09/10] Test enable-only OR semantics of --enable-prometheus-cr-watcher LoadFromCLI OR-es the --enable-prometheus-cr-watcher flag with prometheus_cr.enabled from the config file, so the flag can only enable the watcher: passing =false cannot disable one the config file turns on. Nothing covered that, and replacing the || with a plain assignment would break only the config-file-true rows while leaving every existing test in the package green. Add a table-driven test over the four (config file, CLI) combinations, plus a committed testdata kubeconfig so the test is hermetic. LoadFromCLI builds a client config before it reaches the OR, so with no --kubeconfig-path it falls back to rest.InClusterConfig() and returns "unable to load in-cluster configuration" early. That passes on a developer host that happens to have a kubeconfig and fails in CI, so the flag pointing at testdata/kubeconfig_test.yaml is required rather than decorative. The fixture's server is 127.0.0.1 and is never dialed. pflag.ContinueOnError is a test-level choice so a parse error fails the test instead of exiting the process; the production getFlagSet call keeps pflag.ExitOnError. --- .../config/prometheus_cr_or_semantics_test.go | 36 +++++++++++++++++++ .../config/testdata/kubeconfig_test.yaml | 16 +++++++++ 2 files changed, 52 insertions(+) create mode 100644 cmd/amazon-cloudwatch-agent-target-allocator/config/prometheus_cr_or_semantics_test.go create mode 100644 cmd/amazon-cloudwatch-agent-target-allocator/config/testdata/kubeconfig_test.yaml diff --git a/cmd/amazon-cloudwatch-agent-target-allocator/config/prometheus_cr_or_semantics_test.go b/cmd/amazon-cloudwatch-agent-target-allocator/config/prometheus_cr_or_semantics_test.go new file mode 100644 index 000000000..e377a418f --- /dev/null +++ b/cmd/amazon-cloudwatch-agent-target-allocator/config/prometheus_cr_or_semantics_test.go @@ -0,0 +1,36 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "testing" + + "github.com/spf13/pflag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPrometheusCREnabledOrSemantics(t *testing.T) { + for _, tc := range []struct { + name string + yaml bool + args []string + want bool + }{ + {"yamlFalse_cliAbsent", false, nil, false}, + {"yamlFalse_cliTrue", false, []string{"--enable-prometheus-cr-watcher"}, true}, + {"yamlTrue_cliAbsent", true, nil, true}, + {"yamlTrue_cliFalse", true, []string{"--enable-prometheus-cr-watcher=false"}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + fs := getFlagSet(pflag.ContinueOnError) + args := append([]string{"--kubeconfig-path", "./testdata/kubeconfig_test.yaml"}, tc.args...) + require.NoError(t, fs.Parse(args)) + c := CreateDefaultConfig() + c.PrometheusCR.Enabled = tc.yaml + require.NoError(t, LoadFromCLI(&c, fs)) + assert.Equal(t, tc.want, c.PrometheusCR.Enabled) + }) + } +} diff --git a/cmd/amazon-cloudwatch-agent-target-allocator/config/testdata/kubeconfig_test.yaml b/cmd/amazon-cloudwatch-agent-target-allocator/config/testdata/kubeconfig_test.yaml new file mode 100644 index 000000000..9e7fadf17 --- /dev/null +++ b/cmd/amazon-cloudwatch-agent-target-allocator/config/testdata/kubeconfig_test.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Config +clusters: +- name: test-cluster + cluster: + server: https://127.0.0.1:6443 +contexts: +- name: test-context + context: + cluster: test-cluster + user: test-user +current-context: test-context +users: +- name: test-user + user: + token: test-token From 2d21b9b6bda86eba90cd3de11d67bd9e6fe340db Mon Sep 17 00:00:00 2001 From: Musa Asad Date: Thu, 13 Aug 2026 17:15:47 +0000 Subject: [PATCH 10/10] Name the service account namespace path constant Extract the hardcoded service account namespace path literal into defaultServiceAccountNamespacePath per review feedback. The package-level serviceAccountNamespacePath var remains as the test seam that redirects the read in unit tests. There is no behavior change. --- .../watcher/promOperator.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/amazon-cloudwatch-agent-target-allocator/watcher/promOperator.go b/cmd/amazon-cloudwatch-agent-target-allocator/watcher/promOperator.go index b9321407b..c9ebe6ca6 100644 --- a/cmd/amazon-cloudwatch-agent-target-allocator/watcher/promOperator.go +++ b/cmd/amazon-cloudwatch-agent-target-allocator/watcher/promOperator.go @@ -34,8 +34,10 @@ const defaultCollectorNamespace = "amazon-cloudwatch" const minEventInterval = time.Second * 5 +const defaultServiceAccountNamespacePath = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" + // serviceAccountNamespacePath is a variable so tests can redirect the read. -var serviceAccountNamespacePath = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" +var serviceAccountNamespacePath = defaultServiceAccountNamespacePath func resolveCollectorNamespace(logger logr.Logger) string { if ns := strings.TrimSpace(os.Getenv("OTELCOL_NAMESPACE")); ns != "" {