feat(instrumentation): enable the ADOT Java embedded profiler from java.profiling

.github/workflows/operator-integration-test.yml

@@ -267,6 +267,65 @@ jobs:
           go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/manifests/cmd/ns_instrumentation_env_variables.json app_signals
           kubectl delete instrumentation sample-instrumentation
 
+      - name: Enable Java profiling in the operator
+        run: |
+          # Pass java.profiling the way the Helm chart does, through --auto-instrumentation-config on the manager.
+          CONFIG="$(jq -c . integration-tests/java-profiling/auto-instrumentation-config.json)"
+          INDEX="$(kubectl get deployment cloudwatch-controller-manager -n amazon-cloudwatch -o json | jq '.spec.template.spec.containers | map(.name) | index("manager")')"
+          kubectl patch deployment cloudwatch-controller-manager -n amazon-cloudwatch --type=json \
+            -p "$(jq -cn --arg v "--auto-instrumentation-config=${CONFIG}" --argjson i "${INDEX}" '[{"op":"add","path":"/spec/template/spec/containers/\($i)/args/-","value":$v}]')"
+          kubectl rollout status deployment/cloudwatch-controller-manager -n amazon-cloudwatch --timeout=180s
+          kubectl get deployment cloudwatch-controller-manager -n amazon-cloudwatch -o jsonpath='{.spec.template.spec.containers[*].args}'; echo
+          # A finished rollout does not mean the new manager's webhook serves yet, and the pod webhook ignores
+          # failures. Dry-run a Java pod through admission until the reply carries the profiler variable.
+          cat > /tmp/java-profiling-probe-pod.yaml <<'EOF'
+          apiVersion: v1
+          kind: Pod
+          metadata:
+            name: java-profiling-probe
+            namespace: default
+            annotations:
+              instrumentation.opentelemetry.io/inject-java: "true"
+          spec:
+            containers:
+              - name: app
+                image: nginx:1.14.2
+          EOF
+          for i in $(seq 1 36); do
+            out="$(kubectl create --dry-run=server -o json -f /tmp/java-profiling-probe-pod.yaml 2>/dev/null || true)"
+            if grep -q '"OTEL_AWS_PROFILER_ENABLED"' <<< "$out"; then
+              echo "Webhook injects Java profiling (attempt $i)"
+              exit 0
+            fi
+            echo "Waiting for the webhook to inject Java profiling... ($i/36)"
+            sleep 5
+          done
+          kubectl create --dry-run=server -o yaml -f /tmp/java-profiling-probe-pod.yaml || true
+          kubectl logs -n amazon-cloudwatch -l app.kubernetes.io/name=amazon-cloudwatch-agent-operator --tail=100 || true
+          echo "::error::operator webhook never became ready after enabling Java profiling"
+          exit 1
+
+      - name: Test for Java profiling injection
+        run: |
+          cat integration-tests/java-profiling/sample-deployment-java-profiling.yaml
+          kubectl apply -f integration-tests/java-profiling/sample-deployment-java-profiling.yaml
+          sleep 5
+          kubectl wait --for=condition=Available deployment/nginx-java-profiling -n default --timeout=180s
+          kubectl describe pods -n default -l app=nginx-java-profiling
+          go test ./integration-tests/java-profiling/validate
+          go run ./integration-tests/java-profiling/cmd default integration-tests/java-profiling/java_profiling_expectations.json
+
+      - name: Test for Java profiling opt-out
+        run: |
+          cat integration-tests/java-profiling/sample-deployment-java-profiling-opt-out.yaml
+          kubectl apply -f integration-tests/java-profiling/sample-deployment-java-profiling-opt-out.yaml
+          sleep 5
+          kubectl wait --for=condition=Available deployment/nginx-java-profiling-opt-out -n default --timeout=180s
+          kubectl describe pods -n default -l app=nginx-java-profiling-opt-out
+          go run ./integration-tests/java-profiling/cmd default integration-tests/java-profiling/java_profiling_opt_out_expectations.json
+          kubectl delete -f integration-tests/java-profiling/sample-deployment-java-profiling-opt-out.yaml
+          kubectl delete -f integration-tests/java-profiling/sample-deployment-java-profiling.yaml
+
       # Check pod state/events/operator logs for any failure.
       - name: Dump diagnostics on failure
         if: failure()

integration-tests/java-profiling/auto-instrumentation-config.json

@@ -0,0 +1 @@
+{"java": {"limits": {"cpu": "500m", "memory": "64Mi"}, "requests": {"cpu": "50m", "memory": "64Mi"}, "runtime_metrics": {"enabled": "true"}, "service_events": {}, "dynamic_instrumentation": {}, "profiling": {"enabled": "true", "mode": "cpu", "volume_size_limit": "64Mi"}}}

integration-tests/java-profiling/cmd/main.go

@@ -0,0 +1,80 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+// Command validates the Java profiling injection on the running pods of a cluster.
+// Usage: go run ./integration-tests/java-profiling/cmd <namespace> <expectations.json>
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+
+	"github.com/aws/amazon-cloudwatch-agent-operator/integration-tests/java-profiling/validate"
+)
+
+func main() {
+	if len(os.Args) != 3 {
+		fmt.Println("usage: <namespace> <expectations.json>")
+		os.Exit(2)
+	}
+	if err := run(os.Args[1], os.Args[2]); err != nil {
+		fmt.Println(err)
+		fmt.Println("Java Profiling Injection Test: FAIL")
+		os.Exit(1)
+	}
+	fmt.Println("Java Profiling Injection Test: PASS")
+}
+
+func run(namespace, expectationsPath string) error {
+	exp, err := validate.Load(expectationsPath)
+	if err != nil {
+		return err
+	}
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return fmt.Errorf("error getting user home dir: %w", err)
+	}
+	kubeConfig, err := clientcmd.BuildConfigFromFlags("", filepath.Join(home, ".kube", "config"))
+	if err != nil {
+		return fmt.Errorf("error getting kubernetes config: %w", err)
+	}
+	clientSet, err := kubernetes.NewForConfig(kubeConfig)
+	if err != nil {
+		return fmt.Errorf("error creating kubernetes client: %w", err)
+	}
+	pods, err := clientSet.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: exp.LabelSelector})
+	if err != nil {
+		return fmt.Errorf("error listing pods: %w", err)
+	}
+	checked := 0
+	var failed []string
+	for _, pod := range pods.Items {
+		if pod.DeletionTimestamp != nil || pod.Status.Phase != corev1.PodRunning {
+			continue
+		}
+		checked++
+		problems := validate.Pod(pod, exp)
+		for _, p := range problems {
+			fmt.Printf("Pod %s: %s\n", pod.Name, p)
+		}
+		if len(problems) > 0 {
+			failed = append(failed, pod.Name)
+		} else {
+			fmt.Printf("Pod %s: matches %s\n", pod.Name, expectationsPath)
+		}
+	}
+	if checked == 0 {
+		return fmt.Errorf("no running pods in %s match %q", namespace, exp.LabelSelector)
+	}
+	if len(failed) > 0 {
+		return fmt.Errorf("%d of %d pods do not match: %v", len(failed), checked, failed)
+	}
+	return nil
+}

integration-tests/java-profiling/java_profiling_expectations.json

@@ -0,0 +1,20 @@
+{
+  "labelSelector": "app=nginx-java-profiling",
+  "javaInstrumentedContainers": ["app", "worker"],
+  "profiledContainers": {
+    "app": {
+      "OTEL_AWS_PROFILER_ENABLED": "true",
+      "OTEL_AWS_PROFILER_ENDPOINT": "http://cloudwatch-agent.amazon-cloudwatch:4318/v1development/profiles",
+      "OTEL_AWS_PROFILER_DATA_DIR": "/otel-profiler-data-java/app",
+      "OTEL_AWS_PROFILER_MODE": "cpu"
+    },
+    "worker": {
+      "OTEL_AWS_PROFILER_ENABLED": "true",
+      "OTEL_AWS_PROFILER_ENDPOINT": "http://cloudwatch-agent.amazon-cloudwatch:4318/v1development/profiles",
+      "OTEL_AWS_PROFILER_DATA_DIR": "/otel-profiler-data-java/worker",
+      "OTEL_AWS_PROFILER_MODE": "cpu"
+    }
+  },
+  "unprofiledContainers": ["sidecar"],
+  "volumeSizeLimit": "128Mi"
+}

integration-tests/java-profiling/java_profiling_opt_out_expectations.json

@@ -0,0 +1,8 @@
+{
+  "labelSelector": "app=nginx-java-profiling-opt-out",
+  "javaInstrumentedContainers": ["app"],
+  "profiledContainers": {},
+  "unprofiledContainers": ["app"],
+  "unprofiledAllowedEnv": {"OTEL_AWS_PROFILER_ENABLED": "false"},
+  "volumeSizeLimit": ""
+}

integration-tests/java-profiling/sample-deployment-java-profiling-opt-out.yaml

@@ -0,0 +1,24 @@
+# Java profiling opt-out: the container is Java-instrumented but sets OTEL_AWS_PROFILER_ENABLED=false, so it gets no
+# other profiler variable, volume or mount.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-java-profiling-opt-out
+spec:
+  selector:
+    matchLabels:
+      app: nginx-java-profiling-opt-out
+  template:
+    metadata:
+      labels:
+        app: nginx-java-profiling-opt-out
+      annotations:
+        instrumentation.opentelemetry.io/inject-java: "true"
+    spec:
+      containers:
+        - name: app
+          image: nginx:1.14.2
+          env:
+            - name: OTEL_AWS_PROFILER_ENABLED
+              value: "false"
+      restartPolicy: Always

integration-tests/java-profiling/sample-deployment-java-profiling.yaml

@@ -0,0 +1,28 @@
+# Java profiling: two Java containers (app, worker) are profiled and share one profiler volume; the sidecar is not
+# Java-instrumented and must get nothing from the profiler.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-java-profiling
+spec:
+  selector:
+    matchLabels:
+      app: nginx-java-profiling
+  template:
+    metadata:
+      labels:
+        app: nginx-java-profiling
+      annotations:
+        instrumentation.opentelemetry.io/inject-java: "true"
+        instrumentation.opentelemetry.io/java-container-names: "app,worker"
+    spec:
+      containers:
+        - name: app
+          image: nginx:1.14.2
+        - name: worker
+          image: nginx:1.14.2
+          command: ["sh", "-c", "sleep 3600"]
+        - name: sidecar
+          image: nginx:1.14.2
+          command: ["sh", "-c", "sleep 3600"]
+      restartPolicy: Always