diff --git a/.github/workflows/operator-integration-test.yml b/.github/workflows/operator-integration-test.yml index 28e3186ed..4e238f750 100644 --- a/.github/workflows/operator-integration-test.yml +++ b/.github/workflows/operator-integration-test.yml @@ -267,6 +267,65 @@ jobs: go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/manifests/cmd/ns_instrumentation_env_variables.json app_signals kubectl delete instrumentation sample-instrumentation + - name: Enable Java profiling in the operator + run: | + # Pass java.profiling the way the Helm chart does, through --auto-instrumentation-config on the manager. + CONFIG="$(jq -c . integration-tests/java-profiling/auto-instrumentation-config.json)" + INDEX="$(kubectl get deployment cloudwatch-controller-manager -n amazon-cloudwatch -o json | jq '.spec.template.spec.containers | map(.name) | index("manager")')" + kubectl patch deployment cloudwatch-controller-manager -n amazon-cloudwatch --type=json \ + -p "$(jq -cn --arg v "--auto-instrumentation-config=${CONFIG}" --argjson i "${INDEX}" '[{"op":"add","path":"/spec/template/spec/containers/\($i)/args/-","value":$v}]')" + kubectl rollout status deployment/cloudwatch-controller-manager -n amazon-cloudwatch --timeout=180s + kubectl get deployment cloudwatch-controller-manager -n amazon-cloudwatch -o jsonpath='{.spec.template.spec.containers[*].args}'; echo + # A finished rollout does not mean the new manager's webhook serves yet, and the pod webhook ignores + # failures. Dry-run a Java pod through admission until the reply carries the profiler variable. + cat > /tmp/java-profiling-probe-pod.yaml <<'EOF' + apiVersion: v1 + kind: Pod + metadata: + name: java-profiling-probe + namespace: default + annotations: + instrumentation.opentelemetry.io/inject-java: "true" + spec: + containers: + - name: app + image: nginx:1.14.2 + EOF + for i in $(seq 1 36); do + out="$(kubectl create --dry-run=server -o json -f /tmp/java-profiling-probe-pod.yaml 2>/dev/null || true)" + if grep -q '"OTEL_AWS_PROFILER_ENABLED"' <<< "$out"; then + echo "Webhook injects Java profiling (attempt $i)" + exit 0 + fi + echo "Waiting for the webhook to inject Java profiling... ($i/36)" + sleep 5 + done + kubectl create --dry-run=server -o yaml -f /tmp/java-profiling-probe-pod.yaml || true + kubectl logs -n amazon-cloudwatch -l app.kubernetes.io/name=amazon-cloudwatch-agent-operator --tail=100 || true + echo "::error::operator webhook never became ready after enabling Java profiling" + exit 1 + + - name: Test for Java profiling injection + run: | + cat integration-tests/java-profiling/sample-deployment-java-profiling.yaml + kubectl apply -f integration-tests/java-profiling/sample-deployment-java-profiling.yaml + sleep 5 + kubectl wait --for=condition=Available deployment/nginx-java-profiling -n default --timeout=180s + kubectl describe pods -n default -l app=nginx-java-profiling + go test ./integration-tests/java-profiling/validate + go run ./integration-tests/java-profiling/cmd default integration-tests/java-profiling/java_profiling_expectations.json + + - name: Test for Java profiling opt-out + run: | + cat integration-tests/java-profiling/sample-deployment-java-profiling-opt-out.yaml + kubectl apply -f integration-tests/java-profiling/sample-deployment-java-profiling-opt-out.yaml + sleep 5 + kubectl wait --for=condition=Available deployment/nginx-java-profiling-opt-out -n default --timeout=180s + kubectl describe pods -n default -l app=nginx-java-profiling-opt-out + go run ./integration-tests/java-profiling/cmd default integration-tests/java-profiling/java_profiling_opt_out_expectations.json + kubectl delete -f integration-tests/java-profiling/sample-deployment-java-profiling-opt-out.yaml + kubectl delete -f integration-tests/java-profiling/sample-deployment-java-profiling.yaml + # Check pod state/events/operator logs for any failure. - name: Dump diagnostics on failure if: failure() diff --git a/integration-tests/java-profiling/auto-instrumentation-config.json b/integration-tests/java-profiling/auto-instrumentation-config.json new file mode 100644 index 000000000..91047b355 --- /dev/null +++ b/integration-tests/java-profiling/auto-instrumentation-config.json @@ -0,0 +1 @@ +{"java": {"limits": {"cpu": "500m", "memory": "64Mi"}, "requests": {"cpu": "50m", "memory": "64Mi"}, "runtime_metrics": {"enabled": "true"}, "service_events": {}, "dynamic_instrumentation": {}, "profiling": {"enabled": "true", "mode": "cpu", "volume_size_limit": "64Mi"}}} diff --git a/integration-tests/java-profiling/cmd/main.go b/integration-tests/java-profiling/cmd/main.go new file mode 100644 index 000000000..ee37e4f08 --- /dev/null +++ b/integration-tests/java-profiling/cmd/main.go @@ -0,0 +1,80 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Command validates the Java profiling injection on the running pods of a cluster. +// Usage: go run ./integration-tests/java-profiling/cmd +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + + "github.com/aws/amazon-cloudwatch-agent-operator/integration-tests/java-profiling/validate" +) + +func main() { + if len(os.Args) != 3 { + fmt.Println("usage: ") + os.Exit(2) + } + if err := run(os.Args[1], os.Args[2]); err != nil { + fmt.Println(err) + fmt.Println("Java Profiling Injection Test: FAIL") + os.Exit(1) + } + fmt.Println("Java Profiling Injection Test: PASS") +} + +func run(namespace, expectationsPath string) error { + exp, err := validate.Load(expectationsPath) + if err != nil { + return err + } + home, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("error getting user home dir: %w", err) + } + kubeConfig, err := clientcmd.BuildConfigFromFlags("", filepath.Join(home, ".kube", "config")) + if err != nil { + return fmt.Errorf("error getting kubernetes config: %w", err) + } + clientSet, err := kubernetes.NewForConfig(kubeConfig) + if err != nil { + return fmt.Errorf("error creating kubernetes client: %w", err) + } + pods, err := clientSet.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: exp.LabelSelector}) + if err != nil { + return fmt.Errorf("error listing pods: %w", err) + } + checked := 0 + var failed []string + for _, pod := range pods.Items { + if pod.DeletionTimestamp != nil || pod.Status.Phase != corev1.PodRunning { + continue + } + checked++ + problems := validate.Pod(pod, exp) + for _, p := range problems { + fmt.Printf("Pod %s: %s\n", pod.Name, p) + } + if len(problems) > 0 { + failed = append(failed, pod.Name) + } else { + fmt.Printf("Pod %s: matches %s\n", pod.Name, expectationsPath) + } + } + if checked == 0 { + return fmt.Errorf("no running pods in %s match %q", namespace, exp.LabelSelector) + } + if len(failed) > 0 { + return fmt.Errorf("%d of %d pods do not match: %v", len(failed), checked, failed) + } + return nil +} diff --git a/integration-tests/java-profiling/java_profiling_expectations.json b/integration-tests/java-profiling/java_profiling_expectations.json new file mode 100644 index 000000000..be904ee18 --- /dev/null +++ b/integration-tests/java-profiling/java_profiling_expectations.json @@ -0,0 +1,20 @@ +{ + "labelSelector": "app=nginx-java-profiling", + "javaInstrumentedContainers": ["app", "worker"], + "profiledContainers": { + "app": { + "OTEL_AWS_PROFILER_ENABLED": "true", + "OTEL_AWS_PROFILER_ENDPOINT": "http://cloudwatch-agent.amazon-cloudwatch:4318/v1development/profiles", + "OTEL_AWS_PROFILER_DATA_DIR": "/otel-profiler-data-java/app", + "OTEL_AWS_PROFILER_MODE": "cpu" + }, + "worker": { + "OTEL_AWS_PROFILER_ENABLED": "true", + "OTEL_AWS_PROFILER_ENDPOINT": "http://cloudwatch-agent.amazon-cloudwatch:4318/v1development/profiles", + "OTEL_AWS_PROFILER_DATA_DIR": "/otel-profiler-data-java/worker", + "OTEL_AWS_PROFILER_MODE": "cpu" + } + }, + "unprofiledContainers": ["sidecar"], + "volumeSizeLimit": "128Mi" +} diff --git a/integration-tests/java-profiling/java_profiling_opt_out_expectations.json b/integration-tests/java-profiling/java_profiling_opt_out_expectations.json new file mode 100644 index 000000000..b43c872e8 --- /dev/null +++ b/integration-tests/java-profiling/java_profiling_opt_out_expectations.json @@ -0,0 +1,8 @@ +{ + "labelSelector": "app=nginx-java-profiling-opt-out", + "javaInstrumentedContainers": ["app"], + "profiledContainers": {}, + "unprofiledContainers": ["app"], + "unprofiledAllowedEnv": {"OTEL_AWS_PROFILER_ENABLED": "false"}, + "volumeSizeLimit": "" +} diff --git a/integration-tests/java-profiling/sample-deployment-java-profiling-opt-out.yaml b/integration-tests/java-profiling/sample-deployment-java-profiling-opt-out.yaml new file mode 100644 index 000000000..2f6493195 --- /dev/null +++ b/integration-tests/java-profiling/sample-deployment-java-profiling-opt-out.yaml @@ -0,0 +1,24 @@ +# Java profiling opt-out: the container is Java-instrumented but sets OTEL_AWS_PROFILER_ENABLED=false, so it gets no +# other profiler variable, volume or mount. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-java-profiling-opt-out +spec: + selector: + matchLabels: + app: nginx-java-profiling-opt-out + template: + metadata: + labels: + app: nginx-java-profiling-opt-out + annotations: + instrumentation.opentelemetry.io/inject-java: "true" + spec: + containers: + - name: app + image: nginx:1.14.2 + env: + - name: OTEL_AWS_PROFILER_ENABLED + value: "false" + restartPolicy: Always diff --git a/integration-tests/java-profiling/sample-deployment-java-profiling.yaml b/integration-tests/java-profiling/sample-deployment-java-profiling.yaml new file mode 100644 index 000000000..9cd764fdd --- /dev/null +++ b/integration-tests/java-profiling/sample-deployment-java-profiling.yaml @@ -0,0 +1,28 @@ +# Java profiling: two Java containers (app, worker) are profiled and share one profiler volume; the sidecar is not +# Java-instrumented and must get nothing from the profiler. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-java-profiling +spec: + selector: + matchLabels: + app: nginx-java-profiling + template: + metadata: + labels: + app: nginx-java-profiling + annotations: + instrumentation.opentelemetry.io/inject-java: "true" + instrumentation.opentelemetry.io/java-container-names: "app,worker" + spec: + containers: + - name: app + image: nginx:1.14.2 + - name: worker + image: nginx:1.14.2 + command: ["sh", "-c", "sleep 3600"] + - name: sidecar + image: nginx:1.14.2 + command: ["sh", "-c", "sleep 3600"] + restartPolicy: Always diff --git a/integration-tests/java-profiling/validate/validate.go b/integration-tests/java-profiling/validate/validate.go new file mode 100644 index 000000000..336fbce33 --- /dev/null +++ b/integration-tests/java-profiling/validate/validate.go @@ -0,0 +1,222 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Package validate checks what the operator injects into a pod for Java profiling: the OTEL_AWS_PROFILER_* +// variables of each profiled container, the shared profiler emptyDir and its mounts, and that other containers +// get none of it. +package validate + +import ( + "encoding/json" + "fmt" + "os" + "sort" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +const ( + // ProfilerVolumeName and ProfilerMountPath are the operator's profiler volume and its mount path. + ProfilerVolumeName = "opentelemetry-profiler-data-java" + ProfilerMountPath = "/otel-profiler-data-java" + profilerEnvPrefix = "OTEL_AWS_PROFILER_" +) + +// Expectations is one expectations file, for the pods of one label selector. +type Expectations struct { + // LabelSelector selects the pods to check. + LabelSelector string `json:"labelSelector"` + // JavaInstrumentedContainers must carry the javaagent in JAVA_TOOL_OPTIONS, so that an opt-out or unprofiled + // check cannot pass only because instrumentation did not happen at all. + JavaInstrumentedContainers []string `json:"javaInstrumentedContainers"` + // ProfiledContainers maps a container name to its exact OTEL_AWS_PROFILER_* variables: each set once, with that + // value, and no other OTEL_AWS_PROFILER_* variable. The container must also mount the volume. + ProfiledContainers map[string]map[string]string `json:"profiledContainers"` + // UnprofiledContainers must have no OTEL_AWS_PROFILER_* variable and no profiler mount, apart from the customer's + // own variables listed in UnprofiledAllowedEnv, which must keep exactly that value. + UnprofiledContainers []string `json:"unprofiledContainers"` + // UnprofiledAllowedEnv lists customer-set OTEL_AWS_PROFILER_* variables an unprofiled container keeps, for example + // the opt-out OTEL_AWS_PROFILER_ENABLED=false. + UnprofiledAllowedEnv map[string]string `json:"unprofiledAllowedEnv"` + // VolumeSizeLimit is the profiler volume's expected sizeLimit; empty means the pod must have no profiler volume. + VolumeSizeLimit string `json:"volumeSizeLimit"` +} + +// Load reads an expectations file. +func Load(path string) (Expectations, error) { + var exp Expectations + data, err := os.ReadFile(path) + if err != nil { + return exp, err + } + if err := json.Unmarshal(data, &exp); err != nil { + return exp, fmt.Errorf("parsing %s: %w", path, err) + } + if exp.LabelSelector == "" { + return exp, fmt.Errorf("%s: labelSelector is required", path) + } + if len(exp.JavaInstrumentedContainers) == 0 || (len(exp.ProfiledContainers) == 0 && len(exp.UnprofiledContainers) == 0) { + return exp, fmt.Errorf("%s: expectations check nothing", path) + } + if (exp.VolumeSizeLimit == "") != (len(exp.ProfiledContainers) == 0) { + return exp, fmt.Errorf("%s: volumeSizeLimit must be set exactly when profiledContainers is not empty", path) + } + return exp, nil +} + +// Pod returns one line per mismatch between pod and exp; an empty result means the pod passes. +func Pod(pod corev1.Pod, exp Expectations) []string { + var problems []string + fail := func(format string, args ...interface{}) { problems = append(problems, fmt.Sprintf(format, args...)) } + + containers := map[string]corev1.Container{} + for _, c := range pod.Spec.Containers { + containers[c.Name] = c + } + lookup := func(name string) (corev1.Container, bool) { + c, ok := containers[name] + if !ok { + fail("container %q: not in the pod", name) + } + return c, ok + } + + for _, name := range exp.JavaInstrumentedContainers { + if c, ok := lookup(name); ok && !strings.Contains(envValue(c, "JAVA_TOOL_OPTIONS"), "-javaagent:") { + fail("container %q: not Java-instrumented (no -javaagent in JAVA_TOOL_OPTIONS)", name) + } + } + + names := make([]string, 0, len(exp.ProfiledContainers)) + for name := range exp.ProfiledContainers { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + c, ok := lookup(name) + if !ok { + continue + } + keys := make([]string, 0, len(exp.ProfiledContainers[name])) + for key := range exp.ProfiledContainers[name] { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + want := exp.ProfiledContainers[name][key] + if got := envValues(c, key); len(got) == 0 { + fail("container %q: %s is not set, want %q", name, key, want) + } else if len(got) > 1 { + fail("container %q: %s is set %d times %q, want once", name, key, len(got), got) + } else if got[0] != want { + fail("container %q: %s=%q, want %q", name, key, got[0], want) + } + } + for _, env := range c.Env { + if _, expected := exp.ProfiledContainers[name][env.Name]; strings.HasPrefix(env.Name, profilerEnvPrefix) && !expected { + fail("container %q: unexpected %s=%q", name, env.Name, env.Value) + } + } + if mounts := profilerMounts(c); len(mounts) != 1 || mounts[0].Name != ProfilerVolumeName || + mounts[0].MountPath != ProfilerMountPath || mounts[0].ReadOnly || mounts[0].SubPath != "" { + fail("container %q: want one read-write mount of %s at %s, got %v", name, ProfilerVolumeName, ProfilerMountPath, mounts) + } + } + + for _, name := range exp.UnprofiledContainers { + c, ok := lookup(name) + if !ok { + continue + } + for _, env := range c.Env { + if !strings.HasPrefix(env.Name, profilerEnvPrefix) { + continue + } + if want, allowed := exp.UnprofiledAllowedEnv[env.Name]; !allowed { + fail("container %q: unexpected %s", name, env.Name) + } else if env.Value != want { + fail("container %q: %s=%q, want the customer's %q", name, env.Name, env.Value, want) + } + } + for key, want := range exp.UnprofiledAllowedEnv { + if got := envValues(c, key); len(got) != 1 { + fail("container %q: %s set %d times, want the customer's single %q", name, key, len(got), want) + } + } + if mounts := profilerMounts(c); len(mounts) > 0 { + fail("container %q: unexpected profiler mount %v", name, mounts) + } + } + + // The javaagent init container (and any other) never gets profiler variables or mounts. + for _, c := range pod.Spec.InitContainers { + for _, env := range c.Env { + if strings.HasPrefix(env.Name, profilerEnvPrefix) { + fail("init container %q: unexpected %s", c.Name, env.Name) + } + } + if mounts := profilerMounts(c); len(mounts) > 0 { + fail("init container %q: unexpected profiler mount %v", c.Name, mounts) + } + } + + var volumes []corev1.Volume + for _, v := range pod.Spec.Volumes { + if v.Name == ProfilerVolumeName { + volumes = append(volumes, v) + } + } + switch { + case exp.VolumeSizeLimit == "" && len(volumes) > 0: + fail("unexpected volume %s", ProfilerVolumeName) + case exp.VolumeSizeLimit == "": + case len(volumes) != 1: + fail("want exactly one volume %s, got %d", ProfilerVolumeName, len(volumes)) + case volumes[0].EmptyDir == nil: + fail("volume %s: not an emptyDir", ProfilerVolumeName) + case volumes[0].EmptyDir.Medium != corev1.StorageMediumDefault: + fail("volume %s: medium %q, want disk-backed", ProfilerVolumeName, volumes[0].EmptyDir.Medium) + case volumes[0].EmptyDir.SizeLimit == nil: + fail("volume %s: no sizeLimit, want %s", ProfilerVolumeName, exp.VolumeSizeLimit) + default: + want, err := resource.ParseQuantity(exp.VolumeSizeLimit) + if err != nil { + fail("volumeSizeLimit %q: %v", exp.VolumeSizeLimit, err) + } else if volumes[0].EmptyDir.SizeLimit.Cmp(want) != 0 { + fail("volume %s: sizeLimit %s, want %s", ProfilerVolumeName, volumes[0].EmptyDir.SizeLimit.String(), want.String()) + } + } + return problems +} + +// envValues returns every value the container sets for name, in order (the last one is effective). +func envValues(c corev1.Container, name string) []string { + var values []string + for _, env := range c.Env { + if env.Name == name { + values = append(values, env.Value) + } + } + return values +} + +func envValue(c corev1.Container, name string) string { + values := envValues(c, name) + if len(values) == 0 { + return "" + } + return values[len(values)-1] +} + +// profilerMounts returns the container's mounts of the profiler volume or of anything at the profiler mount path. +func profilerMounts(c corev1.Container) []corev1.VolumeMount { + var mounts []corev1.VolumeMount + for _, m := range c.VolumeMounts { + if m.Name == ProfilerVolumeName || strings.TrimRight(m.MountPath, "/") == ProfilerMountPath { + mounts = append(mounts, m) + } + } + return mounts +} diff --git a/integration-tests/java-profiling/validate/validate_test.go b/integration-tests/java-profiling/validate/validate_test.go new file mode 100644 index 000000000..9db54729d --- /dev/null +++ b/integration-tests/java-profiling/validate/validate_test.go @@ -0,0 +1,116 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package validate + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +func profiledPod() corev1.Pod { + size := resource.MustParse("64Mi") + return corev1.Pod{Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "app", Env: []corev1.EnvVar{ + {Name: "JAVA_TOOL_OPTIONS", Value: " -javaagent:/otel-auto-instrumentation-java/javaagent.jar"}, + {Name: "OTEL_AWS_PROFILER_ENABLED", Value: "true"}, + }, VolumeMounts: []corev1.VolumeMount{{Name: ProfilerVolumeName, MountPath: ProfilerMountPath}}}, + {Name: "sidecar"}, + }, + Volumes: []corev1.Volume{{Name: ProfilerVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: &size}}}}, + }} +} + +var profiledExp = Expectations{ + LabelSelector: "app=x", + JavaInstrumentedContainers: []string{"app"}, + ProfiledContainers: map[string]map[string]string{"app": {"OTEL_AWS_PROFILER_ENABLED": "true"}}, + UnprofiledContainers: []string{"sidecar"}, + VolumeSizeLimit: "67108864", +} + +func TestPodMatches(t *testing.T) { + assert.Empty(t, Pod(profiledPod(), profiledExp)) +} + +func TestPodMismatches(t *testing.T) { + tests := map[string]func(*corev1.Pod){ + "env missing": func(p *corev1.Pod) { p.Spec.Containers[0].Env = p.Spec.Containers[0].Env[:1] }, + "env value differs": func(p *corev1.Pod) { p.Spec.Containers[0].Env[1].Value = "false" }, + "not instrumented": func(p *corev1.Pod) { p.Spec.Containers[0].Env[0].Value = "" }, + "mount missing": func(p *corev1.Pod) { p.Spec.Containers[0].VolumeMounts = nil }, + "mount read-only": func(p *corev1.Pod) { p.Spec.Containers[0].VolumeMounts[0].ReadOnly = true }, + "volume missing": func(p *corev1.Pod) { p.Spec.Volumes = nil }, + "volume size differs": func(p *corev1.Pod) { q := resource.MustParse("128Mi"); p.Spec.Volumes[0].EmptyDir.SizeLimit = &q }, + "volume in memory": func(p *corev1.Pod) { p.Spec.Volumes[0].EmptyDir.Medium = corev1.StorageMediumMemory }, + "sidecar gets env": func(p *corev1.Pod) { + p.Spec.Containers[1].Env = []corev1.EnvVar{{Name: "OTEL_AWS_PROFILER_MODE", Value: "cpu"}} + }, + "sidecar gets the mount": func(p *corev1.Pod) { p.Spec.Containers[1].VolumeMounts = p.Spec.Containers[0].VolumeMounts }, + "container missing": func(p *corev1.Pod) { p.Spec.Containers = p.Spec.Containers[:1] }, + "extra profiler var": func(p *corev1.Pod) { + p.Spec.Containers[0].Env = append(p.Spec.Containers[0].Env, corev1.EnvVar{Name: "OTEL_AWS_PROFILER_VOLUME_SIZE_LIMIT", Value: "64Mi"}) + }, + "duplicate var": func(p *corev1.Pod) { + p.Spec.Containers[0].Env = append(p.Spec.Containers[0].Env, corev1.EnvVar{Name: "OTEL_AWS_PROFILER_ENABLED", Value: "false"}) + }, + "init container gets env": func(p *corev1.Pod) { + p.Spec.InitContainers = []corev1.Container{{Name: "init", Env: []corev1.EnvVar{{Name: "OTEL_AWS_PROFILER_ENABLED", Value: "true"}}}} + }, + "init container gets the mount": func(p *corev1.Pod) { + p.Spec.InitContainers = []corev1.Container{{Name: "init", VolumeMounts: p.Spec.Containers[0].VolumeMounts}} + }, + } + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + pod := profiledPod() + mutate(&pod) + assert.NotEmpty(t, Pod(pod, profiledExp)) + }) + } + // A customer opt-out variable is allowed with its value; another value, or an extra injected variable, is not. + optOutPod := corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "app", Env: []corev1.EnvVar{ + {Name: "JAVA_TOOL_OPTIONS", Value: " -javaagent:/otel-auto-instrumentation-java/javaagent.jar"}, + {Name: "OTEL_AWS_PROFILER_ENABLED", Value: "false"}, + }}}}} + envOptOut := Expectations{LabelSelector: "app=x", JavaInstrumentedContainers: []string{"app"}, UnprofiledContainers: []string{"app"}, + UnprofiledAllowedEnv: map[string]string{"OTEL_AWS_PROFILER_ENABLED": "false"}} + assert.Empty(t, Pod(optOutPod, envOptOut)) + flipped := *optOutPod.DeepCopy() + flipped.Spec.Containers[0].Env[1].Value = "true" + assert.NotEmpty(t, Pod(flipped, envOptOut)) + leaked := *optOutPod.DeepCopy() + leaked.Spec.Containers[0].Env = append(leaked.Spec.Containers[0].Env, corev1.EnvVar{Name: "OTEL_AWS_PROFILER_ENDPOINT", Value: "x"}) + assert.NotEmpty(t, Pod(leaked, envOptOut)) + missing := *optOutPod.DeepCopy() + missing.Spec.Containers[0].Env = missing.Spec.Containers[0].Env[:1] + assert.NotEmpty(t, Pod(missing, envOptOut)) + // With no profiled containers, any profiler volume is a mismatch. + optOut := Expectations{LabelSelector: "app=x", JavaInstrumentedContainers: []string{"app"}, UnprofiledContainers: []string{"app"}} + assert.NotEmpty(t, Pod(profiledPod(), optOut)) +} + +func TestLoad(t *testing.T) { + for _, file := range []string{"../java_profiling_expectations.json", "../java_profiling_opt_out_expectations.json"} { + _, err := Load(file) + assert.NoError(t, err, file) + } + // Files that would check nothing, or a volume size without profiled containers, are rejected. + for name, body := range map[string]string{ + "no selector": `{"javaInstrumentedContainers":["app"],"unprofiledContainers":["app"]}`, + "checks nothing": `{"labelSelector":"app=x","javaInstrumentedContainers":["app"]}`, + "size without pods": `{"labelSelector":"app=x","javaInstrumentedContainers":["app"],"unprofiledContainers":["app"],"volumeSizeLimit":"64Mi"}`, + "pods without a size": `{"labelSelector":"app=x","javaInstrumentedContainers":["app"],"profiledContainers":{"app":{}}}`, + } { + path := filepath.Join(t.TempDir(), "exp.json") + assert.NoError(t, os.WriteFile(path, []byte(body), 0o600)) + _, err := Load(path) + assert.Error(t, err, name) + } +} diff --git a/main.go b/main.go index 719072890..8926cdd51 100644 --- a/main.go +++ b/main.go @@ -114,6 +114,30 @@ func setLangEnvVars(langStr string, cfg map[string]map[string]string) { if dynamicInstrumentation, ok := cfg["dynamic_instrumentation"]; ok { setLangEnvVarsForDynamicInstrumentation(langStr, dynamicInstrumentation) } + if profiling, ok := cfg["profiling"]; ok && langStr == "JAVA" { + setLangEnvVarsForProfiling(langStr, profiling) + } +} + +// setLangEnvVarsForProfiling copies java.profiling.enabled into AUTO_INSTRUMENTATION_JAVA_PROFILING_ENABLED, +// and every other java.profiling key, as JSON, into AUTO_INSTRUMENTATION_JAVA_PROFILING_SETTINGS for the +// pod mutator, which maps them onto OTEL_AWS_PROFILER_*. +func setLangEnvVarsForProfiling(langStr string, cfg map[string]string) { + settings := map[string]string{} + for key, value := range cfg { + if key == "enabled" { + if value != "" { + _ = os.Setenv("AUTO_INSTRUMENTATION_"+langStr+"_PROFILING_ENABLED", value) + } + continue + } + settings[key] = value + } + if len(settings) > 0 { + if raw, err := json.Marshal(settings); err == nil { + _ = os.Setenv("AUTO_INSTRUMENTATION_"+langStr+"_PROFILING_SETTINGS", string(raw)) + } + } } func setLangEnvVarsForServiceEvents(langStr string, cfg map[string]string) { diff --git a/main_test.go b/main_test.go index 9c9e215a4..23d589b6b 100644 --- a/main_test.go +++ b/main_test.go @@ -112,3 +112,26 @@ func assertEnv(t *testing.T, name string, want *string) { t.Errorf("%s: got %q, want %q", name, got, *want) } } + +// Test_setLangEnvVars_profiling verifies that java.profiling.enabled reaches the pod mutator as +// AUTO_INSTRUMENTATION_JAVA_PROFILING_ENABLED, the other keys as JSON in _SETTINGS, and that other languages are ignored. +func Test_setLangEnvVars_profiling(t *testing.T) { + names := []string{"AUTO_INSTRUMENTATION_JAVA_PROFILING_ENABLED", "AUTO_INSTRUMENTATION_JAVA_PROFILING_MODE", "AUTO_INSTRUMENTATION_PYTHON_PROFILING_ENABLED", "AUTO_INSTRUMENTATION_JAVA_PROFILING_SETTINGS", "AUTO_INSTRUMENTATION_PYTHON_PROFILING_SETTINGS"} + unsetAll := func() { + for _, n := range names { + _ = os.Unsetenv(n) + } + } + unsetAll() + t.Cleanup(unsetAll) + + profiling := map[string]string{"enabled": "true", "mode": "wall"} + setLangEnvVars("JAVA", map[string]map[string]string{"profiling": profiling}) + setLangEnvVars("PYTHON", map[string]map[string]string{"profiling": profiling}) + + assertEnv(t, names[0], ptr("true")) + assertEnv(t, names[1], nil) + assertEnv(t, names[2], nil) + assertEnv(t, "AUTO_INSTRUMENTATION_JAVA_PROFILING_SETTINGS", ptr(`{"mode":"wall"}`)) + assertEnv(t, "AUTO_INSTRUMENTATION_PYTHON_PROFILING_SETTINGS", nil) +} diff --git a/pkg/instrumentation/helper.go b/pkg/instrumentation/helper.go index 7f329f2bf..62b0fbb12 100644 --- a/pkg/instrumentation/helper.go +++ b/pkg/instrumentation/helper.go @@ -36,6 +36,12 @@ const ( var defaultSize = resource.MustParse("200Mi") +// defaultProfilerVolumeSize is the default sizeLimit of the Java profiler's data volume (java.profiling.volume_size_limit). +// It covers ADOT's default settings: busy services peaked at 27 to 70 MB (measured), and ADOT caps rotated files at +// 128 MiB. Longer windows or allocation profiling need more, set in the chart. It is a ceiling, not a reservation; +// exceeding it gets the pod evicted. +var defaultProfilerVolumeSize = resource.MustParse("160Mi") + // Calculate if we already inject InitContainers. func isInitContainerMissing(pod corev1.Pod, containerName string) bool { for _, initContainer := range pod.Spec.InitContainers { diff --git a/pkg/instrumentation/javaprofiling.go b/pkg/instrumentation/javaprofiling.go new file mode 100644 index 000000000..ab820bcf2 --- /dev/null +++ b/pkg/instrumentation/javaprofiling.go @@ -0,0 +1,361 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package instrumentation + +import ( + "context" + "encoding/json" + "fmt" + "net" + "os" + "regexp" + "sort" + "strconv" + "strings" + "sync" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/aws/amazon-cloudwatch-agent-operator/internal/manifests/collector/adapters" +) + +// Continuous Java profiling with the async-profiler that the ADOT Java agent embeds +// (v2.32.0 and later). The operator enables the profiler, points it at the agent, gives it a dedicated +// disk-backed emptyDir for its data dir (the native library and rotating JFR files), and maps the chart's +// java.profiling settings onto ADOT's OTEL_AWS_PROFILER_* variables. Profiles go to the agent's OTLP HTTP endpoint +// (opentelemetry.collect.otlp.http_endpoint, 4318 unless its config sets another port), which feeds its profiles +// pipeline. +const ( + envJavaProfilingEnabled = "AUTO_INSTRUMENTATION_JAVA_PROFILING_ENABLED" + envJavaProfilingSettings = "AUTO_INSTRUMENTATION_JAVA_PROFILING_SETTINGS" + envProfilerPrefix = "OTEL_AWS_PROFILER_" + + envProfilerEnabled = "OTEL_AWS_PROFILER_ENABLED" + envProfilerEndpoint = "OTEL_AWS_PROFILER_ENDPOINT" + envProfilerDataDir = "OTEL_AWS_PROFILER_DATA_DIR" + envOTelExporterProtocol = "OTEL_EXPORTER_OTLP_PROTOCOL" + + // The agent's plain-HTTP OTLP receiver (opentelemetry.collect.otlp.http_endpoint) feeds its profiles pipeline; the + // Application Signals receiver (4316) has none. The port follows the cloudwatch-agent agent config, else 4318. + javaProfilerDefaultPort = 4318 + javaProfilerProfilesPath = "/v1development/profiles" + javaProfilerDefaultEndpoint = "http://" + cloudwatchAgentStandardEndpoint + ":4318" + javaProfilerProfilesPath + // fargateSchedulerName and fargateProfileLabel mark an EKS Fargate pod, which has no node-local agent. + fargateSchedulerName = "fargate-scheduler" + fargateProfileLabel = "eks.amazonaws.com/fargate-profile" + + // javaProfilerVolumeName is the pod-level emptyDir that holds every profiled container's data dir. It is + // disk-backed (no medium: Memory), so it is writable with readOnlyRootFilesystem and isolated from the + // javaagent volume; its sizeLimit is javaProfilingConfig.volumeSize. + javaProfilerVolumeName = "opentelemetry-profiler-data-java" + // javaProfilerMountPath is where the volume is mounted in each profiled Java container, and the default + // data_dir base: each container uses /. + javaProfilerMountPath = "/otel-profiler-data-java" +) + +// Reserved java.profiling keys (compared upper-cased): explicit handling, never mapped to OTEL_AWS_PROFILER_. +const ( + profilingKeyEnabled = "ENABLED" // the gate; main.go copies it to envJavaProfilingEnabled + profilingKeyEndpoint = "ENDPOINT" // rejected: profiles always go to the agent + profilingKeyDataDir = "DATA_DIR" // a base path; the operator appends / + profilingKeyVolumeSizeLimit = "VOLUME_SIZE_LIMIT" // the volume's sizeLimit, never an env var +) + +var ( + // javaProfilingKey is the strict form a java.profiling key must have to become an env var name. + javaProfilingKey = regexp.MustCompile(`^[A-Za-z0-9_]+$`) + // javaProfilingValue rejects control characters, which ADOT would echo into the pod's log. + javaProfilingValue = regexp.MustCompile(`^[^\x00-\x1f\x7f]+$`) + // javaProfilingPathSegment keeps a container name safe as one path segment (container names are DNS + // labels already; this is defensive). + javaProfilingPathSegment = regexp.MustCompile(`[^a-z0-9-]`) + // adotProfilerSettings are the OTEL_AWS_PROFILER_* names ADOT v2.32.0 reads that the generic mapping may set + // (ENABLED, ENDPOINT and DATA_DIR are reserved). Other names are still injected (ADOT ignores them) but + // reported, to catch typos. + adotProfilerSettings = map[string]bool{ + "MODE": true, "WALL_INTERVAL_MS": true, "CPU_INTERVAL_MS": true, "MEMORY_ENABLED": true, + "ALLOC_INTERVAL_BYTES": true, "AGGREGATION_MODE": true, "WINDOW_SECONDS": true, + "EXPORT_COMPRESSION": true, "EXPORT_TIMEOUT_MS": true, "MAX_PAYLOAD_BYTES": true, + } + // minProfilerVolumeSize is the smallest accepted volume_size_limit: below it ADOT's native library and first + // window would overflow the volume and get every profiled pod evicted (a typo such as "160m" is 0.16 bytes). + minProfilerVolumeSize = resource.MustParse("16Mi") + javaProfilingConfigOnce sync.Once + javaProfilingConfigData javaProfilingConfig +) + +// javaProfilingConfig is the parsed java.profiling block, other than enabled. +type javaProfilingConfig struct { + // settings are the generic OTEL_AWS_PROFILER_ variables, sorted by name. + settings []corev1.EnvVar + // dataDirBase is the data_dir base; each container's OTEL_AWS_PROFILER_DATA_DIR is /. + dataDirBase string + // volumeSize is the profiler volume's sizeLimit. + volumeSize resource.Quantity + // notes lists one line per skipped, rejected or unknown key, without values. + notes []string +} + +// javaProfilingEnabled reports whether main.go copied java.profiling.enabled "true" from +// --auto-instrumentation-config. +func javaProfilingEnabled() bool { + // Exactly "true", the value the chart requires for enabled ("true" or "false"). + return os.Getenv(envJavaProfilingEnabled) == "true" +} + +// The rule, as documented in the chart README: when java.profiling.enabled is "true", the reserved keys +// enabled, endpoint, data_dir and volume_size_limit have explicit meanings, and every other key under +// java.profiling becomes OTEL_AWS_PROFILER_. A variable already set on the container wins. +// +// parseJavaProfilingConfig parses the java.profiling keys other than enabled, as copied by main.go as JSON. +// Reserved keys, compared after upper-casing: +// - ENABLED is ignored here (main.go handles it); +// - ENDPOINT, and any name containing ENDPOINT, is rejected, so profiles always go to the agent Service; +// - DATA_DIR must be an absolute path without control characters or a ".." segment, else the default base is used; +// - VOLUME_SIZE_LIMIT must parse as a resource.Quantity of whole bytes, at least 16Mi, else the default size +// is used. It is the size per profiled container: the pod's volume gets it once for each. +// +// Generic keys: a key that is not letters, digits and underscores, an empty value (a non-string chart value +// also arrives empty), a value with control characters, or a second key with the same upper-cased name is +// skipped. A name ADOT v2.32.0 does not read is injected and reported. +func parseJavaProfilingConfig(raw string) javaProfilingConfig { + cfg := javaProfilingConfig{dataDirBase: javaProfilerMountPath, volumeSize: defaultProfilerVolumeSize.DeepCopy()} + if raw == "" { + return cfg + } + values := map[string]string{} + if err := json.Unmarshal([]byte(raw), &values); err != nil { + cfg.notes = append(cfg.notes, fmt.Sprintf("java.profiling settings are not a string map: %v", err)) + return cfg + } + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + seen := map[string]bool{} + for _, key := range keys { + value := values[key] + name := strings.ToUpper(key) + switch { + case !javaProfilingKey.MatchString(key): + cfg.notes = append(cfg.notes, fmt.Sprintf("key %q: only letters, digits and underscores", key)) + case name == profilingKeyEnabled: + // main.go reads only the lower-case key as the switch. + if key != "enabled" { + cfg.notes = append(cfg.notes, fmt.Sprintf("key %q: ignored, the switch is the lower-case key \"enabled\"", key)) + } + case strings.Contains(name, profilingKeyEndpoint): + cfg.notes = append(cfg.notes, fmt.Sprintf("key %q: not settable from the chart, profiles always go to the agent", key)) + case seen[name]: + cfg.notes = append(cfg.notes, fmt.Sprintf("key %q: duplicates %s", key, name)) + case value == "": + cfg.notes = append(cfg.notes, fmt.Sprintf("key %q: empty value (chart values must be strings)", key)) + case !javaProfilingValue.MatchString(value): + cfg.notes = append(cfg.notes, fmt.Sprintf("key %q: value has control characters", key)) + case name == profilingKeyDataDir: + seen[name] = true + if !strings.HasPrefix(value, "/") || hasDotDotSegment(value) { + cfg.notes = append(cfg.notes, fmt.Sprintf("key %q: must be an absolute path without a \"..\" segment; using %s", key, javaProfilerMountPath)) + continue + } + cfg.dataDirBase = value + case name == profilingKeyVolumeSizeLimit: + seen[name] = true + quantity, err := resource.ParseQuantity(value) + _, wholeBytes := quantity.AsInt64() + if err != nil || !wholeBytes || quantity.Cmp(minProfilerVolumeSize) < 0 { + cfg.notes = append(cfg.notes, fmt.Sprintf("key %q: not a whole number of bytes of at least %s; using %s", key, minProfilerVolumeSize.String(), defaultProfilerVolumeSize.String())) + continue + } + cfg.volumeSize = quantity + default: + seen[name] = true + cfg.settings = append(cfg.settings, corev1.EnvVar{Name: envProfilerPrefix + name, Value: value}) + if !adotProfilerSettings[name] { + cfg.notes = append(cfg.notes, fmt.Sprintf("key %q: injected, but ADOT v2.32.0 does not read %s%s", key, envProfilerPrefix, name)) + } + } + } + sort.SliceStable(cfg.settings, func(a, b int) bool { return cfg.settings[a].Name < cfg.settings[b].Name }) + return cfg +} + +// javaProfilingConfigFromEnv returns parseJavaProfilingConfig for the settings main.go copied, computed once +// (they do not change while the operator runs). first is true on the first call only, so the caller logs the +// notes once. +func javaProfilingConfigFromEnv() (cfg javaProfilingConfig, first bool) { + javaProfilingConfigOnce.Do(func() { + first = true + javaProfilingConfigData = parseJavaProfilingConfig(os.Getenv(envJavaProfilingSettings)) + }) + return javaProfilingConfigData, first +} + +// envSetByCustomer reports whether the container already defines name: directly (with a value or a +// valueFrom) or through envFrom, as resolved in allEnvs. +func envSetByCustomer(container *corev1.Container, allEnvs []corev1.EnvVar, name string) bool { + if getIndexOfEnv(container.Env, name) > -1 { + return true + } + return getIndexOfEnv(allEnvs, name) > -1 +} + +// javaProfilerVolumePresent reports whether the pod has a volume named javaProfilerVolumeName. Called on the pod +// as it was before any injection, it detects a customer volume with that name (a clash). +func javaProfilerVolumePresent(pod corev1.Pod) bool { + for _, volume := range pod.Spec.Volumes { + if volume.Name == javaProfilerVolumeName { + return true + } + } + return false +} + +// hasDotDotSegment reports whether path has a ".." segment ("/a/../b"), not just ".." inside a name ("/a..b"). +func hasDotDotSegment(path string) bool { + for _, segment := range strings.Split(path, "/") { + if segment == ".." { + return true + } + } + return false +} + +// javaProfilerDataDir returns /, with the container name reduced to one safe segment. +func javaProfilerDataDir(base, containerName string) string { + segment := javaProfilingPathSegment.ReplaceAllString(strings.ToLower(containerName), "-") + if segment == "" { + segment = "container" + } + return strings.TrimRight(base, "/") + "/" + segment +} + +// javaProfilerEndpointForConfig returns the profiles URL for the cloudwatch-agent agent's JSON config: the port of its +// opentelemetry.collect.otlp.http_endpoint on the agent Service, or the default endpoint when the config has no usable +// http_endpoint. The host is the agent Service; the bind address in http_endpoint is not a reachable host. +func javaProfilerEndpointForConfig(agentConfig string) string { + port := javaProfilerDefaultPort + if cfg, err := adapters.ConfigStructFromJSONString(agentConfig); err == nil && cfg != nil && + cfg.OpenTelemetry != nil && cfg.OpenTelemetry.Collect != nil && cfg.OpenTelemetry.Collect.OTLP != nil { + if _, p, splitErr := net.SplitHostPort(cfg.OpenTelemetry.Collect.OTLP.HTTPEndpoint); splitErr == nil { + if n, convErr := strconv.Atoi(p); convErr == nil && n > 0 && n <= 65535 { + port = n + } + } + } + return fmt.Sprintf("http://%s:%d%s", cloudwatchAgentStandardEndpoint, port, javaProfilerProfilesPath) +} + +// javaProfilerEndpointFromAgent reads the cloudwatch-agent AmazonCloudWatchAgent resource, as the default +// Instrumentation does, and returns javaProfilerEndpointForConfig of its config. +func javaProfilerEndpointFromAgent(ctx context.Context, c client.Client) string { + if c == nil { + return javaProfilerDefaultEndpoint + } + cr := GetAmazonCloudWatchAgentResource(ctx, c, amazonCloudWatchAgentName) + return javaProfilerEndpointForConfig(cr.Spec.Config) +} + +// isFargatePod reports whether the pod runs on EKS Fargate, where no node-local cloudwatch-agent receives profiles. +func isFargatePod(pod corev1.Pod) bool { + if pod.Spec.SchedulerName == fargateSchedulerName { + return true + } + _, ok := pod.Labels[fargateProfileLabel] + return ok +} + +// injectJavaProfiling adds OTEL_AWS_PROFILER_ENABLED, OTEL_AWS_PROFILER_ENDPOINT, OTEL_AWS_PROFILER_DATA_DIR and the +// chart's generic settings to the Java container at index after the javaagent was injected, mounts the profiler +// volume in it, and adds the volume to the pod once, growing its sizeLimit by cfg.volumeSize per profiled container. +// A variable the container already sets wins; a customer DATA_DIR is used verbatim. volumeClash is javaProfilerVolumePresent of the pod as it was before any injection, +// a customer volume with the profiler's name. endpoint is javaProfilerEndpointFromAgent. It returns whether it injected. +func injectJavaProfiling(enabled bool, cfg javaProfilingConfig, volumeClash bool, endpoint string, pod corev1.Pod, index int, allEnvs []corev1.EnvVar) (corev1.Pod, bool, error) { + if !enabled { + return pod, false, nil + } + if isFargatePod(pod) { + return pod, false, fmt.Errorf("the pod runs on Fargate, which has no node-local cloudwatch-agent") + } + // isWindowsPod reads only the nodeSelector; spec.os.name also marks a Windows pod. + if isWindowsPod(pod) || (pod.Spec.OS != nil && pod.Spec.OS.Name == corev1.Windows) { + return pod, false, fmt.Errorf("the ADOT Java profiler is linux only") + } + container := &pod.Spec.Containers[index] + // A container that turns the profiler off itself gets no volume or mount (its own value wins anyway). + if idx := getIndexOfEnv(container.Env, envProfilerEnabled); idx > -1 { + if container.Env[idx].ValueFrom == nil && !strings.EqualFold(strings.TrimSpace(container.Env[idx].Value), "true") { + return pod, false, nil + } + } else if idx := getIndexOfEnv(allEnvs, envProfilerEnabled); idx > -1 && !strings.EqualFold(strings.TrimSpace(allEnvs[idx].Value), "true") { + return pod, false, nil + } + // allEnvs resolves only ConfigMaps and ignores prefixes, so a Secret or a prefixed source could hold a + // profiler variable the operator cannot see; injecting would override the customer's value. + for _, from := range container.EnvFrom { + if from.SecretRef != nil || from.Prefix != "" { + return pod, false, fmt.Errorf("the container has an envFrom Secret or prefixed source, which may set OTEL_AWS_PROFILER_* variables the operator cannot read") + } + } + // ADOT sends gRPC when the OTLP protocol is grpc, which the HTTP endpoint does not accept. container.Env + // already holds the Instrumentation's env (injected before this) and allEnvs adds envFrom. A valueFrom + // protocol is treated as gRPC. The -Dotel.exporter.otlp.protocol system property (which ADOT reads first) + // is checked only in JAVA_TOOL_OPTIONS, not in the command, args or JDK_JAVA_OPTIONS. + if !envSetByCustomer(container, allEnvs, envProfilerEndpoint) { + idx := getIndexOfEnv(container.Env, envOTelExporterProtocol) + protocol := getEnvValue(allEnvs, envOTelExporterProtocol) + unknown := false + if idx > -1 { + protocol, unknown = container.Env[idx].Value, container.Env[idx].ValueFrom != nil + } + toolOptions := strings.ToLower(getEnvValue(container.Env, envJavaToolsOptions) + " " + getEnvValue(allEnvs, envJavaToolsOptions)) + if unknown || strings.EqualFold(strings.TrimSpace(protocol), "grpc") || + strings.Contains(toolOptions, "-dotel.exporter.otlp.protocol=grpc") { + return pod, false, fmt.Errorf("the container may use OTLP gRPC (%s or -Dotel.exporter.otlp.protocol), which %s does not accept; set %s to profile it", envOTelExporterProtocol, endpoint, envProfilerEndpoint) + } + } + // Never let the injected volume or mount make the pod invalid: skip this container instead. + if volumeClash { + return pod, false, fmt.Errorf("the pod already has a volume named %s", javaProfilerVolumeName) + } + for _, mount := range container.VolumeMounts { + if mount.Name == javaProfilerVolumeName || strings.TrimRight(mount.MountPath, "/") == javaProfilerMountPath { + return pod, false, fmt.Errorf("the container already mounts %s or %s", javaProfilerVolumeName, javaProfilerMountPath) + } + } + + envs := []corev1.EnvVar{ + {Name: envProfilerEnabled, Value: "true"}, + {Name: envProfilerEndpoint, Value: endpoint}, + {Name: envProfilerDataDir, Value: javaProfilerDataDir(cfg.dataDirBase, container.Name)}, + } + for _, setting := range cfg.settings { + if setting.Name != envProfilerEnabled && setting.Name != envProfilerDataDir && !strings.Contains(setting.Name, "ENDPOINT") { + envs = append(envs, setting) + } + } + for _, env := range envs { + if !envSetByCustomer(container, allEnvs, env.Name) { + container.Env = append(container.Env, env) + } + } + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{Name: javaProfilerVolumeName, MountPath: javaProfilerMountPath}) + // One volume per pod, sized volumeSize per profiled container: each JVM has its own 128 MiB JFR budget. + // A volume present here was added by this function for an earlier container (volumeClash covers the rest). + for i := range pod.Spec.Volumes { + if v := &pod.Spec.Volumes[i]; v.Name == javaProfilerVolumeName && v.EmptyDir != nil && v.EmptyDir.SizeLimit != nil { + v.EmptyDir.SizeLimit.Add(cfg.volumeSize) + return pod, true, nil + } + } + size := cfg.volumeSize.DeepCopy() + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: javaProfilerVolumeName, + VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: &size}}, + }) + return pod, true, nil +} diff --git a/pkg/instrumentation/javaprofiling_endpoint_test.go b/pkg/instrumentation/javaprofiling_endpoint_test.go new file mode 100644 index 000000000..0ae07af38 --- /dev/null +++ b/pkg/instrumentation/javaprofiling_endpoint_test.go @@ -0,0 +1,89 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package instrumentation + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + colfeaturegate "go.opentelemetry.io/collector/featuregate" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/aws/amazon-cloudwatch-agent-operator/apis/v1alpha1" + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/featuregate" +) + +// The profiler endpoint follows the port of the cloudwatch-agent agent's opentelemetry.collect.otlp.http_endpoint, +// rendered through the pod mutator in envtest, and falls back to 4318 when the agent config has none. +func TestJavaProfilingEndpointFollowsAgentConfig(t *testing.T) { + for _, gate := range []*colfeaturegate.Gate{featuregate.EnableMultiInstrumentationSupport, featuregate.SkipMultiInstrumentationContainerValidation} { + was := gate.IsEnabled() + require.NoError(t, colfeaturegate.GlobalRegistry().Set(gate.ID(), true)) + t.Cleanup(func() { require.NoError(t, colfeaturegate.GlobalRegistry().Set(gate.ID(), was)) }) + } + ctx := context.Background() + agentNs := corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: amazonCloudWatchNamespace}} + if err := k8sClient.Create(ctx, &agentNs); err != nil && !apierrors.IsAlreadyExists(err) { + require.NoError(t, err) + } + ns := corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "java-profiling-endpoint"}} + endpointOf := func(t *testing.T) string { + pod := renderITPod(t, "sample-deployment-java-profiling.yaml", ns) + return getEnvValue(pod.Spec.Containers[0].Env, envProfilerEndpoint) + } + + t.Run("no agent resource: default port", func(t *testing.T) { + setJavaProfilingFromITConfig(t, true) + assert.Equal(t, "http://cloudwatch-agent.amazon-cloudwatch:4318/v1development/profiles", endpointOf(t)) + }) + for name, tc := range map[string]struct{ config, want string }{ + "custom port": {`{"opentelemetry":{"collect":{"otlp":{"http_endpoint":"0.0.0.0:4320"}}}}`, "http://cloudwatch-agent.amazon-cloudwatch:4320/v1development/profiles"}, + "default:otel style, both ends": {`{"opentelemetry":{"collect":{"otlp":{"grpc_endpoint":"0.0.0.0:4317","http_endpoint":"0.0.0.0:4318"}}}}`, "http://cloudwatch-agent.amazon-cloudwatch:4318/v1development/profiles"}, + "no otlp receiver: default": {`{"logs":{"metrics_collected":{"application_signals":{}}}}`, "http://cloudwatch-agent.amazon-cloudwatch:4318/v1development/profiles"}, + "no port: default": {`{"opentelemetry":{"collect":{"otlp":{"http_endpoint":"0.0.0.0"}}}}`, "http://cloudwatch-agent.amazon-cloudwatch:4318/v1development/profiles"}, + "port out of range: default": {`{"opentelemetry":{"collect":{"otlp":{"http_endpoint":"0.0.0.0:70000"}}}}`, "http://cloudwatch-agent.amazon-cloudwatch:4318/v1development/profiles"}, + } { + t.Run(name, func(t *testing.T) { + setJavaProfilingFromITConfig(t, true) + agent := v1alpha1.AmazonCloudWatchAgent{ + ObjectMeta: metav1.ObjectMeta{Name: amazonCloudWatchAgentName, Namespace: amazonCloudWatchNamespace}, + Spec: v1alpha1.AmazonCloudWatchAgentSpec{Config: tc.config, UpgradeStrategy: v1alpha1.UpgradeStrategyNone}, + } + require.NoError(t, k8sClient.Create(ctx, &agent)) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, &agent) }) + assert.Equal(t, tc.want, endpointOf(t)) + }) + } +} + +// Fargate pods have no node-local cloudwatch-agent, so they are not profiled. +func TestInjectJavaProfilingSkipsFargate(t *testing.T) { + bySchedulerName := profilingPod() + bySchedulerName.Spec.SchedulerName = "fargate-scheduler" + byProfileLabel := profilingPod() + byProfileLabel.Labels = map[string]string{"eks.amazonaws.com/fargate-profile": "fp-default"} + for name, pod := range map[string]corev1.Pod{"scheduler name": bySchedulerName, "profile label": byProfileLabel} { + t.Run(name, func(t *testing.T) { + out, injected, err := injectJavaProfiling(true, defaultCfg(), false, javaProfilerDefaultEndpoint, pod, 0, nil) + assert.Error(t, err) + assert.False(t, injected) + assert.Equal(t, -1, getIndexOfEnv(out.Spec.Containers[0].Env, envProfilerEnabled)) + assert.Nil(t, profilerVolume(out)) + }) + } +} + +// No pod or namespace annotation opts out of profiling; the container's own OTEL_AWS_PROFILER_ENABLED=false does. +func TestInjectJavaProfilingIgnoresAnnotations(t *testing.T) { + pod := profilingPod() + pod.Annotations = map[string]string{"cloudwatch.aws.amazon.com/java-profiling": "false"} + out, injected, err := injectJavaProfiling(true, defaultCfg(), false, javaProfilerDefaultEndpoint, pod, 0, nil) + require.NoError(t, err) + assert.True(t, injected) + assert.Equal(t, "true", getEnvValue(out.Spec.Containers[0].Env, envProfilerEnabled)) +} diff --git a/pkg/instrumentation/javaprofiling_integration_test.go b/pkg/instrumentation/javaprofiling_integration_test.go new file mode 100644 index 000000000..7306591a9 --- /dev/null +++ b/pkg/instrumentation/javaprofiling_integration_test.go @@ -0,0 +1,120 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package instrumentation + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + colfeaturegate "go.opentelemetry.io/collector/featuregate" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/yaml" + + "github.com/aws/amazon-cloudwatch-agent-operator/integration-tests/java-profiling/validate" + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/featuregate" +) + +// The Java profiling integration case (integration-tests/java-profiling, run in minikube by +// operator-integration-test.yml) checked here against pods the pod mutator renders in envtest: the same +// deployments, operator config and expectations files, and the same validator. +const javaProfilingITDir = "../../integration-tests/java-profiling" + +// setJavaProfilingFromITConfig sets what main.go derives from the case's --auto-instrumentation-config, and resets +// the once-parsed settings so that this config is the one injected. +func setJavaProfilingFromITConfig(t *testing.T, enabled bool) { + t.Helper() + data, err := os.ReadFile(filepath.Join(javaProfilingITDir, "auto-instrumentation-config.json")) + require.NoError(t, err) + var cfg map[string]map[string]map[string]string + require.NoError(t, json.Unmarshal(data, &cfg)) + settings := map[string]string{} + for key, value := range cfg["java"]["profiling"] { + if key != "enabled" { + settings[key] = value + } + } + raw, err := json.Marshal(settings) + require.NoError(t, err) + if enabled { + t.Setenv(envJavaProfilingEnabled, cfg["java"]["profiling"]["enabled"]) + } else { + t.Setenv(envJavaProfilingEnabled, "") + } + t.Setenv(envJavaProfilingSettings, string(raw)) + // The default Instrumentation needs the images the deployed operator gets from its environment. + for _, name := range []string{"AUTO_INSTRUMENTATION_JAVA", "AUTO_INSTRUMENTATION_PYTHON", "AUTO_INSTRUMENTATION_DOTNET", "AUTO_INSTRUMENTATION_NODEJS"} { + t.Setenv(name, "it-image") + } + javaProfilingConfigOnce = sync.Once{} + t.Cleanup(func() { javaProfilingConfigOnce = sync.Once{} }) +} + +func renderITPod(t *testing.T, deploymentFile string, ns corev1.Namespace) corev1.Pod { + t.Helper() + data, err := os.ReadFile(filepath.Join(javaProfilingITDir, deploymentFile)) + require.NoError(t, err) + var deployment appsv1.Deployment + require.NoError(t, yaml.Unmarshal(data, &deployment)) + pod := corev1.Pod{ObjectMeta: deployment.Spec.Template.ObjectMeta, Spec: deployment.Spec.Template.Spec} + pod.Name, pod.Namespace = deployment.Name+"-pod", ns.Name + mutated, err := NewMutator(logr.Discard(), k8sClient, record.NewFakeRecorder(100)).Mutate(context.Background(), ns, pod) + require.NoError(t, err) + return mutated +} + +func loadITExpectations(t *testing.T, file string) validate.Expectations { + t.Helper() + exp, err := validate.Load(filepath.Join(javaProfilingITDir, file)) + require.NoError(t, err) + return exp +} + +func TestJavaProfilingIntegrationCase(t *testing.T) { + // The deployed operator runs with multi-instrumentation on (config/manager), which java-container-names needs; + // other tests in this package may leave the gate off. + for _, gate := range []*colfeaturegate.Gate{featuregate.EnableMultiInstrumentationSupport, featuregate.SkipMultiInstrumentationContainerValidation} { + was := gate.IsEnabled() + require.NoError(t, colfeaturegate.GlobalRegistry().Set(gate.ID(), true)) + t.Cleanup(func() { require.NoError(t, colfeaturegate.GlobalRegistry().Set(gate.ID(), was)) }) + } + // Mutate uses this object as given; the namespace need not exist (envtest cannot delete one). + ns := corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "java-profiling-it"}} + profiled := loadITExpectations(t, "java_profiling_expectations.json") + optOut := loadITExpectations(t, "java_profiling_opt_out_expectations.json") + + t.Run("profiled pod matches", func(t *testing.T) { + setJavaProfilingFromITConfig(t, true) + assert.Empty(t, validate.Pod(renderITPod(t, "sample-deployment-java-profiling.yaml", ns), profiled)) + }) + t.Run("opt-out pod matches", func(t *testing.T) { + setJavaProfilingFromITConfig(t, true) + assert.Empty(t, validate.Pod(renderITPod(t, "sample-deployment-java-profiling-opt-out.yaml", ns), optOut)) + }) + // Controls: the case must fail when the operator does not inject, and the opt-out expectations must fail + // on a profiled pod, so neither check can pass vacuously. + t.Run("profiling off fails the profiled expectations", func(t *testing.T) { + setJavaProfilingFromITConfig(t, false) + pod := renderITPod(t, "sample-deployment-java-profiling.yaml", ns) + problems := strings.Join(validate.Pod(pod, profiled), "\n") + assert.Contains(t, problems, "OTEL_AWS_PROFILER_ENABLED is not set") + assert.Contains(t, problems, "want exactly one volume "+validate.ProfilerVolumeName) + assert.Empty(t, validate.Pod(pod, validate.Expectations{JavaInstrumentedContainers: []string{"app", "worker"}}), + "the pod is still Java-instrumented, so only the profiler checks fail") + }) + t.Run("a profiled pod fails the opt-out expectations", func(t *testing.T) { + setJavaProfilingFromITConfig(t, true) + assert.NotEmpty(t, validate.Pod(renderITPod(t, "sample-deployment-java-profiling.yaml", ns), optOut)) + }) +} diff --git a/pkg/instrumentation/javaprofiling_test.go b/pkg/instrumentation/javaprofiling_test.go new file mode 100644 index 000000000..e53be6607 --- /dev/null +++ b/pkg/instrumentation/javaprofiling_test.go @@ -0,0 +1,315 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package instrumentation + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/aws/amazon-cloudwatch-agent-operator/apis/v1alpha1" +) + +const agentEndpoint = "http://cloudwatch-agent.amazon-cloudwatch:4318/v1development/profiles" + +func profilingPod(env ...corev1.EnvVar) corev1.Pod { + return corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "app", Env: env}}}} +} + +func defaultCfg() javaProfilingConfig { return parseJavaProfilingConfig("") } + +func cfgFrom(t *testing.T, values map[string]string) javaProfilingConfig { + raw, err := json.Marshal(values) + require.NoError(t, err) + return parseJavaProfilingConfig(string(raw)) +} + +func profilerVolume(pod corev1.Pod) *corev1.Volume { + for i := range pod.Spec.Volumes { + if pod.Spec.Volumes[i].Name == "opentelemetry-profiler-data-java" { + return &pod.Spec.Volumes[i] + } + } + return nil +} + +func profilerMounts(c corev1.Container) int { + n := 0 + for _, m := range c.VolumeMounts { + if m.Name == "opentelemetry-profiler-data-java" && m.MountPath == "/otel-profiler-data-java" { + n++ + } + } + return n +} + +func countEnv(envs []corev1.EnvVar, name string) int { + n := 0 + for _, e := range envs { + if e.Name == name { + n++ + } + } + return n +} + +func TestInjectJavaProfilingDefaults(t *testing.T) { + pod, injected, err := injectJavaProfiling(true, defaultCfg(), false, javaProfilerDefaultEndpoint, profilingPod(), 0, nil) + require.NoError(t, err) + assert.True(t, injected) + assert.Equal(t, []corev1.EnvVar{ + {Name: "OTEL_AWS_PROFILER_ENABLED", Value: "true"}, + {Name: "OTEL_AWS_PROFILER_ENDPOINT", Value: agentEndpoint}, + {Name: "OTEL_AWS_PROFILER_DATA_DIR", Value: "/otel-profiler-data-java/app"}, + }, pod.Spec.Containers[0].Env) + v := profilerVolume(pod) + require.NotNil(t, v, "one pod-level profiler volume") + require.NotNil(t, v.EmptyDir) + assert.Equal(t, corev1.StorageMedium(""), v.EmptyDir.Medium, "disk-backed, not memory") + assert.Equal(t, "160Mi", v.EmptyDir.SizeLimit.String()) + assert.Equal(t, 1, profilerMounts(pod.Spec.Containers[0])) +} + +func TestJavaProfilingVolumeSize(t *testing.T) { + cases := map[string]string{"64Mi": "64Mi", "1Gi": "1Gi", "500M": "500M", "16Mi": "16Mi", "abc": "160Mi", "-1Mi": "160Mi", "0": "160Mi", "": "160Mi", + "160m": "160Mi", "1": "160Mi", "15Mi": "160Mi", "1.5": "160Mi", "20000000.5": "160Mi"} + for in, want := range cases { + cfg := cfgFrom(t, map[string]string{"volume_size_limit": in}) + pod, _, err := injectJavaProfiling(true, cfg, false, javaProfilerDefaultEndpoint, profilingPod(), 0, nil) + require.NoError(t, err) + assert.Equal(t, want, profilerVolume(pod).EmptyDir.SizeLimit.String(), in) + assert.Equal(t, -1, getIndexOfEnv(pod.Spec.Containers[0].Env, "OTEL_AWS_PROFILER_VOLUME_SIZE_LIMIT"), "never an env var") + } + cfg := cfgFrom(t, map[string]string{"volume_size_limit": "abc"}) + assert.Len(t, cfg.notes, 1, "an invalid size is logged") + want := resource.MustParse("160Mi") + assert.Equal(t, 0, want.Cmp(defaultProfilerVolumeSize), "the default is 160Mi") +} + +func TestJavaProfilingDataDir(t *testing.T) { + // The chart base gets the container name appended; the volume is still mounted when the base is outside it. + cfg := cfgFrom(t, map[string]string{"data_dir": "/var/profiler/"}) + pod, _, err := injectJavaProfiling(true, cfg, false, javaProfilerDefaultEndpoint, profilingPod(), 0, nil) + require.NoError(t, err) + assert.Equal(t, "/var/profiler/app", getEnvValue(pod.Spec.Containers[0].Env, "OTEL_AWS_PROFILER_DATA_DIR")) + assert.Equal(t, 1, profilerMounts(pod.Spec.Containers[0])) + assert.NotNil(t, profilerVolume(pod)) + + // A relative or ".." base falls back to the default and is logged. + for _, bad := range []string{"relative/dir", "/a/../etc", "/a/.."} { + cfg := cfgFrom(t, map[string]string{"data_dir": bad}) + assert.Len(t, cfg.notes, 1, bad) + pod, _, _ := injectJavaProfiling(true, cfg, false, javaProfilerDefaultEndpoint, profilingPod(), 0, nil) + assert.Equal(t, "/otel-profiler-data-java/app", getEnvValue(pod.Spec.Containers[0].Env, "OTEL_AWS_PROFILER_DATA_DIR"), bad) + } + + // ".." inside a segment is a legitimate name. + cfg = cfgFrom(t, map[string]string{"data_dir": "/data/a..b"}) + assert.Empty(t, cfg.notes) + pod, _, _ = injectJavaProfiling(true, cfg, false, javaProfilerDefaultEndpoint, profilingPod(), 0, nil) + assert.Equal(t, "/data/a..b/app", getEnvValue(pod.Spec.Containers[0].Env, "OTEL_AWS_PROFILER_DATA_DIR")) + + // The pod's own DATA_DIR wins, verbatim, with no container suffix; direct env, valueFrom or envFrom. + own := profilingPod(corev1.EnvVar{Name: "OTEL_AWS_PROFILER_DATA_DIR", Value: "/mine"}) + pod, injected, err := injectJavaProfiling(true, cfg, false, javaProfilerDefaultEndpoint, own, 0, own.Spec.Containers[0].Env) + require.NoError(t, err) + assert.True(t, injected) + assert.Equal(t, 1, countEnv(pod.Spec.Containers[0].Env, "OTEL_AWS_PROFILER_DATA_DIR")) + assert.Equal(t, "/mine", getEnvValue(pod.Spec.Containers[0].Env, "OTEL_AWS_PROFILER_DATA_DIR")) + pod, _, _ = injectJavaProfiling(true, cfg, false, javaProfilerDefaultEndpoint, profilingPod(), 0, []corev1.EnvVar{{Name: "OTEL_AWS_PROFILER_DATA_DIR", Value: "/from-configmap"}}) + assert.Equal(t, -1, getIndexOfEnv(pod.Spec.Containers[0].Env, "OTEL_AWS_PROFILER_DATA_DIR"), "a ConfigMap envFrom value wins") +} + +func TestInjectJavaProfilingTwoContainers(t *testing.T) { + pod := corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "api"}, {Name: "worker_2"}}}} + pod, injected, err := injectJavaProfiling(true, defaultCfg(), false, javaProfilerDefaultEndpoint, pod, 0, nil) + require.NoError(t, err) + assert.True(t, injected) + pod, injected, err = injectJavaProfiling(true, defaultCfg(), false, javaProfilerDefaultEndpoint, pod, 1, nil) + require.NoError(t, err) + assert.True(t, injected) + assert.Equal(t, "/otel-profiler-data-java/api", getEnvValue(pod.Spec.Containers[0].Env, "OTEL_AWS_PROFILER_DATA_DIR")) + assert.Equal(t, "/otel-profiler-data-java/worker-2", getEnvValue(pod.Spec.Containers[1].Env, "OTEL_AWS_PROFILER_DATA_DIR"), "sanitised to one segment") + assert.Equal(t, 1, profilerMounts(pod.Spec.Containers[0])) + assert.Equal(t, 1, profilerMounts(pod.Spec.Containers[1])) + n := 0 + for _, v := range pod.Spec.Volumes { + if v.Name == "opentelemetry-profiler-data-java" { + n++ + } + } + assert.Equal(t, 1, n, "one pod-level volume for both containers") + assert.Equal(t, "320Mi", profilerVolume(pod).EmptyDir.SizeLimit.String(), "the size is per profiled container") +} + +func TestInjectJavaProfilingSkipsAddNoVolume(t *testing.T) { + windows := profilingPod() + windows.Spec.NodeSelector = map[string]string{"kubernetes.io/os": "windows"} + windowsOS := profilingPod() + windowsOS.Spec.OS = &corev1.PodOS{Name: corev1.Windows} + grpc := profilingPod(corev1.EnvVar{Name: "OTEL_EXPORTER_OTLP_PROTOCOL", Value: "grpc"}) + grpcValueFrom := profilingPod(corev1.EnvVar{Name: "OTEL_EXPORTER_OTLP_PROTOCOL", ValueFrom: &corev1.EnvVarSource{ + ConfigMapKeyRef: &corev1.ConfigMapKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "c"}, Key: "p"}}}) + grpcSysprop := profilingPod(corev1.EnvVar{Name: "JAVA_TOOL_OPTIONS", Value: "-Dotel.exporter.otlp.protocol=grpc"}) + secretFrom := profilingPod() + secretFrom.Spec.Containers[0].EnvFrom = []corev1.EnvFromSource{{SecretRef: &corev1.SecretEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "s"}}}} + prefixedFrom := profilingPod() + prefixedFrom.Spec.Containers[0].EnvFrom = []corev1.EnvFromSource{{Prefix: "OTEL_AWS_", ConfigMapRef: &corev1.ConfigMapEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "c"}}}} + ownOff := profilingPod(corev1.EnvVar{Name: "OTEL_AWS_PROFILER_ENABLED", Value: "false"}) + grpcSyspropFrom := []corev1.EnvVar{{Name: "JAVA_TOOL_OPTIONS", Value: "-Xmx1g -Dotel.exporter.otlp.protocol=GRPC"}} + + tests := []struct { + name string + enabled bool + pod corev1.Pod + allEnvs []corev1.EnvVar + wantErr bool + }{ + {name: "disabled", pod: profilingPod()}, + {name: "windows", enabled: true, pod: windows, wantErr: true}, + {name: "windows from spec.os", enabled: true, pod: windowsOS, wantErr: true}, + {name: "grpc on the container", enabled: true, pod: grpc, wantErr: true}, + {name: "grpc from envFrom", enabled: true, pod: profilingPod(), allEnvs: []corev1.EnvVar{{Name: "OTEL_EXPORTER_OTLP_PROTOCOL", Value: "GRPC"}}, wantErr: true}, + {name: "protocol from valueFrom", enabled: true, pod: grpcValueFrom, wantErr: true}, + {name: "grpc system property", enabled: true, pod: grpcSysprop, wantErr: true}, + {name: "grpc system property from envFrom, any case", enabled: true, pod: profilingPod(), allEnvs: grpcSyspropFrom, wantErr: true}, + {name: "container turns the profiler off", enabled: true, pod: ownOff, allEnvs: ownOff.Spec.Containers[0].Env}, + {name: "envFrom turns the profiler off", enabled: true, pod: profilingPod(), allEnvs: []corev1.EnvVar{{Name: "OTEL_AWS_PROFILER_ENABLED", Value: "false"}}}, + {name: "secret envFrom", enabled: true, pod: secretFrom, wantErr: true}, + {name: "prefixed envFrom", enabled: true, pod: prefixedFrom, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + before := len(tt.pod.Spec.Containers[0].Env) + pod, injected, err := injectJavaProfiling(tt.enabled, defaultCfg(), false, javaProfilerDefaultEndpoint, tt.pod, 0, tt.allEnvs) + assert.Equal(t, tt.wantErr, err != nil, err) + assert.False(t, injected) + assert.Len(t, pod.Spec.Containers[0].Env, before, "env unchanged") + assert.Nil(t, profilerVolume(pod), "no volume") + assert.Equal(t, 0, profilerMounts(pod.Spec.Containers[0]), "no mount") + }) + } +} + +func TestInjectJavaProfilingClashes(t *testing.T) { + // A customer volume with the profiler's name: skipped, not rejected by the API server. + pod := profilingPod() + pod.Spec.Volumes = []corev1.Volume{{Name: "opentelemetry-profiler-data-java", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}} + out, injected, err := injectJavaProfiling(true, defaultCfg(), javaProfilerVolumePresent(pod), javaProfilerDefaultEndpoint, pod, 0, nil) + assert.Error(t, err) + assert.False(t, injected) + assert.Len(t, out.Spec.Volumes, 1) + assert.Empty(t, out.Spec.Containers[0].Env) + + // A customer mount at the profiler's path, or of a volume with its name. + for _, mount := range []corev1.VolumeMount{{Name: "mine", MountPath: "/otel-profiler-data-java/"}, {Name: "opentelemetry-profiler-data-java", MountPath: "/elsewhere"}} { + pod := profilingPod() + pod.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{mount} + out, injected, err := injectJavaProfiling(true, defaultCfg(), false, javaProfilerDefaultEndpoint, pod, 0, nil) + assert.Error(t, err, mount.MountPath) + assert.False(t, injected) + assert.Nil(t, profilerVolume(out)) + assert.Len(t, out.Spec.Containers[0].VolumeMounts, 1) + } +} + +func TestJavaProfilingReservedKeys(t *testing.T) { + cfg := cfgFrom(t, map[string]string{ + "enabled": "true", "Enabled": "false", "endpoint": "http://evil:4318", "export_endpoint": "x", "_ENDPOINT": "x", + "data_dir": "/base", "volume_size_limit": "64Mi", "mode": "cpu", + }) + assert.Equal(t, []corev1.EnvVar{{Name: "OTEL_AWS_PROFILER_MODE", Value: "cpu"}}, cfg.settings, "reserved keys never become generic env vars") + assert.Equal(t, "/base", cfg.dataDirBase) + assert.Equal(t, "64Mi", cfg.volumeSize.String()) + notes := strings.Join(cfg.notes, "\n") + assert.Contains(t, notes, `"endpoint"`) + assert.Contains(t, notes, `"export_endpoint"`) + assert.Contains(t, notes, `"Enabled": ignored`, "only the lower-case key is the switch") + assert.NotContains(t, notes, "http://evil", "values are never logged") + + pod, _, err := injectJavaProfiling(true, cfg, false, javaProfilerDefaultEndpoint, profilingPod(), 0, nil) + require.NoError(t, err) + env := pod.Spec.Containers[0].Env + assert.Equal(t, "true", getEnvValue(env, "OTEL_AWS_PROFILER_ENABLED")) + assert.Equal(t, agentEndpoint, getEnvValue(env, "OTEL_AWS_PROFILER_ENDPOINT")) + assert.Equal(t, "/base/app", getEnvValue(env, "OTEL_AWS_PROFILER_DATA_DIR")) + assert.Equal(t, -1, getIndexOfEnv(env, "OTEL_AWS_PROFILER_VOLUME_SIZE_LIMIT")) + assert.Equal(t, -1, getIndexOfEnv(env, "OTEL_AWS_PROFILER_EXPORT_ENDPOINT")) + assert.Equal(t, "cpu", getEnvValue(env, "OTEL_AWS_PROFILER_MODE")) +} + +func TestJavaProfilingGenericKeys(t *testing.T) { + keys := []string{"mode", "wall_interval_ms", "cpu_interval_ms", "memory_enabled", "alloc_interval_bytes", + "window_seconds", "aggregation_mode", "export_compression", "export_timeout_ms", "max_payload_bytes"} + values := map[string]string{} + for _, k := range keys { + values[k] = "v-" + k + } + cfg := cfgFrom(t, values) + assert.Empty(t, cfg.notes, "every settable ADOT v2.32.0 option is known") + require.Len(t, cfg.settings, len(keys)) + for _, k := range keys { + assert.Equal(t, "v-"+k, getEnvValue(cfg.settings, "OTEL_AWS_PROFILER_"+strings.ToUpper(k)), k) + } + assert.Len(t, adotProfilerSettings, len(keys), "the known-name list matches the chart's generic list") + + // Strict names, control characters, empty values, duplicates and unknown names. + cfg = cfgFrom(t, map[string]string{"mode-x": "cpu", "a.b": "1", "empty": "", "MODE": "wall", "mode": "cpu", "window_seconds": "30\nforged", "windows_seconds": "30"}) + assert.Equal(t, []corev1.EnvVar{{Name: "OTEL_AWS_PROFILER_MODE", Value: "wall"}, {Name: "OTEL_AWS_PROFILER_WINDOWS_SECONDS", Value: "30"}}, cfg.settings) + assert.Len(t, cfg.notes, 6) + assert.NotContains(t, strings.Join(cfg.notes, "\n"), "forged") + assert.Len(t, parseJavaProfilingConfig("not json").notes, 1) +} + +func TestInjectJavaProfilingCustomerWins(t *testing.T) { + cfg := cfgFrom(t, map[string]string{"mode": "cpu", "memory_enabled": "true"}) + ref := &corev1.EnvVarSource{ConfigMapKeyRef: &corev1.ConfigMapKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "c"}, Key: "mode"}} + own := []corev1.EnvVar{{Name: "OTEL_AWS_PROFILER_MODE", ValueFrom: ref}} + allEnvs := append(append([]corev1.EnvVar{}, own...), corev1.EnvVar{Name: "OTEL_AWS_PROFILER_MEMORY_ENABLED", Value: ""}) + pod, _, err := injectJavaProfiling(true, cfg, false, javaProfilerDefaultEndpoint, profilingPod(own...), 0, allEnvs) + require.NoError(t, err) + env := pod.Spec.Containers[0].Env + assert.Equal(t, 1, countEnv(env, "OTEL_AWS_PROFILER_MODE"), "a valueFrom on the pod wins") + assert.NotNil(t, env[0].ValueFrom) + assert.Equal(t, -1, getIndexOfEnv(env, "OTEL_AWS_PROFILER_MEMORY_ENABLED"), "an empty ConfigMap value still counts as set") +} + +func TestJavaProfilingEnabled(t *testing.T) { + t.Setenv("AUTO_INSTRUMENTATION_JAVA_PROFILING_ENABLED", "TRUE") + assert.False(t, javaProfilingEnabled(), "exactly \"true\", like the chart") + t.Setenv("AUTO_INSTRUMENTATION_JAVA_PROFILING_ENABLED", "true") + assert.True(t, javaProfilingEnabled()) +} + +// TestInjectJavaProfilingThroughSDKInjector covers the wiring in sdkInjector.inject: per-container decisions, one +// volume sized per profiled container, and a skipped container left without a mount. +func TestInjectJavaProfilingThroughSDKInjector(t *testing.T) { + t.Setenv(envJavaProfilingEnabled, "true") + inst := v1alpha1.Instrumentation{Spec: v1alpha1.InstrumentationSpec{ + Java: v1alpha1.Java{Image: "img:1"}, + Exporter: v1alpha1.Exporter{Endpoint: "http://collector:4316"}, + }} + insts := languageInstrumentations{Java: instrumentationWithContainers{Instrumentation: &inst, Containers: "api,grpc,worker"}} + inj := sdkInjector{logger: logr.Discard()} + pod := inj.inject(context.Background(), insts, corev1.Namespace{}, corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{ + {Name: "api"}, + {Name: "grpc", Env: []corev1.EnvVar{{Name: "OTEL_EXPORTER_OTLP_PROTOCOL", Value: "grpc"}}}, + {Name: "worker"}, + }}}) + require.NotNil(t, profilerVolume(pod)) + assert.Equal(t, "320Mi", profilerVolume(pod).EmptyDir.SizeLimit.String(), "two profiled containers") + assert.Equal(t, "/otel-profiler-data-java/api", getEnvValue(pod.Spec.Containers[0].Env, "OTEL_AWS_PROFILER_DATA_DIR")) + assert.Equal(t, "/otel-profiler-data-java/worker", getEnvValue(pod.Spec.Containers[2].Env, "OTEL_AWS_PROFILER_DATA_DIR")) + assert.Equal(t, 1, profilerMounts(pod.Spec.Containers[0])) + assert.Equal(t, 0, profilerMounts(pod.Spec.Containers[1]), "the gRPC container is skipped") + assert.Equal(t, -1, getIndexOfEnv(pod.Spec.Containers[1].Env, "OTEL_AWS_PROFILER_ENABLED")) + assert.Equal(t, 1, profilerMounts(pod.Spec.Containers[2])) +} diff --git a/pkg/instrumentation/sdk.go b/pkg/instrumentation/sdk.go index 2fbe589b3..e63151f39 100644 --- a/pkg/instrumentation/sdk.go +++ b/pkg/instrumentation/sdk.go @@ -84,6 +84,14 @@ func (i *sdkInjector) inject(ctx context.Context, insts languageInstrumentations i.logger.V(1).Info("injecting Java instrumentation into pod", "otelinst-namespace", otelinst.Namespace, "otelinst-name", otelinst.Name) javaContainers := insts.Java.Containers + // A customer volume named like the profiler's makes every container skip profiling (recorded before the + // first container adds the operator's own volume). + profilerVolumeClash := javaProfilerVolumePresent(pod) + // Read the agent config only when profiling is on, so a disabled operator makes no extra call. + profilerEndpoint := javaProfilerDefaultEndpoint + if javaProfilingEnabled() { + profilerEndpoint = javaProfilerEndpointFromAgent(ctx, i.client) + } for _, container := range strings.Split(javaContainers, ",") { index := getContainerIndex(container, pod) @@ -99,6 +107,18 @@ func (i *sdkInjector) inject(ctx context.Context, insts languageInstrumentations } else { pod = i.injectCommonEnvVar(otelinst, pod, index) pod = i.injectCommonSDKConfig(ctx, otelinst, ns, pod, index, index) + var profiled bool + var profErr error + profilingConfig, first := javaProfilingConfigFromEnv() + if first && len(profilingConfig.notes) > 0 && javaProfilingEnabled() { + i.logger.Info("java.profiling settings: some keys were skipped, rejected or are unknown to ADOT", "keys", profilingConfig.notes) + } + pod, profiled, profErr = injectJavaProfiling(javaProfilingEnabled(), profilingConfig, profilerVolumeClash, profilerEndpoint, pod, index, envs) + if profErr != nil { + i.logger.Info("Skipping Java profiling injection", "reason", profErr.Error(), "container", pod.Spec.Containers[index].Name) + } else if profiled { + i.logger.V(1).Info("enabled the ADOT Java profiler", "container", pod.Spec.Containers[index].Name) + } //disable setting security context in init container due to issue with runAsNonRoot conflict //https://github.com/open-telemetry/opentelemetry-operator/issues/2272 //pod = i.setInitContainerSecurityContext(pod, pod.Spec.Containers[index].SecurityContext, javaInitContainerName)