Skip to content

test(otel): cover the Helm chart's default-on OTLP endpoint on EKS and AKS #802

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
miconeilaws wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

test(otel): cover the Helm chart's default-on OTLP endpoint on EKS and AKS #802

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions generator/test_case_generator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -496,6 +496,20 @@ var testTypeToTestConfig = map[string][]testConfig{
ami: "AL2023_x86_64_STANDARD",
k8sVersion: "1.35",
},
{
testDir: "./test/otel/otlp",
terraformDir: "terraform/eks/daemon/otel-otlp",
targets: map[string]map[string]struct{}{"arc": {"amd64": {}}},
ami: "AL2023_x86_64_STANDARD",
k8sVersion: "1.35",
},
{
testDir: "./test/otel/otlp_disabled",
terraformDir: "terraform/eks/daemon/otel-otlp-disabled",
targets: map[string]map[string]struct{}{"arc": {"amd64": {}}},
ami: "AL2023_x86_64_STANDARD",
k8sVersion: "1.35",
},
{
testDir: "./test/otel/ebs_csi",
terraformDir: "terraform/eks/daemon/otel-ebs-csi",
Expand Down
28 changes: 26 additions & 2 deletions terraform/azure/aks/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -407,7 +407,7 @@ resource "null_resource" "agent_diagnostics" {
EOT
}

depends_on = [kubernetes_daemon_set_v1.cwagent, kubernetes_job_v1.otlp_load, null_resource.ci_helm_chart]
depends_on = [kubernetes_daemon_set_v1.cwagent, kubernetes_job_v1.otlp_load, null_resource.ci_helm_chart, null_resource.ci_otlp_sender]
}

#####################################################################
Expand All @@ -433,7 +433,7 @@ resource "null_resource" "integration_test" {
}
}

depends_on = [kubernetes_daemon_set_v1.cwagent, kubernetes_job_v1.otlp_load, null_resource.ci_helm_chart, null_resource.ci_extra_manifests, null_resource.agent_diagnostics]
depends_on = [kubernetes_daemon_set_v1.cwagent, kubernetes_job_v1.otlp_load, null_resource.ci_helm_chart, null_resource.ci_extra_manifests, null_resource.ci_otlp_sender, null_resource.agent_diagnostics]
}

#####################################################################
Expand Down Expand Up @@ -513,6 +513,30 @@ resource "null_resource" "ci_helm_chart" {
]
}

#####################################################################
# OTLP sender (test_mode == "containerinsights"): an ordinary pod that
# sends OTLP through the cloudwatch-agent Service to the chart's OTLP
# endpoint, which the chart serves by default. default:otel in the
# install values above is an alias of the default config, so the
# endpoint comes from the chart, not from the agent's default:otel.
#####################################################################
resource "null_resource" "ci_otlp_sender" {
count = local.is_ci ? 1 : 0
provisioner "local-exec" {
command = <<-EOT
set -euo pipefail
KC='${local_sensitive_file.kubeconfig.filename}'
kubectl --kubeconfig "$KC" create namespace otlp-e2e --dry-run=client -o yaml | kubectl --kubeconfig "$KC" apply -f -
kubectl --kubeconfig "$KC" -n otlp-e2e create configmap otlp-e2e-sender \
--from-file=sender.sh='${path.module}/../../otlp_service_sender.sh' --dry-run=client -o yaml | kubectl --kubeconfig "$KC" apply -f -
kubectl --kubeconfig "$KC" apply -f '${path.module}/../../../${var.test_dir}/resources/otlp_sender.yaml'
kubectl --kubeconfig "$KC" -n otlp-e2e rollout status deployment/otlp-e2e-app --timeout=5m
EOT
interpreter = ["/bin/bash", "-c"]
}
depends_on = [null_resource.ci_helm_chart]
}

#####################################################################
# keda/karpenter stub emitters + sample app (applied via kubectl).
#####################################################################
Expand Down
40 changes: 34 additions & 6 deletions terraform/eks/daemon/otel-filters/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ locals {
# Namespaces of the stand-in inference servers: one kept and one dropped by
# the namespaces filter.
inference_namespaces = ["filters-keep-a", "filters-keep-secret-b"]

# Run by the OTLP sender container of each test workload.
sender_script = "${path.module}/../../../otlp_service_sender.sh"
}

resource "aws_eks_cluster" "this" {
Expand Down Expand Up @@ -173,6 +176,9 @@ resource "helm_release" "aws_observability" {
# the namespaces filter alone decides whether they arrive. Both solutions
# are on by default. The values set them so that the test does not depend
# on the default.
# metricNames.include names otlp_e2e_requests, one of the two counters the
# OTLP sender container sends, so the other one, otlp_e2e_unlisted, and the
# span metrics computed from the sender's spans are dropped.
# filters-e2e.example.com/keep* is the only include entry that covers the pod
# label filters-e2e.example.com/keep-x, so it tests a dotted prefix include.
# The pod labels app-extra and filters-e2e-keep-secret-x and the metric
Expand All @@ -192,7 +198,7 @@ resource "helm_release" "aws_observability" {
include: ["filters-keep-*", "filters-exact"]
exclude: ["filters-keep-secret*"]
metricNames:
include: ["container_*", "kube_pod_info", "k8s.pod.*", "node_load1", "vllm:num_requests_running", "revision_request_count"]
include: ["container_*", "kube_pod_info", "k8s.pod.*", "node_load1", "vllm:num_requests_running", "revision_request_count", "otlp_e2e_requests"]
exclude: ["container_network_*", "container_memory_usage_bytes"]
podLabels:
include: ["app", "filters-e2e-keep*", "filters-e2e.example.com/keep*", "pod-template-hash"]
Expand Down Expand Up @@ -255,18 +261,21 @@ resource "null_resource" "restart_pods" {
}

# --- Test workloads: one logging deployment per namespace, one pod per node ---
# Each pod also runs an OTLP sender container, which posts OTLP through the
# cloudwatch-agent Service (terraform/otlp_service_sender.sh), so the OTLP
# endpoint's telemetry from each namespace carries the same pod labels as the
# Container Insights telemetry.

resource "null_resource" "workloads" {
for_each = toset(local.namespaces)
depends_on = [aws_eks_node_group.this, null_resource.kubectl]
provisioner "local-exec" {
command = <<-EOT
set -e
kubectl create namespace ${each.key} --dry-run=client -o yaml | kubectl apply -f -
kubectl -n ${each.key} create configmap otlp-e2e-sender \
--from-file=sender.sh=${local.sender_script} --dry-run=client -o yaml | kubectl apply -f -
cat <<'EOF' | kubectl apply -f -
apiVersion: v1
kind: Namespace
metadata:
name: ${each.key}
---
apiVersion: apps/v1
kind: Deployment
metadata:
Expand Down Expand Up @@ -304,6 +313,25 @@ resource "null_resource" "workloads" {
requests:
cpu: 10m
memory: 16Mi
- name: otlp-sender
image: curlimages/curl:8.8.0
command: ["sh", "/sender/sender.sh"]
env:
- name: OTLP_CASE
value: filters
- name: OTLP_METRICS
value: otlp_e2e_requests otlp_e2e_unlisted
resources:
requests:
cpu: 10m
memory: 16Mi
volumeMounts:
- name: sender
mountPath: /sender
volumes:
- name: sender
configMap:
name: otlp-e2e-sender
EOF
kubectl -n ${each.key} rollout status deployment/filters-e2e --timeout=5m
EOT
Expand Down
Loading
Loading