From 44dcbcfb606d04a1080ab25c32d243e48bcfd71f Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 17:13:06 +0000 Subject: [PATCH 1/3] Show EKS Prometheus CPU as request percentage Co-Authored-By: brandon --- compute/eks_service/rvn-eks-cron-definition.yml | 4 ++-- compute/eks_service/rvn-eks-web-definition.yml | 6 +++--- compute/eks_service/rvn-eks-worker-definition.yml | 6 +++--- partials/ui/eks-workload-pod-metrics.yml | 13 +++++++++---- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/compute/eks_service/rvn-eks-cron-definition.yml b/compute/eks_service/rvn-eks-cron-definition.yml index ffc52a9c..3269a783 100644 --- a/compute/eks_service/rvn-eks-cron-definition.yml +++ b/compute/eks_service/rvn-eks-cron-definition.yml @@ -6,7 +6,7 @@ definition: description: Scheduled job on EKS for running a container on a cron schedule, with no Kubernetes manifests or Helm charts to write. release: version: 0.6.0 - description: "Add watch paths, so pipeline runs skip the build and deploy when no watched file changed." + description: Show CPU usage as a percentage of the pods' CPU request. module: inputs: - id: section_cluster @@ -386,7 +386,7 @@ module: Logs and pod charts cover every run of the job, because both match on the pods the CronJob creates rather than on any one Job object. Charts are the same five golden signals whichever store answers: CPU, memory working set, container restarts, network in, and network out. Two things differ on the CloudWatch link of the chain: - - **Container Insights reports per pod, Prometheus reports across pods.** A CloudWatch chart is an average over the run's pods; the Prometheus charts sum them. Same shape, different magnitude. + - **CPU uses different units by provider.** Prometheus shows usage as a percentage of the running pods' CPU requests, matching the autoscaler's CPU target; CloudWatch reports cores. Other Prometheus charts sum across pods; CloudWatch averages across them. - **It needs enhanced observability**, which the add-ons module turns on with CloudWatch by default. Without it, CPU and memory come back empty. **Runs succeeded** and **Runs failed** are Prometheus-only. Container Insights reports on running pods and never on Job completion, so it has no metric that says a run finished, let alone whether it worked. With CloudWatch as the only metrics provider those two charts are empty while the pod charts still work - so if run outcomes matter to you, keep a Prometheus provider selected. diff --git a/compute/eks_service/rvn-eks-web-definition.yml b/compute/eks_service/rvn-eks-web-definition.yml index f798507f..92c8a468 100644 --- a/compute/eks_service/rvn-eks-web-definition.yml +++ b/compute/eks_service/rvn-eks-web-definition.yml @@ -4,8 +4,8 @@ definition: name: EKS Web Service description: Web server on EKS for running any HTTP application on your Kubernetes cluster behind a shared load balancer, with no Kubernetes manifests or Helm charts to write. release: - version: 1.12.1 - description: Show the Spot burst pods' lines in Application logs alongside the On-Demand pods'. + version: 1.12.2 + description: Show CPU usage as a percentage of the pods' CPU request. module: inputs: - id: section_cluster @@ -1000,7 +1000,7 @@ module: Charts are the same five golden signals whichever store answers: CPU, memory working set, container restarts, network in, and network out, plus replicas available and desired. Two things differ on the CloudWatch link of the chain, and are worth knowing before you read a number off it: - - **Container Insights reports per pod, Prometheus reports across pods.** A CloudWatch chart is an average over the workload's pods; the Prometheus charts sum them. Same shape, different magnitude. + - **CPU uses different units by provider.** Prometheus shows usage as a percentage of the running pods' CPU requests, matching the autoscaler's CPU target; CloudWatch reports cores. Other Prometheus charts sum across pods; CloudWatch averages across them. - **It needs enhanced observability**, which the add-ons module turns on with CloudWatch by default. Without it, CPU and memory come back empty. Replicas available falls back to Container Insights' running-pod count for this module's Kubernetes Service. Replicas desired has no CloudWatch equivalent, so with CloudWatch as the only metrics provider that one chart is empty. diff --git a/compute/eks_service/rvn-eks-worker-definition.yml b/compute/eks_service/rvn-eks-worker-definition.yml index 9c0d9fb0..1a8b2337 100644 --- a/compute/eks_service/rvn-eks-worker-definition.yml +++ b/compute/eks_service/rvn-eks-worker-definition.yml @@ -4,8 +4,8 @@ definition: name: EKS Worker description: Background worker on EKS for running a long-lived private process with no exposed port, and no Kubernetes manifests or Helm charts to write. release: - version: 0.14.1 - description: Show the Spot burst pods' lines in Application logs alongside the On-Demand pods'. + version: 0.14.2 + description: Show CPU usage as a percentage of the pods' CPU request. module: inputs: - id: section_cluster @@ -378,7 +378,7 @@ module: Charts are the same five golden signals whichever store answers: CPU, memory working set, container restarts, network in, and network out. Two things differ on the CloudWatch link of the chain, and are worth knowing before you read a number off it: - - **Container Insights reports per pod, Prometheus reports across pods.** A CloudWatch chart is an average over the worker's pods; the Prometheus charts sum them. Same shape, different magnitude. + - **CPU uses different units by provider.** Prometheus shows usage as a percentage of the running pods' CPU requests, matching the autoscaler's CPU target; CloudWatch reports cores. Other Prometheus charts sum across pods; CloudWatch averages across them. - **It needs enhanced observability**, which the add-ons module turns on with CloudWatch by default. Without it, CPU and memory come back empty. Replicas available and desired have no CloudWatch equivalent. Container Insights' nearest metric counts pods behind a Kubernetes Service, and a worker renders no Service, so with CloudWatch as the only metrics provider those two charts are empty while the pod charts still work. diff --git a/partials/ui/eks-workload-pod-metrics.yml b/partials/ui/eks-workload-pod-metrics.yml index 97090744..05f68992 100644 --- a/partials/ui/eks-workload-pod-metrics.yml +++ b/partials/ui/eks-workload-pod-metrics.yml @@ -49,6 +49,11 @@ # `pod_cpu_utilization` / `pod_memory_utilization` because those two are # percentages of the whole node, which would make the CloudWatch link in # the chain plot a different quantity than the Prometheus links. +# +# Prometheus CPU usage is a percentage of the CPU requests for running pods, +# the same basis as the autoscaler's CPU target. The `container!=""` selector +# excludes cAdvisor's pod-level cgroup series to avoid double-counting; +# CloudWatch continues to report CPU in cores. - id: cpu name: CPU usage type: line @@ -58,8 +63,8 @@ aws_account_id: << module.input.aws_account_id >> region: << module.input.amp_workspace_region >> workspace_id: << module.input.amp_workspace_id >> - query: 'sum(rate(container_cpu_usage_seconds_total{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+"}[$__interval]))' - unit: cores + query: '100 * sum(rate(container_cpu_usage_seconds_total{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",container!="",container!="POD"}[$__interval])) / sum(kube_pod_container_resource_requests{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",resource="cpu"} * on(namespace, pod) group_left() (kube_pod_status_phase{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",phase="Running"} == 1))' + unit: Percent - id: cpu name: CPU usage type: line @@ -68,8 +73,8 @@ type: eks_prometheus cluster_arn: << module.input.cluster_arn >> endpoint: << module.input.prometheus_endpoint >> - query: 'sum(rate(container_cpu_usage_seconds_total{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+"}[$__interval]))' - unit: cores + query: '100 * sum(rate(container_cpu_usage_seconds_total{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",container!="",container!="POD"}[$__interval])) / sum(kube_pod_container_resource_requests{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",resource="cpu"} * on(namespace, pod) group_left() (kube_pod_status_phase{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",phase="Running"} == 1))' + unit: Percent - id: cpu name: 'CPU usage<< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? " (On-Demand)" : "" >>' type: line From 8278b682112f01a6254757af1b55973293ad51dc Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 17:14:01 +0000 Subject: [PATCH 2/3] Keep rvn-eks-cron release notes unchanged (unpublished) Co-Authored-By: brandon --- compute/eks_service/rvn-eks-cron-definition.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compute/eks_service/rvn-eks-cron-definition.yml b/compute/eks_service/rvn-eks-cron-definition.yml index 3269a783..f2ee5d0f 100644 --- a/compute/eks_service/rvn-eks-cron-definition.yml +++ b/compute/eks_service/rvn-eks-cron-definition.yml @@ -6,7 +6,7 @@ definition: description: Scheduled job on EKS for running a container on a cron schedule, with no Kubernetes manifests or Helm charts to write. release: version: 0.6.0 - description: Show CPU usage as a percentage of the pods' CPU request. + description: "Add watch paths, so pipeline runs skip the build and deploy when no watched file changed." module: inputs: - id: section_cluster From 67e15d1764363adbf3abd871e8a56c48a7d026ab Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 17:18:59 +0000 Subject: [PATCH 3/3] Count only running pods in CPU usage and skip zero CPU requests Co-Authored-By: brandon --- partials/ui/eks-workload-pod-metrics.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/partials/ui/eks-workload-pod-metrics.yml b/partials/ui/eks-workload-pod-metrics.yml index 05f68992..a9deb030 100644 --- a/partials/ui/eks-workload-pod-metrics.yml +++ b/partials/ui/eks-workload-pod-metrics.yml @@ -54,6 +54,8 @@ # the same basis as the autoscaler's CPU target. The `container!=""` selector # excludes cAdvisor's pod-level cgroup series to avoid double-counting; # CloudWatch continues to report CPU in cores. +# Usage and CPU requests both count only running pods. +# A workload with a zero CPU request shows no CPU data. - id: cpu name: CPU usage type: line @@ -63,7 +65,7 @@ aws_account_id: << module.input.aws_account_id >> region: << module.input.amp_workspace_region >> workspace_id: << module.input.amp_workspace_id >> - query: '100 * sum(rate(container_cpu_usage_seconds_total{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",container!="",container!="POD"}[$__interval])) / sum(kube_pod_container_resource_requests{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",resource="cpu"} * on(namespace, pod) group_left() (kube_pod_status_phase{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",phase="Running"} == 1))' + query: '100 * sum(rate(container_cpu_usage_seconds_total{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",container!="",container!="POD"}[$__interval]) * on(namespace, pod) group_left() (kube_pod_status_phase{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",phase="Running"} == 1)) / (sum(kube_pod_container_resource_requests{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",resource="cpu"} * on(namespace, pod) group_left() (kube_pod_status_phase{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",phase="Running"} == 1)) > 0)' unit: Percent - id: cpu name: CPU usage @@ -73,7 +75,7 @@ type: eks_prometheus cluster_arn: << module.input.cluster_arn >> endpoint: << module.input.prometheus_endpoint >> - query: '100 * sum(rate(container_cpu_usage_seconds_total{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",container!="",container!="POD"}[$__interval])) / sum(kube_pod_container_resource_requests{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",resource="cpu"} * on(namespace, pod) group_left() (kube_pod_status_phase{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",phase="Running"} == 1))' + query: '100 * sum(rate(container_cpu_usage_seconds_total{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",container!="",container!="POD"}[$__interval]) * on(namespace, pod) group_left() (kube_pod_status_phase{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",phase="Running"} == 1)) / (sum(kube_pod_container_resource_requests{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",resource="cpu"} * on(namespace, pod) group_left() (kube_pod_status_phase{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",phase="Running"} == 1)) > 0)' unit: Percent - id: cpu name: 'CPU usage<< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? " (On-Demand)" : "" >>'