diff --git a/compute/eks_service/rvn-eks-cron-definition.yml b/compute/eks_service/rvn-eks-cron-definition.yml index ffc52a9c..f2ee5d0f 100644 --- a/compute/eks_service/rvn-eks-cron-definition.yml +++ b/compute/eks_service/rvn-eks-cron-definition.yml @@ -386,7 +386,7 @@ module: Logs and pod charts cover every run of the job, because both match on the pods the CronJob creates rather than on any one Job object. Charts are the same five golden signals whichever store answers: CPU, memory working set, container restarts, network in, and network out. Two things differ on the CloudWatch link of the chain: - - **Container Insights reports per pod, Prometheus reports across pods.** A CloudWatch chart is an average over the run's pods; the Prometheus charts sum them. Same shape, different magnitude. + - **CPU uses different units by provider.** Prometheus shows usage as a percentage of the running pods' CPU requests, matching the autoscaler's CPU target; CloudWatch reports cores. Other Prometheus charts sum across pods; CloudWatch averages across them. - **It needs enhanced observability**, which the add-ons module turns on with CloudWatch by default. Without it, CPU and memory come back empty. **Runs succeeded** and **Runs failed** are Prometheus-only. Container Insights reports on running pods and never on Job completion, so it has no metric that says a run finished, let alone whether it worked. With CloudWatch as the only metrics provider those two charts are empty while the pod charts still work - so if run outcomes matter to you, keep a Prometheus provider selected. diff --git a/compute/eks_service/rvn-eks-web-definition.yml b/compute/eks_service/rvn-eks-web-definition.yml index f798507f..92c8a468 100644 --- a/compute/eks_service/rvn-eks-web-definition.yml +++ b/compute/eks_service/rvn-eks-web-definition.yml @@ -4,8 +4,8 @@ definition: name: EKS Web Service description: Web server on EKS for running any HTTP application on your Kubernetes cluster behind a shared load balancer, with no Kubernetes manifests or Helm charts to write. release: - version: 1.12.1 - description: Show the Spot burst pods' lines in Application logs alongside the On-Demand pods'. + version: 1.12.2 + description: Show CPU usage as a percentage of the pods' CPU request. module: inputs: - id: section_cluster @@ -1000,7 +1000,7 @@ module: Charts are the same five golden signals whichever store answers: CPU, memory working set, container restarts, network in, and network out, plus replicas available and desired. Two things differ on the CloudWatch link of the chain, and are worth knowing before you read a number off it: - - **Container Insights reports per pod, Prometheus reports across pods.** A CloudWatch chart is an average over the workload's pods; the Prometheus charts sum them. Same shape, different magnitude. + - **CPU uses different units by provider.** Prometheus shows usage as a percentage of the running pods' CPU requests, matching the autoscaler's CPU target; CloudWatch reports cores. Other Prometheus charts sum across pods; CloudWatch averages across them. - **It needs enhanced observability**, which the add-ons module turns on with CloudWatch by default. Without it, CPU and memory come back empty. Replicas available falls back to Container Insights' running-pod count for this module's Kubernetes Service. Replicas desired has no CloudWatch equivalent, so with CloudWatch as the only metrics provider that one chart is empty. diff --git a/compute/eks_service/rvn-eks-worker-definition.yml b/compute/eks_service/rvn-eks-worker-definition.yml index 9c0d9fb0..1a8b2337 100644 --- a/compute/eks_service/rvn-eks-worker-definition.yml +++ b/compute/eks_service/rvn-eks-worker-definition.yml @@ -4,8 +4,8 @@ definition: name: EKS Worker description: Background worker on EKS for running a long-lived private process with no exposed port, and no Kubernetes manifests or Helm charts to write. release: - version: 0.14.1 - description: Show the Spot burst pods' lines in Application logs alongside the On-Demand pods'. + version: 0.14.2 + description: Show CPU usage as a percentage of the pods' CPU request. module: inputs: - id: section_cluster @@ -378,7 +378,7 @@ module: Charts are the same five golden signals whichever store answers: CPU, memory working set, container restarts, network in, and network out. Two things differ on the CloudWatch link of the chain, and are worth knowing before you read a number off it: - - **Container Insights reports per pod, Prometheus reports across pods.** A CloudWatch chart is an average over the worker's pods; the Prometheus charts sum them. Same shape, different magnitude. + - **CPU uses different units by provider.** Prometheus shows usage as a percentage of the running pods' CPU requests, matching the autoscaler's CPU target; CloudWatch reports cores. Other Prometheus charts sum across pods; CloudWatch averages across them. - **It needs enhanced observability**, which the add-ons module turns on with CloudWatch by default. Without it, CPU and memory come back empty. Replicas available and desired have no CloudWatch equivalent. Container Insights' nearest metric counts pods behind a Kubernetes Service, and a worker renders no Service, so with CloudWatch as the only metrics provider those two charts are empty while the pod charts still work. diff --git a/partials/ui/eks-workload-pod-metrics.yml b/partials/ui/eks-workload-pod-metrics.yml index 97090744..a9deb030 100644 --- a/partials/ui/eks-workload-pod-metrics.yml +++ b/partials/ui/eks-workload-pod-metrics.yml @@ -49,6 +49,13 @@ # `pod_cpu_utilization` / `pod_memory_utilization` because those two are # percentages of the whole node, which would make the CloudWatch link in # the chain plot a different quantity than the Prometheus links. +# +# Prometheus CPU usage is a percentage of the CPU requests for running pods, +# the same basis as the autoscaler's CPU target. The `container!=""` selector +# excludes cAdvisor's pod-level cgroup series to avoid double-counting; +# CloudWatch continues to report CPU in cores. +# Usage and CPU requests both count only running pods. +# A workload with a zero CPU request shows no CPU data. - id: cpu name: CPU usage type: line @@ -58,8 +65,8 @@ aws_account_id: << module.input.aws_account_id >> region: << module.input.amp_workspace_region >> workspace_id: << module.input.amp_workspace_id >> - query: 'sum(rate(container_cpu_usage_seconds_total{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+"}[$__interval]))' - unit: cores + query: '100 * sum(rate(container_cpu_usage_seconds_total{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",container!="",container!="POD"}[$__interval]) * on(namespace, pod) group_left() (kube_pod_status_phase{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",phase="Running"} == 1)) / (sum(kube_pod_container_resource_requests{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",resource="cpu"} * on(namespace, pod) group_left() (kube_pod_status_phase{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",phase="Running"} == 1)) > 0)' + unit: Percent - id: cpu name: CPU usage type: line @@ -68,8 +75,8 @@ type: eks_prometheus cluster_arn: << module.input.cluster_arn >> endpoint: << module.input.prometheus_endpoint >> - query: 'sum(rate(container_cpu_usage_seconds_total{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+"}[$__interval]))' - unit: cores + query: '100 * sum(rate(container_cpu_usage_seconds_total{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",container!="",container!="POD"}[$__interval]) * on(namespace, pod) group_left() (kube_pod_status_phase{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",phase="Running"} == 1)) / (sum(kube_pod_container_resource_requests{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",resource="cpu"} * on(namespace, pod) group_left() (kube_pod_status_phase{namespace="<< module.input.namespace >>",pod=~"<< module.input.name >><< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? "(-spot)?" : "" >>-[a-z0-9]+-[a-z0-9]+",phase="Running"} == 1)) > 0)' + unit: Percent - id: cpu name: 'CPU usage<< module.input.spot_burst_enabled == true && module.input.compute_target == "on_demand" ? " (On-Demand)" : "" >>' type: line