From 0f8b1298cbc3e3328b6b02f9ddd073a649ba70a7 Mon Sep 17 00:00:00 2001 From: Fei-Guo Date: Tue, 6 Oct 2026 10:10:55 -0700 Subject: [PATCH] feat: update charts for enabling monitoring --- .../templates/azure/podmonitor.yaml | 56 +++++++++++++++++++ charts/modelharness/values.schema.json | 24 ++++++++ charts/modelharness/values.yaml | 16 ++++++ .../azure/ama-metrics-settings-configmap.yaml | 30 ++++++++++ charts/productionstack/values.yaml | 16 ++++++ 5 files changed, 142 insertions(+) create mode 100644 charts/modelharness/templates/azure/podmonitor.yaml create mode 100644 charts/productionstack/templates/azure/ama-metrics-settings-configmap.yaml diff --git a/charts/modelharness/templates/azure/podmonitor.yaml b/charts/modelharness/templates/azure/podmonitor.yaml new file mode 100644 index 00000000..146932c4 --- /dev/null +++ b/charts/modelharness/templates/azure/podmonitor.yaml @@ -0,0 +1,56 @@ +{{- if and (eq .Values.cloudprovider "azure") .Values.azure.podMonitor.enabled (.Capabilities.APIVersions.Has "azmonitoring.coreos.com/v1/PodMonitor") }} +{{/* +Per-namespace Azure Monitor managed-Prometheus PodMonitor that scrapes +every pod in THIS workload namespace. The `azmonitoring.coreos.com/v1` +PodMonitor CR is reconciled by the AKS Azure Monitor metrics add-on +(distinct from the Prometheus Operator's monitoring.coreos.com +PodMonitor), so it renders only under cloudprovider=azure. + +The `.Capabilities.APIVersions.Has` guard skips this manifest when the +cluster does not serve the azmonitoring.coreos.com/v1 PodMonitor CRD +(i.e. the AKS Azure Monitor managed Prometheus add-on is not installed), +so `helm install/upgrade` against such a cluster succeeds instead of +failing on an unknown kind. NOTE: capability discovery needs a live +cluster — a bare `helm template` (no --validate / no API server) reports +only Helm's built-in API list, so this block renders nothing there even +when the CRD is actually present on the target cluster. + +selector: {} matches every pod in the namespace. The podTargetLabels are +lifted onto the emitted series so downstream queries can group by +inferencepool / gateway / shadow-pod / workspace identity. The three +podMetricsEndpoints cover the metric surfaces a model namespace exposes: +EPP (`metrics` port, /metrics), the Istio gateway sidecar +(`metrics` port, /stats/prometheus), and vLLM inference (`http` +port, /metrics). A port named on a pod that does not expose it is simply +skipped by the scraper. +*/}} +apiVersion: azmonitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: {{ .Values.azure.podMonitor.name }} + namespace: {{ include "modelharness.namespace" . }} + labels: + {{- include "modelharness.labels" . | nindent 4 }} +spec: + labelLimit: 63 + labelNameLengthLimit: 511 + labelValueLengthLimit: 1023 + selector: {} + podTargetLabels: + - inferencepool + - gateway.networking.k8s.io/gateway-name + - kaito.sh/shadow-pod-inferenceset + - kaito.sh/shadow-pod-for + - inferenceset.kaito.sh/created-by + - kaito.sh/workspace + podMetricsEndpoints: + - interval: {{ .Values.azure.podMonitor.interval }} # epp + path: /metrics + port: metrics + - interval: {{ .Values.azure.podMonitor.interval }} # istio gateway + path: /stats/prometheus + port: metrics + - interval: {{ .Values.azure.podMonitor.interval }} # vllm inference + path: /metrics + port: http +{{- end }} diff --git a/charts/modelharness/values.schema.json b/charts/modelharness/values.schema.json index 0c755318..2917ed08 100644 --- a/charts/modelharness/values.schema.json +++ b/charts/modelharness/values.schema.json @@ -368,6 +368,30 @@ } } } + }, + "podMonitor": { + "type": "object", + "additionalProperties": false, + "description": "AKS Azure Monitor managed-Prometheus PodMonitor (azmonitoring.coreos.com/v1) scraping every pod in this workload namespace. Rendered only when cloudprovider=azure. See templates/azure/podmonitor.yaml.", + "properties": { + "enabled": { + "type": "boolean", + "description": "Toggles rendering of the PodMonitor." + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 63, + "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", + "description": "PodMonitor resource name." + }, + "interval": { + "type": "string", + "minLength": 1, + "description": "Scrape interval applied to every pod metrics endpoint (e.g. '30s')." + } + }, + "required": ["enabled"] } }, "required": ["defaultDomain"] diff --git a/charts/modelharness/values.yaml b/charts/modelharness/values.yaml index e5b9053a..df5a323b 100644 --- a/charts/modelharness/values.yaml +++ b/charts/modelharness/values.yaml @@ -428,3 +428,19 @@ azure: jwtEndpoints: issuer: "" jwksUri: "" + + # podMonitor wires THIS namespace's pods into AKS Azure Monitor + # managed Prometheus via an `azmonitoring.coreos.com/v1` PodMonitor + # (reconciled by the Azure Monitor metrics add-on, distinct from the + # Prometheus Operator's monitoring.coreos.com PodMonitor). Rendered + # only when cloudprovider=azure. `selector: {}` scrapes every pod in + # the namespace across the EPP, Istio gateway, and vLLM metric + # surfaces; see templates/azure/podmonitor.yaml. + podMonitor: + # enabled toggles rendering of the PodMonitor. + enabled: true + # name is the PodMonitor resource name. + name: all-model-pods + # interval is the scrape interval applied to every pod metrics + # endpoint. + interval: 30s diff --git a/charts/productionstack/templates/azure/ama-metrics-settings-configmap.yaml b/charts/productionstack/templates/azure/ama-metrics-settings-configmap.yaml new file mode 100644 index 00000000..20a25423 --- /dev/null +++ b/charts/productionstack/templates/azure/ama-metrics-settings-configmap.yaml @@ -0,0 +1,30 @@ +{{- if and (eq .Values.cloudprovider "azure") .Values.azure.amaMetricsSettings.enabled }} +# ama-metrics-settings-configmap: configures the AKS Azure Monitor +# managed-Prometheus metrics add-on (ama-metrics). The add-on reads this +# ConfigMap by its well-known name in kube-system; the name and namespace +# are therefore PINNED, not tunable. Here we enable the built-in +# `dcgmexporter` default scrape target so the add-on collects raw NVIDIA +# DCGM GPU metrics cluster-wide. Rendered once per cluster by the +# productionstack umbrella (cluster-level shared state), alongside the +# DefaultDomainCertificate. +# +# NOTE: this is cluster-global add-on configuration. If the cluster +# already has an `ama-metrics-settings-configmap` managed outside this +# chart, enabling this will conflict — reconcile the two before opting in. +# Harmless when the Azure Monitor managed Prometheus add-on itself is not +# installed: the ConfigMap simply sits unread until the add-on is enabled. +apiVersion: v1 +kind: ConfigMap +metadata: + name: ama-metrics-settings-configmap + namespace: kube-system + labels: + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + kaito.sh/owned-by: productionstack +data: + schema-version: v1 + config-version: ver1 + default-scrape-settings-enabled: |- + dcgmexporter = true +{{- end }} diff --git a/charts/productionstack/values.yaml b/charts/productionstack/values.yaml index b6e32800..dfaf6e31 100644 --- a/charts/productionstack/values.yaml +++ b/charts/productionstack/values.yaml @@ -318,6 +318,22 @@ azure: # charts/modelharness must reference the same name/namespace via # `azure.defaultDomain.secretRef`. secretName: "cert" + + # amaMetricsSettings renders the cluster-wide + # `ama-metrics-settings-configmap` (kube-system) consumed by the AKS + # Azure Monitor managed-Prometheus metrics add-on. It enables the + # built-in `dcgmexporter` default scrape target so the add-on collects + # raw NVIDIA DCGM GPU metrics cluster-wide. The ConfigMap name and + # namespace are fixed by the add-on and PINNED in the template. + # + # Enabled by default (Azure only): the ConfigMap is harmless when the + # add-on is not installed — it simply sits unread until the add-on is + # enabled. Disable only if an `ama-metrics-settings-configmap` is + # already managed on the cluster by the platform or another chart, to + # avoid an ownership conflict. + # See templates/azure/ama-metrics-settings-configmap.yaml. + amaMetricsSettings: + enabled: true # # Control-plane status reporter — the single, leader-elected, read-only # producer of the end-to-end error-handling reason catalogue as Kubernetes