feat: update charts for enabling monitoring

charts/modelharness/templates/azure/podmonitor.yaml

@@ -0,0 +1,56 @@
+{{- if and (eq .Values.cloudprovider "azure") .Values.azure.podMonitor.enabled (.Capabilities.APIVersions.Has "azmonitoring.coreos.com/v1/PodMonitor") }}
+{{/*
+Per-namespace Azure Monitor managed-Prometheus PodMonitor that scrapes
+every pod in THIS workload namespace. The `azmonitoring.coreos.com/v1`
+PodMonitor CR is reconciled by the AKS Azure Monitor metrics add-on
+(distinct from the Prometheus Operator's monitoring.coreos.com
+PodMonitor), so it renders only under cloudprovider=azure.
+
+The `.Capabilities.APIVersions.Has` guard skips this manifest when the
+cluster does not serve the azmonitoring.coreos.com/v1 PodMonitor CRD
+(i.e. the AKS Azure Monitor managed Prometheus add-on is not installed),
+so `helm install/upgrade` against such a cluster succeeds instead of
+failing on an unknown kind. NOTE: capability discovery needs a live
+cluster — a bare `helm template` (no --validate / no API server) reports
+only Helm's built-in API list, so this block renders nothing there even
+when the CRD is actually present on the target cluster.
+
+selector: {} matches every pod in the namespace. The podTargetLabels are
+lifted onto the emitted series so downstream queries can group by
+inferencepool / gateway / shadow-pod / workspace identity. The three
+podMetricsEndpoints cover the metric surfaces a model namespace exposes:
+EPP (`metrics` port, /metrics), the Istio gateway sidecar
+(`metrics` port, /stats/prometheus), and vLLM inference (`http`
+port, /metrics). A port named on a pod that does not expose it is simply
+skipped by the scraper.
+*/}}
+apiVersion: azmonitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: {{ .Values.azure.podMonitor.name }}
+  namespace: {{ include "modelharness.namespace" . }}
+  labels:
+    {{- include "modelharness.labels" . | nindent 4 }}
+spec:
+  labelLimit: 63
+  labelNameLengthLimit: 511
+  labelValueLengthLimit: 1023
+  selector: {}
+  podTargetLabels:
+    - inferencepool
+    - gateway.networking.k8s.io/gateway-name
+    - kaito.sh/shadow-pod-inferenceset
+    - kaito.sh/shadow-pod-for
+    - inferenceset.kaito.sh/created-by
+    - kaito.sh/workspace
+  podMetricsEndpoints:
+    - interval: {{ .Values.azure.podMonitor.interval }}  # epp
+      path: /metrics
+      port: metrics
+    - interval: {{ .Values.azure.podMonitor.interval }}  # istio gateway
+      path: /stats/prometheus
+      port: metrics
+    - interval: {{ .Values.azure.podMonitor.interval }}  # vllm inference
+      path: /metrics
+      port: http
+{{- end }}

charts/modelharness/values.schema.json

@@ -368,6 +368,30 @@
               }
             }
           }
+        },
+        "podMonitor": {
+          "type": "object",
+          "additionalProperties": false,
+          "description": "AKS Azure Monitor managed-Prometheus PodMonitor (azmonitoring.coreos.com/v1) scraping every pod in this workload namespace. Rendered only when cloudprovider=azure. See templates/azure/podmonitor.yaml.",
+          "properties": {
+            "enabled": {
+              "type": "boolean",
+              "description": "Toggles rendering of the PodMonitor."
+            },
+            "name": {
+              "type": "string",
+              "minLength": 1,
+              "maxLength": 63,
+              "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$",
+              "description": "PodMonitor resource name."
+            },
+            "interval": {
+              "type": "string",
+              "minLength": 1,
+              "description": "Scrape interval applied to every pod metrics endpoint (e.g. '30s')."
+            }
+          },
+          "required": ["enabled"]
         }
       },
       "required": ["defaultDomain"]

charts/modelharness/values.yaml

@@ -428,3 +428,19 @@ azure:
     jwtEndpoints:
       issuer: ""
       jwksUri: ""
+
+  # podMonitor wires THIS namespace's pods into AKS Azure Monitor
+  # managed Prometheus via an `azmonitoring.coreos.com/v1` PodMonitor
+  # (reconciled by the Azure Monitor metrics add-on, distinct from the
+  # Prometheus Operator's monitoring.coreos.com PodMonitor). Rendered
+  # only when cloudprovider=azure. `selector: {}` scrapes every pod in
+  # the namespace across the EPP, Istio gateway, and vLLM metric
+  # surfaces; see templates/azure/podmonitor.yaml.
+  podMonitor:
+    # enabled toggles rendering of the PodMonitor.
+    enabled: true
+    # name is the PodMonitor resource name.
+    name: all-model-pods
+    # interval is the scrape interval applied to every pod metrics
+    # endpoint.
+    interval: 30s

charts/productionstack/templates/azure/ama-metrics-settings-configmap.yaml

@@ -0,0 +1,30 @@
+{{- if and (eq .Values.cloudprovider "azure") .Values.azure.amaMetricsSettings.enabled }}
+# ama-metrics-settings-configmap: configures the AKS Azure Monitor
+# managed-Prometheus metrics add-on (ama-metrics). The add-on reads this
+# ConfigMap by its well-known name in kube-system; the name and namespace
+# are therefore PINNED, not tunable. Here we enable the built-in
+# `dcgmexporter` default scrape target so the add-on collects raw NVIDIA
+# DCGM GPU metrics cluster-wide. Rendered once per cluster by the
+# productionstack umbrella (cluster-level shared state), alongside the
+# DefaultDomainCertificate.
+#
+# NOTE: this is cluster-global add-on configuration. If the cluster
+# already has an `ama-metrics-settings-configmap` managed outside this
+# chart, enabling this will conflict — reconcile the two before opting in.
+# Harmless when the Azure Monitor managed Prometheus add-on itself is not
+# installed: the ConfigMap simply sits unread until the add-on is enabled.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ama-metrics-settings-configmap
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    kaito.sh/owned-by: productionstack
+data:
+  schema-version: v1
+  config-version: ver1
+  default-scrape-settings-enabled: |-
+    dcgmexporter = true
+{{- end }}

charts/productionstack/values.yaml

@@ -318,6 +318,22 @@ azure:
     # charts/modelharness must reference the same name/namespace via
     # `azure.defaultDomain.secretRef`.
     secretName: "cert"
+
+  # amaMetricsSettings renders the cluster-wide
+  # `ama-metrics-settings-configmap` (kube-system) consumed by the AKS
+  # Azure Monitor managed-Prometheus metrics add-on. It enables the
+  # built-in `dcgmexporter` default scrape target so the add-on collects
+  # raw NVIDIA DCGM GPU metrics cluster-wide. The ConfigMap name and
+  # namespace are fixed by the add-on and PINNED in the template.
+  #
+  # Enabled by default (Azure only): the ConfigMap is harmless when the
+  # add-on is not installed — it simply sits unread until the add-on is
+  # enabled. Disable only if an `ama-metrics-settings-configmap` is
+  # already managed on the cluster by the platform or another chart, to
+  # avoid an ownership conflict.
+  # See templates/azure/ama-metrics-settings-configmap.yaml.
+  amaMetricsSettings:
+    enabled: true
 #
 # Control-plane status reporter — the single, leader-elected, read-only
 # producer of the end-to-end error-handling reason catalogue as Kubernetes