rambohe-ch · Oct 17, 2026 · Sep 22, 2026 · Sep 23, 2026 · Oct 14, 2026 · Oct 14, 2026
@@ -1,6 +1,8 @@
 name: E2E tests (nightly)
 
-# Runs ONLY specs labeled `Nightly` (slow / destructive scaling cases, etc.).
+# Two jobs on real Karpenter/NAP GPU nodes:
+#   - e2e-nightly-tests: Karpenter scale-from-zero specs (eastus).
+#   - e2e-nightly-scaling-tests: KEDA Scaling specs on H100 (australiaeast).
 # Triggers:
 #   - schedule: every day at 00:00 Sydney time
 #   - workflow_dispatch: manual run
@@ -118,3 +120,84 @@ jobs:
         run: make e2e-teardown
         env:
           RESOURCE_GROUP: ${{ env.RESOURCE_GROUP }}
+
+  # KEDA Scaling specs (1<->2, 0<->1) on real H100 nodes provisioned by
+  # Karpenter/NAP. Uses its own cluster in australiaeast (H100 quota) and runs
+  # after the Karpenter job so the shared self-hosted runner is not contended;
+  # `!cancelled()` keeps it independent of that job's result.
+  e2e-nightly-scaling-tests:
+    needs: e2e-nightly-tests
+    if: ${{ !cancelled() }}
+    runs-on: [ "self-hosted", "hostname:kaito-e2e-github-runner" ]
+    environment: e2e-test
+    # Scaling (240m) + cluster setup/teardown.
+    timeout-minutes: 330
+    permissions:
+      contents: read
+    env:
+      # Short names keep "MC_<RG>_<CLUSTER>_australiaeast" within 80 chars.
+      RESOURCE_GROUP: "kaito-ns-rg-${{ github.run_id }}-${{ github.run_attempt }}"
+      CLUSTER_NAME: "kaito-ns-aks-${{ github.run_id }}-${{ github.run_attempt }}"
+      ACR_NAME: "kaitons${{ github.run_id }}a${{ github.run_attempt }}acr"
+      LOCATION: australiaeast
+      E2E_MODEL_STREAMING: 'true'
+      E2E_MODEL_STREAMING_NAMESPACE: e2e-scaling
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event.inputs.branch || github.ref }}
+
+      - name: E2E base setup
+        uses: ./.github/actions/e2e-base-setup
+        with:
+          resource-group: ${{ env.RESOURCE_GROUP }}
+          cluster-name: ${{ env.CLUSTER_NAME }}
+          acr-name: ${{ env.ACR_NAME }}
+          location: ${{ env.LOCATION }}
+          node-count: ${{ env.NODE_COUNT }}
+          node-vm-size: ${{ env.NODE_VM_SIZE }}
+          node-provisioner: karpenter
+          enable-node-mocker: false
+          node-class: azure
+          provider: ${{ github.event.inputs.provider }}
+          istio-version: ${{ github.event.inputs.istio_version }}
+          gateway-api-version: ${{ github.event.inputs.gateway_api_version }}
+          keda-version: ${{ github.event.inputs.keda_version }}
+
+      - name: Run E2E tests (Scaling — real H100)
+        run: make test-e2e-scaling
+        env:
+          E2E_SCALING_INSTANCE_TYPE: Standard_NC40ads_H100_v5
+
+      - name: Generate Scaling coverage report
+        if: always()
+        run: |
+          go run ./hack/e2e/report \
+            --label-filter 'Scaling' \
+            --workflow 'E2E tests (nightly, scaling, real GPU)' \
+            --results ginkgo-report-scaling.json \
+            --output-md e2e-scaling-coverage-report.md \
+            --output-html e2e-scaling-coverage-report.html
+          echo '## Scaling Nightly (real GPU) Coverage' >> "$GITHUB_STEP_SUMMARY"
+          cat e2e-scaling-coverage-report.md >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload Scaling coverage report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-nightly-scaling-coverage-report
+          path: |
+            e2e-scaling-coverage-report.md
+            e2e-scaling-coverage-report.html
+
+      - name: Dump cluster state
+        if: failure()
+        run: make e2e-dump
+
+      - name: Teardown cluster
+        if: always()
+        run: make e2e-teardown
+        env:
+          RESOURCE_GROUP: ${{ env.RESOURCE_GROUP }}
@@ -1,7 +1,8 @@
 name: E2E tests (daily, upstream)
 
 # Runs the regular E2E suite against upstream KEDA and Istio with the GPU node
-# mocker. Slow Nightly and Perf specs remain in their dedicated workflows.
+# mocker, then the Scaling specs serially on the same cluster. Slow Nightly
+# and Perf specs remain in their dedicated workflows.
 # Triggers:
 #   - schedule: every day at 20:00 Sydney time
 #   - workflow_dispatch: manual run
@@ -50,6 +51,8 @@ jobs:
   e2e-upstream-daily-tests:
     runs-on: [ "self-hosted", "hostname:kaito-e2e-github-runner" ]
     environment: e2e-test
+    # main suite (60m) + Scaling (240m) + cluster setup/teardown.
+    timeout-minutes: 420
     permissions:
       contents: read
 
@@ -60,6 +63,7 @@ jobs:
           ref: ${{ github.event.inputs.branch || github.ref }}
 
       - name: E2E base setup
+        id: setup
         uses: ./.github/actions/e2e-base-setup
         with:
           resource-group: ${{ env.RESOURCE_GROUP }}
@@ -77,13 +81,13 @@ jobs:
       - name: Run E2E tests
         run: make test-e2e
         env:
-          E2E_LABEL: "!Nightly && !Perf"
+          E2E_LABEL: "!Nightly && !Perf && !Scaling"
 
       - name: Generate E2E coverage report
         if: always()
         run: |
           go run ./hack/e2e/report \
-            --label-filter '!Nightly && !Perf' \
+            --label-filter '!Nightly && !Perf && !Scaling' \
             --workflow 'E2E tests (daily, upstream)' \
             --results ginkgo-report.json \
             --output-md e2e-coverage-report.md \
@@ -100,6 +104,33 @@ jobs:
             e2e-coverage-report.md
             e2e-coverage-report.html
 
+      # Runs even when the main suite failed so a regression there does not
+      # hide scaling coverage.
+      - name: Run E2E tests (Scaling)
+        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
+        run: make test-e2e-scaling
+
+      - name: Generate Scaling coverage report
+        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
+        run: |
+          go run ./hack/e2e/report \
+            --label-filter 'Scaling' \
+            --workflow 'E2E tests (daily, upstream, scaling)' \
+            --results ginkgo-report-scaling.json \
+            --output-md e2e-scaling-coverage-report.md \
+            --output-html e2e-scaling-coverage-report.html
+          echo '## Upstream Daily Scaling Coverage' >> "$GITHUB_STEP_SUMMARY"
+          cat e2e-scaling-coverage-report.md >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload Scaling coverage report
+        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-upstream-scaling-coverage-report
+          path: |
+            e2e-scaling-coverage-report.md
+            e2e-scaling-coverage-report.html
+
       - name: Dump cluster state
         if: failure()
         run: make e2e-dump

@@ -100,16 +100,16 @@ jobs:
           node-class: ${{ github.event.inputs.node_class }}
 
       - name: Run E2E tests
-        # Slow Nightly and Perf specs run in their dedicated workflows.
+        # Slow Nightly, Perf, and Scaling specs run in their dedicated workflows.
         run: make test-e2e
         env:
-          E2E_LABEL: "!Nightly && !Perf"
+          E2E_LABEL: "!Nightly && !Perf && !Scaling"
 
       - name: Generate E2E coverage report
         if: always()
         run: |
           go run ./hack/e2e/report \
-            --label-filter '!Nightly && !Perf' \
+            --label-filter '!Nightly && !Perf && !Scaling' \
             --workflow 'E2E tests' \
             --results ginkgo-report.json \
             --output-md e2e-coverage-report.md \

@@ -164,6 +164,10 @@ test-e2e: ## Run e2e tests against a live cluster (requires KUBECONFIG).
 test-e2e-karpenter: ## Run the Karpenter nightly scale-from-zero scenarios with safe swedencentral defaults.
 	$(MAKE) test-e2e E2E_LABEL='Karpenter' E2E_PARALLEL=1 E2E_TIMEOUT=180m
 
+.PHONY: test-e2e-scaling
+test-e2e-scaling: ## Run the KEDA Scaling specs (1<->2, 0<->1) serially. On real GPUs set E2E_SCALING_INSTANCE_TYPE (e.g. Standard_NC40ads_H100_v5).
+	$(MAKE) test-e2e E2E_LABEL='Scaling' E2E_PARALLEL=1 E2E_TIMEOUT=240m E2E_JSON_REPORT=ginkgo-report-scaling.json
+
 .PHONY: test-e2e-perf
 test-e2e-perf: ## Run prefix-cache perf specs serially. Override E2E_TRACE_FIXTURE and E2E_PERF_TIMEOUT as needed.
 	$(MAKE) test-e2e E2E_LABEL='Perf' E2E_PARALLEL=1 E2E_TIMEOUT=$(E2E_PERF_TIMEOUT)

@@ -4,7 +4,7 @@ dependencies:
   version: 1.5.0
 - name: keda-kaito-scaler
   repository: ""
-  version: 0.7.0
+  version: 0.7.1
 - name: gpu-node-mocker
   repository: file://../gpu-node-mocker
   version: 0.4.3
@@ -17,5 +17,5 @@ dependencies:
 - name: productionstack-status-reporter
   repository: ""
   version: 0.4.3
-digest: sha256:e8025c947a3dcb700bd7ba42849b2df9c216af1304f46d5ee70ce92ab67913fd
-generated: "2026-10-15T15:01:28.026668+11:00"
+digest: sha256:094d638a9e4fc5b7139701282c617b3aff9edba3851da2f3067ca11d5bb06727
+generated: "2026-10-16T15:14:52.038649-07:00"
@@ -43,7 +43,7 @@ dependencies:
       - gateway
       - routing
   - name: keda-kaito-scaler
-    version: 0.7.0
+    version: 0.7.1
     tags:
       - autoscaling
   # In-tree gpu-node-mocker chart. Sourced via a `file://` repository

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.7.0
+version: 0.7.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "v0.7.0"
+appVersion: "v0.7.1"
@@ -126,7 +126,7 @@ Set `namespaceOverride: ""` to fall back to the standard Helm
 | `log.level`                | `3`                                      | klog verbosity (`--v`). Bump to `5` for debug logging.                                                        |
 | `image.registry`           | `mcr.microsoft.com`                      | Container image registry.                                                                                    |
 | `image.repository`         | `oss/v2/kaito/keda-kaito-scaler`         | Container image repository.                                                                                  |
-| `image.tag`                | `v0.7.0`                                 | Container image tag.                                                                                         |
+| `image.tag`                | `v0.7.1`                                 | Container image tag.                                                                                         |
 | `image.pullSecrets`        | `[]`                                     | `imagePullSecrets` for the pod spec.                                                                         |
 | `ports.grpc`               | `10450`                                  | gRPC `ExternalScaler` port exposed via the Service.                                                          |
 | `ports.metrics`            | `10451`                                  | Prometheus metrics port (`--metrics-port`). Not currently fronted by a Service.                              |

@@ -21,7 +21,7 @@ namespaceOverride: ""
 image:
     registry: mcr.microsoft.com
     repository: oss/v2/kaito/keda-kaito-scaler
-    tag: v0.7.0
+    tag: v0.7.1
     pullSecrets: []
 
 ports:

@@ -127,6 +127,94 @@ prepare_managed_kaito_crds() {
   KAITO_HELM_ARGS+=(--skip-crds)
 }
 
+retry_azure_identity_not_found() {
+  local attempt output exit_code
+  for attempt in 1 2 3 4 5 6; do
+    if output=$(az "$@" --only-show-errors 2>&1); then
+      printf '%s\n' "${output}"
+      return 0
+    else
+      exit_code=$?
+    fi
+    printf '%s\n' "${output}" >&2
+    if ! grep -Eq '^ERROR: \((Resource)?NotFound\)' <<< "${output}" || [[ "${attempt}" == 6 ]]; then
+      return "${exit_code}"
+    fi
+    echo "Identity operation not yet available (attempt ${attempt}/6); retrying in $((attempt * 10))s..." >&2
+    sleep "$((attempt * 10))"
+  done
+}
+
+prepare_model_streaming() {
+  local namespace="${E2E_MODEL_STREAMING_NAMESPACE:?Set E2E_MODEL_STREAMING_NAMESPACE}"
+  local storage_class="kaito-model-mirror"
+  local service_account="kaito-model-streamer"
+  local identity_name="kaito-model-streamer"
+  local storage_account identity_json identity_client_id identity_principal_id storage_scope oidc_issuer control_plane_id
+  local subscription="${AZURE_SUBSCRIPTION_ID:?Set AZURE_SUBSCRIPTION_ID}"
+  storage_account="kaitomm$(printf '%s' "${CLUSTER_NAME}" | openssl dgst -sha256 | awk '{print substr($NF,1,16)}')"
+
+  echo "=== Preparing ModelMirror and ModelStreaming (${namespace}) ==="
+  echo "  Enabling Blob CSI, OIDC issuer, and workload identity"
+  az aks update --subscription "${subscription}" \
+    --resource-group "${RESOURCE_GROUP}" --name "${CLUSTER_NAME}" \
+    --enable-blob-driver --enable-oidc-issuer --enable-workload-identity \
+    --yes --only-show-errors --output none
+  echo "  Creating model mirror storage account ${storage_account}"
+  az storage account create --subscription "${subscription}" \
+    --resource-group "${RESOURCE_GROUP}" --name "${storage_account}" \
+    --location "${LOCATION}" --sku Premium_LRS --kind BlockBlobStorage \
+    --https-only true --allow-shared-key-access false --output none
+  echo "  Creating streaming identity ${identity_name}"
+  identity_json=$(retry_azure_identity_not_found identity create --subscription "${subscription}" \
+    --resource-group "${RESOURCE_GROUP}" --name "${identity_name}" --output json)
+  identity_client_id=$(jq -er '.clientId | select(type == "string" and length > 0)' <<< "${identity_json}")
+  identity_principal_id=$(jq -er '.principalId | select(type == "string" and length > 0)' <<< "${identity_json}")
+  storage_scope="/subscriptions/${subscription}/resourceGroups/${RESOURCE_GROUP}/providers/Microsoft.Storage/storageAccounts/${storage_account}"
+  echo "  Granting streaming identity access to model blobs"
+  az role assignment create --subscription "${subscription}" \
+    --assignee-object-id "${identity_principal_id}" --assignee-principal-type ServicePrincipal \
+    --role "Storage Blob Data Contributor" --scope "${storage_scope}" --output none
+  echo "  Granting AKS control-plane identity access to the storage account"
+  control_plane_id=$(az aks show --subscription "${subscription}" \
+    --resource-group "${RESOURCE_GROUP}" --name "${CLUSTER_NAME}" --query identity.principalId --output tsv)
+  az role assignment create --subscription "${subscription}" \
+    --assignee-object-id "${control_plane_id}" --assignee-principal-type ServicePrincipal \
+    --role "Storage Account Contributor" --scope "${storage_scope}" --output none
+  oidc_issuer=$(az aks show --subscription "${subscription}" \
+    --resource-group "${RESOURCE_GROUP}" --name "${CLUSTER_NAME}" --query oidcIssuerProfile.issuerUrl --output tsv)
+  echo "  Creating streaming federated credential for ${namespace}/${service_account}"
+  retry_azure_identity_not_found identity federated-credential create --subscription "${subscription}" \
+    --resource-group "${RESOURCE_GROUP}" --identity-name "${identity_name}" \
+    --name model-streaming --issuer "${oidc_issuer}" \
+    --subject "system:serviceaccount:${namespace}:${service_account}" \
+    --audiences api://AzureADTokenExchange --output none
+
+  echo "  Configuring streaming service account and storage class"
+  kubectl create namespace "${namespace}" --dry-run=client --output yaml | kubectl apply -f -
+  kubectl create serviceaccount "${service_account}" --namespace "${namespace}" \
+    --dry-run=client --output yaml | kubectl apply -f -
+  kubectl annotate serviceaccount "${service_account}" --namespace "${namespace}" \
+    "azure.workload.identity/client-id=${identity_client_id}" --overwrite
+  jq -n --arg name "${storage_class}" --arg rg "${RESOURCE_GROUP}" \
+    --arg account "${storage_account}" --arg client "${identity_client_id}" '{
+      apiVersion: "storage.k8s.io/v1", kind: "StorageClass", metadata: {name: $name},
+      provisioner: "blob.csi.azure.com", allowVolumeExpansion: true, reclaimPolicy: "Delete",
+      parameters: {resourceGroup: $rg, storageAccount: $account, protocol: "fuse2",
+        skuName: "Premium_LRS", clientID: $client, mountWithWorkloadIdentityToken: "true"},
+      mountOptions: ["-o allow_other", "--block-cache", "--block-cache-block-size=16",
+        "--block-cache-pool-size=8192", "--block-cache-prefetch=11"]
+    }' | kubectl apply -f -
+  kubectl rollout status daemonset/csi-blob-node --namespace kube-system --timeout=300s
+
+  KAITO_HELM_ARGS+=(
+    --set featureGates.ModelMirror=true
+    --set featureGates.ModelStreaming=true
+    --set "defaultModelMirrorStorageClass=${storage_class}"
+    --set "defaultStreamingServiceAccount=${service_account}"
+  )
+}
+
 install_kaito() {
   echo "=== Installing KAITO workspace operator (image: nightly-latest) ==="
 
@@ -139,6 +227,9 @@ install_kaito() {
 
   KAITO_HELM_ARGS=()
   run_provider_hook prepare_kaito_crds
+  if [[ "${E2E_MODEL_STREAMING:-false}" == "true" ]]; then
+    prepare_model_streaming
+  fi
 
   # Per-model GAIE artifacts are provisioned by charts/modeldeployment; enabling
   # the gate would render a duplicate set of resources via Flux and conflict.