From 5913ad7b4e7142d5781b68abcbd885ac75c887c5 Mon Sep 17 00:00:00 2001
From: Mina Abadir <mina@abadir.email>
Date: Fri, 16 Oct 2026 14:36:24 -0400
Subject: [PATCH 1/2] fix(eks-addons): replace Loki's pod when moving it to a
 volume; roll Alloy faster

Moving Loki from an emptyDir to a PersistentVolumeClaim timed out in prod.
The migration Job deleted the StatefulSet with --cascade=orphan. The
recreated StatefulSet adopted the running pod, tried to add the claim to
it in place, which Kubernetes forbids ("pod updates may not change
fields..."), and never got as far as replacing it. The claim stayed
Pending and Helm waited out its deadline.

statefulset-volume-migration 0.2.0 runs a script instead of one kubectl
call:
- layout differs: POST Loki's /flush, then delete the StatefulSet with
  its pods (--cascade=foreground); the Loki release recreates both;
- layout matches: replace only pods that do not mount their claims, so
  clusters 0.1.0 left wedged recover on the next apply;
- otherwise change nothing.
The Role can get and delete the StatefulSet and its pods by ordinal only.

Alloy rolled one node at a time (~30s each), so on 12 nodes its rollout
outlasted Helm's 5-minute wait and failed the same apply. It now updates
25% of the nodes at a time.

rvn-eks-addons 0.22.1.
---
 .github/workflows/helm-charts.yml             |   3 +
 README.md                                     |   2 +-
 charts/test.sh                                |  27 ++-
 compute/eks/addons/README.md                  |   2 +-
 compute/eks/addons/alloy.tf                   |   8 +
 .../statefulset-volume-migration/Chart.yaml   |   4 +-
 .../ci/default-values.yaml                    |   1 +
 .../files/migrate.sh                          |  79 ++++++++
 .../templates/job.yaml                        |  62 ++++--
 .../templates/rbac.yaml                       |  26 ++-
 .../statefulset-volume-migration/values.yaml  |  41 +++-
 compute/eks/addons/loki_volume_migration.tf   |  18 +-
 .../eks/addons/rvn-eks-addons-definition.yml  |   4 +-
 .../addons/tests/addon_resilience.tftest.hcl  |   8 +
 compute/eks/addons/tests/logs.tftest.hcl      |   7 +
 .../test_statefulset_volume_migration.py      | 181 ++++++++++++++++++
 16 files changed, 434 insertions(+), 39 deletions(-)
 create mode 100644 compute/eks/addons/charts/statefulset-volume-migration/files/migrate.sh
 create mode 100644 compute/eks/addons/tests/test_statefulset_volume_migration.py

diff --git a/.github/workflows/helm-charts.yml b/.github/workflows/helm-charts.yml
index db0ecc93..7750beb5 100644
--- a/.github/workflows/helm-charts.yml
+++ b/.github/workflows/helm-charts.yml
@@ -46,3 +46,6 @@ jobs:
 
       - name: Test load balancer controller CRD retention
         run: python3 -B -m unittest discover -s compute/eks/addons/tests -p 'test_lb_controller_crds_chart.py' -v
+
+      - name: Test the StatefulSet volume migration script
+        run: python3 -B -m unittest discover -s compute/eks/addons/tests -p 'test_statefulset_volume_migration.py' -v
diff --git a/README.md b/README.md
index 5a9bf76a..c96aca9f 100644
--- a/README.md
+++ b/README.md
@@ -82,7 +82,7 @@ sync by `node tools/ravion-modules/dist/src/cli.js readme` (enforced in CI, and
 | `rvn-ecs-web` | ECS Web Service | v2.3.0 | `compute/ecs_service/` |
 | `rvn-ecs-worker` | ECS Worker | v2.2.2 | `compute/ecs_service/` |
 | `rvn-efs` | EFS File System | v1.1.0 | `storage/efs/` |
-| `rvn-eks-addons` | EKS Add-ons | v0.22.0 | `compute/eks/addons/` |
+| `rvn-eks-addons` | EKS Add-ons | v0.22.1 | `compute/eks/addons/` |
 | `rvn-eks-chart` | EKS Helm Chart | v0.3.0 | `compute/eks_service/` |
 | `rvn-eks-cluster` | EKS Cluster | v0.8.1 | `compute/eks/` |
 | `rvn-eks-web` | EKS Web Service | v1.10.0 | `compute/eks_service/` |
diff --git a/charts/test.sh b/charts/test.sh
index 0cbfb2e3..6f5ea340 100755
--- a/charts/test.sh
+++ b/charts/test.sh
@@ -1009,18 +1009,31 @@ test_thanos() {
 
 test_statefulset_volume_migration() {
   local chart="statefulset-volume-migration"
-  local default job
+  local default job ctr role
   default="$(render "${chart}" default --values "$(chart_path "${chart}")/ci/default-values.yaml")"
   job='.[] | select(.kind == "Job")'
-
-  assert_eq "${chart}: the Job orphans the StatefulSet so its pod keeps serving" \
-    "delete statefulset ravion-loki --namespace=ravion-operator --cascade=orphan --ignore-not-found --wait" \
-    "$(q "${default}" "${job} | .spec.template.spec.containers[0].args | join(\" \")")"
+  ctr="${job} | .spec.template.spec.containers[0]"
+  role='.[] | select(.kind == "Role")'
+
+  assert_eq "${chart}: the Job runs the migration script with a busybox shell" \
+    "registry.k8s.io/kubectl:v1.36.4 /tools/sh /scripts/migrate.sh public.ecr.aws/docker/library/busybox:1.37.0-musl" \
+    "$(q "${default}" "${ctr} | .image") $(q "${default}" "${ctr} | .command | join(\" \")") $(q "${default}" "${job} | .spec.template.spec.initContainers[0].image")"
+  assert_eq "${chart}: the Job runs the chart's own script" \
+    "$(cat "$(chart_path "${chart}")/files/migrate.sh")" \
+    "$(q "${default}" '.[] | select(.kind == "ConfigMap") | .data["migrate.sh"]')"
+  assert_eq "${chart}: the Job targets the StatefulSet, its layout and the store's flush endpoint" \
+    "ravion-loki ravion-operator true http://ravion-loki.ravion-operator.svc.cluster.local:3100/flush" \
+    "$(q "${default}" "${ctr} | .env | map(select(.name == \"STATEFULSET\" or .name == \"NAMESPACE\" or .name == \"PERSISTENT\" or .name == \"FLUSH_URL\")) | map(.value) | join(\" \")")"
   assert_eq "${chart}: the Job runs after every install and upgrade of the release" \
     "post-install,post-upgrade" "$(q "${default}" "${job} | .metadata.annotations.\"helm.sh/hook\"")"
-  assert_eq "${chart}: the Job may read and delete that one StatefulSet only" \
+  assert_eq "${chart}: the Job may read and delete that one StatefulSet" \
     "statefulsets ravion-loki delete,get" \
-    "$(q "${default}" '.[] | select(.kind == "Role") | .rules[0] | .resources[0] + " " + .resourceNames[0] + " " + (.verbs | sort | join(","))')"
+    "$(q "${default}" "${role} | .rules[0] | .resources[0] + \" \" + .resourceNames[0] + \" \" + (.verbs | sort | join(\",\"))")"
+  assert_eq "${chart}: and only that StatefulSet's pods, by ordinal" \
+    "pods ravion-loki-0,ravion-loki-1,ravion-loki-2 delete,get" \
+    "$(q "${default}" "${role} | .rules[1] | .resources[0] + \" \" + (.resourceNames | join(\",\")) + \" \" + (.verbs | sort | join(\",\"))")"
+  assert_eq "${chart}: the Job can neither list nor watch" \
+    "0" "$(q "${default}" "[${role} | .rules[].verbs[] | select(. == \"list\" or . == \"watch\")] | length")"
   if helm template test-release "$(chart_path "${chart}")" >/dev/null 2>&1; then
     fail "${chart}: a missing target is rejected" "helm template failure" "helm template succeeded"
   else
diff --git a/compute/eks/addons/README.md b/compute/eks/addons/README.md
index 434a2f16..5bb7562f 100644
--- a/compute/eks/addons/README.md
+++ b/compute/eks/addons/README.md
@@ -415,7 +415,7 @@ Loki runs in **single-binary mode**: every target in one StatefulSet replica, wi
 
 `loki_persistence_enabled` defaults to **on whenever the module's gp3 StorageClass exists** (`ebs_csi_driver_enabled`), and off otherwise, since a PVC with no StorageClass never binds. Loki's chunks are in S3 either way; what the local volume holds is the write-ahead log, the compactor's working directory, and the index cache, so with a volume a restart loses no logs. With persistence off the module mounts an `emptyDir` at `/var/loki` sized by `loki_persistence_size` — necessary, not cosmetic, because the chart only mounts that path when persistence is on and the container runs with a read-only root filesystem.
 
-A StatefulSet's volume layout cannot change in place, so switching an existing Loki between the two runs a small `loki-volume-migration` release first: its Job deletes the Loki StatefulSet with `--cascade=orphan`, leaving the pod serving, and the Loki release recreates it with the new layout in the same apply and replaces the pod. Logs not yet flushed from the old emptyDir are lost once, as on any restart.
+A StatefulSet's volume layout cannot change in place, so switching an existing Loki between the two runs a small `loki-volume-migration` release first: its Job asks Loki to flush to S3, then deletes the Loki StatefulSet together with its pod, and the Loki release recreates both with the new layout in the same apply. Loki is down for about a minute, from the delete until the new pod is ready; Alloy retries in the meantime. The pod is never orphaned: a recreated StatefulSet that adopts a pod without its volume cannot add the volume in place and never replaces the pod, which is what version 0.1.0 of the migration chart did. On a cluster it left that way, the Job replaces the pod on the next apply. When the layout already matches, the Job changes nothing.
 
 #### CloudWatch is a provider, not a mode
 
diff --git a/compute/eks/addons/alloy.tf b/compute/eks/addons/alloy.tf
index b13628f2..5b951d8c 100644
--- a/compute/eks/addons/alloy.tf
+++ b/compute/eks/addons/alloy.tf
@@ -108,6 +108,14 @@ resource "helm_release" "alloy" {
           # for CriticalAddonsOnly) run Karpenter, CoreDNS and the Operator
           # coordinators. The EKS node agents tolerate everything the same way.
           tolerations = [{ operator = "Exists" }]
+
+          # A quarter of the nodes at a time. The chart's default of one node
+          # at a time takes about half a minute per node, so on a dozen nodes a
+          # rollout outlasts Helm's wait and fails the apply.
+          updateStrategy = {
+            type          = "RollingUpdate"
+            rollingUpdate = { maxUnavailable = "25%" }
+          }
         }
 
         alloy = {
diff --git a/compute/eks/addons/charts/statefulset-volume-migration/Chart.yaml b/compute/eks/addons/charts/statefulset-volume-migration/Chart.yaml
index e7091e84..94aea8cd 100644
--- a/compute/eks/addons/charts/statefulset-volume-migration/Chart.yaml
+++ b/compute/eks/addons/charts/statefulset-volume-migration/Chart.yaml
@@ -1,5 +1,5 @@
 apiVersion: v2
 name: statefulset-volume-migration
-description: Delete one StatefulSet with --cascade=orphan, leaving its pods running, so the release that owns it can recreate it with a changed volume layout.
+description: Move one StatefulSet to a changed volume layout by deleting it with its pods so the owning release recreates both, and replace pods an earlier orphaning migration left without their claims.
 type: application
-version: 0.1.0
+version: 0.2.0
diff --git a/compute/eks/addons/charts/statefulset-volume-migration/ci/default-values.yaml b/compute/eks/addons/charts/statefulset-volume-migration/ci/default-values.yaml
index 56823aa5..06a4fbbf 100644
--- a/compute/eks/addons/charts/statefulset-volume-migration/ci/default-values.yaml
+++ b/compute/eks/addons/charts/statefulset-volume-migration/ci/default-values.yaml
@@ -2,3 +2,4 @@ statefulSet:
   name: ravion-loki
   namespace: ravion-operator
 persistent: true
+flushUrl: http://ravion-loki.ravion-operator.svc.cluster.local:3100/flush
diff --git a/compute/eks/addons/charts/statefulset-volume-migration/files/migrate.sh b/compute/eks/addons/charts/statefulset-volume-migration/files/migrate.sh
new file mode 100644
index 00000000..44f4d99e
--- /dev/null
+++ b/compute/eks/addons/charts/statefulset-volume-migration/files/migrate.sh
@@ -0,0 +1,79 @@
+#!/bin/sh
+# Moves one StatefulSet to the volume layout its owning release is about to
+# apply, and recovers pods left behind by an earlier, orphaning migration.
+#
+# A StatefulSet's volumeClaimTemplates are immutable, so the owner cannot switch
+# between an emptyDir and a PersistentVolumeClaim in place. When the layout
+# differs, the StatefulSet is deleted together with its pods, and the owner,
+# upgraded right after this Job, recreates it with fresh pods on the new
+# volumes. Orphaning the pods instead does not work: the recreated StatefulSet
+# adopts them, tries to add the claim to each running pod, which Kubernetes
+# forbids, and never gets as far as replacing them.
+#
+# When the layout already matches, only pods that do not mount their claims are
+# deleted: the ones chart 0.1.0 orphaned. Their StatefulSet recreates them. A
+# StatefulSet in the right shape with healthy pods is left alone.
+#
+# Before any pod goes, FLUSH_URL (when set) is POSTed so the store writes what
+# it holds in memory to object storage.
+set -eu
+
+: "${STATEFULSET:?}" "${NAMESPACE:?}" "${PERSISTENT:?}"
+FLUSH_URL="${FLUSH_URL:-}"
+TIMEOUT="${DELETE_TIMEOUT:-10m}"
+
+log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*"; }
+
+flush() {
+  [ -n "${FLUSH_URL}" ] || return 0
+  log "flushing ${FLUSH_URL}"
+  # Best effort: a store that does not answer usually has nothing left in
+  # memory, and refusing would leave the layout stuck.
+  wget -q -T 120 -O /dev/null --post-data= "${FLUSH_URL}" || log "flush failed; continuing"
+}
+
+# "<replicas> <template>,<template>," or nothing when there is no StatefulSet.
+current="$(kubectl get statefulset "${STATEFULSET}" --namespace="${NAMESPACE}" --ignore-not-found \
+  -o go-template='{{.spec.replicas}} {{range .spec.volumeClaimTemplates}}{{.metadata.name}},{{end}}')"
+if [ -z "${current}" ]; then
+  log "no StatefulSet ${NAMESPACE}/${STATEFULSET}; nothing to migrate"
+  exit 0
+fi
+replicas="${current%% *}"
+templates="$(echo "${current#* }" | tr ',' ' ')"
+
+has_claims=false
+[ -n "$(echo "${templates}" | tr -d ' ')" ] && has_claims=true
+
+if [ "${has_claims}" != "${PERSISTENT}" ]; then
+  log "volume layout changes (persistent: ${has_claims} -> ${PERSISTENT}); deleting ${NAMESPACE}/${STATEFULSET} and its pods"
+  flush
+  kubectl delete statefulset "${STATEFULSET}" --namespace="${NAMESPACE}" \
+    --cascade=foreground --ignore-not-found --wait --timeout="${TIMEOUT}"
+  log "deleted; the owning release recreates it"
+  exit 0
+fi
+
+i=0
+while [ "${i}" -lt "${replicas}" ]; do
+  pod="${STATEFULSET}-${i}"
+  i=$((i + 1))
+  # "present <claim> <claim> ", or nothing when the pod does not exist.
+  mounted="$(kubectl get pod "${pod}" --namespace="${NAMESPACE}" --ignore-not-found \
+    -o go-template='present {{range .spec.volumes}}{{with .persistentVolumeClaim}}{{.claimName}} {{end}}{{end}}')"
+  [ -n "${mounted}" ] || continue
+
+  missing=""
+  for template in ${templates}; do
+    case " ${mounted} " in
+      *" ${template}-${pod} "*) ;;
+      *) missing="${missing} ${template}-${pod}" ;;
+    esac
+  done
+  [ -n "${missing}" ] || continue
+
+  log "${NAMESPACE}/${pod} does not mount${missing}; replacing it"
+  flush
+  kubectl delete pod "${pod}" --namespace="${NAMESPACE}" --ignore-not-found --wait --timeout="${TIMEOUT}"
+done
+log "volume layout up to date"
diff --git a/compute/eks/addons/charts/statefulset-volume-migration/templates/job.yaml b/compute/eks/addons/charts/statefulset-volume-migration/templates/job.yaml
index 39fa7020..c198d014 100644
--- a/compute/eks/addons/charts/statefulset-volume-migration/templates/job.yaml
+++ b/compute/eks/addons/charts/statefulset-volume-migration/templates/job.yaml
@@ -17,6 +17,8 @@ spec:
       labels:
         app.kubernetes.io/name: {{ .Chart.Name }}
         app.kubernetes.io/instance: {{ .Release.Name }}
+      annotations:
+        checksum/script: {{ .Files.Get "files/migrate.sh" | sha256sum }}
     spec:
       serviceAccountName: {{ .Release.Name }}
       restartPolicy: Never
@@ -26,29 +28,65 @@ spec:
         runAsGroup: 65532
         seccompProfile:
           type: RuntimeDefault
+      # The kubectl image has no shell. busybox is statically linked, so it
+      # runs from the shared volume inside the kubectl container.
+      initContainers:
+        - name: tools
+          image: {{ .Values.busyboxImage.repository }}:{{ .Values.busyboxImage.tag }}
+          imagePullPolicy: {{ .Values.busyboxImage.pullPolicy }}
+          command: ["/bin/busybox", "sh", "-c", "cp /bin/busybox /tools/busybox && /tools/busybox --install -s /tools"]
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: tools
+              mountPath: /tools
+          resources:
+            requests:
+              cpu: 10m
+              memory: 16Mi
+            limits:
+              memory: 32Mi
       containers:
-        - name: kubectl
+        - name: migrate
           image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
           imagePullPolicy: {{ .Values.image.pullPolicy }}
-          # Orphaned, the pods keep serving until the owner recreates the
-          # StatefulSet, which adopts and then replaces them.
-          command: ["kubectl"]
-          args:
-            - delete
-            - statefulset
-            - {{ .Values.statefulSet.name }}
-            - --namespace={{ .Values.statefulSet.namespace }}
-            - --cascade=orphan
-            - --ignore-not-found
-            - --wait
+          command: ["/tools/sh", "/scripts/migrate.sh"]
+          env:
+            - name: PATH
+              value: /tools:/bin:/usr/local/bin:/usr/bin
+            - name: STATEFULSET
+              value: {{ .Values.statefulSet.name | quote }}
+            - name: NAMESPACE
+              value: {{ .Values.statefulSet.namespace | quote }}
+            - name: PERSISTENT
+              value: {{ .Values.persistent | toString | quote }}
+            - name: FLUSH_URL
+              value: {{ .Values.flushUrl | quote }}
+            - name: DELETE_TIMEOUT
+              value: {{ .Values.deleteTimeout | quote }}
           securityContext:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true
             capabilities:
               drop: ["ALL"]
+          volumeMounts:
+            - name: tools
+              mountPath: /tools
+            - name: scripts
+              mountPath: /scripts
           resources:
             requests:
               cpu: 10m
               memory: 32Mi
             limits:
               memory: 64Mi
+      volumes:
+        - name: tools
+          emptyDir:
+            sizeLimit: 16Mi
+        - name: scripts
+          configMap:
+            name: {{ .Release.Name }}
diff --git a/compute/eks/addons/charts/statefulset-volume-migration/templates/rbac.yaml b/compute/eks/addons/charts/statefulset-volume-migration/templates/rbac.yaml
index 64befd93..8c2abd41 100644
--- a/compute/eks/addons/charts/statefulset-volume-migration/templates/rbac.yaml
+++ b/compute/eks/addons/charts/statefulset-volume-migration/templates/rbac.yaml
@@ -1,11 +1,28 @@
 {{- if not (and .Values.statefulSet.name .Values.statefulSet.namespace) }}
 {{- fail "statefulSet.name and statefulSet.namespace are required" }}
 {{- end }}
+{{- if lt (int .Values.statefulSet.maxReplicas) 1 }}
+{{- fail "statefulSet.maxReplicas must be at least 1" }}
+{{- end }}
 {{/*
 Ordinary (non-hook) resources: they exist before the post-install and
-post-upgrade hook runs. The Role can read and delete the one StatefulSet only.
+post-upgrade hook runs. The Role can read and delete the one StatefulSet and
+its pods, named by ordinal, and nothing else.
 */}}
 apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ .Chart.Name }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+data:
+  migrate.sh: |
+    {{- .Files.Get "files/migrate.sh" | nindent 4 }}
+---
+apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: {{ .Release.Name }}
@@ -30,6 +47,13 @@ rules:
     resources: ["statefulsets"]
     resourceNames: [{{ .Values.statefulSet.name | quote }}]
     verbs: ["get", "delete"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    resourceNames:
+      {{- range $i := until (int .Values.statefulSet.maxReplicas) }}
+      - {{ printf "%s-%d" $.Values.statefulSet.name $i | quote }}
+      {{- end }}
+    verbs: ["get", "delete"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
diff --git a/compute/eks/addons/charts/statefulset-volume-migration/values.yaml b/compute/eks/addons/charts/statefulset-volume-migration/values.yaml
index 3067baa8..899e11f6 100644
--- a/compute/eks/addons/charts/statefulset-volume-migration/values.yaml
+++ b/compute/eks/addons/charts/statefulset-volume-migration/values.yaml
@@ -1,25 +1,48 @@
 # A StatefulSet's volumeClaimTemplates are immutable, so the release that owns
 # it cannot switch between an emptyDir and a PersistentVolumeClaim in place.
-# This chart's Job deletes the StatefulSet with --cascade=orphan after every
-# install or upgrade of this release; its pods keep running until the owning
-# release recreates the StatefulSet and replaces them. The caller upgrades this
-# release only when the owner's volume layout changes, and upgrades the owner
-# right after it.
+# This chart's Job runs after every install or upgrade of this release (see
+# files/migrate.sh):
+#   - when the StatefulSet's layout differs from `persistent`, it deletes the
+#     StatefulSet together with its pods, and the owning release, upgraded
+#     right after this one, recreates both on the new volumes;
+#   - when the layout already matches, it replaces only pods that do not mount
+#     their claims (left by chart 0.1.0, which orphaned them);
+#   - otherwise it changes nothing.
+# Pods are never orphaned: a recreated StatefulSet that adopts a pod without
+# its claim cannot add the claim in place and never replaces the pod.
 
-# -- The StatefulSet to delete. Missing is fine (a first install).
+# -- The StatefulSet to migrate. Missing is fine (a first install).
 statefulSet:
   name: ""
   namespace: ""
+  # -- Highest replica count the Job may need to handle. The Role names each
+  # pod it may delete, <name>-0 to <name>-(maxReplicas - 1).
+  maxReplicas: 3
 
-# -- The volume layout the owner is moving to. Informational: it names the Job.
+# -- The volume layout the owner is moving to: true for PersistentVolumeClaims,
+# false for none.
 persistent: false
 
-# -- kubectl image the Job runs. Pinned; override for clusters that must pull
-# from a private mirror.
+# -- POSTed before any pod is removed, so the store writes what it holds in
+# memory to object storage (Loki: http://<service>:3100/flush). Empty skips it.
+flushUrl: ""
+
+# -- How long each delete may wait for the pods to go.
+deleteTimeout: 4m
+
+# -- kubectl image the Job runs. It has no shell, so busybox supplies one.
+# Pinned; override for clusters that must pull from a private mirror.
 image:
   repository: registry.k8s.io/kubectl
   tag: v1.36.4
   pullPolicy: IfNotPresent
 
+# -- Statically linked busybox, copied into the Job's pod by an init
+# container to provide sh and wget.
+busyboxImage:
+  repository: public.ecr.aws/docker/library/busybox
+  tag: 1.37.0-musl
+  pullPolicy: IfNotPresent
+
 # -- Give up after this many failed attempts and fail the release.
 backoffLimit: 3
diff --git a/compute/eks/addons/loki_volume_migration.tf b/compute/eks/addons/loki_volume_migration.tf
index bb3d37c1..fb87d23d 100644
--- a/compute/eks/addons/loki_volume_migration.tf
+++ b/compute/eks/addons/loki_volume_migration.tf
@@ -4,10 +4,13 @@
 # Loki runs as a StatefulSet, and a StatefulSet's volume templates cannot be
 # changed in place: switching an existing Loki from an emptyDir to a
 # PersistentVolumeClaim (the default whenever a StorageClass exists) fails the
-# Helm upgrade. This release runs a kubectl Job that deletes the StatefulSet
-# with --cascade=orphan, leaving its pod running, just before the Loki release
-# is upgraded; the upgrade then creates the StatefulSet with the new volume
-# layout, adopts the pod and replaces it.
+# Helm upgrade. This release runs a Job just before the Loki release is
+# upgraded. When the layout changes, the Job flushes Loki to S3 and deletes the
+# StatefulSet together with its pod; the upgrade then creates both on the new
+# volume. Loki is down from the delete until the new pod is ready, about a
+# minute. It never orphans the pod: a recreated StatefulSet that adopts a pod
+# without its claim cannot add the claim in place and never replaces the pod
+# (what chart 0.1.0 did; the Job also replaces pods it left that way).
 #
 # The Job runs whenever this release changes, and every input of this release is
 # part of local.loki_volume_layout, which Loki carries as a pod annotation. So
@@ -22,10 +25,15 @@ locals {
       namespace = local.logs_namespace
     }
     persistent = local.loki_config.persistence_enabled
+    flushUrl   = "http://${local.loki_service_host}:3100/flush"
     image = {
       repository = local.kubectl_image_repository
       tag        = local.kubectl_image_tag
     }
+    busyboxImage = {
+      repository = local.busybox_image_repository
+      tag        = local.busybox_image_tag
+    }
   }
 
   loki_volume_migration_chart_version = yamldecode(file("${path.module}/charts/statefulset-volume-migration/Chart.yaml")).version
@@ -46,6 +54,8 @@ resource "helm_release" "loki_volume_migration" {
   version          = yamldecode(file("${path.module}/charts/statefulset-volume-migration/Chart.yaml")).version
   create_namespace = true
   upgrade_install  = true
+  # The Job's flush (up to 2m) and pod deletion (up to 4m) run inside this.
+  timeout = 600
 
   values = [yamlencode(local.loki_volume_migration_values)]
 }
diff --git a/compute/eks/addons/rvn-eks-addons-definition.yml b/compute/eks/addons/rvn-eks-addons-definition.yml
index 56bcda60..840d4f3e 100644
--- a/compute/eks/addons/rvn-eks-addons-definition.yml
+++ b/compute/eks/addons/rvn-eks-addons-definition.yml
@@ -4,8 +4,8 @@ definition:
   name: EKS Add-ons
   description: Selectable add-ons for an existing EKS cluster.
 release:
-  version: 0.22.0
-  description: "Add Grafana Mimir as an in-cluster metrics destination for clusters that outgrow one Prometheus: three copies of every sample on separate nodes, optionally one per availability zone, history in its own S3 bucket, no limits that reject metrics, and the same Metrics tab, Grafana and KEDA queries through its Prometheus API. Select it alongside Prometheus to move without losing data, then deselect Prometheus."
+  version: 0.22.1
+  description: "Fix applies that time out on Loki and Alloy. Moving Loki onto a persistent volume now flushes it and replaces its pod, instead of leaving the old pod running on its emptyDir while the apply waits; clusters left that way recover on the next apply. Alloy now updates a quarter of the nodes at a time instead of one, so its rollout finishes within Helm's wait on any cluster size."
 module:
   inputs:
     - id: cluster
diff --git a/compute/eks/addons/tests/addon_resilience.tftest.hcl b/compute/eks/addons/tests/addon_resilience.tftest.hcl
index 23aa24c1..b1a940df 100644
--- a/compute/eks/addons/tests/addon_resilience.tftest.hcl
+++ b/compute/eks/addons/tests/addon_resilience.tftest.hcl
@@ -491,6 +491,14 @@ run "loki_keeps_its_wal_on_a_volume" {
     condition     = yamldecode(helm_release.loki_volume_migration[0].values[0]).statefulSet == { name = "ravion-loki", namespace = "ravion-operator" }
     error_message = "The switch from an ephemeral volume needs the StatefulSet recreated; a migration release must do it."
   }
+  assert {
+    condition     = yamldecode(helm_release.loki_volume_migration[0].values[0]).flushUrl == "http://ravion-loki.ravion-operator.svc.cluster.local:3100/flush"
+    error_message = "Loki must flush what it holds in memory before the migration removes its pod."
+  }
+  assert {
+    condition     = yamldecode(helm_release.loki_volume_migration[0].values[0]).busyboxImage == { repository = "public.ecr.aws/docker/library/busybox", tag = "1.37.0-musl" }
+    error_message = "The migration Job runs a script, and the kubectl image has no shell; it needs the module's busybox image."
+  }
   assert {
     condition     = one([for d in helm_release.loki[0].values : yamldecode(d).singleBinary if can(yamldecode(d).singleBinary.podAnnotations["ravion.com/volume-layout"])]).podAnnotations["ravion.com/volume-layout"] == local.loki_volume_layout
     error_message = "Loki must be upgraded whenever the migration recreates its StatefulSet, or it would be left without one."
diff --git a/compute/eks/addons/tests/logs.tftest.hcl b/compute/eks/addons/tests/logs.tftest.hcl
index 8ced4e90..947928e3 100644
--- a/compute/eks/addons/tests/logs.tftest.hcl
+++ b/compute/eks/addons/tests/logs.tftest.hcl
@@ -148,6 +148,13 @@ run "logs_enabled_creates_bucket_loki_and_alloy" {
     error_message = "logs_enabled must install both Loki and Alloy"
   }
 
+  # One node at a time takes about half a minute per node, so on a dozen
+  # nodes a rollout outlasts Helm's wait and fails the apply.
+  assert {
+    condition     = yamldecode(helm_release.alloy[0].values[0]).controller.updateStrategy == { type = "RollingUpdate", rollingUpdate = { maxUnavailable = "25%" } }
+    error_message = "Alloy must roll out a quarter of the nodes at a time, so an upgrade finishes in a few waves whatever the cluster size."
+  }
+
   # In-cluster only: the endpoint is a ClusterIP Service URL, which is what
   # makes Ravion Operator's tunnel the sole route to it.
   assert {
diff --git a/compute/eks/addons/tests/test_statefulset_volume_migration.py b/compute/eks/addons/tests/test_statefulset_volume_migration.py
new file mode 100644
index 00000000..9ddade46
--- /dev/null
+++ b/compute/eks/addons/tests/test_statefulset_volume_migration.py
@@ -0,0 +1,181 @@
+"""Offline tests for the StatefulSet volume migration Job's script.
+
+A fake kubectl answers from a small in-memory description of the cluster and
+records every delete; a fake wget records flushes. Nothing reaches a cluster.
+
+Run from the repository root:
+    python3 -B -m unittest discover -s compute/eks/addons/tests -p 'test_statefulset_volume_migration.py'
+"""
+import json
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+
+SCRIPT = Path(__file__).resolve().parents[1] / "charts/statefulset-volume-migration/files/migrate.sh"
+
+# Answers `kubectl get statefulset|pod NAME ... -o go-template=...` from
+# $CLUSTER (JSON) in the shapes the script's templates print, and records
+# every delete in $CALLS.
+FAKE_KUBECTL = r"""#!/usr/bin/env python3
+import json, os, sys
+args = sys.argv[1:]
+cluster = json.loads(os.environ["CLUSTER"])
+with open(os.environ["CALLS"], "a") as calls:
+    calls.write(" ".join(["kubectl"] + args) + "\n")
+verb, kind, name = args[0], args[1], args[2]
+if verb == "get" and kind == "statefulset":
+    sts = cluster.get("statefulset")
+    if sts is None:
+        sys.exit(0)  # --ignore-not-found prints nothing
+    print(f'{sts["replicas"]} {"".join(t + "," for t in sts["claimTemplates"])}', end="")
+elif verb == "get" and kind == "pod":
+    pod = cluster.get("pods", {}).get(name)
+    if pod is None:
+        sys.exit(0)
+    print("present " + "".join(c + " " for c in pod), end="")
+elif verb == "delete":
+    if os.environ.get("FAIL_DELETE") == "1":
+        sys.exit(1)
+else:
+    sys.exit(2)
+"""
+
+FAKE_WGET = r"""#!/bin/sh
+echo "wget $*" >> "$CALLS"
+[ "${FAIL_FLUSH:-0}" = 1 ] && exit 1
+exit 0
+"""
+
+
+class MigrateTests(unittest.TestCase):
+    def run_job(self, cluster, persistent, flush_url="http://loki:3100/flush", **env_extra):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            for name, body in (("kubectl", FAKE_KUBECTL), ("wget", FAKE_WGET)):
+                tool = root / name
+                tool.write_text(body)
+                tool.chmod(0o755)
+            calls = root / "calls"
+            env = dict(
+                os.environ,
+                PATH=f"{root}:{os.environ['PATH']}",
+                CLUSTER=json.dumps(cluster),
+                CALLS=str(calls),
+                STATEFULSET="ravion-loki",
+                NAMESPACE="ravion-operator",
+                PERSISTENT="true" if persistent else "false",
+                FLUSH_URL=flush_url,
+                **env_extra,
+            )
+            result = subprocess.run(["sh", str(SCRIPT)], env=env, capture_output=True, text=True, timeout=10)
+            lines = calls.read_text().splitlines() if calls.exists() else []
+            return result, [line for line in lines if not line.startswith("kubectl get")]
+
+    def test_missing_statefulset_is_left_alone(self):
+        result, actions = self.run_job({}, persistent=True)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(actions, [])
+
+    def test_layout_change_deletes_the_statefulset_and_its_pods_after_a_flush(self):
+        result, actions = self.run_job(
+            {"statefulset": {"replicas": 1, "claimTemplates": []}, "pods": {"ravion-loki-0": []}},
+            persistent=True,
+        )
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(len(actions), 2, actions)
+        self.assertTrue(actions[0].startswith("wget ") and "http://loki:3100/flush" in actions[0], actions)
+        # Foreground: the pods go with it, so the recreated StatefulSet creates
+        # new ones with the new volumes instead of adopting the old ones.
+        self.assertIn("delete statefulset ravion-loki", actions[1])
+        self.assertIn("--namespace=ravion-operator", actions[1])
+        self.assertIn("--cascade=foreground", actions[1])
+        self.assertNotIn("--cascade=orphan", actions[1])
+        self.assertIn("--wait", actions[1])
+
+    def test_layout_change_back_to_ephemeral_also_recreates(self):
+        result, actions = self.run_job(
+            {"statefulset": {"replicas": 1, "claimTemplates": ["storage"]},
+             "pods": {"ravion-loki-0": ["storage-ravion-loki-0"]}},
+            persistent=False,
+        )
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertTrue(any("delete statefulset ravion-loki" in a and "--cascade=foreground" in a for a in actions), actions)
+
+    def test_matching_layout_with_mounted_claims_changes_nothing(self):
+        result, actions = self.run_job(
+            {"statefulset": {"replicas": 1, "claimTemplates": ["storage"]},
+             "pods": {"ravion-loki-0": ["storage-ravion-loki-0"]}},
+            persistent=True,
+        )
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(actions, [])
+
+    def test_pod_adopted_without_its_claim_is_replaced(self):
+        # What chart 0.1.0 left behind: the StatefulSet has the claim template,
+        # its adopted pod still runs on the old emptyDir, and the controller
+        # cannot add the volume in place, so it never replaces the pod.
+        result, actions = self.run_job(
+            {"statefulset": {"replicas": 1, "claimTemplates": ["storage"]}, "pods": {"ravion-loki-0": []}},
+            persistent=True,
+        )
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(len(actions), 2, actions)
+        self.assertTrue(actions[0].startswith("wget "), actions)
+        self.assertIn("delete pod ravion-loki-0", actions[1])
+        self.assertIn("--namespace=ravion-operator", actions[1])
+        self.assertFalse(any("delete statefulset" in a for a in actions), actions)
+
+    def test_only_the_pods_missing_a_claim_are_replaced(self):
+        result, actions = self.run_job(
+            {"statefulset": {"replicas": 3, "claimTemplates": ["storage"]},
+             "pods": {"ravion-loki-0": ["storage-ravion-loki-0"], "ravion-loki-1": [], "ravion-loki-2": ["storage-ravion-loki-2"]}},
+            persistent=True,
+        )
+        self.assertEqual(result.returncode, 0, result.stderr)
+        deletes = [a for a in actions if a.startswith("kubectl delete")]
+        self.assertEqual(len(deletes), 1, actions)
+        self.assertIn("delete pod ravion-loki-1 ", deletes[0] + " ")
+
+    def test_missing_pod_is_skipped(self):
+        result, actions = self.run_job(
+            {"statefulset": {"replicas": 1, "claimTemplates": ["storage"]}, "pods": {}},
+            persistent=True,
+        )
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(actions, [])
+
+    def test_no_flush_without_a_flush_url(self):
+        result, actions = self.run_job(
+            {"statefulset": {"replicas": 1, "claimTemplates": []}, "pods": {"ravion-loki-0": []}},
+            persistent=True,
+            flush_url="",
+        )
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertFalse(any(a.startswith("wget") for a in actions), actions)
+        self.assertTrue(any("delete statefulset" in a for a in actions), actions)
+
+    def test_failed_flush_is_reported_and_the_migration_continues(self):
+        # A flush that fails usually means the store is already down, with
+        # nothing left in memory to lose; refusing would leave the layout stuck.
+        result, actions = self.run_job(
+            {"statefulset": {"replicas": 1, "claimTemplates": []}, "pods": {"ravion-loki-0": []}},
+            persistent=True,
+            FAIL_FLUSH="1",
+        )
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertIn("flush", (result.stdout + result.stderr).lower())
+        self.assertTrue(any("delete statefulset" in a for a in actions), actions)
+
+    def test_failed_delete_fails_the_job(self):
+        result, _ = self.run_job(
+            {"statefulset": {"replicas": 1, "claimTemplates": []}, "pods": {"ravion-loki-0": []}},
+            persistent=True,
+            FAIL_DELETE="1",
+        )
+        self.assertNotEqual(result.returncode, 0)
+
+
+if __name__ == "__main__":
+    unittest.main()

From be817f6f09f4ca106922635051752cfb2b0361ca Mon Sep 17 00:00:00 2001
From: Mina Abadir <mina@abadir.email>
Date: Fri, 16 Oct 2026 22:26:14 -0400
Subject: [PATCH 2/2] fix(eks-addons): let Loki's volume migration read every
 replica

The migration Job's Role names each pod it may read, up to
statefulSet.maxReplicas, which the module left at the chart's default of
3. A loki_helm_values override with more replicas made the Job fail with
Forbidden on ravion-loki-3 and blocked the Loki upgrade. Pass the most
replicas any loki_helm_values document asks for, fixed or autoscaled.
---
 compute/eks/addons/loki_volume_migration.tf   | 15 +++++++++++--
 .../addons/tests/addon_resilience.tftest.hcl  | 21 ++++++++++++++++++-
 2 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/compute/eks/addons/loki_volume_migration.tf b/compute/eks/addons/loki_volume_migration.tf
index fb87d23d..2d4db302 100644
--- a/compute/eks/addons/loki_volume_migration.tf
+++ b/compute/eks/addons/loki_volume_migration.tf
@@ -19,10 +19,21 @@
 ################################################################################
 
 locals {
+  # The Job's Role names each pod it may read, so it must cover the most
+  # replicas any loki_helm_values document asks for, fixed or autoscaled. The
+  # floor matches the chart's default.
+  loki_volume_migration_max_replicas = max(3, flatten([
+    for doc in var.loki_helm_values : [
+      try(tonumber(yamldecode(doc).singleBinary.replicas), 0),
+      try(tonumber(yamldecode(doc).singleBinary.autoscaling.maxReplicas), 0),
+    ]
+  ])...)
+
   loki_volume_migration_values = {
     statefulSet = {
-      name      = local.loki_release_name
-      namespace = local.logs_namespace
+      name        = local.loki_release_name
+      namespace   = local.logs_namespace
+      maxReplicas = local.loki_volume_migration_max_replicas
     }
     persistent = local.loki_config.persistence_enabled
     flushUrl   = "http://${local.loki_service_host}:3100/flush"
diff --git a/compute/eks/addons/tests/addon_resilience.tftest.hcl b/compute/eks/addons/tests/addon_resilience.tftest.hcl
index c09ed412..75db5e69 100644
--- a/compute/eks/addons/tests/addon_resilience.tftest.hcl
+++ b/compute/eks/addons/tests/addon_resilience.tftest.hcl
@@ -488,7 +488,7 @@ run "loki_keeps_its_wal_on_a_volume" {
     error_message = "With a StorageClass available, Loki's write-ahead log must survive a restart."
   }
   assert {
-    condition     = yamldecode(helm_release.loki_volume_migration[0].values[0]).statefulSet == { name = "ravion-loki", namespace = "ravion-operator" }
+    condition     = yamldecode(helm_release.loki_volume_migration[0].values[0]).statefulSet == { name = "ravion-loki", namespace = "ravion-operator", maxReplicas = 3 }
     error_message = "The switch from an ephemeral volume needs the StatefulSet recreated; a migration release must do it."
   }
   assert {
@@ -505,6 +505,25 @@ run "loki_keeps_its_wal_on_a_volume" {
   }
 }
 
+# The migration Job's Role names each Loki pod it may read, so it must cover
+# every replica loki_helm_values can ask for, or the Job fails with Forbidden.
+run "loki_volume_migration_covers_replicas_from_loki_helm_values" {
+  command = plan
+
+  variables {
+    loki_helm_values = [
+      yamlencode({ singleBinary = { replicas = 5 } }),
+      yamlencode({ singleBinary = { autoscaling = { enabled = true, maxReplicas = 7 } } }),
+      yamlencode({ loki = { auth_enabled = false } }),
+    ]
+  }
+
+  assert {
+    condition     = yamldecode(helm_release.loki_volume_migration[0].values[0]).statefulSet.maxReplicas == 7
+    error_message = "The migration Job must be allowed to read every Loki pod loki_helm_values can create."
+  }
+}
+
 run "loki_stays_ephemeral_without_a_storage_class" {
   command = plan