deploy-cp.yml

name: Deploy CP

# Reusable workflow: provision the CP TDX VM on GCP, wait for it to be
# healthy, verify attestation + dashboard + STONITH, then cascade a
# relaunch of the matching dd-local agent VM and block until it
# re-registers. Called from release.yml's deploy-preview (PR path) and
# deploy-production (main / dispatch path) with env-specific inputs —
# both paths share this exact set of verification steps so every PR
# exercises the prod deploy code.
#
# GitHub Actions allows ≤4 levels of workflow_call nesting. Today's
# chain is `release.yml → deploy-cp.yml` (2). The agent-relaunch
# cascade uses a composite action (same-job, no nesting) to keep
# headroom for future wrapping.

on:
  workflow_call:
    inputs:
      env:
        description: 'DD_ENV (e.g. "production", "pr-42")'
        required: true
        type: string
      hostname:
        description: 'Public hostname (e.g. app.devopsdefender.com)'
        required: true
        type: string
      gcp_environment:
        description: 'GitHub environment name — "production" | "staging"'
        required: true
        type: string
      workload_identity_provider:
        description: 'GCP Workload Identity Federation provider resource name (required only for target=gcp)'
        required: false
        type: string
        default: ''
      service_account:
        description: 'GCP service account email (required only for target=gcp)'
        required: false
        type: string
        default: ''
      release_tag:
        description: 'devopsdefender release tag to deploy (e.g. "latest", "pr-abc123")'
        required: true
        type: string
      comment_on_pr:
        description: 'Leave a PR comment with the preview URL'
        required: false
        type: boolean
        default: false
      relaunch_agent:
        description: 'After CP deploy, cascade a relaunch of dd-local-{env} via SSH'
        required: false
        type: boolean
        default: true
      ref:
        description: 'Git ref the tdx2 host should pull before relaunching the agent VM'
        required: false
        type: string
        default: main
      target:
        description: 'Where the CP runs: "ssh" (libvirt VM on the tdx2 baremetal, default) or "gcp" (fresh TDX VM on GCE)'
        required: false
        type: string
        default: ssh
      ee_tag:
        description: 'Explicit easyenclave-mini release tag to pin this deploy to (pre-flight-test a candidate before promoting staging→stable). Empty = default to channel resolved from env.'
        required: false
        type: string
        default: ''
      owner:
        description: 'Fleet principal — GitHub login or owner/repo path. Baked into the agent.env as EE_OWNER and resolved to id+kind via gh api. No default; the caller (release.yml) sets it explicitly.'
        required: true
        type: string

# Serialize PR-preview deploys on the `preview` bucket: all pr-*
# runs share one libvirt domain (`dd-local-preview`) on tdx2, so
# two concurrent deploys race — the second PR's dd-relaunch.sh
# destroys and rebuilds the VM mid-way through the first PR's
# register-wait, so the first PR's agent ends up registering
# against the WRONG cp (whichever env won the rebuild) and the
# wait times out. Keeping production in its own group so a
# prod deploy isn't blocked by a preview queue.
concurrency:
  group: deploy-cp-${{ inputs.env == 'production' && 'production' || 'preview' }}
  cancel-in-progress: false

jobs:
  deploy:
    runs-on: ubuntu-latest
    environment: ${{ inputs.gcp_environment }}
    permissions:
      contents: read
      id-token: write
      pull-requests: write
    env:
      DD_ENV: ${{ inputs.env }}
      DD_HOSTNAME: ${{ inputs.hostname }}
      # Long-lived PAT (not the run-scoped github.token) so the EE VM can
      # re-fetch its release assets on ANY later boot — a host reboot must
      # self-heal. github.token dies with the job, which made the
      # 2026-05-29 reboot an unrecoverable 401-on-boot outage.
      DD_BOOT_GITHUB_TOKEN: ${{ secrets.DD_MGMT_PAT }}
      GCP_ZONE: us-central1-c
      DD_AUTH_BROKER_URL: https://app.${{ vars.DD_CF_DOMAIN || 'devopsdefender.com' }}
      DD_AUTH_COOKIE_DOMAIN: .${{ vars.DD_CF_DOMAIN || 'devopsdefender.com' }}
      DD_AUTH_COOKIE_SECRET: ${{ secrets.DD_AUTH_COOKIE_SECRET }}
      DD_GITHUB_CLIENT_ID: ${{ inputs.env == 'production' && vars.DD_PRODUCTION_GITHUB_CLIENT_ID || vars.DD_STAGING_GITHUB_CLIENT_ID }}
      DD_GITHUB_CLIENT_SECRET: ${{ inputs.env == 'production' && secrets.DD_PRODUCTION_GITHUB_CLIENT_SECRET || secrets.DD_STAGING_GITHUB_CLIENT_SECRET }}
      DD_STAGING_GITHUB_CLIENT_ID: ${{ vars.DD_STAGING_GITHUB_CLIENT_ID }}
      DD_STAGING_GITHUB_CLIENT_SECRET: ${{ secrets.DD_STAGING_GITHUB_CLIENT_SECRET }}
      DD_PRODUCTION_GITHUB_CLIENT_ID: ${{ vars.DD_PRODUCTION_GITHUB_CLIENT_ID }}
      DD_PRODUCTION_GITHUB_CLIENT_SECRET: ${{ secrets.DD_PRODUCTION_GITHUB_CLIENT_SECRET }}
    steps:
      - uses: actions/checkout@v4

      # Map dd env → easyenclave-mini channel. Prod tracks `stable`
      # when a stable mini image exists; previews track `staging`.
      # Applies to both target=gcp (image family lookup) and target=ssh
      # (release-asset sync on tdx2).
      - name: Resolve easyenclave-mini channel
        id: ee
        env:
          ENV: ${{ inputs.env }}
        run: |
          case "$ENV" in
            production) channel=stable;  family=easyenclave-mini-stable  ;;
            *)          channel=staging; family=easyenclave-mini-staging ;;
          esac
          {
            echo "channel=$channel"
            echo "family=$family"
          } >> "$GITHUB_OUTPUT"

      # GCP auth only on target=gcp. We used to configure it
      # unconditionally so ssh-target deploys could reap GCE orphans
      # from a prior gcp-target deploy, but the resulting ~20s
      # overhead per deploy outweighed the benefit — the orphan path
      # is only reachable during a one-time target=gcp→ssh
      # transition, and `.github/workflows/force-cleanup-tunnels.yml`
      # (and GCP's own console) are better homes for that cleanup.
      - uses: google-github-actions/auth@v2
        if: inputs.target == 'gcp'
        with:
          workload_identity_provider: ${{ inputs.workload_identity_provider }}
          service_account: ${{ inputs.service_account }}
      - uses: google-github-actions/setup-gcloud@v2
        if: inputs.target == 'gcp'

      - name: Relaunch SSH CP VM (target=ssh)
        if: inputs.target == 'ssh'
        uses: ./.github/actions/relaunch-cp
        with:
          env: ${{ inputs.env }}
          hostname: ${{ inputs.hostname }}
          ref: ${{ inputs.ref }}
          ssh-key:       ${{ secrets.DD_LOCAL_SSH_KEY }}
          host:          ${{ inputs.env == 'production' && vars.BAREMETAL_PRODUCTION_HOST || vars.BAREMETAL_STAGING_HOST }}
          ssh-user:      ${{ inputs.env == 'production' && vars.BAREMETAL_PRODUCTION_USER || vars.BAREMETAL_STAGING_USER }}
          repo-root:     ${{ format('/home/{0}/src/dd', inputs.env == 'production' && vars.BAREMETAL_PRODUCTION_USER || vars.BAREMETAL_STAGING_USER) }}
          cf-api-token:  ${{ secrets.DD_CP_CF_API_TOKEN }}
          cf-account-id: ${{ secrets.DD_CP_CF_ACCOUNT_ID }}
          cf-zone-id:    ${{ secrets.DD_CP_CF_ZONE_ID }}
          ita-api-key:   ${{ secrets.DD_ITA_API_KEY }}
          auth-broker-url: ${{ env.DD_AUTH_BROKER_URL }}
          auth-cookie-domain: ${{ env.DD_AUTH_COOKIE_DOMAIN }}
          auth-cookie-secret: ${{ secrets.DD_AUTH_COOKIE_SECRET }}
          github-client-id: ${{ env.DD_GITHUB_CLIENT_ID }}
          github-client-secret: ${{ env.DD_GITHUB_CLIENT_SECRET }}
          staging-github-client-id: ${{ vars.DD_STAGING_GITHUB_CLIENT_ID }}
          staging-github-client-secret: ${{ secrets.DD_STAGING_GITHUB_CLIENT_SECRET }}
          production-github-client-id: ${{ vars.DD_PRODUCTION_GITHUB_CLIENT_ID }}
          production-github-client-secret: ${{ secrets.DD_PRODUCTION_GITHUB_CLIENT_SECRET }}
          release-tag:   ${{ inputs.release_tag }}
          ee-channel:    ${{ steps.ee.outputs.channel }}
          ee-tag:        ${{ inputs.ee_tag }}
          owner:         ${{ inputs.owner }}
          # Long-lived PAT, not run-scoped github.token: EE re-fetches its
          # release assets on every boot, so the token must outlive the job
          # or a reboot 401s and panics. See DD_BOOT_GITHUB_TOKEN above.
          github-token:  ${{ secrets.DD_MGMT_PAT }}

      - name: Create TDX VM (boots from easyenclave-mini, fetches dd from GitHub releases)
        if: inputs.target == 'gcp'
        env:
          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
          DD_DOMAIN: ${{ vars.DD_CF_DOMAIN || 'devopsdefender.com' }}
          CLOUDFLARE_API_TOKEN: ${{ secrets.DD_CP_CF_API_TOKEN }}
          CLOUDFLARE_ACCOUNT_ID: ${{ secrets.DD_CP_CF_ACCOUNT_ID }}
          CLOUDFLARE_ZONE_ID: ${{ secrets.DD_CP_CF_ZONE_ID }}
          DD_AUTH_BROKER_URL: ${{ env.DD_AUTH_BROKER_URL }}
          DD_AUTH_COOKIE_DOMAIN: ${{ env.DD_AUTH_COOKIE_DOMAIN }}
          DD_AUTH_COOKIE_SECRET: ${{ secrets.DD_AUTH_COOKIE_SECRET }}
          DD_GITHUB_CLIENT_ID: ${{ env.DD_GITHUB_CLIENT_ID }}
          DD_GITHUB_CLIENT_SECRET: ${{ env.DD_GITHUB_CLIENT_SECRET }}
          DD_STAGING_GITHUB_CLIENT_ID: ${{ vars.DD_STAGING_GITHUB_CLIENT_ID }}
          DD_STAGING_GITHUB_CLIENT_SECRET: ${{ secrets.DD_STAGING_GITHUB_CLIENT_SECRET }}
          DD_PRODUCTION_GITHUB_CLIENT_ID: ${{ vars.DD_PRODUCTION_GITHUB_CLIENT_ID }}
          DD_PRODUCTION_GITHUB_CLIENT_SECRET: ${{ secrets.DD_PRODUCTION_GITHUB_CLIENT_SECRET }}
          DD_ITA_API_KEY: ${{ secrets.DD_ITA_API_KEY }}
          DD_RELEASE_TAG: ${{ inputs.release_tag }}
          EE_OWNER: ${{ inputs.owner }}
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          EE_IMAGE_FAMILY: ${{ steps.ee.outputs.family }}
          EE_IMAGE_PROJECT: easyenclave
          VM_MACHINE_TYPE: c3-standard-4
          VM_DISK_SIZE: 10GB
          DD_ITA_BASE_URL: https://api.trustauthority.intel.com
          DD_ITA_JWKS_URL: https://portal.trustauthority.intel.com/certs
          DD_ITA_ISSUER: https://portal.trustauthority.intel.com
        run: |
          set -euo pipefail

          VM_NAME="dd-${DD_ENV}-$(date +%s)"
          : "${DD_ITA_API_KEY:?set DD_ITA_API_KEY via secrets.DD_ITA_API_KEY}"
          : "${EE_OWNER:?set inputs.owner from the calling workflow}"

          # Resolve EE_OWNER → numeric id + kind via gh api. Same idiom
          # as apps/_infra/local-{agents,cp}.sh. Hard-fails on lookup
          # miss so a typo doesn't bake a bricked principal into the VM.
          if [[ "$EE_OWNER" == */* ]]; then
            EE_OWNER_ID=$(gh api "repos/$EE_OWNER" -q .id)
            EE_OWNER_KIND=repo
          else
            read -r EE_OWNER_ID GH_TYPE < <(gh api "users/$EE_OWNER" -q '"\(.id) \(.type)"')
            case "$GH_TYPE" in
              User)         EE_OWNER_KIND=user ;;
              Organization) EE_OWNER_KIND=org ;;
              *) echo "::error::unexpected gh api type: $GH_TYPE"; exit 1 ;;
            esac
          fi
          export DD_OWNER="$EE_OWNER" DD_OWNER_ID DD_OWNER_KIND
          DD_OWNER_ID="$EE_OWNER_ID"
          DD_OWNER_KIND="$EE_OWNER_KIND"
          echo "  EE_OWNER=$EE_OWNER (kind=$EE_OWNER_KIND, id=$EE_OWNER_ID)"

          # Bake a workload template: substitute ${VAR} placeholders
          # and strip "KEY=" env entries that ended up with empty values
          # (e.g. OAuth creds in non-prod envs). envsubst is restricted
          # to the uppercase ${VAR} refs the template actually declares
          # so shell locals inside cmd strings ($i, $((…)), etc.)
          # aren't eaten.
          bake() {
            case "$1" in
              *.json.tmpl)
                local vars
                vars=$(grep -oE '\$\{[A-Z_][A-Z0-9_]*\}' "$1" | sort -u | tr -d '\n')
                envsubst "$vars" < "$1" \
                  | jq -c 'if .env then .env |= map(select(test("^[^=]+=.+"))) else . end'
                ;;
              *.json)
                jq -c . "$1"
                ;;
              *)
                echo "::error::unknown workload file type: $1" >&2
                return 1
                ;;
            esac
          }

          # Boot workloads come from apps/<name>/workload.{json,json.tmpl}.
          # cloudflared fetches the binary onto PATH; dd-management runs
          # devopsdefender in DD_MODE=management (CP + dashboard).
          EE_BOOT_WORKLOADS=$({
            bake apps/cloudflared/workload.json
            bake apps/dd-management/workload.json.tmpl
            DD_SESSIOND_DIR=/tmp/dd-shell \
              DD_SESSIOND_SCRATCH_DIR=/tmp/dd-shell/sessions \
              bake apps/dd-sessiond/workload.json.tmpl
            DD_DOMAIN="$DD_DOMAIN" \
              DD_HOSTNAME="$DD_HOSTNAME" \
              DD_ENV="$DD_ENV" \
              DD_OWNER="$EE_OWNER" \
              DD_OWNER_ID="$EE_OWNER_ID" \
              DD_OWNER_KIND="$EE_OWNER_KIND" \
              DD_AUTH_BROKER_URL="$DD_AUTH_BROKER_URL" \
              DD_AUTH_COOKIE_DOMAIN="$DD_AUTH_COOKIE_DOMAIN" \
              DD_AUTH_COOKIE_SECRET="$DD_AUTH_COOKIE_SECRET" \
              bake apps/dd-shell/workload.json.tmpl
          } | jq -cs '.')

          # EE_CAPTURE_SOCKET tells EE (post-capture-socket patch) to tee
          # every spawned workload's stdio to this unix socket. Kept for
          # forward compatibility — a future workload can bind + read it.
          # Unpatched EE images ignore the variable; patched EE falls back
          # to running without capture when nothing is listening.
          jq -c -n \
            --arg workloads "$EE_BOOT_WORKLOADS" \
            --arg owner "$EE_OWNER" \
            --argjson owner_id "$EE_OWNER_ID" \
            --arg owner_kind "$EE_OWNER_KIND" \
            --arg github_token "$DD_BOOT_GITHUB_TOKEN" \
            '{
               "EE_BOOT_WORKLOADS": $workloads,
               "EE_OWNER": $owner,
               "EE_OWNER_ID": ($owner_id | tostring),
               "EE_OWNER_KIND": $owner_kind,
               "EE_GITHUB_TOKEN": $github_token,
               "EE_CAPTURE_SOCKET": "/run/ee/capture.sock"
             }' \
            > /tmp/ee-config.json

          gcloud compute instances create "$VM_NAME" \
            --project="$GCP_PROJECT_ID" \
            --zone="$GCP_ZONE" \
            --machine-type="$VM_MACHINE_TYPE" \
            --confidential-compute-type=TDX \
            --maintenance-policy=TERMINATE \
            --boot-disk-size="$VM_DISK_SIZE" \
            --image-family="$EE_IMAGE_FAMILY" \
            --image-project="$EE_IMAGE_PROJECT" \
            --metadata-from-file=ee-config=/tmp/ee-config.json \
            --labels=devopsdefender=managed,dd_env="${DD_ENV}" \
            --tags=dd-management

          rm -f /tmp/ee-config.json
          echo "VM: $VM_NAME ($DD_HOSTNAME, release $DD_RELEASE_TAG)"

      - name: Wait for CP health (streams GCE serial console)
        if: inputs.target == 'gcp'
        env:
          AGENT_URL: https://${{ inputs.hostname }}
          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
        run: |
          VM_NAME=$(gcloud compute instances list \
            --project="$GCP_PROJECT_ID" \
            --filter="labels.devopsdefender=managed AND labels.dd_env=${DD_ENV}" \
            --format="value(name)" --sort-by=~creationTimestamp | head -1)
          if [ -z "$VM_NAME" ]; then
            echo "::error::no dd-${DD_ENV} VM found — gcp-deploy.sh must have failed"
            exit 1
          fi
          echo "Watching VM: $VM_NAME (zone: $GCP_ZONE)"

          # Readiness gate: don't accept the first lucky 200. cloudflared
          # brings up its HA edge connectors one at a time over ~30s and the
          # tunnel flaps during that window (a probe gets 200, the next hangs
          # on a cold connector). Require N *consecutive* good `/health`
          # payloads so a flap can't masquerade as ready — this is what makes
          # the downstream assertions race-free without blind --retry.
          LAST_LINES=0
          need=3; ok=0
          for i in $(seq 1 90); do
            # Stream serial console so boot failures (DHCP hang, release
            # fetch error, cloudflared exit, etc.) are visible without
            # shelling into GCP.
            gcloud compute instances get-serial-port-output "$VM_NAME" \
              --project="$GCP_PROJECT_ID" --zone="$GCP_ZONE" 2>/dev/null \
              > /tmp/serial.log || true
            TOTAL_LINES=$(wc -l < /tmp/serial.log)
            if [ "$TOTAL_LINES" -gt "$LAST_LINES" ]; then
              tail -n +$((LAST_LINES + 1)) /tmp/serial.log \
                | sed 's/^/[serial] /'
              LAST_LINES=$TOTAL_LINES
            fi

            if grep -qE "FATAL|Kernel panic|Invalid ELF header|/bin/sh: can't access tty" /tmp/serial.log; then
              echo "::error::boot failed — serial log shows fatal pattern"
              exit 1
            fi

            BODY=$(curl -fsS --max-time 8 "${AGENT_URL}/health" 2>/dev/null || true)
            if [ -n "$BODY" ] && echo "$BODY" | jq -e '.ok == true and .service == "cp"' >/dev/null 2>&1; then
              ok=$((ok + 1))
              if [ "$ok" -ge "$need" ]; then
                echo "CP stable at ${AGENT_URL} (${need} consecutive, build=$(echo "$BODY" | jq -r '.build // "?"'))"
                exit 0
              fi
            else
              ok=0
            fi
            echo "  awaiting stable CP (streak ${ok}/${need})... (${i}/90)"
            sleep 5
          done
          echo "::error::CP never stabilized within budget (gcp target)"
          echo "--- final serial tail ---"
          tail -80 /tmp/serial.log | sed 's/^/[serial] /'
          exit 1

      - name: Wait for CP health (SSH CP — streams serial console)
        if: inputs.target == 'ssh'
        env:
          AGENT_URL: https://${{ inputs.hostname }}
          ENV_LABEL: ${{ inputs.env }}
          HOST:      ${{ inputs.env == 'production' && vars.BAREMETAL_PRODUCTION_HOST || vars.BAREMETAL_STAGING_HOST }}
          SSH_USER:  ${{ inputs.env == 'production' && vars.BAREMETAL_PRODUCTION_USER || vars.BAREMETAL_STAGING_USER }}
          SSH_KEY:   ${{ secrets.DD_LOCAL_SSH_KEY }}
        run: |
          set -uo pipefail
          # SSH CP boot fires asynchronously via `virsh start` on the
          # tdx2 host. We own the outside view (poll /health for a
          # real 200) and tail the inside view — the serial console
          # at /var/log/ee-local-${env}-cp.log — so every boot-time
          # eprintln! lands in this job's log in real time. Both views
          # together turn "CP not healthy within 5 min" from an opaque
          # timeout into a traceable failure.
          tail_pid=""
          key=$(mktemp)
          mirror=$(mktemp)
          # `rc=$?` FIRST: capture the script's actual exit status so
          # the `exit $rc` at the end doesn't inherit `pkill`'s exit 1
          # (which fires whenever it has no matches — i.e. a clean
          # state). Without that, every successful run would be
          # reported as failed.
          #
          # `pkill -P $$` kills the ENTIRE tail pipeline (ssh, sed,
          # tee) — not just `$tail_pid`, which is tee (the last
          # command). Without this the ssh hangs on the remote
          # `tail -F`, stdout stays open on the runner, and the step
          # can stall long past the script's `exit` call.
          trap 'rc=$?; rm -f "$key" "$mirror"; pkill -TERM -P $$ 2>/dev/null || true; sleep 0.3; pkill -KILL -P $$ 2>/dev/null || true; exit $rc' EXIT
          printf '%s\n' "$SSH_KEY" > "$key"
          chmod 600 "$key"
          # `stdbuf -oL` on both ends: force tail and sed to line-buffer
          # so each VM boot line lands in this job's log as it's written,
          # instead of getting block-buffered until the pipe closes (which
          # happens long after boot is over). Dropping `2>/dev/null` on
          # tail so "cannot open log" / "retrying" diagnostics surface
          # — if the log file never appears, we want to see why rather
          # than stare at an apparently-silent boot.
          #
          # `tee -a "$mirror"` so the health-poll loop can fail fast
          # when the CP's main.rs emits its terminal
          # `devopsdefender: fatal:` eprintln. Without this the loop
          # would burn its full 5-min budget polling /health on a
          # tunnel that never came up.
          ssh -i "$key" -T \
              -o StrictHostKeyChecking=accept-new \
              -o ServerAliveInterval=30 \
              "$SSH_USER@$HOST" \
              "sudo stdbuf -oL tail -n +1 -F /var/log/ee-local-${ENV_LABEL}-cp.log || true" \
            2>&1 | stdbuf -oL sed -u "s/^/  [dd-local-${ENV_LABEL}-cp console] /" \
                 | stdbuf -oL tee -a "$mirror" &
          tail_pid=$!
          sleep 1
          # Readiness gate: require N *consecutive* good /health payloads, not
          # the first lucky 200. cloudflared's HA connectors warm up over ~30s
          # and the tunnel flaps (200, then a cold-connector hang) in that
          # window — a single 200 then an un-retried verify is exactly how this
          # step used to fail (curl 28, 0 bytes). The streak folds the old
          # separate "Verify CP health payload" step in: payload correctness +
          # stability + build identity, gated once, race-free.
          need=3; ok=0
          for i in $(seq 1 90); do
            BODY=$(curl -fsS --max-time 8 "${AGENT_URL}/health" 2>/dev/null || true)
            if [ -n "$BODY" ] && echo "$BODY" | jq -e '.ok == true and .service == "cp"' >/dev/null 2>&1; then
              ok=$((ok + 1))
              if [ "$ok" -ge "$need" ]; then
                BUILD=$(echo "$BODY" | jq -r '.build // "?"')
                echo "CP stable at ${AGENT_URL} (${need} consecutive, build=${BUILD})"
                if [ "$BUILD" = "dev" ]; then
                  echo "::error::CP /health reports build=dev — running CP is not a CI-built binary"
                  exit 1
                fi
                exit 0
              fi
            else
              ok=0
            fi
            if [ -s "$mirror" ] && grep -q 'devopsdefender: fatal:' "$mirror"; then
              echo "::group::Fatal line from CP console"
              grep -m1 'devopsdefender: fatal:' "$mirror" || true
              echo "::endgroup::"
              echo "::error::CP exited fatal; not polling /health further"
              exit 1
            fi
            echo "  awaiting stable CP (streak ${ok}/${need})... (${i}/90)"
            sleep 5
          done
          echo "::error::CP never stabilized within budget (SSH target)"
          exit 1

      - name: Verify NEW VM health / TDX attestation
        if: inputs.target == 'gcp'
        env:
          AGENT_URL: https://${{ inputs.hostname }}
        run: |
          # The Noise pre-handshake bundle used to live at `/attest`;
          # it's now served inline on `/health` as `.noise.quote_b64`
          # + `.noise.pubkey_hex` so a bastion-app can bootstrap in
          # one fetch. `quote_b64` is an Intel-signed TDX quote whose
          # `report_data` (first 32 bytes) equals the raw Noise static
          # pubkey. MRTD = 48 bytes at offset 184 in TDX quote v4;
          # non-zero means attestation worked.
          for attempt in $(seq 1 60); do
            BODY=$(curl --max-time 10 -s -w '\n%{http_code}' \
              "${AGENT_URL}/health" || echo $'\n000')
            CODE=$(echo "$BODY" | tail -n1)
            JSON=$(echo "$BODY" | sed '$d')
            if [ "$CODE" = "200" ]; then
              ITA_MODE=$(echo "$JSON" | jq -r '.ita_mode // "intel"')
              if [ "$ITA_MODE" = "local" ]; then
                echo "NEW VM healthy in local attestation mode — skipping Intel TDX quote check"
                exit 0
              fi
              QUOTE_B64=$(echo "$JSON" | jq -r '.noise.quote_b64 // empty')
              if [ -n "$QUOTE_B64" ] && [ "$QUOTE_B64" != "null" ]; then
                MRTD=$(echo "$QUOTE_B64" | base64 -d \
                  | dd bs=1 skip=184 count=48 status=none | xxd -p -c 48)
                if [ -n "$MRTD" ] && [ "$MRTD" != "$(printf '00%.0s' {1..48})" ]; then
                  PUBKEY=$(echo "$JSON" | jq -r '.noise.pubkey_hex // empty')
                  echo "NEW VM verified — MRTD: $MRTD, noise_pubkey: $PUBKEY"
                  exit 0
                fi
                echo "  /health .noise 200 but MRTD empty/zero, retrying... (${attempt}/60)"
              else
                echo "  /health 200 but no .noise.quote_b64, retrying... (${attempt}/60)"
              fi
            else
              echo "  /health returned HTTP ${CODE}, retrying... (${attempt}/60)"
            fi
            sleep 10
          done
          echo "::error::/health never returned a valid .noise.quote_b64 — stale tunnel or new VM never came up"
          exit 1

      - name: Verify dashboard requires DD GitHub auth
        env:
          AGENT_URL: https://${{ inputs.hostname }}
          DD_AUTH_BROKER_URL: ${{ env.DD_AUTH_BROKER_URL }}
        run: |
          # The root is routed through Cloudflare, but DD owns browser
          # auth in-app. An unauthenticated browserless curl should
          # redirect to the DD auth broker, not serve the dashboard.
          set +e
          for attempt in $(seq 1 12); do
            out=$(curl -s -o /dev/null -w '%{http_code}|%{redirect_url}' "${AGENT_URL}/")
            code=${out%%|*}
            location=${out#*|}
            if [[ "$code" =~ ^30[1278]$ ]] && [[ "$location" == "$DD_AUTH_BROKER_URL/auth/github/start"* ]]; then
              echo "Dashboard gated by DD GitHub auth (HTTP ${code} → ${location}, attempt ${attempt})"
              exit 0
            fi
            if [ "$code" = "401" ]; then
              echo "Dashboard gated by DD GitHub auth (HTTP 401, attempt ${attempt})"
              exit 0
            fi
            echo "  dashboard returned HTTP ${code} (location=${location:-none}), retrying... (${attempt}/12)"
            sleep 5
          done
          echo "::error::dashboard / never returned a DD auth gate response"
          exit 1

      - name: Comment preview URL on PR
        if: inputs.comment_on_pr && github.event_name == 'pull_request'
        uses: actions/github-script@v7
        with:
          script: |
            const url = `https://${{ inputs.hostname }}`;
            const body = [
              `### DD preview ready`,
              ``,
              `**URL:** ${url}`,
              ``,
              `Browser login: visit ${url} — DD redirects you to`,
              `the GitHub App auth broker. A DD session cookie scoped`,
              `to \`.devopsdefender.com\` lets the preview, fleet, and`,
              `shell hosts share the same login.`,
              ``,
              `Machine-to-machine: GitHub Actions workflows in the`,
              `DD_OWNER org pass their per-job OIDC JWT as`,
              `\`Authorization: Bearer …\` (audience \`dd-agent\`).`,
              ``,
              `Register endpoint for a local agent: \`${url}/register\``,
              `(authenticated by ITA attestation).`,
            ].join('\n');
            const { data: comments } = await github.rest.issues.listComments({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
            });
            const marker = '### DD preview ready';
            const existing = comments.find(c => c.user.type === 'Bot' && c.body && c.body.includes(marker));
            if (existing) {
              await github.rest.issues.updateComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                comment_id: existing.id,
                body,
              });
            } else {
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: context.issue.number,
                body,
              });
            }

      # Cascade a relaunch of the matching dd-local-{env} libvirt domain
      # on the tdx2 host, then block on it registering with the freshly-
      # deployed CP. This is the gate: a release is "done" only when the
      # local agent is back online talking to the new CP.
      - name: Relaunch dd-local-${{ inputs.env == 'production' && 'prod' || 'preview' }}
        if: inputs.relaunch_agent
        timeout-minutes: 15
        uses: ./.github/actions/relaunch-agent
        with:
          kind: ${{ inputs.env == 'production' && 'prod' || 'preview' }}
          url: https://${{ inputs.hostname }}
          ref: ${{ inputs.ref }}
          host: ${{ inputs.env == 'production' && vars.BAREMETAL_PRODUCTION_HOST || vars.BAREMETAL_STAGING_HOST }}
          ssh-user: ${{ inputs.env == 'production' && vars.BAREMETAL_PRODUCTION_USER || vars.BAREMETAL_STAGING_USER }}
          repo-root: ${{ format('/home/{0}/src/dd', inputs.env == 'production' && vars.BAREMETAL_PRODUCTION_USER || vars.BAREMETAL_STAGING_USER) }}
          ssh-key: ${{ secrets.DD_LOCAL_SSH_KEY }}
          ita-api-key: ${{ secrets.DD_ITA_API_KEY }}
          auth-broker-url: ${{ env.DD_AUTH_BROKER_URL }}
          auth-cookie-domain: ${{ env.DD_AUTH_COOKIE_DOMAIN }}
          auth-cookie-secret: ${{ secrets.DD_AUTH_COOKIE_SECRET }}
          release-tag: ${{ inputs.release_tag }}
          ee-channel: ${{ steps.ee.outputs.channel }}
          ee-tag: ${{ inputs.ee_tag }}
          owner: ${{ inputs.owner }}
          # Long-lived PAT, not run-scoped github.token (boot-time asset
          # fetch must survive reboots). See DD_BOOT_GITHUB_TOKEN above.
          github-token: ${{ secrets.DD_MGMT_PAT }}

      # Preview only: deploy hello-world as the end-to-end canary.
      # Exercises the full GH-OIDC auth path (mint → agent verify
      # against GitHub JWKS → repository_owner check) AND proves
      # podman bootstrapped correctly on the agent.
      - name: Deploy hello-world via GH OIDC
        if: inputs.relaunch_agent && inputs.env != 'production'
        uses: ./.github/actions/dd-deploy
        with:
          cp-url: https://${{ inputs.hostname }}
          vm-name: dd-local-preview
          workload: apps/hello-world/workload.json

      # Runs last so the relaunch cascade's own STONITH wave (old agent
      # re-registers → old CP's CF tunnel is deleted → old CP poweroffs)
      # is captured by this verification, in addition to the kill that
      # happens when the new CP first registers its own tunnel. Also
      # keeps the slowest + flakiest verify (24×5s loop + fallback
      # force-delete) behind the user-facing outputs (PR comment,
      # relaunched local agent).
      #
      # target=ssh skips this step entirely — the CP runs on tdx2
      # libvirt, there's no GCE instance to verify. Orphans from a
      # historical gcp-target deploy are cleaned up via the separate
      # `force-cleanup-tunnels.yml` workflow.
      - name: Verify STONITH halted prior VM(s) in this env
        if: inputs.target == 'gcp'
        env:
          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
        run: |
          # dd-register STONITHs the old VM on startup by deleting its
          # CF tunnel → old cloudflared exits → old dd-register poweroffs.
          # Scoped to this env — per-PR previews are hostname-isolated,
          # so this only reaps prior deploys of the same env.
          NEW_VM=$(gcloud compute instances list \
            --project="$GCP_PROJECT_ID" \
            --filter="labels.devopsdefender=managed AND labels.dd_env=${DD_ENV}" \
            --format="value(name)" --sort-by=~creationTimestamp | head -1)
          echo "new VM: $NEW_VM"
          SURVIVORS=""
          for i in $(seq 1 24); do
            SURVIVORS=$(gcloud compute instances list \
              --project="$GCP_PROJECT_ID" \
              --filter="labels.devopsdefender=managed AND labels.dd_env=${DD_ENV} AND status=RUNNING" \
              --format="value(name)" \
              | grep -vx "$NEW_VM" || true)
            if [ -z "$SURVIVORS" ]; then
              echo "STONITH verified — only $NEW_VM running in ${DD_ENV}"
              exit 0
            fi
            echo "  still running besides $NEW_VM: $(echo "$SURVIVORS" | tr '\n' ' ')"
            echo "  waiting for STONITH poweroff... (${i}/24)"
            sleep 5
          done
          echo "::warning::STONITH-by-tunnel-delete timed out; force-deleting zombies:"
          echo "$SURVIVORS"
          # shellcheck disable=SC2086
          gcloud compute instances delete $SURVIVORS \
            --project="$GCP_PROJECT_ID" --zone="$GCP_ZONE" --quiet || true
          echo "zombies reaped; $NEW_VM is the only ${DD_ENV} VM"