deploy-test.yml

name: Deploy test (CFN create + destroy)

on:
  push:
    branches: [ master ]
    paths:
      - 'deploy/aws/template.yaml'
      - 'modules/**'
      - 'scripts/ws_smoke.py'
      - '.github/workflows/deploy-test.yml'
  workflow_dispatch:
    inputs:
      region:
        # The test account's SCP only allows us-west-2 — other choices are kept
        # for forks running against their own account.
        description: Region to deploy into (must be in the template's RegionMap)
        type: choice
        options:
          - us-west-2
          - us-east-1
          - eu-central-1
          - eu-west-1
        default: us-west-2
      instance_type:
        # Must be one of the template's annotated AllowedValues (comma-free,
        # so the ParameterKey=...,ParameterValue=... shorthand still parses).
        description: EC2 instance type (template AllowedValues form)
        type: string
        default: t4g.small (2 vCPU / 2 GiB)
      destroy:
        description: Delete the stack at the end (uncheck to keep it up for debugging)
        type: boolean
        default: true
      web_password:
        description: >-
          Plaintext WebPassword override. Empty = random hex. Only its
          argon2id hash (issue #25) reaches the stack's WebPasswordHash
          parameter; the plaintext still drives this workflow's own login
          smoke tests and WILL appear in the run log, so throwaway values
          only.
        type: string
        default: ''
      use_spot:
        description: Deploy the ipv4-full leg as a Spot instance (the 1-click default is on-demand; this opts the leg into Spot instead)
        type: boolean
        default: false

permissions:
  id-token: write
  contents: read

concurrency:
  group: deploy-test-${{ inputs.region || 'us-west-2' }}
  cancel-in-progress: false

jobs:
  deploy-test:
    runs-on: ubuntu-latest
    # Two legs run in PARALLEL:
    #  - ipv4-full: GH runners are IPv4-only, so this is the only leg that can
    #    actually reach the box; it forces PublicIpv4=true and runs the full
    #    connectivity smoke tests. On-demand (UseSpot=false): a Spot reclaim
    #    mid-test reads as a black-holed EIP, indistinguishable from a real
    #    provisioning bug (two back-to-back runs on 2026-07-07).
    #  - ipv6-outputs: forces PublicIpv4=false, exercising the IPv6-only opt-out
    #    from the template's PublicIpv4=true default. The runner can't
    #    connect over IPv6, so this leg only asserts the stack reaches
    #    CREATE_COMPLETE and its outputs populate. That still catches template
    #    resolution bugs (e.g. the old EIPAssoc dependency) and empty-GetAtt bugs
    #    (e.g. a blank PrimaryIpv6Address -> "https://.sslip.io/").
    #    Re-enabled with SSM diagnostics (issue 190): the leg has failed first
    #    boot on every run since the 26.05 AMI landed (amazon-init dies before
    #    switch-to-configuration, journal-only, no signal on the console), and
    #    it was disabled outright rather than kept red at ~39 min a run with
    #    zero diagnostic value. EnableSsm no longer rides public_ipv4 (below),
    #    so this leg now gets a root shell path even though amazon-init never
    #    switches — the stock AMI's amazon-ssm-agent runs unconditionally
    #    pre-switch, it only needed the instance profile. `continue-on-error`
    #    keeps this leg's still-expected-until-190-is-diagnosed failure from
    #    reddening the workflow conclusion (and hiding the next real
    #    regression on ipv4-full) while it still runs and still captures
    #    evidence every push. Drop continue-on-error once 190 is fixed.
    strategy:
      fail-fast: false
      matrix:
        include:
          - leg: ipv4-full
            public_ipv4: "true"
            use_spot: "false"
          - leg: ipv6-outputs
            public_ipv4: "false"
            use_spot: "false"
    # ipv6-outputs is expected to fail until issue 190 is diagnosed and fixed
    # (see the strategy comment above) — don't let its known-red leg flip the
    # overall workflow conclusion and mask a real ipv4-full regression.
    continue-on-error: ${{ matrix.leg == 'ipv6-outputs' }}
    # Fork? Override with a repo-scoped var AGENT_BOX_ENVIRONMENT, or edit the fallback.
    environment: ${{ vars.AGENT_BOX_ENVIRONMENT || 'defang-agent-box' }}
    # Room for one FirstBootDone-timeout retry (issue 112): two create waits
    # (the WaitCondition alone is 15 min) + a delete + the post-create asserts.
    timeout-minutes: 60
    env:
      AWS_REGION: ${{ inputs.region || 'us-west-2' }}
      ROLE_ARN: ${{ vars.AWS_ROLE_ARN }}
      STACK_NAME: agent-box-e2e-${{ matrix.leg }}-${{ github.run_id }}
    steps:
      - name: Sanity-check required variables
        run: |
          if [ -z "$ROLE_ARN" ]; then
            echo "::error::Missing repo variable: AWS_ROLE_ARN. See deploy/aws/README.md."
            exit 1
          fi

      - uses: actions/checkout@v5

      - name: Configure AWS credentials via OIDC
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ env.ROLE_ARN }}
          aws-region: ${{ env.AWS_REGION }}

      - name: Generate WebPassword
        id: pw
        run: |
          # 48 hex chars by default. Dispatch runs may override the plaintext
          # to reproduce shape-specific auth bugs; the override is
          # intentionally not masked. Only the HASH (issue #25) goes to the
          # stack's WebPasswordHash parameter below - the plaintext never
          # reaches CloudFormation, exactly like the launch page's own
          # client-side generator - but this workflow still needs the
          # plaintext itself for its later login smoke tests, so both outputs
          # are kept. Caddy's own image computes the hash so this step needs
          # no new toolchain beyond the Docker the runner already has.
          if [ -n '${{ inputs.web_password }}' ]; then
            pw='${{ inputs.web_password }}'
            echo "::notice::Using dispatch-provided WebPassword override."
          else
            pw="$(openssl rand -hex 24)"
          fi
          echo "value=$pw" >> "$GITHUB_OUTPUT"
          hash="$(printf '%s\n' "$pw" \
            | docker run --rm -i caddy:2 caddy hash-password --algorithm argon2id)"
          echo "hash=$hash" >> "$GITHUB_OUTPUT"

      - name: Compute module pin (AgentBoxRev + AgentBoxSha256)
        id: pin
        # The source template intentionally has no Default: for these — see
        # publish-template.yml for the same computation on the S3 path. Here
        # we pin to the commit that triggered this run so deploy-test always
        # exercises the module content that just landed.
        run: |
          rev="${{ github.sha }}"
          sha="sha256-$(curl -sSfL "https://raw.githubusercontent.com/${{ github.repository }}/${rev}/modules/agent-box.nix" \
            | openssl dgst -sha256 -binary | base64)"
          echo "rev=$rev" >> "$GITHUB_OUTPUT"
          echo "sha=$sha" >> "$GITHUB_OUTPUT"
          echo "::notice::Pinned AgentBoxRev=$rev AgentBoxSha256=$sha"

      - name: Create stack
        id: create
        env:
          # A hash-shaped value contains '$' and ',' (issue #25) - GitHub's
          # ${{ }} substitution is a textual paste into this script, so
          # interpolating it directly would hand bash a double-quoted string
          # for it to re-parse, and bash treats a paste like "$m=..." as its
          # own variable expansion. Routing it through the environment
          # instead means bash's normal "$VAR" expansion supplies the value
          # verbatim, with nothing in it re-scanned as shell syntax.
          #
          # The commas still have to be dealt with below: aws-cli's shorthand
          # syntax for --parameters splits each list item on ',' to find the
          # next ParameterKey=/ParameterValue= pair, and an argon2id hash's
          # own field separators ("$m=47104,t=1,p=1$...") are exactly that
          # character. Unescaped, aws-cli reads "t=1" and "p=1" as their own
          # parameter structures and rejects them ("Unknown parameter ... t").
          WEB_PASSWORD_HASH: ${{ steps.pw.outputs.hash }}
        run: |
          # PublicIpv4/UseSpot come from the matrix leg (see the job comment).
          # EnableSsm is now unconditionally true on BOTH legs (issue 190):
          # the stock AMI's amazon-ssm-agent runs unconditionally pre-switch
          # (deploy/aws/template.yaml's EnableSsm description), so attaching the
          # instance profile gives a root-shell diagnostic path even on a box
          # whose amazon-init dies before switch-to-configuration ever runs —
          # previously only ipv4-full got this, leaving ipv6-outputs with no
          # way to read its (journal-only) amazon-init failure at all.
          create() {
            # Escape commas the shorthand parser would otherwise treat as
            # separators between parameter structures (see the env comment
            # above) - a literal comma survives as part of the value once
            # backslash-escaped.
            web_password_hash_escaped="${WEB_PASSWORD_HASH//,/\\,}"
            aws cloudformation create-stack \
              --stack-name "$STACK_NAME" \
              --template-body "file://deploy/aws/template.yaml" \
              --parameters \
                "ParameterKey=PublicIpv4,ParameterValue=${{ matrix.public_ipv4 }}" \
                "ParameterKey=UseSpot,ParameterValue=${{ (inputs.use_spot == true && matrix.public_ipv4 == 'true') && 'true' || matrix.use_spot }}" \
                "ParameterKey=WebPasswordHash,ParameterValue=${web_password_hash_escaped}" \
                "ParameterKey=InstanceType,ParameterValue=${{ inputs.instance_type || 't4g.small (2 vCPU / 2 GiB)' }}" \
                "ParameterKey=SeedMainSession,ParameterValue=true" \
                "ParameterKey=EnableSsm,ParameterValue=true" \
                "ParameterKey=AgentBoxRev,ParameterValue=${{ steps.pin.outputs.rev }}" \
                "ParameterKey=AgentBoxSha256,ParameterValue=${{ steps.pin.outputs.sha }}" \
              --on-failure DO_NOTHING \
              --capabilities CAPABILITY_IAM \
              --tags "Key=Purpose,Value=agent-box-e2e" "Key=RunId,Value=${{ github.run_id }}"
            echo "::notice::Waiting for stack ${STACK_NAME} to reach CREATE_COMPLETE..."
            aws cloudformation wait stack-create-complete --stack-name "$STACK_NAME"
          }

          # Pull whatever a FAILED-to-provision instance can still tell us,
          # BEFORE the stack (and the instance) is deleted — this evidence is
          # otherwise destroyed. get-console-output only ever returns amazon-
          # init's terminal "Failed to start ..." line (it logs journal-only,
          # deploy/aws/template.yaml:456,601), so the pre-switch journal itself —
          # where an IPv6-only DNS/resolver failure would actually show up —
          # is only reachable via SSM. EnableSsm=true above means the agent
          # (already running pre-switch on the stock AMI) has a chance to
          # register even though amazon-init never got to switch-to-
          # configuration; best-effort throughout, since a box that also
          # can't reach SSM's endpoints leaves us with the console dump alone.
          dump_diagnostics() {
            local iid="$1" label="$2"
            if [ -z "$iid" ] || [ "$iid" = "None" ]; then
              echo "::warning::No instance id for the $label attempt; nothing to dump."
              return 0
            fi
            echo "::group::Serial console — $label attempt ($iid)"
            aws ec2 get-console-output --instance-id "$iid" --latest \
              --query Output --output text || true
            echo "::endgroup::"
            cid=$(aws ssm send-command --instance-ids "$iid" \
              --document-name AWS-RunShellScript \
              --parameters 'commands=[
                "systemctl status amazon-init --no-pager -l 2>&1 || true",
                "echo ---amazon-init-journal---; journalctl -u amazon-init --no-pager 2>&1 || true",
                "echo ---resolvectl---; resolvectl status 2>&1 || true; cat /etc/resolv.conf 2>&1 || true",
                "echo ---networkd-journal---; journalctl -u systemd-networkd -u systemd-resolved --no-pager 2>&1 | tail -100 || true",
                "echo ---live-fetch-probe---; curl -6 -sSv --max-time 10 -o /dev/null https://raw.githubusercontent.com/ 2>&1 || true"
              ]' \
              --query Command.CommandId --output text 2>/dev/null) || {
                echo "::warning::SSM send-command failed for the $label attempt (agent never registered, or missing IAM perms) - console dump above is all we have."
                return 0
              }
            status=Pending
            for i in $(seq 1 24); do
              status=$(aws ssm get-command-invocation --command-id "$cid" --instance-id "$iid" \
                --query Status --output text 2>/dev/null || echo Pending)
              case "$status" in Success|Failed|Cancelled|TimedOut) break;; esac
              sleep 5
            done
            echo "::group::SSM pre-switch journal — $label attempt ($iid, status: $status)"
            aws ssm get-command-invocation --command-id "$cid" --instance-id "$iid" \
              --query StandardOutputContent --output text 2>/dev/null || true
            aws ssm get-command-invocation --command-id "$cid" --instance-id "$iid" \
              --query StandardErrorContent --output text 2>/dev/null || true
            echo "::endgroup::"
          }

          if create; then exit 0; fi
          first_iid=$(aws cloudformation describe-stack-resources --stack-name "$STACK_NAME" \
            --logical-resource-id Instance \
            --query 'StackResources[0].PhysicalResourceId' --output text 2>/dev/null || true)
          dump_diagnostics "$first_iid" "first"

          # Retry ONCE, and only for the known-intermittent first-boot flake
          # (issue 112): the box occasionally comes up unprovisioned (issue 84,
          # fetch-ec2-metadata failing open), so FirstBootDone never signals
          # and the WaitCondition times out. Any other failure (template bug,
          # quota, IAM) stays a hard failure — retrying those would only mask
          # them.
          reason=$(aws cloudformation describe-stack-events --stack-name "$STACK_NAME" \
            --query 'StackEvents[?LogicalResourceId==`FirstBootDone` && ResourceStatus==`CREATE_FAILED`].ResourceStatusReason | [0]' \
            --output text 2>/dev/null || true)
          case "$reason" in
            *"WaitCondition timed out"*) ;;
            *) echo "::error::Stack creation failed for a reason other than the FirstBootDone timeout; not retrying."; exit 1 ;;
          esac
          # Persistent Spot requests can re-open and snag stack deletion (same
          # hazard the Destroy step guards); cancel best-effort here too so a
          # Spot dispatch run can't wedge the retry.
          if [ -n "$first_iid" ] && [ "$first_iid" != "None" ]; then
            sirs=$(aws ec2 describe-spot-instance-requests \
              --filters "Name=instance-id,Values=$first_iid" \
              --query 'SpotInstanceRequests[].SpotInstanceRequestId' \
              --output text 2>/dev/null || true)
            if [ -n "$sirs" ]; then
              aws ec2 cancel-spot-instance-requests --spot-instance-request-ids $sirs || true
            fi
          fi
          echo "::warning::FirstBootDone timed out (issue 112 flake, or the issue-190 ipv6-outputs first-boot failure); deleting the failed stack and retrying once."
          aws cloudformation delete-stack --stack-name "$STACK_NAME"
          aws cloudformation wait stack-delete-complete --stack-name "$STACK_NAME"

          # Capture the SAME diagnostics for the retry attempt before failing
          # for good — this is the leg's LAST chance to see anything before
          # "Dump failed stack events" (CFN event reasons only, no
          # console/journal) is all that's left.
          if create; then exit 0; fi
          second_iid=$(aws cloudformation describe-stack-resources --stack-name "$STACK_NAME" \
            --logical-resource-id Instance \
            --query 'StackResources[0].PhysicalResourceId' --output text 2>/dev/null || true)
          dump_diagnostics "$second_iid" "second (final)"
          exit 1

      - name: Dump failed stack events
        if: failure() && steps.create.outcome == 'failure'
        run: |
          echo "::group::Stack events (CREATE_FAILED only)"
          aws cloudformation describe-stack-events --stack-name "$STACK_NAME" \
            --query 'reverse(StackEvents[?ResourceStatus==`CREATE_FAILED`].[LogicalResourceId,ResourceStatusReason])' \
            --output table || true
          echo "::endgroup::"
          echo "::group::Stack status"
          aws cloudformation describe-stacks --stack-name "$STACK_NAME" \
            --query 'Stacks[0].[StackStatus,StackStatusReason]' --output table || true
          echo "::endgroup::"

      - name: Fetch outputs
        id: outputs
        run: |
          web_url=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" \
            --query 'Stacks[0].Outputs[?OutputKey==`WebURL`].OutputValue | [0]' --output text)
          public_ip=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" \
            --query 'Stacks[0].Outputs[?OutputKey==`PublicAddress`].OutputValue | [0]' --output text)
          instance_id=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" \
            --query 'Stacks[0].Outputs[?OutputKey==`InstanceId`].OutputValue | [0]' --output text)
          echo "web_url=$web_url"   >> "$GITHUB_OUTPUT"
          echo "public_ip=$public_ip" >> "$GITHUB_OUTPUT"
          echo "instance_id=$instance_id" >> "$GITHUB_OUTPUT"
          echo "::notice::Stack up. WebURL=$web_url  PublicIP=$public_ip  InstanceId=$instance_id"

      - name: Assert outputs are populated
        run: |
          # Runs on BOTH legs. This is the whole point of the ipv6-outputs leg,
          # which can't connect over IPv6 to run the smoke tests below.
          ip='${{ steps.outputs.outputs.public_ip }}'
          url='${{ steps.outputs.outputs.web_url }}'
          fail=0
          if [ -z "$ip" ] || [ "$ip" = "None" ]; then
            echo "::error::PublicAddress output is empty."; fail=1
          fi
          # The host must be present. Catches a blank PrimaryIpv6Address, which
          # renders WebURL as https://.sslip.io/. Also reject any userinfo in the
          # authority: Chrome answers the auth challenge with URL userinfo + an
          # EMPTY password, and typed credentials can't override it (issue 56).
          rest=${url#https://}
          authority=${rest%%/*}
          case "$authority" in *@*)
            echo "::error::WebURL must not contain userinfo (user@): $url"; fail=1 ;;
          esac
          host=${authority%.sslip.io}
          if [ -z "$host" ]; then
            echo "::error::WebURL host is blank: $url"; fail=1
          fi
          if [ "$fail" = 0 ]; then
            echo "::notice::Outputs OK - PublicAddress=$ip host=$host"
          fi
          exit $fail

      - name: Assert amazon-init provisioned the box (serial console)
        # Runs on BOTH legs, including IPv6-only where the runner can't connect
        # over the network. CFN CREATE_COMPLETE does NOT wait for amazon-init
        # (there's no CreationPolicy), so we confirm first-boot provisioning by
        # reading the serial console. systemd logs the amazon-init unit either by
        # its Description ("Finished ... Reconfigure the system from EC2 userdata
        # on startup") or by its unit id ("Started amazon-init.service") depending
        # on the boot - match both. Failure shows "Failed to start ..." or the
        # unit reporting a non-zero exit. THIS is the check that catches the
        # IPv6-only module-fetch failure the ipv4-full leg structurally cannot
        # (that leg has a public IPv4 and reaches IPv4-only hosts directly).
        run: |
          iid='${{ steps.outputs.outputs.instance_id }}'
          if [ -z "$iid" ] || [ "$iid" = "None" ]; then
            echo "::error::No InstanceId output to inspect."; exit 1
          fi
          # Two spellings of "Failed to start", and only one of them is news
          # (issue 186). systemd names a unit by its DESCRIPTION while the unit
          # is still loaded, and by its UNIT ID once it is not — and
          # amazon-init masks itself in the very configuration it is switching
          # into (deploy/aws/template.yaml: `systemd.services.amazon-init.enable =
          # lib.mkForce false`, so a rebooted box does not re-apply user-data).
          # So a HEALTHY first boot ends with its own start job cancelled and
          # the console reading "[FAILED] Failed to start amazon-init.service."
          # — the unit-id form. Treating that as fatal failed a box that had
          # provisioned perfectly, twice on master on 2026-08-24 alone, while
          # its stack signalled FirstBootDone, caddy served a real Let's
          # Encrypt cert and ttyd was listening.
          unit_desc='Reconfigure the system from EC2 userdata'
          fail_re="(Failed to start.*$unit_desc|amazon-init\.service: (Failed|Main process exited))"
          # The benign form, kept only to explain a timeout: a box that masks
          # amazon-init and THEN fails activation still goes red, because
          # nothing below will match ok_re before the poll runs out.
          masked_re='Failed to start.*amazon-init\.service'
          # Success needs a marker a vanilla AMI cannot print. "Finished
          # amazon-init" never appears on a mask-mid-switch boot, so the
          # activation itself is the witness: amazon-init runs
          # switch-to-configuration through systemd-run, and only a box being
          # provisioned from user-data does that.
          switched_re='switch-to-configuration switch'
          ok_re="$switched_re"
          # The stock NixOS AMI's fetch-ec2-metadata.service fails OPEN: an
          # IMDS hiccup logs a complaint, writes no user-data file, and exits
          # 0 — then amazon-init "finishes" having provisioned nothing, so
          # ok_re matches on a vanilla, dead box (issue 84). The fetcher logs
          # with StandardOutput=journal+console, so its terminal give-up line
          # is visible on the serial console. Only that unambiguous line is
          # matched: the fetcher's individual curl error lines also fire on
          # healthy boots (recovered token retries; 404s on optional paths
          # like public-keys), so matching them would false-positive. Checked
          # BEFORE ok_re — the fetcher runs before amazon-init, so on a bad
          # boot the marker is already present when "Finished amazon-init"
          # appears. (A token-OK boot whose single-shot user-data GET fails
          # still slips through to the slow smoke-test path; the WaitCondition
          # proposed in issue 84 is the complete fix.)
          unprov_re='failed to fetch an IMDSv2 token'
          # nixos-rebuild during amazon-init can take a few minutes; console
          # output on Nitro instances is near-real-time. Poll up to ~12 min.
          # get-console-output --latest returns the tail 64 KB. On a chatty
          # boot the actual nix eval/build error can scroll off before we see
          # the "Failed to start" line — so keep the FIRST snapshot around and
          # dump both on failure to give diagnosis the biggest possible window.
          first_out=""
          masked_seen=""
          for i in $(seq 1 48); do
            out=$(aws ec2 get-console-output --instance-id "$iid" --latest \
              --query Output --output text 2>/dev/null || true)
            if [ -z "$first_out" ] && [ -n "$out" ]; then
              first_out="$out"
            fi
            if printf '%s' "$out" | grep -qE "$masked_re"; then
              masked_seen=1
            fi
            if printf '%s' "$out" | grep -qE "$fail_re"; then
              echo "::error::amazon-init FAILED - box did not provision (module fetch / nixos-rebuild)."
              echo "::group::Serial console — LATEST snapshot (contains the fail line + surrounding context)"
              printf '%s\n' "$out"
              echo "::endgroup::"
              echo "::group::Serial console — FIRST snapshot (earliest boot output we captured; likely holds the root-cause nix error)"
              printf '%s\n' "$first_out"
              echo "::endgroup::"
              exit 1
            fi
            if printf '%s' "$out" | grep -qE "$unprov_re"; then
              echo "::error::fetch-ec2-metadata could not read IMDS - user-data was never fetched, so amazon-init provisioned NOTHING (stock-AMI fail-open, issue 84). The box is a vanilla NixOS AMI; the smoke test would burn 10 min on curl 000."
              echo "::group::Serial console — LATEST snapshot (contains the metadata-fetch failure)"
              printf '%s\n' "$out"
              echo "::endgroup::"
              exit 1
            fi
            if printf '%s' "$out" | grep -qE "$ok_re"; then
              echo "::notice::amazon-init finished - box provisioned from user-data."
              exit 0
            fi
            echo "[$i/48] provisioning not yet confirmed from console; waiting..."
            sleep 15
          done
          if [ -n "$masked_seen" ]; then
            echo "::error::amazon-init was masked mid-switch (the benign spelling, issue 186) but activation never completed - no switch-to-configuration on the console. This box did NOT provision."
          fi
          echo "::error::Timed out waiting for amazon-init to finish."
          echo "::group::Serial console — LATEST snapshot"
          printf '%s\n' "$out"
          echo "::endgroup::"
          echo "::group::Serial console — FIRST snapshot"
          printf '%s\n' "$first_out"
          echo "::endgroup::"
          exit 1

      - name: Smoke test — wait for authenticated terminal URL to serve ttyd HTML
        if: matrix.public_ipv4 == 'true'
        env:
          WEB_PASSWORD: ${{ steps.pw.outputs.value }}
        run: |
          url='${{ steps.outputs.outputs.web_url }}'
          terminal_url="$url"
          # NixOS boot + nixos-rebuild + Caddy ACME can add 2-10 min after CFN reports done.
          for i in $(seq 1 60); do
            code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 \
              -u "workspace:${WEB_PASSWORD}" "$terminal_url" || true)
            echo "[$i/60] authenticated GET $terminal_url -> $code"
            if [ "$code" = "200" ]; then
              echo "::notice::Authenticated terminal URL is serving (HTTP 200) - Caddy + ttyd are up."
              break
            fi
            sleep 10
          done
          if [ "$code" != "200" ]; then
            echo "::error::Never got 200 from $terminal_url after 10 min."
            exit 1
          fi

      - name: Smoke test — verify unauthenticated URL returns 401
        if: matrix.public_ipv4 == 'true'
        run: |
          url='${{ steps.outputs.outputs.web_url }}'
          terminal_url="$url"
          code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 5 "$terminal_url" || true)
          if [ "$code" = "401" ]; then
            echo "::notice::Unauthenticated terminal URL returns 401 as expected."
          else
            echo "::error::Expected 401 at $terminal_url (no Basic auth), got $code."
            exit 1
          fi

      - name: Smoke test — root serves the tabbed workspace behind auth
        if: matrix.public_ipv4 == 'true'
        env:
          WEB_PASSWORD: ${{ steps.pw.outputs.value }}
        run: |
          url='${{ steps.outputs.outputs.web_url }}'
          terminal_url="$url"
          root_url="${terminal_url%workspace/}"
          # The root workspace must never be public: it iframes the terminal
          # and can create sessions.
          code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 "$root_url" || true)
          if [ "$code" != "401" ]; then
            echo "::error::Expected 401 at $root_url without credentials, got $code."
            exit 1
          fi
          # -L: a single-user box's root now redirects into that user's own
          # space (#334) rather than serving the workspace at "/" directly.
          body=$(curl -sSL --max-time 10 -u "workspace:${WEB_PASSWORD}" "$root_url" || true)
          if echo "$body" | grep -q 'href="https://[^/"]*@'; then
            echo "::error::Workspace at $root_url embeds URL userinfo (user@) in an href — Chrome auto-sends an empty password and typed credentials can't override it (issue 56):"
            echo "$body" | head -20
            exit 1
          fi
          # The tabbed workspace (issue 119): the tab bar carries the ttyd
          # base for iframe panes, and each session is a ?tab= link. The
          # iframe itself is NOT asserted — it only renders once the session's
          # tmux is live, which can lag boot.
          if ! echo "$body" | grep -qF 'data-term-base="/workspace/"'; then
            echo "::error::Workspace at $root_url isn't wired to the ttyd base /workspace/ (no data-term-base):"
            echo "$body" | head -20
            exit 1
          fi
          # Tab links point at TERM_HOME (this user's own path), not root
          # (#334's render_tabs: href="{home}?tab={name}") — so /workspace/?tab=,
          # not /?tab=.
          if echo "$body" | grep -qF 'href="/workspace/?tab='; then
            echo "::notice::Workspace tab bar lists the sessions (no credentials in the URL)."
          else
            echo "::error::Workspace at $root_url has no session tabs (href=\"/workspace/?tab=\"):"
            echo "$body" | head -20
            exit 1
          fi

      - name: Smoke test — verify ttyd HTML body
        if: matrix.public_ipv4 == 'true'
        env:
          WEB_PASSWORD: ${{ steps.pw.outputs.value }}
        run: |
          url='${{ steps.outputs.outputs.web_url }}'
          terminal_url="$url"
          body=$(curl -sS --max-time 15 -u "workspace:${WEB_PASSWORD}" "$terminal_url")
          if echo "$body" | grep -qi "ttyd\|terminal"; then
            echo "::notice::ttyd HTML served correctly."
          else
            echo "::error::Response body doesn't look like ttyd:"
            echo "$body" | head -20
            exit 1
          fi

      - name: Smoke test — WebSocket upgrade
        if: matrix.public_ipv4 == 'true'
        env:
          WEB_PASSWORD: ${{ steps.pw.outputs.value }}
        run: |
          url='${{ steps.outputs.outputs.web_url }}'
          terminal_url="$url"
          rest=${url#https://}
          authority=${rest%%/*}
          origin="https://${authority}"
          cookie_jar=$(mktemp)
          curl -sS -o /dev/null --max-time 15 -u "workspace:${WEB_PASSWORD}" -c "$cookie_jar" "$terminal_url"
          # 101 = Switching Protocols (success). Anything else means the browser
          # terminal will show 'disconnected'. Browsers send cookies on the
          # upgrade request, so this proves the Basic-auth bootstrap cookie works.
          # The prior step only proves Caddy + the workspace page are up, not
          # that ttyd's own unix socket is bound yet - that can lag a few
          # seconds behind, and Caddy answers 502 until it is, so retry like
          # the "tmux session is live" step below instead of failing on one
          # attempt.
          #
          # ttyd runs with --check-origin (issue #628): it refuses a WebSocket
          # upgrade whose Origin doesn't match the Host, which a real browser
          # always sends and curl does not unless told to. Without this header
          # ttyd logs "User code denied connection" and closes the connection,
          # which Caddy reports upstream as a 502 (agent-box#674) - it is the
          # smoke test failing to look like a browser, not ttyd or Caddy being
          # down.
          for i in $(seq 1 6); do
            code=$(curl -sS -o /dev/null -w "%{http_code}" --http1.1 --max-time 10 \
              -b "$cookie_jar" \
              -H "Connection: Upgrade" \
              -H "Upgrade: websocket" \
              -H "Sec-WebSocket-Version: 13" \
              -H "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==" \
              -H "Sec-WebSocket-Protocol: tty" \
              -H "Origin: ${origin}" \
              "${terminal_url}ws" || true)
            echo "[$i/6] WebSocket upgrade attempt -> HTTP $code"
            if [ "$code" = "101" ]; then
              echo "::notice::WebSocket upgrade succeeded (101 Switching Protocols)."
              break
            fi
            sleep 5
          done
          if [ "$code" != "101" ]; then
            echo "::error::WebSocket upgrade failed (got $code, expected 101) after 6 attempts."
            exit 1
          fi

      - name: Smoke test — tmux session is live (not just ttyd reachable)
        if: matrix.public_ipv4 == 'true'
        env:
          WEB_PASSWORD: ${{ steps.pw.outputs.value }}
        run: |
          # 101 above only proves ttyd upgrades the socket; it says nothing about
          # whether `tmux attach` found a session. Attach for real and assert the
          # terminal isn't showing tmux's "no sessions". Retries because the
          # agent's claude session can take a moment to come up after boot.
          python3 -m pip install --quiet websocket-client
          url='${{ steps.outputs.outputs.web_url }}'
          for i in $(seq 1 12); do
            if python3 scripts/ws_smoke.py "$url" "$WEB_PASSWORD"; then
              exit 0
            fi
            echo "[$i/12] no live session yet; retrying in 10s..."
            sleep 10
          done
          echo "::error::Browser terminal never attached to a live tmux session."
          exit 1

      - name: Diagnose auth state via SSM
        # When a smoke test fails, pull unit status, file listings, and
        # journals off the box before it's destroyed (issues 53/84). Runs on
        # EVERY failure that reaches this point — the failures are
        # intermittent, so dispatch-only diagnostics kept missing them.
        # EnableSsm is unconditionally true now (issue 190), so this also
        # covers the rare case where ipv6-outputs reaches CREATE_COMPLETE but
        # a later assertion (e.g. "Assert amazon-init provisioned the box")
        # fails; the far more common ipv6-outputs failure is a Create-stack
        # timeout, which never reaches this step — that path has its own SSM
        # capture inline (see the "Create stack" step's dump_diagnostics).
        # Deliberately does NOT cat secret files (hash, cookie secret, caddy
        # env) or user-data into the public log — file presence, perms, and
        # journals cover the diagnostic need.
        if: failure()
        run: |
          iid='${{ steps.outputs.outputs.instance_id }}'
          if [ -z "$iid" ] || [ "$iid" = "None" ]; then
            echo "::warning::No InstanceId; skipping SSM diagnostics."; exit 0
          fi
          cid=$(aws ssm send-command --instance-ids "$iid" \
            --document-name AWS-RunShellScript \
            --parameters 'commands=[
              "systemctl --failed --no-pager || true",
              "systemctl status caddy agent-web-auth-secrets agent-box@workspace agent-web-terminal@workspace --no-pager -l 2>&1 | head -120 || true",
              "ls -la /var/lib/agent-box-web/ /run/agent-box-web/ || true",
              "ls -la /run/agent-box-ttyd/ /run/agent-box-ttyd/workspace/ || true",
              "ls -la /etc/ec2-metadata/ || true",
              "ss -ltn || true",
              "ss -lx | grep -i ttyd || true",
              "echo ---fetch-ec2-metadata-journal---; journalctl -u fetch-ec2-metadata --no-pager 2>&1 | tail -40 || true",
              "echo ---caddy-journal---; journalctl -u caddy --no-pager 2>&1 | tail -40 || true",
              "echo ---agent-box-journal---; journalctl -u agent-box@workspace --no-pager 2>&1 | tail -40 || true",
              "echo ---ttyd-journal---; journalctl -u agent-web-terminal@workspace --no-pager 2>&1 | tail -60 || true",
              "echo ---amazon-init-journal---; journalctl -u amazon-init --no-pager 2>&1 | tail -60 || true"
            ]' \
            --query Command.CommandId --output text) || { echo "::warning::SSM send-command failed (missing IAM perms?)."; exit 0; }
          for i in $(seq 1 24); do
            status=$(aws ssm get-command-invocation --command-id "$cid" --instance-id "$iid" \
              --query Status --output text 2>/dev/null || echo Pending)
            case "$status" in Success|Failed|Cancelled|TimedOut) break;; esac
            sleep 5
          done
          echo "::group::SSM diagnostics (status: $status)"
          aws ssm get-command-invocation --command-id "$cid" --instance-id "$iid" \
            --query StandardOutputContent --output text || true
          aws ssm get-command-invocation --command-id "$cid" --instance-id "$iid" \
            --query StandardErrorContent --output text || true
          echo "::endgroup::"

      - name: Destroy stack
        # Push runs (no inputs) always destroy; dispatched runs honor inputs.destroy.
        if: ${{ always() && (github.event_name != 'workflow_dispatch' || inputs.destroy) }}
        run: |
          # Persistent Spot requests re-open when the instance is terminated and
          # can relaunch, orphaning a request and snagging stack deletion. Cancel
          # the request first (this does NOT terminate the instance; delete-stack
          # does). Best-effort so on-demand runs / missing instances don't fail.
          iid=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" \
            --query 'Stacks[0].Outputs[?OutputKey==`InstanceId`].OutputValue | [0]' \
            --output text 2>/dev/null || true)
          if [ -n "$iid" ] && [ "$iid" != "None" ]; then
            sirs=$(aws ec2 describe-spot-instance-requests \
              --filters "Name=instance-id,Values=$iid" \
              --query 'SpotInstanceRequests[].SpotInstanceRequestId' \
              --output text 2>/dev/null || true)
            if [ -n "$sirs" ]; then
              echo "Cancelling Spot request(s) for $iid: $sirs"
              aws ec2 cancel-spot-instance-requests --spot-instance-request-ids $sirs || true
            fi
          fi
          echo "Deleting $STACK_NAME..."
          aws cloudformation delete-stack --stack-name "$STACK_NAME"
          # A teardown that did not happen must not report success. `|| true`
          # here swallowed DELETE_FAILED and a timed-out wait alike, so a stack
          # that survived its own destroy step kept billing with a green job
          # and no signal anywhere. Ask what is actually true afterwards: a
          # deleted stack is no longer describable BY NAME, and anything else
          # is a stack somebody has to delete by hand.
          if ! aws cloudformation wait stack-delete-complete --stack-name "$STACK_NAME"; then
            if desc_err=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" \
              --query 'Stacks[0].StackStatus' --output text 2>&1); then
              echo "::error::$STACK_NAME did not delete (status $desc_err) - it is STILL BILLING. Delete it by hand: aws cloudformation delete-stack --stack-name $STACK_NAME --region $AWS_REGION"
              exit 1
            elif printf '%s' "$desc_err" | grep -qE 'does not exist'; then
              echo "::notice::wait returned non-zero but $STACK_NAME is gone."
            else
              echo "::error::Could not confirm deletion of $STACK_NAME - describe-stacks failed: $desc_err. It may still be billing. Check by hand: aws cloudformation describe-stacks --stack-name $STACK_NAME --region $AWS_REGION"
              exit 1
            fi
          fi

      - name: Flag kept stack
        if: ${{ always() && github.event_name == 'workflow_dispatch' && !inputs.destroy }}
        run: |
          echo "::warning::Stack '$STACK_NAME' was kept (destroy=false). It's still billing."
          echo "::warning::Delete manually: aws cloudformation delete-stack --stack-name $STACK_NAME --region $AWS_REGION"