cloud-agent-e2e-tests.yml

name: Cloud Agent E2E tests

on:
  workflow_dispatch:
    inputs:
      worker_url:
        description: 'HTTPS URL of the deployed cloud-agent-next Worker'
        required: true
        type: string
        default: 'https://cloud-agent-e2e-test.engineering-e11.workers.dev'
      fake_llm_url:
        description: 'HTTPS URL of the deployed fake-LLM Worker'
        required: true
        type: string
        default: 'https://fake-llm.engineering-e11.workers.dev'
      backend_url:
        description: 'HTTPS URL of the Kilo backend that issues stream tickets'
        required: true
        type: string
        default: 'https://api.kilo.ai'

permissions:
  contents: read

concurrency:
  group: cloud-agent-e2e-tests
  cancel-in-progress: false

jobs:
  e2e:
    outputs:
      failed: ${{ steps.summarize.outputs.failed }}
      summary: ${{ steps.summarize.outputs.summary }}
    runs-on: ubuntu-latest
    # Operational ceiling for one whole run, not a registry-derived bound.
    timeout-minutes: 180
    env:
      # Dispatch input overrides; defaults point at the deployed e2e Worker.
      WORKER_URL: ${{ inputs.worker_url || 'https://cloud-agent-e2e-test.engineering-e11.workers.dev' }}
      FAKE_LLM_URL: ${{ inputs.fake_llm_url || 'https://fake-llm.engineering-e11.workers.dev' }}
      E2E_BACKEND_URL: ${{ inputs.backend_url || 'https://api.kilo.ai' }}
      # Secrets are exposed as environment variables only; never echoed.
      E2E_USER_TOKEN: ${{ secrets.E2E_USER_TOKEN }}
      FAKE_LLM_ADMIN_TOKEN: ${{ secrets.FAKE_LLM_ADMIN_TOKEN }}
      E2E_INTERNAL_API_SECRET: ${{ secrets.E2E_INTERNAL_API_SECRET }}
      # The harness's existing DEFAULT_PARALLEL; the runner owns concurrency.
      E2E_PARALLEL: 4
    steps:
      - uses: useblacksmith/checkout@41cdeedae8edb2e684ba22896a5fd2a3cb85db6b # v1
        with:
          lfs: true

      - name: Setup pnpm
        uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0

      - name: Setup Node
        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
        with:
          node-version-file: '.nvmrc'
          cache: 'pnpm'

      - name: Install dependencies
        run: pnpm install --frozen-lockfile

      - name: Run deployed e2e
        run: |
          set -o pipefail
          pnpm --filter cloud-agent-next run e2e:parallel 2>&1 | tee e2e.log

      - name: Summarize results
        id: summarize
        if: always()
        run: |
          # The harness always ends with `Summary: <p> passed, <f> failed, <u> unsupported`.
          summary="$(grep -E '^Summary:' e2e.log | tail -n 1 || true)"
          failed="$(printf '%s\n' "$summary" | grep -oE '[0-9]+ failed' | grep -oE '[0-9]+' || true)"
          if [ -z "$failed" ]; then
            # No summary line (crash/timeout): fall back to the per-scenario failure marker.
            failed="$(grep -c '❌' e2e.log || true)"
          fi
          echo "summary=${summary#Summary: }" >> "$GITHUB_OUTPUT"
          echo "failed=${failed:-unknown}" >> "$GITHUB_OUTPUT"

      - name: Upload log
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: e2e-log
          path: e2e.log
          retention-days: 7

  notify-failure:
    needs: e2e
    if: always() && github.ref == 'refs/heads/main' && contains(join(needs.*.result, ','), 'failure')
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - name: Notify Slack
        continue-on-error: true
        uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1
        with:
          webhook: ${{ secrets.DEPLOY_NOTIFY_SLACK_WEBHOOK_URL }}
          webhook-type: incoming-webhook
          payload: |
            {
              "blocks": [
                {
                  "type": "header",
                  "text": {
                    "type": "plain_text",
                    "text": ":red_circle: Cloud Agent E2E failed on main"
                  }
                },
                {
                  "type": "section",
                  "fields": [
                    {
                      "type": "mrkdwn",
                      "text": "*Commit:*\n<${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|`${{ github.sha }}`>"
                    },
                    {
                      "type": "mrkdwn",
                      "text": "*Author:*\n${{ github.actor }}"
                    },
                    {
                      "type": "mrkdwn",
                      "text": "*Failed tests:*\n${{ needs.e2e.outputs.failed || 'n/a' }}"
                    },
                    {
                      "type": "mrkdwn",
                      "text": "*Summary:*\n${{ needs.e2e.outputs.summary || 'summary unavailable' }}"
                    }
                  ]
                },
                {
                  "type": "context",
                  "elements": [
                    {
                      "type": "mrkdwn",
                      "text": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View workflow run>"
                    }
                  ]
                }
              ]
            }