ci.yml

name: CI

permissions:
  contents: read
  actions: read

on:
  push:
    # `train/**` is a PRECONDITION for CI-gated trains, not a convenience. Under required
    # status checks a push to master is refused unless its sha already carries a green check,
    # and a check only exists if a workflow ran on the branch that produced the sha. Enabling
    # protection while this trigger is master-only makes master unpushable with no check ever
    # arriving -- the fleet-wide shape QTA found, since nearly every seat triggers on its
    # default branch alone. Widen the trigger BEFORE turning protection on.
    branches: [master, 'train/**']
  pull_request:
  workflow_call:
  # Exercise the floating daemon/vault test rigs daily; linked crates are pinned.
  schedule:
    - cron: "0 6 * * *"

concurrency:
  group: ci-${{ github.ref }}
  cancel-in-progress: true

env:
  CARGO_TERM_COLOR: always
  # Workflow-wide, not per step: any step that runs a script importing a shared module under
  # scripts/lib/ leaves a __pycache__ there, and the clean-worktree check at the end rightly
  # refuses it. Set once here, so a new python step cannot forget it; a per-step setting
  # is how the deploy-gate self-test slipped when scripts/lib/wal_gate.py arrived.
  PYTHONDONTWRITEBYTECODE: "1"
  # Line tables keep panics and backtraces readable while cutting the debug target to a
  # fraction of its size. Full debuginfo filled a GitHub-hosted Linux runner's disk and the
  # linker died mid-build.
  CARGO_PROFILE_DEV_DEBUG: line-tables-only

jobs:
  fixtures:
    uses: ./.github/workflows/fixtures.yml

  lint:
    name: Lint (${{ matrix.os }})
    runs-on: ${{ matrix.runner }}
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: ubuntu
            runner: ubuntu-latest
          - os: windows
            runner: windows-latest
    defaults:
      run:
        working-directory: broca
        shell: bash
    steps:
      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
        with:
          persist-credentials: false
          path: broca
      - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
        with:
          components: clippy, rustfmt
      - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
        with:
          workspaces: broca -> target
          shared-key: lint
          cache-on-failure: 'true'
      - name: fmt and clippy (same features as the tests)
        run: |
          source scripts/rust-test-gate.sh
          run_phase fmt cargo fmt --all -- --check
          run_phase clippy cargo clippy --locked --workspace --all-targets "${cargo_feature_args[@]}" -- -D warnings
      - name: Verify clean worktree
        run: test -z "$(git status --porcelain)" || { git status --short; git diff; exit 1; }

  scripts:
    name: Operator scripts
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
        with:
          persist-credentials: false
      - name: Train preconditions
        run: python3 scripts/check-train-preconditions.py
      - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
      - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
        with:
          shared-key: scripts
          cache-on-failure: 'true'
      # Dependency graph suites deliberately use --offline. Fetch their pinned
      # sources here; an independent job cannot borrow another job's Cargo home.
      - name: Prefetch locked dependencies for offline script checks
        run: cargo fetch --locked
      - name: Operator script suites and deploy/export self-tests
        run: scripts/script-suites.sh
      - name: Verify clean worktree
        run: test -z "$(git status --porcelain)" || { git status --short; git diff; exit 1; }

  integration-build:
    name: Archive (${{ matrix.os }})
    runs-on: ${{ matrix.runner }}
    timeout-minutes: 45
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: ubuntu
            runner: ubuntu-latest
          - os: windows
            runner: windows-latest
    defaults:
      run:
        working-directory: broca
        shell: bash
    steps:
      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
        with:
          persist-credentials: false
          path: broca
      - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
      - uses: taiki-e/install-action@2eb6de229bc778f7df20e106e4b34e9a364c73e6 # nextest
      - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
        id: cargo-cache
        with:
          workspaces: broca -> target
          shared-key: archive
          cache-on-failure: 'true'
      - name: Compile and compare independent integration inventories
        env:
          NEXTEST_PROFILE: ci
        run: bash scripts/ci-integration.sh build "$RUNNER_TEMP/integration.tar.zst"
      # Asks nextest which tests the Linux preflight selects, so it runs here where nextest and
      # the freshly built test binaries already exist, not in the operator scripts job.
      - name: Linux preflight excludes every rig-dependent test
        if: matrix.os == 'ubuntu'
        run: PYTHONDONTWRITEBYTECODE=1 python3 scripts/test-portable-test-selection.py
      - name: Cargo cache telemetry
        run: echo "Archive cache exact hit = ${{ steps.cargo-cache.outputs.cache-hit }}" >> "$GITHUB_STEP_SUMMARY"
      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: integration-${{ matrix.os }}
          path: ${{ runner.temp }}/integration.tar.zst
          compression-level: 0
          retention-days: 3
      - name: Verify clean worktree
        run: test -z "$(git status --porcelain)" || { git status --short; git diff; exit 1; }

  integration:
    name: Integration (${{ matrix.os }} ${{ matrix.shard }}/4)
    # Prepare consumers alongside producers; each shard waits only for its own
    # platform's artifacts, not the other platform's matrix jobs or cache saves.
    runs-on: ${{ matrix.runner }}
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        shard: [1, 2, 3, 4]
        include:
          - os: ubuntu
            runner: ubuntu-latest
          - os: windows
            runner: windows-latest
        os: [ubuntu, windows]
    defaults:
      run:
        working-directory: broca
        shell: bash
    env:
      BROCA_REQUIRE_SUBC: '1'
      BROCA_REQUIRE_AIMOCK: '1'
      BROCA_REQUIRE_VAULT: ${{ matrix.os == 'ubuntu' && '1' || '' }}
      SUBC_CORE_BIN: ${{ github.workspace }}/fixtures/bin/ck-subc${{ matrix.os == 'windows' && '.exe' || '' }}
      FAKE_AFT_STUB_BIN: ${{ github.workspace }}/fixtures/bin/fake-aft-stub${{ matrix.os == 'windows' && '.exe' || '' }}
      CK_CLAUSTRUM_BIN: ${{ matrix.os == 'ubuntu' && format('{0}/fixtures/bin/ck-claustrum', github.workspace) || '' }}
      CK_AUTH_BIN: ${{ matrix.os == 'ubuntu' && format('{0}/fixtures/bin/ck-auth', github.workspace) || '' }}
      CK_CLAUSTRUM_CHECKOUT: ${{ matrix.os == 'ubuntu' && format('{0}/fixtures/claustrum', github.workspace) || '' }}
      NEXTEST_TEST_THREADS: num-cpus
    steps:
      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
        with:
          persist-credentials: false
          path: broca
      - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
      - uses: taiki-e/install-action@2eb6de229bc778f7df20e106e4b34e9a364c73e6 # nextest
      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
        with:
          node-version: 20
      - run: npm ci
      - name: Wait for this platform's fixtures and test archive
        env:
          GH_TOKEN: ${{ github.token }}
        run: python3 -B scripts/wait-ci-artifacts.py "fixtures-${{ runner.os }}" "integration-${{ matrix.os }}"
      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          name: fixtures-${{ runner.os }}
          path: ${{ runner.temp }}/fixtures
      - run: bash scripts/unpack-fixtures.sh "$RUNNER_TEMP/fixtures/fixtures-$RUNNER_OS.tar.gz" "$GITHUB_WORKSPACE"
      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          name: integration-${{ matrix.os }}
          path: ${{ runner.temp }}/archive
      - name: Replay integration shard from archive (no compilation)
        run: bash scripts/ci-integration.sh run "$RUNNER_TEMP/archive/integration.tar.zst" "${{ matrix.shard }}/4"
      - name: Upload process cleanup diagnostics
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: rig-survivors-integration-${{ matrix.os }}-${{ matrix.shard }}
          path: broca/target/rig-survivors/*.txt
          if-no-files-found: ignore
      - name: Verify clean worktree
        run: test -z "$(git status --porcelain)" || { git status --short; git diff; exit 1; }

  linux-deploy:
    name: Real Linux ELF deployment
    needs: fixtures
    runs-on: ubuntu-latest
    timeout-minutes: 20
    defaults:
      run:
        working-directory: broca
        shell: bash
    steps:
      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
        with:
          persist-credentials: false
          path: broca
      - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
      - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
        with:
          workspaces: broca -> target
          shared-key: release-deploy
          cache-on-failure: 'true'
      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          name: fixtures-Linux
          path: ${{ runner.temp }}/fixtures
      - run: bash scripts/unpack-fixtures.sh "$RUNNER_TEMP/fixtures/fixtures-Linux.tar.gz" "$GITHUB_WORKSPACE"
      - name: Real Linux ELF deployment under subc
        env:
          SUBC_CORE_BIN: ${{ github.workspace }}/fixtures/bin/ck-subc
          BROCA_DEPLOY_TEST_BIN: ${{ github.workspace }}/broca/target/release/ck-broca
          BROCA_REQUIRE_LINUX_DEPLOY: '1'
        run: |
          cargo build --release --locked -p broca-module-serve
          python3 -B scripts/test-place-release-gate.py LinuxDeployTests.test_real_linux_subc_stage_place_restart_verify -v
      - name: Verify clean worktree
        run: test -z "$(git status --porcelain)" || { git status --short; git diff; exit 1; }

  # Preserve the required status names, and refuse a skipped/red dependency.
  test:
    name: Test (${{ matrix.os }})
    if: always()
    needs: [lint, scripts, unit, integration-build, fixtures, integration, linux-deploy]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        os: [ubuntu, windows]
    steps:
      - name: Require every lane
        env:
          NEEDS_JSON: ${{ toJSON(needs) }}
        run: |
          python3 - <<'PY'
          import json, os, sys
          lanes = json.loads(os.environ['NEEDS_JSON'])
          print(f"Inspected {len(lanes)} required lanes: {lanes}")
          if not lanes or any(lane['result'] != 'success' for lane in lanes.values()):
              sys.exit('A required lane failed, was cancelled, or was skipped')
          PY

  # Linux + Windows matrix. Windows is the load-bearing leg: the cfg(windows) paths
  # (e.g. the lease's ERROR_LOCK_VIOLATION contention check, the subc-binary .exe
  # suffix) cannot be exercised by local macOS/Linux development. macOS is intentionally
  # NOT in CI: it is the development platform (continuously exercised by local cargo test
  # + clippy) and macOS-vs-Linux divergence risk is low (both POSIX).
  #
  # Every job runs on GitHub-hosted runners.
  unit:
    needs: fixtures
    name: Lib/bin/doc (${{ matrix.os }})
    runs-on: ${{ matrix.runner }}
    timeout-minutes: 90
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: ubuntu
            runner: ubuntu-latest
          - os: windows
            runner: windows-latest
    defaults:
      run:
        working-directory: broca
    steps:

      - name: Checkout broca
        uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
        with:
          persist-credentials: false
          path: broca



      - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable branch
        with:
          components: clippy, rustfmt


      # The provider-family e2e tests boot the pinned @copilotkit/aimock devDep (a
      # Node mock LLM server) from node_modules, which is gitignored. Without it the
      # tests can't boot aimock: most skip, but stream_errors_e2e requires it and
      # fails — so the whole mock matrix only runs for real once `npm ci` installs it.
      - name: Set up Node
        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
        with:
          node-version: 20

      - name: Install aimock (npm ci)
        run: npm ci

      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          name: fixtures-${{ runner.os }}
          path: ${{ runner.temp }}/fixtures
      - name: Unpack pinned fixtures
        shell: bash
        run: bash scripts/unpack-fixtures.sh "$RUNNER_TEMP/fixtures/fixtures-$RUNNER_OS.tar.gz" "$GITHUB_WORKSPACE"
      - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
        id: cargo-cache
        with:
          workspaces: broca -> target
          shared-key: units
          cache-on-failure: 'true'
      - name: Cargo cache telemetry
        shell: bash
        run: echo "Unit cache exact hit = ${{ steps.cargo-cache.outputs.cache-hit }}" >> "$GITHUB_STEP_SUMMARY"

      - name: Check linked dependency sources
        run: python3 scripts/check-path-deps.py && python3 scripts/test-check-path-deps.py
        shell: bash

      - name: Lib/bin tests and doctests (live and test-clock)
        timeout-minutes: 45
        env:
          BROCA_REQUIRE_SUBC: "1"
          BROCA_REQUIRE_AIMOCK: "1"
          BROCA_REQUIRE_VAULT: ${{ matrix.os == 'ubuntu' && '1' || '' }}
          SUBC_CORE_BIN: ${{ github.workspace }}/fixtures/bin/ck-subc${{ runner.os == 'Windows' && '.exe' || '' }}
          FAKE_AFT_STUB_BIN: ${{ github.workspace }}/fixtures/bin/fake-aft-stub${{ runner.os == 'Windows' && '.exe' || '' }}
          CK_CLAUSTRUM_BIN: ${{ matrix.os == 'ubuntu' && format('{0}/fixtures/bin/ck-claustrum', github.workspace) || '' }}
          CK_AUTH_BIN: ${{ matrix.os == 'ubuntu' && format('{0}/fixtures/bin/ck-auth', github.workspace) || '' }}
          CK_CLAUSTRUM_CHECKOUT: ${{ matrix.os == 'ubuntu' && format('{0}/fixtures/claustrum', github.workspace) || '' }}
        shell: bash
        run: |
          source scripts/rust-test-gate.sh
          run_phase 'lib/bin tests' cargo test --locked --workspace --lib --bins --no-fail-fast "${cargo_feature_args[@]}" --quiet
          doc_count="$(count_doc_tests)"
          echo "doctests discovered: $doc_count"
          if [[ "$doc_count" -gt 0 ]]; then
            run_phase 'doctests' cargo test --locked --workspace --doc "${cargo_feature_args[@]}" --quiet
          fi


      - name: Upload process cleanup diagnostics
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: rig-survivors-unit-${{ matrix.os }}
          path: broca/target/rig-survivors/*.txt
          if-no-files-found: ignore

      # Exercised 2026-08-12 against a scratch repo, because this gate had been green on
      # every run since the repo existed and nothing showed it COULD go red:
      #   clean tree                     -> passes
      #   modified tracked file          -> fails, naming it
      #   untracked file left behind     -> fails, naming it
      # The third case is the one worth protecting and it rests on a default: porcelain
      # reports untracked files, so a build that emits a generated source nobody added trips
      # this. Adding --untracked-files=no while tidying would silently remove exactly that
      # case, and the remaining two would keep passing.
      # The diff is EMPTY for an untracked file, so it is printed conditionally: an
      # unconditional `git diff` under a porcelain listing makes the listing look redundant,
      # and the listing is the only output naming the file.
      - name: Verify clean worktree
        shell: bash
        run: |
          dirty="$(git status --porcelain)"
          if [ -n "$dirty" ]; then
            echo "::error::Working tree changed during CI (modified tracked files, or new files a build left behind)"
            printf '%s\n' "$dirty"
            diff_out="$(git diff -- . | head -c 20000)"
            if [ -n "$diff_out" ]; then
              echo "git diff (first 20000 bytes):"
              printf '%s\n' "$diff_out"
            else
              echo "(no diff: the change is untracked files the build left behind -- see the list above)"
            fi
            exit 1
          fi


      # Exercised 2026-08-12 against a scratch repo, because this gate had been green on
      # every run since the repo existed and nothing showed it COULD go red:
      #   clean tree                     -> passes
      #   modified tracked file          -> fails, naming it
      #   untracked file left behind     -> fails, naming it
      # The third case is the one worth protecting and it rests on a default: porcelain
      # reports untracked files, so a build that emits a generated source nobody added trips
      # this. Adding --untracked-files=no while tidying would silently remove exactly that
      # case, and the remaining two would keep passing.
      # The diff is EMPTY for an untracked file, so it is printed conditionally: an
      # unconditional `git diff` under a porcelain listing makes the listing look redundant,
      # and the listing is the only output naming the file.
      - name: Verify clean worktree
        shell: bash
        run: |
          dirty="$(git status --porcelain)"
          if [ -n "$dirty" ]; then
            echo "::error::Working tree changed during CI (modified tracked files, or new files a build left behind)"
            printf '%s\n' "$dirty"
            diff_out="$(git diff -- . | head -c 20000)"
            if [ -n "$diff_out" ]; then
              echo "git diff (first 20000 bytes):"
              printf '%s\n' "$diff_out"
            else
              echo "(no diff: the change is untracked files the build left behind -- see the list above)"
            fi
            exit 1
          fi