diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad48a350..753c82fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,12 +71,11 @@ jobs: # (needs systemd=true in wsl.conf), and even with it, `Linger=no` makes # systemd tear down /run/user/ — including the session-bus socket — # when the login session ends, while DBUS_SESSION_BUS_ADDRESS still points - # at the now-dead socket. Real WSL2 isn't worth running on CI (hosted - # windows runners only give WSL1; WSL2 needs nested virtualization), but - # the failure mode reproduces deterministically here: point the bus address - # at a socket that doesn't exist and confirm omac's keychain/Secret Service - # reads degrade gracefully (keychain.IsUnavailable) instead of surfacing - # raw dial errors. + # at the now-dead socket. This job reproduces that failure mode + # deterministically on a plain Ubuntu runner: point the bus address at a + # socket that doesn't exist and confirm omac's keychain/Secret Service reads + # degrade gracefully (keychain.IsUnavailable) instead of surfacing raw dial + # errors. For a full WSL2 run see the wsl2 job below. name: WSL2-style session (dead bus) runs-on: ubuntu-latest steps: @@ -114,6 +113,56 @@ jobs: XDG_RUNTIME_DIR: ${{ runner.temp }}/nonexistent-run-user run: go test -race -count=1 -timeout=5m ./... + wsl2: + # Full test suite on real WSL2: catches WSL-specific osinfo detection, + # sandbox-under-Microsoft-kernel, and keychain degraded-session paths. + # Ubuntu 24.04 needs no AppArmor userns fix (WSL2's kernel doesn't + # enforce it). continue-on-error during rollout, remove after + # consistently successful + name: Test (WSL2 - Ubuntu 24.04) + runs-on: windows-latest + continue-on-error: true + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Strip CRLF from shell and TypeScript files + # actions/checkout defaults to core.autocrlf=true on Windows, which + # converts LF->CRLF. CRLF breaks bash scripts inside WSL (e.g. + # `set -euo pipefail\r` → exit 2) and can cause issues with TypeScript + # tooling. Strip \r from .sh and .ts files so WSL sees clean LF. + shell: bash + run: find . \( -name '*.sh' -o -name '*.ts' \) -not -path './.git/*' -exec sed -i 's/\r$//' {} + + + - name: Set up WSL2 (Ubuntu 24.04) + uses: Vampire/setup-wsl@v7 + with: + distribution: Ubuntu-24.04 + additional-packages: bubblewrap gcc libc6-dev + # update: 'true' # makes it very slow + + - name: Set up Go in WSL2 + shell: wsl-bash {0} + run: | + GO_VERSION=$(grep '^go ' go.mod | tr -d '\r' | awk '{print $2}') + curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" | sudo tar -C /usr/local -xz + + - name: Verify module tidy + shell: wsl-bash {0} + run: /usr/local/go/bin/go mod download && /usr/local/go/bin/go mod verify + + - name: go vet + shell: wsl-bash {0} + run: /usr/local/go/bin/go vet ./... + + - name: Build + shell: wsl-bash {0} + run: /usr/local/go/bin/go build -v ./... + + - name: Test + shell: wsl-bash {0} + run: /usr/local/go/bin/go test -race -count=1 -timeout=10m ./... + lint: name: Lint runs-on: ubuntu-latest @@ -148,6 +197,9 @@ jobs: - name: Workflow shell syntax run: python3 scripts/check-workflow-shell.py + - name: WSL2 coverage + run: python3 scripts/check-wsl2-coverage.py + - name: Release notification policy run: python3 scripts/release-workflow_test.py diff --git a/.github/workflows/doc-drift.yml b/.github/workflows/doc-drift.yml index 1d927c1f..9d08c071 100644 --- a/.github/workflows/doc-drift.yml +++ b/.github/workflows/doc-drift.yml @@ -1,7 +1,7 @@ # Documentation-drift audit. # # An agent (headless opencode against the internal SKAINET gateway, same -# GLM-5.2 as the rest of the harness suite) reads the whole checkout and +# default model as the rest of the harness suite) reads the whole checkout and # reports where the documentation makes a concrete factual claim the code # contradicts — a removed flag, a changed default, a renamed command, a moved # path, a behaviour that no longer holds. Every finding is verified against the @@ -28,9 +28,9 @@ on: type: string default: '2400' model: - description: 'Model id for the audit agent, for THIS RUN ONLY. Leave empty for the opencode pin in internal/e2e/versions.go (zai-org/GLM-5.2)' + description: 'Model id for the audit agent, for THIS RUN ONLY. Set empty to use each harness''s pin in internal/e2e/versions.go (opencode/codex/copilot/pi/codewhale: zai-org/GLM-5.2, claude-code: claude-sonnet-5).' type: string - default: '' + default: 'deepseek-ai/DeepSeek-V4-Flash' context_limit: description: 'Context window declared to the agent, in tokens. Leave empty for the 100000 default, which is safely under every pinned model' type: string @@ -55,7 +55,7 @@ jobs: candidates: ${{ steps.probe.outputs.candidates }} env: SKAINET_TOKEN: ${{ secrets.SKAINET_TOKEN }} - SKAINET_INTERNAL: ${{ secrets.SKAINET_EXTERNAL }} + SKAINET_INTERNAL: ${{ secrets.SKAINET_INTERNAL }} E2E_MODEL: ${{ github.event.inputs.model }} steps: - name: Checkout @@ -89,7 +89,7 @@ jobs: env: DRIFT_MODE: ${{ matrix.mode }} SKAINET_TOKEN: ${{ secrets.SKAINET_TOKEN }} - SKAINET_INTERNAL: ${{ secrets.SKAINET_EXTERNAL }} + SKAINET_INTERNAL: ${{ secrets.SKAINET_INTERNAL }} E2E_VERSION_OPENCODE: ${{ github.event.inputs.opencode_version }} DRIFT_TIMEOUT_SECS: ${{ github.event.inputs.timeout_secs }} # From the preflight job: the input, the gateway probe and the -TEE flip diff --git a/.github/workflows/e2e-readme-onboarding.yml b/.github/workflows/e2e-readme-onboarding.yml index 8322aa66..8687c88c 100644 --- a/.github/workflows/e2e-readme-onboarding.yml +++ b/.github/workflows/e2e-readme-onboarding.yml @@ -24,9 +24,9 @@ on: type: string default: '1200' model: - description: 'Model id for the onboarding agent, for THIS RUN ONLY. Leave empty for the opencode pin in internal/e2e/versions.go (zai-org/GLM-5.2)' + description: 'Model id for the onboarding agent, for THIS RUN ONLY. Set empty to use each harness''s pin in internal/e2e/versions.go (opencode/codex/copilot/pi/codewhale: zai-org/GLM-5.2, claude-code: claude-sonnet-5).' type: string - default: '' + default: 'deepseek-ai/DeepSeek-V4-Flash' context_limit: description: 'Context window declared to the agent, in tokens. Leave empty for the 100000 default, which is safely under every pinned model' type: string @@ -48,7 +48,7 @@ jobs: candidates: ${{ steps.probe.outputs.candidates }} env: SKAINET_TOKEN: ${{ secrets.SKAINET_TOKEN }} - SKAINET_INTERNAL: ${{ secrets.SKAINET_EXTERNAL }} + SKAINET_INTERNAL: ${{ secrets.SKAINET_INTERNAL }} E2E_MODEL: ${{ github.event.inputs.model }} steps: - name: Checkout @@ -81,7 +81,7 @@ jobs: cancel-in-progress: false env: SKAINET_TOKEN: ${{ secrets.SKAINET_TOKEN }} - SKAINET_INTERNAL: ${{ secrets.SKAINET_EXTERNAL }} + SKAINET_INTERNAL: ${{ secrets.SKAINET_INTERNAL }} E2E_VERSION_OPENCODE: ${{ github.event.inputs.opencode_version }} E2E_ONBOARDING_TIMEOUT_SECS: ${{ github.event.inputs.timeout_secs }} # From the preflight job: the input, the gateway probe and the -TEE flip diff --git a/.github/workflows/e2e-smoke.yml b/.github/workflows/e2e-smoke.yml index 1977333f..ab0d4bbe 100644 --- a/.github/workflows/e2e-smoke.yml +++ b/.github/workflows/e2e-smoke.yml @@ -39,9 +39,9 @@ on: type: boolean default: true model: - description: 'Model id for EVERY harness, for THIS RUN ONLY. Leave empty to use each harness''s pin in internal/e2e/versions.go (opencode/codex/copilot/pi/codewhale: zai-org/GLM-5.2, claude-code: claude-sonnet-5)' + description: 'Model id for EVERY harness, for THIS RUN ONLY. Set empty to use each harness''s pin in internal/e2e/versions.go (opencode/codex/copilot/pi/codewhale: zai-org/GLM-5.2, claude-code: claude-sonnet-5).' type: string - default: '' + default: 'deepseek-ai/DeepSeek-V4-Flash' claude_code_model: description: 'Model id for claude-code only — it accepts sonnet or haiku models ONLY. Wins over "model"; leave empty to fall back to it, then to the versions.go pin' type: string @@ -73,7 +73,7 @@ jobs: claude_code: ${{ steps.probe.outputs.claude_code }} env: SKAINET_TOKEN: ${{ secrets.SKAINET_TOKEN }} - SKAINET_INTERNAL: ${{ secrets.SKAINET_EXTERNAL }} + SKAINET_INTERNAL: ${{ secrets.SKAINET_INTERNAL }} E2E_MODEL: ${{ github.event.inputs.model }} E2E_MODEL_CLAUDE_CODE: ${{ github.event.inputs.claude_code_model }} steps: @@ -112,29 +112,30 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] - harness: [opencode, claude-code, codex, copilot, pi, codewhale] + harness: [opencode, claude-code, codex, copilot, pi] # Two omac versions: main (built from source) and release (the latest # published binary — what users actually run). The contract stage is # identical across both (it's compiled from source), but launch + llm # exercise the selected omac binary, catching a released omac that can't # drive a current harness. + # + # codewhale is NOT in the main matrix — it runs in a separate + # non-blocking drift job (smoke-codewhale) so its failures don't + # block releases. To fully remove codewhale, delete that job + + # the Go harness config (harnesses.go, versions.go, etc.). omac: [main, release] exclude: # codex's Rust HTTP client is incompatible with the macOS Seatbelt - # sandbox (see e2e.yml / issue #48). codewhale is excluded on macOS - # by analogy (also a Rust CLI with its own HTTP client) — unverified, - # re-test and drop if it works. + # sandbox (see e2e.yml / issue #48). - os: macos-latest harness: codex - - os: macos-latest - harness: codewhale runs-on: ${{ matrix.os }} concurrency: group: e2e-smoke-${{ matrix.os }}-${{ matrix.harness }}-${{ matrix.omac }} cancel-in-progress: false env: SKAINET_TOKEN: ${{ secrets.SKAINET_TOKEN }} - SKAINET_INTERNAL: ${{ secrets.SKAINET_EXTERNAL }} + SKAINET_INTERNAL: ${{ secrets.SKAINET_INTERNAL }} ANTHROPIC_BASE_URL: ${{ secrets.ANTHROPIC_BASE_URL }} E2E_HARNESS: ${{ matrix.harness }} # Schedule runs always test latest (drift); dispatch honors the input. @@ -146,7 +147,6 @@ jobs: E2E_MODEL_CODEX: ${{ needs.model.outputs.gateway }} E2E_MODEL_COPILOT: ${{ needs.model.outputs.gateway }} E2E_MODEL_PI: ${{ needs.model.outputs.gateway }} - E2E_MODEL_CODEWHALE: ${{ needs.model.outputs.gateway }} E2E_MODEL_CLAUDE_CODE: ${{ needs.model.outputs.claude_code }} E2E_CONTEXT_LIMIT: ${{ github.event.inputs.context_limit }} steps: @@ -379,11 +379,528 @@ jobs: exit 1 fi + smoke-wsl2: + # Drift detection on real WSL2. All 5 harnesses run (GOOS=linux). Both + # omac variants (main + release) via matrix. See the smoke job above for + # the stage definitions; see e2e.yml's e2e-wsl2 job for the WSL2 setup + # pattern. + name: "${{ matrix.harness }} / WSL2 / omac@${{ matrix.omac }}" + needs: model + strategy: + fail-fast: false + matrix: + harness: [opencode, claude-code, codex, copilot, pi] + omac: [main, release] + runs-on: windows-latest + concurrency: + group: e2e-smoke-wsl2-${{ matrix.harness }}-${{ matrix.omac }} + cancel-in-progress: false + env: + SKAINET_TOKEN: ${{ secrets.SKAINET_TOKEN }} + SKAINET_INTERNAL: ${{ secrets.SKAINET_INTERNAL }} + ANTHROPIC_BASE_URL: ${{ secrets.ANTHROPIC_BASE_URL }} + E2E_HARNESS: ${{ matrix.harness }} + E2E_USE_LATEST: ${{ (github.event_name == 'schedule' || github.event.inputs.use_latest != 'false') && '1' || '' }} + E2E_MODEL_OPENCODE: ${{ needs.model.outputs.gateway }} + E2E_MODEL_CODEX: ${{ needs.model.outputs.gateway }} + E2E_MODEL_COPILOT: ${{ needs.model.outputs.gateway }} + E2E_MODEL_PI: ${{ needs.model.outputs.gateway }} + E2E_MODEL_CLAUDE_CODE: ${{ needs.model.outputs.claude_code }} + E2E_CONTEXT_LIMIT: ${{ github.event.inputs.context_limit }} + # WSL only forwards vars listed in WSLENV — include every job/step env + # var the Go tests read via os.Getenv(). Without it, SKAINET_TOKEN, + # E2E_MODEL_*, etc. are unbound inside wsl-bash. OMAC_BIN is set via + # GITHUB_ENV by the "Provision released omac" step. SMOKE_LOG_DIR is + # set via GITHUB_ENV by the "Set log dir" step (runner.temp is not + # available in job-level env — only github/needs/strategy/matrix/ + # vars/secrets/inputs are). + WSLENV: GITHUB_ENV:GITHUB_OUTPUT:GITHUB_STEP_SUMMARY:OMAC_BIN:OMAC_RELEASE_TAG:SKAINET_TOKEN:SKAINET_INTERNAL:ANTHROPIC_BASE_URL:E2E_HARNESS:E2E_USE_LATEST:E2E_MODEL_OPENCODE:E2E_MODEL_CODEX:E2E_MODEL_COPILOT:E2E_MODEL_PI:E2E_MODEL_CLAUDE_CODE:E2E_CONTEXT_LIMIT:E2E_CACHE_DIR:SMOKE_LOG_DIR + steps: + - name: Checkout + uses: actions/checkout@v4 + + # runner.temp is not available in job-level env, so set SMOKE_LOG_DIR + # here via GITHUB_ENV for all subsequent steps (and WSLENV forwards it + # into WSL). RUNNER_TEMP is always available in run steps. + - name: Set log dir + shell: bash + run: echo "SMOKE_LOG_DIR=${RUNNER_TEMP}/e2e-smoke-logs" >> "$GITHUB_ENV" + + - name: Strip CRLF from shell and TypeScript files + # actions/checkout defaults to core.autocrlf=true on Windows, which + # converts LF->CRLF. CRLF breaks bash scripts inside WSL (e.g. + # `set -euo pipefail\r` → exit 2) and can cause issues with TypeScript + # tooling. Strip \r from .sh and .ts files so WSL sees clean LF. + shell: bash + run: find . \( -name '*.sh' -o -name '*.ts' \) -not -path './.git/*' -exec sed -i 's/\r$//' {} + + + - name: Set up WSL2 (Ubuntu 24.04) + uses: Vampire/setup-wsl@v7 + with: + distribution: Ubuntu-24.04 + additional-packages: bubblewrap gcc libc6-dev unzip python3-pip + # Run wsl-bash steps as a non-root user so DAC protects root-owned + # paths (/etc/shadow, /root/.bashrc) — matches Linux CI behavior. + wsl-shell-user: runner + # update: 'true' # makes it very slow + + - name: Set up Go and Node in WSL2 (as root) + shell: wsl-bash -u root {0} + run: | + GO_VERSION=$(grep '^go ' go.mod | tr -d '\r' | awk '{print $2}') + curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" | tar -C /usr/local -xz + curl -fsSL https://deb.nodesource.com/setup_24.x | bash - + apt-get install -y nodejs + + - name: Set up Bun and Rust in WSL2 + shell: wsl-bash {0} + run: | + curl -fsSL https://bun.sh/install | bash + curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable + cat > "$HOME"/wsl-env.sh <<'WEOF' + export PATH="/usr/local/go/bin:$HOME/.bun/bin:$HOME/.cargo/bin:$PATH" + if [ -n "${GITHUB_OUTPUT:-}" ]; then + export GITHUB_OUTPUT=$(wslpath -u "$GITHUB_OUTPUT" 2>/dev/null || echo "$GITHUB_OUTPUT") + fi + if [ -n "${GITHUB_ENV:-}" ]; then + export GITHUB_ENV=$(wslpath -u "$GITHUB_ENV" 2>/dev/null || echo "$GITHUB_ENV") + fi + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + export GITHUB_STEP_SUMMARY=$(wslpath -u "$GITHUB_STEP_SUMMARY" 2>/dev/null || echo "$GITHUB_STEP_SUMMARY") + fi + if [ -n "${OMAC_BIN:-}" ]; then + export OMAC_BIN=$(wslpath -u "$OMAC_BIN" 2>/dev/null || echo "$OMAC_BIN") + fi + if [ -n "${E2E_CACHE_DIR:-}" ]; then + export E2E_CACHE_DIR=$(wslpath -u "$E2E_CACHE_DIR" 2>/dev/null || echo "$E2E_CACHE_DIR") + fi + if [ -n "${SMOKE_LOG_DIR:-}" ]; then + export SMOKE_LOG_DIR=$(wslpath -u "$SMOKE_LOG_DIR" 2>/dev/null || echo "$SMOKE_LOG_DIR") + mkdir -p "$SMOKE_LOG_DIR" + fi + WEOF + + - name: Resolve model + shell: wsl-bash {0} + run: | + source "$HOME"/wsl-env.sh + echo "::notice title=Model::${{ matrix.harness }} runs against $(scripts/resolve-model.sh '${{ matrix.harness }}')" + + - name: Cache harness install packages + uses: actions/cache@v4 + with: + path: ${{ runner.temp }}/e2e-install-cache + key: e2e-smoke-install-Windows-${{ matrix.harness }}-${{ matrix.omac }}-${{ github.run_id }} + restore-keys: | + e2e-smoke-install-Windows-${{ matrix.harness }}-${{ matrix.omac }}- + + - name: Provision released omac (omac@release) + # gh CLI lives on the Windows side, not in WSL. Download the Linux + # tarball to a workspace-relative path (avoids D:\ drive-letter issues + # with tar in Git Bash). Extraction happens in WSL in the next step. + if: matrix.omac == 'release' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + tag=$(gh release view --json tagName --jq .tagName) + echo "OMAC_RELEASE_TAG=$tag" >> "$GITHUB_ENV" + mkdir -p .omac-release + gh release download "$tag" --dir .omac-release --pattern "oh-my-agentic-coder_*_linux_x86_64.tar.gz" + ls -la .omac-release/ + + - name: Extract and provision released omac in WSL + # Extract the Linux ELF binary inside WSL (not Git Bash). Write + # OMAC_BIN directly to "$HOME"/wsl-env.sh (sourced by every wsl-bash + # step) instead of GITHUB_ENV: GITHUB_ENV roundtrips through a + # Windows env var, and wsl-env.sh's wslpath -u corrupts the WSL + # path (/tmp/... → /mnt/d/tmp/...). + if: matrix.omac == 'release' + shell: wsl-bash {0} + run: | + source "$HOME"/wsl-env.sh + f=$(ls .omac-release/*.tar.gz 2>/dev/null | head -1) + [ -z "$f" ] && { echo "no release tarball found"; ls -R .omac-release; exit 1; } + dir=/tmp/omac-release; rm -rf "$dir"; mkdir -p "$dir" + tar -xzf "$f" -C "$dir" + bin=$(find "$dir" -type f -name omac | head -1) + [ -x "$bin" ] || { echo "omac binary missing in release asset"; ls -R "$dir"; exit 1; } + echo "export OMAC_BIN=$bin" >> "$HOME"/wsl-env.sh + echo "provisioned omac@release ${OMAC_RELEASE_TAG:-release} -> $bin"; "$bin" version || true + + - name: Resolve omac version label + id: omac + shell: wsl-bash {0} + run: | + source "$HOME"/wsl-env.sh + if [ "${{ matrix.omac }}" = "release" ]; then + echo "version=${OMAC_RELEASE_TAG:-release}" >> "$GITHUB_OUTPUT" + else + echo "version=main@$(git describe --tags --always --dirty)" >> "$GITHUB_OUTPUT" + fi + + - name: Model-free smoke (contract + launch + serve + system prompt) + id: modelfree + shell: wsl-bash {0} + env: + E2E_CACHE_DIR: ${{ runner.temp }}/e2e-install-cache + run: | + source "$HOME"/wsl-env.sh + set -o pipefail + go test -tags=e2e -timeout=20m -v \ + -run 'TestHarnessCLIContract|TestHarnessLaunchProbe|TestHarnessServeProbe|TestOpenCodeSingleSystemMessage' \ + ./internal/e2e/ 2>&1 | tee "$SMOKE_LOG_DIR/modelfree.log" || true + + - name: LLM turn + id: llm + continue-on-error: true + shell: wsl-bash {0} + env: + E2E_CACHE_DIR: ${{ runner.temp }}/e2e-install-cache + run: | + source "$HOME"/wsl-env.sh + set -o pipefail + attempt=1; max=2 + while :; do + if go test -tags=e2e -timeout=12m -v -run TestE2EEchoRest ./internal/e2e/ 2>&1 | tee "$SMOKE_LOG_DIR/llm.log"; then + break + fi + if [ "$attempt" -ge "$max" ]; then exit 1; fi + attempt=$((attempt + 1)); echo "llm turn flaked — retrying ($attempt/$max)"; sleep 5 + done + + - name: Build compatibility row + id: row + if: always() + shell: wsl-bash {0} + run: | + # wsl-env.sh may not exist if an earlier setup step failed — + # fall back to a minimal env so the row still reports ❌. + if [ -f "$HOME"/wsl-env.sh ]; then + source "$HOME"/wsl-env.sh + else + export PATH="/usr/local/go/bin:$HOME/.bun/bin:$HOME/.cargo/bin:$PATH" + for v in GITHUB_OUTPUT GITHUB_STEP_SUMMARY SMOKE_LOG_DIR; do + eval "val=\${${v}:-}" + [ -n "$val" ] && eval "export $v=\$(wslpath -u \"\$val\" 2>/dev/null || echo \"\$val\")" + done + fi + mkdir -p "${SMOKE_LOG_DIR:-/tmp}/compat" + stage_mark() { + local stage="$1" absent="${2:-❌}" + local line + line=$(grep -oE "OMAC_COMPAT .*stage=$stage .*result=[A-Z]+" "$SMOKE_LOG_DIR/modelfree.log" 2>/dev/null | head -1 || true) + case "$line" in + *result=PASS) echo '✅' ;; + *result=SKIP) echo '➖' ;; + '') echo "$absent" ;; + *) echo '❌' ;; + esac + } + version=$(grep -oE 'OMAC_COMPAT .*stage=contract .*' "$SMOKE_LOG_DIR/modelfree.log" 2>/dev/null \ + | sed -E 's/.*version=([^ ]+).*/\1/' | head -1) + [ -z "$version" ] && version="unknown" + model=$(grep -oE 'OMAC_COMPAT .*stage=contract .*' "$SMOKE_LOG_DIR/modelfree.log" 2>/dev/null \ + | sed -E 's/.*model=([^ ]+).*/\1/' | head -1) + [ -z "$model" ] && model=$(scripts/resolve-model.sh '${{ matrix.harness }}' || echo unknown) + contract=$(stage_mark contract) + launch=$(stage_mark launch) + serve=$(stage_mark serve ➖) + case "${{ steps.llm.outcome }}" in + success) llm='✅' ;; + failure) llm='❌' ;; + *) llm='➖' ;; + esac + date_utc=$(date -u +%Y-%m-%d) + row="| $date_utc | ${{ steps.omac.outputs.version }} | WSL2 | ${{ matrix.harness }} | $version | $contract | $launch | $serve | $llm | $model |" + mkdir -p "$SMOKE_LOG_DIR/compat" + slug="wsl2-${{ matrix.harness }}-${{ matrix.omac }}" + printf '%s\n' "$row" > "$SMOKE_LOG_DIR/compat/$slug.row" + echo "$row" + echo "row<> "$GITHUB_OUTPUT"; echo "$row" >> "$GITHUB_OUTPUT"; echo "EOF" >> "$GITHUB_OUTPUT" + if printf '%s' "$row" | grep -q '❌'; then + stages="" + [ "$contract" = "❌" ] && stages="$stages contract" + [ "$launch" = "❌" ] && stages="$stages launch" + [ "$serve" = "❌" ] && stages="$stages serve" + [ "$llm" = "❌" ] && stages="$stages llm" + { + echo "=== ${{ matrix.harness }} / WSL2 / omac@${{ matrix.omac }} — failing stage(s):$stages ===" + grep -hE 'CLI contract drift|no longer exposes|--- FAIL|omac start .* failed|serve probe failed|omac serve exited|FAIL \[|Error|panic:' \ + "$SMOKE_LOG_DIR/modelfree.log" "$SMOKE_LOG_DIR/llm.log" 2>/dev/null | head -40 || true + tail -n 30 "$SMOKE_LOG_DIR/llm.log" 2>/dev/null || true + echo + } > "$SMOKE_LOG_DIR/compat/$slug.ctx" + fi + { + echo "### ${{ matrix.harness }} / WSL2 / omac@${{ matrix.omac }}" + echo "" + echo "| Timestamp (UTC) | omac | OS | Harness | Version | contract | launch | serve | llm | Model |" + echo "|---|---|---|---|---|:-:|:-:|:-:|:-:|---|" + echo "$row" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload compatibility row + if: always() + uses: actions/upload-artifact@v4 + with: + name: compat-wsl2-${{ matrix.harness }}-${{ matrix.omac }} + path: | + ${{ runner.temp }}/e2e-smoke-logs/compat/*.row + ${{ runner.temp }}/e2e-smoke-logs/compat/*.ctx + if-no-files-found: warn + retention-days: 30 + + - name: Upload logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: smoke-logs-wsl2-${{ matrix.harness }}-${{ matrix.omac }} + path: | + ${{ runner.temp }}/e2e-smoke-logs/modelfree.log + ${{ runner.temp }}/e2e-smoke-logs/llm.log + if-no-files-found: ignore + retention-days: 14 + + - name: Fail this leg on drift + if: always() + shell: bash + run: | + row='${{ steps.row.outputs.row }}' + echo "$row" + if printf '%s' "$row" | grep -q '❌'; then + echo "::error::compatibility drift detected for ${{ matrix.harness }} / WSL2" + exit 1 + fi + + # ── codewhale drift ──────────────────────────────────────────────── + # Non-blocking: runs the smoke suite for codewhale only, on Linux, for + # both omac variants (main + release). Failures are reported in the + # compatibility table but never block releases or the main matrix. + # To fully remove codewhale: delete this job + the report job's + # smoke-codewhale entry + the Go harness config (harnesses.go, etc.). + smoke-codewhale: + name: "codewhale / ubuntu-latest / omac@${{ matrix.omac }} (drift)" + needs: model + # Non-blocking: the report job's ADVISORY_HARNESS_RE env var (report job, + # top-level env) treats codewhale rows as advisory. To make codewhale block + # the run: set continue-on-error to false here AND drop "codewhale" from + # ADVISORY_HARNESS_RE (or empty it) in the report job. + continue-on-error: true + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + omac: [main, release] + concurrency: + group: e2e-smoke-codewhale-${{ matrix.omac }} + cancel-in-progress: false + env: + SKAINET_TOKEN: ${{ secrets.SKAINET_TOKEN }} + SKAINET_INTERNAL: ${{ secrets.SKAINET_INTERNAL }} + ANTHROPIC_BASE_URL: ${{ secrets.ANTHROPIC_BASE_URL }} + E2E_HARNESS: codewhale + E2E_USE_LATEST: ${{ (github.event_name == 'schedule' || github.event.inputs.use_latest != 'false') && '1' || '' }} + E2E_MODEL_CODEWHALE: ${{ needs.model.outputs.gateway }} + E2E_CONTEXT_LIMIT: ${{ github.event.inputs.context_limit }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Resolve model + run: | + echo "::notice title=Model::codewhale runs against $(scripts/resolve-model.sh 'codewhale')" + + - name: Install bubblewrap (Linux only) + if: runner.os == 'Linux' + run: | + sudo apt-get update && sudo apt-get install -y bubblewrap + sudo tee /etc/apparmor.d/bwrap > /dev/null <<'EOF' + abi , + /usr/bin/bwrap flags=(unconfined) { + userns, + } + EOF + sudo apparmor_parser -r /etc/apparmor.d/bwrap + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Set up Bun + uses: oven-sh/setup-bun@v2 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: '24' + + - name: Cache harness install packages + uses: actions/cache@v4 + with: + path: ${{ runner.temp }}/e2e-install-cache + key: e2e-smoke-install-${{ runner.os }}-codewhale-${{ matrix.omac }}-${{ github.run_id }} + restore-keys: | + e2e-smoke-install-${{ runner.os }}-codewhale-${{ matrix.omac }}- + + - name: Provision released omac (omac@release) + if: matrix.omac == 'release' + env: + GH_TOKEN: ${{ github.token }} + run: | + tag=$(gh release view --json tagName --jq .tagName) + echo "OMAC_RELEASE_TAG=$tag" >> "$GITHUB_ENV" + case "$(uname -s)" in Linux) os=linux;; Darwin) os=macOS;; *) echo "unknown os"; exit 1;; esac + case "$(uname -m)" in x86_64|amd64) arch=x86_64;; arm64|aarch64) arch=arm64;; *) echo "unknown arch"; exit 1;; esac + dir="$RUNNER_TEMP/omac-release"; mkdir -p "$dir" + gh release download "$tag" --dir "$dir" --pattern "oh-my-agentic-coder_*_${os}_${arch}.tar.gz" + f=$(ls "$dir"/*.tar.gz | head -1) + tar -xzf "$f" -C "$dir" + bin=$(find "$dir" -type f -name omac | head -1) + [ -x "$bin" ] || { echo "omac binary missing in release asset"; ls -R "$dir"; exit 1; } + echo "OMAC_BIN=$bin" >> "$GITHUB_ENV" + echo "provisioned omac@release $tag -> $bin"; "$bin" version || true + + - name: Resolve omac version label + id: omac + run: | + if [ "${{ matrix.omac }}" = "release" ]; then + echo "version=${OMAC_RELEASE_TAG:-release}" >> "$GITHUB_OUTPUT" + else + echo "version=main@$(git describe --tags --always --dirty)" >> "$GITHUB_OUTPUT" + fi + + - name: Model-free smoke (contract + launch + serve + system prompt) + id: modelfree + env: + E2E_CACHE_DIR: ${{ runner.temp }}/e2e-install-cache + run: | + set -o pipefail + go test -tags=e2e -timeout=20m -v \ + -run 'TestHarnessCLIContract|TestHarnessLaunchProbe|TestHarnessServeProbe|TestOpenCodeSingleSystemMessage' \ + ./internal/e2e/ 2>&1 | tee modelfree.log || true + + - name: LLM turn + id: llm + continue-on-error: true + env: + E2E_CACHE_DIR: ${{ runner.temp }}/e2e-install-cache + run: | + set -o pipefail + attempt=1; max=2 + while :; do + if go test -tags=e2e -timeout=12m -v -run TestE2EEchoRest ./internal/e2e/ 2>&1 | tee llm.log; then + break + fi + if [ "$attempt" -ge "$max" ]; then exit 1; fi + attempt=$((attempt + 1)); echo "llm turn flaked — retrying ($attempt/$max)"; sleep 5 + done + + - name: Build compatibility row + id: row + if: always() + run: | + stage_mark() { + local stage="$1" absent="${2:-❌}" + local line + line=$(grep -oE "OMAC_COMPAT .*stage=$stage .*result=[A-Z]+" modelfree.log 2>/dev/null | head -1 || true) + case "$line" in + *result=PASS) echo '✅' ;; + *result=SKIP) echo '➖' ;; + '') echo "$absent" ;; + *) echo '❌' ;; + esac + } + version=$(grep -oE 'OMAC_COMPAT .*stage=contract .*' modelfree.log 2>/dev/null \ + | sed -E 's/.*version=([^ ]+).*/\1/' | head -1) + [ -z "$version" ] && version="unknown" + model=$(grep -oE 'OMAC_COMPAT .*stage=contract .*' modelfree.log 2>/dev/null \ + | sed -E 's/.*model=([^ ]+).*/\1/' | head -1) + [ -z "$model" ] && model=$(scripts/resolve-model.sh 'codewhale' || echo unknown) + contract=$(stage_mark contract) + launch=$(stage_mark launch) + serve=$(stage_mark serve ➖) + case "${{ steps.llm.outcome }}" in + success) llm='✅' ;; + failure) llm='❌' ;; + *) llm='➖' ;; + esac + date_utc=$(date -u +%Y-%m-%d) + row="| $date_utc | ${{ steps.omac.outputs.version }} | ubuntu-latest | codewhale | $version | $contract | $launch | $serve | $llm | $model |" + mkdir -p compat + slug="ubuntu-latest-codewhale-${{ matrix.omac }}" + printf '%s\n' "$row" > "compat/$slug.row" + echo "$row" + echo "row<> "$GITHUB_OUTPUT"; echo "$row" >> "$GITHUB_OUTPUT"; echo "EOF" >> "$GITHUB_OUTPUT" + if printf '%s' "$row" | grep -q '❌'; then + stages="" + [ "$contract" = "❌" ] && stages="$stages contract" + [ "$launch" = "❌" ] && stages="$stages launch" + [ "$serve" = "❌" ] && stages="$stages serve" + [ "$llm" = "❌" ] && stages="$stages llm" + { + echo "=== codewhale / ubuntu-latest / omac@${{ matrix.omac }} — failing stage(s):$stages ===" + grep -hE 'CLI contract drift|no longer exposes|--- FAIL|omac start .* failed|serve probe failed|omac serve exited|FAIL \[|Error|panic:' \ + modelfree.log llm.log 2>/dev/null | head -40 || true + tail -n 30 llm.log 2>/dev/null || true + echo + } > "compat/$slug.ctx" + fi + { + echo "### codewhale / ubuntu-latest / omac@${{ matrix.omac }}" + echo "" + echo "| Timestamp (UTC) | omac | OS | Harness | Version | contract | launch | serve | llm | Model |" + echo "|---|---|---|---|---|:-:|:-:|:-:|:-:|---|" + echo "$row" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload compatibility row + if: always() + uses: actions/upload-artifact@v4 + with: + name: compat-ubuntu-latest-codewhale-${{ matrix.omac }} + path: | + compat/*.row + compat/*.ctx + if-no-files-found: warn + retention-days: 30 + + - name: Upload logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: smoke-logs-ubuntu-latest-codewhale-${{ matrix.omac }} + path: | + modelfree.log + llm.log + if-no-files-found: ignore + retention-days: 14 + + - name: Fail this leg on drift + if: always() + run: | + row='${{ steps.row.outputs.row }}' + echo "$row" + if printf '%s' "$row" | grep -q '❌'; then + echo "::error::compatibility drift detected for codewhale / ubuntu-latest" + exit 1 + fi + report: name: Record & report - needs: [model, smoke] + needs: [model, smoke, smoke-wsl2, smoke-codewhale] if: always() runs-on: ubuntu-latest + env: + # Advisory harnesses: legs whose rows match this regex should run with + # continue-on-error in their source job and will be reported in the + # dashboard's non-blocking section, never setting has_failures. + # empty → everything is blocking (split disabled) + # regex → matching harnesses are advisory + # Manually keep this in sync with the continue-on-error flags on the source jobs. + ADVISORY_HARNESS_RE: '^codewhale$' steps: - name: Checkout uses: actions/checkout@v4 @@ -412,9 +929,16 @@ jobs: fi printf '%s\n' "$rows" > run.rows echo "has_rows=true" >> "$GITHUB_OUTPUT" - echo "total=$(wc -l < run.rows | tr -d ' ')" >> "$GITHUB_OUTPUT" + # Split rows into blocking (gate the run) and advisory + awk -F'|' -v re="$ADVISORY_HARNESS_RE" 'NF{ + h=$5; gsub(/^[ \t]+|[ \t]+$/,"",h) + if (re!="" && h~re) print > "advisory.rows"; else print > "blocking.rows" + }' run.rows + [ -f blocking.rows ] || cp run.rows blocking.rows + [ -f advisory.rows ] || : > advisory.rows + echo "total=$(wc -l < blocking.rows | tr -d ' ')" >> "$GITHUB_OUTPUT" # Failing rows, in the canonical order (date desc, omac, os, harness). - grep '❌' run.rows | sort -t'|' -k2,2r -k3,3 -k4,4 -k5,5 > failing.rows || true + grep '❌' blocking.rows | sort -t'|' -k2,2r -k3,3 -k4,4 -k5,5 > failing.rows || true fail_count=$(wc -l < failing.rows | tr -d ' ') echo "fail_count=$fail_count" >> "$GITHUB_OUTPUT" if [ "$fail_count" -gt 0 ]; then @@ -422,6 +946,29 @@ jobs: else echo "has_failures=false" >> "$GITHUB_OUTPUT" fi + # Advisory failures: reported, but never set has_failures. + grep '❌' advisory.rows | sort -t'|' -k2,2r -k3,3 -k4,4 -k5,5 > advisory-failing.rows || true + adv_fail=$(wc -l < advisory-failing.rows | tr -d ' ') + echo "advisory_total=$(wc -l < advisory.rows | tr -d ' ')" >> "$GITHUB_OUTPUT" + echo "advisory_fail_count=$adv_fail" >> "$GITHUB_OUTPUT" + if [ "$adv_fail" -gt 0 ]; then + echo "has_advisory_failures=true" >> "$GITHUB_OUTPUT" + else + echo "has_advisory_failures=false" >> "$GITHUB_OUTPUT" + fi + # Move advisory .ctx aside so the failure classifiers (which read + # --ctx-dir compat) only see authoritative failures; harness is extracted from + # each .ctx's "=== / ... ===" header line. + if [ -n "$ADVISORY_HARNESS_RE" ]; then + mkdir -p compat-advisory + for f in compat/*.ctx; do + [ -e "$f" ] || continue + h=$(sed -n 's/^=== \([^ ]*\) \/.*/\1/p' "$f" | head -1) + if printf '%s' "$h" | grep -Eq "$ADVISORY_HARNESS_RE"; then + mv "$f" compat-advisory/ 2>/dev/null || true + fi + done + fi - name: Classify failure reasons (log-based) id: reason @@ -445,7 +992,7 @@ jobs: continue-on-error: true # a summariser hiccup must never mask the failure env: SKAINET_TOKEN: ${{ secrets.SKAINET_TOKEN }} - SKAINET_INTERNAL: ${{ secrets.SKAINET_EXTERNAL }} + SKAINET_INTERNAL: ${{ secrets.SKAINET_INTERNAL }} # The summariser is an LLM call too, so it uses the model the preflight # proved is served — re-resolving the pin here would 422 in exactly # the situation the summary is most needed. @@ -502,6 +1049,8 @@ jobs: HAS_FAILURES: ${{ steps.rows.outputs.has_failures }} FAIL_COUNT: ${{ steps.rows.outputs.fail_count }} TOTAL: ${{ steps.rows.outputs.total }} + HAS_ADVISORY_FAILURES: ${{ steps.rows.outputs.has_advisory_failures }} + ADVISORY_FAIL_COUNT: ${{ steps.rows.outputs.advisory_fail_count }} INFRA_ONLY: ${{ steps.reason.outputs.infra_only }} SUMMARY_MODEL: ${{ steps.summary.outputs.model }} MODEL_FALLBACK: ${{ needs.model.outputs.fallback }} @@ -577,6 +1126,19 @@ jobs: echo "See the summary above and the [run log]($RUN_URL) for the failing stage; a **launch**/**serve**/**llm** ❌ is a sandbox-launch, Desktop serve-mode, or model/agent-turn failure, not a CLI-contract break." fi fi + # Non-blocking failures: reported for visibility + # but never counted in $FAIL_COUNT / $TOTAL above. + if [ "$HAS_ADVISORY_FAILURES" = "true" ]; then + echo "" + echo "## ⚠️ Non-blocking failures" + echo "" + echo "These legs run with \`continue-on-error\` and never fail the run; listed for visibility only." + echo "" + echo "| Timestamp (UTC) | omac | OS | Harness | Version | contract | launch | serve | llm | Model |" + echo "|---|---|---|---|---|:-:|:-:|:-:|:-:|---|" + cat advisory-failing.rows + echo "" + fi echo "" echo "## History (rolling 30 days)" echo "" @@ -654,6 +1216,8 @@ jobs: HAS_FAILURES: ${{ steps.rows.outputs.has_failures }} FAIL_COUNT: ${{ steps.rows.outputs.fail_count }} TOTAL: ${{ steps.rows.outputs.total }} + HAS_ADVISORY_FAILURES: ${{ steps.rows.outputs.has_advisory_failures }} + ADVISORY_FAIL_COUNT: ${{ steps.rows.outputs.advisory_fail_count }} INFRA_ONLY: ${{ steps.reason.outputs.infra_only }} SUMMARY_MODEL: ${{ steps.summary.outputs.model }} MODEL_FALLBACK: ${{ needs.model.outputs.fallback }} @@ -704,6 +1268,9 @@ jobs: "$hdr" "$detail" "$models" "$ISSUE_URL" "$RUN_URL") else text="🟢 omac harness compatibility: all $TOTAL combination(s) green. Model(s): $models. Dashboard: $ISSUE_URL — Run: $RUN_URL" + if [ "$HAS_ADVISORY_FAILURES" = "true" ]; then + text="$text — ($ADVISORY_FAIL_COUNT non-blocking leg(s) failing, see dashboard)" + fi fi # The Slack endpoint is a Workflow Builder webhook: its payload keys # must match the workflow's declared variables, which expect "message" diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 87eed5ff..c0175dbd 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -2,7 +2,7 @@ # # Runs on manual trigger and every Saturday at 10:00 CET (08:00 UTC in # summer / 09:00 UTC in winter). Tests that each harness can start under -# the omac sandbox and that an agent running on GLM-5.2 (or +# the omac sandbox and that an agent running on the selected model (or # claude-sonnet-5 for claude-code) can call the bundled echo-rest skill # through the omac facade. name: "E2E: full" @@ -43,9 +43,9 @@ on: type: string default: 'codewhale@0.9.1' model: - description: 'Model id for EVERY harness, for THIS RUN ONLY (e.g. zai-org/GLM-5.2). Leave empty to use each harness''s pin in internal/e2e/versions.go (opencode/codex/copilot/pi/codewhale: zai-org/GLM-5.2, claude-code: claude-sonnet-5)' + description: 'Model id for EVERY harness, for THIS RUN ONLY. Set empty to use each harness''s pin in internal/e2e/versions.go (opencode/codex/copilot/pi/codewhale: zai-org/GLM-5.2, claude-code: claude-sonnet-5).' type: string - default: '' + default: 'deepseek-ai/DeepSeek-V4-Flash' claude_code_model: description: 'Model id for claude-code only — it accepts sonnet or haiku models ONLY, and bills a different provider than the SKAINET gateway the others use. Wins over "model"; leave empty to fall back to it, then to the versions.go pin' type: string @@ -75,7 +75,7 @@ jobs: claude_code: ${{ steps.probe.outputs.claude_code }} env: SKAINET_TOKEN: ${{ secrets.SKAINET_TOKEN }} - SKAINET_INTERNAL: ${{ secrets.SKAINET_EXTERNAL }} + SKAINET_INTERNAL: ${{ secrets.SKAINET_INTERNAL }} E2E_MODEL: ${{ github.event.inputs.model }} E2E_MODEL_CLAUDE_CODE: ${{ github.event.inputs.claude_code_model }} steps: @@ -120,29 +120,27 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] - harness: [opencode, claude-code, codex, copilot, pi, codewhale] + harness: [opencode, claude-code, codex, copilot, pi] # codex on macOS is excluded — its Rust HTTP client is incompatible # with the macOS Seatbelt sandbox (stream disconnected mid-completion, # even with network=open). `omac start codex` on macOS fails loud; # --no-sandbox would disable the entire omac sandbox, leaving nothing # to assert against. See issue #48. # - # codewhale on macOS is excluded by analogy: it too is a Rust CLI with - # its own HTTP client (the codex failure class). This is precautionary - # and UNVERIFIED for codewhale — re-test on macOS and drop this exclude - # (and the matching ones in internal/e2e) if it works. + # codewhale is NOT in the main matrix — it runs in a separate + # non-blocking drift job (e2e-codewhale) so its failures don't block + # releases. To fully remove codewhale, delete that job + the + # codewhale_version input + the Go harness config. exclude: - os: macos-latest harness: codex - - os: macos-latest - harness: codewhale runs-on: ${{ matrix.os }} concurrency: group: e2e-${{ matrix.os }}-${{ matrix.harness }} cancel-in-progress: false env: SKAINET_TOKEN: ${{ secrets.SKAINET_TOKEN }} - SKAINET_INTERNAL: ${{ secrets.SKAINET_EXTERNAL }} + SKAINET_INTERNAL: ${{ secrets.SKAINET_INTERNAL }} ANTHROPIC_BASE_URL: ${{ secrets.ANTHROPIC_BASE_URL }} E2E_HARNESS: ${{ matrix.harness }} # Drift-detection: install latest instead of the pinned versions below. @@ -158,7 +156,6 @@ jobs: E2E_VERSION_CODEX: ${{ github.event.inputs.codex_version }} E2E_VERSION_COPILOT: ${{ github.event.inputs.copilot_version }} E2E_VERSION_PI: ${{ github.event.inputs.pi_version }} - E2E_VERSION_CODEWHALE: ${{ github.event.inputs.codewhale_version }} # Models come from the preflight job, which already applied the `model` # input, the gateway probe, and the -TEE variant flip. Set PER HARNESS # rather than via cross-harness E2E_MODEL: the probed value is a gateway @@ -169,7 +166,6 @@ jobs: E2E_MODEL_CODEX: ${{ needs.model.outputs.gateway }} E2E_MODEL_COPILOT: ${{ needs.model.outputs.gateway }} E2E_MODEL_PI: ${{ needs.model.outputs.gateway }} - E2E_MODEL_CODEWHALE: ${{ needs.model.outputs.gateway }} E2E_MODEL_CLAUDE_CODE: ${{ needs.model.outputs.claude_code }} steps: - name: Checkout @@ -362,3 +358,393 @@ jobs: path: ${{ runner.temp }}/e2e-logs/ if-no-files-found: ignore retention-days: 14 + + e2e-wsl2: + # Full e2e suite on real WSL2. All 5 harnesses run (WSL2 is GOOS=linux, + # so codex is NOT excluded). Go/bun/node are installed manually + # inside WSL — the setup-* actions install on the Windows side, not in + # WSL. The wsl-bash shell uses --noprofile --norc, so PATH and GITHUB_* + # path translation are set up in "$HOME"/wsl-env.sh and sourced per step. + # Cache and logs use relative paths (relative to the checkout) so they + # work from both the Windows runner (actions/cache, upload-artifact) and + # inside WSL (Go test code) via the automount. + name: "${{ matrix.harness }} / WSL2" + needs: model + strategy: + fail-fast: false + matrix: + harness: [opencode, claude-code, codex, copilot, pi] + runs-on: windows-latest + concurrency: + group: e2e-wsl2-${{ matrix.harness }} + cancel-in-progress: false + env: + SKAINET_TOKEN: ${{ secrets.SKAINET_TOKEN }} + SKAINET_INTERNAL: ${{ secrets.SKAINET_INTERNAL }} + ANTHROPIC_BASE_URL: ${{ secrets.ANTHROPIC_BASE_URL }} + E2E_HARNESS: ${{ matrix.harness }} + E2E_USE_LATEST: ${{ github.event.inputs.use_latest == 'true' && '1' || '' }} + E2E_VERSION_OPENCODE: ${{ github.event.inputs.opencode_version }} + E2E_VERSION_CLAUDE_CODE: ${{ github.event.inputs.claude_code_version }} + E2E_VERSION_CODEX: ${{ github.event.inputs.codex_version }} + E2E_VERSION_COPILOT: ${{ github.event.inputs.copilot_version }} + E2E_VERSION_PI: ${{ github.event.inputs.pi_version }} + E2E_MODEL_OPENCODE: ${{ needs.model.outputs.gateway }} + E2E_MODEL_CODEX: ${{ needs.model.outputs.gateway }} + E2E_MODEL_COPILOT: ${{ needs.model.outputs.gateway }} + E2E_MODEL_PI: ${{ needs.model.outputs.gateway }} + E2E_MODEL_CLAUDE_CODE: ${{ needs.model.outputs.claude_code }} + # WSL only forwards vars listed in WSLENV — include every job/step env + # var the Go tests read via os.Getenv(). Without it, SKAINET_TOKEN, + # E2E_MODEL_*, etc. are unbound inside wsl-bash. + WSLENV: GITHUB_OUTPUT:GITHUB_STEP_SUMMARY:SKAINET_TOKEN:SKAINET_INTERNAL:ANTHROPIC_BASE_URL:E2E_HARNESS:E2E_USE_LATEST:E2E_VERSION_OPENCODE:E2E_VERSION_CLAUDE_CODE:E2E_VERSION_CODEX:E2E_VERSION_COPILOT:E2E_VERSION_PI:E2E_MODEL_OPENCODE:E2E_MODEL_CODEX:E2E_MODEL_COPILOT:E2E_MODEL_PI:E2E_MODEL_CLAUDE_CODE:E2E_LOG_DIR:E2E_CACHE_DIR + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Strip CRLF from shell and TypeScript files + # actions/checkout defaults to core.autocrlf=true on Windows, which + # converts LF->CRLF. CRLF breaks bash scripts inside WSL (e.g. + # `set -euo pipefail\r` → exit 2) and can cause issues with TypeScript + # tooling. Strip \r from .sh and .ts files so WSL sees clean LF. + shell: bash + run: find . \( -name '*.sh' -o -name '*.ts' \) -not -path './.git/*' -exec sed -i 's/\r$//' {} + + + - name: Set up WSL2 (Ubuntu 24.04) + uses: Vampire/setup-wsl@v7 + with: + distribution: Ubuntu-24.04 + additional-packages: bubblewrap gcc libc6-dev unzip python3-pip + # Run wsl-bash steps as a non-root user so DAC protects root-owned + # paths (/etc/shadow, /root/.bashrc) — matches Linux CI behavior. + wsl-shell-user: runner + # update: 'true' # makes it very slow + + - name: Set up Go and Node in WSL2 (as root) + shell: wsl-bash -u root {0} + run: | + GO_VERSION=$(grep '^go ' go.mod | tr -d '\r' | awk '{print $2}') + curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" | tar -C /usr/local -xz + curl -fsSL https://deb.nodesource.com/setup_24.x | bash - + apt-get install -y nodejs + + - name: Set up Bun and Rust in WSL2 + shell: wsl-bash {0} + run: | + curl -fsSL https://bun.sh/install | bash + curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable + # wsl-bash uses --noprofile --norc, so /etc/profile.d is not sourced. + # Make Go, bun, and cargo available to subsequent steps, and translate + # GitHub Actions file paths (Windows backslash paths) to WSL paths. + cat > "$HOME"/wsl-env.sh <<'WEOF' + export PATH="/usr/local/go/bin:$HOME/.bun/bin:$HOME/.cargo/bin:$PATH" + if [ -n "${GITHUB_OUTPUT:-}" ]; then + export GITHUB_OUTPUT=$(wslpath -u "$GITHUB_OUTPUT" 2>/dev/null || echo "$GITHUB_OUTPUT") + fi + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + export GITHUB_STEP_SUMMARY=$(wslpath -u "$GITHUB_STEP_SUMMARY" 2>/dev/null || echo "$GITHUB_STEP_SUMMARY") + fi + if [ -n "${E2E_LOG_DIR:-}" ]; then + export E2E_LOG_DIR=$(wslpath -u "$E2E_LOG_DIR" 2>/dev/null || echo "$E2E_LOG_DIR") + fi + if [ -n "${E2E_CACHE_DIR:-}" ]; then + export E2E_CACHE_DIR=$(wslpath -u "$E2E_CACHE_DIR" 2>/dev/null || echo "$E2E_CACHE_DIR") + fi + WEOF + + - name: Verify module tidy + shell: wsl-bash {0} + run: | + source "$HOME"/wsl-env.sh + go mod download && go mod verify + + - name: Resolve model + id: model + shell: wsl-bash {0} + run: | + source "$HOME"/wsl-env.sh + model=$(scripts/resolve-model.sh '${{ matrix.harness }}') + context="${E2E_CONTEXT_LIMIT:-versions.go default}" + echo "model=$model" >> "$GITHUB_OUTPUT" + echo "context=$context" >> "$GITHUB_OUTPUT" + echo "::notice title=Model::${{ matrix.harness }} runs against $model (context: $context)" + + - name: Cache harness install packages + uses: actions/cache@v4 + with: + path: ${{ runner.temp }}/e2e-install-cache + key: e2e-install-Windows-${{ matrix.harness }}-${{ hashFiles('internal/e2e/versions.go') }}-${{ github.event.inputs.use_latest }}-${{ github.event.inputs.opencode_version }}-${{ github.event.inputs.claude_code_version }}-${{ github.event.inputs.codex_version }}-${{ github.event.inputs.copilot_version }}-${{ github.event.inputs.pi_version }} + restore-keys: | + e2e-install-Windows-${{ matrix.harness }}- + + - name: Deterministic test gate + if: matrix.harness != 'claude-code' || github.event.inputs.skip_claude_code != 'true' + shell: wsl-bash {0} + run: | + source "$HOME"/wsl-env.sh + set -o pipefail + if ! go test -tags=e2e_fast -count=1 ./internal/e2e/ 2>&1 | tee /tmp/e2e-fast.txt; then + echo "::error::deterministic (e2e_fast) test failure — a test defect, not infra; retries cannot clear it" + { + echo "## E2E deterministic gate FAILED (${{ matrix.harness }} / WSL2)" + echo "" + echo "Network-free tests failed against the resolved model/version env." + echo "This is a test defect, not infra — the agent legs were not run." + echo "" + echo '```' + grep -E '^( *--- FAIL|.*_test\.go:)' /tmp/e2e-fast.txt | head -20 + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + exit 1 + fi + + - name: Run e2e test + id: run_e2e + if: matrix.harness != 'claude-code' || github.event.inputs.skip_claude_code != 'true' + shell: wsl-bash {0} + env: + E2E_LOG_DIR: ${{ runner.temp }}/e2e-logs + E2E_CACHE_DIR: ${{ runner.temp }}/e2e-install-cache + run: | + source "$HOME"/wsl-env.sh + set -o pipefail + max_attempts=3 + attempt=1 + while :; do + echo "::group::e2e attempt $attempt/$max_attempts" + if go test -tags=e2e -timeout=30m -v ./internal/e2e/ 2>&1 | tee /tmp/e2e-out.txt; then + echo "::endgroup::" + exit 0 + fi + echo "::endgroup::" + if grep -q '^panic: test timed out after' /tmp/e2e-out.txt; then + hung=$(sed -n '/^panic: test timed out after/,/^$/p' /tmp/e2e-out.txt | head -20) + echo "::error::go test hit its deadline — a hung test, not infra; retries cannot clear it" + echo "test(s) still running at the deadline:" + echo "$hung" + { + echo "## E2E deadline exceeded (${{ matrix.harness }} / WSL2)" + echo '```' + echo "$hung" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + exit 1 + fi + bad=$(grep -oE 'FAIL \[[^]]+\]: (SANDBOX_FAIL|AGENT_REFUSED|AGENT_PARTIAL)' /tmp/e2e-out.txt | sort -u || true) + if [ -n "$bad" ]; then + echo "non-infra failure — not retrying:" + echo "$bad" + exit 1 + fi + if [ "$attempt" -ge "$max_attempts" ]; then + echo "max attempts ($max_attempts) reached — giving up" + exit 1 + fi + attempt=$((attempt + 1)) + echo "infra-only failure — retrying in 15s (attempt $attempt/$max_attempts)" + sleep 15 + done + + - name: Skipped notice + if: steps.run_e2e.outcome == 'skipped' + shell: wsl-bash {0} + run: | + source "$HOME"/wsl-env.sh + echo "## E2E (${{ matrix.harness }} / WSL2): SKIPPED" >> "$GITHUB_STEP_SUMMARY" + echo "skip_claude_code was set on this workflow_dispatch run (model would have been \`${{ steps.model.outputs.model }}\`)." >> "$GITHUB_STEP_SUMMARY" + + - name: Failure classification summary + if: always() && steps.run_e2e.outcome != 'skipped' + shell: wsl-bash {0} + run: | + source "$HOME"/wsl-env.sh + echo "## E2E failure classification (${{ matrix.harness }} / WSL2)" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "Model: \`${{ steps.model.outputs.model }}\` · context: \`${{ steps.model.outputs.context }}\`" >> "$GITHUB_STEP_SUMMARY" + if [ '${{ needs.model.outputs.fallback }}' = 'true' ] && [ '${{ matrix.harness }}' != 'claude-code' ]; then + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "> ⚠️ Fallback model — the intended model is not served by the gateway, so this result does NOT certify it." >> "$GITHUB_STEP_SUMMARY" + fi + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "| Assertion | Mode | Reason |" >> "$GITHUB_STEP_SUMMARY" + echo "|-----------|------|--------|" >> "$GITHUB_STEP_SUMMARY" + if grep -qE 'FAIL \[[^]]+\]: [A-Z_]+ — ' /tmp/e2e-out.txt; then + grep -oE 'FAIL \[[^]]+\]: [A-Z_]+ — [^\\]*' /tmp/e2e-out.txt \ + | sort -u \ + | sed -E 's/FAIL \[([^]]+)\]: ([A-Z_]+) — (.*)/| `\1` | `\2` | \3 |/' \ + >> "$GITHUB_STEP_SUMMARY" + else + echo "| _all passed_ | PASS | no failures |" >> "$GITHUB_STEP_SUMMARY" + fi + cat "$GITHUB_STEP_SUMMARY" + + - name: Upload session artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-logs-${{ matrix.harness }}-wsl2 + path: ${{ runner.temp }}/e2e-logs/ + if-no-files-found: ignore + retention-days: 14 + + # ── codewhale drift ──────────────────────────────────────────────── + # Non-blocking: runs the full e2e suite for codewhale only, on Linux. + # Failures are reported but never block releases or the main matrix. + # To fully remove codewhale: delete this job + the codewhale_version + # input + the Go harness config (harnesses.go, versions.go, etc.). + e2e-codewhale: + name: "codewhale / ubuntu-latest (drift)" + needs: model + continue-on-error: true + runs-on: ubuntu-latest + concurrency: + group: e2e-codewhale + cancel-in-progress: false + env: + SKAINET_TOKEN: ${{ secrets.SKAINET_TOKEN }} + SKAINET_INTERNAL: ${{ secrets.SKAINET_INTERNAL }} + ANTHROPIC_BASE_URL: ${{ secrets.ANTHROPIC_BASE_URL }} + E2E_HARNESS: codewhale + E2E_USE_LATEST: ${{ github.event.inputs.use_latest == 'true' && '1' || '' }} + E2E_VERSION_CODEWHALE: ${{ github.event.inputs.codewhale_version }} + E2E_MODEL_CODEWHALE: ${{ needs.model.outputs.gateway }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Resolve model + id: model + run: | + model=$(scripts/resolve-model.sh 'codewhale') + context="${E2E_CONTEXT_LIMIT:-versions.go default}" + echo "model=$model" >> "$GITHUB_OUTPUT" + echo "context=$context" >> "$GITHUB_OUTPUT" + echo "::notice title=Model::codewhale runs against $model (context: $context)" + + - name: Install bubblewrap (Linux only) + if: runner.os == 'Linux' + run: | + sudo apt-get update && sudo apt-get install -y bubblewrap + sudo tee /etc/apparmor.d/bwrap > /dev/null <<'EOF' + abi , + /usr/bin/bwrap flags=(unconfined) { + userns, + } + EOF + sudo apparmor_parser -r /etc/apparmor.d/bwrap + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Set up Bun + uses: oven-sh/setup-bun@v2 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: '24' + + - name: Cache harness install packages + uses: actions/cache@v4 + with: + path: ${{ runner.temp }}/e2e-install-cache + key: e2e-install-${{ runner.os }}-codewhale-${{ hashFiles('internal/e2e/versions.go') }}-${{ github.event.inputs.use_latest }}-${{ github.event.inputs.codewhale_version }} + restore-keys: | + e2e-install-${{ runner.os }}-codewhale- + + - name: Deterministic test gate + run: | + set -o pipefail + if ! go test -tags=e2e_fast -count=1 ./internal/e2e/ 2>&1 | tee /tmp/e2e-fast.txt; then + echo "::error::deterministic (e2e_fast) test failure — a test defect, not infra; retries cannot clear it" + { + echo "## E2E deterministic gate FAILED (codewhale / ubuntu-latest)" + echo "" + echo "Network-free tests failed against the resolved model/version env." + echo "This is a test defect, not infra — the agent legs were not run." + echo "" + echo '```' + grep -E '^( *--- FAIL|.*_test\.go:)' /tmp/e2e-fast.txt | head -20 + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + exit 1 + fi + + - name: Run e2e test + id: run_e2e + env: + E2E_LOG_DIR: ${{ runner.temp }}/e2e-logs + E2E_CACHE_DIR: ${{ runner.temp }}/e2e-install-cache + run: | + set -o pipefail + max_attempts=3 + attempt=1 + while :; do + echo "::group::e2e attempt $attempt/$max_attempts" + if go test -tags=e2e -timeout=30m -v ./internal/e2e/ 2>&1 | tee /tmp/e2e-out.txt; then + echo "::endgroup::" + exit 0 + fi + echo "::endgroup::" + if grep -q '^panic: test timed out after' /tmp/e2e-out.txt; then + hung=$(sed -n '/^panic: test timed out after/,/^$/p' /tmp/e2e-out.txt | head -20) + echo "::error::go test hit its deadline — a hung test, not infra; retries cannot clear it" + echo "test(s) still running at the deadline:" + echo "$hung" + { + echo "## E2E deadline exceeded (codewhale / ubuntu-latest)" + echo '```' + echo "$hung" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + exit 1 + fi + bad=$(grep -oE 'FAIL \[[^]]+\]: (SANDBOX_FAIL|AGENT_REFUSED|AGENT_PARTIAL)' /tmp/e2e-out.txt | sort -u || true) + if [ -n "$bad" ]; then + echo "non-infra failure — not retrying:" + echo "$bad" + exit 1 + fi + if [ "$attempt" -ge "$max_attempts" ]; then + echo "max attempts ($max_attempts) reached — giving up" + exit 1 + fi + attempt=$((attempt + 1)) + echo "infra-only failure — retrying in 15s (attempt $attempt/$max_attempts)" + sleep 15 + done + + - name: Failure classification summary + if: always() && steps.run_e2e.outcome != 'skipped' + run: | + echo "## E2E failure classification (codewhale / ubuntu-latest)" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "Model: \`${{ steps.model.outputs.model }}\` · context: \`${{ steps.model.outputs.context }}\`" >> "$GITHUB_STEP_SUMMARY" + if [ '${{ needs.model.outputs.fallback }}' = 'true' ]; then + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "> ⚠️ Fallback model — the intended model is not served by the gateway, so this result does NOT certify it." >> "$GITHUB_STEP_SUMMARY" + fi + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "| Assertion | Mode | Reason |" >> "$GITHUB_STEP_SUMMARY" + echo "|-----------|------|--------|" >> "$GITHUB_STEP_SUMMARY" + if grep -qE 'FAIL \[[^]]+\]: [A-Z_]+ — ' /tmp/e2e-out.txt; then + grep -oE 'FAIL \[[^]]+\]: [A-Z_]+ — [^\\]*' /tmp/e2e-out.txt \ + | sort -u \ + | sed -E 's/FAIL \[([^]]+)\]: ([A-Z_]+) — (.*)/| `\1` | `\2` | \3 |/' \ + >> "$GITHUB_STEP_SUMMARY" + else + echo "| _all passed_ | PASS | no failures |" >> "$GITHUB_STEP_SUMMARY" + fi + cat "$GITHUB_STEP_SUMMARY" + + - name: Upload session artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-logs-codewhale-ubuntu-latest + path: ${{ runner.temp }}/e2e-logs/ + if-no-files-found: ignore + retention-days: 14 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b4ed0ff8..7e33d27d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -143,7 +143,7 @@ jobs: # Optional: when present, the thread bullets are rewritten by the LLM # into user-facing highlights. Absent -> deterministic commit bullets. SKAINET_TOKEN: ${{ secrets.SKAINET_TOKEN }} - SKAINET_INTERNAL: ${{ secrets.SKAINET_EXTERNAL }} + SKAINET_INTERNAL: ${{ secrets.SKAINET_INTERNAL }} # Read by scripts/probe-model.sh below. Empty on a tag push (the # normal path), so a real release starts from the committed pin. E2E_MODEL: ${{ github.event.inputs.model }} @@ -207,9 +207,9 @@ jobs: sys="You are the release-notes editor for oh-my-agentic-coder (omac), a CLI that runs coding agents in a sandbox. Below are the raw git commit messages for one release. Rewrite them into concise, user-facing highlights grouped into features (new capabilities) and fixes (bugs resolved). Write for end users: state what they can now do and why it matters. IGNORE anything only project developers care about: PR/issue numbers like (#123), commit SHAs, author or co-author trailers, internal refactors, and test/ci/build/chore changes. Drop internal module and scope names. Merge related commits into one bullet. Output ONLY minified JSON of the form {\"features\":[\"...\"],\"fixes\":[\"...\"]} with at most 6 items per group, each under 140 characters and starting with a capital letter. Use an empty array for a group with nothing user-facing. No preamble, no explanation, no chain-of-thought, no tags." req=$(jq -n --arg m "$model" --arg s "$sys" --arg u "$commits" \ '{model:$m,temperature:0.2,max_tokens:1500,messages:[{role:"system",content:$s},{role:"user",content:$u}]}') - # X-Separate-Reasoning routes GLM reasoning out of message.content. + # X-Separate-Reasoning routes model reasoning out of message.content. # Generous timeout: release notes aren't latency-sensitive, and - # GLM-5.2 reasoning over a full release can take a while. Better a + # the selected model reasoning over a full release can take a while. Better a # slow, good summary than falling back to raw dev-facing bullets. resp=$(curl -sS --max-time 300 "$SKAINET_INTERNAL/chat/completions" \ -H "Authorization: Bearer $SKAINET_TOKEN" -H 'Content-Type: application/json' \ diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 7c52f739..56dd00d6 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -29,9 +29,9 @@ on: options: [quick, standard, deep] default: deep model: - description: 'Model id for the strix agent, for THIS RUN ONLY. Leave empty for the opencode pin in internal/e2e/versions.go. Whatever is resolved here still goes through the gateway probe, so the -TEE variant flip and the fallback chain apply' + description: 'Model id for the strix agent, for THIS RUN ONLY. Set empty to use each harness''s pin in internal/e2e/versions.go. Whatever is resolved here still goes through the gateway probe, so the -TEE variant flip and the fallback chain apply.' type: string - default: '' + default: 'deepseek-ai/DeepSeek-V4-Flash' permissions: contents: read @@ -47,11 +47,9 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 180 env: - # SKAINET_EXTERNAL is the same secret e2e.yml uses for the model - # provider base URL (named for "reachable from GitHub-hosted CI", - # as opposed to a TNG-internal-only address) -- reused here rather - # than duplicating the URL. - LLM_API_BASE: ${{ secrets.SKAINET_EXTERNAL }} + # SKAINET_INTERNAL is the same secret e2e.yml uses for the model + # provider base URL — reused here rather than duplicating the URL. + LLM_API_BASE: ${{ secrets.SKAINET_INTERNAL }} LLM_API_KEY: ${{ secrets.SKAINET_TOKEN }} GH_TOKEN: ${{ github.token }} # Read by scripts/probe-model.sh below; empty on the scheduled run, so @@ -90,7 +88,7 @@ jobs: id: proxy # strix resends the whole conversation on every request and has no # context compaction, so a long scan overflows the model window - # (GLM-5.2: 202752 tokens) and every request then 422s with + # and every request then 422s with # context_length_exceeded -- silently truncating results (a partial run # dir survives, so the scan can look finished). This local proxy # compacts requests in flight (summarising the oldest exchanges); strix diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 12755783..c53f2a9a 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -78,6 +78,9 @@ func Run(args []string, version string) int { fmt.Fprintln(env.Stderr, "omac: cannot absolutize workdir:", err) return ExitIOError } + if resolved, err := filepath.EvalSymlinks(abs); err == nil { + abs = resolved + } env.Workdir = abs if len(subArgs) == 0 { diff --git a/internal/e2e/e2e_test.go b/internal/e2e/e2e_test.go index 43a40a27..8c954923 100644 --- a/internal/e2e/e2e_test.go +++ b/internal/e2e/e2e_test.go @@ -153,13 +153,22 @@ func runE2E(t *testing.T, h harnessConfig) { // it's immune to the agent paraphrasing/summarizing the JSON instead // of reproducing it verbatim. Fall back to agent stdout if the agent // didn't create the file (e.g. it refused or went off-script). + // + // Skip for harnesses that strip OMAC_* from exec_shell subprocesses + // (codewhale exec mode): the curl command can't reach the sidecar + // without OMAC_ECHO_BASE. The workdir write/read and git commit + // assertions below still run — they don't depend on OMAC_*. fileContent, err := os.ReadFile(echoOutputFile) if err != nil { t.Logf("echo-status.txt not found (%v) — falling back to agent stdout", err) } else { t.Logf("echo-status.txt read: %d bytes", len(fileContent)) } - assertEchoOK(t, string(fileContent)+"\n"+stdout) + if h.Sandbox.StripsEnvVars { + t.Logf("SKIP: echoOK — %s strips OMAC_* from exec_shell subprocesses (child_env env_clear in exec mode; hooks only fire in the TUI)", h.Name) + } else { + assertEchoOK(t, string(fileContent)+"\n"+stdout) + } // 9. Assert the workdir write/read and git commit actually happened — // read the files directly rather than trusting the agent's prose, @@ -378,21 +387,34 @@ func runSecurityAudit(t *testing.T, h harnessConfig) { // --- POSITIVE assertions (things that MUST happen) --- - // Sidecar should be reachable regardless of sandbox state. - assertSecretFingerprintPresent(t, stdout) + // Sidecar, env-visibility, and cache-isolation assertions all depend + // on OMAC_* vars reaching the bash subprocess. Harnesses that strip + // OMAC_* in exec mode (codewhale: child_env env_clear, shell_env hook + // doesn't fire in exec) skip these — they'd fail on the harness's env + // stripping, not on a sandbox defect. Filesystem-allow assertions + // don't depend on OMAC_* and still run. + if h.Sandbox.StripsEnvVars { + t.Logf("SKIP: sidecarReachable, envVarsVisible, cacheIsolation — %s strips OMAC_* from exec_shell subprocesses (child_env env_clear in exec mode; hooks only fire in the TUI)", h.Name) + } else { + // Sidecar should be reachable regardless of sandbox state. + assertSecretFingerprintPresent(t, stdout) - // Positive network: the loopback server must be reachable, so a - // deny-all-egress regression is caught. Runs unconditionally. - assertNetworkReachable(t, stdout, netOkMarker) + if sandboxActive { + assertEnvVarsVisible(t, stdout, spec.EnvExpectVisible) + assertCacheIsolation(t, stdout, spec.ExpectedCacheMode) + } + } if sandboxActive { - assertEnvVarsVisible(t, stdout, spec.EnvExpectVisible) assertFilesystemAllowed(t, stdout, spec.FsAllowLabels) - assertCacheIsolation(t, stdout, spec.ExpectedCacheMode) } else { - t.Logf("skipping positive env/fs-allow assertions: %s runs with --no-sandbox", h.Name) + t.Logf("skipping positive fs-allow assertions: %s runs with --no-sandbox", h.Name) } + // Positive network: the loopback server must be reachable, so a + // deny-all-egress regression is caught. Runs unconditionally. + assertNetworkReachable(t, stdout, netOkMarker) + // --- DOCUMENTATION probes (log current behavior, no pass/fail) --- // Exec on read-only mounts: bwrap typically allows exec on read-only diff --git a/internal/e2e/harnesses.go b/internal/e2e/harnesses.go index ee420e51..ca4b03ef 100644 --- a/internal/e2e/harnesses.go +++ b/internal/e2e/harnesses.go @@ -97,6 +97,17 @@ type SandboxConfig struct { // Used when the harness's own runtime is incompatible with the // sandbox mechanism (e.g. codex's Rust HTTP client on macOS). NoSandbox bool + + // StripsEnvVars reports whether the harness strips OMAC_* env vars + // from tool subprocesses in its headless/exec mode. CodeWhale's + // child_env module calls cmd.env_clear() and rebuilds from a fixed + // allowlist that excludes OMAC_* — and its shell_env hook (which + // re-injects them) only fires in the interactive TUI, not in + // `codewhale exec` (the headless mode the e2e tests use). When true, + // sidecar-reachability, cache-isolation, env-var-visibility, and + // echo-rest assertions are skipped: they would all fail on missing + // OMAC_* vars that the harness dropped, not on a sandbox defect. + StripsEnvVars bool } // allHarnesses returns the harnesses eligible on this host. codex is @@ -649,9 +660,12 @@ func piConfig() harnessConfig { // generic OpenAI-compatible route: config.example.toml documents // `provider = "openai"` + a `[providers.openai]` table for "generic // OpenAI-compatible gateways" (config.example.toml:354). We point that -// provider's base_url at SKAINET_INTERNAL and set path_suffix = "/chat/ -// completions" so requests go to /chat/completions — the same shape -// opencode's @ai-sdk/openai-compatible client uses against this gateway. +// provider's base_url at SKAINET_INTERNAL (which already includes /v1) and +// rely on the default chat-completions path: codewhale appends +// /chat/completions to a versioned base_url, yielding /chat/completions +// — the same shape opencode's @ai-sdk/openai-compatible client uses. +// path_suffix is deliberately NOT set: that field strips the version segment +// (e.g. /v1) from base_url before appending, which would drop /v1 and result in 404. // // Env vars: OPENAI_API_KEY carries the bearer token. CodeWhale's openai // provider reads it from the process env (crates/config/src/provider.rs:546), @@ -669,6 +683,20 @@ func piConfig() harnessConfig { // api_key_env = "OPENAI_API_KEY": CodeWhale refuses an ambient key on a custom // base_url unless it is bound explicitly. // +// shell_env hook (EXPERIMENTAL, TUI-only): CodeWhale's child_env calls +// cmd.env_clear() and rebuilds from a fixed allowlist that excludes +// OMAC_* — so bash tool subprocesses can't see OMAC_ECHO_BASE, +// OMAC_CACHE_DIR, etc. The [hooks] shell_env entry re-injects OMAC_* +// and tool cache mappings from CodeWhale's own process env before each +// exec_shell. OPENAI_API_KEY is deliberately NOT re-injected. +// +// Hooks only fire in the interactive TUI, NOT in `codewhale exec` +// (headless mode). The e2e tests use `codewhale exec --auto`, so the +// hook is a no-op during tests and OMAC_* is stripped from bash +// subprocesses. StripsEnvVars=true skips sidecar, cache, env-visibility, +// and echo-rest assertions. Real TUI users get the hook — see +// docs/HARNESSES.md. +// // Sandbox deviations: none expected — the model host (SKAINET_INTERNAL) is // allowed by the base profile and the update check is disabled. This has NOT // been verified against a live gateway run (unlike opencode/pi, which were); @@ -725,11 +753,21 @@ sandbox_mode = "external-sandbox" [providers.openai] base_url = "` + baseURL + `" -path_suffix = "/chat/completions" api_key_env = "OPENAI_API_KEY" [update] check_for_updates = false + +# shell_env hook (EXPERIMENTAL, TUI-only): re-injects OMAC_* and tool cache +# mappings that CodeWhale's child_env strips from bash subprocesses. Only +# fires in the interactive TUI, NOT in codewhale exec (headless mode). +# See docs/HARNESSES.md. +[hooks] +enabled = true + +[[hooks.hooks]] +event = "shell_env" +command = "env | grep -E '^(OMAC_|GOCACHE=|GOMODCACHE=|NPM_CONFIG_CACHE=|NPM_CONFIG_PREFIX=|PIP_CACHE_DIR=|XDG_CACHE_HOME=|XDG_CONFIG_HOME=|XDG_DATA_HOME=|XDG_STATE_HOME=|PWD=|BUN_INSTALL_CACHE_DIR=)' || true" ` if err := os.WriteFile(filepath.Join(cwDir, "config.toml"), []byte(configToml), 0o644); err != nil { t.Fatal(err) @@ -744,7 +782,13 @@ check_for_updates = false // CodeWhale's openai provider reads OPENAI_API_KEY from the env. return []string{"OPENAI_API_KEY=" + token} }, - Sandbox: SandboxConfig{}, // no deviations expected — see the doc comment (unverified live) + Sandbox: SandboxConfig{ + // codewhale exec strips OMAC_* from bash subprocesses (child_env + // env_clear + fixed allowlist). The shell_env hook would re-inject + // them but only fires in the TUI, not in exec mode — see the doc + // comment above. Skip sidecar/cache/echo assertions. + StripsEnvVars: true, + }, RunArgs: func(prompt string) []string { // `exec` leads so the contract deriver captures it as a subcommand // (flagsAndSub only treats a LEADING positional as the subcommand; @@ -759,7 +803,9 @@ check_for_updates = false return []string{"OPENAI_API_KEY"} }, ExpectVisibleEnv: func() []string { - return []string{"OPENAI_API_KEY=", "OMAC_"} + // OMAC_* is NOT visible in bash tool output under exec mode: + // child_env strips it and the shell_env hook doesn't fire. + return nil }, } } diff --git a/internal/e2e/intent_test.go b/internal/e2e/intent_test.go index f3263e3c..1e901362 100644 --- a/internal/e2e/intent_test.go +++ b/internal/e2e/intent_test.go @@ -255,9 +255,9 @@ func TestE2EIntentExplainRecovery(t *testing.T) { tailLines(stderr, 100)) } - // The explain-more hint (distinctive phrase "fuller reason") must reach + // The explain-more hint (distinctive phrase "fuller, specific reason") must reach // the agent via the GET channel — the whole point of the fix. - if !strings.Contains(stdout, "fuller reason") { + if !strings.Contains(stdout, "fuller, specific reason") { t.Errorf("agent output missing explain-more hint from GET /sandbox/intent "+ "(the HTTPS re-ask did not survive)\nSTDOUT (last 200 lines):\n%s", tailLines(stdout, 200)) diff --git a/internal/sandboxrun/integration_worktree_linux_test.go b/internal/sandboxrun/integration_worktree_linux_test.go index abee36ba..987fd9d9 100644 --- a/internal/sandboxrun/integration_worktree_linux_test.go +++ b/internal/sandboxrun/integration_worktree_linux_test.go @@ -9,6 +9,7 @@ import ( "strings" "testing" + "github.com/tngtech/oh-my-agentic-coder/internal/osinfo" "github.com/tngtech/oh-my-agentic-coder/internal/sandboxprofile" ) @@ -169,6 +170,26 @@ func TestIntegrationWorktreeSymlinkEscape(t *testing.T) { } } +// allFatalLinesAreEBUSYPackedRefs reports whether every "fatal:" line in out +// is the WSL2 packed-refs EBUSY quirk (the EBUSY replace + its "failed to run +// pack-refs" consequence). A genuine corruption fatal alongside still fails. +func allFatalLinesAreEBUSYPackedRefs(out string) bool { + any := false + for _, line := range strings.Split(out, "\n") { + if !strings.Contains(strings.ToLower(line), "fatal") { + continue + } + any = true + isEBUSY := strings.Contains(line, "Device or resource busy") && + strings.Contains(line, "packed-refs") + isPackRefsFail := strings.Contains(line, "failed to run pack-refs") + if !isEBUSY && !isPackRefsFail { + return false + } + } + return any +} + // TestIntegrationWorktreeKnownLimitations verifies the PR body's "Known // limitation" claims under real bwrap: branch -d fails cleanly, the // packed-refs.lock EPERM is non-fatal (commit succeeds), gc no-ops, and @@ -232,7 +253,16 @@ func TestIntegrationWorktreeKnownLimitations(t *testing.T) { // fatal failures — a non-zero exit with mere warnings is acceptable. out, code = sandboxGit("gc", "--quiet") if code != 0 && strings.Contains(strings.ToLower(out), "fatal") { - t.Errorf("gc produced fatal output (warnings OK, fatals not):\n%s", out) + // WSL2 returns EBUSY (not EPERM) on read-only-bind rename, so gc's + // atomic replace of packed-refs fails loudly. Tolerate only if every + // fatal line is that quirk; a real corruption fatal still fails. + wslEBUSY := osinfo.Detect() == osinfo.WSL && + allFatalLinesAreEBUSYPackedRefs(out) + if wslEBUSY { + t.Logf("gc only emitted EBUSY fatal errors on WSL2 (known kernel quirk with bind-mount rename); treating as non-fatal:\n%s", out) + } else { + t.Errorf("gc produced fatal output (warnings OK, fatals not):\n%s", out) + } } // 3. branch -d should fail cleanly (exit non-zero), ref preserved. diff --git a/scripts/check-wsl2-coverage.py b/scripts/check-wsl2-coverage.py new file mode 100644 index 00000000..b3b824df --- /dev/null +++ b/scripts/check-wsl2-coverage.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +"""Verify every e2e*.yml workflow has WSL2 coverage. + +For each `.github/workflows/e2e*.yml` (minus the skip list), finds matrix +jobs whose `runs-on` is not Windows ("main" jobs) and requires a WSL2 +counterpart job (matrix + `runs-on` containing `windows`) in the same file. +Each `run:` step in a main job (excluding `if: runner.os == 'Linux'` steps) +must have a name-matched `run:` step in the WSL2 job. `uses:` steps are +not checked (actions run on the host; setup-* is replaced by a manual +install step in the WSL2 job). Job-level `env:` keys must also match — a +new E2E_* var on the main job without a WSL2 counterpart silently uses +the default/unset behavior. Additionally, every env var in the WSL2 job +(job-level + step-level) must appear in the job's WSLENV value — WSL only +forwards vars listed there, so a missing entry is silently unbound. + +Exit 0 when clean, 1 with a per-file/per-step report otherwise. +""" +import glob +import sys + +import yaml + +# Deliberately excluded from the WSL2 coverage requirement. Each value +# is a reason shown in diagnostics so the escape hatch stays visible. +SKIP = { + "e2e-readme-onboarding.yml": ( + "manual-trigger-only doc audit; README's WSL2 content is a pointer " + "to docs/INSTALLATION.md" + ), +} + + +def _basename(path): + return path.rsplit("/", 1)[-1] + + +# Env vars that are legitimately WSL2-only — present in the WSL2 job but not +# the main job, and exempt from the env-key parity check. +WSL2_ONLY_ENV = {"WSLENV", "SMOKE_LOG_DIR"} + + +def _is_windows(job): + runs_on = job.get("runs-on", "") + return isinstance(runs_on, str) and "windows" in runs_on.lower() + + +def _has_matrix(job): + return bool(job.get("strategy", {}).get("matrix")) + + +def _has_harness_matrix(job): + """True when the job's matrix has a `harness` dimension. + + Jobs without a harness dimension (e.g. codewhale drift jobs that only + vary `omac`) are exempt from WSL2 coverage — they're not part of the + main harness matrix and are explicitly Linux-only by design. + """ + matrix = job.get("strategy", {}).get("matrix") or {} + return "harness" in matrix + + +def _run_steps(job): + """Yield (index, name) for each `run:` step, skipping Linux-only steps.""" + for index, step in enumerate(job.get("steps") or []): + if not isinstance(step, dict) or not step.get("run"): + continue + if "runner.os == 'Linux'" in str(step.get("if", "")): + continue + yield index, step.get("name", "") + + +def _step_env_keys(job): + """Collect env var names from step-level `env:` blocks. + + Only steps using `wsl-bash` as their shell need WSLENV forwarding — + other shells (bash, pwsh) run on the Windows side with native env access. + """ + keys = set() + for step in job.get("steps") or []: + if not isinstance(step, dict) or not step.get("run"): + continue + shell = step.get("shell", "") + if "wsl-bash" not in str(shell): + continue + keys.update((step.get("env") or {}).keys()) + return keys + + +# Env vars set by GitHub Actions itself, not the workflow — always present +# inside wsl-bash without a WSLENV entry (the runner injects them), so +# they don't need to be listed. +GITHUB_INJECTED_ENV = {"HOME", "PATH", "PWD", "TMPDIR", "LANG", "TERM"} + + +def check_file(path): + """Yield diagnostic strings for WSL2 coverage gaps in one file.""" + try: + doc = yaml.safe_load(open(path)) or {} + except yaml.YAMLError as exc: + yield f"{path}: not parseable as YAML: {exc}" + return + + fname = _basename(path) + if fname in SKIP: + return + + jobs = doc.get("jobs") or {} + main_jobs, wsl2_jobs = {}, {} + for name, job in jobs.items(): + if not isinstance(job, dict) or not _has_matrix(job): + continue + if not _has_harness_matrix(job): + continue + (wsl2_jobs if _is_windows(job) else main_jobs)[name] = job + + if not main_jobs: + return + + if not wsl2_jobs: + yield ( + f"{path}: no WSL2 job found (matrix + runs-on: windows-latest). " + f"Add a WSL2 counterpart for main job(s): " + f"{', '.join(sorted(main_jobs))} " + f"(or add '{fname}' to SKIP in scripts/check-wsl2-coverage.py)" + ) + return + + wsl2_names = {n for wj in wsl2_jobs.values() for _, n in _run_steps(wj)} + for mname, mjob in sorted(main_jobs.items()): + for index, sname in _run_steps(mjob): + if sname not in wsl2_names: + yield ( + f"{path}: job '{mname}' step {index} ('{sname}'): " + f"no WSL2 counterpart in job(s) {', '.join(sorted(wsl2_jobs))}" + ) + + # Job-level env keys must match — a new E2E_* var on the main job + # without a WSL2 counterpart silently uses the default/unset behavior. + for mname, mjob in sorted(main_jobs.items()): + # Pair with the WSL2 job whose env keys overlap most (handles the + # single-pair case; if there are multiple WSL2 jobs, the closest + # match is the intended counterpart). + wjob = max( + wsl2_jobs.values(), + key=lambda j: len( + set(j.get("env", {})) & set(mjob.get("env", {})) + ), + ) + main_keys = set(mjob.get("env", {})) + wsl_keys = set(wjob.get("env", {})) + only_main = main_keys - wsl_keys + only_wsl = wsl_keys - main_keys + for key in sorted(only_main): + yield ( + f"{path}: job '{mname}' env '{key}': " + f"missing in WSL2 job" + ) + for key in sorted(only_wsl): + if key in WSL2_ONLY_ENV: + continue + yield ( + f"{path}: WSL2 job env '{key}': " + f"missing in main job '{mname}'" + ) + + # WSLENV coverage: every env var in the WSL2 job (job-level + step-level) + # must appear in the WSLENV value. WSL only forwards listed vars, so a + # missing entry is silently unbound inside wsl-bash (set -u aborts). + for wname, wjob in sorted(wsl2_jobs.items()): + wslenv_val = (wjob.get("env") or {}).get("WSLENV", "") + if not wslenv_val: + yield f"{path}: WSL2 job '{wname}' has no WSLENV env var" + continue + forwarded = set(wslenv_val.split(":")) + all_env = set((wjob.get("env") or {}).keys()) | _step_env_keys(wjob) + all_env.discard("WSLENV") + all_env -= GITHUB_INJECTED_ENV + missing = all_env - forwarded + for key in sorted(missing): + yield ( + f"{path}: WSL2 job '{wname}' env '{key}': " + f"missing in WSLENV" + ) + + +def main(): + paths = sorted(glob.glob(".github/workflows/e2e*.yml")) + if not paths: + print("no e2e*.yml workflow files found", file=sys.stderr) + return 1 + problems = [p for path in paths for p in check_file(path)] + for problem in problems: + print(problem, file=sys.stderr) + if problems: + print(f"\n{len(problems)} WSL2 coverage gap(s) found", file=sys.stderr) + return 1 + skipped = [p for p in paths if _basename(p) in SKIP] + checked = len(paths) - len(skipped) + print( + f"{checked} workflow file(s): all run steps have WSL2 coverage" + + (f" ({len(skipped)} skipped)" if skipped else "") + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main())