diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index efbfb361..bebf1fc7 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -752,13 +752,10 @@ jobs: export "STRIX_PROCESS_${budget_suffix}_SECONDS=$process_budget_seconds" export "STRIX_TOTAL_${budget_suffix}_SECONDS=5700" - # Capture the gate exit code plus its console output. The gate returns - # exit 1 both for genuine blocking vulnerabilities AND for - # LLM-backend-unavailable outcomes (GitHub Models "Too many requests" - # rate limits, OpenAI quota starvation, 413 tokens_limit_reached - # token-cap, connection/warm-up failures) that could not complete a scan. A backend outage is CI - # infrastructure noise, not a security finding, so it must not fail - # the required check and block merges. + # Capture the gate exit code plus its console output so the always-run + # artifact steps can preserve exact provider and finding evidence. + # A provider outage without a vulnerability report is incomplete + # security evidence, so the gate's nonzero result must remain nonzero. strix_run_log="$RUNNER_TEMP/strix_gate_console.log" strix_rc=0 set +e @@ -766,36 +763,9 @@ jobs: strix_rc="${PIPESTATUS[0]}" set -e - if [ "$strix_rc" -eq 0 ]; then - exit 0 - fi - - # Preserve configuration failures (exit 2) and any unexpected exit - # code as hard failures — only the scan-failure code (1) can be an - # infrastructure/backend-unavailability outcome. - if [ "$strix_rc" -ne 1 ]; then - exit "$strix_rc" + if [ "$strix_rc" -ne 0 ]; then + echo "::error title=Strix security gate failed::Strix returned exit ${strix_rc}. Provider outages without a vulnerability report are incomplete security evidence and are not converted to success. See the strix-reports artifact and run log." fi - - # Recognized signals that the LLM backend was unavailable / starved. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure' - # Any evidence that a vulnerability was actually reported. Its presence - # forces a hard failure so real findings are NEVER downgraded. Keep the - # severity branch anchored away from identifiers so environment lines - # such as STRIX_FAIL_ON_MIN_SEVERITY do not look like findings. - reported_vulnerability_signal='Vulnerabilities[[:space:]]+[1-9]|(^|[^A-Za-z0-9_])severity[[:space:]]*:' - - # Neutral skip only when ALL hold: a backend-unavailability signal is - # present and no vulnerability was reported anywhere. This preserves - # real security gating while keeping uncontrollable provider outages - # from blocking current-head merge progress. - if grep -Eiq "$backend_unavailable_signal" "$strix_run_log" \ - && ! grep -Eiq "$reported_vulnerability_signal" "$strix_run_log"; then - echo "::warning title=Strix backend unavailable::Strix could not complete because its LLM backend was unavailable (rate limit / token cap / connection or warm-up failure) before producing a vulnerability report. Treating as a neutral skip so an infrastructure outage does not block merges; genuine findings still fail the check. See the strix-reports artifact and the run log." - exit 0 - fi - - echo "Strix reported security findings or failed for a non-backend reason; failing the required check (gate exit ${strix_rc})." >&2 exit "$strix_rc" - name: Collect Strix reports for artifact upload diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index aae63c5d..8272b844 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -284,6 +284,9 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$workflow_file" "vertex_ai/gemini-3.1-pro-preview-customtools | vertex_ai/gemini-2.5-flash)" "strix workflow accepts only exact approved organization Vertex AI models" assert_file_contains "$workflow_file" 'STRIX_VERTEX_FALLBACK_MODELS: ""' "strix workflow disables silent Vertex fallbacks so timeout-class failures fail closed" assert_file_contains "$workflow_file" 'STRIX_FAIL_ON_PROVIDER_SIGNAL: "1"' "strix workflow fails closed on timeout, fatal, warning, denied, or provider failure signals" + assert_file_contains "$workflow_file" 'Provider outages without a vulnerability report are incomplete security evidence and are not converted to success' "strix workflow keeps provider exhaustion fail-closed" + assert_file_not_contains "$workflow_file" "Treating as a neutral skip" "strix workflow must not convert provider exhaustion into a successful security check" + assert_file_not_contains "$workflow_file" "backend_unavailable_signal=" "strix workflow must not recognize provider errors in order to downgrade the gate exit code" assert_file_contains "$workflow_file" 'NPM_CONFIG_IGNORE_SCRIPTS: "true"' "strix workflow disables npm lifecycle scripts for untrusted PR scan data" assert_file_contains "$workflow_file" 'PNPM_CONFIG_IGNORE_SCRIPTS: "true"' "strix workflow disables pnpm lifecycle scripts for untrusted PR scan data" assert_file_contains "$workflow_file" 'YARN_ENABLE_SCRIPTS: "false"' "strix workflow disables yarn lifecycle scripts for untrusted PR scan data" @@ -673,7 +676,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "not a generic model-exhaustion message" "opencode review tells models to return concrete missing-evidence findings instead of progress-only output" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "tokens_limit_reached" "opencode review detects provider context-window overflow" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "skipping remaining attempts for this model" "opencode review skips same-model retries after context-window overflow" - assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" "exceeded your current quota" "strix wrapper neutralizes quota-only provider failures without vulnerability reports" + assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" 'exit "$strix_rc"' "strix wrapper propagates provider-failure and finding exit codes without downgrading them" assert_file_contains "$REPO_ROOT/scripts/ci/strix_quick_gate.sh" "billing details" "strix quick gate classifies provider quota starvation as infrastructure" assert_file_contains "$workflow_file" 'timeout-minutes: 300' "opencode review target contains evidence, the bounded long-review pool, publication, and cleanup overhead" assert_file_contains "$workflow_file" 'timeout-minutes: 12' "opencode evidence preparation fails closed before it ties up the review queue" diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 8e66277f..0cd859b4 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -1,4 +1,5 @@ import json +import os import shlex import shutil import subprocess @@ -885,20 +886,46 @@ def test_optional_strix_workflow_absence_is_logged_without_failing_lookup() -> N assert 'if target_workflow_available "strix.yml"; then' in failed_check_evidence -def test_strix_provider_outage_without_findings_is_neutralized() -> None: +def test_strix_provider_outage_without_findings_fails_closed(tmp_path: Path) -> None: workflow = workflow_text("strix.yml") + run_step = workflow_step(workflow, "Run Strix (quick)") + run_marker = " run: |\n" + run_body = run_step.split(run_marker, 1)[1] + script = textwrap.dedent( + "\n".join(line[10:] for line in run_body.splitlines()) + ) - assert "RateLimitError|Too many requests" in workflow - assert "exceeded your current quota" in workflow - assert "billing details" in workflow - assert "LLM warm-up failed" in workflow - assert "zero_vulnerabilities_signal" not in workflow - assert "(^|[^A-Za-z0-9_])severity[[:space:]]*:" in workflow - assert "STRIX_FAIL_ON_MIN_SEVERITY: MEDIUM" in workflow - assert "before producing a vulnerability report" in workflow - assert "genuine findings still fail the check" in workflow - assert ( - '&& ! grep -Eiq "$reported_vulnerability_signal" "$strix_run_log"' in workflow + fake_gate = tmp_path / "fake-strix-gate.sh" + fake_gate.write_text( + "#!/usr/bin/env bash\n" + "echo 'openai.RateLimitError: exceeded your current quota'\n" + "echo 'Configured model and fallback models were unavailable.'\n" + "exit 1\n", + encoding="utf-8", + ) + fake_gate.chmod(0o700) + + result = subprocess.run( + ["bash"], + input=script, + capture_output=True, + text=True, + env={ + **os.environ, + "RUNNER_TEMP": str(tmp_path), + "TRUSTED_STRIX_GATE": str(fake_gate), + }, + check=False, + ) + + assert result.returncode == 1 + assert "Provider outages without a vulnerability report" in result.stdout + assert "not converted to success" in result.stdout + assert "Treating as a neutral skip" not in workflow + assert "backend_unavailable_signal=" not in workflow + assert (tmp_path / "strix_gate_console.log").read_text(encoding="utf-8") == ( + "openai.RateLimitError: exceeded your current quota\n" + "Configured model and fallback models were unavailable.\n" )