diff --git a/.github/workflows/opencode-review.yml b/.github/workflows/opencode-review.yml index 7010b262..4a083a0b 100644 --- a/.github/workflows/opencode-review.yml +++ b/.github/workflows/opencode-review.yml @@ -6118,6 +6118,7 @@ jobs: | select(((.name // "" | ascii_downcase) as $n | ["opencode-review","coverage-evidence","metadata-only gate evaluation"] | index($n)) | not) | select((.status // "") == "completed") | select((.conclusion // "" | ascii_upcase) as $c | ["FAILURE","TIMED_OUT","ACTION_REQUIRED","CANCELLED","STARTUP_FAILURE"] | index($c)) + | select(((.conclusion // "" | ascii_downcase) == "cancelled" and (.name // "") == "scan-pr-queue") | not) | select(((.conclusion // "" | ascii_downcase) == "cancelled" and ((.name // "") | contains("$" + "{{"))) | not) | select(((.conclusion // "" | ascii_downcase) == "cancelled" and (.name // "") == "noema-review") | not) | "- " + (if (.name // "") == "strix" then "Strix Security Scan/strix" else ((.name // "check") + " check run") end) + ": " + (.conclusion // "unknown") + (if (.details_url // .html_url // "") != "" then " (" + (.details_url // .html_url) + ")" else "" end) diff --git a/scripts/ci/opencode_review_normalize_output.py b/scripts/ci/opencode_review_normalize_output.py index 4045d457..cabfc73f 100755 --- a/scripts/ci/opencode_review_normalize_output.py +++ b/scripts/ci/opencode_review_normalize_output.py @@ -653,6 +653,145 @@ def adversarial_probe_source_receipt_error( return "" +def receipt_verified_evidence_location( + evidence: str, + changed_files: frozenset[str], +) -> tuple[str, int] | None: + """Return one evidence citation uniquely bound to its trusted source receipt. + + A model can place a valid changed-file ``path:line`` citation and its exact + source-line receipt in ``evidence`` while copying a different location into + the redundant structured fields. Consider only safe current-head changed + files cited by the evidence, recompute every cited line receipt from the + sealed source tree, and return a location only when exactly one distinct + citation matches the single model receipt. Ambiguous or unverified evidence + remains unrepairable and fails closed. + """ + receipts = SOURCE_LINE_RECEIPT_RE.findall(evidence) + if len(receipts) != 1: + return None + receipt = receipts[0].casefold() + matches: set[tuple[str, int]] = set() + for path in changed_files: + escaped_path = rf"(? dict[str, Any] | Any: + """Bind trusted source receipts and locations into otherwise valid evidence. + + Models sometimes place the exact changed-file path and positive line in the + structured ``path``/``line`` fields but omit the duplicate ``path:line`` text + from ``evidence``, or cite and receipt-bind one changed source line in + ``evidence`` while copying a different valid changed-file location into the + redundant structured fields. Restore only a missing citation or rebind the + structured location when exactly one cited line matches the existing single + receipt in the sealed current-head tree. Missing, duplicate, ambiguous, or + mismatched receipts, unsafe paths, missing changed-file evidence, and + circular or unobserved claims remain unmodified and fail closed. + """ + if not isinstance(value, dict): + return value + validation = value.get("adversarial_validation") + if not isinstance(validation, dict): + return value + probes = validation.get("probes") + if not isinstance(probes, list): + return value + + changed_files = current_changed_files() + repaired_probes: list[Any] = [] + changed = False + for probe in probes: + if not isinstance(probe, dict): + repaired_probes.append(probe) + continue + path = probe.get("path") + line = probe.get("line") + evidence = probe.get("evidence") + if ( + not isinstance(path, str) + or path not in changed_files + or isinstance(line, bool) + or not isinstance(line, int) + or line <= 0 + or not isinstance(evidence, str) + or not evidence.strip() + or adversarial_probe_location_error(path, line) + ): + repaired_probes.append(probe) + continue + + receipts = SOURCE_LINE_RECEIPT_RE.findall(evidence) + if len(receipts) != 1: + repaired_probes.append(probe) + continue + repaired_evidence = evidence + repaired_path = path + repaired_line = line + rejection = adversarial_evidence_rejection_reason(repaired_evidence, path, line) + if rejection == "must cite the exact probe path and positive line": + if not adversarial_probe_source_receipt_error(evidence, path, line): + repaired_evidence = f"{path}:{line} {repaired_evidence}" + else: + rebound_location = receipt_verified_evidence_location( + evidence, + changed_files, + ) + if rebound_location is None: + repaired_probes.append(probe) + continue + repaired_path, repaired_line = rebound_location + elif rejection: + repaired_probes.append(probe) + continue + if adversarial_probe_source_receipt_error( + repaired_evidence, repaired_path, repaired_line + ) or adversarial_evidence_rejection_reason( + repaired_evidence, + repaired_path, + repaired_line, + ): + repaired_probes.append(probe) + continue + if ( + repaired_evidence == evidence + and repaired_path == path + and repaired_line == line + ): + repaired_probes.append(probe) + else: + repaired_probes.append( + { + **probe, + "path": repaired_path, + "line": repaired_line, + "evidence": repaired_evidence, + } + ) + changed = True + + if not changed: + return value + return { + **value, + "adversarial_validation": {**validation, "probes": repaired_probes}, + } + + def adversarial_validation_error( value: Any, *, @@ -1267,6 +1406,7 @@ def reject(reason: str) -> None: return reject("APPROVE cannot contain findings") if result == "REQUEST_CHANGES" and not findings: return reject("REQUEST_CHANGES requires at least one finding") + value = repair_adversarial_probe_evidence_bindings(value) adversarial_error = adversarial_validation_error( value.get("adversarial_validation"), result=result, diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index aae63c5d..f88c730e 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -1014,6 +1014,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$workflow_file" '(.workflow // "") == "CodeQL"' "opencode approval can distinguish CodeQL dynamic setup checks" assert_file_contains "$workflow_file" '((.isRequired // false) | not) and (.workflow // "") == "CodeQL"' "opencode approval ignores non-required cancelled CodeQL checks without source evidence" assert_file_contains "$workflow_file" '(.name // "") == "scan-pr-queue" and ((.workflow // "") == "PR Review Merge Scheduler" or (.workflow // "") == "Required PR Review Merge Scheduler")' "opencode approval ignores cancelled scheduler queue replacement checks without source evidence" + assert_file_contains "$workflow_file" '(.conclusion // "" | ascii_downcase) == "cancelled" and (.name // "") == "scan-pr-queue") | not' "opencode approval also ignores cancelled scheduler queue replacement checks in the workflow-less REST fallback" assert_file_contains "$workflow_file" 'grep -Fq -- "Strix Security Scan/strix:" "$rollup_file"' "opencode approval avoids duplicate supplemental Strix workflow-run blockers when statusCheckRollup already has the Strix check" assert_file_contains "$workflow_file" 'current_head_manual_strix_success_status()' "opencode approval can identify same-head manual Strix success status evidence" assert_file_contains "$workflow_file" 'manual_run_line="$(latest_current_head_manual_strix_run || true)"' "opencode approval falls back to same-head manual Strix check-run success when commit status publication is unavailable" diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index 71e96add..f56342f5 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -14,6 +14,8 @@ import pytest +from scripts.ci import opencode_review_normalize_output as normalizer + ROOT = Path(__file__).resolve().parents[1] RUNNER = ROOT / "scripts" / "ci" / "run_opencode_review_model_pool.sh" @@ -29,6 +31,16 @@ } +@pytest.fixture(autouse=True) +def clear_normalizer_artifact_caches(): + """Keep trusted artifact caches isolated from later review-gate tests.""" + normalizer.current_changed_files.cache_clear() + normalizer.trusted_execution_receipts.cache_clear() + yield + normalizer.current_changed_files.cache_clear() + normalizer.trusted_execution_receipts.cache_clear() + + def bash_command() -> str: """Return a Bash executable that can run repository shell scripts locally.""" if os.name == "nt": @@ -87,6 +99,258 @@ def seal_artifacts( return hashlib.sha256(manifest.read_bytes()).hexdigest() +def prepare_probe_binding_artifacts( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> tuple[str, int, str]: + """Create one sealed changed source line for normalizer binding tests.""" + runner_temp = tmp_path / "binding-runner-temp" + source_root = tmp_path / "binding-source" + source_path = source_root / "scripts" / "ci" / "example.py" + runner_temp.mkdir() + source_path.parent.mkdir(parents=True) + source_path.write_text( + "return False\nraise SystemExit(1)\nraise SystemExit(1)\n", + encoding="utf-8", + ) + changed_files = runner_temp / "opencode-changed-files.txt" + changed_files.write_text("scripts/ci/example.py\n", encoding="utf-8") + manifest_digest = seal_artifacts( + runner_temp, + head_sha="binding-head", + run_id="binding-run", + run_attempt="1", + paths=(changed_files,), + ) + monkeypatch.setenv("RUNNER_TEMP", str(runner_temp)) + monkeypatch.setenv("OPENCODE_SOURCE_WORKDIR", str(source_root)) + monkeypatch.setenv("OPENCODE_CHANGED_FILES_FILE", str(changed_files)) + monkeypatch.setenv("OPENCODE_ARTIFACT_MANIFEST_SHA256", manifest_digest) + normalizer.current_changed_files.cache_clear() + normalizer.trusted_execution_receipts.cache_clear() + digest = hashlib.sha256(b"return False").hexdigest() + return "scripts/ci/example.py", 1, f"source-line-sha256={digest}" + + +def test_normalizer_binds_only_a_verified_structured_probe_location( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A matching receipt may restore only redundant path:line evidence text.""" + path, line, receipt = prepare_probe_binding_artifacts(tmp_path, monkeypatch) + evidence = ( + f"Regression command rejected malformed input with exit code 1; {receipt}" + ) + value = { + "adversarial_validation": { + "probes": [ + { + "path": path, + "line": line, + "evidence": evidence, + } + ] + } + } + + repaired = normalizer.repair_adversarial_probe_evidence_bindings(value) + + assert repaired is not value + assert repaired["adversarial_validation"]["probes"][0]["evidence"] == ( + f"{path}:{line} {evidence}" + ) + + +def test_normalizer_rebinds_structured_location_to_unique_receipted_citation( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A unique cited changed line may repair redundant structured location drift.""" + path, line, _ = prepare_probe_binding_artifacts(tmp_path, monkeypatch) + cited_line = 2 + receipt = "source-line-sha256=" + hashlib.sha256( + b"raise SystemExit(1)" + ).hexdigest() + evidence = ( + f"Source trace at {path}:{cited_line} rejected malformed input with exit code 1; " + f"{receipt}" + ) + value = { + "adversarial_validation": { + "probes": [ + { + "path": path, + "line": line, + "evidence": evidence, + } + ] + } + } + + repaired = normalizer.repair_adversarial_probe_evidence_bindings(value) + + assert repaired is not value + repaired_probe = repaired["adversarial_validation"]["probes"][0] + assert repaired_probe["path"] == path + assert repaired_probe["line"] == cited_line + assert repaired_probe["evidence"] == evidence + + +def test_normalizer_does_not_rebind_ambiguous_receipted_citations( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Two cited lines with the same trusted bytes remain ambiguous and fail closed.""" + path, line, _ = prepare_probe_binding_artifacts(tmp_path, monkeypatch) + receipt = "source-line-sha256=" + hashlib.sha256( + b"raise SystemExit(1)" + ).hexdigest() + evidence = ( + f"Source traces at {path}:2 and {path}:3 rejected malformed input with exit code 1; " + f"{receipt}" + ) + value = { + "adversarial_validation": { + "probes": [ + { + "path": path, + "line": line, + "evidence": evidence, + } + ] + } + } + + assert normalizer.repair_adversarial_probe_evidence_bindings(value) is value + + +def test_receipt_verified_location_rejects_unverifiable_evidence( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Missing receipts and invalid cited lines cannot yield a trusted location.""" + path, _, receipt = prepare_probe_binding_artifacts(tmp_path, monkeypatch) + changed_files = frozenset({path}) + + assert ( + normalizer.receipt_verified_evidence_location( + f"Source trace at {path}:1 rejected malformed input with exit code 1", + changed_files, + ) + is None + ) + assert ( + normalizer.receipt_verified_evidence_location( + f"Source trace at {path}:99 rejected malformed input with exit code 1; " + f"{receipt}", + changed_files, + ) + is None + ) + + +def test_normalizer_probe_binding_repair_remains_fail_closed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Malformed, untrusted, circular, and already-bound probes are not rewritten.""" + path, line, receipt = prepare_probe_binding_artifacts(tmp_path, monkeypatch) + assert normalizer.repair_adversarial_probe_evidence_bindings(None) is None + + missing_validation: dict[str, object] = {} + assert ( + normalizer.repair_adversarial_probe_evidence_bindings(missing_validation) + is missing_validation + ) + malformed = {"adversarial_validation": {"probes": "not-a-list"}} + assert normalizer.repair_adversarial_probe_evidence_bindings(malformed) is malformed + + invalid_values = [ + { + "adversarial_validation": { + "probes": [ + "not-an-object", + {"path": path, "line": 0, "evidence": receipt}, + ] + } + }, + { + "adversarial_validation": { + "probes": [ + { + "path": path, + "line": line, + "evidence": ( + "Regression command rejected malformed input with exit code 1; " + "source-line-sha256=" + "0" * 64 + ), + } + ] + } + }, + { + "adversarial_validation": { + "probes": [ + { + "path": path, + "line": line, + "evidence": ( + "Regression command rejected malformed input with exit code 1; " + f"{receipt} {receipt}" + ), + } + ] + } + }, + { + "adversarial_validation": { + "probes": [ + { + "path": "scripts/ci/not-changed.py", + "line": line, + "evidence": ( + "Regression command rejected malformed input with exit code 1; " + f"{receipt}" + ), + } + ] + } + }, + { + "adversarial_validation": { + "probes": [ + { + "path": path, + "line": line, + "evidence": receipt, + } + ] + } + }, + { + "adversarial_validation": { + "probes": [ + { + "path": path, + "line": line, + "evidence": f"Source inspection properly handles all cases; {receipt}", + } + ] + } + }, + { + "adversarial_validation": { + "probes": [ + { + "path": path, + "line": line, + "evidence": ( + f"Regression command at {path}:{line} rejected malformed input " + f"with exit code 1; {receipt}" + ), + } + ] + } + }, + ] + for invalid in invalid_values: + assert normalizer.repair_adversarial_probe_evidence_bindings(invalid) is invalid + + def skip_if_windows_bash_is_unresponsive(command: str) -> None: """Skip with a visible reason when local Git Bash cannot start on Windows.""" if os.name != "nt":