diff --git a/.github/workflows/parity-governance.yml b/.github/workflows/parity-governance.yml index d68dc01cab..c8b422427e 100644 --- a/.github/workflows/parity-governance.yml +++ b/.github/workflows/parity-governance.yml @@ -32,6 +32,7 @@ on: - 'Tools/issue_ref.py' - 'Tools/parity_*.py' - 'Tools/parity_*.json' + - 'Tools/parity_case_specs/**' - 'Tools/tests/test_parity_*.py' - 'Tools/tests/test_rr_legacy_preservation_contract.py' - '.github/workflows/parity-governance.yml' @@ -41,6 +42,7 @@ on: - 'Tools/issue_ref.py' - 'Tools/parity_*.py' - 'Tools/parity_*.json' + - 'Tools/parity_case_specs/**' - 'Tools/tests/test_parity_*.py' - 'Tools/tests/test_rr_legacy_preservation_contract.py' - '.github/workflows/parity-governance.yml' @@ -74,11 +76,12 @@ jobs: tests.test_parity_governance_acceptance \ tests.test_rr_legacy_preservation_contract \ tests.test_parity_disposition_kinds \ + tests.test_parity_harness \ 2>&1 | tee "$RUNNER_TEMP/out.txt" ran=$(grep -oE '^Ran [0-9]+ test' "$RUNNER_TEMP/out.txt" | grep -oE '[0-9]+') echo "collected ${ran:-0} tests" - if [ "${ran:-0}" -lt 122 ]; then - echo "::error::expected at least 122 parity-governance tests, collected ${ran:-0} — discovery is broken, not the suite" + if [ "${ran:-0}" -lt 159 ]; then + echo "::error::expected at least 159 parity-governance tests, collected ${ran:-0} — discovery is broken, not the suite" exit 1 fi working-directory: Tools diff --git a/Tools/PARITY_HARNESS.md b/Tools/PARITY_HARNESS.md new file mode 100644 index 0000000000..90c470789a --- /dev/null +++ b/Tools/PARITY_HARNESS.md @@ -0,0 +1,159 @@ +# Shared parity cases and independent validation + +PR2 provides the portable test contract for the parity guard series. A shared +case is written once with its inputs and literal expected result. Each platform +can eventually execute those same cases and validate its own output independently; +a later artifact-only comparison additionally checks platform agreement. + +## Relationship to PR1 and existing native tests + +PR1 inventories Swift/Kotlin counterparts and protects the static parity debt +baseline with ledger/ratchet checks. It does not execute both implementations. +PR2 adds case specifications, canonical transport, splitting/remerging, standalone +expected-result validation, provenance checks, and a differential comparator. +It leaves PR1's rules in place. + +The target is to share **platform-independent behavioral cases**, such as +algorithm inputs and expected scores. Native UI, permissions, Bluetooth-stack +and OS-integration tests remain platform-specific. Existing native tests stay +in place in PR2: no native test has been migrated or deleted. + +Both platforms agreeing is insufficient: they can share a bug. Every case in +this contract therefore requires a checked-in `expected` value. Validation +compares actual output against that literal oracle, never against a dynamically +generated reference answer. Positive comparison validates both sides against the +oracle as well as each other, so agreeing-but-wrong results fail. Oracle quality +still depends on correct domain expectations and suitable case coverage. + +| Layer | Scope | +| --- | --- | +| PR1, merged | Static inventory, debt governance and ledger/ratchet tools | +| PR2 | Shared cases/oracles, portable artifact contracts and tool self-tests | +| PR3 | Native Swift/Kotlin runners and the first migration of existing algorithm cases; broader test data | +| PR4 | Product-source CI enforcement | + +PR3 should start with a small real algorithm, collect relevant cases from both +native test suites, preserve their assertions as common expectations, and prove +that both adapters execute them. Remove redundant native cases only after their +previous checks are demonstrably covered. Continue migrating shared behavior +incrementally; platform-internal tests stay native. + +## Independent builds and later comparison + +An Android job needs only the shared cases and Android implementation. Its +standalone expected-result check needs only those cases and its own result file. +A Swift job can independently do the same on macOS. Neither needs the other +platform's build or artifact to pass its own behavioral tests. Once both artifacts +exist, a separate comparator can consume them without rebuilding either platform. + +PR2 includes only two Python implementations of a small clamp demonstration, +called `core` and `reference`. These exercise the contract and negative controls; +they do not execute Android or Swift product code or establish native parity. +The validator accepts an explicit side label (for example `android` or `swift`), +and the comparator accepts explicit labels for both artifacts. This prepares the +interface; it does not supply native runners or native CI orchestration. + +## Case and provenance contract + +- The specification requires integer `schemaVersion: 1` and a nonblank string + `suiteVersion`, identifying the suite revision. Update the suite version when + changing the shared cases or their intended behavior. This is separate from the + schema version and the product source revision. +- Each case requires `id`, `args` and literal `expected`. The registry in + `parity_shards.py` is the sole ordered shard/operation authority. +- Expanded input records contain `args`, `expected`, `function`, `id`, `shard`, + `suiteVersion` and `caseSha256`. SHA-256 covers the canonical record excluding + `caseSha256` itself, including its expected result and suite version. All rows + of one input artifact must have the same suite version. +- Output records contain `caseSha256`, `function`, `id`, `mutant`, `side`, + `sourceRevision`, `suiteVersion` and `value`. Every row must match the exact + input identity, function, digest and suite version. Missing, duplicated, + unexpected or reordered rows fail closed. +- Runners must supply `sourceRevision`. Every validation, comparison and mutant + verification requires an explicit `--expected-revision`: the intended full + lowercase Git object ID (40 hex digits for SHA-1 or 64 for SHA-256). Every output + row must carry that exact revision. Moving branch names, abbreviated revisions, + mixed revisions, and artifacts from a different revision are rejected. + +The expected revision must come from the caller's intended build/job context, +not be copied from an untrusted result just to make the check pass. The harness +checks metadata consistency; it does **not** attest which code actually ran, +verify a clean checkout, or authenticate artifact producers. Trusted build and +artifact provenance are responsibilities of the later native CI integration. +The source revision is not embedded in persistent cases: the same checked-in +behavioral cases can be reused across product-code changes. Case hashes detect +changed content even if someone forgets to bump the suite version. + +## Run the Python demonstration + +From the repository root, use the revision intended for this demonstration: + +```sh +parity_revision=$(git rev-parse HEAD) +python3 Tools/parity_harness.py expand \ + Tools/parity_case_specs/core_reference.json /tmp/parity-cases.jsonl +python3 Tools/parity_harness.py run core \ + /tmp/parity-cases.jsonl /tmp/parity-core.jsonl \ + --source-revision "$parity_revision" +python3 Tools/parity_harness.py validate core \ + /tmp/parity-cases.jsonl /tmp/parity-core.jsonl \ + --expected-revision "$parity_revision" +``` + +The standalone validation above is complete with no reference artifact. Later: + +```sh +python3 Tools/parity_harness.py run reference \ + /tmp/parity-cases.jsonl /tmp/parity-reference.jsonl \ + --source-revision "$parity_revision" +python3 Tools/parity_harness.py compare \ + /tmp/parity-cases.jsonl /tmp/parity-core.jsonl /tmp/parity-reference.jsonl \ + --expected-revision "$parity_revision" +``` + +For future native artifacts, the same interfaces are: + +```sh +python3 Tools/parity_harness.py validate android cases.jsonl android.jsonl \ + --expected-revision "$parity_revision" +python3 Tools/parity_harness.py validate swift cases.jsonl swift.jsonl \ + --expected-revision "$parity_revision" +python3 Tools/parity_harness.py compare cases.jsonl android.jsonl swift.jsonl \ + --core-side android --reference-side swift \ + --expected-revision "$parity_revision" +``` + +These commands validate already-produced artifacts; PR2 cannot yet produce those +native files. Both comparison sides must have distinct labels. Successful checks +return 0, value/oracle differences return 1, and malformed contracts or provenance +mismatches return 2. `EXPECTED` diagnostics identify oracle failures; `DIFF` +diagnostics identify platform differences. Agreement does not suppress oracle failures. + +## Transport and negative controls + +`split` writes one canonical file per registered shard; `merge` requires the +original expanded input and proves an exact remerge. Shard parts must appear once +each in registry order, with canonical row order. Empty, unknown, duplicate, +missing or changed cases are rejected. Splitting refuses unexpected JSONL files +in the destination before writing. + +Canonical JSON uses UTF-8 Unicode scalar values, sorted object keys, compact +separators, finite floats and signed 64-bit integers. JSONL uses LF only, exactly +one record per line and a final LF; CRLF and blank lines are rejected. Unicode +line/paragraph separators inside strings remain ordinary string content. +Comparison uses exact canonical JSON at every nesting level: booleans, integers +and floats remain distinct, and `-0.0` differs from `0.0`. Float spelling follows +Python's standard JSON encoder; future native adapters must match this contract. +The representative clamp retains the input value (including its number type and +signed zero) when it equals either bound. + +The demonstration `run --mutant` deliberately changes the returned value. +Positive validation/comparison rejects mutant metadata. `verify-mutant` requires +an explicit expected revision, mutant metadata on exactly the named side, a normal +side matching the literal oracle, and every supplied mutant case differing from +that normal result. Controls exist for both sides. + +The existing parity-governance CI runs these tool self-tests as an explicitly +listed module and triggers on case-spec changes. It adds no native build dependency or product-source parity +enforcement. Additional modules, native runners, bulk corpora and enforcement +remain the later series layers described above. diff --git a/Tools/parity_case_specs/core_reference.json b/Tools/parity_case_specs/core_reference.json new file mode 100644 index 0000000000..5495c9ee70 --- /dev/null +++ b/Tools/parity_case_specs/core_reference.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": 1, + "suiteVersion": "core-reference-v1", + "shards": [ + { + "name": "core-reference", + "operations": [ + { + "key": "Portable.clamp/3", + "cases": [ + {"id": "above", "expected": 10, "args": {"lower": 0, "upper": 10, "value": 14}}, + {"id": "inside", "expected": 4, "args": {"lower": 0, "upper": 10, "value": 4}}, + {"id": "lower", "expected": 0, "args": {"lower": 0, "upper": 10, "value": -2}} + ] + } + ] + } + ] +} diff --git a/Tools/parity_harness.py b/Tools/parity_harness.py new file mode 100644 index 0000000000..dc4db050c3 --- /dev/null +++ b/Tools/parity_harness.py @@ -0,0 +1,562 @@ +#!/usr/bin/env python3 +"""Compact, deterministic, standard-library-only differential harness core.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import re +import sys +from pathlib import Path +from typing import Iterable + +import parity_shards + + +class HarnessError(ValueError): + """Raised when a portable harness contract fails closed.""" + + +_CASE_ID = re.compile(r"[a-z][a-z0-9-]*") +_INPUT_FIELDS = {"args", "caseSha256", "expected", "function", "id", "shard", "suiteVersion"} +_OUTPUT_FIELDS = {"caseSha256", "function", "id", "mutant", "side", "sourceRevision", "suiteVersion", "value"} + + +def _nonempty_string(value: object, label: str) -> None: + if not isinstance(value, str) or not value.strip(): + raise HarnessError(f"{label} must be a non-empty string") + + +def _revision(value: object) -> None: + if not isinstance(value, str) or re.fullmatch(r"(?:[0-9a-f]{40}|[0-9a-f]{64})", value) is None: + raise HarnessError("sourceRevision must be a full lowercase Git revision (40 or 64 hex digits)") + + +def _reject_constant(token: str) -> None: + raise HarnessError(f"numbers must be finite, got {token}") + + +def _object_without_duplicate_keys(pairs: list[tuple[str, object]]) -> dict: + value: dict = {} + for key, item in pairs: + if key in value: + raise HarnessError(f"duplicate JSON key {key!r}") + value[key] = item + return value + + +def _validate_portable_json(value: object, path: str = "value") -> None: + if isinstance(value, str): + if any(0xD800 <= ord(char) <= 0xDFFF for char in value): + raise HarnessError(f"{path} strings must contain Unicode scalar values") + return + if value is None or isinstance(value, bool): + return + if isinstance(value, int): + if not -(1 << 63) <= value <= (1 << 63) - 1: + raise HarnessError(f"{path} integer must fit signed 64-bit") + return + if isinstance(value, float): + if not math.isfinite(value): + raise HarnessError(f"{path} number must be finite") + return + if isinstance(value, list): + for number, item in enumerate(value): + _validate_portable_json(item, f"{path}[{number}]") + return + if isinstance(value, dict): + for key, item in value.items(): + if not isinstance(key, str): + raise HarnessError(f"{path} object keys must be strings") + _validate_portable_json(key, f"{path} key") + _validate_portable_json(item, f"{path}.{key}") + return + raise HarnessError(f"{path} has non-JSON type {type(value).__name__}") + + +def parse_json(text: str) -> object: + try: + value = json.loads( + text, + parse_constant=_reject_constant, + object_pairs_hook=_object_without_duplicate_keys, + ) + except json.JSONDecodeError as exc: + raise HarnessError(f"invalid JSON: {exc}") from exc + _validate_portable_json(value) + return value + + +def canonical_json(value: object) -> str: + _validate_portable_json(value) + try: + return json.dumps( + value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False + ) + except (TypeError, ValueError) as exc: + raise HarnessError(f"value is not finite canonical JSON: {exc}") from exc + + +def _exact_fields(value: object, expected: set[str], path: str) -> dict: + if not isinstance(value, dict) or set(value) != expected: + actual = sorted(value) if isinstance(value, dict) else type(value).__name__ + raise HarnessError(f"{path} fields must be exactly {sorted(expected)}, got {actual}") + return value + + +def _operation_index() -> dict[str, tuple[parity_shards.ShardSpec, parity_shards.OperationSpec]]: + parity_shards.validate_registry(parity_shards.REGISTRY) + return parity_shards.operation_index() + + +def validate_spec(value: object) -> dict: + _validate_portable_json(value, "spec") + spec = _exact_fields(value, {"schemaVersion", "shards", "suiteVersion"}, "spec") + _nonempty_string(spec["suiteVersion"], "suiteVersion") + if type(spec["schemaVersion"]) is not int or spec["schemaVersion"] != 1: + raise HarnessError("spec schemaVersion must be 1") + if not isinstance(spec["shards"], list): + raise HarnessError("spec shards must be an array") + expected_shards = [shard.name for shard in parity_shards.REGISTRY] + actual_shards: list[str] = [] + operation_index = _operation_index() + for shard_number, raw_shard in enumerate(spec["shards"]): + path = f"spec.shards[{shard_number}]" + shard = _exact_fields(raw_shard, {"name", "operations"}, path) + if not isinstance(shard["name"], str) or not isinstance(shard["operations"], list): + raise HarnessError(f"{path} requires string name and operations array") + actual_shards.append(shard["name"]) + registered = next( + (item for item in parity_shards.REGISTRY if item.name == shard["name"]), None + ) + if registered is None: + raise HarnessError(f"{path} names unknown shard {shard['name']!r}") + actual_operations: list[str] = [] + for operation_number, raw_operation in enumerate(shard["operations"]): + op_path = f"{path}.operations[{operation_number}]" + operation = _exact_fields(raw_operation, {"key", "cases"}, op_path) + if not isinstance(operation["key"], str) or not isinstance(operation["cases"], list): + raise HarnessError(f"{op_path} requires string key and cases array") + actual_operations.append(operation["key"]) + registered_pair = operation_index.get(operation["key"]) + if registered_pair is None or registered_pair[0].name != shard["name"]: + raise HarnessError(f"{op_path} operation is not registered in shard {shard['name']}") + if not operation["cases"]: + raise HarnessError(f"{op_path} must contain at least one case") + case_ids: list[str] = [] + for case_number, raw_case in enumerate(operation["cases"]): + case_path = f"{op_path}.cases[{case_number}]" + case = _exact_fields(raw_case, {"id", "args", "expected"}, case_path) + if not isinstance(case["id"], str) or _CASE_ID.fullmatch(case["id"]) is None: + raise HarnessError(f"{case_path} id must match {_CASE_ID.pattern}") + if not isinstance(case["args"], dict): + raise HarnessError(f"{case_path} args must be an object") + if tuple(sorted(case["args"])) != registered_pair[1].argument_keys: + raise HarnessError( + f"{case_path} args must contain exactly {list(registered_pair[1].argument_keys)}" + ) + case_ids.append(case["id"]) + try: + registered_pair[1].reference(case["args"]) + except parity_shards.RegistryError as exc: + raise HarnessError(f"{case_path}: {exc}") from exc + if len(case_ids) != len(set(case_ids)): + raise HarnessError(f"{op_path} case ids must be unique") + if case_ids != sorted(case_ids): + raise HarnessError(f"{op_path} cases must be in canonical id order") + expected_operations = [item.key for item in registered.operations] + if actual_operations != expected_operations: + raise HarnessError( + f"{path} operations must exactly match registry order {expected_operations}" + ) + if actual_shards != expected_shards: + raise HarnessError(f"spec shards must exactly match registry order {expected_shards}") + return spec + + +def load_spec(path: Path) -> dict: + try: + raw = path.read_text(encoding="utf-8") + except (OSError, UnicodeError) as exc: + raise HarnessError(f"cannot read case spec {path}: {exc}") from exc + value = parse_json(raw) + return validate_spec(value) + + +def case_digest(row: dict) -> str: + payload = dict(row) + payload.pop("caseSha256", None) + return hashlib.sha256(canonical_json(payload).encode("utf-8")).hexdigest() + + +def expand_spec(value: object) -> list[dict]: + spec = validate_spec(value) + rows: list[dict] = [] + for shard in spec["shards"]: + for operation in shard["operations"]: + for case in operation["cases"]: + row = { + "args": case["args"], + "expected": case["expected"], + "suiteVersion": spec["suiteVersion"], + "function": operation["key"], + "id": f"{shard['name']}/{operation['key']}/{case['id']}", + "shard": shard["name"], + } + row["caseSha256"] = case_digest(row) + rows.append(row) + return sorted(rows, key=lambda item: item["id"]) + + +def canonical_jsonl(records: Iterable[dict]) -> str: + rows = list(records) + if not rows: + raise HarnessError("JSONL must contain at least one record") + return "".join(f"{canonical_json(row)}\n" for row in rows) + + +def parse_jsonl(text: str, label: str) -> list[dict]: + if not text.endswith("\n"): + raise HarnessError(f"{label} JSONL requires a final newline") + lines = text[:-1].split("\n") + if not lines or any(not line for line in lines): + raise HarnessError(f"{label} JSONL must be non-empty without blank lines") + rows: list[dict] = [] + for number, line in enumerate(lines, 1): + row = parse_json(line) + if not isinstance(row, dict) or line != canonical_json(row): + raise HarnessError(f"{label} line {number} is not canonical JSON") + rows.append(row) + return rows + + +def read_jsonl(path: Path, label: str) -> list[dict]: + try: + with path.open(encoding="utf-8", newline="") as stream: + return parse_jsonl(stream.read(), label) + except (OSError, UnicodeError) as exc: + raise HarnessError(f"cannot read {label} {path}: {exc}") from exc + + +def write_jsonl(path: Path, rows: Iterable[dict]) -> None: + text = canonical_jsonl(rows) + try: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8", newline="\n") as stream: + stream.write(text) + except (OSError, UnicodeError) as exc: + raise HarnessError(f"cannot write JSONL {path}: {exc}") from exc + + +def _index(records: Iterable[dict], label: str, fields: set[str]) -> dict[str, dict]: + indexed: dict[str, dict] = {} + for number, raw in enumerate(records, 1): + row = _exact_fields(raw, fields, f"{label}[{number}]") + case_id = row.get("id") + if not isinstance(case_id, str) or not case_id: + raise HarnessError(f"{label}[{number}] requires a non-empty id") + if case_id in indexed: + raise HarnessError(f"duplicate {label} id {case_id}") + indexed[case_id] = row + if not indexed: + raise HarnessError(f"{label} must be non-empty") + return indexed + + +def _input_index(records: Iterable[dict]) -> dict[str, dict]: + indexed = _index(records, "input", _INPUT_FIELDS) + operation_index = _operation_index() + versions = set() + for case_id, row in indexed.items(): + _nonempty_string(row["suiteVersion"], "suiteVersion") + versions.add(row["suiteVersion"]) + if not isinstance(row["function"], str) or not isinstance(row["shard"], str): + raise HarnessError(f"input id={case_id} function and shard must be strings") + if not isinstance(row["caseSha256"], str) or not isinstance(row["args"], dict): + raise HarnessError(f"input id={case_id} requires string caseSha256 and object args") + prefix = f"{row['shard']}/{row['function']}/" + if not case_id.startswith(prefix) or _CASE_ID.fullmatch(case_id[len(prefix):]) is None: + raise HarnessError(f"input id={case_id} is not structurally bound to shard/function") + pair = operation_index.get(row["function"]) + if pair is None or pair[0].name != row["shard"]: + raise HarnessError(f"input id={case_id} targets an unknown operation/shard") + if row["caseSha256"] != case_digest(row): + raise HarnessError(f"input id={case_id} caseSha256 mismatch") + if len(versions) != 1: + raise HarnessError("input suiteVersion must be identical across cases") + if list(indexed) != sorted(indexed): + raise HarnessError("input rows must be in canonical id order") + return indexed + + +def split_records(records: Iterable[dict]) -> dict[str, list[dict]]: + indexed = _input_index(records) + shards = {shard.name: [] for shard in parity_shards.REGISTRY} + for row in indexed.values(): + shards[row["shard"]].append(row) + empty = [name for name, rows in shards.items() if not rows] + if empty: + raise HarnessError(f"every registered shard must be non-empty: {empty}") + return shards + + +def merge_records(parts: Iterable[Iterable[dict]], expected: Iterable[dict]) -> list[dict]: + expected_index = _input_index(expected) + merged: list[dict] = [] + seen: set[str] = set() + part_shards: list[set[str]] = [] + for part in parts: + part = list(part) + part_shards.append({row.get("shard") for row in part if isinstance(row, dict) and isinstance(row.get("shard"), str)}) + for row in part: + case_id = row.get("id") if isinstance(row, dict) else None + if case_id in seen: + raise HarnessError(f"duplicate shard id {case_id}") + if isinstance(case_id, str): + seen.add(case_id) + merged.append(row) + actual_index = _input_index(merged) + missing = sorted(set(expected_index) - set(actual_index)) + unexpected = sorted(set(actual_index) - set(expected_index)) + if missing or unexpected: + raise HarnessError(f"shard IDs mismatch: missing={missing} unexpected={unexpected}") + registered_shards = [{shard.name} for shard in parity_shards.REGISTRY] + if part_shards != registered_shards: + raise HarnessError("merge requires one part per registered shard in registry order") + for case_id in expected_index: + if canonical_json(actual_index[case_id]) != canonical_json(expected_index[case_id]): + raise HarnessError(f"shard id={case_id} content mismatch") + return [actual_index[case_id] for case_id in sorted(actual_index)] + + +def run_records(records: Iterable[dict], side: str, *, source_revision: str, mutant: bool = False) -> list[dict]: + _revision(source_revision) + if side not in {"core", "reference"}: + raise HarnessError("side must be core or reference") + indexed = _input_index(records) + operations = _operation_index() + output: list[dict] = [] + for case_id, row in indexed.items(): + operation = operations[row["function"]][1] + try: + value = (operation.core if side == "core" else operation.reference)(row["args"]) + if mutant: + value = operation.mutant(value) + except parity_shards.RegistryError as exc: + raise HarnessError(f"{side} id={case_id}: {exc}") from exc + canonical_json(value) + output.append( + { + "caseSha256": row["caseSha256"], + "function": row["function"], + "id": case_id, + "mutant": mutant, + "side": side, + "sourceRevision": source_revision, + "suiteVersion": row["suiteVersion"], + "value": value, + } + ) + return output + + +def _output_index(records: Iterable[dict], label: str) -> dict[str, dict]: + indexed = _index(records, label, _OUTPUT_FIELDS) + for case_id, row in indexed.items(): + if not isinstance(row["caseSha256"], str) or not isinstance(row["function"], str): + raise HarnessError(f"{label} id={case_id} function and caseSha256 must be strings") + if not isinstance(row["side"], str) or type(row["mutant"]) is not bool: + raise HarnessError(f"{label} id={case_id} requires string side and boolean mutant") + canonical_json(row["value"]) + if list(indexed) != sorted(indexed): + raise HarnessError(f"{label} rows must be in canonical id order") + return indexed + + +def _bound_output(inputs: dict[str, dict], records: Iterable[dict], side: str, + expected_revision: str, *, allow_mutants: bool = False) -> dict[str, dict]: + _revision(expected_revision) + _nonempty_string(side, "side") + actual = _output_index(records, f"{side} output") + missing = sorted(set(inputs) - set(actual)) + unexpected = sorted(set(actual) - set(inputs)) + if missing or unexpected: + raise HarnessError(f"{side} IDs mismatch: missing={missing} unexpected={unexpected}") + for case_id, row in actual.items(): + if row["side"] != side: + raise HarnessError(f"{side} id={case_id} side mismatch") + if row["sourceRevision"] != expected_revision: + raise HarnessError(f"{side} id={case_id} sourceRevision mismatch") + if row["mutant"] and not allow_mutants: + raise HarnessError(f"{side} id={case_id} mutant output is forbidden in positive compare/validate") + for field in ("caseSha256", "function", "suiteVersion"): + if row[field] != inputs[case_id][field]: + raise HarnessError(f"{side} id={case_id} {field} mismatch") + return actual + + +def _oracle_diffs(inputs: dict[str, dict], output: dict[str, dict], side: str) -> list[str]: + return [ + f"EXPECTED id={case_id} side={side} expected={canonical_json(source['expected'])} " + f"actual={canonical_json(output[case_id]['value'])}" + for case_id, source in inputs.items() + if canonical_json(source["expected"]) != canonical_json(output[case_id]["value"]) + ] + + +def validate_records(inputs: Iterable[dict], output: Iterable[dict], side: str, + *, expected_revision: str) -> list[str]: + """Check one artifact against literal oracles, without a peer build or output.""" + expected = _input_index(inputs) + actual = _bound_output(expected, output, side, expected_revision) + return _oracle_diffs(expected, actual, side) + + +def compare_records(inputs: Iterable[dict], core: Iterable[dict], reference: Iterable[dict], + *, expected_revision: str, core_side: str = "core", + reference_side: str = "reference") -> list[str]: + """Check both oracles and parity against the caller's intended immutable revision.""" + if core_side == reference_side: + raise HarnessError("comparison requires distinct sides") + expected = _input_index(inputs) + left = _bound_output(expected, core, core_side, expected_revision) + right = _bound_output(expected, reference, reference_side, expected_revision) + diffs = _oracle_diffs(expected, left, core_side) + _oracle_diffs(expected, right, reference_side) + for case_id in expected: + if canonical_json(left[case_id]["value"]) != canonical_json(right[case_id]["value"]): + diffs.append(f"DIFF id={case_id} function={expected[case_id]['function']} " + f"{core_side}={canonical_json(left[case_id]['value'])} " + f"{reference_side}={canonical_json(right[case_id]['value'])}") + return diffs + + +def verify_mutant( + inputs: Iterable[dict], core: Iterable[dict], reference: Iterable[dict], expected_side: str, + *, expected_revision: str +) -> int: + if expected_side not in {"core", "reference"}: + raise HarnessError("expected mutant side must be core or reference") + input_rows = list(inputs) + core_rows = list(core) + reference_rows = list(reference) + side_rows = {"core": core_rows, "reference": reference_rows} + other_side = "reference" if expected_side == "core" else "core" + expected_inputs = _input_index(input_rows) + for side, rows in side_rows.items(): + _bound_output(expected_inputs, rows, side, expected_revision, allow_mutants=True) + if ( + not side_rows[expected_side] + or not all(row["mutant"] is True for row in side_rows[expected_side]) + or any(row["mutant"] is not False for row in side_rows[other_side]) + ): + raise HarnessError(f"expected {expected_side} mutant metadata on exactly one side") + normal = _output_index(side_rows[other_side], "normal output") + if _oracle_diffs(expected_inputs, normal, other_side): + raise HarnessError("non-mutant side must match expected results") + mutated = _output_index(side_rows[expected_side], "mutant output") + killed = {case_id for case_id in expected_inputs + if canonical_json(mutated[case_id]["value"]) != canonical_json(normal[case_id]["value"])} + expected = {row["id"] for row in input_rows} + if killed != expected: + raise HarnessError( + f"{expected_side} mutant was not killed exactly: missing={sorted(expected-killed)} " + f"unexpected={sorted(killed-expected)}" + ) + return len(killed) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + expand = commands.add_parser("expand") + expand.add_argument("spec", type=Path) + expand.add_argument("output", type=Path) + split = commands.add_parser("split") + split.add_argument("input", type=Path) + split.add_argument("output_dir", type=Path) + run = commands.add_parser("run") + run.add_argument("side", choices=("core", "reference")) + run.add_argument("input", type=Path) + run.add_argument("output", type=Path) + run.add_argument("--source-revision", required=True) + run.add_argument("--mutant", action="store_true") + merge = commands.add_parser("merge") + merge.add_argument("expected", type=Path) + merge.add_argument("output", type=Path) + merge.add_argument("parts", type=Path, nargs="+") + compare = commands.add_parser("compare") + compare.add_argument("input", type=Path) + compare.add_argument("core", type=Path) + compare.add_argument("reference", type=Path) + compare.add_argument("--core-side", default="core") + compare.add_argument("--reference-side", default="reference") + validate = commands.add_parser("validate") + validate.add_argument("side") + validate.add_argument("input", type=Path) + validate.add_argument("output", type=Path) + verify = commands.add_parser("verify-mutant") + verify.add_argument("side", choices=("core", "reference")) + verify.add_argument("input", type=Path) + verify.add_argument("core", type=Path) + verify.add_argument("reference", type=Path) + for command in (compare, validate, verify): + command.add_argument("--expected-revision", required=True) + args = parser.parse_args(argv) + + if args.command == "expand": + write_jsonl(args.output, expand_spec(load_spec(args.spec))) + elif args.command == "split": + shards = split_records(read_jsonl(args.input, "input")) + unexpected = sorted(path.name for path in args.output_dir.glob("*.jsonl") + if path.name not in {f"{name}.jsonl" for name in shards}) + if unexpected: + raise HarnessError(f"split output directory contains unexpected JSONL: {unexpected}") + for name, rows in shards.items(): + write_jsonl(args.output_dir / f"{name}.jsonl", rows) + elif args.command == "run": + write_jsonl( + args.output, + run_records(read_jsonl(args.input, "input"), args.side, source_revision=args.source_revision, mutant=args.mutant), + ) + elif args.command == "merge": + expected = read_jsonl(args.expected, "expected input") + parts = [read_jsonl(path, f"shard {path}") for path in args.parts] + write_jsonl(args.output, merge_records(parts, expected)) + elif args.command == "validate": + diffs = validate_records(read_jsonl(args.input, "input"), + read_jsonl(args.output, "output"), args.side, + expected_revision=args.expected_revision) + for line in diffs: + print(line) + return 1 if diffs else 0 + elif args.command == "compare": + diffs = compare_records( + read_jsonl(args.input, "input"), + read_jsonl(args.core, "core output"), + read_jsonl(args.reference, "reference output"), + expected_revision=args.expected_revision, + core_side=args.core_side, reference_side=args.reference_side, + ) + for line in diffs: + print(line) + return 1 if diffs else 0 + else: + killed = verify_mutant( + read_jsonl(args.input, "input"), + read_jsonl(args.core, "core output"), + read_jsonl(args.reference, "reference output"), + args.side, + expected_revision=args.expected_revision, + ) + print(f"OK killed {killed} {args.side} mutant case(s)") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except HarnessError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + raise SystemExit(2) diff --git a/Tools/parity_shards.py b/Tools/parity_shards.py new file mode 100644 index 0000000000..202f6b3e22 --- /dev/null +++ b/Tools/parity_shards.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Ordered operation and shard authority for the portable parity harness.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Callable + + +JsonObject = dict[str, object] +Evaluator = Callable[[JsonObject], object] + + +class RegistryError(ValueError): + """Raised when the registry or an operation contract is invalid.""" + + +@dataclass(frozen=True) +class OperationSpec: + key: str + argument_keys: tuple[str, ...] + core: Evaluator + reference: Evaluator + mutant: Callable[[object], object] + + +@dataclass(frozen=True) +class ShardSpec: + name: str + operations: tuple[OperationSpec, ...] + + +def _clamp_arguments(args: JsonObject) -> tuple[int | float, int | float, int | float]: + if set(args) != {"lower", "upper", "value"}: + raise RegistryError("Portable.clamp/3 args must contain exactly lower, upper, value") + values = tuple(args[key] for key in ("value", "lower", "upper")) + if any(isinstance(value, bool) or not isinstance(value, (int, float)) for value in values): + raise RegistryError("Portable.clamp/3 args must be finite numbers") + if any(not math.isfinite(value) for value in values): + raise RegistryError("Portable.clamp/3 args must be finite numbers") + value, lower, upper = values + if lower > upper: + raise RegistryError("Portable.clamp/3 lower must not exceed upper") + return value, lower, upper + + +def _core_clamp(args: JsonObject) -> object: + value, lower, upper = _clamp_arguments(args) + return value if lower <= value <= upper else max(lower, min(upper, value)) + + +def _reference_clamp(args: JsonObject) -> object: + value, lower, upper = _clamp_arguments(args) + if value < lower: + return lower + if value > upper: + return upper + return value + + +def _off_by_one(value: object) -> object: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise RegistryError("representative mutant requires a numeric result") + return value + 1 + + +REGISTRY = ( + ShardSpec( + "core-reference", + ( + OperationSpec( + "Portable.clamp/3", + ("lower", "upper", "value"), + _core_clamp, + _reference_clamp, + _off_by_one, + ), + ), + ), +) + + +def validate_registry(registry: tuple[ShardSpec, ...]) -> None: + if not registry: + raise RegistryError("registry must contain at least one shard") + shard_names = [shard.name for shard in registry] + if shard_names != sorted(shard_names) or len(shard_names) != len(set(shard_names)): + raise RegistryError("shards must have unique names in canonical order") + seen: set[str] = set() + for shard in registry: + if not shard.name or not shard.operations: + raise RegistryError("every shard must have a name and at least one operation") + keys = [operation.key for operation in shard.operations] + if keys != sorted(keys): + raise RegistryError(f"shard {shard.name} operations must be in canonical order") + for operation in shard.operations: + if not operation.key or operation.key in seen: + raise RegistryError(f"duplicate operation {operation.key!r}") + if operation.argument_keys != tuple(sorted(operation.argument_keys)): + raise RegistryError(f"operation {operation.key} argument keys must be canonical") + seen.add(operation.key) + + +validate_registry(REGISTRY) + + +def operation_index() -> dict[str, tuple[ShardSpec, OperationSpec]]: + return { + operation.key: (shard, operation) + for shard in REGISTRY + for operation in shard.operations + } diff --git a/Tools/tests/test_parity_governance_acceptance.py b/Tools/tests/test_parity_governance_acceptance.py index f4ffff3e76..b449d45793 100644 --- a/Tools/tests/test_parity_governance_acceptance.py +++ b/Tools/tests/test_parity_governance_acceptance.py @@ -130,6 +130,7 @@ def test_core_tools_filter_covers_every_governance_tool_path(self) -> None: "Tools/issue_ref.py", "Tools/parity_*.py", "Tools/parity_*.json", + "Tools/parity_case_specs/**", "Tools/tests/test_parity_*.py", "Tools/tests/test_rr_legacy_preservation_contract.py", ".github/workflows/parity-governance.yml", @@ -151,7 +152,7 @@ def test_core_tools_filter_covers_every_governance_tool_path(self) -> None: self.assertIn("tests.test_parity_ledger", governance) self.assertIn("tests.test_parity_governance_acceptance", governance) self.assertIn("tests.test_rr_legacy_preservation_contract", governance) - pull_request_paths = governance_paths[:6] + pull_request_paths = governance_paths[:7] self.assertFalse(any( fnmatchcase("Tools/tests/test_german_today_localization.py", pattern) for pattern in pull_request_paths diff --git a/Tools/tests/test_parity_harness.py b/Tools/tests/test_parity_harness.py new file mode 100644 index 0000000000..57f4e4f719 --- /dev/null +++ b/Tools/tests/test_parity_harness.py @@ -0,0 +1,482 @@ +from __future__ import annotations + +import io +import json +import sys +import subprocess +import tempfile +import unittest +from contextlib import redirect_stdout +from pathlib import Path + + +TOOLS = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(TOOLS)) + +import parity_harness +import parity_shards + + +FIXTURE = TOOLS / "parity_case_specs/core_reference.json" + + +def invoke_cli(args): + if args[0] == "run": + args += ["--source-revision", "a" * 40] + elif args[0] in ("compare", "verify-mutant", "validate"): + args += ["--expected-revision", "a" * 40] + return parity_harness.main(args) + + +def valid_spec() -> dict: + return json.loads(FIXTURE.read_text(encoding="utf-8")) + + +class SharedOracleTests(unittest.TestCase): + REVISION = "a" * 40 + + def setUp(self): + self.rows = parity_harness.expand_spec(valid_spec()) + + def run_side(self, side="core"): + return parity_harness.run_records(self.rows, side, source_revision=self.REVISION) + + def test_one_platform_validates_without_peer(self): + output = self.run_side() + for row in output: + row["side"] = "android" + self.assertEqual([], parity_harness.validate_records( + self.rows, output, "android", expected_revision=self.REVISION)) + + def test_agreeing_but_wrong_outputs_fail_oracle(self): + core, reference = self.run_side(), self.run_side("reference") + core[0]["value"] = reference[0]["value"] = 999 + errors = parity_harness.compare_records( + self.rows, core, reference, expected_revision=self.REVISION) + self.assertEqual(2, len(errors)) + self.assertTrue(all("EXPECTED" in error for error in errors)) + + def test_revision_rejects_stale_and_missing_contract(self): + output = self.run_side() + for revision in ("", "main", "a" * 7, "g" * 40, "b" * 40): + with self.subTest(revision=revision), self.assertRaises(parity_harness.HarnessError): + parity_harness.validate_records( + self.rows, output, "core", expected_revision=revision) + output[0]["sourceRevision"] = "b" * 40 + with self.assertRaisesRegex(parity_harness.HarnessError, "sourceRevision"): + parity_harness.validate_records( + self.rows, output, "core", expected_revision=self.REVISION) + + def test_expected_and_suite_version_are_digest_bound(self): + for field, value in (("expected", 999), ("suiteVersion", "changed")): + rows = [dict(row) for row in self.rows] + rows[0][field] = value + with self.subTest(field=field), self.assertRaises(parity_harness.HarnessError): + parity_harness.validate_records( + rows, self.run_side(), "core", expected_revision=self.REVISION) + + def test_suite_version_required_and_nonempty(self): + for value in (None, "", " ", 1): + spec = valid_spec() + spec["suiteVersion"] = value + with self.subTest(value=value), self.assertRaises(parity_harness.HarnessError): + parity_harness.expand_spec(spec) + + def test_missing_expected_is_rejected(self): + spec = valid_spec() + del spec["shards"][0]["operations"][0]["cases"][0]["expected"] + with self.assertRaisesRegex(parity_harness.HarnessError, "fields must be exactly"): + parity_harness.expand_spec(spec) + + def test_stale_or_mixed_provenance_is_rejected_even_when_values_agree(self): + for side in ("core", "reference", "both"): + core, reference = self.run_side(), self.run_side("reference") + for row in (core if side == "core" else reference if side == "reference" else core + reference): + row["sourceRevision"] = "b" * 40 + with self.subTest(side=side), self.assertRaisesRegex(parity_harness.HarnessError, "sourceRevision"): + parity_harness.compare_records(self.rows, core, reference, expected_revision=self.REVISION) + output = self.run_side() + output[0]["suiteVersion"] = "stale" + with self.assertRaisesRegex(parity_harness.HarnessError, "suiteVersion"): + parity_harness.validate_records(self.rows, output, "core", expected_revision=self.REVISION) + + def test_rehashed_case_changes_still_reject_previous_output(self): + output = self.run_side() + for field, value in (("expected", 999), ("suiteVersion", "new-version")): + rows = [{**row, field: value} for row in self.rows] + for row in rows: + row["caseSha256"] = parity_harness.case_digest(row) + with self.subTest(field=field), self.assertRaisesRegex(parity_harness.HarnessError, "caseSha256"): + parity_harness.validate_records(rows, output, "core", expected_revision=self.REVISION) + + def test_validation_rejects_invalid_single_artifact_and_preserves_exact_oracle_types(self): + output = self.run_side() + for candidate in (output[:-1], [*output, output[0]], + [{**output[0], "side": "swift"}, *output[1:]], + [{**output[0], "mutant": True}, *output[1:]]): + with self.subTest(candidate=candidate), self.assertRaises(parity_harness.HarnessError): + parity_harness.validate_records(self.rows, candidate, "core", expected_revision=self.REVISION) + for expected, actual in ((True, 1), ([1], [1.0]), ({"x": -0.0}, {"x": 0.0})): + rows = [dict(row) for row in self.rows] + rows[0]["expected"] = expected + rows[0]["caseSha256"] = parity_harness.case_digest(rows[0]) + candidate = [dict(row) for row in output] + candidate[0].update(caseSha256=rows[0]["caseSha256"], value=actual) + with self.subTest(expected=expected): + self.assertEqual(1, len(parity_harness.validate_records( + rows, candidate, "core", expected_revision=self.REVISION))) + + def test_mutant_control_rejects_wrong_normal_side(self): + core = parity_harness.run_records(self.rows, "core", source_revision=self.REVISION, mutant=True) + reference = self.run_side("reference") + reference[0]["value"] = 999 + with self.assertRaisesRegex(parity_harness.HarnessError, "non-mutant side"): + parity_harness.verify_mutant(self.rows, core, reference, "core", expected_revision=self.REVISION) + + def test_single_side_cli_exit_codes_and_later_artifact_comparison(self): + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) + inputs, android, swift = (target / name for name in ("cases.jsonl", "android.jsonl", "swift.jsonl")) + parity_harness.write_jsonl(inputs, self.rows) + output = self.run_side() + for row in output: + row["side"] = "android" + parity_harness.write_jsonl(android, output) + command = [sys.executable, str(TOOLS / "parity_harness.py"), "validate", "android", str(inputs), str(android)] + def check(args, code, message=""): + result = subprocess.run(args, capture_output=True, text=True) + self.assertEqual(code, result.returncode, result.stdout + result.stderr) + self.assertIn(message, result.stdout + result.stderr) + check(command, 2, "--expected-revision") + check(command + ["--expected-revision", "main"], 2, "full lowercase Git revision") + command += ["--expected-revision", self.REVISION] + check(command, 0) + self.assertFalse(swift.exists()) # No peer artifact is needed for standalone success. + output[0]["value"] = 999 + parity_harness.write_jsonl(android, output) + check(command, 1, "EXPECTED") + for row in output: + row["side"] = "swift" + parity_harness.write_jsonl(swift, output) + check([sys.executable, str(TOOLS / "parity_harness.py"), "compare", str(inputs), str(android), str(swift), + "--core-side", "android", "--reference-side", "swift", "--expected-revision", self.REVISION], 1, "EXPECTED") + output[0]["sourceRevision"] = "b" * 40 + parity_harness.write_jsonl(swift, output) + check([sys.executable, str(TOOLS / "parity_harness.py"), "validate", "swift", str(inputs), str(swift), + "--expected-revision", self.REVISION], 2, "sourceRevision") + + def test_oracle_is_literal_and_not_reference_generated(self): + spec = valid_spec() + spec["shards"][0]["operations"][0]["cases"][0]["expected"] = 999 + rows = parity_harness.expand_spec(spec) + output = parity_harness.run_records(rows, "reference", source_revision=self.REVISION) + self.assertEqual(1, len(parity_harness.validate_records( + rows, output, "reference", expected_revision=self.REVISION))) + + +class CaseSchemaTests(unittest.TestCase): + def test_checked_fixture_expands_deterministically_and_canonically(self) -> None: + spec = parity_harness.load_spec(FIXTURE) + first = parity_harness.expand_spec(spec) + second = parity_harness.expand_spec(parity_harness.load_spec(FIXTURE)) + self.assertEqual(first, second) + self.assertEqual(parity_harness.canonical_jsonl(first), parity_harness.canonical_jsonl(second)) + self.assertEqual(sorted(row["id"] for row in first), [row["id"] for row in first]) + for row in first: + self.assertEqual( + {"args", "caseSha256", "expected", "function", "id", "shard", "suiteVersion"}, set(row) + ) + self.assertEqual(row["caseSha256"], parity_harness.case_digest(row)) + + def test_schema_rejects_unknown_fields_at_every_level(self) -> None: + paths = [ + (), + ("shards", 0), + ("shards", 0, "operations", 0), + ("shards", 0, "operations", 0, "cases", 0), + ] + for path in paths: + candidate = valid_spec() + node = candidate + for part in path: + node = node[part] + node["unknown"] = True + with self.subTest(path=path), self.assertRaisesRegex( + parity_harness.HarnessError, "fields must be exactly" + ): + parity_harness.validate_spec(candidate) + + def test_schema_rejects_version_order_duplicates_and_nonfinite_values(self) -> None: + candidate = valid_spec() + candidate["schemaVersion"] = 2 + with self.assertRaisesRegex(parity_harness.HarnessError, "schemaVersion"): + parity_harness.validate_spec(candidate) + + candidate = valid_spec() + candidate["shards"][0]["operations"][0]["cases"].reverse() + with self.assertRaisesRegex(parity_harness.HarnessError, "canonical id order"): + parity_harness.validate_spec(candidate) + + candidate = valid_spec() + candidate["shards"][0]["operations"][0]["cases"][1]["id"] = "above" + with self.assertRaisesRegex(parity_harness.HarnessError, "unique"): + parity_harness.validate_spec(candidate) + + with self.assertRaisesRegex(parity_harness.HarnessError, "finite"): + parity_harness.parse_json('{"schemaVersion":NaN,"shards":[]}') + + with self.assertRaisesRegex(parity_harness.HarnessError, "duplicate JSON key"): + parity_harness.parse_json('{"schemaVersion":1,"schemaVersion":1,"shards":[]}') + + candidate = valid_spec() + candidate["shards"][0]["operations"][0]["cases"][0]["args"]["value"] = 1 << 80 + with self.assertRaisesRegex(parity_harness.HarnessError, "signed 64-bit"): + parity_harness.validate_spec(candidate) + + def test_jsonl_requires_exact_canonical_encoding(self) -> None: + rows = parity_harness.expand_spec(valid_spec()) + encoded = parity_harness.canonical_jsonl(rows) + self.assertEqual(rows, parity_harness.parse_jsonl(encoded, "fixture")) + with self.assertRaisesRegex(parity_harness.HarnessError, "canonical"): + parity_harness.parse_jsonl(encoded.replace(":", ": ", 1), "fixture") + with self.assertRaisesRegex(parity_harness.HarnessError, "final newline"): + parity_harness.parse_jsonl(encoded.rstrip("\n"), "fixture") + + +class RegistryAndShardTests(unittest.TestCase): + def test_registry_is_ordered_unique_and_is_the_operation_authority(self) -> None: + parity_shards.validate_registry(parity_shards.REGISTRY) + operation_keys = [ + operation.key for shard in parity_shards.REGISTRY for operation in shard.operations + ] + self.assertEqual(len(operation_keys), len(set(operation_keys))) + self.assertEqual(operation_keys, sorted(operation_keys)) + + duplicate = ( + parity_shards.ShardSpec( + "a", (parity_shards.REGISTRY[0].operations[0],) + ), + parity_shards.ShardSpec( + "b", (parity_shards.REGISTRY[0].operations[0],) + ), + ) + with self.assertRaisesRegex(parity_shards.RegistryError, "duplicate operation"): + parity_shards.validate_registry(duplicate) + + def test_split_is_disjoint_complete_nonempty_canonical_and_exact_remerge(self) -> None: + rows = parity_harness.expand_spec(valid_spec()) + shards = parity_harness.split_records(rows) + self.assertEqual([shard.name for shard in parity_shards.REGISTRY], list(shards)) + id_sets = [{row["id"] for row in part} for part in shards.values()] + self.assertTrue(all(id_set for id_set in id_sets)) + self.assertFalse(any(left & right for i, left in enumerate(id_sets) for right in id_sets[i + 1 :])) + self.assertEqual({row["id"] for row in rows}, set().union(*id_sets)) + self.assertEqual(rows, parity_harness.merge_records(list(shards.values()), rows)) + self.assertEqual( + parity_harness.canonical_jsonl(rows), + parity_harness.canonical_jsonl(parity_harness.merge_records(list(shards.values()), rows)), + ) + + def test_merge_rejects_missing_duplicate_and_unexpected_ids(self) -> None: + rows = parity_harness.expand_spec(valid_spec()) + for parts, message in ( + ([[*rows, rows[0]]], "duplicate"), + ([rows[:-1]], "missing"), + ([[*rows, {**rows[-1], "id": "unexpected"}]], "unexpected"), + ): + with self.subTest(message=message), self.assertRaisesRegex( + parity_harness.HarnessError, message + ): + parity_harness.merge_records(parts, rows) + + def test_merge_rejects_reordered_shard_rows(self) -> None: + rows = parity_harness.expand_spec(valid_spec()) + with self.assertRaisesRegex(parity_harness.HarnessError, "canonical id order"): + parity_harness.merge_records([list(reversed(rows))], rows) + + def test_merge_binds_exactly_one_part_to_each_registered_shard(self) -> None: + rows = parity_harness.expand_spec(valid_spec()) + with self.assertRaisesRegex(parity_harness.HarnessError, "one part per registered shard"): + parity_harness.merge_records([[rows[0]], rows[1:]], rows) + + def test_split_rejects_stale_jsonl_in_output_directory(self) -> None: + rows = parity_harness.expand_spec(valid_spec()) + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) + source = target / "cases.jsonl" + output = target / "shards" + parity_harness.write_jsonl(source, rows) + output.mkdir() + (output / "stale.jsonl").write_text("{}\n", encoding="utf-8") + with self.assertRaisesRegex(parity_harness.HarnessError, "unexpected JSONL"): + invoke_cli(["split", str(source), str(output)]) + + +class ComparatorTests(unittest.TestCase): + def setUp(self) -> None: + self.rows = parity_harness.expand_spec(valid_spec()) + + def test_positive_core_reference_compare(self) -> None: + core = parity_harness.run_records(self.rows, "core", source_revision="a" * 40) + reference = parity_harness.run_records(self.rows, "reference", source_revision="a" * 40) + self.assertEqual([], parity_harness.compare_records(self.rows, core, reference, expected_revision="a" * 40)) + + def test_both_mutant_sides_are_killed(self) -> None: + normal_core = parity_harness.run_records(self.rows, "core", source_revision="a" * 40) + normal_reference = parity_harness.run_records(self.rows, "reference", source_revision="a" * 40) + core_mutant = parity_harness.run_records(self.rows, "core", mutant=True, source_revision="a" * 40) + reference_mutant = parity_harness.run_records(self.rows, "reference", mutant=True, source_revision="a" * 40) + self.assertEqual( + len(self.rows), + parity_harness.verify_mutant(self.rows, core_mutant, normal_reference, "core", expected_revision="a" * 40), + ) + self.assertEqual( + len(self.rows), + parity_harness.verify_mutant(self.rows, normal_core, reference_mutant, "reference", expected_revision="a" * 40), + ) + with self.assertRaisesRegex(parity_harness.HarnessError, "expected core mutant"): + parity_harness.verify_mutant( + self.rows, normal_core, reference_mutant, "core", + expected_revision="a" * 40, + ) + + def test_output_contract_rejects_missing_duplicate_unexpected_and_retargeted_rows(self) -> None: + core = parity_harness.run_records(self.rows, "core", source_revision="a" * 40) + reference = parity_harness.run_records(self.rows, "reference", source_revision="a" * 40) + mutations = [ + (core[:-1], "missing"), + ([*core, core[0]], "duplicate"), + ([*core, {**core[-1], "id": "unexpected"}], "unexpected"), + ([{**core[0], "function": "Other.operation/0"}, *core[1:]], "function mismatch"), + ([{**core[0], "extra": True}, *core[1:]], "fields must be exactly"), + ] + for mutated, message in mutations: + with self.subTest(message=message), self.assertRaisesRegex( + parity_harness.HarnessError, message + ): + parity_harness.compare_records(self.rows, mutated, reference, expected_revision="a" * 40) + + def test_malformed_metadata_always_raises_stable_harness_error(self) -> None: + core = parity_harness.run_records(self.rows, "core", source_revision="a" * 40) + reference = parity_harness.run_records(self.rows, "reference", source_revision="a" * 40) + malformed_input = [{**self.rows[0], "function": []}, *self.rows[1:]] + with self.assertRaises(parity_harness.HarnessError): + parity_harness.compare_records(malformed_input, core, reference, expected_revision="a" * 40) + with self.assertRaises(parity_harness.HarnessError): + parity_harness.verify_mutant(self.rows, [42], reference, "core", expected_revision="a" * 40) + + def test_output_order_and_full_case_identity_are_canonical_and_bound(self) -> None: + core = parity_harness.run_records(self.rows, "core", source_revision="a" * 40) + reference = parity_harness.run_records(self.rows, "reference", source_revision="a" * 40) + with self.assertRaisesRegex(parity_harness.HarnessError, "canonical id order"): + parity_harness.compare_records(self.rows, list(reversed(core)), reference, expected_revision="a" * 40) + + rebound = [dict(row) for row in self.rows] + rebound[0]["id"] = "other/Portable.clamp/3/above" + rebound[0]["caseSha256"] = parity_harness.case_digest(rebound[0]) + rebound.sort(key=lambda row: row["id"]) + with self.assertRaisesRegex(parity_harness.HarnessError, "structurally bound"): + parity_harness.run_records(rebound, "core", source_revision="a" * 40) + + def test_one_reference_shard_end_to_end_and_cli(self) -> None: + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) + expanded = target / "cases.jsonl" + core = target / "core.jsonl" + reference = target / "reference.jsonl" + core_mutant = target / "core-mutant.jsonl" + reference_mutant = target / "reference-mutant.jsonl" + with redirect_stdout(io.StringIO()): + self.assertEqual(0, invoke_cli(["expand", str(FIXTURE), str(expanded)])) + self.assertEqual(0, invoke_cli(["run", "core", str(expanded), str(core)])) + self.assertEqual(0, invoke_cli(["run", "reference", str(expanded), str(reference)])) + self.assertEqual( + 0, + invoke_cli( + ["compare", str(expanded), str(core), str(reference)] + ), + ) + self.assertEqual( + 0, + invoke_cli( + ["run", "core", str(expanded), str(core_mutant), "--mutant"] + ), + ) + self.assertEqual( + 0, + invoke_cli( + ["run", "reference", str(expanded), str(reference_mutant), "--mutant"] + ), + ) + self.assertEqual( + 0, + invoke_cli( + [ + "verify-mutant", "core", str(expanded), + str(core_mutant), str(reference), + ] + ), + ) + self.assertEqual( + 0, + invoke_cli( + [ + "verify-mutant", "reference", str(expanded), + str(core), str(reference_mutant), + ] + ), + ) + self.assertEqual(self.rows, parity_harness.read_jsonl(expanded, "expanded")) + + +class PortableContractRegressionTests(unittest.TestCase): + def test_schema_version_requires_integer_one(self): + for version in (True, 1.0): + spec = valid_spec() + spec["schemaVersion"] = version + with self.subTest(version=version), self.assertRaises(parity_harness.HarnessError): + parity_harness.validate_spec(spec) + + def test_comparison_preserves_nested_types_and_signed_zero(self): + rows = parity_harness.expand_spec(valid_spec()) + for left, right in (({"x": True}, {"x": 1}), ([1], [1.0]), (-0.0, 0.0)): + core = parity_harness.run_records(rows, "core", source_revision="a" * 40) + reference = parity_harness.run_records(rows, "reference", source_revision="a" * 40) + core[0]["value"], reference[0]["value"] = left, right + with self.subTest(left=left, right=right): + self.assertTrue(any(line.startswith("DIFF") for line in parity_harness.compare_records(rows, core, reference, expected_revision="a" * 40))) + + def test_unicode_separators_round_trip(self): + rows = [{"value": "a\u0085b\u2028c\u2029d"}] + self.assertEqual(rows, parity_harness.parse_jsonl(parity_harness.canonical_jsonl(rows), "unicode")) + + def test_crlf_is_rejected_in_memory_and_on_disk(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "input.jsonl" + path.write_bytes(b'{"value":1}\r\n') + with self.assertRaises(parity_harness.HarnessError): + parity_harness.parse_jsonl(path.read_bytes().decode(), "input") + with self.assertRaises(parity_harness.HarnessError): + parity_harness.read_jsonl(path, "input") + + def test_unpaired_surrogates_are_rejected(self): + for value in ("\ud800", {"\udfff": 1}): + with self.subTest(value=repr(value)), self.assertRaises(parity_harness.HarnessError): + parity_harness.canonical_json(value) + + def test_clamp_preserves_value_at_mixed_numeric_boundaries(self): + operation = parity_shards.REGISTRY[0].operations[0] + for value in (0.0, -0.0, 10.0): + args = {"lower": 0, "upper": 10, "value": value} + with self.subTest(value=value): + self.assertEqual(parity_harness.canonical_json(value), parity_harness.canonical_json(operation.core(args))) + self.assertEqual(parity_harness.canonical_json(value), parity_harness.canonical_json(operation.reference(args))) + + def test_write_failure_reports_harness_error(self): + with tempfile.TemporaryDirectory() as directory: + with self.assertRaises(parity_harness.HarnessError): + parity_harness.write_jsonl(Path(directory), [{"value": 1}]) + + +if __name__ == "__main__": + unittest.main()