From a5652922b54c15e42bc46bec8ac36931842709f9 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Fri, 23 Oct 2026 14:35:32 +0800 Subject: [PATCH 1/4] feat(versioning): freeze benchmark editions and compare contracts --- CONTRIBUTING.md | 4 + README.md | 8 ++ docs/README.zh-CN.md | 7 ++ docs/versions.md | 75 +++++++++++++++ src/lexbench_webfetch/cli.py | 22 ++++- src/lexbench_webfetch/versions.py | 153 ++++++++++++++++++++++++++++++ tests/test_release_versions.py | 150 +++++++++++++++++++++++++++++ 7 files changed, 418 insertions(+), 1 deletion(-) create mode 100644 docs/versions.md create mode 100644 src/lexbench_webfetch/versions.py create mode 100644 tests/test_release_versions.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bcc0fb7..ab65b0a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,9 @@ # Contributing +Task, reference and scoring releases follow [benchmark versions](docs/versions.md). +Keep package releases separate from benchmark editions; a new package commit +must not silently replace a report's frozen inputs. + Task definitions and acceptance requirements belong here. Task models, acceptance scoring, file checks and compilation also belong here. Browser adapters, scheduling, reference-capture orchestration and report rendering belong in the selected execution harness. Include a task ID and a reproducible accepted/rejected example when reporting a rule problem. Use Python 3.11 or newer and [uv](https://docs.astral.sh/uv/). Run `uv sync --frozen`, then `make check`. Run `uv sync --frozen --extra test` to install the test tools. Tests use synthetic inputs and need no browser, network access or historical source bundle. Keep the English and Chinese READMEs aligned; other guides use English. diff --git a/README.md b/README.md index ac3a5c6..0ecb625 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,14 @@ uv run --frozen lexbench-webfetch export \ The export includes only manifest-listed files and prints the executable dataset path. Missing files, changed bytes and symlinks fail validation. Existing output directories are never reused. +## Freeze a benchmark version + +Reports bind a benchmark edition, its frozen dataset and its scoring source. +Package upgrades do not change historical results. Use `lexbench-webfetch release` +to freeze an external `benchmark-release.json`, and `compare-releases` to check +whether two editions support direct comparison, require rescoring or belong +to separate major versions. See [benchmark versions](docs/versions.md). + ## Score a saved answer The benchmark includes its own scorer. It needs no Browser Eval installation: diff --git a/docs/README.zh-CN.md b/docs/README.zh-CN.md index 2ccce76..7a318e8 100644 --- a/docs/README.zh-CN.md +++ b/docs/README.zh-CN.md @@ -69,6 +69,13 @@ uv run --frozen lexbench-webfetch export \ 导出工具只复制清单内的文件，并打印可执行数据集路径。缺文件、内容变化或符号链接都会使校验失败，已有输出目录不会被复用。 +## 固定评测版本 + +报告绑定评测版本、冻结题目和评分源码。升级软件包不会改变历史结果。 +用 `lexbench-webfetch release` 在外部输入目录生成 `benchmark-release.json`， +用 `compare-releases` 检查两个版本能否直接比较、是否需要统一重评，或应按不同大版本分别展示。 +具体规则见[评测版本管理](versions.md)。 + ## 对保存的响应评分 本仓库自带评分器，无需安装 Browser Eval： diff --git a/docs/versions.md b/docs/versions.md new file mode 100644 index 0000000..4de87ba --- /dev/null +++ b/docs/versions.md @@ -0,0 +1,75 @@ +# Benchmark versions + +A benchmark edition fixes the task population, request contract, acceptance +rules, frozen references and scoring source. Its version is independent of +the Python package version. A package upgrade does not upgrade a report. + +Use a readable `MAJOR.MINOR.PATCH` together with immutable SHA-256 bindings. +Keep the exact repository commit in the harness dependency lock. A Git tag +such as `benchmark-v2.0.0` gives a released commit a durable name; never move +or overwrite a published release tag. A tag alone does not identify external +reference bytes or a report's selected task population. + +| Change | Version | Comparing results | +| --- | --- | --- | +| Documentation or presentation; measurement contracts and scorer unchanged | Patch | Direct, with the same execution controls | +| Reviewed scoring/reference correction; original tasks and requests retained | Minor | Recompute both saved cohorts under one edition before comparison | +| Added/removed tasks, different URLs, requests, output scope or retrieval route | Major | Start a separate benchmark series of results | + +A minor version is for correcting the existing measurement, not adding a new +scoring dimension under the same name. Expanding stable-label checks into a +main-content benchmark is a major change. The release tool checks concrete +population and request differences; maintainers still review whether a rule +change preserves the intended task. Existing annotation revisions retain +their independent source evidence and executable correction controls. + +## Freeze an edition + +Compile or export the inputs through their existing owner first, including any +registered annotation revision. Freeze the complete selected population before +provider calls: + +```sh +uv run --frozen lexbench-webfetch release \ + --dataset /absolute/inputs/dataset.json \ + --output /absolute/inputs/benchmark-release.json \ + --benchmark lexbench-webfetch --version 2.0.0 +``` + +For a successor, add `--previous /absolute/previous/benchmark-release.json`. +Scoring corrections also require a concrete `--change-note`. The tool rejects +an insufficient version bump and an existing output path. Keep the sidecar +alongside frozen inputs, outside Git. It contains no provider measurements or +local paths. Commit reusable definitions and publish the code through the +normal feature-branch and release process. + +```sh +uv run --frozen lexbench-webfetch compare-releases \ + /absolute/old/benchmark-release.json /absolute/new/benchmark-release.json +``` + +The result is `direct`, `rescore` or `separate`. It checks actual contract +fingerprints in addition to version numbers. Reusing the same version for a +different measurement is an error. A different major edition stays separate +even if some task IDs overlap. An overlap analysis may be diagnostic, but it +does not make the two full-population percentages comparable. + +## Reports and retained calls + +Report builds retain the selected edition, dataset digest and per-result +scoring-source identities. Rebuilding presentation preserves that edition. +Scoring corrections create a new edition and result revision, retaining the +previous report. Do not replace task inputs from an installed package merely +because it is newer. + +For a minor correction, reuse successful and failed saved outputs wherever +the original request contract still applies, and recompute their scores under +one scorer. Changed requests need new calls for the affected scope. Preserve +all physical attempts and use the existing final-attempt selection policy. +Execution budgets, method configurations and timing comparability remain +owned by the harness; a matching dataset edition does not establish those. + +Historical reports without a readable edition keep their existing exact hash +bindings. Naming one does not change its task snapshot, results, permissions +or original scoring source. Never label a historical subset as the latest +full dataset. diff --git a/src/lexbench_webfetch/cli.py b/src/lexbench_webfetch/cli.py index 8ff85e2..ba300d0 100644 --- a/src/lexbench_webfetch/cli.py +++ b/src/lexbench_webfetch/cli.py @@ -10,6 +10,16 @@ def main() -> None: parser = argparse.ArgumentParser(description="Inspect and verify LexBench Web Fetch inputs") sub = parser.add_subparsers(dest="command", required=True) sub.add_parser("list", help="List the versioned suites") + release = sub.add_parser("release", help="Freeze a dataset and scorer as a benchmark edition") + release.add_argument("--dataset", type=Path, required=True) + release.add_argument("--output", type=Path, required=True) + release.add_argument("--benchmark", required=True) + release.add_argument("--version", required=True) + release.add_argument("--previous", type=Path) + release.add_argument("--change-note") + compare = sub.add_parser("compare-releases", help="Check whether two result editions can be compared") + compare.add_argument("left", type=Path) + compare.add_argument("right", type=Path) check = sub.add_parser("validate", help="Check all public definitions and bindings") check.add_argument("--suite") for name in ("verify-inputs", "export"): @@ -43,7 +53,17 @@ def main() -> None: articles.add_argument("--evidence-home", type=Path, required=True) args = parser.parse_args() try: - if args.command == "inspect-revision": + if args.command == "release": + from .versions import freeze_release + previous = json.loads(args.previous.read_text()) if args.previous else None + result = freeze_release(args.dataset, args.output, benchmark=args.benchmark, + version=args.version, previous=previous, + change_note=args.change_note) + elif args.command == "compare-releases": + from .versions import compare_releases + result = compare_releases(json.loads(args.left.read_text()), + json.loads(args.right.read_text())) + elif args.command == "inspect-revision": result = inspect_annotation_revision(args.suite, args.revision) elif args.command == "score": from .evaluation import evaluate_response diff --git a/src/lexbench_webfetch/versions.py b/src/lexbench_webfetch/versions.py new file mode 100644 index 0000000..7b12b9e --- /dev/null +++ b/src/lexbench_webfetch/versions.py @@ -0,0 +1,153 @@ +"""Freeze benchmark editions and classify measurement compatibility.""" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +import re + +from .benchmark import public_task +from .scoring import SCORING_SOURCE_SHA256 + + +def _digest(value) -> str: + raw = json.dumps(value, ensure_ascii=False, sort_keys=True, + separators=(",", ":"), allow_nan=False).encode() + return hashlib.sha256(raw).hexdigest() + + +def _version(value: str) -> tuple[int, int, int]: + if not isinstance(value, str) or not re.fullmatch( + r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)", value + ): + raise ValueError("Release version must be an explicit MAJOR.MINOR.PATCH") + return tuple(int(part) for part in value.split(".")) + + +def _measurement(dataset: dict) -> dict: + cases = dataset.get("cases") + if not isinstance(cases, list) or not cases: + raise ValueError("Release dataset needs nonempty cases") + ids = [case.get("id") for case in cases] + if (any(not isinstance(key, str) or not key for key in ids) + or len(set(ids)) != len(ids)): + raise ValueError("Release dataset requires unique nonempty case IDs") + inputs, contracts = [], [] + for case in sorted(cases, key=lambda item: item["id"]): + task = public_task(case) + request = {key: task[key] for key in ( + "id", "url", "track", "task_kind", "retrieval_hints", "evaluation_url", + ) if key in task} + request["prompt"] = task.get("task_prompt") or task.get("description", "") + route = task.get("source_benchmark", {}) + request["source_route"] = {key: route[key] for key in + ("mode", "hosts", "entry_path") if key in route} + mime = task.get("source_verification", {}).get("mime") + if mime is not None: + request["input_mime"] = mime + inputs.append(request) + ignored = {"name", "category", "description", "task_prompt", "source_review_url", + "url", "track", "task_kind", "retrieval_hints", "evaluation_url"} + contract = {key: value for key, value in task.items() if key not in ignored} + for key in ("ground_truth_main_content", "reference_answer"): + if case.get(key) is not None: + contract[key + "_sha256"] = _digest(case[key]) + contracts.append(contract) + return {"case_count": len(cases), "case_ids_sha256": _digest(sorted(ids)), + "input_contract_sha256": _digest(inputs), + "acceptance_sha256": _digest(contracts)} + + +def _validate(release: dict) -> tuple[int, int, int]: + if release.get("schema_version") != 1 or release.get("kind") != "benchmark_release": + raise ValueError("Unsupported benchmark release") + benchmark = release.get("benchmark") + if not isinstance(benchmark, str) or not re.fullmatch(r"[a-z0-9]+(?:[-_.][a-z0-9]+)*", benchmark): + raise ValueError("Release benchmark needs a stable identifier") + version = _version(release.get("version")) + if type(release.get("case_count")) is not int or release["case_count"] < 1: + raise ValueError("Release case count must be positive") + for key in ("dataset_sha256", "case_ids_sha256", "input_contract_sha256", + "acceptance_sha256", "scoring_source_sha256"): + if not isinstance(release.get(key), str) or not re.fullmatch(r"[0-9a-f]{64}", release[key]): + raise ValueError("Release needs an immutable SHA-256: " + key) + return version + + +def create_release(dataset: dict, *, benchmark: str, version: str, + previous: dict | None = None, change_note: str | None = None, + scoring_source_sha256: str = SCORING_SOURCE_SHA256) -> dict: + """Identify an edition; source qualification and correction review stay upstream.""" + release = {"schema_version": 1, "kind": "benchmark_release", + "benchmark": benchmark, "version": version, + "dataset_sha256": _digest(dataset), **_measurement(dataset), + "scoring_source_sha256": scoring_source_sha256} + new = _validate(release) + if previous is not None: + old = _validate(previous) + if benchmark != previous["benchmark"] or new <= old: + raise ValueError("A successor needs the same benchmark and a newer version") + if new[0] == old[0]: + if any(release[key] != previous[key] for key in ( + "case_count", "case_ids_sha256", "input_contract_sha256", + )): + raise ValueError("Changed task population or input contract requires a major version") + changed = any(release[key] != previous[key] for key in ( + "acceptance_sha256", "scoring_source_sha256", + )) + if changed and new[1] == old[1]: + raise ValueError("Changed acceptance or scoring requires a minor version") + if changed and (not isinstance(change_note, str) or not change_note.strip()): + raise ValueError("A scoring correction requires a change note") + release["previous_release_sha256"] = _digest(previous) + if change_note is not None: + if not isinstance(change_note, str) or not change_note.strip(): + raise ValueError("Release change note must be nonempty") + release["change_note"] = change_note.strip() + return release + + +def verify_release(release: dict, dataset: dict, *, + scoring_source_sha256: str | None = None, + exact_dataset: bool = True) -> None: + """Verify a frozen input, or the same contract in a report's presentation wrapper.""" + _validate(release) + actual = _measurement(dataset) + if any(release[key] != value for key, value in actual.items()): + raise ValueError("Release dataset measurement contract changed") + if exact_dataset and release["dataset_sha256"] != _digest(dataset): + raise ValueError("Release dataset bytes or metadata changed") + if scoring_source_sha256 is not None and release["scoring_source_sha256"] != scoring_source_sha256: + raise ValueError("Release scoring source differs from the selected evaluator") + + +def compare_releases(left: dict, right: dict) -> dict: + """Classify dataset/rule compatibility; execution controls remain report-owned.""" + a, b = _validate(left), _validate(right) + fields = ("case_ids_sha256", "input_contract_sha256", "acceptance_sha256", + "scoring_source_sha256") + differences = [key for key in fields if left[key] != right[key]] + if left["benchmark"] != right["benchmark"] or a[0] != b[0]: + mode, reason = "separate", "Different benchmark or major edition" + elif left["version"] == right["version"] and differences: + raise ValueError("Release version reused for a different measurement contract") + elif any(key in differences for key in ("case_ids_sha256", "input_contract_sha256")): + raise ValueError("Task population or inputs changed within one major version") + elif differences: + if a[:2] == b[:2]: + raise ValueError("Acceptance or scoring changed within one minor version") + mode, reason = "rescore", "Recompute both saved cohorts with one acceptance and scoring edition" + else: + mode, reason = "direct", "Same task population, input contract, acceptance and scorer" + return {"mode": mode, "reason": reason, "differences": differences} + + +def freeze_release(dataset_path: Path, output: Path, **options) -> dict: + """Write one immutable sidecar beside external benchmark inputs.""" + if dataset_path.is_symlink(): + raise ValueError("Release dataset rejects symlinks") + dataset = json.loads(dataset_path.read_text(encoding="utf-8")) + release = create_release(dataset, **options) + with output.open("x", encoding="utf-8") as stream: + stream.write(json.dumps(release, ensure_ascii=False, indent=2) + "\n") + return release diff --git a/tests/test_release_versions.py b/tests/test_release_versions.py new file mode 100644 index 0000000..61e3179 --- /dev/null +++ b/tests/test_release_versions.py @@ -0,0 +1,150 @@ +"""Release compatibility follows workload and scoring, not labels alone.""" +from copy import deepcopy +import json + +import pytest + +from lexbench_webfetch.versions import ( + compare_releases, create_release, freeze_release, verify_release, +) + + +def dataset(track="web"): + return {"id": "sample", "cases": [{ + "id": "first", "url": "https://example.test/a", "track": track, + "task_prompt": "Return the full article.", + "required_facts": [{"id": "topic", "alternatives": ["complete body"]}], + "ground_truth_main_content": "complete body", + }]} + + +def release(data=None, version="1.0.0", **kwargs): + return create_release(data or dataset(), benchmark="sample-benchmark", + version=version, **kwargs) + + +@pytest.mark.parametrize("track", ["web", "dynamic", "challenge", "risk_control", "articles", "document"]) +def test_identical_measurement_remains_comparable_across_documentation_patch(track): + before = dataset(track) + first = release(before) + after = deepcopy(before) + after["name"] = "A clearer report name" + after["cases"][0]["name"] = "A clearer task label" + second = release(after, "1.0.1", previous=first) + assert compare_releases(first, second)["mode"] == "direct" + verify_release(second, after) + + +@pytest.mark.parametrize("change", ["add", "remove", "url", "prompt", "scope", "route"]) +def test_workload_changes_require_a_major_release(change): + before = dataset() + before["cases"].append({**deepcopy(before["cases"][0]), "id": "second"}) + after = deepcopy(before) + if change == "add": + after["cases"].append({**deepcopy(after["cases"][0]), "id": "third"}) + elif change == "remove": + after["cases"].pop() + elif change == "url": + after["cases"][0]["url"] += "/different" + elif change == "prompt": + after["cases"][0]["task_prompt"] = "Return only the heading." + elif change == "scope": + after["cases"][0]["task_kind"] = "file" + else: + after["cases"][0]["retrieval_hints"] = {"wait_until": "selector", "selector": ".body"} + first = release(before) + with pytest.raises(ValueError, match="major"): + release(after, "1.1.0", previous=first, change_note="New task scope") + second = release(after, "2.0.0", previous=first, change_note="New task scope") + assert compare_releases(first, second)["mode"] == "separate" + + +@pytest.mark.parametrize("field,value", [ + ("required_facts", [{"id": "topic", "alternatives": ["complete body", "equivalent body"]}]), + ("ground_truth_main_content", "corrected body"), + ("body_requirements", {"regions": [{"id": "body", "text": "corrected body"}]}), +]) +def test_scoring_corrections_require_minor_version_and_rescore(field, value): + first = release() + after = dataset() + after["cases"][0][field] = value + with pytest.raises(ValueError, match="minor"): + release(after, "1.0.1", previous=first, change_note="Correct independently reviewed text") + with pytest.raises(ValueError, match="change note"): + release(after, "1.1.0", previous=first) + second = release(after, "1.1.0", previous=first, change_note="Correct independently reviewed text") + assert compare_releases(first, second)["mode"] == "rescore" + + +def test_scorer_changes_cannot_silently_relabel_old_scores(): + first = release(scoring_source_sha256="a" * 64) + with pytest.raises(ValueError, match="minor"): + release(version="1.0.1", previous=first, scoring_source_sha256="b" * 64) + second = release(version="1.1.0", previous=first, + scoring_source_sha256="b" * 64, change_note="Fix serialization matching") + assert compare_releases(first, second)["mode"] == "rescore" + with pytest.raises(ValueError, match="scoring source"): + verify_release(first, dataset(), scoring_source_sha256="b" * 64) + + +def test_same_version_cannot_name_different_contracts(): + first = release() + after = dataset() + after["cases"][0]["required_facts"] = [] + with pytest.raises(ValueError, match="reused"): + compare_releases(first, release(after)) + + +def test_different_benchmarks_and_majors_are_separate_even_when_inputs_match(): + first = release() + other = create_release(dataset(), benchmark="another-benchmark", version="1.0.0") + assert compare_releases(first, other)["mode"] == "separate" + assert compare_releases(first, release(version="2.0.0"))["mode"] == "separate" + + +@pytest.mark.parametrize("version", ["latest", "1.0", "01.0.0", "1.0.0-dev", "1.0.-1"]) +def test_mutable_or_ambiguous_version_labels_are_rejected(version): + with pytest.raises(ValueError, match="version"): + release(version=version) + + +def test_dataset_drift_and_duplicate_ids_are_rejected(): + first = release() + after = dataset() + after["cases"][0]["required_facts"] = [] + with pytest.raises(ValueError, match="dataset"): + verify_release(first, after) + after["cases"].append(deepcopy(after["cases"][0])) + with pytest.raises(ValueError, match="unique"): + release(after) + + +def test_freeze_is_exclusive_and_manifest_is_portable(tmp_path): + source = tmp_path / "dataset.json" + source.write_text(json.dumps(dataset())) + target = tmp_path / "benchmark-release.json" + freeze_release(source, target, benchmark="sample-benchmark", version="1.0.0") + frozen = json.loads(target.read_text()) + verify_release(frozen, dataset()) + assert str(tmp_path) not in target.read_text() + with pytest.raises(FileExistsError): + freeze_release(source, target, benchmark="sample-benchmark", version="1.0.1") + link = tmp_path / "link.json" + link.symlink_to(source) + with pytest.raises(ValueError, match="symlink"): + freeze_release(link, tmp_path / "another.json", benchmark="sample-benchmark", version="1.0.0") + + +def test_public_cli_freezes_and_compares_real_files(tmp_path, capsys, monkeypatch): + from lexbench_webfetch.cli import main + source = tmp_path / "dataset.json" + source.write_text(json.dumps(dataset())) + frozen = tmp_path / "release.json" + monkeypatch.setattr("sys.argv", ["lexbench-webfetch", "release", "--dataset", str(source), + "--output", str(frozen), "--benchmark", "sample-benchmark", + "--version", "1.0.0"]) + main() + assert json.loads(capsys.readouterr().out)["version"] == "1.0.0" + monkeypatch.setattr("sys.argv", ["lexbench-webfetch", "compare-releases", str(frozen), str(frozen)]) + main() + assert json.loads(capsys.readouterr().out)["mode"] == "direct" From 04c2b99e0f86aa0264976ad4159bc4e45b2b10ab Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Fri, 23 Oct 2026 14:58:47 +0800 Subject: [PATCH 2/4] feat(versioning): record the immutable scorer commit --- docs/versions.md | 7 +++++-- src/lexbench_webfetch/cli.py | 5 ++++- src/lexbench_webfetch/versions.py | 9 ++++++++- tests/test_release_versions.py | 11 ++++++++++- 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/docs/versions.md b/docs/versions.md index 4de87ba..f66a8c8 100644 --- a/docs/versions.md +++ b/docs/versions.md @@ -33,11 +33,14 @@ provider calls: uv run --frozen lexbench-webfetch release \ --dataset /absolute/inputs/dataset.json \ --output /absolute/inputs/benchmark-release.json \ - --benchmark lexbench-webfetch --version 2.0.0 + --benchmark lexbench-webfetch --version 2.0.0 \ + --scorer-revision ``` For a successor, add `--previous /absolute/previous/benchmark-release.json`. -Scoring corrections also require a concrete `--change-note`. The tool rejects +The scorer commit locates the code; the independently computed scoring-source +hash fixes the actual evaluator bytes. Scoring corrections also require a +concrete `--change-note`. The tool rejects an insufficient version bump and an existing output path. Keep the sidecar alongside frozen inputs, outside Git. It contains no provider measurements or local paths. Commit reusable definitions and publish the code through the diff --git a/src/lexbench_webfetch/cli.py b/src/lexbench_webfetch/cli.py index ba300d0..1c9e06d 100644 --- a/src/lexbench_webfetch/cli.py +++ b/src/lexbench_webfetch/cli.py @@ -15,6 +15,8 @@ def main() -> None: release.add_argument("--output", type=Path, required=True) release.add_argument("--benchmark", required=True) release.add_argument("--version", required=True) + release.add_argument("--scorer-revision", required=True, + help="Full Git commit containing the selected scorer") release.add_argument("--previous", type=Path) release.add_argument("--change-note") compare = sub.add_parser("compare-releases", help="Check whether two result editions can be compared") @@ -58,7 +60,8 @@ def main() -> None: previous = json.loads(args.previous.read_text()) if args.previous else None result = freeze_release(args.dataset, args.output, benchmark=args.benchmark, version=args.version, previous=previous, - change_note=args.change_note) + change_note=args.change_note, + scorer_revision=args.scorer_revision) elif args.command == "compare-releases": from .versions import compare_releases result = compare_releases(json.loads(args.left.read_text()), diff --git a/src/lexbench_webfetch/versions.py b/src/lexbench_webfetch/versions.py index 7b12b9e..3080fc1 100644 --- a/src/lexbench_webfetch/versions.py +++ b/src/lexbench_webfetch/versions.py @@ -71,17 +71,24 @@ def _validate(release: dict) -> tuple[int, int, int]: "acceptance_sha256", "scoring_source_sha256"): if not isinstance(release.get(key), str) or not re.fullmatch(r"[0-9a-f]{64}", release[key]): raise ValueError("Release needs an immutable SHA-256: " + key) + revision = release.get("scorer_revision") + if revision is not None and (not isinstance(revision, str) + or not re.fullmatch(r"[0-9a-f]{40}", revision)): + raise ValueError("Scorer revision must be a full immutable Git commit") return version def create_release(dataset: dict, *, benchmark: str, version: str, previous: dict | None = None, change_note: str | None = None, - scoring_source_sha256: str = SCORING_SOURCE_SHA256) -> dict: + scoring_source_sha256: str = SCORING_SOURCE_SHA256, + scorer_revision: str | None = None) -> dict: """Identify an edition; source qualification and correction review stay upstream.""" release = {"schema_version": 1, "kind": "benchmark_release", "benchmark": benchmark, "version": version, "dataset_sha256": _digest(dataset), **_measurement(dataset), "scoring_source_sha256": scoring_source_sha256} + if scorer_revision is not None: + release["scorer_revision"] = scorer_revision new = _validate(release) if previous is not None: old = _validate(previous) diff --git a/tests/test_release_versions.py b/tests/test_release_versions.py index 61e3179..eafe95a 100644 --- a/tests/test_release_versions.py +++ b/tests/test_release_versions.py @@ -142,9 +142,18 @@ def test_public_cli_freezes_and_compares_real_files(tmp_path, capsys, monkeypatc frozen = tmp_path / "release.json" monkeypatch.setattr("sys.argv", ["lexbench-webfetch", "release", "--dataset", str(source), "--output", str(frozen), "--benchmark", "sample-benchmark", - "--version", "1.0.0"]) + "--version", "1.0.0", "--scorer-revision", "a" * 40]) main() assert json.loads(capsys.readouterr().out)["version"] == "1.0.0" monkeypatch.setattr("sys.argv", ["lexbench-webfetch", "compare-releases", str(frozen), str(frozen)]) main() assert json.loads(capsys.readouterr().out)["mode"] == "direct" + + +def test_scorer_commit_is_a_locator_and_source_hash_owns_compatibility(): + first = release(scorer_revision="a" * 40) + second = release(version="1.0.1", previous=first, scorer_revision="b" * 40) + assert first["scorer_revision"] == "a" * 40 + assert compare_releases(first, second)["mode"] == "direct" + with pytest.raises(ValueError, match="Git commit"): + release(scorer_revision="main") From 56aca0cb9b9b9eef9995f0168e0504a72e309cd1 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Fri, 23 Oct 2026 16:05:18 +0800 Subject: [PATCH 3/4] feat(versioning): publish the frozen benchmark catalog edition --- README.md | 4 ++ docs/README.zh-CN.md | 3 + docs/versions.md | 8 +++ src/lexbench_webfetch/cli.py | 6 +- .../definitions/benchmark-release.json | 72 +++++++++++++++++++ src/lexbench_webfetch/versions.py | 29 ++++++++ tests/test_release_versions.py | 29 +++++++- 7 files changed, 149 insertions(+), 2 deletions(-) create mode 100644 src/lexbench_webfetch/definitions/benchmark-release.json diff --git a/README.md b/README.md index 0ecb625..c25fa17 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,10 @@ to freeze an external `benchmark-release.json`, and `compare-releases` to check whether two editions support direct comparison, require rescoring or belong to separate major versions. See [benchmark versions](docs/versions.md). +`lexbench-webfetch version` verifies and prints the installed catalog edition, +including every suite's task count, frozen input hashes and scoring source. +The immutable `benchmark-vMAJOR.MINOR.PATCH` tag identifies its repository revision. + ## Score a saved answer The benchmark includes its own scorer. It needs no Browser Eval installation: diff --git a/docs/README.zh-CN.md b/docs/README.zh-CN.md index 7a318e8..fe691fc 100644 --- a/docs/README.zh-CN.md +++ b/docs/README.zh-CN.md @@ -53,10 +53,13 @@ cd lexbench-webfetch uv sync --frozen uv run --frozen lexbench-webfetch list uv run --frozen lexbench-webfetch validate +uv run --frozen lexbench-webfetch version ``` 这一步检查公开任务定义，无需浏览器、API Key 或 Browser Eval。它不访问网站，也不计算评测分数。 +`version` 会校验并显示当前题库版本，包括各子集题数、冻结输入哈希和评分器哈希。`benchmark-v主版本.次版本.补丁版本` 标签固定对应的仓库提交；报告另行绑定实际使用的题目和评分规则。 + 复现冻结任务集需要取得获准使用的原始输入。所需文件及哈希列在 `src/lexbench_webfetch/definitions/registry.json` 中。本仓库不分发第三方页面快照，目前也没有公开的完整输入下载包。 ```sh diff --git a/docs/versions.md b/docs/versions.md index f66a8c8..3569f86 100644 --- a/docs/versions.md +++ b/docs/versions.md @@ -25,6 +25,14 @@ their independent source evidence and executable correction controls. ## Freeze an edition +`lexbench-webfetch version` verifies the installed catalog's complete suite list, +task definitions, frozen reference bindings and scorer against +`definitions/benchmark-release.json`. The published `benchmark-vMAJOR.MINOR.PATCH` +tag fixes that catalog. Full reference inputs remain separately hash-bound; +the catalog does not replace the executable datasets or distribute their source +captures. A report binds its actual selected population with the sidecar below; +installing a newer catalog never expands that population. + Compile or export the inputs through their existing owner first, including any registered annotation revision. Freeze the complete selected population before provider calls: diff --git a/src/lexbench_webfetch/cli.py b/src/lexbench_webfetch/cli.py index 1c9e06d..1e9a0d3 100644 --- a/src/lexbench_webfetch/cli.py +++ b/src/lexbench_webfetch/cli.py @@ -10,6 +10,7 @@ def main() -> None: parser = argparse.ArgumentParser(description="Inspect and verify LexBench Web Fetch inputs") sub = parser.add_subparsers(dest="command", required=True) sub.add_parser("list", help="List the versioned suites") + sub.add_parser("version", help="Show and verify the published benchmark catalog edition") release = sub.add_parser("release", help="Freeze a dataset and scorer as a benchmark edition") release.add_argument("--dataset", type=Path, required=True) release.add_argument("--output", type=Path, required=True) @@ -55,7 +56,10 @@ def main() -> None: articles.add_argument("--evidence-home", type=Path, required=True) args = parser.parse_args() try: - if args.command == "release": + if args.command == "version": + from .versions import catalog_release + result = catalog_release() + elif args.command == "release": from .versions import freeze_release previous = json.loads(args.previous.read_text()) if args.previous else None result = freeze_release(args.dataset, args.output, benchmark=args.benchmark, diff --git a/src/lexbench_webfetch/definitions/benchmark-release.json b/src/lexbench_webfetch/definitions/benchmark-release.json new file mode 100644 index 0000000..8f738d2 --- /dev/null +++ b/src/lexbench_webfetch/definitions/benchmark-release.json @@ -0,0 +1,72 @@ +{ + "schema_version": 1, + "kind": "benchmark_catalog_release", + "benchmark": "lexmount-webfetch", + "version": "2.0.0", + "tag": "benchmark-v2.0.0", + "scorer_revision": "d9e2f00c7bca0298960ee42e8d2493ab1e835bb0", + "scoring_source_sha256": "c97158df8941ea55d358f5defd9c3e97869f82ae8aba5ecb6468dc43f1ed1c7a", + "case_count": 550, + "suites": [ + { + "suite": "webfetch", + "case_count": 150, + "dataset_sha256": "5f751146143dcbe9daf46f0fb2511da1147508b3c8c4cd3377a6f3cceb82264e", + "dataset_id": "webfetch", + "directory": "evaluation", + "files": { + "tasks.json": "ae3aec6eab8d2d470cdeb18c4f80d238e5dd79e1108a729589b77a80fe869405", + "dataset-catalog.json": "994fa53f7fc7c3170692a7bb493ebc5af9ad8ffb5b576c022fa123e6e6fb3838", + "source-inputs.json": "c202abc4624811b938a837402bbaf0ecbe716e375fbca70567346d2edb3b1708" + } + }, + { + "suite": "popular-sites", + "case_count": 100, + "dataset_sha256": "85544512d4cfd9459c22af1f70b1eb400844742dbf3179807662b76db2dcfc2e", + "dataset_id": "popular-sites", + "directory": "evaluation/popular-sites", + "files": { + "tasks.json": "82859ffb301740bcf38fee4fe87a675e8fb21af8dd660f92f74e4cbaede35b33", + "dataset-catalog.json": "f2b9dfb550230cdf8dbbba544d58158ac957620c1d064173790ead70754c178d", + "source-inputs.json": "32c31cfd5458a1dfe06d370fc2411a1eea5d1a05da8ef88e0dcd4fd3b0f3c810" + } + }, + { + "suite": "dynamic-content", + "case_count": 100, + "dataset_sha256": "0da2bb30aaabe3ca2b93ec17b4cb88cdc1cf3ea46336bb3777fdb21c1f211b84", + "dataset_id": "dynamic-qualified-100-moli-native-local-audit-20260924", + "directory": "evaluation/dynamic-content-100-moli-webfetch", + "files": { + "tasks.json": "139c62a141183d305d8fa9c3658c546927731461d86aeba5ad3091649bd4ecd7", + "dataset-catalog.json": "ca1572aa80fc2324b2f00800a79b053cfd584d1831ffb16e6906ae0981522c6c", + "source-inputs.json": "710f9c91705b29bb9e4aa15af97dcc856e64e406baa4b9966446408d7e3d26a8" + } + }, + { + "suite": "access-restrictions", + "case_count": 100, + "dataset_sha256": "165b888133e859304b035d74183dd40bf7ab609cac78bb6ae6e75b645c1de426", + "dataset_id": "risk-control-audited-100", + "directory": "evaluation/risk-control/current", + "files": { + "tasks.json": "c67b60ccf5ee0b14e4e23b3efc282e625005f8e096f0bfc978772d312e8f50c2", + "dataset-catalog.json": "52ec968b2ada26e5e061ccb25cbe4a73747539e556fa88f31bece6d7fb79b3cb", + "source-inputs.json": "c61d6825abbff2a92098592f83055347cd0b0101374794023046c0fd0323dd5f" + } + }, + { + "suite": "articles", + "case_count": 100, + "dataset_sha256": "1929d5f702a2f7a0a1248694ca6843fba02ecaf7c559c6bf232865a8fb3c4bb6", + "dataset_id": "articles-v11", + "directory": "evaluation/articles", + "files": { + "tasks.json": "171241ea0bf3cb3f9482de14a7201885733fa717af56d7ea2b28d86a4516a3bb", + "dataset-catalog.json": "8e413e683c309674abf3d297df3f378e76d65ead1e2aa343828e4b0e1d90764a", + "source-inputs.json": "0d9c729f14dae8abb3323a2cf4734736c4733957c9d74d5484033d204642b91b" + } + } + ] +} diff --git a/src/lexbench_webfetch/versions.py b/src/lexbench_webfetch/versions.py index 3080fc1..1b5f072 100644 --- a/src/lexbench_webfetch/versions.py +++ b/src/lexbench_webfetch/versions.py @@ -158,3 +158,32 @@ def freeze_release(dataset_path: Path, output: Path, **options) -> dict: with output.open("x", encoding="utf-8") as stream: stream.write(json.dumps(release, ensure_ascii=False, indent=2) + "\n") return release + + +def catalog_release(path: Path | None = None) -> dict: + """Read the published catalog edition and verify its complete suite bindings.""" + from .benchmark import SUITES, registry, validate + + selected = path if path is not None else SUITES / "benchmark-release.json" + if selected.is_symlink(): + raise ValueError("Catalog release rejects symlinks") + release = json.loads(selected.read_text()) + if release.get("schema_version") != 1 or release.get("kind") != "benchmark_catalog_release": + raise ValueError("Unsupported benchmark catalog release") + _version(release.get("version")) + if release.get("tag") != "benchmark-v" + release["version"]: + raise ValueError("Catalog release tag differs from its version") + if release.get("scoring_source_sha256") != SCORING_SOURCE_SHA256: + raise ValueError("Catalog release scoring source changed") + revision = release.get("scorer_revision", "") + if not re.fullmatch(r"[0-9a-f]{40}", revision): + raise ValueError("Catalog release needs a full scorer commit") + actual = [] + for entry in registry()["suites"]: + validated = validate(entry["id"]) + actual.append({**validated, "dataset_id": entry["dataset_id"], + "directory": entry["directory"], "files": entry["files"]}) + if release.get("suites") != actual or release.get("case_count") != sum( + row["case_count"] for row in actual): + raise ValueError("Catalog release task or frozen reference bindings changed") + return release diff --git a/tests/test_release_versions.py b/tests/test_release_versions.py index eafe95a..2f7ef41 100644 --- a/tests/test_release_versions.py +++ b/tests/test_release_versions.py @@ -5,10 +5,37 @@ import pytest from lexbench_webfetch.versions import ( - compare_releases, create_release, freeze_release, verify_release, + catalog_release, compare_releases, create_release, freeze_release, verify_release, ) +def test_installed_catalog_binds_every_registered_suite(): + from lexbench_webfetch.benchmark import registry + release = catalog_release() + entries = registry()["suites"] + assert [row["suite"] for row in release["suites"]] == [row["id"] for row in entries] + assert release["case_count"] == sum(row["case_count"] for row in entries) + + +@pytest.mark.parametrize("change", ["omitted_suite", "count", "reference", "scorer", "tag"]) +def test_published_catalog_rejects_changed_population_rules_or_reference_bindings(tmp_path, change): + release = catalog_release() + if change == "omitted_suite": + release["suites"].pop() + elif change == "count": + release["case_count"] += 1 + elif change == "reference": + release["suites"][0]["dataset_sha256"] = "0" * 64 + elif change == "scorer": + release["scoring_source_sha256"] = "0" * 64 + else: + release["tag"] = "benchmark-v99.0.0" + path = tmp_path / "release.json" + path.write_text(json.dumps(release)) + with pytest.raises(ValueError): + catalog_release(path) + + def dataset(track="web"): return {"id": "sample", "cases": [{ "id": "first", "url": "https://example.test/a", "track": track, From 2ea8c52df67c39eee2ea5113a63f5da34f102160 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Fri, 23 Oct 2026 17:40:11 +0800 Subject: [PATCH 4/4] fix(versioning): bind all supported tree and list scoring rules --- docs/versions.md | 5 ++ src/lexbench_webfetch/benchmark.py | 10 +++- .../definitions/benchmark-release.json | 4 +- tests/test_release_versions.py | 50 +++++++++++++++++++ 4 files changed, 66 insertions(+), 3 deletions(-) diff --git a/docs/versions.md b/docs/versions.md index 3569f86..f53e6f2 100644 --- a/docs/versions.md +++ b/docs/versions.md @@ -10,6 +10,11 @@ such as `benchmark-v2.0.0` gives a released commit a durable name; never move or overwrite a published release tag. A tag alone does not identify external reference bytes or a report's selected task population. +Use `benchmark-vMAJOR.MINOR.PATCH` in both the benchmark and report repositories. +A report names the benchmark edition it uses; it does not introduce a separate +report-version sequence. The hosting platform's immutable version ID identifies +each published report build. Retain historical tags at their original commits. + | Change | Version | Comparing results | | --- | --- | --- | | Documentation or presentation; measurement contracts and scorer unchanged | Patch | Direct, with the same execution controls | diff --git a/src/lexbench_webfetch/benchmark.py b/src/lexbench_webfetch/benchmark.py index a3178be..6efd435 100644 --- a/src/lexbench_webfetch/benchmark.py +++ b/src/lexbench_webfetch/benchmark.py @@ -24,6 +24,7 @@ "source_review_url", "evaluation_url", "expected_final_url", "page_identity_required", "expected_document_language", "document_language_match", "expected_document_language_present", "target_language", "target_region", + "ordered_tree_min_similarity", "ordered_tree_reference_sha256", ) @@ -49,9 +50,16 @@ def public_task(case: dict) -> dict: task = {key: case[key] for key in PUBLIC_TASK_FIELDS if key in case} if "body_requirements" in task: task["body_requirements"] = reference_bindings(task["body_requirements"]) + if case.get("ordered_tree_reference") is not None: + task["ordered_tree_reference_sha256"] = hashlib.sha256(json.dumps( + case["ordered_tree_reference"], ensure_ascii=False, sort_keys=True, + separators=(",", ":"), allow_nan=False).encode()).hexdigest() benchmark = case.get("source_benchmark", {}) routing = {key: benchmark[key] for key in - ("mode", "hosts", "entry_path", "minimum_entries", "verification_scope") + ("mode", "hosts", "entry_path", "minimum_entries", "verification_scope", + "entry_identity_pattern", "identity_query_keys", "link_base_url", "entry_scope", + "name_validation", "record_name_exclusions", "title_exclusion_patterns", + "price_patterns", "price_exclusion_patterns", "price_unavailable_patterns") if key in benchmark} from .required_components import COMPONENTS for component in COMPONENTS: diff --git a/src/lexbench_webfetch/definitions/benchmark-release.json b/src/lexbench_webfetch/definitions/benchmark-release.json index 8f738d2..3c39a34 100644 --- a/src/lexbench_webfetch/definitions/benchmark-release.json +++ b/src/lexbench_webfetch/definitions/benchmark-release.json @@ -2,8 +2,8 @@ "schema_version": 1, "kind": "benchmark_catalog_release", "benchmark": "lexmount-webfetch", - "version": "2.0.0", - "tag": "benchmark-v2.0.0", + "version": "2.0.1", + "tag": "benchmark-v2.0.1", "scorer_revision": "d9e2f00c7bca0298960ee42e8d2493ab1e835bb0", "scoring_source_sha256": "c97158df8941ea55d358f5defd9c3e97869f82ae8aba5ecb6468dc43f1ed1c7a", "case_count": 550, diff --git a/tests/test_release_versions.py b/tests/test_release_versions.py index 2f7ef41..7a933f8 100644 --- a/tests/test_release_versions.py +++ b/tests/test_release_versions.py @@ -184,3 +184,53 @@ def test_scorer_commit_is_a_locator_and_source_hash_owns_compatibility(): assert compare_releases(first, second)["mode"] == "direct" with pytest.raises(ValueError, match="Git commit"): release(scorer_revision="main") + + +@pytest.mark.parametrize("field", ["ordered_tree_reference", "ordered_tree_min_similarity"]) +@pytest.mark.parametrize("reverse", [False, True]) +def test_tree_verdict_changes_require_rescoring_even_in_relaxed_report_verification(field, reverse): + from lexbench_webfetch.evaluation import evaluate_response + + before = dataset() + case = before["cases"][0] + case.update(ordered_tree_reference={"tag": "p", "text": "hello"}, + ordered_tree_min_similarity=0.5) + after = deepcopy(before) + if field == "ordered_tree_reference": + after["cases"][0][field]["text"] = "world" + else: + case["ordered_tree_reference"]["text"] = "world" + after["cases"][0]["ordered_tree_reference"]["text"] = "world" + case[field] = 0.1 + if reverse: + before, after = after, before + answer = b'{"tag":"p","text":"hello"}' + assert evaluate_response(before["cases"][0], answer)["passed"] is not reverse + assert evaluate_response(after["cases"][0], answer)["passed"] is reverse + first = release(before) + with pytest.raises(ValueError, match="minor"): + release(after, "1.0.1", previous=first) + with pytest.raises(ValueError, match="dataset"): + verify_release(first, after, exact_dataset=False) + corrected = release(after, "1.1.0", previous=first, change_note="Correct frozen tree rule") + assert compare_releases(first, corrected)["mode"] == "rescore" + + +def test_list_name_rule_is_part_of_measurement_not_just_public_display_fields(): + from lexbench_webfetch.evaluation import evaluate_response + + before = dataset() + before["cases"][0]["source_benchmark"] = { + "mode": "dynamic_list_structure", "hosts": ["example.test"], + "entry_path": r"/items/\d+", "minimum_entries": 1, + } + after = deepcopy(before) + after["cases"][0]["source_benchmark"]["name_validation"] = "nonempty" + answer = b'7' + assert evaluate_response(before["cases"][0], answer)["metrics"]["structure_pass"] is False + assert evaluate_response(after["cases"][0], answer)["metrics"]["structure_pass"] is True + first = release(before) + with pytest.raises(ValueError, match="minor"): + release(after, "1.0.1", previous=first) + with pytest.raises(ValueError, match="dataset"): + verify_release(first, after, exact_dataset=False)