From d6f138da9a4eff76b21575f97b2f7ede2ff74863 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Tue, 20 Oct 2026 18:12:26 +0800
Subject: [PATCH 01/51] Enforce scoring and executable consistency at release
boundaries
---
docs/benchmarks.md | 2 +-
docs/metrics.md | 3 ++
evaluation/lexbench-webfetch.json | 4 +--
harness/web_fetch/artifacts.py | 20 ++++++++++++
harness/web_fetch/certification.py | 5 +++
harness/web_fetch/reliability.py | 15 +++++++++
harness/web_fetch/report_package.py | 9 ++++--
harness/web_fetch/rescoring.py | 4 +--
harness/web_fetch/runner.py | 4 ++-
pyproject.toml | 2 +-
tests/package_smoke.py | 2 +-
tests/test_assembly_evidence.py | 6 ++--
tests/test_publication_privacy.py | 2 +-
tests/test_reliability.py | 18 +++++++++++
tests/test_run_certified_benchmark.py | 14 +++++++++
tests/test_scoring.py | 36 +++++++++++++++++++---
tests/test_scoring_revision.py | 22 ++++++-------
tests/test_three_cohort_review_followup.py | 8 ++---
tests/test_webfetch_report_package.py | 12 ++++++++
uv.lock | 12 ++++----
20 files changed, 159 insertions(+), 41 deletions(-)
diff --git a/docs/benchmarks.md b/docs/benchmarks.md
index b7b9cce74..ca442172a 100644
--- a/docs/benchmarks.md
+++ b/docs/benchmarks.md
@@ -41,7 +41,7 @@ uv run --frozen browser-harness-eval run \
--provider-config configs/providers.quickstart.json --providers http
```
-This uses the existing dataset interface. `uv sync --frozen` installs LexBench from the exact Git commit pinned in `pyproject.toml` and `uv.lock`. The pinned commit is merged into LexBench’s main branch. uv retrieves the Git source, builds a wheel and installs the Python package; Git must be available, but no sibling checkout is required. The input checker verifies frozen bytes; a formal live-site comparison still requires the [certified runner](../harness/web_fetch/README.md), independent reference validation before and after calls, and the selected track's release gates. Ordinary CLI runs are diagnostic.
+This uses the existing dataset interface. `uv sync --frozen` installs LexBench from the exact Git commit pinned in `pyproject.toml` and `uv.lock`. uv retrieves the Git source, builds a wheel and installs the Python package; Git must be available, but no sibling checkout is required. The input checker verifies frozen bytes; a formal live-site comparison still requires the [certified runner](../harness/web_fetch/README.md), independent reference validation before and after calls, and the selected track's release gates. Ordinary CLI runs are diagnostic.
## Historical bindings and new revisions
diff --git a/docs/metrics.md b/docs/metrics.md
index b0548b578..4036c25d1 100644
--- a/docs/metrics.md
+++ b/docs/metrics.md
@@ -21,8 +21,11 @@ A returned file URL is not a delivered file. Dynamic list checks measure structu
| Body precision | How much returned text belongs to that reference |
| Body F1 | Harmonic mean of precision and recall |
| Mean / P50 / P95 latency | In the default ranking, timing of task-passing calls only; failures still retain elapsed time in their records |
+| All-call mean / P50 / P95 | The complete call population, including failed calls and timeouts; exposed as `mean_all_attempts_ms`, `all_attempts_p50_ms` and `all_attempts_p95_ms` in scorer summaries |
| Failure class | Request error or unmet task requirement; open the detail for the full error |
Body metrics require a complete, comparable reference. A failed call scores zero; a missing reference leaves the metric unavailable. Check each report's stated timing population before comparing it with another report.
Outputs have SHA-256 hashes. Rebuilding or editing a report must preserve the saved responses and scores. Metric labels come from `report_components.py`; the report does not show a pass rate conditioned only on successful requests.
+
+Formal cohorts reject selected tasks whose acceptance rules are unresolved before capturing references or calling providers. A published method must use one executable fingerprint across its selected results. Compare different builds as separate complete method populations. Offline file rescoring reads the retained file bytes and applies the current file requirements while preserving the original call metadata.
diff --git a/evaluation/lexbench-webfetch.json b/evaluation/lexbench-webfetch.json
index 2c32ac205..c1591b67a 100644
--- a/evaluation/lexbench-webfetch.json
+++ b/evaluation/lexbench-webfetch.json
@@ -2,7 +2,7 @@
"schema_version": 1,
"kind": "external_benchmark_binding",
"repository": "https://github.com/lexmount/lexbench-webfetch",
- "revision": "84c177d97da3b82567d9881e377aa728929cf129",
+ "revision": "51e11917c5b00e1629a1f27c0500391371fc387a",
"ownership": {
"benchmark": "Task populations, task model, acceptance scoring, file checks, compiler and frozen definitions",
"harness": "Execution, reference lifecycle, evidence and reports"
@@ -78,7 +78,7 @@
"evaluation/risk-control/remaining-70-rule-review.md": "301574dddcfddf6d30ed4d882cdd40b1ae859fdc95a0a02cf9868f55e3fd1079",
"evaluation/risk-control/rule-authoring-guide.md": "74363ab502978179e5438a0b992bac7563090e41c91784e492636832dc1cd088",
"evaluation/risk-control/source-inputs.json": "77c5a4b62c07d59ed57154a833e85f9e7a97bc739730f51cd4ca3a698edb9258",
- "evaluation/scoring-regressions.json": "59635a2b9b73681fd12b401f56e532cbaaefe11c891f1bb059da1435b133b02b",
+ "evaluation/scoring-regressions.json": "5edad3eb2e776b947b6ab998aadd89d62332fc9cf5626e5d51a12e29581852be",
"evaluation/source-inputs.json": "c202abc4624811b938a837402bbaf0ecbe716e375fbca70567346d2edb3b1708"
}
}
diff --git a/harness/web_fetch/artifacts.py b/harness/web_fetch/artifacts.py
index ae4cd9360..ac271c2c4 100644
--- a/harness/web_fetch/artifacts.py
+++ b/harness/web_fetch/artifacts.py
@@ -20,6 +20,26 @@
PUBLIC_CALL_LOG_ROOT = "call-logs"
+def file_scoring_metadata(case, metadata: dict, source: Path) -> dict:
+ """Recheck delivered file bytes against current rules without changing call evidence."""
+ from lexbench_webfetch.file_acceptance import evaluate_files
+
+ case = case if isinstance(case, dict) else vars(case)
+ evaluation = metadata.get("file_evaluation")
+ if case.get("task_kind") != "file" or not isinstance(evaluation, dict):
+ return metadata
+ delivered = evaluation.get("files", [])
+ payloads = []
+ for item in delivered:
+ raw = contained_path(source, item["artifact_path"], reject_symlinks=True).read_bytes()
+ if hashlib.sha256(raw).hexdigest() != item.get("sha256"):
+ raise ValueError("Delivered file integrity failure: " + item["artifact_path"])
+ payloads.append(raw)
+ current = evaluate_files(case, payloads)
+ current["files"] = [{**old, **new} for old, new in zip(delivered, current["files"])]
+ return {**metadata, "file_evaluation": current}
+
+
def public_call_log_binding(value: dict[str, Any] | None) -> dict[str, Any] | None:
"""Normalize a public call-log binding and reject partial/unsafe claims."""
if value is None:
diff --git a/harness/web_fetch/certification.py b/harness/web_fetch/certification.py
index 55588030a..88381cb81 100644
--- a/harness/web_fetch/certification.py
+++ b/harness/web_fetch/certification.py
@@ -193,6 +193,11 @@ def main(argv: list[str] | None = None) -> None:
known = {case["id"] for case in dataset["cases"]}
if set(case_ids) - known:
parser.error("Unknown case IDs: " + ", ".join(sorted(set(case_ids) - known)))
+ unresolved = sorted(case["id"] for case in dataset["cases"]
+ if case["id"] in case_ids
+ and (case.get("rule_review") or {}).get("status") == "unresolved")
+ if unresolved and not args.diagnostic:
+ parser.error("Formal cohorts require resolved acceptance rules: " + ", ".join(unresolved))
if args.output.exists() or args.work_root.exists():
parser.error("Output and work root must both be new immutable paths")
diff --git a/harness/web_fetch/reliability.py b/harness/web_fetch/reliability.py
index 456edeecf..4b074d3bd 100644
--- a/harness/web_fetch/reliability.py
+++ b/harness/web_fetch/reliability.py
@@ -6,6 +6,17 @@
from lexbench_webfetch.metrics import task_measured
+def implementation_conflicts(records: list[dict]) -> dict[str, list[str]]:
+ """Find method labels that combine results from distinct executable builds."""
+ implementations = {}
+ for row in records:
+ fingerprint = (row.get("provider_metadata") or {}).get("binary_sha256")
+ if fingerprint:
+ implementations.setdefault(row.get("provider"), set()).add(fingerprint)
+ return {provider: sorted(hashes) for provider, hashes in implementations.items()
+ if len(hashes) > 1}
+
+
def assess_reliability(dataset: dict, records: list[dict]) -> dict:
"""Require exactly one judged result for every planned provider/case pair."""
cases = {case["id"]: case for case in dataset.get("cases", [])}
@@ -26,6 +37,7 @@ def assess_reliability(dataset: dict, records: list[dict]) -> dict:
if row.get("case_id") in cases
and not task_measured(row.get("metrics", {}), cases[row["case_id"]].get("task_kind", "body"))
)
+ conflicts = implementation_conflicts(records)
errors = []
if not cases:
errors.append("评测集没有案例")
@@ -39,6 +51,8 @@ def assess_reliability(dataset: dict, records: list[dict]) -> dict:
errors.append(f"存在 {len(duplicates)} 个重复或多次选入的调用")
if unjudged:
errors.append(f"存在 {len(unjudged)} 个无法判定通过或失败的调用")
+ if conflicts:
+ errors.append("同一方法混用了不同可执行文件: " + ", ".join(sorted(conflicts)))
complete = bool(expected) and not errors
return {
"schema_version": 1,
@@ -51,6 +65,7 @@ def assess_reliability(dataset: dict, records: list[dict]) -> dict:
"extra": [{"provider": p, "case_id": c} for p, c in extra],
"duplicates": [{"provider": p, "case_id": c, "count": counts[(p, c)]} for p, c in duplicates],
"unjudged": [{"provider": p, "case_id": c} for p, c in unjudged],
+ "implementation_conflicts": conflicts,
"errors": errors,
"publication_policy": "all_planned_provider_case_pairs_must_exist_once_and_be_judged",
}
diff --git a/harness/web_fetch/report_package.py b/harness/web_fetch/report_package.py
index 390611ad8..5bfefb5ab 100644
--- a/harness/web_fetch/report_package.py
+++ b/harness/web_fetch/report_package.py
@@ -35,6 +35,10 @@ def validate_release(source: Path, *, presentation_binding: Path | None = None)
return False
dataset_path = source / "dataset.json"
dataset = json.loads(dataset_path.read_text()) if dataset_path.exists() else {}
+ from .reliability import implementation_conflicts
+ records = load_jsonl(source / "results.jsonl")
+ if conflicts := implementation_conflicts(records):
+ raise ValueError("Refusing to package mixed executable versions: " + ", ".join(sorted(conflicts)))
from .reference_lifecycle import require_certifiable_evidence
manifests = []
for name in ('manifest.json', 'execution.json'):
@@ -47,7 +51,7 @@ def validate_release(source: Path, *, presentation_binding: Path | None = None)
if content_binding:
from .report_explainer import current_browser_article_model
current_browser_article_model(source, {case['id']:case for case in dataset['cases']},
- load_jsonl(source/'results.jsonl'), content_binding)
+ records, content_binding)
sampled = dataset.get("comparison", {}).get("kind") == "webmainbench_frozen_html_to_markdown"
if not sampled:
reliability = json.loads((source / "reliability.json").read_text())
@@ -55,7 +59,7 @@ def validate_release(source: Path, *, presentation_binding: Path | None = None)
raise ValueError("Refusing to package an incomplete or unjudged benchmark report")
if dataset.get("comparison", {}).get("require_independent_result_review"):
from .result_review import require_result_review
- require_result_review(source, dataset, load_jsonl(source / "results.jsonl"))
+ require_result_review(source, dataset, records)
return False
review = json.loads((source / "result-review.json").read_text())
if review.get("status") != "passed" or review.get("failures"):
@@ -71,7 +75,6 @@ def validate_release(source: Path, *, presentation_binding: Path | None = None)
or set(review.get("providers", [])) != set(providers)
):
raise ValueError("Report/review scope differs from the declared matrix")
- records = load_jsonl(source / "results.jsonl")
expected = {(provider, case) for provider in providers for case in cases}
if (
len(records) != len(expected)
diff --git a/harness/web_fetch/rescoring.py b/harness/web_fetch/rescoring.py
index 1b300cff3..1871fde11 100644
--- a/harness/web_fetch/rescoring.py
+++ b/harness/web_fetch/rescoring.py
@@ -8,7 +8,7 @@
ROOT = Path(__file__).resolve().parents[2]
from harness.record_io import parse_jsonl
-from harness.web_fetch.artifacts import verify_record_artifacts
+from harness.web_fetch.artifacts import verify_record_artifacts, file_scoring_metadata
from harness.web_fetch.failure_diagnosis import diagnose
from lexbench_webfetch.metrics import summarize
from harness.web_fetch.models import execution_id
@@ -42,7 +42,7 @@ def _rescore_one(arguments):
status_code=r.get("status_code"),
title=r.get("title"),
final_url=r.get("final_url"),
- metadata=r.get("provider_metadata", {}),
+ metadata=file_scoring_metadata(c, r.get("provider_metadata", {}), source),
previous_metrics=r["metrics"],
list_reference=load_list_reference(c, source),
)
diff --git a/harness/web_fetch/runner.py b/harness/web_fetch/runner.py
index a3727bb0c..5bfd300f5 100644
--- a/harness/web_fetch/runner.py
+++ b/harness/web_fetch/runner.py
@@ -1020,10 +1020,12 @@ def rescore_evaluation(
raise ValueError(f"run contains case absent from dataset: {record['case_id']}")
artifact = contained_path(run_dir,record["artifact_path"])
previous_deadline = bool(record.get("metrics", {}).get("deadline_overrun"))
+ from .artifacts import file_scoring_metadata
record["metrics"] = score_attempt(
case, artifact.read_bytes(), ok=bool(record.get("ok")),
title=record.get('title'), final_url=record.get('final_url'),
- metadata=record.get("provider_metadata", {}), previous_metrics=record["metrics"],
+ metadata=file_scoring_metadata(case, record.get("provider_metadata", {}), run_dir),
+ previous_metrics=record["metrics"],
list_reference=load_list_reference(case, dataset_path.parent),
status_code=record.get('status_code'),
)
diff --git a/pyproject.toml b/pyproject.toml
index fa63e419c..5716fc91c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -6,7 +6,7 @@ readme = "README.md"
license = {file = "LICENSE"}
requires-python = ">=3.11"
dependencies = [
- "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@84c177d97da3b82567d9881e377aa728929cf129",
+ "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@51e11917c5b00e1629a1f27c0500391371fc387a",
"beautifulsoup4>=4.12,<5",
"fastcdc==1.7.0",
]
diff --git a/tests/package_smoke.py b/tests/package_smoke.py
index ea4846beb..806ae5171 100644
--- a/tests/package_smoke.py
+++ b/tests/package_smoke.py
@@ -1713,7 +1713,7 @@ def main():
root / 'cohort', root / 'cohort/dataset.json',
root_cause_category='answer-surface',
root_cause='Synthetic installation control preserves the complete answer scoring surface.',
- regression_ids=['answer-title-supplies-required-fact'],
+ regression_ids=['body-only-word-boundary-answer-title-supplies-required-fact'],
)
assert receipt['provider_calls'] == 0 and receipt['task_decision_changes'] == []
revised = json.loads((root / 'cohort/results.jsonl').read_text())
diff --git a/tests/test_assembly_evidence.py b/tests/test_assembly_evidence.py
index ec0f448f2..30f204c83 100644
--- a/tests/test_assembly_evidence.py
+++ b/tests/test_assembly_evidence.py
@@ -77,7 +77,7 @@ def test_canonical_dataset_promotes_an_explicit_full_review_policy(source, tmp_p
(root / 'dataset.json').write_text(json.dumps(baseline))
revise(root, root / 'dataset.json', root_cause_category='answer-surface',
root_cause='Synthetic retained-answer correction is checked on the complete cohort.',
- regression_ids=['answer-title-supplies-required-fact'])
+ regression_ids=['body-only-word-boundary-answer-title-supplies-required-fact'])
baseline = json.loads((root / 'dataset.json').read_text())
canonical = tmp_path / 'canonical.json'
promoted = json.loads(json.dumps(baseline))
@@ -721,7 +721,7 @@ def test_scored_cohort_composition_keeps_authority_without_starting_review(sourc
root, _ = source
revise(root, root / 'dataset.json', root_cause_category='answer-surface',
root_cause='Synthetic retained-answer correction is checked on the complete cohort.',
- regression_ids=['answer-title-supplies-required-fact'])
+ regression_ids=['body-only-word-boundary-answer-title-supplies-required-fact'])
assert valid_scoring_revision(root)
original_bytes = (root / 'results.jsonl').read_bytes()
original_history = (root / 'scoring-revisions.json').read_bytes()
@@ -809,7 +809,7 @@ def test_scored_cohort_composition_rejects_invalid_authority_or_changed_verdict(
(root / 'dataset.json').write_text(json.dumps(metadata))
revise(root, root / 'dataset.json', root_cause_category='answer-surface',
root_cause='Synthetic retained-answer correction is checked on the complete cohort.',
- regression_ids=['answer-title-supplies-required-fact'])
+ regression_ids=['body-only-word-boundary-answer-title-supplies-required-fact'])
dataset = json.loads((root / 'dataset.json').read_text())
case = EvalCase(**dataset['cases'][0])
class Fixture(Provider):
diff --git a/tests/test_publication_privacy.py b/tests/test_publication_privacy.py
index ed39989e6..a9f7dc162 100644
--- a/tests/test_publication_privacy.py
+++ b/tests/test_publication_privacy.py
@@ -605,7 +605,7 @@ def interrupted(*args):
# Reject a different cohort or an attempt/log rewrite.
saved_results = (source / 'results.jsonl').read_bytes()
(source / 'results.jsonl').write_bytes(b'changed results')
- with self.assertRaisesRegex(ValueError, 'cannot replace measured results'):
+ with self.assertRaises(ValueError):
module.build_incremental(source, archive_path, root / 'wrong-cohort')
(source / 'results.jsonl').write_bytes(saved_results)
(source / 'report-manifest.json').write_text(json.dumps({'presentation_refresh': {
diff --git a/tests/test_reliability.py b/tests/test_reliability.py
index 2178ee764..86e8a909b 100644
--- a/tests/test_reliability.py
+++ b/tests/test_reliability.py
@@ -16,6 +16,24 @@ def test_complete_plan_is_reliable(self):
self.assertTrue(result['complete'])
self.assertEqual(result['judged_calls'],1)
+ def test_replacing_failures_with_another_binary_cannot_form_one_version_score(self):
+ dataset = {**self.dataset, 'cases': [self.dataset['cases'][0],
+ {'id': 'b', 'track': 'web', 'task_kind': 'body'}]}
+ rows = [{**self.record, 'provider_metadata': {'binary_sha256': 'first-build'}},
+ {**self.record, 'case_id': 'b',
+ 'provider_metadata': {'binary_sha256': 'replacement-build'}}]
+ with self.assertRaisesRegex(ValueError, 'different|可执行文件'):
+ require_reliable(dataset, rows)
+ rows[1]['provider_metadata']['binary_sha256'] = 'first-build'
+ self.assertTrue(require_reliable(dataset, rows)['complete'])
+
+ def test_separately_named_builds_can_be_compared_on_the_complete_population(self):
+ dataset = {**self.dataset, 'coverage_plan': {'web': ['release', 'candidate']}}
+ rows = [{**self.record, 'provider': name,
+ 'provider_metadata': {'binary_sha256': name + '-build'}}
+ for name in ('release', 'candidate')]
+ self.assertTrue(require_reliable(dataset, rows)['complete'])
+
def test_missing_duplicate_and_unjudged_are_rejected(self):
self.assertFalse(assess_reliability(self.dataset,[])['complete'])
duplicate=assess_reliability(self.dataset,[self.record,self.record])
diff --git a/tests/test_run_certified_benchmark.py b/tests/test_run_certified_benchmark.py
index fefc0448e..b7c824a36 100644
--- a/tests/test_run_certified_benchmark.py
+++ b/tests/test_run_certified_benchmark.py
@@ -22,6 +22,20 @@ def stage_name(command):
class CertifiedBenchmarkTests(unittest.TestCase):
+ def test_unresolved_selected_rules_fail_before_capture_or_provider_calls(self):
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ dataset = root / 'dataset.json'
+ dataset.write_text(json.dumps({'cases': [{
+ 'id': 'provisional', 'rule_review': {'status': 'unresolved'}}]}))
+ with patch('harness.web_fetch.certification.run') as invoked:
+ with self.assertRaises(SystemExit):
+ main(['--dataset', str(dataset), '--output', str(root / 'run'),
+ '--work-root', str(root / 'work'), '--providers', 'fixture',
+ '--case-ids', 'provisional'])
+ invoked.assert_not_called()
+ self.assertFalse((root / 'work').exists())
+
def test_diagnostic_execution_cannot_be_assembled_or_rendered_as_certified(self):
from harness.web_fetch import benchmark_runner as run_benchmark
from harness.web_fetch import reference_freeze as freeze_headless_references
diff --git a/tests/test_scoring.py b/tests/test_scoring.py
index c94a7ee60..f7eddf2e8 100644
--- a/tests/test_scoring.py
+++ b/tests/test_scoring.py
@@ -559,12 +559,12 @@ def test_risk_control_requires_all_declared_stable_page_text(self):
self.assertTrue(complete["stable_task_pass"])
self.assertTrue(task_passed(True, complete))
- def test_source_title_is_answer_evidence_without_inflating_body_length(self):
+ def test_source_title_cannot_supply_required_body_facts(self):
c=case(required_facts=[{'id':'title','alternatives':['Exact page title']},
{'id':'body','alternatives':['Body fact']}],
min_content_chars=20)
measured=score_attempt(c,b'Body fact',ok=True,title='Exact page title')
- self.assertEqual(measured['fact_coverage'],1)
+ self.assertEqual(measured['fact_coverage'],.5)
self.assertEqual(measured['content_chars'],len('Body fact'))
self.assertEqual(measured['title_chars'],len('Exact page title'))
self.assertFalse(measured['length_pass'])
@@ -624,8 +624,9 @@ def test_dataclass_and_dict_inputs_produce_same_metrics_without_mutation(self):
self.assertEqual((mapping, previous, metadata), original)
self.assertEqual(first["content_sha256"], hashlib.sha256(b"Alpha\r\nBeta").hexdigest())
self.assertEqual(first["content_chars"], len("Alpha\r\nBeta"))
- for key in previous:
+ for key in previous.keys() - {"raw_file_pass"}:
self.assertEqual(first[key], previous[key])
+ self.assertNotIn("raw_file_pass", first)
def test_text_coverage_is_declared_groups_not_alternatives_or_word_f1(self):
c = case(required_facts=[{"id": "a", "alternatives": ["Alpha", "Another Alpha"]},
@@ -864,7 +865,7 @@ def test_old_stable_and_list_decisions_cannot_survive_mode_change(self):
self.assertFalse(task_passed(True, measured))
for key in old.keys() - {"raw_file_pass"}:
self.assertNotIn(key, measured)
- self.assertTrue(measured["raw_file_pass"])
+ self.assertNotIn("raw_file_pass", measured)
def test_legacy_list_verification_is_shared_not_just_shape(self):
c = case(task_kind="structure", source_benchmark=legacy_benchmark())
@@ -1139,7 +1140,7 @@ def concurrent_score(*args, **kwargs):
rescore_content.rescore(self.run, case_ids={"web"})
self.assertEqual(path.read_bytes(), concurrent)
- def test_both_rescores_retain_external_file_evidence_and_original_deadline(self):
+ def test_both_rescores_refresh_file_verdict_and_retain_external_identity_and_deadline(self):
records, _ = self.live()
original = next(r for r in records if r["case_id"] == "file")
original["metrics"].update(raw_file_pass=True, raw_file_sha256="independently-verified-file",
@@ -1148,6 +1149,7 @@ def test_both_rescores_retain_external_file_evidence_and_original_deadline(self)
rescored, summary = runner.rescore_evaluation(run_dir=self.run, dataset_meta=self.dataset,
dataset_path=self.path, cases=self.cases)
file_row = next(r for r in rescored if r["case_id"] == "file")
+ original["metrics"]["raw_file_pass"] = False
self.assertEqual(file_row, original)
self.assertEqual(summary["cases"]["file"]["task_pass_rate"], 0)
rescore_content.rescore(self.run, case_ids={"file"})
@@ -1156,6 +1158,30 @@ def test_both_rescores_retain_external_file_evidence_and_original_deadline(self)
self.assertEqual(file_row["metrics"], original["metrics"])
self.assertEqual(summarize(script_rows)["cases"]["file"]["task_pass_rate"], 0)
+ def test_both_rescores_check_current_file_hash_rules_against_retained_bytes(self):
+ raw = b'\x89PNG\r\n\x1a\n' + b'x' * 32
+ self.cases = [case(id='file', task_kind='file', source_verification={
+ 'mime': 'image/png', 'file_sha256': hashlib.sha256(raw).hexdigest()})]
+ self.dataset['cases'] = [asdict(c) for c in self.cases]
+ self.path.write_text(json.dumps(self.dataset))
+ self.provider.fetch.side_effect = lambda c, timeout: ProviderResult(
+ ok=True, content='file attached', files=[{'data': raw}])
+ original, _ = self.live()
+ self.assertTrue(original[0]['metrics']['raw_file_pass'])
+ revised = replace(self.cases[0], source_verification={
+ 'mime': 'image/png', 'file_sha256': hashlib.sha256(raw + b'other').hexdigest()})
+ self.cases = [revised]
+ self.dataset['cases'] = [asdict(revised)]
+ rescored, _ = runner.rescore_evaluation(run_dir=self.run, dataset_meta=self.dataset,
+ dataset_path=self.path, cases=self.cases)
+ self.assertFalse(rescored[0]['metrics']['raw_file_pass'])
+ self.assertEqual(rescored[0]['provider_metadata'], original[0]['provider_metadata'])
+ (self.run / 'dataset.json').write_text(json.dumps(self.dataset))
+ rescore_content.rescore(self.run)
+ row = json.loads((self.run / 'results.jsonl').read_text().strip())
+ self.assertFalse(row['metrics']['raw_file_pass'])
+ self.assertEqual(row['provider_metadata'], original[0]['provider_metadata'])
+
def test_both_rescores_verify_every_declared_evidence_before_writes(self):
records, _ = self.live()
self.prepare_script_references()
diff --git a/tests/test_scoring_revision.py b/tests/test_scoring_revision.py
index 1b01e9bb4..83894ca8b 100644
--- a/tests/test_scoring_revision.py
+++ b/tests/test_scoring_revision.py
@@ -66,7 +66,7 @@ def test_invalid_history_rejects_before_any_evidence_write(tmp_path: Path, histo
canonical,
root_cause_category="answer-surface",
root_cause="Synthetic correction restores a required fact in the answer surface.",
- regression_ids=["answer-title-supplies-required-fact"],
+ regression_ids=["body-only-word-boundary-answer-title-supplies-required-fact"],
)
assert {p.name: p.read_bytes() for p in evidence.iterdir()} == before
@@ -78,7 +78,7 @@ def test_valid_history_can_extend_and_retains_prior_receipts(tmp_path: Path):
authority = {
"root_cause_category": "answer-surface",
"root_cause": "Synthetic correction restores a required fact in the answer surface.",
- "regression_ids": ["answer-title-supplies-required-fact"],
+ "regression_ids": ["body-only-word-boundary-answer-title-supplies-required-fact"],
}
first = revise(evidence, canonical, **authority)
assert first["task_decision_changes"] == [
@@ -131,7 +131,7 @@ def test_reader_prompt_can_follow_scoring_contract_without_changing_saved_call(t
canonical,
root_cause_category="answer-surface",
root_cause="The reader-facing scope described obsolete answer tokens that providers never receive.",
- regression_ids=["answer-title-supplies-required-fact"],
+ regression_ids=["body-only-word-boundary-answer-title-supplies-required-fact"],
)
assert receipt["provider_calls"] == 0
@@ -160,7 +160,7 @@ def test_reader_prompt_revision_cannot_change_provider_url(tmp_path: Path):
canonical,
root_cause_category="answer-surface",
root_cause="The reader-facing scope changed but the physical provider URL must remain frozen.",
- regression_ids=["answer-title-supplies-required-fact"],
+ regression_ids=["body-only-word-boundary-answer-title-supplies-required-fact"],
)
assert {path.name: path.read_bytes() for path in evidence.iterdir()} == before
@@ -179,7 +179,7 @@ def test_scorer_drift_and_task_rule_change_are_rejected_atomically(tmp_path: Pat
canonical,
root_cause_category="answer-surface",
root_cause="A shared answer-surface correction changed the scorer implementation.",
- regression_ids=["answer-title-supplies-required-fact"],
+ regression_ids=["body-only-word-boundary-answer-title-supplies-required-fact"],
)
assert {path.name: path.read_bytes() for path in evidence.iterdir()} == before
@@ -194,7 +194,7 @@ def test_scorer_drift_and_task_rule_change_can_be_recorded_in_two_steps(tmp_path
authority = {
"root_cause_category": "answer-surface",
"root_cause": "A shared answer-surface correction changed the scorer implementation.",
- "regression_ids": ["answer-title-supplies-required-fact"],
+ "regression_ids": ["body-only-word-boundary-answer-title-supplies-required-fact"],
}
scorer_receipt = revise(evidence, evidence / "dataset.json", **authority)
@@ -243,7 +243,7 @@ def test_scorer_only_revision_accepts_an_assembled_mixed_source_cohort(tmp_path:
evidence / "dataset.json",
root_cause_category="answer-surface",
root_cause="An assembled cohort retains outputs scored by two older scorer builds.",
- regression_ids=["answer-title-supplies-required-fact"],
+ regression_ids=["body-only-word-boundary-answer-title-supplies-required-fact"],
)
assert receipt["input_scoring_source_sha256s"] == ["0" * 64, "1" * 64]
@@ -268,7 +268,7 @@ def test_scoring_validation_preserves_complete_saved_attempt_bytes(tmp_path, fai
receipt = revise(evidence, evidence / 'dataset.json', validate_only=True,
root_cause_category='answer-surface',
root_cause='Validate the saved scoring without rewriting a physical-call record.',
- regression_ids=['answer-title-supplies-required-fact'])
+ regression_ids=['body-only-word-boundary-answer-title-supplies-required-fact'])
assert receipt['validation_only'] is True
assert receipt['task_decision_changes'] == []
assert receipt['input_results_sha256'] == receipt['output_results_sha256']
@@ -289,7 +289,7 @@ def test_scoring_validation_rejects_changed_verdict_or_requirements_atomically(t
revise(evidence, canonical, validate_only=True,
root_cause_category='answer-surface',
root_cause='Validation rejects changed scores and requirements without writing evidence.',
- regression_ids=['answer-title-supplies-required-fact'])
+ regression_ids=['body-only-word-boundary-answer-title-supplies-required-fact'])
assert {p.name:p.read_bytes() for p in evidence.iterdir()} == before
@@ -298,7 +298,7 @@ def test_report_assets_can_update_together_without_changing_scored_calls(tmp_pat
evidence, canonical, _ = revision_fixture(tmp_path, b'[]')
revise(evidence, canonical, root_cause_category='answer-surface',
root_cause='Synthetic required-answer correction has an executable acceptance control.',
- regression_ids=['answer-title-supplies-required-fact'])
+ regression_ids=['body-only-word-boundary-answer-title-supplies-required-fact'])
calls = (evidence / 'results.jsonl').read_bytes()
content = evidence / 'report-assets' / 'browser-interpretation.json'
content.parent.mkdir()
@@ -328,7 +328,7 @@ def test_report_asset_update_rejects_unbound_provenance_before_writing(tmp_path,
evidence, canonical, _ = revision_fixture(tmp_path, b'[]')
revise(evidence, canonical, root_cause_category='answer-surface',
root_cause='Synthetic required-answer correction has an executable acceptance control.',
- regression_ids=['answer-title-supplies-required-fact'])
+ regression_ids=['body-only-word-boundary-answer-title-supplies-required-fact'])
content = evidence / 'article.json'
content.write_text('{}')
implementation = evidence / 'implementation-provenance.json'
diff --git a/tests/test_three_cohort_review_followup.py b/tests/test_three_cohort_review_followup.py
index c9aec64a7..d695d4d9c 100644
--- a/tests/test_three_cohort_review_followup.py
+++ b/tests/test_three_cohort_review_followup.py
@@ -145,13 +145,13 @@ def test_legacy_page_text_rules_cannot_take_evidence_from_metadata(rule):
assert score_attempt(case,content.encode(),ok=True,title=title)['content_pass'] is expected
-def test_explicit_title_fact_and_body_scope_are_independent_requirements():
+def test_required_body_facts_and_body_scope_cannot_be_supplied_by_title_metadata():
from lexbench_webfetch.body_coverage import body_requirements
text='Required body paragraph explains independent visible source content and complete record ownership.'
case={'url':'https://example.test/article','task_kind':'body',
'required_facts':[{'id':'title','alternatives':['Exact page title']}],
'body_requirements':body_requirements(text)}
- for content,title,expected in [(text,'Exact page title',True),
- (text,'Wrong title',False),
- ('Other text','Exact page title',False)]:
+ for content,title,expected in [(text+'\nExact page title','Wrong title',True),
+ (text,'Exact page title',False),
+ ('Exact page title','Exact page title',False)]:
assert score_attempt(case,content.encode(),ok=True,title=title)['content_pass'] is expected
diff --git a/tests/test_webfetch_report_package.py b/tests/test_webfetch_report_package.py
index 8cfcd1f72..3dd16fa17 100644
--- a/tests/test_webfetch_report_package.py
+++ b/tests/test_webfetch_report_package.py
@@ -120,6 +120,18 @@ def test_declared_small_matrix_packages_through_public_command(tmp_path):
)
+def test_stale_complete_flag_cannot_publish_mixed_executable_versions(tmp_path):
+ source, archive = report_tree(tmp_path)
+ rows = [{'attempt_id': 'native-' + case, 'provider': 'native', 'case_id': case,
+ 'provider_metadata': {'binary_sha256': fingerprint}}
+ for case, fingerprint in [('alpha', 'original-build'), ('beta', 'new-build')]]
+ (source / 'results.jsonl').write_text(''.join(json.dumps(row) + '\n' for row in rows))
+ archive.write_bytes(b'previous verified release')
+ with pytest.raises(ValueError, match='mixed executable versions'):
+ report_package.build(source, archive)
+ assert archive.read_bytes() == b'previous verified release'
+
+
@pytest.mark.parametrize(
"damage",
[
diff --git a/uv.lock b/uv.lock
index 32b1da58f..5856efc5e 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1,5 +1,5 @@
version = 1
-revision = 3
+revision = 2
requires-python = ">=3.11"
resolution-markers = [
"python_full_version >= '3.13'",
@@ -354,7 +354,7 @@ requires-dist = [
{ name = "jieba", marker = "extra == 'webmainbench-eval'", specifier = "==0.42.1" },
{ name = "json-repair", marker = "extra == 'browser-use-eval'", specifier = ">=0.54,<1" },
{ name = "langchain-community", marker = "extra == 'oss-eval'", specifier = "==0.3.30" },
- { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=84c177d97da3b82567d9881e377aa728929cf129" },
+ { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=51e11917c5b00e1629a1f27c0500391371fc387a" },
{ name = "markdown", marker = "extra == 'webmainbench-eval'", specifier = ">=3.7,<4" },
{ name = "openai", marker = "extra == 'webmainbench-eval'", specifier = ">=2.8,<4" },
{ name = "openpyxl", marker = "extra == 'oss-eval'", specifier = "==3.1.5" },
@@ -1932,8 +1932,8 @@ wheels = [
[[package]]
name = "lexbench-webfetch"
-version = "0.2.0"
-source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=84c177d97da3b82567d9881e377aa728929cf129#84c177d97da3b82567d9881e377aa728929cf129" }
+version = "0.3.0"
+source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=51e11917c5b00e1629a1f27c0500391371fc387a#51e11917c5b00e1629a1f27c0500391371fc387a" }
dependencies = [
{ name = "beautifulsoup4" },
{ name = "soupsieve" },
@@ -3849,7 +3849,7 @@ resolution-markers = [
"python_full_version < '3.12'",
]
dependencies = [
- { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } },
+ { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" }
wheels = [
@@ -3924,7 +3924,7 @@ resolution-markers = [
"python_full_version == '3.12.*'",
]
dependencies = [
- { name = "numpy", version = "2.5.3", source = { registry = "https://pypi.org/simple" } },
+ { name = "numpy", version = "2.5.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/7e/74/66de6258867beb2ef08f35f9f2ac017a52cacd5081714d239ff1a442d458/scipy-1.18.1.tar.gz", hash = "sha256:52c4b7422442aba924d03ad4019852b08a92e64ea187b933135687bfe2747307", size = 30781235, upload-time = "2026-08-21T23:28:50.599Z" }
wheels = [
From 62259de5f4bfe674d166b79b26c7a307fbcba79c Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Tue, 20 Oct 2026 20:16:55 +0800
Subject: [PATCH 02/51] Align native title projection controls with body fact
acceptance
---
.../data/native-projection-regressions.json | 53 +++++++++++++++++--
1 file changed, 50 insertions(+), 3 deletions(-)
diff --git a/harness/web_fetch/data/native-projection-regressions.json b/harness/web_fetch/data/native-projection-regressions.json
index f0705b221..cafa5b990 100644
--- a/harness/web_fetch/data/native-projection-regressions.json
+++ b/harness/web_fetch/data/native-projection-regressions.json
@@ -193,8 +193,8 @@
"category": "answer-surface",
"root_cause": "unbound_native_title_or_final_url",
"ok": true,
- "id": "native-returned-title-remains-answer",
- "failure_mode": "false-negative",
+ "id": "native-returned-title-cannot-supply-missing-body-fact",
+ "failure_mode": "false-positive",
"case": {
"url": "https://wrong.example.test/page",
"task_kind": "body",
@@ -234,7 +234,7 @@
},
"title": "SECRET",
"final_url": "https://wrong.example.test/page",
- "expected_task_pass": true
+ "expected_task_pass": false
},
{
"category": "answer-surface",
@@ -474,6 +474,53 @@
"native_response": "{\"content\":\"\"}",
"expected_task_pass": false,
"expected_validation_error": "Native answer defaults must be empty"
+ },
+ {
+ "category": "answer-surface",
+ "root_cause": "unbound_native_title_or_final_url",
+ "ok": true,
+ "id": "native-returned-body-fact-with-title-remains-valid",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://wrong.example.test/page",
+ "task_kind": "body",
+ "required_facts": [
+ {
+ "alternatives": [
+ "SECRET"
+ ]
+ }
+ ],
+ "expected_any": [],
+ "min_content_chars": 0
+ },
+ "artifact": "SECRET",
+ "native_response": "{\"content\": \"SECRET\", \"title\": \"SECRET\"}",
+ "metadata": {
+ "native_answer_projection": {
+ "schema_version": 2,
+ "content_format": "text",
+ "content_pointer": [
+ "content"
+ ],
+ "result_fields": {
+ "title": {
+ "pointer": [
+ "title"
+ ]
+ },
+ "final_url": {
+ "pointer": [
+ "final_url"
+ ],
+ "fallback": "request_url"
+ }
+ }
+ }
+ },
+ "title": "SECRET",
+ "final_url": "https://wrong.example.test/page",
+ "expected_task_pass": true
}
]
}
From 7d2154485ee4c4f371937205813f023ac21944bb Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Tue, 20 Oct 2026 20:19:31 +0800
Subject: [PATCH 03/51] Cover declared tracking parameters and page-selecting
query differences
---
tests/test_native_fetch_diagnostics.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/tests/test_native_fetch_diagnostics.py b/tests/test_native_fetch_diagnostics.py
index 45abc4e32..ffd6f23b8 100644
--- a/tests/test_native_fetch_diagnostics.py
+++ b/tests/test_native_fetch_diagnostics.py
@@ -28,7 +28,8 @@ def test_ambiguous_response_or_navigation_does_not_supply_status(log):
@pytest.mark.parametrize('requested,observed,page_matches,log_status', [
(URL, URL, True, 200),
- (URL, URL+'?tracking=1', True, None),
+ (URL, URL+'?utm_source=newsletter', True, None),
+ (URL, URL+'?tracking=1', False, None),
(URL+'?page=1', URL+'?page=1', True, 200),
(URL+'?page=1', URL+'?page=2', False, None),
])
From 7c1f130a273fe1c528e13672885a6de2e55dec24 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Tue, 20 Oct 2026 20:23:42 +0800
Subject: [PATCH 04/51] Use current native title control in offline revision
scenarios
---
tests/test_local_revision.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/test_local_revision.py b/tests/test_local_revision.py
index 2a398b17b..47bdd0e57 100644
--- a/tests/test_local_revision.py
+++ b/tests/test_local_revision.py
@@ -499,7 +499,7 @@ def test_native_context_revision_replays_native_fields_and_rejects_tampering(tmp
current.update(title=None, final_url=current['url'])
(tmp_path/'results.jsonl').write_text(json.dumps(current)+'\n')
config = {'cases': [], 'authority': {'description': 'Recover native title and redirect without new calls.'},
- 'regression_ids': ['native-title-must-match-original', 'native-returned-title-remains-answer',
+ 'regression_ids': ['native-title-must-match-original', 'native-returned-body-fact-with-title-remains-valid',
'native-final-url-must-match-original', 'native-returned-redirect-resolves-relative-image'],
'native_answer_corrections': [{'attempt_id': current['attempt_id'],
'native_response': {'path': 'private_artifacts/context.response.bin', 'sha256': hashlib.sha256(native).hexdigest()},
From 407fbb8408fe64ed3300435c3fed625205fd8c62 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Tue, 20 Oct 2026 20:38:06 +0800
Subject: [PATCH 05/51] docs(contributing): define PR and commit title
conventions
---
CONTRIBUTING.md | 27 +++++++++++++++++++++++++++
1 file changed, 27 insertions(+)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 55833a8e2..8afceab95 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -20,6 +20,33 @@ Test both accepted and rejected scenarios. Fix scoring rules rather than editing
Write documentation in English. Keep the [Chinese README](docs/README.zh-CN.md) aligned when setup or usage changes; other docs do not need duplicate translations. Preserve original task text, quotations and frozen acceptance inputs. Reports follow their track's language settings.
+## PR and commit titles
+
+Use `(): ` for PR titles and new commit subjects. The scope is optional; use the module or feature affected, such as `scoring` or `webfetch`.
+
+Choose the type that describes the change:
+
+| Type | Change |
+| --- | --- |
+| `fix` | Correct existing behavior |
+| `feat` | Add supported behavior |
+| `docs` | Change documentation |
+| `refactor` | Restructure code without changing behavior |
+| `test` | Add or correct tests |
+| `perf` | Improve performance |
+| `build` / `ci` | Change packaging, dependencies or automation |
+| `chore` | Other maintenance |
+
+Write the summary in English, start with an imperative verb, and describe the concrete change. Aim for 72 characters or fewer, including the prefix; omit a trailing period. Use `!` before the colon for a breaking change and explain the migration in the description.
+
+Examples:
+
+- `fix(scoring): require facts in the returned body`
+- `fix(webfetch): reject mixed executable versions before release`
+- `docs(contributing): define PR and commit title conventions`
+
+Avoid titles such as `Fix review feedback`, `Update code` or `Misc improvements`; name the behavior or document that changes. A PR title describes the complete diff; each commit subject describes that commit. Update the PR title when its scope changes. If squash merging is used, use a subject that follows the same format. Apply this convention to new work; do not rewrite shared commits solely to rename them.
+
## Run the checks
```bash
From 7b89f689d9b26dea14675514ae354cd6f11ea42f Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Tue, 20 Oct 2026 20:53:33 +0800
Subject: [PATCH 06/51] fix(webfetch): allow retained results in partial repair
runs
---
docs/metrics.md | 2 +-
evaluation/lexbench-webfetch.json | 4 ++--
harness/web_fetch/README.md | 3 +++
harness/web_fetch/reliability.py | 17 ++++++++---------
harness/web_fetch/report_methods.py | 25 ++++++++++++++++++++++---
harness/web_fetch/report_package.py | 3 ---
pyproject.toml | 2 +-
tests/test_reliability.py | 7 ++++---
tests/test_report_methods.py | 17 +++++++++++++++++
tests/test_webfetch_report_package.py | 17 ++++++++---------
uv.lock | 4 ++--
11 files changed, 68 insertions(+), 33 deletions(-)
diff --git a/docs/metrics.md b/docs/metrics.md
index 4036c25d1..ef358c91e 100644
--- a/docs/metrics.md
+++ b/docs/metrics.md
@@ -28,4 +28,4 @@ Body metrics require a complete, comparable reference. A failed call scores zero
Outputs have SHA-256 hashes. Rebuilding or editing a report must preserve the saved responses and scores. Metric labels come from `report_components.py`; the report does not show a pass rate conditioned only on successful requests.
-Formal cohorts reject selected tasks whose acceptance rules are unresolved before capturing references or calling providers. A published method must use one executable fingerprint across its selected results. Compare different builds as separate complete method populations. Offline file rescoring reads the retained file bytes and applies the current file requirements while preserving the original call metadata.
+Formal cohorts reject selected tasks whose acceptance rules are unresolved before capturing references or calling providers. A repair run may retain passed results and rerun only failures. Rescore the complete selected population under one scoring contract, retain every physical call, and disclose each executable fingerprint and its selected result count. A cross-build cumulative result is not a full run by one build. Offline file rescoring reads the retained file bytes and applies the current file requirements while preserving the original call metadata.
diff --git a/evaluation/lexbench-webfetch.json b/evaluation/lexbench-webfetch.json
index c1591b67a..ad1352194 100644
--- a/evaluation/lexbench-webfetch.json
+++ b/evaluation/lexbench-webfetch.json
@@ -2,7 +2,7 @@
"schema_version": 1,
"kind": "external_benchmark_binding",
"repository": "https://github.com/lexmount/lexbench-webfetch",
- "revision": "51e11917c5b00e1629a1f27c0500391371fc387a",
+ "revision": "02027f7d4f3179e48ed2e221229f87a9a204564a",
"ownership": {
"benchmark": "Task populations, task model, acceptance scoring, file checks, compiler and frozen definitions",
"harness": "Execution, reference lifecycle, evidence and reports"
@@ -78,7 +78,7 @@
"evaluation/risk-control/remaining-70-rule-review.md": "301574dddcfddf6d30ed4d882cdd40b1ae859fdc95a0a02cf9868f55e3fd1079",
"evaluation/risk-control/rule-authoring-guide.md": "74363ab502978179e5438a0b992bac7563090e41c91784e492636832dc1cd088",
"evaluation/risk-control/source-inputs.json": "77c5a4b62c07d59ed57154a833e85f9e7a97bc739730f51cd4ca3a698edb9258",
- "evaluation/scoring-regressions.json": "5edad3eb2e776b947b6ab998aadd89d62332fc9cf5626e5d51a12e29581852be",
+ "evaluation/scoring-regressions.json": "7dcdd8771edf2eea783be38b5aa28905317836a526a58f4419f58dfe7b9cd455",
"evaluation/source-inputs.json": "c202abc4624811b938a837402bbaf0ecbe716e375fbca70567346d2edb3b1708"
}
}
diff --git a/harness/web_fetch/README.md b/harness/web_fetch/README.md
index e280571e1..692a5422f 100644
--- a/harness/web_fetch/README.md
+++ b/harness/web_fetch/README.md
@@ -253,6 +253,9 @@ validation, not each measured fetch; a changed binary requires revalidation.
### Retries
+A repair run may retain successful results and rerun only failed or affected cases. Keep every physical call and select the final response under the declared retry/correction policy. Recompute scores for the complete selected result population when the scorer changes; rescoring does not require new provider calls. If executable builds differ, disclose their fingerprints and selected result counts as cumulative results. This does not establish the full-suite result or speed of one build.
+
+
All methods default to one retry after a failed task. Set
`--max-failure-retries N` to any nonnegative integer; `0` disables retries.
There are at most N+1 calls per task, stopping as soon as the task passes.
diff --git a/harness/web_fetch/reliability.py b/harness/web_fetch/reliability.py
index 4b074d3bd..726aefa86 100644
--- a/harness/web_fetch/reliability.py
+++ b/harness/web_fetch/reliability.py
@@ -6,15 +6,16 @@
from lexbench_webfetch.metrics import task_measured
-def implementation_conflicts(records: list[dict]) -> dict[str, list[str]]:
- """Find method labels that combine results from distinct executable builds."""
+def implementation_inventory(records: list[dict]) -> dict[str, dict[str, int]]:
+ """Disclose retained and rerun build populations without requiring new calls."""
implementations = {}
for row in records:
fingerprint = (row.get("provider_metadata") or {}).get("binary_sha256")
if fingerprint:
- implementations.setdefault(row.get("provider"), set()).add(fingerprint)
- return {provider: sorted(hashes) for provider, hashes in implementations.items()
- if len(hashes) > 1}
+ counts = implementations.setdefault(row.get("provider"), {})
+ counts[fingerprint] = counts.get(fingerprint, 0) + 1
+ return {provider: dict(sorted(counts.items()))
+ for provider, counts in implementations.items()}
def assess_reliability(dataset: dict, records: list[dict]) -> dict:
@@ -37,7 +38,7 @@ def assess_reliability(dataset: dict, records: list[dict]) -> dict:
if row.get("case_id") in cases
and not task_measured(row.get("metrics", {}), cases[row["case_id"]].get("task_kind", "body"))
)
- conflicts = implementation_conflicts(records)
+ implementations = implementation_inventory(records)
errors = []
if not cases:
errors.append("评测集没有案例")
@@ -51,8 +52,6 @@ def assess_reliability(dataset: dict, records: list[dict]) -> dict:
errors.append(f"存在 {len(duplicates)} 个重复或多次选入的调用")
if unjudged:
errors.append(f"存在 {len(unjudged)} 个无法判定通过或失败的调用")
- if conflicts:
- errors.append("同一方法混用了不同可执行文件: " + ", ".join(sorted(conflicts)))
complete = bool(expected) and not errors
return {
"schema_version": 1,
@@ -65,7 +64,7 @@ def assess_reliability(dataset: dict, records: list[dict]) -> dict:
"extra": [{"provider": p, "case_id": c} for p, c in extra],
"duplicates": [{"provider": p, "case_id": c, "count": counts[(p, c)]} for p, c in duplicates],
"unjudged": [{"provider": p, "case_id": c} for p, c in unjudged],
- "implementation_conflicts": conflicts,
+ "implementation_inventory": implementations,
"errors": errors,
"publication_policy": "all_planned_provider_case_pairs_must_exist_once_and_be_judged",
}
diff --git a/harness/web_fetch/report_methods.py b/harness/web_fetch/report_methods.py
index 34c6311e3..ca68cb91b 100644
--- a/harness/web_fetch/report_methods.py
+++ b/harness/web_fetch/report_methods.py
@@ -52,6 +52,25 @@ def adoption_counts(data,names):
result.append({'provider':name,'repository_count':len(repos) if sigs is not None else None,'file_count':len(files) if sigs is not None else None,'signatures':sorted(sigs or []),'repositories':list(repos.values())})
return sorted(result,key=lambda x:(x['repository_count'] is None,-(x['repository_count'] or 0),x['provider']))
+def _implementation_scope(records, *, section, table, cell, esc):
+ from .reliability import implementation_inventory
+ from .report import model_text
+ inventories = {name: counts for name, counts in implementation_inventory(records).items()
+ if len(counts) > 1}
+ if not inventories:
+ return ""
+ rows = [("", cell(esc(name)) + cell("" + esc(fingerprint) + "") + cell(str(count)))
+ for name, counts in sorted(inventories.items())
+ for fingerprint, count in counts.items()]
+ explanation = model_text(
+ "这些方法的最终结果包含多个程序版本。允许保留已通过的结果、只重跑失败部分,再统一评分。表中列出各版本采用的结果数;累计成绩不代表某一个版本独立跑完整套任务。",
+ "These methods combine retained results and reruns from multiple executable builds. Passed results may be retained while failures are rerun, then the complete selected population is rescored. Counts below identify each build's selected results; the cumulative score is not a full run by one build.")
+ return section(model_text("局部重跑与结果来源", "Partial reruns and result provenance"),
+ "
" + explanation + "
" + table(
+ [model_text("方法", "Method"), "SHA-256", model_text("采用结果数", "Selected results")],
+ rows, metrics=[]), "implementation-scope")
+
+
def render_methods(out,records,*,hero,section,table,cell,link,esc,authored=True):
if records and all(row.get('provider_type') == 'fixture' for row in records):
body = hero('合成示例适配器', '离线检查安装、评分、证据和报告导航。') + section(
@@ -62,7 +81,7 @@ def render_methods(out,records,*,hero,section,table,cell,link,esc,authored=True)
'
fetch(EvalCase, timeout) → ProviderResult
'
'
输入为任务定义与超时预算,输出为预设的内容或错误。完整输出见该方法的结果详情。
',
'interface-' + name)
- return body
+ return body + _implementation_scope(records, section=section, table=table, cell=cell, esc=esc)
dataset=json.loads((out/'dataset.json').read_text()) if (out/'dataset.json').is_file() else {}
native=measurement_presentation(dataset).native_outputs
if not authored and not native:
@@ -86,7 +105,7 @@ def render_methods(out,records,*,hero,section,table,cell,link,esc,authored=True)
'interface-'+name)
for anchor in sorted(anchors):
body+=section('费用信息','
本轮结果未绑定价格资料;费用取决于调用方配置与服务账单。
',anchor)
- return body
+ return body + _implementation_scope(records, section=section, table=table, cell=cell, esc=esc)
source=Path(__file__).resolve().parents[2]/'research/provider-adoption.json'
data=json.loads(source.read_text());names=sorted({r['provider'] for r in records})
counts=adoption_counts(data,sorted(set(PROFILES)|set(names)))
@@ -137,4 +156,4 @@ def render_methods(out,records,*,hero,section,table,cell,link,esc,authored=True)
('interfaces','接口与示例',render_interfaces(out,records)),
('scope','统计口径',scope),
]))
- return body
+ return body + _implementation_scope(records, section=section, table=table, cell=cell, esc=esc)
diff --git a/harness/web_fetch/report_package.py b/harness/web_fetch/report_package.py
index 5bfefb5ab..df99ee92d 100644
--- a/harness/web_fetch/report_package.py
+++ b/harness/web_fetch/report_package.py
@@ -35,10 +35,7 @@ def validate_release(source: Path, *, presentation_binding: Path | None = None)
return False
dataset_path = source / "dataset.json"
dataset = json.loads(dataset_path.read_text()) if dataset_path.exists() else {}
- from .reliability import implementation_conflicts
records = load_jsonl(source / "results.jsonl")
- if conflicts := implementation_conflicts(records):
- raise ValueError("Refusing to package mixed executable versions: " + ", ".join(sorted(conflicts)))
from .reference_lifecycle import require_certifiable_evidence
manifests = []
for name in ('manifest.json', 'execution.json'):
diff --git a/pyproject.toml b/pyproject.toml
index 5716fc91c..a07f9ee63 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -6,7 +6,7 @@ readme = "README.md"
license = {file = "LICENSE"}
requires-python = ">=3.11"
dependencies = [
- "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@51e11917c5b00e1629a1f27c0500391371fc387a",
+ "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@02027f7d4f3179e48ed2e221229f87a9a204564a",
"beautifulsoup4>=4.12,<5",
"fastcdc==1.7.0",
]
diff --git a/tests/test_reliability.py b/tests/test_reliability.py
index 86e8a909b..495859a3f 100644
--- a/tests/test_reliability.py
+++ b/tests/test_reliability.py
@@ -16,14 +16,15 @@ def test_complete_plan_is_reliable(self):
self.assertTrue(result['complete'])
self.assertEqual(result['judged_calls'],1)
- def test_replacing_failures_with_another_binary_cannot_form_one_version_score(self):
+ def test_failure_only_reruns_retain_successes_and_disclose_build_counts(self):
dataset = {**self.dataset, 'cases': [self.dataset['cases'][0],
{'id': 'b', 'track': 'web', 'task_kind': 'body'}]}
rows = [{**self.record, 'provider_metadata': {'binary_sha256': 'first-build'}},
{**self.record, 'case_id': 'b',
'provider_metadata': {'binary_sha256': 'replacement-build'}}]
- with self.assertRaisesRegex(ValueError, 'different|可执行文件'):
- require_reliable(dataset, rows)
+ result = require_reliable(dataset, rows)
+ self.assertEqual(result['implementation_inventory']['p'],
+ {'first-build': 1, 'replacement-build': 1})
rows[1]['provider_metadata']['binary_sha256'] = 'first-build'
self.assertTrue(require_reliable(dataset, rows)['complete'])
diff --git a/tests/test_report_methods.py b/tests/test_report_methods.py
index e4bbeb7ad..ecabc4ac8 100644
--- a/tests/test_report_methods.py
+++ b/tests/test_report_methods.py
@@ -179,3 +179,20 @@ def unrelated(out,dataset,rows):
unrelated(tmp_path,{},records)
assert provider_label('moli-release')==original
+
+
+def test_method_guide_discloses_retained_success_and_repaired_failure_builds(tmp_path):
+ from harness.web_fetch.report_methods import render_methods
+ from harness.web_fetch.report import hero, section, table, cell, link, esc
+ records = [{'provider': 'sample', 'provider_type': 'fixture',
+ 'provider_metadata': {'binary_sha256': build}}
+ for build in ('retained-build', 'retained-build', 'repair-build')]
+ body = render_methods(tmp_path, records, hero=hero, section=section,
+ table=table, cell=cell, link=link, esc=esc)
+ assert 'implementation-scope' in body
+ assert 'retained-build' in body and 'repair-build' in body
+ assert '>2<' in body and '>1<' in body
+ assert '累计成绩不代表某一个版本' in body
+ single = render_methods(tmp_path, records[:2], hero=hero, section=section,
+ table=table, cell=cell, link=link, esc=esc)
+ assert 'implementation-scope' not in single
diff --git a/tests/test_webfetch_report_package.py b/tests/test_webfetch_report_package.py
index 3dd16fa17..03b3cab8e 100644
--- a/tests/test_webfetch_report_package.py
+++ b/tests/test_webfetch_report_package.py
@@ -120,16 +120,15 @@ def test_declared_small_matrix_packages_through_public_command(tmp_path):
)
-def test_stale_complete_flag_cannot_publish_mixed_executable_versions(tmp_path):
- source, archive = report_tree(tmp_path)
- rows = [{'attempt_id': 'native-' + case, 'provider': 'native', 'case_id': case,
- 'provider_metadata': {'binary_sha256': fingerprint}}
- for case, fingerprint in [('alpha', 'original-build'), ('beta', 'new-build')]]
+def test_retained_success_and_failure_rerun_can_be_packaged_together(tmp_path):
+ source, archive = report_tree(tmp_path, sampled=True)
+ rows = [json.loads(line) for line in (source / 'results.jsonl').read_text().splitlines()]
+ for row in rows:
+ row['provider_metadata'] = {'binary_sha256':
+ 'retained-build' if row['case_id'] == 'alpha' else 'repair-build'}
(source / 'results.jsonl').write_text(''.join(json.dumps(row) + '\n' for row in rows))
- archive.write_bytes(b'previous verified release')
- with pytest.raises(ValueError, match='mixed executable versions'):
- report_package.build(source, archive)
- assert archive.read_bytes() == b'previous verified release'
+ report_package.build(source, archive)
+ assert archive.is_file()
@pytest.mark.parametrize(
diff --git a/uv.lock b/uv.lock
index 5856efc5e..29f3ec4af 100644
--- a/uv.lock
+++ b/uv.lock
@@ -354,7 +354,7 @@ requires-dist = [
{ name = "jieba", marker = "extra == 'webmainbench-eval'", specifier = "==0.42.1" },
{ name = "json-repair", marker = "extra == 'browser-use-eval'", specifier = ">=0.54,<1" },
{ name = "langchain-community", marker = "extra == 'oss-eval'", specifier = "==0.3.30" },
- { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=51e11917c5b00e1629a1f27c0500391371fc387a" },
+ { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=02027f7d4f3179e48ed2e221229f87a9a204564a" },
{ name = "markdown", marker = "extra == 'webmainbench-eval'", specifier = ">=3.7,<4" },
{ name = "openai", marker = "extra == 'webmainbench-eval'", specifier = ">=2.8,<4" },
{ name = "openpyxl", marker = "extra == 'oss-eval'", specifier = "==3.1.5" },
@@ -1933,7 +1933,7 @@ wheels = [
[[package]]
name = "lexbench-webfetch"
version = "0.3.0"
-source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=51e11917c5b00e1629a1f27c0500391371fc387a#51e11917c5b00e1629a1f27c0500391371fc387a" }
+source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=02027f7d4f3179e48ed2e221229f87a9a204564a#02027f7d4f3179e48ed2e221229f87a9a204564a" }
dependencies = [
{ name = "beautifulsoup4" },
{ name = "soupsieve" },
From f19092f275dc12352b1ef01df6f318f02583c7df Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Tue, 20 Oct 2026 20:58:54 +0800
Subject: [PATCH 07/51] fix(webfetch): count missing required identity as a
failed result
---
evaluation/lexbench-webfetch.json | 4 ++--
harness/web_fetch/adjudication_evidence.py | 4 ++--
harness/web_fetch/report.py | 4 ++++
harness/web_fetch/report_i18n.py | 2 ++
pyproject.toml | 2 +-
tests/test_file_report_reason.py | 16 +++++++++++++---
tests/test_local_audit.py | 4 ++--
tests/test_local_measurement_lineage.py | 3 +--
uv.lock | 4 ++--
9 files changed, 29 insertions(+), 14 deletions(-)
diff --git a/evaluation/lexbench-webfetch.json b/evaluation/lexbench-webfetch.json
index ad1352194..d81b867c7 100644
--- a/evaluation/lexbench-webfetch.json
+++ b/evaluation/lexbench-webfetch.json
@@ -2,7 +2,7 @@
"schema_version": 1,
"kind": "external_benchmark_binding",
"repository": "https://github.com/lexmount/lexbench-webfetch",
- "revision": "02027f7d4f3179e48ed2e221229f87a9a204564a",
+ "revision": "42c5e3b0b0ffe2631507ac1b44337ab6b43e853b",
"ownership": {
"benchmark": "Task populations, task model, acceptance scoring, file checks, compiler and frozen definitions",
"harness": "Execution, reference lifecycle, evidence and reports"
@@ -78,7 +78,7 @@
"evaluation/risk-control/remaining-70-rule-review.md": "301574dddcfddf6d30ed4d882cdd40b1ae859fdc95a0a02cf9868f55e3fd1079",
"evaluation/risk-control/rule-authoring-guide.md": "74363ab502978179e5438a0b992bac7563090e41c91784e492636832dc1cd088",
"evaluation/risk-control/source-inputs.json": "77c5a4b62c07d59ed57154a833e85f9e7a97bc739730f51cd4ca3a698edb9258",
- "evaluation/scoring-regressions.json": "7dcdd8771edf2eea783be38b5aa28905317836a526a58f4419f58dfe7b9cd455",
+ "evaluation/scoring-regressions.json": "e5f610d2adfaa6a7fd8d80af3696033c4ccd1a46038d49f7825ebf9dfdcbdcae",
"evaluation/source-inputs.json": "c202abc4624811b938a837402bbaf0ecbe716e375fbca70567346d2edb3b1708"
}
}
diff --git a/harness/web_fetch/adjudication_evidence.py b/harness/web_fetch/adjudication_evidence.py
index 08b294aed..14d2ad439 100644
--- a/harness/web_fetch/adjudication_evidence.py
+++ b/harness/web_fetch/adjudication_evidence.py
@@ -444,7 +444,7 @@ def _require_current_retained_reviews(source, dataset, records, review, receipt)
def score_reviewed_page_identity(source, case, record, annotation, captures, *, historical_scoring_source=None):
"""Apply a bound retrospective content correspondence, never a native URL."""
- from lexbench_webfetch.metrics import normalize_for_match, task_measured
+ from lexbench_webfetch.metrics import normalize_for_match
from lexbench_webfetch.scoring import score_attempt, _same_page_url, SCORING_SOURCE_SHA256
if (annotation.get("attempt_id") != record["attempt_id"]
@@ -462,7 +462,7 @@ def score_reviewed_page_identity(source, case, record, annotation, captures, *,
previous_metrics=record["metrics"], status_code=record.get("status_code"),
title=record.get("title"), final_url=record.get("final_url"),
)
- if not historical and (task_measured(original_metrics)
+ if not historical and (not original_metrics.get("page_identity_required")
or original_metrics.get("page_identity_status") != "unavailable"):
raise ValueError("Page review may only fill an unavailable page identity")
capture = captures[annotation["capture_index"]]
diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py
index 6ab24d128..0c1cf00c4 100644
--- a/harness/web_fetch/report.py
+++ b/harness/web_fetch/report.py
@@ -1568,6 +1568,8 @@ def failure_category(r,c):
if r['metrics'].get('evaluation_excluded'):return '评测环境异常:统一排除比较'
if not measured(r,c):return '无法判定:最终网页身份缺少证据' if r['metrics'].get('page_identity_required') and r['metrics'].get('page_identity_status')=='unavailable' else '未测:缺少验收证据'
if succeeded(r,c):return '—'
+ if r['metrics'].get('page_identity_required') and r['metrics'].get('page_identity_status') != 'verified':
+ return '最终网页身份未核实'
# Classification describes an already failed result; hints never veto success.
metrics=r['metrics']
if (r.get('ok',False) and metrics.get('request_ok',True)
@@ -1658,6 +1660,8 @@ def reason(r,c):
if c.get('task_kind')=='structure':
from .report_components import list_rule_description
return f"检出 {r['metrics'].get('named_entry_count',0)} 个具名详情入口。"+list_rule_description(c)
+ if r['metrics'].get('page_identity_required') and r['metrics'].get('page_identity_status') != 'verified':
+ return '未通过:最终网页身份缺少证据,保留在计划任务分母中。'
if c.get('task_kind')=='file':return '已返回原始文件,格式特征、长度及目标 SHA-256 校验通过。' if succeeded(r,c) else '没有返回可核对的原始文件;文字或重复输入地址不算文件。'
if c.get('task_kind')=='resource':
missing=[x['id'] for x in r['metrics'].get('resource_results',[]) if not x['pass']]
diff --git a/harness/web_fetch/report_i18n.py b/harness/web_fetch/report_i18n.py
index 81bb0c753..339c16abb 100644
--- a/harness/web_fetch/report_i18n.py
+++ b/harness/web_fetch/report_i18n.py
@@ -37,6 +37,8 @@
# Exact editorial translations take priority. Dynamic evidence (task names,
# references and provider output) is deliberately absent from this catalog.
TEXT = {
+ "最终网页身份未核实": "Final page identity is not verified",
+ "未通过:最终网页身份缺少证据,保留在计划任务分母中。": "Failed: final page identity lacks evidence; this result remains in the planned task denominator.",
"仓库内测试适配器;只用于确定性回归测试。": "Repository test adapter; used only for deterministic regression tests.",
"不需要账号或环境变量。": "No account or environment variables required.",
"先检查请求与返回内容,再逐项检查本例声明的稳定文字、正文覆盖与组合规则;诊断文字只显示命中情况。": "Check the request and returned content, then apply the declared stable text, body coverage and composite requirements. Diagnostic text reports matches without changing the verdict.",
diff --git a/pyproject.toml b/pyproject.toml
index a07f9ee63..5cec09fd0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -6,7 +6,7 @@ readme = "README.md"
license = {file = "LICENSE"}
requires-python = ">=3.11"
dependencies = [
- "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@02027f7d4f3179e48ed2e221229f87a9a204564a",
+ "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@42c5e3b0b0ffe2631507ac1b44337ab6b43e853b",
"beautifulsoup4>=4.12,<5",
"fastcdc==1.7.0",
]
diff --git a/tests/test_file_report_reason.py b/tests/test_file_report_reason.py
index ba0f6a0b9..695640534 100644
--- a/tests/test_file_report_reason.py
+++ b/tests/test_file_report_reason.py
@@ -22,10 +22,10 @@ def test_missing_page_identity_is_not_a_file_or_uncalled_result(self):
from harness.web_fetch.report import failure_category, result_label
row={'ok':True,'metrics':{'request_ok':True,'page_identity_required':True,
'page_identity_status':'unavailable'}}
- self.assertEqual(result_label(row,{}),'无法判定')
+ self.assertEqual(result_label(row,{}),'未通过')
self.assertIn('最终网页身份',reason(row,{}))
self.assertNotIn('原文件',reason(row,{}))
- self.assertIn('无法判定',failure_category(row,{}))
+ self.assertIn('身份',failure_category(row,{}))
def test_shared_evidence_lines_preserve_complete_exact_matches(self):
from harness.web_fetch.report import _exact_evidence_lines, _normalized_evidence_lines
@@ -43,13 +43,23 @@ def test_unknown_result_is_visible_but_not_in_acceptance_denominator(self):
rows=[{'provider':'p','case_id':'a','ok':True,'elapsed_ms':1,
'metrics':{'content_pass':True}},
{'provider':'p','case_id':'b','ok':True,'elapsed_ms':1,
- 'metrics':{'page_identity_required':True,'page_identity_status':'unavailable'}}]
+ 'metrics':{'rule_review_status':'unresolved'}}]
rendered=overall_ranking(rows,cases)
self.assertIn('无法判定',rendered)
self.assertIn('1/1',rendered)
self.assertIn('可判定数',rendered)
self.assertNotIn('每题计一次通过或失败',rendered)
+ def test_required_identity_failure_stays_in_the_planned_denominator(self):
+ from harness.web_fetch.report import overall_ranking
+ cases = {key: {'id': key, 'track': 'web'} for key in ('a', 'b')}
+ rows = [{'provider': 'p', 'case_id': key, 'ok': True, 'elapsed_ms': 1,
+ 'metrics': ({'content_pass': True} if key == 'a' else
+ {'page_identity_required': True, 'page_identity_status': 'unavailable'})}
+ for key in cases]
+ self.assertIn('1/2', overall_ranking(rows, cases))
+
+
class BoundedAnchorEvidenceTests(unittest.TestCase):
def test_single_line_html_does_not_repeat_the_answer_for_each_anchor(self):
diff --git a/tests/test_local_audit.py b/tests/test_local_audit.py
index f505e8b3e..c4dc8cd2c 100644
--- a/tests/test_local_audit.py
+++ b/tests/test_local_audit.py
@@ -950,8 +950,8 @@ def test_retrospective_page_review_preserves_call_and_checks_other_requirements(
missing['acceptance_contract']['stable_text_anchors'].append(
{'id': 'missing', 'alternatives': ['A required fact that was not returned']})
bad = {**annotation, 'case_sha256': canonical_json_sha256(missing)}
- with pytest.raises(ValueError, match='unavailable page identity'):
- adjudication_evidence.score_reviewed_page_identity(tmp_path, missing, record, bad, captures)
+ assert not task_passed(True, adjudication_evidence.score_reviewed_page_identity(
+ tmp_path, missing, record, bad, captures))
def test_retrospective_page_review_requires_bound_receipt(tmp_path):
diff --git a/tests/test_local_measurement_lineage.py b/tests/test_local_measurement_lineage.py
index 6733e421d..ee6b4a310 100644
--- a/tests/test_local_measurement_lineage.py
+++ b/tests/test_local_measurement_lineage.py
@@ -48,8 +48,7 @@ def test_unknown_verdict_can_be_shown_locally_without_becoming_publishable():
dataset = {'comparison': {'kind': 'uniform_native_local_measurement',
'publication_state': 'local_audit_only_not_for_release'},
'coverage_plan': {'web': ['p']}, 'cases': [{'id': 'a', 'track': 'web'}]}
- row = {'provider': 'p', 'case_id': 'a', 'metrics': {'page_identity_required': True,
- 'page_identity_status': 'unavailable', 'request_ok': True}}
+ row = {'provider': 'p', 'case_id': 'a', 'metrics': {'rule_review_status': 'unresolved', 'request_ok': True}}
reliability = local_audit.require_complete_local_inventory(dataset, [row])
assert reliability['present_calls'] == 1 and reliability['judged_calls'] == 0
assert not reliability['complete']
diff --git a/uv.lock b/uv.lock
index 29f3ec4af..e8db96e98 100644
--- a/uv.lock
+++ b/uv.lock
@@ -354,7 +354,7 @@ requires-dist = [
{ name = "jieba", marker = "extra == 'webmainbench-eval'", specifier = "==0.42.1" },
{ name = "json-repair", marker = "extra == 'browser-use-eval'", specifier = ">=0.54,<1" },
{ name = "langchain-community", marker = "extra == 'oss-eval'", specifier = "==0.3.30" },
- { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=02027f7d4f3179e48ed2e221229f87a9a204564a" },
+ { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=42c5e3b0b0ffe2631507ac1b44337ab6b43e853b" },
{ name = "markdown", marker = "extra == 'webmainbench-eval'", specifier = ">=3.7,<4" },
{ name = "openai", marker = "extra == 'webmainbench-eval'", specifier = ">=2.8,<4" },
{ name = "openpyxl", marker = "extra == 'oss-eval'", specifier = "==3.1.5" },
@@ -1933,7 +1933,7 @@ wheels = [
[[package]]
name = "lexbench-webfetch"
version = "0.3.0"
-source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=02027f7d4f3179e48ed2e221229f87a9a204564a#02027f7d4f3179e48ed2e221229f87a9a204564a" }
+source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=42c5e3b0b0ffe2631507ac1b44337ab6b43e853b#42c5e3b0b0ffe2631507ac1b44337ab6b43e853b" }
dependencies = [
{ name = "beautifulsoup4" },
{ name = "soupsieve" },
From 76af482771a7a71bc8e994fdd806486359047a11 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Tue, 20 Oct 2026 20:59:33 +0800
Subject: [PATCH 08/51] test(webfetch): keep missing native page identity in
the denominator
---
tests/test_providers.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/test_providers.py b/tests/test_providers.py
index 56c708eff..f8978e436 100644
--- a/tests/test_providers.py
+++ b/tests/test_providers.py
@@ -39,7 +39,7 @@ def test_native_final_url_controls_identity(self, request):
for native_url, passed, measured, identity in (
('https://example.test/final',True,True,'verified'),
('https://example.test/other',False,True,'mismatch'),
- (None,False,False,'unavailable'),
+ (None,False,True,'unavailable'),
):
with self.subTest(native_url=native_url):
metadata={'sourceURL':case.url,'statusCode':200}
From 36dd2d6e9f733eb583b84cbeda5129a46a653b15 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Tue, 20 Oct 2026 21:04:19 +0800
Subject: [PATCH 09/51] build(webfetch): pin scorer revision with corrected
citation metadata
---
evaluation/lexbench-webfetch.json | 2 +-
pyproject.toml | 2 +-
uv.lock | 4 ++--
3 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/evaluation/lexbench-webfetch.json b/evaluation/lexbench-webfetch.json
index d81b867c7..cc690d46e 100644
--- a/evaluation/lexbench-webfetch.json
+++ b/evaluation/lexbench-webfetch.json
@@ -2,7 +2,7 @@
"schema_version": 1,
"kind": "external_benchmark_binding",
"repository": "https://github.com/lexmount/lexbench-webfetch",
- "revision": "42c5e3b0b0ffe2631507ac1b44337ab6b43e853b",
+ "revision": "165947c2a9f269a55810de0086fa86aadb749598",
"ownership": {
"benchmark": "Task populations, task model, acceptance scoring, file checks, compiler and frozen definitions",
"harness": "Execution, reference lifecycle, evidence and reports"
diff --git a/pyproject.toml b/pyproject.toml
index 5cec09fd0..ca38b813e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -6,7 +6,7 @@ readme = "README.md"
license = {file = "LICENSE"}
requires-python = ">=3.11"
dependencies = [
- "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@42c5e3b0b0ffe2631507ac1b44337ab6b43e853b",
+ "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@165947c2a9f269a55810de0086fa86aadb749598",
"beautifulsoup4>=4.12,<5",
"fastcdc==1.7.0",
]
diff --git a/uv.lock b/uv.lock
index e8db96e98..2843b2a2f 100644
--- a/uv.lock
+++ b/uv.lock
@@ -354,7 +354,7 @@ requires-dist = [
{ name = "jieba", marker = "extra == 'webmainbench-eval'", specifier = "==0.42.1" },
{ name = "json-repair", marker = "extra == 'browser-use-eval'", specifier = ">=0.54,<1" },
{ name = "langchain-community", marker = "extra == 'oss-eval'", specifier = "==0.3.30" },
- { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=42c5e3b0b0ffe2631507ac1b44337ab6b43e853b" },
+ { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=165947c2a9f269a55810de0086fa86aadb749598" },
{ name = "markdown", marker = "extra == 'webmainbench-eval'", specifier = ">=3.7,<4" },
{ name = "openai", marker = "extra == 'webmainbench-eval'", specifier = ">=2.8,<4" },
{ name = "openpyxl", marker = "extra == 'oss-eval'", specifier = "==3.1.5" },
@@ -1933,7 +1933,7 @@ wheels = [
[[package]]
name = "lexbench-webfetch"
version = "0.3.0"
-source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=42c5e3b0b0ffe2631507ac1b44337ab6b43e853b#42c5e3b0b0ffe2631507ac1b44337ab6b43e853b" }
+source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=165947c2a9f269a55810de0086fa86aadb749598#165947c2a9f269a55810de0086fa86aadb749598" }
dependencies = [
{ name = "beautifulsoup4" },
{ name = "soupsieve" },
From 52b7035d4b24c3d8cd43e2d0215ba225c1cddcb4 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Tue, 20 Oct 2026 21:12:28 +0800
Subject: [PATCH 10/51] build(webfetch): pin scorer with resource format
preservation
---
evaluation/lexbench-webfetch.json | 4 ++--
pyproject.toml | 2 +-
uv.lock | 4 ++--
3 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/evaluation/lexbench-webfetch.json b/evaluation/lexbench-webfetch.json
index cc690d46e..5f2e204dc 100644
--- a/evaluation/lexbench-webfetch.json
+++ b/evaluation/lexbench-webfetch.json
@@ -2,7 +2,7 @@
"schema_version": 1,
"kind": "external_benchmark_binding",
"repository": "https://github.com/lexmount/lexbench-webfetch",
- "revision": "165947c2a9f269a55810de0086fa86aadb749598",
+ "revision": "5b12816906ad8cd2b55b9804c77e7e6c55f2c54e",
"ownership": {
"benchmark": "Task populations, task model, acceptance scoring, file checks, compiler and frozen definitions",
"harness": "Execution, reference lifecycle, evidence and reports"
@@ -78,7 +78,7 @@
"evaluation/risk-control/remaining-70-rule-review.md": "301574dddcfddf6d30ed4d882cdd40b1ae859fdc95a0a02cf9868f55e3fd1079",
"evaluation/risk-control/rule-authoring-guide.md": "74363ab502978179e5438a0b992bac7563090e41c91784e492636832dc1cd088",
"evaluation/risk-control/source-inputs.json": "77c5a4b62c07d59ed57154a833e85f9e7a97bc739730f51cd4ca3a698edb9258",
- "evaluation/scoring-regressions.json": "e5f610d2adfaa6a7fd8d80af3696033c4ccd1a46038d49f7825ebf9dfdcbdcae",
+ "evaluation/scoring-regressions.json": "f2e8f96001b7a27da22877676625b4c1514b3ffa26f97601623b46582c240a93",
"evaluation/source-inputs.json": "c202abc4624811b938a837402bbaf0ecbe716e375fbca70567346d2edb3b1708"
}
}
diff --git a/pyproject.toml b/pyproject.toml
index ca38b813e..3861cc8e0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -6,7 +6,7 @@ readme = "README.md"
license = {file = "LICENSE"}
requires-python = ">=3.11"
dependencies = [
- "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@165947c2a9f269a55810de0086fa86aadb749598",
+ "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@5b12816906ad8cd2b55b9804c77e7e6c55f2c54e",
"beautifulsoup4>=4.12,<5",
"fastcdc==1.7.0",
]
diff --git a/uv.lock b/uv.lock
index 2843b2a2f..1f7bd9233 100644
--- a/uv.lock
+++ b/uv.lock
@@ -354,7 +354,7 @@ requires-dist = [
{ name = "jieba", marker = "extra == 'webmainbench-eval'", specifier = "==0.42.1" },
{ name = "json-repair", marker = "extra == 'browser-use-eval'", specifier = ">=0.54,<1" },
{ name = "langchain-community", marker = "extra == 'oss-eval'", specifier = "==0.3.30" },
- { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=165947c2a9f269a55810de0086fa86aadb749598" },
+ { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=5b12816906ad8cd2b55b9804c77e7e6c55f2c54e" },
{ name = "markdown", marker = "extra == 'webmainbench-eval'", specifier = ">=3.7,<4" },
{ name = "openai", marker = "extra == 'webmainbench-eval'", specifier = ">=2.8,<4" },
{ name = "openpyxl", marker = "extra == 'oss-eval'", specifier = "==3.1.5" },
@@ -1933,7 +1933,7 @@ wheels = [
[[package]]
name = "lexbench-webfetch"
version = "0.3.0"
-source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=165947c2a9f269a55810de0086fa86aadb749598#165947c2a9f269a55810de0086fa86aadb749598" }
+source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=5b12816906ad8cd2b55b9804c77e7e6c55f2c54e#5b12816906ad8cd2b55b9804c77e7e6c55f2c54e" }
dependencies = [
{ name = "beautifulsoup4" },
{ name = "soupsieve" },
From 695f97c30062e9aa62d3ed3d5840bca2a121fc0b Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Wed, 21 Oct 2026 00:05:28 +0800
Subject: [PATCH 11/51] test(webfetch): cover data-only report instance
boundaries
---
CONTRIBUTING.md | 3 ++-
tests/test_external_local_report.py | 19 +++++++++++++------
2 files changed, 15 insertions(+), 7 deletions(-)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 8afceab95..a31daabae 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -14,6 +14,7 @@ Include the task ID, method and versions, OS, command and sanitized error. Remov
| LexBench Web Fetch task requirements | [Benchmark repository](https://github.com/lexmount/lexbench-webfetch), applied to every method |
| Scoring fix | Shared scorer, scenario regressions and a recorded correction reason |
| New browser, version or sample size | Configuration or registry; reuse the runner and renderer |
+| New report instance | The existing track's profile, external evidence and report content; reuse its build and release commands |
| Reusable logic | `harness/`; `scripts/` stays a thin command entry point |
Test both accepted and rejected scenarios. Fix scoring rules rather than editing saved verdicts. Refactoring must preserve historical measurements and certificates.
@@ -42,7 +43,7 @@ Write the summary in English, start with an imperative verb, and describe the co
Examples:
- `fix(scoring): require facts in the returned body`
-- `fix(webfetch): reject mixed executable versions before release`
+- `fix(webfetch): disclose build provenance for partial reruns`
- `docs(contributing): define PR and commit title conventions`
Avoid titles such as `Fix review feedback`, `Update code` or `Misc improvements`; name the behavior or document that changes. A PR title describes the complete diff; each commit subject describes that commit. Update the PR title when its scope changes. If squash merging is used, use a subject that follows the same format. Apply this convention to new work; do not rewrite shared commits solely to rename them.
diff --git a/tests/test_external_local_report.py b/tests/test_external_local_report.py
index 3fc4405aa..29fa6c24b 100644
--- a/tests/test_external_local_report.py
+++ b/tests/test_external_local_report.py
@@ -19,9 +19,15 @@ def write(path, value):
return hashlib.sha256(path.read_bytes()).hexdigest()
-@pytest.mark.parametrize("case_count,method_count,timeout", [(5, 2, 18), (12, 3, 30)])
+@pytest.mark.parametrize("case_count,method_count,timeout,failures", [
+ (1, 1, 7, True),
+ (1, 1, 7, False),
+ (5, 2, 18, True),
+ (12, 3, 30, True),
+ (7, 4, 45, True),
+])
def test_external_matrix_through_build_and_check_clis(tmp_path, monkeypatch,
- case_count, method_count, timeout):
+ case_count, method_count, timeout, failures):
root = Path(__file__).resolve().parents[1]
evidence = tmp_path / "evidence"
methods = [f"synthetic-{n}" for n in range(method_count)]
@@ -42,9 +48,10 @@ def test_external_matrix_through_build_and_check_clis(tmp_path, monkeypatch,
"source_reverified": False}}
frozen = evidence / "local-evidence/input-dataset.json"
dataset_digest = write(frozen, dataset)
- config = {name: {"type": "fixture", "fail_cases": ["task-0"]} for name in methods}
- # Keep one default-only method and one explicit public control. Private
- # configuration fields must never be copied into the generated report.
+ config = {name: {"type": "fixture", "fail_cases": ["task-0"] if failures else []}
+ for name in methods}
+ # Keep a public control on the first method and defaults on additional methods.
+ # Private configuration fields must never enter the generated report.
config[methods[0]].update(max_bytes=12345, api_key="synthetic-private-credential")
config_path = evidence / "providers.json"
config_digest = write(config_path, {"providers": config})
@@ -95,7 +102,7 @@ def test_external_matrix_through_build_and_check_clis(tmp_path, monkeypatch,
options = load_frozen_method_options(output, saved)
assert options["schema_version"] == 2
assert options["providers"][methods[0]] == {"max_bytes": 12345}
- assert options["providers"][methods[-1]] == {}
+ assert all(options["providers"][name] == {} for name in methods[1:])
assert options["result_count"] == len(saved)
assert options["provider_config_sha256"] == config_digest
projected = (output / options["source_config"]["path"]).read_bytes()
From 95a42ce12d0e343bff70a197e61bb29966928bef Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Wed, 21 Oct 2026 11:26:45 +0800
Subject: [PATCH 12/51] fix(reporting): support real-data instances across
benchmark tracks
---
Makefile | 16 +-
configs/report-templates.json | 19 +-
docs/benchmarks.md | 77 +++++
harness/browser_use/failure_cause_review.py | 14 +
harness/browser_use/report.py | 282 ++++++++---------
harness/browser_use/report_check.py | 47 +--
harness/browser_use/report_i18n.py | 11 +
harness/browser_use/report_package.py | 31 +-
harness/browser_use/selection.py | 34 ++-
harness/reporting/templates.py | 6 +
.../trajectory_replay/report_compaction.py | 283 +++++++++++-------
harness/trajectory_replay/webarena_report.py | 111 +++++--
.../webarena_report_check.py | 30 +-
.../webarena_report_package.py | 26 +-
.../webarena_report_visual.py | 7 +-
harness/web_fetch/wcxb_publication.py | 8 +
harness/web_fetch/wcxb_report_check.py | 15 +-
tests/browser_use/test_report.py | 14 +-
tests/browser_use/test_report_check.py | 28 +-
tests/browser_use/test_selection.py | 24 ++
tests/test_reporting_publication.py | 1 +
tests/test_wcxb_publication.py | 12 +
tests/test_wcxb_report_check.py | 20 ++
.../test_report_compaction.py | 44 +++
.../test_webarena_publish.py | 6 +-
.../trajectory_replay/test_webarena_report.py | 35 ++-
26 files changed, 827 insertions(+), 374 deletions(-)
diff --git a/Makefile b/Makefile
index b8f4e7cc2..3bed2da50 100644
--- a/Makefile
+++ b/Makefile
@@ -108,39 +108,39 @@ wcxb-release-check: runtime-check
browser-use-report: runtime-check
@test -n "$(EVIDENCE)" || (echo 'EVIDENCE="/absolute/path [/absolute/path...]" is required' >&2; exit 2)
- $(PY) scripts/build_browser_use_report.py $(foreach path,$(EVIDENCE),--evidence "$(path)") --output reports/browser-use $(if $(REPORT_LANGUAGE),--language "$(REPORT_LANGUAGE)",) $(if $(filter 1,$(ENGLISH)),--include-english,)
+ $(PY) scripts/build_browser_use_report.py $(foreach path,$(EVIDENCE),--evidence "$(path)") --output "$(or $(OUTPUT),reports/browser-use)" $(if $(PROFILE),--profile "$(PROFILE)",) $(if $(REPORT_LANGUAGE),--language "$(REPORT_LANGUAGE)",) $(if $(filter 1,$(ENGLISH)),--include-english,)
browser-use-review: runtime-check
@test -n "$(EVIDENCE)" || (echo 'EVIDENCE=/absolute/path is required' >&2; exit 2)
$(PY) scripts/review_browser_use_results.py --evidence "$(EVIDENCE)"
browser-use-visual-check: runtime-check
- $(PY) scripts/check_browser_use_visual.py reports/browser-use
+ $(PY) scripts/check_browser_use_visual.py "$(or $(OUTPUT),reports/browser-use)" $(if $(VISUAL_RECEIPT),--receipt "$(VISUAL_RECEIPT)",)
browser-use-report-check: runtime-check
@test -n "$(EVIDENCE)" || (echo 'EVIDENCE="/absolute/path [/absolute/path...]" is required' >&2; exit 2)
- $(PY) scripts/check_browser_use_report.py reports/browser-use $(foreach path,$(EVIDENCE),--evidence "$(path)") --require-complete
- $(PY) scripts/check_report_links.py reports/browser-use
+ $(PY) scripts/check_browser_use_report.py "$(or $(OUTPUT),reports/browser-use)" $(if $(PROFILE),--profile "$(PROFILE)",) $(foreach path,$(EVIDENCE),--evidence "$(path)") --require-complete
+ $(PY) scripts/check_report_links.py "$(or $(OUTPUT),reports/browser-use)"
browser-use-release-check: browser-use-report-check
$(MAKE) browser-use-visual-check
trajectory-replay-report: runtime-check
@test -n "$(EVIDENCE)" || (echo 'EVIDENCE=/absolute/path is required' >&2; exit 2)
- $(PY) scripts/build_webarena_replay_report.py --results "$(EVIDENCE)" $(if $(REPORT_LANGUAGE),--language "$(REPORT_LANGUAGE)",) $(if $(filter 1,$(ENGLISH)),--include-english,)
+ $(PY) scripts/build_webarena_replay_report.py --results "$(EVIDENCE)" --output "$(or $(OUTPUT),reports/trajectory-replay)" $(if $(PROFILE),--profile "$(PROFILE)",) $(if $(REVIEW),--review "$(REVIEW)",) $(if $(REPORT_LANGUAGE),--language "$(REPORT_LANGUAGE)",) $(if $(filter 1,$(ENGLISH)),--include-english,)
trajectory-replay-report-check: runtime-check
- $(PY) scripts/check_webarena_replay_report.py reports/trajectory-replay
+ $(PY) scripts/check_webarena_replay_report.py "$(or $(OUTPUT),reports/trajectory-replay)" $(if $(PROFILE),--profile "$(PROFILE)",)
trajectory-replay-visual-check: runtime-check
- $(PY) scripts/check_webarena_replay_visual.py reports/trajectory-replay
+ $(PY) scripts/check_webarena_replay_visual.py "$(or $(OUTPUT),reports/trajectory-replay)" $(if $(VISUAL_RECEIPT),--receipt "$(VISUAL_RECEIPT)",)
trajectory-replay-release-check: trajectory-replay-report-check
$(MAKE) trajectory-replay-visual-check
trajectory-replay-package: runtime-check
@test -n "$(EVIDENCE)" || (echo 'EVIDENCE=/absolute/path/effective-results.json is required' >&2; exit 2)
- $(PY) scripts/package_webarena_replay_artifact_site.py --evidence "$(EVIDENCE)"
+ $(PY) scripts/package_webarena_replay_artifact_site.py --evidence "$(EVIDENCE)" --report "$(or $(OUTPUT),reports/trajectory-replay)" $(if $(PROFILE),--profile "$(PROFILE)",) $(if $(REVIEW),--review "$(REVIEW)",) $(if $(ARCHIVE),--output "$(ARCHIVE)",) $(if $(ATTESTATION),--attestation "$(ATTESTATION)",) $(if $(VISUAL_RECEIPT),--visual-receipt "$(VISUAL_RECEIPT)",)
trajectory-replay-publish: runtime-check
$(PY) scripts/publish_webarena_replay_artifact_site.py
diff --git a/configs/report-templates.json b/configs/report-templates.json
index 10636a809..d09bfec6b 100644
--- a/configs/report-templates.json
+++ b/configs/report-templates.json
@@ -12,6 +12,7 @@
"wcxb_profile": "evaluation/wcxb/current-report.json",
"wcxb_build": "uv run --frozen python scripts/build_report.py --profile evaluation/wcxb/current-report.json --evidence /absolute/path --output reports/wcxb",
"wcxb_check": "make wcxb-release-check",
+ "wcxb_package": "uv run --frozen python scripts/publish_wcxb_artifact_site.py --package-only --source /absolute/new-report --work /absolute/package",
"tests": "tests/test_headless_report.py",
"variants": ["certified", "wcxb", "local-audit", "diagnostic-cohorts"],
"diagnostic_build": "make webfetch-diagnostic-report DIAGNOSTIC_PROFILE=evaluation/live-webfetch/current-report.json OUTPUT=/absolute/new-output",
@@ -23,18 +24,22 @@
"trajectory-replay": {
"renderer": "harness/trajectory_replay/webarena_report.py",
"entrypoint": "scripts/build_webarena_replay_report.py",
- "profile": "evaluation/trajectory-replay/webarena-verified-full/current-report.json",
- "build": "make trajectory-replay-report EVIDENCE=/absolute/path/results.json",
- "check": "make trajectory-replay-release-check",
- "tests": "tests/trajectory_replay/test_webarena_report.py"
+ "profile": "evaluation/trajectory-replay/webarena-verified/current-run.json",
+ "build": "uv run --frozen python scripts/build_webarena_replay_report.py --profile /absolute/instance-profile.json --results /absolute/results.json --review /absolute/review.json --output /absolute/new-report",
+ "check": "uv run --frozen python scripts/check_webarena_replay_report.py /absolute/new-report --profile /absolute/instance-profile.json",
+ "tests": "tests/trajectory_replay/test_webarena_report.py",
+ "example_profile": "evaluation/trajectory-replay/webarena-verified-full/current-comparison.json",
+ "example": "https://artifact-site.lexmount.cn/s/EFLrNgujgsVc?version=ver_c0f0693e9c08401bb4d2b88393234e55",
+ "package": "uv run --frozen python scripts/package_webarena_replay_artifact_site.py --report /absolute/new-report --profile /absolute/instance-profile.json --evidence /absolute/results.json --review /absolute/review.json --output /absolute/package.zip --attestation /absolute/attestation.json --visual-receipt /absolute/visual.json"
},
"browser-use": {
"renderer": "harness/browser_use/report.py",
"entrypoint": "scripts/build_browser_use_report.py",
"profile": "evaluation/browser-use/current-report.json",
- "build": "make browser-use-report EVIDENCE=/absolute/path",
- "check": "make browser-use-release-check EVIDENCE=/absolute/path",
- "tests": "tests/browser_use/test_report.py"
+ "build": "uv run --frozen python scripts/build_browser_use_report.py --profile /absolute/instance-profile.json --evidence /absolute/evidence --output /absolute/new-report",
+ "check": "uv run --frozen python scripts/check_browser_use_report.py /absolute/new-report --profile /absolute/instance-profile.json --evidence /absolute/evidence --require-complete",
+ "tests": "tests/browser_use/test_report.py",
+ "package": "uv run --frozen python scripts/package_browser_use_artifact_hub.py --report /absolute/new-report --profile /absolute/instance-profile.json --evidence /absolute/evidence --output /absolute/package.zip --attestation /absolute/attestation.json --visual-receipt /absolute/visual.json"
}
}
}
diff --git a/docs/benchmarks.md b/docs/benchmarks.md
index ca442172a..0e28ad562 100644
--- a/docs/benchmarks.md
+++ b/docs/benchmarks.md
@@ -50,3 +50,80 @@ The four initial LexBench suites preserve the original catalogs and source-input
Change task requirements in LexBench, freeze a new dataset/catalog/manifest version there, and select that version in a new experiment profile. Fix execution defects in Browser Eval and scoring defects in LexBench; keep accepted/rejected scenario controls with their owner. Preserve prior measurements and apply corrections across the declared affected scope.
The retained `harness/lexbench/` module concerns historical headless-browser comparisons. It is unrelated to the `lexbench-webfetch` input package and remains an execution/report adapter.
+
+## Report instances
+
+Each report track uses its maintained build, check and package entry points.
+Browser Use and Browser Trajectory Replay accept an instance profile at each boundary. Keep the profile and measured evidence outside
+Git. Select a new output directory; changing participants, sample size or report
+content does not require another renderer or command script.
+
+Browser Use profiles register models, browser labels and measurement controls.
+`task_selection.path` and `task_selection.sha256` bind the selection file; its hash
+must match `selection_sha256`. The selection declares its own `metadata.tier_counts`.
+`official_dataset.path` and `official_dataset.sha256` can bind the exact historical
+official task file, matching `official_data_checksum` and the recorded calls.
+This allows rebuilding an older measurement after the installed benchmark has
+changed. `failure_cause_review.source_path` may name an external, hash-bound review.
+Keep original attempts and task contracts unchanged.
+
+```sh
+uv run --frozen python scripts/build_browser_use_report.py \
+ --profile /absolute/instance/profile.json \
+ --evidence /absolute/model-evidence --output /absolute/instance/report
+uv run --frozen python scripts/check_browser_use_report.py \
+ /absolute/instance/report --profile /absolute/instance/profile.json \
+ --evidence /absolute/model-evidence --require-complete
+uv run --frozen python scripts/package_browser_use_artifact_hub.py \
+ --report /absolute/instance/report --profile /absolute/instance/profile.json \
+ --evidence /absolute/model-evidence --output /absolute/instance/report.zip \
+ --attestation /absolute/instance/attestation.json \
+ --visual-receipt /absolute/instance/visual.json
+```
+
+Repeat `--evidence` for additional model cohorts. The renderer derives comparison
+counts and failure examples from the selected results. Missing task-type or
+execution metadata stays unavailable; it is never inferred from historical IDs.
+
+Trajectory Replay profiles bind the dataset, participant registry, execution
+protocol and reference validity. The failure review must bind the exact saved
+results. Without an authored content binding, the maintained renderer generates
+an interpretation from the current results, including complete failure
+distributions and specimen links. An optional authored interpretation uses
+`--content`, `--content-bindings` and `--interpretation-audit`; these inputs must
+belong to the selected evidence.
+
+```sh
+uv run --frozen python scripts/build_webarena_replay_report.py \
+ --profile /absolute/instance/profile.json --results /absolute/results.json \
+ --review /absolute/failure-review.json --output /absolute/instance/report
+uv run --frozen python scripts/check_webarena_replay_report.py \
+ /absolute/instance/report --profile /absolute/instance/profile.json
+uv run --frozen python scripts/package_webarena_replay_artifact_site.py \
+ --report /absolute/instance/report --profile /absolute/instance/profile.json \
+ --evidence /absolute/results.json --review /absolute/failure-review.json \
+ --output /absolute/instance/report.zip \
+ --attestation /absolute/instance/attestation.json \
+ --visual-receipt /absolute/instance/visual.json
+```
+
+Both packers run integrity, link and browser checks before installing an archive.
+The replay packer automatically uses the existing compact representation above
+its file-count limit. It verifies every task/result page and evidence payload
+against the uncompressed report, then exercises exported routes in every saved
+language. Compression preserves all logical results and physical-call evidence.
+
+The Make targets accept `PROFILE`, `OUTPUT` and `VISUAL_RECEIPT` for these tracks;
+replay build/package targets also accept `REVIEW`, and packaging accepts `ARCHIVE`
+and `ATTESTATION`. Packaging creates a local archive and does not publish it.
+
+For WCXB, select a profile-bound evidence directory and a fresh output with
+`scripts/build_report.py --profile evaluation/wcxb/current-report.json`. Run
+`scripts/check_report.py` with that profile and `scripts/check_wcxb_visual.py`
+against the output. The existing publication entry point also supports local
+packaging without credentials or a network upload:
+
+```sh
+uv run --frozen python scripts/publish_wcxb_artifact_site.py --package-only \
+ --source /absolute/instance/report --work /absolute/instance/package
+```
diff --git a/harness/browser_use/failure_cause_review.py b/harness/browser_use/failure_cause_review.py
index 0649067a8..03e26cfc4 100644
--- a/harness/browser_use/failure_cause_review.py
+++ b/harness/browser_use/failure_cause_review.py
@@ -17,6 +17,20 @@ def sha256(path: Path) -> str | None:
return hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else None
+def review_path(profile: dict[str, Any], *, authority_root: Path | None = None) -> Path:
+ """Resolve an explicitly bound external review or a repository-relative review."""
+ from lexbench_webfetch.paths import contained_path
+
+ from harness.web_fetch.safety import unlinked_path
+ value = (profile.get("failure_cause_review") or {}).get("source_path")
+ if value is None:
+ value = "research/browser-use-failure-cause-review.json"
+ path = Path(value)
+ if path.is_absolute():
+ return unlinked_path(path)
+ return contained_path(authority_root or Path(__file__).parents[2], value, reject_symlinks=True)
+
+
def load_failure_cause_review(path: Path = DEFAULT_REVIEW_PATH) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {}
diff --git a/harness/browser_use/report.py b/harness/browser_use/report.py
index ebcdfbeab..7bc3a02ea 100644
--- a/harness/browser_use/report.py
+++ b/harness/browser_use/report.py
@@ -20,11 +20,9 @@
from harness.record_io import load_jsonl
from harness.report_languages import write_language_contract
+from harness.reporting.templates import template_identity
from harness.web_fetch.report_style import STYLE as SHARED_STYLE
-from .failure_cause_review import (
- DEFAULT_REVIEW_PATH as FAILURE_CAUSE_REVIEW_PATH,
-)
from .failure_cause_review import (
LAYERED_REVIEW_SCHEMA,
extract_failure_cause_review_cohort,
@@ -32,6 +30,9 @@
localized_cause,
validate_failure_cause_review,
)
+from .failure_cause_review import (
+ review_path as failure_review_path,
+)
from .failure_cause_review import (
sha256 as file_sha256,
)
@@ -45,7 +46,7 @@
trajectory,
trajectory_label,
)
-from .selection import PilotSelection, load_pilot, validate_against_official
+from .selection import PilotSelection, load_pilot, report_selection_path, validate_against_official
ENGINE_LABELS = {"chromium": "Chromium", "moli": "Moli 1.1.3"}
MODEL_LABELS = {"gpt-5.6-sol": "GPT-5.6 Sol"}
@@ -57,8 +58,6 @@
"reddit": "Reddit",
"gitlab": "GitLab",
}
-MUTATE_IDS = {399, 404, 441, 506}
-NAVIGATE_IDS = {44, 45, 118}
RESEARCH_ROOT = Path(__file__).parents[2] / "research"
@@ -301,12 +300,6 @@ def _table(
return f'
{cap}
{head}
{body}
'
-def _task_type(task_id: int) -> str:
- return (
- "MUTATE" if task_id in MUTATE_IDS else "NAVIGATE" if task_id in NAVIGATE_IDS else "RETRIEVE"
- )
-
-
def _attempt_id(row: dict[str, Any]) -> str:
model = str(row.get("model") or (next(iter(MODEL_LABELS)) if len(MODEL_LABELS) == 1 else ""))
if not model:
@@ -663,7 +656,8 @@ def _paired_summary(rows: list[dict[str, Any]]) -> dict[str, Any]:
def _resolve_tasks(
- selection: PilotSelection, *, expected_data_checksum: str | None = None
+ selection: PilotSelection, *, expected_data_checksum: str | None = None,
+ dataset_path: Path | None = None
) -> list[dict[str, Any]]:
try:
from harness.environments.webarena.runtime import (
@@ -671,6 +665,12 @@ def _resolve_tasks(
)
benchmark = configure_benchmark()
+ if dataset_path is not None:
+ from webarena_verified import WebArenaVerified
+ if dataset_path.is_symlink() or not dataset_path.is_file():
+ raise ValueError("Frozen official dataset is missing or symlinked")
+ benchmark = WebArenaVerified(config=benchmark.config.model_copy(
+ update={"test_data_file": dataset_path}))
if expected_data_checksum:
dataset_path = Path(benchmark.config.test_data_file)
actual_data_checksum = _sha256(dataset_path)
@@ -713,7 +713,7 @@ def _resolve_tasks(
raise ValueError(f"official task {task['task_id']} has no evaluator contract")
expected = (evals[0].get("expected") or {}) if evals else {}
task["task_type"] = str(
- expected.get("task_type") or _task_type(int(task["task_id"]))
+ expected.get("task_type") or "UNKNOWN"
).upper()
return tasks
@@ -801,7 +801,7 @@ def _failure_cause_table(verified_causes: dict[str, dict[str, Any]]) -> str:
"benchmark": (
"任务 / evaluator",
"排除并修复",
- "修正验收合同,通过独立预检后重跑四种配置。",
+ "修正验收合同,通过独立预检后重跑全部参评配置。",
),
"site": ("自托管站点", "排除并修复", "修复依赖、会话或保存链路,通过读写预检后重跑。"),
"evidence": (
@@ -1127,12 +1127,12 @@ def _summary_comparison_rows(summary: dict[str, dict[str, dict[str, Any]]]) -> l
_winner(
f"{current['passed']}/{current['completed']}",
won=key in higher_metrics["passed"],
- label="四种配置中通过数最高",
+ label="全部配置中通过数最高",
),
_winner(
_pct(current["score"]),
won=key in higher_metrics["score"],
- label="四种配置中通过率最高",
+ label="全部配置中通过率最高",
),
_metric_stack(
[
@@ -2357,28 +2357,6 @@ def _automatic_failure_figure(counts: Counter[str]) -> str:
return '
' + cards + "
"
-def _resource_summary_cards(common_resources: tuple[int, list[list[Any]]]) -> str:
- task_count, rows = common_resources
- if not task_count:
- return '
'
+ chapters = []
+ for model, engine in sorted(configurations):
+ failures = [row for row in failed if row['model'] == model and row['engine'] == engine]
+ if not failures:
+ continue
+ ids = {_attempt_id(row) for row in failures}
+ causes = {key: value for key, value in verified_causes.items() if key in ids}
+ label = f'{MODEL_LABELS[model]} × {ENGINE_LABELS[engine]}'
+ specimen = next(iter(causes), _attempt_id(failures[0]))
+ cause = causes.get(specimen) or {}
+ reason = cause.get('reason') or {}
+ explanation = text(str(reason.get('zh') or '此结果尚无已核验根因。'),
+ str(reason.get('en') or 'This result has no verified root cause.'))
+ chapters.append(
+ f'
{_e(label)}
'
+ + text(f'{len(failures)} 个失败结果,其中 {len(causes)} 个已有根因复核。',
+ f'{len(failures)} failed results; {len(causes)} have a verified cause.')
+ + _failure_cause_table(causes)
+ + f'
'
+ + text('最低值标记只描述各配置的实际运行条件,不能视为统一视口下的资源效率结论。',
+ 'Minimum markers describe the recorded configuration; they do not establish resource efficiency at a common viewport.')
+ + '
共同成功任务的性能与资源
'
+ + text(f'{common_count} 个任务在全部配置中最终通过;下表以每个逻辑结果的全部物理执行计算 P50。',
+ f'All configurations finally passed {common_count} tasks. The table computes P50 over all physical executions within each logical result.')
+ + _table(['模型', '浏览器', 'Agent 链路 P50', 'BrowserTool P50', 'Tokens P50', '峰值内存 P50', 'CPU 时间 P50'], resource_rows,
+ table_id='home-resource-table', classes='core-comparison-table')
+ + text('共同成功子集仍保留各配置记录的运行环境,不能据此推断统一条件下的资源效率。', 'The common-success subset retains each configuration’s recorded environment; it does not establish resource efficiency under identical conditions.')
+ + ''
)
@@ -2702,7 +2669,7 @@ def _attempt_table(
cells = "".join(f"
' + bilingual_text('首页比较全部配置的通过率、耗时、Tokens 和浏览器资源;本页提供同题矩阵与逐结果诊断。', 'The homepage compares pass rates, latency, tokens and browser resources across all configurations. This page owns the same-task matrix and result diagnosis.') + '
' + bilingual_text('账号隔离用于避免用户级对象冲突,不能替代环境隔离。已核验的外部干扰与诊断子集见首页。', 'Account isolation prevents user-object collisions; it does not replace environment isolation. The homepage identifies verified external interference and its diagnostic subset.') + '
',
)
reproduce += _section(
"公开复现入口",
@@ -3769,7 +3748,7 @@ def build_report(*, evidence: Path | Iterable[Path], output: Path, selection_pat
["范围", "本轮做法", "外部付费账号"],
[
["四个自托管站点", "官方 Docker 镜像与内置测试账号;每题前重置", "不需要"],
- ["模型 API", f"{model},reasoning=low", "需要相应 API 权限"],
+ ["模型 API", f"{model}, reasoning={reasoning_effort}", "需要相应 API 权限"],
["Artifact Hub", "只发布脱敏静态报告", "发布者需已有站点权限"],
],
),
@@ -3878,6 +3857,8 @@ def build_report(*, evidence: Path | Iterable[Path], output: Path, selection_pat
)
audit = {
"schema": "lex_browser_use_report/3",
+ "profile_sha256": _sha256(profile_path),
+ "template": template_identity("browser-use"),
"result_count": len(attempts),
"score_authority": {
bundle["model"]: (
@@ -3909,7 +3890,7 @@ def build_report(*, evidence: Path | Iterable[Path], output: Path, selection_pat
model_id: review_value.get("bindings") or {}
for model_id, review_value in failure_reviews.items()
},
- "source_sha256": file_sha256(FAILURE_CAUSE_REVIEW_PATH),
+ "source_sha256": file_sha256(failure_review_path(profile)),
},
"review": {
"coverage": review.get("coverage"),
@@ -3940,7 +3921,7 @@ def build_report(*, evidence: Path | Iterable[Path], output: Path, selection_pat
"models": list(MODEL_LABELS),
"model_labels": MODEL_LABELS,
"baseline_engine": BASELINE_ENGINE,
- "max_task_executions": 2,
+ "max_task_executions": protocol.get("max_task_executions") or profile.get("max_task_executions"),
"environment_reset": (
"official site containers reset before every execution; "
f"{preparation_receipt_count}/{len(source_attempts)} execution-scoped "
@@ -4072,16 +4053,19 @@ def main() -> None:
help="isolated evidence directory for one model; repeat once per model",
)
parser.add_argument("--output", required=True, type=Path)
+ parser.add_argument("--profile", type=Path)
parser.add_argument(
"--selection",
type=Path,
- default=Path("evaluation/browser-use/webarena-verified-pilot.json"),
+ default=None,
)
from harness.report_languages import add_language_argument, primary_report_language
add_language_argument(parser)
parser.add_argument("--include-english", action="store_true", help="Generate both editions; --language selects the homepage")
args = parser.parse_args()
- print(build_report(evidence=args.evidence, output=args.output, selection_path=args.selection, include_english=args.include_english, language=args.language or primary_report_language(args.output)))
+ configuration = _read_json(args.profile or Path("evaluation/browser-use/current-report.json"))
+ selection_path = report_selection_path(configuration, args.selection)
+ print(build_report(evidence=args.evidence, output=args.output, selection_path=selection_path, include_english=args.include_english, language=args.language or primary_report_language(args.output), profile_path=args.profile))
if __name__ == "__main__":
diff --git a/harness/browser_use/report_check.py b/harness/browser_use/report_check.py
index 9d30091fb..11c144c3d 100644
--- a/harness/browser_use/report_check.py
+++ b/harness/browser_use/report_check.py
@@ -14,6 +14,9 @@
load_failure_cause_review,
validate_failure_cause_review,
)
+from harness.browser_use.failure_cause_review import (
+ review_path as failure_review_path,
+)
from harness.browser_use.failure_cause_review import (
sha256 as file_sha256,
)
@@ -28,10 +31,11 @@
_public_attempt,
_public_execution,
)
-from harness.browser_use.selection import load_pilot
+from harness.browser_use.selection import load_pilot, report_selection_path
from harness.record_io import load_jsonl
from harness.report_languages import check_language_contract, generated_languages
from harness.reporting.presentation import visible_text as _visible_text
+from harness.reporting.templates import template_identity
from harness.web_fetch.safety import reject_evidence_symlinks
FORBIDDEN_READER_TERMS = (
@@ -308,14 +312,17 @@ def check(
*,
require_complete: bool,
authority_root: Path | None = None,
+ profile_path: Path | None = None,
) -> None:
reject_evidence_symlinks(report_dir)
for evidence in [evidence_dir] if isinstance(evidence_dir, Path) else evidence_dir:
reject_evidence_symlinks(evidence)
repository = authority_root if authority_root is not None else Path(__file__).parents[2]
- profile_path = contained_path(
+ profile_path = profile_path or contained_path(
repository, "evaluation/browser-use/current-report.json", reject_symlinks=True
)
+ if profile_path.is_symlink():
+ raise ValueError("Symlinked report profile is forbidden")
if not profile_path.is_file():
raise ValueError(
"Browser Use release authority is missing; pass --authority-root "
@@ -324,20 +331,18 @@ def check(
profile = json.loads(profile_path.read_text(encoding="utf-8"))
if profile.get("schema") != "lex_browser_use_current_report/2":
raise ValueError("unsupported Browser Use release authority schema")
- if selection_path is None:
- selection_path = contained_path(
- repository,
- "evaluation/browser-use/webarena-verified-pilot.json",
- reject_symlinks=True,
- )
+ frozen = profile.get("official_dataset")
+ if frozen:
+ dataset_path = Path(frozen["path"])
+ if (dataset_path.is_symlink() or not dataset_path.is_file()
+ or _sha256(dataset_path) != frozen.get("sha256")
+ or frozen.get("sha256") != profile.get("official_data_checksum")):
+ raise ValueError("Frozen official dataset differs from the instance profile")
+ selection_path = report_selection_path(profile, selection_path, authority_root=repository)
if selection_path.is_symlink():
raise ValueError("Symlinked selection is forbidden")
cause_profile = profile.get("failure_cause_review") or {}
- cause_path = contained_path(
- repository,
- cause_profile.get("source_path") or "research/browser-use-failure-cause-review.json",
- reject_symlinks=True,
- )
+ cause_path = failure_review_path(profile, authority_root=repository)
if require_complete and cause_profile.get("required_for_publication"):
configured_hash = cause_profile.get("source_sha256")
if not isinstance(configured_hash, str) or configured_hash != file_sha256(cause_path):
@@ -424,6 +429,10 @@ def check(
raise ValueError("report is missing one or more required report pages")
selection = load_pilot(selection_path)
audit = json.loads(audit_path.read_text(encoding="utf-8"))
+ if audit.get("profile_sha256") and audit.get("template") != template_identity("browser-use"):
+ raise ValueError("report bypasses the maintained Browser Use template")
+ if audit.get("profile_sha256") and audit["profile_sha256"] != _sha256(profile_path):
+ raise ValueError("report was built from a different instance profile")
if audit.get("selection_sha256") != selection.sha256:
raise ValueError("report was not built from the current pre-registered selection")
rows = measured_rows
@@ -534,7 +543,7 @@ def check(
analysis_index = rendered_pages[report_dir / "analysis/index.html"]
for marker in (
"comparison-winner",
- "完整 90 题结果入口",
+ f"完整 {len(selection.tasks)} 题结果入口",
"BrowserTool 动作延迟",
"action-card-grid",
"全部配置 · 单次调用墙钟时间",
@@ -549,14 +558,14 @@ def check(
guide_index = rendered_pages[report_dir / "index.html"]
for marker in (
- "90 题核心指标",
+ f"{len(selection.tasks)} 题核心指标",
"Agent 总耗时",
"BrowserTool 耗时",
"均值",
"P50",
"P95",
"comparison-winner",
- "四种完整配置中统一比较",
+ f"全部 {len(models) * len(engines)} 种配置中比较",
"已通过结果没有再做独立语义盲审",
"程序判分落点与复核根因",
'class="verdict-strip"',
@@ -564,7 +573,6 @@ def check(
'id="home-qualified-results"',
'class="audit-ladder"',
'id="home-resource-table"',
- "results/gpt-6-astra-chromium-699.html#verification",
):
if marker not in guide_index:
raise ValueError(f"interpretation is missing an evidence-backed reading aid: {marker}")
@@ -869,7 +877,8 @@ def check(
external_prefixes = ("benchmark_", "site_", "environment_", "mixed_")
expected_quarantine = {
int(result_id.rsplit("-", 1)[-1])
- for result_id, diagnosis in verified_causes.items()
+ for cohort in default_cause_review.get("cohorts", {}).values()
+ for result_id, diagnosis in (cohort.get("results") or {}).items()
if str((diagnosis.get("subcategory") or {}).get("code") or "").startswith(
external_prefixes
)
@@ -907,6 +916,7 @@ def main() -> None:
help="Explicit selection; defaults to the bound authority root",
)
parser.add_argument("--authority-root", type=Path)
+ parser.add_argument("--profile", type=Path)
parser.add_argument("--require-complete", action="store_true")
args = parser.parse_args()
check(
@@ -915,6 +925,7 @@ def main() -> None:
args.selection,
require_complete=args.require_complete,
authority_root=args.authority_root,
+ profile_path=args.profile,
)
print("Browser Use report integrity check passed")
diff --git a/harness/browser_use/report_i18n.py b/harness/browser_use/report_i18n.py
index 49a4ef90f..6b52bfefb 100644
--- a/harness/browser_use/report_i18n.py
+++ b/harness/browser_use/report_i18n.py
@@ -28,6 +28,10 @@
TEXT = {
+ "全部参评配置在同一张表和任务矩阵中比较;失败结果可打开诊断和证据。": "All configurations appear in one comparison and task matrix; failed results open their diagnosis and evidence.",
+ "全部配置中通过数最高": "Highest pass count across all configurations",
+ "全部配置中通过率最高": "Highest pass rate across all configurations",
+ "修正验收合同,通过独立预检后重跑全部参评配置。": "Correct the acceptance contract, complete independent preflight checks, then rerun every configuration.",
"OpenAI、Anthropic、Google DeepMind、THUDM / ChatGLM、Browser Use。原版 Mind2Web、Online-Mind2Web 与 Mind2Web 2 的任务和评分器不同,不能合并到同一成绩表。": "OpenAI, Anthropic, Google DeepMind, THUDM / ChatGLM and Browser Use. Original Mind2Web, Online-Mind2Web and Mind2Web 2 use different tasks and evaluators; their scores cannot be combined in one table.",
"修复日志": "Fix log",
"配置": "Configuration",
@@ -2326,8 +2330,15 @@ def _header(source: PurePosixPath, english: PurePosixPath) -> str:
)
+def bilingual_text(zh: str, en: str) -> str:
+ """Keep dynamic interface text and its English form beside the same values."""
+ return f'{html.escape(zh)}'
+
+
def localize_html(page: str, source: PurePosixPath) -> str:
english = PurePosixPath("en") / source
+ page = re.sub(r'[^<]*',
+ lambda match: html.escape(html.unescape(match[1])), page)
page = page.replace('', '', 1)
page = re.sub(
r"(.*?) · Browser Use 评测",
diff --git a/harness/browser_use/report_package.py b/harness/browser_use/report_package.py
index 52aa5bfe9..d47233698 100644
--- a/harness/browser_use/report_package.py
+++ b/harness/browser_use/report_package.py
@@ -5,11 +5,14 @@
import argparse
import hashlib
+import json
from pathlib import Path
+from harness.browser_use.failure_cause_review import review_path
from harness.browser_use.report_check import check
from harness.browser_use.report_visual import check as check_visual
from harness.browser_use.report_visual import receipt_is_current
+from harness.browser_use.selection import report_selection_path
from harness.reporting.links import check as check_links
from harness.reporting.publication import package_archive
@@ -38,11 +41,12 @@ def tree_sha256(root: Path) -> str:
def validate_release(
report: Path, evidence: list[Path], visual_receipt: Path,
- *, authority_root: Path | None = None,
+ *, authority_root: Path | None = None, profile_path: Path | None = None,
) -> None:
"""Run every release gate before creating a publishable archive."""
- check(report, evidence, None, require_complete=True, authority_root=authority_root)
+ check(report, evidence, None, require_complete=True,
+ authority_root=authority_root, profile_path=profile_path)
check_links(report)
if not receipt_is_current(report, visual_receipt):
check_visual(report, visual_receipt)
@@ -51,11 +55,20 @@ def validate_release(
def package(
report: Path, evidence: list[Path], output: Path, visual_receipt: Path,
*, authority_root: Path | None = None, attestation_path: Path = ATTESTATION,
+ profile_path: Path | None = None,
) -> dict:
authority = authority_root if authority_root is not None else Path(__file__).parents[2]
- profile = authority / PROFILE
+ profile = profile_path or authority / PROFILE
names = ("manifest.json", "run-config.json", "attempts.jsonl", "result-review.json")
- inputs = [authority / "evaluation", authority / "research"]
+ inputs = [authority / "evaluation", authority / "research", profile]
+ profile_data = json.loads(profile.read_text())
+ inputs.append(report_selection_path(profile_data, authority_root=authority))
+ review = review_path(profile_data, authority_root=authority)
+ if review.is_file():
+ inputs.append(review)
+ frozen = profile_data.get("official_dataset")
+ if frozen:
+ inputs.append(Path(frozen["path"]))
inputs.extend(path / name for path in evidence for name in names)
def attest(archive: Path) -> dict:
@@ -73,7 +86,7 @@ def attest(archive: Path) -> dict:
return package_archive(
report, output, attestation_path, inputs=inputs,
validate=lambda: validate_release(report, evidence, visual_receipt,
- authority_root=authority),
+ authority_root=authority, profile_path=profile_path),
attest=attest,
excluded=(".generated-report",),
)
@@ -86,10 +99,14 @@ def main() -> None:
parser.add_argument("--output", type=Path, default=Path("publish/browser-use-sites.zip"))
parser.add_argument("--visual-receipt", type=Path, default=VISUAL_RECEIPT)
parser.add_argument("--authority-root", type=Path)
+ parser.add_argument("--profile", type=Path)
parser.add_argument("--attestation", type=Path, default=ATTESTATION)
args = parser.parse_args()
- package(args.report, args.evidence, args.output, args.visual_receipt,
- authority_root=args.authority_root, attestation_path=args.attestation)
+ package(
+ args.report, args.evidence, args.output, args.visual_receipt,
+ authority_root=args.authority_root, attestation_path=args.attestation,
+ profile_path=args.profile,
+ )
print(args.output)
diff --git a/harness/browser_use/selection.py b/harness/browser_use/selection.py
index b1e90004f..b98a225cf 100644
--- a/harness/browser_use/selection.py
+++ b/harness/browser_use/selection.py
@@ -223,10 +223,15 @@ def load_pilot(path: str | Path) -> PilotSelection:
if len(ids) != len(set(ids)):
raise ValueError("pilot task IDs must be unique")
counts = {tier: sum(task.tier == tier for task in tasks) for tier in TIER_COUNTS}
- if counts != TIER_COUNTS:
- raise ValueError(f"pilot tier counts must be {TIER_COUNTS}, got {counts}")
- if metadata.get("tier_counts") != TIER_COUNTS:
- raise ValueError("selection metadata tier_counts must match the frozen tier counts")
+ declared = metadata.get("tier_counts")
+ if (not isinstance(declared, dict) or set(declared) != set(TIER_COUNTS)
+ or any(type(value) is not int or value < 0 for value in declared.values())
+ or sum(declared.values()) == 0):
+ raise ValueError("selection metadata must declare nonnegative tier counts")
+ if counts != declared:
+ raise ValueError(
+ f"pilot tier counts must match the frozen declaration {declared}, got {counts}"
+ )
for task in tasks:
low, high = TIER_BOUNDS[task.tier]
if task.estimated_min_steps < low or (high is not None and task.estimated_min_steps > high):
@@ -320,3 +325,24 @@ def validate_against_official(
f"{expected_task_type_counts}, got {dict(task_type_counts)}"
)
return resolved
+
+
+def report_selection_path(
+ profile: dict, explicit: Path | None = None, *, authority_root: Path | None = None
+) -> Path:
+ """Use the same frozen selection in report construction and packaging."""
+ from lexbench_webfetch.paths import contained_path
+
+ from harness.web_fetch.safety import unlinked_path
+ binding = profile.get("task_selection") or {}
+ value = explicit or binding.get("path") or "evaluation/browser-use/webarena-verified-pilot.json"
+ path = Path(value)
+ root = authority_root or Path(__file__).parents[2]
+ path = (unlinked_path(path) if path.is_absolute() else
+ contained_path(root, str(path), reject_symlinks=True))
+ if binding and (
+ _sha256(path) != binding.get("sha256")
+ or binding["sha256"] != profile.get("selection_sha256")
+ ):
+ raise ValueError("report selection differs from its instance profile")
+ return path
diff --git a/harness/reporting/templates.py b/harness/reporting/templates.py
index a08bf19d8..c8834bcff 100644
--- a/harness/reporting/templates.py
+++ b/harness/reporting/templates.py
@@ -28,3 +28,9 @@ def resolve_template(track, *, root=ROOT):
raise ValueError("Report template needs a version-pinned example")
result["example"] = f"{host}/s/{slug}?version={version}"
return {"track": track, **result}
+
+
+def template_identity(track):
+ """Record the maintained owner without borrowing an example's report inputs."""
+ template = resolve_template(track)
+ return {key: template[key] for key in ("track", "renderer", "entrypoint")}
diff --git a/harness/trajectory_replay/report_compaction.py b/harness/trajectory_replay/report_compaction.py
index eb87728c3..5a0af9276 100644
--- a/harness/trajectory_replay/report_compaction.py
+++ b/harness/trajectory_replay/report_compaction.py
@@ -4,10 +4,13 @@
from __future__ import annotations
+import html
import json
+import posixpath
import re
import shutil
from pathlib import Path
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
SHARD_COUNT = 16
@@ -19,33 +22,30 @@ def _task_id(value: str) -> int:
return int(match.group(1))
-def _rewrite_links(text: str, *, detail_kind: str | None = None) -> str:
- text = re.sub(
- r'href="(?:\.\./)?task/(\d+)\.html([#?][^"]*)?"',
- lambda match: f'href="task.html?id={match.group(1)}{match.group(2) or ""}"',
- text,
- )
- text = re.sub(
- r'href="(?:\.\./)?result/([A-Za-z0-9_.-]+)\.html([#?][^"]*)?"',
- lambda match: f'href="result.html?key={match.group(1)}{match.group(2) or ""}"',
- text,
- )
- if detail_kind == "result":
- text = re.sub(
- r'href="([A-Za-z0-9_.-]+-\d+)\.html([#?][^"]*)?"',
- lambda match: f'href="result.html?key={match.group(1)}{match.group(2) or ""}"',
- text,
- )
- text = re.sub(
- r'href="(?:\.\./)*data/evidence/([A-Za-z0-9_.-]+)\.json"',
- lambda match: f'href="evidence-raw.html?key={match.group(1)}"',
- text,
- )
- return (
- text.replace('="../', '="')
- .replace(">下载净化 JSON<", ">查看净化 JSON<")
- .replace(">Download sanitized JSON<", ">Inspect sanitized JSON<")
- )
+def _rewrite_links(text: str, *, detail_kind: str | None = None,
+ source_route: str = 'index.html', target_route: str = 'index.html') -> str:
+ def rewrite(match):
+ name, raw = match[1], html.unescape(match[2])
+ url = urlsplit(raw)
+ if url.scheme or url.netloc or not url.path or raw.startswith('/'):
+ return match[0]
+ resolved = posixpath.normpath(posixpath.join(posixpath.dirname(source_route), url.path))
+ target = resolved
+ query = parse_qsl(url.query, keep_blank_values=True)
+ route = re.fullmatch(r'((?:en/)?)task/(\d+)[.]html', resolved)
+ result = re.fullmatch(r'((?:en/)?)result/([A-Za-z0-9_.-]+)[.]html', resolved)
+ evidence = re.fullmatch(r'data/evidence/([A-Za-z0-9_.-]+)[.]json', resolved)
+ if route or result:
+ item = route or result
+ target = item[1] + ('task.html' if route else 'result.html')
+ query.insert(0, ('id' if route else 'key', item[2]))
+ elif evidence:
+ target = ('en/' if source_route.startswith('en/') else '') + 'evidence-raw.html'
+ query.insert(0, ('key', evidence[1]))
+ relative = posixpath.relpath(target, posixpath.dirname(target_route) or '.')
+ return name + '="' + html.escape(urlunsplit(('', '', relative, urlencode(query), url.fragment)), quote=True) + '"'
+ return re.sub(r'\b(href|src)="([^"]*)"', rewrite, text).replace(
+ '>下载净化 JSON<', '>查看净化 JSON<').replace('>Download sanitized JSON<', '>Inspect sanitized JSON<')
def _javascript_source(source: str) -> str:
@@ -55,9 +55,11 @@ def _javascript_source(source: str) -> str:
return source
-def _loader(kind: str) -> str:
+def _loader(kind: str, lang: str = "zh") -> str:
parameter = "id" if kind == "task" else "key"
- return _javascript_source(f'''轨迹重放详情
''')
def _data_fetch_adapter() -> str:
return _javascript_source(rf'''(()=>{{
const originalFetch=window.fetch.bind(window);const compactCache=new Map();
+const compactBase=new URL("../",document.currentScript.src);
const compactRoute=raw=>{{
const url=new URL(raw,location.href);const match=url.pathname.match(/data\/(trajectory|evidence)\/([A-Za-z0-9_.-]+)\.json$/);
if(!match)return null;
@@ -102,7 +109,7 @@ def _data_fetch_adapter() -> str:
const route=compactRoute(typeof input==='string'?input:input.url);
if(!route)return originalFetch(input,init);
const cacheKey=route.kind+'-'+route.shard;
- if(!compactCache.has(cacheKey))compactCache.set(cacheKey,originalFetch('data/'+route.kind+'-'+route.shard+'.json').then(response=>{{if(!response.ok)throw Error(`HTTP ${{response.status}}`);return response.json()}}));
+ if(!compactCache.has(cacheKey))compactCache.set(cacheKey,originalFetch(new URL('data/'+route.kind+'-'+route.shard+'.json',compactBase)).then(response=>{{if(!response.ok)throw Error(`HTTP ${{response.status}}`);return response.json()}}));
const value=(await compactCache.get(cacheKey))[route.key];
return value===undefined?new Response('',{{status:404}}):new Response(JSON.stringify(value),{{status:200,headers:{{'content-type':'application/json'}}}});
}};
@@ -120,94 +127,138 @@ def compact_report(source: Path, destination: Path) -> dict[str, int]:
if destination.exists():
raise FileExistsError(destination)
destination.mkdir(parents=True)
- excluded = {"task", "result"}
- for path in source.rglob("*"):
+ manifest = json.loads((source / 'report-manifest.json').read_text())
+ languages = manifest.get('languages') or ['zh']
+ prefixes = ['' if language == 'zh' else language + '/' for language in languages]
+ for path in source.rglob('*'):
if not path.is_file():
continue
- relative = path.relative_to(source)
- if relative.parts[0] in excluded or relative.parts[:2] in {
- ("data", "trajectory"),
- ("data", "evidence"),
- }:
+ relative = path.relative_to(source).as_posix()
+ if any(relative.startswith(prefix + kind + '/') for prefix in prefixes for kind in ('task', 'result')):
+ continue
+ if relative.startswith(('data/trajectory/', 'data/evidence/')):
continue
target = destination / relative
target.parent.mkdir(parents=True, exist_ok=True)
- if path.suffix == ".html":
- target.write_text(_rewrite_links(path.read_text(encoding="utf-8")), encoding="utf-8")
+ if path.suffix == '.html':
+ target.write_text(_rewrite_links(path.read_text(), source_route=relative, target_route=relative))
else:
shutil.copy2(path, target)
-
- pages = destination / "pages"
- pages.mkdir()
- for kind in ("task", "result"):
- buckets = [{} for _ in range(SHARD_COUNT)]
- for path in sorted((source / kind).glob("*.html")):
- key = path.stem
- buckets[_task_id(key) % SHARD_COUNT][key] = _rewrite_links(
- path.read_text(encoding="utf-8"), detail_kind=kind
- )
- for index, rows in enumerate(buckets):
- (pages / f"{kind}-{index:02d}.json").write_text(
- json.dumps(rows, ensure_ascii=False, separators=(",", ":")), encoding="utf-8"
- )
- (destination / f"{kind}.html").write_text(_loader(kind), encoding="utf-8")
- (destination / "evidence-raw.html").write_text(
- _raw_evidence_loader(), encoding="utf-8"
- )
-
- (destination / "data").mkdir(exist_ok=True)
- for kind in ("trajectory", "evidence"):
+ for prefix in prefixes:
+ language = 'en' if prefix else 'zh'
+ root = destination / prefix
+ pages = root / 'pages'
+ pages.mkdir()
+ for kind in ('task', 'result'):
+ buckets = [{} for _ in range(SHARD_COUNT)]
+ for path in sorted((source / prefix / kind).glob('*.html')):
+ key = path.stem
+ buckets[_task_id(key) % SHARD_COUNT][key] = _rewrite_links(
+ path.read_text(), source_route=path.relative_to(source).as_posix(),
+ target_route=prefix + kind + '.html')
+ for index, rows in enumerate(buckets):
+ (pages / f'{kind}-{index:02d}.json').write_text(json.dumps(rows, ensure_ascii=False, separators=(',', ':')))
+ (root / f'{kind}.html').write_text(_loader(kind, language))
+ (root / 'evidence-raw.html').write_text(_raw_evidence_loader(language))
+ (destination / 'data').mkdir(exist_ok=True)
+ for kind in ('trajectory', 'evidence'):
shards = [{} for _ in range(SHARD_COUNT)]
- for path in sorted((source / f"data/{kind}").glob("*.json")):
- key = path.stem
- shards[_task_id(key) % SHARD_COUNT][key] = json.loads(
- path.read_text(encoding="utf-8")
- )
+ for path in sorted((source / f'data/{kind}').glob('*.json')):
+ shards[_task_id(path.stem) % SHARD_COUNT][path.stem] = json.loads(path.read_text())
for index, rows in enumerate(shards):
- (destination / f"data/{kind}-{index:02d}.json").write_text(
- json.dumps(rows, ensure_ascii=False, separators=(",", ":")), encoding="utf-8"
- )
- app = destination / "assets/app.js"
- app.write_text(_data_fetch_adapter() + app.read_text(encoding="utf-8"), encoding="utf-8")
- files = [path for path in destination.rglob("*") if path.is_file()]
- return {
- "files": len(files),
- "task_routes": len(list((source / "task").glob("*.html"))),
- "result_routes": len(list((source / "result").glob("*.html"))),
- }
-
-
-def validate_compact_report(report: Path, *, max_files: int) -> dict[str, int]:
- files = [path for path in report.rglob("*") if path.is_file()]
+ (destination / f'data/{kind}-{index:02d}.json').write_text(json.dumps(rows, ensure_ascii=False, separators=(',', ':')))
+ app = destination / 'assets/app.js'
+ app.write_text(_data_fetch_adapter() + app.read_text())
+ return {'files': sum(path.is_file() for path in destination.rglob('*')),
+ 'task_routes': len(list((source / 'task').glob('*.html'))),
+ 'result_routes': len(list((source / 'result').glob('*.html')))}
+
+
+def validate_compact_report(report: Path, *, max_files: int, source: Path | None = None) -> dict[str, int]:
+ files = [path for path in report.rglob('*') if path.is_file()]
if not files or len(files) > max_files:
- raise ValueError(f"compact report file count is outside 1..{max_files}: {len(files)}")
- required = [
- report / "index.html",
- report / "task.html",
- report / "result.html",
- report / "evidence-raw.html",
- ]
- required += [
- report / f"pages/{kind}-{index:02d}.json"
- for kind in ("task", "result")
- for index in range(SHARD_COUNT)
- ]
- required += [
- report / f"data/{kind}-{index:02d}.json"
- for kind in ("trajectory", "evidence")
- for index in range(SHARD_COUNT)
- ]
- if any(not path.is_file() for path in required):
- raise ValueError("compact report route bundles are incomplete")
- manifest = json.loads((report / "report-manifest.json").read_text(encoding="utf-8"))
- task_routes = sum(
- len(json.loads((report / f"pages/task-{index:02d}.json").read_text()))
- for index in range(SHARD_COUNT)
- )
- result_routes = sum(
- len(json.loads((report / f"pages/result-{index:02d}.json").read_text()))
- for index in range(SHARD_COUNT)
- )
- if task_routes != manifest["task_count"] or result_routes != manifest["logical_results"]:
- raise ValueError("compact report route coverage differs from its manifest")
- return {"files": len(files), "task_routes": task_routes, "result_routes": result_routes}
+ raise ValueError(f'compact report file count is outside 1..{max_files}: {len(files)}')
+ manifest = json.loads((report / 'report-manifest.json').read_text())
+ prefixes = ['' if lang == 'zh' else lang + '/' for lang in manifest.get('languages') or ['zh']]
+ for prefix in prefixes:
+ for kind, field in [('task', 'task_count'), ('result', 'logical_results')]:
+ root = report / prefix
+ if not (root / f'{kind}.html').is_file() or not (root / 'evidence-raw.html').is_file():
+ raise ValueError('compact report route bundles are incomplete')
+ values = {}
+ for index in range(SHARD_COUNT):
+ path = root / f'pages/{kind}-{index:02d}.json'
+ if not path.is_file():
+ raise ValueError('compact report route bundles are incomplete')
+ bucket = json.loads(path.read_text())
+ if values.keys() & bucket.keys():
+ raise ValueError('duplicate compact logical route')
+ values.update(bucket)
+ if len(values) != manifest[field]:
+ raise ValueError('compact report route coverage differs from its manifest')
+ if source is not None:
+ expected = {p.stem: _rewrite_links(p.read_text(), source_route=p.relative_to(source).as_posix(),
+ target_route=prefix + kind + '.html') for p in (source / prefix / kind).glob('*.html')}
+ if values != expected:
+ raise ValueError('compact report pages differ from validated source')
+ for kind in ('trajectory', 'evidence'):
+ values = {}
+ for index in range(SHARD_COUNT):
+ path = report / f'data/{kind}-{index:02d}.json'
+ if not path.is_file():
+ raise ValueError('compact evidence bundles are incomplete')
+ bucket = json.loads(path.read_text())
+ if values.keys() & bucket.keys():
+ raise ValueError('duplicate compact evidence key')
+ values.update(bucket)
+ if source is not None:
+ expected = {p.stem: json.loads(p.read_text()) for p in (source / f'data/{kind}').glob('*.json')}
+ if values != expected:
+ raise ValueError('compact evidence differs from validated source')
+ return {'files': len(files), 'task_routes': manifest['task_count'], 'result_routes': manifest['logical_results']}
+
+
+def check_compact_navigation(report: Path) -> None:
+ """Exercise the exported query routes and evidence, in every saved language."""
+ from playwright.sync_api import sync_playwright
+
+ from harness.reporting.visual import browser_executable, local_report_server
+
+ manifest = json.loads((report / 'report-manifest.json').read_text())
+ rows = json.loads((report / 'data/results.json').read_text())
+ samples = [next((row for row in rows if (row['status'] == 'pass') == passed), None)
+ for passed in (True, False)]
+ with local_report_server(report) as origin, sync_playwright() as playwright:
+ browser = playwright.chromium.launch(executable_path=browser_executable(), headless=True)
+ try:
+ for language in manifest.get('languages') or ['zh']:
+ prefix = '' if language == 'zh' else language + '/'
+ page = browser.new_page()
+ for row in samples:
+ if row is None:
+ continue
+ page.goto(f"{origin}/{prefix}result.html?key={row['key']}")
+ page.locator('header.top').wait_for()
+ if page.locator('html').get_attribute('lang') != language:
+ raise ValueError('compact result language differs')
+ step = page.locator('button[data-open-step]').first
+ if step.count():
+ step.click()
+ page.locator('#step-dialog[open]').wait_for()
+ page.wait_for_function("() => document.querySelector('[data-step-planned]')?.textContent.includes('action')")
+ page.keyboard.press('Escape')
+ page.goto(f"{origin}/{prefix}evidence-raw.html?key={row['key']}")
+ page.wait_for_function("() => document.querySelector('pre')?.textContent.trim().startsWith('{')")
+ observed = json.loads(page.locator('pre').inner_text())
+ expected = json.loads((report / f"data/evidence-{int(row['task_id']) % SHARD_COUNT:02d}.json").read_text())[row['key']]
+ if observed != expected:
+ raise ValueError('compact served evidence differs from its bound payload')
+ page.goto(f"{origin}/{prefix}task.html?id={row['task_id']}")
+ page.locator('header.top').wait_for()
+ page.locator('a[href*="result.html?key="]').first.click()
+ page.locator('header.top').wait_for()
+ page.locator('nav a[href*="index.html"]').first.click()
+ page.locator('header.top').wait_for()
+ page.close()
+ finally:
+ browser.close()
diff --git a/harness/trajectory_replay/webarena_report.py b/harness/trajectory_replay/webarena_report.py
index 31925b615..38aeb3059 100644
--- a/harness/trajectory_replay/webarena_report.py
+++ b/harness/trajectory_replay/webarena_report.py
@@ -18,12 +18,14 @@
from harness.record_io import load_jsonl
from harness.reporting.privacy import LOCAL_PATH_RE, WEB_URL_RE
+from harness.reporting.templates import template_identity
from harness.trajectory_replay.catalog_contracts import (
LEGACY_CATALOG_CONTRACTS_SCHEMA,
LEGACY_CATALOG_REPORT_SCHEMA,
)
from harness.trajectory_replay.evaluator_transition import json_sha256, reevaluate_pass
from harness.trajectory_replay.failure_matrix import (
+ browser_failure_profile,
failure_category_matrix,
failure_signature_cards,
)
@@ -337,7 +339,10 @@ def _supplemental_diagnostic_html(diagnostic: dict | None, lang: str) -> str:
network_rows = []
header_rows = []
for method in diagnostic["observations"]:
- label = {"release": "Moli 1.1.8", "pr": "Moli · PR 640"}.get(method["method"], method["method"])
+ label = method.get("label") or method.get("version") or {
+ "release": "正式构建" if zh else "Release build",
+ "pr": "候选构建" if zh else "Candidate build",
+ }.get(method["method"], method["method"])
for sample in method["observations"]:
for request in sample["state"].get("headerRecording", []):
cookie_label = ("有" if request["cookie_present"] else "无") if zh else ("Yes" if request["cookie_present"] else "No")
@@ -346,7 +351,7 @@ def _supplemental_diagnostic_html(diagnostic: dict | None, lang: str) -> str:
request["wire_referer_path"] or missing, request["har_referer_path"] or missing)
header_rows.append("
" + "".join(f"
{_e(value)}
" for value in values) + "
")
for request in sample["state"].get("httpReads", []):
- values = (label, request["path"], request.get("status", "—"),
+ values = (label, request.get("path") or request.get("url") or ("未记录" if zh else "Not recorded"), request.get("status", "—"),
request.get("bytes", "—"), request.get("error", ""))
network_rows.append("
" + "".join(f"
{_e(value)}
" for value in values) + "
")
readiness = sample["state"].get("readiness")
@@ -854,7 +859,7 @@ def _detail_page(
report_root = "../" if zh else "../../"
stage_labels = {
"browser_session_setup": ("浏览器会话初始化", "Browser session setup"),
- "browser_tool_execution": ("Playwright 动作执行", "Playwright action execution"),
+ "browser_tool_execution": ("浏览器动作执行", "Browser action execution"),
"evidence_finalization": ("执行证据收尾", "Evidence finalization"),
"official_evaluator_execution": ("官方评测器执行", "Official evaluator execution"),
"side_effect_not_observed": ("网络验收条件未满足", "Network acceptance not satisfied"),
@@ -1248,7 +1253,7 @@ def _assets(output: Path) -> None:
if(!result)return;
const detail=result.detail_url||'result/'+result.key+'.html';
dialog.querySelector('[data-dialog-title]').textContent=result.browser+' · '+(zh?'任务':'Task')+' #'+result.task_id;
- dialog.querySelector('[data-dialog-stage]').textContent=result.failure_stage==='browser_session_setup'?(zh?'浏览器会话初始化':'Browser session setup'):result.failure_stage==='browser_tool_execution'?(zh?'Playwright 动作':'Playwright action'):result.failure_stage==='evidence_finalization'?(zh?'执行证据收尾':'Evidence finalization'):result.failure_stage==='official_evaluator_execution'?(zh?'官方评测器执行':'Official evaluator execution'):result.failure_stage==='side_effect_not_observed'?(zh?'网络验收条件未满足':'Network acceptance not satisfied'):result.failure_stage==='official_evaluation'?(zh?'官方验收':'Official evaluation'):(zh?'无':'None');
+ dialog.querySelector('[data-dialog-stage]').textContent=result.failure_stage==='browser_session_setup'?(zh?'浏览器会话初始化':'Browser session setup'):result.failure_stage==='browser_tool_execution'?(zh?'浏览器动作':'Browser action'):result.failure_stage==='evidence_finalization'?(zh?'执行证据收尾':'Evidence finalization'):result.failure_stage==='official_evaluator_execution'?(zh?'官方评测器执行':'Official evaluator execution'):result.failure_stage==='side_effect_not_observed'?(zh?'网络验收条件未满足':'Network acceptance not satisfied'):result.failure_stage==='official_evaluation'?(zh?'官方验收':'Official evaluation'):(zh?'无':'None');
dialog.querySelector('[data-dialog-error]').textContent=zh?'正在读取完整证据…':'Loading complete evidence…';
delete dialog.dataset.loaded;
dialog.querySelector('[data-dialog-link]').href=detail;
@@ -1362,6 +1367,7 @@ def _assets(output: Path) -> None:
".interpretation>.presentation-slide{margin:30px 0;padding:clamp(24px,4vw,50px);background:var(--panel);border:1px solid var(--line);border-radius:24px;box-shadow:var(--shadow)}.interpretation>.presentation-slide>h2{max-width:980px;font-size:clamp(1.7rem,3vw,2.7rem);margin-top:0}"
".outcome-chart ol{list-style:none;padding:0;margin:24px 0;display:grid;gap:17px}.outcome-chart li>div:first-child{display:flex;justify-content:space-between;gap:18px}.outcome-chart li strong{font-size:1.08rem}.outcome-track{height:18px;margin-top:7px}.outcome-chart figcaption{text-align:right;color:var(--muted)}"
".browser-failure-profile{display:grid;gap:22px;margin:26px 0}.browser-profile-overview{display:grid;grid-template-columns:1.2fr 1fr;gap:14px}.browser-profile-overview>div{border:1px solid var(--line);border-radius:16px;padding:20px;background:var(--bg)}.browser-profile-overview span,.browser-profile-overview small{display:block}.browser-profile-overview strong{display:block;font-size:clamp(2rem,5vw,4rem);line-height:1.05;margin:8px 0}"
+ ".failure-bars li>*{min-width:0;overflow-wrap:anywhere}.failure-explanations article{min-width:0;overflow-wrap:anywhere}.interpretation{min-width:0;max-width:100%}.interpretation p{overflow-wrap:anywhere}"
".failure-bars{list-style:none;padding:0;margin:0;display:grid;gap:9px}.failure-bars li{display:grid;grid-template-columns:minmax(180px,1.4fr) minmax(180px,2fr) 54px 58px;gap:12px;align-items:center}.failure-bars strong,.failure-bars small{text-align:right}.failure-explanations{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:12px}.failure-explanations article{border:1px solid var(--line);border-radius:15px;padding:18px;background:var(--bg)}.failure-explanations header{display:flex;justify-content:space-between;gap:14px}.failure-explanations h3{font-size:1.05rem;margin:0}.failure-explanations p{font-size:.94rem}.failure-explanations a{font-size:.9rem}"
"\n.diagnostic-observations table{min-width:840px}"
".diagnostic-observations td:first-child{white-space:nowrap;min-width:100px}"
@@ -1370,7 +1376,7 @@ def _assets(output: Path) -> None:
".diagnostic-timeline li{border:1px solid #888;border-radius:10px;padding:12px;min-width:0}"
".diagnostic-timeline strong,.diagnostic-timeline span{display:block}"
".fixed36-diagnostic figure img{display:block;max-width:100%;height:auto;border:1px solid var(--line);border-radius:8px}"
- "@media(max-width:760px){.section-heading{display:block}.section-heading>a{display:inline-block;margin-top:12px}.failure-signature-grid,.failure-explanations,.browser-profile-overview{grid-template-columns:1fr}.failure-bars li{grid-template-columns:1fr 52px}.failure-bars li .failure-bar-track{grid-column:1/-1;grid-row:2}.failure-bars li small{grid-column:2}.interpretation>.presentation-slide{padding:22px 18px}}"
+ "@media(max-width:760px){.section-heading{display:block}.section-heading>a{display:inline-block;margin-top:12px}.failure-signature-grid,.failure-explanations,.browser-profile-overview{grid-template-columns:1fr}.failure-bars li{grid-template-columns:minmax(0,1fr) 52px}.failure-bars li .failure-bar-track{grid-column:1/-1;grid-row:2}.failure-bars li small{grid-column:2}.interpretation>.presentation-slide{padding:22px 18px}}"
"@media print{@page{size:13.333in 7.5in;margin:.42in}.interpretation>.hero,.interpretation>.presentation-slide{break-after:page;break-inside:avoid;box-shadow:none;border-color:#bbb}.failure-explanations article{break-inside:avoid}}"
)
(output / "assets/report.css").write_text(css, encoding="utf-8")
@@ -1695,6 +1701,13 @@ def build_webarena_replay_report(
if zh
else f"WebArena-Verified · {task_count} tasks · model calls 0"
)
+ backend_note = (
+ ("所有方法通过同一 Playwright 重放器执行冻结动作;没有 Agent 决策或模型调用。"
+ if zh else "All methods use the same Playwright replay adapter for frozen actions, with no Agent decisions or model calls.")
+ if execution_backend == "playwright" else
+ ("所有方法重放冻结动作,没有 Agent 决策或模型调用。历史配置没有声明执行后端,报告不推定其具体实现。"
+ if zh else "All methods replay frozen actions without Agent decisions or model calls. The historical profile does not declare an execution backend.")
+ )
failure_participants = tuple(
engine
for engine in methods
@@ -1735,15 +1748,15 @@ def build_webarena_replay_report(
)
core_metrics_note += (
(
- "提前失败会缩短工作量,因此总耗时和资源不按低者判优;单次动作耗时按实际 Playwright 调用统计。"
+ "提前失败会缩短工作量,因此总耗时和资源不按低者判优;单次动作耗时按实际浏览器动作调用统计。"
if has_resource_samples
- else "提前失败会缩短工作量,因此总耗时不按低者判优;单次动作耗时按实际 Playwright 调用统计。"
+ else "提前失败会缩短工作量,因此总耗时不按低者判优;单次动作耗时按实际浏览器动作调用统计。"
)
if zh
else (
- "Early failures reduce work, so total time and resources are not ranked by lower values; per-action latency uses actual Playwright calls."
+ "Early failures reduce work, so total time and resources are not ranked by lower values; per-action latency uses actual browser action calls."
if has_resource_samples
- else "Early failures reduce work, so total time is not ranked by lower values; per-action latency uses actual Playwright calls."
+ else "Early failures reduce work, so total time is not ranked by lower values; per-action latency uses actual browser action calls."
)
)
core_metrics_table = _metrics_table(
@@ -1758,7 +1771,7 @@ def build_webarena_replay_report(
if resource_table
else f'
{"资源数据" if zh else "Resource data"}
{"本轮没有记录进程级 CPU、内存和网络采样,因此这些指标不可用,报告不展示空值表或资源排名。端到端耗时和动作耗时来自现有轨迹记录,无需重新执行任务。" if zh else "This run did not record process-level CPU, memory, or network samples. Those metrics are unavailable, so the report does not show empty tables or resource rankings. End-to-end and action latency come from the retained trajectories and require no rerun."}
'
)
- summary_body = f'''
{eyebrow}
{"浏览器轨迹重放评测" if zh else "Browser Trajectory Replay"}
{f"{browser_count} 款浏览器组成 {method_count} 种方法配置,分别覆盖同一 {task_count} 题的冻结轨迹重放。六种方法均由同一 Playwright 重放器通过 CDP 执行 BrowserTool 动作契约;本轮没有调用 CowAgent BrowserTool,也没有 Agent 或模型参与。结果比较的是 Playwright 重放器与浏览器的组合配置,不能单独归因于浏览器内核。方法版本、轨迹与站点准备见环境说明。" if zh else f"{browser_count} browsers form {method_count} method configurations, each covering the same {task_count} frozen trajectory replays. All six methods use the same Playwright replay adapter over CDP to execute the BrowserTool action contract; this run does not invoke CowAgent BrowserTool, an Agent, or a model. Results compare Playwright-adapter–browser configurations, not browser engines in isolation. Environment documents versions, paths and site preparation."}
{"浏览器轨迹重放评测" if zh else "Browser Trajectory Replay"}
{f"{browser_count} 款浏览器组成 {method_count} 种方法配置,分别覆盖同一 {task_count} 题的冻结轨迹重放。{backend_note} 方法版本、轨迹与站点准备见环境说明。" if zh else f"{browser_count} browsers form {method_count} method configurations, each covering the same {task_count} frozen trajectory replays. {backend_note} Environment documents versions, paths and site preparation."}
')
browser_options = "".join(f'' for key, item in comparison_methods.items())
site_options = "".join(f'' for site in sorted({s for row in public_rows for s in row["sites"]}))
- stage_labels = {"browser_session_setup": ("浏览器会话初始化", "Browser session setup"), "browser_tool_execution": ("Playwright 动作", "Playwright action"), "evidence_finalization": ("执行证据收尾", "Evidence finalization"), "official_evaluator_execution": ("官方评测器执行", "Official evaluator execution"), "side_effect_not_observed": ("网络验收条件未满足", "Network acceptance not satisfied"), "official_evaluation": ("官方验收", "Official evaluation")}
+ stage_labels = {"browser_session_setup": ("浏览器会话初始化", "Browser session setup"), "browser_tool_execution": ("浏览器动作", "Browser action"), "evidence_finalization": ("执行证据收尾", "Evidence finalization"), "official_evaluator_execution": ("官方评测器执行", "Official evaluator execution"), "side_effect_not_observed": ("网络验收条件未满足", "Network acceptance not satisfied"), "official_evaluation": ("官方验收", "Official evaluation")}
present_stages = {row["failure_stage"] for row in public_rows if row["status"] == "fail"}
stage_options = "".join(f'' for code in stage_labels if code in present_stages)
causes = {row["cause_zh"]: row["cause_zh"] if zh else row["cause_en"] for row in public_rows if row["cause_zh"]}
@@ -1834,8 +1847,8 @@ def build_webarena_replay_report(
results_body = f'''
{len(public_rows)} {"条逻辑结果" if zh else "logical outcomes"}
{"结果与失败诊断" if zh else "Results and failure diagnosis"}
{"同题对比支持选择基准和多个方法;下方筛选表默认只显示失败,点击“诊断”查看原始判定、复核原因与执行证据。" if zh else "Choose a baseline and multiple methods for same-task comparison. The inventory starts with failures; open Diagnose for the exact finding, reviewed cause, and execution evidence."}
{result_matrix}
{"任务" if zh else "Task"}
{"方法" if zh else "Method"}
{"站点" if zh else "Site"}
{"判定" if zh else "Verdict"}
{"动作" if zh else "Actions"}
{"失败环节" if zh else "Failure stage"}
{"复核分类" if zh else "Reviewed category"}
{"详情" if zh else "Detail"}
{''.join(result_rows)}
'''
diagnostic_details = (
failure_category_matrix(comparison_methods, public_rows, lang)
- + f'
{"Playwright 动作耗时" if zh else "Playwright action latency"}
'
- + f'
{"均值、P50、P95 按单次 Playwright 调用统计;调用次数只说明工作量。逐题详情保留每个动作的耗时和返回内容。" if zh else "Mean, P50, and P95 are per Playwright call; call counts describe workload. Result details retain every action duration and response."}
'
+ + f'
{"浏览器动作耗时" if zh else "Browser action latency"}
'
+ + f'
{"均值、P50、P95 按单次浏览器动作调用统计;调用次数只说明工作量。逐题详情保留每个动作的耗时和返回内容。" if zh else "Mean, P50, and P95 are per browser action call; call counts describe workload. Result details retain every action duration and response."}
'
+ f'{"展开各类动作耗时" if zh else "Open latency by action type"}{action_table}'
+ f'
{"各方法成功任务的条件分布" if zh else "Conditional distributions on each method’s successful tasks"}
'
+ f'
{"各列仅含该方法成功的任务,任务集合和样本数可能不同。这组数据用于诊断,不参与方法排名。" if zh else "Each column contains only that method’s successful tasks. Task sets and sample counts may differ, so these distributions are diagnostic and not ranked."}
'
@@ -1869,7 +1882,7 @@ def build_webarena_replay_report(
V --> X[{"绑定证据的结果" if zh else "Evidence-bound result"}]
X --> T[{"销毁浏览器与站点" if zh else "Teardown browser and sites"}]
T --> R'''
- environment_body = f'''
{"环境与评测流程" if zh else "Environment and evaluation flow"}
{"192.168.5.2 是 Colima 虚拟机中的宿主网关:浏览器容器通过它访问运行在同一台 Mac 虚拟化环境里的 WebArena 站点。它不是公网测试网站。" if zh else "192.168.5.2 is the host gateway inside the Colima VM. Browser containers use it to reach the WebArena sites running in the same Mac virtualization environment; it is not a public test website."}
{"执行拓扑" if zh else "Execution topology"}
{fallback}{topology}
{"隔离规则" if zh else "Isolation rules"}
1
{"一套固定端口站点环境,因此只允许一个方法运行。" if zh else "One fixed-port site stack permits only one running method."}
{task_count}
{"每种方法覆盖全部任务,方法开始前恢复站点;任务之间不重置。每题只保留一个最终逻辑结果,额外物理执行单独计数,输入未变且已经通过的任务不再执行。" if zh else "Each method covers every task after one site reset, with no reset between tasks. One final logical outcome is retained per task; extra physical calls are counted separately, and an unchanged task is not run again after passing."}
0
{"没有 Agent 决策和模型请求;Playwright 通过 CDP 重放冻结的 BrowserTool 动作契约。" if zh else "No Agent decisions or model calls; Playwright replays the frozen BrowserTool action contract over CDP."}
'''
+ environment_body = f'''
{"环境与评测流程" if zh else "Environment and evaluation flow"}
{"192.168.5.2 是 Colima 虚拟机中的宿主网关:浏览器容器通过它访问运行在同一台 Mac 虚拟化环境里的 WebArena 站点。它不是公网测试网站。" if zh else "192.168.5.2 is the host gateway inside the Colima VM. Browser containers use it to reach the WebArena sites running in the same Mac virtualization environment; it is not a public test website."}
{"执行拓扑" if zh else "Execution topology"}
{fallback}{topology}
{"隔离规则" if zh else "Isolation rules"}
1
{"一套固定端口站点环境,因此只允许一个方法运行。" if zh else "One fixed-port site stack permits only one running method."}
{task_count}
{"每种方法覆盖全部任务,方法开始前恢复站点;任务之间不重置。每题只保留一个最终逻辑结果,额外物理执行单独计数,输入未变且已经通过的任务不再执行。" if zh else "Each method covers every task after one site reset, with no reset between tasks. One final logical outcome is retained per task; extra physical calls are counted separately, and an unchanged task is not run again after passing."}
0
{_e(backend_note)}
'''
environment_body = environment_body.replace("环境与评测流程", "环境与方法").replace("Environment and evaluation flow", "Environment and method")
if "Rainbond" in str((profile.get("execution") or {}).get("site_stack") or ""):
diagram_script = environment_body[environment_body.index('')
+ en.feed('
',
- )
methodology += _section(
"固定协议",
_table(
["项目", "规则"],
[
["Agent / 模型", Safe(bilingual_text(f"CowAgent;{model};推理强度 {reasoning_effort}", f"CowAgent; {model}; reasoning effort {reasoning_effort}"))],
- ["计划与实际对照", "计划固定任务、数据、模型、提示和资源上限,仅替换浏览器;历史执行的视口实际不一致,不能视为浏览器是唯一变量的实验"],
- ["结果口径", "每题×浏览器一个最终逻辑结果;二元通过率;逐题配对"],
+ ["计划与实际对照", Safe(bilingual_text("模型、提示、工具、资源和执行环境以冻结配置及实际执行记录为准。", "Model, prompt, tool, resource and environment controls follow the frozen configuration and recorded executions."))],
+ ["结果口径", Safe(bilingual_text("每题×模型×浏览器一个最终逻辑结果;二元通过率;逐题配对。", "One retained logical result per task, model and browser; binary acceptance and task-paired comparison."))],
[
"执行上限",
- "代码最多允许两次执行;本报告按冻结运行协议选取正式结果,实际执行耗时和模型用量累计",
+ Safe(bilingual_text(
+ f"冻结协议的每题执行上限:{protocol.get('max_task_executions') or profile.get('max_task_executions') or '未记录'};实际执行耗时和模型用量累计。",
+ f"Frozen execution limit per task: {protocol.get('max_task_executions') or profile.get('max_task_executions') or 'not recorded'}; execution latency and model usage are cumulative.")),
],
[
"排除",
- "不纳入无法稳定验收的实时任务;本轮排除需大型数据卷的 Wikipedia / Map",
+ Safe(bilingual_text(
+ "只报告冻结任务清单及其验收合同覆盖的任务。",
+ "Only tasks and acceptance contracts in the frozen selection are reported.")),
],
["裁判", "官方程序化 evaluator;无 LLM judge;人工不得改分"],
],
@@ -3290,7 +3281,9 @@ def build_report(*, evidence: Path | Iterable[Path], output: Path, selection_pat
)
analysis += _section(
f"同题全配置对比:完整 {len(tasks)} 题",
- "
每行是一道预注册任务,四列分别对应模型×浏览器配置;可筛出结果有差异、全部通过或全部失败的任务。
"
+ "
" + bilingual_text(
+ f"每行是一道预注册任务,{len(MODEL_LABELS) * len(ENGINE_LABELS)} 列对应模型×浏览器配置;可筛出结果有差异、全部通过或全部失败的任务。",
+ f"Each row is a registered task, with {len(MODEL_LABELS) * len(ENGINE_LABELS)} columns for model-by-browser configurations. Filter for disagreement, all-pass or all-fail tasks.") + "
"
+ _same_task_comparison(tasks, attempts),
).replace('', '', 1)
analysis += _section(
diff --git a/harness/browser_use/report_i18n.py b/harness/browser_use/report_i18n.py
index 6b52bfefb..7ceff22f6 100644
--- a/harness/browser_use/report_i18n.py
+++ b/harness/browser_use/report_i18n.py
@@ -659,7 +659,6 @@
"共享的长链任务完成错误:30 步上限前没有完成结账。两种浏览器路径细节不同,但都未越过同一验收终点。": "Shared long-horizon completion error: checkout was not completed within the 30-step limit. Browser paths differed in detail, but neither crossed the same acceptance endpoint.",
"共享的长链任务完成错误:30 步上限前没有完成结账。两种浏览器路径细节不同,但都未越过同一验收终点。 本诊断不修改官方分数。": "Shared long-horizon completion error: checkout was not completed within the 30-step limit. Browser paths differed in detail, but neither crossed the same acceptance endpoint. This diagnosis does not change the official score.",
# Local environment and expansion guidance.
- "Mac 上可以继续扩展哪些实例": "Which environments can be expanded on Mac",
"环境": "Environment",
"Apple Silicon Mac 状态": "Apple Silicon Mac status",
"完整集单站题": "Full-set single-site tasks",
@@ -671,12 +670,8 @@
"amd64 镜像;已通过 Colima 转译运行": "amd64 image; already measured through Colima translation",
"arm64 镜像;需约 100 GB 数据;只出现在跨站任务": "arm64 image; about 100 GB of data; appears only in cross-site tasks",
"amd64 镜像;需约 60 GB 数据与 Apple Silicon 转译": "amd64 image; about 60 GB of data plus Apple Silicon translation",
- "Wikipedia 与 Map 的容量来自官方环境说明;状态中的“实跑”只指本项目已有执行证据。": "Wikipedia and Map capacity estimates come from the official environment guide. 'Measured' refers only to evidence already collected by this project.",
- "最小扩展方案:": "Minimum expansion plan:",
- "先从四个已验证站点的 678 个候选中做可完成性预检和分层抽样;Wikipedia、Map 等大数据环境后置。这样扩容不会先引入新的部署变量。": "Preflight solvability and draw a stratified sample from the 678 candidates on the four verified sites first. Defer the large Wikipedia and Map environments so expansion does not begin by adding deployment variables.",
"官方任务集与安装说明": "Official dataset and installation guide",
"官方环境容量说明": "Official environment capacity guide",
- "下一轮轨迹重放固定使用本报告当前 90 题,不扩充任务范围。若未来另行扩充 Browser Use 评测,当前四个已实跑站点可预检 655 个单站任务;计入只依赖这四站的跨站任务后,共有 678 个候选,其中 hard 子集 233 个。": "The next trajectory-replay evaluation is fixed to the report's current 90 tasks; its scope will not expand. If Browser Use is expanded separately later, the four locally measured sites support preflight checks for 655 single-site tasks, or 678 candidates including cross-site tasks that use only these four sites, with 233 in the hard subset.",
"站点确实部署在本机,不是公网代站。": "The sites were deployed locally, not replaced by public websites.",
"为什么五个失败任务会完全相同": "Why the same five tasks failed in both browsers",
"二、为什么五个失败任务会完全相同": "2. Why the same five tasks failed in both browsers",
@@ -737,8 +732,6 @@
"结果口径": "Result unit",
"每题×浏览器一个最终逻辑结果;二元通过率;逐题配对": "One final logical result per task-browser pair; binary pass rate; task-level pairing",
"执行上限": "Execution limit",
- "代码最多允许两次执行;本报告按冻结运行协议选取正式结果,实际执行耗时和模型用量累计": "The code permits at most two executions. The frozen run protocol selects the official result, while actual execution latency and model usage are cumulative.",
- "不纳入无法稳定验收的实时任务;本轮排除需大型数据卷的 Wikipedia / Map": "Exclude live tasks without stable acceptance; this run excludes Wikipedia and Map because they require large data volumes",
"结果分析 · Browser Use 评测": "Results · Browser Use evaluation",
"从总体配对下钻到每个逻辑结果": "Drill down from the paired summary to every logical result",
"先看 30 个同题配对,再定位 10 个失败结果的官方规则、actual回答、轨迹和日志。": "Start with 30 same-task pairs, then inspect the official rules, actual answers, trajectories, and logs for the 10 failed results.",
diff --git a/tests/browser_use/test_report.py b/tests/browser_use/test_report.py
index 74791ee1d..407869098 100644
--- a/tests/browser_use/test_report.py
+++ b/tests/browser_use/test_report.py
@@ -1321,6 +1321,7 @@ def test_report_renders_measured_runtime_identities(tmp_path: Path, monkeypatch)
{
"protocol": {
"cowagent_commit": "abc123",
+ "max_task_executions": 3,
"model": "gpt-5.6-sol",
"reasoning_effort": "low",
},
@@ -1387,10 +1388,15 @@ def test_report_renders_measured_runtime_identities(tmp_path: Path, monkeypatch)
assert "192.168.5.2" in methodology_text
assert "不含点击序列" in methodology_text
assert "计划与实际对照" in methodology_text
- assert "不能视为浏览器是唯一变量的实验" in methodology_text
+ assert "每题执行上限:3" in methodology_text
+ assert "冻结配置及实际执行记录" in methodology_text
+ assert "下一轮轨迹重放" not in methodology_text
+ assert "678 个候选" not in methodology_text
+ assert "Mac 上可以继续扩展哪些实例" not in methodology_text
assert "只替换浏览器" not in methodology_text
english_methodology = localize_html(methodology_text, PurePosixPath("methodology/index.html"))
- assert "the browser was not the only experimental variable" in english_methodology
+ assert "frozen configuration and recorded executions" in english_methodology
+ assert "execution limit per task: 3" in english_methodology
for name in (
"changes/index.html",
"datasets/index.html",
From 889a1cbe2ac6dd1d0001987d5d305d129fe914bf Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Wed, 21 Oct 2026 11:41:50 +0800
Subject: [PATCH 14/51] fix(webfetch): derive WCXB presentation from selected
methods
---
harness/web_fetch/report.py | 36 ++++++++++++++---------------
tests/test_wcxb_report_languages.py | 30 ++++++++++++++++++++++++
2 files changed, 48 insertions(+), 18 deletions(-)
diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py
index 0c1cf00c4..f3bf6d840 100644
--- a/harness/web_fetch/report.py
+++ b/harness/web_fetch/report.py
@@ -95,12 +95,12 @@ def generate_wcxb(out: Path, rows: list[dict], summary: dict, ground_truth: dict
'en':{'home':'Conclusions','results':'Results analysis','method':'Method','guide':'Interpretation','repro':'Reproduce','theme':'Theme','headline':'WCXB main-content extraction comparison','core':'Core results','provider':'Browser','delivered':'Nonempty outputs','precision':'Precision','recall':'Recall','f1':'F1','latency':'Mean end-to-end latency','type':'Page type','case':'Page','diagnosis':'Locator','detail':'Same-page evidence','all':'All','issues':'Issues to inspect','query':'Search page or error','count':'results','reference':'Human main-content reference','output':'Complete browser output','error':'Exact program error','missing':'Frequent unmatched reference words','extra':'Frequent extra output words','summary':'Five results for this page','note':'Low precision often means output contains much text outside the reference; low recall means much reference text is missing. These are locators, not proof of a program defect.','scope':'Population and calculation','limit':'Limits and annotation issues','source':'Frozen source','version':'Measured version or service identity','no_review':f'Targeted anomaly review covers {reviewed_cases} pages and {reviewed_results} outputs; the other {unreviewed_results:,} results have not been individually reviewed. Deterministic locators are not root causes.'}}
L['zh'].update({'dataset':'评测集','review':'核验记录','adoption':'方法介绍','changes':'变更记录','accounts':'账号与安装'})
L['en'].update({'dataset':'Dataset','review':'Verification','adoption':'Methods compared','changes':'Changes','accounts':'Accounts and setup'})
- L['zh']['headline']='WCXB 主内容提取:五款浏览器、六种配置'
- L['en']['headline']='WCXB main-content extraction: five browsers, six configurations'
+ L['zh']['headline']=f'WCXB 主内容提取:{len(providers)} 种配置'
+ L['en']['headline']=f'WCXB main-content extraction: {len(providers)} configurations'
L['zh']['summary']='同一网页的全部方法结果'
L['en']['summary']='All method results for this page'
- L['zh']['intro']=f'五款浏览器的 {len(providers)} 种配置处理同一批 {case_count} 页冻结网页;Moli 正式版与 PR 修复版分别计分。F1 比较提取词语与人工主内容的重合,不衡量 Markdown 结构或链接可用性。'
- L['en']['intro']=f'{len(providers)} configurations across five browsers processed the same {case_count} frozen pages; released and repaired Moli builds have separate results. F1 measures word overlap with human main content, not Markdown structure or link usability.'
+ L['zh']['intro']=f'{len(providers)} 种配置处理同一批 {case_count} 页冻结网页;各配置分别计分。F1 比较提取词语与人工主内容的重合,不衡量 Markdown 结构或链接可用性。'
+ L['en']['intro']=f'{len(providers)} configurations processed the same {case_count} frozen pages; each configuration has separate results. F1 measures word overlap with human main content, not Markdown structure or link usability.'
style+='''body{background:#f5f8fc}body.dark{background:#101923}header{background:#fff;color:#1b2c41;border-bottom:1px solid #dce5ee}body.dark header{background:#172534;color:#eaf3ff;border-color:#31475d}header a{color:#1156ae}body.dark header a{color:#92c3ff}header .wrap{max-width:1220px;padding-top:13px;padding-bottom:13px}header strong{font-family:ui-monospace,monospace;color:#1554a0}.wrap{max-width:1220px}.hero{padding:58px 0 46px}.hero h1{font-size:clamp(34px,4.5vw,56px);letter-spacing:-.035em}.hero p{font-size:17px}.eyebrow{font:700 12px ui-monospace,monospace;letter-spacing:.1em;color:#1457ac}.callout{padding:16px 20px;border-left:4px solid #21845b;background:#eaf8f2;border-radius:0 12px 12px 0;font-weight:650}.panel{box-shadow:0 8px 24px rgba(25,56,88,.04)}.lead-grid{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:14px}.lead-grid .panel{margin:0}.lead-grid b{display:block;color:#1457ac;margin-bottom:7px}.section-lead{max-width:840px;color:#50657c}.winner{color:#117647;font-weight:800}body.dark .winner{color:#8fe3b5}.tabs{display:flex;gap:8px;flex-wrap:wrap}.tabs a{padding:7px 10px;border-radius:8px;background:#e9f0f8;text-decoration:none}.evidence-links{display:flex;gap:12px;flex-wrap:wrap}.reader-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:18px}@media(max-width:700px){.lead-grid,.reader-grid{grid-template-columns:1fr}header nav{gap:8px 14px}header .wrap{display:block}header strong{display:block;margin-bottom:8px}.hero{padding:34px 0 24px}}'''
style+='''.hero{position:relative;isolation:isolate;padding:70px 0 54px}.hero:before{content:"";position:absolute;left:50%;top:0;transform:translateX(-50%);height:100%;width:100vw;background:#f1f6ff;border-bottom:1px solid #dbe5f1;z-index:-1}body.dark .hero:before{background:#152638;border-color:#31465c}body.dark .callout{background:#163b31;color:#e5fff0}body.dark .lead-grid b{color:#9bc6ff}.panel p{overflow-wrap:anywhere}header button{border-color:#cbd9e8;background:#fff;color:#243a52}.scroll{scrollbar-gutter:stable}h2{letter-spacing:-.02em}.hero+.callout{margin-top:26px;margin-bottom:42px}@media(max-width:700px){.hero{padding:42px 0 30px}.hero+.callout{margin-top:18px}}'''
style+='''.flow{overflow:auto;padding:14px;border:1px solid #dbe3eb;border-radius:12px;background:#fff}.flow .mermaid{max-height:none;min-width:920px;background:transparent;color:inherit}.flow-fallback{display:flex;align-items:center;gap:8px;min-width:820px}.flow-fallback .tag{border-radius:9px;padding:8px 12px;white-space:nowrap}@media(max-width:700px){.flow-fallback{min-width:0;display:grid;grid-template-columns:1fr}.flow-fallback .arrow{text-align:center;transform:rotate(90deg)}}'''
@@ -140,8 +140,8 @@ def diag(lang,value):return diag_labels[lang].get(value,value.replace('_',' '))
type_labels={'article':'文章','service':'服务','product':'商品','collection':'合集','forum':'论坛','listing':'列表','documentation':'文档'}
def type_label(lang,value):return type_labels.get(value,value) if lang=='zh' else value
def browser_label(provider):
- if provider=='moli-pr643':return 'Moli PR #643 · 384d1d2a'
- if provider=='moli':return 'Moli 1.1.8'
+ if provider=='moli-pr643':return 'Moli candidate · '+str(profile.get('candidate_source_commit') or '')[:8]
+ if provider=='moli':return 'Moli '+str(identities[provider].get('reported_version') or profile['versions'][provider])
version=identities[provider].get('reported_version') or ''
match=re.search(r'\d+(?:\.\d+)+',version)
number=(match.group(0).split('.')[0] if provider=='chromium' else match.group(0)) if match else ''
@@ -231,9 +231,9 @@ def score_digits(values, direction='max'):
flow_source='flowchart LR\n A["'+flow_labels[0]+'"] --> B["'+flow_labels[1]+'"]\n B --> C["'+flow_labels[2]+'"]\n C --> D["'+flow_labels[3]+'"]\n D --> E["'+flow_labels[4]+'"]\n C -- '+('失败重试' if lang=='zh' else 'failure retry')+' --> B'
flow='
'+('一页怎样变成一行结果' if lang=='zh' else 'From one page to one result')+'
'+' → '.join(f'{esc(label)}' for label in flow_labels)+'
'+esc(flow_source)+'
'
if lang=='zh':
- method=f'
评测方法
{case_count} 份冻结 HTML、六种执行配置、一套官方词语重合评分。
'+flow+f'
输入与运行边界
WCXB v1.0 test 分区中,按冻结输入和人工参考核定的 {case_count} 页进入本报告。六种配置读取完全相同的冻结 HTML;本地五种配置通过隔离的回环 HTTPS 服务读取,Kitesurf 通过 Cloudflare HTML 输入接口读取。外部子资源被阻断。每页一次逻辑调用;失败最多再试一次,全部物理调用保留并累计端到端耗时,最后一次响应是唯一正式结果。
All {case_count} frozen HTML pages, six configurations, one official word-overlap evaluator.
'+flow+f'
Inputs and execution
The {case_count} WCXB v1.0 test pages qualified by frozen input and human reference enter this report. The six configurations read the same frozen HTML. Five local configurations use an isolated loopback HTTPS fixture; Kitesurf receives HTML through Cloudflare’s input field. External subresources are blocked. Each page has one logical result; a failed call may be retried once. Every physical call and its latency are retained, with the final response authoritative.
Kitesurf, Lightpanda, Obscura and Moli return native Markdown. Chromium renders the full DOM and converts it with node-html-markdown 2.0.0. No shared main-content selector is applied.
Denominators and score
The official evaluator lowercases and tokenizes text, then computes word-frequency precision, recall and F1. Quality compares only the {quality_count} references qualified before calls, with equal page weights. F1 does not score paragraph order, Markdown structure or table layout. Required and forbidden snippets are separate diagnostics outside F1.
Quality and latency use the same {case_count} pages. Released and PR-build Moli calls each retain their own immutable evidence.
Locators and latency
Request errors, empty outputs and error pages are separate locators. Precision or recall below 25% on nonempty references triggers a low-value locator; this is a filter, not an official WCXB pass threshold. Latency includes calls and failed retries, not pure browser-core time; local and remote routes differ.
'
+ method=f'
Method
All {case_count} frozen HTML pages, {len(providers)} configurations, one official word-overlap evaluator.
'+flow+f'
Inputs and execution
The {case_count} WCXB v1.0 test pages qualified by frozen input and human reference enter this report. The {len(providers)} configurations read the same frozen HTML. Methods compared lists each interface route and version. External subresources are blocked. Each page has one logical result; a failed call may be retried once. Every physical call and its latency are retained, with the final response authoritative.
Each method retains its native conversion route without a shared main-content selector; Methods compared lists its interface contract and version.
Denominators and score
The official evaluator lowercases and tokenizes text, then computes word-frequency precision, recall and F1. Quality compares only the {quality_count} references qualified before calls, with equal page weights. F1 does not score paragraph order, Markdown structure or table layout. Required and forbidden snippets are separate diagnostics outside F1.
Quality and latency use the same {case_count} pages. Every configuration retains its own immutable call evidence.
Locators and latency
Request errors, empty outputs and error pages are separate locators. Precision or recall below 25% on nonempty references triggers a low-value locator; this is a filter, not an official WCXB pass threshold. Latency includes calls and failed retries, not pure browser-core time; local and remote routes differ.
'
guide=report_content['guide'][lang]
method+='
'+('片段标注边界' if lang=='zh' else 'Snippet annotation boundary')+'
'+((f'{audit["without_present_in_main_content"]} 个应排除片段同时出现在人工主内容参考中。官方片段分照原程序保留,但这类冲突不用于判断浏览器错误;逐页详情会明确标注冲突。') if lang=='zh' else f'{audit["without_present_in_main_content"]} forbidden snippets also appear in their human main-content references. Official snippet values are retained, but these conflicts do not diagnose browser errors; result pages identify them explicitly.')+'
'
page(lang,'methodology/index.html',x['method'],method)
@@ -250,29 +250,29 @@ def score_digits(values, direction='max'):
f'
')
- dataset_intro=(f'本轮任务是从 {case_count} 页冻结 HTML 提取主内容。这里列出输入和人工参考;六种配置的实测分数在结果分析中。' if lang=='zh' else f'The task is to extract main content from {case_count} frozen HTML pages. This inventory owns inputs and human references; the measured browser scores are in Results analysis.')
+ dataset_intro=(f'本轮任务是从 {case_count} 页冻结 HTML 提取主内容。这里列出输入和人工参考;{len(providers)} 种配置的实测分数在结果分析中。' if lang=='zh' else f'The task is to extract main content from {case_count} frozen HTML pages. This inventory owns inputs and human references; the measured browser scores are in Results analysis.')
dataset=f'
{esc(x["dataset"])}
{esc(dataset_intro)}
{"冻结网页与参考" if lang=="zh" else "Frozen pages and references"}
{"人工参考是源评测集标注的主内容纯文本;并非完整 Markdown 标准答案。字符数只描述标注范围,不代表网页正文完整性。" if lang=="zh" else "The source benchmark provides human-selected main-content plain text, not a complete Markdown answer. Character count describes annotation scope, not page completeness."}
{case_count} / {case_count} {"页" if lang=="zh" else "pages"}
ID
{esc(x["type"])}
{"原网页地址" if lang=="zh" else "Original URL"}
{"参考字符数" if lang=="zh" else "Reference characters"}
{"冻结输入" if lang=="zh" else "Frozen input"}
{"人工参考" if lang=="zh" else "Human reference"}
{"评测结果" if lang=="zh" else "Results"}
{"".join(dataset_rows)}
'
page(lang,'datasets/index.html',x['dataset'],dataset)
review_intro=(f'认证覆盖 {result_count:,}/{result_count:,} 条逻辑结果;源文件在调用前后哈希相同。典型异常审查核对 {reviewed_cases} 页、{reviewed_results} 条输出,其余 {unreviewed_results:,} 条未逐项人工审查。' if lang=='zh' else f'Certification covers {result_count:,}/{result_count:,} logical results; source bytes matched before and after all calls. Targeted anomaly review inspected {reviewed_cases} pages and {reviewed_results} outputs; the other {unreviewed_results:,} results were not individually reviewed.')
- review_zh=f"当前评测范围按冻结 HTML 和人工参考的有效性确定,不依据浏览器输出或得分。六种配置在同一批 {case_count} 页上比较。"
- review_en=f"The current evaluation population was qualified from frozen HTML and human references before reading browser outputs or scores. All six configurations are compared on the same {case_count} pages."
+ review_zh=f"当前评测范围按冻结 HTML 和人工参考的有效性确定,不依据浏览器输出或得分。{len(providers)} 种配置在同一批 {case_count} 页上比较。"
+ review_en=f"The current evaluation population was qualified from frozen HTML and human references before reading browser outputs or scores. All {len(providers)} configurations are compared on the same {case_count} pages."
review=f'
{esc(x["review"])}
{esc(review_intro)}
{"源数据资格检查" if lang=="zh" else "Source qualification"}
{esc(profile["versions"][p] if p=="moli-pr643" else identities[p].get("reported_version") or identities[p].get("endpoint") or "service version undisclosed")}
{esc(route_descriptions[p][0 if lang=="zh" else 1])}
{esc(identities[p].get("binary_sha256") or "hosted")}
' for p in providers)
- interface_example='Kitesurf: POST Browser Run /markdown {html: }\nLightpanda / Obscura / Moli: CLI fetch → Markdown\nChromium: Playwright page content → node-html-markdown 2.0.0'
- adoption=f'
{esc(x["adoption"])}
{"六种实际执行配置、版本、输入路线和输出契约。" if lang=="zh" else "Six measured configurations, versions, input routes and output contracts."}
{"本轮实现" if lang=="zh" else "Measured implementations"}
{esc(x["provider"])}
{esc(x["version"])}
{"路线" if lang=="zh" else "Route"}
SHA-256
{versions}
{"接口示例" if lang=="zh" else "Interface examples"}
{esc(interface_example)}
{"此表列的是实测接口,不把公开仓库搜索命中数当作产品采用量。" if lang=="zh" else "These are measured interfaces; repository search hits are not treated as product adoption counts."}
'
- adoption+='
'+('公开代码采用线索' if lang=='zh' else 'Public-code adoption signals')+'
'+((f'{adoption_data["generated_at"][:10]} 的签名核验样本中,Cloudflare /markdown 接口在 {cloudflare_repo_count} 个仓库、{cloudflare_file_count} 个文件出现。该签名没有区分 Kitesurf 后端,也不能推算实际用户数。本轮其余五种 CLI / 浏览器配置没有同口径签名统计,不作采用量排名。') if lang=='zh' else f'In the {adoption_data["generated_at"][:10]} verified signature sample, the Cloudflare /markdown interface appears in {cloudflare_repo_count} repositories and {cloudflare_file_count} files. This signature does not identify the Kitesurf backend or user count. The other five CLI/browser configurations have no matching signature inventory, so no adoption ranking is made.')+'
'
+ interface_example='\n'.join(p+': '+('POST Browser Run /markdown {html: }' if p=='kitesurf' else 'Playwright page content → node-html-markdown 2.0.0' if p=='chromium' else 'CLI fetch → Markdown') for p in providers)
+ adoption=f'
{esc(x["adoption"])}
{"各配置的版本、输入路线和输出契约。" if lang=="zh" else "Measured configurations, versions, input routes and output contracts."}
{"本轮实现" if lang=="zh" else "Measured implementations"}
{esc(x["provider"])}
{esc(x["version"])}
{"路线" if lang=="zh" else "Route"}
SHA-256
{versions}
{"接口示例" if lang=="zh" else "Interface examples"}
{esc(interface_example)}
{"此表列的是实测接口,不把公开仓库搜索命中数当作产品采用量。" if lang=="zh" else "These are measured interfaces; repository search hits are not treated as product adoption counts."}
'
+ adoption+='
'+('公开代码采用线索' if lang=='zh' else 'Public-code adoption signals')+'
'+((f'{adoption_data["generated_at"][:10]} 的签名核验样本中,Cloudflare /markdown 接口在 {cloudflare_repo_count} 个仓库、{cloudflare_file_count} 个文件出现。该签名没有区分 Kitesurf 后端,也不能推算实际用户数。本轮其余配置没有同口径签名统计,不作采用量排名。') if lang=='zh' else f'In the {adoption_data["generated_at"][:10]} verified signature sample, the Cloudflare /markdown interface appears in {cloudflare_repo_count} repositories and {cloudflare_file_count} files. This signature does not identify the Kitesurf backend or user count. The other configurations have no matching signature inventory, so no adoption ranking is made.')+'
'
page(lang,'providers/'+p+'.html',p,profile_body)
- changes_zh=f"六种配置完成全部 {case_count} 页调用;质量展示只使用事前核定的 {quality_count} 页有效参考,共 {visible_count:,} 条结果。端到端耗时保留每款 {case_count} 次调用,以免事后剔除慢或失败的调用。"
- changes_en=f"All six configurations completed {case_count} calls. Quality displays use only {quality_count} references qualified before calls, or {visible_count:,} results. End-to-end latency retains all {case_count} calls per method so slow or failed calls are not removed after measurement."
+ changes_zh=f"{len(providers)} 种配置完成全部 {case_count} 页调用;质量展示只使用事前核定的 {quality_count} 页有效参考,共 {visible_count:,} 条结果。端到端耗时保留每款 {case_count} 次调用,以免事后剔除慢或失败的调用。"
+ changes_en=f"All {len(providers)} configurations completed {case_count} calls. Quality displays use only {quality_count} references qualified before calls, or {visible_count:,} results. End-to-end latency retains all {case_count} calls per method so slow or failed calls are not removed after measurement."
changes=f'
{esc(x["changes"])}
{"当前报告的结果、分母和证据范围。" if lang=="zh" else "Current result, denominator and evidence scope."}
{"有效参考与完整调用" if lang=="zh" else "Qualified references and all calls"}
{esc(HEADLESS_ACCESS[p]["account"] if lang=="zh" else ("Cloudflare Browser Run account and API token required." if p=="kitesurf" else "No provider account or API key required."))}
{esc(HEADLESS_ACCESS[p]["account"] if lang=="zh" else ("Cloudflare Browser Run account and API token required." if p=="kitesurf" else "No provider account or API key required."))}
{"只有 Kitesurf 需要 Cloudflare 账号凭据;本地配置值不进入报告。" if lang=="zh" else "Only Kitesurf needs Cloudflare account credentials; local values are not published."}
{esc(x["provider"])}
{"环境变量" if lang=="zh" else "Environment variables"}
PR #643 separates joined code values and keeps product details when optional UI stripping is used. Per-page F1 rises on 5412 and 5413, falls on 5411 and 5458, and is unchanged on the other {same} pages. Mean recall changes {recall_delta:+.4f} percentage points, mean precision {precision_delta:+.4f}, and mean F1 {f1_delta:+.4f}. F1 is the mean of the {quality_count} page-level F1 values, not a recomputation from the headline mean precision and recall. The two decreases are slightly larger, so mean recall rises while average F1 falls slightly; on 5458, raw scoring also counts the inline HTML tag names in the precision denominator.
')
+ example_case=max((c for c in by_case if c not in exclusion_ids), key=lambda c: abs(by_case[c]['moli-pr643']['f1']-by_case[c]['moli']['f1']))
+ relation_counts={'higher':up,'lower':down,'equal':same}
+ relation_labels=({'higher':'上升','lower':'下降','equal':'不变'} if lang=='zh' else {'higher':'higher','lower':'lower','equal':'unchanged'})
+ change_counts=' · '.join(f'{count} {relation_labels[relation]}' for relation,count in relation_counts.items())
+ paired=(f'
F1 across {quality_count} pages: {change_counts}. Mean recall changes {recall_delta:+.4f} percentage points, mean precision {precision_delta:+.4f}, and mean F1 {f1_delta:+.4f}. F1 is the mean of page-level F1 values, not a recomputation from mean precision and recall.
Moli 1.1.8 and the measured PR were measured in a balanced same-window crossover on all {case_count} pages. The preselected final round gives {moli_before["mean_latency_ms"]:,.0f} ms for the release and {moli_after["mean_latency_ms"]:,.0f} ms for the PR, a {paired_latency["paired_delta_mean_ms"]:+.1f} ms page-paired difference. The codebases diverge, so this difference cannot all be attributed to this PR. Read the timing basis
{esc(browser_label("moli"))} and the measured candidate were measured in a balanced same-window crossover on all {case_count} pages. The preselected final round gives {moli_before["mean_latency_ms"]:,.0f} ms for the release and {moli_after["mean_latency_ms"]:,.0f} ms for the PR, a {paired_latency["paired_delta_mean_ms"]:+.1f} ms page-paired difference. The codebases diverge, so this difference cannot all be attributed to this PR. Read the timing basis
')
lead=(f'{browser_label(best)} 在 {quality_count} 页有效参考上的平均 F1 最高({fmt(summary["providers"][best]["quality_f1"])});{browser_label(fast)} 的 {case_count} 次调用平均端到端耗时最短({summary["providers"][fast]["mean_latency_ms"]:,.0f} ms)。' if lang=='zh' else f'{browser_label(best)} leads mean F1 across {quality_count} qualified pages ({fmt(summary["providers"][best]["quality_f1"])}); {browser_label(fast)} has the shortest mean end-to-end latency across {case_count} calls ({summary["providers"][fast]["mean_latency_ms"]:,.0f} ms).')
type_counts=WordCounter(gt['page_type'] for c,gt in ground_truth.items() if c not in exclusion_ids)
slice_values={t:{p:sum(by_case[c][p]['f1'] for c,g in ground_truth.items() if g['page_type']==t and c not in exclusion_ids)/type_counts[t] for p in providers} for t in type_counts}
@@ -231,9 +235,9 @@ def score_digits(values, direction='max'):
flow_source='flowchart LR\n A["'+flow_labels[0]+'"] --> B["'+flow_labels[1]+'"]\n B --> C["'+flow_labels[2]+'"]\n C --> D["'+flow_labels[3]+'"]\n D --> E["'+flow_labels[4]+'"]\n C -- '+('失败重试' if lang=='zh' else 'failure retry')+' --> B'
flow='
'+('一页怎样变成一行结果' if lang=='zh' else 'From one page to one result')+'
'+' → '.join(f'{esc(label)}' for label in flow_labels)+'
All {case_count} frozen HTML pages, {len(providers)} configurations, one official word-overlap evaluator.
'+flow+f'
Inputs and execution
The {case_count} WCXB v1.0 test pages qualified by frozen input and human reference enter this report. The {len(providers)} configurations read the same frozen HTML. Methods compared lists each interface route and version. External subresources are blocked. Each page has one logical result; a failed call may be retried once. Every physical call and its latency are retained, with the final response authoritative.
Each method retains its native conversion route without a shared main-content selector; Methods compared lists its interface contract and version.
Denominators and score
The official evaluator lowercases and tokenizes text, then computes word-frequency precision, recall and F1. Quality compares only the {quality_count} references qualified before calls, with equal page weights. F1 does not score paragraph order, Markdown structure or table layout. Required and forbidden snippets are separate diagnostics outside F1.
Quality and latency use the same {case_count} pages. Every configuration retains its own immutable call evidence.
Locators and latency
Request errors, empty outputs and error pages are separate locators. Precision or recall below 25% on nonempty references triggers a low-value locator; this is a filter, not an official WCXB pass threshold. Latency includes calls and failed retries, not pure browser-core time; local and remote routes differ.
'
+ method=f'
Method
All {case_count} frozen HTML pages, {len(providers)} configurations, one official word-overlap evaluator.
'+flow+f'
Inputs and execution
The {case_count} WCXB v1.0 test pages qualified by frozen input and human reference enter this report. The {len(providers)} configurations read the same frozen HTML. Methods compared lists each interface route and version. External subresources are blocked. Each page has one logical result; a failed call allows up to {profile.get("max_failure_retries", 0)} retries. Every physical call and its latency are retained, with the final response authoritative.
Each method retains its native conversion route without a shared main-content selector; Methods compared lists its interface contract and version.
Denominators and score
The official evaluator lowercases and tokenizes text, then computes word-frequency precision, recall and F1. Quality compares only the {quality_count} references qualified before calls, with equal page weights. F1 does not score paragraph order, Markdown structure or table layout. Required and forbidden snippets are separate diagnostics outside F1.
Quality and latency use the same {case_count} pages. Every configuration retains its own immutable call evidence.
Locators and latency
Request errors, empty outputs and error pages are separate locators. Precision or recall below 25% on nonempty references triggers a low-value locator; this is a filter, not an official WCXB pass threshold. Latency includes calls and failed retries, not pure browser-core time; local and remote routes differ.
'
guide=report_content['guide'][lang]
method+='
'+('片段标注边界' if lang=='zh' else 'Snippet annotation boundary')+'
'+((f'{audit["without_present_in_main_content"]} 个应排除片段同时出现在人工主内容参考中。官方片段分照原程序保留,但这类冲突不用于判断浏览器错误;逐页详情会明确标注冲突。') if lang=='zh' else f'{audit["without_present_in_main_content"]} forbidden snippets also appear in their human main-content references. Official snippet values are retained, but these conflicts do not diagnose browser errors; result pages identify them explicitly.')+'
'
page(lang,'methodology/index.html',x['method'],method)
diff --git a/harness/web_fetch/wcxb_report_check.py b/harness/web_fetch/wcxb_report_check.py
index 92115bd94..b08f62ea0 100644
--- a/harness/web_fetch/wcxb_report_check.py
+++ b/harness/web_fetch/wcxb_report_check.py
@@ -117,7 +117,7 @@ def require_analysis_controls(html: str | BeautifulSoup, result_count: int) -> B
def require_home_interpretation(html: str | BeautifulSoup, summary: dict,
- paired_latency: dict, lang: str) -> BeautifulSoup:
+ paired_latency: dict, lang: str, *, scores: list[dict]) -> BeautifulSoup:
"""Keep the version comparison readable and its timing caveat beside the table."""
soup = BeautifulSoup(html, 'html.parser') if isinstance(html, str) else html
core = soup.select_one('#wcxb-core')
@@ -138,7 +138,7 @@ def require_home_interpretation(html: str | BeautifulSoup, summary: dict,
require(all(value in context_text for value in required),
'Home timing context disagrees with measured evidence')
require(('交叉顺序' if lang == 'zh' else 'balanced same-window') in context_text
- and ('504 页' if lang == 'zh' else '504 pages') in context_text,
+ and (f'{summary["case_count"]} 页' if lang == 'zh' else f'{summary["case_count"]} pages') in context_text,
'Home timing context does not identify the corrected population')
require(context.select_one('a[href$="guide/index.html"]') is not None,
'Home timing context does not link to the analysis')
@@ -146,15 +146,26 @@ def require_home_interpretation(html: str | BeautifulSoup, summary: dict,
change_text = change.get_text(' ', strip=True)
require(len(change_text) <= (330 if lang == 'zh' else 700),
'Home Moli change summary is too long; move detail to the interpretation page')
- require(len(change.select('p')) == 2 and change.select_one('a[href$="cases/5458.html"]')
+ require(len(change.select('p')) == 2 and change.select_one('a[href*="cases/"]')
and change.select_one('a[href$="guide/index.html"]'),
'Home Moli change summary lacks concise evidence links')
- required_change = (('5412、5413', '5411、5458', '逐页 F1 的平均', '内联 HTML 标签名')
- if lang == 'zh' else
- ('5412 and 5413', '5411 and 5458', 'page-level F1 values',
- 'inline HTML tag names'))
- require(all(marker in change_text for marker in required_change),
- 'Home Moli change summary does not explain recall/F1 divergence')
+ by_case = {}
+ for row in scores:
+ if row.get('quality_eligible', True):
+ by_case.setdefault(row['case_id'], {})[row['provider']] = row['f1']
+ deltas = [values['moli-pr643'] - values['moli'] for values in by_case.values()]
+ expected = {'higher': sum(value > 1e-10 for value in deltas),
+ 'lower': sum(value < -1e-10 for value in deltas),
+ 'equal': sum(abs(value) <= 1e-10 for value in deltas)}
+ actual = {node['data-f1-relation']: int(node['data-count'])
+ for node in change.select('[data-f1-relation][data-count]')}
+ require(actual == expected and sum(actual.values()) == summary['quality_case_count'],
+ 'Home page-level change counts differ from scored evidence')
+ for node in change.select('a[href*="cases/"]'):
+ require(Path(urlsplit(node['href']).path).stem in by_case,
+ 'Home comparison links to a case outside the selected evidence')
+ require(('逐页 F1 的平均' if lang == 'zh' else 'page-level F1 values') in change_text,
+ 'Home comparison omits the page-level F1 definition')
precision_delta = ((summary['providers']['moli-pr643']['quality_precision']
- summary['providers']['moli']['quality_precision']) * 100)
recall_delta = ((summary['providers']['moli-pr643']['quality_recall']
@@ -407,7 +418,7 @@ def check(root: Path, *, authority_root: Path | None = None) -> None:
if route in ('index.html', 'datasets/index.html'):
check_presentation(html)
if route == 'index.html':
- require_home_interpretation(html, summary, paired_latency, lang)
+ require_home_interpretation(html, summary, paired_latency, lang, scores=scores)
require('id="home-type"' in html, 'Page-type filter missing')
require('官方全量 F1' not in html and 'Official full F1' not in html, 'Invalid full-reference score remains visible')
first_card = BeautifulSoup(html, 'html.parser').select_one('.lead-grid .panel')
@@ -574,7 +585,7 @@ def check(root: Path, *, authority_root: Path | None = None) -> None:
home = BeautifulSoup((root / 'index.html').read_text(), 'html.parser')
home.select_one('#moli-latency-context').decompose()
try:
- require_home_interpretation(home, summary, paired_latency, 'zh')
+ require_home_interpretation(home, summary, paired_latency, 'zh', scores=scores)
except ValueError:
pass
else:
diff --git a/tests/test_wcxb_report_languages.py b/tests/test_wcxb_report_languages.py
index a132f7a22..bd2cd655b 100644
--- a/tests/test_wcxb_report_languages.py
+++ b/tests/test_wcxb_report_languages.py
@@ -112,6 +112,7 @@ def test_wcxb_instance_counts_versions_and_interfaces_follow_selected_evidence(t
profile['providers'] = list(providers)
identities['moli']['reported_version'] = '2.3.4'
profile['candidate_source_commit'] = 'abcdef12' + '0' * 32
+ profile['max_failure_retries'] = 3
generate_wcxb(root, rows, summary, truth, identities, profile, content, include_english=True)
for prefix, count_text in [('', f'{len(providers)} 种配置'), ('en/', f'{len(providers)} configurations')]:
home = BeautifulSoup((root / prefix / 'index.html').read_text(), 'html.parser')
@@ -129,3 +130,15 @@ def test_wcxb_instance_counts_versions_and_interfaces_follow_selected_evidence(t
interfaces = BeautifulSoup((root / prefix / 'adoption/index.html').read_text(), 'html.parser').select_one('pre').get_text()
assert 'kitesurf:' not in interfaces
assert ('chromium:' in interfaces) == ('chromium' in providers)
+ from harness.web_fetch.wcxb_latency import load_full_paired_latency, summarize_full_paired_latency
+ from harness.web_fetch.wcxb_report_check import require_home_interpretation
+ _, calls = load_full_paired_latency(root / 'research/wcxb-full-paired-latency', profile, set(truth))
+ paired = summarize_full_paired_latency(calls, set(truth))
+ lang = 'zh' if not prefix else 'en'
+ html = (root / prefix / 'index.html').read_text()
+ checked = require_home_interpretation(html, summary, paired, lang, scores=rows)
+ checked.select_one('[data-f1-relation="higher"]')['data-count'] = '99'
+ with pytest.raises(ValueError, match='counts differ'):
+ require_home_interpretation(checked, summary, paired, lang, scores=rows)
+ assert ('最多重试 3 次' if lang == 'zh' else 'up to 3 retries') in method
+ assert 'cases/5458.html' not in html
From 4f348966c5db808dc3058593c19164d83445620a Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Wed, 21 Oct 2026 11:47:08 +0800
Subject: [PATCH 16/51] fix(webfetch): align WCXB instance counts across
languages
---
harness/web_fetch/report.py | 4 ++--
tests/test_wcxb_report_languages.py | 6 ++++++
2 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py
index d4e47fbeb..f51479398 100644
--- a/harness/web_fetch/report.py
+++ b/harness/web_fetch/report.py
@@ -237,7 +237,7 @@ def score_digits(values, direction='max'):
if lang=='zh':
method=f'
All {case_count} frozen HTML pages, {len(providers)} configurations, one official word-overlap evaluator.
'+flow+f'
Inputs and execution
The {case_count} WCXB v1.0 test pages qualified by frozen input and human reference enter this report. The {len(providers)} configurations read the same frozen HTML. Methods compared lists each interface route and version. External subresources are blocked. Each page has one logical result; a failed call allows up to {profile.get("max_failure_retries", 0)} retries. Every physical call and its latency are retained, with the final response authoritative.
Each method retains its native conversion route without a shared main-content selector; Methods compared lists its interface contract and version.
Denominators and score
The official evaluator lowercases and tokenizes text, then computes word-frequency precision, recall and F1. Quality compares only the {quality_count} references qualified before calls, with equal page weights. F1 does not score paragraph order, Markdown structure or table layout. Required and forbidden snippets are separate diagnostics outside F1.
Quality and latency use the same {case_count} pages. Every configuration retains its own immutable call evidence.
Locators and latency
Request errors, empty outputs and error pages are separate locators. Precision or recall below 25% on nonempty references triggers a low-value locator; this is a filter, not an official WCXB pass threshold. Latency includes calls and failed retries, not pure browser-core time; local and remote routes differ.
'
+ method=f'
Method
All {case_count} frozen HTML pages, {len(providers)} configurations, one official word-overlap evaluator.
'+flow+f'
Inputs and execution
The {case_count} WCXB v1.0 test pages qualified by frozen input and human reference enter this report. The {len(providers)} configurations read the same frozen HTML. Methods compared lists each interface route and version. External subresources are blocked. Each page has one logical result; a failed call allows up to {profile.get("max_failure_retries", 0)} retries. Every physical call and its latency are retained, with the final response authoritative.
Each method retains its native conversion route without a shared main-content selector; Methods compared lists its interface contract and version.
Denominators and score
The official evaluator lowercases and tokenizes text, then computes word-frequency precision, recall and F1. Quality compares only the {quality_count} references qualified before calls, with equal page weights. F1 does not score paragraph order, Markdown structure or table layout. Required and forbidden snippets are separate diagnostics outside F1.
Quality and latency use the same {case_count} pages. Every configuration retains its own immutable call evidence.
Locators and latency
Request errors, empty outputs and error pages are separate locators. Precision or recall below 25% on nonempty references triggers a low-value locator; this is a filter, not an official WCXB pass threshold. Latency includes calls and failed retries, not pure browser-core time; the {len(providers)} configurations use different local and remote routes.
'
guide=report_content['guide'][lang]
method+='
'+('片段标注边界' if lang=='zh' else 'Snippet annotation boundary')+'
'+((f'{audit["without_present_in_main_content"]} 个应排除片段同时出现在人工主内容参考中。官方片段分照原程序保留,但这类冲突不用于判断浏览器错误;逐页详情会明确标注冲突。') if lang=='zh' else f'{audit["without_present_in_main_content"]} forbidden snippets also appear in their human main-content references. Official snippet values are retained, but these conflicts do not diagnose browser errors; result pages identify them explicitly.')+'
'
page(lang,'methodology/index.html',x['method'],method)
@@ -254,7 +254,7 @@ def score_digits(values, direction='max'):
f'
')
- dataset_intro=(f'本轮任务是从 {case_count} 页冻结 HTML 提取主内容。这里列出输入和人工参考;{len(providers)} 种配置的实测分数在结果分析中。' if lang=='zh' else f'The task is to extract main content from {case_count} frozen HTML pages. This inventory owns inputs and human references; the measured browser scores are in Results analysis.')
+ dataset_intro=(f'本轮任务是从 {case_count} 页冻结 HTML 提取主内容。这里列出输入和人工参考;{len(providers)} 种配置的实测分数在结果分析中。' if lang=='zh' else f'The task is to extract main content from {case_count} frozen HTML pages. This inventory owns inputs and human references; scores for the {len(providers)} measured configurations are in Results analysis.')
dataset=f'
{esc(x["dataset"])}
{esc(dataset_intro)}
{"冻结网页与参考" if lang=="zh" else "Frozen pages and references"}
{"人工参考是源评测集标注的主内容纯文本;并非完整 Markdown 标准答案。字符数只描述标注范围,不代表网页正文完整性。" if lang=="zh" else "The source benchmark provides human-selected main-content plain text, not a complete Markdown answer. Character count describes annotation scope, not page completeness."}
{case_count} / {case_count} {"页" if lang=="zh" else "pages"}
ID
{esc(x["type"])}
{"原网页地址" if lang=="zh" else "Original URL"}
{"参考字符数" if lang=="zh" else "Reference characters"}
{"冻结输入" if lang=="zh" else "Frozen input"}
{"人工参考" if lang=="zh" else "Human reference"}
{"评测结果" if lang=="zh" else "Results"}
{"".join(dataset_rows)}
'
page(lang,'datasets/index.html',x['dataset'],dataset)
review_intro=(f'认证覆盖 {result_count:,}/{result_count:,} 条逻辑结果;源文件在调用前后哈希相同。典型异常审查核对 {reviewed_cases} 页、{reviewed_results} 条输出,其余 {unreviewed_results:,} 条未逐项人工审查。' if lang=='zh' else f'Certification covers {result_count:,}/{result_count:,} logical results; source bytes matched before and after all calls. Targeted anomaly review inspected {reviewed_cases} pages and {reviewed_results} outputs; the other {unreviewed_results:,} results were not individually reviewed.')
diff --git a/tests/test_wcxb_report_languages.py b/tests/test_wcxb_report_languages.py
index bd2cd655b..d33e9f8fd 100644
--- a/tests/test_wcxb_report_languages.py
+++ b/tests/test_wcxb_report_languages.py
@@ -125,6 +125,12 @@ def test_wcxb_instance_counts_versions_and_interfaces_follow_selected_evidence(t
assert '六种配置' not in text
assert 'five browsers' not in text
assert '五款浏览器' not in text
+ if not prefix:
+ from harness.web_fetch.wcxb_report_check import ReportShape, require_bilingual_shape
+ zh, en = ReportShape(), ReportShape()
+ zh.feed(text)
+ en.feed((root / 'en' / route).read_text())
+ require_bilingual_shape(zh, en, route)
method = (root / prefix / 'methodology/index.html').read_text()
assert 'Kitesurf receives' not in method
interfaces = BeautifulSoup((root / prefix / 'adoption/index.html').read_text(), 'html.parser').select_one('pre').get_text()
From df6575b1090c4c3673c9b0a219b5c571a944f912 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Wed, 21 Oct 2026 11:50:25 +0800
Subject: [PATCH 17/51] fix(webfetch): normalize measured version labels
---
harness/web_fetch/report.py | 3 +--
tests/test_wcxb_report_languages.py | 3 ++-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py
index f51479398..a29b52bb7 100644
--- a/harness/web_fetch/report.py
+++ b/harness/web_fetch/report.py
@@ -141,8 +141,7 @@ def diag(lang,value):return diag_labels[lang].get(value,value.replace('_',' '))
def type_label(lang,value):return type_labels.get(value,value) if lang=='zh' else value
def browser_label(provider):
if provider=='moli-pr643':return 'Moli candidate · '+str(profile.get('candidate_source_commit') or '')[:8]
- if provider=='moli':return 'Moli '+str(identities[provider].get('reported_version') or profile['versions'][provider])
- version=identities[provider].get('reported_version') or ''
+ version=str(identities[provider].get('reported_version') or profile['versions'].get(provider) or '')
match=re.search(r'\d+(?:\.\d+)+',version)
number=(match.group(0).split('.')[0] if provider=='chromium' else match.group(0)) if match else ''
return provider.capitalize()+(' '+number if number else '')
diff --git a/tests/test_wcxb_report_languages.py b/tests/test_wcxb_report_languages.py
index d33e9f8fd..f9822d48a 100644
--- a/tests/test_wcxb_report_languages.py
+++ b/tests/test_wcxb_report_languages.py
@@ -110,7 +110,7 @@ def test_wcxb_instance_counts_versions_and_interfaces_follow_selected_evidence(t
rows = [row for row in rows if row['provider'] in providers]
summary['providers'] = {p: summary['providers'][p] for p in providers}
profile['providers'] = list(providers)
- identities['moli']['reported_version'] = '2.3.4'
+ identities['moli']['reported_version'] = 'moli 2.3.4'
profile['candidate_source_commit'] = 'abcdef12' + '0' * 32
profile['max_failure_retries'] = 3
generate_wcxb(root, rows, summary, truth, identities, profile, content, include_english=True)
@@ -118,6 +118,7 @@ def test_wcxb_instance_counts_versions_and_interfaces_follow_selected_evidence(t
home = BeautifulSoup((root / prefix / 'index.html').read_text(), 'html.parser')
assert count_text in home.select_one('h1').get_text()
assert 'Moli 2.3.4' in home.get_text()
+ assert 'Moli moli' not in home.get_text()
assert 'abcdef12' in home.get_text()
for route in ('index.html', 'methodology/index.html', 'datasets/index.html', 'changes/index.html'):
text = (root / prefix / route).read_text()
From be79c1a7ef223652c3a7eaf5c12aedc028bfb327 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Wed, 21 Oct 2026 18:25:57 +0800
Subject: [PATCH 18/51] fix(reporting): support external instances and disclose
acceptance scope
---
.agents/skills/benchmark-report/SKILL.md | 7 +
docs/benchmarks.md | 27 +-
evaluation/lexbench-webfetch.json | 4 +-
evaluation/wcxb/current-report.json | 11 +-
harness/web_fetch/README.md | 8 +-
harness/web_fetch/artifacts.py | 18 +-
harness/web_fetch/dataset_assembly.py | 17 +-
harness/web_fetch/reliability.py | 9 +-
harness/web_fetch/report.py | 56 ++-
harness/web_fetch/report_build.py | 126 +++---
harness/web_fetch/report_check.py | 6 +-
harness/web_fetch/report_components.py | 87 ++++
harness/web_fetch/report_methods.py | 9 +-
harness/web_fetch/wcxb_publication.py | 13 +-
harness/web_fetch/wcxb_report_check.py | 502 ++++++++++++-----------
pyproject.toml | 2 +-
scripts/build_report.py | 4 +-
tests/test_acceptance_scope.py | 43 ++
tests/test_dataset_assembly.py | 16 +
tests/test_file_scoring_evidence.py | 37 ++
tests/test_headless_report.py | 4 +-
tests/test_reliability.py | 13 +
tests/test_report_methods.py | 6 +-
tests/test_wcxb_participant_registry.py | 91 ++++
tests/wcxb_fixture.py | 1 +
uv.lock | 4 +-
26 files changed, 775 insertions(+), 346 deletions(-)
create mode 100644 tests/test_acceptance_scope.py
create mode 100644 tests/test_file_scoring_evidence.py
create mode 100644 tests/test_wcxb_participant_registry.py
diff --git a/.agents/skills/benchmark-report/SKILL.md b/.agents/skills/benchmark-report/SKILL.md
index 596f0552d..3f83ad8c7 100644
--- a/.agents/skills/benchmark-report/SKILL.md
+++ b/.agents/skills/benchmark-report/SKILL.md
@@ -17,6 +17,13 @@ do not choose a historical article from provider names or task-family guesses.
Without a content binding, render the maintained data-derived explanation.
Prove reuse through the public build and check entries with a different external
task/method inventory, not only by searching for old numbers in source code.
+Exercise assembly and packaging with that external profile as well. Participant
+IDs, labels, interfaces and comparison roles come from the instance registry;
+a new cohort must not inherit a historical method's account or timing claims.
+Disclose declared literal, fact, body-region, record, resource and identity checks
+separately. A task instruction is not a full-text reference. Preserve frozen
+verdicts while exposing passes without verified page identity, blocked-page
+markers and unexpected resource URLs as diagnostics.
An explicitly requested diagnostic report may be delivered separately from a certified benchmark release. Use the diagnostic build/check entry, label its evidence state and certification gaps on every page, preserve saved scores and outputs, and do not publish rankings or upgrade certification. It does not waive any certified-release reference or review gate. Presentation fixes add no provider calls. Update only the owning code, relevant instructions and demonstrated regression scenarios; do not mechanically modify every skill or copy report-specific conclusions into instructions.
diff --git a/docs/benchmarks.md b/docs/benchmarks.md
index 0e28ad562..7bfabcae6 100644
--- a/docs/benchmarks.md
+++ b/docs/benchmarks.md
@@ -58,6 +58,25 @@ Browser Use and Browser Trajectory Replay accept an instance profile at each bou
Git. Select a new output directory; changing participants, sample size or report
content does not require another renderer or command script.
+The native Web Fetch report also accepts an external profile. Bind the frozen
+dataset, complete participant matrix, execution settings and evidence hashes in
+that profile. Put report prose in the evidence tree and bind it through
+`comparison.report_interpretation`; a report refresh does not require changing
+Python strings. The normal build and check retain certification and review gates.
+The diagnostic build is a separate evidence state and does not certify results.
+
+```sh
+uv run --frozen python scripts/build_report.py \
+ --profile /absolute/instance/profile.json \
+ --evidence /absolute/instance/evidence --output /absolute/instance/report
+uv run --frozen python scripts/check_report.py \
+ /absolute/instance/report --profile /absolute/instance/profile.json
+```
+
+A profile stored outside Git has the same validation rules as a repository profile.
+Repository defaults select the maintained published report; they are optional
+convenience paths, not a requirement to commit new report instances.
+
Browser Use profiles register models, browser labels and measurement controls.
`task_selection.path` and `task_selection.sha256` bind the selection file; its hash
must match `selection_sha256`. The selection declares its own `metadata.tier_counts`.
@@ -118,12 +137,18 @@ replay build/package targets also accept `REVIEW`, and packaging accepts `ARCHIV
and `ATTESTATION`. Packaging creates a local archive and does not publish it.
For WCXB, select a profile-bound evidence directory and a fresh output with
-`scripts/build_report.py --profile evaluation/wcxb/current-report.json`. Run
+`scripts/build_report.py --profile /absolute/instance/profile.json`. Keep its
+`report-content-binding.json` beside the profile. `providers` declares the matrix;
+`participants` can supply labels, bilingual routes, interface examples and account
+requirements. An optional `paired_comparison` names `baseline` and `candidate`
+participants. A declared timing correction still requires complete paired evidence;
+ordinary cohorts do not inherit another report’s timing or optional-feature claims. Run
`scripts/check_report.py` with that profile and `scripts/check_wcxb_visual.py`
against the output. The existing publication entry point also supports local
packaging without credentials or a network upload:
```sh
uv run --frozen python scripts/publish_wcxb_artifact_site.py --package-only \
+ --profile /absolute/instance/profile.json \
--source /absolute/instance/report --work /absolute/instance/package
```
diff --git a/evaluation/lexbench-webfetch.json b/evaluation/lexbench-webfetch.json
index 5f2e204dc..319c524f6 100644
--- a/evaluation/lexbench-webfetch.json
+++ b/evaluation/lexbench-webfetch.json
@@ -2,7 +2,7 @@
"schema_version": 1,
"kind": "external_benchmark_binding",
"repository": "https://github.com/lexmount/lexbench-webfetch",
- "revision": "5b12816906ad8cd2b55b9804c77e7e6c55f2c54e",
+ "revision": "1552f731dd75be26c3b3551c04bf0730e6293335",
"ownership": {
"benchmark": "Task populations, task model, acceptance scoring, file checks, compiler and frozen definitions",
"harness": "Execution, reference lifecycle, evidence and reports"
@@ -78,7 +78,7 @@
"evaluation/risk-control/remaining-70-rule-review.md": "301574dddcfddf6d30ed4d882cdd40b1ae859fdc95a0a02cf9868f55e3fd1079",
"evaluation/risk-control/rule-authoring-guide.md": "74363ab502978179e5438a0b992bac7563090e41c91784e492636832dc1cd088",
"evaluation/risk-control/source-inputs.json": "77c5a4b62c07d59ed57154a833e85f9e7a97bc739730f51cd4ca3a698edb9258",
- "evaluation/scoring-regressions.json": "f2e8f96001b7a27da22877676625b4c1514b3ffa26f97601623b46582c240a93",
+ "evaluation/scoring-regressions.json": "572c140d3cec14a00a2a517949ad2aa118d34e03d6eade07d68dcd8476cd0b10",
"evaluation/source-inputs.json": "c202abc4624811b938a837402bbaf0ecbe716e375fbca70567346d2edb3b1708"
}
}
diff --git a/evaluation/wcxb/current-report.json b/evaluation/wcxb/current-report.json
index 222ec0326..fe1265739 100644
--- a/evaluation/wcxb/current-report.json
+++ b/evaluation/wcxb/current-report.json
@@ -58,5 +58,14 @@
"candidate_binary_sha256": "ba5e80975886f3135098146acec6941cee6d4bc65109d2e576db9dfc1e1c0ade",
"release_binary_sha256": "522f752f30397f3b77910736c4ff76f2bdf4cd1a62a2bd14332bbccc97ce26dc",
"latency_correction_receipt_sha256": "701799a5af8fea1ad02f4f4650c695390d2c6571eb69bda115392160e4e34681",
- "optional_ui_receipt_sha256": "6bb78355ec3343c532023dbc0ba809f41554e74874932b6f95581b87f25d7a81"
+ "optional_ui_receipt_sha256": "6bb78355ec3343c532023dbc0ba809f41554e74874932b6f95581b87f25d7a81",
+ "paired_comparison": {
+ "baseline": "moli",
+ "candidate": "moli-pr643"
+ },
+ "participants": {
+ "moli-pr643": {
+ "label": "Moli candidate · 384d1d2a"
+ }
+ }
}
diff --git a/harness/web_fetch/README.md b/harness/web_fetch/README.md
index 692a5422f..7652b60ce 100644
--- a/harness/web_fetch/README.md
+++ b/harness/web_fetch/README.md
@@ -189,15 +189,15 @@ originals, binds any public redaction and runs the maintained release checks.
### Report instances
-Live WebFetch reports are data instances, not renderer subclasses. An instance's
-`evaluation//current-report.json` binds its complete `cohort_plan`,
+Live WebFetch reports are data instances, not renderer subclasses. A diagnostic instance's
+external profile binds its complete `cohort_plan`,
implementation, external `diagnostic_input` (data-home-relative path and SHA-256),
and publication target. The input manifest owns cohort evidence paths, reviewed
findings, explanatory prose and examples. It does not contain rendering code.
```bash
-uv run --frozen python scripts/build_report.py --profile evaluation//current-report.json --output
-uv run --frozen python scripts/check_diagnostic_report.py --profile evaluation//current-report.json
+uv run --frozen python scripts/build_report.py --profile /absolute/instance/profile.json --output
+uv run --frozen python scripts/check_diagnostic_report.py --profile /absolute/instance/profile.json
```
Set `BROWSER_EVAL_DATA_HOME` when the external evidence uses a nondefault root.
diff --git a/harness/web_fetch/artifacts.py b/harness/web_fetch/artifacts.py
index ac271c2c4..700ff45d2 100644
--- a/harness/web_fetch/artifacts.py
+++ b/harness/web_fetch/artifacts.py
@@ -26,9 +26,21 @@ def file_scoring_metadata(case, metadata: dict, source: Path) -> dict:
case = case if isinstance(case, dict) else vars(case)
evaluation = metadata.get("file_evaluation")
- if case.get("task_kind") != "file" or not isinstance(evaluation, dict):
+ if case.get("task_kind") != "file":
return metadata
+ # No delivered-file claim is a normal failed delivery. A malformed saved
+ # claim is corrupted evidence, not a new provider failure or a stale score.
+ if "file_evaluation" not in metadata:
+ return metadata
+ if not isinstance(evaluation, dict):
+ raise ValueError("Delivered file evaluation must be an object")
delivered = evaluation.get("files", [])
+ if not isinstance(delivered, list):
+ raise ValueError("Delivered files must be a list")
+ for item in delivered:
+ if (not isinstance(item, dict) or not isinstance(item.get("artifact_path"), str)
+ or not item["artifact_path"] or not isinstance(item.get("sha256"), str)):
+ raise ValueError("Delivered file requires an artifact_path and sha256 binding")
payloads = []
for item in delivered:
raw = contained_path(source, item["artifact_path"], reject_symlinks=True).read_bytes()
@@ -36,7 +48,9 @@ def file_scoring_metadata(case, metadata: dict, source: Path) -> dict:
raise ValueError("Delivered file integrity failure: " + item["artifact_path"])
payloads.append(raw)
current = evaluate_files(case, payloads)
- current["files"] = [{**old, **new} for old, new in zip(delivered, current["files"])]
+ if len(current["files"]) != len(delivered):
+ raise ValueError("Delivered file evaluation changed the file count")
+ current["files"] = [{**old, **new} for old, new in zip(delivered, current["files"], strict=True)]
return {**metadata, "file_evaluation": current}
diff --git a/harness/web_fetch/dataset_assembly.py b/harness/web_fetch/dataset_assembly.py
index 37d8c1af8..0b1d3aca8 100644
--- a/harness/web_fetch/dataset_assembly.py
+++ b/harness/web_fetch/dataset_assembly.py
@@ -154,11 +154,16 @@ def assemble_profile_dataset(
declared_profile = definition.get("profile")
if profile_path_override is None and not declared_profile:
raise ValueError("Dataset assembly requires an external report profile")
- profile_path = contained_path(
- repository,
- profile_path_override if profile_path_override is not None else declared_profile,
- reject_symlinks=True,
- )
+ if profile_path_override is None:
+ profile_path = contained_path(repository, declared_profile, reject_symlinks=True)
+ reproduction_profile = str(declared_profile)
+ else:
+ from .safety import reject_evidence_symlinks
+ profile_path = (profile_path_override if profile_path_override.is_absolute()
+ else repository / profile_path_override)
+ reject_evidence_symlinks(profile_path)
+ reproduction_profile = (profile_path.relative_to(repository).as_posix()
+ if profile_path.is_relative_to(repository) else profile_path.name)
profile = json.loads(profile_path.read_text(encoding="utf-8"))
inventory = profile.get("case_inventory")
if not isinstance(inventory, dict) or not inventory:
@@ -234,7 +239,7 @@ def assemble_profile_dataset(
"publication_state": "pre_call_frozen",
"input_sources": input_sources,
"reference_binding_mode": profile["reference_binding_mode"],
- "reproduction_profile": str(profile_path_override or declared_profile),
+ "reproduction_profile": reproduction_profile,
"report_output": profile["report_output"],
}
review_scope = profile.get("independent_result_review_scope")
diff --git a/harness/web_fetch/reliability.py b/harness/web_fetch/reliability.py
index 726aefa86..66e408a10 100644
--- a/harness/web_fetch/reliability.py
+++ b/harness/web_fetch/reliability.py
@@ -11,9 +11,12 @@ def implementation_inventory(records: list[dict]) -> dict[str, dict[str, int]]:
implementations = {}
for row in records:
fingerprint = (row.get("provider_metadata") or {}).get("binary_sha256")
- if fingerprint:
- counts = implementations.setdefault(row.get("provider"), {})
- counts[fingerprint] = counts.get(fingerprint, 0) + 1
+ if not fingerprint:
+ metadata = row.get("provider_metadata") or {}
+ version = metadata.get("reported_version") or metadata.get("browser_version")
+ fingerprint = "reported-version:" + str(version) if version else "unknown"
+ counts = implementations.setdefault(row.get("provider"), {})
+ counts[fingerprint] = counts.get(fingerprint, 0) + 1
return {provider: dict(sorted(counts.items()))
for provider, counts in implementations.items()}
diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py
index a29b52bb7..30bd171db 100644
--- a/harness/web_fetch/report.py
+++ b/harness/web_fetch/report.py
@@ -63,6 +63,11 @@ def generate_wcxb(out: Path, rows: list[dict], summary: dict, ground_truth: dict
raise ValueError('WCXB authored interpretation lacks requested languages')
from collections import Counter as WordCounter
providers=list(summary['providers'])
+ comparison=profile.get('paired_comparison') or {}
+ baseline=comparison.get('baseline')
+ candidate=comparison.get('candidate')
+ if comparison and (baseline == candidate or {baseline,candidate} - set(providers)):
+ raise ValueError('WCXB paired comparison must name two registered participants')
audit=summary['reference_audit']
case_count=summary['case_count']
quality_count=summary['quality_case_count']
@@ -80,8 +85,12 @@ def generate_wcxb(out: Path, rows: list[dict], summary: dict, ground_truth: dict
if len(by_case)!=case_count or any(set(group)!=set(providers) for group in by_case.values()):
raise ValueError('WCXB same-case matrix is incomplete')
from .wcxb_latency import load_full_paired_latency, summarize_full_paired_latency
- _, selected_latency = load_full_paired_latency(out/'research/wcxb-full-paired-latency', profile, set(by_case))
- paired_latency = summarize_full_paired_latency(selected_latency, set(by_case))
+ paired_latency=None
+ if profile.get('latency_correction_receipt_sha256'):
+ if not comparison:
+ raise ValueError('WCXB corrected timing requires declared comparison participants')
+ _, selected_latency = load_full_paired_latency(out/'research/wcxb-full-paired-latency', profile, set(by_case))
+ paired_latency = summarize_full_paired_latency(selected_latency, set(by_case))
adoption_data=json.loads((out/'research/provider-adoption.json').read_text())
cloudflare_matches=[(entry['repo'], match['path']) for entry in adoption_data['included']
for match in entry['matches'] if match['signature']=='cloudflare-markdown']
@@ -140,7 +149,10 @@ def diag(lang,value):return diag_labels[lang].get(value,value.replace('_',' '))
type_labels={'article':'文章','service':'服务','product':'商品','collection':'合集','forum':'论坛','listing':'列表','documentation':'文档'}
def type_label(lang,value):return type_labels.get(value,value) if lang=='zh' else value
def browser_label(provider):
- if provider=='moli-pr643':return 'Moli candidate · '+str(profile.get('candidate_source_commit') or '')[:8]
+ if profile.get('participants',{}).get(provider,{}).get('label'):
+ return profile['participants'][provider]['label']
+ if provider == candidate and profile.get('candidate_source_commit'):
+ return provider.capitalize()+' candidate · '+str(profile['candidate_source_commit'])[:8]
version=str(identities[provider].get('reported_version') or profile['versions'].get(provider) or '')
match=re.search(r'\d+(?:\.\d+)+',version)
number=(match.group(0).split('.')[0] if provider=='chromium' else match.group(0)) if match else ''
@@ -164,7 +176,7 @@ def score_digits(values, direction='max'):
metrics=(('quality_precision','max','precision'),('quality_recall','max','recall'),('quality_f1','max','f1'),('mean_latency_ms','min','mean'),('p50_latency_ms','min','p50'),('p95_latency_ms','min','p95'))
winners={key:unique_winner({p:summary['providers'][p][key] for p in providers},direction) for key,direction,_ in metrics}
metric_digits={key:score_digits({p:summary['providers'][p][key] for p in providers},direction) for key,direction,_ in metrics if direction=='max'}
- if 'moli' in providers and 'moli-pr643' in providers:
+ if baseline and candidate:
metric_digits['quality_f1']=4
core_rows=[]
for p in sorted(providers,key=lambda name:summary['providers'][name]['quality_f1'],reverse=True):
@@ -178,21 +190,21 @@ def score_digits(values, direction='max'):
next_direction=' data-next-direction="asc"' if index==4 else ''
core_header_html.append(f'
' if index else f'
{esc(label)}
')
core='
'+''.join(core_header_html)+'
'+''.join(core_rows)+'
'
- moli_before=summary['providers'].get('moli')
- moli_after=summary['providers'].get('moli-pr643')
+ moli_before=summary['providers'].get(baseline)
+ moli_after=summary['providers'].get(candidate)
if moli_before and moli_after:
- page_deltas=[by_case[c]['moli-pr643']['f1']-by_case[c]['moli']['f1'] for c in by_case if c not in exclusion_ids]
+ page_deltas=[by_case[c][candidate]['f1']-by_case[c][baseline]['f1'] for c in by_case if c not in exclusion_ids]
up=sum(delta>1e-10 for delta in page_deltas);down=sum(delta< -1e-10 for delta in page_deltas);same=quality_count-up-down
precision_delta=(moli_after['quality_precision']-moli_before['quality_precision'])*100
recall_delta=(moli_after['quality_recall']-moli_before['quality_recall'])*100
f1_delta=(moli_after['quality_f1']-moli_before['quality_f1'])*100
- example_case=max((c for c in by_case if c not in exclusion_ids), key=lambda c: abs(by_case[c]['moli-pr643']['f1']-by_case[c]['moli']['f1']))
+ example_case=max((c for c in by_case if c not in exclusion_ids), key=lambda c: abs(by_case[c][candidate]['f1']-by_case[c][baseline]['f1']))
relation_counts={'higher':up,'lower':down,'equal':same}
relation_labels=({'higher':'上升','lower':'下降','equal':'不变'} if lang=='zh' else {'higher':'higher','lower':'lower','equal':'unchanged'})
change_counts=' · '.join(f'{count} {relation_labels[relation]}' for relation,count in relation_counts.items())
- paired=(f'
F1 across {quality_count} pages: {change_counts}. Mean recall changes {recall_delta:+.4f} percentage points, mean precision {precision_delta:+.4f}, and mean F1 {f1_delta:+.4f}. F1 is the mean of page-level F1 values, not a recomputation from mean precision and recall.
F1 across {quality_count} pages: {change_counts}. Mean recall changes {recall_delta:+.4f} percentage points, mean precision {precision_delta:+.4f}, and mean F1 {f1_delta:+.4f}. F1 is the mean of page-level F1 values, not a recomputation from mean precision and recall.
{esc(browser_label("moli"))} and the measured candidate were measured in a balanced same-window crossover on all {case_count} pages. The preselected final round gives {moli_before["mean_latency_ms"]:,.0f} ms for the release and {moli_after["mean_latency_ms"]:,.0f} ms for the PR, a {paired_latency["paired_delta_mean_ms"]:+.1f} ms page-paired difference. The codebases diverge, so this difference cannot all be attributed to this PR. Read the timing basis
{esc(browser_label(baseline))} and the measured candidate were measured in a balanced same-window crossover on all {case_count} pages. The preselected final round gives {moli_before["mean_latency_ms"]:,.0f} ms for the release and {moli_after["mean_latency_ms"]:,.0f} ms for the PR, a {paired_latency["paired_delta_mean_ms"]:+.1f} ms page-paired difference. The codebases diverge, so this difference cannot all be attributed to this PR. Read the timing basis
') if moli_before and moli_after and paired_latency else ''
lead=(f'{browser_label(best)} 在 {quality_count} 页有效参考上的平均 F1 最高({fmt(summary["providers"][best]["quality_f1"])});{browser_label(fast)} 的 {case_count} 次调用平均端到端耗时最短({summary["providers"][fast]["mean_latency_ms"]:,.0f} ms)。' if lang=='zh' else f'{browser_label(best)} leads mean F1 across {quality_count} qualified pages ({fmt(summary["providers"][best]["quality_f1"])}); {browser_label(fast)} has the shortest mean end-to-end latency across {case_count} calls ({summary["providers"][fast]["mean_latency_ms"]:,.0f} ms).')
type_counts=WordCounter(gt['page_type'] for c,gt in ground_truth.items() if c not in exclusion_ids)
slice_values={t:{p:sum(by_case[c][p]['f1'] for c,g in ground_truth.items() if g['page_type']==t and c not in exclusion_ids)/type_counts[t] for p in providers} for t in type_counts}
@@ -262,10 +274,18 @@ def score_digits(values, direction='max'):
review=f'
{esc(x["review"])}
{esc(review_intro)}
{"源数据资格检查" if lang=="zh" else "Source qualification"}
{esc(profile["versions"][p] if p=="moli-pr643" else identities[p].get("reported_version") or identities[p].get("endpoint") or "service version undisclosed")}
{esc(route_descriptions[p][0 if lang=="zh" else 1])}
{esc(identities[p].get("binary_sha256") or "hosted")}
' for p in providers)
- interface_example='\n'.join(p+': '+('POST Browser Run /markdown {html: }' if p=='kitesurf' else 'Playwright page content → node-html-markdown 2.0.0' if p=='chromium' else 'CLI fetch → Markdown') for p in providers)
+ interface_example='\n'.join(p+': '+(profile.get('participants',{}).get(p,{}).get('interface_example') or ('POST Browser Run /markdown {html: }' if p=='kitesurf' else 'Playwright page content → node-html-markdown 2.0.0' if p=='chromium' else 'CLI fetch → Markdown' if p in route_descriptions and p in HEADLESS_ACCESS else ('接口未记录' if lang=='zh' else 'Interface not recorded'))) for p in providers)
adoption=f'
{esc(x["adoption"])}
{"各配置的版本、输入路线和输出契约。" if lang=="zh" else "Measured configurations, versions, input routes and output contracts."}
{"本轮实现" if lang=="zh" else "Measured implementations"}
{esc(x["provider"])}
{esc(x["version"])}
{"路线" if lang=="zh" else "Route"}
SHA-256
{versions}
{"接口示例" if lang=="zh" else "Interface examples"}
{esc(interface_example)}
{"此表列的是实测接口,不把公开仓库搜索命中数当作产品采用量。" if lang=="zh" else "These are measured interfaces; repository search hits are not treated as product adoption counts."}
'
- adoption+='
'+('公开代码采用线索' if lang=='zh' else 'Public-code adoption signals')+'
'+((f'{adoption_data["generated_at"][:10]} 的签名核验样本中,Cloudflare /markdown 接口在 {cloudflare_repo_count} 个仓库、{cloudflare_file_count} 个文件出现。该签名没有区分 Kitesurf 后端,也不能推算实际用户数。本轮其余配置没有同口径签名统计,不作采用量排名。') if lang=='zh' else f'In the {adoption_data["generated_at"][:10]} verified signature sample, the Cloudflare /markdown interface appears in {cloudflare_repo_count} repositories and {cloudflare_file_count} files. This signature does not identify the Kitesurf backend or user count. The other configurations have no matching signature inventory, so no adoption ranking is made.')+'
'+('公开代码采用线索' if lang=='zh' else 'Public-code adoption signals')+'
'+((f'{adoption_data["generated_at"][:10]} 的签名核验样本中,Cloudflare /markdown 接口在 {cloudflare_repo_count} 个仓库、{cloudflare_file_count} 个文件出现。该签名没有区分 Kitesurf 后端,也不能推算实际用户数。本轮其余配置没有同口径签名统计,不作采用量排名。') if lang=='zh' else f'In the {adoption_data["generated_at"][:10]} verified signature sample, the Cloudflare /markdown interface appears in {cloudflare_repo_count} repositories and {cloudflare_file_count} files. This signature does not identify the Kitesurf backend or user count. The other configurations have no matching signature inventory, so no adoption ranking is made.')+'
'+('采用统计' if lang=='zh' else 'Adoption evidence')+'
'+('本报告没有登记这些配置的同口径采用统计。' if lang=='zh' else 'No comparable adoption inventory is registered for these configurations.')+'
'
page(lang,'adoption/index.html',x['adoption'],adoption)
for p in providers:
s=summary['providers'][p]
@@ -275,8 +295,8 @@ def score_digits(values, direction='max'):
changes_en=f"All {len(providers)} configurations completed {case_count} calls. Quality displays use only {quality_count} references qualified before calls, or {visible_count:,} results. End-to-end latency retains all {case_count} calls per method so slow or failed calls are not removed after measurement."
changes=f'
{esc(x["changes"])}
{"当前报告的结果、分母和证据范围。" if lang=="zh" else "Current result, denominator and evidence scope."}
{"有效参考与完整调用" if lang=="zh" else "Qualified references and all calls"}
{esc(HEADLESS_ACCESS[p]["account"] if lang=="zh" else ("Cloudflare Browser Run account and API token required." if p=="kitesurf" else "No provider account or API key required."))}
{"只有 Kitesurf 需要 Cloudflare 账号凭据;本地配置值不进入报告。" if lang=="zh" else "Only Kitesurf needs Cloudflare account credentials; local values are not published."}
{esc(x["provider"])}
{"环境变量" if lang=="zh" else "Environment variables"}
{"账号要求" if lang=="zh" else "Account requirement"}
{"官方入口" if lang=="zh" else "Official source"}
{account_rows}
'
+ account_rows=''.join(f'
{esc(p)}
{esc("、".join(access[p]["env_vars"]))}
{esc(access[p]["account"] if lang=="zh" else (access[p].get("account_en") or ("Cloudflare Browser Run account and API token required." if p=="kitesurf" else "No provider account or API key required." if p in HEADLESS_ACCESS else "Not recorded")))}
{"账号要求来自各方法的声明;未记录的要求会在表中标明。凭据值不进入报告。" if lang=="zh" else "Account requirements follow each method declaration. Missing requirements are marked in the table; credential values are not published."}
{esc(x["provider"])}
{"环境变量" if lang=="zh" else "Environment variables"}
')
methodology=methodology.replace('方法之间并行、同一方法串行。','每种方法内部串行;方法之间的调度见执行批次。')
+ from .report_components import render_acceptance_scope
+ if dataset.get('cases'):
+ methodology+=render_acceptance_scope({case['id']:case for case in dataset['cases']},records)
write(out,'methodology/index.html','评测方法',methodology)
downloads=' · '.join(link('../'+path,label) for path,label in [('dataset.json','评测集与参考'),('results.jsonl','全部调用记录'),('summary.json','分类汇总'),('execution.json','执行批次')]+([('implementation-provenance.json','浏览器版本与二进制来源')] if (out/'implementation-provenance.json').is_file() else [])+([('reference-revisions.json','参考修订收据')] if (out/'reference-revisions.json').is_file() else [])+([('scoring-revisions.json','评分修订收据')] if (out/'scoring-revisions.json').is_file() else [])+([('latency-corrections.json','时延纠正收据')] if (out/'latency-corrections.json').is_file() else [])+([('request-status-corrections.json','请求状态纠正收据')] if (out/'request-status-corrections.json').is_file() else []))
reproduce='
在评测仓库中执行;读取 evaluation/ 的当前实测数据,不调用供应商接口。
make setup\nmake report EVIDENCE=/absolute/path/to/evidence\nmake report-check
'
@@ -2728,8 +2752,8 @@ def write_support(out,records,manifests,*,authored_interpretation=True):
profile_path=dataset.get('comparison',{}).get('reproduction_profile')
if profile_path:
candidate=Path(profile_path)
- if candidate.is_absolute() or not candidate.parts or '..' in candidate.parts or candidate.parts[0]!='evaluation' or candidate.name!='current-report.json':
- raise ValueError('Report reproduction profile must stay under evaluation/')
+ if candidate.is_absolute() or not candidate.parts or '..' in candidate.parts or candidate.suffix!='.json':
+ raise ValueError('Report reproduction profile must be a relative JSON path without parent traversal')
destination=dataset['comparison'].get('report_output','reports/webfetch')
command=f'uv run --frozen python scripts/build_report.py --profile {profile_path} --evidence /absolute/path/to/evidence --output {destination}\nuv run --frozen python scripts/check_report.py {destination} --profile {profile_path}'
reproduce='
'
diff --git a/harness/web_fetch/report_build.py b/harness/web_fetch/report_build.py
index b448376b4..3a322f19b 100644
--- a/harness/web_fetch/report_build.py
+++ b/harness/web_fetch/report_build.py
@@ -1226,8 +1226,17 @@ def build_wcxb_report(source: Path, corpus: Path, sample_path: Path, profile: di
from harness.report_languages import includes_english, select_report_language
include_english = includes_english(language, include_english)
from harness.web_fetch.wcxb_dataset import validate_manifest
- from harness.web_fetch.webmainbench_runner import WCXB_PROVIDERS, load_wcxb_source
+ from harness.web_fetch.webmainbench_runner import load_wcxb_source
source, output = source.resolve(), output.resolve()
+ providers = profile['providers']
+ if not providers or len(set(providers)) != len(providers):
+ raise ValueError('WCXB participant registry must be nonempty and unique')
+ for provider in providers:
+ safe_identifier(provider)
+ comparison = profile.get('paired_comparison') or {}
+ baseline, candidate_id = comparison.get('baseline'), comparison.get('candidate')
+ if comparison and (baseline == candidate_id or {baseline,candidate_id} - set(providers)):
+ raise ValueError('WCXB paired comparison must name two registered participants')
from harness.reporting.assets import load_bound_content
report_content = load_bound_content(
source / "report-assets", content_binding,
@@ -1255,55 +1264,60 @@ def build_wcxb_report(source: Path, corpus: Path, sample_path: Path, profile: di
evaluator_hash = file_sha256(contained_path(corpus, 'evaluate.py'))
if evaluator_hash != profile['official_evaluator_sha256'] or summary.get('official_evaluator_sha256') != evaluator_hash:
raise ValueError('WCXB official evaluator differs from the certified profile')
- if [x['provider'] for x in certificate['providers']] != list(WCXB_PROVIDERS) or profile['providers'] != list(WCXB_PROVIDERS):
+ if [x['provider'] for x in certificate['providers']] != providers:
raise ValueError('WCXB participant registry differs from the certificate')
- candidate = next(x for x in certificate['providers'] if x['provider'] == 'moli-pr643')
- if candidate['identity'].get('binary_sha256') != profile['candidate_binary_sha256']:
- raise ValueError('WCXB candidate binary differs from the current report profile')
- released = next(x for x in certificate['providers'] if x['provider'] == 'moli')
- if released['identity'].get('binary_sha256') != profile['release_binary_sha256']:
- raise ValueError('WCXB released Moli binary differs from the current report profile')
+ if comparison:
+ candidate = next(x for x in certificate['providers'] if x['provider'] == candidate_id)
+ if candidate['identity'].get('binary_sha256') != profile['candidate_binary_sha256']:
+ raise ValueError('WCXB candidate binary differs from the current report profile')
+ released = next(x for x in certificate['providers'] if x['provider'] == baseline)
+ if released['identity'].get('binary_sha256') != profile['release_binary_sha256']:
+ raise ValueError('WCXB baseline binary differs from the current report profile')
rows = load_jsonl(source / 'wcxb-scores.jsonl', skip_blank=True)
case_ids = {r['track_id'] for r in records}
- correction_root = source / 'latency-correction'
- correction_receipt = correction_root / 'receipt.json'
- if (not correction_receipt.is_file()
- or file_sha256(correction_receipt)
- != profile.get('latency_correction_receipt_sha256')):
- raise ValueError('WCXB full-scope latency correction is missing or differs from profile')
- _, corrected_calls = load_full_paired_latency(correction_root, profile, case_ids)
- correction_certificate = json.loads((correction_root / 'certification.json').read_text())
- if (correction_certificate.get('kind') != 'certified_wcxb_paired_latency'
- or correction_certificate.get('complete') is not True
- or correction_certificate.get('receipt_sha256')
- != file_sha256(correction_receipt)):
- raise ValueError('WCXB corrected timing is not certified')
- if any(
- abs(row['total_elapsed_ms'] - corrected_calls[row['case_id'],
- 'release' if row['provider'] == 'moli' else 'pr']['elapsed_ms']) > .01
- for row in rows if row['provider'] in ('moli', 'moli-pr643')
- ):
- raise ValueError('WCXB reported Moli latency differs from the selected paired round')
- optional_root = source / 'optional-ui-4017'
- optional_receipt = optional_root / 'receipt.json'
- if (not optional_receipt.is_file() or optional_receipt.is_symlink()
- or file_sha256(optional_receipt)
- != profile.get('optional_ui_receipt_sha256')):
- raise ValueError('WCXB optional UI-stripping evidence is missing or differs from profile')
- optional = json.loads(optional_receipt.read_text())
- if (optional.get('case_id') != '4017' or optional.get('source_sha256') != corpus_hash
- or optional.get('builds', {}).get('release', {}).get('binary_sha256') != profile['release_binary_sha256']
- or optional.get('builds', {}).get('candidate', {}).get('binary_sha256') != profile['candidate_binary_sha256']
- or optional['builds']['release']['description_present']
- or not optional['builds']['candidate']['description_present']):
- raise ValueError('WCXB optional UI-stripping comparison does not bind both builds')
- for name in ('release', 'candidate'):
- for ext, key in (('md', 'output_sha256'), ('stderr', 'stderr_sha256')):
- artifact = optional_root / f'{name}.{ext}'
- if (not artifact.is_file() or artifact.is_symlink()
- or file_sha256(artifact) != optional['builds'][name][key]):
- raise ValueError('WCXB optional UI-stripping artifact changed: ' + artifact.name)
- if len(rows) != len(records) * len(WCXB_PROVIDERS) or {(r['provider'], r['case_id']) for r in rows} != {(p,c) for p in WCXB_PROVIDERS for c in case_ids}:
+ if profile.get('latency_correction_receipt_sha256'):
+ if not comparison:
+ raise ValueError('WCXB corrected timing requires declared comparison participants')
+ correction_root = source / 'latency-correction'
+ correction_receipt = correction_root / 'receipt.json'
+ if (not correction_receipt.is_file()
+ or file_sha256(correction_receipt)
+ != profile.get('latency_correction_receipt_sha256')):
+ raise ValueError('WCXB full-scope latency correction is missing or differs from profile')
+ _, corrected_calls = load_full_paired_latency(correction_root, profile, case_ids)
+ correction_certificate = json.loads((correction_root / 'certification.json').read_text())
+ if (correction_certificate.get('kind') != 'certified_wcxb_paired_latency'
+ or correction_certificate.get('complete') is not True
+ or correction_certificate.get('receipt_sha256')
+ != file_sha256(correction_receipt)):
+ raise ValueError('WCXB corrected timing is not certified')
+ if any(
+ abs(row['total_elapsed_ms'] - corrected_calls[row['case_id'],
+ 'release' if row['provider'] == baseline else 'pr']['elapsed_ms']) > .01
+ for row in rows if row['provider'] in (baseline,candidate_id)
+ ):
+ raise ValueError('WCXB reported paired latency differs from the selected paired round')
+ if profile.get('optional_ui_receipt_sha256'):
+ optional_root = source / 'optional-ui-4017'
+ optional_receipt = optional_root / 'receipt.json'
+ if (not optional_receipt.is_file() or optional_receipt.is_symlink()
+ or file_sha256(optional_receipt)
+ != profile.get('optional_ui_receipt_sha256')):
+ raise ValueError('WCXB optional UI-stripping evidence is missing or differs from profile')
+ optional = json.loads(optional_receipt.read_text())
+ if (optional.get('case_id') != '4017' or optional.get('source_sha256') != corpus_hash
+ or optional.get('builds', {}).get('release', {}).get('binary_sha256') != profile['release_binary_sha256']
+ or optional.get('builds', {}).get('candidate', {}).get('binary_sha256') != profile['candidate_binary_sha256']
+ or optional['builds']['release']['description_present']
+ or not optional['builds']['candidate']['description_present']):
+ raise ValueError('WCXB optional UI-stripping comparison does not bind both builds')
+ for name in ('release', 'candidate'):
+ for ext, key in (('md', 'output_sha256'), ('stderr', 'stderr_sha256')):
+ artifact = optional_root / f'{name}.{ext}'
+ if (not artifact.is_file() or artifact.is_symlink()
+ or file_sha256(artifact) != optional['builds'][name][key]):
+ raise ValueError('WCXB optional UI-stripping artifact changed: ' + artifact.name)
+ if len(rows) != len(records) * len(providers) or {(r['provider'], r['case_id']) for r in rows} != {(p,c) for p in providers for c in case_ids}:
raise ValueError('WCXB score matrix is incomplete')
if output.exists() and not (output / '.generated-report').is_file():
raise ValueError('Refusing to replace an unmarked report')
@@ -1314,12 +1328,14 @@ def build_wcxb_report(source: Path, corpus: Path, sample_path: Path, profile: di
shutil.copy2(source / name, staging / name)
shutil.copy2(sample_path, staging / 'sample-manifest.json')
(staging / 'research').mkdir()
- (staging / 'research/wcxb-full-paired-latency').mkdir()
- for name in ('receipt.json', 'certification.json', 'calls.jsonl'):
- shutil.copy2(correction_root / name, staging / 'research/wcxb-full-paired-latency' / name)
- (staging / 'research/wcxb-optional-ui-4017').mkdir()
- for name in ('receipt.json', 'release.md', 'candidate.md', 'release.stderr', 'candidate.stderr'):
- shutil.copy2(optional_root / name, staging / 'research/wcxb-optional-ui-4017' / name)
+ if profile.get('latency_correction_receipt_sha256'):
+ (staging / 'research/wcxb-full-paired-latency').mkdir()
+ for name in ('receipt.json', 'certification.json', 'calls.jsonl'):
+ shutil.copy2(correction_root / name, staging / 'research/wcxb-full-paired-latency' / name)
+ if profile.get('optional_ui_receipt_sha256'):
+ (staging / 'research/wcxb-optional-ui-4017').mkdir()
+ for name in ('receipt.json', 'release.md', 'candidate.md', 'release.stderr', 'candidate.stderr'):
+ shutil.copy2(optional_root / name, staging / 'research/wcxb-optional-ui-4017' / name)
shutil.copy2(Path(__file__).resolve().parents[2] / 'research/report-changes.json',
staging / 'research/report-changes.json')
shutil.copy2(Path(__file__).resolve().parents[2] / 'research/provider-adoption.json',
@@ -1385,7 +1401,7 @@ def build_wcxb_report(source: Path, corpus: Path, sample_path: Path, profile: di
case_id = record['track_id']
outputs = {}
retries = {}
- for provider in WCXB_PROVIDERS:
+ for provider in providers:
outputs[provider] = (staging / 'artifacts' / provider / f'{case_id}.txt').read_text(encoding='utf-8')
retries[provider] = [
{**retry, 'output': contained_path(staging, retry['artifact_path']).read_text(encoding='utf-8')}
@@ -1409,7 +1425,7 @@ def build_wcxb_report(source: Path, corpus: Path, sample_path: Path, profile: di
raise ValueError('WCXB representative review scope differs from measured matrix')
for case_review in review['cases']:
case_id = case_review['case_id']
- if case_id not in rows_by_case or not set(case_review['outputs']) <= set(WCXB_PROVIDERS):
+ if case_id not in rows_by_case or not set(case_review['outputs']) <= set(providers):
raise ValueError('WCXB review case or method inventory differs')
evidence = json.loads((evidence_root / f'{case_id}.json').read_text())
if (hashlib.sha256(evidence['source_html'].encode()).hexdigest() != case_review['source_html_sha256']
diff --git a/harness/web_fetch/report_check.py b/harness/web_fetch/report_check.py
index 7effbde07..a0682fbb8 100644
--- a/harness/web_fetch/report_check.py
+++ b/harness/web_fetch/report_check.py
@@ -309,14 +309,10 @@ def check(root: Path, *, authority_root: Path, profile: Path | None = None, pres
profile_path = (
(repository / profile).resolve() if not profile.is_absolute() else profile.resolve()
)
- if not profile_path.is_relative_to((repository / "evaluation").resolve()):
- raise ValueError("--profile must stay under evaluation/")
if (root / "wcxb-score-summary.json").is_file():
- if profile_path and profile_path != repository / "evaluation/wcxb/current-report.json":
- raise ValueError("WCXB report requires its own current profile")
from harness.web_fetch.wcxb_report_check import check as check_wcxb
- check_wcxb(root, authority_root=repository)
+ check_wcxb(root, authority_root=repository, profile_path=profile_path)
return None
if profile_path:
require_current_report_profile(root, profile_path)
diff --git a/harness/web_fetch/report_components.py b/harness/web_fetch/report_components.py
index 54c8e3c2a..22d0b6854 100644
--- a/harness/web_fetch/report_components.py
+++ b/harness/web_fetch/report_components.py
@@ -521,3 +521,90 @@ def sources_panel():
('PDF.js','直接 PDF 与嵌入文档阅读器案例。'),
('MDN 媒体示例','图片和视频入口案例。'),
])
+
+
+def acceptance_scope(cases, records):
+ """Describe frozen requirements and saved outcomes without changing either."""
+ from collections import Counter, defaultdict
+ from urllib.parse import urlsplit
+ from lexbench_webfetch.metrics import task_passed
+
+ families = defaultdict(Counter)
+ affiliated = []
+ for case in cases.values():
+ counts = families[case.get('track', 'web')]
+ contract = case.get('acceptance_contract') or {}
+ counts['tasks'] += 1
+ for key, present in {
+ 'identity': case.get('page_identity_required', False),
+ 'literal': bool(case.get('expected_any') or contract.get('stable_text_anchors')),
+ 'facts': bool(case.get('required_facts')),
+ 'body_regions': bool((case.get('body_requirements') or {}).get('regions')),
+ 'records': bool(contract.get('minimum_records', 0)),
+ 'resources': bool(case.get('expected_resources')),
+ 'full_reference': bool(case.get('ground_truth_main_content') and case.get('content_score_applicable', True)),
+ 'short_length_floor': case.get('min_content_chars', 1) <= 1,
+ }.items():
+ counts[key] += bool(present)
+ url = urlsplit(case.get('url', ''))
+ if url.hostname == 'github.com' and (url.path.rstrip('/') == '/lexmount/moli' or url.path.startswith('/lexmount/moli/')):
+ # The benchmark maintainer and this evaluated repository share an
+ # organization. The frozen task remains in the original population.
+ affiliated.append(case['id'])
+ methods = defaultdict(Counter)
+ for row in records:
+ case = cases[row['case_id']]
+ metrics = row.get('metrics') or {}
+ counts = methods[row['provider']]
+ counts['results'] += 1
+ passed = task_passed(row['ok'], metrics, case.get('task_kind', 'body'))
+ counts['passes'] += passed
+ counts['identity_verified'] += metrics.get('page_identity_status') == 'verified'
+ counts['unverified_identity_passes'] += bool(passed and metrics.get('page_identity_status') != 'verified')
+ counts['blocked_diagnostics'] += bool(metrics.get('blocked_page') or metrics.get('stable_blocked_page'))
+ if case.get('expected_resources'):
+ counts['resource_results'] += 1
+ counts['unexpected_resources'] += len(metrics.get('unexpected_resource_urls') or [])
+ precision = metrics.get('resource_precision')
+ if isinstance(precision, (int, float)) and not isinstance(precision, bool):
+ counts['resource_precision_sum'] += precision
+ counts['resource_precision_count'] += 1
+ return {'families': dict(families), 'methods': dict(methods), 'affiliated_cases': affiliated}
+
+
+def render_acceptance_scope(cases, records):
+ from .report import model_text as h, section, table, cell, esc, pct, provider_label, link
+ scope = acceptance_scope(cases, records)
+ fields = ('tasks', 'identity', 'literal', 'facts', 'body_regions', 'records', 'resources', 'full_reference', 'short_length_floor')
+ rows = [('', cell(esc(track)) + ''.join(cell(str(counts[key])) for key in fields))
+ for track, counts in sorted(scope['families'].items())]
+ body = '
' + h(
+ '下表统计每类冻结任务声明了哪些要求。一题可以使用多项要求。固定文字、事实组和列表结构分别列出;没有正文范围或全文参考的任务,不据此声称测过完整正文质量。',
+ 'The table counts requirements declared by frozen tasks. A task may use several checks. Literal text, fact groups and record structure are separate; tasks without body regions or a full reference do not establish full-body quality.') + '
' + h(
+ '通过但未核验页身份的结果仍按原合同计分,表中以全部通过结果为分母列出比例。拦截提示仅作诊断。资源多余地址数和平均精确率按资源任务统计;召回合同不会因此增加事后门槛。语言默认比较主标签,只有任务声明 exact 时才要求地区标签完全一致。',
+ 'Passes without verified page identity retain their frozen verdicts; their share uses all passes as its denominator. Blocked-page markers are diagnostic. Unexpected URLs and mean precision describe resource tasks without adding a new recall threshold. Language checks compare the primary tag unless a task declares exact regional matching.') + '
' + h('利益关系:以下任务涉及评测维护方同一组织下的产品仓库;保留原任务和原分母。',
+ 'Affiliation: these tasks concern a product repository in the benchmark maintainer’s organization. The original tasks and denominator are retained.') + ' ' + ' · '.join(
+ link('../datasets/' + identifier + '.html', esc(identifier))
+ for identifier in scope['affiliated_cases']) + '
'
+ return section(h('验收覆盖与诊断范围', 'Acceptance coverage and diagnostics'), body, 'acceptance-scope')
diff --git a/harness/web_fetch/report_methods.py b/harness/web_fetch/report_methods.py
index ca68cb91b..c80f05539 100644
--- a/harness/web_fetch/report_methods.py
+++ b/harness/web_fetch/report_methods.py
@@ -55,19 +55,18 @@ def adoption_counts(data,names):
def _implementation_scope(records, *, section, table, cell, esc):
from .reliability import implementation_inventory
from .report import model_text
- inventories = {name: counts for name, counts in implementation_inventory(records).items()
- if len(counts) > 1}
+ inventories = implementation_inventory(records)
if not inventories:
return ""
rows = [("", cell(esc(name)) + cell("" + esc(fingerprint) + "") + cell(str(count)))
for name, counts in sorted(inventories.items())
for fingerprint, count in counts.items()]
explanation = model_text(
- "这些方法的最终结果包含多个程序版本。允许保留已通过的结果、只重跑失败部分,再统一评分。表中列出各版本采用的结果数;累计成绩不代表某一个版本独立跑完整套任务。",
- "These methods combine retained results and reruns from multiple executable builds. Passed results may be retained while failures are rerun, then the complete selected population is rescored. Counts below identify each build's selected results; the cumulative score is not a full run by one build.")
+ "表中列出每个方法采用的结果数及其版本依据。unknown 表示未记录版本;reported-version 表示仅记录了版本名称,未核验二进制。多个版本合并的累计成绩不代表某一个版本独立跑完整套任务。",
+ "Every selected result is counted below. unknown means no version was recorded; reported-version identifies a recorded version label without a verified binary. When several builds are combined, the cumulative score is not a complete run by any single build.")
return section(model_text("局部重跑与结果来源", "Partial reruns and result provenance"),
"
" + explanation + "
" + table(
- [model_text("方法", "Method"), "SHA-256", model_text("采用结果数", "Selected results")],
+ [model_text("方法", "Method"), model_text("版本依据", "Version evidence"), model_text("采用结果数", "Selected results")],
rows, metrics=[]), "implementation-scope")
diff --git a/harness/web_fetch/wcxb_publication.py b/harness/web_fetch/wcxb_publication.py
index 1feb1b351..24a090128 100644
--- a/harness/web_fetch/wcxb_publication.py
+++ b/harness/web_fetch/wcxb_publication.py
@@ -57,12 +57,12 @@ def inventory(report: Path, profile: dict) -> dict[str, str]:
return {**expected, INVENTORY: actual[INVENTORY]}
-def package(source: Path, archive: Path, receipt: Path, *, authority_root: Path) -> dict:
+def package(source: Path, archive: Path, receipt: Path, *, authority_root: Path, profile_path: Path | None = None) -> dict:
from harness.reporting.links import check as check_links
from harness.web_fetch.wcxb_report_check import check as check_report
- profile_path = authority_root / 'evaluation/wcxb/current-report.json'
- binding_path = authority_root / 'evaluation/wcxb/report-content-binding.json'
+ profile_path = profile_path or authority_root / 'evaluation/wcxb/current-report.json'
+ binding_path = profile_path.with_name('report-content-binding.json')
reject_evidence_symlinks(profile_path)
reject_evidence_symlinks(binding_path)
profile = json.loads(profile_path.read_bytes())
@@ -71,7 +71,7 @@ def package(source: Path, archive: Path, receipt: Path, *, authority_root: Path)
def validate():
inventory(source, profile)
- check_report(source, authority_root=authority_root)
+ check_report(source, authority_root=authority_root, profile_path=profile_path)
check_links(source)
def attest(staged):
@@ -200,14 +200,17 @@ def main(argv=None, *, root: Path | None = None) -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--authority-root', type=Path, default=root or Path.cwd())
parser.add_argument('--source', type=Path)
+ parser.add_argument('--profile', type=Path, help='external WCXB instance profile for local packaging')
parser.add_argument('--package-only', action='store_true', help='validate and create a local archive without publishing')
parser.add_argument('--work', type=Path, default=data_home() / 'publication')
args = parser.parse_args(argv)
authority = args.authority_root.absolute()
+ if args.profile and not args.package_only:
+ parser.error('--profile currently requires --package-only')
if args.package_only:
archive, receipt = args.work / 'wcxb-current.zip', args.work / 'wcxb-package.json'
proof = package(args.source or authority / 'reports/wcxb', archive, receipt,
- authority_root=authority)
+ authority_root=authority, profile_path=args.profile)
print(json.dumps({'archive': str(archive), 'archive_sha256': proof['archive_sha256'],
'files': len(proof['files'])}))
return
diff --git a/harness/web_fetch/wcxb_report_check.py b/harness/web_fetch/wcxb_report_check.py
index b08f62ea0..c7bdd3fbd 100644
--- a/harness/web_fetch/wcxb_report_check.py
+++ b/harness/web_fetch/wcxb_report_check.py
@@ -14,6 +14,7 @@
from bs4 import BeautifulSoup
+from harness.web_fetch.safety import reject_evidence_symlinks
from harness.record_io import load_jsonl
from harness.report_languages import localized_report_routes, visible_numbers
from harness.reporting.presentation import check_presentation
@@ -117,43 +118,56 @@ def require_analysis_controls(html: str | BeautifulSoup, result_count: int) -> B
def require_home_interpretation(html: str | BeautifulSoup, summary: dict,
- paired_latency: dict, lang: str, *, scores: list[dict]) -> BeautifulSoup:
+ paired_latency: dict | None, lang: str, *, scores: list[dict], comparison: dict | None = None) -> BeautifulSoup:
"""Keep the version comparison readable and its timing caveat beside the table."""
soup = BeautifulSoup(html, 'html.parser') if isinstance(html, str) else html
core = soup.select_one('#wcxb-core')
- contexts = soup.select('#moli-latency-context')
- changes = soup.select('#moli-change')
- require(core is not None and len(contexts) == len(changes) == 1,
- 'Home Moli comparison or timing context is missing or duplicated')
- context = contexts[0]
- require(core.find_parent('section') == context.find_parent('section'),
- 'Moli timing context must be beside the core comparison')
- require(core.find_next(id='moli-latency-context') == context,
- 'Moli timing context must follow the core comparison')
- release = summary['providers']['moli']['mean_latency_ms']
- candidate = summary['providers']['moli-pr643']['mean_latency_ms']
- required = (f'{release:,.0f} ms', f'{candidate:,.0f} ms',
- f'{paired_latency["paired_delta_mean_ms"]:+.1f} ms')
- context_text = context.get_text(' ', strip=True)
- require(all(value in context_text for value in required),
- 'Home timing context disagrees with measured evidence')
- require(('交叉顺序' if lang == 'zh' else 'balanced same-window') in context_text
- and (f'{summary["case_count"]} 页' if lang == 'zh' else f'{summary["case_count"]} pages') in context_text,
- 'Home timing context does not identify the corrected population')
- require(context.select_one('a[href$="guide/index.html"]') is not None,
- 'Home timing context does not link to the analysis')
+ contexts = soup.select('#paired-latency-context')
+ changes = soup.select('#configuration-change')
+ require(core is not None, 'Home core comparison is missing')
+ if comparison == {}:
+ require(not changes and not contexts, 'Home has an undeclared paired comparison')
+ return soup
+ require(len(changes) == 1, 'Home configuration comparison is missing or duplicated')
+ baseline, candidate = changes[0].get('data-baseline'), changes[0].get('data-candidate')
+ require(baseline != candidate and {baseline,candidate} <= set(summary['providers']),
+ 'Home comparison does not identify two registered participants')
+ if comparison is not None:
+ require((baseline,candidate) == (comparison['baseline'],comparison['candidate']),
+ 'Home comparison differs from its declared participants')
+ if paired_latency is None:
+ require(not contexts, 'Home claims a same-window comparison without timing evidence')
+ else:
+ require(len(contexts) == 1, 'Paired timing context is missing or duplicated')
+ context = contexts[0]
+ require(core.find_parent('section') == context.find_parent('section'),
+ 'Paired timing context must be beside the core comparison')
+ require(core.find_next(id='paired-latency-context') == context,
+ 'Paired timing context must follow the core comparison')
+ release = summary['providers'][baseline]['mean_latency_ms']
+ candidate_latency = summary['providers'][candidate]['mean_latency_ms']
+ required = (f'{release:,.0f} ms', f'{candidate_latency:,.0f} ms',
+ f'{paired_latency["paired_delta_mean_ms"]:+.1f} ms')
+ context_text = context.get_text(' ', strip=True)
+ require(all(value in context_text for value in required),
+ 'Home timing context disagrees with measured evidence')
+ require(('交叉顺序' if lang == 'zh' else 'balanced same-window') in context_text
+ and (f'{summary["case_count"]} 页' if lang == 'zh' else f'{summary["case_count"]} pages') in context_text,
+ 'Home timing context does not identify the corrected population')
+ require(context.select_one('a[href$="guide/index.html"]') is not None,
+ 'Home timing context does not link to the analysis')
change = changes[0]
change_text = change.get_text(' ', strip=True)
require(len(change_text) <= (330 if lang == 'zh' else 700),
- 'Home Moli change summary is too long; move detail to the interpretation page')
+ 'Home configuration change summary is too long; move detail to the interpretation page')
require(len(change.select('p')) == 2 and change.select_one('a[href*="cases/"]')
and change.select_one('a[href$="guide/index.html"]'),
- 'Home Moli change summary lacks concise evidence links')
+ 'Home configuration change summary lacks concise evidence links')
by_case = {}
for row in scores:
if row.get('quality_eligible', True):
by_case.setdefault(row['case_id'], {})[row['provider']] = row['f1']
- deltas = [values['moli-pr643'] - values['moli'] for values in by_case.values()]
+ deltas = [values[candidate] - values[baseline] for values in by_case.values()]
expected = {'higher': sum(value > 1e-10 for value in deltas),
'lower': sum(value < -1e-10 for value in deltas),
'equal': sum(abs(value) <= 1e-10 for value in deltas)}
@@ -166,27 +180,29 @@ def require_home_interpretation(html: str | BeautifulSoup, summary: dict,
'Home comparison links to a case outside the selected evidence')
require(('逐页 F1 的平均' if lang == 'zh' else 'page-level F1 values') in change_text,
'Home comparison omits the page-level F1 definition')
- precision_delta = ((summary['providers']['moli-pr643']['quality_precision']
- - summary['providers']['moli']['quality_precision']) * 100)
- recall_delta = ((summary['providers']['moli-pr643']['quality_recall']
- - summary['providers']['moli']['quality_recall']) * 100)
- f1_delta = ((summary['providers']['moli-pr643']['quality_f1']
- - summary['providers']['moli']['quality_f1']) * 100)
+ precision_delta = ((summary['providers'][candidate]['quality_precision']
+ - summary['providers'][baseline]['quality_precision']) * 100)
+ recall_delta = ((summary['providers'][candidate]['quality_recall']
+ - summary['providers'][baseline]['quality_recall']) * 100)
+ f1_delta = ((summary['providers'][candidate]['quality_f1']
+ - summary['providers'][baseline]['quality_f1']) * 100)
require(all(f'{value:+.4f}' in change_text
for value in (precision_delta, recall_delta, f1_delta)),
- 'Home Moli change summary disagrees with aggregate deltas')
+ 'Home configuration change summary disagrees with aggregate deltas')
require(not any(f'{summary["providers"][provider]["quality_f1"] * 100:.4f}%'
- in change_text for provider in ('moli', 'moli-pr643')),
- 'Home Moli change summary repeats exact table values')
+ in change_text for provider in (baseline,candidate)),
+ 'Home configuration change summary repeats exact table values')
return soup
def require_guide_interpretation(html: str | BeautifulSoup, summary: dict,
- providers: list[str], lang: str) -> BeautifulSoup:
+ providers: list[str], lang: str, *, repair_analysis: bool = True) -> BeautifulSoup:
"""A guide must work as a standalone explanation, with traceable examples."""
soup = BeautifulSoup(html, 'html.parser') if isinstance(html, str) else html
anchors = ('guide-results', 'guide-score', 'guide-cases', 'guide-repair', 'guide-methods',
'guide-timing', 'guide-conclusion')
+ if not repair_analysis:
+ anchors = tuple(anchor for anchor in anchors if anchor != 'guide-repair')
toc = soup.select_one('nav.essay-toc')
require(toc is not None and all(soup.select_one(f'#{anchor}') is not None
and toc.select_one(f'a[href="#{anchor}"]') is not None
@@ -207,48 +223,53 @@ def require_guide_interpretation(html: str | BeautifulSoup, summary: dict,
f'Guide overview disagrees with {provider} evidence')
specimen = soup.select_one('#guide-cases')
require(specimen is not None, 'Guide lacks a direct source-example entry')
- repair = soup.select_one('#guide-repair')
- images = ('code-block.png', 'description.png', 'adjacent-code.png')
- require(repair is not None and len(repair.select('.repair-case')) == len(images),
- 'Guide must explain each Moli repair in its own figure')
- for case, image in zip(repair.select('.repair-case'), images):
- graphic = case.select_one('figure img')
- require(graphic is not None and graphic.get('src') == f'assets/wcxb-repair/{image}'
- and len(graphic.get('alt', '')) >= 30
- and len(case.select('figcaption span')) == 3
- and len(case.select('.repair-samples > div')) == 3
- and case.select_one('.repair-mechanism a[href*="github.com/lexmount/moli/blob/384d1d2a/"]'),
- f'Guide repair figure {image} lacks its image, code samples or current source')
- captions = [node.get_text(' ', strip=True) for node in case.select('figcaption span')]
- scope = case.select_one('.repair-scope')
- require('1.1.8' in captions[1] and '384d1d2a' in captions[2] and scope is not None,
- 'Guide figure does not compare the tagged release with current PR')
- if image == 'description.png':
- mode = '可选 UI 清理' if lang == 'zh' else 'optional UI stripping'
- require(all(mode in caption for caption in captions[1:]),
- 'Optional stripping captions omit the measured mode')
- require('tsfunction identity(arg: number)' in repair.select('.repair-case')[0].get_text(' ', strip=True)
- and '= 30
+ and len(case.select('figcaption span')) == 3
+ and len(case.select('.repair-samples > div')) == 3
+ and case.select_one('.repair-mechanism a[href*="github.com/lexmount/moli/blob/384d1d2a/"]'),
+ f'Guide repair figure {image} lacks its image, code samples or current source')
+ captions = [node.get_text(' ', strip=True) for node in case.select('figcaption span')]
+ scope = case.select_one('.repair-scope')
+ require('1.1.8' in captions[1] and '384d1d2a' in captions[2] and scope is not None,
+ 'Guide figure does not compare the tagged release with current PR')
+ if image == 'description.png':
+ mode = '可选 UI 清理' if lang == 'zh' else 'optional UI stripping'
+ require(all(mode in caption for caption in captions[1:]),
+ 'Optional stripping captions omit the measured mode')
+ require('tsfunction identity(arg: number)' in repair.select('.repair-case')[0].get_text(' ', strip=True)
+ and ' 200,
'Guide conclusion lacks a substantive finding')
return soup
@@ -275,17 +296,21 @@ def check_bound_interpretation(root: Path, binding: dict) -> None:
raise ValueError('WCXB interpretation differs from bound content: ' + language)
-def check(root: Path, *, authority_root: Path | None = None) -> None:
+def check(root: Path, *, authority_root: Path | None = None, profile_path: Path | None = None) -> None:
root = root.resolve()
repository = (authority_root or ROOT).resolve()
- profile_path = repository / "evaluation/wcxb/current-report.json"
+ profile_path = profile_path or repository / "evaluation/wcxb/current-report.json"
if not profile_path.is_file():
raise ValueError("WCXB release authority is missing; pass --authority-root pointing to the repository")
- profile = json.loads((repository / 'evaluation/wcxb/current-report.json').read_text())
+ reject_evidence_symlinks(profile_path)
+ profile = json.loads(profile_path.read_text())
+ comparison = profile.get('paired_comparison') or {}
+ baseline, candidate_id = comparison.get('baseline'), comparison.get('candidate')
+ repair_analysis = bool(profile.get('optional_ui_receipt_sha256'))
from .wcxb_publication import inventory
inventory(root, profile)
- binding_path = contained_path(repository, 'evaluation/wcxb/report-content-binding.json',
- reject_symlinks=True)
+ binding_path = profile_path.with_name('report-content-binding.json')
+ reject_evidence_symlinks(binding_path)
check_bound_interpretation(root, json.loads(binding_path.read_bytes()))
sample_file = root / 'sample-manifest.json'
sample = json.loads(sample_file.read_text())
@@ -303,52 +328,59 @@ def check(root: Path, *, authority_root: Path | None = None) -> None:
case_ids = {row['file_id'] for row in sample['cases']}
excluded = {row['file_id'] for row in sample.get('exclusions', [])}
qualified = case_ids - excluded
- require(len(case_ids) == profile['case_count'] == 504, 'WCXB case count differs')
+ require(len(case_ids) == profile['case_count'], 'WCXB case count differs')
require(len(scores) == len(case_ids) * len(providers), 'WCXB score count differs')
- require(summary['quality_case_count'] == profile['primary_quality_case_count'] == sample['primary_quality_case_count'] == 504, 'WCXB quality denominator differs')
+ require(summary['quality_case_count'] == profile['primary_quality_case_count'] == sample['primary_quality_case_count'], 'WCXB quality denominator differs')
require(summary['exclusions'] == sample.get('exclusions', []) == [], 'WCXB current cohort contains excluded cases')
require(summary['benchmark'] == sample['dataset_id'] == certificate['benchmark'], 'WCXB certified dataset identity differs')
require({(r['provider'], r['case_id']) for r in scores} == {(p, c) for p in providers for c in case_ids}, 'WCXB method-case matrix differs')
require([r['provider'] for r in certificate['providers']] == providers, 'WCXB certificate participants differ')
- candidate = next(r for r in certificate['providers'] if r['provider'] == 'moli-pr643')
- require(candidate['identity'].get('binary_sha256') == profile['candidate_binary_sha256'],
- 'WCXB candidate binary differs from the current report profile')
- released = next(r for r in certificate['providers'] if r['provider'] == 'moli')
- require(released['identity'].get('binary_sha256') == profile['release_binary_sha256'],
- 'WCXB released Moli binary differs from the current report profile')
- optional_root = root / 'research/wcxb-optional-ui-4017'
- optional_receipt = optional_root / 'receipt.json'
- require(optional_receipt.is_file() and not optional_receipt.is_symlink()
- and digest(optional_receipt) == profile['optional_ui_receipt_sha256'],
- 'WCXB optional UI-stripping evidence differs from current profile')
- optional = json.loads(optional_receipt.read_text())
- for name, binary_hash in (('release', profile['release_binary_sha256']),
- ('candidate', profile['candidate_binary_sha256'])):
- build = optional['builds'][name]
- artifact = optional_root / f'{name}.md'
- require(build['binary_sha256'] == binary_hash and build['returncode'] == 0
- and artifact.is_file() and not artifact.is_symlink()
- and digest(artifact) == build['output_sha256'],
- 'WCXB optional UI-stripping output is not bound to its binary')
- require(not optional['builds']['release']['description_present']
- and optional['builds']['candidate']['description_present'],
- 'WCXB optional UI-stripping finding is not supported')
+ if comparison:
+ require(baseline != candidate_id and {baseline,candidate_id} <= set(providers),
+ 'WCXB paired comparison names invalid participants')
+ candidate = next(r for r in certificate['providers'] if r['provider'] == candidate_id)
+ require(candidate['identity'].get('binary_sha256') == profile['candidate_binary_sha256'],
+ 'WCXB candidate binary differs from the current report profile')
+ released = next(r for r in certificate['providers'] if r['provider'] == baseline)
+ require(released['identity'].get('binary_sha256') == profile['release_binary_sha256'],
+ 'WCXB released Moli binary differs from the current report profile')
+ if repair_analysis:
+ optional_root = root / 'research/wcxb-optional-ui-4017'
+ optional_receipt = optional_root / 'receipt.json'
+ require(optional_receipt.is_file() and not optional_receipt.is_symlink()
+ and digest(optional_receipt) == profile['optional_ui_receipt_sha256'],
+ 'WCXB optional UI-stripping evidence differs from current profile')
+ optional = json.loads(optional_receipt.read_text())
+ for name, binary_hash in (('release', profile['release_binary_sha256']),
+ ('candidate', profile['candidate_binary_sha256'])):
+ build = optional['builds'][name]
+ artifact = optional_root / f'{name}.md'
+ require(build['binary_sha256'] == binary_hash and build['returncode'] == 0
+ and artifact.is_file() and not artifact.is_symlink()
+ and digest(artifact) == build['output_sha256'],
+ 'WCXB optional UI-stripping output is not bound to its binary')
+ require(not optional['builds']['release']['description_present']
+ and optional['builds']['candidate']['description_present'],
+ 'WCXB optional UI-stripping finding is not supported')
from harness.web_fetch.wcxb_latency import (
load_full_paired_latency,
summarize_full_paired_latency,
)
- _, selected_latency = load_full_paired_latency(root / 'research/wcxb-full-paired-latency', profile, case_ids)
- paired_latency = summarize_full_paired_latency(selected_latency, case_ids)
- for provider, method in (('moli', 'release'), ('moli-pr643', 'pr')):
- values = sorted(selected_latency[case_id, method]['elapsed_ms'] for case_id in case_ids)
- position = (len(values) - 1) * .95
- low = int(position)
- p95 = values[low] + (values[min(low + 1, len(values) - 1)] - values[low]) * (position - low)
- measured = summary['providers'][provider]
- require(abs(measured['mean_latency_ms'] - paired_latency[method]['mean_ms']) < .01
- and abs(measured['p50_latency_ms'] - paired_latency[method]['p50_ms']) < .01
- and abs(measured['p95_latency_ms'] - p95) < .01,
- 'WCXB Moli timing summary differs from certified full paired evidence')
+ paired_latency = None
+ if profile.get('latency_correction_receipt_sha256'):
+ require(bool(comparison), 'WCXB corrected timing has no declared participants')
+ _, selected_latency = load_full_paired_latency(root / 'research/wcxb-full-paired-latency', profile, case_ids)
+ paired_latency = summarize_full_paired_latency(selected_latency, case_ids)
+ for provider, method in ((baseline, 'release'), (candidate_id, 'pr')):
+ values = sorted(selected_latency[case_id, method]['elapsed_ms'] for case_id in case_ids)
+ position = (len(values) - 1) * .95
+ low = int(position)
+ p95 = values[low] + (values[min(low + 1, len(values) - 1)] - values[low]) * (position - low)
+ measured = summary['providers'][provider]
+ require(abs(measured['mean_latency_ms'] - paired_latency[method]['mean_ms']) < .01
+ and abs(measured['p50_latency_ms'] - paired_latency[method]['p50_ms']) < .01
+ and abs(measured['p95_latency_ms'] - p95) < .01,
+ 'WCXB Moli timing summary differs from certified full paired evidence')
evidence_hashes = json.loads((root / 'evidence-manifest.json').read_text())
require(set(evidence_hashes) == case_ids, 'WCXB evidence inventory differs')
by_case = {}
@@ -401,24 +433,25 @@ def check(root: Path, *, authority_root: Path | None = None) -> None:
require(all(f'data-case="{case}"' not in html for case in excluded), 'Invalid references entered result inventory')
if route == 'guide/index.html':
guide = BeautifulSoup(html, 'html.parser')
- require_guide_interpretation(guide, summary, providers, lang)
- for image in ('code-block.png', 'description.png', 'adjacent-code.png'):
- asset = root / 'assets/wcxb-repair' / image
- require(asset.is_file() and asset.read_bytes()[:8] == b'\x89PNG\r\n\x1a\n',
- f'Repair graphic missing or invalid: {image}')
- headings = {node.get_text(' ', strip=True) for node in guide.select('h2')}
- require(('Obscura 为什么分数更高' if lang == 'zh' else 'Why Obscura scores higher') in headings,
- 'Obscura content analysis is missing')
- require(('Moli 两版耗时' if lang == 'zh' else 'Moli latency') in headings,
- 'Paired Moli latency analysis is missing')
- require(guide.select_one('a[href="research/wcxb-full-paired-latency/calls.jsonl"]') is not None,
- 'Paired latency evidence link is missing')
- require(guide.select_one('a[href$="cases/4461.html#obscura"]') is not None,
- 'Obscura boundary counterexample is missing')
+ require_guide_interpretation(guide, summary, providers, lang, repair_analysis=repair_analysis)
+ if repair_analysis:
+ for image in ('code-block.png', 'description.png', 'adjacent-code.png'):
+ asset = root / 'assets/wcxb-repair' / image
+ require(asset.is_file() and asset.read_bytes()[:8] == b'\x89PNG\r\n\x1a\n',
+ f'Repair graphic missing or invalid: {image}')
+ headings = {node.get_text(' ', strip=True) for node in guide.select('h2')}
+ require(('Obscura 为什么分数更高' if lang == 'zh' else 'Why Obscura scores higher') in headings,
+ 'Obscura content analysis is missing')
+ require(('Moli 两版耗时' if lang == 'zh' else 'Moli latency') in headings,
+ 'Paired Moli latency analysis is missing')
+ require(guide.select_one('a[href="research/wcxb-full-paired-latency/calls.jsonl"]') is not None,
+ 'Paired latency evidence link is missing')
+ require(guide.select_one('a[href$="cases/4461.html#obscura"]') is not None,
+ 'Obscura boundary counterexample is missing')
if route in ('index.html', 'datasets/index.html'):
check_presentation(html)
if route == 'index.html':
- require_home_interpretation(html, summary, paired_latency, lang, scores=scores)
+ require_home_interpretation(html, summary, paired_latency, lang, scores=scores, comparison=comparison)
require('id="home-type"' in html, 'Page-type filter missing')
require('官方全量 F1' not in html and 'Official full F1' not in html, 'Invalid full-reference score remains visible')
first_card = BeautifulSoup(html, 'html.parser').select_one('.lead-grid .panel')
@@ -474,26 +507,27 @@ def check(root: Path, *, authority_root: Path | None = None) -> None:
# Negative controls ensure the semantic contract rejects missing or wrong highlights.
home = BeautifulSoup((root / 'index.html').read_text(), 'html.parser')
winner = home.select_one('#wcxb-core .comparison-winner')
- require(winner is not None, 'No numeric winner for negative control')
- winner.unwrap()
- try:
- check_presentation(home)
- except ValueError:
- pass
- else:
- raise AssertionError('A missing comparison highlight was accepted')
+ if winner is not None:
+ winner.unwrap()
+ try:
+ check_presentation(home)
+ except ValueError:
+ pass
+ else:
+ raise AssertionError('A missing comparison highlight was accepted')
home = BeautifulSoup((root / 'index.html').read_text(), 'html.parser')
- first = home.select_one('#wcxb-core .comparison-winner').find_parent('td')
- metric = first['data-metric']
- other = next(cell for cell in home.select('#wcxb-core td[data-metric]')
- if cell['data-metric'] == metric and cell is not first)
- other.append(BeautifulSoup('wrong', 'html.parser'))
- try:
- check_presentation(home)
- except ValueError:
- pass
- else:
- raise AssertionError('An incorrect comparison highlight was accepted')
+ if len(providers) > 1:
+ first = home.select_one('#wcxb-core td[data-metric]')
+ metric = first['data-metric']
+ other = next(cell for cell in home.select('#wcxb-core td[data-metric]')
+ if cell['data-metric'] == metric and cell is not first)
+ other.append(BeautifulSoup('wrong', 'html.parser'))
+ try:
+ check_presentation(home)
+ except ValueError:
+ pass
+ else:
+ raise AssertionError('An incorrect comparison highlight was accepted')
analysis = require_analysis_controls((root / 'analysis/index.html').read_text(), len(qualified) * len(providers))
analysis.select_one('#diagnosis-dialog [data-diagnosis-link]').decompose()
try:
@@ -513,108 +547,111 @@ def check(root: Path, *, authority_root: Path | None = None) -> None:
pass
else:
raise AssertionError('A duplicate result summary was accepted')
- for lang, prefix in language_pairs:
- guide = BeautifulSoup((root / prefix / 'guide/index.html').read_text(), 'html.parser')
- captions = guide.select('#guide-repair .repair-case')[1].select('figcaption span')
- captions[1].string = 'Moli 1.1.8'
- try:
- require_guide_interpretation(guide, summary, providers, lang)
- except ValueError:
- pass
- else:
- raise AssertionError(f'A {lang} optional-feature caption without its mode was accepted')
+ if repair_analysis:
+ for lang, prefix in language_pairs:
+ guide = BeautifulSoup((root / prefix / 'guide/index.html').read_text(), 'html.parser')
+ captions = guide.select('#guide-repair .repair-case')[1].select('figcaption span')
+ captions[1].string = 'Moli 1.1.8'
+ try:
+ require_guide_interpretation(guide, summary, providers, lang, repair_analysis=repair_analysis)
+ except ValueError:
+ pass
+ else:
+ raise AssertionError(f'A {lang} optional-feature caption without its mode was accepted')
guide = BeautifulSoup((root / 'guide/index.html').read_text(), 'html.parser')
guide.select_one('#guide-results tbody tr').decompose()
try:
- require_guide_interpretation(guide, summary, providers, 'zh')
+ require_guide_interpretation(guide, summary, providers, 'zh', repair_analysis=repair_analysis)
except ValueError:
pass
else:
raise AssertionError('A guide with a missing measured method was accepted')
- guide = BeautifulSoup((root / 'guide/index.html').read_text(), 'html.parser')
- guide.select_one('#guide-repair a[href$="cases/5458.html#moli-pr643"]').decompose()
- try:
- require_guide_interpretation(guide, summary, providers, 'zh')
- except ValueError:
- pass
- else:
- raise AssertionError('A guide with broken worked evidence was accepted')
- for lang, prefix in language_pairs:
- guide = BeautifulSoup((root / prefix / 'guide/index.html').read_text(), 'html.parser')
- guide.select_one('#guide-repair .repair-case').decompose()
- try:
- require_guide_interpretation(guide, summary, providers, lang)
- except ValueError:
- pass
- else:
- raise AssertionError(f'A {lang} guide missing a repair diagram was accepted')
- guide = BeautifulSoup((root / prefix / 'guide/index.html').read_text(), 'html.parser')
- guide.select_one('#guide-repair figure img').decompose()
- try:
- require_guide_interpretation(guide, summary, providers, lang)
- except ValueError:
- pass
- else:
- raise AssertionError(f'A {lang} guide missing a repair image was accepted')
- guide = BeautifulSoup((root / prefix / 'guide/index.html').read_text(), 'html.parser')
- guide.select('#guide-repair .repair-case')[2].select('figcaption span')[1].string = (
- '正式版 1.1.7' if lang == 'zh' else 'Release 1.1.7')
+ if repair_analysis:
+ guide = BeautifulSoup((root / 'guide/index.html').read_text(), 'html.parser')
+ guide.select_one('#guide-repair a[href$="cases/5458.html#moli-pr643"]').decompose()
try:
- require_guide_interpretation(guide, summary, providers, lang)
+ require_guide_interpretation(guide, summary, providers, 'zh', repair_analysis=repair_analysis)
except ValueError:
pass
else:
- raise AssertionError(f'A {lang} guide mislabeled the tagged release')
- guide = BeautifulSoup((root / prefix / 'guide/index.html').read_text(), 'html.parser')
- for node in guide.find_all(string=lambda value: value and 'gap-2' in value):
- node.replace_with(node.replace('gap-2', ''))
- try:
- require_guide_interpretation(guide, summary, providers, lang)
- except ValueError:
- pass
- else:
- raise AssertionError(f'A {lang} guide missing the CSS-spacing limit was accepted')
+ raise AssertionError('A guide with broken worked evidence was accepted')
+ for lang, prefix in language_pairs:
+ guide = BeautifulSoup((root / prefix / 'guide/index.html').read_text(), 'html.parser')
+ guide.select_one('#guide-repair .repair-case').decompose()
+ try:
+ require_guide_interpretation(guide, summary, providers, lang, repair_analysis=repair_analysis)
+ except ValueError:
+ pass
+ else:
+ raise AssertionError(f'A {lang} guide missing a repair diagram was accepted')
+ guide = BeautifulSoup((root / prefix / 'guide/index.html').read_text(), 'html.parser')
+ guide.select_one('#guide-repair figure img').decompose()
+ try:
+ require_guide_interpretation(guide, summary, providers, lang, repair_analysis=repair_analysis)
+ except ValueError:
+ pass
+ else:
+ raise AssertionError(f'A {lang} guide missing a repair image was accepted')
+ guide = BeautifulSoup((root / prefix / 'guide/index.html').read_text(), 'html.parser')
+ guide.select('#guide-repair .repair-case')[2].select('figcaption span')[1].string = (
+ '正式版 1.1.7' if lang == 'zh' else 'Release 1.1.7')
+ try:
+ require_guide_interpretation(guide, summary, providers, lang, repair_analysis=repair_analysis)
+ except ValueError:
+ pass
+ else:
+ raise AssertionError(f'A {lang} guide mislabeled the tagged release')
+ guide = BeautifulSoup((root / prefix / 'guide/index.html').read_text(), 'html.parser')
+ for node in guide.find_all(string=lambda value: value and 'gap-2' in value):
+ node.replace_with(node.replace('gap-2', ''))
+ try:
+ require_guide_interpretation(guide, summary, providers, lang, repair_analysis=repair_analysis)
+ except ValueError:
+ pass
+ else:
+ raise AssertionError(f'A {lang} guide missing the CSS-spacing limit was accepted')
guide = BeautifulSoup((root / 'guide/index.html').read_text(), 'html.parser')
- guide.select_one('#guide-results .comparison-winner').unwrap()
- try:
- require_guide_interpretation(guide, summary, providers, 'zh')
- except ValueError:
- pass
- else:
- raise AssertionError('A guide without a winning-column highlight was accepted')
- home = BeautifulSoup((root / 'index.html').read_text(), 'html.parser')
- home.select_one('#moli-latency-context').decompose()
- try:
- require_home_interpretation(home, summary, paired_latency, 'zh', scores=scores)
- except ValueError:
- pass
- else:
- raise AssertionError('A home page without adjacent timing context was accepted')
- with tempfile.TemporaryDirectory(prefix='wcxb-latency-negative-') as directory:
- invalid = Path(directory)
- evidence = root / 'research/wcxb-full-paired-latency'
- shutil.copy2(evidence / 'receipt.json', invalid / 'receipt.json')
- shutil.copy2(evidence / 'calls.jsonl', invalid / 'calls.jsonl')
- calls = (invalid / 'calls.jsonl').read_bytes()
- (invalid / 'calls.jsonl').write_bytes(calls.rsplit(b'\n', 2)[0] + b'\n')
+ if guide.select_one('#guide-results .comparison-winner') is not None:
+ guide.select_one('#guide-results .comparison-winner').unwrap()
try:
- load_full_paired_latency(invalid, profile, case_ids)
+ require_guide_interpretation(guide, summary, providers, 'zh', repair_analysis=repair_analysis)
except ValueError:
pass
else:
- raise AssertionError('An incomplete paired latency cohort was accepted')
- (invalid / 'calls.jsonl').write_bytes(calls)
- receipt = json.loads((invalid / 'receipt.json').read_text())
- receipt['binary_sha256']['pr'] = '0' * 64
- (invalid / 'receipt.json').write_text(json.dumps(receipt))
+ raise AssertionError('A guide without a winning-column highlight was accepted')
+ if paired_latency is not None:
+ home = BeautifulSoup((root / 'index.html').read_text(), 'html.parser')
+ home.select_one('#paired-latency-context').decompose()
try:
- load_full_paired_latency(invalid, profile, case_ids)
+ require_home_interpretation(home, summary, paired_latency, 'zh', scores=scores)
except ValueError:
pass
else:
- raise AssertionError('A wrong paired binary identity was accepted')
+ raise AssertionError('A home page without adjacent timing context was accepted')
+ with tempfile.TemporaryDirectory(prefix='wcxb-latency-negative-') as directory:
+ invalid = Path(directory)
+ evidence = root / 'research/wcxb-full-paired-latency'
+ shutil.copy2(evidence / 'receipt.json', invalid / 'receipt.json')
+ shutil.copy2(evidence / 'calls.jsonl', invalid / 'calls.jsonl')
+ calls = (invalid / 'calls.jsonl').read_bytes()
+ (invalid / 'calls.jsonl').write_bytes(calls.rsplit(b'\n', 2)[0] + b'\n')
+ try:
+ load_full_paired_latency(invalid, profile, case_ids)
+ except ValueError:
+ pass
+ else:
+ raise AssertionError('An incomplete paired latency cohort was accepted')
+ (invalid / 'calls.jsonl').write_bytes(calls)
+ receipt = json.loads((invalid / 'receipt.json').read_text())
+ receipt['binary_sha256']['pr'] = '0' * 64
+ (invalid / 'receipt.json').write_text(json.dumps(receipt))
+ try:
+ load_full_paired_latency(invalid, profile, case_ids)
+ except ValueError:
+ pass
+ else:
+ raise AssertionError('A wrong paired binary identity was accepted')
require((root / 'assets/style.css').is_file() and (root / 'assets/report.js').is_file(), 'WCXB report assets missing')
- require(len([path for path in root.rglob('*') if path.is_file()]) < 2000, 'WCXB report tree unexpectedly large')
print(f'PASS: WCXB {len(case_ids)}-case, {len(providers)}-method, declared-language evidence and route inventory')
@@ -622,8 +659,9 @@ def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('report', type=Path)
parser.add_argument('--authority-root', type=Path)
+ parser.add_argument('--profile', type=Path)
args = parser.parse_args()
- check(args.report, authority_root=args.authority_root)
+ check(args.report, authority_root=args.authority_root, profile_path=args.profile)
def binary_from_env(path: Path) -> str:
diff --git a/pyproject.toml b/pyproject.toml
index 3861cc8e0..9bf1058a1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -6,7 +6,7 @@ readme = "README.md"
license = {file = "LICENSE"}
requires-python = ">=3.11"
dependencies = [
- "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@5b12816906ad8cd2b55b9804c77e7e6c55f2c54e",
+ "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@1552f731dd75be26c3b3551c04bf0730e6293335",
"beautifulsoup4>=4.12,<5",
"fastcdc==1.7.0",
]
diff --git a/scripts/build_report.py b/scripts/build_report.py
index 07c0b056c..3fff89ec3 100644
--- a/scripts/build_report.py
+++ b/scripts/build_report.py
@@ -16,7 +16,7 @@
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('--evidence',type=Path)
parser.add_argument('--profile',type=Path,
- help='current-report profile under evaluation/; requires an explicit output unless the profile declares report_output')
+ help='report profile in the repository or external storage; requires an explicit output unless the profile declares report_output')
parser.add_argument('--public-benchmark-research',type=Path)
parser.add_argument('--output',type=Path)
add_language_argument(parser)
@@ -58,8 +58,6 @@ def requested_language(output, supported=SUPPORTED_REPORT_LANGUAGES):
if args.profile:
args.profile=(ROOT/args.profile).resolve() if not args.profile.is_absolute() else args.profile.resolve()
- if not args.profile.is_relative_to((ROOT/'evaluation').resolve()):
- parser.error('--profile must stay under evaluation/')
if json.loads(args.profile.read_text()).get('variant') == 'diagnostic-cohorts':
args.diagnostic_cohorts=True
if args.diagnostic_cohorts:
diff --git a/tests/test_acceptance_scope.py b/tests/test_acceptance_scope.py
new file mode 100644
index 000000000..1e99b2248
--- /dev/null
+++ b/tests/test_acceptance_scope.py
@@ -0,0 +1,43 @@
+"""Report coverage describes declarations, not inferred retrieval capabilities."""
+from copy import deepcopy
+from harness.web_fetch.report_components import acceptance_scope
+
+
+def test_scope_separates_literal_facts_body_and_records_without_changing_verdicts():
+ cases = {
+ 'navigation': {'id': 'navigation', 'track': 'popular', 'expected_any': ['Home'], 'reference_answer': 'Instruction to read the page'},
+ 'article': {'id': 'article', 'track': 'dynamic', 'page_identity_required': True,
+ 'ground_truth_main_content': 'Full independent article', 'required_facts': [{'id': 'fact'}], 'body_requirements': {'regions': ['body']},
+ 'acceptance_contract': {'minimum_records': 3}},
+ 'resource': {'id': 'resource', 'track': 'web', 'task_kind': 'resource',
+ 'expected_resources': [{'id': 'video'}]},
+ }
+ rows = [
+ {'provider': 'service', 'case_id': 'navigation', 'ok': True, 'metrics': {'content_pass': True}},
+ {'provider': 'service', 'case_id': 'article', 'ok': True,
+ 'metrics': {'content_pass': True, 'forbidden_requirement_hits': [], 'page_identity_required': True, 'page_identity_status': 'verified', 'blocked_page': True}},
+ {'provider': 'service', 'case_id': 'resource', 'ok': True,
+ 'metrics': {'resource_pass': True, 'resource_precision': .5, 'unexpected_resource_urls': ['other']}},
+ ]
+ before = deepcopy((cases, rows))
+ scope = acceptance_scope(cases, rows)
+ assert scope['families']['popular']['literal'] == 1
+ assert scope['families']['popular']['body_regions'] == scope['families']['popular']['full_reference'] == 0
+ assert scope['families']['dynamic']['full_reference'] == 1
+ assert scope['families']['dynamic']['facts'] == scope['families']['dynamic']['body_regions'] == 1
+ assert scope['families']['dynamic']['records'] == scope['families']['dynamic']['identity'] == 1
+ method = scope['methods']['service']
+ assert method['passes'] == 3 and method['unverified_identity_passes'] == 2
+ assert method['blocked_diagnostics'] == method['resource_results'] == method['unexpected_resources'] == 1
+ assert method['resource_precision_sum'] == .5
+ assert (cases, rows) == before
+
+
+def test_affiliation_is_exact_repository_scope_not_a_similar_name():
+ cases = {name: {'id': name, 'url': url} for name, url in {
+ 'owned': 'https://github.com/lexmount/moli',
+ 'readme': 'https://github.com/lexmount/moli/blob/main/README.md',
+ 'other': 'https://github.com/lexmount/molina',
+ 'external': 'https://github.com/another/moli',
+ }.items()}
+ assert acceptance_scope(cases, [])['affiliated_cases'] == ['owned', 'readme']
diff --git a/tests/test_dataset_assembly.py b/tests/test_dataset_assembly.py
index d545111c9..0efa5595d 100644
--- a/tests/test_dataset_assembly.py
+++ b/tests/test_dataset_assembly.py
@@ -320,3 +320,19 @@ def test_input_changing_case_correction_requires_all_method_rerun(tmp_path):
assert dataset["cases"][0]["category"] == "current_homepage"
assert dataset["cases"][0]["tags"] == ["current"]
assert dataset["comparison"]["case_corrections"]["rerun_all_methods"] == ["plain"]
+
+
+def test_explicit_external_profile_is_portable_and_preserves_input_hash_checks(tmp_path):
+ repository = tmp_path / 'checkout'
+ path, assembly = _fixture(repository)
+ profile = tmp_path / 'instance/profile.json'
+ _write(profile, json.loads((repository / 'profile.json').read_text()))
+ dataset, raw = assemble_profile_dataset(path, repository=repository,
+ profile_path_override=profile, require_output_hash=False)
+ assert dataset['comparison']['reproduction_profile'] == 'profile.json'
+ assert str(tmp_path).encode() not in raw
+ source = repository / 'plain.json'
+ source.write_text('{"cases": []}')
+ with pytest.raises(ValueError, match='source hash changed'):
+ assemble_profile_dataset(path, repository=repository,
+ profile_path_override=profile, require_output_hash=False)
diff --git a/tests/test_file_scoring_evidence.py b/tests/test_file_scoring_evidence.py
new file mode 100644
index 000000000..02a365424
--- /dev/null
+++ b/tests/test_file_scoring_evidence.py
@@ -0,0 +1,37 @@
+"""Saved file evidence must be complete before it can be rescored."""
+import hashlib
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from harness.web_fetch.artifacts import file_scoring_metadata
+
+
+class FileScoringEvidenceTests(unittest.TestCase):
+ def test_missing_delivery_is_not_malformed_evidence(self):
+ self.assertEqual(file_scoring_metadata({'task_kind': 'file'}, {}, Path('.')), {})
+
+ def test_malformed_saved_claims_fail_before_scoring(self):
+ for evaluation in (None, [], 'invalid', {'files': {}}, {'files': [{}]},
+ {'files': [{'artifact_path': 'file'}]}):
+ with self.subTest(evaluation=evaluation), self.assertRaises(ValueError):
+ file_scoring_metadata({'task_kind': 'file'},
+ {'file_evaluation': evaluation}, Path('.'))
+
+ def test_rescore_preserves_each_delivered_file_binding(self):
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ raw = b'retained delivered bytes'
+ (root / 'file.bin').write_bytes(raw)
+ binding = {'artifact_path': 'file.bin', 'sha256': hashlib.sha256(raw).hexdigest()}
+ metadata = {'file_evaluation': {'files': [binding]}}
+ with patch('lexbench_webfetch.file_acceptance.evaluate_files',
+ return_value={'files': [{'passed': False}]}):
+ result = file_scoring_metadata({'task_kind': 'file'}, metadata, root)
+ self.assertEqual(result['file_evaluation']['files'], [{**binding, 'passed': False}])
+ self.assertEqual(metadata['file_evaluation']['files'], [binding])
+ with patch('lexbench_webfetch.file_acceptance.evaluate_files',
+ return_value={'files': []}):
+ with self.assertRaisesRegex(ValueError, 'file count'):
+ file_scoring_metadata({'task_kind': 'file'}, metadata, root)
diff --git a/tests/test_headless_report.py b/tests/test_headless_report.py
index db2108690..61d3cb795 100644
--- a/tests/test_headless_report.py
+++ b/tests/test_headless_report.py
@@ -1059,9 +1059,9 @@ def test_public_builder_materializes_bound_prose_and_preserves_scores(self):
for track in ['web','risk_control']},
'frozen_inputs':frozen_input_bindings(source)}
repository=Path(__file__).resolve().parents[1]
- with tempfile.TemporaryDirectory(prefix='report-profile-test-',dir=repository/'evaluation') as profile_dir:
+ with tempfile.TemporaryDirectory(prefix='report-profile-test-') as profile_dir:
profile_path=Path(profile_dir)/'current-report.json'
- measured['comparison']['reproduction_profile']=profile_path.relative_to(repository).as_posix()
+ measured['comparison']['reproduction_profile']='instances/current-report.json'
measured['comparison']['report_output']='reports/synthetic-browser'
(source/'dataset.json').write_text(json.dumps(measured))
profile['frozen_inputs']=frozen_input_bindings(source)
diff --git a/tests/test_reliability.py b/tests/test_reliability.py
index 495859a3f..4f656a794 100644
--- a/tests/test_reliability.py
+++ b/tests/test_reliability.py
@@ -28,6 +28,19 @@ def test_failure_only_reruns_retain_successes_and_disclose_build_counts(self):
rows[1]['provider_metadata']['binary_sha256'] = 'first-build'
self.assertTrue(require_reliable(dataset, rows)['complete'])
+ def test_inventory_counts_cloud_unknown_and_retained_build_results(self):
+ from harness.web_fetch.reliability import implementation_inventory
+ rows = [
+ {'provider': 'cloud', 'provider_metadata': {}},
+ {'provider': 'cloud', 'provider_metadata': {'reported_version': 'service-v2'}},
+ {'provider': 'local', 'provider_metadata': {'binary_sha256': 'build-a'}},
+ {'provider': 'local', 'provider_metadata': None},
+ ]
+ inventory = implementation_inventory(rows)
+ self.assertEqual(inventory['cloud'], {'unknown': 1, 'reported-version:service-v2': 1})
+ self.assertEqual(inventory['local'], {'build-a': 1, 'unknown': 1})
+ self.assertEqual(sum(sum(counts.values()) for counts in inventory.values()), len(rows))
+
def test_separately_named_builds_can_be_compared_on_the_complete_population(self):
dataset = {**self.dataset, 'coverage_plan': {'web': ['release', 'candidate']}}
rows = [{**self.record, 'provider': name,
diff --git a/tests/test_report_methods.py b/tests/test_report_methods.py
index ecabc4ac8..437bba929 100644
--- a/tests/test_report_methods.py
+++ b/tests/test_report_methods.py
@@ -195,4 +195,8 @@ def test_method_guide_discloses_retained_success_and_repaired_failure_builds(tmp
assert '累计成绩不代表某一个版本' in body
single = render_methods(tmp_path, records[:2], hero=hero, section=section,
table=table, cell=cell, link=link, esc=esc)
- assert 'implementation-scope' not in single
+ assert 'implementation-scope' in single
+ assert 'retained-build' in single and '>2<' in single
+ unknown = render_methods(tmp_path, [{'provider': 'cloud', 'provider_type': 'fixture'}],
+ hero=hero, section=section, table=table, cell=cell, link=link, esc=esc)
+ assert 'unknown' in unknown and '>1<' in unknown
diff --git a/tests/test_wcxb_participant_registry.py b/tests/test_wcxb_participant_registry.py
new file mode 100644
index 000000000..96489fa1e
--- /dev/null
+++ b/tests/test_wcxb_participant_registry.py
@@ -0,0 +1,91 @@
+"""A report instance may use participant IDs absent from historical cohorts."""
+from harness.web_fetch.report import generate_wcxb
+from tests.wcxb_fixture import create_report_inputs
+
+
+def test_wcxb_renderer_accepts_entirely_new_participant_ids(tmp_path):
+ root, rows, summary, ground_truth, identities, profile, content = create_report_inputs(tmp_path)
+ names = {name: 'participant-' + str(i) for i, name in enumerate(summary['providers'])}
+ for row in rows:
+ row['provider'] = names[row['provider']]
+ summary['providers'] = {names[name]: values for name, values in summary['providers'].items()}
+ identities = {names[name]: values for name, values in identities.items()}
+ profile['providers'] = [names[name] for name in profile['providers']]
+ profile['versions'] = {names[name]: value for name, value in profile['versions'].items()}
+ profile['paired_comparison'] = {'baseline': names['moli'], 'candidate': names['moli-pr643']}
+ for name, replacement in names.items():
+ (root / 'artifacts' / name).rename(root / 'artifacts' / replacement)
+ generate_wcxb(root, rows, summary, ground_truth, identities, profile, content, include_english=True)
+ for prefix in ('', 'en/'):
+ home = (root / prefix / 'index.html').read_text()
+ assert all('providers/' + name + '.html' in home for name in names.values())
+ assert 'data-baseline="' + names['moli'] + '"' in home
+ assert 'data-candidate="' + names['moli-pr643'] + '"' in home
+ assert 'Only Kitesurf' not in (root / prefix / 'accounts/index.html').read_text()
+
+
+def test_public_wcxb_build_check_package_accept_external_nonhistorical_cohort(tmp_path):
+ import hashlib
+ import json
+ import subprocess
+ import sys
+ from pathlib import Path
+ from harness.web_fetch.wcxb_scoring import score
+ from tests.wcxb_fixture import create_cohort, write_json
+
+ repository = Path(__file__).resolve().parents[1]
+ cohort = create_cohort(tmp_path / 'cohort', providers=('alpha', 'beta', 'gamma'))
+ summary = score(**cohort)
+ evidence = cohort['evidence_root']
+ profile = json.loads(cohort['profile_path'].read_bytes())
+ review_path = evidence / 'report-assets/research/wcxb-representative-review.json'
+ write_json(review_path, {'scope': 'targeted_representative_anomaly_review',
+ 'case_count': 0, 'result_count': 0, 'total_results': 6, 'unreviewed_results': 6, 'cases': []})
+ profile.update(source_repository='https://example.invalid/benchmark', source_commit='a' * 40,
+ versions={name: 'synthetic' for name in profile['providers']},
+ report_assets={'research/wcxb-representative-review.json':
+ hashlib.sha256(review_path.read_bytes()).hexdigest()})
+ anchors = ('guide-results', 'guide-score', 'guide-cases', 'guide-methods',
+ 'guide-timing', 'guide-conclusion')
+ rows = []
+ for name, values in summary['providers'].items():
+ rows.append('
The complete population contains two frozen pages and three methods. '
+ 'One method has a retained request failure, which stays in the score and latency population. '
+ 'The other methods tie on these synthetic inputs. These fixtures test evidence handling; '
+ 'they do not measure browser capability or support an adoption recommendation.
')
+ body += '
' + anchor + '
' + detail + ''
+ content_path = evidence / 'report-assets/report-content.json'
+ write_json(content_path, {'schema_version': 1, 'guide': {'zh': body, 'en': body}})
+ profile_path = tmp_path / 'instance/profile.json'
+ write_json(profile_path, profile)
+ binding = {'schema_version': 1, 'relative_path': 'report-content.json',
+ 'sha256': hashlib.sha256(content_path.read_bytes()).hexdigest(),
+ 'evidence': {'wcxb-score-summary.json': hashlib.sha256((evidence / 'wcxb-score-summary.json').read_bytes()).hexdigest()},
+ 'authority': {'configs/report-templates.json': hashlib.sha256((repository / 'configs/report-templates.json').read_bytes()).hexdigest()},
+ 'rendered_sha256': {'guide': {lang: hashlib.sha256(body.encode()).hexdigest() for lang in ('zh', 'en')}}}
+ write_json(profile_path.with_name('report-content-binding.json'), binding)
+ output = tmp_path / 'report'
+ commands = [
+ ['scripts/build_report.py', '--profile', str(profile_path), '--evidence', str(evidence),
+ '--wcxb-source', str(cohort['source']), '--wcxb-sample', str(cohort['sample_manifest']),
+ '--output', str(output), '--language', 'en'],
+ ['scripts/check_report.py', str(output), '--profile', str(profile_path)],
+ ['scripts/publish_wcxb_artifact_site.py', '--package-only', '--source', str(output),
+ '--profile', str(profile_path), '--work', str(tmp_path / 'package')],
+ ]
+ for command in commands:
+ result = subprocess.run([sys.executable, *command], cwd=repository, capture_output=True, text=True)
+ assert result.returncode == 0, result.stdout + result.stderr
+ assert (tmp_path / 'package/wcxb-current.zip').is_file()
diff --git a/tests/wcxb_fixture.py b/tests/wcxb_fixture.py
index 9be2ee1c3..9fd1a231c 100644
--- a/tests/wcxb_fixture.py
+++ b/tests/wcxb_fixture.py
@@ -262,6 +262,7 @@ def create_report_inputs(tmp_path, *, english_source=True):
original = load_jsonl(cohort['evidence_root'] / 'wcxb-scores.jsonl')
profile = json.loads(cohort['profile_path'].read_bytes())
profile.update(source_repository='https://example.invalid/synthetic', source_commit='a' * 40,
+ paired_comparison={'baseline':'moli','candidate':'moli-pr643'},
candidate_source_commit='b' * 40,
versions={provider: 'synthetic' for provider in PROVIDERS})
root = tmp_path / 'report'
diff --git a/uv.lock b/uv.lock
index 1f7bd9233..4f5b88f06 100644
--- a/uv.lock
+++ b/uv.lock
@@ -354,7 +354,7 @@ requires-dist = [
{ name = "jieba", marker = "extra == 'webmainbench-eval'", specifier = "==0.42.1" },
{ name = "json-repair", marker = "extra == 'browser-use-eval'", specifier = ">=0.54,<1" },
{ name = "langchain-community", marker = "extra == 'oss-eval'", specifier = "==0.3.30" },
- { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=5b12816906ad8cd2b55b9804c77e7e6c55f2c54e" },
+ { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=1552f731dd75be26c3b3551c04bf0730e6293335" },
{ name = "markdown", marker = "extra == 'webmainbench-eval'", specifier = ">=3.7,<4" },
{ name = "openai", marker = "extra == 'webmainbench-eval'", specifier = ">=2.8,<4" },
{ name = "openpyxl", marker = "extra == 'oss-eval'", specifier = "==3.1.5" },
@@ -1933,7 +1933,7 @@ wheels = [
[[package]]
name = "lexbench-webfetch"
version = "0.3.0"
-source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=5b12816906ad8cd2b55b9804c77e7e6c55f2c54e#5b12816906ad8cd2b55b9804c77e7e6c55f2c54e" }
+source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=1552f731dd75be26c3b3551c04bf0730e6293335#1552f731dd75be26c3b3551c04bf0730e6293335" }
dependencies = [
{ name = "beautifulsoup4" },
{ name = "soupsieve" },
From 45b05a344ca0e8290a034413f40c85009c6ad8fd Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Wed, 21 Oct 2026 18:33:54 +0800
Subject: [PATCH 19/51] test(certification): reject diagnostic evidence during
packaging
---
tests/test_run_certified_benchmark.py | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/tests/test_run_certified_benchmark.py b/tests/test_run_certified_benchmark.py
index b7c824a36..bde4e5353 100644
--- a/tests/test_run_certified_benchmark.py
+++ b/tests/test_run_certified_benchmark.py
@@ -104,6 +104,14 @@ def execute(stage, arguments):
with self.assertRaisesRegex(ReferenceLifecycleError,'[Dd]iagnostic'):
build_report(output,root/'formal-report')
self.assertFalse((root/'formal-report').exists())
+ # A copied diagnostic run must not become a formal package,
+ # even after its dataset display label is removed.
+ from harness.web_fetch.report_package import validate_release
+ with self.assertRaisesRegex(ReferenceLifecycleError,'[Dd]iagnostic'):
+ validate_release(output)
+ (output/'dataset.json').write_text(json.dumps(saved))
+ with self.assertRaisesRegex(ReferenceLifecycleError,'[Dd]iagnostic'):
+ validate_release(output)
self.assertEqual(dataset.read_bytes(),original)
def test_frozen_contract_execution_needs_no_historical_qualification_files(self):
From 51c9c6a85a4180f5bfcd694351655a9a433fa1ce Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Wed, 21 Oct 2026 20:46:47 +0800
Subject: [PATCH 20/51] fix(webfetch): audit saved-output scoring through
shared reports
---
.../references/audit-and-revision.md | 11 +
configs/report-templates.json | 17 +-
docs/benchmarks.md | 25 ++
evaluation/lexbench-webfetch.json | 12 +-
evaluation/risk-control/run-profile.json | 2 +-
harness/web_fetch/dataset_assembly.py | 3 +
harness/web_fetch/methods.py | 96 +++---
harness/web_fetch/publication_check.py | 7 +-
harness/web_fetch/published_snapshot.py | 319 ++++++++++++++++++
harness/web_fetch/reference_lifecycle.py | 4 +-
harness/web_fetch/report.py | 71 +++-
harness/web_fetch/report_check.py | 11 +-
harness/web_fetch/report_explainer.py | 7 +-
harness/web_fetch/report_package.py | 25 +-
harness/web_fetch/rescoring.py | 111 ++++++
harness/web_fetch/scoring_revision.py | 1 +
pyproject.toml | 2 +-
scripts/build_report.py | 7 +
tests/test_publication_privacy.py | 22 ++
tests/test_published_snapshot_report.py | 99 ++++++
tests/test_published_snapshot_rescoring.py | 74 ++++
tests/test_report_methods.py | 16 +
uv.lock | 4 +-
23 files changed, 878 insertions(+), 68 deletions(-)
create mode 100644 harness/web_fetch/published_snapshot.py
create mode 100644 tests/test_published_snapshot_report.py
create mode 100644 tests/test_published_snapshot_rescoring.py
diff --git a/.agents/skills/benchmark-report/references/audit-and-revision.md b/.agents/skills/benchmark-report/references/audit-and-revision.md
index e11928961..6293a85d2 100644
--- a/.agents/skills/benchmark-report/references/audit-and-revision.md
+++ b/.agents/skills/benchmark-report/references/audit-and-revision.md
@@ -98,3 +98,14 @@ message. Reproject retained full responses without new product calls when only
this selection changed; missing historical envelopes require fresh calls across
the full declared correction scope before claiming revised benchmark scores.
Reruns of unchanged compiled acceptance-contract tasks freeze the existing executable dataset. They do not require recovering historical qualification raw files or repeating admission review. Source qualification is part of task authoring and reference/rule correction, not a prerequisite for every provider-version comparison. Preserve historical receipts without applying their evidence-file requirements to a new execution protocol.
+
+A published-snapshot score audit uses the maintained Web Fetch profile adapter,
+not a copied renderer. Bind the complete original export, full historical matrix,
+initial and corrected scores, per-input original/public/scored hashes, and every
+changed decision. Preserve collection facts and timing. Evaluate a suspected
+privacy effect with the historical scorer on the same public bytes; a redaction
+receipt by itself does not establish causation. Keep source-backed annotation
+corrections and executable regression provenance together. Reuse the native
+report's routes, interpretation schema, structured history and packer, and run its
+explicit snapshot audit browser gate before packaging. The catalog provides the
+runnable example and commands. Store instance evidence and prose outside Git.
diff --git a/configs/report-templates.json b/configs/report-templates.json
index d09bfec6b..fe5fc7097 100644
--- a/configs/report-templates.json
+++ b/configs/report-templates.json
@@ -14,12 +14,25 @@
"wcxb_check": "make wcxb-release-check",
"wcxb_package": "uv run --frozen python scripts/publish_wcxb_artifact_site.py --package-only --source /absolute/new-report --work /absolute/package",
"tests": "tests/test_headless_report.py",
- "variants": ["certified", "wcxb", "local-audit", "diagnostic-cohorts"],
+ "variants": [
+ "certified",
+ "wcxb",
+ "local-audit",
+ "diagnostic-cohorts",
+ "published-snapshot-audit"
+ ],
"diagnostic_build": "make webfetch-diagnostic-report DIAGNOSTIC_PROFILE=evaluation/live-webfetch/current-report.json OUTPUT=/absolute/new-output",
"diagnostic_profile": "evaluation/live-webfetch/current-report.json",
"diagnostic_check": "uv run --frozen python scripts/check_diagnostic_report.py /absolute/output --profile evaluation/live-webfetch/current-report.json",
"diagnostic_example": "tests/test_report_templates.py:diagnostic_example",
- "diagnostic_example_check": "uv run --frozen python -m unittest tests.test_report_templates"
+ "diagnostic_example_check": "uv run --frozen python -m unittest tests.test_report_templates",
+ "snapshot_audit_build": "uv run --frozen python scripts/build_report.py --profile /absolute/profile.json --evidence /absolute/published-snapshot --output /absolute/new-report --language zh",
+ "snapshot_audit_check": "uv run --frozen python scripts/check_report.py /absolute/new-report --profile /absolute/profile.json",
+ "snapshot_audit_tests": "tests/test_published_snapshot_rescoring.py",
+ "snapshot_audit_example": "tests/test_published_snapshot_report.py:example",
+ "snapshot_audit_example_check": "uv run --frozen python -m pytest tests/test_published_snapshot_report.py tests/test_published_snapshot_rescoring.py",
+ "snapshot_audit_release_check": "uv run --frozen python scripts/check_report.py /absolute/new-report --profile /absolute/profile.json --visual-receipt /absolute/private/visual.json",
+ "snapshot_audit_package": "uv run --frozen python scripts/package_artifact_hub.py /absolute/new-report /absolute/package.zip --snapshot-audit"
},
"trajectory-replay": {
"renderer": "harness/trajectory_replay/webarena_report.py",
diff --git a/docs/benchmarks.md b/docs/benchmarks.md
index 7bfabcae6..522b84b54 100644
--- a/docs/benchmarks.md
+++ b/docs/benchmarks.md
@@ -152,3 +152,28 @@ uv run --frozen python scripts/publish_wcxb_artifact_site.py --package-only \
--profile /absolute/instance/profile.json \
--source /absolute/instance/report --work /absolute/instance/package
```
+
+### Re-score a published snapshot
+
+A `published-snapshot-audit` profile rebuilds a saved Web Fetch report through the
+current renderer. Bind its complete exported file manifest, original and corrected
+results, historical verdicts, initial recomputation, scoring receipt, review,
+interpretation and diagram under `inputs`. An optional bound `method_registry`
+provides historical participant IDs, engines and labels; `release_history` supplies
+the report's structured history. Keep these instance files outside Git.
+
+`score_published_snapshot` verifies the full task-by-method matrix, preserves call
+identities and timing, and computes scores without contacting providers. It accepts
+an exact hash-verified recovered original, otherwise identifies a privacy-projected
+input separately. A projected file alone is not evidence that redaction changed a
+verdict. Compare the old scorer on the same bytes before assigning that cause.
+
+Build with `scripts/build_report.py --profile --evidence
+--output --language zh`. This historical format supports Chinese and
+provides an English not-generated notice. Check with `scripts/check_report.py
+ --profile --visual-receipt `, then package
+through `scripts/package_artifact_hub.py --snapshot-audit`.
+Run these Python entries through `uv run --frozen python`. The mode retains the
+original routes and metric populations and labels the new report as a saved-input
+audit; it does not certify new calls or replace the original release. The ordinary
+packaging path rejects this evidence state without the explicit audit flag.
diff --git a/evaluation/lexbench-webfetch.json b/evaluation/lexbench-webfetch.json
index 319c524f6..0793aeae5 100644
--- a/evaluation/lexbench-webfetch.json
+++ b/evaluation/lexbench-webfetch.json
@@ -2,7 +2,7 @@
"schema_version": 1,
"kind": "external_benchmark_binding",
"repository": "https://github.com/lexmount/lexbench-webfetch",
- "revision": "1552f731dd75be26c3b3551c04bf0730e6293335",
+ "revision": "4d73ada2fa1305fab61f0c9d222f7a315361281c",
"ownership": {
"benchmark": "Task populations, task model, acceptance scoring, file checks, compiler and frozen definitions",
"harness": "Execution, reference lifecycle, evidence and reports"
@@ -13,7 +13,7 @@
"dataset_id": "webfetch",
"case_count": 150,
"files": {
- "tasks.json": "ae3aec6eab8d2d470cdeb18c4f80d238e5dd79e1108a729589b77a80fe869405",
+ "tasks.json": "02f887ae8645102eb780a456fd2ee4c0c00b8855c9ea9b1043218225dcc5a834",
"dataset-catalog.json": "994fa53f7fc7c3170692a7bb493ebc5af9ad8ffb5b576c022fa123e6e6fb3838",
"source-inputs.json": "c202abc4624811b938a837402bbaf0ecbe716e375fbca70567346d2edb3b1708"
},
@@ -37,7 +37,7 @@
"dataset_id": "dynamic-qualified-100-moli-native-local-audit-20260924",
"case_count": 100,
"files": {
- "tasks.json": "139c62a141183d305d8fa9c3658c546927731461d86aeba5ad3091649bd4ecd7",
+ "tasks.json": "f76077e57746e942932f5eee1914e4286bcdd97c4ba23f4038dbe8bf7f9b82a0",
"dataset-catalog.json": "ca1572aa80fc2324b2f00800a79b053cfd584d1831ffb16e6906ae0981522c6c",
"source-inputs.json": "710f9c91705b29bb9e4aa15af97dcc856e64e406baa4b9966446408d7e3d26a8"
},
@@ -49,7 +49,7 @@
"dataset_id": "risk-control-audited-100",
"case_count": 100,
"files": {
- "tasks.json": "c67b60ccf5ee0b14e4e23b3efc282e625005f8e096f0bfc978772d312e8f50c2",
+ "tasks.json": "23c2c7e0b470527dfd848146dd94bb47188d2dbdeb7b9105d04664b9b02b9085",
"dataset-catalog.json": "52ec968b2ada26e5e061ccb25cbe4a73747539e556fa88f31bece6d7fb79b3cb",
"source-inputs.json": "c61d6825abbff2a92098592f83055347cd0b0101374794023046c0fd0323dd5f"
},
@@ -70,7 +70,7 @@
"evaluation/risk-control/dataset-catalog.json": "02e30a5c408de3b5113e28f27359cf5e454814de612f860e6b7d7536c892efb0",
"evaluation/risk-control/first-30-rule-review.md": "6945a6aa7b0d8d5990d8abf5d24fc41b5a42a22c70940a7c996d8698c6e9bf81",
"evaluation/risk-control/history/v5/dataset-catalog.json": "73b1a3cd945c535d5043f0ba01316b98a26af25ed5156d900cdff79f2cbad4ef",
- "evaluation/risk-control/list-contracts.json": "a566fb13328bd1a88b189b5a49fb7b3565a8d037c22aaa19c2b86087cbbe0881",
+ "evaluation/risk-control/list-contracts.json": "f263be741c90c24801e4449fb238e5957ab9bf60c5a7b843f1b72e32899caaeb",
"evaluation/risk-control/qualification/cohort-policy.json": "4835abe67852adf1609cb3033b19cbef839650d14879c0e24dd76116af2acd2e",
"evaluation/risk-control/qualification/cohort/cohort-manifest.json": "aeec7ec26d24650896263574135a27fc4848f4d362bd38f0a5b5c167f82abc26",
"evaluation/risk-control/qualification/cohort/cohort-receipt.json": "a5929b62ffb8b64da1ac902c5c5be61a75fa01ed171a497f115de14442a9a872",
@@ -78,7 +78,7 @@
"evaluation/risk-control/remaining-70-rule-review.md": "301574dddcfddf6d30ed4d882cdd40b1ae859fdc95a0a02cf9868f55e3fd1079",
"evaluation/risk-control/rule-authoring-guide.md": "74363ab502978179e5438a0b992bac7563090e41c91784e492636832dc1cd088",
"evaluation/risk-control/source-inputs.json": "77c5a4b62c07d59ed57154a833e85f9e7a97bc739730f51cd4ca3a698edb9258",
- "evaluation/scoring-regressions.json": "572c140d3cec14a00a2a517949ad2aa118d34e03d6eade07d68dcd8476cd0b10",
+ "evaluation/scoring-regressions.json": "39efcdb355b9744baecf4eeabfa839b6db2c5d17cfb9dd8aebf23faf87443a51",
"evaluation/source-inputs.json": "c202abc4624811b938a837402bbaf0ecbe716e375fbca70567346d2edb3b1708"
}
}
diff --git a/evaluation/risk-control/run-profile.json b/evaluation/risk-control/run-profile.json
index 4e434371c..04325ad52 100644
--- a/evaluation/risk-control/run-profile.json
+++ b/evaluation/risk-control/run-profile.json
@@ -33,7 +33,7 @@
},
"list_contracts": {
"path": "evaluation/risk-control/list-contracts.json",
- "sha256": "a566fb13328bd1a88b189b5a49fb7b3565a8d037c22aaa19c2b86087cbbe0881"
+ "sha256": "f263be741c90c24801e4449fb238e5957ab9bf60c5a7b843f1b72e32899caaeb"
},
"body_contracts": {
"path": "evaluation/risk-control/body-contracts.json",
diff --git a/harness/web_fetch/dataset_assembly.py b/harness/web_fetch/dataset_assembly.py
index 0b1d3aca8..00bdd26b8 100644
--- a/harness/web_fetch/dataset_assembly.py
+++ b/harness/web_fetch/dataset_assembly.py
@@ -24,7 +24,10 @@
"acceptance_contract",
"composite_acceptance",
"forbidden_any",
+ "diagnostic_any",
"mutable_body_policy",
+ "expected_resources",
+ "resource_output_contract",
}
_RERUN_ONLY_CASE_FIELDS = {
diff --git a/harness/web_fetch/methods.py b/harness/web_fetch/methods.py
index e4fd3acdb..de4bb0ef2 100644
--- a/harness/web_fetch/methods.py
+++ b/harness/web_fetch/methods.py
@@ -11,7 +11,7 @@
@lru_cache(maxsize=1)
-def browser_method_registry():
+def _execution_browser_method_registry():
"""Use the execution registry for release and source-build participants."""
path = Path(__file__).resolve().parents[2] / "configs/providers.headless.json"
if not path.is_file():
@@ -27,6 +27,14 @@ def browser_method_registry():
return json.loads(path.read_text())["providers"]
+_REPORT_METHODS = ContextVar("webfetch_report_methods", default={})
+
+
+def browser_method_registry():
+ """Use a bound report registry while rendering historical or external methods."""
+ return {**_execution_browser_method_registry(), **_REPORT_METHODS.get()}
+
+
def native_browser_engine(name):
return browser_method_registry().get(name, {}).get("engine")
@@ -48,48 +56,60 @@ def with_measured_method_labels(render):
"""Scope Moli version labels to the actual report's hash-bound binaries."""
@wraps(render)
def wrapped(out, dataset, records, *args, **kwargs):
- path = Path(out) / "implementation-provenance.json"
- provenance = json.loads(path.read_text()) if path.is_file() else {}
- identities = {
- item["binary_sha256"]: item["version"]
- for item in provenance.get("implementations", [])
- if item.get("version") and item.get("binary_sha256")
- }
- labels = {}
- for name in {r["provider"] for r in records}:
- if native_browser_engine(name) != "moli":
- continue
- subset = [r for r in records if r["provider"] == name]
- versions = [
- identities.get(r.get("provider_metadata", {}).get("binary_sha256"))
- for r in subset
- ]
- if versions and all(versions):
- counts = Counter(versions)
- if len(counts) == 1:
- labels[name] = "Moli " + next(iter(counts))
- else:
- releases = [
- match.group(1) if (match := re.match(r"^(\d+\.\d+\.\d+)", version)) else None
- for version in counts
- ]
- if None not in releases and len(set(releases)) == 1:
- release = releases[0]
- configured = browser_method_registry().get(name, {}).get("label", "")
- qualifier = "(候选版)" if "candidate" in configured.lower() else ""
- labels[name] = f"Moli {release}{qualifier}"
- else:
- labels[name] = "Moli 多构建结果(" + ";".join(
- f"{count} 例 {version}" for version, count in sorted(counts.items())
- ) + ")"
- token = _MEASURED_LABELS.set(labels)
+ comparison = dataset.get("comparison")
+ saved = Path(out) / "dataset.json"
+ if comparison is None and saved.is_file():
+ comparison = json.loads(saved.read_text()).get("comparison", {})
+ registry_token = _REPORT_METHODS.set((comparison or {}).get("method_registry", _REPORT_METHODS.get()))
try:
- return render(out, dataset, records, *args, **kwargs)
+ return _render_with_labels(render, out, dataset, records, *args, **kwargs)
finally:
- _MEASURED_LABELS.reset(token)
+ _REPORT_METHODS.reset(registry_token)
return wrapped
+def _render_with_labels(render, out, dataset, records, *args, **kwargs):
+ path = Path(out) / "implementation-provenance.json"
+ provenance = json.loads(path.read_text()) if path.is_file() else {}
+ identities = {
+ item["binary_sha256"]: item["version"]
+ for item in provenance.get("implementations", [])
+ if item.get("version") and item.get("binary_sha256")
+ }
+ labels = {}
+ for name in {r["provider"] for r in records}:
+ if native_browser_engine(name) != "moli":
+ continue
+ subset = [r for r in records if r["provider"] == name]
+ versions = [
+ identities.get(r.get("provider_metadata", {}).get("binary_sha256"))
+ for r in subset
+ ]
+ if versions and all(versions):
+ counts = Counter(versions)
+ if len(counts) == 1:
+ labels[name] = "Moli " + next(iter(counts))
+ else:
+ releases = [
+ match.group(1) if (match := re.match(r"^(\d+\.\d+\.\d+)", version)) else None
+ for version in counts
+ ]
+ if None not in releases and len(set(releases)) == 1:
+ release = releases[0]
+ configured = browser_method_registry().get(name, {}).get("label", "")
+ qualifier = "(候选版)" if "candidate" in configured.lower() else ""
+ labels[name] = f"Moli {release}{qualifier}"
+ else:
+ labels[name] = "Moli 多构建结果(" + ";".join(
+ f"{count} 例 {version}" for version, count in sorted(counts.items())
+ ) + ")"
+ token = _MEASURED_LABELS.set(labels)
+ try:
+ return render(out, dataset, records, *args, **kwargs)
+ finally:
+ _MEASURED_LABELS.reset(token)
+
+
METHOD_REGISTRY: dict[str, dict[str, Any]] = {
"http": {"endpoint":"HTTP GET","contract":"已知 URL → HTML 文本或文件任务的原始字节","output":"HTML 提取文本;文件任务保留响应字节","freshness":"每次独立请求,不执行页面 JavaScript","leaderboard":"独立 HTTP 对照"},
"headless_html": {"endpoint":"browser HTML → node-html-markdown 2.0.0","contract":"已知 URL → DOM HTML → Markdown","output":"Markdown;独立保存 HTML 和诊断","freshness":"每次新进程/会话;domcontentloaded;45 秒外层预算","leaderboard":"六类任务分别验收","report":{"observed_routes":True}},
diff --git a/harness/web_fetch/publication_check.py b/harness/web_fetch/publication_check.py
index d08c9c41e..e9eaaaa60 100644
--- a/harness/web_fetch/publication_check.py
+++ b/harness/web_fetch/publication_check.py
@@ -38,9 +38,9 @@ def scored_output_hash(row: dict) -> str:
return expected
-def check(archive, source, overlay=None, *, presentation_binding=None, external_archives=None, allow_pending_external=False):
+def check(archive, source, overlay=None, *, presentation_binding=None, external_archives=None, allow_pending_external=False, snapshot_audit=False):
from .report_package import validate_release, DATA_ATTACHMENT_HTML
- validate_release(source, presentation_binding=presentation_binding)
+ validate_release(source, presentation_binding=presentation_binding, snapshot_audit=snapshot_audit)
source_files = evidence_tree_identity(source)
physical = archive_inventory(archive)
if overlay:
@@ -327,5 +327,6 @@ def main(*, root: Path | None = None):
parser.add_argument("source", nargs="?", type=Path, default=root / "reports/webfetch")
parser.add_argument("overlay", nargs="?", type=Path)
parser.add_argument("--presentation-binding", type=Path)
+ parser.add_argument("--snapshot-audit",action="store_true")
args = parser.parse_args()
- check(args.archive, args.source, args.overlay, presentation_binding=args.presentation_binding)
+ check(args.archive, args.source, args.overlay, presentation_binding=args.presentation_binding,snapshot_audit=args.snapshot_audit)
diff --git a/harness/web_fetch/published_snapshot.py b/harness/web_fetch/published_snapshot.py
new file mode 100644
index 000000000..00a3847bd
--- /dev/null
+++ b/harness/web_fetch/published_snapshot.py
@@ -0,0 +1,319 @@
+"""Build a score audit of a published WebFetch snapshot with the native renderer."""
+from __future__ import annotations
+
+import json
+import shutil
+import tempfile
+from pathlib import Path
+
+from harness.record_io import canonical_json_sha256, file_sha256, load_jsonl
+from lexbench_webfetch.metrics import task_passed
+from lexbench_webfetch.paths import contained_path
+from lexbench_webfetch.scoring import SCORING_SOURCE_SHA256
+from .scoring_revision import ALLOWED_FIELDS, correction_provenance
+
+
+def _bound(root, binding):
+ path = contained_path(root, binding['path'], reject_symlinks=True)
+ if file_sha256(path) != binding['sha256']:
+ raise ValueError('Snapshot audit input changed: ' + binding['path'])
+ return path
+
+
+def validate_snapshot_audit(source, profile_path):
+ """Validate the full matrix, immutable calls and every score-bearing view."""
+ source, profile_path = Path(source).resolve(), Path(profile_path).resolve()
+ config = json.loads(profile_path.read_text())
+ if config.get('variant') != 'published-snapshot-audit' or config.get('certified') is not False:
+ raise ValueError('Published snapshot audits are explicitly diagnostic')
+ inputs = {key: _bound(profile_path.parent, value) for key, value in config['inputs'].items()}
+ snapshot = json.loads(inputs['snapshot_manifest'].read_text())
+ paths = list(source.rglob('*'))
+ if any(path.is_symlink() for path in paths):
+ raise ValueError('Published snapshots reject symlinks')
+ actual = {path.relative_to(source).as_posix() for path in paths if path.is_file()}
+ if actual - {'.generated-report'} != set(snapshot['files']):
+ raise ValueError('Published snapshot has missing or undeclared files')
+ if set(config.get('source',{})) != {'url','version','archive_sha256'}:
+ raise ValueError('Snapshot source needs a pinned public identity')
+ for relative, item in snapshot['files'].items():
+ if file_sha256(contained_path(source, relative, reject_symlinks=True)) != item['sha256']:
+ raise ValueError('Published snapshot changed: '+relative)
+ baseline = json.loads(contained_path(source, 'dataset.json').read_text())
+ previous = load_jsonl(contained_path(source, 'results.jsonl'))
+ dataset = json.loads(inputs['dataset'].read_text())
+ records = load_jsonl(inputs['results'])
+ receipt = json.loads(inputs['receipt'].read_text())
+ audit = json.loads(inputs['review'].read_text())
+ old_cases = {row['id']: row for row in baseline['cases']}
+ cases = {row['id']: row for row in dataset['cases']}
+ old_rows = {row['attempt_id']: row for row in previous}
+ rows = {row['attempt_id']: row for row in records}
+ if (len(cases) != len(dataset['cases']) or cases.keys() != old_cases.keys()
+ or dataset.get('coverage_plan') != baseline.get('coverage_plan')
+ or len(rows) != len(records) or rows.keys() != old_rows.keys()):
+ raise ValueError('Snapshot audit changed the task-by-method matrix')
+ if (receipt.get('kind') != 'published_snapshot_rescore' or receipt.get('provider_calls_added') != 0
+ or receipt.get('input_dataset_sha256') != file_sha256(source / 'dataset.json')
+ or receipt.get('input_results_sha256') != file_sha256(source / 'results.jsonl')
+ or receipt.get('scoring_source_sha256') != SCORING_SOURCE_SHA256
+ or receipt.get('corrected_dataset_sha256') != canonical_json_sha256(dataset)):
+ raise ValueError('Snapshot audit does not bind the saved calls and current scorer')
+ correction_provenance(**config['correction_provenance'])
+ for identifier, old in old_cases.items():
+ changed = {key for key in old.keys() | cases[identifier].keys() if old.get(key) != cases[identifier].get(key)}
+ if changed - ALLOWED_FIELDS:
+ raise ValueError('Snapshot audit changed collection inputs: ' + identifier)
+ views = {item['attempt_id']: item for item in receipt['inputs']}
+ if len(views) != len(records) or views.keys() != rows.keys():
+ raise ValueError('Snapshot audit lacks complete scoring input provenance')
+ for identifier, old in old_rows.items():
+ row, view = rows[identifier], views[identifier]
+ facts = lambda value: {key: item for key, item in value.items()
+ if key not in {'metrics', 'scoring', 'diagnosis', 'failure_category'}}
+ if facts(old) != facts(row):
+ raise ValueError('Snapshot audit changed a physical call: ' + identifier)
+ artifact = contained_path(source, old['artifact_path'], reject_symlinks=True)
+ if (view['path'] != old['artifact_path'] or view['published_sha256'] != file_sha256(artifact)
+ or view['original_sha256'] != old['metrics']['content_sha256']
+ or row['metrics']['content_sha256'] != view['scored_sha256']
+ or row['metrics']['scoring_source_sha256'] != SCORING_SOURCE_SHA256
+ or row.get('scoring', {}).get('artifact_view') != view['view']):
+ raise ValueError('Snapshot audit artifact identity changed: ' + identifier)
+ expected = {'original': view['original_sha256'], 'recovered_original': view['original_sha256'],
+ 'published_projection': view['published_sha256']}.get(view['view'])
+ if (view['scored_sha256'] != expected
+ or view['view'] == 'original' and view['original_sha256'] != view['published_sha256']
+ or view['view'] == 'published_projection' and view['original_sha256'] == view['published_sha256']):
+ raise ValueError('Snapshot audit confuses original and public bytes')
+ if view['view'] not in {'original', 'recovered_original', 'published_projection'}:
+ raise ValueError('Unknown snapshot scoring view')
+ historic = json.loads(inputs['historical_verdicts'].read_text())['verdicts']
+ initial = {row['attempt_id']:row for row in load_jsonl(inputs['initial_results'])}
+ if set(historic) != rows.keys() or initial.keys() != rows.keys() or any(type(v) is not bool for v in historic.values()):
+ raise ValueError('Snapshot audit comparison lacks the complete original matrix')
+ current = {key:task_passed(row['ok'],row['metrics'],cases[row['case_id']].get('task_kind','body')) for key,row in rows.items()}
+ first = {key:task_passed(row['ok'],row['metrics'],old_cases[row['case_id']].get('task_kind','body')) for key,row in initial.items()}
+ required_reviews = {key for key in rows if historic[key] != first[key] or historic[key] != current[key]}
+ if {item['attempt_id'] for item in audit.get('changes',[])} != required_reviews:
+ raise ValueError('Snapshot audit omits a changed decision')
+ expected_providers = {row['provider'] for row in rows.values()}
+ if {item['provider'] for item in audit['providers']} != expected_providers:
+ raise ValueError('Snapshot audit omits a participant')
+ for item in audit['providers']:
+ keys = [key for key,row in rows.items() if row['provider']==item['provider']]
+ expected = {'total':len(keys),'historical_passes':sum(historic[k] for k in keys),
+ 'initial_rescore_passes':sum(first[k] for k in keys),'corrected_passes':sum(current[k] for k in keys)}
+ if any(item.get(key) != value for key,value in expected.items()):
+ raise ValueError('Snapshot audit comparison counts differ: '+item['provider'])
+ if 'method_registry' in inputs:
+ methods=json.loads(inputs['method_registry'].read_text())
+ if set(methods) != expected_providers or any(not m.get('engine') or not m.get('label') for m in methods.values()):
+ raise ValueError('Snapshot audit method registry is incomplete')
+ for row in records:
+ engine=row.get('provider_metadata',{}).get('engine')
+ if engine and engine != methods[row['provider']]['engine']:
+ raise ValueError('Snapshot audit method registry changes the measured engine')
+ reviewed = audit.get('changes', [])
+ review_ids = [item['attempt_id'] for item in reviewed]
+ if len(review_ids) != len(set(review_ids)) or not set(review_ids) <= rows.keys():
+ raise ValueError('Snapshot audit review scope is invalid')
+ for item in reviewed:
+ row = rows[item['attempt_id']]
+ if (item.get('after_fix_pass') != task_passed(row['ok'], row['metrics'], cases[row['case_id']].get('task_kind','body'))
+ or any(not item.get(key) for key in ('requirement', 'actual', 'reason', 'treatment', 'classification'))):
+ raise ValueError('Snapshot audit lacks a concrete, current disposition: ' + item['attempt_id'])
+ for binding in item.get('evidence', []):
+ _bound(source, binding)
+ if audit.get('total_attempts') != len(records) or audit.get('reviewed_attempts') != len(reviewed):
+ raise ValueError('Snapshot audit review counts differ')
+ return config, inputs, dataset, records, receipt, audit
+
+
+def build_snapshot_audit(source, output, *, profile):
+ """Keep the original report routes and metric populations, adding score review."""
+ from .report import generate
+ from .report_build import _install_report_tree, write_frozen_result_summary
+ from harness.reporting.templates import resolve_template
+ from harness.report_languages import generated_languages
+ source, output = Path(source).resolve(), Path(output).resolve()
+ if source == output or source in output.parents or output in source.parents:
+ raise ValueError('Snapshot audit needs a separate output tree')
+ if output.exists() and not (output / '.generated-report').is_file():
+ raise ValueError('Refusing to replace an unmarked output')
+ config, inputs, dataset, records, receipt, audit = validate_snapshot_audit(source, profile)
+ output.parent.mkdir(parents=True, exist_ok=True)
+ with tempfile.TemporaryDirectory(prefix='.snapshot-audit-', dir=output.parent) as work:
+ staging = Path(work) / 'report'
+ shutil.copytree(source, staging)
+ shutil.rmtree(staging / 'en', ignore_errors=True)
+ history = staging / 'audit/history'
+ history.mkdir(parents=True)
+ for name in ('dataset.json', 'results.jsonl', 'result-review.json'):
+ if (staging / name).is_file():
+ shutil.copy2(staging / name, history / name)
+ for item in audit['changes']:
+ for evidence in item.get('evidence',[]):
+ if evidence.get('historical'):
+ target=contained_path(history,evidence['path'])
+ target.parent.mkdir(parents=True,exist_ok=True)
+ shutil.copy2(contained_path(source,evidence['path']),target)
+ (staging / 'result-review.json').unlink(missing_ok=True)
+ (staging / 'archived-reading.json').unlink(missing_ok=True)
+ for key, name in [('historical_verdicts','historical-verdicts.json'), ('initial_results','initial-results.jsonl'), ('receipt','scoring-inputs.json'), ('review','score-changes.json'), ('figure','scoring-explained.png'), ('interpretation','interpretation.json')]:
+ shutil.copy2(inputs[key], staging / 'audit' / name)
+ if 'release_history' in inputs:
+ (staging/'report-assets').mkdir(exist_ok=True)
+ shutil.copy2(inputs['release_history'],staging/'report-assets/report-changes.json')
+ comparison = dataset.setdefault('comparison', {})
+ if 'method_registry' in inputs:
+ comparison['method_registry'] = json.loads(inputs['method_registry'].read_text())
+ for key in ('independent_review', 'report_interpretation'):
+ comparison.pop(key, None)
+ comparison['report_interpretation'] = {'path':'audit/interpretation.json','sha256':file_sha256(staging/'audit/interpretation.json')}
+ comparison['require_independent_result_review'] = False
+ comparison['publication_state'] = 'published_snapshot_score_audit'
+ comparison['score_audit'] = {'path': 'audit/score-changes.json',
+ 'sha256': file_sha256(staging / 'audit/score-changes.json'),
+ 'scoring_receipt': 'audit/scoring-inputs.json',
+ 'source': config['source'], 'reviewed_attempts': len(audit['changes']),
+ 'total_attempts': len(records), 'projected_scoring_inputs': receipt['projected_scoring_inputs'],
+ 'recovered_originals': receipt['recovered_originals'],
+ 'figure': 'audit/scoring-explained.png'}
+ (staging / 'dataset.json').write_text(json.dumps(dataset, ensure_ascii=False, indent=2)+'\n')
+ shutil.copy2(inputs['results'], staging / 'results.jsonl')
+ batches = json.loads((staging / 'execution.json').read_text())['batches']
+ generate(staging, dataset, records, batches, authored_interpretation=True, frozen_scores=True)
+ write_frozen_result_summary(staging, dataset, records)
+ (staging / '.generated-report').write_text('webfetch\n')
+ manifest = {'schema_version':1, 'template':resolve_template('webfetch'),
+ 'evidence_state':'published_snapshot_score_audit', 'certified':False,
+ 'dataset_sha256':file_sha256(staging/'dataset.json'),
+ 'results_sha256':file_sha256(staging/'results.jsonl'),
+ 'generated_languages':list(generated_languages(staging)),
+ 'score_audit':comparison['score_audit'], 'profile_sha256':file_sha256(profile)}
+ (staging/'report-manifest.json').write_text(json.dumps(manifest,ensure_ascii=False,indent=2)+'\n')
+ check_snapshot_audit(staging)
+ _install_report_tree(staging, output)
+ return output / 'index.html'
+
+
+def check_snapshot_audit(root):
+ """Deterministic checks for matrix values, disclosure and reviewed evidence."""
+ from bs4 import BeautifulSoup
+ from .report_check import check_saved_result_summary
+ root = Path(root)
+ manifest=json.loads((root/'report-manifest.json').read_text())
+ if manifest.get('evidence_state')!='published_snapshot_score_audit' or manifest.get('certified') is not False:
+ raise ValueError('Snapshot audit lost its diagnostic identity')
+ for name in ['dataset','results']:
+ relative='dataset.json' if name=='dataset' else 'results.jsonl'
+ if manifest.get(name+'_sha256')!=file_sha256(root/relative):
+ raise ValueError('Snapshot audit input changed: '+name)
+ dataset = json.loads((root/'dataset.json').read_text())
+ rows = load_jsonl(root/'results.jsonl')
+ binding = dataset['comparison']['score_audit']
+ audit = json.loads(_bound(root, binding).read_text())
+ check_saved_result_summary(root, dataset, rows)
+ for relative in ['index.html','guide/index.html','audit/index.html','analysis/index.html']:
+ soup = BeautifulSoup((root/relative).read_text(), 'html.parser')
+ if not soup.select_one('[data-evidence-state="published_snapshot_score_audit"]'):
+ raise ValueError('Snapshot report lost its evidence disclosure: '+relative)
+ audit_html = BeautifulSoup((root/'audit/index.html').read_text(), 'html.parser')
+ from .report_explainer import check_current_browser_article
+ from harness.reporting.presentation import check_presentation
+ check_current_browser_article(root,{case['id']:case for case in dataset['cases']},rows,dataset['comparison']['report_interpretation'])
+ for route in ['index.html','guide/index.html','analysis/index.html','audit/index.html']:
+ check_presentation((root/route).read_text())
+ for item in audit['providers']:
+ element = audit_html.select_one('[data-audit-summary-provider="'+item['provider']+'"]')
+ expected = [str(item[key]) for key in ['published_passes','historical_passes','initial_rescore_passes','corrected_passes','total']]
+ if element is None or [cell.get_text(' ',strip=True) for cell in element.select('td')][-5:] != expected:
+ raise ValueError('Snapshot audit rendered comparison differs: '+item['provider'])
+ cases = {case['id']:case for case in dataset['cases']}
+ indexed = {row['attempt_id']:row for row in rows}
+ for item in audit['changes']:
+ identifier=item['attempt_id'];row=indexed[identifier]
+ article=audit_html.find(id=identifier)
+ if article is None or item['reason'] not in article.get_text():
+ raise ValueError('Snapshot audit lost a reviewed disposition: '+identifier)
+ if item['after_fix_pass'] != task_passed(row['ok'],row['metrics'],cases[row['case_id']].get('task_kind','body')):
+ raise ValueError('Snapshot audit verdict differs: '+identifier)
+ for evidence in item.get('evidence',[]):
+ archived={**evidence,'path':'audit/history/'+evidence['path']} if evidence.get('historical') else evidence
+ _bound(root,archived)
+ detail=BeautifulSoup((root/'attempts'/f'{identifier}.html').read_text(),'html.parser')
+ if not detail.select_one('[data-score-audit]'):
+ raise ValueError('Snapshot result lost its audit explanation: '+identifier)
+ return f'PASS: {len(cases)} tasks, {len(rows)} saved calls, {len(audit["changes"])} reviewed changes; zero new calls'
+
+
+def read_snapshot_public_file(root, binding):
+ """Inspect original bytes or a hash-bound historical publication projection."""
+ path = contained_path(root, binding['path'], reject_symlinks=True)
+ observed = file_sha256(path)
+ if observed == binding['sha256']:
+ return path
+ privacy = json.loads(contained_path(root, 'publication-privacy.json').read_text())
+ proof = next((item for item in privacy['files'] if item['path'] == binding['path']), {})
+ if (proof.get('original_sha256') != binding['sha256']
+ or proof.get('published_sha256') != observed
+ or proof.get('published_bytes') != path.stat().st_size):
+ raise ValueError('Unbound historical publication view: '+binding['path'])
+ return path
+
+
+def check_snapshot_visual(root, receipt):
+ """Exercise native route classes, themes, narrow layouts and evidence dialogs."""
+ from playwright.sync_api import sync_playwright
+ from harness.reporting.visual import browser_executable, local_report_server, settle_theme
+ root, receipt = Path(root).resolve(), Path(receipt).resolve()
+ if receipt == root or root in receipt.parents:
+ raise ValueError('Visual receipts belong outside the public report')
+ check_snapshot_audit(root)
+ dataset=json.loads((root/'dataset.json').read_text())
+ rows=load_jsonl(root/'results.jsonl');cases={c['id']:c for c in dataset['cases']}
+ routes=['index.html','guide/index.html','audit/index.html','analysis/index.html',
+ 'datasets/index.html','methodology/index.html','review/index.html',
+ 'changes/index.html','adoption/index.html','reproduce/index.html','accounts/index.html','en/index.html']
+ for folder in ['cases','providers']:
+ pages=sorted((root/folder).glob('*.html'))
+ if pages:routes.append(pages[0].relative_to(root).as_posix())
+ for passed in [False,True]:
+ row=next(r for r in rows if task_passed(r['ok'],r['metrics'],cases[r['case_id']].get('task_kind','body')) is passed)
+ routes.append('attempts/'+row['attempt_id']+'.html')
+ states=[];interactions=[]
+ receipt.parent.mkdir(parents=True,exist_ok=True)
+ with local_report_server(root) as origin, sync_playwright() as pw:
+ browser=pw.chromium.launch(executable_path=browser_executable(),headless=True)
+ page=browser.new_page();errors=[];page.on('pageerror',lambda error:errors.append(str(error)))
+ for route in routes:
+ for width,height in [(1440,1000),(390,844)]:
+ page.set_viewport_size({'width':width,'height':height})
+ for theme in ['light','dark']:
+ page.goto(origin+'/'+route,wait_until='networkidle')
+ page.evaluate('(t)=>{document.documentElement.dataset.theme=t;document.body.classList.toggle("dark",t==="dark");localStorage.setItem("theme",t)}',theme)
+ settle_theme(page)
+ state=page.evaluate('''() => ({overflow:document.documentElement.scrollWidth-innerWidth,
+ brokenImages:[...document.images].filter(i=>!i.complete||i.naturalWidth===0).map(i=>i.getAttribute('src')),
+ style:document.styleSheets.length})''')
+ state.update(route=route,width=width,height=height,theme=theme);states.append(state)
+ if abs(state['overflow'])>2 or state['brokenImages'] or not state['style'] or errors:
+ raise ValueError('Snapshot visual check failed: '+json.dumps({'state':state,'errors':errors}))
+ if route in ['index.html','guide/index.html','audit/index.html']:
+ page.screenshot(path=str(receipt.parent/(route.split('/')[0]+'-'+str(width)+'-'+theme+'.png')),full_page=False)
+ if route.startswith('attempts/'):
+ page.goto(origin+'/'+route,wait_until='networkidle')
+ button=page.locator('[data-call-log-open]').first
+ if button.count():
+ button.evaluate('(b)=>{for(let e=b.parentElement;e;e=e.parentElement)if(e.tagName==="DETAILS")e.open=true}')
+ button.click();page.locator('dialog[open]').wait_for()
+ if not page.locator('dialog[open]').inner_text().strip():raise ValueError('Empty evidence dialog')
+ page.keyboard.press('Escape')
+ if page.locator('dialog[open]').count():raise ValueError('Evidence dialog did not close')
+ interactions.append({'route':route,'open_close':'passed'})
+ page.locator('header a[href]').first.click();page.wait_for_load_state('networkidle')
+ if not page.url.endswith('/index.html'):raise ValueError('Nested report navigation failed')
+ browser.close()
+ receipt.write_text(json.dumps({'schema':'webfetch_snapshot_visual/1','states':states,'interactions':interactions,'errors':errors},ensure_ascii=False,indent=2)+'\n')
+ return f'PASS: {len(states)} visual states and {len(interactions)} evidence dialogs'
diff --git a/harness/web_fetch/reference_lifecycle.py b/harness/web_fetch/reference_lifecycle.py
index 0a021671c..b9bc835a9 100644
--- a/harness/web_fetch/reference_lifecycle.py
+++ b/harness/web_fetch/reference_lifecycle.py
@@ -1094,7 +1094,7 @@ def valid_assembled_binding(row, index):
allowed_fields = {
"page_identity_required",
'body_requirements', 'required_patterns', 'required_facts',
- 'acceptance_contract', 'composite_acceptance', 'forbidden_any', 'expected_resources',
+ 'acceptance_contract', 'composite_acceptance', 'forbidden_any', 'diagnostic_any', 'expected_resources',
'resource_output_contract', 'task_prompt', 'reference_answer',
'mutable_body_policy',
}
@@ -1301,7 +1301,7 @@ def receipt_transitions(receipt):
return {}
allowed = {
"body_requirements", "required_patterns", "required_facts",
- "acceptance_contract", "composite_acceptance", "forbidden_any", "expected_resources",
+ "acceptance_contract", "composite_acceptance", "forbidden_any", "diagnostic_any", "expected_resources",
"resource_output_contract", "task_prompt", "reference_answer",
"mutable_body_policy",
}
diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py
index 30bd171db..bcedea9d5 100644
--- a/harness/web_fetch/report.py
+++ b/harness/web_fetch/report.py
@@ -641,8 +641,12 @@ def run_log_section(out,record,output):
path=contained_path(out,binding['path'])
if not path.is_file():continue
raw=path.read_bytes();observed=hashlib.sha256(raw).hexdigest()
- if binding.get('sha256') and binding['sha256']!=observed:raise ValueError(f'Call log hash mismatch: {record["attempt_id"]}')
- if binding.get('bytes') is not None and binding['bytes']!=len(raw):raise ValueError(f'Call log size mismatch: {record["attempt_id"]}')
+ if _RENDER_CONTEXT.get().score_audit:
+ from .published_snapshot import read_snapshot_public_file
+ read_snapshot_public_file(out,binding)
+ else:
+ if binding.get('sha256') and binding['sha256']!=observed:raise ValueError(f'Call log hash mismatch: {record["attempt_id"]}')
+ if binding.get('bytes') is not None and binding['bytes']!=len(raw):raise ValueError(f'Call log size mismatch: {record["attempt_id"]}')
try:payload=json.loads(raw)
except (UnicodeDecodeError,json.JSONDecodeError):payload={}
logs.append({'label':label,'path':binding['path'],'sha256':observed,
@@ -747,6 +751,8 @@ def run_log_section(out,record,output):
f'({diagnostic_size:,} 字节 · SHA-256:{esc(diagnostic_sha)} · 截断:{diagnostic_truncated})')
if not raw_record:raw_record='这次历史调用没有可下载的原始记录'
evidence_identity_note=(model_text('评分原件私下保留;公开下载隐藏目录或敏感值,校验记录分别标明原件与公开副本。','Scoring originals remain private; public downloads hide paths or sensitive values and bind original/public identities separately.') if any(view.get('redactions') for view in metadata.get('publication_views',{}).values()) else '原始输出保持调用后保存的字节。')
+ if record.get('scoring',{}).get('mode')=='published_snapshot_rescore':
+ evidence_identity_note='本次只重新评分。调用日志保留原调用信息;本次评分输入及其原始、公开哈希在实际输出处分别列出。'
return (f'
运行日志与证据
'
f'
{fields}
{runtime_error}'
f'查看完整运行字段
{full_fields}
'
@@ -830,6 +836,7 @@ def table(headers,rows,id='',sortable=False,cls='',metrics=None,index_header='
class RenderContext:
banner: str = ''
english_generated: bool = False
+ score_audit: dict | None = None
_RENDER_CONTEXT=ContextVar('webfetch_render_context',default=RenderContext())
@@ -850,10 +857,16 @@ def write(out,path,title,body,*,root_links=False):
nav=[('datasets/index.html','评测集'),('methodology/index.html','评测方法'),('analysis/index.html','结果分析'),('review/index.html','核验记录'),('guide/index.html','报告解读'),('adoption/index.html','方法介绍'),('reproduce/index.html','环境与复现'),('changes/index.html','变更记录'),('accounts/index.html','账号与安装')]
brand=link(base+'index.html','Web Fetch')
context=_RENDER_CONTEXT.get()
+ if context.score_audit and path.startswith('attempts/'):
+ identifier=Path(path).stem
+ reviewed=next((item for item in context.score_audit['changes'] if item['attempt_id']==identifier),None)
+ if reviewed:
+ body+=section('评分复核','
')
counterpart=(base+'en/'+path if context.english_generated else base+'en/index.html')
language=f'English'
footer='所有分数由本轮调用记录计算。网页与服务可能变化;结论仅适用于本轮样本和配置。'
- page=f'{document_base}{esc(title)} · Web Fetch{context.banner}{body}'
+ banner=context.banner.replace('href="audit/index.html"','href="'+base+'audit/index.html"')
+ page=f'{document_base}{esc(title)} · Web Fetch{banner}{body}'
target=contained_path(out,path);target.parent.mkdir(parents=True,exist_ok=True);target.write_text(page,encoding='utf-8')
_RESEARCH_ROUTES=[
@@ -2071,7 +2084,9 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
unknown=set(dataset_case_ids)-{case['id'] for case in dataset['cases']}
if unknown:raise ValueError('Unknown dataset refresh cases: '+', '.join(sorted(unknown)))
local_snapshot=dataset.get('comparison',{}).get('publication_state')=='local_audit_only_not_for_release'
- diagnostic=local_snapshot and not local_pass_rate_ranking(dataset)
+ score_binding=dataset.get('comparison',{}).get('score_audit')
+ score_audit=json.loads(contained_path(out,score_binding['path']).read_text()) if score_binding else None
+ diagnostic=bool(score_audit) or (local_snapshot and not local_pass_rate_ranking(dataset))
if local_snapshot and not show_local_status_banners:
raise ValueError('Uncertified local diagnostic status cannot be hidden')
from .reliability import assess_reliability
@@ -2101,7 +2116,9 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
reliability_banner += '
'
+ _RENDER_CONTEXT.set(RenderContext(reliability_banner,include_english,score_audit))
validate_report_ids(dataset,records)
for path in ['assets/style.css','assets/report.js','references']:
contained_path(out,path)
@@ -2125,6 +2142,8 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
for track,label in tracks:
cards+=overview_card(records,cases,track,label,names,diagnostic=diagnostic)
description=(f'{len(cases)} 个案例 · {len(names)} 种方法 · 本地未认证诊断快照。' if diagnostic else f'{len(cases)} 个案例 · {len(names)} 种方法 · {track_label} 类分榜与一个综合通过率总榜。')
+ if score_audit:
+ description=f'{len(cases)} 个案例 · {len(names)} 种方法 · 沿用原报告的任务范围、调用和指标定义。'
if local_pass_rate_ranking(dataset):
description=model_text(f'{len(cases)} 个案例 · {len(names)} 种方法 · 按本轮任务通过率排序。',f'{len(cases)} cases · {len(names)} methods · Ordered by task pass rate for this sample.')
home='
'+hero('Web Fetch 评测报告',description)
@@ -2424,6 +2443,11 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
if answer_view and answer_view['original_sha256']!=answer_view['published_sha256']:
output_note=(f'{len(output):,} 字符 · '+link('../'+answer_view['path'],'下载公开副本','Download public view')+' · '+link('../'+answer_view['receipt'],'查看脱敏记录','View redaction record')+
' '+model_text('目录或敏感值已隐藏;成绩来自评分原件。原件与公开副本的校验值见运行字段。','Paths or sensitive values are hidden; scores come from the original. Original/public hashes are listed in the run fields.'))
+ if r.get('scoring',{}).get('mode')=='published_snapshot_rescore':
+ view=r['scoring']['artifact_view']
+ view_label={'original':'与原哈希相同的保存输出','recovered_original':'按原哈希恢复的输出(本页展示公开副本)','published_projection':'公开脱敏副本'}[view]
+ output_note=f'{len(output):,} 字符 · 本次评分输入:'+esc(view_label)+' · '+link('../'+r['artifact_path'],'下载公开输出')
+ output_note+=' 原始调用输出 SHA-256:'+esc(r['scoring']['original_content_sha256'])+' 本次评分输入 SHA-256:'+esc(r['scoring']['scored_content_sha256'])+''
if readable_output(output)!=output:output_note+=' · 控制字节在页面中显示为 \\xNN'
actual_body=(corruption+f'
')
+ inventory=[]
+ for item in audit['changes']:
+ identifier=item['attempt_id']
+ inventory.append(('',cell(esc(item['provider']))+cell(esc(item['case_id']))+cell(verdict(item['historical_pass']))+cell(verdict(item['initial_rescore_pass']))+cell(verdict(item['after_fix_pass']))+cell(esc(item['classification']))+cell(link('#'+identifier,'原因与证据'))))
+ controls=''
+ markup=table(['方法','题目','旧判定','修复前','修复后','原因分类','证据'],inventory,metrics=[])
+ # Keep paragraphs in the evidence sections, outside the fixed-height inventory.
+ from bs4 import BeautifulSoup
+ document=BeautifulSoup(markup,'html.parser')
+ for element,item in zip(document.select('tbody tr'),audit['changes']):element['data-audit-provider']=item['provider']
+ body+=section('逐项判定',controls+'
'+str(document)+'
')
+ body+=section('脱敏副本的影响','
'+esc(audit['privacy_summary'])+'
'+''.join('
'+esc(text)+'
' for text in audit['privacy_findings']))
+ details=[]
+ for item in audit['changes']:
+ identifier=item['attempt_id']
+ facts=''.join('
'+label+'
'+esc(item[key])+'
' for key,label in [('requirement','验收要求'),('actual','实际返回'),('reason','变化原因'),('treatment','处理结果'),('source_scope','来源范围')])
+ evidence=' · '.join(link('../'+('audit/history/' if evidence.get('historical') else '')+evidence['path'],evidence.get('label') or evidence['path']) for evidence in item.get('evidence',[]))
+ quotations=''.join('
'+esc(quote['label'])+'
'+esc(quote['text'])+'
' for quote in item.get('quotes',[]))
+ details.append('
'))
diff --git a/harness/web_fetch/report_check.py b/harness/web_fetch/report_check.py
index a0682fbb8..0659243f7 100644
--- a/harness/web_fetch/report_check.py
+++ b/harness/web_fetch/report_check.py
@@ -9,7 +9,7 @@
from pathlib import Path
from urllib.parse import parse_qsl, urlencode, urlsplit
-from harness.record_io import load_jsonl
+from harness.record_io import file_sha256, load_jsonl
from harness.report_languages import check_language_contract, generated_languages, visible_numbers
from harness.reporting.article_assets import ArticleAssetParity
from harness.reporting.presentation import check_presentation
@@ -309,6 +309,11 @@ def check(root: Path, *, authority_root: Path, profile: Path | None = None, pres
profile_path = (
(repository / profile).resolve() if not profile.is_absolute() else profile.resolve()
)
+ if (root / 'report-manifest.json').is_file() and json.loads((root / 'report-manifest.json').read_text()).get('evidence_state') == 'published_snapshot_score_audit':
+ from .published_snapshot import check_snapshot_audit
+ if profile_path and file_sha256(profile_path) != json.loads((root / 'report-manifest.json').read_text()).get('profile_sha256'):
+ raise ValueError('Snapshot audit profile changed')
+ return check_snapshot_audit(root)
if (root / "wcxb-score-summary.json").is_file():
from harness.web_fetch.wcxb_report_check import check as check_wcxb
@@ -781,7 +786,11 @@ def main(argv=None, *, authority_root: Path | None = None):
parser.add_argument('--source-dataset', type=Path)
parser.add_argument("--authority-root", type=Path, default=authority_root or Path.cwd())
parser.add_argument("--presentation-binding", type=Path)
+ parser.add_argument("--visual-receipt",type=Path,help="Full snapshot-audit browser checks; save receipt outside the report")
args = parser.parse_args(argv)
+ if args.visual_receipt:
+ from .published_snapshot import check_snapshot_visual
+ print(check_snapshot_visual(args.root,args.visual_receipt))
result = check(args.root, authority_root=args.authority_root, profile=args.profile, presentation_binding=args.presentation_binding, source_dataset=args.source_dataset)
if result is not None:
print(result)
diff --git a/harness/web_fetch/report_explainer.py b/harness/web_fetch/report_explainer.py
index ad5043c2a..a2d20f0b2 100644
--- a/harness/web_fetch/report_explainer.py
+++ b/harness/web_fetch/report_explainer.py
@@ -406,7 +406,12 @@ def evidence(path, anchor, title, payload):
payload = '
完整实际返回
'+escape(raw)+'
'
payload += '
结果与评分
'+escape(json.dumps({key:row[key] for key in ('attempt_id','provider','case_id','run_id','started_at','elapsed_ms','ok','status_code','error_type','metrics') if key in row},ensure_ascii=False,indent=2))+'
'
for binding in record_public_call_log_bindings(row):
- log = verify_public_call_log(out,binding).read_text()
+ snapshot = json.loads((out/'dataset.json').read_text()).get('comparison',{}).get('score_audit') if (out/'dataset.json').is_file() else None
+ if snapshot:
+ from .published_snapshot import read_snapshot_public_file
+ log = read_snapshot_public_file(out,binding).read_text()
+ else:
+ log = verify_public_call_log(out,binding).read_text()
payload += '
调用日志
'+escape(log)+'
'
evidence('../attempts/'+row['attempt_id']+'.html', 'offline-result-'+row['attempt_id'],
'完整结果与调用证据 · '+name, payload)
diff --git a/harness/web_fetch/report_package.py b/harness/web_fetch/report_package.py
index df99ee92d..afb4ce275 100644
--- a/harness/web_fetch/report_package.py
+++ b/harness/web_fetch/report_package.py
@@ -28,7 +28,17 @@ def _require(condition, detail):
raise ValueError(str(detail))
-def validate_release(source: Path, *, presentation_binding: Path | None = None) -> bool:
+def validate_release(source: Path, *, presentation_binding: Path | None = None, snapshot_audit=False) -> bool:
+ manifest_path=source/'report-manifest.json'
+ audit_state=manifest_path.is_file() and json.loads(manifest_path.read_text()).get('evidence_state')=='published_snapshot_score_audit'
+ if snapshot_audit:
+ if not audit_state or presentation_binding is not None:
+ raise ValueError('Snapshot packaging requires its explicit diagnostic report')
+ from .published_snapshot import check_snapshot_audit
+ check_snapshot_audit(source)
+ return False
+ if audit_state:
+ raise ValueError('Published snapshot audits require --snapshot-audit; they are not certified releases')
if presentation_binding is not None:
from .report_build import validate_local_presentation
validate_local_presentation(source, presentation_binding)
@@ -534,7 +544,7 @@ def _archive_identity(path: Path):
return hashlib.file_digest(stream, "sha256").hexdigest()
-def build(source: Path, archive: Path, *, presentation_binding: Path | None = None, cached_content: Path | None = None, parts_directory: Path | None = None, refresh_privacy=False):
+def build(source: Path, archive: Path, *, presentation_binding: Path | None = None, cached_content: Path | None = None, parts_directory: Path | None = None, refresh_privacy=False, snapshot_audit=False):
if parts_directory is not None:
from .safety import unlinked_path
parts_directory = unlinked_path(parts_directory)
@@ -551,7 +561,7 @@ def build(source: Path, archive: Path, *, presentation_binding: Path | None = No
archive.parent.mkdir(parents=True, exist_ok=True)
target = archive.resolve()
previous = _archive_identity(target)
- sampled = validate_release(source, presentation_binding=presentation_binding)
+ sampled = validate_release(source, presentation_binding=presentation_binding, snapshot_audit=snapshot_audit)
with tempfile.TemporaryDirectory(prefix=".webfetch-package-", dir=archive.parent) as work:
staging = Path(work)
staged_archive = staging / "package.zip"
@@ -564,7 +574,7 @@ def build(source: Path, archive: Path, *, presentation_binding: Path | None = No
if parts_directory is None:
_write_archive(staging/'content', staged_archive)
from .publication_check import check
- check(staged_archive, source, presentation_binding=presentation_binding)
+ check(staged_archive, source, presentation_binding=presentation_binding, snapshot_audit=snapshot_audit)
if evidence_tree_identity(cached_content) != cached_identity:
raise ValueError('Cached public content changed during packaging')
if parts_directory is not None:
@@ -572,7 +582,7 @@ def build(source: Path, archive: Path, *, presentation_binding: Path | None = No
partition_content(staging/'content',staged_parts)
_write_archive(staging/'content',staged_archive)
from .publication_check import check
- check(staged_archive,source,presentation_binding=presentation_binding,
+ check(staged_archive,source,presentation_binding=presentation_binding,snapshot_audit=snapshot_audit,
external_archives=part_archives(staged_parts/'parts.json'),allow_pending_external=True)
manifest = _wire_manifest(staging/'content')
if evidence_tree_identity(source) != before:
@@ -751,16 +761,19 @@ def main(*, root: Path | None = None):
parser.add_argument(
"archive", nargs="?", type=Path, default=root / "publish/webfetch-sites.zip"
)
+ parser.add_argument("--snapshot-audit",action="store_true",help="Package an explicitly marked published-snapshot score audit")
parser.add_argument("--incremental", type=Path, help="Write an explanatory update against the baseline archive")
parser.add_argument("--presentation-binding", type=Path, help="Validate a frozen diagnostic snapshot without certifying it")
parser.add_argument("--cached-content", type=Path, help="Reuse cached public files and refresh source-bound result projections")
parser.add_argument("--refresh-privacy", action="store_true", help="Refresh cached public projections with the current privacy policy")
parser.add_argument("--parts-directory", type=Path, help="Prepare capacity-bound data attachments before publication")
args = parser.parse_args()
+ if args.incremental and args.snapshot_audit:
+ parser.error('Snapshot score revisions require a complete package')
if args.incremental:
print(json.dumps(build_incremental(args.source, args.archive, args.incremental, presentation_binding=args.presentation_binding), ensure_ascii=False))
return
- build(args.source, args.archive, presentation_binding=args.presentation_binding, cached_content=args.cached_content, parts_directory=args.parts_directory, refresh_privacy=args.refresh_privacy)
+ build(args.source, args.archive, presentation_binding=args.presentation_binding, cached_content=args.cached_content, parts_directory=args.parts_directory, refresh_privacy=args.refresh_privacy, snapshot_audit=args.snapshot_audit)
print(args.archive.resolve())
diff --git a/harness/web_fetch/rescoring.py b/harness/web_fetch/rescoring.py
index 1871fde11..237ddba4f 100644
--- a/harness/web_fetch/rescoring.py
+++ b/harness/web_fetch/rescoring.py
@@ -202,3 +202,114 @@ def main(argv=None):
if __name__ == "__main__":
main()
+
+
+def _score_published_row(arguments):
+ """Score verified saved bytes without changing a physical call's facts."""
+ record, case, path, digest, view = arguments
+ raw = Path(path).read_bytes()
+ if hashlib.sha256(raw).hexdigest() != digest:
+ raise ValueError('Published scoring input changed: ' + record['attempt_id'])
+ result = deepcopy(record)
+ result['metrics'] = score_attempt(case, raw, ok=record['ok'],
+ metadata=record.get('provider_metadata'), previous_metrics=record['metrics'],
+ status_code=record.get('status_code'), title=record.get('title'),
+ final_url=record.get('final_url'))
+ result.pop('diagnosis', None)
+ result.pop('failure_category', None)
+ result['scoring'] = {'mode': 'published_snapshot_rescore',
+ 'source_sha256': SCORING_SOURCE_SHA256, 'provider_calls_added': 0,
+ 'artifact_view': view, 'original_content_sha256': record['metrics']['content_sha256'],
+ 'scored_content_sha256': digest}
+ return result
+
+
+def score_published_snapshot(source, dataset, *, jobs=1, recovered=None, progress=None):
+ """Recompute a published matrix, disclosing privacy views per result.
+
+ This explicit diagnostic boundary cannot certify a cohort or replace private
+ evidence. Every changed artifact must match the saved publication receipt;
+ an exact recovered original takes precedence over its public view.
+ """
+ from concurrent.futures import ProcessPoolExecutor
+ from collections import deque
+ from harness.record_io import file_sha256
+ from lexbench_webfetch.metrics import task_passed
+ if type(jobs) is not int or not 1 <= jobs <= 8:
+ raise ValueError('Offline scoring jobs must be between 1 and 8')
+ source = Path(source).resolve()
+ records = parse_jsonl(contained_path(source, 'results.jsonl').read_bytes())
+ original = json.loads(contained_path(source, 'dataset.json').read_text())
+ validate_report_ids(original, records)
+ cases = {case['id']: case for case in dataset['cases']}
+ old_cases = {case['id']: case for case in original['cases']}
+ from .scoring_revision import ALLOWED_FIELDS
+ if cases.keys() != old_cases.keys() or dataset.get('coverage_plan') != original.get('coverage_plan'):
+ raise ValueError('Published rescoring must retain the complete matrix')
+ for key, prior in old_cases.items():
+ changes = {field for field in prior.keys() | cases[key].keys()
+ if prior.get(field) != cases[key].get(field)}
+ if changes - ALLOWED_FIELDS:
+ raise ValueError('Published rescoring changed collection inputs: ' + key)
+ privacy_path = contained_path(source, 'publication-privacy.json')
+ privacy = json.loads(privacy_path.read_text()) if privacy_path.is_file() else {'files': []}
+ views = {item['path']: item for item in privacy['files'] if item.get('scored_output')}
+ recovered = recovered or {}
+ unknown = set(recovered) - {row['artifact_path'] for row in records}
+ if unknown:
+ raise ValueError('Recovered output is outside the matrix')
+ requests, inputs = [], []
+ for row in records:
+ relative = row['artifact_path']
+ path = contained_path(source, relative, reject_symlinks=True)
+ published_hash = file_sha256(path)
+ original_hash = row['metrics']['content_sha256']
+ if published_hash != original_hash:
+ proof = views.get(relative, {})
+ if (proof.get('original_sha256') != original_hash
+ or proof.get('published_sha256') != published_hash
+ or proof.get('published_bytes') != path.stat().st_size):
+ raise ValueError('Output lacks an exact privacy binding: ' + row['attempt_id'])
+ view = 'published_projection' if published_hash != original_hash else 'original'
+ if relative in recovered:
+ path = Path(recovered[relative])
+ if path.is_symlink() or file_sha256(path) != original_hash:
+ raise ValueError('Recovered original hash differs: ' + row['attempt_id'])
+ digest, view = original_hash, 'recovered_original'
+ else:
+ digest = published_hash
+ inputs.append({'attempt_id': row['attempt_id'], 'path': relative,
+ 'original_sha256': original_hash, 'published_sha256': published_hash,
+ 'scored_sha256': digest, 'view': view})
+ requests.append((row, cases[row['case_id']], str(path), digest, view))
+ results = []
+ def accept(result):
+ results.append(result)
+ if progress:
+ progress(len(results), len(records))
+ if jobs == 1:
+ for request in requests:
+ accept(_score_published_row(request))
+ else:
+ with ProcessPoolExecutor(max_workers=jobs) as pool:
+ remaining = iter(requests)
+ pending = deque(pool.submit(_score_published_row, request)
+ for request in [next(remaining, None) for _ in range(jobs)]
+ if request is not None)
+ while pending:
+ accept(pending.popleft().result())
+ following = next(remaining, None)
+ if following is not None:
+ pending.append(pool.submit(_score_published_row, following))
+ from harness.record_io import canonical_json_sha256
+ receipt = {'kind': 'published_snapshot_rescore', 'provider_calls_added': 0,
+ 'corrected_dataset_sha256': canonical_json_sha256(dataset),
+ 'input_dataset_sha256': file_sha256(source / 'dataset.json'),
+ 'input_results_sha256': file_sha256(source / 'results.jsonl'),
+ 'scoring_source_sha256': SCORING_SOURCE_SHA256, 'inputs': inputs,
+ 'result_count': len(results), 'case_count': len(cases),
+ 'projected_scoring_inputs': sum(item['view'] == 'published_projection' for item in inputs),
+ 'recovered_originals': sum(item['view'] == 'recovered_original' for item in inputs),
+ 'task_passes': sum(task_passed(row['ok'], row['metrics'], cases[row['case_id']].get('task_kind','body'))
+ for row in results)}
+ return results, receipt
diff --git a/harness/web_fetch/scoring_revision.py b/harness/web_fetch/scoring_revision.py
index dbb15f43a..adfe0d137 100644
--- a/harness/web_fetch/scoring_revision.py
+++ b/harness/web_fetch/scoring_revision.py
@@ -35,6 +35,7 @@
"acceptance_contract",
"composite_acceptance",
"forbidden_any",
+ "diagnostic_any",
"expected_resources",
"resource_output_contract",
"task_prompt",
diff --git a/pyproject.toml b/pyproject.toml
index 9bf1058a1..167137f93 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -6,7 +6,7 @@ readme = "README.md"
license = {file = "LICENSE"}
requires-python = ">=3.11"
dependencies = [
- "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@1552f731dd75be26c3b3551c04bf0730e6293335",
+ "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@4d73ada2fa1305fab61f0c9d222f7a315361281c",
"beautifulsoup4>=4.12,<5",
"fastcdc==1.7.0",
]
diff --git a/scripts/build_report.py b/scripts/build_report.py
index 3fff89ec3..70300b7af 100644
--- a/scripts/build_report.py
+++ b/scripts/build_report.py
@@ -58,6 +58,13 @@ def requested_language(output, supported=SUPPORTED_REPORT_LANGUAGES):
if args.profile:
args.profile=(ROOT/args.profile).resolve() if not args.profile.is_absolute() else args.profile.resolve()
+ if json.loads(args.profile.read_text()).get('variant') == 'published-snapshot-audit':
+ if not args.evidence or not args.output or args.local_audit or args.wcxb or args.include_english:
+ parser.error('snapshot audit requires a separate --evidence, --output and bound --profile')
+ requested_language(args.output, supported=('zh',))
+ from harness.web_fetch.published_snapshot import build_snapshot_audit
+ print(build_snapshot_audit(args.evidence, args.output, profile=args.profile))
+ raise SystemExit(0)
if json.loads(args.profile.read_text()).get('variant') == 'diagnostic-cohorts':
args.diagnostic_cohorts=True
if args.diagnostic_cohorts:
diff --git a/tests/test_publication_privacy.py b/tests/test_publication_privacy.py
index a9f7dc162..0c575b4ff 100644
--- a/tests/test_publication_privacy.py
+++ b/tests/test_publication_privacy.py
@@ -933,3 +933,25 @@ def test_review_projection_does_not_hide_risks_in_unrelated_metadata():
files = {'local-evidence/review/supplement.json': raw}
public, _ = project_host_metadata_files(files, ['/Users/fixture'], public_excerpt_views=True)
assert public == files and publication_risk(public['local-evidence/review/supplement.json'])
+
+
+def test_signed_thumbnail_redaction_preserves_outer_record_and_following_text():
+ """A private image query must not consume the record's closing Markdown."""
+ from lexbench_webfetch.scoring import score_attempt
+ from lexbench_webfetch.metrics import task_passed
+ case={'url':'https://example.test/','acceptance_contract':{'version':1,
+ 'schema':'named_link','minimum_records':2,'hosts':['example.test'],
+ 'entry_path':r'/boards/[0-9]+','stable_text_anchors':[{'id':'feature','alternatives':['Collect and organize']} ]}}
+ original=(
+ '[Collection One](https://example.test/boards/1)\n'
+ '[Collection Two](https://example.test/boards/2)\n'
+ 'Collect and organize\n').encode()
+ projected,receipt=project_file('artifacts/sample/boards.txt',original)
+ assert b'fictional-one' not in projected and b'fictional-two' not in projected
+ assert receipt['replacements']==2
+ assert task_passed(True,score_attempt(case,original,ok=True))
+ scored=score_attempt(case,projected,ok=True)
+ assert task_passed(True,scored)
+ assert scored['stable_unique_record_count']==2
+ broken=projected.replace(b')Collection One](https://example.test/boards/1)',b'')
+ assert not task_passed(True,score_attempt(case,broken,ok=True))
diff --git a/tests/test_published_snapshot_report.py b/tests/test_published_snapshot_report.py
new file mode 100644
index 000000000..8540092f7
--- /dev/null
+++ b/tests/test_published_snapshot_report.py
@@ -0,0 +1,99 @@
+"""A saved report matrix rebuilds through the maintained WebFetch renderer."""
+import json
+
+import pytest
+
+from harness.record_io import canonical_json_sha256, file_sha256
+from harness.web_fetch.published_snapshot import build_snapshot_audit, check_snapshot_audit, validate_snapshot_audit
+from harness.web_fetch.report_explainer import _headless_article_input
+from harness.web_fetch.rescoring import score_published_snapshot
+from lexbench_webfetch.scoring import score_attempt
+
+
+def example(root):
+ source=root/'source';source.mkdir()
+ cases=[{'id':key,'name':key,'url':'https://example.test/'+key,'track':'web','category':'web',
+ 'required_facts':[{'id':'body','alternatives':['complete answer']}]} for key in ['complete','missing']]
+ dataset={'id':'published-example','name':'Published example','cases':cases,
+ 'coverage_plan':{'web':['chromium']},'comparison':{'kind':'headless_markdown',
+ 'timeout_seconds':5,'runs_per_case':1,'concurrency':1,'max_failure_retries':0}}
+ rows=[]
+ for case in cases:
+ raw=b'complete answer' if case['id']=='complete' else b'partial output'
+ path=source/'artifacts'/f'{case["id"]}.txt';path.parent.mkdir(exist_ok=True);path.write_bytes(raw)
+ reference=source/'references'/f'{case["id"]}.txt';reference.parent.mkdir(exist_ok=True);reference.write_text('complete answer')
+ rows.append({'provider':'chromium','provider_type':'fixture','case_id':case['id'],'case_name':case['name'],
+ 'category':'web','attempt_id':'chromium__'+case['id']+'__r001','run':1,'run_id':'call-'+case['id'],
+ 'url':case['url'],'started_at':'2026-10-01T00:00:00Z','elapsed_ms':100,
+ 'ok':True,'status_code':200,'title':None,'final_url':case['url'],'error_type':None,
+ 'error_message':None,'provider_metadata':{},'artifact_path':path.relative_to(source).as_posix(),
+ 'metrics':score_attempt(case,raw,ok=True)})
+ rows[0]['metrics']['content_pass']=False
+ (source/'dataset.json').write_text(json.dumps(dataset))
+ (source/'results.jsonl').write_text(''.join(json.dumps(row)+'\n' for row in rows))
+ (source/'execution.json').write_text(json.dumps({'batches':[{'timeout_seconds':5,'attempt_ids':[row['attempt_id'] for row in rows]}]}))
+ result,receipt=score_published_snapshot(source,dataset)
+ manifest={'files':{p.relative_to(source).as_posix():{'sha256':file_sha256(p)} for p in source.rglob('*') if p.is_file()}}
+ def save(name,value):
+ path=root/name;path.write_text(json.dumps(value,ensure_ascii=False));return {'path':name,'sha256':file_sha256(path)}
+ review={'title':'Saved output review','summary':'One corrected serialization decision.',
+ 'figure_caption':'Saved output and acceptance rules.','metric_note':'Two saved calls, no new calls.',
+ 'focus_heading':'Sample configuration','focus_summary':'One correction.',
+ 'privacy_summary':'All original bytes are present.','privacy_findings':[],
+ 'review_scope':'One of two results reviewed.','total_attempts':2,'reviewed_attempts':1,
+ 'providers':[{'provider':'chromium','published_passes':0,'historical_passes':0,'initial_rescore_passes':0,'corrected_passes':1,'total':2}],
+ 'changes':[{'attempt_id':result[0]['attempt_id'],'provider':'chromium','case_id':'complete',
+ 'historical_pass':False,'initial_rescore_pass':False,'after_fix_pass':True,
+ 'classification':'Serialization defect','requirement':'Return the complete answer.',
+ 'actual':'complete answer','reason':'The saved answer includes the required text.',
+ 'treatment':'Use the shared scorer.','source_scope':'Frozen fixture reference.','evidence':[],'quotes':[]}]}
+ article={'schema':'webfetch_browser_interpretation/1','evidence_sha256':canonical_json_sha256(_headless_article_input({c['id']:c for c in cases},result,['chromium'])),
+ 'title':'Saved result interpretation','intro':'Two saved calls under the same contract.',
+ 'scoring':'Required text must occur in the delivered answer.','conclusions':'One answer meets its contract.',
+ 'providers':{'chromium':{'specimen_case_id':'missing','explanation':'The partial output lacks the required text.'}}}
+ inputs={'dataset':save('corrected.json',dataset),'receipt':save('receipt.json',receipt),
+ 'review':save('review.json',review),'interpretation':save('interpretation.json',article),
+ 'snapshot_manifest':save('manifest.json',manifest),
+ 'historical_verdicts':save('historic.json',{'verdicts':{r['attempt_id']:False for r in rows}})}
+ initial=root/'initial.jsonl';initial.write_text(''.join(json.dumps(row)+'\n' for row in rows))
+ inputs['initial_results']={'path':initial.name,'sha256':file_sha256(initial)}
+ result_path=root/'corrected.jsonl';result_path.write_text(''.join(json.dumps(row)+'\n' for row in result))
+ inputs['results']={'path':result_path.name,'sha256':file_sha256(result_path)}
+ figure=root/'figure.png';figure.write_bytes(b'fixture image');inputs['figure']={'path':figure.name,'sha256':file_sha256(figure)}
+ profile={'variant':'published-snapshot-audit','certified':False,'inputs':inputs,
+ 'source':{'url':'https://example.test/report','version':'frozen','archive_sha256':'0'*64},
+ 'correction_provenance':{'category':'format-normalization','root_cause':'Visible native label boundaries must preserve complete declared words.',
+ 'regression_ids':['native-serialization-stock-label','native-serialization-stock-word-fragment']}}
+ path=root/'profile.json';path.write_text(json.dumps(profile));return source,path
+
+
+def test_saved_matrix_keeps_native_routes_and_inspectable_review(tmp_path):
+ source,profile=example(tmp_path);before={p.relative_to(source):p.read_bytes() for p in source.rglob('*') if p.is_file()}
+ out=tmp_path/'report';build_snapshot_audit(source,out,profile=profile)
+ assert '2 saved calls' in check_snapshot_audit(out)
+ from harness.web_fetch.report_package import validate_release
+ with pytest.raises(ValueError,match='require --snapshot-audit'):validate_release(out)
+ assert validate_release(out,snapshot_audit=True) is False
+ for path in ['index.html','guide/index.html','datasets/index.html','analysis/index.html','attempts/chromium__complete__r001.html','attempts/chromium__missing__r001.html','en/index.html']:
+ assert (out/path).is_file()
+ assert {p.relative_to(source):p.read_bytes() for p in source.rglob('*') if p.is_file()}==before
+ detail=out/'attempts/chromium__complete__r001.html';detail.write_text(detail.read_text().replace('data-score-audit','data-deleted-audit'))
+ with pytest.raises(ValueError,match='lost its audit explanation'):check_snapshot_audit(out)
+
+
+@pytest.mark.parametrize('defect',['extra-file','changed-source','changed-call','wrong-view','wrong-count','review-omission'])
+def test_snapshot_boundary_rejects_unbound_evidence_and_call_changes(tmp_path,defect):
+ source,profile=example(tmp_path);config=json.loads(profile.read_text())
+ if defect=='extra-file':(source/'private-note.txt').write_text('undeclared')
+ elif defect=='changed-source':(source/'artifacts/complete.txt').write_text('tampered')
+ elif defect in {'wrong-count','review-omission'}:
+ p=tmp_path/config['inputs']['review']['path'];review=json.loads(p.read_text())
+ if defect=='wrong-count':review['providers'][0]['corrected_passes']=2
+ else:review['changes']=[];review['reviewed_attempts']=0
+ p.write_text(json.dumps(review));config['inputs']['review']['sha256']=file_sha256(p);profile.write_text(json.dumps(config))
+ else:
+ p=tmp_path/config['inputs']['results']['path'];rows=[json.loads(x) for x in p.read_text().splitlines()]
+ if defect=='changed-call':rows[0]['elapsed_ms']=1
+ else:rows[0]['scoring']['artifact_view']='published_projection'
+ p.write_text(''.join(json.dumps(row)+'\n' for row in rows));config['inputs']['results']['sha256']=file_sha256(p);profile.write_text(json.dumps(config))
+ with pytest.raises(ValueError):validate_snapshot_audit(source,profile)
diff --git a/tests/test_published_snapshot_rescoring.py b/tests/test_published_snapshot_rescoring.py
new file mode 100644
index 000000000..7b2cee1ea
--- /dev/null
+++ b/tests/test_published_snapshot_rescoring.py
@@ -0,0 +1,74 @@
+"""Public-copy rescoring must disclose changed bytes and preserve call facts."""
+import hashlib
+import json
+
+import pytest
+
+from harness.web_fetch.rescoring import score_published_snapshot
+from lexbench_webfetch.metrics import task_passed
+from lexbench_webfetch.scoring import score_attempt
+
+
+def snapshot(tmp_path):
+ case = {'id': 'one', 'name': 'One', 'url': 'https://example.test/',
+ 'category': 'web', 'track': 'web', 'required_facts': [
+ {'id': 'fact', 'alternatives': ['complete answer']}]}
+ raw = b'complete answer'
+ public = b'[REDACTED]'
+ original = tmp_path / 'original.txt'
+ original.write_bytes(raw)
+ source = tmp_path / 'source'
+ source.mkdir()
+ (source / 'answer.txt').write_bytes(public)
+ dataset = {'cases': [case], 'coverage_plan': {'web': ['sample']}}
+ record = {'attempt_id': 'sample__one__r001', 'provider': 'sample',
+ 'case_id': 'one', 'case_name': 'One', 'category': 'web',
+ 'url': case['url'], 'run': 1, 'ok': True, 'elapsed_ms': 43.5,
+ 'run_id': 'unchanged-call', 'artifact_path': 'answer.txt',
+ 'provider_metadata': {}, 'metrics': score_attempt(case, raw, ok=True)}
+ (source / 'dataset.json').write_text(json.dumps(dataset))
+ (source / 'results.jsonl').write_text(json.dumps(record) + '\n')
+ receipt = {'files': [{'path': 'answer.txt', 'scored_output': True,
+ 'original_sha256': hashlib.sha256(raw).hexdigest(),
+ 'published_sha256': hashlib.sha256(public).hexdigest(),
+ 'published_bytes': len(public)}]}
+ (source / 'publication-privacy.json').write_text(json.dumps(receipt))
+ return source, dataset, record, original
+
+
+@pytest.mark.parametrize('recover', [False, True])
+def test_exact_public_or_recovered_input_is_explicit(tmp_path, recover):
+ source, dataset, old, original = snapshot(tmp_path)
+ before = {p.name: p.read_bytes() for p in source.iterdir()}
+ rows, receipt = score_published_snapshot(source, dataset,
+ recovered={'answer.txt': original} if recover else None)
+ row = rows[0]
+ assert task_passed(row['ok'], row['metrics']) is recover
+ assert row['scoring']['artifact_view'] == ('recovered_original' if recover else 'published_projection')
+ assert receipt['projected_scoring_inputs'] == (0 if recover else 1)
+ assert receipt['provider_calls_added'] == 0
+ assert {k: v for k, v in old.items() if k != 'metrics'} == {
+ k: v for k, v in row.items() if k not in {'metrics', 'scoring'}}
+ assert {p.name: p.read_bytes() for p in source.iterdir()} == before
+
+
+@pytest.mark.parametrize('defect', ['public_hash', 'original_hash', 'size', 'recovered_hash', 'matrix', 'input'])
+def test_unbound_or_changed_evidence_cannot_be_scored(tmp_path, defect):
+ source, dataset, _, original = snapshot(tmp_path)
+ recovered = None
+ receipt_path = source / 'publication-privacy.json'
+ receipt = json.loads(receipt_path.read_text())
+ if defect in {'public_hash', 'original_hash'}:
+ receipt['files'][0]['published_sha256' if defect == 'public_hash' else 'original_sha256'] = '0' * 64
+ elif defect == 'size':
+ receipt['files'][0]['published_bytes'] += 1
+ elif defect == 'recovered_hash':
+ original.write_bytes(b'other answer')
+ recovered = {'answer.txt': original}
+ elif defect == 'matrix':
+ dataset['cases'] = []
+ else:
+ dataset['cases'][0]['url'] = 'https://example.test/other'
+ receipt_path.write_text(json.dumps(receipt))
+ with pytest.raises(ValueError):
+ score_published_snapshot(source, dataset, recovered=recovered)
diff --git a/tests/test_report_methods.py b/tests/test_report_methods.py
index 437bba929..4f9d11bb9 100644
--- a/tests/test_report_methods.py
+++ b/tests/test_report_methods.py
@@ -200,3 +200,19 @@ def test_method_guide_discloses_retained_success_and_repaired_failure_builds(tmp
unknown = render_methods(tmp_path, [{'provider': 'cloud', 'provider_type': 'fixture'}],
hero=hero, section=section, table=table, cell=cell, link=link, esc=esc)
assert 'unknown' in unknown and '>1<' in unknown
+
+
+def test_bound_external_method_registry_is_scoped_to_its_report(tmp_path):
+ from harness.web_fetch.methods import with_measured_method_labels, native_browser_engine, browser_method_label, browser_method_registry
+ @with_measured_method_labels
+ def render(out,dataset,records):
+ assert native_browser_engine('external-build') == 'moli'
+ assert browser_method_label('external-build') == 'External Moli build'
+ assert browser_method_registry()['external-build']['source_url'] == 'https://example.test/build'
+ return 'rendered'
+ dataset={'comparison':{'method_registry':{'external-build':{'engine':'moli','label':'External Moli build','source_url':'https://example.test/build'}}}}
+ assert render(tmp_path,dataset,[]) == 'rendered'
+ import json
+ (tmp_path/'dataset.json').write_text(json.dumps(dataset))
+ assert render(tmp_path,{'external-case':{'id':'external-case'}},[]) == 'rendered'
+ assert native_browser_engine('external-build') is None
diff --git a/uv.lock b/uv.lock
index 4f5b88f06..ef79d41c7 100644
--- a/uv.lock
+++ b/uv.lock
@@ -354,7 +354,7 @@ requires-dist = [
{ name = "jieba", marker = "extra == 'webmainbench-eval'", specifier = "==0.42.1" },
{ name = "json-repair", marker = "extra == 'browser-use-eval'", specifier = ">=0.54,<1" },
{ name = "langchain-community", marker = "extra == 'oss-eval'", specifier = "==0.3.30" },
- { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=1552f731dd75be26c3b3551c04bf0730e6293335" },
+ { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=4d73ada2fa1305fab61f0c9d222f7a315361281c" },
{ name = "markdown", marker = "extra == 'webmainbench-eval'", specifier = ">=3.7,<4" },
{ name = "openai", marker = "extra == 'webmainbench-eval'", specifier = ">=2.8,<4" },
{ name = "openpyxl", marker = "extra == 'oss-eval'", specifier = "==3.1.5" },
@@ -1933,7 +1933,7 @@ wheels = [
[[package]]
name = "lexbench-webfetch"
version = "0.3.0"
-source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=1552f731dd75be26c3b3551c04bf0730e6293335#1552f731dd75be26c3b3551c04bf0730e6293335" }
+source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=4d73ada2fa1305fab61f0c9d222f7a315361281c#4d73ada2fa1305fab61f0c9d222f7a315361281c" }
dependencies = [
{ name = "beautifulsoup4" },
{ name = "soupsieve" },
From 4185cca3024f781743329d8778def7a3ca1d05c2 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Wed, 21 Oct 2026 20:48:36 +0800
Subject: [PATCH 21/51] test(webfetch): load deferred images in snapshot visual
checks
---
harness/web_fetch/published_snapshot.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/harness/web_fetch/published_snapshot.py b/harness/web_fetch/published_snapshot.py
index 00a3847bd..4c13a9754 100644
--- a/harness/web_fetch/published_snapshot.py
+++ b/harness/web_fetch/published_snapshot.py
@@ -294,11 +294,12 @@ def check_snapshot_visual(root, receipt):
page.goto(origin+'/'+route,wait_until='networkidle')
page.evaluate('(t)=>{document.documentElement.dataset.theme=t;document.body.classList.toggle("dark",t==="dark");localStorage.setItem("theme",t)}',theme)
settle_theme(page)
+ page.evaluate('() => Promise.all([...document.images].map(i=>{i.loading="eager";return i.decode().catch(()=>{})}))')
state=page.evaluate('''() => ({overflow:document.documentElement.scrollWidth-innerWidth,
brokenImages:[...document.images].filter(i=>!i.complete||i.naturalWidth===0).map(i=>i.getAttribute('src')),
style:document.styleSheets.length})''')
state.update(route=route,width=width,height=height,theme=theme);states.append(state)
- if abs(state['overflow'])>2 or state['brokenImages'] or not state['style'] or errors:
+ if abs(state['overflow'])>2 or state['brokenImages'] or (not state['style'] and route!='en/index.html') or errors:
raise ValueError('Snapshot visual check failed: '+json.dumps({'state':state,'errors':errors}))
if route in ['index.html','guide/index.html','audit/index.html']:
page.screenshot(path=str(receipt.parent/(route.split('/')[0]+'-'+str(width)+'-'+theme+'.png')),full_page=False)
From 05f0931bdf7bd61cb11610b4c8809796c84ece10 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Wed, 21 Oct 2026 20:52:38 +0800
Subject: [PATCH 22/51] fix(webfetch): retain historical privacy receipts when
repacking audits
---
harness/web_fetch/published_snapshot.py | 22 +++++++++++++++++-----
harness/web_fetch/report_package.py | 2 ++
tests/test_published_snapshot_report.py | 7 +++++++
3 files changed, 26 insertions(+), 5 deletions(-)
diff --git a/harness/web_fetch/published_snapshot.py b/harness/web_fetch/published_snapshot.py
index 4c13a9754..73210e4f3 100644
--- a/harness/web_fetch/published_snapshot.py
+++ b/harness/web_fetch/published_snapshot.py
@@ -149,6 +149,10 @@ def build_snapshot_audit(source, output, *, profile):
shutil.rmtree(staging / 'en', ignore_errors=True)
history = staging / 'audit/history'
history.mkdir(parents=True)
+ historical_privacy=None
+ if (staging/'publication-privacy.json').is_file():
+ shutil.move(staging/'publication-privacy.json',history/'publication-privacy.json')
+ historical_privacy={'path':'audit/history/publication-privacy.json','sha256':file_sha256(history/'publication-privacy.json')}
for name in ('dataset.json', 'results.jsonl', 'result-review.json'):
if (staging / name).is_file():
shutil.copy2(staging / name, history / name)
@@ -180,6 +184,8 @@ def build_snapshot_audit(source, output, *, profile):
'total_attempts': len(records), 'projected_scoring_inputs': receipt['projected_scoring_inputs'],
'recovered_originals': receipt['recovered_originals'],
'figure': 'audit/scoring-explained.png'}
+ if historical_privacy:
+ comparison['score_audit']['historical_privacy']=historical_privacy
(staging / 'dataset.json').write_text(json.dumps(dataset, ensure_ascii=False, indent=2)+'\n')
shutil.copy2(inputs['results'], staging / 'results.jsonl')
batches = json.loads((staging / 'execution.json').read_text())['batches']
@@ -214,6 +220,7 @@ def check_snapshot_audit(root):
rows = load_jsonl(root/'results.jsonl')
binding = dataset['comparison']['score_audit']
audit = json.loads(_bound(root, binding).read_text())
+ if binding.get('historical_privacy'):_bound(root,binding['historical_privacy'])
check_saved_result_summary(root, dataset, rows)
for relative in ['index.html','guide/index.html','audit/index.html','analysis/index.html']:
soup = BeautifulSoup((root/relative).read_text(), 'html.parser')
@@ -254,7 +261,10 @@ def read_snapshot_public_file(root, binding):
observed = file_sha256(path)
if observed == binding['sha256']:
return path
- privacy = json.loads(contained_path(root, 'publication-privacy.json').read_text())
+ dataset=json.loads(contained_path(root,'dataset.json').read_text())
+ historical=dataset.get('comparison',{}).get('score_audit',{}).get('historical_privacy')
+ privacy_path=_bound(root,historical) if historical else contained_path(root,'publication-privacy.json')
+ privacy = json.loads(privacy_path.read_text())
proof = next((item for item in privacy['files'] if item['path'] == binding['path']), {})
if (proof.get('original_sha256') != binding['sha256']
or proof.get('published_sha256') != observed
@@ -276,8 +286,8 @@ def check_snapshot_visual(root, receipt):
routes=['index.html','guide/index.html','audit/index.html','analysis/index.html',
'datasets/index.html','methodology/index.html','review/index.html',
'changes/index.html','adoption/index.html','reproduce/index.html','accounts/index.html','en/index.html']
- for folder in ['cases','providers']:
- pages=sorted((root/folder).glob('*.html'))
+ for folder in ['datasets','providers']:
+ pages=sorted(p for p in (root/folder).glob('*.html') if p.name!='index.html')
if pages:routes.append(pages[0].relative_to(root).as_posix())
for passed in [False,True]:
row=next(r for r in rows if task_passed(r['ok'],r['metrics'],cases[r['case_id']].get('task_kind','body')) is passed)
@@ -309,10 +319,12 @@ def check_snapshot_visual(root, receipt):
if button.count():
button.evaluate('(b)=>{for(let e=b.parentElement;e;e=e.parentElement)if(e.tagName==="DETAILS")e.open=true}')
button.click();page.locator('dialog[open]').wait_for()
- if not page.locator('dialog[open]').inner_text().strip():raise ValueError('Empty evidence dialog')
+ page.wait_for_function('''() => {const d=document.querySelector('dialog[open]');return d && [...d.querySelectorAll('pre[data-file],pre[data-evidence-file]')].every(p=>p.dataset.loaded==='1') && (d.querySelector('textarea')?.value.length>0 || [...d.querySelectorAll('pre')].some(p=>p.textContent.trim().length>0))}''')
+ page.screenshot(path=str(receipt.parent/('detail-'+route.split('/')[-1]+'.png')),full_page=False)
page.keyboard.press('Escape')
if page.locator('dialog[open]').count():raise ValueError('Evidence dialog did not close')
- interactions.append({'route':route,'open_close':'passed'})
+ page.wait_for_function('(id)=>document.activeElement?.dataset.callLogOpen===id',arg=button.get_attribute('data-call-log-open'))
+ interactions.append({'route':route,'load_open_close_focus':'passed'})
page.locator('header a[href]').first.click();page.wait_for_load_state('networkidle')
if not page.url.endswith('/index.html'):raise ValueError('Nested report navigation failed')
browser.close()
diff --git a/harness/web_fetch/report_package.py b/harness/web_fetch/report_package.py
index afb4ce275..2fd9b22eb 100644
--- a/harness/web_fetch/report_package.py
+++ b/harness/web_fetch/report_package.py
@@ -129,6 +129,8 @@ def _build_content(source, destination, archive, *, webmainbench, raw_blocks, de
footprint_projections = []
source_paths = sorted(path for path in source.rglob("*") if path.is_file())
source_names = {path.relative_to(source).as_posix() for path in source_paths}
+ if "publication-privacy.json" in source_names:
+ raise ValueError("Publication source contains a reserved generated file: publication-privacy.json")
source_file_count = sum(path.name != ".generated-report" for path in source_paths)
logical_route_count = sum(path.suffix == ".html" for path in source_paths)
diff --git a/tests/test_published_snapshot_report.py b/tests/test_published_snapshot_report.py
index 8540092f7..47ba5b193 100644
--- a/tests/test_published_snapshot_report.py
+++ b/tests/test_published_snapshot_report.py
@@ -32,6 +32,7 @@ def example(root):
(source/'dataset.json').write_text(json.dumps(dataset))
(source/'results.jsonl').write_text(''.join(json.dumps(row)+'\n' for row in rows))
(source/'execution.json').write_text(json.dumps({'batches':[{'timeout_seconds':5,'attempt_ids':[row['attempt_id'] for row in rows]}]}))
+ (source/'publication-privacy.json').write_text(json.dumps({'files':[]}))
result,receipt=score_published_snapshot(source,dataset)
manifest={'files':{p.relative_to(source).as_posix():{'sha256':file_sha256(p)} for p in source.rglob('*') if p.is_file()}}
def save(name,value):
@@ -74,6 +75,12 @@ def test_saved_matrix_keeps_native_routes_and_inspectable_review(tmp_path):
from harness.web_fetch.report_package import validate_release
with pytest.raises(ValueError,match='require --snapshot-audit'):validate_release(out)
assert validate_release(out,snapshot_audit=True) is False
+ assert not (out/'publication-privacy.json').exists()
+ assert (out/'audit/history/publication-privacy.json').is_file()
+ from harness.web_fetch.report_package import build
+ from harness.web_fetch.publication_check import check
+ archive=tmp_path/'report.zip';build(out,archive,snapshot_audit=True)
+ check(archive,out,snapshot_audit=True)
for path in ['index.html','guide/index.html','datasets/index.html','analysis/index.html','attempts/chromium__complete__r001.html','attempts/chromium__missing__r001.html','en/index.html']:
assert (out/path).is_file()
assert {p.relative_to(source):p.read_bytes() for p in source.rglob('*') if p.is_file()}==before
From d0761d2d1aac335977696a62711535e9dbe223c1 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Wed, 21 Oct 2026 20:56:10 +0800
Subject: [PATCH 23/51] feat(webfetch): publish saved-output audits through the
shared entry
---
configs/report-templates.json | 3 ++-
docs/benchmarks.md | 5 +++++
harness/web_fetch/report_publish.py | 10 ++++++----
tests/test_published_snapshot_report.py | 18 ++++++++++++++++++
4 files changed, 31 insertions(+), 5 deletions(-)
diff --git a/configs/report-templates.json b/configs/report-templates.json
index fe5fc7097..91e7d4c27 100644
--- a/configs/report-templates.json
+++ b/configs/report-templates.json
@@ -32,7 +32,8 @@
"snapshot_audit_example": "tests/test_published_snapshot_report.py:example",
"snapshot_audit_example_check": "uv run --frozen python -m pytest tests/test_published_snapshot_report.py tests/test_published_snapshot_rescoring.py",
"snapshot_audit_release_check": "uv run --frozen python scripts/check_report.py /absolute/new-report --profile /absolute/profile.json --visual-receipt /absolute/private/visual.json",
- "snapshot_audit_package": "uv run --frozen python scripts/package_artifact_hub.py /absolute/new-report /absolute/package.zip --snapshot-audit"
+ "snapshot_audit_package": "uv run --frozen python scripts/package_artifact_hub.py /absolute/new-report /absolute/package.zip --snapshot-audit",
+ "snapshot_audit_publish": "uv run --frozen python scripts/publish_artifact_hub.py --create --profile /absolute/publication-target.json --source /absolute/new-report --archive /absolute/package.zip --receipt /absolute/receipt.json --operation-journal /absolute/pending.json --snapshot-audit --visibility private"
},
"trajectory-replay": {
"renderer": "harness/trajectory_replay/webarena_report.py",
diff --git a/docs/benchmarks.md b/docs/benchmarks.md
index 522b84b54..f466b1c9f 100644
--- a/docs/benchmarks.md
+++ b/docs/benchmarks.md
@@ -177,3 +177,8 @@ Run these Python entries through `uv run --frozen python`. The mode retains the
original routes and metric populations and labels the new report as a saved-input
audit; it does not certify new calls or replace the original release. The ordinary
packaging path rejects this evidence state without the explicit audit flag.
+
+Publish through `scripts/publish_artifact_hub.py --snapshot-audit` with a separate
+publication target, receipt and operation journal. Use `--create` for a new report
+and set visibility explicitly. A later update uses that report's own site and
+expected version. Keep the original report's target unchanged.
diff --git a/harness/web_fetch/report_publish.py b/harness/web_fetch/report_publish.py
index f18c61947..e5ef35a3a 100644
--- a/harness/web_fetch/report_publish.py
+++ b/harness/web_fetch/report_publish.py
@@ -47,6 +47,7 @@ class PublishConfig:
presentation_binding: Path | None = None
visibility: str | None = None
parts_manifest: Path | None = None
+ snapshot_audit: bool = False
def response_json(status, raw, *, operation):
@@ -107,7 +108,7 @@ def publish(config: PublishConfig, *, transport=None, package_check=None, valida
raise ValueError("Unknown publication mode")
if config.operation_namespace == 'webfetch':
from .report_package import validate_release
- validate_release(config.source, presentation_binding=config.presentation_binding)
+ validate_release(config.source, presentation_binding=config.presentation_binding,snapshot_audit=config.snapshot_audit)
if config.visibility not in {None, "private", "unlisted", "public"}:
raise ValueError("Unknown publication visibility")
if config.presentation_binding is not None:
@@ -118,9 +119,9 @@ def publish(config: PublishConfig, *, transport=None, package_check=None, valida
attachments = part_archives(config.parts_manifest)
parts_plan = json.loads(config.parts_manifest.read_text())
validation_inputs = (*validation_inputs,config.parts_manifest,*attachments.values())
- if package_check is None and (config.presentation_binding is not None or config.parts_manifest is not None):
+ if package_check is None and (config.presentation_binding is not None or config.parts_manifest is not None or config.snapshot_audit):
from functools import partial
- package_check = partial(check,presentation_binding=config.presentation_binding,external_archives=attachments)
+ package_check = partial(check,presentation_binding=config.presentation_binding,external_archives=attachments,snapshot_audit=config.snapshot_audit)
protected = {config.profile.resolve(), config.archive.resolve(),
*(Path(path).resolve() for path in validation_inputs)}
if config.receipt.resolve() in protected or config.journal.resolve() in protected:
@@ -353,6 +354,7 @@ def main(*, root: Path | None = None):
parser.add_argument("--presentation-binding", type=Path)
parser.add_argument("--parts-manifest", type=Path)
parser.add_argument("--visibility", choices=("private", "unlisted", "public"))
+ parser.add_argument("--snapshot-audit",action="store_true",help="Publish a validated saved-output score audit")
args = parser.parse_args()
result = publish(
PublishConfig(
@@ -365,7 +367,7 @@ def main(*, root: Path | None = None):
args.title,
args.base_url,
args.expected_version,
- presentation_binding=args.presentation_binding, visibility=args.visibility, parts_manifest=args.parts_manifest,
+ presentation_binding=args.presentation_binding, visibility=args.visibility, parts_manifest=args.parts_manifest,snapshot_audit=args.snapshot_audit,
)
)
print(json.dumps(result, ensure_ascii=False), flush=True)
diff --git a/tests/test_published_snapshot_report.py b/tests/test_published_snapshot_report.py
index 47ba5b193..791386cd9 100644
--- a/tests/test_published_snapshot_report.py
+++ b/tests/test_published_snapshot_report.py
@@ -104,3 +104,21 @@ def test_snapshot_boundary_rejects_unbound_evidence_and_call_changes(tmp_path,de
else:rows[0]['scoring']['artifact_view']='published_projection'
p.write_text(''.join(json.dumps(row)+'\n' for row in rows));config['inputs']['results']['sha256']=file_sha256(p);profile.write_text(json.dumps(config))
with pytest.raises(ValueError):validate_snapshot_audit(source,profile)
+
+
+def test_snapshot_package_uses_the_shared_publisher_without_certification(tmp_path):
+ from dataclasses import replace
+ from harness.web_fetch.report_package import build
+ from harness.web_fetch.report_publish import PublishConfig, publish
+ source,profile=example(tmp_path);out=tmp_path/'report'
+ build_snapshot_audit(source,out,profile=profile);archive=tmp_path/'report.zip'
+ build(out,archive,snapshot_audit=True)
+ target=tmp_path/'publication.json';target.write_text(json.dumps({'artifact_hub_base_url':'https://example.test'}))
+ config=PublishConfig(target,archive,out,tmp_path/'publication-receipt.json',tmp_path/'operation.json',mode='create',visibility='private')
+ def transport(path,data=None,content_type=None,**kwargs):
+ if path=='/api/sites':return 200,json.dumps({'slug':'new-audit','versionId':'ver_new'}).encode()
+ if path=='/api/sites/new-audit/sharing':return 200,b'{"visibility":"private"}'
+ raise AssertionError(path)
+ with pytest.raises(ValueError,match='require --snapshot-audit'):publish(config,transport=transport)
+ result=publish(replace(config,snapshot_audit=True),transport=transport)
+ assert result['visibility']=='private' and result['url']=='https://example.test/s/new-audit'
From 44e079f7525a7fa8d77a8cdb3548ca8def8c3ab6 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Wed, 21 Oct 2026 21:02:26 +0800
Subject: [PATCH 24/51] fix(webfetch): verify retained public views of
recovered scoring inputs
---
harness/web_fetch/publication_check.py | 7 ++++++-
tests/test_published_snapshot_report.py | 18 +++++++++++++-----
2 files changed, 19 insertions(+), 6 deletions(-)
diff --git a/harness/web_fetch/publication_check.py b/harness/web_fetch/publication_check.py
index e9eaaaa60..f1aa38f5b 100644
--- a/harness/web_fetch/publication_check.py
+++ b/harness/web_fetch/publication_check.py
@@ -249,7 +249,12 @@ def data(name):
expected_hash = scored_output_hash(row)
retained_view = False
if hashlib.sha256(original).hexdigest() != expected_hash:
- public = bound_public_view(source,{'path':row['artifact_path'],'sha256':expected_hash})
+ binding={'path':row['artifact_path'],'sha256':expected_hash}
+ public = bound_public_view(source,binding)
+ if public is None and snapshot_audit:
+ from .published_snapshot import read_snapshot_public_file
+ retained=read_snapshot_public_file(source,binding)
+ public=(retained,{'role':'scored_answer','published_sha256':hashlib.sha256(original).hexdigest()})
if (public is None or public[1]['role'] != 'scored_answer'
or public[1]['published_sha256'] != hashlib.sha256(original).hexdigest()):
raise ValueError("Original scored output hash differs")
diff --git a/tests/test_published_snapshot_report.py b/tests/test_published_snapshot_report.py
index 791386cd9..d910390e5 100644
--- a/tests/test_published_snapshot_report.py
+++ b/tests/test_published_snapshot_report.py
@@ -10,7 +10,7 @@
from lexbench_webfetch.scoring import score_attempt
-def example(root):
+def example(root, recover_original=False):
source=root/'source';source.mkdir()
cases=[{'id':key,'name':key,'url':'https://example.test/'+key,'track':'web','category':'web',
'required_facts':[{'id':'body','alternatives':['complete answer']}]} for key in ['complete','missing']]
@@ -32,8 +32,15 @@ def example(root):
(source/'dataset.json').write_text(json.dumps(dataset))
(source/'results.jsonl').write_text(''.join(json.dumps(row)+'\n' for row in rows))
(source/'execution.json').write_text(json.dumps({'batches':[{'timeout_seconds':5,'attempt_ids':[row['attempt_id'] for row in rows]}]}))
- (source/'publication-privacy.json').write_text(json.dumps({'files':[]}))
- result,receipt=score_published_snapshot(source,dataset)
+ privacy=[];recovered={}
+ if recover_original:
+ artifact=source/rows[0]['artifact_path'];original=root/'recovered.txt';original.write_bytes(artifact.read_bytes())
+ artifact.write_bytes(original.read_bytes()+b' [REDACTED]')
+ privacy.append({'path':rows[0]['artifact_path'],'original_sha256':file_sha256(original),
+ 'published_sha256':file_sha256(artifact),'published_bytes':artifact.stat().st_size,'scored_output':True})
+ recovered[rows[0]['artifact_path']]=original
+ (source/'publication-privacy.json').write_text(json.dumps({'files':privacy}))
+ result,receipt=score_published_snapshot(source,dataset,recovered=recovered)
manifest={'files':{p.relative_to(source).as_posix():{'sha256':file_sha256(p)} for p in source.rglob('*') if p.is_file()}}
def save(name,value):
path=root/name;path.write_text(json.dumps(value,ensure_ascii=False));return {'path':name,'sha256':file_sha256(path)}
@@ -68,8 +75,9 @@ def save(name,value):
path=root/'profile.json';path.write_text(json.dumps(profile));return source,path
-def test_saved_matrix_keeps_native_routes_and_inspectable_review(tmp_path):
- source,profile=example(tmp_path);before={p.relative_to(source):p.read_bytes() for p in source.rglob('*') if p.is_file()}
+@pytest.mark.parametrize("recover_original",[False,True])
+def test_saved_matrix_keeps_native_routes_and_inspectable_review(tmp_path,recover_original):
+ source,profile=example(tmp_path,recover_original=recover_original);before={p.relative_to(source):p.read_bytes() for p in source.rglob('*') if p.is_file()}
out=tmp_path/'report';build_snapshot_audit(source,out,profile=profile)
assert '2 saved calls' in check_snapshot_audit(out)
from harness.web_fetch.report_package import validate_release
From 0afd182ba036ab2e3c543e9253ddb50cf954aa32 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Wed, 21 Oct 2026 21:52:28 +0800
Subject: [PATCH 25/51] fix(report): present saved results as a normal report
update
---
harness/web_fetch/published_snapshot.py | 5 +++++
harness/web_fetch/report.py | 15 +++++++++------
tests/test_published_snapshot_report.py | 10 ++++++++++
3 files changed, 24 insertions(+), 6 deletions(-)
diff --git a/harness/web_fetch/published_snapshot.py b/harness/web_fetch/published_snapshot.py
index 73210e4f3..fd71702f2 100644
--- a/harness/web_fetch/published_snapshot.py
+++ b/harness/web_fetch/published_snapshot.py
@@ -226,6 +226,11 @@ def check_snapshot_audit(root):
soup = BeautifulSoup((root/relative).read_text(), 'html.parser')
if not soup.select_one('[data-evidence-state="published_snapshot_score_audit"]'):
raise ValueError('Snapshot report lost its evidence disclosure: '+relative)
+ methodology = BeautifulSoup((root/'methodology/index.html').read_text(), 'html.parser')
+ scope = methodology.select_one('[data-scoring-input-scope]')
+ expected_scope = [binding[key] for key in ['total_attempts','projected_scoring_inputs','recovered_originals','reviewed_attempts']]
+ if scope is None or any(str(value) not in scope.get_text() for value in expected_scope):
+ raise ValueError('Snapshot report lost its scoring input scope')
audit_html = BeautifulSoup((root/'audit/index.html').read_text(), 'html.parser')
from .report_explainer import check_current_browser_article
from harness.reporting.presentation import check_presentation
diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py
index bcedea9d5..ec9c2bc1a 100644
--- a/harness/web_fetch/report.py
+++ b/harness/web_fetch/report.py
@@ -861,12 +861,14 @@ def write(out,path,title,body,*,root_links=False):
identifier=Path(path).stem
reviewed=next((item for item in context.score_audit['changes'] if item['attempt_id']==identifier),None)
if reviewed:
- body+=section('评分复核','
'
_RENDER_CONTEXT.set(RenderContext(reliability_banner,include_english,score_audit))
validate_report_ids(dataset,records)
for path in ['assets/style.css','assets/report.js','references']:
@@ -2770,6 +2770,9 @@ def write_support(out,records,manifests,*,authored_interpretation=True):
from .report_components import render_acceptance_scope
if dataset.get('cases'):
methodology+=render_acceptance_scope({case['id']:case for case in dataset['cases']},records)
+ score_binding=dataset.get('comparison',{}).get('score_audit')
+ if score_binding:
+ methodology+=section('数据范围','
')
write(out,'methodology/index.html','评测方法',methodology)
downloads=' · '.join(link('../'+path,label) for path,label in [('dataset.json','评测集与参考'),('results.jsonl','全部调用记录'),('summary.json','分类汇总'),('execution.json','执行批次')]+([('implementation-provenance.json','浏览器版本与二进制来源')] if (out/'implementation-provenance.json').is_file() else [])+([('reference-revisions.json','参考修订收据')] if (out/'reference-revisions.json').is_file() else [])+([('scoring-revisions.json','评分修订收据')] if (out/'scoring-revisions.json').is_file() else [])+([('latency-corrections.json','时延纠正收据')] if (out/'latency-corrections.json').is_file() else [])+([('request-status-corrections.json','请求状态纠正收据')] if (out/'request-status-corrections.json').is_file() else []))
reproduce='
在评测仓库中执行;读取 evaluation/ 的当前实测数据,不调用供应商接口。
make setup\nmake report EVIDENCE=/absolute/path/to/evidence\nmake report-check
')
inventory=[]
for item in audit['changes']:
@@ -4394,4 +4397,4 @@ def verdict(value):return '通过' if value else '失败'
body+=section('每条变化的依据',''.join(details))
body+=section('审查范围','
'))
diff --git a/tests/test_published_snapshot_report.py b/tests/test_published_snapshot_report.py
index d910390e5..cbf652324 100644
--- a/tests/test_published_snapshot_report.py
+++ b/tests/test_published_snapshot_report.py
@@ -92,6 +92,16 @@ def test_saved_matrix_keeps_native_routes_and_inspectable_review(tmp_path,recove
for path in ['index.html','guide/index.html','datasets/index.html','analysis/index.html','attempts/chromium__complete__r001.html','attempts/chromium__missing__r001.html','en/index.html']:
assert (out/path).is_file()
assert {p.relative_to(source):p.read_bytes() for p in source.rglob('*') if p.is_file()}==before
+ from bs4 import BeautifulSoup
+ for route in ['index.html','guide/index.html','analysis/index.html','audit/index.html']:
+ page=BeautifulSoup((out/route).read_text(),'html.parser')
+ assert page.select_one('main[data-evidence-state]') is not None
+ assert not page.select('[role="status"][data-evidence-state]')
+ assert '保存输出的评分复核' not in page.get_text()
+ method=out/'methodology/index.html';original=method.read_text()
+ method.write_text(original.replace('data-scoring-input-scope','data-removed-scope'))
+ with pytest.raises(ValueError,match='lost its scoring input scope'):check_snapshot_audit(out)
+ method.write_text(original)
detail=out/'attempts/chromium__complete__r001.html';detail.write_text(detail.read_text().replace('data-score-audit','data-deleted-audit'))
with pytest.raises(ValueError,match='lost its audit explanation'):check_snapshot_audit(out)
From dfdc990f1a944c9c40beac1955243d7c0180b366 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Wed, 21 Oct 2026 21:53:59 +0800
Subject: [PATCH 26/51] fix(report): label version comparisons by measured
configuration
---
harness/web_fetch/report_explainer.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/harness/web_fetch/report_explainer.py b/harness/web_fetch/report_explainer.py
index a2d20f0b2..2e24c892b 100644
--- a/harness/web_fetch/report_explainer.py
+++ b/harness/web_fetch/report_explainer.py
@@ -804,7 +804,7 @@ def append_html(parent, markup):
for figure in figures_by_section['conclusion']:parts['conclusions'].insert(1,figure)
if previous:
both=sum(new[cid] and old[cid] for cid in cases);neither=sum(not new[cid] and not old[cid] for cid in cases)
- changes=[('两版都通过',both),('修复版变为通过',gains),('修复版变为失败',losses),('两版都失败',neither)]
+ changes=[('两种配置都通过',both),('仅'+label(primary)+'通过',gains),('仅'+label(previous)+'通过',losses),('两种配置都失败',neither)]
append_html(parts['conclusions'],'
同一批题的两版结果 · '+str(model['cases'])+' 题
'+''.join('
'+title+'
'+str(count)+'题
' for i,(title,count) in enumerate(changes))+'
同一验收标准,所有任务都保留在对照中;变化是在线调用的实际结果,不单独证明代码因果。')
chapters=[('scope','01 这次测什么',['scope']),('results','02 通过率、耗时与资源',['result-decomposition','all-latency','timing-boundary','successful-latency','resources']),('method','03 测试方法与实例',['scoring','examples']),('configuration','04 浏览器与测试配置',['interfaces']),('failures','05 失败任务与正文缺失',['body-quality','failure-summary']+['browser-'+name for name in order]),('conclusion','06 结论与适用范围',['conclusions'])]
toc=''
From ab0117b6f88c23581b082bb930a3bc881548b8ee Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Wed, 21 Oct 2026 21:57:25 +0800
Subject: [PATCH 27/51] fix(report): remove snapshot status callouts from
result summaries
---
harness/web_fetch/report.py | 10 +++++++---
tests/test_published_snapshot_report.py | 3 ++-
2 files changed, 9 insertions(+), 4 deletions(-)
diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py
index ec9c2bc1a..7ad939901 100644
--- a/harness/web_fetch/report.py
+++ b/harness/web_fetch/report.py
@@ -1974,6 +1974,9 @@ def overall_ranking(records,cases,audit=None,*,diagnostic=False,call_budget_seco
if diagnostic:
intro='
'))
overall_title=('任务判定汇总(非排名)' if diagnostic else model_text('本轮任务通过率排名','Task pass-rate ranking for this sample') if local_pass_rate_ranking(dataset) else '综合通过率总榜')
+ if score_audit:overall_title='任务通过情况'
budgets={m['timeout_seconds'] for m in manifests if m.get('timeout_seconds') is not None}
home+=section(overall_title,overall_ranking(records,cases,audit,diagnostic=diagnostic,call_budget_seconds=next(iter(budgets)) if len(budgets)==1 else None),'overall-ranking')
- if diagnostic:
+ if diagnostic and not score_audit:
home+=result_ledger_section(out,records)
low=[r for r in records if not r['metrics'].get('evaluation_excluded') and succeeded(r,cases[r['case_id']]) and cases[r['case_id']].get('ground_truth_main_content') and r['metrics'].get('word_precision') is not None and r['metrics']['word_precision']<.3]
home+=section('通过结果的内容诊断',f'
')
+ input_rows=[]
+ for name in sorted({row['provider'] for row in records}):
+ selected=[row for row in records if row['provider']==name]
+ counts=Counter(row.get('scoring',{}).get('artifact_view','original') for row in selected)
+ input_rows.append(('',cell(esc(provider_label(name)))+cell(str(counts['original']))+cell(str(counts['recovered_original']))+cell(str(counts['published_projection']))+cell(str(len(selected)))))
+ methodology+=section('评分输入来源',table(['方法','原始输出','按哈希恢复','公开脱敏副本','合计'],input_rows,metrics=[]))
+ revision=dataset.get('definition_revision')
+ if revision:
+ methodology+=section('计分定义','
')
write(out,'methodology/index.html','评测方法',methodology)
downloads=' · '.join(link('../'+path,label) for path,label in [('dataset.json','评测集与参考'),('results.jsonl','全部调用记录'),('summary.json','分类汇总'),('execution.json','执行批次')]+([('implementation-provenance.json','浏览器版本与二进制来源')] if (out/'implementation-provenance.json').is_file() else [])+([('reference-revisions.json','参考修订收据')] if (out/'reference-revisions.json').is_file() else [])+([('scoring-revisions.json','评分修订收据')] if (out/'scoring-revisions.json').is_file() else [])+([('latency-corrections.json','时延纠正收据')] if (out/'latency-corrections.json').is_file() else [])+([('request-status-corrections.json','请求状态纠正收据')] if (out/'request-status-corrections.json').is_file() else []))
reproduce='
在评测仓库中执行;读取 evaluation/ 的当前实测数据,不调用供应商接口。
make setup\nmake report EVIDENCE=/absolute/path/to/evidence\nmake report-check
'+table(['方法','账号要求','官方获取入口','变量名称','本轮配置'],access_rows,cls='accounts-table',metrics=[]))
@@ -3628,7 +3636,7 @@ def evidence_links(urls):
for name in names:
example=next((r for r in records if r['provider']==name and r['ok']),next(r for r in records if r['provider']==name))
entry,input_route,output_route=routes[name]
- official=PROFILES[name][2]
+ official=browser_method_registry().get(name,{}).get('source_url',PROFILES.get(name,PROFILES[native_browser_engine(name) or name])[2])
interface_rows.append(('',cell(link('../providers/'+name+'.html',labels[name]))+cell(esc(entry))+cell(esc(input_route))+cell(esc(output_route))+cell(link('../attempts/'+example['attempt_id']+'.html',h('查看实际输入、返回和逐项评分','Actual input, output and component scores')))+cell(link(official,h('官方获取入口','Official source')))))
adoption+=section(h('本轮输入、输出与实例','Run inputs, outputs and examples'),'
'+h('这里列的是公开集冻结 HTML 的实际路线,不是原 URL 的实时抓取接口。实例链接对应存档调用;账号字段只展示变量名和占位符。','These are the frozen-HTML routes used for this public benchmark, not live fetching of original URLs. Example links open archived calls; account fields show variable names or placeholders only.')+'
'+table([h('方法','Method'),h('接口 / 程序','Interface / program'),h('本轮输入','Run input'),h('本轮输出','Run output'),h('真实调用实例','Archived call example'),h('官方来源','Official source')],interface_rows,cls='wide-prose-table',metrics=[]))
if research:
@@ -3677,7 +3685,7 @@ def evidence_links(urls):
account_rows=[]
for name in names:
- access=HEADLESS_ACCESS[name]
+ access=method_access(name,records)
account_rows.append(('',cell(labels[name])+cell(esc(access['account']))+cell(' '.join(''+esc(v)+'' for v in access['env_vars']))+cell(' '.join(link(url,label) for label,url in access['links']))))
accounts=hero(h('账号与安装','Accounts and setup'),h('只有 Kitesurf 的 Cloudflare 路线需要账号凭据;报告不公开值。','Only the Kitesurf Cloudflare route needs account credentials; values are never published.'))
accounts+=section(h('配置','Configuration'),table([h('方法','Method'),h('账号','Account'),h('变量','Variables'),h('官方入口','Official source')],account_rows,cls='accounts-table',metrics=[]))
@@ -4374,10 +4382,10 @@ def render_score_audit(out, audit, binding):
def verdict(value):return '通过' if value else '失败'
rows=[]
for item in audit['providers']:
- rows.append((' data-audit-summary-provider="'+esc(item['provider'])+'"',cell(esc(provider_label(item['provider'])))+cell(str(item['published_passes']))+cell(str(item['historical_passes']))+cell(str(item['initial_rescore_passes']))+cell(str(item['corrected_passes']))+cell(str(item['total']))))
+ rows.append((' data-audit-summary-provider="'+esc(item['provider'])+'"',cell(esc(provider_label(item['provider'])))+cell(str(item['published_passes']))+cell(str(item['historical_passes']))+cell(str(item['initial_rescore_passes']))+(cell(str(item['original_definition_passes'])) if 'original_definition_passes' in item else '')+cell(str(item['corrected_passes']))+cell(str(item['total']))))
body=hero(audit['title'],audit['summary'])
body+=section('判定变化来自哪里',''+esc(audit['figure_caption'])+'')
- body+=section('版本结果对照','
'+table(['方法','原页面汇总','按旧规则核对旧记录','修复前重算']+(['原定义+当前评分器'] if all('original_definition_passes' in item for item in audit['providers']) else [])+['修订定义+当前评分器','题数'],rows,id='score-audit-comparison',metrics=[]))
body+=section(esc(audit['focus_heading']), '
'+esc(audit['focus_summary'])+'
')
inventory=[]
for item in audit['changes']:
diff --git a/harness/web_fetch/report_i18n.py b/harness/web_fetch/report_i18n.py
index 339c16abb..b8a29586b 100644
--- a/harness/web_fetch/report_i18n.py
+++ b/harness/web_fetch/report_i18n.py
@@ -37,6 +37,16 @@
# Exact editorial translations take priority. Dynamic evidence (task names,
# references and provider output) is deliberately absent from this catalog.
TEXT = {
+ '计分定义': 'Scoring definitions',
+ '评分输入来源': 'Scoring input sources',
+ '按哈希恢复': 'Recovered by hash',
+ '公开脱敏副本': 'Public redacted copies',
+ '原定义+当前评分器': 'Original definitions + current scorer',
+ '修订定义+当前评分器': 'Revised definitions + current scorer',
+ '数据集:': 'Dataset:',
+ '定义版本:': 'Definition revision:',
+ '原始定义数据集:': 'Original definition dataset:',
+ "安装与来源": "Source and installation",
"最终网页身份未核实": "Final page identity is not verified",
"未通过:最终网页身份缺少证据,保留在计划任务分母中。": "Failed: final page identity lacks evidence; this result remains in the planned task denominator.",
"仓库内测试适配器;只用于确定性回归测试。": "Repository test adapter; used only for deterministic regression tests.",
diff --git a/harness/web_fetch/report_interfaces.py b/harness/web_fetch/report_interfaces.py
index 162c79ade..255a0c160 100644
--- a/harness/web_fetch/report_interfaces.py
+++ b/harness/web_fetch/report_interfaces.py
@@ -4,7 +4,7 @@
from html import escape
from lexbench_webfetch.paths import contained_path
from .methods import native_browser_engine, browser_method_registry, browser_method_label
-from .report_components import measurement_presentation
+from .report_components import measurement_presentation, bilingual_text
URL='https://example.com/'
HEADLESS_ACCESS={
@@ -16,12 +16,7 @@
},
'lightpanda':{'account':'不需要供应商账号或 API Key。','configuration':'从官方项目获取二进制,配置 LIGHTPANDA_BINARY;不使用托管服务凭据。','env_vars':['LIGHTPANDA_BINARY'],'links':[('官方项目与安装说明','https://github.com/lightpanda-io/browser')]},
'obscura':{'account':'不需要供应商账号或 API Key。','configuration':'从官方项目获取程序及配套 worker,配置 OBSCURA_BINARY。','env_vars':['OBSCURA_BINARY'],'links':[('官方项目与安装说明','https://github.com/h4ckf0r0day/obscura')]},
- 'moli-1.0.6':{'account':'不需要供应商账号或 API Key。','configuration':'从官方 v1.0.6 release 获取二进制,配置 MOLI_1_0_6_BINARY;发布归档与可执行文件均校验 SHA-256。','env_vars':['MOLI_1_0_6_BINARY'],'links':[('Moli 1.0.6 正式发布','https://github.com/lexmount/moli/releases/tag/v1.0.6')]},
'moli':{'account':'不需要供应商账号或 API Key。','configuration':'安装与本报告版本和 SHA-256 一致的二进制,配置 MOLI_BINARY;详见“环境与复现”。','env_vars':['MOLI_BINARY'],'links':[('官方项目与安装说明','https://github.com/lexmount/moli')]},
- 'moli-1.1.7':{'account':'不需要供应商账号或 API Key。','configuration':'从官方 v1.1.7 release 获取二进制,配置 MOLI_1_1_7_BINARY。','env_vars':['MOLI_1_1_7_BINARY'],'links':[('Moli 1.1.7 正式发布','https://github.com/lexmount/moli/releases/tag/v1.1.7')]},
- 'moli-1.1.11':{'account':'不需要供应商账号或 API Key。','configuration':'从官方 v1.1.11 release 获取二进制,配置 MOLI_1_1_11_BINARY。','env_vars':['MOLI_1_1_11_BINARY'],'links':[('Moli 1.1.11 正式发布','https://github.com/lexmount/moli/releases/tag/v1.1.11')]},
- 'moli-pr643':{'account':'不需要供应商账号或 API Key。','configuration':'从 Moli PR #643 的固定源码提交构建,配置 MOLI_PR643_BINARY;此方法是候选构建版。','env_vars':['MOLI_PR643_BINARY'],'links':[('Moli 源码 PR','https://github.com/lexmount/moli/pull/643')]},
- 'moli-1.1.3':{'account':'不需要供应商账号或 API Key。','configuration':'从官方 v1.1.3 release 获取二进制,配置 MOLI_1_1_3_BINARY;发布包与可执行文件均校验 SHA-256。','env_vars':['MOLI_1_1_3_BINARY'],'links':[('Moli 1.1.3 正式发布','https://github.com/lexmount/moli/releases/tag/v1.1.3')]},
'chromium':{'account':'不需要 Google 账号或 API Key;不加载个人登录会话。','configuration':'安装与本报告版本和 SHA-256 一致的浏览器,配置 CHROMIUM_BINARY;详见“环境与复现”。','env_vars':['CHROMIUM_BINARY'],'links':[('Chrome for Testing','https://googlechromelabs.github.io/chrome-for-testing/'),('官方浏览器安装说明','https://playwright.dev/python/docs/browsers')]},
}
def spec(entry,auth,params,fields,request,source,notes):
@@ -89,19 +84,15 @@ def spec(entry,auth,params,fields,request,source,notes):
[('stdout','UTF-8 string','浏览器原生 Markdown → content'),('native_markdown_ms','number','CLI 调用及原生序列化阶段耗时'),('browser_stderr','string','浏览器诊断;不进入正文')],
{'url':URL,'engine':engine,'timeout':45,'wait_until':wait,'dump':'markdown'},'harness/web_fetch/providers/headless_worker.py',
'网页正文、嵌入文档以及网页中的图片或视频信息都使用浏览器原生 Markdown。直接文件使用浏览器已有的原始响应接口。不在失败后切换输出方式、调用共同 DOM 脚本,或请求外部服务补正文。接口未提供 HTTP 状态或最终网址时保留空值。')
-def moli_release_spec(version):
+def moli_version_spec(version=None):
base=SPECS['moli']
return {**base,
- 'entry':f'CLI · Moli {version} 原生 Markdown / 原始文件接口',
+ 'entry':'CLI · Moli'+(' '+version if version else '')+' 原生 Markdown / 原始文件接口',
'params':[('URL','string · 必填','唯一任务输入'),('--dump markdown','网页、文档正文、结构与网页内嵌媒体任务','直接取得 Markdown;直接 PDF/DOCX 不提供可读正文时失败'),('--layout / --with-frames','动态网站正文整类;网页内嵌媒体任务使用 --with-frames','动态正文统一启用 Moli 页面布局能力,并在同一次原生 Markdown 调用中包含已加载子文档;不按题目设置 selector'),('--wait-selector / --wait-script','旧版预声明完成条件','只等待,不抽取或拼接正文'),('外层截止','seconds','外层统一为 45 秒。内部参数以冻结的调用记录为准:显式模式留出收尾和清理时间;uniform 模式沿用产品默认值')],
'fields':[('stdout','UTF-8 string / bytes','网页与文档任务为原生 Markdown;直接文件为原始字节'),('raw_stdout_path / sha256','string','保存的原始文档或媒体证据'),('file_evaluation','object','文件格式、长度和目标 SHA-256 验收')],
- 'request':{'page_or_document_body':'moli fetch --dump markdown --wait-until done URL','dynamic_body':'moli fetch --dump markdown --layout --with-frames --wait-until domstable URL','webpage_media':'moli fetch --dump markdown --with-frames --wait-until done URL','direct_file':'原始响应或附件接口;具体路线见逐次调用记录','version':version},
- 'notes':f'本方法只使用官方 {version} 发行二进制,与其他 Moli 版本分开计分。网页正文、文档正文、结构题和网页内嵌媒体题统一使用原生 Markdown;动态网站正文整类统一启用 --layout、--with-frames 和 domstable,不按题目设置 selector。网页媒体只核对这份输出中的目标地址,不下载,也不切换 HTML、JSON 或网络跟踪接口。直接 PDF/DOCX 没有交付可读正文时按失败记录;直接媒体文件验收原始字节。评测端不使用外部 PDF/DOCX 解析器补正文,也不在失败后切换引擎或输出模式。'}
-SPECS['moli-1.0.6']=moli_release_spec('1.0.6')
-SPECS['moli']=moli_release_spec('1.1.5')
-SPECS['moli-1.1.7']=moli_release_spec('1.1.7')
-SPECS['moli-1.1.3']=moli_release_spec('1.1.3')
-SPECS['moli-1.1.11']=moli_release_spec('1.1.11')
+ 'request':{'page_or_document_body':'moli fetch --dump markdown --wait-until done URL','dynamic_body':'moli fetch --dump markdown --layout --with-frames --wait-until domstable URL','webpage_media':'moli fetch --dump markdown --with-frames --wait-until done URL','direct_file':'原始响应或附件接口;具体路线见逐次调用记录',**({'version':version} if version else {})},
+ 'notes':'实际构建和 SHA-256 见环境与复现。网页正文、文档正文、结构题和网页内嵌媒体题统一使用原生 Markdown;动态网站正文整类统一启用 --layout、--with-frames 和 domstable,不按题目设置 selector。网页媒体只核对这份输出中的目标地址,不下载,也不切换 HTML、JSON 或网络跟踪接口。直接 PDF/DOCX 没有交付可读正文时按失败记录;直接媒体文件验收原始字节。评测端不使用外部 PDF/DOCX 解析器补正文,也不在失败后切换引擎或输出模式。'}
+SPECS['moli']=moli_version_spec()
SPECS['chromium']=spec(
'Playwright · Chromium → DOM HTML → node-html-markdown','本地独立浏览器;不需要 Google 账号,不加载个人会话',
[('url','string','唯一任务输入'),('wait_until','string','load'),('timeout','seconds','45 秒端到端截止'),('converter','string','node-html-markdown 2.0.0')],
@@ -115,37 +106,42 @@ def method_provenance(out):
def method_version(record, provenance=None):
- """Use executable identity, never a different engine's CDP compatibility number."""
+ """Use measured executable/service identity instead of compatibility versions."""
meta=record.get('provider_metadata') or {}
name=record.get('provider')
- if name=='kitesurf':return 'Beta · 托管后端版本未公开'
bound=(provenance or {}).get(meta.get('binary_sha256'),{})
if bound.get('version'):return bound['version']
if meta.get('reported_version'):return meta['reported_version']
+ registration=browser_method_registry().get(name,{})
+ if registration.get('version_source')=='reported':return None
value=meta.get('browser_version')
- if name in {'moli','lightpanda','obscura'} and re.fullmatch(r'\d+\.\d+\.\d+\.\d+',str(value)):
+ if (native_browser_engine(name) or name) in {'moli','lightpanda','obscura'} and re.fullmatch(r'\d+\.\d+\.\d+\.\d+',str(value)):
return None
return value or meta.get('binary_version') or meta.get('moli_version') or meta.get('library_version') or meta.get('version')
def method_access(name, records):
+ registration=browser_method_registry().get(name,{})
+ if registration.get('report_access') is not None:
+ return registration['report_access']
engine=native_browser_engine(name) or name
- access=HEADLESS_ACCESS.get(name) or HEADLESS_ACCESS.get(engine)
- if access and engine!=name:
- source=browser_method_registry()[name].get('source_url')
- return {**access,'configuration':'使用本轮绑定提交构建的二进制;提交、构建环境与 SHA-256 见环境与复现。','links':[(name+' · 源码分支',source)] if source else access['links']}
- if access is None or name!='moli':return access
- versions=set()
- for row in records:
- if row['provider']!=name:continue
- match=re.fullmatch(r'moli (\d+\.\d+\.\d+)',row.get('provider_metadata',{}).get('reported_version',''))
- if match:versions.add(match.group(1))
- if len(versions)!=1:return access
- version=next(iter(versions))
- return {**access,'links':[(f'Moli {version}',f'https://github.com/lexmount/moli/releases/tag/v{version}')]}
+ access=HEADLESS_ACCESS.get(engine)
+ if access is None:return None
+ result={**access}
+ variables=re.findall(r'\$\{([A-Z][A-Z0-9_]*)',registration.get('binary',''))
+ if variables:
+ result['env_vars']=list(dict.fromkeys(variables))
+ result['configuration']='安装与本报告版本和 SHA-256 一致的二进制;配置 '+', '.join(result['env_vars'])+'。'
+ result['configuration_en']='Install the binary matching this report’s version and SHA-256; set '+', '.join(result['env_vars'])+'.'
+ source=registration.get('source_url')
+ if source:result['links']=[('安装与来源',source)]
+ return result
def method_spec(name, record, *, timeout_seconds=None):
+ registration=browser_method_registry().get(name,{})
+ if registration.get('report_interface') is not None:
+ return registration['report_interface']
engine=native_browser_engine(name) or name
if record and record.get('provider_type')=='moli_webfetch':
base=SPECS['moli-webfetch']
@@ -169,13 +165,13 @@ def method_spec(name, record, *, timeout_seconds=None):
return {**base,'entry':entry,'params':params,'notes':base['notes']+' '+timeout_note}
base=SPECS.get(name) or SPECS[engine]
meta=(record or {}).get('provider_metadata',{})
- if name=='moli':
+ if engine=='moli':
match=re.fullmatch(r'moli (\d+\.\d+\.\d+)',meta.get('reported_version',''))
- if match:base=moli_release_spec(match.group(1))
+ if match:base=moli_version_spec(match.group(1))
else:
base={**base,'entry':'CLI · Moli','request':{k:v for k,v in base['request'].items() if k!='version'}}
return {**base,'notes':'实际版本、配置和调用路线见“环境与复现”及逐次调用记录。'}
- if name=='chromium':
+ if engine=='chromium':
request={**base['request']}
params=list(base['params'])
wait=meta.get('navigation_wait_until')
@@ -426,11 +422,10 @@ def fields(rows):return '
'+ ' · '.join(''+escape(label)+'' for label,url in access['links'])+'
实际版本和接口见运行记录;下方链接指向官方说明。
'
+ body+='
账号与获取入口
'+escape(access['account'])+'
'+(bilingual_text(access['configuration'],access['configuration_en']) if access.get('configuration_en') else escape(access['configuration']))+'
'+ ' · '.join(''+escape(label)+'' for label,url in access['links'])+'
实际版本和接口见运行记录;下方链接指向官方说明。
'
sample_label='本轮输出节选 · 统一记录' if r else '本轮未执行'
body+='
输入
'+fields(s['params'])+('
本轮保存的调用字段
' if bindings else '
调用输入示例
')+pre(s['request'])+'
输出
'+fields(s['fields'])+'
'+sample_label+'
'+pre(sample)+'
'
measured_link='该次完整输出与判定 →' if r else '未纳入本轮;参数说明为仓库登记模式,不计入本轮结果'
- from .report_components import bilingual_text
example_note='' if bindings else bilingual_text('以下接口说明与数值属于仓库登记示例,不是这次调用的请求记录。', 'The following interface descriptions and values are registered examples, not this call’s request record.')
body+='
'
return body
diff --git a/harness/web_fetch/report_methods.py b/harness/web_fetch/report_methods.py
index c80f05539..cf976ef3c 100644
--- a/harness/web_fetch/report_methods.py
+++ b/harness/web_fetch/report_methods.py
@@ -11,11 +11,7 @@
'kitesurf':('远程账号 API','Cloudflare Browser Run 账户级 Markdown 端点,指定 browser=kitesurf,直接返回 Markdown。','https://developers.cloudflare.com/browser-run/kitesurf/'),
'lightpanda':('本地浏览器 CLI','Lightpanda fetch --dump markdown,直接输出浏览器原生 Markdown。','https://github.com/lightpanda-io/browser'),
'obscura':('本地浏览器 CLI','Obscura fetch --dump markdown;标准模式,不启用 stealth 或备用引擎。','https://github.com/h4ckf0r0day/obscura'),
- 'moli-1.0.6':('本地浏览器 CLI','普通网页使用 fetch --dump markdown;直接文档和媒体使用裸 fetch。原始文档的本地正文解析计入耗时,文件任务直接校验字节。','https://github.com/lexmount/moli/releases/tag/v1.0.6'),
'moli':('本地浏览器 CLI','普通网页使用 fetch --dump markdown;直接文档和媒体使用裸 fetch。原始文档的本地正文解析计入耗时,文件任务直接校验字节。','https://github.com/lexmount/moli'),
- 'moli-1.1.7':('本地浏览器 CLI','普通网页使用 fetch --dump markdown;直接文档和媒体使用裸 fetch。原始文档的本地正文解析计入耗时,文件任务直接校验字节。','https://github.com/lexmount/moli/releases/tag/v1.1.7'),
- 'moli-1.1.11':('本地浏览器 CLI','普通网页使用 fetch --dump markdown;本轮风险网站评测固定 debug 日志并逐次完整留存。','https://github.com/lexmount/moli/releases/tag/v1.1.11'),
- 'moli-1.1.3':('本地浏览器 CLI','普通网页使用 fetch --dump markdown;直接文档和媒体使用裸 fetch。原始文档的本地正文解析计入耗时,文件任务直接校验字节。','https://github.com/lexmount/moli/releases/tag/v1.1.3'),
'chromium':('本地浏览器 CDP','Playwright 启动 Chromium,取完整渲染 DOM,再由 node-html-markdown 2.0.0 生成 Markdown。','https://playwright.dev/python/docs/browsers'),
'moli-webfetch':('本地 CLI','Moli:网页正文、文档正文、结构和网页内嵌媒体统一使用原生 Markdown;直接媒体文件使用原始响应。完成条件只控制官方等待参数;不调用模型。','https://github.com/lexmount/moli/tree/main/skills/moli-webfetch'),
'cowagent-built-in':('项目内置','CowAgent 内的 HTTP 抓取和文本、文档解析工具。','https://github.com/zhayujie/CowAgent/blob/main/agent/tools/web_fetch/web_fetch.py'),
@@ -30,8 +26,7 @@
'langchain-webbase':('本地开源库','WebBaseLoader 使用 HTTP 与 BeautifulSoup 取得网页文字。','https://reference.langchain.com/python/langchain-community/document_loaders/web_base/WebBaseLoader'),
'crawl4ai':('本地开源库','本地浏览器渲染,再抽取 Markdown、媒体及链接信息。','https://docs.crawl4ai.com/'),
}
-DISPLAY_NAMES={'moli-1.0.6':'Moli 1.0.6','moli':'Moli','moli-1.1.3':'Moli 1.1.3','moli-1.1.7':'Moli 1.1.7','moli-1.1.11':'Moli 1.1.11'}
-def display_name(name):return measured_method_label(name,DISPLAY_NAMES.get(name,browser_method_label(name)))
+def display_name(name):return measured_method_label(name,browser_method_label(name))
def adoption_counts(data,names):
result=[]
diff --git a/pyproject.toml b/pyproject.toml
index 167137f93..49d3da730 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -6,7 +6,7 @@ readme = "README.md"
license = {file = "LICENSE"}
requires-python = ">=3.11"
dependencies = [
- "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@4d73ada2fa1305fab61f0c9d222f7a315361281c",
+ "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@27fb1908e297e431c913124842badbbd3b254f5f",
"beautifulsoup4>=4.12,<5",
"fastcdc==1.7.0",
]
diff --git a/tests/test_headless_access.py b/tests/test_headless_access.py
index 2ee994183..8ede1558a 100644
--- a/tests/test_headless_access.py
+++ b/tests/test_headless_access.py
@@ -8,8 +8,10 @@
class HeadlessAccessTests(unittest.TestCase):
def test_all_measured_engines_have_account_and_official_sources(self):
- self.assertEqual(set(HEADLESS_ACCESS),
- {'kitesurf','lightpanda','obscura','moli-1.0.6','moli','moli-1.1.3','moli-1.1.7','moli-1.1.11','moli-pr643','chromium'})
+ from harness.web_fetch.methods import browser_method_registry
+ from harness.web_fetch.report_interfaces import method_access
+ for name in browser_method_registry():
+ self.assertTrue(method_access(name, [])['links'])
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
(root / 'adoption').mkdir()
@@ -56,7 +58,8 @@ def test_executable_version_is_not_replaced_by_cdp_compatibility_version(self):
row['provider_metadata']['reported_version']=version
self.assertEqual(method_version(row),version)
self.assertEqual(method_version({'provider':'chromium','provider_metadata':{'browser_version':'155.0.8059.39'}}),'155.0.8059.39')
- self.assertEqual(method_version({'provider':'kitesurf','provider_metadata':{'browser_version':'145.0.0.0'}}),'Beta · 托管后端版本未公开')
+ self.assertIsNone(method_version({'provider':'kitesurf','provider_metadata':{'browser_version':'145.0.0.0'}}))
+ self.assertEqual(method_version({'provider':'kitesurf','provider_metadata':{'reported_version':'service-build-9'}}),'service-build-9')
if __name__ == '__main__':
diff --git a/tests/test_published_snapshot_report.py b/tests/test_published_snapshot_report.py
index 40b0068f9..9ae8ebca1 100644
--- a/tests/test_published_snapshot_report.py
+++ b/tests/test_published_snapshot_report.py
@@ -141,3 +141,35 @@ def transport(path,data=None,content_type=None,**kwargs):
with pytest.raises(ValueError,match='require --snapshot-audit'):publish(config,transport=transport)
result=publish(replace(config,snapshot_audit=True),transport=transport)
assert result['visibility']=='private' and result['url']=='https://example.test/s/new-audit'
+
+
+@pytest.mark.parametrize('defect',[None,'same-id','different-rule','different-parent'])
+def test_annotation_version_is_distinct_and_matches_report_rules(tmp_path,defect):
+ source,profile=example(tmp_path);config=json.loads(profile.read_text())
+ dataset_path=tmp_path/config['inputs']['dataset']['path'];dataset=json.loads(dataset_path.read_text())
+ parent=dataset['id'];dataset['id']=parent+'--annotation-v2'
+ dataset['cases'][0]['diagnostic_any']=['source terminology']
+ revision={'id':'annotation-v2','changes':[{'case_id':'complete','set':{'diagnostic_any':['source terminology']},'remove':[]}]}
+ revision_path=tmp_path/'annotation.json';revision_path.write_text(json.dumps(revision))
+ config['inputs']['annotation_revision']={'path':revision_path.name,'sha256':file_sha256(revision_path)}
+ dataset['definition_revision']={'id':'annotation-v2','path':'audit/annotation-definition.json',
+ 'sha256':file_sha256(revision_path),'parent_dataset_id':parent,
+ 'parent_dataset_sha256':file_sha256(source/'dataset.json')}
+ if defect=='same-id':dataset['id']=parent
+ elif defect=='different-rule':dataset['cases'][0]['diagnostic_any']=['unbound terminology']
+ elif defect=='different-parent':dataset['definition_revision']['parent_dataset_sha256']='0'*64
+ dataset_path.write_text(json.dumps(dataset));config['inputs']['dataset']['sha256']=file_sha256(dataset_path)
+ receipt=tmp_path/config['inputs']['receipt']['path'];data=json.loads(receipt.read_text())
+ data['corrected_dataset_sha256']=canonical_json_sha256(dataset);receipt.write_text(json.dumps(data));config['inputs']['receipt']['sha256']=file_sha256(receipt)
+ article=tmp_path/config['inputs']['interpretation']['path'];data=json.loads(article.read_text())
+ rows=[json.loads(line) for line in (tmp_path/config['inputs']['results']['path']).read_text().splitlines()]
+ data['evidence_sha256']=canonical_json_sha256(_headless_article_input({c['id']:c for c in dataset['cases']},rows,['chromium']))
+ article.write_text(json.dumps(data));config['inputs']['interpretation']['sha256']=file_sha256(article);profile.write_text(json.dumps(config))
+ output=tmp_path/'versioned-report'
+ if defect:
+ with pytest.raises(ValueError,match='Annotation revision|annotation version'):build_snapshot_audit(source,output,profile=profile)
+ assert not output.exists()
+ else:
+ build_snapshot_audit(source,output,profile=profile)
+ assert '2 saved calls' in check_snapshot_audit(output)
+ assert (output/'audit/annotation-definition.json').read_bytes()==revision_path.read_bytes()
diff --git a/tests/test_report_capabilities.py b/tests/test_report_capabilities.py
index c22fb086a..e8326d8c9 100644
--- a/tests/test_report_capabilities.py
+++ b/tests/test_report_capabilities.py
@@ -52,7 +52,7 @@ def test_source_backed_capabilities_link_to_current_pinned_revisions(self):
self.assertTrue(SOURCES['moli-release'][1].endswith('/releases/tag/v1.1.4'))
def test_all_evaluated_interfaces_have_scoped_evidence(self):
- self.assertEqual(set(PROFILES), set(CAPABILITIES))
+ self.assertTrue(set(PROFILES) <= set(CAPABILITIES))
for name, row in CAPABILITIES.items():
self.assertTrue(row['scope'], name)
self.assertTrue(row['tested'], name)
diff --git a/tests/test_report_methods.py b/tests/test_report_methods.py
index 4f9d11bb9..ad7a65633 100644
--- a/tests/test_report_methods.py
+++ b/tests/test_report_methods.py
@@ -216,3 +216,35 @@ def render(out,dataset,records):
(tmp_path/'dataset.json').write_text(json.dumps(dataset))
assert render(tmp_path,{'external-case':{'id':'external-case'}},[]) == 'rendered'
assert native_browser_engine('external-build') is None
+
+
+def test_new_participant_interface_and_access_are_bound_data(tmp_path):
+ import json
+ from harness.web_fetch.methods import with_measured_method_labels
+ from harness.web_fetch.report_interfaces import render_interfaces
+ (tmp_path/'answer.txt').write_text('A complete saved answer')
+ (tmp_path/'adoption').mkdir()
+ contract={'entry':'Future CLI v9 fetch URL','auth':'No account',
+ 'params':[['url','string','Target page']],
+ 'fields':[['body','string','Returned text']],
+ 'request':{'url':'https://example.test/'},
+ 'source':'harness/web_fetch/providers/headless_worker.py',
+ 'notes':'Explicit instance contract.'}
+ access={'account':'No account','configuration':'Set FUTURE_BINARY.',
+ 'env_vars':['FUTURE_BINARY'],'links':[['Source','https://example.test/install']]}
+ dataset={'comparison':{'method_registry':{'future-version':{
+ 'engine':'moli','label':'Future participant','report_interface':contract,
+ 'report_access':access}}}}
+ rows=[{'provider':'future-version','provider_type':'headless_markdown',
+ 'provider_metadata':{'reported_version':'future 9.0.0'},
+ 'case_id':'example','attempt_id':'future__example',
+ 'artifact_path':'answer.txt','ok':True}]
+ @with_measured_method_labels
+ def render(out, definition, records):
+ return render_interfaces(out,records)
+ html=render(tmp_path,dataset,rows)
+ output=json.loads((tmp_path/'adoption/interfaces/future-version.json').read_text())
+ assert output['interface']==contract and output['access']==access
+ assert 'Future CLI v9 fetch URL' in html
+ assert 'MOLI_BINARY' not in json.dumps(output)
+ assert '1.1.5' not in json.dumps(output)
diff --git a/tests/test_report_presentation.py b/tests/test_report_presentation.py
index 4994d44c1..fc470a1db 100644
--- a/tests/test_report_presentation.py
+++ b/tests/test_report_presentation.py
@@ -1,9 +1,8 @@
import tempfile
-import json
from pathlib import Path
import unittest
from harness.web_fetch.report import EXTRA,active_tracks,failure_category,filter_controls,case_attrs,failure_filters,overall_ranking,overview_matched_latency,provider_configuration,review_cell,table,write
-from harness.web_fetch.report_interfaces import SPECS,HEADLESS_ACCESS
+from harness.web_fetch.report_interfaces import SPECS
from harness.web_fetch.report_style import STYLE
class ReportRevisionTests(unittest.TestCase):
@@ -97,7 +96,8 @@ def test_pass_no_failure(self):
self.assertEqual(failure_category(self.row(metrics={'content_pass':True}),{}),'—')
def test_all_interfaces(self):
from harness.web_fetch.report_methods import PROFILES
- self.assertEqual(set(SPECS),set(PROFILES))
+ from harness.web_fetch.report_interfaces import method_spec
+ for name in PROFILES:self.assertTrue(method_spec(name,None)['entry'])
self.assertTrue({'kitesurf','lightpanda','obscura','moli','chromium'} <= set(SPECS))
for s in SPECS.values():
self.assertTrue(s['params'] and s['fields'] and s['request'] and s['source'])
@@ -131,8 +131,10 @@ def test_review_label_links_records_without_claiming_fresh_verification(self):
self.assertIn('1/1 · 复核记录',row)
self.assertNotIn('已核验',row)
def test_headless_access_covers_every_method_and_lists_setup_variables(self):
- self.assertEqual(set(HEADLESS_ACCESS),{'kitesurf','lightpanda','obscura','moli-1.0.6','moli','moli-1.1.3','moli-1.1.7','moli-1.1.11','moli-pr643','chromium'})
- for access in HEADLESS_ACCESS.values():
+ from harness.web_fetch.report_interfaces import method_access
+ from harness.web_fetch.methods import browser_method_registry
+ for name in browser_method_registry():
+ access=method_access(name,[])
self.assertTrue(access['env_vars'])
self.assertTrue(access['links'])
def test_language_toggle_uses_one_shared_report_tree(self):
@@ -320,13 +322,15 @@ def render(self, records, name='moli'):
def test_generic_moli_uses_observed_release_and_task_adapter(self):
record={'provider':'moli','provider_type':'moli_webfetch','provider_metadata':{'reported_version':'moli 1.1.15'}}
output=self.render([record]);access=output['access']
- self.assertIn('/tag/v1.1.15',access['links'][0][1])
+ self.assertIn('github.com/lexmount/moli',access['links'][0][1])
+ self.assertNotIn('/tag/',access['links'][0][1])
self.assertNotIn('1.1.5',str(access))
self.assertEqual(output['interface']['source'],'harness/web_fetch/providers/moli.py')
old={'provider':'moli-1.0.6','provider_type':'headless_markdown','provider_metadata':{'reported_version':'moli 1.0.6'}}
output=self.render([old],'moli-1.0.6')
- self.assertIn('1.0.6',output['access']['links'][0][1])
- self.assertEqual(output['interface'],json.loads(json.dumps(SPECS['moli-1.0.6'])))
+ self.assertEqual(output['access']['env_vars'],['MOLI_1_0_6_BINARY'])
+ self.assertIn('1.0.6',output['interface']['entry'])
+ self.assertNotIn('1.1.5',str(output['interface']))
def test_unknown_or_mixed_generic_versions_do_not_invent_a_release(self):
rows=[{'provider':'moli','provider_metadata':{'reported_version':'moli '+v}} for v in ('1.1.14','1.1.15')]
for records in ([],rows):
diff --git a/uv.lock b/uv.lock
index ef79d41c7..9fbcc12b0 100644
--- a/uv.lock
+++ b/uv.lock
@@ -354,7 +354,7 @@ requires-dist = [
{ name = "jieba", marker = "extra == 'webmainbench-eval'", specifier = "==0.42.1" },
{ name = "json-repair", marker = "extra == 'browser-use-eval'", specifier = ">=0.54,<1" },
{ name = "langchain-community", marker = "extra == 'oss-eval'", specifier = "==0.3.30" },
- { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=4d73ada2fa1305fab61f0c9d222f7a315361281c" },
+ { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=27fb1908e297e431c913124842badbbd3b254f5f" },
{ name = "markdown", marker = "extra == 'webmainbench-eval'", specifier = ">=3.7,<4" },
{ name = "openai", marker = "extra == 'webmainbench-eval'", specifier = ">=2.8,<4" },
{ name = "openpyxl", marker = "extra == 'oss-eval'", specifier = "==3.1.5" },
@@ -1933,7 +1933,7 @@ wheels = [
[[package]]
name = "lexbench-webfetch"
version = "0.3.0"
-source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=4d73ada2fa1305fab61f0c9d222f7a315361281c#4d73ada2fa1305fab61f0c9d222f7a315361281c" }
+source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=27fb1908e297e431c913124842badbbd3b254f5f#27fb1908e297e431c913124842badbbd3b254f5f" }
dependencies = [
{ name = "beautifulsoup4" },
{ name = "soupsieve" },
From 6428f56352126ec498f6aab0be8dfd16a662b4fb Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Thu, 22 Oct 2026 11:24:38 +0800
Subject: [PATCH 29/51] fix(report): sort comparisons and highlight unique
winners
---
harness/web_fetch/report.py | 20 +++++++----------
harness/web_fetch/report_check.py | 2 +-
harness/web_fetch/report_i18n.py | 2 +-
tests/test_local_diagnostic_presentation.py | 9 ++++----
tests/test_report_presentation.py | 24 +++++++++++++++++++++
5 files changed, 39 insertions(+), 18 deletions(-)
diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py
index e135550cb..e80cd04b5 100644
--- a/harness/web_fetch/report.py
+++ b/harness/web_fetch/report.py
@@ -763,10 +763,7 @@ def pct(n):return '—' if n is None else f'{n*100:.1f}%'
def ms(n):return '—' if n is None else f'{n:,.0f} ms'
def link(url,label,english=None):return f'{model_text(label,english)}'
def cell(value,sort=None):return '
{value}
'
-EXTRA+='''
-table.ranking:not([data-row-index="false"]) tbody td:nth-child(4),
-table.ranking[data-row-index="false"] tbody td:nth-child(3){color:var(--good);font-weight:800}
-'''
+
def model_text(value,english=None):
return esc(value) if english is None else ''+esc(value)+''
def rate(a,b):return pct(a/b) if b else '—'
@@ -1742,7 +1739,6 @@ def ranking(records,cases,track,base='',id=None,compact=False,audit=None,*,diagn
if report_family(cases[r['case_id']])==track:groups[r['provider']].append(r)
def sort_key(item):
name,rs=item
- if diagnostic:return name
n=sum(measured(r,cases[r['case_id']]) for r in rs)
return (-sum(succeeded(r,cases[r['case_id']]) for r in rs)/n if n else 1,name)
rows=[];text_scopes={}
@@ -1808,7 +1804,7 @@ def avg(key):
directions={i:('quality_'+str(i),'max') for i in range(1,len(heads)) if heads[i] not in {'均值','P50','P95','判分完整性','审查结果'}}
directions.update({i:('latency_'+str(i),'min') for i,h in enumerate(heads) if h in {'均值','P50','P95'}})
rows=comparative_rows(rows,directions)
- result=table(heads,rows,id or 'ranking-'+track,True,'ranking',metrics=[] if compact else None,include_index=not diagnostic)
+ result=table(heads,rows,id or 'ranking-'+track,True,'ranking',metrics=[] if compact else None,include_index=not diagnostic,default_sort=(2+int(not diagnostic),'desc'))
if text_scopes and len(set(text_scopes.values()))>1:
result='文字核对覆盖范围
'+''.join('
'+esc(name)+f':{len(scope)}/{len(selected)} 例;'+esc('、'.join(selected[k]['name'] for k in sorted(scope)) or '无文字核对记录')+'
' for name,scope in sorted(text_scopes.items()))+'
'+result
if compact:
@@ -1816,7 +1812,7 @@ def avg(key):
if title_scope:result+=title_diagnostics(groups,selected,base,id or 'ranking-'+track)
result+=matched_latency(groups,cases,base)
from .report_components import task_pass_guide
- return task_pass_guide(track,base,selected)+'
'+note+'
'+(result if diagnostic else result.replace('
'+note+''+result
def matched_latency(groups,cases,base):
scopes=[{r['case_id'] for r in rows if succeeded(r,cases[r['case_id']])} for rows in groups.values()]
@@ -1944,8 +1940,8 @@ def overall_ranking(records,cases,audit=None,*,diagnostic=False,call_budget_seco
values+=cell(ms(percentile(times,.5)),percentile(times,.5))
values+=cell(ms(percentile(times,.95)),percentile(times,.95))
if audit:values+=review_cell(audit,rs)
- rows.append((passed,name,('',values)))
- rows=[row for _,_,row in sorted(rows,key=lambda x:x[1] if diagnostic else (-x[0],x[1]))]
+ rows.append((passed/n if n else -1,name,('',values)))
+ rows=[row for _,_,row in sorted(rows,key=lambda x:(-x[0],x[1]))]
tracks=active_tracks(cases)
heads=['方法','请求成功率','综合任务通过率']+(['无法判定'] if has_unknown else [])+[label for _,label in tracks]+['总耗时均值','总耗时 P50','总耗时 P95']
if audit:heads+=['审查结果']
@@ -1976,8 +1972,8 @@ def overall_ranking(records,cases,audit=None,*,diagnostic=False,call_budget_seco
if _RENDER_CONTEXT.get().score_audit:
intro=''
help=help.replace('诊断指标说明','指标说明')
- rendered=table(heads,rows,'ranking-overall',True,'ranking',metrics=[],include_index=not diagnostic)
- return intro+timing_note+help+(rendered if diagnostic else rendered.replace('
任务通过率、内容质量和耗时分别展示。' if score_audit else '
方法按名称排列,不提供名次。任务判定、内容质量和耗时分别展示。
' if diagnostic else '
分榜用于判断具体能力。默认按任务通过率排序,点击列名可切换;质量分数与通过率分别计算。
')+tabs('home-rankings',panels),'results')
+ home+=section('按任务类别查看结果' if score_audit else '按任务类别查看诊断结果' if diagnostic else f'{track_label} 类分榜',('
任务通过率、内容质量和耗时分别展示。
' if score_audit else '
默认按任务通过率降序排列,不提供名次。任务判定、内容质量和耗时分别展示。
' if diagnostic else '
分榜用于判断具体能力。默认按任务通过率排序,点击列名可切换;质量分数与通过率分别计算。
')+tabs('home-rankings',panels),'results')
write(out,'index.html','评测报告',home+'')
controls=filter_controls([('track','类型',tracks),('category','类别',[(x,x) for x in dict.fromkeys(c['category'] for c in cases.values())])])
rows=[]
diff --git a/harness/web_fetch/report_check.py b/harness/web_fetch/report_check.py
index 0659243f7..1d17b8507 100644
--- a/harness/web_fetch/report_check.py
+++ b/harness/web_fetch/report_check.py
@@ -296,7 +296,7 @@ def check_local_diagnostic_presentation(root: Path, *, ranked=False):
if ranked and 'ranking' not in disclosure.visible:
raise ValueError('Sample ranking scope missing: ' + str(path.relative_to(root)))
for table in re.findall(r'
]*\bid="ranking-[^"]+".*?
', page, re.S):
- if not ranked and ('data-dir="desc"' in table or re.search(r'
\s*(?:序号|Rank|No\.)\s*
', table)):
+ if not ranked and re.search(r'
\s*(?:序号|Rank|No\.)\s*
', table):
raise ValueError('Uncertified diagnostic comparison assigns ranks: ' + str(path.relative_to(root)))
diff --git a/harness/web_fetch/report_i18n.py b/harness/web_fetch/report_i18n.py
index b8a29586b..2afe758a6 100644
--- a/harness/web_fetch/report_i18n.py
+++ b/harness/web_fetch/report_i18n.py
@@ -164,7 +164,7 @@
"本地诊断结果,不提供名次。": "Local diagnostic results; no ranks are assigned.",
"任务判定汇总(非排名)": "Task judgments (not a ranking)",
"按任务类别查看诊断结果": "Diagnostic results by task family",
- "方法按名称排列,不提供名次。任务判定、内容质量和耗时分别展示。": "Methods are listed by name without ranks. Task judgments, content quality and latency are shown separately.",
+ "默认按任务通过率降序排列,不提供名次。任务判定、内容质量和耗时分别展示。": "Methods are sorted by task pass rate in descending order without ranks. Task judgments, content quality and latency are shown separately.",
"本地未认证诊断结果,仅展示保存的任务判定与耗时,不用于产品排名。通过不代表返回内容干净;请同时查看全文指标和挑战提示。": "Local uncertified diagnostic results show saved task judgments and latency, not product rankings. Passing does not establish clean content; also inspect full-text metrics and challenge indicators.",
"诊断指标说明": "Diagnostic metric definitions",
"无命中": "No matches",
diff --git a/tests/test_local_diagnostic_presentation.py b/tests/test_local_diagnostic_presentation.py
index 874325440..49e5516e9 100644
--- a/tests/test_local_diagnostic_presentation.py
+++ b/tests/test_local_diagnostic_presentation.py
@@ -9,7 +9,7 @@
from tests.test_report_body_scope import case, record
-def test_diagnostic_tables_do_not_assign_ranks_or_initial_score_order():
+def test_diagnostic_tables_sort_saved_pass_rates_without_assigning_ranks():
c = case('body', kind='body', track='web')
high = record(c)
high['provider'] = 'z-high'
@@ -19,10 +19,11 @@ def test_diagnostic_tables_do_not_assign_ranks_or_initial_score_order():
ranking([high, low], {c['id']: c}, 'web', diagnostic=True)):
soup = BeautifulSoup(rendered, 'html.parser')
table = soup.select_one('table.ranking')
- assert table.get('data-col') is None
+ assert table.get('data-col') == '2'
+ assert table.get('data-dir') == 'desc'
assert table.select_one('th').get_text(strip=True).startswith('方法')
rows = table.select('tbody tr')
- assert ['a-low', 'z-high'] == [row.select_one('td a')['href'].split('/')[-1].removesuffix('.html') for row in rows]
+ assert ['z-high', 'a-low'] == [row.select_one('td a')['href'].split('/')[-1].removesuffix('.html') for row in rows]
assert len(rows[0].select('td')) == len(table.select('th'))
assert '0)' in table.select_one('th button')['onclick']
# Certified comparisons retain their existing score order and ordinal column.
@@ -85,7 +86,7 @@ def test_release_check_rejects_hidden_status_or_ranked_diagnostic(tmp_path):
page.write_text(prefix + banner + '