Enforce scoring and executable consistency at release boundaries

docs/benchmarks.md

@@ -41,7 +41,7 @@ uv run --frozen browser-harness-eval run \
   --provider-config configs/providers.quickstart.json --providers http
 ```
 
-This uses the existing dataset interface. `uv sync --frozen` installs LexBench from the exact Git commit pinned in `pyproject.toml` and `uv.lock`. The pinned commit is merged into LexBench’s main branch. uv retrieves the Git source, builds a wheel and installs the Python package; Git must be available, but no sibling checkout is required. The input checker verifies frozen bytes; a formal live-site comparison still requires the [certified runner](../harness/web_fetch/README.md), independent reference validation before and after calls, and the selected track's release gates. Ordinary CLI runs are diagnostic.
+This uses the existing dataset interface. `uv sync --frozen` installs LexBench from the exact Git commit pinned in `pyproject.toml` and `uv.lock`. uv retrieves the Git source, builds a wheel and installs the Python package; Git must be available, but no sibling checkout is required. The input checker verifies frozen bytes; a formal live-site comparison still requires the [certified runner](../harness/web_fetch/README.md), independent reference validation before and after calls, and the selected track's release gates. Ordinary CLI runs are diagnostic.
 
 ## Historical bindings and new revisions
 

docs/metrics.md

@@ -21,8 +21,11 @@ A returned file URL is not a delivered file. Dynamic list checks measure structu
 | Body precision | How much returned text belongs to that reference |
 | Body F1 | Harmonic mean of precision and recall |
 | Mean / P50 / P95 latency | In the default ranking, timing of task-passing calls only; failures still retain elapsed time in their records |
+| All-call mean / P50 / P95 | The complete call population, including failed calls and timeouts; exposed as `mean_all_attempts_ms`, `all_attempts_p50_ms` and `all_attempts_p95_ms` in scorer summaries |
 | Failure class | Request error or unmet task requirement; open the detail for the full error |
 
 Body metrics require a complete, comparable reference. A failed call scores zero; a missing reference leaves the metric unavailable. Check each report's stated timing population before comparing it with another report.
 
 Outputs have SHA-256 hashes. Rebuilding or editing a report must preserve the saved responses and scores. Metric labels come from `report_components.py`; the report does not show a pass rate conditioned only on successful requests.
+
+Formal cohorts reject selected tasks whose acceptance rules are unresolved before capturing references or calling providers. A published method must use one executable fingerprint across its selected results. Compare different builds as separate complete method populations. Offline file rescoring reads the retained file bytes and applies the current file requirements while preserving the original call metadata.

evaluation/lexbench-webfetch.json

@@ -2,7 +2,7 @@
   "schema_version": 1,
   "kind": "external_benchmark_binding",
   "repository": "https://github.com/lexmount/lexbench-webfetch",
-  "revision": "6d8d72637bb96f4938b9a2f7c46d0c7dffd74945",
+  "revision": "c9183abce7cdecc377e229a6c323add1d8cdbc13",
   "ownership": {
     "benchmark": "Task populations, task model, acceptance scoring, file checks, compiler and frozen definitions",
     "harness": "Execution, reference lifecycle, evidence and reports"
@@ -78,7 +78,7 @@
     "evaluation/risk-control/remaining-70-rule-review.md": "301574dddcfddf6d30ed4d882cdd40b1ae859fdc95a0a02cf9868f55e3fd1079",
     "evaluation/risk-control/rule-authoring-guide.md": "74363ab502978179e5438a0b992bac7563090e41c91784e492636832dc1cd088",
     "evaluation/risk-control/source-inputs.json": "77c5a4b62c07d59ed57154a833e85f9e7a97bc739730f51cd4ca3a698edb9258",
-    "evaluation/scoring-regressions.json": "d519f86e327d9e2fd1c884670642eb31714435864e834592e2c7b65ef21ee655",
+    "evaluation/scoring-regressions.json": "a9271915ac3baf3c0ab88341b29ad4a7b62e5964563f168729814e1102f8c771",
     "evaluation/source-inputs.json": "c202abc4624811b938a837402bbaf0ecbe716e375fbca70567346d2edb3b1708"
   }
 }

harness/web_fetch/artifacts.py

@@ -19,6 +19,26 @@
 PUBLIC_CALL_LOG_ROOT = "call-logs"
 
 
+def file_scoring_metadata(case, metadata: dict, source: Path) -> dict:
+    """Recheck delivered file bytes against current rules without changing call evidence."""
+    from lexbench_webfetch.file_acceptance import evaluate_files
+
+    case = case if isinstance(case, dict) else vars(case)
+    evaluation = metadata.get("file_evaluation")
+    if case.get("task_kind") != "file" or not isinstance(evaluation, dict):
+        return metadata
+    delivered = evaluation.get("files", [])
+    payloads = []
+    for item in delivered:
+        raw = contained_path(source, item["artifact_path"], reject_symlinks=True).read_bytes()
+        if hashlib.sha256(raw).hexdigest() != item.get("sha256"):
+            raise ValueError("Delivered file integrity failure: " + item["artifact_path"])
+        payloads.append(raw)
+    current = evaluate_files(case, payloads)
+    current["files"] = [{**old, **new} for old, new in zip(delivered, current["files"])]
+    return {**metadata, "file_evaluation": current}
+
+
 def public_call_log_binding(value: dict[str, Any] | None) -> dict[str, Any] | None:
     """Normalize a public call-log binding and reject partial/unsafe claims."""
     if value is None:

harness/web_fetch/certification.py

@@ -188,6 +188,11 @@ def main(argv: list[str] | None = None) -> None:
     known = {case["id"] for case in dataset["cases"]}
     if set(case_ids) - known:
         parser.error("Unknown case IDs: " + ", ".join(sorted(set(case_ids) - known)))
+    unresolved = sorted(case["id"] for case in dataset["cases"]
+                        if case["id"] in case_ids
+                        and (case.get("rule_review") or {}).get("status") == "unresolved")
+    if unresolved and not args.diagnostic:
+        parser.error("Formal cohorts require resolved acceptance rules: " + ", ".join(unresolved))
     if args.output.exists() or args.work_root.exists():
         parser.error("Output and work root must both be new immutable paths")
 

harness/web_fetch/reliability.py

@@ -6,6 +6,17 @@
 from lexbench_webfetch.metrics import task_measured
 
 
+def implementation_conflicts(records: list[dict]) -> dict[str, list[str]]:
+    """Find method labels that combine results from distinct executable builds."""
+    implementations = {}
+    for row in records:
+        fingerprint = (row.get("provider_metadata") or {}).get("binary_sha256")
+        if fingerprint:
+            implementations.setdefault(row.get("provider"), set()).add(fingerprint)
+    return {provider: sorted(hashes) for provider, hashes in implementations.items()
+            if len(hashes) > 1}
+
+
 def assess_reliability(dataset: dict, records: list[dict]) -> dict:
     """Require exactly one judged result for every planned provider/case pair."""
     cases = {case["id"]: case for case in dataset.get("cases", [])}
@@ -26,6 +37,7 @@ def assess_reliability(dataset: dict, records: list[dict]) -> dict:
         if row.get("case_id") in cases
         and not task_measured(row.get("metrics", {}), cases[row["case_id"]].get("task_kind", "body"))
     )
+    conflicts = implementation_conflicts(records)
     errors = []
     if not cases:
         errors.append("评测集没有案例")
@@ -39,6 +51,8 @@ def assess_reliability(dataset: dict, records: list[dict]) -> dict:
         errors.append(f"存在 {len(duplicates)} 个重复或多次选入的调用")
     if unjudged:
         errors.append(f"存在 {len(unjudged)} 个无法判定通过或失败的调用")
+    if conflicts:
+        errors.append("同一方法混用了不同可执行文件: " + ", ".join(sorted(conflicts)))
     complete = bool(expected) and not errors
     return {
         "schema_version": 1,
@@ -51,6 +65,7 @@ def assess_reliability(dataset: dict, records: list[dict]) -> dict:
         "extra": [{"provider": p, "case_id": c} for p, c in extra],
         "duplicates": [{"provider": p, "case_id": c, "count": counts[(p, c)]} for p, c in duplicates],
         "unjudged": [{"provider": p, "case_id": c} for p, c in unjudged],
+        "implementation_conflicts": conflicts,
         "errors": errors,
         "publication_policy": "all_planned_provider_case_pairs_must_exist_once_and_be_judged",
     }

harness/web_fetch/report_package.py

@@ -29,6 +29,10 @@ def _require(condition, detail):
 def validate_release(source: Path) -> bool:
     dataset_path = source / "dataset.json"
     dataset = json.loads(dataset_path.read_text()) if dataset_path.exists() else {}
+    from .reliability import implementation_conflicts
+    records = load_jsonl(source / "results.jsonl")
+    if conflicts := implementation_conflicts(records):
+        raise ValueError("Refusing to package mixed executable versions: " + ", ".join(sorted(conflicts)))
     from .reference_lifecycle import require_certifiable_evidence
     manifests = []
     for name in ('manifest.json', 'execution.json'):
@@ -41,15 +45,15 @@ def validate_release(source: Path) -> bool:
     if content_binding:
         from .report_explainer import current_browser_article_model
         current_browser_article_model(source, {case['id']:case for case in dataset['cases']},
-                                      load_jsonl(source/'results.jsonl'), content_binding)
+                                      records, content_binding)
     sampled = dataset.get("comparison", {}).get("kind") == "webmainbench_frozen_html_to_markdown"
     if not sampled:
         reliability = json.loads((source / "reliability.json").read_text())
         if reliability.get("complete") is not True:
             raise ValueError("Refusing to package an incomplete or unjudged benchmark report")
         if dataset.get("comparison", {}).get("require_independent_result_review"):
             from .result_review import require_result_review
-            require_result_review(source, dataset, load_jsonl(source / "results.jsonl"))
+            require_result_review(source, dataset, records)
         return False
     review = json.loads((source / "result-review.json").read_text())
     if review.get("status") != "passed" or review.get("failures"):
@@ -65,7 +69,6 @@ def validate_release(source: Path) -> bool:
         or set(review.get("providers", [])) != set(providers)
     ):
         raise ValueError("Report/review scope differs from the declared matrix")
-    records = load_jsonl(source / "results.jsonl")
     expected = {(provider, case) for provider in providers for case in cases}
     if (
         len(records) != len(expected)

harness/web_fetch/rescoring.py

@@ -8,7 +8,7 @@
 
 ROOT = Path(__file__).resolve().parents[2]
 from harness.record_io import parse_jsonl
-from harness.web_fetch.artifacts import verify_record_artifacts
+from harness.web_fetch.artifacts import verify_record_artifacts, file_scoring_metadata
 from harness.web_fetch.failure_diagnosis import diagnose
 from lexbench_webfetch.metrics import summarize
 from harness.web_fetch.models import execution_id
@@ -42,7 +42,7 @@ def _rescore_one(arguments):
         status_code=r.get("status_code"),
         title=r.get("title"),
         final_url=r.get("final_url"),
-        metadata=r.get("provider_metadata", {}),
+        metadata=file_scoring_metadata(c, r.get("provider_metadata", {}), source),
         previous_metrics=r["metrics"],
         list_reference=load_list_reference(c, source),
     )

harness/web_fetch/runner.py

@@ -937,10 +937,12 @@ def rescore_evaluation(
             raise ValueError(f"run contains case absent from dataset: {record['case_id']}")
         artifact = contained_path(run_dir,record["artifact_path"])
         previous_deadline = bool(record.get("metrics", {}).get("deadline_overrun"))
+        from .artifacts import file_scoring_metadata
         record["metrics"] = score_attempt(
             case, artifact.read_bytes(), ok=bool(record.get("ok")),
             title=record.get('title'), final_url=record.get('final_url'),
-            metadata=record.get("provider_metadata", {}), previous_metrics=record["metrics"],
+            metadata=file_scoring_metadata(case, record.get("provider_metadata", {}), run_dir),
+            previous_metrics=record["metrics"],
             list_reference=load_list_reference(case, dataset_path.parent),
             status_code=record.get('status_code'),
         )

pyproject.toml

@@ -6,7 +6,7 @@ readme = "README.md"
 license = {file = "LICENSE"}
 requires-python = ">=3.11"
 dependencies = [
-  "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@6d8d72637bb96f4938b9a2f7c46d0c7dffd74945",
+  "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@c9183abce7cdecc377e229a6c323add1d8cdbc13",
   "beautifulsoup4>=4.12,<5",
 ]
 

tests/package_smoke.py

@@ -1709,7 +1709,7 @@ def main():
             root / 'cohort', root / 'cohort/dataset.json',
             root_cause_category='answer-surface',
             root_cause='Synthetic installation control preserves the complete answer scoring surface.',
-            regression_ids=['answer-title-supplies-required-fact'],
+            regression_ids=['body-only-word-boundary-answer-title-supplies-required-fact'],
         )
         assert receipt['provider_calls'] == 0 and receipt['task_decision_changes'] == []
         revised = json.loads((root / 'cohort/results.jsonl').read_text())

tests/test_assembly_evidence.py

@@ -77,7 +77,7 @@ def test_canonical_dataset_promotes_an_explicit_full_review_policy(source, tmp_p
     (root / 'dataset.json').write_text(json.dumps(baseline))
     revise(root, root / 'dataset.json', root_cause_category='answer-surface',
         root_cause='Synthetic retained-answer correction is checked on the complete cohort.',
-        regression_ids=['answer-title-supplies-required-fact'])
+        regression_ids=['body-only-word-boundary-answer-title-supplies-required-fact'])
     baseline = json.loads((root / 'dataset.json').read_text())
     canonical = tmp_path / 'canonical.json'
     promoted = json.loads(json.dumps(baseline))
@@ -721,7 +721,7 @@ def test_scored_cohort_composition_keeps_authority_without_starting_review(sourc
     root, _ = source
     revise(root, root / 'dataset.json', root_cause_category='answer-surface',
         root_cause='Synthetic retained-answer correction is checked on the complete cohort.',
-        regression_ids=['answer-title-supplies-required-fact'])
+        regression_ids=['body-only-word-boundary-answer-title-supplies-required-fact'])
     assert valid_scoring_revision(root)
     original_bytes = (root / 'results.jsonl').read_bytes()
     original_history = (root / 'scoring-revisions.json').read_bytes()
@@ -809,7 +809,7 @@ def test_scored_cohort_composition_rejects_invalid_authority_or_changed_verdict(
         (root / 'dataset.json').write_text(json.dumps(metadata))
     revise(root, root / 'dataset.json', root_cause_category='answer-surface',
         root_cause='Synthetic retained-answer correction is checked on the complete cohort.',
-        regression_ids=['answer-title-supplies-required-fact'])
+        regression_ids=['body-only-word-boundary-answer-title-supplies-required-fact'])
     dataset = json.loads((root / 'dataset.json').read_text())
     case = EvalCase(**dataset['cases'][0])
     class Fixture(Provider):

tests/test_publication_privacy.py

@@ -548,7 +548,7 @@ def interrupted(*args):
             # Reject a different cohort or an attempt/log rewrite.
             saved_results = (source / 'results.jsonl').read_bytes()
             (source / 'results.jsonl').write_bytes(b'changed results')
-            with self.assertRaisesRegex(ValueError, 'cannot replace measured results'):
+            with self.assertRaises(ValueError):
                 module.build_incremental(source, archive_path, root / 'wrong-cohort')
             (source / 'results.jsonl').write_bytes(saved_results)
             (source / 'report-manifest.json').write_text(json.dumps({'presentation_refresh': {

tests/test_reliability.py

@@ -16,6 +16,24 @@ def test_complete_plan_is_reliable(self):
         self.assertTrue(result['complete'])
         self.assertEqual(result['judged_calls'],1)
 
+    def test_replacing_failures_with_another_binary_cannot_form_one_version_score(self):
+        dataset = {**self.dataset, 'cases': [self.dataset['cases'][0],
+                   {'id': 'b', 'track': 'web', 'task_kind': 'body'}]}
+        rows = [{**self.record, 'provider_metadata': {'binary_sha256': 'first-build'}},
+                {**self.record, 'case_id': 'b',
+                 'provider_metadata': {'binary_sha256': 'replacement-build'}}]
+        with self.assertRaisesRegex(ValueError, 'different|可执行文件'):
+            require_reliable(dataset, rows)
+        rows[1]['provider_metadata']['binary_sha256'] = 'first-build'
+        self.assertTrue(require_reliable(dataset, rows)['complete'])
+
+    def test_separately_named_builds_can_be_compared_on_the_complete_population(self):
+        dataset = {**self.dataset, 'coverage_plan': {'web': ['release', 'candidate']}}
+        rows = [{**self.record, 'provider': name,
+                 'provider_metadata': {'binary_sha256': name + '-build'}}
+                for name in ('release', 'candidate')]
+        self.assertTrue(require_reliable(dataset, rows)['complete'])
+
     def test_missing_duplicate_and_unjudged_are_rejected(self):
         self.assertFalse(assess_reliability(self.dataset,[])['complete'])
         duplicate=assess_reliability(self.dataset,[self.record,self.record])

tests/test_run_certified_benchmark.py

@@ -22,6 +22,20 @@ def stage_name(command):
 
 
 class CertifiedBenchmarkTests(unittest.TestCase):
+    def test_unresolved_selected_rules_fail_before_capture_or_provider_calls(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            dataset = root / 'dataset.json'
+            dataset.write_text(json.dumps({'cases': [{
+                'id': 'provisional', 'rule_review': {'status': 'unresolved'}}]}))
+            with patch('harness.web_fetch.certification.run') as invoked:
+                with self.assertRaises(SystemExit):
+                    main(['--dataset', str(dataset), '--output', str(root / 'run'),
+                          '--work-root', str(root / 'work'), '--providers', 'fixture',
+                          '--case-ids', 'provisional'])
+            invoked.assert_not_called()
+            self.assertFalse((root / 'work').exists())
+
     def test_diagnostic_execution_cannot_be_assembled_or_rendered_as_certified(self):
         from harness.web_fetch import benchmark_runner as run_benchmark
         from harness.web_fetch import reference_freeze as freeze_headless_references