diff --git a/docs/benchmarks.md b/docs/benchmarks.md index b7b9cce74..ca442172a 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -41,7 +41,7 @@ uv run --frozen browser-harness-eval run \ --provider-config configs/providers.quickstart.json --providers http ``` -This uses the existing dataset interface. `uv sync --frozen` installs LexBench from the exact Git commit pinned in `pyproject.toml` and `uv.lock`. The pinned commit is merged into LexBench’s main branch. uv retrieves the Git source, builds a wheel and installs the Python package; Git must be available, but no sibling checkout is required. The input checker verifies frozen bytes; a formal live-site comparison still requires the [certified runner](../harness/web_fetch/README.md), independent reference validation before and after calls, and the selected track's release gates. Ordinary CLI runs are diagnostic. +This uses the existing dataset interface. `uv sync --frozen` installs LexBench from the exact Git commit pinned in `pyproject.toml` and `uv.lock`. uv retrieves the Git source, builds a wheel and installs the Python package; Git must be available, but no sibling checkout is required. The input checker verifies frozen bytes; a formal live-site comparison still requires the [certified runner](../harness/web_fetch/README.md), independent reference validation before and after calls, and the selected track's release gates. Ordinary CLI runs are diagnostic. ## Historical bindings and new revisions diff --git a/docs/metrics.md b/docs/metrics.md index b0548b578..4036c25d1 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -21,8 +21,11 @@ A returned file URL is not a delivered file. Dynamic list checks measure structu | Body precision | How much returned text belongs to that reference | | Body F1 | Harmonic mean of precision and recall | | Mean / P50 / P95 latency | In the default ranking, timing of task-passing calls only; failures still retain elapsed time in their records | +| All-call mean / P50 / P95 | The complete call population, including failed calls and timeouts; exposed as `mean_all_attempts_ms`, `all_attempts_p50_ms` and `all_attempts_p95_ms` in scorer summaries | | Failure class | Request error or unmet task requirement; open the detail for the full error | Body metrics require a complete, comparable reference. A failed call scores zero; a missing reference leaves the metric unavailable. Check each report's stated timing population before comparing it with another report. Outputs have SHA-256 hashes. Rebuilding or editing a report must preserve the saved responses and scores. Metric labels come from `report_components.py`; the report does not show a pass rate conditioned only on successful requests. + +Formal cohorts reject selected tasks whose acceptance rules are unresolved before capturing references or calling providers. A published method must use one executable fingerprint across its selected results. Compare different builds as separate complete method populations. Offline file rescoring reads the retained file bytes and applies the current file requirements while preserving the original call metadata. diff --git a/evaluation/lexbench-webfetch.json b/evaluation/lexbench-webfetch.json index 8e8d8b6b7..d13b948e5 100644 --- a/evaluation/lexbench-webfetch.json +++ b/evaluation/lexbench-webfetch.json @@ -2,7 +2,7 @@ "schema_version": 1, "kind": "external_benchmark_binding", "repository": "https://github.com/lexmount/lexbench-webfetch", - "revision": "6d8d72637bb96f4938b9a2f7c46d0c7dffd74945", + "revision": "c9183abce7cdecc377e229a6c323add1d8cdbc13", "ownership": { "benchmark": "Task populations, task model, acceptance scoring, file checks, compiler and frozen definitions", "harness": "Execution, reference lifecycle, evidence and reports" @@ -78,7 +78,7 @@ "evaluation/risk-control/remaining-70-rule-review.md": "301574dddcfddf6d30ed4d882cdd40b1ae859fdc95a0a02cf9868f55e3fd1079", "evaluation/risk-control/rule-authoring-guide.md": "74363ab502978179e5438a0b992bac7563090e41c91784e492636832dc1cd088", "evaluation/risk-control/source-inputs.json": "77c5a4b62c07d59ed57154a833e85f9e7a97bc739730f51cd4ca3a698edb9258", - "evaluation/scoring-regressions.json": "d519f86e327d9e2fd1c884670642eb31714435864e834592e2c7b65ef21ee655", + "evaluation/scoring-regressions.json": "a9271915ac3baf3c0ab88341b29ad4a7b62e5964563f168729814e1102f8c771", "evaluation/source-inputs.json": "c202abc4624811b938a837402bbaf0ecbe716e375fbca70567346d2edb3b1708" } } diff --git a/harness/web_fetch/artifacts.py b/harness/web_fetch/artifacts.py index 80f9eb792..439b69cee 100644 --- a/harness/web_fetch/artifacts.py +++ b/harness/web_fetch/artifacts.py @@ -19,6 +19,26 @@ PUBLIC_CALL_LOG_ROOT = "call-logs" +def file_scoring_metadata(case, metadata: dict, source: Path) -> dict: + """Recheck delivered file bytes against current rules without changing call evidence.""" + from lexbench_webfetch.file_acceptance import evaluate_files + + case = case if isinstance(case, dict) else vars(case) + evaluation = metadata.get("file_evaluation") + if case.get("task_kind") != "file" or not isinstance(evaluation, dict): + return metadata + delivered = evaluation.get("files", []) + payloads = [] + for item in delivered: + raw = contained_path(source, item["artifact_path"], reject_symlinks=True).read_bytes() + if hashlib.sha256(raw).hexdigest() != item.get("sha256"): + raise ValueError("Delivered file integrity failure: " + item["artifact_path"]) + payloads.append(raw) + current = evaluate_files(case, payloads) + current["files"] = [{**old, **new} for old, new in zip(delivered, current["files"])] + return {**metadata, "file_evaluation": current} + + def public_call_log_binding(value: dict[str, Any] | None) -> dict[str, Any] | None: """Normalize a public call-log binding and reject partial/unsafe claims.""" if value is None: diff --git a/harness/web_fetch/certification.py b/harness/web_fetch/certification.py index 48c411b5d..d5f3c76c6 100644 --- a/harness/web_fetch/certification.py +++ b/harness/web_fetch/certification.py @@ -188,6 +188,11 @@ def main(argv: list[str] | None = None) -> None: known = {case["id"] for case in dataset["cases"]} if set(case_ids) - known: parser.error("Unknown case IDs: " + ", ".join(sorted(set(case_ids) - known))) + unresolved = sorted(case["id"] for case in dataset["cases"] + if case["id"] in case_ids + and (case.get("rule_review") or {}).get("status") == "unresolved") + if unresolved and not args.diagnostic: + parser.error("Formal cohorts require resolved acceptance rules: " + ", ".join(unresolved)) if args.output.exists() or args.work_root.exists(): parser.error("Output and work root must both be new immutable paths") diff --git a/harness/web_fetch/reliability.py b/harness/web_fetch/reliability.py index 456edeecf..4b074d3bd 100644 --- a/harness/web_fetch/reliability.py +++ b/harness/web_fetch/reliability.py @@ -6,6 +6,17 @@ from lexbench_webfetch.metrics import task_measured +def implementation_conflicts(records: list[dict]) -> dict[str, list[str]]: + """Find method labels that combine results from distinct executable builds.""" + implementations = {} + for row in records: + fingerprint = (row.get("provider_metadata") or {}).get("binary_sha256") + if fingerprint: + implementations.setdefault(row.get("provider"), set()).add(fingerprint) + return {provider: sorted(hashes) for provider, hashes in implementations.items() + if len(hashes) > 1} + + def assess_reliability(dataset: dict, records: list[dict]) -> dict: """Require exactly one judged result for every planned provider/case pair.""" cases = {case["id"]: case for case in dataset.get("cases", [])} @@ -26,6 +37,7 @@ def assess_reliability(dataset: dict, records: list[dict]) -> dict: if row.get("case_id") in cases and not task_measured(row.get("metrics", {}), cases[row["case_id"]].get("task_kind", "body")) ) + conflicts = implementation_conflicts(records) errors = [] if not cases: errors.append("评测集没有案例") @@ -39,6 +51,8 @@ def assess_reliability(dataset: dict, records: list[dict]) -> dict: errors.append(f"存在 {len(duplicates)} 个重复或多次选入的调用") if unjudged: errors.append(f"存在 {len(unjudged)} 个无法判定通过或失败的调用") + if conflicts: + errors.append("同一方法混用了不同可执行文件: " + ", ".join(sorted(conflicts))) complete = bool(expected) and not errors return { "schema_version": 1, @@ -51,6 +65,7 @@ def assess_reliability(dataset: dict, records: list[dict]) -> dict: "extra": [{"provider": p, "case_id": c} for p, c in extra], "duplicates": [{"provider": p, "case_id": c, "count": counts[(p, c)]} for p, c in duplicates], "unjudged": [{"provider": p, "case_id": c} for p, c in unjudged], + "implementation_conflicts": conflicts, "errors": errors, "publication_policy": "all_planned_provider_case_pairs_must_exist_once_and_be_judged", } diff --git a/harness/web_fetch/report_package.py b/harness/web_fetch/report_package.py index 8a623af39..a4348250a 100644 --- a/harness/web_fetch/report_package.py +++ b/harness/web_fetch/report_package.py @@ -29,6 +29,10 @@ def _require(condition, detail): def validate_release(source: Path) -> bool: dataset_path = source / "dataset.json" dataset = json.loads(dataset_path.read_text()) if dataset_path.exists() else {} + from .reliability import implementation_conflicts + records = load_jsonl(source / "results.jsonl") + if conflicts := implementation_conflicts(records): + raise ValueError("Refusing to package mixed executable versions: " + ", ".join(sorted(conflicts))) from .reference_lifecycle import require_certifiable_evidence manifests = [] for name in ('manifest.json', 'execution.json'): @@ -41,7 +45,7 @@ def validate_release(source: Path) -> bool: if content_binding: from .report_explainer import current_browser_article_model current_browser_article_model(source, {case['id']:case for case in dataset['cases']}, - load_jsonl(source/'results.jsonl'), content_binding) + records, content_binding) sampled = dataset.get("comparison", {}).get("kind") == "webmainbench_frozen_html_to_markdown" if not sampled: reliability = json.loads((source / "reliability.json").read_text()) @@ -49,7 +53,7 @@ def validate_release(source: Path) -> bool: raise ValueError("Refusing to package an incomplete or unjudged benchmark report") if dataset.get("comparison", {}).get("require_independent_result_review"): from .result_review import require_result_review - require_result_review(source, dataset, load_jsonl(source / "results.jsonl")) + require_result_review(source, dataset, records) return False review = json.loads((source / "result-review.json").read_text()) if review.get("status") != "passed" or review.get("failures"): @@ -65,7 +69,6 @@ def validate_release(source: Path) -> bool: or set(review.get("providers", [])) != set(providers) ): raise ValueError("Report/review scope differs from the declared matrix") - records = load_jsonl(source / "results.jsonl") expected = {(provider, case) for provider in providers for case in cases} if ( len(records) != len(expected) diff --git a/harness/web_fetch/rescoring.py b/harness/web_fetch/rescoring.py index 12049cf32..064281cfc 100644 --- a/harness/web_fetch/rescoring.py +++ b/harness/web_fetch/rescoring.py @@ -8,7 +8,7 @@ ROOT = Path(__file__).resolve().parents[2] from harness.record_io import parse_jsonl -from harness.web_fetch.artifacts import verify_record_artifacts +from harness.web_fetch.artifacts import verify_record_artifacts, file_scoring_metadata from harness.web_fetch.failure_diagnosis import diagnose from lexbench_webfetch.metrics import summarize from harness.web_fetch.models import execution_id @@ -42,7 +42,7 @@ def _rescore_one(arguments): status_code=r.get("status_code"), title=r.get("title"), final_url=r.get("final_url"), - metadata=r.get("provider_metadata", {}), + metadata=file_scoring_metadata(c, r.get("provider_metadata", {}), source), previous_metrics=r["metrics"], list_reference=load_list_reference(c, source), ) diff --git a/harness/web_fetch/runner.py b/harness/web_fetch/runner.py index 9744af599..1dcfd6822 100644 --- a/harness/web_fetch/runner.py +++ b/harness/web_fetch/runner.py @@ -937,10 +937,12 @@ def rescore_evaluation( raise ValueError(f"run contains case absent from dataset: {record['case_id']}") artifact = contained_path(run_dir,record["artifact_path"]) previous_deadline = bool(record.get("metrics", {}).get("deadline_overrun")) + from .artifacts import file_scoring_metadata record["metrics"] = score_attempt( case, artifact.read_bytes(), ok=bool(record.get("ok")), title=record.get('title'), final_url=record.get('final_url'), - metadata=record.get("provider_metadata", {}), previous_metrics=record["metrics"], + metadata=file_scoring_metadata(case, record.get("provider_metadata", {}), run_dir), + previous_metrics=record["metrics"], list_reference=load_list_reference(case, dataset_path.parent), status_code=record.get('status_code'), ) diff --git a/pyproject.toml b/pyproject.toml index 1f6be06a6..4f827f2fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ readme = "README.md" license = {file = "LICENSE"} requires-python = ">=3.11" dependencies = [ - "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@6d8d72637bb96f4938b9a2f7c46d0c7dffd74945", + "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@c9183abce7cdecc377e229a6c323add1d8cdbc13", "beautifulsoup4>=4.12,<5", ] diff --git a/tests/package_smoke.py b/tests/package_smoke.py index 5d1607572..4c3002ca2 100644 --- a/tests/package_smoke.py +++ b/tests/package_smoke.py @@ -1709,7 +1709,7 @@ def main(): root / 'cohort', root / 'cohort/dataset.json', root_cause_category='answer-surface', root_cause='Synthetic installation control preserves the complete answer scoring surface.', - regression_ids=['answer-title-supplies-required-fact'], + regression_ids=['body-only-word-boundary-answer-title-supplies-required-fact'], ) assert receipt['provider_calls'] == 0 and receipt['task_decision_changes'] == [] revised = json.loads((root / 'cohort/results.jsonl').read_text()) diff --git a/tests/test_assembly_evidence.py b/tests/test_assembly_evidence.py index ec0f448f2..30f204c83 100644 --- a/tests/test_assembly_evidence.py +++ b/tests/test_assembly_evidence.py @@ -77,7 +77,7 @@ def test_canonical_dataset_promotes_an_explicit_full_review_policy(source, tmp_p (root / 'dataset.json').write_text(json.dumps(baseline)) revise(root, root / 'dataset.json', root_cause_category='answer-surface', root_cause='Synthetic retained-answer correction is checked on the complete cohort.', - regression_ids=['answer-title-supplies-required-fact']) + regression_ids=['body-only-word-boundary-answer-title-supplies-required-fact']) baseline = json.loads((root / 'dataset.json').read_text()) canonical = tmp_path / 'canonical.json' promoted = json.loads(json.dumps(baseline)) @@ -721,7 +721,7 @@ def test_scored_cohort_composition_keeps_authority_without_starting_review(sourc root, _ = source revise(root, root / 'dataset.json', root_cause_category='answer-surface', root_cause='Synthetic retained-answer correction is checked on the complete cohort.', - regression_ids=['answer-title-supplies-required-fact']) + regression_ids=['body-only-word-boundary-answer-title-supplies-required-fact']) assert valid_scoring_revision(root) original_bytes = (root / 'results.jsonl').read_bytes() original_history = (root / 'scoring-revisions.json').read_bytes() @@ -809,7 +809,7 @@ def test_scored_cohort_composition_rejects_invalid_authority_or_changed_verdict( (root / 'dataset.json').write_text(json.dumps(metadata)) revise(root, root / 'dataset.json', root_cause_category='answer-surface', root_cause='Synthetic retained-answer correction is checked on the complete cohort.', - regression_ids=['answer-title-supplies-required-fact']) + regression_ids=['body-only-word-boundary-answer-title-supplies-required-fact']) dataset = json.loads((root / 'dataset.json').read_text()) case = EvalCase(**dataset['cases'][0]) class Fixture(Provider): diff --git a/tests/test_publication_privacy.py b/tests/test_publication_privacy.py index 49b588e4a..51303613d 100644 --- a/tests/test_publication_privacy.py +++ b/tests/test_publication_privacy.py @@ -548,7 +548,7 @@ def interrupted(*args): # Reject a different cohort or an attempt/log rewrite. saved_results = (source / 'results.jsonl').read_bytes() (source / 'results.jsonl').write_bytes(b'changed results') - with self.assertRaisesRegex(ValueError, 'cannot replace measured results'): + with self.assertRaises(ValueError): module.build_incremental(source, archive_path, root / 'wrong-cohort') (source / 'results.jsonl').write_bytes(saved_results) (source / 'report-manifest.json').write_text(json.dumps({'presentation_refresh': { diff --git a/tests/test_reliability.py b/tests/test_reliability.py index 2178ee764..86e8a909b 100644 --- a/tests/test_reliability.py +++ b/tests/test_reliability.py @@ -16,6 +16,24 @@ def test_complete_plan_is_reliable(self): self.assertTrue(result['complete']) self.assertEqual(result['judged_calls'],1) + def test_replacing_failures_with_another_binary_cannot_form_one_version_score(self): + dataset = {**self.dataset, 'cases': [self.dataset['cases'][0], + {'id': 'b', 'track': 'web', 'task_kind': 'body'}]} + rows = [{**self.record, 'provider_metadata': {'binary_sha256': 'first-build'}}, + {**self.record, 'case_id': 'b', + 'provider_metadata': {'binary_sha256': 'replacement-build'}}] + with self.assertRaisesRegex(ValueError, 'different|可执行文件'): + require_reliable(dataset, rows) + rows[1]['provider_metadata']['binary_sha256'] = 'first-build' + self.assertTrue(require_reliable(dataset, rows)['complete']) + + def test_separately_named_builds_can_be_compared_on_the_complete_population(self): + dataset = {**self.dataset, 'coverage_plan': {'web': ['release', 'candidate']}} + rows = [{**self.record, 'provider': name, + 'provider_metadata': {'binary_sha256': name + '-build'}} + for name in ('release', 'candidate')] + self.assertTrue(require_reliable(dataset, rows)['complete']) + def test_missing_duplicate_and_unjudged_are_rejected(self): self.assertFalse(assess_reliability(self.dataset,[])['complete']) duplicate=assess_reliability(self.dataset,[self.record,self.record]) diff --git a/tests/test_run_certified_benchmark.py b/tests/test_run_certified_benchmark.py index d3164dbd3..d177e48b6 100644 --- a/tests/test_run_certified_benchmark.py +++ b/tests/test_run_certified_benchmark.py @@ -22,6 +22,20 @@ def stage_name(command): class CertifiedBenchmarkTests(unittest.TestCase): + def test_unresolved_selected_rules_fail_before_capture_or_provider_calls(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + dataset = root / 'dataset.json' + dataset.write_text(json.dumps({'cases': [{ + 'id': 'provisional', 'rule_review': {'status': 'unresolved'}}]})) + with patch('harness.web_fetch.certification.run') as invoked: + with self.assertRaises(SystemExit): + main(['--dataset', str(dataset), '--output', str(root / 'run'), + '--work-root', str(root / 'work'), '--providers', 'fixture', + '--case-ids', 'provisional']) + invoked.assert_not_called() + self.assertFalse((root / 'work').exists()) + def test_diagnostic_execution_cannot_be_assembled_or_rendered_as_certified(self): from harness.web_fetch import benchmark_runner as run_benchmark from harness.web_fetch import reference_freeze as freeze_headless_references diff --git a/tests/test_scoring.py b/tests/test_scoring.py index 8867032c0..b27bf7495 100644 --- a/tests/test_scoring.py +++ b/tests/test_scoring.py @@ -402,12 +402,12 @@ def test_risk_control_requires_all_declared_stable_page_text(self): self.assertTrue(complete["stable_task_pass"]) self.assertTrue(task_passed(True, complete)) - def test_source_title_is_answer_evidence_without_inflating_body_length(self): + def test_source_title_cannot_supply_required_body_facts(self): c=case(required_facts=[{'id':'title','alternatives':['Exact page title']}, {'id':'body','alternatives':['Body fact']}], min_content_chars=20) measured=score_attempt(c,b'Body fact',ok=True,title='Exact page title') - self.assertEqual(measured['fact_coverage'],1) + self.assertEqual(measured['fact_coverage'],.5) self.assertEqual(measured['content_chars'],len('Body fact')) self.assertEqual(measured['title_chars'],len('Exact page title')) self.assertFalse(measured['length_pass']) @@ -467,8 +467,9 @@ def test_dataclass_and_dict_inputs_produce_same_metrics_without_mutation(self): self.assertEqual((mapping, previous, metadata), original) self.assertEqual(first["content_sha256"], hashlib.sha256(b"Alpha\r\nBeta").hexdigest()) self.assertEqual(first["content_chars"], len("Alpha\r\nBeta")) - for key in previous: + for key in previous.keys() - {"raw_file_pass"}: self.assertEqual(first[key], previous[key]) + self.assertNotIn("raw_file_pass", first) def test_text_coverage_is_declared_groups_not_alternatives_or_word_f1(self): c = case(required_facts=[{"id": "a", "alternatives": ["Alpha", "Another Alpha"]}, @@ -707,7 +708,7 @@ def test_old_stable_and_list_decisions_cannot_survive_mode_change(self): self.assertFalse(task_passed(True, measured)) for key in old.keys() - {"raw_file_pass"}: self.assertNotIn(key, measured) - self.assertTrue(measured["raw_file_pass"]) + self.assertNotIn("raw_file_pass", measured) def test_legacy_list_verification_is_shared_not_just_shape(self): c = case(task_kind="structure", source_benchmark=legacy_benchmark()) @@ -971,7 +972,7 @@ def concurrent_score(*args, **kwargs): rescore_content.rescore(self.run, case_ids={"web"}) self.assertEqual(path.read_bytes(), concurrent) - def test_both_rescores_retain_external_file_evidence_and_original_deadline(self): + def test_both_rescores_refresh_file_verdict_and_retain_external_identity_and_deadline(self): records, _ = self.live() original = next(r for r in records if r["case_id"] == "file") original["metrics"].update(raw_file_pass=True, raw_file_sha256="independently-verified-file", @@ -980,6 +981,7 @@ def test_both_rescores_retain_external_file_evidence_and_original_deadline(self) rescored, summary = runner.rescore_evaluation(run_dir=self.run, dataset_meta=self.dataset, dataset_path=self.path, cases=self.cases) file_row = next(r for r in rescored if r["case_id"] == "file") + original["metrics"]["raw_file_pass"] = False self.assertEqual(file_row, original) self.assertEqual(summary["cases"]["file"]["task_pass_rate"], 0) rescore_content.rescore(self.run, case_ids={"file"}) @@ -988,6 +990,30 @@ def test_both_rescores_retain_external_file_evidence_and_original_deadline(self) self.assertEqual(file_row["metrics"], original["metrics"]) self.assertEqual(summarize(script_rows)["cases"]["file"]["task_pass_rate"], 0) + def test_both_rescores_check_current_file_hash_rules_against_retained_bytes(self): + raw = b'\x89PNG\r\n\x1a\n' + b'x' * 32 + self.cases = [case(id='file', task_kind='file', source_verification={ + 'mime': 'image/png', 'file_sha256': hashlib.sha256(raw).hexdigest()})] + self.dataset['cases'] = [asdict(c) for c in self.cases] + self.path.write_text(json.dumps(self.dataset)) + self.provider.fetch.side_effect = lambda c, timeout: ProviderResult( + ok=True, content='file attached', files=[{'data': raw}]) + original, _ = self.live() + self.assertTrue(original[0]['metrics']['raw_file_pass']) + revised = replace(self.cases[0], source_verification={ + 'mime': 'image/png', 'file_sha256': hashlib.sha256(raw + b'other').hexdigest()}) + self.cases = [revised] + self.dataset['cases'] = [asdict(revised)] + rescored, _ = runner.rescore_evaluation(run_dir=self.run, dataset_meta=self.dataset, + dataset_path=self.path, cases=self.cases) + self.assertFalse(rescored[0]['metrics']['raw_file_pass']) + self.assertEqual(rescored[0]['provider_metadata'], original[0]['provider_metadata']) + (self.run / 'dataset.json').write_text(json.dumps(self.dataset)) + rescore_content.rescore(self.run) + row = json.loads((self.run / 'results.jsonl').read_text().strip()) + self.assertFalse(row['metrics']['raw_file_pass']) + self.assertEqual(row['provider_metadata'], original[0]['provider_metadata']) + def test_both_rescores_verify_every_declared_evidence_before_writes(self): records, _ = self.live() self.prepare_script_references() diff --git a/tests/test_scoring_revision.py b/tests/test_scoring_revision.py index 1b01e9bb4..83894ca8b 100644 --- a/tests/test_scoring_revision.py +++ b/tests/test_scoring_revision.py @@ -66,7 +66,7 @@ def test_invalid_history_rejects_before_any_evidence_write(tmp_path: Path, histo canonical, root_cause_category="answer-surface", root_cause="Synthetic correction restores a required fact in the answer surface.", - regression_ids=["answer-title-supplies-required-fact"], + regression_ids=["body-only-word-boundary-answer-title-supplies-required-fact"], ) assert {p.name: p.read_bytes() for p in evidence.iterdir()} == before @@ -78,7 +78,7 @@ def test_valid_history_can_extend_and_retains_prior_receipts(tmp_path: Path): authority = { "root_cause_category": "answer-surface", "root_cause": "Synthetic correction restores a required fact in the answer surface.", - "regression_ids": ["answer-title-supplies-required-fact"], + "regression_ids": ["body-only-word-boundary-answer-title-supplies-required-fact"], } first = revise(evidence, canonical, **authority) assert first["task_decision_changes"] == [ @@ -131,7 +131,7 @@ def test_reader_prompt_can_follow_scoring_contract_without_changing_saved_call(t canonical, root_cause_category="answer-surface", root_cause="The reader-facing scope described obsolete answer tokens that providers never receive.", - regression_ids=["answer-title-supplies-required-fact"], + regression_ids=["body-only-word-boundary-answer-title-supplies-required-fact"], ) assert receipt["provider_calls"] == 0 @@ -160,7 +160,7 @@ def test_reader_prompt_revision_cannot_change_provider_url(tmp_path: Path): canonical, root_cause_category="answer-surface", root_cause="The reader-facing scope changed but the physical provider URL must remain frozen.", - regression_ids=["answer-title-supplies-required-fact"], + regression_ids=["body-only-word-boundary-answer-title-supplies-required-fact"], ) assert {path.name: path.read_bytes() for path in evidence.iterdir()} == before @@ -179,7 +179,7 @@ def test_scorer_drift_and_task_rule_change_are_rejected_atomically(tmp_path: Pat canonical, root_cause_category="answer-surface", root_cause="A shared answer-surface correction changed the scorer implementation.", - regression_ids=["answer-title-supplies-required-fact"], + regression_ids=["body-only-word-boundary-answer-title-supplies-required-fact"], ) assert {path.name: path.read_bytes() for path in evidence.iterdir()} == before @@ -194,7 +194,7 @@ def test_scorer_drift_and_task_rule_change_can_be_recorded_in_two_steps(tmp_path authority = { "root_cause_category": "answer-surface", "root_cause": "A shared answer-surface correction changed the scorer implementation.", - "regression_ids": ["answer-title-supplies-required-fact"], + "regression_ids": ["body-only-word-boundary-answer-title-supplies-required-fact"], } scorer_receipt = revise(evidence, evidence / "dataset.json", **authority) @@ -243,7 +243,7 @@ def test_scorer_only_revision_accepts_an_assembled_mixed_source_cohort(tmp_path: evidence / "dataset.json", root_cause_category="answer-surface", root_cause="An assembled cohort retains outputs scored by two older scorer builds.", - regression_ids=["answer-title-supplies-required-fact"], + regression_ids=["body-only-word-boundary-answer-title-supplies-required-fact"], ) assert receipt["input_scoring_source_sha256s"] == ["0" * 64, "1" * 64] @@ -268,7 +268,7 @@ def test_scoring_validation_preserves_complete_saved_attempt_bytes(tmp_path, fai receipt = revise(evidence, evidence / 'dataset.json', validate_only=True, root_cause_category='answer-surface', root_cause='Validate the saved scoring without rewriting a physical-call record.', - regression_ids=['answer-title-supplies-required-fact']) + regression_ids=['body-only-word-boundary-answer-title-supplies-required-fact']) assert receipt['validation_only'] is True assert receipt['task_decision_changes'] == [] assert receipt['input_results_sha256'] == receipt['output_results_sha256'] @@ -289,7 +289,7 @@ def test_scoring_validation_rejects_changed_verdict_or_requirements_atomically(t revise(evidence, canonical, validate_only=True, root_cause_category='answer-surface', root_cause='Validation rejects changed scores and requirements without writing evidence.', - regression_ids=['answer-title-supplies-required-fact']) + regression_ids=['body-only-word-boundary-answer-title-supplies-required-fact']) assert {p.name:p.read_bytes() for p in evidence.iterdir()} == before @@ -298,7 +298,7 @@ def test_report_assets_can_update_together_without_changing_scored_calls(tmp_pat evidence, canonical, _ = revision_fixture(tmp_path, b'[]') revise(evidence, canonical, root_cause_category='answer-surface', root_cause='Synthetic required-answer correction has an executable acceptance control.', - regression_ids=['answer-title-supplies-required-fact']) + regression_ids=['body-only-word-boundary-answer-title-supplies-required-fact']) calls = (evidence / 'results.jsonl').read_bytes() content = evidence / 'report-assets' / 'browser-interpretation.json' content.parent.mkdir() @@ -328,7 +328,7 @@ def test_report_asset_update_rejects_unbound_provenance_before_writing(tmp_path, evidence, canonical, _ = revision_fixture(tmp_path, b'[]') revise(evidence, canonical, root_cause_category='answer-surface', root_cause='Synthetic required-answer correction has an executable acceptance control.', - regression_ids=['answer-title-supplies-required-fact']) + regression_ids=['body-only-word-boundary-answer-title-supplies-required-fact']) content = evidence / 'article.json' content.write_text('{}') implementation = evidence / 'implementation-provenance.json' diff --git a/tests/test_three_cohort_review_followup.py b/tests/test_three_cohort_review_followup.py index 7b782a948..beb78daa2 100644 --- a/tests/test_three_cohort_review_followup.py +++ b/tests/test_three_cohort_review_followup.py @@ -145,13 +145,13 @@ def test_legacy_page_text_rules_cannot_take_evidence_from_metadata(rule): assert score_attempt(case,content.encode(),ok=True,title=title)['content_pass'] is expected -def test_explicit_title_fact_and_body_scope_are_independent_requirements(): +def test_required_body_facts_and_body_scope_cannot_be_supplied_by_title_metadata(): from lexbench_webfetch.body_coverage import body_requirements text='Required body paragraph explains independent visible source content and complete record ownership.' case={'url':'https://example.test/article','task_kind':'body', 'required_facts':[{'id':'title','alternatives':['Exact page title']}], 'body_requirements':body_requirements(text)} - for content,title,expected in [(text,'Exact page title',True), - (text,'Wrong title',False), - ('Other text','Exact page title',False)]: + for content,title,expected in [(text+'\nExact page title','Wrong title',True), + (text,'Exact page title',False), + ('Exact page title','Exact page title',False)]: assert score_attempt(case,content.encode(),ok=True,title=title)['content_pass'] is expected diff --git a/tests/test_webfetch_report_package.py b/tests/test_webfetch_report_package.py index 4d0927c7b..276f9c60e 100644 --- a/tests/test_webfetch_report_package.py +++ b/tests/test_webfetch_report_package.py @@ -120,6 +120,18 @@ def test_declared_small_matrix_packages_through_public_command(tmp_path): ) +def test_stale_complete_flag_cannot_publish_mixed_executable_versions(tmp_path): + source, archive = report_tree(tmp_path) + rows = [{'attempt_id': 'native-' + case, 'provider': 'native', 'case_id': case, + 'provider_metadata': {'binary_sha256': fingerprint}} + for case, fingerprint in [('alpha', 'original-build'), ('beta', 'new-build')]] + (source / 'results.jsonl').write_text(''.join(json.dumps(row) + '\n' for row in rows)) + archive.write_bytes(b'previous verified release') + with pytest.raises(ValueError, match='mixed executable versions'): + report_package.build(source, archive) + assert archive.read_bytes() == b'previous verified release' + + @pytest.mark.parametrize( "damage", [ diff --git a/uv.lock b/uv.lock index c364e7e56..d79461680 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 5 +revision = 2 requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.13'", @@ -352,7 +352,7 @@ requires-dist = [ { name = "jieba", marker = "extra == 'webmainbench-eval'", specifier = "==0.42.1" }, { name = "json-repair", marker = "extra == 'browser-use-eval'", specifier = ">=0.54,<1" }, { name = "langchain-community", marker = "extra == 'oss-eval'", specifier = "==0.3.30" }, - { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=6d8d72637bb96f4938b9a2f7c46d0c7dffd74945" }, + { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=c9183abce7cdecc377e229a6c323add1d8cdbc13" }, { name = "markdown", marker = "extra == 'webmainbench-eval'", specifier = ">=3.7,<4" }, { name = "openai", marker = "extra == 'webmainbench-eval'", specifier = ">=2.8,<4" }, { name = "openpyxl", marker = "extra == 'oss-eval'", specifier = "==3.1.5" }, @@ -1882,8 +1882,8 @@ wheels = [ [[package]] name = "lexbench-webfetch" -version = "0.2.0" -source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=6d8d72637bb96f4938b9a2f7c46d0c7dffd74945#6d8d72637bb96f4938b9a2f7c46d0c7dffd74945" } +version = "0.3.0" +source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=c9183abce7cdecc377e229a6c323add1d8cdbc13#c9183abce7cdecc377e229a6c323add1d8cdbc13" } dependencies = [ { name = "beautifulsoup4" }, { name = "soupsieve" }, @@ -3790,7 +3790,7 @@ resolution-markers = [ "python_full_version < '3.12'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -3865,7 +3865,7 @@ resolution-markers = [ "python_full_version == '3.12.*'", ] dependencies = [ - { name = "numpy", version = "2.5.3", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.5.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7e/74/66de6258867beb2ef08f35f9f2ac017a52cacd5081714d239ff1a442d458/scipy-1.18.1.tar.gz", hash = "sha256:52c4b7422442aba924d03ad4019852b08a92e64ea187b933135687bfe2747307", size = 30781235, upload-time = "2026-08-21T23:28:50.599Z" } wheels = [