Transparent menu
+
+ Template payload is not visible
Visible duplicateVisible duplicate
Translucent text
Inline formatting preserved
@@ -256,6 +258,8 @@ def test_live_multielement_completion_preserves_declared_content_scope(rendered_ if mode == "visible": assert "Inline formatting preserved" in row["content_text"] assert row["content_text"].count("Visible duplicate") == 1 + assert "Fallback code" not in row["content_text"] + assert "Template payload" not in row["content_text"] assert ("Hidden panel" in row["content_text"]) == (mode != "visible") assert ("Excluded panel" in row["content_text"]) == (mode == "dom") From 739135920681e5d40ad7f5722a2b3ca1e16e18e5 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:43:26 +0800 Subject: [PATCH 09/19] fix(webfetch): unify declared source checks and bound capture workers --- .../references/evaluation-design.md | 2 +- harness/web_fetch/README.md | 12 ++++ harness/web_fetch/browser_observations.py | 72 ++++++++++++++++++- harness/web_fetch/reference_lifecycle.py | 9 +-- harness/web_fetch/render_evidence.py | 13 +++- tests/test_browser_observation_contracts.py | 27 ++++++- tests/test_browser_reference_scope.py | 6 +- tests/test_dynamic_entry_matching.py | 32 ++++----- tests/test_dynamic_reference.py | 42 +++++++++-- tests/test_freeze_headless_references.py | 10 +-- tests/test_reference_lifecycle.py | 39 ++++++++-- tests/test_render_evidence.py | 27 +++++++ 12 files changed, 245 insertions(+), 46 deletions(-) diff --git a/.agents/skills/benchmark-report/references/evaluation-design.md b/.agents/skills/benchmark-report/references/evaluation-design.md index ea412c7ea..73e1c5a85 100644 --- a/.agents/skills/benchmark-report/references/evaluation-design.md +++ b/.agents/skills/benchmark-report/references/evaluation-design.md @@ -38,7 +38,7 @@ When comparing builds of one implementation, do not infer a code-induced latency Prefer invariant contracts for changing pages: stable labels, required fields, record shape, types, count bounds, uniqueness, and independently observed explanatory text. Do not freeze ranks, prices, member counts, or current titles unless the benchmark is explicitly a time-bound snapshot. -Ground truth must be independent of the provider under test. Retain source location, bytes or a durable observation, hash, capture method, and scope. Partial annotations support only the requirements they cover. They are not a full-body reference. +Ground truth must be independent of the provider under test. Retain source location, bytes or a durable observation, hash, capture method, and scope. Partial annotations support only the requirements they cover. They are not a full-body reference. When authoring or expanding a task, inspect its actual reference text against the reader's main content needs. A dashboard's labels do not establish its values; a feed's navigation does not establish its entries. Select applicable fact, body-region, record, pattern, resource and file checks through the shared scorer. Give the source-complete answer and a page shell or incomplete answer opposite outcomes before freezing the new version. Preserve earlier input versions. ## Machine-readable current profile diff --git a/harness/web_fetch/README.md b/harness/web_fetch/README.md index 72f91d9f8..e88ad8bdb 100644 --- a/harness/web_fetch/README.md +++ b/harness/web_fetch/README.md @@ -114,6 +114,18 @@ a changed experiment. Checkpoints and final outputs are created exclusively and cannot replace an existing observation. Store them under the configured data home, outside Git. +The batch isolates each browser worker in a child process. Its wall-clock budget +covers all declared attempts plus two seconds for startup and cleanup, including +browser transport calls that do not honor Playwright's page timeout. A stuck or +crashed worker produces a failed observation and leaves other cases intact. + +Explicit `source_verification.reference_mode: live_exact` applies to every track +and output kind, including stable contracts and mixed text/resource deliveries. +A required live reference must exist before execution. Without that declaration, +compiled references remain frozen. Rendered-reference checks apply the declared +body regions and patterns as well as facts; a fact group is optional when another +content requirement establishes the source scope. + ## WCXB scoring `wcxb_scoring.py` owns certified frozen-HTML scoring. The maintained command is diff --git a/harness/web_fetch/browser_observations.py b/harness/web_fetch/browser_observations.py index a7b211c4a..e9e510a8f 100644 --- a/harness/web_fetch/browser_observations.py +++ b/harness/web_fetch/browser_observations.py @@ -7,6 +7,9 @@ import hashlib import ipaddress import json +import os +import signal +import subprocess import socket import sys import tempfile @@ -566,6 +569,11 @@ def capture_with_retries( "message": str(error)[:1000], } ) + return _failed_observation(case, observer, errors, attempts, timeout_ms, last_row) + + +def _failed_observation(case, observer, errors, attempts, timeout_ms, last_row=None): + last_row = last_row or {} return { **last_row, "case_id": case["id"], @@ -601,6 +609,68 @@ def capture_source_identity() -> dict: "lexbench_webfetch/scoring": SCORING_SOURCE_SHA256} + +_CAPTURE_PROGRAM = ( + "import json,sys; from harness.web_fetch.browser_observations import capture_with_retries; " + "print(json.dumps(capture_with_retries(**json.load(sys.stdin))))" +) + + +def _capture_in_process(case, timeout_ms, attempts, browser_binary=None, resolved_ip=None, + *, observation_scope="reference", dns_mode="system"): + """Bound browser transport and cleanup as well as page operations.""" + payload = dict(case=case, timeout_ms=timeout_ms, attempts=attempts, + browser_binary=browser_binary, resolved_ip=resolved_ip, + observation_scope=observation_scope, dns_mode=dns_mode) + budget = timeout_ms * attempts / 1000 + 2 + began = time.monotonic() + diagnostics = "" + with subprocess.Popen( + [sys.executable, "-c", _CAPTURE_PROGRAM], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + cwd=Path(__file__).resolve().parents[2], start_new_session=os.name == "posix", + env={**os.environ, "DEBUG": "pw:api"}, + ) as process: + try: + stdout, stderr = process.communicate(json.dumps(payload), timeout=budget) + if process.returncode: + raise RuntimeError(stderr[-1000:] or f"Capture worker exited {process.returncode}") + result = json.loads(stdout) + if not isinstance(result, dict) or result.get('case_id') != case['id']: + raise ValueError("Capture worker returned another case") + return result + except subprocess.TimeoutExpired: + # Each POSIX worker owns a new process group, including its driver. + # Terminate only that group so a stuck transport cannot hold pipes open. + signals = (signal.SIGTERM, signal.SIGKILL) if os.name == "posix" else (None, None) + for sig in signals: + try: + if os.name == "posix": + os.killpg(process.pid, sig) + else: + process.kill() + except ProcessLookupError: + pass + try: + _stdout, diagnostics = process.communicate(timeout=1) + break + except subprocess.TimeoutExpired: + continue + message = f"Browser observation worker exceeded {budget:g}s wall-clock budget" + error_type = "TimeoutError" + except (RuntimeError, ValueError) as error: + message, error_type = str(error), type(error).__name__ + row = _failed_observation( + case, {"browser": "Chromium", "binary": "explicit" if browser_binary else "Playwright default", + "dns_mode": dns_mode}, + [{"attempt": None, "type": error_type, "message": message}], None, timeout_ms) + row['completion']['elapsed_ms'] = round((time.monotonic() - began) * 1000, 2) + row['worker_wall_budget_ms'] = budget * 1000 + if diagnostics: + row['worker_diagnostics'] = diagnostics[-4000:] + return row + + def capture_batch( cases, *, @@ -685,7 +755,7 @@ def observe(case): ) if dns_mode != "system": options["dns_mode"] = dns_mode - row = capture_with_retries( + row = _capture_in_process( case, timeout_ms, attempts, browser_binary, resolved_ip, **options ) guard() diff --git a/harness/web_fetch/reference_lifecycle.py b/harness/web_fetch/reference_lifecycle.py index b9bc835a9..1ed901676 100644 --- a/harness/web_fetch/reference_lifecycle.py +++ b/harness/web_fetch/reference_lifecycle.py @@ -143,16 +143,11 @@ def requires_fresh_reference(case) -> bool: if contract: from lexbench_webfetch.stable_tasks import validate_acceptance_contract validate_acceptance_contract(contract) - return False verification = _get(case, "source_verification", {}) or {} mode = verification.get("reference_mode", "frozen") if mode not in {"frozen", "live_exact"}: raise ReferenceLifecycleError("Unknown reference mode; use frozen or live_exact") - return (mode == "live_exact" - and _get(case, "track", "web") in {"web", "dynamic", "challenge"} - and _get(case, "task_kind", "body") == "body" - and bool((_get(case, "ground_truth_main_content", "") or "").strip()) - and bool(verification)) + return mode == "live_exact" def expected_providers(dataset, case): @@ -593,6 +588,8 @@ def _binding(case, dataset_path: Path, now: datetime) -> dict[str, Any]: identifier = safe_identifier(_get(case, "id")) verification = _get(case, "source_verification") reference = _get(case, "ground_truth_main_content") + if not isinstance(reference, str) or not reference.strip(): + raise ReferenceLifecycleError(f"{identifier}: required live reference is missing") if not isinstance(verification, dict) or verification.get("state") != "valid": raise ReferenceLifecycleError(f"{identifier}: independent source verification is not valid") if not isinstance(verification.get("method"), str) or not verification["method"].strip(): diff --git a/harness/web_fetch/render_evidence.py b/harness/web_fetch/render_evidence.py index 36ca672f8..c1e61cdfb 100644 --- a/harness/web_fetch/render_evidence.py +++ b/harness/web_fetch/render_evidence.py @@ -117,7 +117,9 @@ def validate_render_reference( errors.append(f"duplicate_fact_id:{identifier}") seen.add(identifier) facts.append({"id": identifier, "alternatives": alternatives}) - if not facts: + body_requirements = get("body_requirements", {}) or {} + required_patterns = get("required_patterns", []) or [] + if not facts and not (body_requirements or required_patterns or get("acceptance_contract")): errors.append("required_facts_empty") texts = {"initial_visible": _initial_visible(html), "initial_payload": _payload_candidates(html), @@ -147,6 +149,15 @@ def validate_render_reference( for group in coverage[key]["groups"]: if not group["pass"]: errors.append(f"{key}_missing_fact:{group['id']}") + if body_requirements or required_patterns: + from lexbench_webfetch.metrics import evaluate_content + for key in ("reference", "rendered"): + checks = evaluate_content(texts[key], [], body_requirements=body_requirements, + required_patterns=required_patterns) + if not checks["body_coverage_pass"]: + errors.append(f"{key}_body_requirements_not_met") + if not all(row['pass'] for row in checks['pattern_results']): + errors.append(f"{key}_pattern_requirements_not_met") evidence = (get("source_benchmark", {}) or {}).get("render_evidence", {}) completion = evidence.get("completion", {}) condition = completion.get("condition") diff --git a/tests/test_browser_observation_contracts.py b/tests/test_browser_observation_contracts.py index bb2506175..39c56bf15 100644 --- a/tests/test_browser_observation_contracts.py +++ b/tests/test_browser_observation_contracts.py @@ -36,7 +36,7 @@ def test_invalid_policy_rejected_before_browser_calls(rule, monkeypatch): def unexpected(*args, **kwargs): pytest.fail("Invalid policy reached browser capture") - monkeypatch.setattr(capture, "capture_with_retries", unexpected) + monkeypatch.setattr(capture, "_capture_in_process", unexpected) with pytest.raises(ValueError, match="policy"): capture.capture_batch([case_with(rule)], timeout_ms=1000, attempts=1) @@ -93,7 +93,7 @@ def observe(case, *args, **kwargs): calls.append(case["id"]) return {"case_id": case["id"], "status": "success", "content_text": "Article"} - monkeypatch.setattr(capture, "capture_with_retries", observe) + monkeypatch.setattr(capture, "_capture_in_process", observe) options = dict( timeout_ms=1000, attempts=1, @@ -133,7 +133,7 @@ def observe(case, *args, **kwargs): binary.write_bytes(b"changed during observation") return {"case_id": case["id"]} - monkeypatch.setattr(capture, "capture_with_retries", observe) + monkeypatch.setattr(capture, "_capture_in_process", observe) with pytest.raises(ValueError, match="implementation changed"): capture.capture_batch([case_with()], **options) assert not (options["checkpoints"] / "new-task.json").exists() @@ -272,3 +272,24 @@ def test_live_ambiguous_content_scope_is_rejected_instead_of_truncated(rendered_ task["url"] = rendered_page with pytest.raises(ValueError, match="exactly one element"): capture.capture(task, 10000, os.environ["BROWSER_EVAL_TEST_BROWSER_BINARY"]) + + +def test_stuck_capture_worker_is_bounded_and_remains_an_unqualified_observation(monkeypatch): + import time + monkeypatch.setattr(capture, '_CAPTURE_PROGRAM', 'import time; time.sleep(30)') + began = time.monotonic() + row = capture._capture_in_process(case_with(), 20, 1) + assert time.monotonic() - began < 8 + assert row['status'] == 'failure' + assert row['completion']['satisfied'] is False + assert row['attempt_count'] is None + assert row['attempt_errors'][0]['type'] == 'TimeoutError' + assert 'wall-clock budget' in row['attempt_errors'][0]['message'] + + +def test_crashed_capture_worker_does_not_erase_other_case_results(monkeypatch): + monkeypatch.setattr(capture, '_CAPTURE_PROGRAM', "raise RuntimeError('controlled worker failure')") + row = capture._capture_in_process(case_with(), 1000, 1) + assert row['case_id'] == 'new-task' + assert row['status'] == 'failure' + assert 'controlled worker failure' in row['attempt_errors'][0]['message'] diff --git a/tests/test_browser_reference_scope.py b/tests/test_browser_reference_scope.py index c7d9f7dbc..7ccf545ae 100644 --- a/tests/test_browser_reference_scope.py +++ b/tests/test_browser_reference_scope.py @@ -85,7 +85,7 @@ def observe(case, *args, **kwargs): calls.append(case['id']) return {'case_id':case['id'],'status':'failure' if case['id']=='blocked' else 'success', 'content_text':'Verify you are human' if case['id']=='blocked' else 'Article body'} - monkeypatch.setattr(browser_capture,'capture_with_retries',observe) + monkeypatch.setattr(browser_capture,'_capture_in_process',observe) cases=[{'id':k,'url':'https://example.com/'+k} for k in ('article','blocked')] binary = tmp_path / 'browser' binary.write_bytes(b'synthetic browser identity') @@ -109,7 +109,7 @@ def observe(case, *args, **kwargs): def test_checkpoint_cannot_cross_dns_modes(tmp_path, monkeypatch): binary = tmp_path / 'browser' binary.write_bytes(b'synthetic browser identity') - monkeypatch.setattr(browser_capture, 'capture_with_retries', lambda case, *args, **kwargs: { + monkeypatch.setattr(browser_capture, '_capture_in_process', lambda case, *args, **kwargs: { 'case_id': case['id'], 'status': 'success', 'completion': {'satisfied': True}, }) options = dict(timeout_ms=1000, attempts=1, checkpoints=tmp_path / 'checkpoints', @@ -403,7 +403,7 @@ def fake_capture(case, timeout_ms, attempts, browser_binary, resolved_ip): 'completion': {'satisfied': succeeded}, } - monkeypatch.setattr(browser_capture, 'capture_with_retries', fake_capture) + monkeypatch.setattr(browser_capture, '_capture_in_process', fake_capture) monkeypatch.setattr('sys.argv', [ 'capture_headless_browser_observations.py', '--dataset', str(dataset), '--output', str(output), diff --git a/tests/test_dynamic_entry_matching.py b/tests/test_dynamic_entry_matching.py index acc2ffca0..064968064 100644 --- a/tests/test_dynamic_entry_matching.py +++ b/tests/test_dynamic_entry_matching.py @@ -4,7 +4,7 @@ from pathlib import Path from lexbench_webfetch.dynamic_entry_matching import matching_entry_facts -from lexbench_webfetch.dynamic_reference import entry_facts +from tests.test_dynamic_reference import _entry_facts from lexbench_webfetch.metrics import fact_in_text, normalize_for_match @@ -25,7 +25,7 @@ def assert_complete(self, text, facts): def test_corrupt_one_price_and_shipping_with_repeated_values(self): entries = ['First Shirt\n$10.90\nor 9 x$1.21\nFree shipping', 'Second Shirt\n$10.90\nor 9 x$1.21\nFree shipping'] - facts = entry_facts('dynamic-react-cart', entries) + facts = _entry_facts('dynamic-react-cart', entries) reference = '\n\n'.join(entries) for corrupted in [reference.replace('$10.90', '$999.99', 1), reference.replace('Free shipping', '', 1), @@ -36,7 +36,7 @@ def test_corrupt_one_price_and_shipping_with_repeated_values(self): self.assertTrue(hits['entry-2-name']) def test_prefix_shipping_cannot_be_borrowed_from_next_card(self): - facts = entry_facts('dynamic-react-cart', [ + facts = _entry_facts('dynamic-react-cart', [ 'First\n$10\nFree shipping', 'Second\n$10\nFree shipping']) self.assert_complete('Free shipping First $10 Add to cart Free shipping Second $10 Add to cart', facts) hits = self.hits('First $10 Add to cart Free shipping Second $10 Add to cart', facts) @@ -44,7 +44,7 @@ def test_prefix_shipping_cannot_be_borrowed_from_next_card(self): self.assertTrue(hits['entry-2-shipping']) def test_swagger_method_and_path_belong_to_summary(self): - facts = entry_facts('dynamic-swagger', [ + facts = _entry_facts('dynamic-swagger', [ 'POST\n/pet/{petId}/uploadImage\nuploads an image', 'GET\n/pet\nList pets', 'POST\n/user\nCreate user']) text = 'GET /pet/{petId}/uploadImage uploads an image GET /pet List pets POST /user Create user' @@ -54,7 +54,7 @@ def test_swagger_method_and_path_belong_to_summary(self): self.assertFalse(self.hits(text.replace('/user', '/user/login'), facts)['entry-3-path']) def test_native_markdown_link_can_adjoin_method_path_and_summary(self): - facts = entry_facts('dynamic-swagger', [ + facts = _entry_facts('dynamic-swagger', [ 'POST\n/pet\nAdd a new pet to the store', 'GET\n/pet/{petId}\nFind pet by ID']) self.assert_complete( @@ -62,7 +62,7 @@ def test_native_markdown_link_can_adjoin_method_path_and_summary(self): 'GET[/pet/{petId}](#/pet/getPetById)Find pet by ID', facts) def test_overlapping_titles_do_not_invent_missing_record(self): - facts = entry_facts('dynamic-react-cart', [ + facts = _entry_facts('dynamic-react-cart', [ 'Blue T-Shirt\n$10', 'Marine Blue T-shirt\n$10']) hits = self.hits('Marine Blue T-shirt $10', facts) self.assertFalse(hits['entry-1-name']) @@ -73,30 +73,30 @@ def test_overlapping_titles_do_not_invent_missing_record(self): def test_shuffled_and_identical_titles_with_distinct_descriptions(self): entries = ['$299\nSame title\nDark grey laptop\n9 reviews', '$299\nSame title\nGold laptop\n9 reviews'] - facts = entry_facts('dynamic-catalog', entries) + facts = _entry_facts('dynamic-catalog', entries) self.assert_complete('\n\n'.join(reversed(entries)), facts) hits = self.hits(entries[1], facts) self.assertFalse(hits['entry-1-description']) self.assertTrue(hits['entry-2-description']) def test_duplicate_identical_records_need_disjoint_tuples(self): - facts = entry_facts('dynamic-product-reviews', ['2022-07-22\nGood', '2022-07-22\nGood']) + facts = _entry_facts('dynamic-product-reviews', ['2022-07-22\nGood', '2022-07-22\nGood']) hits = self.hits('2022-07-22 Good', facts) self.assertEqual(sum(bool(v) for k, v in hits.items() if k.endswith('-text')), 1) self.assert_complete('2022-07-22 Good 2022-07-22 Good', facts) def test_labels_separators_and_explicit_layouts(self): - facts = entry_facts('dynamic-swagger', ['POST\n/pet\nCreate pet']) + facts = _entry_facts('dynamic-swagger', ['POST\n/pet\nCreate pet']) self.assert_complete('Method: POST | Path: /pet | Summary: Create pet', facts) - facts = entry_facts('dynamic-catalog', ['$10\nTitle\nDescription\n1 reviews']) + facts = _entry_facts('dynamic-catalog', ['$10\nTitle\nDescription\n1 reviews']) self.assert_complete('Title | Price: $10 | Description: Description | 1reviews', facts) - facts = entry_facts('dynamic-quotes-api', ['“Hello.”\nby Someone\nTags: one two']) + facts = _entry_facts('dynamic-quotes-api', ['“Hello.”\nby Someone\nTags: one two']) self.assert_complete('“Hello.”by Someone Tags: one two', facts) - facts = entry_facts('dynamic-products-api', ['Hoodie\n$52']) + facts = _entry_facts('dynamic-products-api', ['Hoodie\n$52']) self.assert_complete('Hoodie\\\\ \\ $52', facts) def test_arbitrary_windows_order_and_numeric_fragments_rejected(self): - facts = entry_facts('dynamic-products-api', ['Shirt\n$10']) + facts = _entry_facts('dynamic-products-api', ['Shirt\n$10']) for text in ['Shirt unrelated product $10', 'Shirt $100', 'Shirt $10.90', 'Shirt -$10', 'Shirt $99 Other $10', '$10 Shirt']: self.assertFalse(any(self.hits(text, facts).values()), text) @@ -107,7 +107,7 @@ def test_fails_closed_without_mutating_frozen_facts(self): before = copy.deepcopy(facts) self.assertEqual(self.hits('Hi', facts), {'entry-1-unknown': []}) self.assertEqual(facts, before) - facts = entry_facts('dynamic-products-api', ['Shirt\n$10']) + facts = _entry_facts('dynamic-products-api', ['Shirt\n$10']) self.assertFalse(any(self.hits('Shirt $10', facts + [facts[0]]).values())) def test_all_frozen_captures_and_missing_records(self): @@ -115,7 +115,7 @@ def test_all_frozen_captures_and_missing_records(self): for filename in ('dynamic-browser-captures.json', 'dynamic-expansion-captures.json'): captures += json.loads((ROOT/'research'/filename).read_text()) for capture in captures: - facts = entry_facts(capture['id'], capture['entries']) + facts = _entry_facts(capture['id'], capture['entries']) with self.subTest(case=capture['id']): self.assert_complete(capture['rendered_text'], facts) self.assert_complete('\n\n'.join(reversed(capture['entries'])), facts) @@ -125,7 +125,7 @@ def test_all_frozen_captures_and_missing_records(self): def test_every_new_capture_field_corruption_is_rejected(self): captures = json.loads((ROOT/'research/dynamic-expansion-captures.json').read_text()) for capture in captures: - facts = entry_facts(capture['id'], capture['entries']) + facts = _entry_facts(capture['id'], capture['entries']) for fact in facts: index = int(fact['id'].split('-')[1]) - 1 entries = capture['entries'][:] diff --git a/tests/test_dynamic_reference.py b/tests/test_dynamic_reference.py index 2eb026ec7..fc0eb526c 100644 --- a/tests/test_dynamic_reference.py +++ b/tests/test_dynamic_reference.py @@ -1,18 +1,50 @@ import copy import json +import re import unittest from pathlib import Path -from lexbench_webfetch.dynamic_reference import entry_facts from lexbench_webfetch.metrics import evaluate_content ROOT=Path(__file__).resolve().parents[1] + +def _entry_facts(case_id, entries): + fields = (['quote', 'author', 'tags'] if case_id in {'dynamic-quotes-api','dynamic-delayed'} + else ['name','price'] if case_id=='dynamic-products-api' + else ['price','title','description','reviews'] if case_id=='dynamic-catalog' + else ['date','text'] if case_id in {'dynamic-reviews-graphql','dynamic-product-reviews'} + else ['name','description'] if case_id=='dynamic-angular-phones' + else ['method','path','summary'] if case_id=='dynamic-swagger' + else ['family'] if case_id=='dynamic-iframe' + else ['name','price','financing','shipping'] if case_id=='dynamic-react-cart' else None) + if not fields:raise ValueError('Unknown dynamic case') + facts=[] + for i,entry in enumerate(entries,1): + values=[line.strip() for line in entry.splitlines() if line.strip()] + entry_fields = fields + if case_id == 'dynamic-react-cart': + # Shipping and financing are optional visible fields on these cards. + entry_fields = ['name', 'price'] + if len(values)>2 and values[2].startswith('or '):entry_fields.append('financing') + if values[-1]=='Free shipping':entry_fields.append('shipping') + if len(values)!=len(entry_fields):raise ValueError(f'Incomplete or unexpected fields: {case_id} entry {i}') + for field,value in zip(entry_fields,values): + if field=='author':value=value.removeprefix('by ') + if field=='tags':value=value.removeprefix('Tags: ') + alternatives=[value] + # Numeric count + its label can be adjacent in Markdown extraction. + # Keep the whole number/unit, never remove arbitrary word boundaries. + if field=='reviews' and re.fullmatch(r'\d+\s+reviews?',value): + alternatives.append(re.sub(r'\s+','',value)) + facts.append({'id':f'entry-{i}-{field}','alternatives':alternatives}) + return facts + class DynamicReferenceTests(unittest.TestCase): def test_all_observed_fields_are_required(self): captures=json.loads((ROOT/'research/dynamic-browser-captures.json').read_text()) captures+=json.loads((ROOT/'research/dynamic-expansion-captures.json').read_text()) for c in captures: - facts=entry_facts(c['id'],c['entries']) + facts=_entry_facts(c['id'],c['entries']) ref=c['rendered_text'] score=lambda text:evaluate_content(text,[],required_facts=facts,min_content_chars=1) self.assertTrue(score(ref)['content_pass'],c['id']) @@ -23,10 +55,10 @@ def test_all_observed_fields_are_required(self): def test_placeholder_or_incomplete_card_is_rejected(self): for entries in [[''],['Only a product name']]: - with self.assertRaises(ValueError):entry_facts('dynamic-products-api',entries) + with self.assertRaises(ValueError):_entry_facts('dynamic-products-api',entries) def test_count_label_spacing_preserves_numeric_identity(self): - facts=entry_facts('dynamic-catalog',['$10\nTitle\nDescription\n1 reviews']) + facts=_entry_facts('dynamic-catalog',['$10\nTitle\nDescription\n1 reviews']) score=lambda text:evaluate_content(text,[],required_facts=facts,min_content_chars=1)['content_pass'] self.assertTrue(score('$10 Title Description 1reviews')) self.assertFalse(score('$10 Title Description 11reviews')) @@ -39,7 +71,7 @@ def test_new_observed_values_replace_old_fact_values(self): updated = copy.deepcopy(case['entries']) old = updated[0] updated[0] = 'Updated family name' - facts = entry_facts(case['id'], updated) + facts = _entry_facts(case['id'], updated) fresh = '\n'.join(updated) stale = '\n'.join(case['entries']) score = lambda text: evaluate_content(text, [], required_facts=facts, min_content_chars=1) diff --git a/tests/test_freeze_headless_references.py b/tests/test_freeze_headless_references.py index b5780f3d5..921357061 100644 --- a/tests/test_freeze_headless_references.py +++ b/tests/test_freeze_headless_references.py @@ -165,7 +165,7 @@ def test_unavailable_body_reference_recovers_from_complete_clean_capture(self): self.assertEqual(frozen['cases'][0]['source_verification']['state'],'valid') self.assertEqual(changes[0]['case_id'],case['id']) - def test_unavailable_body_reference_stays_unavailable_when_capture_is_blocked(self): + def test_required_live_reference_cannot_freeze_a_blocked_empty_capture(self): with tempfile.TemporaryDirectory() as directory: captures=Path(directory) case={'id':'blocked-reference','name':'Blocked reference', @@ -179,10 +179,10 @@ def test_unavailable_body_reference_stays_unavailable_when_capture_is_blocked(se 'sha256':'b'*64,'quality_issues':['http_status_403']} (captures/'audit.json').write_text(json.dumps([audit])) - frozen,changes,_=compute_reference_update({'cases':[case]},captures) - - self.assertEqual(frozen['cases'][0],case) - self.assertEqual(changes,[]) + with self.assertRaisesRegex(ValueError, 'requires independent browser observation'): + compute_reference_update({'cases':[case]},captures) + self.assertEqual(case['ground_truth_main_content'], '') + self.assertEqual(case['source_verification']['state'], 'unavailable') def test_freeze_writes_nothing_until_changed_scope_is_promoted(self): with tempfile.TemporaryDirectory() as directory: diff --git a/tests/test_reference_lifecycle.py b/tests/test_reference_lifecycle.py index cb0ac631e..be8250d5e 100644 --- a/tests/test_reference_lifecycle.py +++ b/tests/test_reference_lifecycle.py @@ -1,4 +1,5 @@ """Synthetic clocks and captures only: no network, browsers or real providers.""" +import pytest import json import tempfile import unittest @@ -147,10 +148,11 @@ def test_equivalent_capture_allows_selector_wording_but_not_scoring_scope(self): with self.assertRaises(ReferenceLifecycleError): check_case_equivalent_reference_binding(replace(equivalent, task_prompt="Different task"), binding) - def test_only_body_with_independent_metadata_is_required(self): - for case in (replace(self.case, source_verification={}), - replace(self.case, task_kind="structure"), replace(self.case, ground_truth_main_content="")): - self.assertIsNone(self.checks(case)) + def test_explicit_reference_requirements_are_independent_of_output_kind(self): + self.assertIsNone(self.checks(replace(self.case, source_verification={}))) + self.assertIsNotNone(self.checks(replace(self.case, task_kind="structure"))) + with self.assertRaisesRegex(ReferenceLifecycleError, "required live reference is missing"): + self.checks(replace(self.case, ground_truth_main_content="")) with self.assertRaisesRegex(ReferenceLifecycleError, "resource evidence binding"): self.checks(replace(self.case, track="image", task_kind="resource", expected_resources=[{"id": "image", "urls": ["https://example.test/a.png"]}])) @@ -622,6 +624,7 @@ def test_public_guard_allows_complete_current_batch_without_live_references(self stable = replace( self.case, id="stable", + source_verification={**self.case.source_verification, "reference_mode": "frozen"}, acceptance_contract={ "version": 1, "schema": "product", @@ -629,7 +632,8 @@ def test_public_guard_allows_complete_current_batch_without_live_references(self "stable_text_anchors": [{"id": "section", "alternatives": ["Catalog"]}], }, ) - resource = replace(self.case, id="resource", track="image", task_kind="resource") + resource = replace(self.case, id="resource", track="image", task_kind="resource", + source_verification={**self.case.source_verification, "reference_mode": "frozen"}) records = [ {"attempt_id": "p__stable__r001", "case_id": "stable", "provider": "p", "started_at": "2026-01-01T00:00:01+00:00", "elapsed_ms": 10, @@ -655,6 +659,7 @@ def test_public_guard_rejects_incomplete_current_batch_without_live_references(s stable = replace( self.case, id="stable", + source_verification={**self.case.source_verification, "reference_mode": "frozen"}, acceptance_contract={ "version": 1, "schema": "product", @@ -687,6 +692,7 @@ def test_current_batch_is_strict_inside_mixed_historical_manifests(self): stable = replace( self.case, id="stable", + source_verification={**self.case.source_verification, "reference_mode": "frozen"}, acceptance_contract={ "version": 1, "schema": "product", @@ -799,3 +805,26 @@ def test_legacy_rescore_still_allowed_without_claiming_protocol(self): if __name__ == "__main__": unittest.main() + + +@pytest.mark.parametrize('track', ['web', 'dynamic', 'challenge', 'risk_control', 'document', 'image', 'video']) +@pytest.mark.parametrize('kind', ['body', 'structure', 'resource', 'file']) +def test_explicit_live_reference_is_selected_for_every_mixed_contract(track, kind): + from harness.web_fetch.reference_lifecycle import requires_fresh_reference + case = {'id': 'live', 'url': 'https://example.test/', 'track': track, 'task_kind': kind, + 'ground_truth_main_content': 'Substantive independently observed article', + 'acceptance_contract': {'version': 1, 'schema': 'body_text', 'minimum_records': 0, + 'stable_text_anchors': [{'id': 'title', 'alternatives': ['Article']}]}, + 'source_verification': {'reference_mode': 'live_exact'}} + assert requires_fresh_reference(case) + case['source_verification'] = {} + assert not requires_fresh_reference(case) + + +def test_unknown_reference_mode_is_not_hidden_by_a_stable_contract(): + from harness.web_fetch.reference_lifecycle import requires_fresh_reference + with pytest.raises(ReferenceLifecycleError, match='reference mode'): + requires_fresh_reference({'acceptance_contract': { + 'version': 1, 'schema': 'body_text', 'minimum_records': 0, + 'stable_text_anchors': [{'id': 'title', 'alternatives': ['Article']}]}, + 'source_verification': {'reference_mode': 'live_excat'}}) diff --git a/tests/test_render_evidence.py b/tests/test_render_evidence.py index 6dcd1c640..e5e9faac3 100644 --- a/tests/test_render_evidence.py +++ b/tests/test_render_evidence.py @@ -163,3 +163,30 @@ def test_elapsed_zero_exact_deadline_and_omission_are_allowed(self): if __name__ == "__main__": unittest.main() + + +class DeclaredContentReferenceTests(unittest.TestCase): + def test_body_scope_is_sufficient_without_an_artificial_fact_group(self): + from lexbench_webfetch.body_coverage import body_requirements + text = 'An article explains how batteries store energy and how charging changes their capacity.' + case = {'id': 'article', 'url': 'https://example.test/', 'required_facts': [], + 'ground_truth_main_content': text, 'body_requirements': body_requirements(text), + 'source_benchmark': {'render_evidence': { + 'scope': 'Complete article body', 'rendered_truncated': False, + 'completion': {'condition': 'Article loaded', 'satisfied': True, 'timeout_ms': 10000}}}} + self.assertTrue(validate_render_reference(case, '该诊断回答“已经返回的条目标题是否完整、干净”，不衡量榜单召回率、实时性或整页正文质量。
'+steps +table(['方法','标题完整度','标题纯净度','标题 F1','标题参考覆盖（可评分 / 返回）'],rows,identifier+'-titles',True,metrics=[])+ '') +def report_scope_disclosure(dataset, *, prefix=''): + """Render profile-authored scope notes without embedding cohort facts in code.""" + scope=dataset.get('comparison',{}).get('report_scope_disclosure',{}) + if not scope:return '' + paragraphs=''.join(''+esc(model_text(item['zh'],item['en']))+'
' for item in scope['paragraphs']) + attachments=' · '.join(link(prefix+item['path'],model_text(item['zh'],item['en'])) for item in scope.get('attachments',[])) + return section(model_text('样本与复现范围','Sample and reproduction scope'), + ''+paragraphs+('
',
+ 'report-scope-disclosure')
+
+
def input_suite_comparison(dataset, cases, records):
"""Compare the selected input groups declared by the frozen source inventory."""
sources=dataset.get('comparison',{}).get('input_sources',[])
@@ -2233,6 +2244,7 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
guide=render_explainer(out,cases,records,table=table,cell=cell,link=link,authored=authored_interpretation)
from .report_reading import render_archived_reading
guide=render_archived_reading(out) or guide
+ guide=report_scope_disclosure(dataset,prefix='../')+guide
write(out,'guide/index.html','报告解读',guide)
panels=[]
for track,label in tracks:
@@ -2243,6 +2255,7 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
if score_audit:overall_title='任务通过情况'
budgets={m['timeout_seconds'] for m in manifests if m.get('timeout_seconds') is not None}
home+=section(overall_title,overall_ranking(records,cases,audit,diagnostic=diagnostic,call_budget_seconds=next(iter(budgets)) if len(budgets)==1 else None),'overall-ranking')
+ home+=report_scope_disclosure(dataset)
home+=input_suite_comparison(dataset,cases,records)
if diagnostic and not score_audit:
home+=result_ledger_section(out,records)
@@ -2909,6 +2922,7 @@ def write_support(out,records,manifests,*,authored_interpretation=True):
from .report_components import render_acceptance_scope
if dataset.get('cases'):
methodology+=render_acceptance_scope({case['id']:case for case in dataset['cases']},records)
+ methodology+=report_scope_disclosure(dataset,prefix='../')
score_binding=dataset.get('comparison',{}).get('score_audit')
if score_binding:
methodology+=section('数据范围',''+attachments+'
' if attachments else '')+''+esc(f'本报告使用原有 {score_binding["total_attempts"]} 条调用记录和耗时，没有新增浏览器调用。其中 {score_binding["projected_scoring_inputs"]} 条评分输入为公开脱敏副本，{score_binding["recovered_originals"]} 条为按原哈希恢复的输出。逐项核对范围为 {score_binding["reviewed_attempts"]}/{score_binding["total_attempts"]} 条判定变化。')+'
'+link('../audit/index.html','输入差异与逐题验收记录')+'
'+esc(model_text(scope_summary['zh'],scope_summary['en']))+'
'+esc(model_text(scope_summary['zh'],scope_summary['en']))+'
该诊断回答“已经返回的条目标题是否完整、干净”，不衡量榜单召回率、实时性或整页正文质量。
'+steps +table(['方法','标题完整度','标题纯净度','标题 F1','标题参考覆盖（可评分 / 返回）'],rows,identifier+'-titles',True,metrics=[])+ '') -def report_scope_disclosure(dataset, *, prefix=''): - """Render profile-authored scope notes without embedding cohort facts in code.""" - scope=dataset.get('comparison',{}).get('report_scope_disclosure',{}) - if not scope:return '' - paragraphs=''.join(''+esc(model_text(item['zh'],item['en']))+'
' for item in scope['paragraphs']) - attachments=' · '.join(link(prefix+item['path'],model_text(item['zh'],item['en'])) for item in scope.get('attachments',[])) - return section(model_text('样本与复现范围','Sample and reproduction scope'), - ''+paragraphs+('
',
- 'report-scope-disclosure')
-
-
def input_suite_comparison(dataset, cases, records):
"""Compare the selected input groups declared by the frozen source inventory."""
sources=dataset.get('comparison',{}).get('input_sources',[])
@@ -2244,7 +2233,6 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
guide=render_explainer(out,cases,records,table=table,cell=cell,link=link,authored=authored_interpretation)
from .report_reading import render_archived_reading
guide=render_archived_reading(out) or guide
- guide=report_scope_disclosure(dataset,prefix='../')+guide
write(out,'guide/index.html','报告解读',guide)
panels=[]
for track,label in tracks:
@@ -2255,7 +2243,6 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
if score_audit:overall_title='任务通过情况'
budgets={m['timeout_seconds'] for m in manifests if m.get('timeout_seconds') is not None}
home+=section(overall_title,overall_ranking(records,cases,audit,diagnostic=diagnostic,call_budget_seconds=next(iter(budgets)) if len(budgets)==1 else None),'overall-ranking')
- home+=report_scope_disclosure(dataset)
home+=input_suite_comparison(dataset,cases,records)
if diagnostic and not score_audit:
home+=result_ledger_section(out,records)
@@ -2922,7 +2909,6 @@ def write_support(out,records,manifests,*,authored_interpretation=True):
from .report_components import render_acceptance_scope
if dataset.get('cases'):
methodology+=render_acceptance_scope({case['id']:case for case in dataset['cases']},records)
- methodology+=report_scope_disclosure(dataset,prefix='../')
score_binding=dataset.get('comparison',{}).get('score_audit')
if score_binding:
methodology+=section('数据范围',''+attachments+'
' if attachments else '')+''+esc(f'本报告使用原有 {score_binding["total_attempts"]} 条调用记录和耗时，没有新增浏览器调用。其中 {score_binding["projected_scoring_inputs"]} 条评分输入为公开脱敏副本，{score_binding["recovered_originals"]} 条为按原哈希恢复的输出。逐项核对范围为 {score_binding["reviewed_attempts"]}/{score_binding["total_attempts"]} 条判定变化。')+'
'+link('../audit/index.html','输入差异与逐题验收记录')+'