From c4ece943049f80902a4de3b1700a033075d38fd3 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Thu, 22 Oct 2026 14:07:15 +0800
Subject: [PATCH 1/7] fix(scoring): combine every declared acceptance
requirement
---
CONTRIBUTING.md | 33 ++
.../evaluation/scoring-regressions.json | 351 ++++++++++++++++++
src/lexbench_webfetch/dynamic_reference.py | 31 --
src/lexbench_webfetch/metrics.py | 21 +-
src/lexbench_webfetch/scoring.py | 35 +-
tests/test_stable_tasks.py | 8 +-
tests/test_unified_requirements.py | 72 ++++
7 files changed, 508 insertions(+), 43 deletions(-)
create mode 100644 tests/test_unified_requirements.py
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 326a6b3..481ed35 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -53,3 +53,36 @@ uv run --frozen lexbench-webfetch inspect-revision \
`export --revision` writes `annotation-review.json` beside the dataset, including
the field differences, source hashes and executed controls. It preserves the
original input files and the revision's existing dataset identity.
+
+## Main content and shared acceptance
+
+Choose the required content from the reader's task and an independent page
+observation before writing matchers. An article needs its substantive sections;
+a feed needs the visible entries and their task-relevant fields; a product list
+needs the information the task asks to retrieve. A search landing page can
+legitimately have no article body. Navigation and footer words do not establish
+that a news feed or article was retrieved.
+
+Use the existing fact, body-region, record, pattern, resource and file checks.
+Combine the applicable requirements in the task definition. Do not implement a
+separate scorer for a site or report family. `score_attempt` exposes the selected
+`acceptance_checks`; `task_passed` requires all of them. Undeclared checks are
+omitted. Missing evidence for a declared requirement is unavailable and cannot
+pass. Full-text overlap remains diagnostic unless an explicit acceptance rule
+requires the content.
+
+`required_facts` are requirements even when a stable record contract is present.
+Do not put obsolete prices, rankings or changing membership in this field just
+to obtain diagnostic overlap metrics. Keep historical reference text in its
+reference field and declare current, independently supported requirements.
+A contract can select stable field rules for changing records without freezing
+old membership, but nonempty labels alone do not prove the records are truthful
+or that all main page sections are covered.
+
+Test a complete answer, the page shell without its main content, and an answer
+missing each required section or record field. Include equivalent supported
+serializations. Compare source, reference and output separately; an extractor's
+own output cannot qualify its reference. Preserve frozen task versions when
+adding coverage: new source observations do not retroactively describe an old
+provider call. A scoring change requires a fresh scoring identity and an audited
+offline recomputation before any published numbers change.
diff --git a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json
index 5e24b59..f1167b1 100644
--- a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json
+++ b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json
@@ -56770,6 +56770,357 @@
"artifact": "",
"ok": true,
"expected_task_pass": false
+ },
+ {
+ "id": "unified-body-complete-delivery-passes",
+ "category": "structured-extraction",
+ "root_cause": "declared_requirement_ignored_by_task_kind",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://example.test/",
+ "acceptance_contract": {
+ "version": 1,
+ "schema": "named_link",
+ "minimum_records": 1,
+ "hosts": [
+ "example.test"
+ ],
+ "entry_path": "/item/\\d+",
+ "stable_text_anchors": []
+ },
+ "required_facts": [
+ {
+ "id": "description",
+ "alternatives": [
+ "Battery lasts twelve hours"
+ ]
+ }
+ ],
+ "expected_resources": [
+ {
+ "id": "manual",
+ "urls": [
+ "/manual.pdf"
+ ]
+ }
+ ],
+ "task_kind": "body"
+ },
+ "artifact": "[Portable speaker](/item/1)\nBattery lasts twelve hours\n[Manual](/manual.pdf)",
+ "ok": true,
+ "expected_task_pass": true
+ },
+ {
+ "id": "unified-body-missing-fact-fails",
+ "category": "structured-extraction",
+ "root_cause": "declared_requirement_ignored_by_task_kind",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://example.test/",
+ "acceptance_contract": {
+ "version": 1,
+ "schema": "named_link",
+ "minimum_records": 1,
+ "hosts": [
+ "example.test"
+ ],
+ "entry_path": "/item/\\d+",
+ "stable_text_anchors": []
+ },
+ "required_facts": [
+ {
+ "id": "description",
+ "alternatives": [
+ "Battery lasts twelve hours"
+ ]
+ }
+ ],
+ "expected_resources": [
+ {
+ "id": "manual",
+ "urls": [
+ "/manual.pdf"
+ ]
+ }
+ ],
+ "task_kind": "body"
+ },
+ "artifact": "[Portable speaker](/item/1)\n[Manual](/manual.pdf)",
+ "ok": true,
+ "expected_task_pass": false
+ },
+ {
+ "id": "unified-body-missing-resource-fails",
+ "category": "structured-extraction",
+ "root_cause": "declared_requirement_ignored_by_task_kind",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://example.test/",
+ "acceptance_contract": {
+ "version": 1,
+ "schema": "named_link",
+ "minimum_records": 1,
+ "hosts": [
+ "example.test"
+ ],
+ "entry_path": "/item/\\d+",
+ "stable_text_anchors": []
+ },
+ "required_facts": [
+ {
+ "id": "description",
+ "alternatives": [
+ "Battery lasts twelve hours"
+ ]
+ }
+ ],
+ "expected_resources": [
+ {
+ "id": "manual",
+ "urls": [
+ "/manual.pdf"
+ ]
+ }
+ ],
+ "task_kind": "body"
+ },
+ "artifact": "[Portable speaker](/item/1)\nBattery lasts twelve hours",
+ "ok": true,
+ "expected_task_pass": false
+ },
+ {
+ "id": "unified-structure-complete-delivery-passes",
+ "category": "structured-extraction",
+ "root_cause": "declared_requirement_ignored_by_task_kind",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://example.test/",
+ "acceptance_contract": {
+ "version": 1,
+ "schema": "named_link",
+ "minimum_records": 1,
+ "hosts": [
+ "example.test"
+ ],
+ "entry_path": "/item/\\d+",
+ "stable_text_anchors": []
+ },
+ "required_facts": [
+ {
+ "id": "description",
+ "alternatives": [
+ "Battery lasts twelve hours"
+ ]
+ }
+ ],
+ "expected_resources": [
+ {
+ "id": "manual",
+ "urls": [
+ "/manual.pdf"
+ ]
+ }
+ ],
+ "task_kind": "structure"
+ },
+ "artifact": "[Portable speaker](/item/1)\nBattery lasts twelve hours\n[Manual](/manual.pdf)",
+ "ok": true,
+ "expected_task_pass": true
+ },
+ {
+ "id": "unified-structure-missing-fact-fails",
+ "category": "structured-extraction",
+ "root_cause": "declared_requirement_ignored_by_task_kind",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://example.test/",
+ "acceptance_contract": {
+ "version": 1,
+ "schema": "named_link",
+ "minimum_records": 1,
+ "hosts": [
+ "example.test"
+ ],
+ "entry_path": "/item/\\d+",
+ "stable_text_anchors": []
+ },
+ "required_facts": [
+ {
+ "id": "description",
+ "alternatives": [
+ "Battery lasts twelve hours"
+ ]
+ }
+ ],
+ "expected_resources": [
+ {
+ "id": "manual",
+ "urls": [
+ "/manual.pdf"
+ ]
+ }
+ ],
+ "task_kind": "structure"
+ },
+ "artifact": "[Portable speaker](/item/1)\n[Manual](/manual.pdf)",
+ "ok": true,
+ "expected_task_pass": false
+ },
+ {
+ "id": "unified-structure-missing-resource-fails",
+ "category": "structured-extraction",
+ "root_cause": "declared_requirement_ignored_by_task_kind",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://example.test/",
+ "acceptance_contract": {
+ "version": 1,
+ "schema": "named_link",
+ "minimum_records": 1,
+ "hosts": [
+ "example.test"
+ ],
+ "entry_path": "/item/\\d+",
+ "stable_text_anchors": []
+ },
+ "required_facts": [
+ {
+ "id": "description",
+ "alternatives": [
+ "Battery lasts twelve hours"
+ ]
+ }
+ ],
+ "expected_resources": [
+ {
+ "id": "manual",
+ "urls": [
+ "/manual.pdf"
+ ]
+ }
+ ],
+ "task_kind": "structure"
+ },
+ "artifact": "[Portable speaker](/item/1)\nBattery lasts twelve hours",
+ "ok": true,
+ "expected_task_pass": false
+ },
+ {
+ "id": "unified-resource-complete-delivery-passes",
+ "category": "structured-extraction",
+ "root_cause": "declared_requirement_ignored_by_task_kind",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://example.test/",
+ "acceptance_contract": {
+ "version": 1,
+ "schema": "named_link",
+ "minimum_records": 1,
+ "hosts": [
+ "example.test"
+ ],
+ "entry_path": "/item/\\d+",
+ "stable_text_anchors": []
+ },
+ "required_facts": [
+ {
+ "id": "description",
+ "alternatives": [
+ "Battery lasts twelve hours"
+ ]
+ }
+ ],
+ "expected_resources": [
+ {
+ "id": "manual",
+ "urls": [
+ "/manual.pdf"
+ ]
+ }
+ ],
+ "task_kind": "resource"
+ },
+ "artifact": "[Portable speaker](/item/1)\nBattery lasts twelve hours\n[Manual](/manual.pdf)",
+ "ok": true,
+ "expected_task_pass": true
+ },
+ {
+ "id": "unified-resource-missing-fact-fails",
+ "category": "structured-extraction",
+ "root_cause": "declared_requirement_ignored_by_task_kind",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://example.test/",
+ "acceptance_contract": {
+ "version": 1,
+ "schema": "named_link",
+ "minimum_records": 1,
+ "hosts": [
+ "example.test"
+ ],
+ "entry_path": "/item/\\d+",
+ "stable_text_anchors": []
+ },
+ "required_facts": [
+ {
+ "id": "description",
+ "alternatives": [
+ "Battery lasts twelve hours"
+ ]
+ }
+ ],
+ "expected_resources": [
+ {
+ "id": "manual",
+ "urls": [
+ "/manual.pdf"
+ ]
+ }
+ ],
+ "task_kind": "resource"
+ },
+ "artifact": "[Portable speaker](/item/1)\n[Manual](/manual.pdf)",
+ "ok": true,
+ "expected_task_pass": false
+ },
+ {
+ "id": "unified-resource-missing-resource-fails",
+ "category": "structured-extraction",
+ "root_cause": "declared_requirement_ignored_by_task_kind",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://example.test/",
+ "acceptance_contract": {
+ "version": 1,
+ "schema": "named_link",
+ "minimum_records": 1,
+ "hosts": [
+ "example.test"
+ ],
+ "entry_path": "/item/\\d+",
+ "stable_text_anchors": []
+ },
+ "required_facts": [
+ {
+ "id": "description",
+ "alternatives": [
+ "Battery lasts twelve hours"
+ ]
+ }
+ ],
+ "expected_resources": [
+ {
+ "id": "manual",
+ "urls": [
+ "/manual.pdf"
+ ]
+ }
+ ],
+ "task_kind": "resource"
+ },
+ "artifact": "[Portable speaker](/item/1)\nBattery lasts twelve hours",
+ "ok": true,
+ "expected_task_pass": false
}
],
"superseded_cases": {
diff --git a/src/lexbench_webfetch/dynamic_reference.py b/src/lexbench_webfetch/dynamic_reference.py
index e9c2a61..33381af 100644
--- a/src/lexbench_webfetch/dynamic_reference.py
+++ b/src/lexbench_webfetch/dynamic_reference.py
@@ -25,34 +25,3 @@ def entry_field_orders(fields):
return [body + ('shipping',), ('shipping',) + body] if 'shipping' in fields else [body]
return [layout for layout in layouts if frozenset(layout) == fields]
-
-def entry_facts(case_id, entries):
- fields = (['quote', 'author', 'tags'] if case_id in {'dynamic-quotes-api','dynamic-delayed'}
- else ['name','price'] if case_id=='dynamic-products-api'
- else ['price','title','description','reviews'] if case_id=='dynamic-catalog'
- else ['date','text'] if case_id in {'dynamic-reviews-graphql','dynamic-product-reviews'}
- else ['name','description'] if case_id=='dynamic-angular-phones'
- else ['method','path','summary'] if case_id=='dynamic-swagger'
- else ['family'] if case_id=='dynamic-iframe'
- else ['name','price','financing','shipping'] if case_id=='dynamic-react-cart' else None)
- if not fields:raise ValueError('Unknown dynamic case')
- facts=[]
- for i,entry in enumerate(entries,1):
- values=[line.strip() for line in entry.splitlines() if line.strip()]
- entry_fields = fields
- if case_id == 'dynamic-react-cart':
- # Shipping and financing are optional visible fields on these cards.
- entry_fields = ['name', 'price']
- if len(values)>2 and values[2].startswith('or '):entry_fields.append('financing')
- if values[-1]=='Free shipping':entry_fields.append('shipping')
- if len(values)!=len(entry_fields):raise ValueError(f'Incomplete or unexpected fields: {case_id} entry {i}')
- for field,value in zip(entry_fields,values):
- if field=='author':value=value.removeprefix('by ')
- if field=='tags':value=value.removeprefix('Tags: ')
- alternatives=[value]
- # Numeric count + its label can be adjacent in Markdown extraction.
- # Keep the whole number/unit, never remove arbitrary word boundaries.
- if field=='reviews' and re.fullmatch(r'\d+\s+reviews?',value):
- alternatives.append(re.sub(r'\s+','',value))
- facts.append({'id':f'entry-{i}-{field}','alternatives':alternatives})
- return facts
diff --git a/src/lexbench_webfetch/metrics.py b/src/lexbench_webfetch/metrics.py
index 258b8f7..829325e 100644
--- a/src/lexbench_webfetch/metrics.py
+++ b/src/lexbench_webfetch/metrics.py
@@ -459,6 +459,8 @@ def task_passed(ok: bool, metrics: dict, task_kind: str | None = None) -> bool:
if ('acceptance_policy_version' in metrics or 'forbidden_hits' in metrics
or ('blocked_page' in metrics and 'forbidden_requirement_hits' not in metrics)):
raise ValueError('Historical scoring metrics require their frozen scorer or a complete offline rescore')
+ if metrics.get('request_ok') is False:
+ return False
if metrics.get('forbidden_requirement_hits') or metrics.get('forbidden_requirement_pass') is False:
return False
if (metrics.get('page_identity_required')
@@ -466,14 +468,13 @@ def task_passed(ok: bool, metrics: dict, task_kind: str | None = None) -> bool:
return False
if not task_measured(metrics, task_kind) or metrics.get('deadline_overrun'):
return False
+ if "acceptance_checks" in metrics:
+ checks = metrics["acceptance_checks"]
+ return bool(ok and checks and all(value is True for value in checks.values()))
if "stable_task_pass" in metrics:
combined = (metrics.get("content_pass")
- if ("composite_acceptance_pass" in metrics
- or metrics.get("body_region_results")
- or metrics.get("pattern_results"))
- else True)
- return bool(ok and metrics["stable_task_pass"]
- and combined
+ if metrics.get("content_score_applicable", True) else True)
+ return bool(ok and metrics["stable_task_pass"] and combined
and metrics.get("body_coverage_pass") is not False)
field = {'structure':'list_task_pass', 'resource':'resource_pass', 'file':'raw_file_pass'}.get(task_kind, 'content_pass')
return bool(ok and metrics.get(field))
@@ -487,6 +488,9 @@ def task_measured(metrics: dict, task_kind: str | None = None) -> bool:
kind = task_kind or metrics.get('task_kind', 'body')
if metrics.get('page_identity_required') and metrics.get('page_identity_status') != 'verified':
return True
+ if 'acceptance_checks' in metrics and any(
+ value is None for value in metrics['acceptance_checks'].values()):
+ return False
return kind != 'file' or isinstance(metrics.get('raw_file_pass'), bool)
@@ -810,6 +814,8 @@ def evaluate_content(
"fact_total": len(fact_results),
"fact_coverage": fact_coverage,
"fact_coverage_threshold": fact_coverage_threshold,
+ "fact_coverage_pass": fact_coverage >= fact_coverage_threshold,
+ "pattern_pass": all(item["pass"] for item in pattern_results),
"forbidden_requirement_hits": forbidden,
"diagnostic_hits": [item for item in (diagnostic_any or [])
if (text := normalize_for_match(item)) and forbidden_in_text(folded, text)],
@@ -883,7 +889,8 @@ def evaluate_content(
if "content_too_short" not in result["stable_task_errors"]:
result["stable_task_errors"].append("content_too_short")
result["content_pass"] = bool(
- declared_body_pass and declared_pattern_pass and result["stable_task_pass"] and not forbidden
+ declared_body_pass and declared_pattern_pass and result["stable_task_pass"]
+ and result["fact_coverage_pass"] and not forbidden
)
result["content_decision_basis"] = (
"declared_content_requirements_with_stable_text_diagnostics"
diff --git a/src/lexbench_webfetch/scoring.py b/src/lexbench_webfetch/scoring.py
index d5cdf13..fe11c83 100644
--- a/src/lexbench_webfetch/scoring.py
+++ b/src/lexbench_webfetch/scoring.py
@@ -321,8 +321,41 @@ def score_attempt(
# A threshold is only an explicit task requirement, not a benchmark-wide
# interpretation of continuous similarity as a pass rate.
if 'ordered_tree_min_similarity' in case:
+ result['ordered_tree_pass'] = bool(
+ tree_result['score'] >= case['ordered_tree_min_similarity'])
result['content_pass'] = bool(
- ok and tree_result['score'] >= case['ordered_tree_min_similarity']
+ result['content_pass'] and ok and result['ordered_tree_pass']
and not result['forbidden_requirement_hits'])
+ result["acceptance_checks"] = acceptance_checks(case, result)
result["scoring_source_sha256"] = SCORING_SOURCE_SHA256
return result
+
+
+def acceptance_checks(case: dict, metrics: dict) -> dict[str, bool | None]:
+ """Select declared requirements, never a website- or provider-specific scorer.
+
+ Missing evidence for a selected requirement remains unavailable. Undeclared
+ requirements are omitted; similarity diagnostics never become pass gates.
+ """
+ kind = case.get('task_kind', 'body')
+ checks = {}
+ if case.get('acceptance_contract'):
+ checks['records_and_anchors'] = metrics.get('stable_task_pass')
+ elif kind in {'body', 'structure'}:
+ checks['content' if kind == 'body' else 'records'] = metrics.get(
+ 'content_pass' if kind == 'body' else 'list_task_pass')
+ if case.get('required_facts'):
+ checks['facts'] = metrics.get('fact_coverage_pass')
+ if case.get('required_patterns'):
+ checks['patterns'] = metrics.get('pattern_pass')
+ if case.get('body_requirements'):
+ checks['body'] = metrics.get('body_coverage_pass')
+ if case.get('composite_acceptance'):
+ checks['composite'] = metrics.get('composite_acceptance_pass')
+ if case.get('expected_resources') or case.get('resource_output_contract') or kind == 'resource':
+ checks['resources'] = metrics.get('resource_pass')
+ if kind == 'file':
+ checks['file'] = metrics.get('raw_file_pass')
+ if 'ordered_tree_min_similarity' in case:
+ checks['tree'] = metrics.get('ordered_tree_pass')
+ return checks
diff --git a/tests/test_stable_tasks.py b/tests/test_stable_tasks.py
index e9a1768..cd0224f 100644
--- a/tests/test_stable_tasks.py
+++ b/tests/test_stable_tasks.py
@@ -467,16 +467,16 @@ def test_strict_contract_validation_before_snapshot_exemption(self):
with self.subTest(value=value), self.assertRaises(ValueError):
validate_acceptance_contract(value)
- def test_contract_metrics_decision_is_independent_of_frozen_literal_metrics(self):
+ def test_declared_facts_remain_required_with_a_record_contract(self):
rules = contract("product")
before = deepcopy(rules)
result = evaluate_content("Alpha\n$10\n\nBeta\n$20", ["Old name"], required_facts=[
{"id": "frozen", "alternatives": ["Old title $50"]}], min_content_chars=10000,
ground_truth_main_content="Old title $50", acceptance_contract=rules)
- self.assertTrue(result["content_pass"])
+ self.assertFalse(result["content_pass"])
self.assertEqual(result["fact_coverage"], 0)
self.assertIsNotNone(result["word_f1"])
- self.assertTrue(task_passed(True, result, "structure"))
+ self.assertFalse(task_passed(True, result, "structure"))
self.assertFalse(task_passed(False, result))
self.assertEqual(rules, before)
self.assertIsNone(result["factual_correctness"])
@@ -493,7 +493,7 @@ def test_stable_text_and_structure_are_separate_from_word_overlap(self):
{"id": "section", "alternatives": ["Product catalog", "Available products"]},
{"id": "explanation", "alternatives": ["Prices include tax"]}])
text = "# Available products\nPrices include tax\n\nNew coat\n$19\n\nNew scarf\n$99"
- result = evaluate_content(text, ["Old title"], required_facts=[{"id": "old", "alternatives": ["Old price"]}],
+ result = evaluate_content(text, ["Old title"],
ground_truth_main_content="Old title Old price", acceptance_contract=rules)
self.assertTrue(result["stable_structure_pass"])
self.assertTrue(result["stable_content_pass"])
diff --git a/tests/test_unified_requirements.py b/tests/test_unified_requirements.py
new file mode 100644
index 0000000..8c749da
--- /dev/null
+++ b/tests/test_unified_requirements.py
@@ -0,0 +1,72 @@
+"""Declared requirements compose independently of a task's reporting family."""
+import pytest
+from lexbench_webfetch.scoring import score_attempt
+from lexbench_webfetch.metrics import task_passed
+
+
+CONTRACT = {'version': 1, 'schema': 'named_link', 'minimum_records': 1,
+ 'hosts': ['example.test'], 'entry_path': r'/item/\d+',
+ 'stable_text_anchors': []}
+
+
+@pytest.mark.parametrize('kind', ['body', 'structure', 'resource'])
+@pytest.mark.parametrize('missing', [None, 'fact', 'resource', 'record'])
+def test_every_declared_requirement_contributes_to_acceptance(kind, missing):
+ case = {'url': 'https://example.test/', 'task_kind': kind,
+ 'acceptance_contract': CONTRACT,
+ 'required_facts': [{'id': 'description', 'alternatives': ['Battery lasts twelve hours']}],
+ 'expected_resources': [{'id': 'manual', 'urls': ['/manual.pdf']}]}
+ parts = {'fact': 'Battery lasts twelve hours', 'record': '[Portable speaker](/item/1)',
+ 'resource': '[Manual](/manual.pdf)'}
+ answer = '\n'.join(text for key, text in parts.items() if key != missing)
+ metrics = score_attempt(case, answer.encode(), ok=True)
+ assert task_passed(True, metrics) is (missing is None)
+
+
+@pytest.mark.parametrize('answer,passed', [
+ ('Home\n[Item](/item/1)\nBattery lasts twelve hours', True),
+ ('Home\n[Item](/item/1)', False)])
+def test_required_facts_remain_required_next_to_a_stable_contract(answer, passed):
+ case = {'url': 'https://example.test/', 'acceptance_contract': CONTRACT,
+ 'required_facts': [{'id': 'detail', 'alternatives': ['Battery lasts twelve hours']}]}
+ assert task_passed(True, score_attempt(case, answer.encode(), ok=True)) is passed
+
+
+@pytest.mark.parametrize('kind', ['body', 'structure', 'resource', 'file'])
+@pytest.mark.parametrize('missing', [None, 'fact', 'resource', 'record', 'file'])
+def test_file_and_text_requirements_use_the_same_combination(kind, missing):
+ from lexbench_webfetch.evaluation import evaluate_response
+ case = {'url': 'https://example.test/', 'task_kind': kind,
+ 'acceptance_contract': CONTRACT,
+ 'required_facts': [{'id': 'description', 'alternatives': ['Battery lasts twelve hours']}],
+ 'expected_resources': [{'id': 'manual', 'urls': ['/manual.pdf']}]}
+ parts = {'fact': 'Battery lasts twelve hours', 'record': '[Portable speaker](/item/1)',
+ 'resource': '[Manual](/manual.pdf)'}
+ answer = '\n'.join(text for key, text in parts.items() if key != missing).encode()
+ files = [] if missing == 'file' else [b'\x89PNG\r\n\x1a\n' + b'x' * 32]
+ result = evaluate_response(case, answer, files=files)
+ assert result['passed'] is (missing is None or (missing == 'file' and kind != 'file'))
+
+
+def test_absent_requirements_are_skipped_but_missing_required_evidence_is_not():
+ from lexbench_webfetch.evaluation import evaluate_response
+ task = {'url': 'https://example.test/', 'task_kind': 'resource'}
+ unavailable = evaluate_response(task, b'No reference or resource returned')
+ assert not unavailable['passed']
+ assert not unavailable['measured']
+ assert unavailable['metrics']['acceptance_checks'] == {'resources': None}
+ task = {'url': 'https://example.test/', 'required_facts': [
+ {'id': 'body', 'alternatives': ['Article body']} ]}
+ accepted = evaluate_response(task, b'Article body')
+ assert accepted['passed']
+ assert set(accepted['metrics']['acceptance_checks']) == {'content', 'facts'}
+
+
+def test_perfect_tree_does_not_override_a_missing_content_requirement():
+ import json
+ case = {'url': 'https://example.test/', 'ordered_tree_reference': {'tag': 'table'},
+ 'ordered_tree_min_similarity': 1.0,
+ 'required_facts': [{'id': 'value', 'alternatives': ['Quarterly revenue']} ]}
+ metrics = score_attempt(case, json.dumps(case['ordered_tree_reference']).encode(), ok=True)
+ assert metrics['ordered_tree_similarity'] == 1.0
+ assert not task_passed(True, metrics)
From da92898f9c0c8a36cb8d73d97631163ee2f62431 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Thu, 22 Oct 2026 14:15:09 +0800
Subject: [PATCH 2/7] fix(scoring): honor content and identity checks across
task families
---
.../evaluation/scoring-regressions.json | 136 ++++++++++++++++++
src/lexbench_webfetch/dynamic_reference.py | 31 ++++
src/lexbench_webfetch/metrics.py | 1 +
src/lexbench_webfetch/scoring.py | 28 ++--
tests/test_acceptance_isolation.py | 2 +-
tests/test_unified_requirements.py | 39 +++++
6 files changed, 224 insertions(+), 13 deletions(-)
diff --git a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json
index f1167b1..f285ba7 100644
--- a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json
+++ b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json
@@ -57121,6 +57121,142 @@
"artifact": "[Portable speaker](/item/1)\nBattery lasts twelve hours",
"ok": true,
"expected_task_pass": false
+ },
+ {
+ "id": "unified-expected_any-complete-passes",
+ "category": "structured-extraction",
+ "root_cause": "declared_requirement_ignored_by_task_kind",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://example.test/",
+ "acceptance_contract": {
+ "version": 1,
+ "schema": "named_link",
+ "minimum_records": 1,
+ "hosts": [
+ "example.test"
+ ],
+ "entry_path": "/item/\\d+",
+ "stable_text_anchors": []
+ },
+ "expected_any": [
+ "Required introduction"
+ ]
+ },
+ "artifact": "Required introduction\n[Item](/item/1)",
+ "ok": true,
+ "expected_task_pass": true
+ },
+ {
+ "id": "unified-expected_any-missing-fails",
+ "category": "structured-extraction",
+ "root_cause": "declared_requirement_ignored_by_task_kind",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://example.test/",
+ "acceptance_contract": {
+ "version": 1,
+ "schema": "named_link",
+ "minimum_records": 1,
+ "hosts": [
+ "example.test"
+ ],
+ "entry_path": "/item/\\d+",
+ "stable_text_anchors": []
+ },
+ "expected_any": [
+ "Required introduction"
+ ]
+ },
+ "artifact": "[Item](/item/1)",
+ "ok": true,
+ "expected_task_pass": false
+ },
+ {
+ "id": "unified-min_content_chars-complete-passes",
+ "category": "structured-extraction",
+ "root_cause": "declared_requirement_ignored_by_task_kind",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://example.test/",
+ "acceptance_contract": {
+ "version": 1,
+ "schema": "named_link",
+ "minimum_records": 1,
+ "hosts": [
+ "example.test"
+ ],
+ "entry_path": "/item/\\d+",
+ "stable_text_anchors": []
+ },
+ "min_content_chars": 80
+ },
+ "artifact": "Required introduction and a substantive explanation of a portable device lasting twelve hours.\n[Item](/item/1)",
+ "ok": true,
+ "expected_task_pass": true
+ },
+ {
+ "id": "unified-min_content_chars-missing-fails",
+ "category": "structured-extraction",
+ "root_cause": "declared_requirement_ignored_by_task_kind",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://example.test/",
+ "acceptance_contract": {
+ "version": 1,
+ "schema": "named_link",
+ "minimum_records": 1,
+ "hosts": [
+ "example.test"
+ ],
+ "entry_path": "/item/\\d+",
+ "stable_text_anchors": []
+ },
+ "min_content_chars": 80
+ },
+ "artifact": "[Item](/item/1)",
+ "ok": true,
+ "expected_task_pass": false
+ },
+ {
+ "id": "unified-document-language-matching-passes",
+ "category": "structured-extraction",
+ "root_cause": "declared_requirement_ignored_by_task_kind",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://example.test/",
+ "track": "web",
+ "expected_final_url": "https://example.test/",
+ "expected_document_language": "en",
+ "page_identity_required": true
+ },
+ "artifact": "Article body",
+ "metadata": {
+ "document_language": "en"
+ },
+ "final_url": "https://example.test/",
+ "ok": true,
+ "expected_task_pass": true
+ },
+ {
+ "id": "unified-document-language-wrong-fails",
+ "category": "structured-extraction",
+ "root_cause": "declared_requirement_ignored_by_task_kind",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://example.test/",
+ "track": "web",
+ "expected_final_url": "https://example.test/",
+ "expected_document_language": "en",
+ "page_identity_required": true
+ },
+ "artifact": "Article body",
+ "metadata": {
+ "document_language": "fr"
+ },
+ "final_url": "https://example.test/",
+ "ok": true,
+ "expected_task_pass": false
}
],
"superseded_cases": {
diff --git a/src/lexbench_webfetch/dynamic_reference.py b/src/lexbench_webfetch/dynamic_reference.py
index 33381af..e9c2a61 100644
--- a/src/lexbench_webfetch/dynamic_reference.py
+++ b/src/lexbench_webfetch/dynamic_reference.py
@@ -25,3 +25,34 @@ def entry_field_orders(fields):
return [body + ('shipping',), ('shipping',) + body] if 'shipping' in fields else [body]
return [layout for layout in layouts if frozenset(layout) == fields]
+
+def entry_facts(case_id, entries):
+ fields = (['quote', 'author', 'tags'] if case_id in {'dynamic-quotes-api','dynamic-delayed'}
+ else ['name','price'] if case_id=='dynamic-products-api'
+ else ['price','title','description','reviews'] if case_id=='dynamic-catalog'
+ else ['date','text'] if case_id in {'dynamic-reviews-graphql','dynamic-product-reviews'}
+ else ['name','description'] if case_id=='dynamic-angular-phones'
+ else ['method','path','summary'] if case_id=='dynamic-swagger'
+ else ['family'] if case_id=='dynamic-iframe'
+ else ['name','price','financing','shipping'] if case_id=='dynamic-react-cart' else None)
+ if not fields:raise ValueError('Unknown dynamic case')
+ facts=[]
+ for i,entry in enumerate(entries,1):
+ values=[line.strip() for line in entry.splitlines() if line.strip()]
+ entry_fields = fields
+ if case_id == 'dynamic-react-cart':
+ # Shipping and financing are optional visible fields on these cards.
+ entry_fields = ['name', 'price']
+ if len(values)>2 and values[2].startswith('or '):entry_fields.append('financing')
+ if values[-1]=='Free shipping':entry_fields.append('shipping')
+ if len(values)!=len(entry_fields):raise ValueError(f'Incomplete or unexpected fields: {case_id} entry {i}')
+ for field,value in zip(entry_fields,values):
+ if field=='author':value=value.removeprefix('by ')
+ if field=='tags':value=value.removeprefix('Tags: ')
+ alternatives=[value]
+ # Numeric count + its label can be adjacent in Markdown extraction.
+ # Keep the whole number/unit, never remove arbitrary word boundaries.
+ if field=='reviews' and re.fullmatch(r'\d+\s+reviews?',value):
+ alternatives.append(re.sub(r'\s+','',value))
+ facts.append({'id':f'entry-{i}-{field}','alternatives':alternatives})
+ return facts
diff --git a/src/lexbench_webfetch/metrics.py b/src/lexbench_webfetch/metrics.py
index 829325e..16ebec9 100644
--- a/src/lexbench_webfetch/metrics.py
+++ b/src/lexbench_webfetch/metrics.py
@@ -804,6 +804,7 @@ def evaluate_content(
"content_pass": content_pass,
"pattern_results": pattern_results,
"legacy_marker_pass": legacy_pass,
+ "content_length_pass": length_pass,
"matched_markers": matched,
"expected_markers": expected,
"marker_hits": len(matched),
diff --git a/src/lexbench_webfetch/scoring.py b/src/lexbench_webfetch/scoring.py
index fe11c83..9996e55 100644
--- a/src/lexbench_webfetch/scoring.py
+++ b/src/lexbench_webfetch/scoring.py
@@ -157,26 +157,21 @@ def score_attempt(
))
result.update(request_ok=ok, request_status_basis=status_basis,
normalized_named_link_count=len(links))
- dynamic_page_identity = bool(
- case.get("track") == "dynamic"
- and case.get("evaluation_url")
- and case.get("expected_final_url")
- and case.get("target_language")
- and case.get("target_region")
+ language_page_identity = bool(
+ case.get("expected_final_url")
+ and (case.get("expected_document_language") or case.get("target_language"))
)
risk_navigation = (
((case.get("source_benchmark") or {}).get("risk_control") or {}).get(
"navigation_contract"
)
- if case.get("track") == "risk_control"
- else None
)
risk_page_identity = bool(
isinstance(risk_navigation, dict)
and isinstance(risk_navigation.get("requested_url"), str)
and isinstance(risk_navigation.get("observed_final_url"), str)
)
- if dynamic_page_identity:
+ if language_page_identity:
observed_language = (metadata or {}).get("document_language")
language_present = case.get("expected_document_language_present", True)
if not isinstance(language_present, bool):
@@ -184,7 +179,7 @@ def score_attempt(
if not language_present and case.get("expected_document_language"):
raise ValueError("An absent source language cannot have an expected language tag")
expected_document_language = (
- case.get("expected_document_language") or case["target_language"]
+ case.get("expected_document_language") or case.get("target_language")
)
final_consistent = bool(
isinstance(final_url, str) and final_url
@@ -256,7 +251,8 @@ def score_attempt(
and result.get("content_pass")
and (not case.get("acceptance_contract") or result.get("stable_task_pass"))
)
- if (result["task_kind"] == "body" and result["declared_truncation_hits"]
+ if ((result["task_kind"] == "body" or declared_regions)
+ and result["declared_truncation_hits"]
and not exact_body_proven):
result["content_pass"] = False
if contract := case.get("acceptance_contract"):
@@ -264,6 +260,9 @@ def score_attempt(
result["stable_content_pass"] = False
if "declared_truncation" not in result["stable_task_errors"]:
result["stable_task_errors"].append("declared_truncation")
+ result['body_delivery_pass'] = bool(
+ result['body_coverage_pass']
+ and (not result['declared_truncation_hits'] or exact_body_proven))
file_evaluation = (metadata or {}).get('file_evaluation')
if result['task_kind'] == 'file' and isinstance(file_evaluation, dict) and ('raw_file_pass' not in result or delivered_files is not None):
result['raw_file_pass'] = bool(file_evaluation.get('pass'))
@@ -344,12 +343,17 @@ def acceptance_checks(case: dict, metrics: dict) -> dict[str, bool | None]:
elif kind in {'body', 'structure'}:
checks['content' if kind == 'body' else 'records'] = metrics.get(
'content_pass' if kind == 'body' else 'list_task_pass')
+ if case.get('content_score_applicable', True):
+ if case.get('expected_any'):
+ checks['literals'] = metrics.get('legacy_marker_pass')
+ if 'min_content_chars' in case:
+ checks['length'] = metrics.get('content_length_pass')
if case.get('required_facts'):
checks['facts'] = metrics.get('fact_coverage_pass')
if case.get('required_patterns'):
checks['patterns'] = metrics.get('pattern_pass')
if case.get('body_requirements'):
- checks['body'] = metrics.get('body_coverage_pass')
+ checks['body'] = metrics.get('body_delivery_pass')
if case.get('composite_acceptance'):
checks['composite'] = metrics.get('composite_acceptance_pass')
if case.get('expected_resources') or case.get('resource_output_contract') or kind == 'resource':
diff --git a/tests/test_acceptance_isolation.py b/tests/test_acceptance_isolation.py
index f917b5e..337dd52 100644
--- a/tests/test_acceptance_isolation.py
+++ b/tests/test_acceptance_isolation.py
@@ -118,7 +118,7 @@ def test_xml_declarations_cannot_crash_body_or_resource_scoring(declaration):
body_case = {"url": "https://example.test/spec", "expected_any": ["Specification"]}
raw = ("Specification\n" + declaration).encode()
assert task_passed(True, score_attempt(body_case, raw, ok=True))
- resource_case = {**body_case, "task_kind": "resource",
+ resource_case = {**body_case, "task_kind": "resource", "content_score_applicable": False,
"expected_resources": [{"id": "image", "urls": ["/target.png"]}]}
for answer, passed in [(declaration + '\n\n'
+ '[Target](/target.png)', True), (declaration + '\n', False)]:
diff --git a/tests/test_unified_requirements.py b/tests/test_unified_requirements.py
index 8c749da..96f1ddd 100644
--- a/tests/test_unified_requirements.py
+++ b/tests/test_unified_requirements.py
@@ -70,3 +70,42 @@ def test_perfect_tree_does_not_override_a_missing_content_requirement():
metrics = score_attempt(case, json.dumps(case['ordered_tree_reference']).encode(), ok=True)
assert metrics['ordered_tree_similarity'] == 1.0
assert not task_passed(True, metrics)
+
+
+@pytest.mark.parametrize('field,value,short_answer', [
+ ('expected_any', ['Required introduction'], '[Item](/item/1)'),
+ ('min_content_chars', 80, '[Item](/item/1)')])
+def test_declared_literal_and_length_requirements_survive_a_record_contract(field, value, short_answer):
+ case = {'url': 'https://example.test/', 'acceptance_contract': CONTRACT, field: value}
+ assert not task_passed(True, score_attempt(case, short_answer.encode(), ok=True))
+ full = 'Required introduction with a full substantive explanation of the portable device.\n[Item](/item/1)'
+ assert task_passed(True, score_attempt(case, full.encode(), ok=True))
+
+
+@pytest.mark.parametrize('track', ['web', 'dynamic', 'challenge', 'risk_control', 'document', 'image', 'video'])
+@pytest.mark.parametrize('language,passed', [('en', True), ('fr', False), (None, False)])
+def test_declared_document_language_check_does_not_depend_on_track(track, language, passed):
+ case = {'url': 'https://example.test/', 'track': track,
+ 'expected_final_url': 'https://example.test/',
+ 'expected_document_language': 'en', 'page_identity_required': True}
+ metrics = score_attempt(case, b'Article body', ok=True, final_url=case['url'],
+ metadata={'document_language': language})
+ assert task_passed(True, metrics) is passed
+
+
+@pytest.mark.parametrize('kind', ['body', 'structure', 'resource', 'file'])
+@pytest.mark.parametrize('complete', [True, False])
+def test_declared_body_scope_keeps_truncation_protection_across_output_types(kind, complete):
+ from lexbench_webfetch.evaluation import evaluate_response
+ task = {'url': 'https://example.test/', 'task_kind': kind,
+ 'acceptance_contract': CONTRACT,
+ 'expected_resources': [{'id': 'image', 'urls': ['/image.png']}],
+ 'body_requirements': {'version': 1, 'unit': 'region_token_recall', 'minimum_coverage': 0.8,
+ 'regions': [{'id': 'body', 'text': 'alpha beta gamma delta epsilon zeta eta theta iota kappa'}]}}
+ answer = ('alpha beta gamma delta epsilon zeta eta theta iota\n'
+ '[Item](/item/1)\n\n'
+ '(Content truncated. Fetch a more specific URL or section for the full text.)')
+ if complete:
+ answer = answer.replace('theta iota\n', 'theta iota kappa\n')
+ result = evaluate_response(task, answer.encode(), files=[b'\x89PNG\r\n\x1a\n'+b'x'*32])
+ assert result['passed'] is complete
From 9008719de74659a0e48189aa5fefb623aca785c2 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Thu, 22 Oct 2026 15:08:09 +0800
Subject: [PATCH 3/7] fix(scoring)!: apply one acceptance contract across input
forms
---
CONTRIBUTING.md | 5 +-
.../evaluation/scoring-regressions.json | 36 ++++++++++++++
src/lexbench_webfetch/metrics.py | 2 +-
src/lexbench_webfetch/scoring.py | 12 ++---
tests/test_stable_tasks.py | 2 +-
tests/test_unified_requirements.py | 49 ++++++++++++++++++-
6 files changed, 94 insertions(+), 12 deletions(-)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 481ed35..bc46dbc 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -64,7 +64,10 @@ legitimately have no article body. Navigation and footer words do not establish
that a news feed or article was retrieved.
Use the existing fact, body-region, record, pattern, resource and file checks.
-Combine the applicable requirements in the task definition. Do not implement a
+Combine the applicable requirements in the task definition. For record-only or
+resource-only deliveries, set `content_score_applicable: false` to omit the default
+body-length and literal checks; explicit facts and body-region requirements still
+apply. Do not implement a
separate scorer for a site or report family. `score_attempt` exposes the selected
`acceptance_checks`; `task_passed` requires all of them. Undeclared checks are
omitted. Missing evidence for a declared requirement is unavailable and cannot
diff --git a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json
index f285ba7..a791fdc 100644
--- a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json
+++ b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json
@@ -57257,6 +57257,42 @@
"final_url": "https://example.test/",
"ok": true,
"expected_task_pass": false
+ },
+ {
+ "id": "unified-absent-document-language-empty-tag-passes",
+ "category": "structured-extraction",
+ "root_cause": "declared_requirement_ignored_by_task_kind",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://example.test/",
+ "expected_document_language_present": false,
+ "page_identity_required": true
+ },
+ "artifact": "Article body",
+ "metadata": {
+ "document_language": ""
+ },
+ "final_url": "https://example.test/",
+ "ok": true,
+ "expected_task_pass": true
+ },
+ {
+ "id": "unified-absent-document-language-unexpected-tag-fails",
+ "category": "structured-extraction",
+ "root_cause": "declared_requirement_ignored_by_task_kind",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://example.test/",
+ "expected_document_language_present": false,
+ "page_identity_required": true
+ },
+ "artifact": "Article body",
+ "metadata": {
+ "document_language": "en"
+ },
+ "final_url": "https://example.test/",
+ "ok": true,
+ "expected_task_pass": false
}
],
"superseded_cases": {
diff --git a/src/lexbench_webfetch/metrics.py b/src/lexbench_webfetch/metrics.py
index 16ebec9..89d1ace 100644
--- a/src/lexbench_webfetch/metrics.py
+++ b/src/lexbench_webfetch/metrics.py
@@ -891,7 +891,7 @@ def evaluate_content(
result["stable_task_errors"].append("content_too_short")
result["content_pass"] = bool(
declared_body_pass and declared_pattern_pass and result["stable_task_pass"]
- and result["fact_coverage_pass"] and not forbidden
+ and result["fact_coverage_pass"] and legacy_pass and length_pass and not forbidden
)
result["content_decision_basis"] = (
"declared_content_requirements_with_stable_text_diagnostics"
diff --git a/src/lexbench_webfetch/scoring.py b/src/lexbench_webfetch/scoring.py
index 9996e55..1bbdd17 100644
--- a/src/lexbench_webfetch/scoring.py
+++ b/src/lexbench_webfetch/scoring.py
@@ -158,8 +158,8 @@ def score_attempt(
result.update(request_ok=ok, request_status_basis=status_basis,
normalized_named_link_count=len(links))
language_page_identity = bool(
- case.get("expected_final_url")
- and (case.get("expected_document_language") or case.get("target_language"))
+ case.get("expected_document_language") or case.get("target_language")
+ or case.get("expected_document_language_present") is False
)
risk_navigation = (
((case.get("source_benchmark") or {}).get("risk_control") or {}).get(
@@ -183,7 +183,7 @@ def score_attempt(
)
final_consistent = bool(
isinstance(final_url, str) and final_url
- and _same_page_url(final_url, case["expected_final_url"])
+ and _same_page_url(final_url, case.get("expected_final_url") or case["url"])
)
language_consistent = (
observed_language == "" if not language_present else bool(
@@ -248,8 +248,7 @@ def score_attempt(
declared_regions
and len(measured_regions) == len(declared_regions)
and all(region.get("coverage") == 1.0 for region in measured_regions)
- and result.get("content_pass")
- and (not case.get("acceptance_contract") or result.get("stable_task_pass"))
+ and result.get("body_coverage_pass")
)
if ((result["task_kind"] == "body" or declared_regions)
and result["declared_truncation_hits"]
@@ -346,8 +345,7 @@ def acceptance_checks(case: dict, metrics: dict) -> dict[str, bool | None]:
if case.get('content_score_applicable', True):
if case.get('expected_any'):
checks['literals'] = metrics.get('legacy_marker_pass')
- if 'min_content_chars' in case:
- checks['length'] = metrics.get('content_length_pass')
+ checks['length'] = metrics.get('content_length_pass')
if case.get('required_facts'):
checks['facts'] = metrics.get('fact_coverage_pass')
if case.get('required_patterns'):
diff --git a/tests/test_stable_tasks.py b/tests/test_stable_tasks.py
index cd0224f..4324362 100644
--- a/tests/test_stable_tasks.py
+++ b/tests/test_stable_tasks.py
@@ -493,7 +493,7 @@ def test_stable_text_and_structure_are_separate_from_word_overlap(self):
{"id": "section", "alternatives": ["Product catalog", "Available products"]},
{"id": "explanation", "alternatives": ["Prices include tax"]}])
text = "# Available products\nPrices include tax\n\nNew coat\n$19\n\nNew scarf\n$99"
- result = evaluate_content(text, ["Old title"],
+ result = evaluate_content(text, [],
ground_truth_main_content="Old title Old price", acceptance_contract=rules)
self.assertTrue(result["stable_structure_pass"])
self.assertTrue(result["stable_content_pass"])
diff --git a/tests/test_unified_requirements.py b/tests/test_unified_requirements.py
index 96f1ddd..64b14a5 100644
--- a/tests/test_unified_requirements.py
+++ b/tests/test_unified_requirements.py
@@ -54,12 +54,14 @@ def test_absent_requirements_are_skipped_but_missing_required_evidence_is_not():
unavailable = evaluate_response(task, b'No reference or resource returned')
assert not unavailable['passed']
assert not unavailable['measured']
- assert unavailable['metrics']['acceptance_checks'] == {'resources': None}
+ assert unavailable['metrics']['acceptance_checks']['resources'] is None
task = {'url': 'https://example.test/', 'required_facts': [
{'id': 'body', 'alternatives': ['Article body']} ]}
accepted = evaluate_response(task, b'Article body')
assert accepted['passed']
- assert set(accepted['metrics']['acceptance_checks']) == {'content', 'facts'}
+ assert 'facts' in accepted['metrics']['acceptance_checks']
+ assert 'resources' not in accepted['metrics']['acceptance_checks']
+ assert 'file' not in accepted['metrics']['acceptance_checks']
def test_perfect_tree_does_not_override_a_missing_content_requirement():
@@ -109,3 +111,46 @@ def test_declared_body_scope_keeps_truncation_protection_across_output_types(kin
answer = answer.replace('theta iota\n', 'theta iota kappa\n')
result = evaluate_response(task, answer.encode(), files=[b'\x89PNG\r\n\x1a\n'+b'x'*32])
assert result['passed'] is complete
+
+
+@pytest.mark.parametrize('track', ['web', 'dynamic', 'document'])
+@pytest.mark.parametrize('language,passed', [('', True), ('en', False), (None, False)])
+def test_declared_absent_language_tag_is_checked_without_locale_or_redirect_fields(track, language, passed):
+ case = {'url': 'https://example.test/', 'track': track,
+ 'expected_document_language_present': False, 'page_identity_required': True}
+ metrics = score_attempt(case, b'Article body', ok=True, final_url=case['url'],
+ metadata={'document_language': language})
+ assert task_passed(True, metrics) is passed
+
+
+def test_dict_and_dataclass_apply_the_same_default_text_requirement():
+ from lexbench_webfetch.models import EvalCase
+ links = {'links': [{'text': 'Portable speaker', 'href': '/item/1'}]}
+ common = {'id': 'sample', 'name': 'Sample', 'url': 'https://example.test/',
+ 'category': 'test', 'expected_any': [], 'task_kind': 'structure',
+ 'acceptance_contract': CONTRACT}
+ for applicable, passed in [(True, False), (False, True)]:
+ task = {**common, 'content_score_applicable': applicable}
+ for case in (task, EvalCase(**task)):
+ assert task_passed(True, score_attempt(case, b'', ok=True, metadata=links)) is passed
+
+
+def test_low_level_content_checks_keep_declared_literals_and_length():
+ from lexbench_webfetch.metrics import evaluate_content
+ assert evaluate_content('[Portable speaker](/item/1)', [], acceptance_contract=CONTRACT,
+ base_url='https://example.test/')['content_pass']
+ for fields in ({'expected': ['Required introduction']}, {'min_content_chars': 1000}):
+ arguments = {'expected': [], **fields}
+ metrics = evaluate_content('[Portable speaker](/item/1)', acceptance_contract=CONTRACT,
+ base_url='https://example.test/', **arguments)
+ assert not metrics['content_pass']
+
+
+def test_omitted_literal_check_cannot_invalidate_proven_complete_body():
+ from lexbench_webfetch.evaluation import evaluate_response
+ case = {'url': 'https://example.test/', 'acceptance_contract': CONTRACT,
+ 'content_score_applicable': False, 'expected_any': ['Optional page shell'],
+ 'body_requirements': {'version': 1, 'unit': 'region_token_recall',
+ 'minimum_coverage': 0.8, 'regions': [{'id': 'body', 'text': 'alpha beta gamma delta epsilon'}]}}
+ result = evaluate_response(case, b'alpha beta gamma delta epsilon\n[Speaker](/item/1)\n(Content truncated. Fetch a more specific URL or section for the full text.)')
+ assert result['passed']
From 73fd5193cbf0d53971289aaefaded88b070d51f6 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Thu, 22 Oct 2026 15:17:26 +0800
Subject: [PATCH 4/7] fix(scoring): reject silent reinterpretation of frozen
metrics
---
src/lexbench_webfetch/metrics.py | 7 ++++++-
tests/test_unified_requirements.py | 8 ++++++++
2 files changed, 14 insertions(+), 1 deletion(-)
diff --git a/src/lexbench_webfetch/metrics.py b/src/lexbench_webfetch/metrics.py
index 89d1ace..a95d185 100644
--- a/src/lexbench_webfetch/metrics.py
+++ b/src/lexbench_webfetch/metrics.py
@@ -459,6 +459,8 @@ def task_passed(ok: bool, metrics: dict, task_kind: str | None = None) -> bool:
if ('acceptance_policy_version' in metrics or 'forbidden_hits' in metrics
or ('blocked_page' in metrics and 'forbidden_requirement_hits' not in metrics)):
raise ValueError('Historical scoring metrics require their frozen scorer or a complete offline rescore')
+ if 'scoring_source_sha256' in metrics and 'acceptance_checks' not in metrics:
+ raise ValueError('Stored metrics require their frozen scorer or a complete offline rescore')
if metrics.get('request_ok') is False:
return False
if metrics.get('forbidden_requirement_hits') or metrics.get('forbidden_requirement_pass') is False:
@@ -473,7 +475,10 @@ def task_passed(ok: bool, metrics: dict, task_kind: str | None = None) -> bool:
return bool(ok and checks and all(value is True for value in checks.values()))
if "stable_task_pass" in metrics:
combined = (metrics.get("content_pass")
- if metrics.get("content_score_applicable", True) else True)
+ if ("fact_coverage_pass" in metrics
+ or "composite_acceptance_pass" in metrics
+ or metrics.get("body_region_results") or metrics.get("pattern_results"))
+ else True)
return bool(ok and metrics["stable_task_pass"] and combined
and metrics.get("body_coverage_pass") is not False)
field = {'structure':'list_task_pass', 'resource':'resource_pass', 'file':'raw_file_pass'}.get(task_kind, 'content_pass')
diff --git a/tests/test_unified_requirements.py b/tests/test_unified_requirements.py
index 64b14a5..09d5a21 100644
--- a/tests/test_unified_requirements.py
+++ b/tests/test_unified_requirements.py
@@ -154,3 +154,11 @@ def test_omitted_literal_check_cannot_invalidate_proven_complete_body():
'minimum_coverage': 0.8, 'regions': [{'id': 'body', 'text': 'alpha beta gamma delta epsilon'}]}}
result = evaluate_response(case, b'alpha beta gamma delta epsilon\n[Speaker](/item/1)\n(Content truncated. Fetch a more specific URL or section for the full text.)')
assert result['passed']
+
+
+def test_stored_metrics_without_declared_check_results_require_the_frozen_scorer():
+ with pytest.raises(ValueError, match='frozen scorer'):
+ task_passed(True, {'scoring_source_sha256': 'a' * 64,
+ 'stable_task_pass': True, 'content_pass': True})
+ metrics = score_attempt({'url': 'https://example.test/'}, b'Article body', ok=True)
+ assert task_passed(True, metrics)
From 4146808b4f798330a1f912e20b5688612c6750f6 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Thu, 22 Oct 2026 15:43:24 +0800
Subject: [PATCH 5/7] fix(scoring): preserve frozen results and generalize
anchor roles
---
CONTRIBUTING.md | 4 ++
.../evaluation/scoring-regressions.json | 60 +++++++++++++++++++
src/lexbench_webfetch/dynamic_reference.py | 35 +----------
src/lexbench_webfetch/metrics.py | 9 ++-
src/lexbench_webfetch/risk_control_dataset.py | 2 +
src/lexbench_webfetch/stable_tasks.py | 2 -
tests/test_body_text_acceptance_roles.py | 13 ----
tests/test_unified_requirements.py | 35 +++++++++--
8 files changed, 100 insertions(+), 60 deletions(-)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index bc46dbc..7214b0a 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -74,6 +74,10 @@ omitted. Missing evidence for a declared requirement is unavailable and cannot
pass. Full-text overlap remains diagnostic unless an explicit acceptance rule
requires the content.
+A positive record-count requirement can own acceptance while page-shell anchors
+remain diagnostic, using `stable_text_role: "diagnostic"`. The same role applies
+to every record schema.
+
`required_facts` are requirements even when a stable record contract is present.
Do not put obsolete prices, rankings or changing membership in this field just
to obtain diagnostic overlap metrics. Keep historical reference text in its
diff --git a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json
index a791fdc..2711e59 100644
--- a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json
+++ b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json
@@ -57293,6 +57293,66 @@
"final_url": "https://example.test/",
"ok": true,
"expected_task_pass": false
+ },
+ {
+ "id": "diagnostic-shell-with-required-record-record",
+ "category": "structured-extraction",
+ "root_cause": "page_shell_role_tied_to_record_schema",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://example.test/",
+ "acceptance_contract": {
+ "version": 1,
+ "schema": "named_link",
+ "minimum_records": 1,
+ "hosts": [
+ "example.test"
+ ],
+ "entry_path": "/item/\\d+",
+ "stable_text_role": "diagnostic",
+ "stable_text_anchors": [
+ {
+ "id": "shell",
+ "alternatives": [
+ "Account Privacy Terms"
+ ]
+ }
+ ]
+ }
+ },
+ "artifact": "[Speaker](/item/1)",
+ "ok": true,
+ "expected_task_pass": true
+ },
+ {
+ "id": "diagnostic-shell-with-required-record-shell",
+ "category": "structured-extraction",
+ "root_cause": "page_shell_role_tied_to_record_schema",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://example.test/",
+ "acceptance_contract": {
+ "version": 1,
+ "schema": "named_link",
+ "minimum_records": 1,
+ "hosts": [
+ "example.test"
+ ],
+ "entry_path": "/item/\\d+",
+ "stable_text_role": "diagnostic",
+ "stable_text_anchors": [
+ {
+ "id": "shell",
+ "alternatives": [
+ "Account Privacy Terms"
+ ]
+ }
+ ]
+ }
+ },
+ "artifact": "Account Privacy Terms",
+ "ok": true,
+ "expected_task_pass": false
}
],
"superseded_cases": {
diff --git a/src/lexbench_webfetch/dynamic_reference.py b/src/lexbench_webfetch/dynamic_reference.py
index e9c2a61..d96dec3 100644
--- a/src/lexbench_webfetch/dynamic_reference.py
+++ b/src/lexbench_webfetch/dynamic_reference.py
@@ -1,5 +1,4 @@
-"""Field contracts for independently observed, finite dynamic practice pages."""
-import re
+"""Shared field layouts for owned record matching."""
def entry_field_orders(fields):
@@ -24,35 +23,3 @@ def entry_field_orders(fields):
body = tuple(f for f in ('name', 'price', 'financing') if f in fields)
return [body + ('shipping',), ('shipping',) + body] if 'shipping' in fields else [body]
return [layout for layout in layouts if frozenset(layout) == fields]
-
-
-def entry_facts(case_id, entries):
- fields = (['quote', 'author', 'tags'] if case_id in {'dynamic-quotes-api','dynamic-delayed'}
- else ['name','price'] if case_id=='dynamic-products-api'
- else ['price','title','description','reviews'] if case_id=='dynamic-catalog'
- else ['date','text'] if case_id in {'dynamic-reviews-graphql','dynamic-product-reviews'}
- else ['name','description'] if case_id=='dynamic-angular-phones'
- else ['method','path','summary'] if case_id=='dynamic-swagger'
- else ['family'] if case_id=='dynamic-iframe'
- else ['name','price','financing','shipping'] if case_id=='dynamic-react-cart' else None)
- if not fields:raise ValueError('Unknown dynamic case')
- facts=[]
- for i,entry in enumerate(entries,1):
- values=[line.strip() for line in entry.splitlines() if line.strip()]
- entry_fields = fields
- if case_id == 'dynamic-react-cart':
- # Shipping and financing are optional visible fields on these cards.
- entry_fields = ['name', 'price']
- if len(values)>2 and values[2].startswith('or '):entry_fields.append('financing')
- if values[-1]=='Free shipping':entry_fields.append('shipping')
- if len(values)!=len(entry_fields):raise ValueError(f'Incomplete or unexpected fields: {case_id} entry {i}')
- for field,value in zip(entry_fields,values):
- if field=='author':value=value.removeprefix('by ')
- if field=='tags':value=value.removeprefix('Tags: ')
- alternatives=[value]
- # Numeric count + its label can be adjacent in Markdown extraction.
- # Keep the whole number/unit, never remove arbitrary word boundaries.
- if field=='reviews' and re.fullmatch(r'\d+\s+reviews?',value):
- alternatives.append(re.sub(r'\s+','',value))
- facts.append({'id':f'entry-{i}-{field}','alternatives':alternatives})
- return facts
diff --git a/src/lexbench_webfetch/metrics.py b/src/lexbench_webfetch/metrics.py
index a95d185..625e55d 100644
--- a/src/lexbench_webfetch/metrics.py
+++ b/src/lexbench_webfetch/metrics.py
@@ -459,10 +459,6 @@ def task_passed(ok: bool, metrics: dict, task_kind: str | None = None) -> bool:
if ('acceptance_policy_version' in metrics or 'forbidden_hits' in metrics
or ('blocked_page' in metrics and 'forbidden_requirement_hits' not in metrics)):
raise ValueError('Historical scoring metrics require their frozen scorer or a complete offline rescore')
- if 'scoring_source_sha256' in metrics and 'acceptance_checks' not in metrics:
- raise ValueError('Stored metrics require their frozen scorer or a complete offline rescore')
- if metrics.get('request_ok') is False:
- return False
if metrics.get('forbidden_requirement_hits') or metrics.get('forbidden_requirement_pass') is False:
return False
if (metrics.get('page_identity_required')
@@ -472,7 +468,8 @@ def task_passed(ok: bool, metrics: dict, task_kind: str | None = None) -> bool:
return False
if "acceptance_checks" in metrics:
checks = metrics["acceptance_checks"]
- return bool(ok and checks and all(value is True for value in checks.values()))
+ return bool(ok and metrics.get("request_ok", ok) and checks
+ and all(value is True for value in checks.values()))
if "stable_task_pass" in metrics:
combined = (metrics.get("content_pass")
if ("fact_coverage_pass" in metrics
@@ -861,6 +858,8 @@ def evaluate_content(
body_requirements
or (composite_acceptance and composite_acceptance.get("groups"))
or acceptance_contract.get("page_patterns")
+ or (type(acceptance_contract.get("minimum_records")) is int
+ and acceptance_contract["minimum_records"] > 0)
):
raise ValueError(
"Diagnostic stable text requires body, composite, or page-pattern acceptance"
diff --git a/src/lexbench_webfetch/risk_control_dataset.py b/src/lexbench_webfetch/risk_control_dataset.py
index 16dbd0a..54333fd 100644
--- a/src/lexbench_webfetch/risk_control_dataset.py
+++ b/src/lexbench_webfetch/risk_control_dataset.py
@@ -53,6 +53,8 @@ def assign_content_acceptance_roles(cases: list[dict]) -> None:
and case["composite_acceptance"].get("groups")
)
or contract.get("page_patterns")
+ or (type(contract.get("minimum_records")) is int
+ and contract["minimum_records"] > 0)
):
raise ValueError(
"Diagnostic stable text requires body, composite, or page-pattern acceptance"
diff --git a/src/lexbench_webfetch/stable_tasks.py b/src/lexbench_webfetch/stable_tasks.py
index c6867f4..8a58911 100644
--- a/src/lexbench_webfetch/stable_tasks.py
+++ b/src/lexbench_webfetch/stable_tasks.py
@@ -805,8 +805,6 @@ def validate_acceptance_contract(contract: dict) -> None:
raise ValueError("Unknown stable acceptance contract fields")
if contract.get("stable_text_role", "required") not in {"required", "diagnostic"}:
raise ValueError("Invalid stable text role")
- if contract.get("stable_text_role") == "diagnostic" and contract["schema"] != "body_text":
- raise ValueError("Diagnostic stable text requires a body-text contract")
if contract.get('content_scope', 'legacy') not in {'legacy', 'target_content', 'full_page_stable_coverage'}:
raise ValueError('Invalid stable content scope')
if contract.get('content_scope') == 'target_content' and any(
diff --git a/tests/test_body_text_acceptance_roles.py b/tests/test_body_text_acceptance_roles.py
index 399c21b..7d948e3 100644
--- a/tests/test_body_text_acceptance_roles.py
+++ b/tests/test_body_text_acceptance_roles.py
@@ -84,19 +84,6 @@ def test_empty_composite_owner_cannot_make_stable_text_diagnostic():
assign_content_acceptance_roles([case])
-def test_structured_contract_cannot_relax_its_stable_text():
- contract = {
- "version": 1,
- "schema": "named_link",
- "minimum_records": 1,
- "hosts": ["example.test"],
- "entry_path": "/item/[0-9]+",
- "stable_text_role": "diagnostic",
- }
- with pytest.raises(ValueError, match="body-text"):
- validate_acceptance_contract(contract)
-
-
def test_compiler_policy_does_not_infer_diagnostic_from_gate_presence():
body = body_case()
del body["acceptance_contract"]["stable_text_role"]
diff --git a/tests/test_unified_requirements.py b/tests/test_unified_requirements.py
index 09d5a21..f6fe2a5 100644
--- a/tests/test_unified_requirements.py
+++ b/tests/test_unified_requirements.py
@@ -156,9 +156,32 @@ def test_omitted_literal_check_cannot_invalidate_proven_complete_body():
assert result['passed']
-def test_stored_metrics_without_declared_check_results_require_the_frozen_scorer():
- with pytest.raises(ValueError, match='frozen scorer'):
- task_passed(True, {'scoring_source_sha256': 'a' * 64,
- 'stable_task_pass': True, 'content_pass': True})
- metrics = score_attempt({'url': 'https://example.test/'}, b'Article body', ok=True)
- assert task_passed(True, metrics)
+def test_frozen_metric_projection_keeps_its_recorded_acceptance_semantics():
+ frozen = {'scoring_source_sha256': 'a' * 64, 'stable_task_pass': True,
+ 'content_pass': False, 'content_score_applicable': False,
+ 'stable_anchor_results': [{'pass': True}]}
+ assert task_passed(True, frozen)
+ # A failed declared pattern remained a failure in the historical format.
+ assert not task_passed(True, {**frozen, 'pattern_results': [{'pass': False}]})
+ current = score_attempt({'url': 'https://example.test/',
+ 'acceptance_contract': CONTRACT, 'required_facts': [
+ {'id': 'required', 'alternatives': ['Battery lasts twelve hours']}]},
+ b'[Speaker](/item/1)', ok=True)
+ assert not task_passed(True, current)
+ assert 'acceptance_checks' not in frozen
+
+
+@pytest.mark.parametrize('answer,passed', [
+ ('[Speaker](/item/1)', True), ('Account Privacy Terms', False)])
+def test_page_shell_literals_can_be_diagnostic_when_records_own_acceptance(answer, passed):
+ case = {'url': 'https://example.test/', 'acceptance_contract': {
+ **CONTRACT, 'stable_text_role': 'diagnostic',
+ 'stable_text_anchors': [{'id': 'shell', 'alternatives': ['Account Privacy Terms']}]}}
+ assert task_passed(True, score_attempt(case, answer.encode(), ok=True)) is passed
+
+
+def test_imported_tasks_apply_the_same_record_owned_acceptance_role():
+ from lexbench_webfetch.risk_control_dataset import assign_content_acceptance_roles
+ case = {'acceptance_contract': {**CONTRACT, 'stable_text_role': 'diagnostic'}}
+ assign_content_acceptance_roles([case])
+ assert case['acceptance_contract']['stable_text_role'] == 'diagnostic'
From 926714ae75f35740f9bda7fef496a9159844c3f4 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Thu, 22 Oct 2026 15:52:32 +0800
Subject: [PATCH 6/7] fix(scoring): skip undeclared record requirements
---
.../evaluation/scoring-regressions.json | 42 +++++++++++++++++++
src/lexbench_webfetch/scoring.py | 7 ++--
tests/test_unified_requirements.py | 10 +++++
3 files changed, 56 insertions(+), 3 deletions(-)
diff --git a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json
index 2711e59..7fa0aa2 100644
--- a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json
+++ b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json
@@ -57353,6 +57353,48 @@
"artifact": "Account Privacy Terms",
"ok": true,
"expected_task_pass": false
+ },
+ {
+ "id": "facts-without-undeclared-records-complete",
+ "category": "structured-extraction",
+ "root_cause": "undeclared_record_check_from_output_kind",
+ "failure_mode": "false-negative",
+ "case": {
+ "url": "https://example.test/",
+ "task_kind": "structure",
+ "required_facts": [
+ {
+ "id": "duration",
+ "alternatives": [
+ "Battery lasts twelve hours"
+ ]
+ }
+ ]
+ },
+ "artifact": "Battery lasts twelve hours",
+ "ok": true,
+ "expected_task_pass": true
+ },
+ {
+ "id": "facts-without-undeclared-records-missing",
+ "category": "structured-extraction",
+ "root_cause": "undeclared_record_check_from_output_kind",
+ "failure_mode": "false-positive",
+ "case": {
+ "url": "https://example.test/",
+ "task_kind": "structure",
+ "required_facts": [
+ {
+ "id": "duration",
+ "alternatives": [
+ "Battery lasts twelve hours"
+ ]
+ }
+ ]
+ },
+ "artifact": "Home Terms",
+ "ok": true,
+ "expected_task_pass": false
}
],
"superseded_cases": {
diff --git a/src/lexbench_webfetch/scoring.py b/src/lexbench_webfetch/scoring.py
index 1bbdd17..c90eb73 100644
--- a/src/lexbench_webfetch/scoring.py
+++ b/src/lexbench_webfetch/scoring.py
@@ -339,9 +339,10 @@ def acceptance_checks(case: dict, metrics: dict) -> dict[str, bool | None]:
checks = {}
if case.get('acceptance_contract'):
checks['records_and_anchors'] = metrics.get('stable_task_pass')
- elif kind in {'body', 'structure'}:
- checks['content' if kind == 'body' else 'records'] = metrics.get(
- 'content_pass' if kind == 'body' else 'list_task_pass')
+ elif (case.get('source_benchmark') or {}).get('mode') == 'dynamic_list_structure':
+ checks['records'] = metrics.get('list_task_pass')
+ elif kind == 'body':
+ checks['content'] = metrics.get('content_pass')
if case.get('content_score_applicable', True):
if case.get('expected_any'):
checks['literals'] = metrics.get('legacy_marker_pass')
diff --git a/tests/test_unified_requirements.py b/tests/test_unified_requirements.py
index f6fe2a5..a3ee159 100644
--- a/tests/test_unified_requirements.py
+++ b/tests/test_unified_requirements.py
@@ -185,3 +185,13 @@ def test_imported_tasks_apply_the_same_record_owned_acceptance_role():
case = {'acceptance_contract': {**CONTRACT, 'stable_text_role': 'diagnostic'}}
assign_content_acceptance_roles([case])
assert case['acceptance_contract']['stable_text_role'] == 'diagnostic'
+
+
+@pytest.mark.parametrize('kind', ['body', 'structure'])
+@pytest.mark.parametrize('answer,passed', [('Battery lasts twelve hours', True), ('Home Terms', False)])
+def test_facts_do_not_require_an_undeclared_record_check(kind, answer, passed):
+ case = {'url': 'https://example.test/', 'task_kind': kind,
+ 'required_facts': [{'id': 'duration', 'alternatives': ['Battery lasts twelve hours']}]}
+ result = score_attempt(case, answer.encode(), ok=True)
+ assert task_passed(True, result) is passed
+ assert 'records' not in result['acceptance_checks']
From 28d757fc4cf9558e4003f7e1f1e9cd721d5e43e4 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Thu, 22 Oct 2026 22:45:02 +0800
Subject: [PATCH 7/7] test(scoring): cover post checks in shared diagnostics
---
tests/test_article_post_coverage_integration.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/tests/test_article_post_coverage_integration.py b/tests/test_article_post_coverage_integration.py
index 75fa610..a45f16d 100644
--- a/tests/test_article_post_coverage_integration.py
+++ b/tests/test_article_post_coverage_integration.py
@@ -25,12 +25,16 @@ def test_current_collection_coverage_without_runtime_reference(number, minimum):
assert case['source_benchmark']['post_contract']['minimum_posts'] == minimum
full = evaluate_response(case, (stable+'\n\n'+'\n\n'.join(posts)).encode())
assert full['passed'] and full['measured']
+ assert full['metrics']['acceptance_checks']['article_posts'] is True
assert 'article_reference_status' not in full['metrics']
assert full['metrics']['content_decision_basis'] == 'base_acceptance_and_required_components'
- for text in (stable, stable+'\n\n'+'\n\n'.join(posts[:-1]), '\n\n'.join(posts)):
+ for text, posts_pass in ((stable, False),
+ (stable+'\n\n'+'\n\n'.join(posts[:-1]), False),
+ ('\n\n'.join(posts), True)):
result = evaluate_response(case, text.encode())
assert not result['passed']
assert result['measured']
+ assert result['metrics']['acceptance_checks']['article_posts'] is posts_pass
def test_cli_accepts_collection_without_reference_arguments(tmp_path):