From c9183abce7cdecc377e229a6c323add1d8cdbc13 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:08:09 +0800 Subject: [PATCH] Fix acceptance isolation and explicit page identity contracts --- README.md | 10 ++ docs/README.zh-CN.md | 10 ++ pyproject.toml | 2 +- src/lexbench_webfetch/benchmark.py | 3 +- .../evaluation/scoring-regressions.json | 50 +++++- src/lexbench_webfetch/metrics.py | 28 +++- src/lexbench_webfetch/models.py | 2 + src/lexbench_webfetch/scoring.py | 72 +++++---- tests/test_acceptance_isolation.py | 146 ++++++++++++++++++ uv.lock | 4 +- 10 files changed, 287 insertions(+), 40 deletions(-) create mode 100644 tests/test_acceptance_isolation.py diff --git a/README.md b/README.md index ffd47d9..17d45d8 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,16 @@ uv run --frozen browser-harness-eval run \ Run these two commands from Browser Eval. Ordinary runs are diagnostic; a formal live-site comparison additionally requires its pre-call/post-call reference certification. The benchmark input checker verifies frozen bytes, not present-day website validity. See the [Browser Eval integration guide](https://github.com/lexmount/browser-eval/blob/main/docs/benchmarks.md). +## Scoring behavior + +Version 0.3 matches Latin words at word boundaries for facts, markers and stable anchors. Required body facts use the returned content; a separate document title cannot supply missing body text. Chinese phrases retain literal matching, and numeric and currency checks keep their value boundaries. + +A task can set `page_identity_required: true` to require its final URL observations. Missing or mismatched observations then fail acceptance. Dynamic tasks can also set `document_language_match: "exact"` when the complete language tag matters; the default is `"primary"`. These checks apply across interfaces. Frozen tasks that do not require URL metadata keep their content-based acceptance rules. + +Offline scoring recomputes derived fields. It retains the original reference hash, deadline evidence and file fingerprints. A file verdict requires current file verification; an old pass flag cannot certify a rescore. Summaries expose `mean_all_attempts_ms`, `all_attempts_p50_ms` and `all_attempts_p95_ms` alongside the existing successful-call timings. The all-attempt population includes failed calls and timeouts. + +These changes do not revise frozen tasks or certify historical results. Score corrections require a new evidence revision tied to the scorer and its regression controls. + ## Repository contents | Path | Purpose | diff --git a/docs/README.zh-CN.md b/docs/README.zh-CN.md index a34e08b..d22c41d 100644 --- a/docs/README.zh-CN.md +++ b/docs/README.zh-CN.md @@ -106,6 +106,16 @@ uv run --frozen browser-harness-eval run \ 普通运行用于诊断。正式比较实时网站时，还需要完成调用前后的参考内容认证。输入校验只能证明文件版本相符，不能证明网站今天仍然符合旧参考。详见 [Browser Eval 接入说明](https://github.com/lexmount/browser-eval/blob/main/docs/benchmarks.md)。 +## 评分规则 + +0.3 版对事实、标记和固定锚点中的拉丁文使用单词边界匹配。正文事实只匹配返回内容，单独返回的文档标题不能补足正文。中文短语仍按原文匹配，数字和货币保留原有的数值边界检查。 + +任务可设置 `page_identity_required: true`，要求返回的最终网址符合任务声明；缺少网址或网址不符都会判为未通过。动态任务需要区分完整语言标签时，可再设置 `document_language_match: "exact"`，默认值为 `"primary"`，只比较主语言。这些检查适用于所有接口。没有要求网址元数据的冻结任务，继续按原有内容规则验收。 + +离线评分会重新计算派生字段，保留原始参考哈希、调用时限证据和文件指纹。文件结论需要当前文件验收依据；旧的通过标记不能认证本次重评分。汇总在原有成功调用耗时之外，提供全部调用的 `mean_all_attempts_ms`、`all_attempts_p50_ms` 和 `all_attempts_p95_ms`，包含失败调用与超时。 + +上述修改不改写冻结题目，也不自动认证历史成绩。更正分数仍需建立新的证据修订，绑定评分器版本和回归用例。 + ## 仓库内容 | 路径 | 用途 | diff --git a/pyproject.toml b/pyproject.toml index d3d39a7..18e2499 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "lexbench-webfetch" -version = "0.2.0" +version = "0.3.0" description = "Versioned Web Fetch tasks, acceptance contracts and frozen reference bindings" readme = "README.md" requires-python = ">=3.11" diff --git a/src/lexbench_webfetch/benchmark.py b/src/lexbench_webfetch/benchmark.py index f8b7be4..d3fdfe5 100644 --- a/src/lexbench_webfetch/benchmark.py +++ b/src/lexbench_webfetch/benchmark.py @@ -20,7 +20,8 @@ "fact_coverage_threshold", "expected_resources", "resource_output_contract", "content_score_applicable", "required_patterns", "acceptance_contract", "composite_acceptance", "mutable_body_policy", "body_requirements", "retrieval_hints", - "source_review_url", "evaluation_url", "expected_final_url", "expected_document_language", + "source_review_url", "evaluation_url", "expected_final_url", "page_identity_required", + "expected_document_language", "document_language_match", "expected_document_language_present", "target_language", "target_region", ) diff --git a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json index 378cdf5..8a57b2e 100644 --- a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json +++ b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json @@ -24225,9 +24225,57 @@ {"id": "html-record-nested-title-list-uses-enclosing-card", "category": "structured-extraction", "root_cause": "html_nested_title_layout_owner", "failure_mode": "false-negative", "case": {"url": "https://example.test/catalog", "task_kind": "body", "min_content_chars": 1, "composite_acceptance": {"version": 1, "groups": [{"id": "houses", "kind": "named_link", "minimum_records": 1, "hosts": ["example.test"], "entry_path": "/item/[0-9]+", "required_fields": [{"id": "price", "kind": "price"}, {"id": "area", "kind": "area"}]}]}}, "artifact": "

$100,000

900 sqft

", "ok": true, "expected_task_pass": true, "expected_metrics": {"composite_acceptance_pass": true}}, {"id": "html-record-nested-distinct-title-stays-independent", "category": "structured-extraction", "root_cause": "html_nested_title_layout_owner", "failure_mode": "false-positive", "case": {"url": "https://example.test/catalog", "task_kind": "body", "min_content_chars": 1, "composite_acceptance": {"version": 1, "groups": [{"id": "houses", "kind": "named_link", "minimum_records": 2, "hosts": ["example.test"], "entry_path": "/item/[0-9]+", "required_fields": [{"id": "price", "kind": "price"}, {"id": "area", "kind": "area"}]}]}}, "artifact": "

$100,000

900 sqft

", "ok": true, "expected_task_pass": false, "expected_metrics": {"composite_acceptance_pass": false}}, {"id": "html-record-duplicate-identity-validates-before-dedup", "category": "structured-extraction", "root_cause": "html_record_validation_before_identity_deduplication", "failure_mode": "false-negative", "case": {"url": "https://example.test/catalog", "task_kind": "body", "min_content_chars": 1, "composite_acceptance": {"version": 1, "groups": [{"id": "houses", "kind": "named_link", "minimum_records": 1, "hosts": ["example.test"], "entry_path": "/item/[0-9]+", "required_fields": [{"id": "price", "kind": "price"}, {"id": "area", "kind": "area"}]}]}}, "artifact": "
First House

$100,000

First House

$100,000

900 sqft

", "ok": true, "expected_task_pass": true, "expected_metrics": {"composite_acceptance_pass": true}}, - {"id": "html-record-duplicate-identity-cannot-combine-name-and-fields", "category": "structured-extraction", "root_cause": "html_record_validation_before_identity_deduplication", "failure_mode": "false-positive", "case": {"url": "https://example.test/catalog", "task_kind": "body", "min_content_chars": 1, "composite_acceptance": {"version": 1, "groups": [{"id": "houses", "kind": "named_link", "minimum_records": 1, "hosts": ["example.test"], "entry_path": "/item/[0-9]+", "required_fields": [{"id": "price", "kind": "price"}, {"id": "area", "kind": "area"}]}]}}, "artifact": "
First House

$100,000

900 sqft

First House
", "ok": true, "expected_task_pass": false, "expected_metrics": {"composite_acceptance_pass": false}} + {"id": "html-record-duplicate-identity-cannot-combine-name-and-fields", "category": "structured-extraction", "root_cause": "html_record_validation_before_identity_deduplication", "failure_mode": "false-positive", "case": {"url": "https://example.test/catalog", "task_kind": "body", "min_content_chars": 1, "composite_acceptance": {"version": 1, "groups": [{"id": "houses", "kind": "named_link", "minimum_records": 1, "hosts": ["example.test"], "entry_path": "/item/[0-9]+", "required_fields": [{"id": "price", "kind": "price"}, {"id": "area", "kind": "area"}]}]}}, "artifact": "
First House

$100,000

900 sqft

First House
", "ok": true, "expected_task_pass": false, "expected_metrics": {"composite_acceptance_pass": false}}, + {"id": "body-only-word-boundary-answer-title-supplies-required-fact", "category": "answer-surface", "root_cause": "metadata_title_supplies_missing_body_fact", "failure_mode": "false-positive", "case": {"url": "https://example.test/page", "required_facts": [{"id": "title", "alternatives": ["Exact page title"]}]}, "artifact": "Body", "title": "Exact page title", "ok": true, "expected_task_pass": false, "expected_metrics": {"fact_coverage": 0.0, "content_chars": 4}}, + {"id": "body-only-word-boundary-answer-title-does-not-inflate-body-length", "category": "answer-surface", "root_cause": "answer_surface_omission", "failure_mode": "false-positive", "case": {"url": "https://example.test/page", "required_facts": [{"id": "title", "alternatives": ["Exact page title"]}], "min_content_chars": 20}, "artifact": "Body", "title": "Exact page title", "ok": true, "expected_task_pass": false, "expected_metrics": {"fact_coverage": 0.0, "length_pass": false, "content_chars": 4}}, + {"id": "body-only-word-boundary-archive-news-and-footer-cannot-replace-introduction", "category": "acceptance-contract", "root_cause": "archive_introduction_scope_not_legacy_ui_copy", "failure_mode": "false-positive", "case": {"url": "https://example.test/archive", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "body_text", "stable_text_role": "diagnostic", "minimum_records": 0, "page_patterns": ["(?is)\\bibiblio(?:\\.org)?\\b.{0,500}\\b(?:collections? of collections|public digital (?:library|archive)|digital (?:library|archive|repository))\\b.{0,700}\\b(?:freely available information|free(?:ly)? available (?:digital )?(?:information|resources|collections))\\b.{0,700}\\b(?:software|music|literature|art|history|science|politics|cultural studies)\\b.{0,700}\\b(?:UNC|University of North Carolina(?: at|-)? Chapel Hill)\\b"], "stable_text_anchors": [{"id": "legacy-navigation", "alternatives": ["Logos"]}]}}, "artifact": "ibiblio News\\nFirst update\\nSecond update\\nThird update\\nLogos\\nHosted by UNC Chapel Hill", "ok": true, "expected_task_pass": false, "expected_metrics": {"stable_task_pass": false, "stable_anchors_required": false, "stable_anchor_hits": 0}}, + {"id": "latin-boundary-fact-Home-whole", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-negative", "case": {"url": "https://example.test/article", "expected_any": [], "required_facts": [{"id": "answer", "alternatives": ["Home"]}]}, "artifact": "(Home)", "ok": true, "expected_task_pass": true}, + {"id": "latin-boundary-fact-Home-fragment", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-positive", "case": {"url": "https://example.test/article", "expected_any": [], "required_facts": [{"id": "answer", "alternatives": ["Home"]}]}, "artifact": "homepage", "ok": true, "expected_task_pass": false}, + {"id": "latin-boundary-marker-Home-whole", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-negative", "case": {"url": "https://example.test/article", "expected_any": ["Home"]}, "artifact": "(Home)", "ok": true, "expected_task_pass": true}, + {"id": "latin-boundary-marker-Home-fragment", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-positive", "case": {"url": "https://example.test/article", "expected_any": ["Home"]}, "artifact": "homepage", "ok": true, "expected_task_pass": false}, + {"id": "latin-boundary-anchor-Home-whole", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-negative", "case": {"url": "https://example.test/article", "expected_any": [], "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Home"]}]}}, "artifact": "(Home)", "ok": true, "expected_task_pass": true}, + {"id": "latin-boundary-anchor-Home-fragment", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-positive", "case": {"url": "https://example.test/article", "expected_any": [], "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Home"]}]}}, "artifact": "homepage", "ok": true, "expected_task_pass": false}, + {"id": "latin-boundary-fact-Live-whole", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-negative", "case": {"url": "https://example.test/article", "expected_any": [], "required_facts": [{"id": "answer", "alternatives": ["Live"]}]}, "artifact": "(Live)", "ok": true, "expected_task_pass": true}, + {"id": "latin-boundary-fact-Live-fragment", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-positive", "case": {"url": "https://example.test/article", "expected_any": [], "required_facts": [{"id": "answer", "alternatives": ["Live"]}]}, "artifact": "deliver", "ok": true, "expected_task_pass": false}, + {"id": "latin-boundary-marker-Live-whole", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-negative", "case": {"url": "https://example.test/article", "expected_any": ["Live"]}, "artifact": "(Live)", "ok": true, "expected_task_pass": true}, + {"id": "latin-boundary-marker-Live-fragment", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-positive", "case": {"url": "https://example.test/article", "expected_any": ["Live"]}, "artifact": "deliver", "ok": true, "expected_task_pass": false}, + {"id": "latin-boundary-anchor-Live-whole", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-negative", "case": {"url": "https://example.test/article", "expected_any": [], "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Live"]}]}}, "artifact": "(Live)", "ok": true, "expected_task_pass": true}, + {"id": "latin-boundary-anchor-Live-fragment", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-positive", "case": {"url": "https://example.test/article", "expected_any": [], "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Live"]}]}}, "artifact": "deliver", "ok": true, "expected_task_pass": false}, + {"id": "latin-boundary-fact-News-whole", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-negative", "case": {"url": "https://example.test/article", "expected_any": [], "required_facts": [{"id": "answer", "alternatives": ["News"]}]}, "artifact": "(News)", "ok": true, "expected_task_pass": true}, + {"id": "latin-boundary-fact-News-fragment", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-positive", "case": {"url": "https://example.test/article", "expected_any": [], "required_facts": [{"id": "answer", "alternatives": ["News"]}]}, "artifact": "newsletter", "ok": true, "expected_task_pass": false}, + {"id": "latin-boundary-marker-News-whole", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-negative", "case": {"url": "https://example.test/article", "expected_any": ["News"]}, "artifact": "(News)", "ok": true, "expected_task_pass": true}, + {"id": "latin-boundary-marker-News-fragment", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-positive", "case": {"url": "https://example.test/article", "expected_any": ["News"]}, "artifact": "newsletter", "ok": true, "expected_task_pass": false}, + {"id": "latin-boundary-anchor-News-whole", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-negative", "case": {"url": "https://example.test/article", "expected_any": [], "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["News"]}]}}, "artifact": "(News)", "ok": true, "expected_task_pass": true}, + {"id": "latin-boundary-anchor-News-fragment", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-positive", "case": {"url": "https://example.test/article", "expected_any": [], "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["News"]}]}}, "artifact": "newsletter", "ok": true, "expected_task_pass": false}, + {"id": "latin-boundary-fact-Print-whole", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-negative", "case": {"url": "https://example.test/article", "expected_any": [], "required_facts": [{"id": "answer", "alternatives": ["Print"]}]}, "artifact": "(Print)", "ok": true, "expected_task_pass": true}, + {"id": "latin-boundary-fact-Print-fragment", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-positive", "case": {"url": "https://example.test/article", "expected_any": [], "required_facts": [{"id": "answer", "alternatives": ["Print"]}]}, "artifact": "blueprint", "ok": true, "expected_task_pass": false}, + {"id": "latin-boundary-marker-Print-whole", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-negative", "case": {"url": "https://example.test/article", "expected_any": ["Print"]}, "artifact": "(Print)", "ok": true, "expected_task_pass": true}, + {"id": "latin-boundary-marker-Print-fragment", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-positive", "case": {"url": "https://example.test/article", "expected_any": ["Print"]}, "artifact": "blueprint", "ok": true, "expected_task_pass": false}, + {"id": "latin-boundary-anchor-Print-whole", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-negative", "case": {"url": "https://example.test/article", "expected_any": [], "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Print"]}]}}, "artifact": "(Print)", "ok": true, "expected_task_pass": true}, + {"id": "latin-boundary-anchor-Print-fragment", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-positive", "case": {"url": "https://example.test/article", "expected_any": [], "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Print"]}]}}, "artifact": "blueprint", "ok": true, "expected_task_pass": false}, + {"id": "latin-boundary-fact-url-whole", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-negative", "case": {"url": "https://example.test/article", "expected_any": [], "required_facts": [{"id": "answer", "alternatives": ["url"]}]}, "artifact": "(url)", "ok": true, "expected_task_pass": true}, + {"id": "latin-boundary-fact-url-fragment", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-positive", "case": {"url": "https://example.test/article", "expected_any": [], "required_facts": [{"id": "answer", "alternatives": ["url"]}]}, "artifact": "curl", "ok": true, "expected_task_pass": false}, + {"id": "latin-boundary-marker-url-whole", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-negative", "case": {"url": "https://example.test/article", "expected_any": ["url"]}, "artifact": "(url)", "ok": true, "expected_task_pass": true}, + {"id": "latin-boundary-marker-url-fragment", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-positive", "case": {"url": "https://example.test/article", "expected_any": ["url"]}, "artifact": "curl", "ok": true, "expected_task_pass": false}, + {"id": "latin-boundary-anchor-url-whole", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-negative", "case": {"url": "https://example.test/article", "expected_any": [], "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["url"]}]}}, "artifact": "(url)", "ok": true, "expected_task_pass": true}, + {"id": "latin-boundary-anchor-url-fragment", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-positive", "case": {"url": "https://example.test/article", "expected_any": [], "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["url"]}]}}, "artifact": "curl", "ok": true, "expected_task_pass": false}, + {"id": "latin-boundary-fact-café-whole", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-negative", "case": {"url": "https://example.test/article", "expected_any": [], "required_facts": [{"id": "answer", "alternatives": ["café"]}]}, "artifact": "(café)", "ok": true, "expected_task_pass": true}, + {"id": "latin-boundary-fact-café-fragment", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-positive", "case": {"url": "https://example.test/article", "expected_any": [], "required_facts": [{"id": "answer", "alternatives": ["café"]}]}, "artifact": "caféteria", "ok": true, "expected_task_pass": false}, + {"id": "latin-boundary-marker-café-whole", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-negative", "case": {"url": "https://example.test/article", "expected_any": ["café"]}, "artifact": "(café)", "ok": true, "expected_task_pass": true}, + {"id": "latin-boundary-marker-café-fragment", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-positive", "case": {"url": "https://example.test/article", "expected_any": ["café"]}, "artifact": "caféteria", "ok": true, "expected_task_pass": false}, + {"id": "latin-boundary-anchor-café-whole", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-negative", "case": {"url": "https://example.test/article", "expected_any": [], "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["café"]}]}}, "artifact": "(café)", "ok": true, "expected_task_pass": true}, + {"id": "latin-boundary-anchor-café-fragment", "category": "format-normalization", "root_cause": "latin_word_fragment_false_match", "failure_mode": "false-positive", "case": {"url": "https://example.test/article", "expected_any": [], "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["café"]}]}}, "artifact": "caféteria", "ok": true, "expected_task_pass": false}, + {"id": "body-fact-title-absent", "category": "answer-surface", "root_cause": "metadata_title_supplies_missing_body_fact", "failure_mode": "false-positive", "case": {"url": "https://example.test/article", "required_facts": [{"id": "install", "alternatives": ["widget --start"]}]}, "artifact": "unrelated text", "title": "widget --start", "ok": true, "expected_task_pass": false}, + {"id": "body-fact-title-present", "category": "answer-surface", "root_cause": "metadata_title_supplies_missing_body_fact", "failure_mode": "false-negative", "case": {"url": "https://example.test/article", "required_facts": [{"id": "install", "alternatives": ["widget --start"]}]}, "artifact": "widget --start", "title": "widget --start", "ok": true, "expected_task_pass": true}, + {"id": "query-sensitive-dynamic-matching-final-page-and-language-remain-eligible", "category": "acceptance-contract", "root_cause": "undeclared_query_parameters_may_select_other_content", "failure_mode": "false-negative", "case": {"url": "https://example.test/start", "evaluation_url": "https://example.test/start", "expected_final_url": "https://example.test/en/catalog", "target_language": "en-US", "target_region": "US", "track": "dynamic", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "identity", "alternatives": ["Catalog"]}]}}, "artifact": "Catalog", "ok": true, "final_url": "https://example.test/en/catalog?session=123#items", "metadata": {"route": "dynamic_selected_text", "document_language": "en"}, "expected_task_pass": true, "expected_metrics": {"page_identity_status": "mismatch", "final_url_consistent": false, "language_consistent": true}}, + {"id": "answer-body-fact-remains-valid-with-an-unrelated-document-title", "category": "answer-surface", "root_cause": "answer_surface_omission", "failure_mode": "false-negative", "case": {"url": "https://example.test/page", "required_facts": [{"id": "heading", "alternatives": ["Exact page title"]}]}, "artifact": "Exact page title\nRequired body", "title": "Unrelated title metadata", "ok": true, "expected_task_pass": true, "expected_metrics": {"fact_coverage": 1.0}} ], "superseded_cases": { + "dynamic-matching-final-page-and-language-remain-eligible": "query-sensitive-dynamic-matching-final-page-and-language-remain-eligible", + "answer-title-supplies-required-fact": "body-only-word-boundary-answer-title-supplies-required-fact", + "answer-title-does-not-inflate-body-length": "body-only-word-boundary-answer-title-does-not-inflate-body-length", + "archive-news-and-footer-cannot-replace-introduction": "body-only-word-boundary-archive-news-and-footer-cannot-replace-introduction", + "owned-specific-image-alt-is-an-accessible-record-name": "specific-image-alternative-does-not-create-visible-record-name", "nested-image-accessible-names-identify-linked-records": "image-alternatives-do-not-create-visible-record-names", "dynamic-final-page-mismatch-cannot-be-counted-as-a-pass": "content-policy-dynamic-final-page-mismatch-cannot-be-counted-as-a-pass", diff --git a/src/lexbench_webfetch/metrics.py b/src/lexbench_webfetch/metrics.py index 9038d86..372974c 100644 --- a/src/lexbench_webfetch/metrics.py +++ b/src/lexbench_webfetch/metrics.py @@ -223,8 +223,16 @@ def _inline_frame_candidates( URLs that the browser serialized, and ignores hidden synchronization or ad frames. The same parser applies to any provider returning inline frames. """ + if not re.search(r' bool: if (not task_measured(metrics, task_kind) or metrics.get('deadline_overrun') or metrics.get('blocked_page')): return False + if metrics.get('page_identity_required') and metrics.get('page_identity_status') != 'verified': + return False if "stable_task_pass" in metrics: combined = (metrics.get("content_pass") if ("composite_acceptance_pass" in metrics @@ -566,6 +576,13 @@ def fact_in_text(text: str, fact: str) -> bool: return False left = r'(? dict[str, Any]: "mean_content_ms": round(statistics.mean(content_successful_elapsed), 2) if content_successful_elapsed else None, "transport_p50_ms": percentile(transport_successful_elapsed, 0.50), "transport_p95_ms": percentile(transport_successful_elapsed, 0.95), + "mean_all_attempts_ms": round(statistics.mean(elapsed), 2), + "all_attempts_p50_ms": percentile(elapsed, 0.50), "all_attempts_p95_ms": percentile(elapsed, 0.95), "median_content_chars": round( statistics.median(item["metrics"]["content_chars"] for item in items), 1 diff --git a/src/lexbench_webfetch/models.py b/src/lexbench_webfetch/models.py index ab86ad6..c589625 100644 --- a/src/lexbench_webfetch/models.py +++ b/src/lexbench_webfetch/models.py @@ -46,7 +46,9 @@ class EvalCase: source_review_url: str = "" evaluation_url: str = "" expected_final_url: str = "" + page_identity_required: bool = False expected_document_language: str = "" + document_language_match: str = "primary" expected_document_language_present: bool = True target_language: str = "" target_region: str = "" diff --git a/src/lexbench_webfetch/scoring.py b/src/lexbench_webfetch/scoring.py index b0ef47f..aae8665 100644 --- a/src/lexbench_webfetch/scoring.py +++ b/src/lexbench_webfetch/scoring.py @@ -42,39 +42,37 @@ def load_list_reference(case: EvalCase | dict[str, Any], source: Path) -> dict | return None -# These conditional outputs must not survive a change of acceptance mode. All -# unconditional content/resource fields are overwritten below. Other evidence -# (e.g. raw_file_*, original reference binding and deadline) is retained, not -# re-certified by text scoring. -_CONDITIONAL_FIELDS = { - "acceptance_contract_sha256", "content_decision_basis", "factual_correctness", - "resource_acceptance_contract_sha256", - "structure_pass", "named_entry_count", "minimum_entries", "snapshot_f1", - "dynamic_titles", "verification_note", "confirmed_entries", "checked_entries", - "page_identity_required", "page_identity_status", "final_url_consistent", - "document_language_required", - "language_consistent", "observed_document_language", - "acceptable_final_urls", - "ordered_tree", "ordered_tree_similarity", - "composite_acceptance_sha256", - "body_requirements_sha256", +# Only collection evidence survives an offline score. Derived acceptance fields +# are recomputed, including fields no longer emitted by the current scorer. +# File fingerprints stay readable; a verdict needs current file verification. +_RETAINED_EVIDENCE_FIELDS = { + "reference_sha256", "deadline_overrun", "deadline_budget_ms", "file_evidence", + "raw_file_sha256", } def _same_page_url(actual: str, expected: str) -> bool: - actual_parts, expected_parts = urlsplit(actual), urlsplit(expected) + if not isinstance(actual, str) or not isinstance(expected, str): + return False + try: + actual_parts, expected_parts = urlsplit(actual), urlsplit(expected) + actual_port = actual_parts.port or {"http": 80, "https": 443}.get(actual_parts.scheme) + expected_port = expected_parts.port or {"http": 80, "https": 443}.get(expected_parts.scheme) + except ValueError: + return False actual_path = actual_parts.path.rstrip("/") or "/" expected_path = expected_parts.path.rstrip("/") or "/" - if ((actual_parts.scheme, actual_parts.hostname, actual_path) - != (expected_parts.scheme, expected_parts.hostname, expected_path)): + if ((actual_parts.scheme, actual_parts.hostname, actual_port, actual_path) + != (expected_parts.scheme, expected_parts.hostname, expected_port, expected_path)): return False # A declared query can select a different page on the same path (for # example, two distinct search results). Compare its decoded parameters; # a fragment does not change the fetched page. - expected_query = parse_qsl(expected_parts.query, keep_blank_values=True) - return (not expected_query or - sorted(parse_qsl(actual_parts.query, keep_blank_values=True)) - == sorted(expected_query)) + def page_query(parts): + return sorted((key, value) for key, value in parse_qsl(parts.query, keep_blank_values=True) + if not key.casefold().startswith('utm_') + and key.casefold() not in {'gclid', 'fbclid'}) + return page_query(actual_parts) == page_query(expected_parts) def score_attempt( @@ -100,14 +98,19 @@ def score_attempt( evidence is preserved; the caller must have validated its artifact binding. """ case = _case_dict(case) + identity_required = case.get("page_identity_required", False) + if not isinstance(identity_required, bool): + raise ValueError("page_identity_required must be boolean") + language_match = case.get("document_language_match", "primary") + if language_match not in {"primary", "exact"}: + raise ValueError("document_language_match must be primary or exact") from .response_input import answer_input # Live callers forward the conversion completed inside the timed boundary. # Offline callers reconstruct it from immutable evidence by the same function. content, links, ok, status_basis = (prepared_answer if prepared_answer is not None else answer_input(artifact, metadata, ok=ok, status_code=status_code)) result = {key: value for key, value in (previous_metrics or {}).items() - if key not in _CONDITIONAL_FIELDS - and not key.startswith(("stable_", "list_", "page_context_"))} + if key in _RETAINED_EVIDENCE_FIELDS} keys = ("required_facts", "forbidden_any", "min_content_chars", "fact_coverage_threshold", "ground_truth_main_content", "required_patterns", "acceptance_contract", "body_requirements", "composite_acceptance") @@ -143,7 +146,6 @@ def score_attempt( normalized_named_link_count=len(links)) dynamic_page_identity = bool( case.get("track") == "dynamic" - and (metadata or {}).get("route") == "dynamic_selected_text" and case.get("evaluation_url") and case.get("expected_final_url") and case.get("target_language") @@ -178,8 +180,10 @@ def score_attempt( language_consistent = ( observed_language == "" if not language_present else bool( isinstance(observed_language, str) and observed_language.strip() - and observed_language.split("-", 1)[0].casefold() - == expected_document_language.split("-", 1)[0].casefold() + and (observed_language.casefold() == expected_document_language.casefold() + if language_match == "exact" else + observed_language.split("-", 1)[0].casefold() + == expected_document_language.split("-", 1)[0].casefold()) ) ) identity_status = ( @@ -189,7 +193,7 @@ def score_attempt( else "unavailable" ) result.update( - page_identity_required=False, + page_identity_required=identity_required, page_identity_status=identity_status, final_url_consistent=final_consistent, language_consistent=language_consistent, @@ -207,13 +211,19 @@ def score_attempt( and any(_same_page_url(final_url, expected) for expected in acceptable_urls) ) result.update( - page_identity_required=False, + page_identity_required=identity_required, page_identity_status=( "verified" if final_consistent else "mismatch" if final_url else "unavailable" ), final_url_consistent=final_consistent, acceptable_final_urls=acceptable_urls, ) + elif identity_required: + consistent = bool(final_url and _same_page_url( + final_url, case.get("expected_final_url") or case["url"])) + result.update(page_identity_required=True, final_url_consistent=consistent, + page_identity_status="verified" if consistent else + "mismatch" if final_url else "unavailable") else: result.update(page_identity_required=False, page_identity_status="not_required") result["task_kind"] = task_kind @@ -242,7 +252,7 @@ def score_attempt( if "declared_truncation" not in result["stable_task_errors"]: result["stable_task_errors"].append("declared_truncation") file_evaluation = (metadata or {}).get('file_evaluation') - if result['task_kind'] == 'file' and isinstance(file_evaluation, dict) and 'raw_file_pass' not in result: + if result['task_kind'] == 'file' and isinstance(file_evaluation, dict): result['raw_file_pass'] = bool(file_evaluation.get('pass')) result["content_score_applicable"] = case.get("content_score_applicable", True) contract = scoring_values.get("acceptance_contract") diff --git a/tests/test_acceptance_isolation.py b/tests/test_acceptance_isolation.py new file mode 100644 index 0000000..6d50c73 --- /dev/null +++ b/tests/test_acceptance_isolation.py @@ -0,0 +1,146 @@ +"""Task acceptance must come from delivered content and current file evidence.""" +import pytest + +from lexbench_webfetch.metrics import summarize, task_passed +from lexbench_webfetch.scoring import score_attempt + + +@pytest.mark.parametrize("literal,other", [ + ("Home", "homepage"), ("Live", "deliver"), ("News", "newsletter"), + ("Print", "blueprint"), ("url", "curl"), ("café", "caféteria"), +]) +@pytest.mark.parametrize("gate", ["fact", "marker", "anchor"]) +def test_whole_word_requirements_reject_longer_words(literal, other, gate): + case = {"url": "https://example.test/article", "expected_any": []} + if gate == "fact": + case["required_facts"] = [{"id": "answer", "alternatives": [literal]}] + elif gate == "marker": + case["expected_any"] = [literal] + else: + case["acceptance_contract"] = {"version": 1, "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [{"id": "answer", "alternatives": [literal]}]} + for answer, passed in [(other, False), (f"({literal})", True)]: + metrics = score_attempt(case, answer.encode(), ok=True) + assert task_passed(True, metrics) is passed + + +@pytest.mark.parametrize("body,passed", [(b"unrelated text", False), + (b"Install with widget --start.", True), (b"", False)]) +def test_document_title_cannot_supply_a_required_body_fact(body, passed): + case = {"url": "https://example.test/article", "expected_any": [], + "required_facts": [{"id": "installation", "alternatives": ["widget --start"]}]} + metrics = score_attempt(case, body, ok=True, title="widget --start") + assert task_passed(True, metrics) is passed + + +@pytest.mark.parametrize("current_pass", [True, False]) +def test_current_file_validation_supersedes_a_previous_verdict(current_pass): + case = {"url": "https://example.test/file", "task_kind": "file"} + metadata = {"file_evaluation": {"pass": current_pass}} + fresh = score_attempt(case, b"file attached", ok=True, metadata=metadata) + rescored = score_attempt(case, b"file attached", ok=True, metadata=metadata, + previous_metrics={"raw_file_pass": not current_pass, "rule_review_reason": "obsolete"}) + assert rescored == fresh + assert task_passed(True, rescored) is current_pass + + +def test_removed_derived_metrics_do_not_survive_rescoring(): + case = {"url": "https://example.test/article", "expected_any": ["Article"]} + fresh = score_attempt(case, b"Article", ok=True) + rescored = score_attempt(case, b"Article", ok=True, previous_metrics={ + "retired_acceptance_pass": True, "rule_review_reason": "obsolete", + "composite_acceptance_pass": False, "ordered_tree_similarity": 1.0}) + assert rescored == fresh + + +def test_a_legacy_file_verdict_without_current_file_evidence_cannot_certify_a_rescore(): + case = {"url": "https://example.test/file", "task_kind": "file"} + metrics = score_attempt(case, b"file attached", ok=True, + previous_metrics={"raw_file_pass": True, "raw_file_sha256": "legacy-hash"}) + assert "raw_file_pass" not in metrics + assert metrics["raw_file_sha256"] == "legacy-hash" + assert not task_passed(True, metrics) + + +def test_rescoring_preserves_collection_identity_and_deadline(): + case = {"url": "https://example.test/article"} + collected = {"reference_sha256": "frozen-capture", "deadline_overrun": True, + "deadline_budget_ms": 45000, + "raw_file_sha256": "verified-file", "file_evidence": {"sha256": "verified-file"}} + metrics = score_attempt(case, b"Article", ok=True, previous_metrics=collected) + assert all(metrics[k] == v for k, v in collected.items()) + assert not task_passed(True, metrics) + + +def test_all_call_latency_includes_fast_success_slow_failure_and_timeout(): + rows = [] + for index, (elapsed, answer, ok) in enumerate([ + (10, b"Article", True), (1000, b"Wrong page", True), (45000, b"", False)]): + case = {"url": "https://example.test/article", "expected_any": ["Article"]} + rows.append({"provider": "browser", "provider_type": "local", "case_id": str(index), + "case_name": "Article", "track": "web", "category": "article", + "ok": ok, "elapsed_ms": elapsed, + "metrics": score_attempt(case, answer, ok=ok)}) + metrics = summarize(rows)["providers"]["browser"] + assert metrics["mean_content_ms"] == 10 + assert metrics["mean_all_attempts_ms"] == 15336.67 + assert metrics["all_attempts_p50_ms"] == 1000 + assert metrics["all_attempts_p95_ms"] == 45000 + + +@pytest.mark.parametrize("track", ["web", "dynamic", "risk_control"]) +@pytest.mark.parametrize("route", [None, "dynamic_selected_text", "another_interface"]) +@pytest.mark.parametrize("destination,passed", [ + ("https://example.test/article", True), ("https://example.test/wrong", False), (None, False)]) +def test_declared_page_identity_is_required_on_every_interface(track, route, destination, passed): + case = {"url": "https://example.test/article", "track": track, + "expected_any": ["Article"], "page_identity_required": True} + if track == "dynamic": + case.update(evaluation_url=case["url"], expected_final_url=case["url"], + target_language="en", target_region="US") + if track == "risk_control": + case["source_benchmark"] = {"risk_control": {"navigation_contract": { + "requested_url": case["url"], "observed_final_url": case["url"]}}} + metadata = {"document_language": "en"} + if route: + metadata["route"] = route + metrics = score_attempt(case, b"Article", ok=True, metadata=metadata, final_url=destination) + assert task_passed(True, metrics) is passed + + +@pytest.mark.parametrize("declaration", ["\n]]>", + ""]) +def test_xml_declarations_cannot_crash_body_or_resource_scoring(declaration): + body_case = {"url": "https://example.test/spec", "expected_any": ["Specification"]} + raw = ("Specification\n" + declaration).encode() + assert task_passed(True, score_attempt(body_case, raw, ok=True)) + resource_case = {**body_case, "task_kind": "resource", + "expected_resources": [{"id": "image", "urls": ["/target.png"]}]} + for answer, passed in [(declaration + '\n\n' + + '[Target](/target.png)', True), (declaration + '\n', False)]: + metrics = score_attempt(resource_case, answer.encode(), ok=True) + assert task_passed(True, metrics) is passed + + +@pytest.mark.parametrize("destination,passed", [ + ("https://example.test:443/article/#section", True), + ("https://example.test/article?utm_source=link", True), + ("https://example.test/article?q=another", False), + ("https://example.test:8443/article", False), + ("https://example.test:invalid/article", False)]) +def test_identity_distinguishes_page_queries_and_ports(destination, passed): + case = {"url": "https://example.test/article", "page_identity_required": True} + assert task_passed(True, score_attempt(case, b"Article", ok=True, final_url=destination)) is passed + + +@pytest.mark.parametrize("mode,language,passed", [ + ("primary", "zh-TW", True), ("exact", "zh-TW", False), ("exact", "zh-CN", True)]) +def test_tasks_can_require_the_complete_document_locale(mode, language, passed): + case = {"url": "https://example.test/article", "track": "dynamic", + "evaluation_url": "https://example.test/article", "expected_final_url": "https://example.test/article", + "target_language": "zh-CN", "target_region": "CN", "page_identity_required": True, + "document_language_match": mode} + metrics = score_attempt(case, b"Article", ok=True, final_url=case["url"], + metadata={"document_language": language}) + assert task_passed(True, metrics) is passed diff --git a/uv.lock b/uv.lock index 60c24c1..8ddc0c5 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 5 +revision = 2 requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.11.5'", @@ -48,7 +48,7 @@ wheels = [ [[package]] name = "lexbench-webfetch" -version = "0.2.0" +version = "0.3.0" source = { editable = "." } dependencies = [ { name = "beautifulsoup4" },