From 51e11917c5b00e1629a1f27c0500391371fc387a Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:08:09 +0800 Subject: [PATCH 01/20] Fix acceptance isolation and explicit page identity contracts --- README.md | 10 + docs/README.zh-CN.md | 10 + pyproject.toml | 2 +- src/lexbench_webfetch/benchmark.py | 3 +- .../evaluation/scoring-regressions.json | 908 +++++++++++++++++- src/lexbench_webfetch/metrics.py | 30 +- src/lexbench_webfetch/models.py | 2 + src/lexbench_webfetch/scoring.py | 65 +- src/lexbench_webfetch/stable_tasks.py | 3 +- tests/test_acceptance_isolation.py | 146 +++ uv.lock | 4 +- 11 files changed, 1141 insertions(+), 42 deletions(-) create mode 100644 tests/test_acceptance_isolation.py diff --git a/README.md b/README.md index ffd47d9..17d45d8 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,16 @@ uv run --frozen browser-harness-eval run \ Run these two commands from Browser Eval. Ordinary runs are diagnostic; a formal live-site comparison additionally requires its pre-call/post-call reference certification. The benchmark input checker verifies frozen bytes, not present-day website validity. See the [Browser Eval integration guide](https://github.com/lexmount/browser-eval/blob/main/docs/benchmarks.md). +## Scoring behavior + +Version 0.3 matches Latin words at word boundaries for facts, markers and stable anchors. Required body facts use the returned content; a separate document title cannot supply missing body text. Chinese phrases retain literal matching, and numeric and currency checks keep their value boundaries. + +A task can set `page_identity_required: true` to require its final URL observations. Missing or mismatched observations then fail acceptance. Dynamic tasks can also set `document_language_match: "exact"` when the complete language tag matters; the default is `"primary"`. These checks apply across interfaces. Frozen tasks that do not require URL metadata keep their content-based acceptance rules. + +Offline scoring recomputes derived fields. It retains the original reference hash, deadline evidence and file fingerprints. A file verdict requires current file verification; an old pass flag cannot certify a rescore. Summaries expose `mean_all_attempts_ms`, `all_attempts_p50_ms` and `all_attempts_p95_ms` alongside the existing successful-call timings. The all-attempt population includes failed calls and timeouts. + +These changes do not revise frozen tasks or certify historical results. Score corrections require a new evidence revision tied to the scorer and its regression controls. + ## Repository contents | Path | Purpose | diff --git a/docs/README.zh-CN.md b/docs/README.zh-CN.md index a34e08b..d22c41d 100644 --- a/docs/README.zh-CN.md +++ b/docs/README.zh-CN.md @@ -106,6 +106,16 @@ uv run --frozen browser-harness-eval run \ 普通运行用于诊断。正式比较实时网站时，还需要完成调用前后的参考内容认证。输入校验只能证明文件版本相符，不能证明网站今天仍然符合旧参考。详见 [Browser Eval 接入说明](https://github.com/lexmount/browser-eval/blob/main/docs/benchmarks.md)。 +## 评分规则 + +0.3 版对事实、标记和固定锚点中的拉丁文使用单词边界匹配。正文事实只匹配返回内容，单独返回的文档标题不能补足正文。中文短语仍按原文匹配，数字和货币保留原有的数值边界检查。 + +任务可设置 `page_identity_required: true`，要求返回的最终网址符合任务声明；缺少网址或网址不符都会判为未通过。动态任务需要区分完整语言标签时，可再设置 `document_language_match: "exact"`，默认值为 `"primary"`，只比较主语言。这些检查适用于所有接口。没有要求网址元数据的冻结任务，继续按原有内容规则验收。 + +离线评分会重新计算派生字段，保留原始参考哈希、调用时限证据和文件指纹。文件结论需要当前文件验收依据；旧的通过标记不能认证本次重评分。汇总在原有成功调用耗时之外，提供全部调用的 `mean_all_attempts_ms`、`all_attempts_p50_ms` 和 `all_attempts_p95_ms`，包含失败调用与超时。 + +上述修改不改写冻结题目，也不自动认证历史成绩。更正分数仍需建立新的证据修订，绑定评分器版本和回归用例。 + ## 仓库内容 | 路径 | 用途 | diff --git a/pyproject.toml b/pyproject.toml index d3d39a7..18e2499 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "lexbench-webfetch" -version = "0.2.0" +version = "0.3.0" description = "Versioned Web Fetch tasks, acceptance contracts and frozen reference bindings" readme = "README.md" requires-python = ">=3.11" diff --git a/src/lexbench_webfetch/benchmark.py b/src/lexbench_webfetch/benchmark.py index f8b7be4..d3fdfe5 100644 --- a/src/lexbench_webfetch/benchmark.py +++ b/src/lexbench_webfetch/benchmark.py @@ -20,7 +20,8 @@ "fact_coverage_threshold", "expected_resources", "resource_output_contract", "content_score_applicable", "required_patterns", "acceptance_contract", "composite_acceptance", "mutable_body_policy", "body_requirements", "retrieval_hints", - "source_review_url", "evaluation_url", "expected_final_url", "expected_document_language", + "source_review_url", "evaluation_url", "expected_final_url", "page_identity_required", + "expected_document_language", "document_language_match", "expected_document_language_present", "target_language", "target_region", ) diff --git a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json index af4f90a..7836406 100644 --- a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json +++ b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json @@ -54197,6 +54197,909 @@ "artifact": "# [Orchard Live](https://example.test/item/one)\n- Date: October 10, 2026\n- Venue: [Grand Hall](https://foreign.test/venue/one)\n- City: Springfield", "ok": true, "expected_task_pass": true + }, + { + "id": "body-only-word-boundary-answer-title-supplies-required-fact", + "category": "answer-surface", + "root_cause": "metadata_title_supplies_missing_body_fact", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/page", + "required_facts": [ + { + "id": "title", + "alternatives": [ + "Exact page title" + ] + } + ] + }, + "artifact": "Body", + "title": "Exact page title", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "fact_coverage": 0.0, + "content_chars": 4 + } + }, + { + "id": "body-only-word-boundary-answer-title-does-not-inflate-body-length", + "category": "answer-surface", + "root_cause": "answer_surface_omission", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/page", + "required_facts": [ + { + "id": "title", + "alternatives": [ + "Exact page title" + ] + } + ], + "min_content_chars": 20 + }, + "artifact": "Body", + "title": "Exact page title", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "fact_coverage": 0.0, + "length_pass": false, + "content_chars": 4 + } + }, + { + "id": "latin-boundary-fact-Home-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "Home" + ] + } + ] + }, + "artifact": "(Home)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-fact-Home-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "Home" + ] + } + ] + }, + "artifact": "homepage", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-marker-Home-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [ + "Home" + ] + }, + "artifact": "(Home)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-marker-Home-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [ + "Home" + ] + }, + "artifact": "homepage", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-anchor-Home-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "answer", + "alternatives": [ + "Home" + ] + } + ] + } + }, + "artifact": "(Home)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-anchor-Home-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "answer", + "alternatives": [ + "Home" + ] + } + ] + } + }, + "artifact": "homepage", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-fact-Live-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "Live" + ] + } + ] + }, + "artifact": "(Live)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-fact-Live-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "Live" + ] + } + ] + }, + "artifact": "deliver", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-marker-Live-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [ + "Live" + ] + }, + "artifact": "(Live)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-marker-Live-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [ + "Live" + ] + }, + "artifact": "deliver", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-anchor-Live-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "answer", + "alternatives": [ + "Live" + ] + } + ] + } + }, + "artifact": "(Live)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-anchor-Live-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "answer", + "alternatives": [ + "Live" + ] + } + ] + } + }, + "artifact": "deliver", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-fact-News-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "News" + ] + } + ] + }, + "artifact": "(News)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-fact-News-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "News" + ] + } + ] + }, + "artifact": "newsletter", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-marker-News-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [ + "News" + ] + }, + "artifact": "(News)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-marker-News-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [ + "News" + ] + }, + "artifact": "newsletter", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-anchor-News-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "answer", + "alternatives": [ + "News" + ] + } + ] + } + }, + "artifact": "(News)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-anchor-News-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "answer", + "alternatives": [ + "News" + ] + } + ] + } + }, + "artifact": "newsletter", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-fact-Print-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "Print" + ] + } + ] + }, + "artifact": "(Print)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-fact-Print-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "Print" + ] + } + ] + }, + "artifact": "blueprint", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-marker-Print-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [ + "Print" + ] + }, + "artifact": "(Print)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-marker-Print-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [ + "Print" + ] + }, + "artifact": "blueprint", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-anchor-Print-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "answer", + "alternatives": [ + "Print" + ] + } + ] + } + }, + "artifact": "(Print)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-anchor-Print-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "answer", + "alternatives": [ + "Print" + ] + } + ] + } + }, + "artifact": "blueprint", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-fact-url-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "url" + ] + } + ] + }, + "artifact": "(url)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-fact-url-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "url" + ] + } + ] + }, + "artifact": "curl", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-marker-url-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [ + "url" + ] + }, + "artifact": "(url)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-marker-url-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [ + "url" + ] + }, + "artifact": "curl", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-anchor-url-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "answer", + "alternatives": [ + "url" + ] + } + ] + } + }, + "artifact": "(url)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-anchor-url-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "answer", + "alternatives": [ + "url" + ] + } + ] + } + }, + "artifact": "curl", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-fact-café-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "café" + ] + } + ] + }, + "artifact": "(café)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-fact-café-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "café" + ] + } + ] + }, + "artifact": "caféteria", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-marker-café-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [ + "café" + ] + }, + "artifact": "(café)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-marker-café-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [ + "café" + ] + }, + "artifact": "caféteria", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-anchor-café-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "answer", + "alternatives": [ + "café" + ] + } + ] + } + }, + "artifact": "(café)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-anchor-café-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "answer", + "alternatives": [ + "café" + ] + } + ] + } + }, + "artifact": "caféteria", + "ok": true, + "expected_task_pass": false + }, + { + "id": "body-fact-title-absent", + "category": "answer-surface", + "root_cause": "metadata_title_supplies_missing_body_fact", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "required_facts": [ + { + "id": "install", + "alternatives": [ + "widget --start" + ] + } + ] + }, + "artifact": "unrelated text", + "title": "widget --start", + "ok": true, + "expected_task_pass": false + }, + { + "id": "body-fact-title-present", + "category": "answer-surface", + "root_cause": "metadata_title_supplies_missing_body_fact", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "required_facts": [ + { + "id": "install", + "alternatives": [ + "widget --start" + ] + } + ] + }, + "artifact": "widget --start", + "title": "widget --start", + "ok": true, + "expected_task_pass": true + }, + { + "id": "query-sensitive-dynamic-matching-final-page-and-language-remain-eligible", + "category": "acceptance-contract", + "root_cause": "undeclared_query_parameters_may_select_other_content", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/start", + "evaluation_url": "https://example.test/start", + "expected_final_url": "https://example.test/en/catalog", + "target_language": "en-US", + "target_region": "US", + "track": "dynamic", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "identity", + "alternatives": [ + "Catalog" + ] + } + ] + } + }, + "artifact": "Catalog", + "ok": true, + "final_url": "https://example.test/en/catalog?session=123#items", + "metadata": { + "route": "dynamic_selected_text", + "document_language": "en" + }, + "expected_task_pass": true, + "expected_metrics": { + "page_identity_status": "mismatch", + "final_url_consistent": false, + "language_consistent": true + } + }, + { + "id": "answer-body-fact-remains-valid-with-an-unrelated-document-title", + "category": "answer-surface", + "root_cause": "answer_surface_omission", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/page", + "required_facts": [ + { + "id": "heading", + "alternatives": [ + "Exact page title" + ] + } + ] + }, + "artifact": "Exact page title\nRequired body", + "title": "Unrelated title metadata", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "fact_coverage": 1.0 + } } ], "superseded_cases": { @@ -54210,7 +55113,10 @@ "required-final-url-present-keeps-valid-body": "content-policy-required-final-url-present-keeps-valid-body", "inline-linked-card-visible-fields": "inline-linked-card-visible-fields-ambiguous-boundaries-rejected", "inline-accessible-card-visible-fields": "inline-accessible-card-visible-fields-ambiguous-boundaries-rejected", - "body-unfenced-native-table-code-retained": "body-unfenced-table-is-markup-not-literal-code" + "body-unfenced-native-table-code-retained": "body-unfenced-table-is-markup-not-literal-code", + "dynamic-matching-final-page-and-language-remain-eligible": "query-sensitive-dynamic-matching-final-page-and-language-remain-eligible", + "answer-title-supplies-required-fact": "body-only-word-boundary-answer-title-supplies-required-fact", + "answer-title-does-not-inflate-body-length": "body-only-word-boundary-answer-title-does-not-inflate-body-length" }, "superseded_controls": { "frozen_scorer_sha256": "ceb62fb821e6777d24a015ff713f406faecfff6181b0601b8e3ea2adffbf7c10", diff --git a/src/lexbench_webfetch/metrics.py b/src/lexbench_webfetch/metrics.py index 9664199..6140901 100644 --- a/src/lexbench_webfetch/metrics.py +++ b/src/lexbench_webfetch/metrics.py @@ -204,8 +204,16 @@ def _inline_frame_candidates( URLs that the browser serialized, and ignores hidden synchronization or ad frames. The same parser applies to any provider returning inline frames. """ + if not re.search(r' *(?=\w)', '>', normalized) -def fact_in_text(text: str, fact: str) -> bool: +def fact_in_text(text: str, fact: str, *, latin_boundaries: bool = True) -> bool: """Match an explicitly registered normalized fact without fuzzy semantics.""" if not fact: return False left = r'(? dict[str, Any]: "mean_content_ms": round(statistics.mean(content_successful_elapsed), 2) if content_successful_elapsed else None, "transport_p50_ms": percentile(transport_successful_elapsed, 0.50), "transport_p95_ms": percentile(transport_successful_elapsed, 0.95), + "mean_all_attempts_ms": round(statistics.mean(elapsed), 2), + "all_attempts_p50_ms": percentile(elapsed, 0.50), "all_attempts_p95_ms": percentile(elapsed, 0.95), "median_content_chars": round( statistics.median(item["metrics"]["content_chars"] for item in items), 1 diff --git a/src/lexbench_webfetch/models.py b/src/lexbench_webfetch/models.py index ff1a711..13f01d2 100644 --- a/src/lexbench_webfetch/models.py +++ b/src/lexbench_webfetch/models.py @@ -48,7 +48,9 @@ class EvalCase: evaluation_url: str = "" page_identity_required: bool = field(default=False, kw_only=True) expected_final_url: str = "" + page_identity_required: bool = False expected_document_language: str = "" + document_language_match: str = "primary" expected_document_language_present: bool = True target_language: str = "" target_region: str = "" diff --git a/src/lexbench_webfetch/scoring.py b/src/lexbench_webfetch/scoring.py index 3f20cf9..d5cdf13 100644 --- a/src/lexbench_webfetch/scoring.py +++ b/src/lexbench_webfetch/scoring.py @@ -43,39 +43,37 @@ def load_list_reference(case: EvalCase | dict[str, Any], source: Path) -> dict | return None -# These conditional outputs must not survive a change of acceptance mode. All -# unconditional content/resource fields are overwritten below. Other evidence -# (e.g. raw_file_*, original reference binding and deadline) is retained, not -# re-certified by text scoring. -_CONDITIONAL_FIELDS = { - "acceptance_contract_sha256", "content_decision_basis", "factual_correctness", - "resource_acceptance_contract_sha256", - "structure_pass", "named_entry_count", "minimum_entries", "snapshot_f1", - "dynamic_titles", "verification_note", "confirmed_entries", "checked_entries", - "page_identity_required", "page_identity_status", "final_url_consistent", - "document_language_required", - "language_consistent", "observed_document_language", - "acceptable_final_urls", - "ordered_tree", "ordered_tree_similarity", - "composite_acceptance_sha256", - "body_requirements_sha256", +# Only collection evidence survives an offline score. Derived acceptance fields +# are recomputed, including fields no longer emitted by the current scorer. +# File fingerprints stay readable; a verdict needs current file verification. +_RETAINED_EVIDENCE_FIELDS = { + "reference_sha256", "deadline_overrun", "deadline_budget_ms", "file_evidence", + "raw_file_sha256", } def _same_page_url(actual: str, expected: str) -> bool: - actual_parts, expected_parts = urlsplit(actual), urlsplit(expected) + if not isinstance(actual, str) or not isinstance(expected, str): + return False + try: + actual_parts, expected_parts = urlsplit(actual), urlsplit(expected) + actual_port = actual_parts.port or {"http": 80, "https": 443}.get(actual_parts.scheme) + expected_port = expected_parts.port or {"http": 80, "https": 443}.get(expected_parts.scheme) + except ValueError: + return False actual_path = actual_parts.path.rstrip("/") or "/" expected_path = expected_parts.path.rstrip("/") or "/" - if ((actual_parts.scheme, actual_parts.hostname, actual_path) - != (expected_parts.scheme, expected_parts.hostname, expected_path)): + if ((actual_parts.scheme, actual_parts.hostname, actual_port, actual_path) + != (expected_parts.scheme, expected_parts.hostname, expected_port, expected_path)): return False # A declared query can select a different page on the same path (for # example, two distinct search results). Compare its decoded parameters; # a fragment does not change the fetched page. - expected_query = parse_qsl(expected_parts.query, keep_blank_values=True) - return (not expected_query or - sorted(parse_qsl(actual_parts.query, keep_blank_values=True)) - == sorted(expected_query)) + def page_query(parts): + return sorted((key, value) for key, value in parse_qsl(parts.query, keep_blank_values=True) + if not key.casefold().startswith('utm_') + and key.casefold() not in {'gclid', 'fbclid'}) + return page_query(actual_parts) == page_query(expected_parts) def score_attempt( @@ -116,16 +114,16 @@ def score_attempt( if (contract.get('minimum_records', 0) == 0 and not (case.get('body_requirements') or {}).get('regions')): raise ValueError('Full-page coverage requires target records or body regions') + language_match = case.get("document_language_match", "primary") + if language_match not in {"primary", "exact"}: + raise ValueError("document_language_match must be primary or exact") from .response_input import answer_input # Live callers forward the conversion completed inside the timed boundary. # Offline callers reconstruct it from immutable evidence by the same function. content, links, ok, status_basis = (prepared_answer if prepared_answer is not None else answer_input(artifact, metadata, ok=ok, status_code=status_code)) result = {key: value for key, value in (previous_metrics or {}).items() - if key not in _CONDITIONAL_FIELDS - and not key.startswith(("stable_", "list_", "page_context_"))} - for retired in ('acceptance_policy_version', 'forbidden_hits', 'reference_term_exemptions'): - result.pop(retired, None) + if key in _RETAINED_EVIDENCE_FIELDS} keys = ("required_facts", "forbidden_any", "diagnostic_any", "min_content_chars", "fact_coverage_threshold", "ground_truth_main_content", "required_patterns", "acceptance_contract", "body_requirements", "composite_acceptance") @@ -161,7 +159,6 @@ def score_attempt( normalized_named_link_count=len(links)) dynamic_page_identity = bool( case.get("track") == "dynamic" - and (metadata or {}).get("route") == "dynamic_selected_text" and case.get("evaluation_url") and case.get("expected_final_url") and case.get("target_language") @@ -196,8 +193,10 @@ def score_attempt( language_consistent = ( observed_language == "" if not language_present else bool( isinstance(observed_language, str) and observed_language.strip() - and observed_language.split("-", 1)[0].casefold() - == expected_document_language.split("-", 1)[0].casefold() + and (observed_language.casefold() == expected_document_language.casefold() + if language_match == "exact" else + observed_language.split("-", 1)[0].casefold() + == expected_document_language.split("-", 1)[0].casefold()) ) ) identity_status = ( @@ -232,6 +231,12 @@ def score_attempt( final_url_consistent=final_consistent, acceptable_final_urls=acceptable_urls, ) + elif identity_required: + consistent = bool(final_url and _same_page_url( + final_url, case.get("expected_final_url") or case["url"])) + result.update(page_identity_required=True, final_url_consistent=consistent, + page_identity_status="verified" if consistent else + "mismatch" if final_url else "unavailable") else: result.update(page_identity_required=False, page_identity_status="not_required") result["task_kind"] = task_kind diff --git a/src/lexbench_webfetch/stable_tasks.py b/src/lexbench_webfetch/stable_tasks.py index 0be8d7a..2195ee9 100644 --- a/src/lexbench_webfetch/stable_tasks.py +++ b/src/lexbench_webfetch/stable_tasks.py @@ -706,7 +706,8 @@ def _stable_literal_match(surface, value, registered_words=()): from .metrics import fact_in_text, normalize_for_match expected = normalize_for_match(_visible(value)) folded = normalize_for_match(surface) - if not fact_in_text(folded, expected): + if not fact_in_text(folded, expected, latin_boundaries=bool( + re.search(r"[^\x00-\x7f]", value))): return False # Arabic labels are whitespace-delimited words too. A longer inflected # word cannot supply an independently registered ordinary navigation label. diff --git a/tests/test_acceptance_isolation.py b/tests/test_acceptance_isolation.py new file mode 100644 index 0000000..6d50c73 --- /dev/null +++ b/tests/test_acceptance_isolation.py @@ -0,0 +1,146 @@ +"""Task acceptance must come from delivered content and current file evidence.""" +import pytest + +from lexbench_webfetch.metrics import summarize, task_passed +from lexbench_webfetch.scoring import score_attempt + + +@pytest.mark.parametrize("literal,other", [ + ("Home", "homepage"), ("Live", "deliver"), ("News", "newsletter"), + ("Print", "blueprint"), ("url", "curl"), ("café", "caféteria"), +]) +@pytest.mark.parametrize("gate", ["fact", "marker", "anchor"]) +def test_whole_word_requirements_reject_longer_words(literal, other, gate): + case = {"url": "https://example.test/article", "expected_any": []} + if gate == "fact": + case["required_facts"] = [{"id": "answer", "alternatives": [literal]}] + elif gate == "marker": + case["expected_any"] = [literal] + else: + case["acceptance_contract"] = {"version": 1, "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [{"id": "answer", "alternatives": [literal]}]} + for answer, passed in [(other, False), (f"({literal})", True)]: + metrics = score_attempt(case, answer.encode(), ok=True) + assert task_passed(True, metrics) is passed + + +@pytest.mark.parametrize("body,passed", [(b"unrelated text", False), + (b"Install with widget --start.", True), (b"", False)]) +def test_document_title_cannot_supply_a_required_body_fact(body, passed): + case = {"url": "https://example.test/article", "expected_any": [], + "required_facts": [{"id": "installation", "alternatives": ["widget --start"]}]} + metrics = score_attempt(case, body, ok=True, title="widget --start") + assert task_passed(True, metrics) is passed + + +@pytest.mark.parametrize("current_pass", [True, False]) +def test_current_file_validation_supersedes_a_previous_verdict(current_pass): + case = {"url": "https://example.test/file", "task_kind": "file"} + metadata = {"file_evaluation": {"pass": current_pass}} + fresh = score_attempt(case, b"file attached", ok=True, metadata=metadata) + rescored = score_attempt(case, b"file attached", ok=True, metadata=metadata, + previous_metrics={"raw_file_pass": not current_pass, "rule_review_reason": "obsolete"}) + assert rescored == fresh + assert task_passed(True, rescored) is current_pass + + +def test_removed_derived_metrics_do_not_survive_rescoring(): + case = {"url": "https://example.test/article", "expected_any": ["Article"]} + fresh = score_attempt(case, b"Article", ok=True) + rescored = score_attempt(case, b"Article", ok=True, previous_metrics={ + "retired_acceptance_pass": True, "rule_review_reason": "obsolete", + "composite_acceptance_pass": False, "ordered_tree_similarity": 1.0}) + assert rescored == fresh + + +def test_a_legacy_file_verdict_without_current_file_evidence_cannot_certify_a_rescore(): + case = {"url": "https://example.test/file", "task_kind": "file"} + metrics = score_attempt(case, b"file attached", ok=True, + previous_metrics={"raw_file_pass": True, "raw_file_sha256": "legacy-hash"}) + assert "raw_file_pass" not in metrics + assert metrics["raw_file_sha256"] == "legacy-hash" + assert not task_passed(True, metrics) + + +def test_rescoring_preserves_collection_identity_and_deadline(): + case = {"url": "https://example.test/article"} + collected = {"reference_sha256": "frozen-capture", "deadline_overrun": True, + "deadline_budget_ms": 45000, + "raw_file_sha256": "verified-file", "file_evidence": {"sha256": "verified-file"}} + metrics = score_attempt(case, b"Article", ok=True, previous_metrics=collected) + assert all(metrics[k] == v for k, v in collected.items()) + assert not task_passed(True, metrics) + + +def test_all_call_latency_includes_fast_success_slow_failure_and_timeout(): + rows = [] + for index, (elapsed, answer, ok) in enumerate([ + (10, b"Article", True), (1000, b"Wrong page", True), (45000, b"", False)]): + case = {"url": "https://example.test/article", "expected_any": ["Article"]} + rows.append({"provider": "browser", "provider_type": "local", "case_id": str(index), + "case_name": "Article", "track": "web", "category": "article", + "ok": ok, "elapsed_ms": elapsed, + "metrics": score_attempt(case, answer, ok=ok)}) + metrics = summarize(rows)["providers"]["browser"] + assert metrics["mean_content_ms"] == 10 + assert metrics["mean_all_attempts_ms"] == 15336.67 + assert metrics["all_attempts_p50_ms"] == 1000 + assert metrics["all_attempts_p95_ms"] == 45000 + + +@pytest.mark.parametrize("track", ["web", "dynamic", "risk_control"]) +@pytest.mark.parametrize("route", [None, "dynamic_selected_text", "another_interface"]) +@pytest.mark.parametrize("destination,passed", [ + ("https://example.test/article", True), ("https://example.test/wrong", False), (None, False)]) +def test_declared_page_identity_is_required_on_every_interface(track, route, destination, passed): + case = {"url": "https://example.test/article", "track": track, + "expected_any": ["Article"], "page_identity_required": True} + if track == "dynamic": + case.update(evaluation_url=case["url"], expected_final_url=case["url"], + target_language="en", target_region="US") + if track == "risk_control": + case["source_benchmark"] = {"risk_control": {"navigation_contract": { + "requested_url": case["url"], "observed_final_url": case["url"]}}} + metadata = {"document_language": "en"} + if route: + metadata["route"] = route + metrics = score_attempt(case, b"Article", ok=True, metadata=metadata, final_url=destination) + assert task_passed(True, metrics) is passed + + +@pytest.mark.parametrize("declaration", ["\n]]>", + ""]) +def test_xml_declarations_cannot_crash_body_or_resource_scoring(declaration): + body_case = {"url": "https://example.test/spec", "expected_any": ["Specification"]} + raw = ("Specification\n" + declaration).encode() + assert task_passed(True, score_attempt(body_case, raw, ok=True)) + resource_case = {**body_case, "task_kind": "resource", + "expected_resources": [{"id": "image", "urls": ["/target.png"]}]} + for answer, passed in [(declaration + '\n\n' + + '[Target](/target.png)', True), (declaration + '\n', False)]: + metrics = score_attempt(resource_case, answer.encode(), ok=True) + assert task_passed(True, metrics) is passed + + +@pytest.mark.parametrize("destination,passed", [ + ("https://example.test:443/article/#section", True), + ("https://example.test/article?utm_source=link", True), + ("https://example.test/article?q=another", False), + ("https://example.test:8443/article", False), + ("https://example.test:invalid/article", False)]) +def test_identity_distinguishes_page_queries_and_ports(destination, passed): + case = {"url": "https://example.test/article", "page_identity_required": True} + assert task_passed(True, score_attempt(case, b"Article", ok=True, final_url=destination)) is passed + + +@pytest.mark.parametrize("mode,language,passed", [ + ("primary", "zh-TW", True), ("exact", "zh-TW", False), ("exact", "zh-CN", True)]) +def test_tasks_can_require_the_complete_document_locale(mode, language, passed): + case = {"url": "https://example.test/article", "track": "dynamic", + "evaluation_url": "https://example.test/article", "expected_final_url": "https://example.test/article", + "target_language": "zh-CN", "target_region": "CN", "page_identity_required": True, + "document_language_match": mode} + metrics = score_attempt(case, b"Article", ok=True, final_url=case["url"], + metadata={"document_language": language}) + assert task_passed(True, metrics) is passed diff --git a/uv.lock b/uv.lock index 60c24c1..8ddc0c5 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 5 +revision = 2 requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.11.5'", @@ -48,7 +48,7 @@ wheels = [ [[package]] name = "lexbench-webfetch" -version = "0.2.0" +version = "0.3.0" source = { editable = "." } dependencies = [ { name = "beautifulsoup4" }, From 8135ef741d7de205eef986a17697759a12d597df Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:38:03 +0800 Subject: [PATCH 02/20] docs(contributing): define PR and commit title conventions --- CONTRIBUTING.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9b1284a..264a5d2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,3 +7,30 @@ Use Python 3.11 or newer and [uv](https://docs.astral.sh/uv/). Run `uv sync --fr A task revision must explain the intended requirement, preserve the prior input version, and include scenarios that should pass and fail. Update the original dataset, catalog, public task definitions, input manifest and registry together after independent source verification. Do not regenerate old hashes to disguise an input correction. Frozen quotes retain their original language. Keep measured outputs, captures, credentials and internal review notes outside Git. A result proposal must supply the evidence listed in the [protocol](docs/protocol.md); opening a PR does not add it to a leaderboard. See [THIRD_PARTY](THIRD_PARTY.md) before sharing source material. + +## PR and commit titles + +Use `(): ` for PR titles and new commit subjects. The scope is optional; use the module or feature affected, such as `scoring` or `webfetch`. + +Choose the type that describes the change: + +| Type | Change | +| --- | --- | +| `fix` | Correct existing behavior | +| `feat` | Add supported behavior | +| `docs` | Change documentation | +| `refactor` | Restructure code without changing behavior | +| `test` | Add or correct tests | +| `perf` | Improve performance | +| `build` / `ci` | Change packaging, dependencies or automation | +| `chore` | Other maintenance | + +Write the summary in English, start with an imperative verb, and describe the concrete change. Aim for 72 characters or fewer, including the prefix; omit a trailing period. Use `!` before the colon for a breaking change and explain the migration in the description. + +Examples: + +- `fix(scoring): require facts in the returned body` +- `fix(webfetch): reject mixed executable versions before release` +- `docs(contributing): define PR and commit title conventions` + +Avoid titles such as `Fix review feedback`, `Update code` or `Misc improvements`; name the behavior or document that changes. A PR title describes the complete diff; each commit subject describes that commit. Update the PR title when its scope changes. If squash merging is used, use a subject that follows the same format. Apply this convention to new work; do not rewrite shared commits solely to rename them. From 02027f7d4f3179e48ed2e221229f87a9a204564a Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:51:13 +0800 Subject: [PATCH 03/20] fix(scoring): exclude diagnostic URLs and keep identity failures measured --- .../evaluation/scoring-regressions.json | 166 +++++++++++++++++- src/lexbench_webfetch/metrics.py | 86 ++++----- tests/test_acceptance_isolation.py | 51 ++++++ 3 files changed, 251 insertions(+), 52 deletions(-) diff --git a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json index 7836406..71cb77c 100644 --- a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json +++ b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json @@ -55100,6 +55100,169 @@ "expected_metrics": { "fact_coverage": 1.0 } + }, + { + "id": "explicit-contract-missing-page-identity-counts-as-failure", + "category": "task-predicate", + "root_cause": "unknown_identity_masks_independent_requirement_failure", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/start", + "track": "risk_control", + "task_kind": "body", + "expected_any": [ + "Expected body" + ], + "min_content_chars": 1, + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "body", + "alternatives": [ + "Expected body" + ] + } + ] + }, + "source_benchmark": { + "risk_control": { + "navigation_contract": { + "requested_url": "https://example.test/start", + "observed_final_url": "https://example.test/final" + } + } + }, + "page_identity_required": true + }, + "artifact": "Expected body", + "ok": true, + "expected_task_pass": false, + "expected_task_measured": true, + "expected_metrics": { + "page_identity_status": "unavailable", + "length_pass": true + } + }, + { + "id": "resource-log-with-no-answer", + "category": "answer-surface", + "root_cause": "resource_network_diagnostics_are_not_answers", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/page", + "task_kind": "resource", + "content_score_applicable": false, + "expected_resources": [ + { + "id": "video", + "urls": [ + "https://example.test/target.mp4" + ] + } + ] + }, + "artifact": "{\"network\": [{\"url\": \"https://example.test/target.mp4\"}]}", + "ok": true, + "expected_task_pass": false + }, + { + "id": "resource-log-with-delivered-answer", + "category": "answer-surface", + "root_cause": "resource_network_diagnostics_are_not_answers", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/page", + "task_kind": "resource", + "content_score_applicable": false, + "expected_resources": [ + { + "id": "video", + "urls": [ + "https://example.test/target.mp4" + ] + } + ] + }, + "artifact": "{\"network\": [{\"url\": \"https://example.test/target.mp4\"}], \"resources\": [\"https://example.test/target.mp4\"]}", + "ok": true, + "expected_task_pass": true + }, + { + "id": "forbidden-latin-word-fragment", + "category": "format-normalization", + "root_cause": "forbidden_accented_latin_word_boundaries", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/page", + "forbidden_any": [ + "café" + ] + }, + "artifact": "caféteria", + "ok": true, + "expected_task_pass": true + }, + { + "id": "forbidden-latin-whole-word", + "category": "format-normalization", + "root_cause": "forbidden_accented_latin_word_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/page", + "forbidden_any": [ + "café" + ] + }, + "artifact": "A café.", + "ok": true, + "expected_task_pass": false + }, + { + "id": "neutral-frame-resource-absent", + "category": "answer-surface", + "root_cause": "neutral_inline_frame_document_url", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/page", + "task_kind": "resource", + "content_score_applicable": false, + "expected_resources": [ + { + "id": "frame", + "urls": [ + "https://frames.test/path/clip.mp4" + ] + } + ] + }, + "artifact": "", + "ok": true, + "expected_task_pass": false + }, + { + "id": "neutral-frame-resource-present", + "category": "answer-surface", + "root_cause": "neutral_inline_frame_document_url", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/page", + "task_kind": "resource", + "content_score_applicable": false, + "expected_resources": [ + { + "id": "frame", + "urls": [ + "https://frames.test/path/clip.mp4" + ] + } + ] + }, + "artifact": "", + "ok": true, + "expected_task_pass": true } ], "superseded_cases": { @@ -55116,7 +55279,8 @@ "body-unfenced-native-table-code-retained": "body-unfenced-table-is-markup-not-literal-code", "dynamic-matching-final-page-and-language-remain-eligible": "query-sensitive-dynamic-matching-final-page-and-language-remain-eligible", "answer-title-supplies-required-fact": "body-only-word-boundary-answer-title-supplies-required-fact", - "answer-title-does-not-inflate-body-length": "body-only-word-boundary-answer-title-does-not-inflate-body-length" + "answer-title-does-not-inflate-body-length": "body-only-word-boundary-answer-title-does-not-inflate-body-length", + "explicit-contract-missing-page-identity-with-valid-body-stays-unknown": "explicit-contract-missing-page-identity-counts-as-failure" }, "superseded_controls": { "frozen_scorer_sha256": "ceb62fb821e6777d24a015ff713f406faecfff6181b0601b8e3ea2adffbf7c10", diff --git a/src/lexbench_webfetch/metrics.py b/src/lexbench_webfetch/metrics.py index 6140901..b6ac841 100644 --- a/src/lexbench_webfetch/metrics.py +++ b/src/lexbench_webfetch/metrics.py @@ -78,6 +78,8 @@ def _metadata_resource_candidates( if isinstance(value, dict): for child_key, child in value.items(): child_key = str(child_key) + if child_key.casefold() in {"network", "request", "requests", "logs", "debug", "diagnostics"}: + continue candidates.extend(_metadata_resource_candidates( child, key=child_key.casefold(), path=f"{path}.{child_key}", )) @@ -188,7 +190,7 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None return source = values.get("srcdoc") if source: - self.documents.append((source, values.get("data-moli-frame-url"))) + self.documents.append((source, values.get("data-frame-url") or values.get("data-moli-frame-url"))) def _inline_frame_candidates( @@ -199,10 +201,9 @@ def _inline_frame_candidates( ) -> list[tuple[str, str]]: """Read resource URLs from visible native ``srcdoc`` serialization. - Moli's official ``--with-frames`` output keeps child documents inside the - returned HTML. This is still provider output; the scorer only resolves - URLs that the browser serialized, and ignores hidden synchronization or ad - frames. The same parser applies to any provider returning inline frames. + Returned inline documents are provider output. Resolve their declared + document URLs and ignore hidden synchronization or advertising frames. + Historical serialized attributes remain readable without rewriting evidence. """ if not re.search(r' list[tuple[str, str]]: - """Read URLs from a browser's native JSON page and network record. - - The accepted surface is deliberately structural: serialized page HTML and - network entries named ``url`` or ``final_url``. It never reads expected - answers to choose a field or invents a URL from a playback identifier. - """ + """Read declared answer fields, keeping request and network logs diagnostic.""" try: payload = json.loads(content) except (json.JSONDecodeError, TypeError): @@ -268,7 +264,7 @@ def _native_json_resource_candidates( if not isinstance(payload, dict): return [] candidates: list[tuple[str, str]] = [] - html = payload.get("html") + html = payload.get("html") or payload.get("content") if isinstance(html, str): candidates.extend( (match[1], "content.native_json_html_url") @@ -283,17 +279,10 @@ def _native_json_resource_candidates( candidates.extend(_inline_frame_candidates( html, base_url=base_url, images_only=False, )) - pending = [payload.get("network")] - while pending: - value = pending.pop() - if isinstance(value, dict): - for key, item in value.items(): - if key in {"url", "final_url"} and isinstance(item, str): - candidates.append((item, f"content.native_json_network.{key}")) - elif isinstance(item, (dict, list)): - pending.append(item) - elif isinstance(value, list): - pending.extend(value) + for key in ("resources", "links", "images", "videos", "media", "extras"): + if key in payload: + candidates.extend(_metadata_resource_candidates( + payload[key], key="links", path=f"content.native_json.{key}")) return candidates @@ -365,17 +354,23 @@ def evaluate_resources( answer_fields = {key: metadata[key] for key in ('links','images','videos','media','extras') if key in metadata} candidates.extend(_metadata_resource_candidates(answer_fields)) if not typed_json_answer: - from .reference_quality import markdown_link_destinations - candidates.extend((url, "content.bare_url") for url in - re.findall(r"https?://[^\s<>'\")\]]+", content)) - candidates.extend((match[1], "content.html_url") for match in - re.findall(r"(?:src|href)\s*=\s*([\"'])(.*?)\1", content, re.I)) - candidates.extend((url, "content.markdown_link") - for url in markdown_link_destinations(content)) - candidates.extend((url, "content.visible_file") - for url in _visible_relative_resource_urls(content)) - candidates.extend(_inline_frame_candidates(content, base_url=base_url, images_only=False)) - candidates.extend(_native_json_resource_candidates(content, base_url=base_url)) + try: + structured = json.loads(content) + except (ValueError, TypeError): + structured = None + if isinstance(structured, (dict, list)): + candidates.extend(_native_json_resource_candidates(content, base_url=base_url)) + else: + from .reference_quality import markdown_link_destinations + candidates.extend((url, "content.bare_url") for url in + re.findall(r"https?://[^\s<>'\")\]]+", content)) + candidates.extend((match[1], "content.html_url") for match in + re.findall(r"(?:src|href)\s*=\s*([\"'])(.*?)\1", content, re.I)) + candidates.extend((url, "content.markdown_link") + for url in markdown_link_destinations(content)) + candidates.extend((url, "content.visible_file") + for url in _visible_relative_resource_urls(content)) + candidates.extend(_inline_frame_candidates(content, base_url=base_url, images_only=False)) normalized_candidates, candidate_evidence = _normalized_candidates(candidates, base_url=base_url) results = [] allowed_urls: set[str] = set() @@ -474,20 +469,6 @@ def task_measured(metrics: dict, task_kind: str | None = None) -> bool: if metrics.get('rule_review_status') == 'unresolved': return False kind = task_kind or metrics.get('task_kind', 'body') - if (metrics.get('page_identity_required') - and metrics.get('page_identity_status') == 'unavailable' - and metrics.get('request_ok') is not False): - # Combined pass flags already include identity. Only independent - # failed requirements can prove failure while identity is unknown. - definite_failure = bool( - metrics.get('deadline_overrun') - or any(error != 'page_identity_not_verified' - for error in metrics.get('stable_task_errors', [])) - or (kind == 'body' and (metrics.get('length_pass') is False - or metrics.get('body_coverage_pass') is False)) - ) - if not definite_failure: - return False return kind != 'file' or isinstance(metrics.get('raw_file_pass'), bool) @@ -594,11 +575,14 @@ def fact_in_text(text: str, fact: str, *, latin_boundaries: bool = True) -> bool def forbidden_in_text(text: str, phrase: str) -> bool: - """Match ASCII word boundaries without segmenting CJK phrases.""" + """Match Latin word boundaries without segmenting CJK phrases.""" if not phrase: return False - left = r'(?Article"} + if delivered: + payload["resources"] = [target] + if surface == "json_log_list" and not delivered: + payload = diagnostics["network"] + artifact, metadata = json.dumps(payload).encode(), {} + else: + artifact = b"No resource URL in the answer" + metadata = {"extras": diagnostics} + if delivered: + metadata["videos"] = [target] + metrics = score_attempt(case, artifact, ok=True, metadata=metadata) + assert task_passed(True, metrics) is delivered + assert bool(metrics["resource_hits"]) is delivered + + +@pytest.mark.parametrize("answer", ["caféteria", "décafé"]) +def test_forbidden_latin_words_do_not_match_fragments(answer): + case = {"url": "https://example.test/page", "forbidden_any": ["café"]} + metrics = score_attempt(case, answer.encode(), ok=True) + assert task_passed(True, metrics) + assert not task_passed(True, score_attempt(case, b"A caf\xc3\xa9.", ok=True)) + + +def test_required_missing_identity_is_a_failure_in_the_full_population(): + from lexbench_webfetch.metrics import task_measured + case = {"url": "https://example.test/page", "page_identity_required": True} + metrics = score_attempt(case, b"Article", ok=True) + assert task_measured(metrics) + assert not task_passed(True, metrics) + + +@pytest.mark.parametrize("attribute", ["data-frame-url", "data-moli-frame-url"]) +def test_visible_frame_relative_resources_use_the_declared_document_url(attribute): + case = {"url": "https://example.test/page", "task_kind": "resource", + "expected_resources": [{"id": "frame", "urls": ["https://frames.test/path/clip.mp4"]}]} + artifact = (f'').encode() + assert task_passed(True, score_attempt(case, artifact, ok=True)) From 42c5e3b0b0ffe2631507ac1b44337ab6b43e853b Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:57:25 +0800 Subject: [PATCH 04/20] fix(scoring): retain failed file identity checks in the population --- .../evaluation/scoring-regressions.json | 36 +++++++++++++++++++ src/lexbench_webfetch/metrics.py | 2 ++ tests/test_acceptance_isolation.py | 5 +-- 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json index 71cb77c..5554378 100644 --- a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json +++ b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json @@ -55263,6 +55263,42 @@ "artifact": "", "ok": true, "expected_task_pass": true + }, + { + "category": "acceptance-contract", + "root_cause": "required_file_identity_missing_must_not_shrink_population", + "case": { + "url": "https://example.test/file", + "task_kind": "file", + "page_identity_required": true + }, + "artifact": "File delivered", + "ok": true, + "expected_task_measured": true, + "id": "file-missing-required-identity-is-measured-failure", + "failure_mode": "false-positive", + "expected_task_pass": false + }, + { + "category": "acceptance-contract", + "root_cause": "required_file_identity_missing_must_not_shrink_population", + "case": { + "url": "https://example.test/file", + "task_kind": "file", + "page_identity_required": true + }, + "artifact": "File delivered", + "ok": true, + "expected_task_measured": true, + "id": "file-with-required-identity-and-current-verdict-passes", + "failure_mode": "false-negative", + "final_url": "https://example.test/file", + "metadata": { + "file_evaluation": { + "pass": true + } + }, + "expected_task_pass": true } ], "superseded_cases": { diff --git a/src/lexbench_webfetch/metrics.py b/src/lexbench_webfetch/metrics.py index b6ac841..484eab7 100644 --- a/src/lexbench_webfetch/metrics.py +++ b/src/lexbench_webfetch/metrics.py @@ -469,6 +469,8 @@ def task_measured(metrics: dict, task_kind: str | None = None) -> bool: if metrics.get('rule_review_status') == 'unresolved': return False kind = task_kind or metrics.get('task_kind', 'body') + if metrics.get('page_identity_required') and metrics.get('page_identity_status') != 'verified': + return True return kind != 'file' or isinstance(metrics.get('raw_file_pass'), bool) diff --git a/tests/test_acceptance_isolation.py b/tests/test_acceptance_isolation.py index be7e85a..b13de84 100644 --- a/tests/test_acceptance_isolation.py +++ b/tests/test_acceptance_isolation.py @@ -180,9 +180,10 @@ def test_forbidden_latin_words_do_not_match_fragments(answer): assert not task_passed(True, score_attempt(case, b"A caf\xc3\xa9.", ok=True)) -def test_required_missing_identity_is_a_failure_in_the_full_population(): +@pytest.mark.parametrize("kind", ["body", "resource", "file"]) +def test_required_missing_identity_is_a_failure_in_the_full_population(kind): from lexbench_webfetch.metrics import task_measured - case = {"url": "https://example.test/page", "page_identity_required": True} + case = {"url": "https://example.test/page", "page_identity_required": True, "task_kind": kind} metrics = score_attempt(case, b"Article", ok=True) assert task_measured(metrics) assert not task_passed(True, metrics) From 28fc344cb1e85df4aba0efc5dc018fb1999ace00 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:02:01 +0800 Subject: [PATCH 05/20] docs(citation): align CFF metadata and add bilingual BibTeX examples --- CITATION.cff | 3 ++- README.md | 21 +++++++++++++++++++-- docs/README.zh-CN.md | 21 +++++++++++++++++++-- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index e077256..a5481a5 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -5,5 +5,6 @@ title: LexBench Web Fetch authors: - name: LexBench Web Fetch contributors repository-code: "https://github.com/lexmount/lexbench-webfetch" +url: "https://github.com/lexmount/lexbench-webfetch" license: MIT -version: 0.2.0 +version: 0.3.0 diff --git a/README.md b/README.md index 17d45d8..a4428fc 100644 --- a/README.md +++ b/README.md @@ -129,8 +129,25 @@ These changes do not revise frozen tasks or certify historical results. Score co Browser adapters, credentials, measured runs, report renderers and hosting configuration belong to the execution harness or its evidence storage. This repository owns task models, deterministic acceptance scoring, file-byte checks and task compilation. Browser Eval imports these implementations as a pinned dependency. This repository publishes no measured leaderboard. -## Contribute and cite +## Contribute Read [CONTRIBUTING](CONTRIBUTING.md) before changing a task or proposing a result. Run `make check` to validate the definitions and test the input tools. Report security issues through [SECURITY](SECURITY.md). -Cite [LexBench Web Fetch](CITATION.cff) and record the exact commit and suite. Cite any execution harness separately. Project-owned code and definitions use the [MIT license](LICENSE); third-party source material retains its own terms, described in [THIRD_PARTY](THIRD_PARTY.md). +Project-owned code and definitions use the [MIT license](LICENSE); third-party source material retains its own terms, described in [THIRD_PARTY](THIRD_PARTY.md). + +## Cite LexBench Web Fetch + +If you use the task definitions or scoring tools in published work, cite this repository and record the exact commit and suite used. GitHub's citation menu reads [CITATION.cff](CITATION.cff). For version 0.3.0: + +```bibtex +@software{lexbench_webfetch_2026, + author = {{LexBench Web Fetch contributors}}, + title = {LexBench Web Fetch}, + year = {2026}, + version = {0.3.0}, + url = {https://github.com/lexmount/lexbench-webfetch} +} +``` + +Cite [Browser Eval](https://github.com/lexmount/browser-eval#cite-browser-eval) separately when using its execution harness, and cite upstream sources or benchmarks according to their attribution terms. This repository citation does not replace those citations. When using another version, update the version and year in the example and identify its tag or commit. + diff --git a/docs/README.zh-CN.md b/docs/README.zh-CN.md index d22c41d..090a5ca 100644 --- a/docs/README.zh-CN.md +++ b/docs/README.zh-CN.md @@ -129,8 +129,25 @@ uv run --frozen browser-harness-eval run \ 浏览器适配器、凭据、实测结果、报告生成器和托管配置由执行脚手架或外部证据存储管理。任务模型、确定性验收评分、文件字节校验和任务编译由本仓库维护，Browser Eval 通过固定版本的依赖直接调用。本仓库没有实测排行榜。 -## 参与贡献与引用 +## 参与贡献 修改任务或提交结果前，请阅读 [CONTRIBUTING](../CONTRIBUTING.md)。`make check` 会检查任务定义和输入工具。安全问题请参阅 [SECURITY](../SECURITY.md)。 -引用 [LexBench Web Fetch](../CITATION.cff) 时，请记录具体提交版本和任务集；执行脚手架单独引用。项目自有代码和任务定义采用 [MIT 许可](../LICENSE)，第三方原始材料保留各自使用条件，详见 [THIRD_PARTY](../THIRD_PARTY.md)。 +项目自有代码和任务定义采用 [MIT 许可](../LICENSE)，第三方原始材料保留各自使用条件，详见 [THIRD_PARTY](../THIRD_PARTY.md)。 + +## 引用 LexBench Web Fetch + +在公开作品中使用本仓库的任务定义或评分工具时，请引用仓库，并记录具体提交和任务集。GitHub 的引用入口读取 [CITATION.cff](../CITATION.cff)。0.3.0 版可这样引用： + +```bibtex +@software{lexbench_webfetch_2026, + author = {{LexBench Web Fetch contributors}}, + title = {LexBench Web Fetch}, + year = {2026}, + version = {0.3.0}, + url = {https://github.com/lexmount/lexbench-webfetch} +} +``` + +如果还使用了 [Browser Eval](https://github.com/lexmount/browser-eval#cite-browser-eval) 执行脚手架，请单独引用。上游来源或 benchmark 也应按各自的署名要求引用，本仓库的引用不能替代它们。使用其他版本时，请相应修改示例中的版本号和年份，并注明实际 tag 或提交。 + From 165947c2a9f269a55810de0086fa86aadb749598 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:03:28 +0800 Subject: [PATCH 06/20] docs(citation): keep bilingual examples consistently formatted --- README.md | 1 - docs/README.zh-CN.md | 1 - 2 files changed, 2 deletions(-) diff --git a/README.md b/README.md index a4428fc..d6cdccd 100644 --- a/README.md +++ b/README.md @@ -150,4 +150,3 @@ If you use the task definitions or scoring tools in published work, cite this re ``` Cite [Browser Eval](https://github.com/lexmount/browser-eval#cite-browser-eval) separately when using its execution harness, and cite upstream sources or benchmarks according to their attribution terms. This repository citation does not replace those citations. When using another version, update the version and year in the example and identify its tag or commit. - diff --git a/docs/README.zh-CN.md b/docs/README.zh-CN.md index 090a5ca..039035b 100644 --- a/docs/README.zh-CN.md +++ b/docs/README.zh-CN.md @@ -150,4 +150,3 @@ uv run --frozen browser-harness-eval run \ ``` 如果还使用了 [Browser Eval](https://github.com/lexmount/browser-eval#cite-browser-eval) 执行脚手架，请单独引用。上游来源或 benchmark 也应按各自的署名要求引用，本仓库的引用不能替代它们。使用其他版本时，请相应修改示例中的版本号和年份，并注明实际 tag 或提交。 - From 5b12816906ad8cd2b55b9804c77e7e6c55f2c54e Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:10:58 +0800 Subject: [PATCH 07/20] fix(scoring): preserve explicit JSON resource URL lists --- .../evaluation/scoring-regressions.json | 42 +++++++++++++++++++ src/lexbench_webfetch/metrics.py | 4 ++ tests/test_acceptance_isolation.py | 8 ++++ 3 files changed, 54 insertions(+) diff --git a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json index 5554378..370df9b 100644 --- a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json +++ b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json @@ -55299,6 +55299,48 @@ } }, "expected_task_pass": true + }, + { + "id": "json-url-list-answer-matches", + "category": "answer-surface", + "root_cause": "resource_network_diagnostics_are_not_answers", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/page", + "task_kind": "resource", + "expected_resources": [ + { + "id": "video", + "urls": [ + "/target.mp4" + ] + } + ] + }, + "artifact": "[\"/target.mp4\"]", + "ok": true, + "expected_task_pass": true + }, + { + "id": "json-url-list-answer-does-not-match", + "category": "answer-surface", + "root_cause": "resource_network_diagnostics_are_not_answers", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/page", + "task_kind": "resource", + "expected_resources": [ + { + "id": "video", + "urls": [ + "/target.mp4" + ] + } + ] + }, + "artifact": "[\"/other.mp4\"]", + "ok": true, + "expected_task_pass": false } ], "superseded_cases": { diff --git a/src/lexbench_webfetch/metrics.py b/src/lexbench_webfetch/metrics.py index 484eab7..942bf8e 100644 --- a/src/lexbench_webfetch/metrics.py +++ b/src/lexbench_webfetch/metrics.py @@ -261,6 +261,10 @@ def _native_json_resource_candidates( payload = json.loads(content) except (json.JSONDecodeError, TypeError): return [] + if isinstance(payload, list): + # A URL list is itself an answer; a list of request/log objects is not. + return [(value, "content.native_json_resource_list") for value in payload + if isinstance(value, str)] if all(isinstance(value, str) for value in payload) else [] if not isinstance(payload, dict): return [] candidates: list[tuple[str, str]] = [] diff --git a/tests/test_acceptance_isolation.py b/tests/test_acceptance_isolation.py index b13de84..8e4ca0c 100644 --- a/tests/test_acceptance_isolation.py +++ b/tests/test_acceptance_isolation.py @@ -196,3 +196,11 @@ def test_visible_frame_relative_resources_use_the_declared_document_url(attribut artifact = (f'').encode() assert task_passed(True, score_attempt(case, artifact, ok=True)) + + +@pytest.mark.parametrize("returned,passed", [("/target.mp4", True), ("/other.mp4", False)]) +def test_a_returned_json_url_list_remains_a_resource_answer(returned, passed): + import json + case = {"url": "https://example.test/page", "task_kind": "resource", + "expected_resources": [{"id": "video", "urls": ["/target.mp4"]}]} + assert task_passed(True, score_attempt(case, json.dumps([returned]).encode(), ok=True)) is passed From 2cd160deac7096595d286cb8e4adab5100b1787d Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Wed, 21 Oct 2026 00:05:36 +0800 Subject: [PATCH 08/20] docs(contributing): use a supported partial rerun example --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 264a5d2..d339779 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,7 +30,7 @@ Write the summary in English, start with an imperative verb, and describe the co Examples: - `fix(scoring): require facts in the returned body` -- `fix(webfetch): reject mixed executable versions before release` +- `fix(webfetch): disclose build provenance for partial reruns` - `docs(contributing): define PR and commit title conventions` Avoid titles such as `Fix review feedback`, `Update code` or `Misc improvements`; name the behavior or document that changes. A PR title describes the complete diff; each commit subject describes that commit. Update the PR title when its scope changes. If squash merging is used, use a subject that follows the same format. Apply this convention to new work; do not rewrite shared commits solely to rename them. From 1552f731dd75be26c3b3551c04bf0730e6293335 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Wed, 21 Oct 2026 17:32:40 +0800 Subject: [PATCH 09/20] fix(scoring): preserve frame resources and alphabetic word boundaries --- .../evaluation/scoring-regressions.json | 162 ++++++++++++++++++ src/lexbench_webfetch/metrics.py | 40 +++-- src/lexbench_webfetch/stable_tasks.py | 6 +- tests/test_acceptance_isolation.py | 23 +++ 4 files changed, 215 insertions(+), 16 deletions(-) diff --git a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json index 370df9b..297bdc8 100644 --- a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json +++ b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json @@ -55341,6 +55341,168 @@ "artifact": "[\"/other.mp4\"]", "ok": true, "expected_task_pass": false + }, + { + "id": "word-boundary-mixed-script-complete", + "category": "format-normalization", + "root_cause": "alphabetic_boundary_omits_greek_cyrillic", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "stats" + ] + } + ] + }, + "artifact": "(stats)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "word-boundary-mixed-script-fragment", + "category": "format-normalization", + "root_cause": "alphabetic_boundary_omits_greek_cyrillic", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "stats" + ] + } + ] + }, + "artifact": "αstats", + "ok": true, + "expected_task_pass": false + }, + { + "id": "word-boundary-greek-complete", + "category": "format-normalization", + "root_cause": "alphabetic_boundary_omits_greek_cyrillic", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "Αθήνα" + ] + } + ] + }, + "artifact": "(Αθήνα)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "word-boundary-greek-fragment", + "category": "format-normalization", + "root_cause": "alphabetic_boundary_omits_greek_cyrillic", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "Αθήνα" + ] + } + ] + }, + "artifact": "Αθήνας", + "ok": true, + "expected_task_pass": false + }, + { + "id": "word-boundary-cyrillic-complete", + "category": "format-normalization", + "root_cause": "alphabetic_boundary_omits_greek_cyrillic", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "новости" + ] + } + ] + }, + "artifact": "(новости)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "word-boundary-cyrillic-fragment", + "category": "format-normalization", + "root_cause": "alphabetic_boundary_omits_greek_cyrillic", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "новости" + ] + } + ] + }, + "artifact": "новостик", + "ok": true, + "expected_task_pass": false + }, + { + "id": "xml-declaration-frame-target", + "category": "structured-extraction", + "root_cause": "xml_declaration_discards_visible_frame_resources", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/spec", + "task_kind": "resource", + "expected_resources": [ + { + "id": "image", + "urls": [ + "/target.png" + ] + } + ] + }, + "artifact": "", + "ok": true, + "expected_task_pass": true + }, + { + "id": "xml-declaration-frame-non-target", + "category": "structured-extraction", + "root_cause": "xml_declaration_discards_visible_frame_resources", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/spec", + "task_kind": "resource", + "expected_resources": [ + { + "id": "image", + "urls": [ + "/target.png" + ] + } + ] + }, + "artifact": "", + "ok": true, + "expected_task_pass": false } ], "superseded_cases": { diff --git a/src/lexbench_webfetch/metrics.py b/src/lexbench_webfetch/metrics.py index 942bf8e..f6f3c48 100644 --- a/src/lexbench_webfetch/metrics.py +++ b/src/lexbench_webfetch/metrics.py @@ -176,6 +176,16 @@ def __init__(self) -> None: super().__init__(convert_charrefs=True) self.documents: list[tuple[str, str | None]] = [] + def parse_marked_section(self, i: int, report: int = 1) -> int: + # HTMLParser rejects XML conditional sections that are valid opaque + # declarations in returned documents. Skip their declared extent while + # retaining visible frames both before and after them. + try: + return super().parse_marked_section(i, report) + except AssertionError: + end = self.rawdata.find("]>", i + 3) + return end + 2 if end >= 0 else -1 + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: if tag.casefold() != "iframe": return @@ -208,13 +218,8 @@ def _inline_frame_candidates( if not re.search(r' *(?=\w)', '>', normalized) +# Alphabets whose adjacent letters form a word. CJK keeps declared phrase +# containment because those writing systems do not require word separators. +WORD_CHARACTERS = r'a-z0-9_\u00c0-\u024f\u1e00-\u1eff\u0370-\u052f\u1f00-\u1fff\u2de0-\u2dff\ua640-\ua69f' + + +def _word_letter(char: str) -> bool: + return any(script in unicodedata.name(char, '') for script in ('LATIN', 'GREEK', 'CYRILLIC')) + + def fact_in_text(text: str, fact: str, *, latin_boundaries: bool = True) -> bool: """Match an explicitly registered normalized fact without fuzzy semantics.""" if not fact: @@ -568,10 +582,10 @@ def fact_in_text(text: str, fact: str, *, latin_boundaries: bool = True) -> bool right = r'(?![0-9]|\.[0-9])' if fact[-1].isdigit() else '' # Latin words must not match fragments of longer words. Keep CJK phrase # matching and the stricter numeric/currency boundaries intact. - latin_word = r'a-z0-9_\u00c0-\u024f\u1e00-\u1eff' - if latin_boundaries and 'LATIN' in unicodedata.name(fact[0], ''): + latin_word = WORD_CHARACTERS + if latin_boundaries and _word_letter(fact[0]): left = rf'(? bool: """Match Latin word boundaries without segmenting CJK phrases.""" if not phrase: return False - latin_word = r'a-z0-9_\u00c0-\u024f\u1e00-\u1eff' + latin_word = WORD_CHARACTERS left = rf'(?", + "", "]]>"]) +@pytest.mark.parametrize("position", ["before", "after"]) +def test_xml_declarations_preserve_visible_frame_resource_answers(declaration, position): + case = {"url": "https://example.test/spec", "task_kind": "resource", + "expected_resources": [{"id": "image", "urls": ["/target.png"]}]} + for filename, passed in [("target.png", True), ("other.png", False)]: + frame = '' + answer = declaration + frame if position == "before" else frame + declaration + assert task_passed(True, score_attempt(case, answer.encode(), ok=True)) is passed + + +@pytest.mark.parametrize("literal,fragment", [("stats", "αstats"), ("stats", "statsя"), + ("Αθήνα", "Αθήνας"), ("новости", "новостик")]) +def test_forbidden_words_use_the_same_alphabetic_boundaries(literal, fragment): + case = {"url": "https://example.test/article", "forbidden_any": [literal]} + for answer, passed in [(fragment, True), ('(' + literal + ')', False)]: + assert task_passed(True, score_attempt(case, answer.encode(), ok=True)) is passed From 1942cd7917bed75d69ec60737720595b5ab22984 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Wed, 21 Oct 2026 19:45:41 +0800 Subject: [PATCH 10/20] fix(scoring): preserve native labels and owned card fields --- src/lexbench_webfetch/composite_tasks.py | 48 +- .../risk-control/current/tasks.json | 3 +- .../evaluation/scoring-regressions.json | 644 ++++++++++++++++++ .../definitions/evaluation/tasks.json | 34 +- .../definitions/registry.json | 4 +- src/lexbench_webfetch/dynamic_lists.py | 12 +- src/lexbench_webfetch/metrics.py | 17 +- src/lexbench_webfetch/reference_quality.py | 22 +- src/lexbench_webfetch/stable_tasks.py | 18 +- tests/test_named_scope_heading.py | 29 + tests/test_native_card_fields.py | 22 + tests/test_resource_link_serialization.py | 32 + tests/test_stable_text_serialization.py | 56 ++ 13 files changed, 897 insertions(+), 44 deletions(-) create mode 100644 tests/test_named_scope_heading.py create mode 100644 tests/test_native_card_fields.py create mode 100644 tests/test_resource_link_serialization.py create mode 100644 tests/test_stable_text_serialization.py diff --git a/src/lexbench_webfetch/composite_tasks.py b/src/lexbench_webfetch/composite_tasks.py index f04d0bf..4a02c27 100644 --- a/src/lexbench_webfetch/composite_tasks.py +++ b/src/lexbench_webfetch/composite_tasks.py @@ -433,7 +433,22 @@ def _card_section_context(content: str, begin: int, end: int, title_start: int, peer_boundary = (peer_item and ( (item_indent is not None and peer_indent <= item_indent) or (item_indent is None and own_heading is None and peer_indent == 0))) - if (peer_boundary and not owned_item and absolute > title_start + field_item = False + if peer_item and not markdown_links(line): + # Native cards may serialize their typed fields as sibling + # bullets. A complete declared value belongs to this card; + # arbitrary peer prose still starts a new section. + values = _context_lines(line, "") + for spec in group.get("required_fields", []): + if not (spec.get("aliases") or _field_token_pattern(spec)): + continue + found = _field_value(values, spec) + if found: + number, _, start, end = found + if not (values[number][:start] + values[number][end:]).strip(): + field_item = True + break + if (peer_boundary and not owned_item and not field_item and absolute > title_start and not any(link.start <= absolute < link.end for link in links)): return context[:offset] owned_heading = heading and any( @@ -912,7 +927,7 @@ def serialized_link_card_fields(link): field = re.sub(r"\\+$", "", raw.strip()).strip() heading = re.match(r"^#{1,6}[ \t]+\S", field) field = re.sub(r"^#{1,6}(?:[ \t]+|$)", "", field).strip(" *_") - if field: + if field and not re.fullmatch(r"[-+*_ \t]+", field): if not fields: explicit_name = heading is not None fields.append(field) @@ -1012,6 +1027,8 @@ def _field_token_pattern(spec: dict): """Typed card fields own a distinct value span, not an entire layout line.""" if spec.get('aliases'): return None + if spec['kind'] == 'text' and spec.get('pattern'): + return re.compile(spec['pattern'], re.I) if spec['kind'] == 'integer' and spec.get('pattern'): # A number elsewhere in the card cannot prove a numeric labeled field. return re.compile(rf"\b{_AMOUNT}\s*(?:{spec['pattern']})|(?:{spec['pattern']})", re.I) @@ -1021,6 +1038,7 @@ def _field_token_pattern(spec: dict): def _text_role_value(value: str, spec: dict) -> str | None: """Explicit metadata belongs to its declared role, not another text field.""" + value = value.strip() roles = {"title", "name", "author", "creator", "owner", "source", "publisher", "description", "summary", "date", "time", "version", "downloads", "views", "likes", "count", "price", "rating", "size"} @@ -1124,7 +1142,14 @@ def _field_value( continue if token_pattern := _field_token_pattern(spec): for match in token_pattern.finditer(value): - if _value_matches_spec(match[0], spec): + if spec["kind"] == "text" and ( + match.start() and value[match.start() - 1].isalnum() and match[0][:1].isalnum() + or match.end() < len(value) and value[match.end()].isalnum() and match[0][-1:].isalnum() + ): + continue + checked = {key: value for key, value in spec.items() + if key != "pattern"} if spec["kind"] == "text" else spec + if _value_matches_spec(match[0], checked): return index, match[0], start + match.start(), start + match.end() continue checked_spec = {**spec, "minimum_chars": 1} if aliases and spec["kind"] == "text" and "minimum_chars" not in spec else spec @@ -1176,20 +1201,6 @@ def _fixed_field_value_key(value: str, spec: dict) -> str: def _required_fields(context: str, title: str, *, field_specs: list[dict]) -> bool: """Require every declared field inside the record block which owns the link.""" lines = _labeled_field_lines(_context_lines(context, title), field_specs) - separated = [] - patterns = [re.compile(spec["pattern"], re.I) for spec in field_specs - if spec["kind"] == "text" and spec.get("pattern") and not spec.get("aliases")] - for line in lines: - for pattern in patterns: - match = pattern.match(line) - if match and match.end() and (match.end() == len(line) or not line[match.end()].isalnum()): - rest = line[match.end():].strip(" |\t") - if rest: - separated.extend((match[0], rest)) - break - else: - separated.append(line) - lines = separated used: set[int] = set() typed_lines: set[int] = set() declared = _declared_aliases(field_specs) @@ -1208,7 +1219,8 @@ def _required_fields(context: str, title: str, *, field_specs: list[dict]) -> bo # physical line, but one value cannot satisfy two required fields. lines[index] = lines[index][:start] + " " * (end - start) + lines[index][end:] if typed: - typed_lines.add(index) + if spec["kind"] != "text": + typed_lines.add(index) elif not spec.get("aliases"): used.add(index) return True diff --git a/src/lexbench_webfetch/definitions/evaluation/risk-control/current/tasks.json b/src/lexbench_webfetch/definitions/evaluation/risk-control/current/tasks.json index 09404eb..984a3f2 100644 --- a/src/lexbench_webfetch/definitions/evaluation/risk-control/current/tasks.json +++ b/src/lexbench_webfetch/definitions/evaluation/risk-control/current/tasks.json @@ -5057,7 +5057,8 @@ { "id": "city_or_region", "kind": "text", - "minimum_chars": 2 + "minimum_chars": 2, + "pattern": ",\\s*(?:AL|AK|AZ|AR|CA|CO|CT|DE|DC|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY)\\b" } ] } diff --git a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json index 297bdc8..3a85d6d 100644 --- a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json +++ b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json @@ -55503,6 +55503,650 @@ "artifact": "", "ok": true, "expected_task_pass": false + }, + { + "id": "native-serialization-complete-notice", + "category": "format-normalization", + "root_cause": "visible_label_boundaries_in_native_serialization", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "required_facts": [ + { + "id": "label", + "alternatives": [ + "Example Corp. Avenida N°5, Santiago de Chile" + ] + } + ] + }, + "artifact": "ALL RIGHTS RESERVEDExample Corp. Avenida N°5, Santiago de Chile", + "ok": true, + "expected_task_pass": true + }, + { + "id": "native-serialization-changed-notice", + "category": "format-normalization", + "root_cause": "visible_label_boundaries_in_native_serialization", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "required_facts": [ + { + "id": "label", + "alternatives": [ + "Example Corp. Avenida N°5, Santiago de Chile" + ] + } + ] + }, + "artifact": "ALL RIGHTS RESERVEDExample Corp. Avenida N°6, Santiago de Chile", + "ok": true, + "expected_task_pass": false + }, + { + "id": "native-serialization-stock-label", + "category": "format-normalization", + "root_cause": "visible_label_boundaries_in_native_serialization", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "required_facts": [ + { + "id": "label", + "alternatives": [ + "In stock" + ] + } + ] + }, + "artifact": "In stockAdd to basket", + "ok": true, + "expected_task_pass": true + }, + { + "id": "native-serialization-stock-word-fragment", + "category": "format-normalization", + "root_cause": "visible_label_boundaries_in_native_serialization", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "required_facts": [ + { + "id": "label", + "alternatives": [ + "In stock" + ] + } + ] + }, + "artifact": "In stockholders", + "ok": true, + "expected_task_pass": false + }, + { + "id": "native-serialization-carousel-label", + "category": "format-normalization", + "root_cause": "visible_label_boundaries_in_native_serialization", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "required_facts": [ + { + "id": "label", + "alternatives": [ + "Careers" + ] + } + ] + }, + "artifact": "[Careers9 of 9](/careers)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "native-serialization-carousel-digit-word", + "category": "format-normalization", + "root_cause": "visible_label_boundaries_in_native_serialization", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "required_facts": [ + { + "id": "label", + "alternatives": [ + "Careers" + ] + } + ] + }, + "artifact": "[Careers9](/careers)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "native-serialization-cyrillic-label", + "category": "format-normalization", + "root_cause": "visible_label_boundaries_in_native_serialization", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "required_facts": [ + { + "id": "label", + "alternatives": [ + "Наука" + ] + } + ] + }, + "artifact": "[Технологии](/technology)[Наука](/science)Все темы", + "ok": true, + "expected_task_pass": true + }, + { + "id": "native-serialization-cyrillic-word-fragment", + "category": "format-normalization", + "root_cause": "visible_label_boundaries_in_native_serialization", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "required_facts": [ + { + "id": "label", + "alternatives": [ + "Наука" + ] + } + ] + }, + "artifact": "Наукам", + "ok": true, + "expected_task_pass": false + }, + { + "id": "native-serialization-punctuation-markup", + "category": "format-normalization", + "root_cause": "visible_label_boundaries_in_native_serialization", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "required_facts": [ + { + "id": "label", + "alternatives": [ + "toute l’actu dès 7h30" + ] + } + ] + }, + "artifact": "_:_toute l’actu dès 7h30", + "ok": true, + "expected_task_pass": true + }, + { + "id": "resource-serialization-markup-entity", + "category": "format-normalization", + "root_cause": "markdown_destination_entities_decoded_once", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/gallery", + "task_kind": "resource", + "expected_resources": [ + { + "id": "photo", + "urls": [ + "https://cdn.example.test/photo.jpg?width=640&height=480" + ] + } + ], + "resource_output_contract": { + "candidate_scope": "image_answer_fields", + "unexpected": "reject" + } + }, + "artifact": "![Photo](https://cdn.example.test/photo.jpg?width=640&height=480)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "resource-serialization-different-size", + "category": "format-normalization", + "root_cause": "markdown_destination_entities_decoded_once", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/gallery", + "task_kind": "resource", + "expected_resources": [ + { + "id": "photo", + "urls": [ + "https://cdn.example.test/photo.jpg?width=640&height=480" + ] + } + ], + "resource_output_contract": { + "candidate_scope": "image_answer_fields", + "unexpected": "reject" + } + }, + "artifact": "![Photo](https://cdn.example.test/photo.jpg?width=641&height=480)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "named-scope-heading-owns-records", + "category": "structured-extraction", + "root_cause": "heading_scope_and_empty_nested_card_layout", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/search", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "entry_scope": { + "html_selector": ".results a[href]", + "start_pattern": "^Find games$", + "end_pattern": "^See all" + } + } + }, + "artifact": "Find games\n[Help](/help)\nSee all\n### Find games\n[Alpha](/item/1)\n[Beta](/item/2)\nSee all", + "ok": true, + "expected_task_pass": true + }, + { + "id": "named-scope-navigation-cannot-supply-records", + "category": "structured-extraction", + "root_cause": "heading_scope_and_empty_nested_card_layout", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/search", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "entry_scope": { + "html_selector": ".results a[href]", + "start_pattern": "^Find games$", + "end_pattern": "^See all" + } + } + }, + "artifact": "Find games\n[Alpha](/item/1)\n[Beta](/item/2)\nSee all\n### Find games\n[Gamma](/item/3)\nSee all\n[Delta](/item/4)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "named-scope-empty-layout-is-not-name", + "category": "structured-extraction", + "root_cause": "heading_scope_and_empty_nested_card_layout", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/search", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "entry_scope": { + "html_selector": ".results a[href]", + "start_pattern": "^Find games$", + "end_pattern": "^See all" + } + } + }, + "artifact": "### Find games\n- [![](/cover.jpg)\n\n - **\n\nAlpha game\n2026.3.26\n8,500円\nPublisher\n**](/item/1)\n- [Beta](/item/2)\nSee all", + "ok": true, + "expected_task_pass": true + }, + { + "id": "named-scope-image-without-visible-name", + "category": "structured-extraction", + "root_cause": "heading_scope_and_empty_nested_card_layout", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/search", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "entry_scope": { + "html_selector": ".results a[href]", + "start_pattern": "^Find games$", + "end_pattern": "^See all" + } + } + }, + "artifact": "### Find games\n- [![](/cover.jpg)\n\n - **\n\n2026.3.26\n**](/item/1)\n- [Beta](/item/2)\nSee all", + "ok": true, + "expected_task_pass": false + }, + { + "id": "native-card-bullet-complete", + "category": "structured-extraction", + "root_cause": "typed_card_fields_serialized_as_sibling_bullets", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "homes", + "kind": "named_link", + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "minimum_records": 1, + "required_fields": [ + { + "id": "price", + "kind": "price" + }, + { + "id": "bedrooms", + "kind": "integer", + "pattern": "\\bbds\\b" + }, + { + "id": "bathrooms", + "kind": "integer", + "pattern": "\\bba\\b" + }, + { + "id": "area", + "kind": "area" + } + ] + } + ] + } + }, + "artifact": "- [$900](/item/1)\n- **3** bds\n- **2** ba\n- **900** sqft\nActive[House A](/item/1)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "native-card-bullet-missing-area", + "category": "structured-extraction", + "root_cause": "typed_card_fields_serialized_as_sibling_bullets", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "homes", + "kind": "named_link", + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "minimum_records": 1, + "required_fields": [ + { + "id": "price", + "kind": "price" + }, + { + "id": "bedrooms", + "kind": "integer", + "pattern": "\\bbds\\b" + }, + { + "id": "bathrooms", + "kind": "integer", + "pattern": "\\bba\\b" + }, + { + "id": "area", + "kind": "area" + } + ] + } + ] + } + }, + "artifact": "- [House A](/item/1)\n- [House B](/item/2)\n- $900\n- 3 bds\n- 2 ba", + "ok": true, + "expected_task_pass": false + }, + { + "id": "native-card-bullet-other-section", + "category": "structured-extraction", + "root_cause": "typed_card_fields_serialized_as_sibling_bullets", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "homes", + "kind": "named_link", + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "minimum_records": 1, + "required_fields": [ + { + "id": "price", + "kind": "price" + }, + { + "id": "bedrooms", + "kind": "integer", + "pattern": "\\bbds\\b" + }, + { + "id": "bathrooms", + "kind": "integer", + "pattern": "\\bba\\b" + }, + { + "id": "area", + "kind": "area" + } + ] + } + ] + } + }, + "artifact": "- [House A](/item/1)\n# Other section\n- $900\n- 3 bds\n- 2 ba\n- 900 sqft", + "ok": true, + "expected_task_pass": false + }, + { + "id": "native-card-text-same-line-region", + "category": "structured-extraction", + "root_cause": "declared_text_patterns_consume_distinct_value_spans", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "events", + "kind": "named_link", + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "minimum_records": 1, + "required_fields": [ + { + "id": "venue", + "kind": "text", + "minimum_chars": 2 + }, + { + "id": "region", + "kind": "text", + "pattern": ",\\s*(?:TX|AL)\\b" + }, + { + "id": "date", + "kind": "date_or_time" + } + ] + } + ] + } + }, + "artifact": "[Concert](/item/1)\n2026-10-16\nExample Hall Tuscaloosa, AL", + "ok": true, + "expected_task_pass": true + }, + { + "id": "native-card-text-region-without-venue", + "category": "structured-extraction", + "root_cause": "declared_text_patterns_consume_distinct_value_spans", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "events", + "kind": "named_link", + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "minimum_records": 1, + "required_fields": [ + { + "id": "venue", + "kind": "text", + "minimum_chars": 2 + }, + { + "id": "region", + "kind": "text", + "pattern": ",\\s*(?:TX|AL)\\b" + }, + { + "id": "date", + "kind": "date_or_time" + } + ] + } + ] + } + }, + "artifact": "[Concert](/item/1)\n2026-10-16\n, AL", + "ok": true, + "expected_task_pass": false + }, + { + "id": "native-card-text-no-region", + "category": "structured-extraction", + "root_cause": "declared_text_patterns_consume_distinct_value_spans", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "events", + "kind": "named_link", + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "minimum_records": 1, + "required_fields": [ + { + "id": "venue", + "kind": "text", + "minimum_chars": 2 + }, + { + "id": "region", + "kind": "text", + "pattern": ",\\s*(?:TX|AL)\\b" + }, + { + "id": "date", + "kind": "date_or_time" + } + ] + } + ] + } + }, + "artifact": "[Concert](/item/1)\n2026-10-16\nExample Hall", + "ok": true, + "expected_task_pass": false + }, + { + "id": "source-backed-diagnostic-source-discusses-status", + "category": "acceptance-contract", + "root_cause": "source_body_term_is_not_an_explicit_exclusion", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/spec", + "required_facts": [ + { + "id": "content", + "alternatives": [ + "response specification" + ] + } + ], + "diagnostic_any": [ + "404 Not Found" + ] + }, + "artifact": "The HTTP standard explains 404 Not Found in the response specification.", + "ok": true, + "expected_task_pass": true + }, + { + "id": "source-backed-diagnostic-status-heading", + "category": "acceptance-contract", + "root_cause": "source_body_term_is_not_an_explicit_exclusion", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/spec", + "required_facts": [ + { + "id": "content", + "alternatives": [ + "response specification" + ] + } + ], + "diagnostic_any": [ + "404 Not Found" + ] + }, + "artifact": "404 Not Found", + "ok": true, + "expected_task_pass": false } ], "superseded_cases": { diff --git a/src/lexbench_webfetch/definitions/evaluation/tasks.json b/src/lexbench_webfetch/definitions/evaluation/tasks.json index 03a5624..ee3567b 100644 --- a/src/lexbench_webfetch/definitions/evaluation/tasks.json +++ b/src/lexbench_webfetch/definitions/evaluation/tasks.json @@ -2142,7 +2142,6 @@ "forbidden_any": [ "Access denied", "Attention Required! | Cloudflare", - "CAPTCHA", "Page not found", "404 Not Found", "无法访问此站点" @@ -2191,7 +2190,10 @@ } } ] - } + }, + "diagnostic_any": [ + "CAPTCHA" + ] }, { "id": "live-012-navigation-menu", @@ -3146,7 +3148,6 @@ "forbidden_any": [ "Access denied", "Attention Required! | Cloudflare", - "CAPTCHA", "Page not found", "404 Not Found", "无法访问此站点" @@ -3195,7 +3196,10 @@ } } ] - } + }, + "diagnostic_any": [ + "CAPTCHA" + ] }, { "id": "live-024-cascading-style-sheets-level-2-r", @@ -3735,7 +3739,6 @@ "Attention Required! | Cloudflare", "CAPTCHA", "Page not found", - "404 Not Found", "无法访问此站点" ], "min_content_chars": 300, @@ -3782,7 +3785,10 @@ } } ] - } + }, + "diagnostic_any": [ + "404 Not Found" + ] }, { "id": "stable-xml10-recommendation", @@ -9419,7 +9425,9 @@ "urls": [ "https://upload.wikimedia.org/wikipedia/commons/thumb/4/41/A_Man_on_the_Moon%2C_AS11-40-5903_%28cropped%29.jpg/330px-A_Man_on_the_Moon%2C_AS11-40-5903_%28cropped%29.jpg?utm_source=en.wikipedia.org&utm_campaign=parser&utm_content=thumbnail" ], - "url_patterns": [] + "url_patterns": [ + "^(?:https://upload\\.wikimedia\\.org/wikipedia/commons/4/41/A_Man_on_the_Moon,_AS11\\-40\\-5903_\\(cropped\\)\\.jpg|https://(?:upload|thumb)\\.wikimedia\\.org/wikipedia/commons/thumb/4/41/A_Man_on_the_Moon,_AS11\\-40\\-5903_\\(cropped\\)\\.jpg/[1-9][0-9]*px-A_Man_on_the_Moon,_AS11\\-40\\-5903_\\(cropped\\)\\.jpg)(?:\\?(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+)(?:&(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+))*)?$" + ] }, { "id": "image-2", @@ -9427,7 +9435,9 @@ "urls": [ "https://upload.wikimedia.org/wikipedia/commons/thumb/2/27/Apollo_11_insignia.png/250px-Apollo_11_insignia.png?utm_source=en.wikipedia.org&utm_campaign=parser&utm_content=thumbnail" ], - "url_patterns": [] + "url_patterns": [ + "^(?:https://upload\\.wikimedia\\.org/wikipedia/commons/2/27/Apollo_11_insignia\\.png|https://(?:upload|thumb)\\.wikimedia\\.org/wikipedia/commons/thumb/2/27/Apollo_11_insignia\\.png/[1-9][0-9]*px-Apollo_11_insignia\\.png)(?:\\?(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+)(?:&(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+))*)?$" + ] } ], "resource_output_contract": { @@ -9526,7 +9536,9 @@ "https://images.unsplash.com/photo-1493673272479-a20888bcee10", "https://unsplash.com/photos/mSK5nNsAsLY/download" ], - "url_patterns": [] + "url_patterns": [ + "^https://images\\.unsplash\\.com/photo\\-1493673272479\\-a20888bcee10(?:\\?[^#]*)?$" + ] } ], "resource_output_contract": { @@ -9559,7 +9571,9 @@ "urls": [ "https://images.pexels.com/photos/12377231/pexels-photo-12377231.jpeg" ], - "url_patterns": [] + "url_patterns": [ + "^https://images\\.pexels\\.com/photos/12377231/pexels\\-photo\\-12377231\\.jpeg(?:\\?[^#]*)?$" + ] } ], "resource_output_contract": { diff --git a/src/lexbench_webfetch/definitions/registry.json b/src/lexbench_webfetch/definitions/registry.json index fccf498..629861f 100644 --- a/src/lexbench_webfetch/definitions/registry.json +++ b/src/lexbench_webfetch/definitions/registry.json @@ -6,7 +6,7 @@ "dataset_id": "webfetch", "case_count": 150, "files": { - "tasks.json": "ae3aec6eab8d2d470cdeb18c4f80d238e5dd79e1108a729589b77a80fe869405", + "tasks.json": "0c97913e763a017c2d9151f00688b871473519f479dd028d5e8c8d60fc905d8e", "dataset-catalog.json": "994fa53f7fc7c3170692a7bb493ebc5af9ad8ffb5b576c022fa123e6e6fb3838", "source-inputs.json": "c202abc4624811b938a837402bbaf0ecbe716e375fbca70567346d2edb3b1708" }, @@ -39,7 +39,7 @@ "dataset_id": "risk-control-audited-100", "case_count": 100, "files": { - "tasks.json": "c67b60ccf5ee0b14e4e23b3efc282e625005f8e096f0bfc978772d312e8f50c2", + "tasks.json": "5487a8ab6f9334da6aa2a70ce61151d7773eff31515e4068180081846f191e96", "dataset-catalog.json": "52ec968b2ada26e5e061ccb25cbe4a73747539e556fa88f31bece6d7fb79b3cb", "source-inputs.json": "c61d6825abbff2a92098592f83055347cd0b0101374794023046c0fd0323dd5f" }, diff --git a/src/lexbench_webfetch/dynamic_lists.py b/src/lexbench_webfetch/dynamic_lists.py index 4bb5bb9..7f842b5 100644 --- a/src/lexbench_webfetch/dynamic_lists.py +++ b/src/lexbench_webfetch/dynamic_lists.py @@ -263,7 +263,7 @@ def _note_malformed_markdown_destinations(content, base_url, diagnostics): break -def _text_scope_boundary(content, pattern): +def _text_scope_boundary(content, pattern, *, prefer_heading=False): """Match literal text or an equivalent standalone Markdown heading. Return offsets in the original answer so record syntax and boundaries are @@ -272,6 +272,7 @@ def _text_scope_boundary(content, pattern): """ from .reference_quality import visible_markdown_links candidates=[] + headings=[] raw=re.search(pattern,content,re.M) if raw:candidates.append((raw.start(),raw.end())) offset=0 @@ -285,8 +286,13 @@ def _text_scope_boundary(content, pattern): visible=re.sub(r'([*_]{1,2})(.+?)\1',r'\2',visible) visible=re.sub(r'\s+#+\s*$','',visible).strip() if re.match(pattern,visible,re.M): - candidates.append((offset,offset+len(line))) + candidate = (offset, offset + len(line)) + candidates.append(candidate) + if re.match(r"^[ \t]{0,3}#{1,6}[ \t]+", decoded_line): + headings.append(candidate) offset+=len(line) + if prefer_heading and headings: + return min(headings) return min(candidates) if candidates else None @@ -430,7 +436,7 @@ def _scope_named_content(content, scope, *, contract=None, base_url=''): owned_items.append((destinations.pop(), titles[0].get_text(' ', strip=True))) return '', owned_items, True if scope.get('start_pattern'): - start = _text_scope_boundary(content, scope['start_pattern']) + start = _text_scope_boundary(content, scope['start_pattern'], prefer_heading=True) if not start: return '', [], False content = content[start[1]:] diff --git a/src/lexbench_webfetch/metrics.py b/src/lexbench_webfetch/metrics.py index f6f3c48..6a7040d 100644 --- a/src/lexbench_webfetch/metrics.py +++ b/src/lexbench_webfetch/metrics.py @@ -156,7 +156,14 @@ def _image_content_candidates(content: str) -> list[tuple[str, str]]: for url in srcset_urls(html.unescape(match[1]))) # HTML attributes are decoded above. Do not also admit their raw entity # spelling as a second bare candidate; plain URL answers remain literal. + from .reference_quality import markdown_links bare_content = re.sub(r"]*>", "", content, flags=re.I | re.S) + markup_links = markdown_links(bare_content) + for link in markup_links: + if not link.is_image and _looks_like_image_url(link.destination): + candidates.append((link.destination, "content.markdown_image_file")) + for link in reversed(markup_links): + bare_content = bare_content[:link.start] + " " + bare_content[link.end:] # Explicit image markup is typed evidence. A standalone HTTP URL may also # be the entire answer, including an extensionless CDN address. Every URL # with that syntax is admitted alike, without consulting target resources. @@ -499,6 +506,9 @@ def normalize_for_match(value: str) -> str: # escapes are presentation syntax, not part of the page's wording. normalized = re.sub(r'\\([!"#$%&\'()*+,\-./:;<=>?@\[\\\]^_`{|}~])', r'\1', normalized) normalized = re.sub(r'`([^`\n]+)`', r'\1', normalized) + # Standalone punctuation emitted with emphasis is formatting, including + # when the following word is adjacent. Preserve numeric/operator syntax. + normalized = re.sub(r'(? str: inner = match.group(2) @@ -704,8 +714,9 @@ def evaluate_content( truncation_hits = declared_truncation_hits(answer_text) folded = normalize_for_match(answer_text) content_folded = normalize_for_match(visible_content) + from .stable_tasks import _stable_literal_match matched = [marker for marker in expected - if fact_in_text(content_folded, normalize_for_match(marker))] + if _stable_literal_match(visible_content, marker, phrase_boundaries=True)] facts = required_facts or [] from .dynamic_entry_matching import matching_entry_facts entry_hits = matching_entry_facts(content_folded, facts, normalize_for_match, fact_in_text) @@ -719,9 +730,7 @@ def evaluate_content( identifier = str(fact.get("id") or f"fact-{index + 1}") hits = (entry_hits[identifier] if identifier in entry_hits else [item for item in alternatives - if (_stable_literal_match(visible_content, item, registered_fact_words) - if re.fullmatch(r'[A-Za-z]+|[\u0600-\u06ff]+', item) - else fact_in_text(content_folded, normalize_for_match(item)))]) + if _stable_literal_match(visible_content, item, registered_fact_words, phrase_boundaries=True)]) fact_results.append({ "id": identifier, "alternatives": alternatives, diff --git a/src/lexbench_webfetch/reference_quality.py b/src/lexbench_webfetch/reference_quality.py index d07b179..1021046 100644 --- a/src/lexbench_webfetch/reference_quality.py +++ b/src/lexbench_webfetch/reference_quality.py @@ -5,6 +5,21 @@ from html import unescape +# Cover the alphabets whose word boundaries the literal scorer enforces. +_CASE_CODEPOINTS = (*range(0x0250), *range(0x0370, 0x0530), *range(0x1E00, 0x2000), + *range(0x2DE0, 0x2E00), *range(0xA640, 0xA6A0)) +_LOWER_LETTERS = re.escape(''.join(chr(n) for n in _CASE_CODEPOINTS if chr(n).islower())) +_UPPER_LETTERS = re.escape(''.join(chr(n) for n in _CASE_CODEPOINTS if chr(n).isupper())) +_CASE_BOUNDARY = re.compile( + rf'(?<=[{_LOWER_LETTERS}])(?=[{_UPPER_LETTERS}])|' + rf'(?<=[{_UPPER_LETTERS}])(?=[{_UPPER_LETTERS}][{_LOWER_LETTERS}])') + + +def separate_case_labels(text: str) -> str: + """Restore cased label boundaries without splitting lowercase word fragments.""" + return _CASE_BOUNDARY.sub(' ', text) + + HTML_BLOCK_ELEMENTS = frozenset({ "address", "article", "aside", "blockquote", "caption", "colgroup", "dd", "details", "dialog", "div", "dl", "dt", "fieldset", "figcaption", "figure", @@ -253,7 +268,9 @@ def markdown_links(text: str) -> list[MarkdownLink]: titled = re.match(r"(?s)^(.*?)\s+(['\"])(.*)\2$", raw_destination) destination = re.sub(r"\s+", "", titled[1] if titled else raw_destination) native_title = titled[3] if titled else "" - links.append(MarkdownLink(raw_label, destination, image, index, destination_end + 1, native_title)) + # Character references in a Markdown destination are rendered URL + # syntax. Decode once here; bare URL answers retain their literal bytes. + links.append(MarkdownLink(raw_label, unescape(destination), image, index, destination_end + 1, native_title)) index = destination_end + 1 return links @@ -288,6 +305,9 @@ def visible_markdown_links(text: str) -> str: if match.start < start: continue label = re.sub(r"\s+", " ", visible_markdown_links(match.label)).strip() + # A native carousel can append its accessibility position directly + # to the label. Keep both values but restore their observable boundary. + label = re.sub(r'(?<=[^\W\d_])(?=\d+\s+of\s+\d+$)', ' ', label) # Distinct adjacent link labels retain a token boundary. Some native # serializers also drop a real DOM text boundary immediately before a # path link or a multi-word link label (``out[/r/x]`` / ``or[Stack diff --git a/src/lexbench_webfetch/stable_tasks.py b/src/lexbench_webfetch/stable_tasks.py index 77c1bc2..c6867f4 100644 --- a/src/lexbench_webfetch/stable_tasks.py +++ b/src/lexbench_webfetch/stable_tasks.py @@ -696,7 +696,7 @@ def is_record(url): return None -def _stable_literal_match(surface, value, registered_words=()): +def _stable_literal_match(surface, value, registered_words=(), *, phrase_boundaries=False): """An atomic Latin label is a word, not part of another word. Native camel boundaries and fully covered chains of registered labels @@ -704,10 +704,20 @@ def _stable_literal_match(surface, value, registered_words=()): Longer declared text retains the existing exact-phrase containment rule. """ from .metrics import WORD_CHARACTERS, fact_in_text, normalize_for_match + from .reference_quality import visible_markdown_links + surface = visible_markdown_links(surface) + surface = re.sub(r'(?<=[^\W\d_])(?=\d+\s+of\s+\d+\b)', ' ', surface) expected = normalize_for_match(_visible(value)) folded = normalize_for_match(surface) - if not fact_in_text(folded, expected, latin_boundaries=bool( - re.search(r"[^\x00-\x7f]", value))): + from .reference_quality import separate_case_labels as camel + alphabetic_boundaries = (not re.fullmatch(r'[A-Za-z]+', value) + and (phrase_boundaries or bool(re.search(r'[^\x00-\x7f]', value)))) + surfaces = (folded, normalize_for_match(camel(surface))) if alphabetic_boundaries else (folded,) + # Native serializers can join an all-caps preceding label to a complete + # notice. Apply the same case boundary used for atomic labels before the + # Unicode phrase gate, while retaining complete-word matching on both ends. + if not any(fact_in_text(text, expected, latin_boundaries=alphabetic_boundaries) + for text in surfaces): return False # Arabic labels are whitespace-delimited words too. A longer inflected # word cannot supply an independently registered ordinary navigation label. @@ -720,8 +730,6 @@ def _stable_literal_match(surface, value, registered_words=()): rf"(?']) +def test_entity_escaped_link_destination_is_one_resource(template): + result = score(template.format(url=TARGET.replace('&', '&'))) + assert task_passed(True, result) + assert result['resource_candidate_urls'] == [TARGET] + assert result['resource_precision'] == 1.0 + + +@pytest.mark.parametrize('output', [ + '![Photo](https://cdn.example.test/photo.jpg?width=641&height=480)', + 'https://cdn.example.test/photo.jpg?width=640&height=480', + '![Photo](https://cdn.example.test/photo.jpg?width=640%26amp%3Bheight=480)', +]) +def test_markup_decoding_does_not_equate_different_or_literal_url_values(output): + assert not task_passed(True, score(output)) diff --git a/tests/test_stable_text_serialization.py b/tests/test_stable_text_serialization.py new file mode 100644 index 0000000..c8529d9 --- /dev/null +++ b/tests/test_stable_text_serialization.py @@ -0,0 +1,56 @@ +"""Stable notices survive native text concatenation without accepting word fragments.""" +import pytest + +from lexbench_webfetch.metrics import task_passed +from lexbench_webfetch.scoring import score_attempt + + +@pytest.mark.parametrize('prefix', ['', 'ALL RIGHTS RESERVED', 'TODOS LOS DERECHOS RESERVADOS']) +def test_complete_notice_after_an_adjacent_capitalized_label(prefix): + notice = 'Example Corp. Avenida N°5, Santiago de Chile' + case = {'url': 'https://example.test/', 'acceptance_contract': { + 'version': 1, 'schema': 'body_text', 'minimum_records': 0, + 'stable_text_anchors': [{'id': 'notice', 'alternatives': [notice]}]}} + result = score_attempt(case, (prefix + notice).encode(), ok=True) + assert task_passed(True, result) + + +@pytest.mark.parametrize('notice', [ + 'Examples Corp. Avenida N°5, Santiago de Chile', + 'Example Corp. Avenida N°5, Santiago de Chilean', + 'Example Corp. Avenida N°6, Santiago de Chile', +]) +def test_adjacent_notice_still_requires_the_complete_declared_words(notice): + case = {'url': 'https://example.test/', 'acceptance_contract': { + 'version': 1, 'schema': 'body_text', 'minimum_records': 0, + 'stable_text_anchors': [{'id': 'notice', 'alternatives': [ + 'Example Corp. Avenida N°5, Santiago de Chile']} ]}} + result = score_attempt(case, ('ALL RIGHTS RESERVED' + notice).encode(), ok=True) + assert not task_passed(True, result) + + +@pytest.mark.parametrize('gate', ['fact', 'anchor']) +@pytest.mark.parametrize('expected,output', [ + ('In stock', 'In stockAdd to basket'), + ('Наука', '[Технологии](/technology)[Наука](/science)Все темы'), + ('Careers', '[Careers9 of 9](/careers)'), + ('toute l’actu dès 7h30', '_:_toute l’actu dès 7h30'), +]) +def test_native_label_boundaries_are_preserved_for_facts_and_anchors(gate, expected, output): + case = {'url': 'https://example.test/'} + if gate == 'fact': + case['required_facts'] = [{'id': 'label', 'alternatives': [expected]}] + else: + case['acceptance_contract'] = {'version': 1, 'schema': 'body_text', 'minimum_records': 0, + 'stable_text_anchors': [{'id': 'label', 'alternatives': [expected]}]} + assert task_passed(True, score_attempt(case, output.encode(), ok=True)) + + +@pytest.mark.parametrize('expected,output', [ + ('In stock', 'In stockholders'), ('Наука', '[Наукам](/science)'), + ('Careers', '[Careers9](/careers)'), ('stats', 'statsmodels'), +]) +def test_native_label_recovery_does_not_accept_a_different_word(expected, output): + case = {'url': 'https://example.test/', 'required_facts': [ + {'id': 'label', 'alternatives': [expected]}]} + assert not task_passed(True, score_attempt(case, output.encode(), ok=True)) From 5e3b64f1ab71e2213fdae7b2efacbed42e6b022b Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Wed, 21 Oct 2026 20:04:50 +0800 Subject: [PATCH 11/20] fix(scoring): retain fields in complete linked cards --- docs/protocol.md | 4 + src/lexbench_webfetch/composite_tasks.py | 31 +- .../risk-control/current/tasks.json | 4 +- .../risk-control/list-contracts.json | 12 +- .../evaluation/scoring-regressions.json | 313 ++++++++++++++++++ .../definitions/registry.json | 2 +- tests/test_card_preludes.py | 19 ++ tests/test_declared_card_labels.py | 15 + 8 files changed, 392 insertions(+), 8 deletions(-) create mode 100644 tests/test_card_preludes.py create mode 100644 tests/test_declared_card_labels.py diff --git a/docs/protocol.md b/docs/protocol.md index 3e77a8e..590a1d7 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -38,4 +38,8 @@ Historical `fetch_log_final_url` and `video_values` formats remain replay reader A flattened link label establishes multiple record fields only when its visible text retains field separators, such as line breaks or spaced vertical bars. URL slugs, English ticket-action labels and US city/state spelling cannot establish missing field boundaries. Ambiguous labels remain unqualified even when a reader can guess their intended layout. Named table columns with duplicate matching headers remain ambiguous; a navigation line without a table structure is not a column header. +A named-link group may declare `markdown_label_pattern` with a named `label` capture when a card places metadata before its visible name. The capture selects text already present in that link; it cannot supply a name from the URL. `record_start_pattern` bounds cards whose fields precede their title link. Fields before a later card start or another target link cannot complete the current record. + +Text field patterns consume their matched spans, so distinct declared fields may share a physical line. Matching one field cannot reuse its value for a second requirement. Unlabeled text still needs its own evidence; relative-date controls such as “Today” do not establish a venue or source. + Contract-declared field aliases support Unicode labels. The built-in English metadata-role recognition is a limited legacy fallback, not an automatic translation dictionary. Declare the language-specific labels needed by a task rather than treating every colon in prose as a metadata separator. Body matching treats unfenced table tags as HTML, including in mixed prose. Literal code needs an explicit code fence or escape; a standalone table tag cannot disable HTML visibility rules for hidden elements or attributes. diff --git a/src/lexbench_webfetch/composite_tasks.py b/src/lexbench_webfetch/composite_tasks.py index 4a02c27..acbe7ff 100644 --- a/src/lexbench_webfetch/composite_tasks.py +++ b/src/lexbench_webfetch/composite_tasks.py @@ -154,6 +154,8 @@ def _validate_group(group: dict) -> None: "link_base_url", "record_name_exclusions", "record_line_pattern", + "record_start_pattern", + "markdown_label_pattern", "start_pattern", "end_pattern", "required_fields", @@ -180,11 +182,13 @@ def _validate_group(group: dict) -> None: } ): raise ValueError("Invalid composite record group") - for key in ("start_pattern", "end_pattern", "record_line_pattern"): + for key in ("start_pattern", "end_pattern", "record_line_pattern", "record_start_pattern", "markdown_label_pattern"): if key in group: if not isinstance(group[key], str) or not group[key]: raise ValueError("Invalid composite scope pattern") - re.compile(group[key]) + compiled = re.compile(group[key]) + if key == "markdown_label_pattern" and "label" not in compiled.groupindex: + raise ValueError("Markdown card label pattern requires a label group") kind = group["kind"] if "combined_columns" in group and kind != "table": raise ValueError("Combined columns require a table group") @@ -694,6 +698,12 @@ def previous_scalar(link): return previous_start, scalar def visible_label(link): + if pattern := group.get("markdown_label_pattern"): + surface = link.label + for image in reversed([item for item in markdown_links(surface) if item.is_image]): + surface = surface[:image.start] + surface[image.end:] + if match := re.search(pattern, visible_markdown_links(surface)): + return match["label"].strip() card_fields = serialized_link_card_fields(link) label = card_fields[0] if card_fields else link.label for image in reversed([item for item in markdown_links(label) if item.is_image]): @@ -806,6 +816,16 @@ def owner_for(position): if index and ordered[index - 1][6] > line_start: line_start = start next_line = max(next_line, link_end) + if not owner and group.get("record_start_pattern"): + starts = list(re.finditer(group["record_start_pattern"], content[:link_start], re.M)) + if starts: + marker = starts[-1].start() + prior_end = ordered[index - 1][6] if index else 0 + if marker >= prior_end: + line_start = marker + following = re.search(group["record_start_pattern"], content[link_end:], re.M) + if following: + next_line = min(next_line, link_end + following.start()) if owner: begin, end = owner # The first link can have fields before it in the same card. @@ -959,7 +979,9 @@ def _context_lines(context: str, title: str) -> list[str]: for image in reversed(markdown_links(label)): if image.is_image: label = label[:image.start] + " " + label[image.end:] - if _key(visible_markdown_links(label)) != title_key: + visible_label = visible_markdown_links(label) + if (_key(visible_label) != title_key + and title_key not in {_key(line) for line in visible_label.splitlines()}): continue header = re.match(r"[ \t]*(?P\*\*|__)(?P[^\n]+?)(?P=mark)(?:\\)*[ \t]*\n", label) if not header: @@ -1039,6 +1061,9 @@ def _field_token_pattern(spec: dict): def _text_role_value(value: str, spec: dict) -> str | None: """Explicit metadata belongs to its declared role, not another text field.""" value = value.strip() + if (re.fullmatch(r"today|tomorrow|yesterday", value, re.I) + and spec["id"] not in {"date", "date_or_time", "time", "status"}): + return None roles = {"title", "name", "author", "creator", "owner", "source", "publisher", "description", "summary", "date", "time", "version", "downloads", "views", "likes", "count", "price", "rating", "size"} diff --git a/src/lexbench_webfetch/definitions/evaluation/risk-control/current/tasks.json b/src/lexbench_webfetch/definitions/evaluation/risk-control/current/tasks.json index 984a3f2..27b778c 100644 --- a/src/lexbench_webfetch/definitions/evaluation/risk-control/current/tasks.json +++ b/src/lexbench_webfetch/definitions/evaluation/risk-control/current/tasks.json @@ -5060,7 +5060,9 @@ "minimum_chars": 2, "pattern": ",\\s*(?:AL|AK|AZ|AR|CA|CO|CT|DE|DC|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY)\\b" } - ] + ], + "markdown_label_pattern": "(?m)^(?:(?:Today|Tomorrow)[ \\t]*\\n)?(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday),[^\\n]*\\b20[0-9]{2}\\b[^\\n]*\\n(?P<label>[^\\n]+)\\n", + "record_start_pattern": "(?m)^[-*+][ \\t]+(?:January|February|March|April|May|June|July|August|September|October|November|December|Multiple Dates)\\b" } ] }, diff --git a/src/lexbench_webfetch/definitions/evaluation/risk-control/list-contracts.json b/src/lexbench_webfetch/definitions/evaluation/risk-control/list-contracts.json index 27950f1..42dcec7 100644 --- a/src/lexbench_webfetch/definitions/evaluation/risk-control/list-contracts.json +++ b/src/lexbench_webfetch/definitions/evaluation/risk-control/list-contracts.json @@ -311,9 +311,12 @@ { "id": "city_or_region", "kind": "text", - "minimum_chars": 2 + "minimum_chars": 2, + "pattern": ",\\s*(?:AL|AK|AZ|AR|CA|CO|CT|DE|DC|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY)\\b" } - ] + ], + "markdown_label_pattern": "(?m)^(?:(?:Today|Tomorrow)[ \\t]*\\n)?(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday),[^\\n]*\\b20[0-9]{2}\\b[^\\n]*\\n(?P<label>[^\\n]+)\\n", + "record_start_pattern": "(?m)^[-*+][ \\t]+(?:January|February|March|April|May|June|July|August|September|October|November|December|Multiple Dates)\\b" } ] }, @@ -468,7 +471,10 @@ "end_pattern": "(?m)^\\s*(?:#{1,6}\\s*)?(?:第六章|Chapter\\s+6\\b|Section\\s+6\\.)", "combined_columns": [ { - "aliases": ["Problem", "题目"], + "aliases": [ + "Problem", + "题目" + ], "pattern": "^(?P<number>[A-Z]?[0-9]+)\\s+(?P<name>.+)$" } ], diff --git a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json index 3a85d6d..59d2389 100644 --- a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json +++ b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json @@ -56147,6 +56147,319 @@ "artifact": "404 Not Found", "ok": true, "expected_task_pass": false + }, + { + "id": "declared-whole-card-complete", + "category": "structured-extraction", + "root_cause": "declared_visible_title_and_fields_inside_one_card_link", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "events", + "kind": "named_link", + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "minimum_records": 1, + "markdown_label_pattern": "(?m)^(?:Today\\n)?2026-10-16\\n(?P<label>[^\\n]+)\\n", + "required_fields": [ + { + "id": "date", + "kind": "date_or_time" + }, + { + "id": "venue", + "kind": "text", + "minimum_chars": 2 + }, + { + "id": "region", + "kind": "text", + "pattern": ",\\s*AL\\b" + } + ] + } + ] + } + }, + "artifact": "[![](/cover.jpg)\nToday\n2026-10-16\nEvening Concert\nExample Hall Tuscaloosa, AL](/item/1)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "declared-whole-card-no-name", + "category": "structured-extraction", + "root_cause": "declared_visible_title_and_fields_inside_one_card_link", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "events", + "kind": "named_link", + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "minimum_records": 1, + "markdown_label_pattern": "(?m)^(?:Today\\n)?2026-10-16\\n(?P<label>[^\\n]+)\\n", + "required_fields": [ + { + "id": "date", + "kind": "date_or_time" + }, + { + "id": "venue", + "kind": "text", + "minimum_chars": 2 + }, + { + "id": "region", + "kind": "text", + "pattern": ",\\s*AL\\b" + } + ] + } + ] + } + }, + "artifact": "[![](/cover.jpg)\nToday\n2026-10-16\nExample Hall Tuscaloosa, AL](/item/1)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "declared-whole-card-no-venue", + "category": "structured-extraction", + "root_cause": "declared_visible_title_and_fields_inside_one_card_link", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "events", + "kind": "named_link", + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "minimum_records": 1, + "markdown_label_pattern": "(?m)^(?:Today\\n)?2026-10-16\\n(?P<label>[^\\n]+)\\n", + "required_fields": [ + { + "id": "date", + "kind": "date_or_time" + }, + { + "id": "venue", + "kind": "text", + "minimum_chars": 2 + }, + { + "id": "region", + "kind": "text", + "pattern": ",\\s*AL\\b" + } + ] + } + ] + } + }, + "artifact": "[![](/cover.jpg)\nToday\n2026-10-16\nEvening Concert\n, AL](/item/1)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "declared-whole-card-no-region", + "category": "structured-extraction", + "root_cause": "declared_visible_title_and_fields_inside_one_card_link", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "events", + "kind": "named_link", + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "minimum_records": 1, + "markdown_label_pattern": "(?m)^(?:Today\\n)?2026-10-16\\n(?P<label>[^\\n]+)\\n", + "required_fields": [ + { + "id": "date", + "kind": "date_or_time" + }, + { + "id": "venue", + "kind": "text", + "minimum_chars": 2 + }, + { + "id": "region", + "kind": "text", + "pattern": ",\\s*AL\\b" + } + ] + } + ] + } + }, + "artifact": "[![](/cover.jpg)\nToday\n2026-10-16\nEvening Concert\nExample Hall](/item/1)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "declared-card-prelude-owned", + "category": "structured-extraction", + "root_cause": "declared_card_start_owns_pre_title_fields", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "events", + "kind": "named_link", + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "minimum_records": 1, + "record_start_pattern": "(?m)^- 2026-10-[0-9]{2}$", + "required_fields": [ + { + "id": "date", + "kind": "date_or_time" + }, + { + "id": "venue", + "kind": "text", + "aliases": [ + "Venue" + ] + }, + { + "id": "region", + "kind": "text", + "aliases": [ + "Region" + ] + } + ] + } + ] + } + }, + "artifact": "- 2026-10-16\nVenue: Hall A\nRegion: Alabama\n[Concert](/item/1)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "declared-card-prelude-separate-card", + "category": "structured-extraction", + "root_cause": "declared_card_start_owns_pre_title_fields", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "events", + "kind": "named_link", + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "minimum_records": 1, + "record_start_pattern": "(?m)^- 2026-10-[0-9]{2}$", + "required_fields": [ + { + "id": "date", + "kind": "date_or_time" + }, + { + "id": "venue", + "kind": "text", + "aliases": [ + "Venue" + ] + }, + { + "id": "region", + "kind": "text", + "aliases": [ + "Region" + ] + } + ] + } + ] + } + }, + "artifact": "- 2026-10-16\nVenue: Hall A\nRegion: Alabama\n- 2026-10-17\n[Concert](/item/1)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "declared-card-prelude-next-card-fields", + "category": "structured-extraction", + "root_cause": "declared_card_start_owns_pre_title_fields", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "events", + "kind": "named_link", + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "minimum_records": 1, + "record_start_pattern": "(?m)^- 2026-10-[0-9]{2}$", + "required_fields": [ + { + "id": "date", + "kind": "date_or_time" + }, + { + "id": "venue", + "kind": "text", + "aliases": [ + "Venue" + ] + }, + { + "id": "region", + "kind": "text", + "aliases": [ + "Region" + ] + } + ] + } + ] + } + }, + "artifact": "- 2026-10-16\nVenue: Hall A\n[Concert](/item/1)\n- 2026-10-17\nRegion: Alabama\n[Other](/item/2)", + "ok": true, + "expected_task_pass": false } ], "superseded_cases": { diff --git a/src/lexbench_webfetch/definitions/registry.json b/src/lexbench_webfetch/definitions/registry.json index 629861f..391d36f 100644 --- a/src/lexbench_webfetch/definitions/registry.json +++ b/src/lexbench_webfetch/definitions/registry.json @@ -39,7 +39,7 @@ "dataset_id": "risk-control-audited-100", "case_count": 100, "files": { - "tasks.json": "5487a8ab6f9334da6aa2a70ce61151d7773eff31515e4068180081846f191e96", + "tasks.json": "23c2c7e0b470527dfd848146dd94bb47188d2dbdeb7b9105d04664b9b02b9085", "dataset-catalog.json": "52ec968b2ada26e5e061ccb25cbe4a73747539e556fa88f31bece6d7fb79b3cb", "source-inputs.json": "c61d6825abbff2a92098592f83055347cd0b0101374794023046c0fd0323dd5f" }, diff --git a/tests/test_card_preludes.py b/tests/test_card_preludes.py new file mode 100644 index 0000000..91c8901 --- /dev/null +++ b/tests/test_card_preludes.py @@ -0,0 +1,19 @@ +"""Declared card starts own fields before a linked title without borrowing peers.""" +import pytest +from lexbench_webfetch.scoring import score_attempt +from lexbench_webfetch.metrics import task_passed + +CASE={'url':'https://example.test/','composite_acceptance':{'version':1,'groups':[{ + 'id':'events','kind':'named_link','hosts':['example.test'],'entry_path':r'/item/[0-9]+', + 'minimum_records':1,'record_start_pattern':r'(?m)^- 2026-10-[0-9]{2}$', + 'required_fields':[{'id':'date','kind':'date_or_time'}, + {'id':'venue','kind':'text','aliases':['Venue']}, + {'id':'region','kind':'text','aliases':['Region']}]}]}} + +@pytest.mark.parametrize('body,expected',[ + ('- 2026-10-16\nVenue: Hall A\nRegion: Alabama\n[Concert](/item/1)',True), + ('- 2026-10-16\nVenue: Hall A\nRegion: Alabama\n- 2026-10-17\n[Concert](/item/1)',False), + ('- 2026-10-16\nVenue: Hall A\n[Concert](/item/1)\n- 2026-10-17\nRegion: Alabama\n[Other](/item/2)',False), +]) +def test_fields_before_link_belong_only_to_the_declared_card(body,expected): + assert task_passed(True,score_attempt(CASE,body.encode(),ok=True)) is expected diff --git a/tests/test_declared_card_labels.py b/tests/test_declared_card_labels.py new file mode 100644 index 0000000..a8dcf86 --- /dev/null +++ b/tests/test_declared_card_labels.py @@ -0,0 +1,15 @@ +import pytest +from lexbench_webfetch.scoring import score_attempt +from lexbench_webfetch.metrics import task_passed + +CASE={'url':'https://example.test/','composite_acceptance':{'version':1,'groups':[{'id':'events','kind':'named_link','hosts':['example.test'],'entry_path':r'/item/[0-9]+','minimum_records':1,'markdown_label_pattern':r'(?m)^(?:Today\n)?2026-10-16\n(?P<label>[^\n]+)\n','required_fields':[{'id':'date','kind':'date_or_time'},{'id':'venue','kind':'text','minimum_chars':2},{'id':'region','kind':'text','pattern':r',\s*AL\b'}]}]}} +@pytest.mark.parametrize('label,passed',[ + ('Today\n2026-10-16\nEvening Concert\nExample Hall Tuscaloosa, AL',True), + ('2026-10-16\nEvening Concert\nExample Hall Tuscaloosa, AL',True), + ('Today\n2026-10-16\nExample Hall Tuscaloosa, AL',False), + ('Today\n2026-10-16\nEvening Concert\n, AL',False), + ('Today\n2026-10-16\nEvening Concert\nExample Hall',False), +]) +def test_declared_multiline_card_label_and_owned_fields(label,passed): + result=score_attempt(CASE,('[![](/cover.jpg)\n'+label+'](/item/1)').encode(),ok=True) + assert task_passed(True,result) is passed From d62129b03cbd664e72e794b2c8bc1f5b7dc10e07 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Wed, 21 Oct 2026 20:16:48 +0800 Subject: [PATCH 12/20] fix(scoring): allow declared title information fields --- docs/protocol.md | 2 + .../evaluation/scoring-regressions.json | 81 +++++++++++++++++++ .../definitions/evaluation/tasks.json | 3 +- .../definitions/registry.json | 2 +- src/lexbench_webfetch/metrics.py | 11 ++- src/lexbench_webfetch/reference_quality.py | 30 +++++++ tests/test_information_title_scope.py | 36 +++++++++ 7 files changed, 161 insertions(+), 4 deletions(-) create mode 100644 tests/test_information_title_scope.py diff --git a/docs/protocol.md b/docs/protocol.md index 590a1d7..a6e6e1d 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -42,4 +42,6 @@ A named-link group may declare `markdown_label_pattern` with a named `label` cap Text field patterns consume their matched spans, so distinct declared fields may share a physical line. Matching one field cannot reuse its value for a second requirement. Unlabeled text still needs its own evidence; relative-date controls such as “Today” do not establish a venue or source. +Required facts use body text by default. An information task may declare `match_scope: "body_or_title"` for a fact that explicitly requests a page title. This permits the delivered title field, an HTML document title, or a scalar top-level Markdown front-matter title. Keywords and descriptions cannot supply the title, and title metadata cannot supply other facts. + Contract-declared field aliases support Unicode labels. The built-in English metadata-role recognition is a limited legacy fallback, not an automatic translation dictionary. Declare the language-specific labels needed by a task rather than treating every colon in prose as a metadata separator. Body matching treats unfenced table tags as HTML, including in mixed prose. Literal code needs an explicit code fence or escape; a standalone table tag cannot disable HTML visibility rules for hidden elements or attributes. diff --git a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json index 59d2389..6da45fd 100644 --- a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json +++ b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json @@ -56460,6 +56460,87 @@ "artifact": "- 2026-10-16\nVenue: Hall A\n[Concert](/item/1)\n- 2026-10-17\nRegion: Alabama\n[Other](/item/2)", "ok": true, "expected_task_pass": false + }, + { + "id": "declared-information-explicit-title", + "category": "acceptance-contract", + "root_cause": "information_title_scope_does_not_promote_unrelated_metadata", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/video", + "required_facts": [ + { + "id": "video-title", + "alternatives": [ + "Example video" + ], + "match_scope": "body_or_title" + }, + { + "id": "publisher", + "alternatives": [ + "Example Channel" + ] + } + ] + }, + "artifact": "---\ntitle: \"Example video\"\n---\nExample Channel", + "ok": true, + "expected_task_pass": true + }, + { + "id": "declared-information-keywords-not-title", + "category": "acceptance-contract", + "root_cause": "information_title_scope_does_not_promote_unrelated_metadata", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/video", + "required_facts": [ + { + "id": "video-title", + "alternatives": [ + "Example video" + ], + "match_scope": "body_or_title" + }, + { + "id": "publisher", + "alternatives": [ + "Example Channel" + ] + } + ] + }, + "artifact": "---\ntitle: \"Other video\"\nkeywords: \"Example video\"\n---\nExample Channel", + "ok": true, + "expected_task_pass": false + }, + { + "id": "declared-information-keywords-not-publisher", + "category": "acceptance-contract", + "root_cause": "information_title_scope_does_not_promote_unrelated_metadata", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/video", + "required_facts": [ + { + "id": "video-title", + "alternatives": [ + "Example video" + ], + "match_scope": "body_or_title" + }, + { + "id": "publisher", + "alternatives": [ + "Example Channel" + ] + } + ] + }, + "artifact": "---\ntitle: \"Example video\"\nkeywords: \"Example Channel\"\n---\nOther body", + "ok": true, + "expected_task_pass": false } ], "superseded_cases": { diff --git a/src/lexbench_webfetch/definitions/evaluation/tasks.json b/src/lexbench_webfetch/definitions/evaluation/tasks.json index ee3567b..2eb48ec 100644 --- a/src/lexbench_webfetch/definitions/evaluation/tasks.json +++ b/src/lexbench_webfetch/definitions/evaluation/tasks.json @@ -9869,7 +9869,8 @@ "id": "视频标题", "alternatives": [ "Me at the zoo" - ] + ], + "match_scope": "body_or_title" }, { "id": "发布者", diff --git a/src/lexbench_webfetch/definitions/registry.json b/src/lexbench_webfetch/definitions/registry.json index 391d36f..5f92985 100644 --- a/src/lexbench_webfetch/definitions/registry.json +++ b/src/lexbench_webfetch/definitions/registry.json @@ -6,7 +6,7 @@ "dataset_id": "webfetch", "case_count": 150, "files": { - "tasks.json": "0c97913e763a017c2d9151f00688b871473519f479dd028d5e8c8d60fc905d8e", + "tasks.json": "02f887ae8645102eb780a456fd2ee4c0c00b8855c9ea9b1043218225dcc5a834", "dataset-catalog.json": "994fa53f7fc7c3170692a7bb493ebc5af9ad8ffb5b576c022fa123e6e6fb3838", "source-inputs.json": "c202abc4624811b938a837402bbaf0ecbe716e375fbca70567346d2edb3b1708" }, diff --git a/src/lexbench_webfetch/metrics.py b/src/lexbench_webfetch/metrics.py index 6a7040d..258b8f7 100644 --- a/src/lexbench_webfetch/metrics.py +++ b/src/lexbench_webfetch/metrics.py @@ -728,15 +728,22 @@ def evaluate_content( for index, fact in enumerate(facts): alternatives = [str(item) for item in fact.get("alternatives", [])] identifier = str(fact.get("id") or f"fact-{index + 1}") + scope = fact.get("match_scope", "body") + if scope not in {"body", "body_or_title"}: + raise ValueError("Unknown required-fact matching scope") + targets = [visible_content] + if scope == "body_or_title": + from .reference_quality import delivered_page_title + targets.append(delivered_page_title(content, title)) hits = (entry_hits[identifier] if identifier in entry_hits else [item for item in alternatives - if _stable_literal_match(visible_content, item, registered_fact_words, phrase_boundaries=True)]) + if any(_stable_literal_match(target, item, registered_fact_words, phrase_boundaries=True) for target in targets)]) fact_results.append({ "id": identifier, "alternatives": alternatives, "matched": hits, "pass": bool(hits), - **({'match_scope': 'entry'} if identifier in entry_hits else {}), + **({'match_scope': 'entry'} if identifier in entry_hits else {'match_scope': scope} if scope != 'body' else {}), }) fact_hits = sum(item["pass"] for item in fact_results) fact_coverage = round(fact_hits / len(fact_results), 4) if fact_results else 1.0 diff --git a/src/lexbench_webfetch/reference_quality.py b/src/lexbench_webfetch/reference_quality.py index 1021046..540764c 100644 --- a/src/lexbench_webfetch/reference_quality.py +++ b/src/lexbench_webfetch/reference_quality.py @@ -60,6 +60,36 @@ def markdown_body_content(text: str) -> str: +def delivered_page_title(content: str, explicit_title: str | None = None) -> str: + """Read a declared title field, never keywords or a description snippet.""" + if explicit_title and explicit_title.strip(): + return explicit_title.strip() + if is_html_document_or_fragment(content): + from bs4 import BeautifulSoup + title = BeautifulSoup(content, "html.parser").find("title") + return title.get_text(" ", strip=True) if title else "" + body = markdown_body_content(content) + if body == content: + return "" + header = content[:len(content)-len(body)] + values = re.findall(r"(?m)^title:[ \t]*([^\r\n]*)$", header) + if len(values) != 1: + return "" + value = values[0].strip() + if value.startswith('"'): + import json + try: + parsed = json.loads(value) + except ValueError: + return "" + return parsed if isinstance(parsed, str) else "" + if value.startswith("'"): + return value[1:-1].replace("''", "'") if value.endswith("'") else "" + if not value or value[0] in "[{|>" or value.casefold() in {"null", "true", "false", "~"}: + return "" + return value.split(" #", 1)[0].strip() + + def markdown_heading_text(text: str) -> str: """Remove ATX presentation markers while retaining visible heading words.""" output = [] diff --git a/tests/test_information_title_scope.py b/tests/test_information_title_scope.py new file mode 100644 index 0000000..0143507 --- /dev/null +++ b/tests/test_information_title_scope.py @@ -0,0 +1,36 @@ +"""Information requests may use a delivered page title, never unrelated metadata.""" +import pytest +from lexbench_webfetch.scoring import score_attempt +from lexbench_webfetch.metrics import task_passed + + +def evaluate(output,scope='body_or_title',title=None): + return score_attempt({'url':'https://example.test/video','required_facts':[ + {'id':'video-title','alternatives':['Example video'],'match_scope':scope}, + {'id':'publisher','alternatives':['Example Channel']} ]},output.encode(),ok=True,title=title) + +@pytest.mark.parametrize('output,title',[ + ('Example video\nExample Channel',None), + ('---\ntitle: "Example video - Player"\n---\nExample Channel',None), + ("---\ntitle: 'Example video'\n---\nExample Channel",None), + ('<html><head><title>Example videoExample Channel',None), + ('Example Channel','Example video'), +]) +def test_declared_title_information_can_use_a_real_page_title(output,title): + assert task_passed(True,evaluate(output,title=title)) + +@pytest.mark.parametrize('output',[ + '---\ntitle: "Other video"\nkeywords: "Example video"\n---\nExample Channel', + '---\ntitle: "Example video"\nkeywords: "Example Channel"\n---\nOther body', + '---\ntitle: "Example video"\ntitle: "Other video"\n---\nExample Channel', +]) +def test_keywords_or_ambiguous_metadata_cannot_supply_information(output): + assert not task_passed(True,evaluate(output)) + + +def test_body_requirement_still_rejects_title_only_evidence(): + assert not task_passed(True,evaluate('---\ntitle: "Example video"\n---\nExample Channel',scope='body')) + + +def test_fact_cannot_be_joined_across_body_and_title(): + assert not task_passed(True,evaluate('Example Channel\nExample',title='video')) From 4d73ada2fa1305fab61f0c9d222f7a315361281c Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Wed, 21 Oct 2026 20:22:04 +0800 Subject: [PATCH 13/20] fix(tasks): bound software results to their own section --- .../tasks.json | 4 +- .../evaluation/scoring-regressions.json | 52 +++++++++++++++++++ .../definitions/registry.json | 2 +- 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/src/lexbench_webfetch/definitions/evaluation/dynamic-content-100-moli-webfetch/tasks.json b/src/lexbench_webfetch/definitions/evaluation/dynamic-content-100-moli-webfetch/tasks.json index 31ab96f..3c119af 100644 --- a/src/lexbench_webfetch/definitions/evaluation/dynamic-content-100-moli-webfetch/tasks.json +++ b/src/lexbench_webfetch/definitions/evaluation/dynamic-content-100-moli-webfetch/tasks.json @@ -30581,7 +30581,7 @@ "entry_scope": { "html_selector": ".nc3-as-result--soft:not(.is-loading) a[href]", "start_pattern": "^ゲームソフトをさがす\\s*$", - "end_pattern": "^すべて見る" + "end_pattern": "^(?:すべて見る|ページをさがす)" } }, "body_requirements": {}, @@ -30643,7 +30643,7 @@ "entry_scope": { "html_selector": ".nc3-as-result--soft:not(.is-loading) a[href]", "start_pattern": "^ゲームソフトをさがす\\s*$", - "end_pattern": "^すべて見る" + "end_pattern": "^(?:すべて見る|ページをさがす)" } }, "body_requirements": {}, diff --git a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json index 6da45fd..052c881 100644 --- a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json +++ b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json @@ -56541,6 +56541,58 @@ "artifact": "---\ntitle: \"Example video\"\nkeywords: \"Example Channel\"\n---\nOther body", "ok": true, "expected_task_pass": false + }, + { + "id": "named-section-boundary-complete-software", + "category": "acceptance-contract", + "root_cause": "declared_result_section_ends_at_the_next_independent_section", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/search", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "entry_scope": { + "html_selector": ".results a[href]", + "start_pattern": "^Find games$", + "end_pattern": "^Find pages$" + } + } + }, + "artifact": "### Find games\n[Alpha](/item/1)\n[Beta](/item/2)\n### Find pages\n[Other page](/item/3)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "named-section-boundary-next-section-not-software", + "category": "acceptance-contract", + "root_cause": "declared_result_section_ends_at_the_next_independent_section", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/search", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "entry_scope": { + "html_selector": ".results a[href]", + "start_pattern": "^Find games$", + "end_pattern": "^Find pages$" + } + } + }, + "artifact": "### Find games\n[Alpha](/item/1)\n### Find pages\n[Other page](/item/2)", + "ok": true, + "expected_task_pass": false } ], "superseded_cases": { diff --git a/src/lexbench_webfetch/definitions/registry.json b/src/lexbench_webfetch/definitions/registry.json index 5f92985..1fcc1d8 100644 --- a/src/lexbench_webfetch/definitions/registry.json +++ b/src/lexbench_webfetch/definitions/registry.json @@ -28,7 +28,7 @@ "dataset_id": "dynamic-qualified-100-moli-native-local-audit-20260924", "case_count": 100, "files": { - "tasks.json": "139c62a141183d305d8fa9c3658c546927731461d86aeba5ad3091649bd4ecd7", + "tasks.json": "f76077e57746e942932f5eee1914e4286bcdd97c4ba23f4038dbe8bf7f9b82a0", "dataset-catalog.json": "ca1572aa80fc2324b2f00800a79b053cfd584d1831ffb16e6906ae0981522c6c", "source-inputs.json": "710f9c91705b29bb9e4aa15af97dcc856e64e406baa4b9966446408d7e3d26a8" }, From 27fb1908e297e431c913124842badbbd3b254f5f Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Wed, 21 Oct 2026 22:24:24 +0800 Subject: [PATCH 14/20] fix(inputs): version annotation corrections without rewriting frozen suites --- docs/data.md | 19 + pyproject.toml | 2 +- src/lexbench_webfetch/benchmark.py | 61 ++- src/lexbench_webfetch/cli.py | 3 +- .../serialization-20261021.json | 360 ++++++++++++++++++ .../tasks.json | 4 +- .../risk-control/current/tasks.json | 7 +- .../risk-control/list-contracts.json | 12 +- .../evaluation/scoring-regressions.json | 177 +++++++++ .../definitions/evaluation/tasks.json | 37 +- .../definitions/registry.json | 6 +- tests/test_benchmark.py | 59 +++ 12 files changed, 697 insertions(+), 50 deletions(-) create mode 100644 src/lexbench_webfetch/definitions/annotation-revisions/serialization-20261021.json diff --git a/docs/data.md b/docs/data.md index 379a546..661690f 100644 --- a/docs/data.md +++ b/docs/data.md @@ -19,3 +19,22 @@ An approved input directory contains the exact `dataset.json` and, when listed b The exported directory intentionally has no `source-inputs.json`: it is a directly materialized, verified input tree. Browser Eval can consume its dataset through `--dataset` without relying on Browser Eval's historical storage paths. Keep the suite manifest and benchmark revision with your experiment metadata. Complete third-party text, PDFs, media, rendered pages and provider outputs are excluded from this repository. Obtain required inputs under their original terms. There is currently no hosted public input bundle; the public definitions can be inspected independently, while exact historical runs require separately acquired matching inputs. A new capture is a new version and must not be presented as the old frozen suite. + +## Score-only annotation versions + +Frozen `tasks.json`, catalogs and input manifests retain their original bytes. +Corrections live in `definitions/annotation-revisions/` with a distinct revision +ID, parent task and source-manifest hashes, per-task predecessor hashes and +registered scoring controls. They change acceptance annotations, not collection +URLs, prompts or saved provider outputs. + +To export a corrected definition, select its version explicitly: + +```sh +uv run --frozen lexbench-webfetch export --suite webfetch --source /path/to/frozen-inputs --output /path/to/new-inputs --revision serialization-20261021 +``` + +The exported dataset receives a new ID and a `definition_revision` binding. +Without `--revision`, export reproduces the original frozen dataset exactly. +A score recomputed with revised annotations must retain both definition hashes +and its original call identities; it is not a new browser run. diff --git a/pyproject.toml b/pyproject.toml index 18e2499..4cf31a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,4 +26,4 @@ build-backend = "setuptools.build_meta" where = ["src"] [tool.setuptools.package-data] -lexbench_webfetch = ["definitions/*.json", "definitions/evaluation/*.json", "definitions/evaluation/*/*.json", "definitions/evaluation/*/*.md", "definitions/evaluation/*/*/*.json", "definitions/evaluation/*/*/*/*.json"] +lexbench_webfetch = ["definitions/*.json", "definitions/annotation-revisions/*.json", "definitions/evaluation/*.json", "definitions/evaluation/*/*.json", "definitions/evaluation/*/*.md", "definitions/evaluation/*/*/*.json", "definitions/evaluation/*/*/*/*.json"] diff --git a/src/lexbench_webfetch/benchmark.py b/src/lexbench_webfetch/benchmark.py index d3fdfe5..0a31aa9 100644 --- a/src/lexbench_webfetch/benchmark.py +++ b/src/lexbench_webfetch/benchmark.py @@ -2,13 +2,14 @@ from __future__ import annotations import hashlib +from copy import deepcopy import json import os import re import shutil import tempfile from pathlib import Path -from .paths import contained_path +from .paths import contained_path, safe_identifier from .inputs import digest, load_manifest, verify_catalog SUITES = Path(__file__).parent / "definitions" @@ -16,7 +17,7 @@ # Full reference bodies and source-capture/review payloads stay in frozen inputs. PUBLIC_TASK_FIELDS = ( "id", "name", "url", "category", "track", "task_kind", "description", "task_prompt", - "expected_any", "required_facts", "forbidden_any", "min_content_chars", + "expected_any", "required_facts", "forbidden_any", "diagnostic_any", "min_content_chars", "fact_coverage_threshold", "expected_resources", "resource_output_contract", "content_score_applicable", "required_patterns", "acceptance_contract", "composite_acceptance", "mutable_body_policy", "body_requirements", "retrieval_hints", @@ -122,6 +123,10 @@ def validate(identifier: str) -> dict: contained_path(root, relative, reject_symlinks=True) if not re.fullmatch(r"[0-9a-f]{64}", expected): raise ValueError(f"Invalid input SHA-256: {relative}") + for path in sorted((SUITES/'annotation-revisions').glob('*.json')): + revision = read(path) + if identifier in revision.get('parents', {}): + apply_annotation_revision(identifier, tasks, revision['id']) return {"suite": identifier, "case_count": len(ids), "dataset_sha256": catalog["dataset_sha256"]} @@ -153,7 +158,53 @@ def verify_inputs(identifier: str, source: Path) -> dict: return result -def export(identifier: str, source: Path, output: Path) -> Path: +def apply_annotation_revision(identifier: str, dataset: dict, revision: str) -> dict: + """Apply an explicit score-only version; keep its frozen parent unchanged.""" + safe_identifier(revision) + path = contained_path(SUITES, f'annotation-revisions/{revision}.json', reject_symlinks=True) + definition = read(path) + if (definition.get('schema_version') != 1 + or definition.get('kind') != 'score_only_annotation_revision' + or definition.get('id') != revision): + raise ValueError('Unsupported annotation revision') + from .scoring_regressions import load_scoring_regressions + controls = definition.get('regression_ids', []) + if not controls or set(controls) - load_scoring_regressions().keys(): + raise ValueError('Annotation revision requires registered regression controls') + entry, root = suite(identifier) + parent = definition.get('parents', {}).get(identifier) + if parent != {'tasks_sha256': digest(root/'tasks.json'), + 'source_inputs_sha256': digest(root/'source-inputs.json')}: + raise ValueError('Annotation revision has a different frozen parent') + if dataset.get('id') != entry['dataset_id']: + raise ValueError('Annotation revision dataset identity differs') + result = deepcopy(dataset) + cases = {case['id']: case for case in result['cases']} + allowed = {'expected_resources', 'required_facts', 'forbidden_any', 'diagnostic_any', + 'composite_acceptance', 'acceptance_contract', 'required_patterns'} + changes = [row for row in definition['changes'] if row['suite'] == identifier] + if not changes or len({row['case_id'] for row in changes}) != len(changes): + raise ValueError('Annotation revision has an invalid task inventory') + for change in changes: + case = cases.get(change['case_id']) + public = public_task(case) if case is not None else None + encoded = json.dumps(public, ensure_ascii=False, sort_keys=True, separators=(',', ':')).encode() + if hashlib.sha256(encoded).hexdigest() != change['before_sha256']: + raise ValueError('Annotation revision task differs from frozen parent') + if (set(change['set']) | set(change['remove'])) - allowed: + raise ValueError('Annotation revision cannot change collection inputs') + case.update(deepcopy(change['set'])) + for field in change['remove']: + case.pop(field, None) + result['id'] = dataset['id'] + '--' + revision + result['definition_revision'] = { + 'id': revision, 'sha256': digest(path), 'parent_dataset_id': dataset['id'], + 'parent_tasks_sha256': parent['tasks_sha256'], 'provider_calls_added': 0, + } + return result + + +def export(identifier: str, source: Path, output: Path, *, revision: str | None = None) -> Path: """Copy only declared inputs after verification; never overwrite a run or input tree.""" verify_inputs(identifier, source) if output.exists() or output.is_symlink(): @@ -169,6 +220,10 @@ def export(identifier: str, source: Path, output: Path) -> Path: target.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(contained_path(source, relative, reject_symlinks=True), target) verify_inputs(identifier, stage) + if revision is not None: + dataset_path = stage / manifest['dataset'] + revised = apply_annotation_revision(identifier, read(dataset_path), revision) + dataset_path.write_text(json.dumps(revised, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') # A materialized dataset is intentionally independent of the historical # Browser Eval external-input resolver; do not copy source-inputs.json. os.rename(stage, output) diff --git a/src/lexbench_webfetch/cli.py b/src/lexbench_webfetch/cli.py index 5df6781..4997ade 100644 --- a/src/lexbench_webfetch/cli.py +++ b/src/lexbench_webfetch/cli.py @@ -18,6 +18,7 @@ def main() -> None: command.add_argument("--source", type=Path, required=True) if name == "export": command.add_argument("--output", type=Path, required=True) + command.add_argument("--revision", help="Apply a registered score-only annotation version") score = sub.add_parser("score", help="Evaluate a saved response and delivered files") score.add_argument("--task", type=Path, required=True, help="Complete task JSON including its frozen reference requirements") @@ -60,7 +61,7 @@ def main() -> None: elif args.command == "verify-inputs": result = verify_inputs(args.suite, args.source) else: - result = {"dataset": str(export(args.suite, args.source, args.output).resolve())} + result = {"dataset": str(export(args.suite, args.source, args.output, revision=args.revision).resolve())} except (ValueError, KeyError, OSError) as error: parser.exit(1, f"{error}\n") print(json.dumps(result, ensure_ascii=False, indent=2)) diff --git a/src/lexbench_webfetch/definitions/annotation-revisions/serialization-20261021.json b/src/lexbench_webfetch/definitions/annotation-revisions/serialization-20261021.json new file mode 100644 index 0000000..9a8d04b --- /dev/null +++ b/src/lexbench_webfetch/definitions/annotation-revisions/serialization-20261021.json @@ -0,0 +1,360 @@ +{ + "schema_version": 1, + "kind": "score_only_annotation_revision", + "id": "serialization-20261021", + "reason": "Preserve source terms, declared resource identities, native record boundaries and information-title scope without changing requests or collected outputs.", + "parents": { + "webfetch": { + "tasks_sha256": "ae3aec6eab8d2d470cdeb18c4f80d238e5dd79e1108a729589b77a80fe869405", + "source_inputs_sha256": "c202abc4624811b938a837402bbaf0ecbe716e375fbca70567346d2edb3b1708" + }, + "dynamic-content": { + "tasks_sha256": "139c62a141183d305d8fa9c3658c546927731461d86aeba5ad3091649bd4ecd7", + "source_inputs_sha256": "710f9c91705b29bb9e4aa15af97dcc856e64e406baa4b9966446408d7e3d26a8" + }, + "access-restrictions": { + "tasks_sha256": "c67b60ccf5ee0b14e4e23b3efc282e625005f8e096f0bfc978772d312e8f50c2", + "source_inputs_sha256": "c61d6825abbff2a92098592f83055347cd0b0101374794023046c0fd0323dd5f" + } + }, + "regression_ids": [ + "native-serialization-complete-notice", + "native-serialization-changed-notice", + "native-serialization-stock-label", + "native-serialization-stock-word-fragment", + "native-serialization-carousel-label", + "native-serialization-carousel-digit-word", + "native-serialization-cyrillic-label", + "native-serialization-cyrillic-word-fragment", + "native-serialization-punctuation-markup", + "resource-serialization-markup-entity", + "resource-serialization-different-size", + "named-scope-heading-owns-records", + "named-scope-navigation-cannot-supply-records", + "named-scope-empty-layout-is-not-name", + "named-scope-image-without-visible-name", + "native-card-bullet-complete", + "native-card-bullet-missing-area", + "native-card-bullet-other-section", + "native-card-text-same-line-region", + "native-card-text-region-without-venue", + "native-card-text-no-region", + "source-backed-diagnostic-source-discusses-status", + "source-backed-diagnostic-status-heading", + "declared-whole-card-complete", + "declared-whole-card-no-name", + "declared-whole-card-no-venue", + "declared-whole-card-no-region", + "declared-card-prelude-owned", + "declared-card-prelude-separate-card", + "declared-card-prelude-next-card-fields", + "declared-information-explicit-title", + "declared-information-keywords-not-title", + "declared-information-keywords-not-publisher", + "named-section-boundary-complete-software", + "named-section-boundary-next-section-not-software", + "declared-photo-query-unsplash-resize", + "declared-photo-query-unsplash-overlay", + "declared-photo-query-unsplash-other-asset", + "declared-photo-query-pexels-resize", + "declared-photo-query-pexels-overlay", + "declared-photo-query-pexels-other-asset" + ], + "changes": [ + { + "suite": "webfetch", + "case_id": "stable-browser-use-readme", + "before_sha256": "05cfc4d5ac1bbf1189cdc81459601dca7ef8a45a6d886a25eb07ebe409c8968b", + "set": { + "forbidden_any": [ + "Access denied", + "Attention Required! | Cloudflare", + "Page not found", + "404 Not Found", + "无法访问此站点" + ], + "diagnostic_any": [ + "CAPTCHA" + ] + }, + "remove": [] + }, + { + "suite": "webfetch", + "case_id": "stable-wcag21-recommendation", + "before_sha256": "c980470f54a9b4ebb4d91bb3d734df8647c0b234c926cc380edb31919b7dea63", + "set": { + "forbidden_any": [ + "Access denied", + "Attention Required! | Cloudflare", + "Page not found", + "404 Not Found", + "无法访问此站点" + ], + "diagnostic_any": [ + "CAPTCHA" + ] + }, + "remove": [] + }, + { + "suite": "webfetch", + "case_id": "live-030-rfc-9110", + "before_sha256": "58b5a6368da964807b43b3743ac37b91bf09bf678c7afb7553339128cfb742ca", + "set": { + "forbidden_any": [ + "Access denied", + "Attention Required! | Cloudflare", + "CAPTCHA", + "Page not found", + "无法访问此站点" + ], + "diagnostic_any": [ + "404 Not Found" + ] + }, + "remove": [] + }, + { + "suite": "webfetch", + "case_id": "image-04", + "before_sha256": "604ccebd58a7ec60175da2f6c509ababf5a6be3ad1fcc9b258bf39114b304001", + "set": { + "expected_resources": [ + { + "id": "image-1", + "kind": "image-1", + "urls": [ + "https://upload.wikimedia.org/wikipedia/commons/thumb/4/41/A_Man_on_the_Moon%2C_AS11-40-5903_%28cropped%29.jpg/330px-A_Man_on_the_Moon%2C_AS11-40-5903_%28cropped%29.jpg?utm_source=en.wikipedia.org&utm_campaign=parser&utm_content=thumbnail" + ], + "url_patterns": [ + "^(?:https://upload\\.wikimedia\\.org/wikipedia/commons/4/41/A_Man_on_the_Moon,_AS11\\-40\\-5903_\\(cropped\\)\\.jpg|https://(?:upload|thumb)\\.wikimedia\\.org/wikipedia/commons/thumb/4/41/A_Man_on_the_Moon,_AS11\\-40\\-5903_\\(cropped\\)\\.jpg/[1-9][0-9]*px-A_Man_on_the_Moon,_AS11\\-40\\-5903_\\(cropped\\)\\.jpg)(?:\\?(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+)(?:&(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+))*)?$" + ] + }, + { + "id": "image-2", + "kind": "image-2", + "urls": [ + "https://upload.wikimedia.org/wikipedia/commons/thumb/2/27/Apollo_11_insignia.png/250px-Apollo_11_insignia.png?utm_source=en.wikipedia.org&utm_campaign=parser&utm_content=thumbnail" + ], + "url_patterns": [ + "^(?:https://upload\\.wikimedia\\.org/wikipedia/commons/2/27/Apollo_11_insignia\\.png|https://(?:upload|thumb)\\.wikimedia\\.org/wikipedia/commons/thumb/2/27/Apollo_11_insignia\\.png/[1-9][0-9]*px-Apollo_11_insignia\\.png)(?:\\?(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+)(?:&(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+))*)?$" + ] + } + ] + }, + "remove": [] + }, + { + "suite": "webfetch", + "case_id": "image-unsplash-detail", + "before_sha256": "5d219be254b0145cd19f833958e8e6826b2540fc3eed9d92c541b2a47f634354", + "set": { + "expected_resources": [ + { + "id": "main-image", + "kind": "image", + "urls": [ + "https://images.unsplash.com/photo-1493673272479-a20888bcee10", + "https://unsplash.com/photos/mSK5nNsAsLY/download" + ], + "url_patterns": [ + "^https://images\\.unsplash\\.com/photo\\-1493673272479\\-a20888bcee10(?:\\?(?:(?:w|h)=[1-9][0-9]*|q=(?:100|[1-9][0-9]?)|dpr=[1-9][0-9]*(?:\\.[0-9]+)?|fm=(?:jpg|jpeg|png|webp|avif|gif)|fit=(?:crop|clip|fill|fillmax|max|min|scale)|crop=(?:faces|edges|entropy|top|bottom|left|right|focalpoint)|cs=(?:srgb|tinysrgb)|auto=format|ixlib=[A-Za-z0-9._-]+|ixid=[A-Za-z0-9_+/=%-]+)(?:&(?:(?:w|h)=[1-9][0-9]*|q=(?:100|[1-9][0-9]?)|dpr=[1-9][0-9]*(?:\\.[0-9]+)?|fm=(?:jpg|jpeg|png|webp|avif|gif)|fit=(?:crop|clip|fill|fillmax|max|min|scale)|crop=(?:faces|edges|entropy|top|bottom|left|right|focalpoint)|cs=(?:srgb|tinysrgb)|auto=format|ixlib=[A-Za-z0-9._-]+|ixid=[A-Za-z0-9_+/=%-]+))*)?$" + ] + } + ] + }, + "remove": [] + }, + { + "suite": "webfetch", + "case_id": "image-pexels-detail", + "before_sha256": "a3e0ca87dcadb509a7e3c23371eac30cb10adfb5c00e96774d3b6369ef024651", + "set": { + "expected_resources": [ + { + "id": "main-image", + "kind": "image", + "urls": [ + "https://images.pexels.com/photos/12377231/pexels-photo-12377231.jpeg" + ], + "url_patterns": [ + "^https://images\\.pexels\\.com/photos/12377231/pexels\\-photo\\-12377231\\.jpeg(?:\\?(?:(?:w|h)=[1-9][0-9]*|q=(?:100|[1-9][0-9]?)|dpr=[1-9][0-9]*(?:\\.[0-9]+)?|fm=(?:jpg|jpeg|png|webp|avif|gif)|fit=(?:crop|clip|fill|fillmax|max|min|scale)|crop=(?:faces|edges|entropy|top|bottom|left|right|focalpoint)|cs=(?:srgb|tinysrgb)|auto=(?:compress|format|compress,format)|dl=[A-Za-z0-9._-]+)(?:&(?:(?:w|h)=[1-9][0-9]*|q=(?:100|[1-9][0-9]?)|dpr=[1-9][0-9]*(?:\\.[0-9]+)?|fm=(?:jpg|jpeg|png|webp|avif|gif)|fit=(?:crop|clip|fill|fillmax|max|min|scale)|crop=(?:faces|edges|entropy|top|bottom|left|right|focalpoint)|cs=(?:srgb|tinysrgb)|auto=(?:compress|format|compress,format)|dl=[A-Za-z0-9._-]+))*)?$" + ] + } + ] + }, + "remove": [] + }, + { + "suite": "webfetch", + "case_id": "video-09", + "before_sha256": "78f981fb66c5c47e4b90f57a6f6dec2c1a5b284374f706dde0958dbcabfa11a9", + "set": { + "required_facts": [ + { + "id": "视频标题", + "alternatives": [ + "Me at the zoo" + ], + "match_scope": "body_or_title" + }, + { + "id": "发布者", + "alternatives": [ + "jawed" + ] + } + ] + }, + "remove": [] + }, + { + "suite": "dynamic-content", + "case_id": "replacement-ja-nintendo-mario", + "before_sha256": "c93d075172fc993788626460f9786a41ddff9f1f5955146cfb463e4bd8e21c94", + "set": { + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 7, + "hosts": [ + "www.nintendo.com", + "www.nintendo.co.jp", + "store-jp.nintendo.com" + ], + "entry_path": "/(?:jp/(?:games/(?:switch2|zelda)/[^?#]+|switch/[^?#]+|my-mario/app/index\\.html)|item/software/D[0-9]+|zelda/[^?#]+|character/zelda/[^?#]+|switch/[^?#]+)", + "stable_text_anchors": [ + { + "id": "rule25-740e4325c2c2", + "alternatives": [ + "ゲームソフトをさがす" + ], + "match_scope": "non_record_lines" + }, + { + "id": "rule25-112f4528726b", + "alternatives": [ + "ページをさがす" + ], + "match_scope": "non_record_lines" + }, + { + "id": "rule25-0e3206dde854", + "alternatives": [ + "すべてのページ" + ], + "match_scope": "non_record_lines" + } + ], + "entry_scope": { + "html_selector": ".nc3-as-result--soft:not(.is-loading) a[href]", + "start_pattern": "^ゲームソフトをさがす\\s*$", + "end_pattern": "^(?:すべて見る|ページをさがす)" + } + } + }, + "remove": [] + }, + { + "suite": "dynamic-content", + "case_id": "replacement-ja-nintendo-zelda", + "before_sha256": "6170d2206529d52ed0f84702e3fc444dee90086152c391d6b6da519a3d3408c0", + "set": { + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 8, + "hosts": [ + "www.nintendo.com", + "www.nintendo.co.jp", + "store-jp.nintendo.com" + ], + "entry_path": "/(?:jp/(?:games/(?:switch2|zelda)/[^?#]+|switch/[^?#]+|my-mario/app/index\\.html)|item/software/D[0-9]+|zelda/[^?#]+|character/zelda/[^?#]+|switch/[^?#]+)", + "stable_text_anchors": [ + { + "id": "rule25-04f69a7c4127", + "alternatives": [ + "ゲームソフトをさがす" + ], + "match_scope": "non_record_lines" + }, + { + "id": "rule25-744c35c56ada", + "alternatives": [ + "ページをさがす" + ], + "match_scope": "non_record_lines" + }, + { + "id": "rule25-36f85bd9f532", + "alternatives": [ + "すべてのページ" + ], + "match_scope": "non_record_lines" + } + ], + "entry_scope": { + "html_selector": ".nc3-as-result--soft:not(.is-loading) a[href]", + "start_pattern": "^ゲームソフトをさがす\\s*$", + "end_pattern": "^(?:すべて見る|ページをさがす)" + } + } + }, + "remove": [] + }, + { + "suite": "access-restrictions", + "case_id": "risk-diverse-074", + "before_sha256": "45c0608f91f9666378ba036ac877aa34998d9b71803e89d0c7a0c8c49ceff29e", + "set": { + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "concerts", + "kind": "named_link", + "minimum_records": 26, + "hosts": [ + "www.ticketmaster.com", + "www.ticketweb.com" + ], + "entry_path": "/.*event/[A-Za-z0-9_-]+", + "record_name_exclusions": [ + "BLUE FRIDAY SPECIAL! Entry to All Shows at the Blue Moon for 2026!", + "I'm On the List! + 1 (Entry to all shows at the Blue Moon in 2026...)" + ], + "required_fields": [ + { + "id": "date_or_time", + "kind": "date_or_time" + }, + { + "id": "venue", + "kind": "text", + "minimum_chars": 2 + }, + { + "id": "city_or_region", + "kind": "text", + "minimum_chars": 2, + "pattern": ",\\s*(?:AL|AK|AZ|AR|CA|CO|CT|DE|DC|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY)\\b" + } + ], + "markdown_label_pattern": "(?m)^(?:(?:Today|Tomorrow)[ \\t]*\\n)?(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday),[^\\n]*\\b20[0-9]{2}\\b[^\\n]*\\n(?P