diff --git a/CITATION.cff b/CITATION.cff index e077256..a5481a5 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -5,5 +5,6 @@ title: LexBench Web Fetch authors: - name: LexBench Web Fetch contributors repository-code: "https://github.com/lexmount/lexbench-webfetch" +url: "https://github.com/lexmount/lexbench-webfetch" license: MIT -version: 0.2.0 +version: 0.3.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9b1284a..bcc0fb7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,3 +7,53 @@ Use Python 3.11 or newer and [uv](https://docs.astral.sh/uv/). Run `uv sync --fr A task revision must explain the intended requirement, preserve the prior input version, and include scenarios that should pass and fail. Update the original dataset, catalog, public task definitions, input manifest and registry together after independent source verification. Do not regenerate old hashes to disguise an input correction. Frozen quotes retain their original language. Keep measured outputs, captures, credentials and internal review notes outside Git. A result proposal must supply the evidence listed in the [protocol](docs/protocol.md); opening a PR does not add it to a leaderboard. See [THIRD_PARTY](THIRD_PARTY.md) before sharing source material. + +## PR and commit titles + +Use `(): ` for PR titles and new commit subjects. The scope is optional; use the module or feature affected, such as `scoring` or `webfetch`. + +Choose the type that describes the change: + +| Type | Change | +| --- | --- | +| `fix` | Correct existing behavior | +| `feat` | Add supported behavior | +| `docs` | Change documentation | +| `refactor` | Restructure code without changing behavior | +| `test` | Add or correct tests | +| `perf` | Improve performance | +| `build` / `ci` | Change packaging, dependencies or automation | +| `chore` | Other maintenance | + +Write the summary in English, start with an imperative verb, and describe the concrete change. Aim for 72 characters or fewer, including the prefix; omit a trailing period. Use `!` before the colon for a breaking change and explain the migration in the description. + +Examples: + +- `fix(scoring): require facts in the returned body` +- `fix(webfetch): disclose build provenance for partial reruns` +- `docs(contributing): define PR and commit title conventions` + +Avoid titles such as `Fix review feedback`, `Update code` or `Misc improvements`; name the behavior or document that changes. A PR title describes the complete diff; each commit subject describes that commit. Update the PR title when its scope changes. If squash merging is used, use a subject that follows the same format. Apply this convention to new work; do not rewrite shared commits solely to rename them. + +## Score-only annotation revisions + +Keep an existing revision's bytes unchanged. Bind each new correction in +`definitions/annotation-revisions/reviews.json` to its revision hash, exact field +difference, frozen source inputs, reason, and accepted/rejected regression controls. +These controls run before a revision is applied. Changing another field requires +updating its review binding; a registered control ID alone is insufficient. + +Inspect a revision without private measured outputs: + +```sh +uv run --frozen lexbench-webfetch inspect-revision \ + --suite dynamic-content --revision serialization-20261021 +``` + +`export --revision` writes `annotation-review.json` beside the dataset, including +the field differences, source hashes and executed controls. It preserves the +original input files and the revision's existing dataset identity. + +## License + +Contributions use the [Apache License 2.0](LICENSE). Preserve third-party licenses, copyright notices and source attributions described in [THIRD_PARTY](THIRD_PARTY.md). diff --git a/LICENSE b/LICENSE index cfb4b00..d645695 100644 --- a/LICENSE +++ b/LICENSE @@ -1,21 +1,202 @@ -MIT License - -Copyright (c) 2026 LexMount - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..03d34a5 --- /dev/null +++ b/NOTICE @@ -0,0 +1,2 @@ +LexBench Web Fetch +Copyright 2026 LexMount diff --git a/README.md b/README.md index cd5de7e..ac3a5c6 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,11 @@
-LexBench Web Fetch task suites and acceptance rules - -# LexBench Web Fetch - -### What does a fetch actually return? - -Versioned tasks and acceptance rules for retrieving web content, structured records and files. +LexBench Web Fetch — Measure what comes back. [English](README.md) · [简体中文](docs/README.zh-CN.md) [![CI](https://github.com/lexmount/lexbench-webfetch/actions/workflows/ci.yml/badge.svg)](https://github.com/lexmount/lexbench-webfetch/actions/workflows/ci.yml) -[![License: MIT](https://img.shields.io/badge/code-MIT-51735b)](LICENSE) +[![License: Apache 2.0](https://img.shields.io/badge/code-Apache--2.0-51735b)](LICENSE) [![Python 3.11+](https://img.shields.io/badge/Python-3.11%2B-356a91)](pyproject.toml) [Tasks](#task-suites) · [Quickstart](#quickstart) · [Evaluation protocol](docs/protocol.md) · [Contributing](CONTRIBUTING.md) @@ -110,6 +104,16 @@ uv run --frozen browser-harness-eval run \ Run these two commands from Browser Eval. Ordinary runs are diagnostic; a formal live-site comparison additionally requires its pre-call/post-call reference certification. The benchmark input checker verifies frozen bytes, not present-day website validity. See the [Browser Eval integration guide](https://github.com/lexmount/browser-eval/blob/main/docs/benchmarks.md). +## Scoring behavior + +Version 0.3 matches Latin words at word boundaries for facts, markers and stable anchors. Required body facts use the returned content; a separate document title cannot supply missing body text. Chinese phrases retain literal matching, and numeric and currency checks keep their value boundaries. + +A task can set `page_identity_required: true` to require its final URL observations. Missing or mismatched observations then fail acceptance. Dynamic tasks can also set `document_language_match: "exact"` when the complete language tag matters; the default is `"primary"`. These checks apply across interfaces. Frozen tasks that do not require URL metadata keep their content-based acceptance rules. + +Offline scoring recomputes derived fields. It retains the original reference hash, deadline evidence and file fingerprints. A file verdict requires current file verification; an old pass flag cannot certify a rescore. Summaries expose `mean_all_attempts_ms`, `all_attempts_p50_ms` and `all_attempts_p95_ms` alongside the existing successful-call timings. The all-attempt population includes failed calls and timeouts. + +These changes do not revise frozen tasks or certify historical results. Score corrections require a new evidence revision tied to the scorer and its regression controls. + ## Repository contents | Path | Purpose | @@ -123,8 +127,24 @@ Run these two commands from Browser Eval. Ordinary runs are diagnostic; a formal Browser adapters, credentials, measured runs, report renderers and hosting configuration belong to the execution harness or its evidence storage. This repository owns task models, deterministic acceptance scoring, file-byte checks and task compilation. Browser Eval imports these implementations as a pinned dependency. This repository publishes no measured leaderboard. -## Contribute and cite +## Contribute Read [CONTRIBUTING](CONTRIBUTING.md) before changing a task or proposing a result. Run `make check` to validate the definitions and test the input tools. Report security issues through [SECURITY](SECURITY.md). -Cite [LexBench Web Fetch](CITATION.cff) and record the exact commit and suite. Cite any execution harness separately. Project-owned code and definitions use the [MIT license](LICENSE); third-party source material retains its own terms, described in [THIRD_PARTY](THIRD_PARTY.md). +Project-owned code and definitions use the [Apache 2.0 license](LICENSE); third-party source material retains its own terms, described in [THIRD_PARTY](THIRD_PARTY.md). + +## Cite LexBench Web Fetch + +If you use the task definitions or scoring tools in published work, cite this repository and record the exact commit and suite used. GitHub's citation menu reads [CITATION.cff](CITATION.cff). For version 0.3.0: + +```bibtex +@software{lexbench_webfetch_2026, + author = {{LexBench Web Fetch contributors}}, + title = {LexBench Web Fetch}, + year = {2026}, + version = {0.3.0}, + url = {https://github.com/lexmount/lexbench-webfetch} +} +``` + +Cite [Browser Eval](https://github.com/lexmount/browser-eval#cite-browser-eval) separately when using its execution harness, and cite upstream sources or benchmarks according to their attribution terms. This repository citation does not replace those citations. When using another version, update the version and year in the example and identify its tag or commit. diff --git a/THIRD_PARTY.md b/THIRD_PARTY.md index 2f47296..e3156af 100644 --- a/THIRD_PARTY.md +++ b/THIRD_PARTY.md @@ -1,6 +1,6 @@ # Third-party content -The MIT license covers project-owned code, documentation and task requirements. Initial definitions and input-checking concepts originate in [Browser Eval](https://github.com/lexmount/browser-eval), also MIT licensed; retain its copyright notice in LICENSE. +The Apache-2.0 license covers project-owned code, documentation and task requirements. Initial definitions and input-checking concepts originate in [Browser Eval](https://github.com/lexmount/browser-eval); its LexMount copyright notice is retained in NOTICE. Original website text, document files, media, screenshots and frozen reference captures retain their source terms and are not included here. Public accessibility and a source URL do not grant redistribution permission. Task definitions retain source URLs and compact acceptance requirements; full historical reference inputs must be obtained separately under an appropriate authorization basis. diff --git a/docs/README.zh-CN.md b/docs/README.zh-CN.md index 0b090f9..2ccce76 100644 --- a/docs/README.zh-CN.md +++ b/docs/README.zh-CN.md @@ -1,17 +1,11 @@
-LexBench Web Fetch task suites and acceptance rules - -# LexBench Web Fetch - -### 抓取成功后，究竟拿到了什么？ - -为网页正文、结构化记录和文件抓取提供版本固定的任务与验收规则。 +LexBench Web Fetch — Measure what comes back. [English](../README.md) · 简体中文 [![CI](https://github.com/lexmount/lexbench-webfetch/actions/workflows/ci.yml/badge.svg)](https://github.com/lexmount/lexbench-webfetch/actions/workflows/ci.yml) -[![License: MIT](https://img.shields.io/badge/code-MIT-51735b)](../LICENSE) +[![License: Apache 2.0](https://img.shields.io/badge/code-Apache--2.0-51735b)](../LICENSE) [![Python 3.11+](https://img.shields.io/badge/Python-3.11%2B-356a91)](../pyproject.toml) [任务集](#任务集) · [快速开始](#快速开始) · [评测协议](protocol.md) · [参与贡献](../CONTRIBUTING.md) @@ -110,6 +104,16 @@ uv run --frozen browser-harness-eval run \ 普通运行用于诊断。正式比较实时网站时，还需要完成调用前后的参考内容认证。输入校验只能证明文件版本相符，不能证明网站今天仍然符合旧参考。详见 [Browser Eval 接入说明](https://github.com/lexmount/browser-eval/blob/main/docs/benchmarks.md)。 +## 评分规则 + +0.3 版对事实、标记和固定锚点中的拉丁文使用单词边界匹配。正文事实只匹配返回内容，单独返回的文档标题不能补足正文。中文短语仍按原文匹配，数字和货币保留原有的数值边界检查。 + +任务可设置 `page_identity_required: true`，要求返回的最终网址符合任务声明；缺少网址或网址不符都会判为未通过。动态任务需要区分完整语言标签时，可再设置 `document_language_match: "exact"`，默认值为 `"primary"`，只比较主语言。这些检查适用于所有接口。没有要求网址元数据的冻结任务，继续按原有内容规则验收。 + +离线评分会重新计算派生字段，保留原始参考哈希、调用时限证据和文件指纹。文件结论需要当前文件验收依据；旧的通过标记不能认证本次重评分。汇总在原有成功调用耗时之外，提供全部调用的 `mean_all_attempts_ms`、`all_attempts_p50_ms` 和 `all_attempts_p95_ms`，包含失败调用与超时。 + +上述修改不改写冻结题目，也不自动认证历史成绩。更正分数仍需建立新的证据修订，绑定评分器版本和回归用例。 + ## 仓库内容 | 路径 | 用途 | @@ -123,8 +127,24 @@ uv run --frozen browser-harness-eval run \ 浏览器适配器、凭据、实测结果、报告生成器和托管配置由执行脚手架或外部证据存储管理。任务模型、确定性验收评分、文件字节校验和任务编译由本仓库维护，Browser Eval 通过固定版本的依赖直接调用。本仓库没有实测排行榜。 -## 参与贡献与引用 +## 参与贡献 修改任务或提交结果前，请阅读 [CONTRIBUTING](../CONTRIBUTING.md)。`make check` 会检查任务定义和输入工具。安全问题请参阅 [SECURITY](../SECURITY.md)。 -引用 [LexBench Web Fetch](../CITATION.cff) 时，请记录具体提交版本和任务集；执行脚手架单独引用。项目自有代码和任务定义采用 [MIT 许可](../LICENSE)，第三方原始材料保留各自使用条件，详见 [THIRD_PARTY](../THIRD_PARTY.md)。 +项目自有代码和任务定义采用 [Apache 2.0 许可](../LICENSE)，第三方原始材料保留各自使用条件，详见 [THIRD_PARTY](../THIRD_PARTY.md)。 + +## 引用 LexBench Web Fetch + +在公开作品中使用本仓库的任务定义或评分工具时，请引用仓库，并记录具体提交和任务集。GitHub 的引用入口读取 [CITATION.cff](../CITATION.cff)。0.3.0 版可这样引用： + +```bibtex +@software{lexbench_webfetch_2026, + author = {{LexBench Web Fetch contributors}}, + title = {LexBench Web Fetch}, + year = {2026}, + version = {0.3.0}, + url = {https://github.com/lexmount/lexbench-webfetch} +} +``` + +如果还使用了 [Browser Eval](https://github.com/lexmount/browser-eval#cite-browser-eval) 执行脚手架，请单独引用。上游来源或 benchmark 也应按各自的署名要求引用，本仓库的引用不能替代它们。使用其他版本时，请相应修改示例中的版本号和年份，并注明实际 tag 或提交。 diff --git a/docs/assets/overview.png b/docs/assets/overview.png new file mode 100644 index 0000000..8cb774d Binary files /dev/null and b/docs/assets/overview.png differ diff --git a/docs/assets/overview.svg b/docs/assets/overview.svg deleted file mode 100644 index 613aa61..0000000 --- a/docs/assets/overview.svg +++ /dev/null @@ -1,22 +0,0 @@ - -LexBench Web Fetch -Benchmark definitions: 550 TASK ENTRIES — 150 general · 100 dynamic · 100 popular · 100 restricted · 100 articles; ACCEPTANCE RULES — Body text · Structured records · Resources · Files; FROZEN INPUTS — Versioned catalogs · SHA-256 reference bindings - - -BENCHMARK DEFINITIONS -LexBench Web Fetch - -550 TASK ENTRIES -150 general · 100 dynamic -100 popular · 100 restricted · 100 articles - - -ACCEPTANCE RULES -Body text · Structured records -Resources · Files - - -FROZEN INPUTS -Versioned catalogs -SHA-256 reference bindings - diff --git a/docs/data.md b/docs/data.md index 379a546..661690f 100644 --- a/docs/data.md +++ b/docs/data.md @@ -19,3 +19,22 @@ An approved input directory contains the exact `dataset.json` and, when listed b The exported directory intentionally has no `source-inputs.json`: it is a directly materialized, verified input tree. Browser Eval can consume its dataset through `--dataset` without relying on Browser Eval's historical storage paths. Keep the suite manifest and benchmark revision with your experiment metadata. Complete third-party text, PDFs, media, rendered pages and provider outputs are excluded from this repository. Obtain required inputs under their original terms. There is currently no hosted public input bundle; the public definitions can be inspected independently, while exact historical runs require separately acquired matching inputs. A new capture is a new version and must not be presented as the old frozen suite. + +## Score-only annotation versions + +Frozen `tasks.json`, catalogs and input manifests retain their original bytes. +Corrections live in `definitions/annotation-revisions/` with a distinct revision +ID, parent task and source-manifest hashes, per-task predecessor hashes and +registered scoring controls. They change acceptance annotations, not collection +URLs, prompts or saved provider outputs. + +To export a corrected definition, select its version explicitly: + +```sh +uv run --frozen lexbench-webfetch export --suite webfetch --source /path/to/frozen-inputs --output /path/to/new-inputs --revision serialization-20261021 +``` + +The exported dataset receives a new ID and a `definition_revision` binding. +Without `--revision`, export reproduces the original frozen dataset exactly. +A score recomputed with revised annotations must retain both definition hashes +and its original call identities; it is not a new browser run. diff --git a/docs/protocol.md b/docs/protocol.md index 126a569..9b8baa8 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -38,4 +38,25 @@ Historical `fetch_log_final_url` and `video_values` formats remain replay reader A flattened link label establishes multiple record fields only when its visible text retains field separators, such as line breaks or spaced vertical bars. URL slugs, English ticket-action labels and US city/state spelling cannot establish missing field boundaries. Ambiguous labels remain unqualified even when a reader can guess their intended layout. Named table columns with duplicate matching headers remain ambiguous; a navigation line without a table structure is not a column header. +A named-link group may declare `markdown_label_pattern` with a named `label` capture when a card places metadata before its visible name. The capture selects text already present in that link; it cannot supply a name from the URL. `record_start_pattern` bounds cards whose fields precede their title link. Fields before a later card start or another target link cannot complete the current record. + +Text field patterns consume their matched spans, so distinct declared fields may share a physical line. Matching one field cannot reuse its value for a second requirement. Unlabeled text still needs its own evidence; relative-date controls such as “Today” do not establish a venue or source. + +Required facts use body text by default. An information task may declare `match_scope: "body_or_title"` for a fact that explicitly requests a page title. This permits the delivered title field, an HTML document title, or a scalar top-level Markdown front-matter title. Keywords and descriptions cannot supply the title, and title metadata cannot supply other facts. + Contract-declared field aliases support Unicode labels. The built-in English metadata-role recognition is a limited legacy fallback, not an automatic translation dictionary. Declare the language-specific labels needed by a task rather than treating every colon in prose as a metadata separator. Body matching treats unfenced table tags as HTML, including in mixed prose. Literal code needs an explicit code fence or escape; a standalone table tag cannot disable HTML visibility rules for hidden elements or attributes. + +### Reference controls for annotation revisions + +A revision export runs the frozen reference through the public scorer and task +predicate, then checks that an error page fails. The resulting +`annotation-review.json` records the source and answer hashes, outcomes and scorer +identity. Resource references use URLs retained by an independent source capture, +serialized in the requested answer format; they are not generated from expected +URLs. A known reference that fails the revised contract blocks export. + +Some historical input bundles retain observation hashes without the original +page bytes or an executable reference answer. Their control is `unavailable`, +never passed, and the review's `complete` flag remains false. Exporting these +frozen inputs preserves their historical definition; it does not certify the +missing evidence or create a new independent source observation. diff --git a/pyproject.toml b/pyproject.toml index d3d39a7..c8988ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,11 @@ [project] name = "lexbench-webfetch" -version = "0.2.0" +version = "0.3.0" description = "Versioned Web Fetch tasks, acceptance contracts and frozen reference bindings" readme = "README.md" requires-python = ">=3.11" -license = "MIT" +license = "Apache-2.0" +license-files = ["LICENSE", "NOTICE", "THIRD_PARTY.md"] dependencies = ["beautifulsoup4==4.14.2", "soupsieve==2.9.2"] [project.optional-dependencies] @@ -26,4 +27,4 @@ build-backend = "setuptools.build_meta" where = ["src"] [tool.setuptools.package-data] -lexbench_webfetch = ["definitions/*.json", "definitions/evaluation/*.json", "definitions/evaluation/*/*.json", "definitions/evaluation/*/*.md", "definitions/evaluation/*/*/*.json", "definitions/evaluation/*/*/*/*.json"] +lexbench_webfetch = ["definitions/*.json", "definitions/annotation-revisions/*.json", "definitions/evaluation/*.json", "definitions/evaluation/*/*.json", "definitions/evaluation/*/*.md", "definitions/evaluation/*/*/*.json", "definitions/evaluation/*/*/*/*.json"] diff --git a/src/lexbench_webfetch/benchmark.py b/src/lexbench_webfetch/benchmark.py index 42c2e53..a3178be 100644 --- a/src/lexbench_webfetch/benchmark.py +++ b/src/lexbench_webfetch/benchmark.py @@ -2,13 +2,14 @@ from __future__ import annotations import hashlib +from copy import deepcopy import json import os import re import shutil import tempfile from pathlib import Path -from .paths import contained_path +from .paths import contained_path, safe_identifier from .inputs import digest, load_manifest, verify_catalog SUITES = Path(__file__).parent / "definitions" @@ -16,11 +17,12 @@ # Full reference bodies and source-capture/review payloads stay in frozen inputs. PUBLIC_TASK_FIELDS = ( "id", "name", "url", "category", "track", "task_kind", "description", "task_prompt", - "expected_any", "required_facts", "forbidden_any", "min_content_chars", + "expected_any", "required_facts", "forbidden_any", "diagnostic_any", "min_content_chars", "fact_coverage_threshold", "expected_resources", "resource_output_contract", "content_score_applicable", "required_patterns", "acceptance_contract", "composite_acceptance", "mutable_body_policy", "body_requirements", "retrieval_hints", - "source_review_url", "evaluation_url", "expected_final_url", "expected_document_language", + "source_review_url", "evaluation_url", "expected_final_url", "page_identity_required", + "expected_document_language", "document_language_match", "expected_document_language_present", "target_language", "target_region", ) @@ -125,6 +127,10 @@ def validate(identifier: str) -> dict: contained_path(root, relative, reject_symlinks=True) if not re.fullmatch(r"[0-9a-f]{64}", expected): raise ValueError(f"Invalid input SHA-256: {relative}") + for path in sorted((SUITES/'annotation-revisions').glob('*.json')): + revision = read(path) + if identifier in revision.get('parents', {}): + apply_annotation_revision(identifier, tasks, revision['id']) return {"suite": identifier, "case_count": len(ids), "dataset_sha256": catalog["dataset_sha256"]} @@ -156,7 +162,184 @@ def verify_inputs(identifier: str, source: Path) -> dict: return result -def export(identifier: str, source: Path, output: Path) -> Path: +def annotation_changes(before, after, path=''): + """Describe changed leaves; lists retain their order and exact values.""" + if isinstance(before, dict) and isinstance(after, dict): + rows = [] + for key in sorted(before.keys() | after.keys()): + child = path + '/' + key.replace('~', '~0').replace('/', '~1') + if key not in before: rows.append({'path': child, 'after': after[key]}) + elif key not in after: rows.append({'path': child, 'before': before[key]}) + else: rows.extend(annotation_changes(before[key], after[key], child)) + return rows + if isinstance(before, list) and isinstance(after, list) and len(before) == len(after): + return [row for index, (old, new) in enumerate(zip(before, after)) + for row in annotation_changes(old, new, path + '/' + str(index))] + return [] if before == after else [{'path': path, 'before': before, 'after': after}] + + +def _review_annotation(definition, revision_path, identifier, root, changes): + from .scoring_regressions import check_scoring_regressions + reviews_path = contained_path(SUITES, 'annotation-revisions/reviews.json', reject_symlinks=True) + if not reviews_path.is_file(): + raise ValueError('Annotation revision requires per-task evidence and controls') + reviews = read(reviews_path) + review = reviews.get('revisions', {}).get(definition['id'], {}) + if reviews.get('schema_version') != 1 or review.get('revision_sha256') != digest(revision_path): + raise ValueError('Annotation review differs from the declared revision') + entries = review.get('changes', []) + keys = [(item['suite'], item['case_id']) for item in entries] + if len(keys) != len(set(keys)) or set(keys) != {(item['suite'], item['case_id']) for item in definition['changes']}: + raise ValueError('Annotation review task inventory differs') + declared_inputs = input_files(read(root/'source-inputs.json')) + records = [] + for change, old, new in changes: + item = entries[keys.index((identifier, change['case_id']))] + delta = annotation_changes(public_task(old), public_task(new)) + encoded = json.dumps(delta, ensure_ascii=False, sort_keys=True, separators=(',', ':')).encode() + if item.get('changes_sha256') != hashlib.sha256(encoded).hexdigest(): + raise ValueError('Annotation includes unreviewed field changes: ' + change['case_id']) + if not isinstance(item.get('reason'), str) or not item['reason'].strip(): + raise ValueError('Annotation requires a per-task reason') + sources = item.get('sources', []) + if not sources or any(source.get('path') not in declared_inputs + or declared_inputs[source['path']] != source.get('sha256') + or not isinstance(source.get('scope'), str) or not source['scope'].strip() + for source in sources): + raise ValueError('Annotation source is not bound by its frozen inputs') + controls = item.get('regression_ids', []) + if not controls or set(controls) - set(definition['regression_ids']): + raise ValueError('Annotation controls differ from its declared revision') + check_scoring_regressions(controls) + records.append({'suite': identifier, 'case_id': change['case_id'], + 'reason': item['reason'], 'sources': sources, 'changes': delta, + 'regression_ids': controls, + 'source_coverage': ('reference_source_declared' if any(source['path'] != 'dataset.json' for source in sources) + else 'task_and_source_hashes_only')}) + return {'revision': definition['id'], 'revision_sha256': digest(revision_path), + 'review_sha256': digest(reviews_path), 'changes': records} + + +def apply_annotation_revision(identifier: str, dataset: dict, revision: str) -> dict: + """Apply a reviewed score-only version without changing its frozen parent.""" + return _apply_annotation_revision(identifier, dataset, revision)[0] + + +def inspect_annotation_revision(identifier: str, revision: str) -> dict: + """Inspect exact field differences, frozen sources and passing controls.""" + _, root = suite(identifier) + return _apply_annotation_revision(identifier, read(root/'tasks.json'), revision)[1] + + +def _apply_annotation_revision(identifier: str, dataset: dict, revision: str): + safe_identifier(revision) + path = contained_path(SUITES, f'annotation-revisions/{revision}.json', reject_symlinks=True) + definition = read(path) + if (definition.get('schema_version') != 1 + or definition.get('kind') != 'score_only_annotation_revision' + or definition.get('id') != revision): + raise ValueError('Unsupported annotation revision') + from .scoring_regressions import load_scoring_regressions + controls = definition.get('regression_ids', []) + if not controls or set(controls) - load_scoring_regressions().keys(): + raise ValueError('Annotation revision requires registered regression controls') + entry, root = suite(identifier) + parent = definition.get('parents', {}).get(identifier) + if parent != {'tasks_sha256': digest(root/'tasks.json'), + 'source_inputs_sha256': digest(root/'source-inputs.json')}: + raise ValueError('Annotation revision has a different frozen parent') + if dataset.get('id') != entry['dataset_id']: + raise ValueError('Annotation revision dataset identity differs') + result = deepcopy(dataset) + cases = {case['id']: case for case in result['cases']} + allowed = {'expected_resources', 'required_facts', 'forbidden_any', 'diagnostic_any', + 'composite_acceptance', 'acceptance_contract', 'required_patterns'} + changes = [row for row in definition['changes'] if row['suite'] == identifier] + if not changes or len({row['case_id'] for row in changes}) != len(changes): + raise ValueError('Annotation revision has an invalid task inventory') + reviewed_changes = [] + for change in changes: + case = cases.get(change['case_id']) + public = public_task(case) if case is not None else None + encoded = json.dumps(public, ensure_ascii=False, sort_keys=True, separators=(',', ':')).encode() + if hashlib.sha256(encoded).hexdigest() != change['before_sha256']: + raise ValueError('Annotation revision task differs from frozen parent') + if (set(change['set']) | set(change['remove'])) - allowed: + raise ValueError('Annotation revision cannot change collection inputs') + old = deepcopy(case) + case.update(deepcopy(change['set'])) + for field in change['remove']: + case.pop(field, None) + reviewed_changes.append((change, old, case)) + review = _review_annotation(definition, path, identifier, root, reviewed_changes) + result['id'] = dataset['id'] + '--' + revision + result['definition_revision'] = { + 'id': revision, 'sha256': digest(path), 'parent_dataset_id': dataset['id'], + 'parent_tasks_sha256': parent['tasks_sha256'], 'provider_calls_added': 0, + } + return result, review + + +def check_revision_references(dataset: dict, review: dict, source: Path) -> dict: + """Run frozen reference and error-page controls without treating absent text as a pass.""" + from html import escape + from .scoring import score_attempt, SCORING_SOURCE_SHA256 + from .metrics import task_passed + cases = {case['id']: case for case in dataset['cases']} + results = [] + for item in review['changes']: + case = cases[item['case_id']] + candidates = [] + for field in ('ground_truth_main_content', 'reference_answer'): + if isinstance(case.get(field), str) and case[field].strip(): + candidates.append((case[field].encode(), {'path': 'dataset.json', 'field': field})) + for binding in item['sources']: + if binding['path'] == 'dataset.json': + continue + path = contained_path(source, binding['path'], reject_symlinks=True) + if digest(path) != binding['sha256']: + raise ValueError('Reference control source hash mismatch: ' + binding['path']) + raw = path.read_bytes() + if path.suffix == '.json': + payload = json.loads(raw) + resources = payload.get('resource_candidates') if isinstance(payload, dict) else None + if isinstance(resources, list): + image_answer = bool(case.get('expected_resources')) and all( + resource.get('kind') == 'image' for resource in case['expected_resources']) + lines = [] + for resource in resources: + url = resource.get('url') + if isinstance(url, str): + lines.append(('' if image_answer else '">Reference')) + raw = '\n'.join(lines).encode() + elif isinstance(payload, dict) and isinstance(payload.get('reference'), str): + raw = payload['reference'].encode() + else: + continue + candidates.append((raw, {'path': binding['path'], 'sha256': binding['sha256']})) + controls = [] + for raw, binding in candidates: + metrics = score_attempt(case, raw, ok=True) + controls.append({'source': binding, 'answer_sha256': hashlib.sha256(raw).hexdigest(), + 'passed': task_passed(True, metrics, case.get('task_kind', 'body'))}) + if controls[-1]['passed']: + break + negative = b'Verify you are human. Access Denied. CAPTCHA.' + metrics = score_attempt(case, negative, ok=True) + negative_pass = task_passed(True, metrics, case.get('task_kind', 'body')) + status = ('failed' if negative_pass + else 'passed' if any(control['passed'] for control in controls) + else 'failed' if controls else 'unavailable') + results.append({'case_id': case['id'], 'status': status, 'reference_candidates': controls, + 'error_page_passed': negative_pass, + 'reason': ('The frozen inputs retain source hashes but no executable reference answer.' + if not controls else '')}) + return {'scoring_source_sha256': SCORING_SOURCE_SHA256, + 'complete': all(result['status'] == 'passed' for result in results), 'cases': results} + + +def export(identifier: str, source: Path, output: Path, *, revision: str | None = None) -> Path: """Copy only declared inputs after verification; never overwrite a run or input tree.""" verify_inputs(identifier, source) if output.exists() or output.is_symlink(): @@ -172,6 +355,14 @@ def export(identifier: str, source: Path, output: Path) -> Path: target.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(contained_path(source, relative, reject_symlinks=True), target) verify_inputs(identifier, stage) + if revision is not None: + dataset_path = stage / manifest['dataset'] + revised, review = _apply_annotation_revision(identifier, read(dataset_path), revision) + review['reference_controls'] = check_revision_references(revised, review, stage) + if any(item['status'] == 'failed' for item in review['reference_controls']['cases']): + raise ValueError('Revised contract rejects its frozen reference or accepts an error page') + (stage/'annotation-review.json').write_text(json.dumps(review, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + dataset_path.write_text(json.dumps(revised, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') # A materialized dataset is intentionally independent of the historical # Browser Eval external-input resolver; do not copy source-inputs.json. os.rename(stage, output) diff --git a/src/lexbench_webfetch/cli.py b/src/lexbench_webfetch/cli.py index f8aabd7..8ff85e2 100644 --- a/src/lexbench_webfetch/cli.py +++ b/src/lexbench_webfetch/cli.py @@ -3,7 +3,7 @@ import json from pathlib import Path -from .benchmark import export, registry, validate, verify_inputs +from .benchmark import export, inspect_annotation_revision, registry, validate, verify_inputs def main() -> None: @@ -18,6 +18,10 @@ def main() -> None: command.add_argument("--source", type=Path, required=True) if name == "export": command.add_argument("--output", type=Path, required=True) + command.add_argument("--revision", help="Apply a registered score-only annotation version") + inspect = sub.add_parser("inspect-revision", help="Show reviewed field differences, sources and acceptance controls") + inspect.add_argument("--suite", required=True) + inspect.add_argument("--revision", required=True) score = sub.add_parser("score", help="Evaluate a saved response and delivered files") score.add_argument("--task", type=Path, required=True, help="Complete task JSON including its frozen reference requirements") @@ -39,7 +43,9 @@ def main() -> None: articles.add_argument("--evidence-home", type=Path, required=True) args = parser.parse_args() try: - if args.command == "score": + if args.command == "inspect-revision": + result = inspect_annotation_revision(args.suite, args.revision) + elif args.command == "score": from .evaluation import evaluate_response metadata = json.loads(args.metadata.read_text(encoding="utf-8")) if args.metadata else None if args.metadata and not isinstance(metadata, dict): @@ -74,7 +80,7 @@ def main() -> None: elif args.command == "verify-inputs": result = verify_inputs(args.suite, args.source) else: - result = {"dataset": str(export(args.suite, args.source, args.output).resolve())} + result = {"dataset": str(export(args.suite, args.source, args.output, revision=args.revision).resolve())} except (ValueError, KeyError, OSError) as error: parser.exit(1, f"{error}\n") print(json.dumps(result, ensure_ascii=False, indent=2)) diff --git a/src/lexbench_webfetch/composite_tasks.py b/src/lexbench_webfetch/composite_tasks.py index 84ca463..c347193 100644 --- a/src/lexbench_webfetch/composite_tasks.py +++ b/src/lexbench_webfetch/composite_tasks.py @@ -154,6 +154,8 @@ def _validate_group(group: dict) -> None: "link_base_url", "record_name_exclusions", "record_line_pattern", + "record_start_pattern", + "markdown_label_pattern", "start_pattern", "end_pattern", "required_fields", @@ -180,11 +182,13 @@ def _validate_group(group: dict) -> None: } ): raise ValueError("Invalid composite record group") - for key in ("start_pattern", "end_pattern", "record_line_pattern"): + for key in ("start_pattern", "end_pattern", "record_line_pattern", "record_start_pattern", "markdown_label_pattern"): if key in group: if not isinstance(group[key], str) or not group[key]: raise ValueError("Invalid composite scope pattern") - re.compile(group[key]) + compiled = re.compile(group[key]) + if key == "markdown_label_pattern" and "label" not in compiled.groupindex: + raise ValueError("Markdown card label pattern requires a label group") kind = group["kind"] if "combined_columns" in group and kind != "table": raise ValueError("Combined columns require a table group") @@ -433,7 +437,22 @@ def _card_section_context(content: str, begin: int, end: int, title_start: int, peer_boundary = (peer_item and ( (item_indent is not None and peer_indent <= item_indent) or (item_indent is None and own_heading is None and peer_indent == 0))) - if (peer_boundary and not owned_item and absolute > title_start + field_item = False + if peer_item and not markdown_links(line): + # Native cards may serialize their typed fields as sibling + # bullets. A complete declared value belongs to this card; + # arbitrary peer prose still starts a new section. + values = _context_lines(line, "") + for spec in group.get("required_fields", []): + if not (spec.get("aliases") or _field_token_pattern(spec)): + continue + found = _field_value(values, spec) + if found: + number, _, start, end = found + if not (values[number][:start] + values[number][end:]).strip(): + field_item = True + break + if (peer_boundary and not owned_item and not field_item and absolute > title_start and not any(link.start <= absolute < link.end for link in links)): return context[:offset] owned_heading = heading and any( @@ -679,6 +698,12 @@ def previous_scalar(link): return previous_start, scalar def visible_label(link): + if pattern := group.get("markdown_label_pattern"): + surface = link.label + for image in reversed([item for item in markdown_links(surface) if item.is_image]): + surface = surface[:image.start] + surface[image.end:] + if match := re.search(pattern, visible_markdown_links(surface)): + return match["label"].strip() card_fields = serialized_link_card_fields(link) label = card_fields[0] if card_fields else link.label for image in reversed([item for item in markdown_links(label) if item.is_image]): @@ -791,6 +816,16 @@ def owner_for(position): if index and ordered[index - 1][6] > line_start: line_start = start next_line = max(next_line, link_end) + if not owner and group.get("record_start_pattern"): + starts = list(re.finditer(group["record_start_pattern"], content[:link_start], re.M)) + if starts: + marker = starts[-1].start() + prior_end = ordered[index - 1][6] if index else 0 + if marker >= prior_end: + line_start = marker + following = re.search(group["record_start_pattern"], content[link_end:], re.M) + if following: + next_line = min(next_line, link_end + following.start()) if owner: begin, end = owner # The first link can have fields before it in the same card. @@ -912,7 +947,7 @@ def serialized_link_card_fields(link): field = re.sub(r"\\+$", "", raw.strip()).strip() heading = re.match(r"^#{1,6}[ \t]+\S", field) field = re.sub(r"^#{1,6}(?:[ \t]+|$)", "", field).strip(" *_") - if field: + if field and not re.fullmatch(r"[-+*_ \t]+", field): if not fields: explicit_name = heading is not None fields.append(field) @@ -944,7 +979,9 @@ def _context_lines(context: str, title: str) -> list[str]: for image in reversed(markdown_links(label)): if image.is_image: label = label[:image.start] + " " + label[image.end:] - if _key(visible_markdown_links(label)) != title_key: + visible_label = visible_markdown_links(label) + if (_key(visible_label) != title_key + and title_key not in {_key(line) for line in visible_label.splitlines()}): continue header = re.match(r"[ \t]*(?P\*\*|__)(?P[^\n]+?)(?P=mark)(?:\\)*[ \t]*\n", label) if not header: @@ -1012,6 +1049,8 @@ def _field_token_pattern(spec: dict): """Typed card fields own a distinct value span, not an entire layout line.""" if spec.get('aliases'): return None + if spec['kind'] == 'text' and spec.get('pattern'): + return re.compile(spec['pattern'], re.I) if spec['kind'] == 'integer' and spec.get('pattern'): # A number elsewhere in the card cannot prove a numeric labeled field. return re.compile(rf"\b{_AMOUNT}\s*(?:{spec['pattern']})|(?:{spec['pattern']})", re.I) @@ -1021,6 +1060,10 @@ def _field_token_pattern(spec: dict): def _text_role_value(value: str, spec: dict) -> str | None: """Explicit metadata belongs to its declared role, not another text field.""" + value = value.strip() + if (re.fullmatch(r"today|tomorrow|yesterday", value, re.I) + and spec["id"] not in {"date", "date_or_time", "time", "status"}): + return None roles = {"title", "name", "author", "creator", "owner", "source", "publisher", "description", "summary", "date", "time", "version", "downloads", "views", "likes", "count", "price", "rating", "size"} @@ -1124,7 +1167,14 @@ def _field_value( continue if token_pattern := _field_token_pattern(spec): for match in token_pattern.finditer(value): - if _value_matches_spec(match[0], spec): + if spec["kind"] == "text" and ( + match.start() and value[match.start() - 1].isalnum() and match[0][:1].isalnum() + or match.end() < len(value) and value[match.end()].isalnum() and match[0][-1:].isalnum() + ): + continue + checked = {key: value for key, value in spec.items() + if key != "pattern"} if spec["kind"] == "text" else spec + if _value_matches_spec(match[0], checked): return index, match[0], start + match.start(), start + match.end() continue checked_spec = {**spec, "minimum_chars": 1} if aliases and spec["kind"] == "text" and "minimum_chars" not in spec else spec @@ -1176,20 +1226,6 @@ def _fixed_field_value_key(value: str, spec: dict) -> str: def _required_fields(context: str, title: str, *, field_specs: list[dict]) -> bool: """Require every declared field inside the record block which owns the link.""" lines = _labeled_field_lines(_context_lines(context, title), field_specs) - separated = [] - patterns = [re.compile(spec["pattern"], re.I) for spec in field_specs - if spec["kind"] == "text" and spec.get("pattern") and not spec.get("aliases")] - for line in lines: - for pattern in patterns: - match = pattern.match(line) - if match and match.end() and (match.end() == len(line) or not line[match.end()].isalnum()): - rest = line[match.end():].strip(" |\t") - if rest: - separated.extend((match[0], rest)) - break - else: - separated.append(line) - lines = separated used: set[int] = set() typed_lines: set[int] = set() declared = _declared_aliases(field_specs) @@ -1208,7 +1244,8 @@ def _required_fields(context: str, title: str, *, field_specs: list[dict]) -> bo # physical line, but one value cannot satisfy two required fields. lines[index] = lines[index][:start] + " " * (end - start) + lines[index][end:] if typed: - typed_lines.add(index) + if spec["kind"] != "text": + typed_lines.add(index) elif not spec.get("aliases"): used.add(index) return True diff --git a/src/lexbench_webfetch/definitions/annotation-revisions/reviews.json b/src/lexbench_webfetch/definitions/annotation-revisions/reviews.json new file mode 100644 index 0000000..cec227e --- /dev/null +++ b/src/lexbench_webfetch/definitions/annotation-revisions/reviews.json @@ -0,0 +1,242 @@ +{ + "schema_version": 1, + "revisions": { + "serialization-20261021": { + "revision_sha256": "e0fe42179e584abb73782cbc459f521cab9cb09d3442de1ec4a6ff54d1b1235a", + "changes": [ + { + "suite": "webfetch", + "case_id": "stable-browser-use-readme", + "reason": "The frozen README discusses CAPTCHA as a feature; that source term must not reject otherwise complete content. Missing source content still fails.", + "changes_sha256": "0d6d2572e0a1fbd6e5900689f32c36041c4460fb60a67c99a55793ec18b57ddb", + "sources": [ + { + "path": "dataset.json", + "sha256": "5f751146143dcbe9daf46f0fb2511da1147508b3c8c4cd3377a6f3cceb82264e", + "scope": "Frozen task and independently captured reference inputs for stable-browser-use-readme" + }, + { + "path": "sources/stable-browser-use-readme/current-source.html.txt", + "sha256": "0b77929bbdff3b79a3e41cad10e005a834855787945d3ced295bc81fda1561d4", + "scope": "Frozen source/reference for https://github.com/browser-use/browser-use/blob/2b1f9d377999a59fe7627c1a5aa88c12aa42e11f/README.md" + }, + { + "path": "sources/stable-browser-use-readme/reference.txt", + "sha256": "e0125e5233473fcf1266c1bdfb5fb3d5e3545ae027b5f954dcceb3d874e50765", + "scope": "Frozen source/reference for https://github.com/browser-use/browser-use/blob/2b1f9d377999a59fe7627c1a5aa88c12aa42e11f/README.md" + } + ], + "regression_ids": [ + "source-backed-diagnostic-source-discusses-status", + "source-backed-diagnostic-status-heading" + ] + }, + { + "suite": "webfetch", + "case_id": "stable-wcag21-recommendation", + "reason": "The recommendation discusses CAPTCHA accessibility. Keep that source term diagnostic while retaining its independent content requirements.", + "changes_sha256": "0d6d2572e0a1fbd6e5900689f32c36041c4460fb60a67c99a55793ec18b57ddb", + "sources": [ + { + "path": "dataset.json", + "sha256": "5f751146143dcbe9daf46f0fb2511da1147508b3c8c4cd3377a6f3cceb82264e", + "scope": "Frozen task and independently captured reference inputs for stable-wcag21-recommendation" + }, + { + "path": "sources/stable-wcag21-recommendation/current-source.html.txt", + "sha256": "a8793e95937866bacd03d9b421e4e889bc7811b84d0ddec6ed8c790f8e7f67dd", + "scope": "Frozen source/reference for https://www.w3.org/TR/2018/REC-WCAG21-20180605/" + }, + { + "path": "sources/stable-wcag21-recommendation/reference.txt", + "sha256": "4e427c638b8372ae21dec876596004451c1bfe43a23ac4331fc3ce8e6f1a1a5c", + "scope": "Frozen source/reference for https://www.w3.org/TR/2018/REC-WCAG21-20180605/" + } + ], + "regression_ids": [ + "source-backed-diagnostic-source-discusses-status", + "source-backed-diagnostic-status-heading" + ] + }, + { + "suite": "webfetch", + "case_id": "live-030-rfc-9110", + "reason": "RFC 9110 defines 404 Not Found. Its occurrence in the declared source is content, while a standalone error page lacks the required specification text.", + "changes_sha256": "16deb54070e7a3124c1a27b3e202579e33f0cd6e7860bb320b05cd7e2ff445c0", + "sources": [ + { + "path": "dataset.json", + "sha256": "5f751146143dcbe9daf46f0fb2511da1147508b3c8c4cd3377a6f3cceb82264e", + "scope": "Frozen task and independently captured reference inputs for live-030-rfc-9110" + }, + { + "path": "sources/live-030-rfc-9110/current-source.html.txt", + "sha256": "d431760660ea44e130f6e919dab216df2d0b3a490567a98089267523368fe1e5", + "scope": "Frozen source/reference for https://www.rfc-editor.org/rfc/rfc9110.html" + }, + { + "path": "sources/live-030-rfc-9110/reference.txt", + "sha256": "f8ca256a4b0c0c0dce03f544daf37930fdcd78ef90fd28b506a3b7684f9f5df2", + "scope": "Frozen source/reference for https://www.rfc-editor.org/rfc/rfc9110.html" + }, + { + "path": "sources/live-030-rfc-9110/source.html.txt", + "sha256": "d431760660ea44e130f6e919dab216df2d0b3a490567a98089267523368fe1e5", + "scope": "Frozen source/reference for https://www.rfc-editor.org/rfc/rfc9110.html" + } + ], + "regression_ids": [ + "source-backed-diagnostic-source-discusses-status", + "source-backed-diagnostic-status-heading" + ] + }, + { + "suite": "webfetch", + "case_id": "image-04", + "reason": "The declared image asset and dimensions are unchanged. Accept serialized equivalents of its explicit frozen URL aliases, not a different image size.", + "changes_sha256": "8b5f8ce6244bbf32574c1a30eaf58f3b97c13f0e9207a3f676a418f356b5128c", + "sources": [ + { + "path": "dataset.json", + "sha256": "5f751146143dcbe9daf46f0fb2511da1147508b3c8c4cd3377a6f3cceb82264e", + "scope": "Frozen task and independently captured reference inputs for image-04" + }, + { + "path": "sources/image-04/source.html.txt", + "sha256": "a88a957464642829b46cd764ab99a8baac990cf200eed516c88282de3310405a", + "scope": "Frozen source/reference for https://en.wikipedia.org/wiki/Apollo_11" + } + ], + "regression_ids": [ + "resource-serialization-markup-entity", + "resource-serialization-different-size" + ] + }, + { + "suite": "webfetch", + "case_id": "image-unsplash-detail", + "reason": "Accept documented presentation parameters for the same Unsplash photo. Asset identity and unsupported overlay transformations remain restricted.", + "changes_sha256": "6ed1819e40083d373591d289e8c32a9e27b577bcc6207fe603c2270770068fff", + "sources": [ + { + "path": "dataset.json", + "sha256": "5f751146143dcbe9daf46f0fb2511da1147508b3c8c4cd3377a6f3cceb82264e", + "scope": "Frozen task and independently captured reference inputs for image-unsplash-detail" + }, + { + "path": "sources/image-unsplash-detail/browser-observation.json", + "sha256": "1d4824d1f09b8deeef0f846b07d95fefa74ae15ed6a2d70e563d064ae531f84b", + "scope": "Frozen source/reference for https://unsplash.com/photos/green-leaf-plants-mSK5nNsAsLY" + } + ], + "regression_ids": [ + "declared-photo-query-unsplash-resize", + "declared-photo-query-unsplash-overlay", + "declared-photo-query-unsplash-other-asset" + ] + }, + { + "suite": "webfetch", + "case_id": "image-pexels-detail", + "reason": "Accept documented presentation parameters for the same Pexels photo. A different photo or unsupported overlay remains invalid.", + "changes_sha256": "2145ee63ff8a5fcdfa5bc8705743bc0fd29b1e0c5b840367f0b543ae0a280de7", + "sources": [ + { + "path": "dataset.json", + "sha256": "5f751146143dcbe9daf46f0fb2511da1147508b3c8c4cd3377a6f3cceb82264e", + "scope": "Frozen task and independently captured reference inputs for image-pexels-detail" + }, + { + "path": "sources/image-pexels-detail/browser-observation.json", + "sha256": "5a74a7ad4d78609a9163ce6addc3f4542d287b8e09c88a5cb8cdf22a01bc178d", + "scope": "Frozen source/reference for https://www.pexels.com/photo/a-green-trees-in-the-forest-12377231/" + } + ], + "regression_ids": [ + "declared-photo-query-pexels-resize", + "declared-photo-query-pexels-overlay", + "declared-photo-query-pexels-other-asset" + ] + }, + { + "suite": "webfetch", + "case_id": "video-09", + "reason": "The frozen task requests video title and publisher. Its actual title field may supply the title; keywords cannot supply the publisher.", + "changes_sha256": "300626be546a061612fc8ce029a0b7ea33443996df7c0919c298f541d06bbd23", + "sources": [ + { + "path": "dataset.json", + "sha256": "5f751146143dcbe9daf46f0fb2511da1147508b3c8c4cd3377a6f3cceb82264e", + "scope": "Frozen task and independently captured reference inputs for video-09" + }, + { + "path": "sources/video-09/source.html.txt", + "sha256": "000cd8ff3290a2c5f2446cf339c7dea1afb4faedbfa57f8467eaaf2103c6f380", + "scope": "Frozen source/reference for https://www.youtube.com/watch?v=jNQXAC9IVRw" + } + ], + "regression_ids": [ + "declared-information-explicit-title", + "declared-information-keywords-not-title", + "declared-information-keywords-not-publisher" + ] + }, + { + "suite": "dynamic-content", + "case_id": "replacement-ja-nintendo-mario", + "reason": "The frozen task selects Mario software results. The next independent page-search section ends that scope; it cannot contribute records. Keep seven required records, hosts and detail paths unchanged.", + "changes_sha256": "77f55cf810ebf779775ea9e31bbdb682ee14854fc0d471e229c5a4af1f510aa6", + "sources": [ + { + "path": "dataset.json", + "sha256": "0da2bb30aaabe3ca2b93ec17b4cb88cdc1cf3ea46336bb3777fdb21c1f211b84", + "scope": "Frozen task contract and independent observation hashes for replacement-ja-nintendo-mario; the original page bytes are not retained in this input bundle." + } + ], + "regression_ids": [ + "named-section-boundary-complete-software", + "named-section-boundary-next-section-not-software" + ] + }, + { + "suite": "dynamic-content", + "case_id": "replacement-ja-nintendo-zelda", + "reason": "The frozen task selects Zelda software results. The next independent page-search section ends that scope; it cannot contribute records. Keep eight required records, hosts and detail paths unchanged.", + "changes_sha256": "77f55cf810ebf779775ea9e31bbdb682ee14854fc0d471e229c5a4af1f510aa6", + "sources": [ + { + "path": "dataset.json", + "sha256": "0da2bb30aaabe3ca2b93ec17b4cb88cdc1cf3ea46336bb3777fdb21c1f211b84", + "scope": "Frozen task contract and independent observation hashes for replacement-ja-nintendo-zelda; the original page bytes are not retained in this input bundle." + } + ], + "regression_ids": [ + "named-section-boundary-complete-software", + "named-section-boundary-next-section-not-software" + ] + }, + { + "suite": "access-restrictions", + "case_id": "risk-diverse-074", + "reason": "Keep 26 named event links with date, venue and region. Their fields may share a visible card or line; fields from another card cannot complete an event.", + "changes_sha256": "dce4c44e8a5a5867ebdebf6a04fc1f954a00b40875cd1e975a6fb36a12b1d25f", + "sources": [ + { + "path": "dataset.json", + "sha256": "165b888133e859304b035d74183dd40bf7ab609cac78bb6ae6e75b645c1de426", + "scope": "Frozen task contract and independent observation hashes for risk-diverse-074; the original page bytes are not retained in this input bundle." + } + ], + "regression_ids": [ + "declared-whole-card-complete", + "declared-whole-card-no-name", + "declared-whole-card-no-venue", + "declared-whole-card-no-region", + "native-card-text-same-line-region", + "native-card-text-region-without-venue" + ] + } + ] + } + } +} diff --git a/src/lexbench_webfetch/definitions/annotation-revisions/serialization-20261021.json b/src/lexbench_webfetch/definitions/annotation-revisions/serialization-20261021.json new file mode 100644 index 0000000..9a8d04b --- /dev/null +++ b/src/lexbench_webfetch/definitions/annotation-revisions/serialization-20261021.json @@ -0,0 +1,360 @@ +{ + "schema_version": 1, + "kind": "score_only_annotation_revision", + "id": "serialization-20261021", + "reason": "Preserve source terms, declared resource identities, native record boundaries and information-title scope without changing requests or collected outputs.", + "parents": { + "webfetch": { + "tasks_sha256": "ae3aec6eab8d2d470cdeb18c4f80d238e5dd79e1108a729589b77a80fe869405", + "source_inputs_sha256": "c202abc4624811b938a837402bbaf0ecbe716e375fbca70567346d2edb3b1708" + }, + "dynamic-content": { + "tasks_sha256": "139c62a141183d305d8fa9c3658c546927731461d86aeba5ad3091649bd4ecd7", + "source_inputs_sha256": "710f9c91705b29bb9e4aa15af97dcc856e64e406baa4b9966446408d7e3d26a8" + }, + "access-restrictions": { + "tasks_sha256": "c67b60ccf5ee0b14e4e23b3efc282e625005f8e096f0bfc978772d312e8f50c2", + "source_inputs_sha256": "c61d6825abbff2a92098592f83055347cd0b0101374794023046c0fd0323dd5f" + } + }, + "regression_ids": [ + "native-serialization-complete-notice", + "native-serialization-changed-notice", + "native-serialization-stock-label", + "native-serialization-stock-word-fragment", + "native-serialization-carousel-label", + "native-serialization-carousel-digit-word", + "native-serialization-cyrillic-label", + "native-serialization-cyrillic-word-fragment", + "native-serialization-punctuation-markup", + "resource-serialization-markup-entity", + "resource-serialization-different-size", + "named-scope-heading-owns-records", + "named-scope-navigation-cannot-supply-records", + "named-scope-empty-layout-is-not-name", + "named-scope-image-without-visible-name", + "native-card-bullet-complete", + "native-card-bullet-missing-area", + "native-card-bullet-other-section", + "native-card-text-same-line-region", + "native-card-text-region-without-venue", + "native-card-text-no-region", + "source-backed-diagnostic-source-discusses-status", + "source-backed-diagnostic-status-heading", + "declared-whole-card-complete", + "declared-whole-card-no-name", + "declared-whole-card-no-venue", + "declared-whole-card-no-region", + "declared-card-prelude-owned", + "declared-card-prelude-separate-card", + "declared-card-prelude-next-card-fields", + "declared-information-explicit-title", + "declared-information-keywords-not-title", + "declared-information-keywords-not-publisher", + "named-section-boundary-complete-software", + "named-section-boundary-next-section-not-software", + "declared-photo-query-unsplash-resize", + "declared-photo-query-unsplash-overlay", + "declared-photo-query-unsplash-other-asset", + "declared-photo-query-pexels-resize", + "declared-photo-query-pexels-overlay", + "declared-photo-query-pexels-other-asset" + ], + "changes": [ + { + "suite": "webfetch", + "case_id": "stable-browser-use-readme", + "before_sha256": "05cfc4d5ac1bbf1189cdc81459601dca7ef8a45a6d886a25eb07ebe409c8968b", + "set": { + "forbidden_any": [ + "Access denied", + "Attention Required! | Cloudflare", + "Page not found", + "404 Not Found", + "无法访问此站点" + ], + "diagnostic_any": [ + "CAPTCHA" + ] + }, + "remove": [] + }, + { + "suite": "webfetch", + "case_id": "stable-wcag21-recommendation", + "before_sha256": "c980470f54a9b4ebb4d91bb3d734df8647c0b234c926cc380edb31919b7dea63", + "set": { + "forbidden_any": [ + "Access denied", + "Attention Required! | Cloudflare", + "Page not found", + "404 Not Found", + "无法访问此站点" + ], + "diagnostic_any": [ + "CAPTCHA" + ] + }, + "remove": [] + }, + { + "suite": "webfetch", + "case_id": "live-030-rfc-9110", + "before_sha256": "58b5a6368da964807b43b3743ac37b91bf09bf678c7afb7553339128cfb742ca", + "set": { + "forbidden_any": [ + "Access denied", + "Attention Required! | Cloudflare", + "CAPTCHA", + "Page not found", + "无法访问此站点" + ], + "diagnostic_any": [ + "404 Not Found" + ] + }, + "remove": [] + }, + { + "suite": "webfetch", + "case_id": "image-04", + "before_sha256": "604ccebd58a7ec60175da2f6c509ababf5a6be3ad1fcc9b258bf39114b304001", + "set": { + "expected_resources": [ + { + "id": "image-1", + "kind": "image-1", + "urls": [ + "https://upload.wikimedia.org/wikipedia/commons/thumb/4/41/A_Man_on_the_Moon%2C_AS11-40-5903_%28cropped%29.jpg/330px-A_Man_on_the_Moon%2C_AS11-40-5903_%28cropped%29.jpg?utm_source=en.wikipedia.org&utm_campaign=parser&utm_content=thumbnail" + ], + "url_patterns": [ + "^(?:https://upload\\.wikimedia\\.org/wikipedia/commons/4/41/A_Man_on_the_Moon,_AS11\\-40\\-5903_\\(cropped\\)\\.jpg|https://(?:upload|thumb)\\.wikimedia\\.org/wikipedia/commons/thumb/4/41/A_Man_on_the_Moon,_AS11\\-40\\-5903_\\(cropped\\)\\.jpg/[1-9][0-9]*px-A_Man_on_the_Moon,_AS11\\-40\\-5903_\\(cropped\\)\\.jpg)(?:\\?(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+)(?:&(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+))*)?$" + ] + }, + { + "id": "image-2", + "kind": "image-2", + "urls": [ + "https://upload.wikimedia.org/wikipedia/commons/thumb/2/27/Apollo_11_insignia.png/250px-Apollo_11_insignia.png?utm_source=en.wikipedia.org&utm_campaign=parser&utm_content=thumbnail" + ], + "url_patterns": [ + "^(?:https://upload\\.wikimedia\\.org/wikipedia/commons/2/27/Apollo_11_insignia\\.png|https://(?:upload|thumb)\\.wikimedia\\.org/wikipedia/commons/thumb/2/27/Apollo_11_insignia\\.png/[1-9][0-9]*px-Apollo_11_insignia\\.png)(?:\\?(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+)(?:&(?:utm_source=(?:en\\.wikipedia\\.org|commons\\.wikimedia\\.org)|utm_campaign=(?:parser|index)|utm_content=thumbnail|_=[0-9]+))*)?$" + ] + } + ] + }, + "remove": [] + }, + { + "suite": "webfetch", + "case_id": "image-unsplash-detail", + "before_sha256": "5d219be254b0145cd19f833958e8e6826b2540fc3eed9d92c541b2a47f634354", + "set": { + "expected_resources": [ + { + "id": "main-image", + "kind": "image", + "urls": [ + "https://images.unsplash.com/photo-1493673272479-a20888bcee10", + "https://unsplash.com/photos/mSK5nNsAsLY/download" + ], + "url_patterns": [ + "^https://images\\.unsplash\\.com/photo\\-1493673272479\\-a20888bcee10(?:\\?(?:(?:w|h)=[1-9][0-9]*|q=(?:100|[1-9][0-9]?)|dpr=[1-9][0-9]*(?:\\.[0-9]+)?|fm=(?:jpg|jpeg|png|webp|avif|gif)|fit=(?:crop|clip|fill|fillmax|max|min|scale)|crop=(?:faces|edges|entropy|top|bottom|left|right|focalpoint)|cs=(?:srgb|tinysrgb)|auto=format|ixlib=[A-Za-z0-9._-]+|ixid=[A-Za-z0-9_+/=%-]+)(?:&(?:(?:w|h)=[1-9][0-9]*|q=(?:100|[1-9][0-9]?)|dpr=[1-9][0-9]*(?:\\.[0-9]+)?|fm=(?:jpg|jpeg|png|webp|avif|gif)|fit=(?:crop|clip|fill|fillmax|max|min|scale)|crop=(?:faces|edges|entropy|top|bottom|left|right|focalpoint)|cs=(?:srgb|tinysrgb)|auto=format|ixlib=[A-Za-z0-9._-]+|ixid=[A-Za-z0-9_+/=%-]+))*)?$" + ] + } + ] + }, + "remove": [] + }, + { + "suite": "webfetch", + "case_id": "image-pexels-detail", + "before_sha256": "a3e0ca87dcadb509a7e3c23371eac30cb10adfb5c00e96774d3b6369ef024651", + "set": { + "expected_resources": [ + { + "id": "main-image", + "kind": "image", + "urls": [ + "https://images.pexels.com/photos/12377231/pexels-photo-12377231.jpeg" + ], + "url_patterns": [ + "^https://images\\.pexels\\.com/photos/12377231/pexels\\-photo\\-12377231\\.jpeg(?:\\?(?:(?:w|h)=[1-9][0-9]*|q=(?:100|[1-9][0-9]?)|dpr=[1-9][0-9]*(?:\\.[0-9]+)?|fm=(?:jpg|jpeg|png|webp|avif|gif)|fit=(?:crop|clip|fill|fillmax|max|min|scale)|crop=(?:faces|edges|entropy|top|bottom|left|right|focalpoint)|cs=(?:srgb|tinysrgb)|auto=(?:compress|format|compress,format)|dl=[A-Za-z0-9._-]+)(?:&(?:(?:w|h)=[1-9][0-9]*|q=(?:100|[1-9][0-9]?)|dpr=[1-9][0-9]*(?:\\.[0-9]+)?|fm=(?:jpg|jpeg|png|webp|avif|gif)|fit=(?:crop|clip|fill|fillmax|max|min|scale)|crop=(?:faces|edges|entropy|top|bottom|left|right|focalpoint)|cs=(?:srgb|tinysrgb)|auto=(?:compress|format|compress,format)|dl=[A-Za-z0-9._-]+))*)?$" + ] + } + ] + }, + "remove": [] + }, + { + "suite": "webfetch", + "case_id": "video-09", + "before_sha256": "78f981fb66c5c47e4b90f57a6f6dec2c1a5b284374f706dde0958dbcabfa11a9", + "set": { + "required_facts": [ + { + "id": "视频标题", + "alternatives": [ + "Me at the zoo" + ], + "match_scope": "body_or_title" + }, + { + "id": "发布者", + "alternatives": [ + "jawed" + ] + } + ] + }, + "remove": [] + }, + { + "suite": "dynamic-content", + "case_id": "replacement-ja-nintendo-mario", + "before_sha256": "c93d075172fc993788626460f9786a41ddff9f1f5955146cfb463e4bd8e21c94", + "set": { + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 7, + "hosts": [ + "www.nintendo.com", + "www.nintendo.co.jp", + "store-jp.nintendo.com" + ], + "entry_path": "/(?:jp/(?:games/(?:switch2|zelda)/[^?#]+|switch/[^?#]+|my-mario/app/index\\.html)|item/software/D[0-9]+|zelda/[^?#]+|character/zelda/[^?#]+|switch/[^?#]+)", + "stable_text_anchors": [ + { + "id": "rule25-740e4325c2c2", + "alternatives": [ + "ゲームソフトをさがす" + ], + "match_scope": "non_record_lines" + }, + { + "id": "rule25-112f4528726b", + "alternatives": [ + "ページをさがす" + ], + "match_scope": "non_record_lines" + }, + { + "id": "rule25-0e3206dde854", + "alternatives": [ + "すべてのページ" + ], + "match_scope": "non_record_lines" + } + ], + "entry_scope": { + "html_selector": ".nc3-as-result--soft:not(.is-loading) a[href]", + "start_pattern": "^ゲームソフトをさがす\\s*$", + "end_pattern": "^(?:すべて見る|ページをさがす)" + } + } + }, + "remove": [] + }, + { + "suite": "dynamic-content", + "case_id": "replacement-ja-nintendo-zelda", + "before_sha256": "6170d2206529d52ed0f84702e3fc444dee90086152c391d6b6da519a3d3408c0", + "set": { + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 8, + "hosts": [ + "www.nintendo.com", + "www.nintendo.co.jp", + "store-jp.nintendo.com" + ], + "entry_path": "/(?:jp/(?:games/(?:switch2|zelda)/[^?#]+|switch/[^?#]+|my-mario/app/index\\.html)|item/software/D[0-9]+|zelda/[^?#]+|character/zelda/[^?#]+|switch/[^?#]+)", + "stable_text_anchors": [ + { + "id": "rule25-04f69a7c4127", + "alternatives": [ + "ゲームソフトをさがす" + ], + "match_scope": "non_record_lines" + }, + { + "id": "rule25-744c35c56ada", + "alternatives": [ + "ページをさがす" + ], + "match_scope": "non_record_lines" + }, + { + "id": "rule25-36f85bd9f532", + "alternatives": [ + "すべてのページ" + ], + "match_scope": "non_record_lines" + } + ], + "entry_scope": { + "html_selector": ".nc3-as-result--soft:not(.is-loading) a[href]", + "start_pattern": "^ゲームソフトをさがす\\s*$", + "end_pattern": "^(?:すべて見る|ページをさがす)" + } + } + }, + "remove": [] + }, + { + "suite": "access-restrictions", + "case_id": "risk-diverse-074", + "before_sha256": "45c0608f91f9666378ba036ac877aa34998d9b71803e89d0c7a0c8c49ceff29e", + "set": { + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "concerts", + "kind": "named_link", + "minimum_records": 26, + "hosts": [ + "www.ticketmaster.com", + "www.ticketweb.com" + ], + "entry_path": "/.*event/[A-Za-z0-9_-]+", + "record_name_exclusions": [ + "BLUE FRIDAY SPECIAL! Entry to All Shows at the Blue Moon for 2026!", + "I'm On the List! + 1 (Entry to all shows at the Blue Moon in 2026...)" + ], + "required_fields": [ + { + "id": "date_or_time", + "kind": "date_or_time" + }, + { + "id": "venue", + "kind": "text", + "minimum_chars": 2 + }, + { + "id": "city_or_region", + "kind": "text", + "minimum_chars": 2, + "pattern": ",\\s*(?:AL|AK|AZ|AR|CA|CO|CT|DE|DC|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY)\\b" + } + ], + "markdown_label_pattern": "(?m)^(?:(?:Today|Tomorrow)[ \\t]*\\n)?(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday),[^\\n]*\\b20[0-9]{2}\\b[^\\n]*\\n(?P<label>[^\\n]+)\\n", + "record_start_pattern": "(?m)^[-*+][ \\t]+(?:January|February|March|April|May|June|July|August|September|October|November|December|Multiple Dates)\\b" + } + ] + } + }, + "remove": [] + } + ], + "authority": { + "source_contract": "The tasks request a named image asset, without an exact display size. Only documented resizing, encoding, download and attribution parameters are aliases; image composition and unknown query keys are excluded.", + "references": [ + "https://unsplash.com/documentation#dynamically-resizable-images", + "https://www.pexels.com/api/documentation/" + ] + } +} diff --git a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json index 857f310..fb1d2cc 100644 --- a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json +++ b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json @@ -54198,6 +54198,2859 @@ "ok": true, "expected_task_pass": true }, + { + "id": "body-only-word-boundary-answer-title-supplies-required-fact", + "category": "answer-surface", + "root_cause": "metadata_title_supplies_missing_body_fact", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/page", + "required_facts": [ + { + "id": "title", + "alternatives": [ + "Exact page title" + ] + } + ] + }, + "artifact": "Body", + "title": "Exact page title", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "fact_coverage": 0.0, + "content_chars": 4 + } + }, + { + "id": "body-only-word-boundary-answer-title-does-not-inflate-body-length", + "category": "answer-surface", + "root_cause": "answer_surface_omission", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/page", + "required_facts": [ + { + "id": "title", + "alternatives": [ + "Exact page title" + ] + } + ], + "min_content_chars": 20 + }, + "artifact": "Body", + "title": "Exact page title", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "fact_coverage": 0.0, + "length_pass": false, + "content_chars": 4 + } + }, + { + "id": "latin-boundary-fact-Home-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "Home" + ] + } + ] + }, + "artifact": "(Home)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-fact-Home-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "Home" + ] + } + ] + }, + "artifact": "homepage", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-marker-Home-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [ + "Home" + ] + }, + "artifact": "(Home)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-marker-Home-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [ + "Home" + ] + }, + "artifact": "homepage", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-anchor-Home-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "answer", + "alternatives": [ + "Home" + ] + } + ] + } + }, + "artifact": "(Home)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-anchor-Home-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "answer", + "alternatives": [ + "Home" + ] + } + ] + } + }, + "artifact": "homepage", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-fact-Live-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "Live" + ] + } + ] + }, + "artifact": "(Live)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-fact-Live-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "Live" + ] + } + ] + }, + "artifact": "deliver", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-marker-Live-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [ + "Live" + ] + }, + "artifact": "(Live)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-marker-Live-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [ + "Live" + ] + }, + "artifact": "deliver", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-anchor-Live-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "answer", + "alternatives": [ + "Live" + ] + } + ] + } + }, + "artifact": "(Live)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-anchor-Live-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "answer", + "alternatives": [ + "Live" + ] + } + ] + } + }, + "artifact": "deliver", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-fact-News-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "News" + ] + } + ] + }, + "artifact": "(News)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-fact-News-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "News" + ] + } + ] + }, + "artifact": "newsletter", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-marker-News-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [ + "News" + ] + }, + "artifact": "(News)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-marker-News-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [ + "News" + ] + }, + "artifact": "newsletter", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-anchor-News-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "answer", + "alternatives": [ + "News" + ] + } + ] + } + }, + "artifact": "(News)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-anchor-News-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "answer", + "alternatives": [ + "News" + ] + } + ] + } + }, + "artifact": "newsletter", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-fact-Print-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "Print" + ] + } + ] + }, + "artifact": "(Print)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-fact-Print-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "Print" + ] + } + ] + }, + "artifact": "blueprint", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-marker-Print-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [ + "Print" + ] + }, + "artifact": "(Print)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-marker-Print-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [ + "Print" + ] + }, + "artifact": "blueprint", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-anchor-Print-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "answer", + "alternatives": [ + "Print" + ] + } + ] + } + }, + "artifact": "(Print)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-anchor-Print-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "answer", + "alternatives": [ + "Print" + ] + } + ] + } + }, + "artifact": "blueprint", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-fact-url-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "url" + ] + } + ] + }, + "artifact": "(url)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-fact-url-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "url" + ] + } + ] + }, + "artifact": "curl", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-marker-url-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [ + "url" + ] + }, + "artifact": "(url)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-marker-url-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [ + "url" + ] + }, + "artifact": "curl", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-anchor-url-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "answer", + "alternatives": [ + "url" + ] + } + ] + } + }, + "artifact": "(url)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-anchor-url-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "answer", + "alternatives": [ + "url" + ] + } + ] + } + }, + "artifact": "curl", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-fact-café-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "café" + ] + } + ] + }, + "artifact": "(café)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-fact-café-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "café" + ] + } + ] + }, + "artifact": "caféteria", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-marker-café-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [ + "café" + ] + }, + "artifact": "(café)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-marker-café-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [ + "café" + ] + }, + "artifact": "caféteria", + "ok": true, + "expected_task_pass": false + }, + { + "id": "latin-boundary-anchor-café-whole", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "answer", + "alternatives": [ + "café" + ] + } + ] + } + }, + "artifact": "(café)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "latin-boundary-anchor-café-fragment", + "category": "format-normalization", + "root_cause": "latin_word_fragment_false_match", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "answer", + "alternatives": [ + "café" + ] + } + ] + } + }, + "artifact": "caféteria", + "ok": true, + "expected_task_pass": false + }, + { + "id": "body-fact-title-absent", + "category": "answer-surface", + "root_cause": "metadata_title_supplies_missing_body_fact", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "required_facts": [ + { + "id": "install", + "alternatives": [ + "widget --start" + ] + } + ] + }, + "artifact": "unrelated text", + "title": "widget --start", + "ok": true, + "expected_task_pass": false + }, + { + "id": "body-fact-title-present", + "category": "answer-surface", + "root_cause": "metadata_title_supplies_missing_body_fact", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "required_facts": [ + { + "id": "install", + "alternatives": [ + "widget --start" + ] + } + ] + }, + "artifact": "widget --start", + "title": "widget --start", + "ok": true, + "expected_task_pass": true + }, + { + "id": "query-sensitive-dynamic-matching-final-page-and-language-remain-eligible", + "category": "acceptance-contract", + "root_cause": "undeclared_query_parameters_may_select_other_content", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/start", + "evaluation_url": "https://example.test/start", + "expected_final_url": "https://example.test/en/catalog", + "target_language": "en-US", + "target_region": "US", + "track": "dynamic", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "identity", + "alternatives": [ + "Catalog" + ] + } + ] + } + }, + "artifact": "Catalog", + "ok": true, + "final_url": "https://example.test/en/catalog?session=123#items", + "metadata": { + "route": "dynamic_selected_text", + "document_language": "en" + }, + "expected_task_pass": true, + "expected_metrics": { + "page_identity_status": "mismatch", + "final_url_consistent": false, + "language_consistent": true + } + }, + { + "id": "answer-body-fact-remains-valid-with-an-unrelated-document-title", + "category": "answer-surface", + "root_cause": "answer_surface_omission", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/page", + "required_facts": [ + { + "id": "heading", + "alternatives": [ + "Exact page title" + ] + } + ] + }, + "artifact": "Exact page title\nRequired body", + "title": "Unrelated title metadata", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "fact_coverage": 1.0 + } + }, + { + "id": "explicit-contract-missing-page-identity-counts-as-failure", + "category": "task-predicate", + "root_cause": "unknown_identity_masks_independent_requirement_failure", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/start", + "track": "risk_control", + "task_kind": "body", + "expected_any": [ + "Expected body" + ], + "min_content_chars": 1, + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "body", + "alternatives": [ + "Expected body" + ] + } + ] + }, + "source_benchmark": { + "risk_control": { + "navigation_contract": { + "requested_url": "https://example.test/start", + "observed_final_url": "https://example.test/final" + } + } + }, + "page_identity_required": true + }, + "artifact": "Expected body", + "ok": true, + "expected_task_pass": false, + "expected_task_measured": true, + "expected_metrics": { + "page_identity_status": "unavailable", + "length_pass": true + } + }, + { + "id": "resource-log-with-no-answer", + "category": "answer-surface", + "root_cause": "resource_network_diagnostics_are_not_answers", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/page", + "task_kind": "resource", + "content_score_applicable": false, + "expected_resources": [ + { + "id": "video", + "urls": [ + "https://example.test/target.mp4" + ] + } + ] + }, + "artifact": "{\"network\": [{\"url\": \"https://example.test/target.mp4\"}]}", + "ok": true, + "expected_task_pass": false + }, + { + "id": "resource-log-with-delivered-answer", + "category": "answer-surface", + "root_cause": "resource_network_diagnostics_are_not_answers", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/page", + "task_kind": "resource", + "content_score_applicable": false, + "expected_resources": [ + { + "id": "video", + "urls": [ + "https://example.test/target.mp4" + ] + } + ] + }, + "artifact": "{\"network\": [{\"url\": \"https://example.test/target.mp4\"}], \"resources\": [\"https://example.test/target.mp4\"]}", + "ok": true, + "expected_task_pass": true + }, + { + "id": "forbidden-latin-word-fragment", + "category": "format-normalization", + "root_cause": "forbidden_accented_latin_word_boundaries", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/page", + "forbidden_any": [ + "café" + ] + }, + "artifact": "caféteria", + "ok": true, + "expected_task_pass": true + }, + { + "id": "forbidden-latin-whole-word", + "category": "format-normalization", + "root_cause": "forbidden_accented_latin_word_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/page", + "forbidden_any": [ + "café" + ] + }, + "artifact": "A café.", + "ok": true, + "expected_task_pass": false + }, + { + "id": "neutral-frame-resource-absent", + "category": "answer-surface", + "root_cause": "neutral_inline_frame_document_url", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/page", + "task_kind": "resource", + "content_score_applicable": false, + "expected_resources": [ + { + "id": "frame", + "urls": [ + "https://frames.test/path/clip.mp4" + ] + } + ] + }, + "artifact": "<iframe data-frame-url=\"https://frames.test/path/index.html\" srcdoc=\"<video src="other.mp4"></video>\"></iframe>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "neutral-frame-resource-present", + "category": "answer-surface", + "root_cause": "neutral_inline_frame_document_url", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/page", + "task_kind": "resource", + "content_score_applicable": false, + "expected_resources": [ + { + "id": "frame", + "urls": [ + "https://frames.test/path/clip.mp4" + ] + } + ] + }, + "artifact": "<iframe data-frame-url=\"https://frames.test/path/index.html\" srcdoc=\"<video src="clip.mp4"></video>\"></iframe>", + "ok": true, + "expected_task_pass": true + }, + { + "category": "acceptance-contract", + "root_cause": "required_file_identity_missing_must_not_shrink_population", + "case": { + "url": "https://example.test/file", + "task_kind": "file", + "page_identity_required": true + }, + "artifact": "File delivered", + "ok": true, + "expected_task_measured": true, + "id": "file-missing-required-identity-is-measured-failure", + "failure_mode": "false-positive", + "expected_task_pass": false + }, + { + "category": "acceptance-contract", + "root_cause": "required_file_identity_missing_must_not_shrink_population", + "case": { + "url": "https://example.test/file", + "task_kind": "file", + "page_identity_required": true + }, + "artifact": "File delivered", + "ok": true, + "expected_task_measured": true, + "id": "file-with-required-identity-and-current-verdict-passes", + "failure_mode": "false-negative", + "final_url": "https://example.test/file", + "metadata": { + "file_evaluation": { + "pass": true + } + }, + "expected_task_pass": true + }, + { + "id": "json-url-list-answer-matches", + "category": "answer-surface", + "root_cause": "resource_network_diagnostics_are_not_answers", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/page", + "task_kind": "resource", + "expected_resources": [ + { + "id": "video", + "urls": [ + "/target.mp4" + ] + } + ] + }, + "artifact": "[\"/target.mp4\"]", + "ok": true, + "expected_task_pass": true + }, + { + "id": "json-url-list-answer-does-not-match", + "category": "answer-surface", + "root_cause": "resource_network_diagnostics_are_not_answers", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/page", + "task_kind": "resource", + "expected_resources": [ + { + "id": "video", + "urls": [ + "/target.mp4" + ] + } + ] + }, + "artifact": "[\"/other.mp4\"]", + "ok": true, + "expected_task_pass": false + }, + { + "id": "word-boundary-mixed-script-complete", + "category": "format-normalization", + "root_cause": "alphabetic_boundary_omits_greek_cyrillic", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "stats" + ] + } + ] + }, + "artifact": "(stats)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "word-boundary-mixed-script-fragment", + "category": "format-normalization", + "root_cause": "alphabetic_boundary_omits_greek_cyrillic", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "stats" + ] + } + ] + }, + "artifact": "αstats", + "ok": true, + "expected_task_pass": false + }, + { + "id": "word-boundary-greek-complete", + "category": "format-normalization", + "root_cause": "alphabetic_boundary_omits_greek_cyrillic", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "Αθήνα" + ] + } + ] + }, + "artifact": "(Αθήνα)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "word-boundary-greek-fragment", + "category": "format-normalization", + "root_cause": "alphabetic_boundary_omits_greek_cyrillic", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "Αθήνα" + ] + } + ] + }, + "artifact": "Αθήνας", + "ok": true, + "expected_task_pass": false + }, + { + "id": "word-boundary-cyrillic-complete", + "category": "format-normalization", + "root_cause": "alphabetic_boundary_omits_greek_cyrillic", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "новости" + ] + } + ] + }, + "artifact": "(новости)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "word-boundary-cyrillic-fragment", + "category": "format-normalization", + "root_cause": "alphabetic_boundary_omits_greek_cyrillic", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "required_facts": [ + { + "id": "answer", + "alternatives": [ + "новости" + ] + } + ] + }, + "artifact": "новостик", + "ok": true, + "expected_task_pass": false + }, + { + "id": "xml-declaration-frame-target", + "category": "structured-extraction", + "root_cause": "xml_declaration_discards_visible_frame_resources", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/spec", + "task_kind": "resource", + "expected_resources": [ + { + "id": "image", + "urls": [ + "/target.png" + ] + } + ] + }, + "artifact": "<![UNRECOGNIZED[ XML source ]]><iframe srcdoc=\"<img src="/target.png">\"></iframe>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "xml-declaration-frame-non-target", + "category": "structured-extraction", + "root_cause": "xml_declaration_discards_visible_frame_resources", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/spec", + "task_kind": "resource", + "expected_resources": [ + { + "id": "image", + "urls": [ + "/target.png" + ] + } + ] + }, + "artifact": "<![UNRECOGNIZED[ XML source ]]><iframe srcdoc=\"<img src="/other.png">\"></iframe>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "native-serialization-complete-notice", + "category": "format-normalization", + "root_cause": "visible_label_boundaries_in_native_serialization", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "required_facts": [ + { + "id": "label", + "alternatives": [ + "Example Corp. Avenida N°5, Santiago de Chile" + ] + } + ] + }, + "artifact": "ALL RIGHTS RESERVEDExample Corp. Avenida N°5, Santiago de Chile", + "ok": true, + "expected_task_pass": true + }, + { + "id": "native-serialization-changed-notice", + "category": "format-normalization", + "root_cause": "visible_label_boundaries_in_native_serialization", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "required_facts": [ + { + "id": "label", + "alternatives": [ + "Example Corp. Avenida N°5, Santiago de Chile" + ] + } + ] + }, + "artifact": "ALL RIGHTS RESERVEDExample Corp. Avenida N°6, Santiago de Chile", + "ok": true, + "expected_task_pass": false + }, + { + "id": "native-serialization-stock-label", + "category": "format-normalization", + "root_cause": "visible_label_boundaries_in_native_serialization", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "required_facts": [ + { + "id": "label", + "alternatives": [ + "In stock" + ] + } + ] + }, + "artifact": "In stockAdd to basket", + "ok": true, + "expected_task_pass": true + }, + { + "id": "native-serialization-stock-word-fragment", + "category": "format-normalization", + "root_cause": "visible_label_boundaries_in_native_serialization", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "required_facts": [ + { + "id": "label", + "alternatives": [ + "In stock" + ] + } + ] + }, + "artifact": "In stockholders", + "ok": true, + "expected_task_pass": false + }, + { + "id": "native-serialization-carousel-label", + "category": "format-normalization", + "root_cause": "visible_label_boundaries_in_native_serialization", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "required_facts": [ + { + "id": "label", + "alternatives": [ + "Careers" + ] + } + ] + }, + "artifact": "[Careers9 of 9](/careers)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "native-serialization-carousel-digit-word", + "category": "format-normalization", + "root_cause": "visible_label_boundaries_in_native_serialization", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "required_facts": [ + { + "id": "label", + "alternatives": [ + "Careers" + ] + } + ] + }, + "artifact": "[Careers9](/careers)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "native-serialization-cyrillic-label", + "category": "format-normalization", + "root_cause": "visible_label_boundaries_in_native_serialization", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "required_facts": [ + { + "id": "label", + "alternatives": [ + "Наука" + ] + } + ] + }, + "artifact": "[Технологии](/technology)[Наука](/science)Все темы", + "ok": true, + "expected_task_pass": true + }, + { + "id": "native-serialization-cyrillic-word-fragment", + "category": "format-normalization", + "root_cause": "visible_label_boundaries_in_native_serialization", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "required_facts": [ + { + "id": "label", + "alternatives": [ + "Наука" + ] + } + ] + }, + "artifact": "Наукам", + "ok": true, + "expected_task_pass": false + }, + { + "id": "native-serialization-punctuation-markup", + "category": "format-normalization", + "root_cause": "visible_label_boundaries_in_native_serialization", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "required_facts": [ + { + "id": "label", + "alternatives": [ + "toute l’actu dès 7h30" + ] + } + ] + }, + "artifact": "_:_toute l’actu dès 7h30", + "ok": true, + "expected_task_pass": true + }, + { + "id": "resource-serialization-markup-entity", + "category": "format-normalization", + "root_cause": "markdown_destination_entities_decoded_once", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/gallery", + "task_kind": "resource", + "expected_resources": [ + { + "id": "photo", + "urls": [ + "https://cdn.example.test/photo.jpg?width=640&height=480" + ] + } + ], + "resource_output_contract": { + "candidate_scope": "image_answer_fields", + "unexpected": "reject" + } + }, + "artifact": "![Photo](https://cdn.example.test/photo.jpg?width=640&height=480)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "resource-serialization-different-size", + "category": "format-normalization", + "root_cause": "markdown_destination_entities_decoded_once", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/gallery", + "task_kind": "resource", + "expected_resources": [ + { + "id": "photo", + "urls": [ + "https://cdn.example.test/photo.jpg?width=640&height=480" + ] + } + ], + "resource_output_contract": { + "candidate_scope": "image_answer_fields", + "unexpected": "reject" + } + }, + "artifact": "![Photo](https://cdn.example.test/photo.jpg?width=641&height=480)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "named-scope-heading-owns-records", + "category": "structured-extraction", + "root_cause": "heading_scope_and_empty_nested_card_layout", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/search", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "entry_scope": { + "html_selector": ".results a[href]", + "start_pattern": "^Find games$", + "end_pattern": "^See all" + } + } + }, + "artifact": "Find games\n[Help](/help)\nSee all\n### Find games\n[Alpha](/item/1)\n[Beta](/item/2)\nSee all", + "ok": true, + "expected_task_pass": true + }, + { + "id": "named-scope-navigation-cannot-supply-records", + "category": "structured-extraction", + "root_cause": "heading_scope_and_empty_nested_card_layout", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/search", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "entry_scope": { + "html_selector": ".results a[href]", + "start_pattern": "^Find games$", + "end_pattern": "^See all" + } + } + }, + "artifact": "Find games\n[Alpha](/item/1)\n[Beta](/item/2)\nSee all\n### Find games\n[Gamma](/item/3)\nSee all\n[Delta](/item/4)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "named-scope-empty-layout-is-not-name", + "category": "structured-extraction", + "root_cause": "heading_scope_and_empty_nested_card_layout", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/search", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "entry_scope": { + "html_selector": ".results a[href]", + "start_pattern": "^Find games$", + "end_pattern": "^See all" + } + } + }, + "artifact": "### Find games\n- [![](/cover.jpg)\n\n - **\n\nAlpha game\n2026.3.26\n8,500円\nPublisher\n**](/item/1)\n- [Beta](/item/2)\nSee all", + "ok": true, + "expected_task_pass": true + }, + { + "id": "named-scope-image-without-visible-name", + "category": "structured-extraction", + "root_cause": "heading_scope_and_empty_nested_card_layout", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/search", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "entry_scope": { + "html_selector": ".results a[href]", + "start_pattern": "^Find games$", + "end_pattern": "^See all" + } + } + }, + "artifact": "### Find games\n- [![](/cover.jpg)\n\n - **\n\n2026.3.26\n**](/item/1)\n- [Beta](/item/2)\nSee all", + "ok": true, + "expected_task_pass": false + }, + { + "id": "native-card-bullet-complete", + "category": "structured-extraction", + "root_cause": "typed_card_fields_serialized_as_sibling_bullets", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "homes", + "kind": "named_link", + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "minimum_records": 1, + "required_fields": [ + { + "id": "price", + "kind": "price" + }, + { + "id": "bedrooms", + "kind": "integer", + "pattern": "\\bbds\\b" + }, + { + "id": "bathrooms", + "kind": "integer", + "pattern": "\\bba\\b" + }, + { + "id": "area", + "kind": "area" + } + ] + } + ] + } + }, + "artifact": "- [$900](/item/1)\n- **3** bds\n- **2** ba\n- **900** sqft\nActive[House A](/item/1)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "native-card-bullet-missing-area", + "category": "structured-extraction", + "root_cause": "typed_card_fields_serialized_as_sibling_bullets", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "homes", + "kind": "named_link", + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "minimum_records": 1, + "required_fields": [ + { + "id": "price", + "kind": "price" + }, + { + "id": "bedrooms", + "kind": "integer", + "pattern": "\\bbds\\b" + }, + { + "id": "bathrooms", + "kind": "integer", + "pattern": "\\bba\\b" + }, + { + "id": "area", + "kind": "area" + } + ] + } + ] + } + }, + "artifact": "- [House A](/item/1)\n- [House B](/item/2)\n- $900\n- 3 bds\n- 2 ba", + "ok": true, + "expected_task_pass": false + }, + { + "id": "native-card-bullet-other-section", + "category": "structured-extraction", + "root_cause": "typed_card_fields_serialized_as_sibling_bullets", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "homes", + "kind": "named_link", + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "minimum_records": 1, + "required_fields": [ + { + "id": "price", + "kind": "price" + }, + { + "id": "bedrooms", + "kind": "integer", + "pattern": "\\bbds\\b" + }, + { + "id": "bathrooms", + "kind": "integer", + "pattern": "\\bba\\b" + }, + { + "id": "area", + "kind": "area" + } + ] + } + ] + } + }, + "artifact": "- [House A](/item/1)\n# Other section\n- $900\n- 3 bds\n- 2 ba\n- 900 sqft", + "ok": true, + "expected_task_pass": false + }, + { + "id": "native-card-text-same-line-region", + "category": "structured-extraction", + "root_cause": "declared_text_patterns_consume_distinct_value_spans", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "events", + "kind": "named_link", + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "minimum_records": 1, + "required_fields": [ + { + "id": "venue", + "kind": "text", + "minimum_chars": 2 + }, + { + "id": "region", + "kind": "text", + "pattern": ",\\s*(?:TX|AL)\\b" + }, + { + "id": "date", + "kind": "date_or_time" + } + ] + } + ] + } + }, + "artifact": "[Concert](/item/1)\n2026-10-16\nExample Hall Tuscaloosa, AL", + "ok": true, + "expected_task_pass": true + }, + { + "id": "native-card-text-region-without-venue", + "category": "structured-extraction", + "root_cause": "declared_text_patterns_consume_distinct_value_spans", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "events", + "kind": "named_link", + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "minimum_records": 1, + "required_fields": [ + { + "id": "venue", + "kind": "text", + "minimum_chars": 2 + }, + { + "id": "region", + "kind": "text", + "pattern": ",\\s*(?:TX|AL)\\b" + }, + { + "id": "date", + "kind": "date_or_time" + } + ] + } + ] + } + }, + "artifact": "[Concert](/item/1)\n2026-10-16\n, AL", + "ok": true, + "expected_task_pass": false + }, + { + "id": "native-card-text-no-region", + "category": "structured-extraction", + "root_cause": "declared_text_patterns_consume_distinct_value_spans", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "events", + "kind": "named_link", + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "minimum_records": 1, + "required_fields": [ + { + "id": "venue", + "kind": "text", + "minimum_chars": 2 + }, + { + "id": "region", + "kind": "text", + "pattern": ",\\s*(?:TX|AL)\\b" + }, + { + "id": "date", + "kind": "date_or_time" + } + ] + } + ] + } + }, + "artifact": "[Concert](/item/1)\n2026-10-16\nExample Hall", + "ok": true, + "expected_task_pass": false + }, + { + "id": "source-backed-diagnostic-source-discusses-status", + "category": "acceptance-contract", + "root_cause": "source_body_term_is_not_an_explicit_exclusion", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/spec", + "required_facts": [ + { + "id": "content", + "alternatives": [ + "response specification" + ] + } + ], + "diagnostic_any": [ + "404 Not Found" + ] + }, + "artifact": "The HTTP standard explains 404 Not Found in the response specification.", + "ok": true, + "expected_task_pass": true + }, + { + "id": "source-backed-diagnostic-status-heading", + "category": "acceptance-contract", + "root_cause": "source_body_term_is_not_an_explicit_exclusion", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/spec", + "required_facts": [ + { + "id": "content", + "alternatives": [ + "response specification" + ] + } + ], + "diagnostic_any": [ + "404 Not Found" + ] + }, + "artifact": "404 Not Found", + "ok": true, + "expected_task_pass": false + }, + { + "id": "declared-whole-card-complete", + "category": "structured-extraction", + "root_cause": "declared_visible_title_and_fields_inside_one_card_link", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "events", + "kind": "named_link", + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "minimum_records": 1, + "markdown_label_pattern": "(?m)^(?:Today\\n)?2026-10-16\\n(?P<label>[^\\n]+)\\n", + "required_fields": [ + { + "id": "date", + "kind": "date_or_time" + }, + { + "id": "venue", + "kind": "text", + "minimum_chars": 2 + }, + { + "id": "region", + "kind": "text", + "pattern": ",\\s*AL\\b" + } + ] + } + ] + } + }, + "artifact": "[![](/cover.jpg)\nToday\n2026-10-16\nEvening Concert\nExample Hall Tuscaloosa, AL](/item/1)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "declared-whole-card-no-name", + "category": "structured-extraction", + "root_cause": "declared_visible_title_and_fields_inside_one_card_link", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "events", + "kind": "named_link", + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "minimum_records": 1, + "markdown_label_pattern": "(?m)^(?:Today\\n)?2026-10-16\\n(?P<label>[^\\n]+)\\n", + "required_fields": [ + { + "id": "date", + "kind": "date_or_time" + }, + { + "id": "venue", + "kind": "text", + "minimum_chars": 2 + }, + { + "id": "region", + "kind": "text", + "pattern": ",\\s*AL\\b" + } + ] + } + ] + } + }, + "artifact": "[![](/cover.jpg)\nToday\n2026-10-16\nExample Hall Tuscaloosa, AL](/item/1)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "declared-whole-card-no-venue", + "category": "structured-extraction", + "root_cause": "declared_visible_title_and_fields_inside_one_card_link", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "events", + "kind": "named_link", + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "minimum_records": 1, + "markdown_label_pattern": "(?m)^(?:Today\\n)?2026-10-16\\n(?P<label>[^\\n]+)\\n", + "required_fields": [ + { + "id": "date", + "kind": "date_or_time" + }, + { + "id": "venue", + "kind": "text", + "minimum_chars": 2 + }, + { + "id": "region", + "kind": "text", + "pattern": ",\\s*AL\\b" + } + ] + } + ] + } + }, + "artifact": "[![](/cover.jpg)\nToday\n2026-10-16\nEvening Concert\n, AL](/item/1)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "declared-whole-card-no-region", + "category": "structured-extraction", + "root_cause": "declared_visible_title_and_fields_inside_one_card_link", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "events", + "kind": "named_link", + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "minimum_records": 1, + "markdown_label_pattern": "(?m)^(?:Today\\n)?2026-10-16\\n(?P<label>[^\\n]+)\\n", + "required_fields": [ + { + "id": "date", + "kind": "date_or_time" + }, + { + "id": "venue", + "kind": "text", + "minimum_chars": 2 + }, + { + "id": "region", + "kind": "text", + "pattern": ",\\s*AL\\b" + } + ] + } + ] + } + }, + "artifact": "[![](/cover.jpg)\nToday\n2026-10-16\nEvening Concert\nExample Hall](/item/1)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "declared-card-prelude-owned", + "category": "structured-extraction", + "root_cause": "declared_card_start_owns_pre_title_fields", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "events", + "kind": "named_link", + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "minimum_records": 1, + "record_start_pattern": "(?m)^- 2026-10-[0-9]{2}$", + "required_fields": [ + { + "id": "date", + "kind": "date_or_time" + }, + { + "id": "venue", + "kind": "text", + "aliases": [ + "Venue" + ] + }, + { + "id": "region", + "kind": "text", + "aliases": [ + "Region" + ] + } + ] + } + ] + } + }, + "artifact": "- 2026-10-16\nVenue: Hall A\nRegion: Alabama\n[Concert](/item/1)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "declared-card-prelude-separate-card", + "category": "structured-extraction", + "root_cause": "declared_card_start_owns_pre_title_fields", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "events", + "kind": "named_link", + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "minimum_records": 1, + "record_start_pattern": "(?m)^- 2026-10-[0-9]{2}$", + "required_fields": [ + { + "id": "date", + "kind": "date_or_time" + }, + { + "id": "venue", + "kind": "text", + "aliases": [ + "Venue" + ] + }, + { + "id": "region", + "kind": "text", + "aliases": [ + "Region" + ] + } + ] + } + ] + } + }, + "artifact": "- 2026-10-16\nVenue: Hall A\nRegion: Alabama\n- 2026-10-17\n[Concert](/item/1)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "declared-card-prelude-next-card-fields", + "category": "structured-extraction", + "root_cause": "declared_card_start_owns_pre_title_fields", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "events", + "kind": "named_link", + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "minimum_records": 1, + "record_start_pattern": "(?m)^- 2026-10-[0-9]{2}$", + "required_fields": [ + { + "id": "date", + "kind": "date_or_time" + }, + { + "id": "venue", + "kind": "text", + "aliases": [ + "Venue" + ] + }, + { + "id": "region", + "kind": "text", + "aliases": [ + "Region" + ] + } + ] + } + ] + } + }, + "artifact": "- 2026-10-16\nVenue: Hall A\n[Concert](/item/1)\n- 2026-10-17\nRegion: Alabama\n[Other](/item/2)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "declared-information-explicit-title", + "category": "acceptance-contract", + "root_cause": "information_title_scope_does_not_promote_unrelated_metadata", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/video", + "required_facts": [ + { + "id": "video-title", + "alternatives": [ + "Example video" + ], + "match_scope": "body_or_title" + }, + { + "id": "publisher", + "alternatives": [ + "Example Channel" + ] + } + ] + }, + "artifact": "---\ntitle: \"Example video\"\n---\nExample Channel", + "ok": true, + "expected_task_pass": true + }, + { + "id": "declared-information-keywords-not-title", + "category": "acceptance-contract", + "root_cause": "information_title_scope_does_not_promote_unrelated_metadata", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/video", + "required_facts": [ + { + "id": "video-title", + "alternatives": [ + "Example video" + ], + "match_scope": "body_or_title" + }, + { + "id": "publisher", + "alternatives": [ + "Example Channel" + ] + } + ] + }, + "artifact": "---\ntitle: \"Other video\"\nkeywords: \"Example video\"\n---\nExample Channel", + "ok": true, + "expected_task_pass": false + }, + { + "id": "declared-information-keywords-not-publisher", + "category": "acceptance-contract", + "root_cause": "information_title_scope_does_not_promote_unrelated_metadata", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/video", + "required_facts": [ + { + "id": "video-title", + "alternatives": [ + "Example video" + ], + "match_scope": "body_or_title" + }, + { + "id": "publisher", + "alternatives": [ + "Example Channel" + ] + } + ] + }, + "artifact": "---\ntitle: \"Example video\"\nkeywords: \"Example Channel\"\n---\nOther body", + "ok": true, + "expected_task_pass": false + }, + { + "id": "named-section-boundary-complete-software", + "category": "acceptance-contract", + "root_cause": "declared_result_section_ends_at_the_next_independent_section", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/search", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "entry_scope": { + "html_selector": ".results a[href]", + "start_pattern": "^Find games$", + "end_pattern": "^Find pages$" + } + } + }, + "artifact": "### Find games\n[Alpha](/item/1)\n[Beta](/item/2)\n### Find pages\n[Other page](/item/3)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "named-section-boundary-next-section-not-software", + "category": "acceptance-contract", + "root_cause": "declared_result_section_ends_at_the_next_independent_section", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/search", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "entry_scope": { + "html_selector": ".results a[href]", + "start_pattern": "^Find games$", + "end_pattern": "^Find pages$" + } + } + }, + "artifact": "### Find games\n[Alpha](/item/1)\n### Find pages\n[Other page](/item/2)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "declared-photo-query-unsplash-resize", + "category": "acceptance-contract", + "root_cause": "resource_alias_must_bind_asset_and_declared_presentation_parameters", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/photo", + "task_kind": "resource", + "expected_resources": [ + { + "id": "main-image", + "kind": "image", + "urls": [ + "https://images.unsplash.com/photo-1493673272479-a20888bcee10", + "https://unsplash.com/photos/mSK5nNsAsLY/download" + ], + "url_patterns": [ + "^https://images\\.unsplash\\.com/photo\\-1493673272479\\-a20888bcee10(?:\\?(?:(?:w|h)=[1-9][0-9]*|q=(?:100|[1-9][0-9]?)|dpr=[1-9][0-9]*(?:\\.[0-9]+)?|fm=(?:jpg|jpeg|png|webp|avif|gif)|fit=(?:crop|clip|fill|fillmax|max|min|scale)|crop=(?:faces|edges|entropy|top|bottom|left|right|focalpoint)|cs=(?:srgb|tinysrgb)|auto=format|ixlib=[A-Za-z0-9._-]+|ixid=[A-Za-z0-9_+/=%-]+)(?:&(?:(?:w|h)=[1-9][0-9]*|q=(?:100|[1-9][0-9]?)|dpr=[1-9][0-9]*(?:\\.[0-9]+)?|fm=(?:jpg|jpeg|png|webp|avif|gif)|fit=(?:crop|clip|fill|fillmax|max|min|scale)|crop=(?:faces|edges|entropy|top|bottom|left|right|focalpoint)|cs=(?:srgb|tinysrgb)|auto=format|ixlib=[A-Za-z0-9._-]+|ixid=[A-Za-z0-9_+/=%-]+))*)?$" + ] + } + ], + "resource_output_contract": { + "candidate_scope": "image_answer_fields", + "unexpected": "allow" + } + }, + "artifact": "![Photo](https://images.unsplash.com/photo-1493673272479-a20888bcee10?auto=format&w=640&q=80)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "declared-photo-query-unsplash-overlay", + "category": "acceptance-contract", + "root_cause": "resource_alias_must_bind_asset_and_declared_presentation_parameters", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/photo", + "task_kind": "resource", + "expected_resources": [ + { + "id": "main-image", + "kind": "image", + "urls": [ + "https://images.unsplash.com/photo-1493673272479-a20888bcee10", + "https://unsplash.com/photos/mSK5nNsAsLY/download" + ], + "url_patterns": [ + "^https://images\\.unsplash\\.com/photo\\-1493673272479\\-a20888bcee10(?:\\?(?:(?:w|h)=[1-9][0-9]*|q=(?:100|[1-9][0-9]?)|dpr=[1-9][0-9]*(?:\\.[0-9]+)?|fm=(?:jpg|jpeg|png|webp|avif|gif)|fit=(?:crop|clip|fill|fillmax|max|min|scale)|crop=(?:faces|edges|entropy|top|bottom|left|right|focalpoint)|cs=(?:srgb|tinysrgb)|auto=format|ixlib=[A-Za-z0-9._-]+|ixid=[A-Za-z0-9_+/=%-]+)(?:&(?:(?:w|h)=[1-9][0-9]*|q=(?:100|[1-9][0-9]?)|dpr=[1-9][0-9]*(?:\\.[0-9]+)?|fm=(?:jpg|jpeg|png|webp|avif|gif)|fit=(?:crop|clip|fill|fillmax|max|min|scale)|crop=(?:faces|edges|entropy|top|bottom|left|right|focalpoint)|cs=(?:srgb|tinysrgb)|auto=format|ixlib=[A-Za-z0-9._-]+|ixid=[A-Za-z0-9_+/=%-]+))*)?$" + ] + } + ], + "resource_output_contract": { + "candidate_scope": "image_answer_fields", + "unexpected": "allow" + } + }, + "artifact": "![Photo](https://images.unsplash.com/photo-1493673272479-a20888bcee10?w=640&mark=https://other.test/logo.png)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "declared-photo-query-unsplash-other-asset", + "category": "acceptance-contract", + "root_cause": "resource_alias_must_bind_asset_and_declared_presentation_parameters", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/photo", + "task_kind": "resource", + "expected_resources": [ + { + "id": "main-image", + "kind": "image", + "urls": [ + "https://images.unsplash.com/photo-1493673272479-a20888bcee10", + "https://unsplash.com/photos/mSK5nNsAsLY/download" + ], + "url_patterns": [ + "^https://images\\.unsplash\\.com/photo\\-1493673272479\\-a20888bcee10(?:\\?(?:(?:w|h)=[1-9][0-9]*|q=(?:100|[1-9][0-9]?)|dpr=[1-9][0-9]*(?:\\.[0-9]+)?|fm=(?:jpg|jpeg|png|webp|avif|gif)|fit=(?:crop|clip|fill|fillmax|max|min|scale)|crop=(?:faces|edges|entropy|top|bottom|left|right|focalpoint)|cs=(?:srgb|tinysrgb)|auto=format|ixlib=[A-Za-z0-9._-]+|ixid=[A-Za-z0-9_+/=%-]+)(?:&(?:(?:w|h)=[1-9][0-9]*|q=(?:100|[1-9][0-9]?)|dpr=[1-9][0-9]*(?:\\.[0-9]+)?|fm=(?:jpg|jpeg|png|webp|avif|gif)|fit=(?:crop|clip|fill|fillmax|max|min|scale)|crop=(?:faces|edges|entropy|top|bottom|left|right|focalpoint)|cs=(?:srgb|tinysrgb)|auto=format|ixlib=[A-Za-z0-9._-]+|ixid=[A-Za-z0-9_+/=%-]+))*)?$" + ] + } + ], + "resource_output_contract": { + "candidate_scope": "image_answer_fields", + "unexpected": "allow" + } + }, + "artifact": "![Photo](https://images.unsplash.com/photo-1493673272480-a20888bcee10?w=640)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "declared-photo-query-pexels-resize", + "category": "acceptance-contract", + "root_cause": "resource_alias_must_bind_asset_and_declared_presentation_parameters", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/photo", + "task_kind": "resource", + "expected_resources": [ + { + "id": "main-image", + "kind": "image", + "urls": [ + "https://images.pexels.com/photos/12377231/pexels-photo-12377231.jpeg" + ], + "url_patterns": [ + "^https://images\\.pexels\\.com/photos/12377231/pexels\\-photo\\-12377231\\.jpeg(?:\\?(?:(?:w|h)=[1-9][0-9]*|q=(?:100|[1-9][0-9]?)|dpr=[1-9][0-9]*(?:\\.[0-9]+)?|fm=(?:jpg|jpeg|png|webp|avif|gif)|fit=(?:crop|clip|fill|fillmax|max|min|scale)|crop=(?:faces|edges|entropy|top|bottom|left|right|focalpoint)|cs=(?:srgb|tinysrgb)|auto=(?:compress|format|compress,format)|dl=[A-Za-z0-9._-]+)(?:&(?:(?:w|h)=[1-9][0-9]*|q=(?:100|[1-9][0-9]?)|dpr=[1-9][0-9]*(?:\\.[0-9]+)?|fm=(?:jpg|jpeg|png|webp|avif|gif)|fit=(?:crop|clip|fill|fillmax|max|min|scale)|crop=(?:faces|edges|entropy|top|bottom|left|right|focalpoint)|cs=(?:srgb|tinysrgb)|auto=(?:compress|format|compress,format)|dl=[A-Za-z0-9._-]+))*)?$" + ] + } + ], + "resource_output_contract": { + "candidate_scope": "image_answer_fields", + "unexpected": "allow" + } + }, + "artifact": "![Photo](https://images.pexels.com/photos/12377231/pexels-photo-12377231.jpeg?auto=compress&w=640&q=80)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "declared-photo-query-pexels-overlay", + "category": "acceptance-contract", + "root_cause": "resource_alias_must_bind_asset_and_declared_presentation_parameters", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/photo", + "task_kind": "resource", + "expected_resources": [ + { + "id": "main-image", + "kind": "image", + "urls": [ + "https://images.pexels.com/photos/12377231/pexels-photo-12377231.jpeg" + ], + "url_patterns": [ + "^https://images\\.pexels\\.com/photos/12377231/pexels\\-photo\\-12377231\\.jpeg(?:\\?(?:(?:w|h)=[1-9][0-9]*|q=(?:100|[1-9][0-9]?)|dpr=[1-9][0-9]*(?:\\.[0-9]+)?|fm=(?:jpg|jpeg|png|webp|avif|gif)|fit=(?:crop|clip|fill|fillmax|max|min|scale)|crop=(?:faces|edges|entropy|top|bottom|left|right|focalpoint)|cs=(?:srgb|tinysrgb)|auto=(?:compress|format|compress,format)|dl=[A-Za-z0-9._-]+)(?:&(?:(?:w|h)=[1-9][0-9]*|q=(?:100|[1-9][0-9]?)|dpr=[1-9][0-9]*(?:\\.[0-9]+)?|fm=(?:jpg|jpeg|png|webp|avif|gif)|fit=(?:crop|clip|fill|fillmax|max|min|scale)|crop=(?:faces|edges|entropy|top|bottom|left|right|focalpoint)|cs=(?:srgb|tinysrgb)|auto=(?:compress|format|compress,format)|dl=[A-Za-z0-9._-]+))*)?$" + ] + } + ], + "resource_output_contract": { + "candidate_scope": "image_answer_fields", + "unexpected": "allow" + } + }, + "artifact": "![Photo](https://images.pexels.com/photos/12377231/pexels-photo-12377231.jpeg?w=640&mark=https://other.test/logo.png)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "declared-photo-query-pexels-other-asset", + "category": "acceptance-contract", + "root_cause": "resource_alias_must_bind_asset_and_declared_presentation_parameters", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/photo", + "task_kind": "resource", + "expected_resources": [ + { + "id": "main-image", + "kind": "image", + "urls": [ + "https://images.pexels.com/photos/12377231/pexels-photo-12377231.jpeg" + ], + "url_patterns": [ + "^https://images\\.pexels\\.com/photos/12377231/pexels\\-photo\\-12377231\\.jpeg(?:\\?(?:(?:w|h)=[1-9][0-9]*|q=(?:100|[1-9][0-9]?)|dpr=[1-9][0-9]*(?:\\.[0-9]+)?|fm=(?:jpg|jpeg|png|webp|avif|gif)|fit=(?:crop|clip|fill|fillmax|max|min|scale)|crop=(?:faces|edges|entropy|top|bottom|left|right|focalpoint)|cs=(?:srgb|tinysrgb)|auto=(?:compress|format|compress,format)|dl=[A-Za-z0-9._-]+)(?:&(?:(?:w|h)=[1-9][0-9]*|q=(?:100|[1-9][0-9]?)|dpr=[1-9][0-9]*(?:\\.[0-9]+)?|fm=(?:jpg|jpeg|png|webp|avif|gif)|fit=(?:crop|clip|fill|fillmax|max|min|scale)|crop=(?:faces|edges|entropy|top|bottom|left|right|focalpoint)|cs=(?:srgb|tinysrgb)|auto=(?:compress|format|compress,format)|dl=[A-Za-z0-9._-]+))*)?$" + ] + } + ], + "resource_output_contract": { + "candidate_scope": "image_answer_fields", + "unexpected": "allow" + } + }, + "artifact": "![Photo](https://images.pexels.com/photos/12377232/pexels-photo-12377232.jpeg?w=640)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "localized-required-navigation-english", + "category": "acceptance-contract", + "root_cause": "Equivalent observed navigation labels must satisfy their declared group without dropping required navigation or substantive records.", + "failure_mode": "false-negative", + "case": { + "id": "localized-training", + "url": "https://example.test/training", + "task_kind": "body", + "min_content_chars": 1, + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "title", + "alternatives": [ + "Training exercises" + ] + }, + { + "id": "home", + "alternatives": [ + "首页", + "Home" + ] + }, + { + "id": "login", + "alternatives": [ + "登录", + "Login" + ] + } + ] + }, + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "exercises", + "kind": "table", + "minimum_records": 2, + "required_columns": [ + { + "id": "number", + "kind": "text", + "aliases": [ + "ID" + ], + "pattern": "^[A-Z][0-9]+$" + }, + { + "id": "name", + "kind": "text", + "aliases": [ + "Name" + ] + } + ], + "identity_column": "number" + } + ] + } + }, + "artifact": "Training exercises\nHome\nLogin\n\n| ID | Name |\n| --- | --- |\n| A1 | Addition |\n| A2 | Multiplication |\n", + "ok": true, + "expected_task_pass": true + }, + { + "id": "localized-required-navigation-chinese", + "category": "acceptance-contract", + "root_cause": "Equivalent observed navigation labels must satisfy their declared group without dropping required navigation or substantive records.", + "failure_mode": "false-negative", + "case": { + "id": "localized-training", + "url": "https://example.test/training", + "task_kind": "body", + "min_content_chars": 1, + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "title", + "alternatives": [ + "Training exercises" + ] + }, + { + "id": "home", + "alternatives": [ + "首页", + "Home" + ] + }, + { + "id": "login", + "alternatives": [ + "登录", + "Login" + ] + } + ] + }, + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "exercises", + "kind": "table", + "minimum_records": 2, + "required_columns": [ + { + "id": "number", + "kind": "text", + "aliases": [ + "ID" + ], + "pattern": "^[A-Z][0-9]+$" + }, + { + "id": "name", + "kind": "text", + "aliases": [ + "Name" + ] + } + ], + "identity_column": "number" + } + ] + } + }, + "artifact": "Training exercises\n首页\n登录\n\n| ID | Name |\n| --- | --- |\n| A1 | Addition |\n| A2 | Multiplication |\n", + "ok": true, + "expected_task_pass": true + }, + { + "id": "localized-required-navigation-missing", + "category": "acceptance-contract", + "root_cause": "Equivalent observed navigation labels must satisfy their declared group without dropping required navigation or substantive records.", + "failure_mode": "false-positive", + "case": { + "id": "localized-training", + "url": "https://example.test/training", + "task_kind": "body", + "min_content_chars": 1, + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "title", + "alternatives": [ + "Training exercises" + ] + }, + { + "id": "home", + "alternatives": [ + "首页", + "Home" + ] + }, + { + "id": "login", + "alternatives": [ + "登录", + "Login" + ] + } + ] + }, + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "exercises", + "kind": "table", + "minimum_records": 2, + "required_columns": [ + { + "id": "number", + "kind": "text", + "aliases": [ + "ID" + ], + "pattern": "^[A-Z][0-9]+$" + }, + { + "id": "name", + "kind": "text", + "aliases": [ + "Name" + ] + } + ], + "identity_column": "number" + } + ] + } + }, + "artifact": "Training exercises\nHome\n\n| ID | Name |\n| --- | --- |\n| A1 | Addition |\n| A2 | Multiplication |\n", + "ok": true, + "expected_task_pass": false + }, + { + "id": "localized-required-navigation-without-records", + "category": "acceptance-contract", + "root_cause": "Equivalent observed navigation labels must satisfy their declared group without dropping required navigation or substantive records.", + "failure_mode": "false-positive", + "case": { + "id": "localized-training", + "url": "https://example.test/training", + "task_kind": "body", + "min_content_chars": 1, + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "title", + "alternatives": [ + "Training exercises" + ] + }, + { + "id": "home", + "alternatives": [ + "首页", + "Home" + ] + }, + { + "id": "login", + "alternatives": [ + "登录", + "Login" + ] + } + ] + }, + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "exercises", + "kind": "table", + "minimum_records": 2, + "required_columns": [ + { + "id": "number", + "kind": "text", + "aliases": [ + "ID" + ], + "pattern": "^[A-Z][0-9]+$" + }, + { + "id": "name", + "kind": "text", + "aliases": [ + "Name" + ] + } + ], + "identity_column": "number" + } + ] + } + }, + "artifact": "Training exercises\nHome\nLogin", + "ok": true, + "expected_task_pass": false + }, { "id": "visible-control-visible-button", "category": "answer-surface", @@ -54368,17 +57221,21 @@ } ], "superseded_cases": { - "owned-specific-image-alt-is-an-accessible-record-name": "specific-image-alternative-does-not-create-visible-record-name", - "nested-image-accessible-names-identify-linked-records": "image-alternatives-do-not-create-visible-record-names", - "dynamic-final-page-mismatch-cannot-be-counted-as-a-pass": "content-policy-dynamic-final-page-mismatch-cannot-be-counted-as-a-pass", + "inline-linked-card-visible-fields": "inline-linked-card-visible-fields-ambiguous-boundaries-rejected", + "explicit-contract-missing-page-identity-with-valid-body-stays-unknown": "explicit-contract-missing-page-identity-counts-as-failure", + "answer-title-does-not-inflate-body-length": "body-only-word-boundary-answer-title-does-not-inflate-body-length", + "required-final-url-missing-is-a-measured-failure": "content-policy-required-final-url-missing-is-a-measured-failure", "dynamic-search-query-mismatch-is-a-different-page": "content-policy-dynamic-search-query-mismatch-is-a-different-page", + "inline-accessible-card-visible-fields": "inline-accessible-card-visible-fields-ambiguous-boundaries-rejected", "dynamic-source-document-language-mismatch-is-not-verified": "content-policy-dynamic-source-document-language-mismatch-is-not-verified", "dynamic-undeclared-source-language-rejects-declared-language": "content-policy-dynamic-undeclared-source-language-rejects-declared-language", - "required-final-url-missing-is-a-measured-failure": "content-policy-required-final-url-missing-is-a-measured-failure", + "answer-title-supplies-required-fact": "body-only-word-boundary-answer-title-supplies-required-fact", + "dynamic-final-page-mismatch-cannot-be-counted-as-a-pass": "content-policy-dynamic-final-page-mismatch-cannot-be-counted-as-a-pass", + "body-unfenced-native-table-code-retained": "body-unfenced-table-is-markup-not-literal-code", + "nested-image-accessible-names-identify-linked-records": "image-alternatives-do-not-create-visible-record-names", + "owned-specific-image-alt-is-an-accessible-record-name": "specific-image-alternative-does-not-create-visible-record-name", "required-final-url-present-keeps-valid-body": "content-policy-required-final-url-present-keeps-valid-body", - "inline-linked-card-visible-fields": "inline-linked-card-visible-fields-ambiguous-boundaries-rejected", - "inline-accessible-card-visible-fields": "inline-accessible-card-visible-fields-ambiguous-boundaries-rejected", - "body-unfenced-native-table-code-retained": "body-unfenced-table-is-markup-not-literal-code" + "dynamic-matching-final-page-and-language-remain-eligible": "query-sensitive-dynamic-matching-final-page-and-language-remain-eligible" }, "superseded_controls": { "frozen_scorer_sha256": "ceb62fb821e6777d24a015ff713f406faecfff6181b0601b8e3ea2adffbf7c10", diff --git a/src/lexbench_webfetch/dynamic_lists.py b/src/lexbench_webfetch/dynamic_lists.py index f4b76c9..a503026 100644 --- a/src/lexbench_webfetch/dynamic_lists.py +++ b/src/lexbench_webfetch/dynamic_lists.py @@ -272,7 +272,7 @@ def _note_malformed_markdown_destinations(content, base_url, diagnostics): break -def _text_scope_boundary(content, pattern): +def _text_scope_boundary(content, pattern, *, prefer_heading=False): """Match literal text or an equivalent standalone Markdown heading. Return offsets in the original answer so record syntax and boundaries are @@ -281,6 +281,7 @@ def _text_scope_boundary(content, pattern): """ from .reference_quality import visible_markdown_links candidates=[] + headings=[] raw=re.search(pattern,content,re.M) if raw:candidates.append((raw.start(),raw.end())) offset=0 @@ -294,8 +295,13 @@ def _text_scope_boundary(content, pattern): visible=re.sub(r'([*_]{1,2})(.+?)\1',r'\2',visible) visible=re.sub(r'\s+#+\s*$','',visible).strip() if re.match(pattern,visible,re.M): - candidates.append((offset,offset+len(line))) + candidate = (offset, offset + len(line)) + candidates.append(candidate) + if re.match(r"^[ \t]{0,3}#{1,6}[ \t]+", decoded_line): + headings.append(candidate) offset+=len(line) + if prefer_heading and headings: + return min(headings) return min(candidates) if candidates else None @@ -439,7 +445,7 @@ def _scope_named_content(content, scope, *, contract=None, base_url=''): owned_items.append((destinations.pop(), titles[0].get_text(' ', strip=True))) return '', owned_items, True if scope.get('start_pattern'): - start = _text_scope_boundary(content, scope['start_pattern']) + start = _text_scope_boundary(content, scope['start_pattern'], prefer_heading=True) if not start: return '', [], False content = content[start[1]:] diff --git a/src/lexbench_webfetch/metrics.py b/src/lexbench_webfetch/metrics.py index 3a86997..95b7202 100644 --- a/src/lexbench_webfetch/metrics.py +++ b/src/lexbench_webfetch/metrics.py @@ -78,6 +78,8 @@ def _metadata_resource_candidates( if isinstance(value, dict): for child_key, child in value.items(): child_key = str(child_key) + if child_key.casefold() in {"network", "request", "requests", "logs", "debug", "diagnostics"}: + continue candidates.extend(_metadata_resource_candidates( child, key=child_key.casefold(), path=f"{path}.{child_key}", )) @@ -154,7 +156,14 @@ def _image_content_candidates(content: str) -> list[tuple[str, str]]: for url in srcset_urls(html.unescape(match[1]))) # HTML attributes are decoded above. Do not also admit their raw entity # spelling as a second bare candidate; plain URL answers remain literal. + from .reference_quality import markdown_links bare_content = re.sub(r"<img\b[^>]*>", "", content, flags=re.I | re.S) + markup_links = markdown_links(bare_content) + for link in markup_links: + if not link.is_image and _looks_like_image_url(link.destination): + candidates.append((link.destination, "content.markdown_image_file")) + for link in reversed(markup_links): + bare_content = bare_content[:link.start] + " " + bare_content[link.end:] # Explicit image markup is typed evidence. A standalone HTTP URL may also # be the entire answer, including an extensionless CDN address. Every URL # with that syntax is admitted alike, without consulting target resources. @@ -174,6 +183,16 @@ def __init__(self) -> None: super().__init__(convert_charrefs=True) self.documents: list[tuple[str, str | None]] = [] + def parse_marked_section(self, i: int, report: int = 1) -> int: + # HTMLParser rejects XML conditional sections that are valid opaque + # declarations in returned documents. Skip their declared extent while + # retaining visible frames both before and after them. + try: + return super().parse_marked_section(i, report) + except AssertionError: + end = self.rawdata.find("]>", i + 3) + return end + 2 if end >= 0 else -1 + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: if tag.casefold() != "iframe": return @@ -188,7 +207,7 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None return source = values.get("srcdoc") if source: - self.documents.append((source, values.get("data-moli-frame-url"))) + self.documents.append((source, values.get("data-frame-url") or values.get("data-moli-frame-url"))) def _inline_frame_candidates( @@ -199,13 +218,15 @@ def _inline_frame_candidates( ) -> list[tuple[str, str]]: """Read resource URLs from visible native ``srcdoc`` serialization. - Moli's official ``--with-frames`` output keeps child documents inside the - returned HTML. This is still provider output; the scorer only resolves - URLs that the browser serialized, and ignores hidden synchronization or ad - frames. The same parser applies to any provider returning inline frames. + Returned inline documents are provider output. Resolve their declared + document URLs and ignore hidden synchronization or advertising frames. + Historical serialized attributes remain readable without rewriting evidence. """ + if not re.search(r'<iframe\b', content, re.I): + return [] parser = _VisibleInlineFrames() parser.feed(content) + parser.close() candidates: list[tuple[str, str]] = [] for document, declared_base in parser.documents[:100]: try: @@ -247,20 +268,19 @@ def _native_json_resource_candidates( *, base_url: str, ) -> list[tuple[str, str]]: - """Read URLs from a browser's native JSON page and network record. - - The accepted surface is deliberately structural: serialized page HTML and - network entries named ``url`` or ``final_url``. It never reads expected - answers to choose a field or invents a URL from a playback identifier. - """ + """Read declared answer fields, keeping request and network logs diagnostic.""" try: payload = json.loads(content) except (json.JSONDecodeError, TypeError): return [] + if isinstance(payload, list): + # A URL list is itself an answer; a list of request/log objects is not. + return [(value, "content.native_json_resource_list") for value in payload + if isinstance(value, str)] if all(isinstance(value, str) for value in payload) else [] if not isinstance(payload, dict): return [] candidates: list[tuple[str, str]] = [] - html = payload.get("html") + html = payload.get("html") or payload.get("content") if isinstance(html, str): candidates.extend( (match[1], "content.native_json_html_url") @@ -275,17 +295,10 @@ def _native_json_resource_candidates( candidates.extend(_inline_frame_candidates( html, base_url=base_url, images_only=False, )) - pending = [payload.get("network")] - while pending: - value = pending.pop() - if isinstance(value, dict): - for key, item in value.items(): - if key in {"url", "final_url"} and isinstance(item, str): - candidates.append((item, f"content.native_json_network.{key}")) - elif isinstance(item, (dict, list)): - pending.append(item) - elif isinstance(value, list): - pending.extend(value) + for key in ("resources", "links", "images", "videos", "media", "extras"): + if key in payload: + candidates.extend(_metadata_resource_candidates( + payload[key], key="links", path=f"content.native_json.{key}")) return candidates @@ -357,17 +370,23 @@ def evaluate_resources( answer_fields = {key: metadata[key] for key in ('links','images','videos','media','extras') if key in metadata} candidates.extend(_metadata_resource_candidates(answer_fields)) if not typed_json_answer: - from .reference_quality import markdown_link_destinations - candidates.extend((url, "content.bare_url") for url in - re.findall(r"https?://[^\s<>'\")\]]+", content)) - candidates.extend((match[1], "content.html_url") for match in - re.findall(r"(?:src|href)\s*=\s*([\"'])(.*?)\1", content, re.I)) - candidates.extend((url, "content.markdown_link") - for url in markdown_link_destinations(content)) - candidates.extend((url, "content.visible_file") - for url in _visible_relative_resource_urls(content)) - candidates.extend(_inline_frame_candidates(content, base_url=base_url, images_only=False)) - candidates.extend(_native_json_resource_candidates(content, base_url=base_url)) + try: + structured = json.loads(content) + except (ValueError, TypeError): + structured = None + if isinstance(structured, (dict, list)): + candidates.extend(_native_json_resource_candidates(content, base_url=base_url)) + else: + from .reference_quality import markdown_link_destinations + candidates.extend((url, "content.bare_url") for url in + re.findall(r"https?://[^\s<>'\")\]]+", content)) + candidates.extend((match[1], "content.html_url") for match in + re.findall(r"(?:src|href)\s*=\s*([\"'])(.*?)\1", content, re.I)) + candidates.extend((url, "content.markdown_link") + for url in markdown_link_destinations(content)) + candidates.extend((url, "content.visible_file") + for url in _visible_relative_resource_urls(content)) + candidates.extend(_inline_frame_candidates(content, base_url=base_url, images_only=False)) normalized_candidates, candidate_evidence = _normalized_candidates(candidates, base_url=base_url) results = [] allowed_urls: set[str] = set() @@ -468,20 +487,8 @@ def task_measured(metrics: dict, task_kind: str | None = None) -> bool: if metrics.get('rule_review_status') == 'unresolved': return False kind = task_kind or metrics.get('task_kind', 'body') - if (metrics.get('page_identity_required') - and metrics.get('page_identity_status') == 'unavailable' - and metrics.get('request_ok') is not False): - # Combined pass flags already include identity. Only independent - # failed requirements can prove failure while identity is unknown. - definite_failure = bool( - metrics.get('deadline_overrun') - or any(error != 'page_identity_not_verified' - for error in metrics.get('stable_task_errors', [])) - or (kind == 'body' and (metrics.get('length_pass') is False - or metrics.get('body_coverage_pass') is False)) - ) - if not definite_failure: - return False + if metrics.get('page_identity_required') and metrics.get('page_identity_status') != 'verified': + return True return kind != 'file' or isinstance(metrics.get('raw_file_pass'), bool) @@ -501,6 +508,9 @@ def normalize_for_match(value: str) -> str: # escapes are presentation syntax, not part of the page's wording. normalized = re.sub(r'\\([!"#$%&\'()*+,\-./:;<=>?@\[\\\]^_`{|}~])', r'\1', normalized) normalized = re.sub(r'`([^`\n]+)`', r'\1', normalized) + # Standalone punctuation emitted with emphasis is formatting, including + # when the following word is adjacent. Preserve numeric/operator syntax. + normalized = re.sub(r'(?<!\w)([_*])([:;,.!?])\1(?=\w)', r'\2 ', normalized) normalized = re.sub(r'\*\*(?=\S)([^*\n]*\S)\*\*', r'\1', normalized) def strip_single_emphasis(match: re.Match) -> str: inner = match.group(2) @@ -567,12 +577,28 @@ def quoted_pair(match): return re.sub(r'(?<=\w) *> *(?=\w)', '>', normalized) -def fact_in_text(text: str, fact: str) -> bool: +# Alphabets whose adjacent letters form a word. CJK keeps declared phrase +# containment because those writing systems do not require word separators. +WORD_CHARACTERS = r'a-z0-9_\u00c0-\u024f\u1e00-\u1eff\u0370-\u052f\u1f00-\u1fff\u2de0-\u2dff\ua640-\ua69f' + + +def _word_letter(char: str) -> bool: + return any(script in unicodedata.name(char, '') for script in ('LATIN', 'GREEK', 'CYRILLIC')) + + +def fact_in_text(text: str, fact: str, *, latin_boundaries: bool = True) -> bool: """Match an explicitly registered normalized fact without fuzzy semantics.""" if not fact: return False left = r'(?<![0-9.+\-])' if fact[0].isdigit() or fact[0] in '$£€¥' else '' right = r'(?![0-9]|\.[0-9])' if fact[-1].isdigit() else '' + # Latin words must not match fragments of longer words. Keep CJK phrase + # matching and the stricter numeric/currency boundaries intact. + latin_word = WORD_CHARACTERS + if latin_boundaries and _word_letter(fact[0]): + left = rf'(?<![{latin_word}])' + if latin_boundaries and _word_letter(fact[-1]): + right = rf'(?![{latin_word}])' for match in re.finditer(left + re.escape(fact) + right, text): if fact[0] in '$£€¥' and text[:match.start()].rstrip().endswith(('-', '+')): continue @@ -581,11 +607,14 @@ def fact_in_text(text: str, fact: str) -> bool: def forbidden_in_text(text: str, phrase: str) -> bool: - """Match ASCII word boundaries without segmenting CJK phrases.""" + """Match Latin word boundaries without segmenting CJK phrases.""" if not phrase: return False - left = r'(?<![a-z0-9_])' if phrase[0].isascii() and phrase[0].isalnum() else '' - right = r'(?![a-z0-9_])' if phrase[-1].isascii() and phrase[-1].isalnum() else '' + latin_word = WORD_CHARACTERS + left = rf'(?<![{latin_word}])' if (phrase[0].isascii() and phrase[0].isalnum() + or _word_letter(phrase[0])) else '' + right = rf'(?![{latin_word}])' if (phrase[-1].isascii() and phrase[-1].isalnum() + or _word_letter(phrase[-1])) else '' return bool(re.search(left + re.escape(phrase) + right, text)) @@ -687,10 +716,12 @@ def evaluate_content( truncation_hits = declared_truncation_hits(answer_text) folded = normalize_for_match(answer_text) content_folded = normalize_for_match(visible_content) - matched = [marker for marker in expected if (text := normalize_for_match(marker)) and text in content_folded] + from .stable_tasks import _stable_literal_match + matched = [marker for marker in expected + if _stable_literal_match(visible_content, marker, phrase_boundaries=True)] facts = required_facts or [] from .dynamic_entry_matching import matching_entry_facts - entry_hits = matching_entry_facts(folded, facts, normalize_for_match, fact_in_text) + entry_hits = matching_entry_facts(content_folded, facts, normalize_for_match, fact_in_text) fact_results = [] from .stable_tasks import _stable_literal_match registered_fact_words = tuple(value for fact in facts @@ -699,17 +730,22 @@ def evaluate_content( for index, fact in enumerate(facts): alternatives = [str(item) for item in fact.get("alternatives", [])] identifier = str(fact.get("id") or f"fact-{index + 1}") + scope = fact.get("match_scope", "body") + if scope not in {"body", "body_or_title"}: + raise ValueError("Unknown required-fact matching scope") + targets = [visible_content] + if scope == "body_or_title": + from .reference_quality import delivered_page_title + targets.append(delivered_page_title(content, title)) hits = (entry_hits[identifier] if identifier in entry_hits else [item for item in alternatives - if (_stable_literal_match(answer_text, item, registered_fact_words) - if re.fullmatch(r'[A-Za-z]+|[\u0600-\u06ff]+', item) - else fact_in_text(folded, normalize_for_match(item)))]) + if any(_stable_literal_match(target, item, registered_fact_words, phrase_boundaries=True) for target in targets)]) fact_results.append({ "id": identifier, "alternatives": alternatives, "matched": hits, "pass": bool(hits), - **({'match_scope': 'entry'} if identifier in entry_hits else {}), + **({'match_scope': 'entry'} if identifier in entry_hits else {'match_scope': scope} if scope != 'body' else {}), }) fact_hits = sum(item["pass"] for item in fact_results) fact_coverage = round(fact_hits / len(fact_results), 4) if fact_results else 1.0 @@ -923,6 +959,8 @@ def summarize(records: list[dict[str, Any]]) -> dict[str, Any]: "mean_content_ms": round(statistics.mean(content_successful_elapsed), 2) if content_successful_elapsed else None, "transport_p50_ms": percentile(transport_successful_elapsed, 0.50), "transport_p95_ms": percentile(transport_successful_elapsed, 0.95), + "mean_all_attempts_ms": round(statistics.mean(elapsed), 2), + "all_attempts_p50_ms": percentile(elapsed, 0.50), "all_attempts_p95_ms": percentile(elapsed, 0.95), "median_content_chars": round( statistics.median(item["metrics"]["content_chars"] for item in items), 1 diff --git a/src/lexbench_webfetch/models.py b/src/lexbench_webfetch/models.py index a569dde..6546579 100644 --- a/src/lexbench_webfetch/models.py +++ b/src/lexbench_webfetch/models.py @@ -48,7 +48,9 @@ class EvalCase: evaluation_url: str = "" page_identity_required: bool = field(default=False, kw_only=True) expected_final_url: str = "" + page_identity_required: bool = False expected_document_language: str = "" + document_language_match: str = "primary" expected_document_language_present: bool = True target_language: str = "" target_region: str = "" diff --git a/src/lexbench_webfetch/reference_quality.py b/src/lexbench_webfetch/reference_quality.py index d07b179..540764c 100644 --- a/src/lexbench_webfetch/reference_quality.py +++ b/src/lexbench_webfetch/reference_quality.py @@ -5,6 +5,21 @@ from html import unescape +# Cover the alphabets whose word boundaries the literal scorer enforces. +_CASE_CODEPOINTS = (*range(0x0250), *range(0x0370, 0x0530), *range(0x1E00, 0x2000), + *range(0x2DE0, 0x2E00), *range(0xA640, 0xA6A0)) +_LOWER_LETTERS = re.escape(''.join(chr(n) for n in _CASE_CODEPOINTS if chr(n).islower())) +_UPPER_LETTERS = re.escape(''.join(chr(n) for n in _CASE_CODEPOINTS if chr(n).isupper())) +_CASE_BOUNDARY = re.compile( + rf'(?<=[{_LOWER_LETTERS}])(?=[{_UPPER_LETTERS}])|' + rf'(?<=[{_UPPER_LETTERS}])(?=[{_UPPER_LETTERS}][{_LOWER_LETTERS}])') + + +def separate_case_labels(text: str) -> str: + """Restore cased label boundaries without splitting lowercase word fragments.""" + return _CASE_BOUNDARY.sub(' ', text) + + HTML_BLOCK_ELEMENTS = frozenset({ "address", "article", "aside", "blockquote", "caption", "colgroup", "dd", "details", "dialog", "div", "dl", "dt", "fieldset", "figcaption", "figure", @@ -45,6 +60,36 @@ def markdown_body_content(text: str) -> str: +def delivered_page_title(content: str, explicit_title: str | None = None) -> str: + """Read a declared title field, never keywords or a description snippet.""" + if explicit_title and explicit_title.strip(): + return explicit_title.strip() + if is_html_document_or_fragment(content): + from bs4 import BeautifulSoup + title = BeautifulSoup(content, "html.parser").find("title") + return title.get_text(" ", strip=True) if title else "" + body = markdown_body_content(content) + if body == content: + return "" + header = content[:len(content)-len(body)] + values = re.findall(r"(?m)^title:[ \t]*([^\r\n]*)$", header) + if len(values) != 1: + return "" + value = values[0].strip() + if value.startswith('"'): + import json + try: + parsed = json.loads(value) + except ValueError: + return "" + return parsed if isinstance(parsed, str) else "" + if value.startswith("'"): + return value[1:-1].replace("''", "'") if value.endswith("'") else "" + if not value or value[0] in "[{|>" or value.casefold() in {"null", "true", "false", "~"}: + return "" + return value.split(" #", 1)[0].strip() + + def markdown_heading_text(text: str) -> str: """Remove ATX presentation markers while retaining visible heading words.""" output = [] @@ -253,7 +298,9 @@ def markdown_links(text: str) -> list[MarkdownLink]: titled = re.match(r"(?s)^(.*?)\s+(['\"])(.*)\2$", raw_destination) destination = re.sub(r"\s+", "", titled[1] if titled else raw_destination) native_title = titled[3] if titled else "" - links.append(MarkdownLink(raw_label, destination, image, index, destination_end + 1, native_title)) + # Character references in a Markdown destination are rendered URL + # syntax. Decode once here; bare URL answers retain their literal bytes. + links.append(MarkdownLink(raw_label, unescape(destination), image, index, destination_end + 1, native_title)) index = destination_end + 1 return links @@ -288,6 +335,9 @@ def visible_markdown_links(text: str) -> str: if match.start < start: continue label = re.sub(r"\s+", " ", visible_markdown_links(match.label)).strip() + # A native carousel can append its accessibility position directly + # to the label. Keep both values but restore their observable boundary. + label = re.sub(r'(?<=[^\W\d_])(?=\d+\s+of\s+\d+$)', ' ', label) # Distinct adjacent link labels retain a token boundary. Some native # serializers also drop a real DOM text boundary immediately before a # path link or a multi-word link label (``out[/r/x]`` / ``or[Stack diff --git a/src/lexbench_webfetch/scoring.py b/src/lexbench_webfetch/scoring.py index 04312a1..3fafbfd 100644 --- a/src/lexbench_webfetch/scoring.py +++ b/src/lexbench_webfetch/scoring.py @@ -44,39 +44,38 @@ def load_list_reference(case: EvalCase | dict[str, Any], source: Path) -> dict | return None -# These conditional outputs must not survive a change of acceptance mode. All -# unconditional content/resource fields are overwritten below. Other evidence -# (e.g. raw_file_*, original reference binding and deadline) is retained, not -# re-certified by text scoring. -_CONDITIONAL_FIELDS = { - "acceptance_contract_sha256", "content_decision_basis", "factual_correctness", - "resource_acceptance_contract_sha256", - "structure_pass", "named_entry_count", "minimum_entries", "snapshot_f1", - "dynamic_titles", "verification_note", "confirmed_entries", "checked_entries", - "page_identity_required", "page_identity_status", "final_url_consistent", - "document_language_required", - "language_consistent", "observed_document_language", - "acceptable_final_urls", - "ordered_tree", "ordered_tree_similarity", - "composite_acceptance_sha256", - "body_requirements_sha256", "required_components", +# Only collection evidence survives an offline score. Derived acceptance fields +# are recomputed, including fields no longer emitted by the current scorer. +# File fingerprints stay readable; a verdict needs current file verification. +_RETAINED_EVIDENCE_FIELDS = { + "reference_sha256", "deadline_overrun", "deadline_budget_ms", "file_evidence", + "raw_file_sha256", + } def _same_page_url(actual: str, expected: str) -> bool: - actual_parts, expected_parts = urlsplit(actual), urlsplit(expected) + if not isinstance(actual, str) or not isinstance(expected, str): + return False + try: + actual_parts, expected_parts = urlsplit(actual), urlsplit(expected) + actual_port = actual_parts.port or {"http": 80, "https": 443}.get(actual_parts.scheme) + expected_port = expected_parts.port or {"http": 80, "https": 443}.get(expected_parts.scheme) + except ValueError: + return False actual_path = actual_parts.path.rstrip("/") or "/" expected_path = expected_parts.path.rstrip("/") or "/" - if ((actual_parts.scheme, actual_parts.hostname, actual_path) - != (expected_parts.scheme, expected_parts.hostname, expected_path)): + if ((actual_parts.scheme, actual_parts.hostname, actual_port, actual_path) + != (expected_parts.scheme, expected_parts.hostname, expected_port, expected_path)): return False # A declared query can select a different page on the same path (for # example, two distinct search results). Compare its decoded parameters; # a fragment does not change the fetched page. - expected_query = parse_qsl(expected_parts.query, keep_blank_values=True) - return (not expected_query or - sorted(parse_qsl(actual_parts.query, keep_blank_values=True)) - == sorted(expected_query)) + def page_query(parts): + return sorted((key, value) for key, value in parse_qsl(parts.query, keep_blank_values=True) + if not key.casefold().startswith('utm_') + and key.casefold() not in {'gclid', 'fbclid'}) + return page_query(actual_parts) == page_query(expected_parts) def score_attempt( @@ -117,16 +116,17 @@ def score_attempt( if (contract.get('minimum_records', 0) == 0 and not (case.get('body_requirements') or {}).get('regions')): raise ValueError('Full-page coverage requires target records or body regions') + language_match = case.get("document_language_match", "primary") + if language_match not in {"primary", "exact"}: + raise ValueError("document_language_match must be primary or exact") from .response_input import answer_input # Live callers forward the conversion completed inside the timed boundary. # Offline callers reconstruct it from immutable evidence by the same function. content, links, ok, status_basis = (prepared_answer if prepared_answer is not None else answer_input(artifact, metadata, ok=ok, status_code=status_code)) result = {key: value for key, value in (previous_metrics or {}).items() - if key not in _CONDITIONAL_FIELDS - and not key.startswith(("stable_", "list_", "page_context_", "article_"))} - for retired in ('acceptance_policy_version', 'forbidden_hits', 'reference_term_exemptions'): - result.pop(retired, None) + if key in _RETAINED_EVIDENCE_FIELDS} + keys = ("required_facts", "forbidden_any", "diagnostic_any", "min_content_chars", "fact_coverage_threshold", "ground_truth_main_content", "required_patterns", "acceptance_contract", "body_requirements", "composite_acceptance") @@ -162,7 +162,6 @@ def score_attempt( normalized_named_link_count=len(links)) dynamic_page_identity = bool( case.get("track") == "dynamic" - and (metadata or {}).get("route") == "dynamic_selected_text" and case.get("evaluation_url") and case.get("expected_final_url") and case.get("target_language") @@ -197,8 +196,10 @@ def score_attempt( language_consistent = ( observed_language == "" if not language_present else bool( isinstance(observed_language, str) and observed_language.strip() - and observed_language.split("-", 1)[0].casefold() - == expected_document_language.split("-", 1)[0].casefold() + and (observed_language.casefold() == expected_document_language.casefold() + if language_match == "exact" else + observed_language.split("-", 1)[0].casefold() + == expected_document_language.split("-", 1)[0].casefold()) ) ) identity_status = ( @@ -233,6 +234,12 @@ def score_attempt( final_url_consistent=final_consistent, acceptable_final_urls=acceptable_urls, ) + elif identity_required: + consistent = bool(final_url and _same_page_url( + final_url, case.get("expected_final_url") or case["url"])) + result.update(page_identity_required=True, final_url_consistent=consistent, + page_identity_status="verified" if consistent else + "mismatch" if final_url else "unavailable") else: result.update(page_identity_required=False, page_identity_status="not_required") result["task_kind"] = task_kind diff --git a/src/lexbench_webfetch/scoring_regressions.py b/src/lexbench_webfetch/scoring_regressions.py index e6695e5..311285c 100644 --- a/src/lexbench_webfetch/scoring_regressions.py +++ b/src/lexbench_webfetch/scoring_regressions.py @@ -2,6 +2,7 @@ from __future__ import annotations import json +import re from pathlib import Path from typing import Any @@ -173,3 +174,27 @@ def valid_correction_provenance(receipt: dict[str, Any]) -> bool: or provenance.get("root_cause_categories") == source_categories) and provenance.get("verdict_corrections") == expected ) + + +def check_scoring_regressions(identifiers): + """Execute declared acceptance controls before applying an input revision.""" + from .scoring import score_attempt + from .metrics import task_passed, task_measured + registry = load_scoring_regressions() + rows = [registry[key] for key in require_scoring_regressions(identifiers)] + if {row['expected_task_pass'] for row in rows} != {False, True}: + raise ValueError('Annotation controls require accepted and rejected scenarios') + for row in rows: + kwargs = {key: row.get(key) for key in ('metadata', 'title', 'final_url')} + try: + metrics = score_attempt(row['case'], row['artifact'].encode(), ok=row['ok'], **kwargs) + except ValueError as error: + if row.get('expected_validation_error') and re.search(row['expected_validation_error'], str(error)): + continue + raise ValueError('Annotation regression failed: ' + row['id']) from error + expected_error = row.get('expected_validation_error') + if (expected_error or task_passed(row['ok'], metrics, row['case'].get('task_kind', 'body')) != row['expected_task_pass'] + or ('expected_task_measured' in row and task_measured(metrics) != row['expected_task_measured']) + or any(metrics.get(key) != value for key, value in row.get('expected_metrics', {}).items())): + raise ValueError('Annotation regression failed: ' + row['id']) + return list(dict.fromkeys(identifiers)) diff --git a/src/lexbench_webfetch/stable_tasks.py b/src/lexbench_webfetch/stable_tasks.py index aa085fb..eaa0744 100644 --- a/src/lexbench_webfetch/stable_tasks.py +++ b/src/lexbench_webfetch/stable_tasks.py @@ -696,17 +696,28 @@ def is_record(url): return None -def _stable_literal_match(surface, value, registered_words=()): +def _stable_literal_match(surface, value, registered_words=(), *, phrase_boundaries=False): """An atomic Latin label is a word, not part of another word. Native camel boundaries and fully covered chains of registered labels preserve adjacent UI text without accepting arbitrary word fragments. Longer declared text retains the existing exact-phrase containment rule. """ - from .metrics import fact_in_text, normalize_for_match + from .metrics import WORD_CHARACTERS, fact_in_text, normalize_for_match + from .reference_quality import visible_markdown_links + surface = visible_markdown_links(surface) + surface = re.sub(r'(?<=[^\W\d_])(?=\d+\s+of\s+\d+\b)', ' ', surface) expected = normalize_for_match(_visible(value)) folded = normalize_for_match(surface) - if not fact_in_text(folded, expected): + from .reference_quality import separate_case_labels as camel + alphabetic_boundaries = (not re.fullmatch(r'[A-Za-z]+', value) + and (phrase_boundaries or bool(re.search(r'[^\x00-\x7f]', value)))) + surfaces = (folded, normalize_for_match(camel(surface))) if alphabetic_boundaries else (folded,) + # Native serializers can join an all-caps preceding label to a complete + # notice. Apply the same case boundary used for atomic labels before the + # Unicode phrase gate, while retaining complete-word matching on both ends. + if not any(fact_in_text(text, expected, latin_boundaries=alphabetic_boundaries) + for text in surfaces): return False # Arabic labels are whitespace-delimited words too. A longer inflected # word cannot supply an independently registered ordinary navigation label. @@ -716,16 +727,14 @@ def _stable_literal_match(surface, value, registered_words=()): if not re.fullmatch(r"[A-Za-z]+", value): return True boundary = lambda needle, text: bool(re.search( - r"(?<![a-z])" + re.escape(needle) + r"(?![a-z])", text)) + rf"(?<![{WORD_CHARACTERS}])" + re.escape(needle) + rf"(?![{WORD_CHARACTERS}])", text)) if boundary(expected, folded): return True - camel = lambda text: re.sub( - r"(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])", " ", text) if boundary(normalize_for_match(camel(value)), normalize_for_match(camel(surface))): return True labels = {normalize_for_match(word) for word in registered_words} - for token in set(re.findall(r"[a-z]+", folded)): + for token in set(re.findall(rf"[{WORD_CHARACTERS}]+", folded)): if expected not in token: continue states = {(0, False)} diff --git a/tests/test_acceptance_isolation.py b/tests/test_acceptance_isolation.py new file mode 100644 index 0000000..f917b5e --- /dev/null +++ b/tests/test_acceptance_isolation.py @@ -0,0 +1,229 @@ +"""Task acceptance must come from delivered content and current file evidence.""" +import pytest + +from lexbench_webfetch.metrics import summarize, task_passed +from lexbench_webfetch.scoring import score_attempt + + +@pytest.mark.parametrize("literal,other", [ + ("Home", "homepage"), ("Live", "deliver"), ("News", "newsletter"), + ("Print", "blueprint"), ("url", "curl"), ("café", "caféteria"), + ("Clone", "Cloned"), ("stats", "statsmodels"), + ("stats", "αstats"), ("stats", "statsя"), + ("Αθήνα", "Αθήνας"), ("новости", "новостик"), +]) +@pytest.mark.parametrize("gate", ["fact", "marker", "anchor"]) +def test_whole_word_requirements_reject_longer_words(literal, other, gate): + case = {"url": "https://example.test/article", "expected_any": []} + if gate == "fact": + case["required_facts"] = [{"id": "answer", "alternatives": [literal]}] + elif gate == "marker": + case["expected_any"] = [literal] + else: + case["acceptance_contract"] = {"version": 1, "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [{"id": "answer", "alternatives": [literal]}]} + for answer, passed in [(other, False), (f"({literal})", True)]: + metrics = score_attempt(case, answer.encode(), ok=True) + assert task_passed(True, metrics) is passed + + +@pytest.mark.parametrize("body,passed", [(b"unrelated text", False), + (b"Install with widget --start.", True), (b"", False)]) +def test_document_title_cannot_supply_a_required_body_fact(body, passed): + case = {"url": "https://example.test/article", "expected_any": [], + "required_facts": [{"id": "installation", "alternatives": ["widget --start"]}]} + metrics = score_attempt(case, body, ok=True, title="widget --start") + assert task_passed(True, metrics) is passed + + +@pytest.mark.parametrize("current_pass", [True, False]) +def test_current_file_validation_supersedes_a_previous_verdict(current_pass): + case = {"url": "https://example.test/file", "task_kind": "file"} + metadata = {"file_evaluation": {"pass": current_pass}} + fresh = score_attempt(case, b"file attached", ok=True, metadata=metadata) + rescored = score_attempt(case, b"file attached", ok=True, metadata=metadata, + previous_metrics={"raw_file_pass": not current_pass, "rule_review_reason": "obsolete"}) + assert rescored == fresh + assert task_passed(True, rescored) is current_pass + + +def test_removed_derived_metrics_do_not_survive_rescoring(): + case = {"url": "https://example.test/article", "expected_any": ["Article"]} + fresh = score_attempt(case, b"Article", ok=True) + rescored = score_attempt(case, b"Article", ok=True, previous_metrics={ + "retired_acceptance_pass": True, "rule_review_reason": "obsolete", + "composite_acceptance_pass": False, "ordered_tree_similarity": 1.0}) + assert rescored == fresh + + +def test_a_legacy_file_verdict_without_current_file_evidence_cannot_certify_a_rescore(): + case = {"url": "https://example.test/file", "task_kind": "file"} + metrics = score_attempt(case, b"file attached", ok=True, + previous_metrics={"raw_file_pass": True, "raw_file_sha256": "legacy-hash"}) + assert "raw_file_pass" not in metrics + assert metrics["raw_file_sha256"] == "legacy-hash" + assert not task_passed(True, metrics) + + +def test_rescoring_preserves_collection_identity_and_deadline(): + case = {"url": "https://example.test/article"} + collected = {"reference_sha256": "frozen-capture", "deadline_overrun": True, + "deadline_budget_ms": 45000, + "raw_file_sha256": "verified-file", "file_evidence": {"sha256": "verified-file"}} + metrics = score_attempt(case, b"Article", ok=True, previous_metrics=collected) + assert all(metrics[k] == v for k, v in collected.items()) + assert not task_passed(True, metrics) + + +def test_all_call_latency_includes_fast_success_slow_failure_and_timeout(): + rows = [] + for index, (elapsed, answer, ok) in enumerate([ + (10, b"Article", True), (1000, b"Wrong page", True), (45000, b"", False)]): + case = {"url": "https://example.test/article", "expected_any": ["Article"]} + rows.append({"provider": "browser", "provider_type": "local", "case_id": str(index), + "case_name": "Article", "track": "web", "category": "article", + "ok": ok, "elapsed_ms": elapsed, + "metrics": score_attempt(case, answer, ok=ok)}) + metrics = summarize(rows)["providers"]["browser"] + assert metrics["mean_content_ms"] == 10 + assert metrics["mean_all_attempts_ms"] == 15336.67 + assert metrics["all_attempts_p50_ms"] == 1000 + assert metrics["all_attempts_p95_ms"] == 45000 + + +@pytest.mark.parametrize("track", ["web", "dynamic", "risk_control"]) +@pytest.mark.parametrize("route", [None, "dynamic_selected_text", "another_interface"]) +@pytest.mark.parametrize("destination,passed", [ + ("https://example.test/article", True), ("https://example.test/wrong", False), (None, False)]) +def test_declared_page_identity_is_required_on_every_interface(track, route, destination, passed): + case = {"url": "https://example.test/article", "track": track, + "expected_any": ["Article"], "page_identity_required": True} + if track == "dynamic": + case.update(evaluation_url=case["url"], expected_final_url=case["url"], + target_language="en", target_region="US") + if track == "risk_control": + case["source_benchmark"] = {"risk_control": {"navigation_contract": { + "requested_url": case["url"], "observed_final_url": case["url"]}}} + metadata = {"document_language": "en"} + if route: + metadata["route"] = route + metrics = score_attempt(case, b"Article", ok=True, metadata=metadata, final_url=destination) + assert task_passed(True, metrics) is passed + + +@pytest.mark.parametrize("declaration", ["<![%draft;[\n<!ELEMENT note ANY>\n]]>", + "<![UNRECOGNIZED[ XML source ]]>"]) +def test_xml_declarations_cannot_crash_body_or_resource_scoring(declaration): + body_case = {"url": "https://example.test/spec", "expected_any": ["Specification"]} + raw = ("Specification\n" + declaration).encode() + assert task_passed(True, score_attempt(body_case, raw, ok=True)) + resource_case = {**body_case, "task_kind": "resource", + "expected_resources": [{"id": "image", "urls": ["/target.png"]}]} + for answer, passed in [(declaration + '\n<iframe srcdoc="example"></iframe>\n' + + '[Target](/target.png)', True), (declaration + '\n<iframe></iframe>', False)]: + metrics = score_attempt(resource_case, answer.encode(), ok=True) + assert task_passed(True, metrics) is passed + + +@pytest.mark.parametrize("destination,passed", [ + ("https://example.test:443/article/#section", True), + ("https://example.test/article?utm_source=link", True), + ("https://example.test/article?q=another", False), + ("https://example.test:8443/article", False), + ("https://example.test:invalid/article", False)]) +def test_identity_distinguishes_page_queries_and_ports(destination, passed): + case = {"url": "https://example.test/article", "page_identity_required": True} + assert task_passed(True, score_attempt(case, b"Article", ok=True, final_url=destination)) is passed + + +@pytest.mark.parametrize("mode,language,passed", [ + ("primary", "zh-TW", True), ("exact", "zh-TW", False), ("exact", "zh-CN", True)]) +def test_tasks_can_require_the_complete_document_locale(mode, language, passed): + case = {"url": "https://example.test/article", "track": "dynamic", + "evaluation_url": "https://example.test/article", "expected_final_url": "https://example.test/article", + "target_language": "zh-CN", "target_region": "CN", "page_identity_required": True, + "document_language_match": mode} + metrics = score_attempt(case, b"Article", ok=True, final_url=case["url"], + metadata={"document_language": language}) + assert task_passed(True, metrics) is passed + + +@pytest.mark.parametrize("surface", ["native_json", "nested_metadata", "json_log_list"]) +@pytest.mark.parametrize("delivered", [False, True]) +def test_resource_answers_exclude_network_diagnostics(surface, delivered): + import json + target = "https://example.test/target.mp4" + case = {"url": "https://example.test/page", "task_kind": "resource", + "content_score_applicable": False, + "expected_resources": [{"id": "video", "urls": [target]}]} + diagnostics = {"network": [{"url": target, "final_url": target}]} + if surface in {"native_json", "json_log_list"}: + payload = {**diagnostics, "html": "<main>Article</main>"} + if delivered: + payload["resources"] = [target] + if surface == "json_log_list" and not delivered: + payload = diagnostics["network"] + artifact, metadata = json.dumps(payload).encode(), {} + else: + artifact = b"No resource URL in the answer" + metadata = {"extras": diagnostics} + if delivered: + metadata["videos"] = [target] + metrics = score_attempt(case, artifact, ok=True, metadata=metadata) + assert task_passed(True, metrics) is delivered + assert bool(metrics["resource_hits"]) is delivered + + +@pytest.mark.parametrize("answer", ["caféteria", "décafé"]) +def test_forbidden_latin_words_do_not_match_fragments(answer): + case = {"url": "https://example.test/page", "forbidden_any": ["café"]} + metrics = score_attempt(case, answer.encode(), ok=True) + assert task_passed(True, metrics) + assert not task_passed(True, score_attempt(case, b"A caf\xc3\xa9.", ok=True)) + + +@pytest.mark.parametrize("kind", ["body", "resource", "file"]) +def test_required_missing_identity_is_a_failure_in_the_full_population(kind): + from lexbench_webfetch.metrics import task_measured + case = {"url": "https://example.test/page", "page_identity_required": True, "task_kind": kind} + metrics = score_attempt(case, b"Article", ok=True) + assert task_measured(metrics) + assert not task_passed(True, metrics) + + +@pytest.mark.parametrize("attribute", ["data-frame-url", "data-moli-frame-url"]) +def test_visible_frame_relative_resources_use_the_declared_document_url(attribute): + case = {"url": "https://example.test/page", "task_kind": "resource", + "expected_resources": [{"id": "frame", "urls": ["https://frames.test/path/clip.mp4"]}]} + artifact = (f'<iframe {attribute}="https://frames.test/path/index.html" ' + 'srcdoc="<video src="clip.mp4"></video>"></iframe>').encode() + assert task_passed(True, score_attempt(case, artifact, ok=True)) + + +@pytest.mark.parametrize("returned,passed", [("/target.mp4", True), ("/other.mp4", False)]) +def test_a_returned_json_url_list_remains_a_resource_answer(returned, passed): + import json + case = {"url": "https://example.test/page", "task_kind": "resource", + "expected_resources": [{"id": "video", "urls": ["/target.mp4"]}]} + assert task_passed(True, score_attempt(case, json.dumps([returned]).encode(), ok=True)) is passed + + +@pytest.mark.parametrize("declaration", ["<?xml version=\"1.0\"?>", + "<![UNRECOGNIZED[ XML source ]]>", "<![%draft;[<!ELEMENT note ANY>]]>"]) +@pytest.mark.parametrize("position", ["before", "after"]) +def test_xml_declarations_preserve_visible_frame_resource_answers(declaration, position): + case = {"url": "https://example.test/spec", "task_kind": "resource", + "expected_resources": [{"id": "image", "urls": ["/target.png"]}]} + for filename, passed in [("target.png", True), ("other.png", False)]: + frame = '<iframe srcdoc="<img src="/' + filename + '">"></iframe>' + answer = declaration + frame if position == "before" else frame + declaration + assert task_passed(True, score_attempt(case, answer.encode(), ok=True)) is passed + + +@pytest.mark.parametrize("literal,fragment", [("stats", "αstats"), ("stats", "statsя"), + ("Αθήνα", "Αθήνας"), ("новости", "новостик")]) +def test_forbidden_words_use_the_same_alphabetic_boundaries(literal, fragment): + case = {"url": "https://example.test/article", "forbidden_any": [literal]} + for answer, passed in [(fragment, True), ('(' + literal + ')', False)]: + assert task_passed(True, score_attempt(case, answer.encode(), ok=True)) is passed diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index e0305bd..5b4e2bc 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -44,6 +44,20 @@ def test_file_acceptance_metadata_is_public_without_capture_payloads(self): self.assertEqual(public['source_verification'], {'mime': 'image/png', 'file_sha256': 'a'*64}) self.assertNotIn('private source capture', json.dumps(public)) + def test_registered_revision_keeps_original_suites_and_has_new_dataset_ids(self): + import copy + revision = bench.read(bench.SUITES/'annotation-revisions/serialization-20261021.json') + changed = 0 + for identifier in revision['parents']: + _, root = bench.suite(identifier) + data = bench.read(root/'tasks.json');before=copy.deepcopy(data) + new = bench.apply_annotation_revision(identifier, data, revision['id']) + self.assertEqual(data, before) + self.assertNotEqual(new['id'], data['id']) + self.assertEqual(len(new['cases']), len(data['cases'])) + changed += sum(a != b for a,b in zip(new['cases'],data['cases'])) + self.assertEqual(changed, len(revision['changes'])) + def test_unknown_suite_is_rejected(self): with self.assertRaisesRegex(ValueError, 'Unknown suite'): bench.validate('not-a-suite') @@ -96,6 +110,101 @@ def test_complete_inputs_export_without_private_extras(self): {'dataset.json', 'sources/article.json'}) self.assertEqual(bench.verify_inputs('synthetic', output)['case_count'], 1) + def annotation_revision(self): + from lexbench_webfetch.scoring_regressions import load_scoring_regressions + task = bench.read(self.suites/'synthetic/tasks.json')['cases'][0] + encoded = json.dumps(task, ensure_ascii=False, sort_keys=True, separators=(',', ':')).encode() + data = {'schema_version': 1, 'kind': 'score_only_annotation_revision', + 'id': 'source-annotation-v2', 'parents': {'synthetic': { + 'tasks_sha256': bench.digest(self.suites/'synthetic/tasks.json'), + 'source_inputs_sha256': bench.digest(self.suites/'synthetic/source-inputs.json')}}, + 'regression_ids': [next(iter(load_scoring_regressions()))], + 'changes': [{'suite': 'synthetic', 'case_id': 'article', + 'before_sha256': hashlib.sha256(encoded).hexdigest(), + 'set': {'required_facts': [{'id': 'main', 'alternatives': ['unique fact']}]}, + 'remove': []}]} + revision_path = self.suites/'annotation-revisions/source-annotation-v2.json' + write(revision_path, data) + delta = [{'path': '/required_facts', 'after': [{'id': 'main', 'alternatives': ['unique fact']}]}] + data['regression_ids'] = ['source-backed-diagnostic-source-discusses-status', + 'source-backed-diagnostic-status-heading'] + write(revision_path, data) + write(self.suites/'annotation-revisions/reviews.json', {'schema_version': 1, 'revisions': { + data['id']: {'revision_sha256': bench.digest(revision_path), 'changes': [{ + 'suite': 'synthetic', 'case_id': 'article', + 'reason': 'The frozen independent reference explicitly requires unique fact.', + 'changes_sha256': hashlib.sha256(json.dumps(delta, ensure_ascii=False, sort_keys=True, separators=(',', ':')).encode()).hexdigest(), + 'sources': [{'path': 'sources/article.json', + 'sha256': bench.digest(self.source/'sources/article.json'), + 'scope': 'Independent synthetic article reference'}], + 'regression_ids': data['regression_ids']}]} }}) + return data + + def test_revised_export_has_a_distinct_identity_and_preserves_frozen_inputs(self): + self.annotation_revision() + before = (self.source/'dataset.json').read_bytes() + path = bench.export('synthetic', self.source, self.root/'revised', revision='source-annotation-v2') + revised = bench.read(path) + self.assertEqual(revised['id'], 'test--source-annotation-v2') + self.assertEqual(revised['definition_revision']['parent_dataset_id'], 'test') + self.assertEqual(revised['cases'][0]['required_facts'][0]['alternatives'], ['unique fact']) + review = bench.read(path.parent/'annotation-review.json') + self.assertEqual(review['changes'][0]['changes'][0]['path'], '/required_facts') + self.assertTrue(review['changes'][0]['sources']) + self.assertTrue(review['reference_controls']['complete']) + self.assertEqual(review['reference_controls']['cases'][0]['status'], 'passed') + self.assertEqual((self.source/'dataset.json').read_bytes(), before) + self.assertEqual(bench.verify_inputs('synthetic', self.source)['case_count'], 1) + old = bench.export('synthetic', self.source, self.root/'unchanged') + self.assertEqual(old.read_bytes(), before) + + def test_revised_export_rejects_wrong_parent_unknown_controls_and_collection_changes(self): + import copy + definition = self.annotation_revision() + for defect in ['parent', 'task', 'controls', 'collection']: + with self.subTest(defect=defect): + bad = copy.deepcopy(definition) + if defect == 'parent': bad['parents']['synthetic']['tasks_sha256'] = '0'*64 + elif defect == 'task': bad['changes'][0]['before_sha256'] = '0'*64 + elif defect == 'controls': bad['regression_ids'] = ['unknown'] + else: bad['changes'][0]['set']['url'] = 'https://other.test/' + write(self.suites/'annotation-revisions/source-annotation-v2.json', bad) + output = self.root/defect + with self.assertRaises(ValueError): + bench.export('synthetic', self.source, output, revision='source-annotation-v2') + self.assertFalse(output.exists()) + + def test_revised_export_rejects_unreviewed_or_unproven_changes(self): + import copy + definition = self.annotation_revision() + for defect in ('missing-review', 'diff', 'source', 'missing-source-binding', 'reason', 'one-direction', 'failing-control'): + with self.subTest(defect=defect): + self.annotation_revision() + review_path = self.suites/'annotation-revisions/reviews.json' + review = bench.read(review_path) if review_path.exists() else {'revisions': {}} + if defect == 'missing-review': + if review_path.exists(): review_path.unlink() + elif review['revisions']: + item = review['revisions']['source-annotation-v2']['changes'][0] + if defect == 'diff': item['changes_sha256'] = '0'*64 + elif defect == 'source': item['sources'][0]['sha256'] = '0'*64 + elif defect == 'missing-source-binding': item['sources'] = [{'scope': 'No bound input'}] + elif defect == 'reason': item['reason'] = '' + elif defect == 'one-direction': item['regression_ids'] = ['source-backed-diagnostic-status-heading'] + write(review_path, review) + output = self.root/('invalid-'+defect) + if defect == 'failing-control': + from lexbench_webfetch.scoring_regressions import load_scoring_regressions + controls = copy.deepcopy(load_scoring_regressions()) + controls['source-backed-diagnostic-source-discusses-status']['expected_task_pass'] = False + with patch('lexbench_webfetch.scoring_regressions.load_scoring_regressions', return_value=controls): + with self.assertRaises(ValueError): + bench.export('synthetic', self.source, output, revision=definition['id']) + else: + with self.assertRaises(ValueError): + bench.export('synthetic', self.source, output, revision=definition['id']) + self.assertFalse(output.exists()) + def test_existing_output_is_preserved(self): output = self.root / 'existing' output.mkdir() @@ -181,5 +290,48 @@ def test_escaping_paths_are_rejected(self): bench.contained_path(self.source, value, reject_symlinks=True) + +class ReferenceControls(unittest.TestCase): + def test_reference_checks_reject_missing_required_content_and_report_missing_sources(self): + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) + case = {'id': 'article', 'url': 'https://example.test/', + 'required_facts': [{'id': 'body', 'alternatives': ['substantive article']}], + 'ground_truth_main_content': 'substantive article'} + review = {'changes': [{'case_id': 'article', 'sources': [{'path': 'dataset.json'}]}]} + result = bench.check_revision_references({'cases': [case]}, review, source) + self.assertTrue(result['complete']) + self.assertFalse(result['cases'][0]['error_page_passed']) + case['ground_truth_main_content'] = 'Home Terms Privacy' + self.assertEqual(bench.check_revision_references({'cases': [case]}, review, source)['cases'][0]['status'], 'failed') + del case['ground_truth_main_content'] + missing = bench.check_revision_references({'cases': [case]}, review, source) + self.assertFalse(missing['complete']) + self.assertEqual(missing['cases'][0]['status'], 'unavailable') + + def test_missing_reference_cannot_hide_an_accepted_error_page(self): + with tempfile.TemporaryDirectory() as directory: + case = {'id': 'unchecked', 'url': 'https://example.test/', 'min_content_chars': 1} + review = {'changes': [{'case_id': 'unchecked', 'sources': [{'path': 'dataset.json'}]}]} + result = bench.check_revision_references({'cases': [case]}, review, Path(directory)) + self.assertTrue(result['cases'][0]['error_page_passed']) + self.assertEqual(result['cases'][0]['status'], 'failed') + self.assertFalse(result['complete']) + + def test_media_reference_uses_observed_urls_in_an_image_answer(self): + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) + digest = write(source / 'observed.json', {'resource_candidates': [ + {'tag': 'a', 'attribute': 'href', 'url': 'https://example.test/download/42'}]}) + case = {'id': 'image', 'url': 'https://example.test/', 'task_kind': 'resource', + 'content_score_applicable': False, + 'expected_resources': [{'id': 'image', 'kind': 'image', 'urls': ['https://example.test/download/42']}], + 'resource_output_contract': {'candidate_scope': 'image_answer_fields', 'unexpected': 'allow'}} + review = {'changes': [{'case_id': 'image', 'sources': [{'path': 'observed.json', 'sha256': digest}]}]} + self.assertTrue(bench.check_revision_references({'cases': [case]}, review, source)['complete']) + case['expected_resources'][0]['urls'] = ['https://example.test/download/99'] + self.assertFalse(bench.check_revision_references({'cases': [case]}, review, source)['complete']) + + if __name__ == '__main__': unittest.main() diff --git a/tests/test_card_preludes.py b/tests/test_card_preludes.py new file mode 100644 index 0000000..91c8901 --- /dev/null +++ b/tests/test_card_preludes.py @@ -0,0 +1,19 @@ +"""Declared card starts own fields before a linked title without borrowing peers.""" +import pytest +from lexbench_webfetch.scoring import score_attempt +from lexbench_webfetch.metrics import task_passed + +CASE={'url':'https://example.test/','composite_acceptance':{'version':1,'groups':[{ + 'id':'events','kind':'named_link','hosts':['example.test'],'entry_path':r'/item/[0-9]+', + 'minimum_records':1,'record_start_pattern':r'(?m)^- 2026-10-[0-9]{2}$', + 'required_fields':[{'id':'date','kind':'date_or_time'}, + {'id':'venue','kind':'text','aliases':['Venue']}, + {'id':'region','kind':'text','aliases':['Region']}]}]}} + +@pytest.mark.parametrize('body,expected',[ + ('- 2026-10-16\nVenue: Hall A\nRegion: Alabama\n[Concert](/item/1)',True), + ('- 2026-10-16\nVenue: Hall A\nRegion: Alabama\n- 2026-10-17\n[Concert](/item/1)',False), + ('- 2026-10-16\nVenue: Hall A\n[Concert](/item/1)\n- 2026-10-17\nRegion: Alabama\n[Other](/item/2)',False), +]) +def test_fields_before_link_belong_only_to_the_declared_card(body,expected): + assert task_passed(True,score_attempt(CASE,body.encode(),ok=True)) is expected diff --git a/tests/test_declared_card_labels.py b/tests/test_declared_card_labels.py new file mode 100644 index 0000000..a8dcf86 --- /dev/null +++ b/tests/test_declared_card_labels.py @@ -0,0 +1,15 @@ +import pytest +from lexbench_webfetch.scoring import score_attempt +from lexbench_webfetch.metrics import task_passed + +CASE={'url':'https://example.test/','composite_acceptance':{'version':1,'groups':[{'id':'events','kind':'named_link','hosts':['example.test'],'entry_path':r'/item/[0-9]+','minimum_records':1,'markdown_label_pattern':r'(?m)^(?:Today\n)?2026-10-16\n(?P<label>[^\n]+)\n','required_fields':[{'id':'date','kind':'date_or_time'},{'id':'venue','kind':'text','minimum_chars':2},{'id':'region','kind':'text','pattern':r',\s*AL\b'}]}]}} +@pytest.mark.parametrize('label,passed',[ + ('Today\n2026-10-16\nEvening Concert\nExample Hall Tuscaloosa, AL',True), + ('2026-10-16\nEvening Concert\nExample Hall Tuscaloosa, AL',True), + ('Today\n2026-10-16\nExample Hall Tuscaloosa, AL',False), + ('Today\n2026-10-16\nEvening Concert\n, AL',False), + ('Today\n2026-10-16\nEvening Concert\nExample Hall',False), +]) +def test_declared_multiline_card_label_and_owned_fields(label,passed): + result=score_attempt(CASE,('[![](/cover.jpg)\n'+label+'](/item/1)').encode(),ok=True) + assert task_passed(True,result) is passed diff --git a/tests/test_information_title_scope.py b/tests/test_information_title_scope.py new file mode 100644 index 0000000..0143507 --- /dev/null +++ b/tests/test_information_title_scope.py @@ -0,0 +1,36 @@ +"""Information requests may use a delivered page title, never unrelated metadata.""" +import pytest +from lexbench_webfetch.scoring import score_attempt +from lexbench_webfetch.metrics import task_passed + + +def evaluate(output,scope='body_or_title',title=None): + return score_attempt({'url':'https://example.test/video','required_facts':[ + {'id':'video-title','alternatives':['Example video'],'match_scope':scope}, + {'id':'publisher','alternatives':['Example Channel']} ]},output.encode(),ok=True,title=title) + +@pytest.mark.parametrize('output,title',[ + ('Example video\nExample Channel',None), + ('---\ntitle: "Example video - Player"\n---\nExample Channel',None), + ("---\ntitle: 'Example video'\n---\nExample Channel",None), + ('<html><head><title>Example videoExample Channel',None), + ('Example Channel','Example video'), +]) +def test_declared_title_information_can_use_a_real_page_title(output,title): + assert task_passed(True,evaluate(output,title=title)) + +@pytest.mark.parametrize('output',[ + '---\ntitle: "Other video"\nkeywords: "Example video"\n---\nExample Channel', + '---\ntitle: "Example video"\nkeywords: "Example Channel"\n---\nOther body', + '---\ntitle: "Example video"\ntitle: "Other video"\n---\nExample Channel', +]) +def test_keywords_or_ambiguous_metadata_cannot_supply_information(output): + assert not task_passed(True,evaluate(output)) + + +def test_body_requirement_still_rejects_title_only_evidence(): + assert not task_passed(True,evaluate('---\ntitle: "Example video"\n---\nExample Channel',scope='body')) + + +def test_fact_cannot_be_joined_across_body_and_title(): + assert not task_passed(True,evaluate('Example Channel\nExample',title='video')) diff --git a/tests/test_named_scope_heading.py b/tests/test_named_scope_heading.py new file mode 100644 index 0000000..9007d31 --- /dev/null +++ b/tests/test_named_scope_heading.py @@ -0,0 +1,29 @@ +"""A section heading owns records; a matching navigation label does not.""" +from lexbench_webfetch.scoring import score_attempt +from lexbench_webfetch.metrics import task_passed + +CASE = {'url': 'https://example.test/search', 'acceptance_contract': { + 'version': 1, 'schema': 'named_link', 'minimum_records': 2, + 'hosts': ['example.test'], 'entry_path': r'/item/[0-9]+', + 'entry_scope': {'html_selector': '.results a[href]', + 'start_pattern': '^Find games$', 'end_pattern': '^See all'}}} + + +def test_section_heading_beats_the_same_label_in_navigation(): + text = 'Find games\n[Help](/help)\nSee all\n\n### Find games\n[Alpha](/item/1)\n[Beta](/item/2)\nSee all' + assert task_passed(True, score_attempt(CASE, text.encode(), ok=True)) + + +def test_section_cannot_borrow_records_from_navigation_or_after_its_end(): + text = 'Find games\n[Alpha](/item/1)\n[Beta](/item/2)\nSee all\n\n### Find games\n[Gamma](/item/3)\nSee all\n[Delta](/item/4)' + assert not task_passed(True, score_attempt(CASE, text.encode(), ok=True)) + + +def test_nested_image_card_does_not_use_empty_list_markup_as_its_name(): + text = '### Find games\n- [![](/cover.jpg)\n\n - **\n\nAlpha game\n2026.3.26\n8,500円\nPublisher\n**](/item/1)\n- [Beta](/item/2)\nSee all' + assert task_passed(True, score_attempt(CASE, text.encode(), ok=True)) + + +def test_empty_card_with_only_artwork_and_layout_has_no_visible_name(): + text = '### Find games\n- [![](/cover.jpg)\n\n - **\n\n2026.3.26\n**](/item/1)\n- [Beta](/item/2)\nSee all' + assert not task_passed(True, score_attempt(CASE, text.encode(), ok=True)) diff --git a/tests/test_native_card_fields.py b/tests/test_native_card_fields.py new file mode 100644 index 0000000..d89731d --- /dev/null +++ b/tests/test_native_card_fields.py @@ -0,0 +1,22 @@ +import pytest +from lexbench_webfetch.scoring import score_attempt +from lexbench_webfetch.metrics import task_passed + +FIELDS=[{'id':'price','kind':'price'},{'id':'bedrooms','kind':'integer','pattern':r'\bbds\b'},{'id':'bathrooms','kind':'integer','pattern':r'\bba\b'},{'id':'area','kind':'area'}] +def score(body, fields=FIELDS): + return score_attempt({'url':'https://example.test/','min_content_chars':1,'composite_acceptance':{'version':1,'groups':[{'id':'cards','kind':'named_link','hosts':['example.test'],'entry_path':r'/item/[0-9]+','minimum_records':1,'required_fields':fields}]}},body.encode(),ok=True) + +@pytest.mark.parametrize('body,expected',[ + ('- [$900](/item/1)\n\n- **3** bds\n- **2** ba\n- **900** sqft\n\nActive[House A](/item/1)',True), + ('- [House A](/item/1)\n\n- [House B](/item/2)\n- $900\n- 3 bds\n- 2 ba\n',False), + ('- [House A](/item/1)\n\n# Other section\n- $900\n- 3 bds\n- 2 ba\n- 900 sqft',False), + ('- [House A](/item/1)\n\n- unrelated prose\n- $900\n- 3 bds\n- 2 ba\n- 900 sqft',False), +]) +def test_owned_field_bullets(body,expected): + assert task_passed(True,score(body)) is expected + +def test_text_pattern_consumes_distinct_spans_without_losing_lookahead(): + fields=[{'id':'venue','kind':'text','pattern':r'^Hall A(?=\s+Town, TX$)'},{'id':'city','kind':'text','pattern':r'Town, TX$'}] + assert task_passed(True,score('[Event](/item/1)\nHall A Town, TX',fields)) + assert not task_passed(True,score('[Event](/item/1)\nHall A',fields)) + assert not task_passed(True,score('[Event](/item/1)\nTown, TX',fields)) diff --git a/tests/test_resource_link_serialization.py b/tests/test_resource_link_serialization.py new file mode 100644 index 0000000..b471e8a --- /dev/null +++ b/tests/test_resource_link_serialization.py @@ -0,0 +1,32 @@ +"""Rendered link syntax must not change the identity or count of a resource.""" +import pytest + +from lexbench_webfetch.scoring import score_attempt +from lexbench_webfetch.metrics import task_passed + + +TARGET = 'https://cdn.example.test/photo.jpg?width=640&height=480' + + +def score(output, target=TARGET): + return score_attempt({'url': 'https://example.test/gallery', 'task_kind': 'resource', + 'expected_resources': [{'id': 'photo', 'urls': [target]}], + 'resource_output_contract': {'candidate_scope': 'image_answer_fields', 'unexpected': 'reject'}}, + output.encode(), ok=True) + + +@pytest.mark.parametrize('template', ['![Photo]({url})', '[Download]({url})', '']) +def test_entity_escaped_link_destination_is_one_resource(template): + result = score(template.format(url=TARGET.replace('&', '&'))) + assert task_passed(True, result) + assert result['resource_candidate_urls'] == [TARGET] + assert result['resource_precision'] == 1.0 + + +@pytest.mark.parametrize('output', [ + '![Photo](https://cdn.example.test/photo.jpg?width=641&height=480)', + 'https://cdn.example.test/photo.jpg?width=640&height=480', + '![Photo](https://cdn.example.test/photo.jpg?width=640%26amp%3Bheight=480)', +]) +def test_markup_decoding_does_not_equate_different_or_literal_url_values(output): + assert not task_passed(True, score(output)) diff --git a/tests/test_source_language_policy.py b/tests/test_source_language_policy.py new file mode 100644 index 0000000..15043ed --- /dev/null +++ b/tests/test_source_language_policy.py @@ -0,0 +1,33 @@ +"""Maintainer code uses English; original-language task evidence stays intact.""" +import ast +import io +from pathlib import Path +import re +import tokenize + +import lexbench_webfetch + + +HAN = re.compile(r"[\u3400-\u9fff]") + + +def language_findings(source): + findings = [token.start[0] for token in tokenize.generate_tokens(io.StringIO(source).readline) + if token.type in (tokenize.NAME, tokenize.COMMENT) and HAN.search(token.string)] + for node in ast.walk(ast.parse(source)): + if isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + if HAN.search(ast.get_docstring(node) or ""): + findings.append(getattr(node, 'lineno', 1)) + return findings + + +def test_original_language_evidence_is_allowed_but_maintainer_text_is_english(): + assert language_findings('value = "原文引述"\n') == [] + assert language_findings('"""维护说明"""\n# 实现注释\n结果 = "原文引述"\n') == [2, 3, 1] + + +def test_installed_package_identifiers_comments_and_docstrings_use_english(): + root = Path(lexbench_webfetch.__file__).resolve().parent + findings = [(path.relative_to(root), line) for path in root.rglob('*.py') + for line in language_findings(path.read_text(encoding='utf-8'))] + assert findings == [] diff --git a/tests/test_stable_text_serialization.py b/tests/test_stable_text_serialization.py new file mode 100644 index 0000000..c8529d9 --- /dev/null +++ b/tests/test_stable_text_serialization.py @@ -0,0 +1,56 @@ +"""Stable notices survive native text concatenation without accepting word fragments.""" +import pytest + +from lexbench_webfetch.metrics import task_passed +from lexbench_webfetch.scoring import score_attempt + + +@pytest.mark.parametrize('prefix', ['', 'ALL RIGHTS RESERVED', 'TODOS LOS DERECHOS RESERVADOS']) +def test_complete_notice_after_an_adjacent_capitalized_label(prefix): + notice = 'Example Corp. Avenida N°5, Santiago de Chile' + case = {'url': 'https://example.test/', 'acceptance_contract': { + 'version': 1, 'schema': 'body_text', 'minimum_records': 0, + 'stable_text_anchors': [{'id': 'notice', 'alternatives': [notice]}]}} + result = score_attempt(case, (prefix + notice).encode(), ok=True) + assert task_passed(True, result) + + +@pytest.mark.parametrize('notice', [ + 'Examples Corp. Avenida N°5, Santiago de Chile', + 'Example Corp. Avenida N°5, Santiago de Chilean', + 'Example Corp. Avenida N°6, Santiago de Chile', +]) +def test_adjacent_notice_still_requires_the_complete_declared_words(notice): + case = {'url': 'https://example.test/', 'acceptance_contract': { + 'version': 1, 'schema': 'body_text', 'minimum_records': 0, + 'stable_text_anchors': [{'id': 'notice', 'alternatives': [ + 'Example Corp. Avenida N°5, Santiago de Chile']} ]}} + result = score_attempt(case, ('ALL RIGHTS RESERVED' + notice).encode(), ok=True) + assert not task_passed(True, result) + + +@pytest.mark.parametrize('gate', ['fact', 'anchor']) +@pytest.mark.parametrize('expected,output', [ + ('In stock', 'In stockAdd to basket'), + ('Наука', '[Технологии](/technology)[Наука](/science)Все темы'), + ('Careers', '[Careers9 of 9](/careers)'), + ('toute l’actu dès 7h30', '_:_toute l’actu dès 7h30'), +]) +def test_native_label_boundaries_are_preserved_for_facts_and_anchors(gate, expected, output): + case = {'url': 'https://example.test/'} + if gate == 'fact': + case['required_facts'] = [{'id': 'label', 'alternatives': [expected]}] + else: + case['acceptance_contract'] = {'version': 1, 'schema': 'body_text', 'minimum_records': 0, + 'stable_text_anchors': [{'id': 'label', 'alternatives': [expected]}]} + assert task_passed(True, score_attempt(case, output.encode(), ok=True)) + + +@pytest.mark.parametrize('expected,output', [ + ('In stock', 'In stockholders'), ('Наука', '[Наукам](/science)'), + ('Careers', '[Careers9](/careers)'), ('stats', 'statsmodels'), +]) +def test_native_label_recovery_does_not_accept_a_different_word(expected, output): + case = {'url': 'https://example.test/', 'required_facts': [ + {'id': 'label', 'alternatives': [expected]}]} + assert not task_passed(True, score_attempt(case, output.encode(), ok=True)) diff --git a/uv.lock b/uv.lock index 60c24c1..8ddc0c5 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 5 +revision = 2 requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.11.5'", @@ -48,7 +48,7 @@ wheels = [ [[package]] name = "lexbench-webfetch" -version = "0.2.0" +version = "0.3.0" source = { editable = "." } dependencies = [ { name = "beautifulsoup4" },