From f65e6abd83d078a5ff4807c9cddbe729db3f3866 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Fri, 23 Oct 2026 19:35:59 +0800 Subject: [PATCH 1/4] docs: simplify citations and keep benchmark tags upstream --- CITATION.cff | 2 +- CONTRIBUTING.md | 11 +++++++++++ README.md | 10 +--------- docs/README.zh-CN.md | 10 +--------- docs/benchmarks.md | 7 ++++--- 5 files changed, 18 insertions(+), 22 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index bc204f8c4..95f11a8df 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -8,4 +8,4 @@ repository-code: "https://github.com/lexmount/browser-eval" url: "https://github.com/lexmount/browser-eval" version: 0.1.0 date-released: 2026-10-10 -license: MIT +license: Apache-2.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3a6bac756..acfda560c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,6 +32,17 @@ Keep human-readable editions, immutable dataset/scorer hashes and the package commit together. A minor scoring correction requires common-edition rescoring; changed task populations or requests require a major edition. +## Update Web Fetch dependencies and methods + +Update the pinned evaluator with `make bump-lexbench REV=`. +It verifies upstream main ancestry and updates the lock and definition hashes. +Historical report certificates retain their original evaluator identities. + +Register Web Fetch products in `harness/web_fetch/data/report-methods.json`; +executable versions and run parameters come from saved evidence. See +[method metadata](harness/web_fetch/README.md#report-method-metadata) and +[adding a method or release](harness/web_fetch/README.md#adding-a-method-or-release). + ## PR and commit titles Use `(): ` for PR titles and new commit subjects. The scope is optional; use the module or feature affected, such as `scoring` or `webfetch`. diff --git a/README.md b/README.md index be06ae515..3646a2a24 100644 --- a/README.md +++ b/README.md @@ -179,7 +179,7 @@ Project code is [Apache 2.0 licensed](LICENSE). [THIRD_PARTY](THIRD_PARTY.md) co ## Cite Browser Eval -If you use the harness in published work, cite this repository and record the tag or commit used. Cite LexBench Web Fetch or other benchmark owners separately for their task definitions and methodology. GitHub's citation menu reads [CITATION.cff](CITATION.cff). For the initial release: +Use this citation or [CITATION.cff](CITATION.cff), and record the commit used. ```bibtex @software{browser_eval_2026, @@ -190,11 +190,3 @@ If you use the harness in published work, cite this repository and record the ta url = {https://github.com/lexmount/browser-eval} } ``` - -Cite upstream benchmarks and evaluators separately when using their data or methods; this repository citation does not replace their attribution requirements. - -Update the pinned LexBench evaluator with `make bump-lexbench REV=`. The command verifies upstream main ancestry, regenerates the lock and definition hashes, and prints changed bindings. Historical report certificates retain their original evaluator identities. - -Web Fetch method descriptions, interfaces, account requirements, capability sources, and their English/Chinese text share one registry: `harness/web_fetch/data/report-methods.json`. Register a new product there; executable versions and run parameters come from saved evidence. See [method metadata](harness/web_fetch/README.md#report-method-metadata) for the distinction between registered examples and measured calls. - -For the exact configuration and code steps for a new product or release, see [Adding a method or release](harness/web_fetch/README.md#adding-a-method-or-release). diff --git a/docs/README.zh-CN.md b/docs/README.zh-CN.md index 7b2bceeb8..3c9f0ffde 100644 --- a/docs/README.zh-CN.md +++ b/docs/README.zh-CN.md @@ -178,7 +178,7 @@ make ci ## 引用 Browser Eval -在公开作品中使用本脚手架时，请引用仓库，并记录实际使用的 tag 或提交。任务定义与评测方法请另行引用 LexBench Web Fetch 或对应 benchmark。GitHub 的引用入口读取 [CITATION.cff](../CITATION.cff)。首个版本可这样引用： +使用以下引用或 [CITATION.cff](../CITATION.cff)，并注明实际使用的提交。 ```bibtex @software{browser_eval_2026, @@ -189,11 +189,3 @@ make ci url = {https://github.com/lexmount/browser-eval} } ``` - -使用上游 benchmark 的数据、评估器或方法时，还应分别引用原项目；引用本仓库不能代替上游的署名要求。 - -更新锁定的 LexBench 评分库时，运行 `make bump-lexbench REV=<已合入提交的完整哈希>`。命令会核对提交是否属于上游 main、重新生成锁文件与定义文件摘要，并列出变化。历史报告证书保留当时的评分库标识。 - -Web Fetch 的方法介绍、接口、账号要求、能力来源及中英文文案统一维护在 `harness/web_fetch/data/report-methods.json`。新增产品时修改这份资料；程序版本和运行参数读取保存的证据。登记示例与实测调用的区别见[方法资料说明](../harness/web_fetch/README.md#report-method-metadata)。 - -新增产品或更换版本需要修改哪些配置和代码，见[新增方法或版本步骤](../harness/web_fetch/README.md#adding-a-method-or-release)。 diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 113e435ff..c4743a726 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -57,9 +57,10 @@ Use a benchmark version together with exact task/reference and scoring-source hashes. Keep the package's full Git commit pinned separately. A package update does not select a new dataset for an existing report. -Use the same `benchmark-vMAJOR.MINOR.PATCH` tag prefix in both repositories. -Reports reference their benchmark edition; report rebuilds use the hosting -platform's version ID rather than a separate semantic-version sequence. +Create `benchmark-vMAJOR.MINOR.PATCH` tags only in the benchmark repository. +Browser Eval uses full Git commits to identify harness and report-building code. +Reports reference their benchmark edition and the hosting platform's immutable +version ID for each published build. | Change | Edition | Result comparison | | --- | --- | --- | From ceac92b492a620a762dfdb16c8ab0b15a9ef1315 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:00:17 +0800 Subject: [PATCH 2/4] feat(reporting): identify report builds and prepare v0.2.0 --- CITATION.cff | 4 +- README.md | 160 +++-------------- docs/README.zh-CN.md | 161 +++--------------- docs/benchmarks.md | 13 +- docs/report-hosting.md | 2 +- evaluation/lexbench-webfetch.json | 4 +- harness/browser_use/report.py | 4 + harness/browser_use/report_check.py | 2 + harness/reporting/versions.py | 112 ++++++++++++ harness/trajectory_replay/webarena_report.py | 2 + .../webarena_report_check.py | 2 + harness/web_fetch/__init__.py | 2 +- harness/web_fetch/published_snapshot.py | 2 + harness/web_fetch/report.py | 2 +- harness/web_fetch/report_build.py | 12 ++ harness/web_fetch/report_diagnostic.py | 2 + harness/web_fetch/report_version.py | 7 +- harness/web_fetch/runner.py | 2 + pyproject.toml | 6 +- tests/test_report_software_versions.py | 93 ++++++++++ uv.lock | 6 +- 21 files changed, 317 insertions(+), 283 deletions(-) create mode 100644 harness/reporting/versions.py create mode 100644 tests/test_report_software_versions.py diff --git a/CITATION.cff b/CITATION.cff index 95f11a8df..3e44e0c1f 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -6,6 +6,6 @@ authors: - name: Browser Eval contributors repository-code: "https://github.com/lexmount/browser-eval" url: "https://github.com/lexmount/browser-eval" -version: 0.1.0 -date-released: 2026-10-10 +version: 0.2.0 +date-released: 2026-10-23 license: Apache-2.0 diff --git a/README.md b/README.md index 3646a2a24..7fd868d40 100644 --- a/README.md +++ b/README.md @@ -8,174 +8,68 @@ [![License: Apache 2.0](https://img.shields.io/badge/code-Apache--2.0-51735b)](LICENSE) [![Python 3.11+](https://img.shields.io/badge/Python-3.11%2B-356a91)](pyproject.toml) -[Quickstart](#quickstart) · [Benchmarks](docs/benchmarks.md) · [Reports](#read-and-share-a-report) · [Contributing](CONTRIBUTING.md) +[Quickstart](#quickstart) · [Tracks](#evaluation-tracks) · [Reports](#reports) · [Contributing](CONTRIBUTING.md) ---- +Browser Eval is an extensible evaluation harness for browsers and web agents. It combines task contracts, adapters, saved evidence and scoring to produce inspectable reports. -Browser Eval connects benchmark tasks to browser and service adapters, executes declared comparisons, calls the benchmark scorer on saved outputs and builds inspectable reports. It supports Web Fetch, Browser Trajectory Replay and Browser Use. Task populations and acceptance requirements come from the selected benchmark. +## Evaluation tracks -[LexBench Web Fetch](https://github.com/lexmount/lexbench-webfetch) owns the self-authored Web Fetch task suites. WebMainBench, WCXB and WebArena supply other benchmark inputs. Browser Eval provides the execution and reporting machinery; each benchmark retains its own scope, versions and attribution. See [repository boundaries and integration](docs/benchmarks.md). +The following workflows have runnable guides. Additional workflows, including Web Search, can use the same adapter and task-contract model; see [contributing](CONTRIBUTING.md). -You get an HTML report with comparison tables, task details, responses and logs. Read it locally or publish it through [Artifact Site](https://github.com/lexmount/artifact-site) so others can explore the results online. - -[What it tests](#what-it-tests) · [Quickstart](#quickstart) · [Reports](#read-and-share-a-report) · [Related projects](#related-repositories) · [Contribute](#contribute) - -```mermaid -flowchart LR - A[Benchmark tasks and contracts] --> B[Browser Eval] - B --> C[Browser or provider adapters] - C --> D[Saved outputs and timings] - D --> E[Scoring and evidence checks] - E --> F[Inspectable HTML report] -``` - -## What it tests - -Supported execution tracks: - -| Track | Question | Start here | -| --- | --- | --- | -| Web Fetch | Can a method return the required page text, records, resources or files? How long does it take? | [Local example](examples/web_fetch/README.md) · [Full guide](harness/web_fetch/README.md) | -| Browser Trajectory Replay | Can different browsers complete the same recorded actions? What resources do they use? | [Curated suite](examples/trajectory_replay/README.md) · [WebArena recordings](examples/trajectory_replay/public_webarena/README.md) | -| Browser Use | Can an Agent complete a task when only the browser changes? | [CowAgent + WebArena guide](examples/browser_use/README.md) | - -Each track reports its own task population and scores. HTTP is a fetch baseline; trajectory replay uses fixed actions; Browser Use includes Agent decisions. - -
-Supported browsers, services and tools - -| Track | Implemented participants | Preparation | -| --- | --- | --- | -| Web Fetch: local browsers | Moli, Chromium, Lightpanda, Obscura | Install the selected binary; Chromium also needs Playwright and Node.js conversion dependencies | -| Web Fetch: services | Lexmount, Firecrawl, Tavily Extract, Exa Contents, Cloudflare Markdown/Content; hosted Kitesurf via Cloudflare | Provider account and credentials in the repository `.env` | -| Web Fetch: Agent tools and extractors | CowAgent, OpenClaw, DeepSeek Harness, LangChain WebBaseLoader, Crawl4AI | Upstream checkout/runtime or the `oss-eval` extra, according to the adapter | -| Browser Trajectory Replay / Browser Use | Moli and Chromium in the current browser registry | Docker/Linux environment; Browser Use also needs CowAgent and a model account | -| Controls | Plain HTTP and offline fixture | Base Python installation | - -Support varies by task type. Check [adapter requirements](docs/provider-contract.md) and [dependency versions](docs/dependencies.md) before selecting a method. - -
- -
-Benchmarks and datasets - -| Suite | What it measures | Availability and entry | +| Track | What it tests | Guide | | --- | --- | --- | -| Offline fixture / local article | Harness operation, selection, scoring and evidence; no performance ranking | Included; commands below | -| LexBench Web Fetch | Ordinary pages, dynamic content, documents, images/video, popular sites and access restrictions | [Benchmark repository](https://github.com/lexmount/lexbench-webfetch) · [Integration](docs/benchmarks.md); complete historical references require approved external inputs | -| WebMainBench / WCXB | Extraction fidelity on independently bound frozen HTML and reference content | [Frozen-HTML guide](harness/web_fetch/WEBMAINBENCH.md); obtain source data under its terms and use the suite's certification pipeline | -| Curated trajectory suite | Identical fixed actions on selected public pages | [Trajectory run guide](examples/trajectory_replay/README.md); uses Docker and live sites | -| WebArena human-recording replay | Qualified fixed trajectories and official task evaluation on self-hosted sites | [Reproduction guide](docs/webarena-fixed-trajectory-reproduction.md); external datasets and substantial deployment resources | -| WebArena-Verified Browser Use | Autonomous CowAgent task completion against the official evaluator | [Browser Use guide](examples/browser_use/README.md); current selection and environment are declared separately | +| Web Fetch | Retrieve the requested page content, records, resources or files | [Local example](examples/web_fetch/README.md) · [Full guide](harness/web_fetch/README.md) | +| Browser Trajectory Replay | Complete the same recorded browser actions across implementations | [Curated suite](examples/trajectory_replay/README.md) · [WebArena replay](examples/trajectory_replay/public_webarena/README.md) | +| Browser Use | Let an Agent use different browsers to complete tasks | [CowAgent + WebArena](examples/browser_use/README.md) | -The Web Fetch CLI takes a dataset file. Replay and Browser Use have separate runners. +Web Fetch supports local browsers, hosted services and extraction tools. See [adapters](docs/provider-contract.md) for available methods and [dependencies](docs/dependencies.md) for installation requirements. Replay and Browser Use have separate runners and environments. -
+Task definitions and scoring come from the selected benchmark: [LexBench Web Fetch](https://github.com/lexmount/lexbench-webfetch), WebMainBench, WCXB or WebArena. See [benchmark integration](docs/benchmarks.md). ## Quickstart -Clone the repository, open a terminal in its root, and install [uv](https://docs.astral.sh/uv/getting-started/installation/). The base CLI supports Python 3.11–3.14; repository development defaults to 3.11. +Install [uv](https://docs.astral.sh/uv/getting-started/installation/), then run the offline example: -```bash +```sh +git clone https://github.com/lexmount/browser-eval.git +cd browser-eval uv sync --frozen uv run --frozen browser-harness-eval validate uv run --frozen browser-harness-eval run --delay 0 ``` -Open the report path printed in the terminal. This offline example needs no browser, Docker or API key. It contains one expected pass and one expected failure so you can see both kinds of result. Installation needs network access. - -Next, [run HTTP, Moli or Chromium against a local article](examples/web_fetch/README.md). That guide covers browser installation, selecting several methods and choosing your own tasks. Remote services and Agent runs need their own accounts; their calls may incur charges. +Open the report path printed in the terminal. The example has one expected pass and one expected failure; it needs no browser, Docker or API key. Python 3.11–3.14 is supported; development uses 3.11. -
-Choose another Python version or install an optional adapter +Next, [try HTTP, Moli or Chromium on a local article](examples/web_fetch/README.md). Optional adapters, service credentials and live evaluation commands are covered in the track guides above. -To use Python 3.12, add `--python 3.12` to both `uv sync --frozen` and subsequent `uv run --frozen` commands. Optional adapters and historical experiments have additional version constraints; the full development environment uses 3.11. +## Reports -| Extra | Used for | -| --- | --- | -| `headless-eval` | Chromium, PDF parsing and container measurements | -| `webmainbench-eval` | WebMainBench scoring | -| `browser-use-eval` | WebArena and Agent execution; requires Git for the pinned evaluator | -| `oss-eval` | Open-source extraction libraries | +Reports include comparisons, task results, full responses and failure evidence. Runs are saved under `~/.browser-eval/runs/`; use `--output /absolute/path/new-run` to choose another directory. -Install an extra with `uv sync --frozen --extra NAME`. Browser binaries, Docker, site datasets and accounts are installed separately. See [dependencies](docs/dependencies.md). +Rebuild from saved evidence without repeating calls: -
- -## Read and share a report - -Reports retain the selected benchmark edition and exact dataset/scorer hashes. -Package upgrades leave historical results unchanged. See -[benchmark editions](docs/benchmarks.md#benchmark-editions) for patch, minor and -major comparison rules. - -| Output | What you will find | -| --- | --- | -| `report/en/index.html` | English comparisons, task results and evidence links | -| `report/index.html` | Report entry; opens the selected language | -| `results.jsonl`, `summary.json` | Scores and aggregates for scripts or further analysis | -| `manifest.json`, `dataset.json` | Methods, versions, settings and tasks used | -| `artifacts/` | Saved responses and other evidence | - -Runs go to `~/.browser-eval/runs/` by default. Set `BROWSER_EVAL_DATA_HOME` to change the root, or `--output /absolute/path/new-run` for one run. Each run needs a new directory. - -Rebuild a report from saved evidence without repeating browser or API calls: - -```bash +```sh uv run --frozen browser-harness-eval report /absolute/path/to/run ``` -The Web Fetch CLI opens English by default, with a switchable Chinese edition. Choose Chinese when starting a run, or change the language of a saved report: - -```bash -uv run --frozen browser-harness-eval run --delay 0 --language zh -uv run --frozen browser-harness-eval report /absolute/path/to/run --language en -``` - -Rebuilding without `--language` keeps the previous choice. Language changes do not repeat calls or change scores; task wording, responses and logs retain their original language. English (`en`) and Chinese (`zh`) are supported; other language codes are rejected. See [report language support](docs/report-hosting.md#report-languages) for the standalone builders and historical reports. - -Frozen local snapshots use their own external `presentation-binding.json`; keep report-specific profiles and publication targets with their saved evidence, outside Git. Follow the [frozen local report workflow](harness/web_fetch/README.md#frozen-local-report-inputs) to check or rebuild them without changing saved scores. - -To publish online: - -1. Sign in or create an account on the [China site](https://artifact-site.lexmount.cn/) or [global site](https://yours.lexmount.com/artifact-site). -2. Publish the run's `report/` directory using the [CLI or MCP guide](docs/report-hosting.md). -3. Choose who can open the report in its sharing settings. +English is the default; add `--language zh` when running or rebuilding for Chinese. A rebuild without that option keeps the saved language. -[Artifact Site](https://github.com/lexmount/artifact-site) is open source and supports self-hosting, report versions and access controls. Local reports need no hosting account and never upload automatically. +Each new report records its Browser Eval build and selected benchmark edition, with exact code and input hashes. See [versioning](docs/benchmarks.md#benchmark-editions) for comparison rules and [reference inputs](docs/reference-inputs.md) for historical data. -Example runs are diagnostic. For a formal benchmark release, follow the selected track's certification and release checks; uploading a report does not certify it. - -## Related repositories - -| Repository | Role in this work | -| --- | --- | -| [LexBench Web Fetch](https://github.com/lexmount/lexbench-webfetch) | Independent task suites and acceptance contracts for self-authored Web Fetch evaluations | -| [Moli](https://github.com/lexmount/moli) | Browser evaluated alongside the other implementations | -| [Artifact Site](https://github.com/lexmount/artifact-site) | Report hosting, versions and sharing; self-hosting supported | -| [Lexbench-Headless-Browser](https://github.com/lexmount/Lexbench-Headless-Browser) | Separate benchmark for protocols, automation drivers and web-platform semantics | -| [WebArena-Verified fork](https://github.com/lexmount/webarena-verified/tree/lex-main) | Tasks and evaluator for Browser Use and public WebArena replay; maintained on `lex-main` | -| [WebMainBench fork](https://github.com/lexmount/WebMainBench/tree/lex-main) · [upstream](https://github.com/opendatalab/WebMainBench) | Frozen-HTML extraction benchmark; the fork maintains declared scoring corrections | -| [WCXB](https://github.com/Murrough-Foley/web-content-extraction-benchmark) | Independent content-extraction benchmark | -| [CowAgent](https://github.com/zhayujie/CowAgent) | Browser Use Agent runner and Web Fetch tool baseline | -| [OpenClaw](https://github.com/openclaw/openclaw) · [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) | Web Fetch tool baselines | - -See [dependency sources and pins](docs/dependencies.md) for exact versions. Install only the components your selected method needs. +To share a report online, publish its `report/` directory through [Artifact Site](https://github.com/lexmount/artifact-site), following the [hosting guide](docs/report-hosting.md). Choose access permissions there. Formal benchmark releases also require the checks in their track guide. ## Contribute -PRs are welcome for adapters, execution fixes, tests, documentation and translations. Changes to LexBench task requirements belong in [LexBench Web Fetch](https://github.com/lexmount/lexbench-webfetch). Start with [CONTRIBUTING](CONTRIBUTING.md), or [open an issue](https://github.com/lexmount/browser-eval/issues) with a reproducible example. +Read [CONTRIBUTING](CONTRIBUTING.md) for code ownership, tests and PR conventions, or [open an issue](https://github.com/lexmount/browser-eval/issues) with a reproducible example. -```bash +```sh make setup-test make ci ``` -Historical pages and measured outputs live outside Git. For reproducing past runs, see [reference inputs](docs/reference-inputs.md). For code layout and release internals, see [architecture](docs/architecture.md). - -Project code is [Apache 2.0 licensed](LICENSE). [THIRD_PARTY](THIRD_PARTY.md) covers data rights and attribution; [SECURITY](SECURITY.md) covers credentials and untrusted content. +[Architecture](docs/architecture.md) · [Security](SECURITY.md) · [Third-party content](THIRD_PARTY.md) · [Apache 2.0](LICENSE) ## Cite Browser Eval @@ -186,7 +80,7 @@ Use this citation or [CITATION.cff](CITATION.cff), and record the commit used. author = {{Browser Eval contributors}}, title = {Browser Eval}, year = {2026}, - version = {0.1.0}, + version = {0.2.0}, url = {https://github.com/lexmount/browser-eval} } ``` diff --git a/docs/README.zh-CN.md b/docs/README.zh-CN.md index 3c9f0ffde..23cf003ec 100644 --- a/docs/README.zh-CN.md +++ b/docs/README.zh-CN.md @@ -1,6 +1,6 @@
-Browser Eval — Same tasks. Clear comparisons. +Browser Eval — 相同任务，清晰比较。 [English](../README.md) · **简体中文** @@ -8,173 +8,68 @@ [![License: Apache 2.0](https://img.shields.io/badge/code-Apache--2.0-51735b)](../LICENSE) [![Python 3.11+](https://img.shields.io/badge/Python-3.11%2B-356a91)](../pyproject.toml) -[快速开始](#快速开始) · [Benchmark 接入](benchmarks.md) · [报告](#查看与分享报告) · [参与贡献](../CONTRIBUTING.md) +[快速开始](#快速开始) · [评测流程](#评测流程) · [报告](#报告) · [参与贡献](../CONTRIBUTING.md)
---- +Browser Eval 是面向浏览器和 Web Agent 的可扩展评测框架，将任务规则、适配器、保存的证据和评分结果汇集为可查阅的报告。 -Browser Eval 把 benchmark 任务接到浏览器或服务适配器上，执行预先声明的比较，调用 benchmark 评分器检查保存的输出，并生成可查看证据的报告。它支持 Web Fetch、轨迹重放和 Browser Use；任务范围和验收要求由所选 benchmark 决定。 +## 评测流程 -自建的 Web Fetch 任务集由 [LexBench Web Fetch](https://github.com/lexmount/lexbench-webfetch) 维护。WebMainBench、WCXB 和 WebArena 提供其他评测输入。Browser Eval 负责执行与报告，各 benchmark 分别维护范围、版本和来源说明。详见[仓库分工与接入](benchmarks.md)。 +以下流程已有可运行的指南。Web Search 等其他流程也可按同一套适配器和任务规则接入，见[贡献指南](../CONTRIBUTING.md)。 -运行后得到 HTML 报告，里面有比较表、任务详情、响应和日志。可以在本地查看，也可以通过 [Artifact Site](https://github.com/lexmount/artifact-site) 发布，让其他人在线查看结果。 - -[评测内容](#评测内容) · [快速开始](#快速开始) · [查看与分享报告](#查看与分享报告) · [相关项目](#相关仓库导航) · [参与贡献](#参与贡献) - -```mermaid -flowchart LR - A[Benchmark 任务与验收规则] --> B[Browser Eval] - B --> C[浏览器或服务适配器] - C --> D[保存的响应与耗时] - D --> E[评分与证据检查] - E --> F[可查看证据的 HTML 报告] -``` - -## 评测内容 - -支持以下执行方式： - -| 轨道 | 回答什么问题 | 从这里开始 | -| --- | --- | --- | -| Web Fetch | 能否返回任务要求的正文、记录、资源或文件？需要多久？ | [本地示例](../examples/web_fetch/README.md) · [完整说明](../harness/web_fetch/README.md) | -| Browser Trajectory Replay（轨迹重放） | 不同浏览器能否完成同一组已记录的操作？消耗多少资源？ | [项目任务集](../examples/trajectory_replay/README.md) · [WebArena 人工轨迹](../examples/trajectory_replay/public_webarena/README.md) | -| Browser Use | 只更换浏览器时，Agent 能否完成任务？ | [CowAgent + WebArena 说明](../examples/browser_use/README.md) | - -各轨道分别报告任务范围和分数。HTTP 是抓取基线；轨迹重放执行固定操作；Browser Use 包含 Agent 决策。 - -
-支持的浏览器、服务和工具 - -| 轨道 | 已实现的参与方 | 准备条件 | -| --- | --- | --- | -| Web Fetch：本地浏览器 | Moli、Chromium、Lightpanda、Obscura | 安装所选浏览器；Chromium 还需要 Playwright 和 Node.js 转换依赖 | -| Web Fetch：服务 | Lexmount、Firecrawl、Tavily Extract、Exa Contents、Cloudflare Markdown/Content；通过 Cloudflare 调用托管 Kitesurf | 供应商账户；凭据写入仓库 `.env` | -| Web Fetch：Agent 工具与提取库 | CowAgent、OpenClaw、DeepSeek Harness、LangChain WebBaseLoader、Crawl4AI | 按适配器要求准备上游源码、运行环境或 `oss-eval` 依赖 | -| 轨迹重放 / Browser Use | 当前浏览器注册表中的 Moli、Chromium | Docker/Linux 环境；Browser Use 还需要 CowAgent 和模型账户 | -| 对照方法 | 普通 HTTP、离线 fixture | 基础 Python 安装 | - -各方法支持的任务类型不同，选择前请查看[适配器要求](../docs/provider-contract.md)和[依赖版本](../docs/dependencies.md)。 - -
- -
-Benchmark 与数据集 - -| 任务集 | 评测内容 | 可用条件与入口 | +| 流程 | 测什么 | 使用指南 | | --- | --- | --- | -| 离线 fixture / 本地文章 | 检查运行、任务选择、评分和证据，不提供性能排名 | 随仓库提供，命令见下文 | -| LexBench Web Fetch | 普通网页、动态内容、文档、图片/视频、热门网站、访问限制 | [Benchmark 仓库](https://github.com/lexmount/lexbench-webfetch) · [接入说明](benchmarks.md)；完整历史参考需要获准使用的外部输入 | -| WebMainBench / WCXB | 独立绑定的冻结 HTML 与参考内容上的提取质量 | [冻结 HTML 说明](../harness/web_fetch/WEBMAINBENCH.md)；按原数据许可获取输入，并使用对应认证流程 | -| 项目维护的轨迹任务集 | 在指定公开网页上执行相同固定操作 | [轨迹运行说明](../examples/trajectory_replay/README.md)；需要 Docker 和网站访问 | -| WebArena 人工轨迹重放 | 自建网站上的已通过资格检查的固定轨迹与官方任务评估 | [复现说明](../docs/webarena-fixed-trajectory-reproduction.md)；需要外部数据和较多部署资源 | -| WebArena-Verified Browser Use | CowAgent 自主完成任务，由官方评估器判定 | [Browser Use 说明](../examples/browser_use/README.md)；任务范围和环境分别声明 | +| Web Fetch | 获取任务要求的网页正文、记录、资源或文件 | [本地示例](../examples/web_fetch/README.md) · [完整指南](../harness/web_fetch/README.md) | +| 轨迹重放 | 比较不同浏览器能否完成同一组录制动作 | [自建任务集](../examples/trajectory_replay/README.md) · [WebArena 重放](../examples/trajectory_replay/public_webarena/README.md) | +| Browser Use | 让 Agent 使用不同浏览器完成任务 | [CowAgent + WebArena](../examples/browser_use/README.md) | -Web Fetch CLI 使用数据集文件；轨迹重放和 Browser Use 使用各自的执行入口。 +Web Fetch 支持本地浏览器、在线服务和提取工具。可用方法见[适配器说明](../docs/provider-contract.md)，安装要求见[依赖说明](../docs/dependencies.md)。轨迹重放和 Browser Use 各有自己的运行入口与环境。 -
+题目和评分规则来自选定的评测集：[LexBench Web Fetch](https://github.com/lexmount/lexbench-webfetch)、WebMainBench、WCXB 或 WebArena。接入方式见[评测集指南](../docs/benchmarks.md)。 ## 快速开始 -克隆仓库，在根目录打开终端，安装 [uv](https://docs.astral.sh/uv/getting-started/installation/)。基础 CLI 支持 Python 3.11–3.14，仓库开发默认使用 3.11。 +安装 [uv](https://docs.astral.sh/uv/getting-started/installation/)，运行离线示例： -```bash +```sh +git clone https://github.com/lexmount/browser-eval.git +cd browser-eval uv sync --frozen uv run --frozen browser-harness-eval validate uv run --frozen browser-harness-eval run --delay 0 ``` -打开终端打印的报告路径。这个离线示例无需浏览器、Docker 或 API Key，预设一个通过结果和一个失败结果，方便查看两种情况。安装依赖时需要联网。 - -接下来，可以[用 HTTP、Moli 或 Chromium 抓取本地文章](../examples/web_fetch/README.md)。说明中包含浏览器安装、多方法比较和自选任务。远程服务和 Agent 运行需要相应账户，调用可能产生费用。 +打开终端打印的报告路径。示例包含一个预期通过和一个预期失败，无需浏览器、Docker 或 API 密钥。支持 Python 3.11–3.14，仓库开发使用 3.11。 -
-选择其他 Python 版本或安装可选适配器 +接下来可以[用 HTTP、Moli 或 Chromium 读取本地文章](../examples/web_fetch/README.md)。可选适配器、服务凭据和在线评测命令见上面的各流程指南。 -使用 Python 3.12 时，在 `uv sync --frozen` 和后续 `uv run --frozen` 命令中都加上 `--python 3.12`。可选适配器和历史实验另有版本要求，完整开发环境使用 3.11。 +## 报告 -| 额外依赖 | 用途 | -| --- | --- | -| `headless-eval` | Chromium、PDF 解析和容器测量 | -| `webmainbench-eval` | WebMainBench 评分 | -| `browser-use-eval` | WebArena 和 Agent；安装固定提交的评估器需要 Git | -| `oss-eval` | 开源提取库 | +报告包含结果对比、逐题结果、完整响应和失败证据。运行结果默认保存在 `~/.browser-eval/runs/`，也可用 `--output /absolute/path/new-run` 指定新目录。 -用 `uv sync --frozen --extra NAME` 安装。浏览器、Docker、网站数据和账户分别准备，见[依赖说明](../docs/dependencies.md)。 +从保存的证据重建报告，无需重新调用浏览器或服务： -
- -## 查看与分享报告 - -报告固定绑定所选评测版本，以及题目和评分源码的精确哈希。升级软件包不会改变历史结果。 -补丁版可以直接比较；评分修正的小版本要统一重评；题目或任务要求改变的大版本分别展示。 -具体规则见[评测版本](benchmarks.md#benchmark-editions)。 - -| 输出 | 内容 | -| --- | --- | -| `report/en/index.html` | 英文比较、任务结果和证据入口 | -| `report/index.html` | 报告入口，打开所选语言 | -| `results.jsonl`、`summary.json` | 供程序分析的评分和汇总 | -| `manifest.json`、`dataset.json` | 使用的方法、版本、设置和任务 | -| `artifacts/` | 保存的响应及其他证据 | - -默认保存在 `~/.browser-eval/runs/`。用 `BROWSER_EVAL_DATA_HOME` 改数据根目录，或用 `--output /absolute/path/new-run` 指定本次输出。每次运行需要一个新目录。 - -从保存的证据重建报告，无需再次调用浏览器或 API： - -```bash +```sh uv run --frozen browser-harness-eval report /absolute/path/to/run ``` -Web Fetch 命令默认打开英文报告，并保留可切换的中文版。开始评测时可以选中文，也可以更改已有报告的语言： - -```bash -uv run --frozen browser-harness-eval run --delay 0 --language zh -uv run --frozen browser-harness-eval report /absolute/path/to/run --language en -``` - -重建时不传 `--language`，会沿用上次的选择。切换语言不重新调用浏览器或服务，也不改变评分；任务原文、响应和日志保留原来的语言。目前支持英文（`en`）和中文（`zh`），其他语言代码会报错。独立构建器和历史报告的支持范围见[报告语言说明](report-hosting.md#report-languages)。 - -冻结的本地报告使用随证据保存的 `presentation-binding.json`。报告专用配置和发布目标保存在 Git 之外，与该报告的证据放在一起。按照[冻结本地报告流程](../harness/web_fetch/README.md#frozen-local-report-inputs)检查或重建，不改变保存的评分。 - -在线发布： - -1. 在[中国站](https://artifact-site.lexmount.cn/)或[全球站](https://yours.lexmount.com/artifact-site)登录或创建账户。 -2. 按 [CLI 或 MCP 说明](../docs/report-hosting.md)发布本次运行的 `report/` 目录。 -3. 在分享设置中选择谁能访问报告。 +默认生成英文；运行或重建时加 `--language zh` 可选中文。重建时不传该选项，会沿用已保存的语言。 -[Artifact Site](https://github.com/lexmount/artifact-site) 已开源，支持自行部署、报告版本管理和访问权限。本地报告无需托管账户，也不会自动上传。 +新报告记录 Browser Eval 构建版本和选定的评测版本，并保留代码与输入哈希。结果比较规则见[版本说明](../docs/benchmarks.md#benchmark-editions)，历史数据见[参考输入](../docs/reference-inputs.md)。 -示例运行生成诊断报告。正式 benchmark 发布需要通过所选轨道的认证和发布检查；上传报告本身不代表结果已通过认证。 - -## 相关仓库导航 - -| 仓库 | 在这项工作中的用途 | -| --- | --- | -| [LexBench Web Fetch](https://github.com/lexmount/lexbench-webfetch) | 自建 Web Fetch 评测使用的独立任务集和验收规则 | -| [Moli](https://github.com/lexmount/moli) | 与其他实现一起接受评测的浏览器 | -| [Artifact Site](https://github.com/lexmount/artifact-site) | 报告托管、版本管理和分享，支持自行部署 | -| [Lexbench-Headless-Browser](https://github.com/lexmount/Lexbench-Headless-Browser) | 独立评测浏览器协议、自动化驱动和 Web 平台语义 | -| [WebArena-Verified fork](https://github.com/lexmount/webarena-verified/tree/lex-main) | Browser Use 与公开 WebArena 重放的任务和评估器，在 `lex-main` 维护 | -| [WebMainBench fork](https://github.com/lexmount/WebMainBench/tree/lex-main) · [上游](https://github.com/opendatalab/WebMainBench) | 冻结 HTML 的正文提取 benchmark；fork 维护已声明的评分修正 | -| [WCXB](https://github.com/Murrough-Foley/web-content-extraction-benchmark) | 独立的网页正文提取 benchmark | -| [CowAgent](https://github.com/zhayujie/CowAgent) | Browser Use 的 Agent 执行器和 Web Fetch 工具基线 | -| [OpenClaw](https://github.com/openclaw/openclaw) · [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) | Web Fetch 工具基线 | - -具体版本见[依赖来源与固定提交](../docs/dependencies.md)，按所选方法安装需要的组件即可。 +在线分享时，按[托管指南](../docs/report-hosting.md)将 `report/` 目录发布到 [Artifact Site](https://github.com/lexmount/artifact-site)，并设置访问权限。正式评测发布还需通过对应流程的检查。 ## 参与贡献 -欢迎为适配器、执行修复、测试、文档和翻译提交 PR。LexBench 任务要求的修改请提交到 [LexBench Web Fetch](https://github.com/lexmount/lexbench-webfetch)。先看[贡献指南](../CONTRIBUTING.md)，或提交带有复现示例的 [issue](https://github.com/lexmount/browser-eval/issues)。 +代码归属、测试和 PR 规范见 [CONTRIBUTING](../CONTRIBUTING.md)。反馈问题请[提交可复现的示例](https://github.com/lexmount/browser-eval/issues)。 -```bash +```sh make setup-test make ci ``` -历史网页和测量输出保存在 Git 之外。复现历史运行见[参考输入](../docs/reference-inputs.md)，代码结构和发布内部流程见[架构说明](../docs/architecture.md)。 - -项目代码采用 [Apache 2.0 许可](../LICENSE)。数据权利和署名见[第三方说明](../THIRD_PARTY.md)，凭据和不可信内容处理见[安全说明](../SECURITY.md)。 +[代码结构](../docs/architecture.md) · [安全](../SECURITY.md) · [第三方内容](../THIRD_PARTY.md) · [Apache 2.0](../LICENSE) ## 引用 Browser Eval @@ -185,7 +80,7 @@ make ci author = {{Browser Eval contributors}}, title = {Browser Eval}, year = {2026}, - version = {0.1.0}, + version = {0.2.0}, url = {https://github.com/lexmount/browser-eval} } ``` diff --git a/docs/benchmarks.md b/docs/benchmarks.md index c4743a726..d11d7ab98 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -57,11 +57,18 @@ Use a benchmark version together with exact task/reference and scoring-source hashes. Keep the package's full Git commit pinned separately. A package update does not select a new dataset for an existing report. -Create `benchmark-vMAJOR.MINOR.PATCH` tags only in the benchmark repository. -Browser Eval uses full Git commits to identify harness and report-building code. +Create `bench-vMAJOR.MINOR.PATCH` tags only in the benchmark repository. +Browser Eval publishes software versions as `vMAJOR.MINOR.PATCH` and records the +full Git commit for execution and report-building code. Reports reference their benchmark edition and the hosting platform's immutable version ID for each published build. +Report home pages link to the Browser Eval build version and the selected bench +edition. Their JSON bindings retain code commits and source/input hashes. +Rebuilding with a newer Browser Eval release changes the build identity, not the +saved call identity or benchmark edition. Custom datasets without a frozen +edition retain their hashes rather than inheriting the installed catalog version. + | Change | Edition | Result comparison | | --- | --- | --- | | Documentation or presentation only | Patch | Direct when execution controls also match | @@ -69,7 +76,7 @@ version ID for each published build. | Added/removed cases, changed requests, output scope or routes | Major | Separate full-population results | The task package owns `lexbench-webfetch release` and `compare-releases`. -See its [version contract](https://github.com/lexmount/lexbench-webfetch/blob/benchmark-v2.0.1/docs/versions.md). +See its [version contract](https://github.com/lexmount/lexbench-webfetch/blob/bench-v2.0.1/docs/versions.md). Freeze `benchmark-release.json` beside the compiled inputs before calls. The certified Web Fetch runner accepts `--benchmark-release`; it also discovers the sidecar beside the selected dataset. Release creation records the full scorer diff --git a/docs/report-hosting.md b/docs/report-hosting.md index 2a539bae4..e49614767 100644 --- a/docs/report-hosting.md +++ b/docs/report-hosting.md @@ -85,7 +85,7 @@ Example runs remain diagnostic after upload. Publishing does not certify benchma ## Maintained benchmark releases -Formal releases use the selected track's package/publish commands after certification, required review and release checks. See [architecture](architecture.md#reports-and-publication) and the [track guides](../README.md#what-it-tests). +Formal releases use the selected track's package/publish commands after certification, required review and release checks. See [architecture](architecture.md#reports-and-publication) and the [track guides](../README.md#evaluation-tracks). | Requirement | Why it matters | | --- | --- | diff --git a/evaluation/lexbench-webfetch.json b/evaluation/lexbench-webfetch.json index 14379fbb0..940f11aa0 100644 --- a/evaluation/lexbench-webfetch.json +++ b/evaluation/lexbench-webfetch.json @@ -2,7 +2,7 @@ "schema_version": 1, "kind": "external_benchmark_binding", "repository": "https://github.com/lexmount/lexbench-webfetch", - "revision": "2ea8c52df67c39eee2ea5113a63f5da34f102160", + "revision": "d0e78e71cf5145217cb1f2b71420a7c4194d80dd", "ownership": { "benchmark": "Task populations, task model, acceptance scoring, file checks, compiler and frozen definitions", "harness": "Execution, reference lifecycle, evidence and reports" @@ -81,6 +81,6 @@ "evaluation/scoring-regressions.json": "432ddabe23d045d0ce4c23aef69ca3337ba4d240fe3c2061932cee26b7dc00dc", "evaluation/source-inputs.json": "c202abc4624811b938a837402bbaf0ecbe716e375fbca70567346d2edb3b1708", "annotation-revisions/serialization-20261021.json": "e0fe42179e584abb73782cbc459f521cab9cb09d3442de1ec4a6ff54d1b1235a", - "benchmark-release.json": "96ac7f1044601f4a3f768be814a49a48ff46e3ad9802f7f9e17a524460b57c4b" + "benchmark-release.json": "456be14df670cb9d973415ed87d4de6f45d7b5021e3284180c8a0c8b5daee773" } } diff --git a/harness/browser_use/report.py b/harness/browser_use/report.py index e45b2c331..f8fc0c489 100644 --- a/harness/browser_use/report.py +++ b/harness/browser_use/report.py @@ -3991,6 +3991,10 @@ def translated(zh, en): write_language_contract(stage, include_english=include_english) if language is not None: select_report_language(stage, language) + from harness.reporting.versions import bind_build_version + audit['browser_eval'] = bind_build_version(stage) + _write_json(stage / "data/summary.json", audit) + _write_json(stage / "report-data.json", audit) (stage / ".generated-report").write_text("browser-use-report/2\n", encoding="utf-8") backup: Path | None = None if output.exists(): diff --git a/harness/browser_use/report_check.py b/harness/browser_use/report_check.py index e9768ecb7..118bbf66f 100644 --- a/harness/browser_use/report_check.py +++ b/harness/browser_use/report_check.py @@ -478,6 +478,8 @@ def check( raise ValueError("report is missing one or more required report pages") selection = load_pilot(selection_path) audit = json.loads(audit_path.read_text(encoding="utf-8")) + from harness.reporting.versions import check_build_version + check_build_version(report_dir, audit) if audit.get("profile_sha256") and audit.get("template") != template_identity("browser-use"): raise ValueError("report bypasses the maintained Browser Use template") if audit.get("profile_sha256") and audit["profile_sha256"] != _sha256(profile_path): diff --git a/harness/reporting/versions.py b/harness/reporting/versions.py new file mode 100644 index 000000000..88922584f --- /dev/null +++ b/harness/reporting/versions.py @@ -0,0 +1,112 @@ +"""Identify report-building software without changing measured inputs.""" + +import hashlib +import json +import re +import subprocess +from functools import cache +from html import escape +from pathlib import Path + +from harness.web_fetch import __version__ + +FILENAME = "browser-eval-version.json" + + +@cache +def software_identity(): + """Use the package's checkout, never an unrelated working directory.""" + package = Path(__file__).resolve().parents[1] + root = package.parent + revision = None + modified = False + if (root / ".git").exists(): + try: + revision = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=root, text=True, + stderr=subprocess.DEVNULL, + ).strip() + modified = bool(subprocess.check_output( + ["git", "status", "--porcelain", "--untracked-files=normal", "--", "harness"], + cwd=root, text=True, stderr=subprocess.DEVNULL, + ).strip()) + except (OSError, subprocess.CalledProcessError): + revision = None + digest = hashlib.sha256() + for path in sorted(package.rglob("*.py")): + digest.update(path.relative_to(package).as_posix().encode() + b"\0" + path.read_bytes() + b"\0") + return {"version": __version__, "revision": revision, + "source_sha256": digest.hexdigest(), "modified": modified} + + +def bind_build_version(root): + """Bind this build and add one compact link to each generated home page.""" + identity = {"role": "report_builder", **software_identity()} + raw = (json.dumps(identity, indent=2) + "\n").encode() + (root / FILENAME).write_bytes(raw) + for route in ("index.html", "en/index.html"): + page = root / route + if not page.is_file(): + continue + content = page.read_text() + if "data-english-not-generated" in content: + continue + content = re.sub(r'<(p|span) data-report-build-version>.*?', '', content, flags=re.S) + href = ("../" if route.startswith("en/") else "") + FILENAME + link = ('Browser Eval v' + + escape(identity["version"]) + '') + benchmark = re.search(r']*\bdata-benchmark-release=[^>]*>.*?', content, re.S) + if benchmark: + label = '' + link + ' · ' + page.write_text(content[:benchmark.start()] + label + content[benchmark.start():]) + continue + label = '

' + link + '

' + end = '' if '' in content else '' + if end not in content: + raise ValueError("Report home has no body: " + route) + page.write_text(content.replace(end, label + end, 1)) + return {"file": FILENAME, "sha256": hashlib.sha256(raw).hexdigest(), + "version": identity["version"]} + + +def check_build_version(root, manifest): + """Keep legacy reports readable; verify every declared build and label.""" + from bs4 import BeautifulSoup + + binding = manifest.get("browser_eval") + path = root / FILENAME + pages = [(route, BeautifulSoup((root / route).read_text(), "html.parser")) + for route in ("index.html", "en/index.html") if (root / route).is_file()] + if binding is None: + if path.exists() or any(page.select_one('[data-browser-eval-release]') for _, page in pages): + raise ValueError("Report lost its Browser Eval version binding") + return + if path.is_symlink() or binding.get("file") != FILENAME or not path.is_file(): + raise ValueError("Report Browser Eval version is missing") + raw = path.read_bytes() + identity = json.loads(raw) + if (hashlib.sha256(raw).hexdigest() != binding.get("sha256") + or identity.get("role") != "report_builder" + or identity.get("version") != binding.get("version") + or not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", identity.get("version", "")) + or not re.fullmatch(r"[0-9a-f]{64}", identity.get("source_sha256", "")) + or (identity.get("revision") is not None + and not re.fullmatch(r"[0-9a-f]{40}", identity["revision"]))): + raise ValueError("Report Browser Eval version changed") + for route, page in pages: + if page.select_one('[data-english-not-generated]'): + continue + labels = page.select('[data-browser-eval-release]') + if (len(labels) != 1 or labels[0].get("data-browser-eval-release") != identity["version"] + or labels[0].get_text(" ", strip=True) != "Browser Eval v" + identity["version"] + or labels[0].get("href") != ("../" if route.startswith("en/") else "") + FILENAME): + raise ValueError("Report lost its Browser Eval version label: " + route) + + +def record_build_version(root): + """Stamp builders whose report manifest is already complete.""" + path = root / "report-manifest.json" + manifest = json.loads(path.read_text()) if path.is_file() else {"schema_version": 1} + manifest["browser_eval"] = bind_build_version(root) + path.write_text(json.dumps(manifest, indent=2) + "\n") diff --git a/harness/trajectory_replay/webarena_report.py b/harness/trajectory_replay/webarena_report.py index 0e72c0eea..59d0149ac 100644 --- a/harness/trajectory_replay/webarena_report.py +++ b/harness/trajectory_replay/webarena_report.py @@ -2183,6 +2183,8 @@ def build_webarena_replay_report( "measurement_notes": {lang: profile["measurement_notes"][lang] for lang in languages} if profile.get("measurement_notes") else {}, } + from harness.reporting.versions import bind_build_version + manifest['browser_eval'] = bind_build_version(output) _write_json(output / "report-manifest.json", manifest) if language is not None: entry = select_report_language(output, language) diff --git a/harness/trajectory_replay/webarena_report_check.py b/harness/trajectory_replay/webarena_report_check.py index d6eae7675..5919c6462 100644 --- a/harness/trajectory_replay/webarena_report_check.py +++ b/harness/trajectory_replay/webarena_report_check.py @@ -526,6 +526,8 @@ def check(report_dir: Path, *, profile_path: Path | None = None) -> dict[str, in reject_evidence_symlinks(report_dir) root = report_dir.resolve() manifest = json.loads((root / "report-manifest.json").read_text(encoding="utf-8")) + from harness.reporting.versions import check_build_version + check_build_version(root, manifest) if (manifest.get("profile_sha256") and manifest.get("template") != template_identity("trajectory-replay")): raise ValueError("report bypasses the maintained replay template") diff --git a/harness/web_fetch/__init__.py b/harness/web_fetch/__init__.py index 478b919b6..6de505ec3 100644 --- a/harness/web_fetch/__init__.py +++ b/harness/web_fetch/__init__.py @@ -1,3 +1,3 @@ """Browser harness evaluation toolkit.""" -__version__ = "0.1.0" +__version__ = "0.2.0" diff --git a/harness/web_fetch/published_snapshot.py b/harness/web_fetch/published_snapshot.py index 2cef0b001..13310e47d 100644 --- a/harness/web_fetch/published_snapshot.py +++ b/harness/web_fetch/published_snapshot.py @@ -230,6 +230,8 @@ def build_snapshot_audit(source, output, *, profile): 'score_audit':comparison['score_audit'], 'profile_sha256':file_sha256(profile)} if release_binding: manifest['benchmark_release'] = release_binding + from harness.reporting.versions import bind_build_version + manifest['browser_eval'] = bind_build_version(staging) (staging/'report-manifest.json').write_text(json.dumps(manifest,ensure_ascii=False,indent=2)+'\n') check_snapshot_audit(staging) _install_report_tree(staging, output) diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py index b67ae17c8..b31590f7b 100644 --- a/harness/web_fetch/report.py +++ b/harness/web_fetch/report.py @@ -2218,7 +2218,7 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english release_path=out/'benchmark-release.json' if release_path.is_file(): release=json.loads(release_path.read_text()) - description=''+description+' · '+esc(release['benchmark']+' v'+release['version'])+'' + description=''+description+' · '+esc('bench v'+release['version'])+'' home='
'+hero('Web Fetch 评测报告',description) from .report_explainer import render_explainer TITLE='本轮调用链检查与日志范围' if dataset.get('comparison',{}).get('call_chain_audit') else '本报告的任务范围与验收规则' diff --git a/harness/web_fetch/report_build.py b/harness/web_fetch/report_build.py index b1b7fd4ff..34b11ecaf 100644 --- a/harness/web_fetch/report_build.py +++ b/harness/web_fetch/report_build.py @@ -5,6 +5,8 @@ import tempfile from pathlib import Path +from harness.reporting.versions import record_build_version + from harness.record_io import file_sha256, load_jsonl from harness.reporting.assets import copy_bound_assets @@ -139,6 +141,8 @@ def build_diagnostic_cohorts(source: Path | None, output: Path, profile: Path | "input_sha256": input_sha256, "artifact_hashes": {path: hashlib.sha256(raw).hexdigest() for path, raw in files.items()}} + from harness.reporting.versions import bind_build_version + manifest['browser_eval'] = bind_build_version(staging) (staging / "report-manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2)) check_diagnostic_report(staging, profile=profile) staging.rename(output) @@ -214,6 +218,7 @@ def build_public_benchmark_research(source: Path, output: Path, *, include_engli staging = Path(tempfile.mkdtemp(prefix='.research-report-build-', dir=output.parent)) try: generate_public_benchmark_research(staging, research, include_english=include_english) + record_build_version(staging) if sample is not None: shutil.copy2(sample_path, staging / 'sample-manifest.json') if language is not None: @@ -334,6 +339,8 @@ def write_frozen_presentation_manifest(output, manifest, *, show_local_status_ba current.update(template=resolve_template('webfetch'), languages=languages, generated_languages=languages, presentation={"show_local_status_banners": show_local_status_banners, "frozen_scores": True, "authored_interpretation": authored_interpretation}) + from harness.reporting.versions import bind_build_version + current['browser_eval'] = bind_build_version(output) (output / "report-manifest.json").write_text(json.dumps(current, ensure_ascii=False, indent=2) + "\n") @@ -793,6 +800,8 @@ def build_report(source: Path, output: Path, *, include_english: bool = False, if dataset.get('comparison', {}).get('kind') == 'uniform_native_local_measurement': template_manifest.update(evidence_state='local_measurement', certified=False, languages=template_manifest['generated_languages'], provider_calls_added=0) + from harness.reporting.versions import bind_build_version + template_manifest['browser_eval'] = bind_build_version(staging) (staging/'report-manifest.json').write_text(json.dumps(template_manifest, ensure_ascii=False, indent=2)+'\n') (staging/marker).write_text('Generated from persisted evidence. Rebuild; do not edit.\n') # Replace only an explicitly marked generated directory; never merge stale pages. @@ -981,6 +990,7 @@ def build_webmainbench_report(source: Path, output: Path, *, include_english: bo content_binding['relative_path']: content_binding['sha256']}) shutil.copy2(content_binding_path, staging / 'report-content-binding.json') generate_webmainbench(staging, dataset, records, summaries, manifests, review, research, corrections, adoption, fixture_correction, teds_revision, anomaly_review, latency_diagnostic, include_english=include_english, report_content=report_content) + record_build_version(staging) if language is not None: select_report_language(staging, language) (staging / '.generated-report').write_text('Generated from persisted evidence. Rebuild; do not edit.\n') @@ -1213,6 +1223,7 @@ def build_webmainbench_full_report(source: Path, output: Path, *, authority_root shutil.copy2(asset_source, staging / 'assets' / asset['name']) generate_webmainbench_full(staging, dataset, records, summaries, display_profile) + record_build_version(staging) from harness.web_fetch.webmainbench_report_check import ( check as check_webmainbench_full, ) @@ -1459,6 +1470,7 @@ def build_wcxb_report(source: Path, corpus: Path, sample_path: Path, profile: di if language is not None: select_report_language(staging, language) from .wcxb_publication import write_inventory + record_build_version(staging) write_inventory(staging, profile) if output.exists(): backup = Path(tempfile.mkdtemp(prefix='.wcxb-report-old-', dir=output.parent)) / 'report' diff --git a/harness/web_fetch/report_diagnostic.py b/harness/web_fetch/report_diagnostic.py index 6e6eefa09..53b5bceb6 100644 --- a/harness/web_fetch/report_diagnostic.py +++ b/harness/web_fetch/report_diagnostic.py @@ -442,6 +442,8 @@ def check_diagnostic_report(output, profile=None): if profile is not None: require_diagnostic_scope(model, json.loads(Path(profile).read_text())) manifest = json.loads((output / "report-manifest.json").read_text()) + from harness.reporting.versions import check_build_version + check_build_version(output, manifest) if profile is not None: binding = json.loads(Path(profile).read_text()).get('diagnostic_input') if binding and manifest.get('input_sha256') != binding['sha256']: diff --git a/harness/web_fetch/report_version.py b/harness/web_fetch/report_version.py index d926f2474..e3f5b377e 100644 --- a/harness/web_fetch/report_version.py +++ b/harness/web_fetch/report_version.py @@ -104,6 +104,8 @@ def check_release(root, dataset, records, *, profile=None): manifest_path = root / "report-manifest.json" manifest = json.loads(manifest_path.read_text()) if manifest_path.is_file() else {} + from harness.reporting.versions import check_build_version + check_build_version(root, manifest) binding = manifest.get("benchmark_release") expected = _profile_binding(profile) measured = _input_release(root) @@ -136,6 +138,9 @@ def check_release(root, dataset, records, *, profile=None): label is None or label.get("href") != ("../" if route.startswith("en/") else "") + FILENAME or label.get("data-benchmark-release") != release["version"] - or label.get_text(" ", strip=True) != release["benchmark"] + " v" + release["version"] + or label.get_text(" ", strip=True) not in ( + {"bench v" + release["version"]} if manifest.get("browser_eval") else + {"bench v" + release["version"], release["benchmark"] + " v" + release["version"]} + ) ): raise ValueError("Report lost its benchmark version label: " + route) diff --git a/harness/web_fetch/runner.py b/harness/web_fetch/runner.py index 5bfd300f5..99a6e8490 100644 --- a/harness/web_fetch/runner.py +++ b/harness/web_fetch/runner.py @@ -568,6 +568,7 @@ def run_evaluation( started_at = utc_now() records: list[dict[str, Any]] = [] + from harness.reporting.versions import software_identity manifest = { "schema_version": 1, "runtime_source_sha256": hashlib.sha256(SCORING_SOURCE_SHA256.encode()+b'\0'+b'\0'.join( @@ -583,6 +584,7 @@ def run_evaluation( "deadline_policy": "end_to_end_answer; non_cancellable_adapter_quarantined", "public_call_log_schema_version": 1, "tool": "web_fetch", + "browser_eval": software_identity(), "started_at": started_at, "dataset": { "id": dataset_meta.get("id"), diff --git a/pyproject.toml b/pyproject.toml index 92d2ceb82..fc1c34499 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,13 @@ [project] name = "browser-harness-eval" -version = "0.1.0" -description = "Evaluation harness for web fetching, browser trajectory replay and autonomous agents" +version = "0.2.0" +description = "Extensible evaluation harness for browsers and web agents, with shared adapters, evidence, scoring and reports" readme = "README.md" license = "Apache-2.0" license-files = ["LICENSE", "NOTICE", "THIRD_PARTY.md"] requires-python = ">=3.11" dependencies = [ - "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@2ea8c52df67c39eee2ea5113a63f5da34f102160", + "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@d0e78e71cf5145217cb1f2b71420a7c4194d80dd", "beautifulsoup4>=4.12,<5", "fastcdc==1.7.0", ] diff --git a/tests/test_report_software_versions.py b/tests/test_report_software_versions.py new file mode 100644 index 000000000..650e50cc6 --- /dev/null +++ b/tests/test_report_software_versions.py @@ -0,0 +1,93 @@ +"""Readers can identify a report build independently of its benchmark edition.""" + +import json +import pytest +from bs4 import BeautifulSoup + +from harness.reporting import versions +from harness.web_fetch import __version__ +from harness.web_fetch.published_snapshot import build_snapshot_audit, check_snapshot_audit +from tests.test_benchmark_release_binding import configured + + +def test_build_records_software_without_relabeling_saved_benchmark(tmp_path, monkeypatch): + source, profile, _ = configured(tmp_path) + before = {path.relative_to(source): path.read_bytes() + for path in source.rglob('*') if path.is_file()} + first = tmp_path / 'first' + build_snapshot_audit(source, first, profile=profile) + identity = json.loads((first / versions.FILENAME).read_text()) + assert identity['version'] == __version__ + assert identity['role'] == 'report_builder' + assert len(identity['revision']) == 40 + assert len(identity['source_sha256']) == 64 + page = BeautifulSoup((first / 'index.html').read_text(), 'html.parser') + assert page.select_one('[data-browser-eval-release]').get_text() == 'Browser Eval v' + __version__ + assert page.select_one('[data-benchmark-release]').get_text() == 'bench v1.0.0' + + # A newer builder must not change the selected tasks, scorer or saved calls. + monkeypatch.setattr(versions, 'software_identity', lambda: {**identity, 'version': '0.3.0'}) + second = tmp_path / 'second' + build_snapshot_audit(source, second, profile=profile) + assert 'Browser Eval v0.3.0' in (second / 'index.html').read_text() + for file in ('benchmark-release.json', 'results.jsonl', 'dataset.json'): + assert (first / file).read_bytes() == (second / file).read_bytes() + assert before == {path.relative_to(source): path.read_bytes() + for path in source.rglob('*') if path.is_file()} + check_snapshot_audit(first) + check_snapshot_audit(second) + + +@pytest.mark.parametrize('defect', ['file', 'version', 'commit', 'binding', 'label', 'href']) +def test_release_gate_rejects_missing_or_conflicting_build_identity(tmp_path, defect): + source, profile, _ = configured(tmp_path) + output = tmp_path / 'report' + build_snapshot_audit(source, output, profile=profile) + path = output / versions.FILENAME + if defect == 'file': + path.unlink() + elif defect in {'version', 'commit'}: + identity = json.loads(path.read_text()) + identity['version' if defect == 'version' else 'revision'] = 'wrong' + path.write_text(json.dumps(identity)) + elif defect == 'binding': + path = output / 'report-manifest.json' + manifest = json.loads(path.read_text()) + del manifest['browser_eval'] + path.write_text(json.dumps(manifest)) + else: + path = output / 'index.html' + content = path.read_text() + content = content.replace('data-browser-eval-release=', 'data-lost=') if defect == 'label' else content.replace( + 'href="browser-eval-version.json"', 'href="missing-version.json"') + path.write_text(content) + with pytest.raises(ValueError, match='Browser Eval'): + check_snapshot_audit(output) + + +def test_shared_identity_handles_both_languages_and_repeat_builds(tmp_path): + (tmp_path / 'en').mkdir() + for path in (tmp_path / 'index.html', tmp_path / 'en/index.html'): + path.write_text('
Results
') + versions.bind_build_version(tmp_path) + binding = versions.bind_build_version(tmp_path) + versions.check_build_version(tmp_path, {'browser_eval': binding}) + for route in ('index.html', 'en/index.html'): + page = BeautifulSoup((tmp_path / route).read_text(), 'html.parser') + assert len(page.select('[data-browser-eval-release]')) == 1 + assert '../browser-eval-version.json' in (tmp_path / 'en/index.html').read_text() + + +def test_installed_package_does_not_borrow_an_unrelated_checkout(tmp_path, monkeypatch): + package = tmp_path / 'installed/harness/reporting' + package.mkdir(parents=True) + module = package / 'versions.py' + module.write_text('installed package source') + monkeypatch.setattr(versions, '__file__', str(module)) + # This must work without Git and without a repository around the package. + monkeypatch.setattr(versions.subprocess, 'check_output', lambda *a, **kw: pytest.fail('unrelated Git lookup')) + identity = versions.software_identity.__wrapped__() + assert identity['revision'] is None + assert not identity['modified'] + assert identity['version'] == __version__ + assert len(identity['source_sha256']) == 64 diff --git a/uv.lock b/uv.lock index 3c0aac23d..0be4dc936 100644 --- a/uv.lock +++ b/uv.lock @@ -289,7 +289,7 @@ wheels = [ [[package]] name = "browser-harness-eval" -version = "0.1.0" +version = "0.2.0" source = { editable = "." } dependencies = [ { name = "beautifulsoup4" }, @@ -354,7 +354,7 @@ requires-dist = [ { name = "jieba", marker = "extra == 'webmainbench-eval'", specifier = "==0.42.1" }, { name = "json-repair", marker = "extra == 'browser-use-eval'", specifier = ">=0.54,<1" }, { name = "langchain-community", marker = "extra == 'oss-eval'", specifier = "==0.3.30" }, - { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=2ea8c52df67c39eee2ea5113a63f5da34f102160" }, + { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=d0e78e71cf5145217cb1f2b71420a7c4194d80dd" }, { name = "markdown", marker = "extra == 'webmainbench-eval'", specifier = ">=3.7,<4" }, { name = "openai", marker = "extra == 'webmainbench-eval'", specifier = ">=2.8,<4" }, { name = "openpyxl", marker = "extra == 'oss-eval'", specifier = "==3.1.5" }, @@ -1933,7 +1933,7 @@ wheels = [ [[package]] name = "lexbench-webfetch" version = "0.3.0" -source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=2ea8c52df67c39eee2ea5113a63f5da34f102160#2ea8c52df67c39eee2ea5113a63f5da34f102160" } +source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=d0e78e71cf5145217cb1f2b71420a7c4194d80dd#d0e78e71cf5145217cb1f2b71420a7c4194d80dd" } dependencies = [ { name = "beautifulsoup4" }, { name = "soupsieve" }, From 30645af1ba2520b4d089cbef32979e088cffe277 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:03:52 +0800 Subject: [PATCH 3/4] fix(reporting): resolve version links against each document base --- harness/reporting/versions.py | 16 +++++++++++++--- tests/test_report_software_versions.py | 15 +++++++++++---- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/harness/reporting/versions.py b/harness/reporting/versions.py index 88922584f..b7c98f833 100644 --- a/harness/reporting/versions.py +++ b/harness/reporting/versions.py @@ -2,17 +2,27 @@ import hashlib import json +import posixpath import re import subprocess from functools import cache -from html import escape +from html import escape, unescape from pathlib import Path +from urllib.parse import urljoin, urlsplit from harness.web_fetch import __version__ FILENAME = "browser-eval-version.json" +def _version_href(route, content): + base = re.search(r']*\bhref=["\']([^"\']+)', content) + document = 'https://report.invalid/' + route + resolved = urljoin(document, unescape(base[1])) if base else document + directory = posixpath.dirname(urlsplit(resolved).path) + return posixpath.relpath('/' + FILENAME, directory or '/') + + @cache def software_identity(): """Use the package's checkout, never an unrelated working directory.""" @@ -52,7 +62,7 @@ def bind_build_version(root): if "data-english-not-generated" in content: continue content = re.sub(r'<(p|span) data-report-build-version>.*?', '', content, flags=re.S) - href = ("../" if route.startswith("en/") else "") + FILENAME + href = _version_href(route, content) link = ('Browser Eval v' + escape(identity["version"]) + '') @@ -100,7 +110,7 @@ def check_build_version(root, manifest): labels = page.select('[data-browser-eval-release]') if (len(labels) != 1 or labels[0].get("data-browser-eval-release") != identity["version"] or labels[0].get_text(" ", strip=True) != "Browser Eval v" + identity["version"] - or labels[0].get("href") != ("../" if route.startswith("en/") else "") + FILENAME): + or labels[0].get("href") != _version_href(route, str(page))): raise ValueError("Report lost its Browser Eval version label: " + route) diff --git a/tests/test_report_software_versions.py b/tests/test_report_software_versions.py index 650e50cc6..cac247da9 100644 --- a/tests/test_report_software_versions.py +++ b/tests/test_report_software_versions.py @@ -65,17 +65,24 @@ def test_release_gate_rejects_missing_or_conflicting_build_identity(tmp_path, de check_snapshot_audit(output) -def test_shared_identity_handles_both_languages_and_repeat_builds(tmp_path): +@pytest.mark.parametrize('bases', [(None, None), ('./', '../'), ('./?preview=token', '../?preview=token')]) +def test_shared_identity_handles_both_languages_and_repeat_builds(tmp_path, bases): (tmp_path / 'en').mkdir() - for path in (tmp_path / 'index.html', tmp_path / 'en/index.html'): - path.write_text('
Results
') + for path, base in zip((tmp_path / 'index.html', tmp_path / 'en/index.html'), bases): + path.write_text('' + (f'' if base else '') + + '
Results
') versions.bind_build_version(tmp_path) binding = versions.bind_build_version(tmp_path) versions.check_build_version(tmp_path, {'browser_eval': binding}) for route in ('index.html', 'en/index.html'): page = BeautifulSoup((tmp_path / route).read_text(), 'html.parser') assert len(page.select('[data-browser-eval-release]')) == 1 - assert '../browser-eval-version.json' in (tmp_path / 'en/index.html').read_text() + from urllib.parse import urljoin + base = page.select_one('base') + document = (tmp_path / route).as_uri() + target = urljoin(urljoin(document, base['href']) if base else document, + page.select_one('[data-browser-eval-release]')['href']) + assert target == (tmp_path / versions.FILENAME).as_uri() def test_installed_package_does_not_borrow_an_unrelated_checkout(tmp_path, monkeypatch): From 76c2c6b07d230d07a35d30a3993579cb70fbe778 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:38:24 +0800 Subject: [PATCH 4/4] fix(reporting): include runtime assets in software fingerprints --- harness/reporting/versions.py | 7 +++- tests/test_report_software_versions.py | 50 ++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/harness/reporting/versions.py b/harness/reporting/versions.py index b7c98f833..10991f26c 100644 --- a/harness/reporting/versions.py +++ b/harness/reporting/versions.py @@ -13,6 +13,8 @@ from harness.web_fetch import __version__ FILENAME = "browser-eval-version.json" +_RUNTIME_SUFFIXES = {".py", ".js", ".cjs", ".mjs", ".mts", ".c", ".json", + ".html", ".css", ".ini", ".txt", ".png"} def _version_href(route, content): @@ -43,7 +45,10 @@ def software_identity(): except (OSError, subprocess.CalledProcessError): revision = None digest = hashlib.sha256() - for path in sorted(package.rglob("*.py")): + for path in sorted(package.rglob("*")): + if (not path.is_file() or path.suffix not in _RUNTIME_SUFFIXES + or "__pycache__" in path.relative_to(package).parts): + continue digest.update(path.relative_to(package).as_posix().encode() + b"\0" + path.read_bytes() + b"\0") return {"version": __version__, "revision": revision, "source_sha256": digest.hexdigest(), "modified": modified} diff --git a/tests/test_report_software_versions.py b/tests/test_report_software_versions.py index cac247da9..eb08c8990 100644 --- a/tests/test_report_software_versions.py +++ b/tests/test_report_software_versions.py @@ -98,3 +98,53 @@ def test_installed_package_does_not_borrow_an_unrelated_checkout(tmp_path, monke assert not identity['modified'] assert identity['version'] == __version__ assert len(identity['source_sha256']) == 64 + + +@pytest.mark.parametrize('asset', [ + 'web_fetch/assets/report-loader.js', + 'lexbench/assets/render_contract.cjs', + 'web_fetch/html_to_markdown.mjs', + 'web_fetch/workers/openclaw_webfetch_worker.mts', + 'web_fetch/workers/native_dns_darwin.c', + 'web_fetch/report-translations.json', + 'browser_use/report-translations.json', + 'trajectory_replay/messages.json', + 'environments/webarena/runtime/php-fpm-qemu.ini', + 'web_fetch/templates/interpretation-headless.css', +]) +def test_installed_runtime_asset_changes_identify_a_different_build(tmp_path, monkeypatch, asset): + from pathlib import Path + + source = Path(versions.__file__).resolve().parents[1] + package = tmp_path / 'installed/harness' + module = package / 'reporting/versions.py' + module.parent.mkdir(parents=True) + module.write_bytes(Path(versions.__file__).read_bytes()) + target = package / asset + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes((source / asset).read_bytes()) + monkeypatch.setattr(versions, '__file__', str(module)) + original = versions.software_identity.__wrapped__() + assert original['revision'] is None + target.write_bytes(target.read_bytes() + b'\nchanged runtime asset\n') + changed = versions.software_identity.__wrapped__() + assert changed['source_sha256'] != original['source_sha256'] + assert {k: v for k, v in changed.items() if k != 'source_sha256'} == { + k: v for k, v in original.items() if k != 'source_sha256'} + target.unlink() + assert versions.software_identity.__wrapped__()['source_sha256'] not in { + original['source_sha256'], changed['source_sha256']} + + +def test_generated_caches_do_not_change_installed_build_identity(tmp_path, monkeypatch): + module = tmp_path / 'harness/reporting/versions.py' + module.parent.mkdir(parents=True) + module.write_text('runtime source') + monkeypatch.setattr(versions, '__file__', str(module)) + original = versions.software_identity.__wrapped__() + cache = module.parent / '__pycache__' + cache.mkdir() + (cache / 'versions.pyc').write_bytes(b'compiled cache') + (cache / 'metadata.json').write_text('{}') + (module.parent / 'versions.pyc').write_bytes(b'compiled cache') + assert versions.software_identity.__wrapped__() == original