feat(reporting): identify report builds and prepare v0.2.0

CITATION.cff

@@ -6,6 +6,6 @@ authors:
   - name: Browser Eval contributors
 repository-code: "https://github.com/lexmount/browser-eval"
 url: "https://github.com/lexmount/browser-eval"
-version: 0.1.0
-date-released: 2026-10-10
-license: MIT
+version: 0.2.0
+date-released: 2026-10-23
+license: Apache-2.0

CONTRIBUTING.md

@@ -32,6 +32,17 @@ Keep human-readable editions, immutable dataset/scorer hashes and the package
 commit together. A minor scoring correction requires common-edition rescoring;
 changed task populations or requests require a major edition.
 
+## Update Web Fetch dependencies and methods
+
+Update the pinned evaluator with `make bump-lexbench REV=<full merged commit>`.
+It verifies upstream main ancestry and updates the lock and definition hashes.
+Historical report certificates retain their original evaluator identities.
+
+Register Web Fetch products in `harness/web_fetch/data/report-methods.json`;
+executable versions and run parameters come from saved evidence. See
+[method metadata](harness/web_fetch/README.md#report-method-metadata) and
+[adding a method or release](harness/web_fetch/README.md#adding-a-method-or-release).
+
 ## PR and commit titles
 
 Use `<type>(<scope>): <summary>` for PR titles and new commit subjects. The scope is optional; use the module or feature affected, such as `scoring` or `webfetch`.

docs/benchmarks.md

@@ -57,9 +57,17 @@ Use a benchmark version together with exact task/reference and scoring-source
 hashes. Keep the package's full Git commit pinned separately. A package update
 does not select a new dataset for an existing report.
 
-Use the same `benchmark-vMAJOR.MINOR.PATCH` tag prefix in both repositories.
-Reports reference their benchmark edition; report rebuilds use the hosting
-platform's version ID rather than a separate semantic-version sequence.
+Create `bench-vMAJOR.MINOR.PATCH` tags only in the benchmark repository.
+Browser Eval publishes software versions as `vMAJOR.MINOR.PATCH` and records the
+full Git commit for execution and report-building code.
+Reports reference their benchmark edition and the hosting platform's immutable
+version ID for each published build.
+
+Report home pages link to the Browser Eval build version and the selected bench
+edition. Their JSON bindings retain code commits and source/input hashes.
+Rebuilding with a newer Browser Eval release changes the build identity, not the
+saved call identity or benchmark edition. Custom datasets without a frozen
+edition retain their hashes rather than inheriting the installed catalog version.
 
 | Change | Edition | Result comparison |
 | --- | --- | --- |
@@ -68,7 +76,7 @@ platform's version ID rather than a separate semantic-version sequence.
 | Added/removed cases, changed requests, output scope or routes | Major | Separate full-population results |
 
 The task package owns `lexbench-webfetch release` and `compare-releases`.
-See its [version contract](https://github.com/lexmount/lexbench-webfetch/blob/benchmark-v2.0.1/docs/versions.md).
+See its [version contract](https://github.com/lexmount/lexbench-webfetch/blob/bench-v2.0.1/docs/versions.md).
 Freeze `benchmark-release.json` beside the compiled inputs before calls.
 The certified Web Fetch runner accepts `--benchmark-release`; it also discovers
 the sidecar beside the selected dataset. Release creation records the full scorer

docs/report-hosting.md

@@ -85,7 +85,7 @@ Example runs remain diagnostic after upload. Publishing does not certify benchma
 
 ## Maintained benchmark releases
 
-Formal releases use the selected track's package/publish commands after certification, required review and release checks. See [architecture](architecture.md#reports-and-publication) and the [track guides](../README.md#what-it-tests).
+Formal releases use the selected track's package/publish commands after certification, required review and release checks. See [architecture](architecture.md#reports-and-publication) and the [track guides](../README.md#evaluation-tracks).
 
 | Requirement | Why it matters |
 | --- | --- |

evaluation/lexbench-webfetch.json

@@ -2,7 +2,7 @@
   "schema_version": 1,
   "kind": "external_benchmark_binding",
   "repository": "https://github.com/lexmount/lexbench-webfetch",
-  "revision": "2ea8c52df67c39eee2ea5113a63f5da34f102160",
+  "revision": "d0e78e71cf5145217cb1f2b71420a7c4194d80dd",
   "ownership": {
     "benchmark": "Task populations, task model, acceptance scoring, file checks, compiler and frozen definitions",
     "harness": "Execution, reference lifecycle, evidence and reports"
@@ -81,6 +81,6 @@
     "evaluation/scoring-regressions.json": "432ddabe23d045d0ce4c23aef69ca3337ba4d240fe3c2061932cee26b7dc00dc",
     "evaluation/source-inputs.json": "c202abc4624811b938a837402bbaf0ecbe716e375fbca70567346d2edb3b1708",
     "annotation-revisions/serialization-20261021.json": "e0fe42179e584abb73782cbc459f521cab9cb09d3442de1ec4a6ff54d1b1235a",
-    "benchmark-release.json": "96ac7f1044601f4a3f768be814a49a48ff46e3ad9802f7f9e17a524460b57c4b"
+    "benchmark-release.json": "456be14df670cb9d973415ed87d4de6f45d7b5021e3284180c8a0c8b5daee773"
   }
 }

harness/browser_use/report.py

@@ -3991,6 +3991,10 @@ def translated(zh, en):
         write_language_contract(stage, include_english=include_english)
         if language is not None:
             select_report_language(stage, language)
+        from harness.reporting.versions import bind_build_version
+        audit['browser_eval'] = bind_build_version(stage)
+        _write_json(stage / "data/summary.json", audit)
+        _write_json(stage / "report-data.json", audit)
         (stage / ".generated-report").write_text("browser-use-report/2\n", encoding="utf-8")
         backup: Path | None = None
         if output.exists():

harness/browser_use/report_check.py

@@ -478,6 +478,8 @@ def check(
         raise ValueError("report is missing one or more required report pages")
     selection = load_pilot(selection_path)
     audit = json.loads(audit_path.read_text(encoding="utf-8"))
+    from harness.reporting.versions import check_build_version
+    check_build_version(report_dir, audit)
     if audit.get("profile_sha256") and audit.get("template") != template_identity("browser-use"):
         raise ValueError("report bypasses the maintained Browser Use template")
     if audit.get("profile_sha256") and audit["profile_sha256"] != _sha256(profile_path):

harness/reporting/versions.py

@@ -0,0 +1,127 @@
+"""Identify report-building software without changing measured inputs."""
+
+import hashlib
+import json
+import posixpath
+import re
+import subprocess
+from functools import cache
+from html import escape, unescape
+from pathlib import Path
+from urllib.parse import urljoin, urlsplit
+
+from harness.web_fetch import __version__
+
+FILENAME = "browser-eval-version.json"
+_RUNTIME_SUFFIXES = {".py", ".js", ".cjs", ".mjs", ".mts", ".c", ".json",
+                     ".html", ".css", ".ini", ".txt", ".png"}
+
+
+def _version_href(route, content):
+    base = re.search(r'<base\b[^>]*\bhref=["\']([^"\']+)', content)
+    document = 'https://report.invalid/' + route
+    resolved = urljoin(document, unescape(base[1])) if base else document
+    directory = posixpath.dirname(urlsplit(resolved).path)
+    return posixpath.relpath('/' + FILENAME, directory or '/')
+
+
+@cache
+def software_identity():
+    """Use the package's checkout, never an unrelated working directory."""
+    package = Path(__file__).resolve().parents[1]
+    root = package.parent
+    revision = None
+    modified = False
+    if (root / ".git").exists():
+        try:
+            revision = subprocess.check_output(
+                ["git", "rev-parse", "HEAD"], cwd=root, text=True,
+                stderr=subprocess.DEVNULL,
+            ).strip()
+            modified = bool(subprocess.check_output(
+                ["git", "status", "--porcelain", "--untracked-files=normal", "--", "harness"],
+                cwd=root, text=True, stderr=subprocess.DEVNULL,
+            ).strip())
+        except (OSError, subprocess.CalledProcessError):
+            revision = None
+    digest = hashlib.sha256()
+    for path in sorted(package.rglob("*")):
+        if (not path.is_file() or path.suffix not in _RUNTIME_SUFFIXES
+                or "__pycache__" in path.relative_to(package).parts):
+            continue
+        digest.update(path.relative_to(package).as_posix().encode() + b"\0" + path.read_bytes() + b"\0")
+    return {"version": __version__, "revision": revision,
+            "source_sha256": digest.hexdigest(), "modified": modified}
+
+
+def bind_build_version(root):
+    """Bind this build and add one compact link to each generated home page."""
+    identity = {"role": "report_builder", **software_identity()}
+    raw = (json.dumps(identity, indent=2) + "\n").encode()
+    (root / FILENAME).write_bytes(raw)
+    for route in ("index.html", "en/index.html"):
+        page = root / route
+        if not page.is_file():
+            continue
+        content = page.read_text()
+        if "data-english-not-generated" in content:
+            continue
+        content = re.sub(r'<(p|span) data-report-build-version>.*?</\1>', '', content, flags=re.S)
+        href = _version_href(route, content)
+        link = ('<a data-browser-eval-release="'
+                 + escape(identity["version"]) + '" href="' + href + '">Browser Eval v'
+                 + escape(identity["version"]) + '</a>')
+        benchmark = re.search(r'<a\b[^>]*\bdata-benchmark-release=[^>]*>.*?</a>', content, re.S)
+        if benchmark:
+            label = '<span data-report-build-version>' + link + ' · </span>'
+            page.write_text(content[:benchmark.start()] + label + content[benchmark.start():])
+            continue
+        label = '<p data-report-build-version>' + link + '</p>'
+        end = '</footer>' if '</footer>' in content else '</body>'
+        if end not in content:
+            raise ValueError("Report home has no body: " + route)
+        page.write_text(content.replace(end, label + end, 1))
+    return {"file": FILENAME, "sha256": hashlib.sha256(raw).hexdigest(),
+            "version": identity["version"]}
+
+
+def check_build_version(root, manifest):
+    """Keep legacy reports readable; verify every declared build and label."""
+    from bs4 import BeautifulSoup
+
+    binding = manifest.get("browser_eval")
+    path = root / FILENAME
+    pages = [(route, BeautifulSoup((root / route).read_text(), "html.parser"))
+             for route in ("index.html", "en/index.html") if (root / route).is_file()]
+    if binding is None:
+        if path.exists() or any(page.select_one('[data-browser-eval-release]') for _, page in pages):
+            raise ValueError("Report lost its Browser Eval version binding")
+        return
+    if path.is_symlink() or binding.get("file") != FILENAME or not path.is_file():
+        raise ValueError("Report Browser Eval version is missing")
+    raw = path.read_bytes()
+    identity = json.loads(raw)
+    if (hashlib.sha256(raw).hexdigest() != binding.get("sha256")
+            or identity.get("role") != "report_builder"
+            or identity.get("version") != binding.get("version")
+            or not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", identity.get("version", ""))
+            or not re.fullmatch(r"[0-9a-f]{64}", identity.get("source_sha256", ""))
+            or (identity.get("revision") is not None
+                and not re.fullmatch(r"[0-9a-f]{40}", identity["revision"]))):
+        raise ValueError("Report Browser Eval version changed")
+    for route, page in pages:
+        if page.select_one('[data-english-not-generated]'):
+            continue
+        labels = page.select('[data-browser-eval-release]')
+        if (len(labels) != 1 or labels[0].get("data-browser-eval-release") != identity["version"]
+                or labels[0].get_text(" ", strip=True) != "Browser Eval v" + identity["version"]
+                or labels[0].get("href") != _version_href(route, str(page))):
+            raise ValueError("Report lost its Browser Eval version label: " + route)
+
+
+def record_build_version(root):
+    """Stamp builders whose report manifest is already complete."""
+    path = root / "report-manifest.json"
+    manifest = json.loads(path.read_text()) if path.is_file() else {"schema_version": 1}
+    manifest["browser_eval"] = bind_build_version(root)
+    path.write_text(json.dumps(manifest, indent=2) + "\n")

harness/trajectory_replay/webarena_report.py

@@ -2183,6 +2183,8 @@ def build_webarena_replay_report(
         "measurement_notes": {lang: profile["measurement_notes"][lang] for lang in languages}
         if profile.get("measurement_notes") else {},
     }
+    from harness.reporting.versions import bind_build_version
+    manifest['browser_eval'] = bind_build_version(output)
     _write_json(output / "report-manifest.json", manifest)
     if language is not None:
         entry = select_report_language(output, language)

harness/trajectory_replay/webarena_report_check.py

@@ -526,6 +526,8 @@ def check(report_dir: Path, *, profile_path: Path | None = None) -> dict[str, in
     reject_evidence_symlinks(report_dir)
     root = report_dir.resolve()
     manifest = json.loads((root / "report-manifest.json").read_text(encoding="utf-8"))
+    from harness.reporting.versions import check_build_version
+    check_build_version(root, manifest)
     if (manifest.get("profile_sha256")
             and manifest.get("template") != template_identity("trajectory-replay")):
         raise ValueError("report bypasses the maintained replay template")

harness/web_fetch/__init__.py

@@ -1,3 +1,3 @@
 """Browser harness evaluation toolkit."""
 
-__version__ = "0.1.0"
+__version__ = "0.2.0"

harness/web_fetch/published_snapshot.py

@@ -230,6 +230,8 @@ def build_snapshot_audit(source, output, *, profile):
             'score_audit':comparison['score_audit'], 'profile_sha256':file_sha256(profile)}
         if release_binding:
             manifest['benchmark_release'] = release_binding
+        from harness.reporting.versions import bind_build_version
+        manifest['browser_eval'] = bind_build_version(staging)
         (staging/'report-manifest.json').write_text(json.dumps(manifest,ensure_ascii=False,indent=2)+'\n')
         check_snapshot_audit(staging)
         _install_report_tree(staging, output)

harness/web_fetch/report.py

@@ -2218,7 +2218,7 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english
  release_path=out/'benchmark-release.json'
  if release_path.is_file():
   release=json.loads(release_path.read_text())
-  description='<span>'+description+'</span> · <a data-benchmark-release="'+esc(release['version'])+'" href="benchmark-release.json">'+esc(release['benchmark']+' v'+release['version'])+'</a>'
+  description='<span>'+description+'</span> · <a data-benchmark-release="'+esc(release['version'])+'" href="benchmark-release.json">'+esc('bench v'+release['version'])+'</a>'
  home='<div class="home-page">'+hero('Web Fetch 评测报告',description)
  from .report_explainer import render_explainer
  TITLE='本轮调用链检查与日志范围' if dataset.get('comparison',{}).get('call_chain_audit') else '本报告的任务范围与验收规则'

harness/web_fetch/report_build.py

@@ -5,6 +5,8 @@
 import tempfile
 from pathlib import Path
 
+from harness.reporting.versions import record_build_version
+
 from harness.record_io import file_sha256, load_jsonl
 from harness.reporting.assets import copy_bound_assets
 
@@ -139,6 +141,8 @@ def build_diagnostic_cohorts(source: Path | None, output: Path, profile: Path |
                     "input_sha256": input_sha256,
                     "artifact_hashes": {path: hashlib.sha256(raw).hexdigest()
                                         for path, raw in files.items()}}
+        from harness.reporting.versions import bind_build_version
+        manifest['browser_eval'] = bind_build_version(staging)
         (staging / "report-manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
         check_diagnostic_report(staging, profile=profile)
         staging.rename(output)
@@ -214,6 +218,7 @@ def build_public_benchmark_research(source: Path, output: Path, *, include_engli
     staging = Path(tempfile.mkdtemp(prefix='.research-report-build-', dir=output.parent))
     try:
         generate_public_benchmark_research(staging, research, include_english=include_english)
+        record_build_version(staging)
         if sample is not None:
             shutil.copy2(sample_path, staging / 'sample-manifest.json')
         if language is not None:
@@ -334,6 +339,8 @@ def write_frozen_presentation_manifest(output, manifest, *, show_local_status_ba
     current.update(template=resolve_template('webfetch'), languages=languages, generated_languages=languages,
                    presentation={"show_local_status_banners": show_local_status_banners,
                                  "frozen_scores": True, "authored_interpretation": authored_interpretation})
+    from harness.reporting.versions import bind_build_version
+    current['browser_eval'] = bind_build_version(output)
     (output / "report-manifest.json").write_text(json.dumps(current, ensure_ascii=False, indent=2) + "\n")
 
 
@@ -793,6 +800,8 @@ def build_report(source: Path, output: Path, *, include_english: bool = False,
         if dataset.get('comparison', {}).get('kind') == 'uniform_native_local_measurement':
             template_manifest.update(evidence_state='local_measurement', certified=False,
                 languages=template_manifest['generated_languages'], provider_calls_added=0)
+        from harness.reporting.versions import bind_build_version
+        template_manifest['browser_eval'] = bind_build_version(staging)
         (staging/'report-manifest.json').write_text(json.dumps(template_manifest, ensure_ascii=False, indent=2)+'\n')
         (staging/marker).write_text('Generated from persisted evidence. Rebuild; do not edit.\n')
         # Replace only an explicitly marked generated directory; never merge stale pages.
@@ -981,6 +990,7 @@ def build_webmainbench_report(source: Path, output: Path, *, include_english: bo
             content_binding['relative_path']: content_binding['sha256']})
         shutil.copy2(content_binding_path, staging / 'report-content-binding.json')
         generate_webmainbench(staging, dataset, records, summaries, manifests, review, research, corrections, adoption, fixture_correction, teds_revision, anomaly_review, latency_diagnostic, include_english=include_english, report_content=report_content)
+        record_build_version(staging)
         if language is not None:
             select_report_language(staging, language)
         (staging / '.generated-report').write_text('Generated from persisted evidence. Rebuild; do not edit.\n')
@@ -1213,6 +1223,7 @@ def build_webmainbench_full_report(source: Path, output: Path, *, authority_root
                 shutil.copy2(asset_source, staging / 'assets' / asset['name'])
         generate_webmainbench_full(staging, dataset, records,
                                    summaries, display_profile)
+        record_build_version(staging)
         from harness.web_fetch.webmainbench_report_check import (
             check as check_webmainbench_full,
         )
@@ -1459,6 +1470,7 @@ def build_wcxb_report(source: Path, corpus: Path, sample_path: Path, profile: di
         if language is not None:
             select_report_language(staging, language)
         from .wcxb_publication import write_inventory
+        record_build_version(staging)
         write_inventory(staging, profile)
         if output.exists():
             backup = Path(tempfile.mkdtemp(prefix='.wcxb-report-old-', dir=output.parent)) / 'report'

harness/web_fetch/report_diagnostic.py

@@ -442,6 +442,8 @@ def check_diagnostic_report(output, profile=None):
     if profile is not None:
         require_diagnostic_scope(model, json.loads(Path(profile).read_text()))
     manifest = json.loads((output / "report-manifest.json").read_text())
+    from harness.reporting.versions import check_build_version
+    check_build_version(output, manifest)
     if profile is not None:
         binding = json.loads(Path(profile).read_text()).get('diagnostic_input')
         if binding and manifest.get('input_sha256') != binding['sha256']: