Fix acceptance isolation and explicit page identity contracts

README.md

@@ -106,6 +106,16 @@ uv run --frozen browser-harness-eval run \
 
 Run these two commands from Browser Eval. Ordinary runs are diagnostic; a formal live-site comparison additionally requires its pre-call/post-call reference certification. The benchmark input checker verifies frozen bytes, not present-day website validity. See the [Browser Eval integration guide](https://github.com/lexmount/browser-eval/blob/main/docs/benchmarks.md).
 
+## Scoring behavior
+
+Version 0.3 matches Latin words at word boundaries for facts, markers and stable anchors. Required body facts use the returned content; a separate document title cannot supply missing body text. Chinese phrases retain literal matching, and numeric and currency checks keep their value boundaries.
+
+A task can set `page_identity_required: true` to require its final URL observations. Missing or mismatched observations then fail acceptance. Dynamic tasks can also set `document_language_match: "exact"` when the complete language tag matters; the default is `"primary"`. These checks apply across interfaces. Frozen tasks that do not require URL metadata keep their content-based acceptance rules.
+
+Offline scoring recomputes derived fields. It retains the original reference hash, deadline evidence and file fingerprints. A file verdict requires current file verification; an old pass flag cannot certify a rescore. Summaries expose `mean_all_attempts_ms`, `all_attempts_p50_ms` and `all_attempts_p95_ms` alongside the existing successful-call timings. The all-attempt population includes failed calls and timeouts.
+
+These changes do not revise frozen tasks or certify historical results. Score corrections require a new evidence revision tied to the scorer and its regression controls.
+
 ## Repository contents
 
 | Path | Purpose |

docs/README.zh-CN.md

@@ -106,6 +106,16 @@ uv run --frozen browser-harness-eval run \
 
 普通运行用于诊断。正式比较实时网站时，还需要完成调用前后的参考内容认证。输入校验只能证明文件版本相符，不能证明网站今天仍然符合旧参考。详见 [Browser Eval 接入说明](https://github.com/lexmount/browser-eval/blob/main/docs/benchmarks.md)。
 
+## 评分规则
+
+0.3 版对事实、标记和固定锚点中的拉丁文使用单词边界匹配。正文事实只匹配返回内容，单独返回的文档标题不能补足正文。中文短语仍按原文匹配，数字和货币保留原有的数值边界检查。
+
+任务可设置 `page_identity_required: true`，要求返回的最终网址符合任务声明；缺少网址或网址不符都会判为未通过。动态任务需要区分完整语言标签时，可再设置 `document_language_match: "exact"`，默认值为 `"primary"`，只比较主语言。这些检查适用于所有接口。没有要求网址元数据的冻结任务，继续按原有内容规则验收。
+
+离线评分会重新计算派生字段，保留原始参考哈希、调用时限证据和文件指纹。文件结论需要当前文件验收依据；旧的通过标记不能认证本次重评分。汇总在原有成功调用耗时之外，提供全部调用的 `mean_all_attempts_ms`、`all_attempts_p50_ms` 和 `all_attempts_p95_ms`，包含失败调用与超时。
+
+上述修改不改写冻结题目，也不自动认证历史成绩。更正分数仍需建立新的证据修订，绑定评分器版本和回归用例。
+
 ## 仓库内容
 
 | 路径 | 用途 |

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "lexbench-webfetch"
-version = "0.2.0"
+version = "0.3.0"
 description = "Versioned Web Fetch tasks, acceptance contracts and frozen reference bindings"
 readme = "README.md"
 requires-python = ">=3.11"

src/lexbench_webfetch/benchmark.py

@@ -20,7 +20,8 @@
     "fact_coverage_threshold", "expected_resources", "resource_output_contract",
     "content_score_applicable", "required_patterns", "acceptance_contract",
     "composite_acceptance", "mutable_body_policy", "body_requirements", "retrieval_hints",
-    "source_review_url", "evaluation_url", "expected_final_url", "expected_document_language",
+    "source_review_url", "evaluation_url", "expected_final_url", "page_identity_required",
+    "expected_document_language", "document_language_match",
     "expected_document_language_present", "target_language", "target_region",
 )
 

src/lexbench_webfetch/metrics.py

@@ -223,8 +223,16 @@ def _inline_frame_candidates(
     URLs that the browser serialized, and ignores hidden synchronization or ad
     frames.  The same parser applies to any provider returning inline frames.
     """
+    if not re.search(r'<iframe\b', content, re.I):
+        return []
     parser = _VisibleInlineFrames()
-    parser.feed(content)
+    try:
+        parser.feed(content)
+    except AssertionError:
+        # The stdlib HTML scanner rejects XML conditional declarations that
+        # can occur in returned documents or code examples. Frame extraction
+        # is supplementary; direct resource links remain independently checked.
+        return []
     candidates: list[tuple[str, str]] = []
     for document, declared_base in parser.documents[:100]:
         try:
@@ -463,6 +471,8 @@ def task_passed(ok: bool, metrics: dict, task_kind: str | None = None) -> bool:
     if (not task_measured(metrics, task_kind) or metrics.get('deadline_overrun')
             or metrics.get('blocked_page')):
         return False
+    if metrics.get('page_identity_required') and metrics.get('page_identity_status') != 'verified':
+        return False
     if "stable_task_pass" in metrics:
         combined = (metrics.get("content_pass")
                     if ("composite_acceptance_pass" in metrics
@@ -566,6 +576,13 @@ def fact_in_text(text: str, fact: str) -> bool:
         return False
     left = r'(?<![0-9.+\-])' if fact[0].isdigit() or fact[0] in '$£€¥' else ''
     right = r'(?![0-9]|\.[0-9])' if fact[-1].isdigit() else ''
+    # Latin words must not match fragments of longer words. Keep CJK phrase
+    # matching and the stricter numeric/currency boundaries intact.
+    latin_word = r'a-z0-9_\u00c0-\u024f\u1e00-\u1eff'
+    if 'LATIN' in unicodedata.name(fact[0], ''):
+        left = rf'(?<![{latin_word}])'
+    if 'LATIN' in unicodedata.name(fact[-1], ''):
+        right = rf'(?![{latin_word}])'
     for match in re.finditer(left + re.escape(fact) + right, text):
         if fact[0] in '$£€¥' and text[:match.start()].rstrip().endswith(('-', '+')):
             continue
@@ -645,16 +662,17 @@ def evaluate_content(
     truncation_hits = declared_truncation_hits(answer_text)
     folded = normalize_for_match(answer_text)
     content_folded = normalize_for_match(visible_content)
-    matched = [marker for marker in expected if (text := normalize_for_match(marker)) and text in content_folded]
+    matched = [marker for marker in expected
+               if fact_in_text(content_folded, normalize_for_match(marker))]
     facts = required_facts or []
     from .dynamic_entry_matching import matching_entry_facts
-    entry_hits = matching_entry_facts(folded, facts, normalize_for_match, fact_in_text)
+    entry_hits = matching_entry_facts(content_folded, facts, normalize_for_match, fact_in_text)
     fact_results = []
     for index, fact in enumerate(facts):
         alternatives = [str(item) for item in fact.get("alternatives", [])]
         identifier = str(fact.get("id") or f"fact-{index + 1}")
         hits = (entry_hits[identifier] if identifier in entry_hits else
-                [item for item in alternatives if fact_in_text(folded, normalize_for_match(item))])
+                [item for item in alternatives if fact_in_text(content_folded, normalize_for_match(item))])
         fact_results.append({
             "id": identifier,
             "alternatives": alternatives,
@@ -872,6 +890,8 @@ def summarize(records: list[dict[str, Any]]) -> dict[str, Any]:
             "mean_content_ms": round(statistics.mean(content_successful_elapsed), 2) if content_successful_elapsed else None,
             "transport_p50_ms": percentile(transport_successful_elapsed, 0.50),
             "transport_p95_ms": percentile(transport_successful_elapsed, 0.95),
+            "mean_all_attempts_ms": round(statistics.mean(elapsed), 2),
+            "all_attempts_p50_ms": percentile(elapsed, 0.50),
             "all_attempts_p95_ms": percentile(elapsed, 0.95),
             "median_content_chars": round(
                 statistics.median(item["metrics"]["content_chars"] for item in items), 1

src/lexbench_webfetch/models.py

@@ -46,7 +46,9 @@ class EvalCase:
     source_review_url: str = ""
     evaluation_url: str = ""
     expected_final_url: str = ""
+    page_identity_required: bool = False
     expected_document_language: str = ""
+    document_language_match: str = "primary"
     expected_document_language_present: bool = True
     target_language: str = ""
     target_region: str = ""

src/lexbench_webfetch/scoring.py

@@ -42,39 +42,37 @@ def load_list_reference(case: EvalCase | dict[str, Any], source: Path) -> dict |
     return None
 
 
-# These conditional outputs must not survive a change of acceptance mode. All
-# unconditional content/resource fields are overwritten below. Other evidence
-# (e.g. raw_file_*, original reference binding and deadline) is retained, not
-# re-certified by text scoring.
-_CONDITIONAL_FIELDS = {
-    "acceptance_contract_sha256", "content_decision_basis", "factual_correctness",
-    "resource_acceptance_contract_sha256",
-    "structure_pass", "named_entry_count", "minimum_entries", "snapshot_f1",
-    "dynamic_titles", "verification_note", "confirmed_entries", "checked_entries",
-    "page_identity_required", "page_identity_status", "final_url_consistent",
-    "document_language_required",
-    "language_consistent", "observed_document_language",
-    "acceptable_final_urls",
-    "ordered_tree", "ordered_tree_similarity",
-    "composite_acceptance_sha256",
-    "body_requirements_sha256",
+# Only collection evidence survives an offline score. Derived acceptance fields
+# are recomputed, including fields no longer emitted by the current scorer.
+# File fingerprints stay readable; a verdict needs current file verification.
+_RETAINED_EVIDENCE_FIELDS = {
+    "reference_sha256", "deadline_overrun", "deadline_budget_ms", "file_evidence",
+    "raw_file_sha256",
 }
 
 
 def _same_page_url(actual: str, expected: str) -> bool:
-    actual_parts, expected_parts = urlsplit(actual), urlsplit(expected)
+    if not isinstance(actual, str) or not isinstance(expected, str):
+        return False
+    try:
+        actual_parts, expected_parts = urlsplit(actual), urlsplit(expected)
+        actual_port = actual_parts.port or {"http": 80, "https": 443}.get(actual_parts.scheme)
+        expected_port = expected_parts.port or {"http": 80, "https": 443}.get(expected_parts.scheme)
+    except ValueError:
+        return False
     actual_path = actual_parts.path.rstrip("/") or "/"
     expected_path = expected_parts.path.rstrip("/") or "/"
-    if ((actual_parts.scheme, actual_parts.hostname, actual_path)
-            != (expected_parts.scheme, expected_parts.hostname, expected_path)):
+    if ((actual_parts.scheme, actual_parts.hostname, actual_port, actual_path)
+            != (expected_parts.scheme, expected_parts.hostname, expected_port, expected_path)):
         return False
     # A declared query can select a different page on the same path (for
     # example, two distinct search results). Compare its decoded parameters;
     # a fragment does not change the fetched page.
-    expected_query = parse_qsl(expected_parts.query, keep_blank_values=True)
-    return (not expected_query or
-            sorted(parse_qsl(actual_parts.query, keep_blank_values=True))
-            == sorted(expected_query))
+    def page_query(parts):
+        return sorted((key, value) for key, value in parse_qsl(parts.query, keep_blank_values=True)
+                      if not key.casefold().startswith('utm_')
+                      and key.casefold() not in {'gclid', 'fbclid'})
+    return page_query(actual_parts) == page_query(expected_parts)
 
 
 def score_attempt(
@@ -100,14 +98,19 @@ def score_attempt(
     evidence is preserved; the caller must have validated its artifact binding.
     """
     case = _case_dict(case)
+    identity_required = case.get("page_identity_required", False)
+    if not isinstance(identity_required, bool):
+        raise ValueError("page_identity_required must be boolean")
+    language_match = case.get("document_language_match", "primary")
+    if language_match not in {"primary", "exact"}:
+        raise ValueError("document_language_match must be primary or exact")
     from .response_input import answer_input
     # Live callers forward the conversion completed inside the timed boundary.
     # Offline callers reconstruct it from immutable evidence by the same function.
     content, links, ok, status_basis = (prepared_answer if prepared_answer is not None else
         answer_input(artifact, metadata, ok=ok, status_code=status_code))
     result = {key: value for key, value in (previous_metrics or {}).items()
-              if key not in _CONDITIONAL_FIELDS
-              and not key.startswith(("stable_", "list_", "page_context_"))}
+              if key in _RETAINED_EVIDENCE_FIELDS}
     keys = ("required_facts", "forbidden_any", "min_content_chars",
             "fact_coverage_threshold", "ground_truth_main_content", "required_patterns",
             "acceptance_contract", "body_requirements", "composite_acceptance")
@@ -143,7 +146,6 @@ def score_attempt(
                   normalized_named_link_count=len(links))
     dynamic_page_identity = bool(
         case.get("track") == "dynamic"
-        and (metadata or {}).get("route") == "dynamic_selected_text"
         and case.get("evaluation_url")
         and case.get("expected_final_url")
         and case.get("target_language")
@@ -178,8 +180,10 @@ def score_attempt(
         language_consistent = (
             observed_language == "" if not language_present else bool(
                 isinstance(observed_language, str) and observed_language.strip()
-                and observed_language.split("-", 1)[0].casefold()
-                == expected_document_language.split("-", 1)[0].casefold()
+                and (observed_language.casefold() == expected_document_language.casefold()
+                     if language_match == "exact" else
+                     observed_language.split("-", 1)[0].casefold()
+                     == expected_document_language.split("-", 1)[0].casefold())
             )
         )
         identity_status = (
@@ -189,7 +193,7 @@ def score_attempt(
             else "unavailable"
         )
         result.update(
-            page_identity_required=False,
+            page_identity_required=identity_required,
             page_identity_status=identity_status,
             final_url_consistent=final_consistent,
             language_consistent=language_consistent,
@@ -207,13 +211,19 @@ def score_attempt(
             and any(_same_page_url(final_url, expected) for expected in acceptable_urls)
         )
         result.update(
-            page_identity_required=False,
+            page_identity_required=identity_required,
             page_identity_status=(
                 "verified" if final_consistent else "mismatch" if final_url else "unavailable"
             ),
             final_url_consistent=final_consistent,
             acceptable_final_urls=acceptable_urls,
         )
+    elif identity_required:
+        consistent = bool(final_url and _same_page_url(
+            final_url, case.get("expected_final_url") or case["url"]))
+        result.update(page_identity_required=True, final_url_consistent=consistent,
+                      page_identity_status="verified" if consistent else
+                      "mismatch" if final_url else "unavailable")
     else:
         result.update(page_identity_required=False, page_identity_status="not_required")
     result["task_kind"] = task_kind
@@ -242,7 +252,7 @@ def score_attempt(
             if "declared_truncation" not in result["stable_task_errors"]:
                 result["stable_task_errors"].append("declared_truncation")
     file_evaluation = (metadata or {}).get('file_evaluation')
-    if result['task_kind'] == 'file' and isinstance(file_evaluation, dict) and 'raw_file_pass' not in result:
+    if result['task_kind'] == 'file' and isinstance(file_evaluation, dict):
         result['raw_file_pass'] = bool(file_evaluation.get('pass'))
     result["content_score_applicable"] = case.get("content_score_applicable", True)
     contract = scoring_values.get("acceptance_contract")