From 38d224ece97a2c22e73ab56672e47644a4e22d7f Mon Sep 17 00:00:00 2001 From: Maisiqi <324751762+Maisiqi@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:34:31 +0800 Subject: [PATCH 01/18] Add reusable ten-method native Web Fetch cohort --- .env.example | 7 + README.md | 4 +- configs/providers.example.json | 135 +++++- docs/README.zh-CN.md | 4 +- docs/provider-contract.md | 189 ++++++++ harness/web_fetch/benchmark_runner.py | 8 +- harness/web_fetch/call_boundary.py | 24 +- harness/web_fetch/certification.py | 8 +- harness/web_fetch/cli.py | 16 +- harness/web_fetch/config.py | 20 +- harness/web_fetch/data/report-methods.json | 489 ++++++++++++++++++-- harness/web_fetch/diagnostics.py | 2 +- harness/web_fetch/native_projection.py | 22 +- harness/web_fetch/native_routes.py | 26 +- harness/web_fetch/providers/base.py | 7 +- harness/web_fetch/providers/hosted_fetch.py | 221 +++++++++ harness/web_fetch/report_build.py | 3 + harness/web_fetch/runner.py | 52 ++- tests/fixtures/native-route-baseline.json | 36 +- tests/test_accounts.py | 5 +- tests/test_hosted_fetch.py | 362 +++++++++++++++ tests/test_native_route_contract.py | 8 +- tests/test_report_methods.py | 20 + tests/test_scoring.py | 4 +- tests/test_selected_native_policy.py | 171 +++++++ 25 files changed, 1730 insertions(+), 113 deletions(-) create mode 100644 harness/web_fetch/providers/hosted_fetch.py create mode 100644 tests/test_hosted_fetch.py create mode 100644 tests/test_selected_native_policy.py diff --git a/.env.example b/.env.example index 8fc860082..feea29872 100644 --- a/.env.example +++ b/.env.example @@ -6,6 +6,8 @@ OPENCLAW_REPO= OPENCLAW_NODE=~/.browser-eval/toolchains/openclaw-node/node_modules/node/bin/node OPENCLAW_PNPM=~/.browser-eval/toolchains/openclaw-node/node_modules/.bin/pnpm DEEPSEEK_HARNESS_REPO= +# DeepSeek standalone Fetch API: https://platform.deepseek.com/api_keys +DEEPSEEK_API_KEY= OSS_PYTHON=.venv/bin/python # Official Moli CLIs; each measured release binary is version- and SHA-256-gated. # Use an executable path or a command on PATH. No API key is needed. @@ -66,7 +68,9 @@ FIRECRAWL_API_KEY= TAVILY_API_KEY= EXA_API_KEY= BROWSERBASE_API_KEY= +# Optional for browser sessions; not required by the Fetch adapter. BROWSERBASE_PROJECT_ID= +BROWSER_USE_API_KEY= # Query-to-search-results and model-bound retrieval BOCHA_API_KEY= @@ -87,3 +91,6 @@ OXYLABS_PASSWORD= # Rainbond OpenAPI token used only by the disposable WebArena slice manager. # Create it in Rainbond: Personal Center -> Access Token. RAINBOND_API_TOKEN= + +# Crawl4AI Cloud: https://crawl4ai.com/dashboard/ (distinct from the local library) +CRAWL4AI_CLOUD_API_KEY= diff --git a/README.md b/README.md index 586f61ee4..d2429fc24 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Each track reports its own task population and scores. HTTP is a fetch baseline; | Track | Implemented participants | Preparation | | --- | --- | --- | | Web Fetch: local browsers | Moli, Chromium, Lightpanda, Obscura | Install the selected binary; Chromium also needs Playwright and Node.js conversion dependencies | -| Web Fetch: services | Lexmount, Firecrawl, Tavily Extract, Exa Contents, Cloudflare Markdown/Content; hosted Kitesurf via Cloudflare | Provider account and credentials in the repository `.env` | +| Web Fetch: services | Lexmount, Firecrawl, Tavily Extract, Exa Contents, Cloudflare Markdown/Content, Bright Data Web Unlocker, Browser Use fetch-use, Browserbase Fetch, DeepSeek Fetch API, Crawl4AI Cloud; hosted Kitesurf via Cloudflare | Provider account and credentials in the repository `.env` | | Web Fetch: Agent tools and extractors | CowAgent, OpenClaw, DeepSeek Harness, LangChain WebBaseLoader, Crawl4AI | Upstream checkout/runtime or the `oss-eval` extra, according to the adapter | | Browser Trajectory Replay / Browser Use | Moli and Chromium in the current browser registry | Docker/Linux environment; Browser Use also needs CowAgent and a model account | | Controls | Plain HTTP and offline fixture | Base Python installation | @@ -193,3 +193,5 @@ Update the pinned LexBench evaluator with `make bump-lexbench REV= timeout: value = TimeoutError('End-to-end answer deadline exceeded') if isinstance(value, TimeoutError): + sink = getattr(provider, "_call_diagnostics", None) + if sink: + sink.emit("caller", "answer_timeout", type=type(value).__name__, message=str(value), + timeout_seconds=timeout, worker_still_running=worker.is_alive()) # Completion/cancellation cleanup is not additional answer budget. answer_elapsed_ms = round((completed - begin) * 1000, 2) provider.cancel() @@ -86,4 +96,12 @@ def work(): ), worker if isinstance(value, Exception): raise value + if response_only: + # Stop the user-observed timer before scorer input preparation/storage. + value.metadata['_answer_elapsed_ms'] = round((completed - begin) * 1000, 2) + value.metadata['latency_scope'] = 'native_response_received' + with diagnostic_context(getattr(provider, "_call_diagnostics", None)): + emit("provider_returned", ok=value.ok, content_chars=len(value.content)) + value = prepare_answer(value, case) + emit("answer_prepared", content_chars=len(value.prepared_answer[0])) return value, worker diff --git a/harness/web_fetch/certification.py b/harness/web_fetch/certification.py index 9a46ab084..45045fa72 100644 --- a/harness/web_fetch/certification.py +++ b/harness/web_fetch/certification.py @@ -21,7 +21,7 @@ webmainbench_runner, ) from harness.web_fetch.reference_inputs import read_dataset_bytes -from harness.web_fetch.config import DEFAULT_CALL_TIMEOUT_SECONDS, add_failure_retry_argument +from harness.web_fetch.config import DEFAULT_CALL_TIMEOUT_SECONDS, add_failure_retry_argument, resolve_provider_selection ROOT = Path(__file__).resolve().parents[2] @@ -142,7 +142,7 @@ def main(argv: list[str] | None = None) -> None: parser.add_argument("--dataset", type=Path, default=ROOT / "evaluation/dataset.json") parser.add_argument("--output", type=Path, required=True) parser.add_argument("--work-root", type=Path, required=True) - parser.add_argument("--providers", required=True) + parser.add_argument("--providers", required=True, help="Comma-separated names or configured with --provider-config") parser.add_argument("--case-ids", required=True) parser.add_argument("--workers", type=int, default=1) parser.add_argument("--timeout", type=float, default=DEFAULT_CALL_TIMEOUT_SECONDS, @@ -179,6 +179,10 @@ def main(argv: list[str] | None = None) -> None: parser.error("--timeout must be positive and finite") if args.provider_config and not args.provider_config.is_file(): parser.error("--provider-config must be an existing file") + try: + args.providers = resolve_provider_selection(args.providers, args.provider_config) + except ValueError as error: + parser.error(str(error)) if args.call_order == "case-alternating" and (args.workers != 1 or not args.provider_config): parser.error("Alternating case calls require --workers 1 and --provider-config") if args.browser_capture_attempts < 1: diff --git a/harness/web_fetch/cli.py b/harness/web_fetch/cli.py index a01c3b00e..d07204ed0 100644 --- a/harness/web_fetch/cli.py +++ b/harness/web_fetch/cli.py @@ -5,7 +5,7 @@ from pathlib import Path from .accounts import print_account_checklist -from .config import load_dataset, load_provider_config, load_runtime_env, select_diagnostic_scope, add_failure_retry_argument, DEFAULT_CALL_TIMEOUT_SECONDS +from .config import load_dataset, load_provider_config, load_runtime_env, select_diagnostic_scope, resolve_provider_selection, add_failure_retry_argument, DEFAULT_CALL_TIMEOUT_SECONDS from lexbench_webfetch.metrics import task_passed from lexbench_webfetch.models import TASK_KINDS from .providers import build_provider @@ -28,7 +28,8 @@ def command_run(args: argparse.Namespace) -> int: load_runtime_env(args.env_file) dataset_meta, cases = load_dataset(args.dataset) provider_configs = load_provider_config(args.provider_config) - names = [name.strip() for name in args.providers.split(",") if name.strip()] + selection = resolve_provider_selection(args.providers, args.provider_config) + names = [name.strip() for name in selection.split(",") if name.strip()] unknown = [name for name in names if name not in provider_configs] if unknown: raise SystemExit(f"Unknown providers: {', '.join(unknown)}") @@ -50,7 +51,11 @@ def command_run(args: argparse.Namespace) -> int: delay=args.delay, provider_concurrency=args.provider_concurrency, max_failure_retries=args.max_failure_retries, + collect_only=getattr(args, "collect_only", False), ) + if getattr(args, "collect_only", False): + print(f"Collected responses and logs: {args.output.resolve()}") + return 0 report_path = build_report(args.output, args.output / 'report', diagnostic_preview=True, language=args.language) print(f"Report: {report_path.resolve()}") @@ -81,7 +86,8 @@ def command_validate(args: argparse.Namespace) -> int: load_runtime_env(args.env_file) _, cases = load_dataset(args.dataset) configs = load_provider_config(args.provider_config) - names = [name.strip() for name in args.providers.split(",") if name.strip()] + selection = resolve_provider_selection(args.providers, args.provider_config) + names = [name.strip() for name in selection.split(",") if name.strip()] errors = [] for name in names: if name not in configs: @@ -114,12 +120,14 @@ def build_parser() -> argparse.ArgumentParser: add_language_argument(run, default="en") run.add_argument("--dataset", type=Path, default=EXAMPLES / "fixture-dataset.json") run.add_argument("--provider-config", type=Path, default=EXAMPLES / "fixture-providers.json") - run.add_argument("--providers", default="fixture", help="Comma-separated configured provider names") + run.add_argument("--providers", default="fixture", help="Comma-separated names, or configured for the provider config default cohort") run.add_argument("--case-ids", default="", help="Optional comma-separated case IDs") run.add_argument("--task-kind", choices=TASK_KINDS, help="Select one task kind; intersects with --case-ids") run.add_argument("--runs", type=int, default=1) run.add_argument("--timeout", type=float, default=DEFAULT_CALL_TIMEOUT_SECONDS) + run.add_argument("--collect-only", action="store_true", + help="Collect one call per case without scoring, retries or a scored report") run.add_argument("--delay", type=float, default=0.5) run.add_argument( "--provider-concurrency", diff --git a/harness/web_fetch/config.py b/harness/web_fetch/config.py index 923e5bee1..76a6db8bf 100644 --- a/harness/web_fetch/config.py +++ b/harness/web_fetch/config.py @@ -18,7 +18,7 @@ ROOT = Path(__file__).resolve().parents[2] DEFAULT_FAILURE_RETRIES = 1 -DEFAULT_CALL_TIMEOUT_SECONDS = 45 +DEFAULT_CALL_TIMEOUT_SECONDS = 30 def nonnegative_int(value: str) -> int: @@ -185,3 +185,21 @@ def load_provider_settings(path: Path) -> dict[str, Any]: def load_provider_config(path: Path) -> dict[str, dict[str, Any]]: return load_provider_settings(path)["providers"] + + +def resolve_provider_selection(selection: str | None, config_path: Path | None) -> str | None: + """Resolve an explicitly requested configured cohort without changing other defaults.""" + if selection != 'configured': + return selection + if config_path is None: + raise ValueError('--providers configured requires --provider-config') + settings = load_provider_settings(config_path) + names = settings.get('default_providers') + if (not isinstance(names, list) or not names + or any(not isinstance(name, str) or not name.strip() or ',' in name for name in names) + or len(names) != len(set(names))): + raise ValueError('Provider config must declare nonempty, unique default_providers') + unknown = set(names) - set(settings['providers']) + if unknown: + raise ValueError('Configured cohort contains unknown providers: ' + ', '.join(sorted(unknown))) + return ','.join(names) diff --git a/harness/web_fetch/data/report-methods.json b/harness/web_fetch/data/report-methods.json index 0371aca83..71b820099 100644 --- a/harness/web_fetch/data/report-methods.json +++ b/harness/web_fetch/data/report-methods.json @@ -404,6 +404,91 @@ "zh": "离线诊断", "en": "Offline diagnostics" } + }, + "brightdata_unlocker": { + "endpoint": "POST https://api.brightdata.com/request", + "interface_method": "brightdata", + "contract": { + "zh": "已知网址交给 Web Unlocker，返回原生 Markdown。", + "en": "Send a known URL to Web Unlocker and retain native Markdown." + }, + "output": "Markdown", + "freshness": { + "zh": "抓取、缓存及内部重试由服务负责；实际参数和耗时以调用记录为准；测试程序的失败重试次数以冻结的运行配置为准。", + "en": "Retrieval, caching and internal retries are service-owned; actual parameters and timing come from call evidence; caller failure retries follow the frozen run configuration." + }, + "leaderboard": { + "zh": "网页文字提取", + "en": "Webpage text retrieval" + } + }, + "browserbase_fetch": { + "endpoint": "POST https://api.browserbase.com/v1/fetch", + "interface_method": "browserbase", + "contract": { + "zh": "通过 Fetch 接口获取网页文字，不替换为浏览器会话。", + "en": "Retrieve webpage text through Fetch without substituting a browser session." + }, + "output": "Markdown", + "freshness": { + "zh": "抓取、缓存及内部重试由服务负责；实际参数和耗时以调用记录为准；测试程序的失败重试次数以冻结的运行配置为准。", + "en": "Retrieval, caching and internal retries are service-owned; actual parameters and timing come from call evidence; caller failure retries follow the frozen run configuration." + }, + "leaderboard": { + "zh": "网页文字提取", + "en": "Webpage text retrieval" + } + }, + "browser_use_fetch": { + "endpoint": "POST https://fetch.browser-use.com/fetch", + "interface_method": "browser-use-fetch", + "contract": { + "zh": "通过 fetch-use 获取网页文字，不启动 Agent。", + "en": "Retrieve webpage text through fetch-use without an Agent." + }, + "output": "Markdown", + "freshness": { + "zh": "抓取、缓存及内部重试由服务负责；实际参数和耗时以调用记录为准；测试程序的失败重试次数以冻结的运行配置为准。", + "en": "Retrieval, caching and internal retries are service-owned; actual parameters and timing come from call evidence; caller failure retries follow the frozen run configuration." + }, + "leaderboard": { + "zh": "网页文字提取", + "en": "Webpage text retrieval" + } + }, + "deepseek_fetch": { + "endpoint": "POST https://api.deepseek.com/v1/tools/fetch", + "interface_method": "deepseek-fetch-api", + "contract": { + "zh": "只发送网址；正文与标题分别保留。", + "en": "Send only the URL; retain content and title separately." + }, + "output": "Markdown", + "freshness": { + "zh": "抓取、缓存及内部重试由服务负责；实际参数和耗时以调用记录为准；测试程序的失败重试次数以冻结的运行配置为准。", + "en": "Retrieval, caching and internal retries are service-owned; actual parameters and timing come from call evidence; caller failure retries follow the frozen run configuration." + }, + "leaderboard": { + "zh": "网页文字提取", + "en": "Webpage text retrieval" + } + }, + "crawl4ai_cloud": { + "endpoint": "POST https://api.crawl4ai.com/scrape", + "interface_method": "crawl4ai-cloud", + "contract": { + "zh": "调用 Cloud 抓取接口，与本机开源库分别登记。", + "en": "Call the Cloud scrape API, registered separately from the local library." + }, + "output": "Markdown", + "freshness": { + "zh": "抓取、缓存及内部重试由服务负责；实际参数和耗时以调用记录为准；测试程序的失败重试次数以冻结的运行配置为准。", + "en": "Retrieval, caching and internal retries are service-owned; actual parameters and timing come from call evidence; caller failure retries follow the frozen run configuration." + }, + "leaderboard": { + "zh": "网页文字提取", + "en": "Webpage text retrieval" + } } }, "route_labels": { @@ -5180,22 +5265,14 @@ "zh": "Browserbase Fetch", "en": "Browserbase Fetch" }, - "status": "planned", + "status": "implemented", "source_url": "https://www.browserbase.com/sign-up", "access": { "type": "hosted", "env_vars": [ - "BROWSERBASE_API_KEY", - "BROWSERBASE_PROJECT_ID" + "BROWSERBASE_API_KEY" ], "links": [ - [ - { - "zh": "注册 / 登录", - "en": "Register / sign in" - }, - "https://www.browserbase.com/sign-up" - ], [ { "zh": "获取凭据", @@ -5205,17 +5282,67 @@ ] ], "account": { - "zh": "官方页面存在契约冲突；打分前必须先以账号内实际接口核验。", - "en": "Official pages describe conflicting contracts; verify the account-specific interface before scoring." + "zh": "需要服务账号及可用额度。", + "en": "Requires a service account and available quota." }, "configuration": { - "zh": "注册后创建或选择项目，从 Overview 控制台复制 API key 与 Project ID。", - "en": "After registering, create or select a project and copy its API key and Project ID from Overview." + "zh": "在本地 .env 中配置所列变量。", + "en": "Set the listed variables in the local .env file." }, "required": true, "priority": { - "zh": "B · 已知 URL 抓取候选", - "en": "B · Known-URL fetch candidate" + "zh": "B · 已知 URL 抓取", + "en": "B · Known-URL fetch" + } + }, + "kind": "adapter", + "adapter_types": [ + "browserbase_fetch" + ], + "profile": { + "kind": { + "zh": "托管 API", + "en": "Managed API" + }, + "description": { + "zh": "通过原生接口获取网页文字，保存原始响应和调用日志。", + "en": "Retrieve webpage text through the native API, retaining raw responses and call logs." + } + }, + "interface": { + "entry": "POST https://api.browserbase.com/v1/fetch", + "auth": "X-BB-API-Key: ", + "params": [ + [ + "url", + { + "zh": "string · 必填", + "en": "string · required" + }, + { + "zh": "本题提供的网址", + "en": "The task URL" + } + ] + ], + "fields": [ + [ + "content", + "string", + { + "zh": "原生返回的正文；不根据标准答案补全", + "en": "Native returned text; never completed from reference answers" + } + ] + ], + "request": { + "url": "https://example.com/", + "format": "markdown" + }, + "source": "harness/web_fetch/providers/hosted_fetch.py", + "notes": { + "zh": "这是接口示例，不代表历史调用参数。渲染、代理、引擎、输出格式和时间预算以保存的配置及请求日志为准。保存完整原始响应，不补抓或补写正文；调用端日志不等于服务内部日志。", + "en": "This interface example does not establish historical call parameters. Rendering, proxy, engine, output format and time budget come from saved configuration and request logs. Preserve the complete response without supplemental retrieval or content repair; caller logs are not service-internal traces." } } }, @@ -5418,7 +5545,7 @@ "zh": "Bright Data Web Unlocker", "en": "Bright Data Web Unlocker" }, - "status": "planned", + "status": "implemented", "source_url": "https://brightdata.com/cp/start", "access": { "type": "hosted", @@ -5427,13 +5554,6 @@ "BRIGHTDATA_UNLOCKER_ZONE" ], "links": [ - [ - { - "zh": "注册 / 登录", - "en": "Register / sign in" - }, - "https://brightdata.com/cp/start" - ], [ { "zh": "获取凭据", @@ -5443,17 +5563,69 @@ ] ], "account": { - "zh": "需要账号、计费或试用资格，以及 Web Unlocker zone。", - "en": "Requires an account, billing or trial eligibility, and a Web Unlocker zone." + "zh": "需要服务账号及可用额度。", + "en": "Requires a service account and available quota." }, "configuration": { - "zh": "注册账号，创建 Web Unlocker zone，再复制 API token 与 zone 名称。", - "en": "Register, create a Web Unlocker zone and copy the API token and zone name." + "zh": "在本地 .env 中配置所列变量。", + "en": "Set the listed variables in the local .env file." }, "required": true, "priority": { - "zh": "C · 解锁场景数据集", - "en": "C · Access-unblocking dataset" + "zh": "B · 已知 URL 抓取", + "en": "B · Known-URL fetch" + } + }, + "kind": "adapter", + "adapter_types": [ + "brightdata_unlocker" + ], + "profile": { + "kind": { + "zh": "托管 API", + "en": "Managed API" + }, + "description": { + "zh": "通过原生接口获取网页文字，保存原始响应和调用日志。", + "en": "Retrieve webpage text through the native API, retaining raw responses and call logs." + } + }, + "interface": { + "entry": "POST https://api.brightdata.com/request", + "auth": "Authorization: Bearer ", + "params": [ + [ + "url", + { + "zh": "string · 必填", + "en": "string · required" + }, + { + "zh": "本题提供的网址", + "en": "The task URL" + } + ] + ], + "fields": [ + [ + "HTTP response body", + "string", + { + "zh": "原生返回的正文；不根据标准答案补全", + "en": "Native returned text; never completed from reference answers" + } + ] + ], + "request": { + "url": "https://example.com/", + "zone": "", + "format": "raw", + "data_format": "markdown" + }, + "source": "harness/web_fetch/providers/hosted_fetch.py", + "notes": { + "zh": "这是接口示例，不代表历史调用参数。渲染、代理、引擎、输出格式和时间预算以保存的配置及请求日志为准。保存完整原始响应，不补抓或补写正文；调用端日志不等于服务内部日志。", + "en": "This interface example does not establish historical call parameters. Rendering, proxy, engine, output format and time budget come from saved configuration and request logs. Preserve the complete response without supplemental retrieval or content repair; caller logs are not service-internal traces." } } }, @@ -5665,6 +5837,263 @@ "en": "C · Fetch API comparison" } } + }, + "browser-use-fetch": { + "label": { + "zh": "Browser Use Fetch", + "en": "Browser Use Fetch" + }, + "source_url": "https://cloud.browser-use.com/settings?tab=api-keys", + "status": "implemented", + "kind": "adapter", + "adapter_types": [ + "browser_use_fetch" + ], + "access": { + "type": "hosted", + "env_vars": [ + "BROWSER_USE_API_KEY" + ], + "required": true, + "priority": { + "zh": "B · 已知 URL 抓取", + "en": "B · Known-URL fetch" + }, + "links": [ + [ + { + "zh": "获取凭据", + "en": "Get credentials" + }, + "https://cloud.browser-use.com/settings?tab=api-keys" + ] + ], + "configuration": { + "zh": "在本地 .env 中配置所列变量。", + "en": "Set the listed variables in the local .env file." + }, + "account": { + "zh": "需要服务账号及可用额度。", + "en": "Requires a service account and available quota." + } + }, + "profile": { + "kind": { + "zh": "托管 API", + "en": "Managed API" + }, + "description": { + "zh": "通过原生接口获取网页文字，保存原始响应和调用日志。", + "en": "Retrieve webpage text through the native API, retaining raw responses and call logs." + } + }, + "interface": { + "entry": "POST https://fetch.browser-use.com/fetch", + "auth": "X-Browser-Use-API-Key: ", + "params": [ + [ + "url", + { + "zh": "string · 必填", + "en": "string · required" + }, + { + "zh": "本题提供的网址", + "en": "The task URL" + } + ] + ], + "fields": [ + [ + "body", + "string", + { + "zh": "原生返回的正文；不根据标准答案补全", + "en": "Native returned text; never completed from reference answers" + } + ] + ], + "request": { + "url": "https://example.com/", + "output_format": "markdown" + }, + "source": "harness/web_fetch/providers/hosted_fetch.py", + "notes": { + "zh": "这是接口示例，不代表历史调用参数。渲染、代理、引擎、输出格式和时间预算以保存的配置及请求日志为准。保存完整原始响应，不补抓或补写正文；调用端日志不等于服务内部日志。", + "en": "This interface example does not establish historical call parameters. Rendering, proxy, engine, output format and time budget come from saved configuration and request logs. Preserve the complete response without supplemental retrieval or content repair; caller logs are not service-internal traces." + } + } + }, + "deepseek-fetch-api": { + "label": { + "zh": "DeepSeek Fetch API", + "en": "DeepSeek Fetch API" + }, + "source_url": "https://platform.deepseek.com/api_keys", + "status": "implemented", + "kind": "adapter", + "adapter_types": [ + "deepseek_fetch" + ], + "access": { + "type": "hosted", + "env_vars": [ + "DEEPSEEK_API_KEY" + ], + "required": true, + "priority": { + "zh": "B · 已知 URL 抓取", + "en": "B · Known-URL fetch" + }, + "links": [ + [ + { + "zh": "获取凭据", + "en": "Get credentials" + }, + "https://platform.deepseek.com/api_keys" + ] + ], + "configuration": { + "zh": "在本地 .env 中配置所列变量。", + "en": "Set the listed variables in the local .env file." + }, + "account": { + "zh": "需要服务账号及可用额度。", + "en": "Requires a service account and available quota." + } + }, + "profile": { + "kind": { + "zh": "托管 API", + "en": "Managed API" + }, + "description": { + "zh": "通过原生接口获取网页文字，保存原始响应和调用日志。", + "en": "Retrieve webpage text through the native API, retaining raw responses and call logs." + } + }, + "interface": { + "entry": "POST https://api.deepseek.com/v1/tools/fetch", + "auth": "Authorization: Bearer ", + "params": [ + [ + "url", + { + "zh": "string · 必填", + "en": "string · required" + }, + { + "zh": "本题提供的网址", + "en": "The task URL" + } + ] + ], + "fields": [ + [ + "content; title", + "string", + { + "zh": "原生返回的正文；不根据标准答案补全", + "en": "Native returned text; never completed from reference answers" + } + ] + ], + "request": { + "url": "https://example.com/" + }, + "source": "harness/web_fetch/providers/hosted_fetch.py", + "notes": { + "zh": "这是接口示例，不代表历史调用参数。渲染、代理、引擎、输出格式和时间预算以保存的配置及请求日志为准。保存完整原始响应，不补抓或补写正文；调用端日志不等于服务内部日志。", + "en": "This interface example does not establish historical call parameters. Rendering, proxy, engine, output format and time budget come from saved configuration and request logs. Preserve the complete response without supplemental retrieval or content repair; caller logs are not service-internal traces." + } + } + }, + "crawl4ai-cloud": { + "label": { + "zh": "Crawl4AI Cloud", + "en": "Crawl4AI Cloud" + }, + "source_url": "https://api.crawl4ai.com/docs/", + "status": "implemented", + "kind": "adapter", + "adapter_types": [ + "crawl4ai_cloud" + ], + "access": { + "type": "hosted", + "env_vars": [ + "CRAWL4AI_CLOUD_API_KEY" + ], + "required": true, + "priority": { + "zh": "B · 已知 URL 抓取", + "en": "B · Known-URL fetch" + }, + "links": [ + [ + { + "zh": "获取凭据", + "en": "Get credentials" + }, + "https://crawl4ai.com/dashboard/" + ] + ], + "configuration": { + "zh": "在本地 .env 中配置所列变量。", + "en": "Set the listed variables in the local .env file." + }, + "account": { + "zh": "需要服务账号及可用额度。", + "en": "Requires a service account and available quota." + } + }, + "profile": { + "kind": { + "zh": "托管 API", + "en": "Managed API" + }, + "description": { + "zh": "通过原生接口获取网页文字，保存原始响应和调用日志。", + "en": "Retrieve webpage text through the native API, retaining raw responses and call logs." + } + }, + "interface": { + "entry": "POST https://api.crawl4ai.com/scrape", + "auth": "Authorization: Bearer ", + "params": [ + [ + "url", + { + "zh": "string · 必填", + "en": "string · required" + }, + { + "zh": "本题提供的网址", + "en": "The task URL" + } + ] + ], + "fields": [ + [ + "markdown", + "string", + { + "zh": "原生返回的正文；不根据标准答案补全", + "en": "Native returned text; never completed from reference answers" + } + ] + ], + "request": { + "url": "https://example.com/", + "format": "md" + }, + "source": "harness/web_fetch/providers/hosted_fetch.py", + "notes": { + "zh": "这是接口示例，不代表历史调用参数。渲染、代理、引擎、输出格式和时间预算以保存的配置及请求日志为准。保存完整原始响应，不补抓或补写正文；调用端日志不等于服务内部日志。", + "en": "This interface example does not establish historical call parameters. Rendering, proxy, engine, output format and time budget come from saved configuration and request logs. Preserve the complete response without supplemental retrieval or content repair; caller logs are not service-internal traces." + } + } } }, "sources": { diff --git a/harness/web_fetch/diagnostics.py b/harness/web_fetch/diagnostics.py index 09d5568fd..d195be3ca 100644 --- a/harness/web_fetch/diagnostics.py +++ b/harness/web_fetch/diagnostics.py @@ -9,7 +9,7 @@ from pathlib import Path _LOCAL = threading.local() -_SECRET = {"authorization", "proxy-authorization", "cookie", "set-cookie", "api_key", "apikey", "api-key", "x-api-key", "token", "password", "secret"} +_SECRET = {"authorization", "proxy-authorization", "cookie", "set-cookie", "api_key", "apikey", "api-key", "x-api-key", "x-bb-api-key", "x-browser-use-api-key", "token", "password", "secret"} def redact(value): if isinstance(value, dict): diff --git a/harness/web_fetch/native_projection.py b/harness/web_fetch/native_projection.py index e24745f04..d232899e2 100644 --- a/harness/web_fetch/native_projection.py +++ b/harness/web_fetch/native_projection.py @@ -47,6 +47,24 @@ def native_answer_projection(provider_type, case, payload, status_code=None, *, track, task_kind = get('track', 'web'), get('task_kind', 'body') payload = _object(payload) successful_http = type(status_code) is int and 200 <= status_code < 300 + if provider_type == 'brightdata_unlocker': + return {'schema_version': 2, 'source_format': 'bytes', + 'content_format': 'utf8' if successful_http else 'empty'} + if provider_type in {'browser_use_fetch', 'browserbase_fetch', 'deepseek_fetch', 'crawl4ai_cloud'}: + field = ('body' if provider_type == 'browser_use_fetch' else + 'markdown' if provider_type == 'crawl4ai_cloud' else 'content') + value = payload.get(field) + fmt = ('text' if isinstance(value, str) else + 'json' if provider_type == 'browserbase_fetch' and interface == 'json' + and isinstance(value, (dict, list)) else 'empty') + declaration = {'schema_version': 2, 'content_pointer': [field], + 'content_format': fmt if successful_http else 'empty', + 'optional_content': True, 'result_fields': {}} + if provider_type == 'deepseek_fetch': + declaration['result_fields']['title'] = {'pointer': ['title'], 'format': 'text_or_none'} + if provider_type == 'browser_use_fetch': + declaration['result_fields']['final_url'] = {'pointer': ['final_url'], 'format': 'text_or_none'} + return declaration if provider_type in {'tavily_extract', 'exa_contents'}: results = payload.get('results') result = _object(results[0]) if isinstance(results, list) and results else {} @@ -545,7 +563,9 @@ def check_source(spec): if 'legacy' in spec: check_source(spec['legacy']) if declaration.get('source_format') == 'bytes': - required = 'original_bytes' if declaration.get('files') else 'stdout' + required = ('original_bytes' if declaration.get('files') else + 'response_body' if 'response_body' in allowed + else 'stdout') if required not in allowed: raise ValueError('Native byte projection differs from its frozen answer fields') else: diff --git a/harness/web_fetch/native_routes.py b/harness/web_fetch/native_routes.py index abccf1281..ccdde06ee 100644 --- a/harness/web_fetch/native_routes.py +++ b/harness/web_fetch/native_routes.py @@ -157,6 +157,14 @@ def normalized_http_parameters(case, arguments): body['urls'] = [''] else: raise ValueError('Native HTTP input differs from the frozen task URL') + # Native fetch-use requires a fresh opaque UUID per physical invocation. + # Normalize only this session token, never task or extraction parameters. + if 'session_id' in body and 'X-Browser-Use-API-Key' in arguments.get('headers', {}): + import uuid + token = body['session_id'] + if not isinstance(token, str) or str(uuid.UUID(token)) != token: + raise ValueError('Native fetch session must be a canonical UUID') + body['session_id'] = '' return {'method': arguments.get('method', 'GET'), 'body': body, 'headers_sha256': canonical_json_sha256(arguments.get('headers', {})), 'timeout_seconds': arguments.get('timeout'), 'max_bytes': arguments.get('max_bytes')} @@ -491,13 +499,19 @@ def _expected_native_declaration(root, record, raw, cases_by_id): if case is None or case.get('url') != record.get('url'): raise ValueError('Native answer selection lacks its frozen request identity') try: - payload = json.loads(raw) if record['provider_type'] != 'moli_webfetch' else {} + payload = json.loads(raw) if record['provider_type'] not in {'moli_webfetch', 'brightdata_unlocker'} else {} except (ValueError, UnicodeDecodeError): + payload = None + if record['provider_type'] in {'browser_use_fetch', 'browserbase_fetch', 'deepseek_fetch', 'crawl4ai_cloud'}: + if not isinstance(payload, dict): + return None # Malformed API envelopes are evidence, never answers. + if payload is None: payload = {} requests = record['provider_metadata']['native_route_receipt'].get('requests', []) interface = requests[0].get('signature', {}).get('interface') if requests else None return native_answer_projection(record['provider_type'], case, payload, - record.get('status_code'), interface=interface) + record['provider_metadata'].get('api_http_status', record.get('status_code')), + interface=record['provider_metadata'].get('output_contract', interface)) def verify_saved_native_answer(root, record, *, cases_by_id=None): @@ -531,7 +545,7 @@ def verify_saved_native_answer(root, record, *, cases_by_id=None): # A timed-out empty answer is deliberately discarded. Network or # pre-transport failures may have no raw response to select from. if record.get('error_type') != 'timeout': - bytes_source = record['provider_type'] == 'moli_webfetch' + bytes_source = record.get('provider_type') == 'moli_webfetch' raw_path = metadata.get('raw_stdout_path' if bytes_source else 'raw_response_path') raw_hash = metadata.get('raw_stdout_sha256' if bytes_source else 'raw_response_sha256') if raw_path or raw_hash: @@ -547,8 +561,10 @@ def verify_saved_native_answer(root, record, *, cases_by_id=None): if any(metadata.get(key) for key in SCORE_ANSWER_FIELDS) or metadata.get('file_evaluation', {}).get('files'): raise ValueError('Discarded late native answer retains score-bearing fields') return - raw_path = metadata.get('raw_stdout_path') if declaration.get('source_format') == 'bytes' else metadata.get('raw_response_path') - raw_hash = metadata.get('raw_stdout_sha256') if declaration.get('source_format') == 'bytes' else metadata.get('raw_response_sha256') + stdout_source = (declaration.get('source_format') == 'bytes' + and record.get('provider_type') != 'brightdata_unlocker') + raw_path = metadata.get('raw_stdout_path' if stdout_source else 'raw_response_path') + raw_hash = metadata.get('raw_stdout_sha256' if stdout_source else 'raw_response_sha256') if not raw_path or not raw_hash: raise ValueError('Declared native answer lacks its retained raw response') raw = read_evidence_binding(root, {'path': raw_path, 'sha256': raw_hash}).read_bytes() diff --git a/harness/web_fetch/providers/base.py b/harness/web_fetch/providers/base.py index e5090a931..c4a6bb8d9 100644 --- a/harness/web_fetch/providers/base.py +++ b/harness/web_fetch/providers/base.py @@ -78,6 +78,8 @@ def terminate_owned_process_tree(process): class Provider(ABC): + allow_failure_retries = True + # Public configuration diagnostics only. Credentials, commands and local # paths remain covered by the full configuration hash, never displayed. # Adapters may extend this declaration for their own safe options. @@ -116,7 +118,7 @@ def check_native_call(self, case: EvalCase) -> None: return if case.retrieval_hints and not (case.track == "dynamic" and case.retrieval_hints == NATIVE_RETRIEVAL_HINTS): raise ValueError("fixed native calls prohibit per-case retrieval hints") - if case.track not in {"web", "dynamic", "challenge", "risk_control", "document", "image", "video", "file"}: + if case.track not in {"web", "dynamic", "challenge", "risk_control", "articles", "document", "image", "video", "file"}: raise ValueError("fixed native calls require a declared task route") if case.track == "document" and case.task_kind == "resource": raise ValueError("fixed native calls require a declared task route") @@ -197,6 +199,9 @@ def begin_call(self) -> None: def build_provider(name: str, config: dict[str, Any]) -> Provider: provider_type = config.get("type") + if provider_type in {"brightdata_unlocker", "browser_use_fetch", "browserbase_fetch", "deepseek_fetch", "crawl4ai_cloud"}: + from .hosted_fetch import HostedFetchProvider + return HostedFetchProvider(name, config) if provider_type in {"headless_html", "headless_markdown"}: from .headless import HeadlessProvider return HeadlessProvider(name, config) diff --git a/harness/web_fetch/providers/hosted_fetch.py b/harness/web_fetch/providers/hosted_fetch.py new file mode 100644 index 000000000..a5a62f2bf --- /dev/null +++ b/harness/web_fetch/providers/hosted_fetch.py @@ -0,0 +1,221 @@ +"""URL-only hosted Markdown retrieval; no Agent or browser-session fallback.""" +from __future__ import annotations + +import base64 +import hashlib +import json +import uuid + +from lexbench_webfetch.models import EvalCase +from ..models import ProviderResult +from ..native_routes import NativeRoute +from ..native_projection import native_answer_projection +from ..network_dns import public_dns_context, validate_dns_mode +from .base import Provider +from .http_client import inner_timeout_budget, request + + +ENDPOINTS = { + "crawl4ai_cloud": "https://api.crawl4ai.com/scrape", + "brightdata_unlocker": "https://api.brightdata.com/request", + "browser_use_fetch": "https://fetch.browser-use.com/fetch", + "browserbase_fetch": "https://api.browserbase.com/v1/fetch", + "deepseek_fetch": "https://api.deepseek.com/v1/tools/fetch", +} + + +def _http_status(value: object) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) and 100 <= value <= 599 else None + + +class HostedFetchProvider(Provider): + # The shared runner owns declared failure retries; the adapter calls once. + native_configuration_fields = (*Provider.native_configuration_fields, + 'render', 'debug', 'proxies', 'proxy_country', 'engine', 'proxy', 'output_format') + + def __init__(self, name, config): + super().__init__(name, config) + self._session_id = str(uuid.uuid4()) + + def begin_call(self): + self._session_id = str(uuid.uuid4()) + + def native_route(self, case): + fields = {'brightdata_unlocker': ('response_body',), 'browser_use_fetch': ('body', 'final_url'), + 'browserbase_fetch': ('content',), 'deepseek_fetch': ('content', 'title'), + 'crawl4ai_cloud': ('markdown',)} + return NativeRoute('http', ENDPOINTS[self.type], fields[self.type]) + + def validate(self) -> list[str]: + errors = [] if self.config.get("api_key") else ["api_key"] + if self.type == "brightdata_unlocker" and not self.config.get("zone"): + errors.append("zone") + if self.type == "brightdata_unlocker" and "render" in self.config and not isinstance(self.config["render"], bool): + errors.append("render must be boolean") + if self.type == "brightdata_unlocker" and "debug" in self.config and not isinstance(self.config["debug"], bool): + errors.append("debug must be boolean") + try: + validate_dns_mode(self.config.get("dns_mode", "system")) + except ValueError as error: + errors.append(str(error)) + if self.type == "browserbase_fetch" and not isinstance(self.config.get("proxies", False), bool): + errors.append("proxies must be boolean") + if self.type == "crawl4ai_cloud": + if self.config.get("engine", "browser") not in {"auto", "browser-light", "browser"}: + errors.append("unsupported engine") + if self.config.get("proxy", "none") not in {"none", "isp", "residential"}: + errors.append("unsupported proxy") + output_format = self.config.get("output_format", "markdown") + formats = {"crawl4ai_cloud": {"markdown"}, "browserbase_fetch": {"markdown", "json"}, + "browser_use_fetch": {"markdown", "structured"}, + "brightdata_unlocker": {"markdown"}, "deepseek_fetch": {"markdown"}} + if not isinstance(output_format, str) or output_format not in formats[self.type]: + errors.append("unsupported output_format") + if self.type == "browserbase_fetch" and output_format == "json" and not isinstance(self.config.get("schema"), dict): + errors.append("JSON output requires schema") + return errors + + def native_http_arguments(self, case, timeout): + budget = inner_timeout_budget(timeout, native_default=True) + output_format = self.config.get("output_format", "markdown") + key = str(self.config.get("api_key", "")) + body: dict = {"url": case.url} + if self.type == "brightdata_unlocker": + headers = {"Authorization": f"Bearer {key}"} + body.update(zone=self.config["zone"], format="raw", data_format="markdown") + # Native Web Unlocker option, applied uniformly from the declared + # config. It adds no caller-side browser, wait, or follow-up fetch. + if "render" in self.config: + body["render"] = "true" if self.config["render"] else "false" + if "debug" in self.config: + body["debug"] = self.config["debug"] + elif self.type == "browserbase_fetch": + headers = {"X-BB-API-Key": key} + body.update(format=output_format, allowRedirects=True, + proxies=self.config.get("proxies", True)) + if output_format == "json": + body["schema"] = self.config["schema"] + elif self.type == "crawl4ai_cloud": + headers = {"Authorization": f"Bearer {key}"} + body.update(format="md", engine=self.config.get("engine", "browser"), + proxy=self.config.get("proxy", "none")) + elif self.type == "deepseek_fetch": + headers = {"Authorization": f"Bearer {key}"} + else: + headers = {"X-Browser-Use-API-Key": key} + body.update(method="GET", output_format=output_format, timeout_ms=min(120000, int(budget * 1000)), + follow_redirects=True, max_redirects=10, + proxy_country=self.config.get("proxy_country", "US"), + session_id=self._session_id) + return self.config.get("endpoint", ENDPOINTS[self.type]), self.native_http_options(body, headers, budget) + + def fetch(self, case: EvalCase, timeout: float, *, native_route_ledger=None) -> ProviderResult: + self.check_native_call(case) + if case.task_kind not in {"body", "structure"} or case.track not in {"web", "dynamic", "challenge", "risk_control", "articles"}: + return ProviderResult(False, "", error_type="unsupported_task", + error_message="This adapter supports webpage text tasks only") + # Direct diagnostic invocations also get a fresh session. + if self.type == "browser_use_fetch": + self.begin_call() + budget = inner_timeout_budget(timeout, native_default=True) + endpoint, arguments = self.native_http_arguments(case, budget) + body = arguments['json_body'] + output_format = self.config.get("output_format", "markdown") + dns_mode = self.config.get("dns_mode", "system") + with public_dns_context(dns_mode, budget) as dns_trace: + response = self.native_http_request(case, request, endpoint, + native_route_ledger=native_route_ledger, **arguments) + metadata = { + "_raw_response_b64": base64.b64encode(response.body).decode("ascii"), + "raw_response_sha256": hashlib.sha256(response.body).hexdigest(), + "raw_response_bytes": len(response.body), + "api_http_status": response.status, "output_contract": output_format, + "dns_mode": dns_mode, "dns_trace": dns_trace, + "client_retries": 0, "retrieval_hints_applied": False, + "service_retry_policy": "provider_default_not_overridden", + "request_timeout_seconds": budget, + } + # API envelopes are evidence, never page content. Bright Data's raw + # mode deliberately returns the selected Markdown directly. + api_ok = 200 <= response.status < 300 + content, status, final_url, title = "", None, None, None + malformed = False + service_error = False + if self.type == "brightdata_unlocker": + content = response.body.decode("utf-8") if api_ok else "" + metadata["native_answer_projection"] = native_answer_projection( + self.type, case, {}, response.status) + metadata["upstream_http_status_available"] = False + else: + try: + payload = response.json() + except ValueError: + payload = None + if not isinstance(payload, dict): + malformed = True + else: + if self.type == "browserbase_fetch": + content = payload.get("content") + if output_format == "json" and isinstance(content, (dict, list)): + content = json.dumps(content, ensure_ascii=False, separators=(",", ":")) + status = _http_status(payload.get("statusCode", payload.get("status_code"))) + service_error = payload.get("encoding") == "base64" or bool(payload.get("error")) + elif self.type == "crawl4ai_cloud": + content = payload.get("markdown") + status = _http_status(payload.get("origin_status")) + service_error = (bool(payload.get("error")) or payload.get("success") is False + or payload.get("ok") is False) + metadata["upstream_http_status_available"] = status is not None + elif self.type == "deepseek_fetch": + content = payload.get("content") + title = payload.get("title") + service_error = bool(payload.get("error")) + metadata["upstream_http_status_available"] = False + else: + content = payload.get("body") + status = _http_status(payload.get("status_code")) + final_url = payload.get("final_url") if isinstance(payload.get("final_url"), str) else None + service_error = bool(payload.get("error")) or bool(payload.get("is_binary")) + metadata["request_id"] = payload.get("id") if isinstance(payload.get("id"), str) else None + malformed = (not isinstance(content, str) or + (not isinstance(title, str) if self.type == "deepseek_fetch" else + False if self.type == "crawl4ai_cloud" else status is None)) + content = content if isinstance(content, str) else "" + if self.type == "deepseek_fetch" and isinstance(payload.get("error"), dict): + metadata["service_error"] = payload["error"] + if not api_ok: + # Keep error envelopes in raw evidence, not in scored page text. + content = "" + error_type = ("access_denied" if response.status == 403 and self.type == "crawl4ai_cloud" else + "security_policy" if response.status == 403 and self.type == "deepseek_fetch" else + "authentication" if response.status in {401, 403} else + "rate_limit" if response.status == 429 else + "quota" if response.status == 402 else "api_error") + message = f"Fetch API returned HTTP {response.status}; inspect retained response" + elif malformed: + error_type, message = "invalid_response", "Fetch API returned an invalid text response envelope" + elif service_error: + error_type, message = "api_error", "Fetch service reported an error or binary output; inspect retained response" + elif status is not None and not 200 <= status < 300: + error_type, message = "http_error", f"Target page returned HTTP {status}" + elif not content.strip(): + error_type, message = "empty_content", "Fetch API returned empty Markdown" + else: + error_type = message = None + if self.type != "brightdata_unlocker" and isinstance(payload, dict): + metadata['native_answer_projection'] = native_answer_projection( + self.type, case, payload, response.status, interface=output_format) + if self.type == "crawl4ai_cloud": + metadata["request_parameters"] = body + metadata["native_browser_logs_available"] = False + if self.type == "browserbase_fetch": + metadata["proxies"] = self.config.get("proxies", True) + if self.type == "brightdata_unlocker": + metadata["render"] = self.config.get("render") + if self.type == "browser_use_fetch": + metadata["proxy_country"] = body["proxy_country"] + return ProviderResult( + ok=error_type is None, content=content, status_code=status, + final_url=final_url, title=title if isinstance(title, str) else None, + error_type=error_type, error_message=message, metadata=metadata, + ) diff --git a/harness/web_fetch/report_build.py b/harness/web_fetch/report_build.py index c0765fbcb..653baaaba 100644 --- a/harness/web_fetch/report_build.py +++ b/harness/web_fetch/report_build.py @@ -369,6 +369,9 @@ def build_report(source: Path, output: Path, *, include_english: bool = False, from harness.report_languages import includes_english, select_report_language include_english = includes_english(language, include_english) source, output = source.resolve(), output.resolve() + manifest_path = contained_path(source, 'manifest.json') + if manifest_path.is_file() and json.loads(manifest_path.read_text()).get('collection_only'): + raise ValueError('Collection-only evidence has no scores; explicitly rescore before building a scored report') if source == output or output in source.parents: raise ValueError('Report output must not contain its evidence source') marker = '.generated-report' diff --git a/harness/web_fetch/runner.py b/harness/web_fetch/runner.py index 5bfd300f5..700642821 100644 --- a/harness/web_fetch/runner.py +++ b/harness/web_fetch/runner.py @@ -25,6 +25,7 @@ PRIVATE_ARTIFACT_ROOT, PUBLIC_CALL_LOG_ROOT, diagnostic_log_binding, + materialize_task_references, raw_response_binding, verify_record_artifacts, verify_record_private_artifacts, @@ -36,7 +37,7 @@ from harness.web_fetch.models import AttemptRecord, ProviderResult from lexbench_webfetch.models import EvalCase, validate_case from .providers import Provider -from .reference_inputs import read_dataset_bytes, benchmark_input_path +from .reference_inputs import read_dataset_bytes, benchmark_input_path, input_tree from lexbench_webfetch.scoring import SCORING_SOURCE_SHA256 from .reference_lifecycle import ( ReferenceLifecycleError, @@ -339,7 +340,8 @@ def persist_public_call_log( "events": [ {"stage": "call_started", "at": started_at, "status_code": None, "error_type": None}, - {"stage": "exception_raised" if exception is not None else "response_received", + {"stage": ("exception_raised" if exception is not None else + "answer_deadline_exceeded" if result.error_type == "timeout" else "response_received"), "at": completed_at, "status_code": result.status_code, "error_type": _public_text(result.error_type, limit=256)}, {"stage": "call_completed", "at": completed_at, @@ -394,7 +396,7 @@ def persist_result_artifacts( require_native_projection_schema(result.metadata['native_answer_projection']) raw = result.metadata.get('_raw_response_b64') if result.metadata.get('native_answer_projection', {}).get('source_format') == 'bytes': - raw = result.metadata.get('_raw_stdout_b64') + raw = result.metadata.get('_raw_stdout_b64', raw) needs_projection = (result.metadata.get('native_answer_required') or result.metadata.get('native_answer_projection') or any(result.metadata.get(key) for key in ('links', 'images', 'videos', 'media', 'extras')) @@ -497,6 +499,7 @@ def run_evaluation( reference_policy: str = "fresh", case_admission: Any | None = None, call_order: str = "provider", + collect_only: bool = False, ) -> tuple[list[dict[str, Any]], dict[str, Any]]: output_dir = output_dir.resolve() for name, value, minimum in ( @@ -509,6 +512,8 @@ def run_evaluation( if (not isinstance(value, (int, float)) or isinstance(value, bool) or not math.isfinite(value) or value < minimum or (name == 'timeout' and value == 0)): raise ValueError(f'{name} must be finite and {"positive" if name == "timeout" else "nonnegative"}') + if collect_only and (runs != 1 or max_failure_retries != 0): + raise ValueError('Collection-only diagnostics require one run and zero caller retries') if call_order not in {"provider", "case-alternating"}: raise ValueError("Unsupported call order") if call_order == "case-alternating" and provider_concurrency != 1: @@ -520,6 +525,8 @@ def run_evaluation( if not cases or not providers: raise ValueError('Execution requires at least one task and provider') for provider in providers: + if max_failure_retries and not provider.allow_failure_retries: + raise ValueError(f"provider {provider.name!r} prohibits caller failure retries") if provider.config.get("uniform_native_parameters"): if runs != 1: raise ValueError("fixed native calls require one run") @@ -542,7 +549,9 @@ def run_evaluation( contained_path(output_dir, filename) if reference_policy not in {"fresh", "frozen_existing"}: raise ValueError("unknown reference policy") - if reference_policy == "frozen_existing": + if collect_only: + reference_checks = None + elif reference_policy == "frozen_existing": # Explicit local measurement of existing rules; no live-source claim. from .reference_lifecycle import validate_resource_source_evidence for case in cases: @@ -560,6 +569,9 @@ def run_evaluation( from .network_preflight import preflight_guarded_providers network_audit = preflight_guarded_providers(providers, cases) output_dir.mkdir(parents=True, exist_ok=True) + materialize_task_references( + output_dir, {name: input_tree(dataset_path, name) for name in ("sources", "list-references")}, + [case.id for case in cases]) if network_audit is not None: (output_dir / 'network-preflight.json').write_text( json.dumps(network_audit, ensure_ascii=False, indent=2) + '\n') @@ -570,6 +582,7 @@ def run_evaluation( manifest = { "schema_version": 1, + "collection_only": collect_only, "runtime_source_sha256": hashlib.sha256(SCORING_SOURCE_SHA256.encode()+b'\0'+b'\0'.join( p.relative_to(Path(__file__).parent).as_posix().encode()+b'\0'+p.read_bytes() for p in sorted(Path(__file__).parent.rglob('*.py')))).hexdigest(), @@ -672,12 +685,14 @@ def run_provider(provider: Provider, selected_cases=None, selected_runs=None) -> binding = reference_checks["cases"].get(case.id) if reference_checks else None # Outside provider exception handling: expired references # stop measurement, never create a supplier failure record. - if reference_policy == "frozen_existing": + if collect_only: + pass + elif reference_policy == "frozen_existing": if hashlib.sha256(read_dataset_bytes(dataset_path)).hexdigest() != frozen_dataset_sha256: raise ReferenceLifecycleError("Frozen rules changed during the run", "reference_changed") else: check_case_reference_fresh(case, dataset_path, binding) - # Admission waits before the answer timer and 45-second budget. + # Admission waits before the per-call answer timer. admission_token = (case_admission.acquire(provider, case) if case_admission is not None else None) pending = None @@ -685,6 +700,7 @@ def run_provider(provider: Provider, selected_cases=None, selected_runs=None) -> try: started = utc_now() physical_attempts = [] + sequence_begin = time.monotonic() stopped_provider = False for physical_attempt in range(1, max_failure_retries + 2): retry_throttle_wait_ms = 0.0 @@ -752,18 +768,19 @@ def run_provider(provider: Provider, selected_cases=None, selected_runs=None) -> result.metadata['native_answer_required'] = True artifact = persist_result_artifacts(output_dir, artifact_rel, result, request_url=case.url) artifact_bytes = result.content.encode('utf-8', errors='replace') - measured = score_attempt( + measured = ({'content_sha256': hashlib.sha256(artifact_bytes).hexdigest(), + 'scoring_available': False} if collect_only else score_attempt( case, artifact_bytes, ok=result.ok, metadata=result.metadata, title=result.title, final_url=result.final_url, list_reference=load_list_reference(case, dataset_path.parent), status_code=result.status_code, prepared_answer=result.prepared_answer, - ) + )) measured['deadline_overrun'] = physical_elapsed_ms > timeout * 1000 measured['deadline_budget_ms'] = timeout * 1000 if binding is not None: measured['reference_sha256'] = binding['reference_sha256'] - physical_passed = task_passed(result.ok, measured, case.task_kind) + physical_passed = None if collect_only else task_passed(result.ok, measured, case.task_kind) physical_completed = utc_now() call_log = persist_public_call_log( output_dir, provider=provider, case=case, attempt_id=attempt_id, @@ -822,6 +839,7 @@ def run_provider(provider: Provider, selected_cases=None, selected_runs=None) -> and pending.is_alive()) if physical_passed or stopped_provider or physical_attempt > max_failure_retries: break + result.metadata['call_sequence_elapsed_ms'] = round((time.monotonic() - sequence_begin) * 1000, 2) elapsed_ms = physical_attempts[-1]['elapsed_ms'] result.metadata['call_log'] = physical_attempts[-1]['call_log'] if len(physical_attempts) > 1: @@ -861,12 +879,18 @@ def run_provider(provider: Provider, selected_cases=None, selected_runs=None) -> "client_throttle_wait_ms": throttle_wait_ms, }, ).to_dict() - apply_request_status(record) + if not collect_only: + apply_request_status(record) + else: + record['scoring'] = {'mode': 'not_scored', 'reason': 'collection_only_diagnostic'} record["completed_at"] = finished from .models import execution_id record['run_id'] = execution_id(record) - passed = task_passed(record['ok'],record['metrics'],case.task_kind) - state = "PASS" if passed else "FAIL" + if collect_only: + state = "RECV" if record["ok"] else "ERR" + else: + passed = task_passed(record['ok'],record['metrics'],case.task_kind) + state = "PASS" if passed else "FAIL" with result_lock: records.append(record) results_file.write(json.dumps(record, ensure_ascii=True) + "\n") @@ -934,7 +958,9 @@ def run_provider(provider: Provider, selected_cases=None, selected_runs=None) -> encoding="utf-8", ) - summary = summarize(records) + summary = ({'mode': 'collection_only', 'scoring_state': 'not_scored', + 'attempt_count': len(records), 'responses_ok': sum(bool(r['ok']) for r in records), + 'task_pass_rate': None} if collect_only else summarize(records)) manifest["completed_at"] = utc_now() manifest["attempt_count"] = len(records) manifest["attempt_ids"] = [r['attempt_id'] for r in records] diff --git a/tests/fixtures/native-route-baseline.json b/tests/fixtures/native-route-baseline.json index a85b41f96..081ae9367 100644 --- a/tests/fixtures/native-route-baseline.json +++ b/tests/fixtures/native-route-baseline.json @@ -1,74 +1,74 @@ { "cloudflare_content:configured": { "cells": 192, - "sha256": "c501d817b59dbefc362964386736f67ae4c46a7f2981aebf84dae53f10ae677b" + "sha256": "f7c345477ff7ced84b450586cb2b83720fac64edf22dd5bc7725c012cda89beb" }, "cloudflare_content:defaults": { "cells": 192, - "sha256": "287739522a6a6c57e529380e909472a5d515a846a2979bd659a67222b0f9dfd6" + "sha256": "ac5cabdd2f59cb366241c54fa6ac0d71d87b8e268f8b94c76e602264004f981c" }, "cloudflare_markdown:configured": { "cells": 192, - "sha256": "4905efb447fcdb4aec57bc73dd26880bf1c0e9084a71fca5374e442416db60ae" + "sha256": "c56aef9437389a1325df9b915ff84dff97d6699e00a54ad02c554cc25e0647d0" }, "cloudflare_markdown:defaults": { "cells": 192, - "sha256": "2e9171663d9791377f107794667707836d65572ed9e06077d275b0e0a00c791f" + "sha256": "c86a678efcd9710c92eb5a731d485865ef1f87d4971e1873452536093871a6bb" }, "cowagent_builtin:configured": { "cells": 192, - "sha256": "709a83a64fc0b36e9ae446f3c430c42223627d7332c6853e4d30091a9ed0c992" + "sha256": "cdb320a627bdca00381b7ecf42b504606c197128584dfe666dff20273da3d39c" }, "cowagent_builtin:defaults": { "cells": 192, - "sha256": "f952cc5a5f0e59975b18403da67bbbc138d886febdabd817ccd2682a4fdef39b" + "sha256": "f0ccf993d06ba6d37a4d3b194a49257a4423da733f6e5a480a8e9744e8e8be60" }, "exa_contents:configured": { "cells": 192, - "sha256": "671cdcdd4902ec034caee226164861e3503bac84c8ce9832ec8413896f0b16f8" + "sha256": "9f45c203c8c619df4f49d55cfa2979a439b7e166a474ed03020856cbcb817719" }, "exa_contents:defaults": { "cells": 192, - "sha256": "d65bc8289d165286e6341568e7009233549cfb641bf6b6ad293a410675fb8c20" + "sha256": "f54d453a98061790439f2286147832a4703f6d63f9e2d9e0564478d3a57b7efc" }, "firecrawl:configured": { "cells": 192, - "sha256": "7b3a41efd52dbe26075e1b7fb4525adba0ae2edf1bebb5c84ba6391e0392e079" + "sha256": "0b75cdbdec79a64649f320ab9c10344033930a405fece521e232e5f541d43198" }, "firecrawl:defaults": { "cells": 192, - "sha256": "b8d94e892531dc7f5d3ea8a805a96bf556de609e02fb4d50abe8877723186330" + "sha256": "9e42b2a3667bb2bc31709c80bf23102d81aff562d66747778dda6b56050cc960" }, "fixture:configured": { "cells": 192, - "sha256": "c95ef4b78f9df35c12713d7dac41f1c3abc675925653f43a4a22ee09152489e2" + "sha256": "d72000f903b1751542e71463ab158c2789db593bc58f67ed6b84b50fa61c649f" }, "fixture:defaults": { "cells": 192, - "sha256": "0ec20a64f8c447a737c303c02eafdb218a94e3f4930680cc34ab9fa33d8e744b" + "sha256": "47324b12cd3951ae1ddd5711704c375632cc5be68272d2c4bfd128ac4f0bca82" }, "moli_webfetch:configured": { "cells": 192, - "sha256": "940e90b1de64c65270d1db7726eec53680c68ca03dc5e43193b5eff4628c6338" + "sha256": "ae9c2025dadbc73057b0be857032e5b27f9f9a487eee524add19e40eae543676" }, "moli_webfetch:defaults": { "cells": 192, - "sha256": "551cfed573d065b40f58f53e20e78e5c6a1d7b77fd3808c67e6400c785fe892a" + "sha256": "8e9297fa0e7f40fc73c559fb801d34ce3d6ece280359ac8da9c008698d187091" }, "subprocess_jsonl:configured": { "cells": 192, - "sha256": "7d03a82682e0ac27139633c071fe119841baebf889f224fc5eea7c745088fcd1" + "sha256": "088970f3a05691f115e10ed915187f808c80a058e648da5f16dc76edb19c5d37" }, "subprocess_jsonl:defaults": { "cells": 192, - "sha256": "c18e2cacbb00b1759156548a62b499fcd9c0dbcffd531fb3f454ef0a1f82900c" + "sha256": "0491ed2a04e5cb1fd263a86dade57de3e8988ef62d4ff99198bdafecde2d64c3" }, "tavily_extract:configured": { "cells": 192, - "sha256": "aecbb4eacbdefc6a9726e97d30b263d4f94c39012d9829afbab8d936e0c22287" + "sha256": "69de47e62c3abb20c3349e077a37774851023c5c9614bbb5996473cde0822548" }, "tavily_extract:defaults": { "cells": 192, - "sha256": "84a39ad2141dca513b6d4c0e89293d8993facd3f47b449c8094321d171262930" + "sha256": "8b172f5af0cfcfd99e82972ffa8a0f8c1cf59a45f142cc56860267b7ccf3cf61" } } diff --git a/tests/test_accounts.py b/tests/test_accounts.py index 593d2bf87..01b1934bb 100644 --- a/tests/test_accounts.py +++ b/tests/test_accounts.py @@ -74,8 +74,9 @@ def test_cloudflare_retains_both_product_permissions(): def test_unimplemented_candidates_remain_marked_planned(): entries = account_checklist() planned = {item['id'] for item in entries if not item['implemented']} - assert 'browserbase' in planned - assert len(planned) == 12 + assert 'scrapingbee' in planned + assert not {'browserbase', 'brightdata', 'browser-use-fetch', 'deepseek-fetch-api'} & planned + assert len(planned) == 10 catalog = report_method_catalog()['methods'] assert all(catalog[name]['status'] == 'planned' for name in planned) diff --git a/tests/test_hosted_fetch.py b/tests/test_hosted_fetch.py new file mode 100644 index 000000000..c16ef4f60 --- /dev/null +++ b/tests/test_hosted_fetch.py @@ -0,0 +1,362 @@ +import base64 +import hashlib +import json +import tempfile +import unittest +import threading +import urllib.error +from pathlib import Path +from unittest.mock import Mock, patch + +from harness.web_fetch.diagnostics import CallDiagnostics +from harness.web_fetch.methods import method_info +from lexbench_webfetch.models import EvalCase +from harness.web_fetch.providers import build_provider +from harness.web_fetch.providers.http_client import HttpResponse +from harness.web_fetch.runner import run_evaluation +from lexbench_webfetch.metrics import task_passed + + +TYPES = ('brightdata_unlocker', 'browser_use_fetch', 'browserbase_fetch') + + +class HostedFetchTest(unittest.TestCase): + def setUp(self): + self.case = EvalCase('article', 'Article', 'https://example.com', 'static', ['Required article text']) + + def provider(self, kind, **config): + return build_provider(kind, {'type': kind, 'api_key': 'private-test-key', 'zone': 'test-zone', **config}) + + def response(self, kind, text='Required article text', status=200): + payload = ({'body': text, 'status_code': status, 'final_url': self.case.url} + if kind == 'browser_use_fetch' else {'content': text, 'statusCode': status}) + body = text.encode() if kind == 'brightdata_unlocker' else json.dumps(payload).encode() + return HttpResponse(200, 'https://service.example/', {}, body) + + + @patch('harness.web_fetch.providers.hosted_fetch.request') + def test_crawl4ai_cloud_contract_and_projection(self, call): + from harness.web_fetch.native_projection import require_native_answer_projection + provider = self.provider('crawl4ai_cloud') + self.assertEqual(provider.validate(), []) + for payload, status, expected in [ + ({'markdown':'Native body'}, 200, None), + ({'markdown':'Native body','origin_status':404}, 200, 'http_error'), + ({'markdown':'','error':'blocked'}, 403, 'access_denied'), + ({'content':'Wrong field'}, 200, 'invalid_response'), + ({'markdown':''}, 200, 'empty_content'), + ({'markdown':'body','success':False}, 200, 'api_error'), + ({'markdown':'body','ok':False}, 200, 'api_error'), + ]: + call.reset_mock() + raw=json.dumps(payload).encode() + call.return_value=HttpResponse(status,'https://api.crawl4ai.com/scrape',{},raw) + result=provider.fetch(self.case,45) + self.assertEqual(result.error_type,expected) + self.assertEqual(base64.b64decode(result.metadata['_raw_response_b64']),raw) + call.assert_called_once() + self.assertEqual(call.call_args.kwargs['json_body'], + {'url':self.case.url,'format':'md','engine':'browser','proxy':'none'}) + self.assertEqual(call.call_args.kwargs['timeout'],45) + require_native_answer_projection(result.content,result.metadata,raw, + title=result.title,final_url=result.final_url,request_url=self.case.url) + for option in ({'engine':'invalid'},{'proxy':'invalid'},{'output_format':'html'}): + self.assertTrue(self.provider('crawl4ai_cloud',**option).validate()) + + @patch('harness.web_fetch.providers.hosted_fetch.request') + def test_latest_article_track_preserves_native_answer_and_rejects_tampering(self, call): + from harness.web_fetch.native_projection import require_native_answer_projection + case = EvalCase('article-new', 'Article', 'https://example.com/article', 'article', [], track='articles') + for kind in (*TYPES, 'deepseek_fetch'): + with self.subTest(kind=kind): + response = (HttpResponse(200, 'https://service.example/', {}, + b'{"title":"Title","content":"Required article text"}') + if kind == 'deepseek_fetch' else self.response(kind)) + call.return_value = response + result = self.provider(kind).fetch(case, 45) + self.assertTrue(result.ok) + self.assertEqual(result.content, 'Required article text') + if kind != 'brightdata_unlocker': + args = dict(title=result.title, final_url=result.final_url, request_url=case.url) + require_native_answer_projection(result.content, result.metadata, response.body, **args) + with self.assertRaises(ValueError): + require_native_answer_projection('Changed answer', result.metadata, response.body, **args) + + @patch('harness.web_fetch.providers.hosted_fetch.request') + def test_routes_and_preserved_evidence(self, call): + for kind in TYPES: + with self.subTest(kind=kind): + response = self.response(kind) + call.return_value = response + result = self.provider(kind).fetch(self.case, 45) + self.assertTrue(result.ok) + self.assertEqual(result.content, 'Required article text') + self.assertEqual(base64.b64decode(result.metadata['_raw_response_b64']), response.body) + self.assertNotIn('private-test-key', json.dumps(result.metadata)) + self.assertNotEqual(method_info(kind)['contract'], '未知') + body = call.call_args.kwargs['json_body'] + self.assertEqual(body['url'], self.case.url) + self.assertEqual(call.call_args.kwargs['timeout'], 45) + self.assertEqual(result.metadata['request_timeout_seconds'], 45) + self.assertEqual(result.metadata['dns_mode'], 'system') + self.assertEqual(result.metadata['dns_trace'], []) + self.assertEqual(result.metadata['service_retry_policy'], 'provider_default_not_overridden') + if kind == 'brightdata_unlocker': + self.assertEqual(body['data_format'], 'markdown') + self.assertEqual(body['format'], 'raw') + self.assertIsNone(result.status_code) + elif kind == 'browser_use_fetch': + self.assertNotIn('retry', body) + self.assertEqual(body['timeout_ms'], 45000) + self.assertEqual(body['output_format'], 'markdown') + session = body['session_id'] + self.provider(kind).fetch(self.case, 45) + self.assertNotEqual(session, call.call_args.kwargs['json_body']['session_id']) + else: + self.assertTrue(body['allowRedirects']) + self.assertTrue(body['proxies']) + self.assertNotIn('projectId', body) + + @patch('harness.web_fetch.providers.hosted_fetch.request') + def test_brightdata_native_render_option_is_explicit(self, call): + call.return_value = self.response('brightdata_unlocker') + for value in (None, True, False): + config = {} if value is None else {'render': value} + result = self.provider('brightdata_unlocker', **config).fetch(self.case, 45) + body = call.call_args.kwargs['json_body'] + self.assertEqual(body.get('render'), None if value is None else str(value).lower()) + self.assertEqual(result.metadata['render'], value) + self.assertNotIn('wait', body) + self.assertNotIn('selector', body) + self.assertIn('render must be boolean', self.provider('brightdata_unlocker', render='true').validate()) + + @patch('harness.web_fetch.providers.hosted_fetch.request') + def test_only_brightdata_receives_documented_native_debug_flag(self, call): + for kind in TYPES: + call.return_value = self.response(kind) + self.provider(kind, log_level='debug', debug=True).fetch(self.case, 45) + body = call.call_args.kwargs['json_body'] + if kind == 'brightdata_unlocker': + self.assertIs(body['debug'], True) + else: + self.assertNotIn('debug', body) + self.assertIn('debug must be boolean', self.provider(TYPES[0], debug='true').validate()) + + @patch('harness.web_fetch.providers.hosted_fetch.request') + def test_native_structured_formats_preserve_only_returned_answers(self, call): + schema = {'type':'object','properties':{'records':{'type':'array','items':{'type':'object'}}}} + answer = {'records':[{'text':'Returned by service'}]} + call.return_value = HttpResponse(200,'https://service.example/',{},json.dumps({'content':answer,'statusCode':200}).encode()) + result = self.provider('browserbase_fetch', output_format='json', schema=schema).fetch(self.case,45) + self.assertTrue(result.ok) + self.assertEqual(json.loads(result.content),answer) + self.assertEqual(call.call_args.kwargs['json_body']['schema'],schema) + self.assertEqual(result.metadata['output_contract'],'json') + call.return_value = self.response('browser_use_fetch', json.dumps(answer)) + result = self.provider('browser_use_fetch', output_format='structured').fetch(self.case,45) + self.assertTrue(result.ok) + self.assertEqual(json.loads(result.content),answer) + self.assertEqual(call.call_args.kwargs['json_body']['output_format'],'structured') + self.assertTrue(self.provider('browserbase_fetch', output_format='json').validate()) + self.assertTrue(self.provider('browser_use_fetch', output_format='invented').validate()) + + @patch('harness.web_fetch.providers.hosted_fetch.request') + def test_http_200_is_not_enough(self, call): + for kind in TYPES: + call.return_value = self.response(kind, '') + result = self.provider(kind).fetch(self.case, 45) + self.assertFalse(result.ok) + self.assertEqual(result.error_type, 'empty_content') + for kind in TYPES[1:]: + call.return_value = self.response(kind, 'Required article text', 403) + self.assertEqual(self.provider(kind).fetch(self.case, 45).error_type, 'http_error') + for raw in (b'not json', b'{"message":"Required article text"}', + b'{"content":{},"statusCode":200}'): + call.return_value = HttpResponse(200, 'https://service.example/', {}, raw) + self.assertFalse(self.provider(kind).fetch(self.case, 45).ok) + for kind, payload in ( + ('browser_use_fetch', {'body':'Required article text','status_code':200,'error':'failed'}), + ('browserbase_fetch', {'content':'Required article text','statusCode':200,'encoding':'base64'}), + ): + call.return_value = HttpResponse(200, 'https://service.example/', {}, json.dumps(payload).encode()) + self.assertFalse(self.provider(kind).fetch(self.case, 45).ok) + + @patch('harness.web_fetch.providers.hosted_fetch.request') + def test_api_errors_are_separate_from_content(self, call): + for kind in TYPES: + for status, label in ((401,'authentication'), (402,'quota'), (429,'rate_limit'), (500,'api_error')): + call.return_value = HttpResponse(status, 'https://service.example/', {}, b'{"error":"Required article text"}') + result = self.provider(kind).fetch(self.case, 45) + self.assertFalse(result.ok) + self.assertEqual(result.content, '') + self.assertEqual(result.error_type, label) + + @patch('harness.web_fetch.providers.hosted_fetch.request') + def test_unsupported_delivery_never_calls_service(self, call): + case = EvalCase('file', 'File', 'https://example.com/file.pdf', 'document', [], track='document', task_kind='file') + for kind in TYPES: + self.assertEqual(self.provider(kind).fetch(case, 45).error_type, 'unsupported_task') + self.assertIn('api_key', build_provider(kind, {'type':kind}).validate()) + call.assert_not_called() + + @patch('harness.web_fetch.providers.hosted_fetch.request') + @patch('harness.web_fetch.network_dns.lookup_public_dns') + def test_single_request_without_dns_fallback_or_task_assistance(self, lookup, call): + self.case.retrieval_hints.update(version=1, completion={ + 'selector':'#answer', 'minimum_count':10, 'timeout_ms':30000}) + for kind in TYPES: + with self.subTest(kind=kind): + call.reset_mock() + call.return_value = HttpResponse(503, 'https://service.example/', {}, b'Unavailable') + result = self.provider(kind).fetch(self.case, 45) + self.assertFalse(result.ok) + call.assert_called_once() + body = call.call_args.kwargs['json_body'] + self.assertNotIn('#answer', json.dumps(body)) + self.assertNotIn('retry', body) + self.assertFalse(result.metadata['retrieval_hints_applied']) + lookup.assert_not_called() + + @patch('harness.web_fetch.providers.http_client.urllib.request.build_opener') + def test_runner_scores_and_saves_all_three_methods(self, call): + def respond(request, **kwargs): + url = request.full_url + if 'brightdata' in url: + kind = TYPES[0] + elif 'browser-use' in url: + kind = TYPES[1] + else: + kind = TYPES[2] + response = self.response(kind) + return Mock(status=response.status, headers={'x-brd-debug':'req_id=test-native-id; render=true'}, + read=Mock(return_value=response.body), geturl=Mock(return_value=response.url)) + call.return_value.open.side_effect = respond + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + reference = root/'sources/article/reference.txt' + reference.parent.mkdir(parents=True) + reference.write_text('Required article text') + self.case.source_benchmark.update(stable_annotation={ + 'anchor_evidence': 'sources/article/reference.txt', + 'anchor_evidence_sha256': hashlib.sha256(reference.read_bytes()).hexdigest(), + }) + self.case.acceptance_contract.update(version=1, schema='body_text', minimum_records=0, + stable_text_anchors=[{'id':'article','alternatives':['Required article text']}]) + dataset = {'tool':'web_fetch','id':'hosted-test','name':'Test','cases':[vars(self.case)], + 'coverage_plan':{'web':list(TYPES)}, + 'comparison':{'evidence_state':'diagnostic','kind':'web_fetch_diagnostic'}} + dataset_path = root/'dataset.json' + dataset_path.write_text(json.dumps(dataset)) + config_path = root/'providers.json' + config_path.write_text('{}') + records, _ = run_evaluation(output_dir=root/'run', dataset_meta=dataset, + dataset_path=dataset_path, cases=[self.case], providers=[self.provider(k, log_level='debug') for k in TYPES], + provider_config_path=config_path, runs=1, timeout=45, delay=0, provider_concurrency=1,max_failure_retries=0) + self.assertEqual(len(records),3) + self.assertTrue(all(task_passed(r['ok'],r['metrics'],'body') for r in records)) + for record in records: + self.assertTrue((root/'run'/record['provider_metadata']['raw_response_path']).is_file()) + log_path = root/'run'/record['provider_metadata']['diagnostic_log_path'] + raw_log = log_path.read_bytes() + self.assertEqual(hashlib.sha256(raw_log).hexdigest(), record['provider_metadata']['diagnostic_log_sha256']) + self.assertNotIn(b'private-test-key', raw_log) + for event in ('http_request_started', 'http_response_received', 'collection_completed'): + self.assertIn(event, raw_log.decode()) + self.assertIn('req_id=test-native-id', raw_log.decode()) + # Reporting must work from the saved run after original references + # disappear; it must never need another provider call. + reference.unlink() + from harness.web_fetch.report_build import build_report + report = build_report(root/'run', root/'run/report', diagnostic_preview=True, language='en') + self.assertTrue(report.is_file()) + self.assertEqual(call.call_count, 3) + + def test_diagnostics_redact_new_auth_headers(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory)/'log.jsonl' + log = CallDiagnostics(path, 'test') + log.emit('caller','request',headers={'X-BB-API-Key':'secret-bb','X-Browser-Use-API-Key':'secret-bu'}) + log.file.close() + self.assertNotIn('secret-bb',path.read_text()) + self.assertNotIn('secret-bu',path.read_text()) + + @patch('harness.web_fetch.providers.hosted_fetch.request') + def test_deepseek_url_only_and_exact_response(self, call): + raw = b'{"title":"Separate title","content":"Unchanged body"}' + call.return_value = HttpResponse(200, 'https://service.example/', {}, raw) + provider = self.provider('deepseek_fetch', log_level='debug') + result = provider.fetch(self.case, 45) + self.assertTrue(result.ok) + self.assertEqual(result.title, 'Separate title') + self.assertEqual(result.content, 'Unchanged body') + self.assertIsNone(result.status_code) + self.assertEqual(base64.b64decode(result.metadata['_raw_response_b64']), raw) + self.assertEqual(call.call_args.kwargs['json_body'], {'url':self.case.url}) + self.assertEqual(call.call_args.kwargs['timeout'], 45) + self.assertEqual(call.call_args.kwargs['headers'], {'Authorization':'Bearer private-test-key'}) + self.assertEqual(call.call_count, 1) + call.return_value = HttpResponse(429, 'https://service.example/', {}, b'{"error":{"code":"rate_limit","message":"actual limit error"}}') + failed = provider.fetch(self.case, 45) + self.assertFalse(failed.ok) + self.assertEqual(failed.error_type,'rate_limit') + self.assertEqual(failed.metadata['service_error']['message'],'actual limit error') + + @patch('harness.web_fetch.runner.score_attempt', side_effect=AssertionError('Scorer must not run')) + @patch('harness.web_fetch.providers.http_client.urllib.request.build_opener') + def test_collection_retains_actual_http_errors_and_exception_trace_without_scoring(self, opener, scorer): + responses = [Mock(status=200, headers={}, read=Mock(return_value=b'{"title":"Title","content":"Body"}'), + geturl=Mock(return_value='https://service.example/')), + Mock(status=500, headers={}, read=Mock(return_value=b'{"error":{"message":"native failure"}}'), + geturl=Mock(return_value='https://service.example/')), + urllib.error.URLError('synthetic DNS failure')] + opener.return_value.open.side_effect = responses + cases = [EvalCase(str(i),'Control',self.case.url,'static',[]) for i in range(3)] + with tempfile.TemporaryDirectory() as directory: + root=Path(directory) + rows,summary=run_evaluation(output_dir=root/'run',dataset_meta={'cases':[vars(c) for c in cases]}, + dataset_path=root/'dataset.json', cases=cases, + providers=[self.provider('deepseek_fetch',log_level='debug')],provider_config_path=root/'providers.json', + runs=1,timeout=45,delay=0,collect_only=True,max_failure_retries=0) + self.assertEqual([r['ok'] for r in rows],[True,False,False]) + self.assertIsNone(summary['task_pass_rate']) + self.assertEqual(opener.return_value.open.call_count,3) + for row in rows: + self.assertNotIn('content_pass',row['metrics']) + log=(root/'run'/row['provider_metadata']['diagnostic_log_path']).read_text() + self.assertNotIn('private-test-key',log) + self.assertIn('http_request_started',log) + failure_log=(root/'run'/rows[-1]['provider_metadata']['diagnostic_log_path']).read_text() + self.assertIn('synthetic DNS failure',failure_log) + self.assertIn('Traceback',failure_log) + server_log=(root/'run'/rows[1]['provider_metadata']['diagnostic_log_path']).read_text() + self.assertIn('native failure',server_log) + from harness.web_fetch.report_build import build_report + with self.assertRaisesRegex(ValueError,'Collection-only evidence'): + build_report(root/'run',root/'report') + scorer.assert_not_called() + + @patch('harness.web_fetch.providers.http_client.urllib.request.build_opener') + def test_collection_timeout_has_actual_deadline_event_and_no_retry(self, opener): + release=threading.Event() + def blocked(*args,**kwargs): + release.wait(1) + raise TimeoutError('simulated socket timeout') + opener.return_value.open.side_effect=blocked + try: + with tempfile.TemporaryDirectory() as directory: + root=Path(directory) + rows,_=run_evaluation(output_dir=root/'run',dataset_meta={'cases':[vars(self.case)]}, + dataset_path=root/'dataset.json',cases=[self.case], + providers=[self.provider('deepseek_fetch',log_level='debug')],provider_config_path=root/'providers.json', + runs=1,timeout=0.02,delay=0,collect_only=True,max_failure_retries=0) + self.assertEqual(rows[0]['error_type'],'timeout') + log=(root/'run'/rows[0]['provider_metadata']['diagnostic_log_path']).read_text() + self.assertIn('answer_timeout',log) + self.assertIn('End-to-end answer deadline exceeded',log) + self.assertEqual(opener.return_value.open.call_count,1) + finally: + release.set() + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_native_route_contract.py b/tests/test_native_route_contract.py index d7fbc94d2..17964c69f 100644 --- a/tests/test_native_route_contract.py +++ b/tests/test_native_route_contract.py @@ -71,7 +71,7 @@ def _route_http_fixture(tmp_path): cases = [EvalCase(name, name, 'https://example.test/'+name, 'control', [], source_benchmark={'input_media_kind': 'html_page'}) for name in ('one', 'two')] - ledger = NativeRouteLedger(cases, [provider]) + ledger = NativeRouteLedger(cases, [provider], timeout_seconds=45) binding = ledger.write(tmp_path, dataset_sha256='dataset', runtime_source_sha256='runtime') manifest = {'native_routes_required': True, 'native_route_contract': binding, 'dataset_sha256': 'dataset', 'runtime_source_sha256': 'runtime', @@ -257,7 +257,7 @@ def test_custom_firecrawl_without_auth_still_matches_actual_request(tmp_path): 'uniform_native_parameters': True}) case = EvalCase('case', 'Case', 'https://example.test/', 'control', [], source_benchmark={'input_media_kind': 'html_page'}) - ledger = NativeRouteLedger([case], [provider]) + ledger = NativeRouteLedger([case], [provider], timeout_seconds=45) ledger.write(tmp_path, dataset_sha256='fixture', runtime_source_sha256='fixture') with patch('harness.web_fetch.providers.firecrawl.request', return_value=HttpResponse(200, case.url, {}, b'{}')) as transport: @@ -443,7 +443,7 @@ def test_http_native_answers_survive_persistence_and_reject_tampering(tmp_path, 'include_resource_links': True, 'rich_links_limit': 12, 'formats': ['markdown', 'images', 'links']}) case = EvalCase('case', 'Case', 'https://example.test/', 'control', [], track=track, task_kind=kind, source_benchmark={'input_media_kind': 'html_page'}) - ledger = NativeRouteLedger([case], [provider]) + ledger = NativeRouteLedger([case], [provider], timeout_seconds=45) ledger.write(tmp_path, dataset_sha256='fixture', runtime_source_sha256='fixture') with patch('harness.web_fetch.providers.'+module+'.request', return_value=HttpResponse(200, case.url, {}, json.dumps(payload).encode())): @@ -935,7 +935,7 @@ def test_explicit_ledgers_do_not_replace_each_others_provider_state(tmp_path): from unittest.mock import Mock from harness.web_fetch.native_routes import NativeRouteLedger provider, cases, first, _, _ = _route_http_fixture(tmp_path) - second = NativeRouteLedger(cases, [provider]) + second = NativeRouteLedger(cases, [provider], timeout_seconds=45) other = tmp_path / 'other' other.mkdir() second.write(other, dataset_sha256='dataset', runtime_source_sha256='runtime') diff --git a/tests/test_report_methods.py b/tests/test_report_methods.py index 3708abdeb..92cfd95ac 100644 --- a/tests/test_report_methods.py +++ b/tests/test_report_methods.py @@ -269,3 +269,23 @@ def test_method_model_selects_direct_english_fields_without_reverse_translation( assert '1/1 records' in observed['configuration'] assert not re.search(r'[\u4e00-\u9fff]', observed['configuration']) + + +def test_hosted_interfaces_follow_adapter_identity_in_external_cohorts(): + from harness.web_fetch.report_interfaces import method_spec + from harness.web_fetch.methods import method_info + adapters = ('brightdata_unlocker', 'browser_use_fetch', 'browserbase_fetch', + 'deepseek_fetch', 'crawl4ai_cloud') + for adapter in adapters: + record = {'provider': 'external-participant', 'provider_type': adapter, + 'provider_metadata': {}} + for language in ('zh', 'en'): + spec = method_spec('external-participant', record, language=language) + assert spec['entry'] == method_info(adapter, language=language)['endpoint'] + assert spec['request']['url'] == 'https://example.com/' + assert spec['source'] == 'harness/web_fetch/providers/hosted_fetch.py' + assert spec['fields'] + assert 'API_KEY' in spec['auth'] + english = method_spec('external-participant', record, language='en') + assert 'historical call parameters' in english['notes'] + assert not any('\u3400' <= char <= '\u9fff' for char in str(english)) diff --git a/tests/test_scoring.py b/tests/test_scoring.py index f7eddf2e8..5c2b0b01b 100644 --- a/tests/test_scoring.py +++ b/tests/test_scoring.py @@ -949,7 +949,7 @@ def setUp(self): self.path = self.root / "dataset.json" self.path.write_text(json.dumps(self.dataset)) refs = self.root / "list-references" - refs.mkdir() + refs.mkdir(exist_ok=True) for name in ("legacy", "unverified"): (refs / f"{name}.json").write_text(json.dumps({**REFERENCE, "case_id": name})) texts = {"web": "Alpha\r\n", "dynamic": "Catalog\nCategory filter\n" + LINKS, @@ -985,7 +985,7 @@ def live(self): def prepare_script_references(self): refs = self.run / "list-references" - refs.mkdir() + refs.mkdir(exist_ok=True) for name in ("legacy", "unverified"): (refs / f"{name}.json").write_bytes((self.root / "list-references" / f"{name}.json").read_bytes()) diff --git a/tests/test_selected_native_policy.py b/tests/test_selected_native_policy.py new file mode 100644 index 000000000..d3a781eb7 --- /dev/null +++ b/tests/test_selected_native_policy.py @@ -0,0 +1,171 @@ +"""Offline behavior checks for the selected ten-method configuration.""" +import json +import time +from dataclasses import asdict, replace +from pathlib import Path +from unittest.mock import patch, Mock + +import pytest + +from lexbench_webfetch.models import EvalCase +from harness.web_fetch.cli import build_parser +from harness.web_fetch.providers.base import build_provider +from harness.web_fetch.native_routes import normalized_http_parameters, verify_saved_native_answer +from harness.web_fetch.runner import run_evaluation + +SELECTED = ('moli-webfetch', 'firecrawl', 'cloudflare-markdown', 'tavily', 'exa', + 'brightdata', 'browser-use-fetch', 'browserbase', 'deepseek-fetch-api', 'crawl4ai-cloud') +CONFIG = json.loads(Path('configs/providers.example.json').read_text())['providers'] + + +def provider(name): + return build_provider(name, {**CONFIG[name], 'api_key': 'offline-key', + 'api_token': 'offline-token', 'account_id': 'offline-account', 'zone': 'offline-zone'}) + + +def case(): + return EvalCase('one', 'One', 'https://example.test/', 'static', ['Expected'], + source_benchmark={'input_media_kind': 'html_page'}) + + +def test_defaults(): + args = build_parser().parse_args(['run']) + assert args.timeout == 30 + assert args.max_failure_retries == 1 + + +@pytest.mark.parametrize('name', SELECTED) +def test_selected_methods_reject_case_assistance_before_transport(name): + p = provider(name) + assert p.config['uniform_native_parameters'] is True + hinted = replace(case(), retrieval_hints={'completion': {'selector': '#answer'}}) + with pytest.raises(ValueError, match='per-case'): + p.fetch(hinted, 30) + + +@pytest.mark.parametrize('name', SELECTED[1:]) +def test_request_is_independent_of_grading_requirements(name): + p = provider(name) + first = case() + second = replace(first, expected_any=['Different answer'], acceptance_contract={ + 'schema': 'named_link', 'required_links': [{'url': 'https://answer.test/'}]}) + endpoint, args = p.native_http_arguments(first, 30) + other_endpoint, other = p.native_http_arguments(second, 30) + assert endpoint == other_endpoint + assert normalized_http_parameters(first, args) == normalized_http_parameters(second, other) + assert args['timeout'] == 30 + if name == 'exa': + assert args['json_body']['extras']['richLinks'] == 100 + if name == 'cloudflare-markdown': + document = replace(first, url='https://example.test/file.pdf', track='document') + doc_endpoint, doc_args = p.native_http_arguments(document, 30) + assert doc_endpoint.endswith('/markdown') + assert doc_args['json_body']['url'] == document.url + assert 'multipart' not in doc_args + + +@pytest.mark.parametrize('name', SELECTED[5:]) +@pytest.mark.parametrize('answers', [('Expected',), ('wrong', 'Expected'), ('wrong', 'still wrong'), ('HTTP_ERROR', 'Expected')]) +def test_hosted_native_retry_persists_every_physical_answer(tmp_path, name, answers): + p, c = provider(name), case() + texts = iter(answers) + def respond(request, **kwargs): + text = next(texts) + payload = ({'body': text, 'status_code': 200, 'final_url': c.url} if name == 'browser-use-fetch' + else {'markdown': text} if name == 'crawl4ai-cloud' + else {'content': text, 'title': 'Title', 'statusCode': 200}) + raw = text.encode() if name == 'brightdata' else json.dumps(payload).encode() + assert kwargs['timeout'] <= 30 + return Mock(status=503 if text == 'HTTP_ERROR' else 200, headers={}, read=Mock(return_value=raw), + geturl=Mock(return_value=request.full_url)) + with patch('harness.web_fetch.providers.http_client.urllib.request.build_opener') as opener: + opener.return_value.open.side_effect = respond + rows, _ = run_evaluation(output_dir=tmp_path, dataset_meta={'id':'offline', 'cases':[asdict(c)]}, + dataset_path=tmp_path/'dataset.json', cases=[c], providers=[p], + provider_config_path=tmp_path/'providers.json', runs=1, timeout=30, delay=0) + assert opener.return_value.open.call_count == len(answers) + row = rows[0] + assert (tmp_path/row['artifact_path']).read_text() == answers[-1] + assert row['provider_metadata']['latency_scope'] == 'native_response_received' + assert (tmp_path/row['provider_metadata']['raw_response_path']).is_file() + assert (tmp_path/row['provider_metadata']['diagnostic_log_path']).is_file() + verify_saved_native_answer(tmp_path, row, cases_by_id={c.id: asdict(c)}) + # Raw responses and caller logs remain separate for each paid physical call. + assert len(list(tmp_path.rglob('*.response.bin'))) == len(answers) + + +def test_native_preparation_does_not_consume_response_budget(): + from harness.web_fetch.call_boundary import fetch_with_deadline, prepare_answer + from harness.web_fetch.models import ProviderResult + p = build_provider('fixture', {'type':'fixture', 'uniform_native_parameters':True}) + p.fetch = Mock(return_value=ProviderResult(True, 'Expected')) + def slow_prepare(result, c): + time.sleep(.08) + return prepare_answer(result, c) + with patch('harness.web_fetch.call_boundary.prepare_answer', side_effect=slow_prepare): + result, _ = fetch_with_deadline(p, case(), .05) + assert result.ok + assert result.metadata['_answer_elapsed_ms'] < 50 + + +@pytest.mark.parametrize('name', SELECTED[6:]) +def test_malformed_api_response_is_retained_as_failure(tmp_path, name): + p, c = provider(name), case() + with patch('harness.web_fetch.providers.http_client.urllib.request.build_opener') as opener: + opener.return_value.open.return_value = Mock(status=200, headers={}, + read=Mock(return_value=b'not valid JSON'), geturl=Mock(return_value='https://service.test/')) + rows, _ = run_evaluation(output_dir=tmp_path, dataset_meta={'id':'offline', 'cases':[asdict(c)]}, + dataset_path=tmp_path/'dataset.json', cases=[c], providers=[p], + provider_config_path=tmp_path/'providers.json', runs=1, timeout=30, delay=0) + assert opener.return_value.open.call_count == 2 + row = rows[0] + assert not row['ok'] and row['error_type'] == 'invalid_response' + assert (tmp_path/row['artifact_path']).read_text() == '' + assert (tmp_path/row['provider_metadata']['raw_response_path']).read_bytes() == b'not valid JSON' + verify_saved_native_answer(tmp_path, row, cases_by_id={c.id: asdict(c)}) + + +def test_moli_native_command_has_no_case_hint_or_reduced_timer(): + p = provider('moli-webfetch') + first = case() + second = replace(first, expected_any=['Another answer'], acceptance_contract={'schema':'named_link'}) + assert p.native_cli_command(first) == p.native_cli_command(second) + command = p.native_cli_command(first) + assert '--wait-until' in command # Fixed product-native completion option. + assert not any('timeout' in arg or 'selector' in arg for arg in command) + + +def test_configured_cohort_and_local_configuration_guards(): + from harness.web_fetch.config import resolve_provider_selection + names = resolve_provider_selection('configured', Path('configs/providers.example.json')).split(',') + assert tuple(names) == SELECTED + assert resolve_provider_selection('fixture', None) == 'fixture' + assert resolve_provider_selection(None, None) is None + for name in names: + assert CONFIG[name]['log_level'] == 'debug' + p = provider('moli-webfetch') + assert not p._configuration_errors('/offline/moli') + + +@pytest.mark.parametrize('names', [[], ['missing'], ['fixture', 'fixture']]) +def test_configured_cohort_rejects_invalid_selection(tmp_path, names): + from harness.web_fetch.config import resolve_provider_selection + config = tmp_path/'providers.json' + config.write_text(json.dumps({'providers':{'fixture':{'type':'fixture'}}, 'default_providers':names})) + with pytest.raises(ValueError): + resolve_provider_selection('configured', config) + + +def test_configured_cli_selection_reaches_runner(tmp_path): + from harness.web_fetch.cli import command_run + config = tmp_path/'providers.json' + config.write_text(json.dumps({'providers':{'fixture':{'type':'fixture'}}, 'default_providers':['fixture']})) + args = build_parser().parse_args(['run','--providers','configured','--provider-config',str(config), + '--output',str(tmp_path/'run')]) + with patch('harness.web_fetch.cli.load_runtime_env'), \ + patch('harness.web_fetch.cli.run_evaluation', return_value=([], {})) as run, \ + patch('harness.web_fetch.cli.build_report', return_value=tmp_path/'report'): + command_run(args) + assert [p.name for p in run.call_args.kwargs['providers']] == ['fixture'] + assert run.call_args.kwargs['timeout'] == 30 + assert run.call_args.kwargs['max_failure_retries'] == 1 From 15ff30366c1d0e22f4ff70da53994eed4fea70cc Mon Sep 17 00:00:00 2001 From: Maisiqi <324751762+Maisiqi@users.noreply.github.com> Date: Fri, 23 Oct 2026 19:22:28 +0800 Subject: [PATCH 02/18] Align native cohort evidence and capability checks --- harness/web_fetch/artifacts.py | 5 +- harness/web_fetch/data/report-methods.json | 439 +++++++++++++++++++++ tests/test_assembly_evidence.py | 2 +- tests/test_dynamic_provider_waits.py | 5 +- tests/test_interrupted_evidence.py | 6 +- tests/test_providers.py | 10 +- 6 files changed, 459 insertions(+), 8 deletions(-) diff --git a/harness/web_fetch/artifacts.py b/harness/web_fetch/artifacts.py index 700ff45d2..96cb41ccb 100644 --- a/harness/web_fetch/artifacts.py +++ b/harness/web_fetch/artifacts.py @@ -157,9 +157,12 @@ def verify_public_call_log(root: Path, binding: dict[str, Any], *, allow_public_ for event in events) or events[0]["stage"] != "call_started" or events[-1]["stage"] != "call_completed" - or not any(event["stage"] in {"response_received", "exception_raised"} + or not any(event["stage"] in {"response_received", "exception_raised", "answer_deadline_exceeded"} for event in events[1:-1])): raise ValueError("Public call log has an unexpected event schema") + if any(event["stage"] == "answer_deadline_exceeded" and event["error_type"] != "timeout" + for event in events): + raise ValueError("Deadline event must identify a timeout") summary = payload["request"]["summary"] if summary is not None and not isinstance(summary, dict): raise ValueError("Public call log has an unexpected request summary") diff --git a/harness/web_fetch/data/report-methods.json b/harness/web_fetch/data/report-methods.json index 71b820099..e74dc837f 100644 --- a/harness/web_fetch/data/report-methods.json +++ b/harness/web_fetch/data/report-methods.json @@ -5344,6 +5344,92 @@ "zh": "这是接口示例，不代表历史调用参数。渲染、代理、引擎、输出格式和时间预算以保存的配置及请求日志为准。保存完整原始响应，不补抓或补写正文；调用端日志不等于服务内部日志。", "en": "This interface example does not establish historical call parameters. Rendering, proxy, engine, output format and time budget come from saved configuration and request logs. Preserve the complete response without supplemental retrieval or content repair; caller logs are not service-internal traces." } + }, + "capabilities": { + "scope": "POST https://api.browserbase.com/v1/fetch", + "sources": [ + "hosted-fetch-adapter" + ], + "web": { + "status": { + "zh": "有限支持", + "en": "Limited support" + }, + "detail": { + "zh": "当前接入支持网页正文和结构任务；不代表实测通过率。", + "en": "The adapter supports webpage body and structure tasks; this is not a measured pass rate." + } + }, + "input": { + "zh": "已知网址；使用预先固定的原生接口参数。", + "en": "Known URL with native interface parameters fixed before calls." + }, + "structure": { + "zh": "保留原始响应，仅按声明字段读取正文；不补抓内容。", + "en": "Retain raw responses and read declared answer fields without supplemental fetching." + }, + "access": { + "zh": "需要服务账号及可用额度。", + "en": "Requires a service account and available quota." + }, + "notes": { + "zh": "这里登记的是当前接入范围，不推断服务商其他产品的能力。", + "en": "This describes the current adapter, not other products from the provider." + }, + "tested": { + "zh": "接入已通过离线模拟响应检查；不代表完成同范围正式评测。", + "en": "Adapter checked with offline simulated responses; no same-scope formal benchmark is implied." + }, + "document": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "image": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "av": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "file": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "js": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "渲染行为由原生服务及所选配置决定；未据离线检查确认页面执行能力。", + "en": "Rendering depends on the native service and selected configuration; offline checks do not establish page execution capability." + } + } } }, "bocha-web-search": { @@ -5627,6 +5713,92 @@ "zh": "这是接口示例，不代表历史调用参数。渲染、代理、引擎、输出格式和时间预算以保存的配置及请求日志为准。保存完整原始响应，不补抓或补写正文；调用端日志不等于服务内部日志。", "en": "This interface example does not establish historical call parameters. Rendering, proxy, engine, output format and time budget come from saved configuration and request logs. Preserve the complete response without supplemental retrieval or content repair; caller logs are not service-internal traces." } + }, + "capabilities": { + "scope": "POST https://api.brightdata.com/request", + "sources": [ + "hosted-fetch-adapter" + ], + "web": { + "status": { + "zh": "有限支持", + "en": "Limited support" + }, + "detail": { + "zh": "当前接入支持网页正文和结构任务；不代表实测通过率。", + "en": "The adapter supports webpage body and structure tasks; this is not a measured pass rate." + } + }, + "input": { + "zh": "已知网址；使用预先固定的原生接口参数。", + "en": "Known URL with native interface parameters fixed before calls." + }, + "structure": { + "zh": "保留原始响应，仅按声明字段读取正文；不补抓内容。", + "en": "Retain raw responses and read declared answer fields without supplemental fetching." + }, + "access": { + "zh": "需要服务账号及可用额度。", + "en": "Requires a service account and available quota." + }, + "notes": { + "zh": "这里登记的是当前接入范围，不推断服务商其他产品的能力。", + "en": "This describes the current adapter, not other products from the provider." + }, + "tested": { + "zh": "接入已通过离线模拟响应检查；不代表完成同范围正式评测。", + "en": "Adapter checked with offline simulated responses; no same-scope formal benchmark is implied." + }, + "document": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "image": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "av": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "file": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "js": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "渲染行为由原生服务及所选配置决定；未据离线检查确认页面执行能力。", + "en": "Rendering depends on the native service and selected configuration; offline checks do not establish page execution capability." + } + } } }, "scrapingbee": { @@ -5922,6 +6094,92 @@ "zh": "这是接口示例，不代表历史调用参数。渲染、代理、引擎、输出格式和时间预算以保存的配置及请求日志为准。保存完整原始响应，不补抓或补写正文；调用端日志不等于服务内部日志。", "en": "This interface example does not establish historical call parameters. Rendering, proxy, engine, output format and time budget come from saved configuration and request logs. Preserve the complete response without supplemental retrieval or content repair; caller logs are not service-internal traces." } + }, + "capabilities": { + "scope": "POST https://fetch.browser-use.com/fetch", + "sources": [ + "hosted-fetch-adapter" + ], + "web": { + "status": { + "zh": "有限支持", + "en": "Limited support" + }, + "detail": { + "zh": "当前接入支持网页正文和结构任务；不代表实测通过率。", + "en": "The adapter supports webpage body and structure tasks; this is not a measured pass rate." + } + }, + "input": { + "zh": "已知网址；使用预先固定的原生接口参数。", + "en": "Known URL with native interface parameters fixed before calls." + }, + "structure": { + "zh": "保留原始响应，仅按声明字段读取正文；不补抓内容。", + "en": "Retain raw responses and read declared answer fields without supplemental fetching." + }, + "access": { + "zh": "需要服务账号及可用额度。", + "en": "Requires a service account and available quota." + }, + "notes": { + "zh": "这里登记的是当前接入范围，不推断服务商其他产品的能力。", + "en": "This describes the current adapter, not other products from the provider." + }, + "tested": { + "zh": "接入已通过离线模拟响应检查；不代表完成同范围正式评测。", + "en": "Adapter checked with offline simulated responses; no same-scope formal benchmark is implied." + }, + "document": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "image": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "av": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "file": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "js": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "渲染行为由原生服务及所选配置决定；未据离线检查确认页面执行能力。", + "en": "Rendering depends on the native service and selected configuration; offline checks do not establish page execution capability." + } + } } }, "deepseek-fetch-api": { @@ -6007,6 +6265,92 @@ "zh": "这是接口示例，不代表历史调用参数。渲染、代理、引擎、输出格式和时间预算以保存的配置及请求日志为准。保存完整原始响应，不补抓或补写正文；调用端日志不等于服务内部日志。", "en": "This interface example does not establish historical call parameters. Rendering, proxy, engine, output format and time budget come from saved configuration and request logs. Preserve the complete response without supplemental retrieval or content repair; caller logs are not service-internal traces." } + }, + "capabilities": { + "scope": "POST https://api.deepseek.com/v1/tools/fetch", + "sources": [ + "hosted-fetch-adapter" + ], + "web": { + "status": { + "zh": "有限支持", + "en": "Limited support" + }, + "detail": { + "zh": "当前接入支持网页正文和结构任务；不代表实测通过率。", + "en": "The adapter supports webpage body and structure tasks; this is not a measured pass rate." + } + }, + "input": { + "zh": "已知网址；使用预先固定的原生接口参数。", + "en": "Known URL with native interface parameters fixed before calls." + }, + "structure": { + "zh": "保留原始响应，仅按声明字段读取正文；不补抓内容。", + "en": "Retain raw responses and read declared answer fields without supplemental fetching." + }, + "access": { + "zh": "需要服务账号及可用额度。", + "en": "Requires a service account and available quota." + }, + "notes": { + "zh": "这里登记的是当前接入范围，不推断服务商其他产品的能力。", + "en": "This describes the current adapter, not other products from the provider." + }, + "tested": { + "zh": "接入已通过离线模拟响应检查；不代表完成同范围正式评测。", + "en": "Adapter checked with offline simulated responses; no same-scope formal benchmark is implied." + }, + "document": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "image": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "av": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "file": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "js": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "渲染行为由原生服务及所选配置决定；未据离线检查确认页面执行能力。", + "en": "Rendering depends on the native service and selected configuration; offline checks do not establish page execution capability." + } + } } }, "crawl4ai-cloud": { @@ -6093,6 +6437,92 @@ "zh": "这是接口示例，不代表历史调用参数。渲染、代理、引擎、输出格式和时间预算以保存的配置及请求日志为准。保存完整原始响应，不补抓或补写正文；调用端日志不等于服务内部日志。", "en": "This interface example does not establish historical call parameters. Rendering, proxy, engine, output format and time budget come from saved configuration and request logs. Preserve the complete response without supplemental retrieval or content repair; caller logs are not service-internal traces." } + }, + "capabilities": { + "scope": "POST https://api.crawl4ai.com/scrape", + "sources": [ + "hosted-fetch-adapter" + ], + "web": { + "status": { + "zh": "有限支持", + "en": "Limited support" + }, + "detail": { + "zh": "当前接入支持网页正文和结构任务；不代表实测通过率。", + "en": "The adapter supports webpage body and structure tasks; this is not a measured pass rate." + } + }, + "input": { + "zh": "已知网址；使用预先固定的原生接口参数。", + "en": "Known URL with native interface parameters fixed before calls." + }, + "structure": { + "zh": "保留原始响应，仅按声明字段读取正文；不补抓内容。", + "en": "Retain raw responses and read declared answer fields without supplemental fetching." + }, + "access": { + "zh": "需要服务账号及可用额度。", + "en": "Requires a service account and available quota." + }, + "notes": { + "zh": "这里登记的是当前接入范围，不推断服务商其他产品的能力。", + "en": "This describes the current adapter, not other products from the provider." + }, + "tested": { + "zh": "接入已通过离线模拟响应检查；不代表完成同范围正式评测。", + "en": "Adapter checked with offline simulated responses; no same-scope formal benchmark is implied." + }, + "document": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "image": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "av": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "file": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "js": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "渲染行为由原生服务及所选配置决定；未据离线检查确认页面执行能力。", + "en": "Rendering depends on the native service and selected configuration; offline checks do not establish page execution capability." + } + } } } }, @@ -6399,6 +6829,15 @@ "url": "https://github.com/lexmount/moli/releases/tag/v1.1.5", "checked_at": "2026-09-11", "checked_version": "1.1.5" + }, + "hosted-fetch-adapter": { + "label": { + "zh": "托管 Fetch 接入代码（能力范围）", + "en": "Hosted Fetch adapter (implemented scope)" + }, + "url": "https://github.com/lexmount/browser-eval/blob/38d224ec/harness/web_fetch/providers/hosted_fetch.py", + "checked_at": "2026-10-23", + "checked_version": "38d224ec" } }, "output_contract_labels": { diff --git a/tests/test_assembly_evidence.py b/tests/test_assembly_evidence.py index 30f204c83..e3a9726d7 100644 --- a/tests/test_assembly_evidence.py +++ b/tests/test_assembly_evidence.py @@ -520,7 +520,7 @@ def fetch(self, case, timeout): (root / 'execution.json').write_text(json.dumps({'batches': [manifest]})) (root / 'manifest.json').unlink() for folder in ('sources', 'list-references'): - (root / folder).mkdir() + (root / folder).mkdir(exist_ok=True) (root / 'reference-checks/private-note.txt').write_text('synthetic private note') output = tmp_path / 'assembled' relative = manifest['reference_checks']['postcheck']['evidence_path'] diff --git a/tests/test_dynamic_provider_waits.py b/tests/test_dynamic_provider_waits.py index 391ac36fb..1fe98c964 100644 --- a/tests/test_dynamic_provider_waits.py +++ b/tests/test_dynamic_provider_waits.py @@ -35,7 +35,10 @@ def dynamic_case(**completion_overrides): def test_release_configs_select_firecrawl_actions_route(): for filename in ("configs/providers.example.json", "configs/providers.resources.json"): providers = json.loads(Path(filename).read_text())["providers"] - assert providers["firecrawl"]["dynamic_wait_strategy"] == "actions" + if providers["firecrawl"].get("uniform_native_parameters"): + assert "dynamic_wait_strategy" not in providers["firecrawl"] + else: + assert providers["firecrawl"]["dynamic_wait_strategy"] == "actions" assert providers["firecrawl-resources"]["dynamic_wait_strategy"] == "actions" assert providers["firecrawl"]["proxy"] == "auto" assert providers["firecrawl-resources"]["proxy"] == "auto" diff --git a/tests/test_interrupted_evidence.py b/tests/test_interrupted_evidence.py index ade1e9d21..2266b26f3 100644 --- a/tests/test_interrupted_evidence.py +++ b/tests/test_interrupted_evidence.py @@ -98,8 +98,8 @@ def test_adding_a_provider_preserves_the_declared_review_policy(self): run_evaluation(output_dir=output, dataset_meta=dataset, dataset_path=path, cases=[case], providers=[FixtureProvider(provider, {"type": "fixture"})], provider_config_path=config, runs=1, timeout=1, delay=0, max_failure_retries=0) - (output / "sources").mkdir() - (output / "list-references").mkdir() + (output / "sources").mkdir(exist_ok=True) + (output / "list-references").mkdir(exist_ok=True) runs.append(output) target = root / "assembled" assemble(runs[0], root / "unused", [], target, provider_additions=[runs[1]]) @@ -133,7 +133,7 @@ def test_case_addition_retains_every_certified_batch_and_call_log(self): manifest=json.loads((output/'manifest.json').read_text()) (output/'execution.json').write_text(json.dumps({'batches':[manifest]})) (output/'manifest.json').unlink() - (output/'sources').mkdir();(output/'list-references').mkdir();runs.append(output) + (output/'sources').mkdir(exist_ok=True);(output/'list-references').mkdir(exist_ok=True);runs.append(output) target=root/'assembled' assemble(runs[0],root/'unused',[],target,additions=[runs[1]]) rows=parse_jsonl((target/'results.jsonl').read_bytes()) diff --git a/tests/test_providers.py b/tests/test_providers.py index f8978e436..7bf3c0ae4 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -924,11 +924,17 @@ def test_release_configs_bind_strongest_tavily_and_exa_text_routes(self) -> None with self.subTest(filename=filename): self.assertEqual(providers["tavily"]["extract_depth"], "advanced") self.assertEqual(providers["tavily-resources"]["extract_depth"], "advanced") - self.assertEqual(providers["tavily"]["extract_timeout_seconds"], 60.0) + if providers["tavily"].get("uniform_native_parameters"): + self.assertNotIn("extract_timeout_seconds", providers["tavily"]) + else: + self.assertEqual(providers["tavily"]["extract_timeout_seconds"], 60.0) self.assertEqual(providers["tavily-resources"]["extract_timeout_seconds"], 60.0) self.assertEqual(providers["exa"]["text_verbosity"], "full") self.assertEqual(providers["exa-resources"]["text_verbosity"], "full") - self.assertEqual(providers["exa"]["livecrawl_timeout_ms"], 15_000) + if providers["exa"].get("uniform_native_parameters"): + self.assertNotIn("livecrawl_timeout_ms", providers["exa"]) + else: + self.assertEqual(providers["exa"]["livecrawl_timeout_ms"], 15_000) self.assertEqual(providers["exa-resources"]["livecrawl_timeout_ms"], 15_000) @patch("harness.web_fetch.providers.cloudflare.request") From 67c52d3cab1f6c70508a677efab3b0bb6c582f40 Mon Sep 17 00:00:00 2001 From: Maisiqi <324751762+Maisiqi@users.noreply.github.com> Date: Fri, 23 Oct 2026 19:45:33 +0800 Subject: [PATCH 03/18] Deduplicate hosted method documentation using existing templates --- docs/provider-contract.md | 176 +--- harness/web_fetch/data/report-methods.json | 911 +++++---------------- 2 files changed, 250 insertions(+), 837 deletions(-) diff --git a/docs/provider-contract.md b/docs/provider-contract.md index 990da840c..560932ee1 100644 --- a/docs/provider-contract.md +++ b/docs/provider-contract.md @@ -27,115 +27,45 @@ Use the existing scorer and report renderer. ### Hosted Markdown fetch services -`configs/providers.example.json` includes `brightdata`, `browser-use-fetch`, -`browserbase`, `deepseek-fetch-api` and `crawl4ai-cloud`. These adapters use `providers/hosted_fetch.py`, the shared HTTP client, -and the existing runner/scorer. They support webpage body/structure tasks only; -file and resource tasks fail explicitly without making a provider call. +Five methods share `providers/hosted_fetch.py` and the existing HTTP client, +runner and scorer. Their credentials and fixed options are declared in +`configs/providers.example.json`; credentials come from the repository `.env`. -| Method | Credentials in repository `.env` | Selected route | +| Method | Credentials | Native route and example options | | --- | --- | --- | -| Bright Data | `BRIGHTDATA_API_TOKEN`, `BRIGHTDATA_UNLOCKER_ZONE` | Web Unlocker `/request`, `format=raw`, `data_format=markdown`, native `render="true"` in the example configuration | -| Browser Use | `BROWSER_USE_API_KEY` | fetch-use `/fetch`, `output_format=markdown`, new session, US proxy, service retry defaults preserved; no Agent | -| Browserbase | `BROWSERBASE_API_KEY` | `/v1/fetch`, `format=markdown`, redirects enabled, native proxies enabled by default; no Project ID required | -| DeepSeek Fetch API | `DEEPSEEK_API_KEY` | `/v1/tools/fetch`, request body contains only `url`; title and Markdown content are preserved separately | - -DeepSeek Fetch exposes no documented render, wait, retry or debug request option. -Its example configuration enables the caller's observed HTTP log, not invented -service internals. Actual exceptions include their runtime traceback; deadline -events are recorded when the caller enforces its answer cutoff. The API HTTP -status is not the target website's status. The complete response envelope is -retained even for failed API requests. - -For connectivity/capability checks while the scorer is being revised, use the -existing `run` command with `--collect-only --runs 1 --max-failure-retries 0`. -This mode retains the same calls, deadline, raw artifacts and logs, but never -invokes scoring or creates a scored report. Its summary has no task pass rate; -references are copied for later use without claiming current qualification. -Collection-only evidence must be explicitly rescored before a scored report can -be built. This diagnostic mode does not certify a benchmark cohort. - -Browserbase Markdown uses Fetch Extract billing and does not execute JavaScript. -Bright Data supports a native `render` flag on the same Web Unlocker endpoint. -The adapter accepts an optional Boolean `render` configuration and sends the -documented string `"true"`/`"false"`; omission preserves provider selection. -The example enables it uniformly before calls, without caller-supplied selectors, -extra waiting or browser code. Rendering may increase latency and does not -guarantee that Markdown excludes unrelated page sections. Browser Use's fetch-use -SDK exposes no render/wait selector option; its guide directs JavaScript-dependent -tasks to the separate browser product. Do not silently replace either Fetch -method with a browser session or Agent and keep the same method identity. -Native output alternatives can be declared separately: `output_format=structured` -for fetch-use, or `output_format=json` with a JSON Schema `schema` for Browserbase. -The latter uses the service's own extraction; the adapter only serializes the -returned object, never fills missing fields. Freeze schemas before calls, without -reference values, answer counts or source scripts. These are separate output -contracts from the default Markdown comparison; do not silently substitute a -later alternative for an earlier failed response. Record the selected format in -each result. A missing JavaScript renderer alone does not prove a task impossible: -embedded HTML data may be extracted through another documented native route. -Bright Data manages unlocking/retries internally; its raw response does not -expose the target HTTP status, so that field stays unavailable. The API HTTP -status is recorded separately for these hosted methods. Complete response envelopes -are retained as private hash-bound evidence, separate from score-bearing text. -Empty, malformed, binary and failed target responses cannot become successful -page retrievals merely because the service returned HTTP 200. - -The example configurations enable `log_level: debug` for these hosted methods. -This activates the shared caller log: timestamped request parameters (with API -credentials redacted), actual HTTP response headers/body, and completion/errors. -The runner retains these private logs separately from scored answers. This is -observed client traffic, not a trace of undisclosed service internals. Bright Data -also receives its documented native `debug: true` option; any returned -`x-brd-debug` and error headers are retained verbatim in the HTTP log. Browser Use -Fetch and Browserbase Fetch have no documented equivalent in the inspected Fetch -contracts; no invented debug flag or browser-session fallback is sent. Logs can -identify exposed request IDs, API failures and returned content, but cannot prove -an internal rendering or retry cause that the service does not expose. Previously -collected runs without this setting have no retroactive verbose log. - -These adapters use the full caller timeout (30 seconds in the CLI by default), -without reserving 10 percent. The outer answer deadline includes API connection, -transfer and answer decoding; scoring and report generation run afterward. -Browser Use receives that same timeout through its native `timeout_ms` parameter; -the other APIs keep their service-side defaults and are bounded by the caller. -The shared runner allows one declared failure retry by default. Each physical -call gets its own 30-second deadline; the final response is authoritative and all -physical calls and cumulative execution cost are retained. The adapters add no -hidden caller retries. Service-internal retries are not overridden; their actual count -is unknown unless exposed by the service. A successful retry inside the service -still has to return within the same outer deadline. - -No adapter applies task-specific retrieval hints or silently switches to a -browser or Agent. No caller code fetches the target URL, alternate URLs or data -endpoints to complete an answer. Product-native redirect handling, Markdown -conversion, unlocking and configured proxies are allowed. Freeze one configuration -for the whole declared scope before calls; do not tune individual tasks from -reference answers or observed failures. A diagnostic run can therefore expose -dynamic-page limitations. -Select a predeclared task scope with the existing CLI, for example: - -```bash -uv run --frozen browser-harness-eval run \ - --dataset /absolute/path/to/dataset.json \ - --provider-config configs/providers.example.json \ - --providers brightdata,browser-use-fetch,browserbase \ - --case-ids case-a,case-b --provider-concurrency 1 --max-failure-retries 0 -``` - -`dns_mode` defaults to `system`; the example configurations use the local system -resolver without extra public-DNS queries or automatic DNS fallback. Only API -hostnames are resolved by the caller; target-site DNS stays with each service. -An explicitly declared `public_doh` setting -uses the existing thread-scoped DNS helper, retaining HTTPS hostname/certificate -verification and DNS observations. It does not change system settings or add a -retrieval fallback. Apply the same DNS policy to compared methods and include -its overhead in observed latency. Historical runs preserve their original DNS, -timeout and proxy configuration; changing defaults does not update their scores -or make their results measurements of the new configuration. - -API contracts: [Bright Data request and render parameter](https://docs.brightdata.com/api-reference/rest-api/unlocker/unlock-website), +| Bright Data | `BRIGHTDATA_API_TOKEN`, `BRIGHTDATA_UNLOCKER_ZONE` | `/request`, raw Markdown, rendering and native debug enabled | +| Browser Use Fetch | `BROWSER_USE_API_KEY` | `/fetch`, Markdown, fresh session, US proxy; no Agent | +| Browserbase Fetch | `BROWSERBASE_API_KEY` | `/v1/fetch`, Markdown, redirects and native proxy enabled; no Project ID | +| DeepSeek Fetch API | `DEEPSEEK_API_KEY` | `/v1/tools/fetch`, URL-only request; retain content and title | +| Crawl4AI Cloud | `CRAWL4AI_CLOUD_API_KEY` | `/scrape`, `format=md`, `engine=browser`, `proxy=none` | + +These adapters implement webpage body/structure tasks. Unsupported document, +media and file tasks fail before transport, without helper downloads or a +substitute browser/Agent. Rendering and internal retries remain service-owned; +offline adapter checks do not establish JavaScript capability or task pass rates. +Alternative structured formats require a separately declared output contract; +never derive schemas or parameters from expected answers. + +Retain complete native responses separately from score-bearing answer fields. +An API HTTP 200 does not prove target retrieval succeeded: explicit service or +origin errors, empty content and malformed responses remain failures. API status +is distinct from target-site status. `log_level: debug` saves redacted caller +requests, response headers/body and errors; it cannot expose undisclosed service +internals. Only Bright Data receives the native debug option. + +`dns_mode: system` uses the local resolver for API hosts; target-site DNS belongs +to the service. No automatic DNS fallback is added. Browser Use receives the call +budget through `timeout_ms`; the other services retain their native defaults +within the outer deadline. Historical runs keep their recorded configuration. + +For diagnostics without scoring, use `run --collect-only --runs 1 +--max-failure-retries 0`. It preserves responses and logs, but cannot produce a +scored report without explicit rescoring and does not certify a cohort. + +API references: [Bright Data](https://docs.brightdata.com/api-reference/rest-api/unlocker/unlock-website), [fetch-use](https://browser-use.com/posts/fetch-use), -[Browserbase Fetch](https://docs.browserbase.com/platform/fetch/overview). +[Browserbase Fetch](https://docs.browserbase.com/platform/fetch/overview), +[Crawl4AI Cloud](https://api.crawl4ai.com/docs/). | Output contract | Comparison scope | | --- | --- | @@ -206,32 +136,14 @@ rendered-page condition was satisfied while fetching only response bytes. ## Crawl4AI Cloud and the local library -`crawl4ai-cloud` uses the shared hosted adapter with `CRAWL4AI_CLOUD_API_KEY` -from the repository `.env`. It is a separate method identity from `crawl4ai`; -local measurements must not be relabeled as cloud measurements. -The example configuration selects `POST /scrape` with `format=md`, -`engine=browser`, and `proxy=none`. Browser mode requests a fresh browser fetch -instead of the cloud's automatic cache/HTTP/browser routing. No country is forced. -Caller DNS is `system`; this does not configure the service's target-site DNS. -The call budget is a run parameter, defaulting to 30 seconds per physical call. -The shared runner permits one failure retry by default, with a fresh call budget. -Internal service retries remain service-owned. Historical 45-second calls remain unchanged. - -The local worker uses headless Chrome, cache bypass, `domcontentloaded`, -overlay removal, iframe processing, and `raw_markdown`. Cloud exposes no exact -counterparts for those wait/extraction settings and returns service-cleaned -`markdown`; do not claim identical browser versions, output, or site access. -Only webpage body/structure tasks are implemented by this adapter; PDF/media/file -support needs its own declared and tested contract before use. - -Retain the complete native JSON, including its engine/cache/usage fields, alongside -unchanged Markdown and caller debug logs. No undocumented native debug flag is -sent. An API HTTP 200 alone is insufficient: an explicit error/false success, -invalid Markdown field, empty content, or explicit origin failure is a failure. -Cloud 403 is access denial, not necessarily invalid credentials: documented -causes also include target-site blocks. Native browser traces are not promised. - -Official contract: https://api.crawl4ai.com/docs/ (checked 2026-10-23). +`crawl4ai-cloud` and local `crawl4ai` are separate methods: old local results +cannot be relabeled as Cloud results. The local worker uses headless Chrome, +cache bypass, `domcontentloaded`, overlay removal, iframe processing and +`raw_markdown`. Cloud returns service-cleaned Markdown with no exact counterpart +for every local option. Browser versions, extraction and site access may differ. +Retain Cloud engine/cache/usage fields in the raw response. A Cloud 403 can mean +target access denial, not only invalid credentials; native browser traces are +not promised. ### Selected fixed-native comparison policy diff --git a/harness/web_fetch/data/report-methods.json b/harness/web_fetch/data/report-methods.json index e74dc837f..b04bcaac2 100644 --- a/harness/web_fetch/data/report-methods.json +++ b/harness/web_fetch/data/report-methods.json @@ -5261,14 +5261,13 @@ "status": "implemented" }, "browserbase": { + "extends": "hosted_fetch", "label": { "zh": "Browserbase Fetch", "en": "Browserbase Fetch" }, - "status": "implemented", "source_url": "https://www.browserbase.com/sign-up", "access": { - "type": "hosted", "env_vars": [ "BROWSERBASE_API_KEY" ], @@ -5280,51 +5279,14 @@ }, "https://www.browserbase.com/overview" ] - ], - "account": { - "zh": "需要服务账号及可用额度。", - "en": "Requires a service account and available quota." - }, - "configuration": { - "zh": "在本地 .env 中配置所列变量。", - "en": "Set the listed variables in the local .env file." - }, - "required": true, - "priority": { - "zh": "B · 已知 URL 抓取", - "en": "B · Known-URL fetch" - } + ] }, - "kind": "adapter", "adapter_types": [ "browserbase_fetch" ], - "profile": { - "kind": { - "zh": "托管 API", - "en": "Managed API" - }, - "description": { - "zh": "通过原生接口获取网页文字，保存原始响应和调用日志。", - "en": "Retrieve webpage text through the native API, retaining raw responses and call logs." - } - }, "interface": { "entry": "POST https://api.browserbase.com/v1/fetch", "auth": "X-BB-API-Key: ", - "params": [ - [ - "url", - { - "zh": "string · 必填", - "en": "string · required" - }, - { - "zh": "本题提供的网址", - "en": "The task URL" - } - ] - ], "fields": [ [ "content", @@ -5336,100 +5298,11 @@ ] ], "request": { - "url": "https://example.com/", "format": "markdown" - }, - "source": "harness/web_fetch/providers/hosted_fetch.py", - "notes": { - "zh": "这是接口示例，不代表历史调用参数。渲染、代理、引擎、输出格式和时间预算以保存的配置及请求日志为准。保存完整原始响应，不补抓或补写正文；调用端日志不等于服务内部日志。", - "en": "This interface example does not establish historical call parameters. Rendering, proxy, engine, output format and time budget come from saved configuration and request logs. Preserve the complete response without supplemental retrieval or content repair; caller logs are not service-internal traces." } }, "capabilities": { - "scope": "POST https://api.browserbase.com/v1/fetch", - "sources": [ - "hosted-fetch-adapter" - ], - "web": { - "status": { - "zh": "有限支持", - "en": "Limited support" - }, - "detail": { - "zh": "当前接入支持网页正文和结构任务；不代表实测通过率。", - "en": "The adapter supports webpage body and structure tasks; this is not a measured pass rate." - } - }, - "input": { - "zh": "已知网址；使用预先固定的原生接口参数。", - "en": "Known URL with native interface parameters fixed before calls." - }, - "structure": { - "zh": "保留原始响应，仅按声明字段读取正文；不补抓内容。", - "en": "Retain raw responses and read declared answer fields without supplemental fetching." - }, - "access": { - "zh": "需要服务账号及可用额度。", - "en": "Requires a service account and available quota." - }, - "notes": { - "zh": "这里登记的是当前接入范围，不推断服务商其他产品的能力。", - "en": "This describes the current adapter, not other products from the provider." - }, - "tested": { - "zh": "接入已通过离线模拟响应检查；不代表完成同范围正式评测。", - "en": "Adapter checked with offline simulated responses; no same-scope formal benchmark is implied." - }, - "document": { - "status": { - "zh": "未声明", - "en": "Undeclared" - }, - "detail": { - "zh": "当前接入未实现此类任务，不提供调用端补救。", - "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." - } - }, - "image": { - "status": { - "zh": "未声明", - "en": "Undeclared" - }, - "detail": { - "zh": "当前接入未实现此类任务，不提供调用端补救。", - "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." - } - }, - "av": { - "status": { - "zh": "未声明", - "en": "Undeclared" - }, - "detail": { - "zh": "当前接入未实现此类任务，不提供调用端补救。", - "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." - } - }, - "file": { - "status": { - "zh": "未声明", - "en": "Undeclared" - }, - "detail": { - "zh": "当前接入未实现此类任务，不提供调用端补救。", - "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." - } - }, - "js": { - "status": { - "zh": "未声明", - "en": "Undeclared" - }, - "detail": { - "zh": "渲染行为由原生服务及所选配置决定；未据离线检查确认页面执行能力。", - "en": "Rendering depends on the native service and selected configuration; offline checks do not establish page execution capability." - } - } + "scope": "POST https://api.browserbase.com/v1/fetch" } }, "bocha-web-search": { @@ -5627,14 +5500,13 @@ } }, "brightdata": { + "extends": "hosted_fetch", "label": { "zh": "Bright Data Web Unlocker", "en": "Bright Data Web Unlocker" }, - "status": "implemented", "source_url": "https://brightdata.com/cp/start", "access": { - "type": "hosted", "env_vars": [ "BRIGHTDATA_API_TOKEN", "BRIGHTDATA_UNLOCKER_ZONE" @@ -5647,51 +5519,14 @@ }, "https://brightdata.com/cp/zones" ] - ], - "account": { - "zh": "需要服务账号及可用额度。", - "en": "Requires a service account and available quota." - }, - "configuration": { - "zh": "在本地 .env 中配置所列变量。", - "en": "Set the listed variables in the local .env file." - }, - "required": true, - "priority": { - "zh": "B · 已知 URL 抓取", - "en": "B · Known-URL fetch" - } + ] }, - "kind": "adapter", "adapter_types": [ "brightdata_unlocker" ], - "profile": { - "kind": { - "zh": "托管 API", - "en": "Managed API" - }, - "description": { - "zh": "通过原生接口获取网页文字，保存原始响应和调用日志。", - "en": "Retrieve webpage text through the native API, retaining raw responses and call logs." - } - }, "interface": { "entry": "POST https://api.brightdata.com/request", "auth": "Authorization: Bearer ", - "params": [ - [ - "url", - { - "zh": "string · 必填", - "en": "string · required" - }, - { - "zh": "本题提供的网址", - "en": "The task URL" - } - ] - ], "fields": [ [ "HTTP response body", @@ -5703,139 +5538,50 @@ ] ], "request": { - "url": "https://example.com/", "zone": "", "format": "raw", "data_format": "markdown" - }, - "source": "harness/web_fetch/providers/hosted_fetch.py", - "notes": { - "zh": "这是接口示例，不代表历史调用参数。渲染、代理、引擎、输出格式和时间预算以保存的配置及请求日志为准。保存完整原始响应，不补抓或补写正文；调用端日志不等于服务内部日志。", - "en": "This interface example does not establish historical call parameters. Rendering, proxy, engine, output format and time budget come from saved configuration and request logs. Preserve the complete response without supplemental retrieval or content repair; caller logs are not service-internal traces." } }, "capabilities": { - "scope": "POST https://api.brightdata.com/request", - "sources": [ - "hosted-fetch-adapter" + "scope": "POST https://api.brightdata.com/request" + } + }, + "scrapingbee": { + "label": { + "zh": "ScrapingBee", + "en": "ScrapingBee" + }, + "status": "planned", + "source_url": "https://app.scrapingbee.com/account/register", + "access": { + "type": "hosted", + "env_vars": [ + "SCRAPINGBEE_API_KEY" ], - "web": { - "status": { - "zh": "有限支持", - "en": "Limited support" - }, - "detail": { - "zh": "当前接入支持网页正文和结构任务；不代表实测通过率。", - "en": "The adapter supports webpage body and structure tasks; this is not a measured pass rate." - } - }, - "input": { - "zh": "已知网址；使用预先固定的原生接口参数。", - "en": "Known URL with native interface parameters fixed before calls." - }, - "structure": { - "zh": "保留原始响应，仅按声明字段读取正文；不补抓内容。", - "en": "Retain raw responses and read declared answer fields without supplemental fetching." - }, - "access": { - "zh": "需要服务账号及可用额度。", - "en": "Requires a service account and available quota." - }, - "notes": { - "zh": "这里登记的是当前接入范围，不推断服务商其他产品的能力。", - "en": "This describes the current adapter, not other products from the provider." - }, - "tested": { - "zh": "接入已通过离线模拟响应检查；不代表完成同范围正式评测。", - "en": "Adapter checked with offline simulated responses; no same-scope formal benchmark is implied." + "links": [ + [ + { + "zh": "注册 / 登录", + "en": "Register / sign in" + }, + "https://app.scrapingbee.com/account/register" + ], + [ + { + "zh": "获取凭据", + "en": "Get credentials" + }, + "https://app.scrapingbee.com/account/manage/api_key" + ] + ], + "account": { + "zh": "受试用/账户限额约束；生产套餐有月度最低消费。", + "en": "Trial and account limits apply; production plans have a monthly minimum charge." }, - "document": { - "status": { - "zh": "未声明", - "en": "Undeclared" - }, - "detail": { - "zh": "当前接入未实现此类任务，不提供调用端补救。", - "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." - } - }, - "image": { - "status": { - "zh": "未声明", - "en": "Undeclared" - }, - "detail": { - "zh": "当前接入未实现此类任务，不提供调用端补救。", - "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." - } - }, - "av": { - "status": { - "zh": "未声明", - "en": "Undeclared" - }, - "detail": { - "zh": "当前接入未实现此类任务，不提供调用端补救。", - "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." - } - }, - "file": { - "status": { - "zh": "未声明", - "en": "Undeclared" - }, - "detail": { - "zh": "当前接入未实现此类任务，不提供调用端补救。", - "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." - } - }, - "js": { - "status": { - "zh": "未声明", - "en": "Undeclared" - }, - "detail": { - "zh": "渲染行为由原生服务及所选配置决定；未据离线检查确认页面执行能力。", - "en": "Rendering depends on the native service and selected configuration; offline checks do not establish page execution capability." - } - } - } - }, - "scrapingbee": { - "label": { - "zh": "ScrapingBee", - "en": "ScrapingBee" - }, - "status": "planned", - "source_url": "https://app.scrapingbee.com/account/register", - "access": { - "type": "hosted", - "env_vars": [ - "SCRAPINGBEE_API_KEY" - ], - "links": [ - [ - { - "zh": "注册 / 登录", - "en": "Register / sign in" - }, - "https://app.scrapingbee.com/account/register" - ], - [ - { - "zh": "获取凭据", - "en": "Get credentials" - }, - "https://app.scrapingbee.com/account/manage/api_key" - ] - ], - "account": { - "zh": "受试用/账户限额约束；生产套餐有月度最低消费。", - "en": "Trial and account limits apply; production plans have a monthly minimum charge." - }, - "configuration": { - "zh": "注册后打开 Account → API Key，复制 key。", - "en": "Register and copy the key from Account → API Key." + "configuration": { + "zh": "注册后打开 Account → API Key，复制 key。", + "en": "Register and copy the key from Account → API Key." }, "required": true, "priority": { @@ -6011,26 +5757,19 @@ } }, "browser-use-fetch": { + "extends": "hosted_fetch", "label": { "zh": "Browser Use Fetch", "en": "Browser Use Fetch" }, "source_url": "https://cloud.browser-use.com/settings?tab=api-keys", - "status": "implemented", - "kind": "adapter", "adapter_types": [ "browser_use_fetch" ], "access": { - "type": "hosted", "env_vars": [ "BROWSER_USE_API_KEY" ], - "required": true, - "priority": { - "zh": "B · 已知 URL 抓取", - "en": "B · Known-URL fetch" - }, "links": [ [ { @@ -6039,42 +5778,11 @@ }, "https://cloud.browser-use.com/settings?tab=api-keys" ] - ], - "configuration": { - "zh": "在本地 .env 中配置所列变量。", - "en": "Set the listed variables in the local .env file." - }, - "account": { - "zh": "需要服务账号及可用额度。", - "en": "Requires a service account and available quota." - } - }, - "profile": { - "kind": { - "zh": "托管 API", - "en": "Managed API" - }, - "description": { - "zh": "通过原生接口获取网页文字，保存原始响应和调用日志。", - "en": "Retrieve webpage text through the native API, retaining raw responses and call logs." - } + ] }, "interface": { "entry": "POST https://fetch.browser-use.com/fetch", "auth": "X-Browser-Use-API-Key: ", - "params": [ - [ - "url", - { - "zh": "string · 必填", - "en": "string · required" - }, - { - "zh": "本题提供的网址", - "en": "The task URL" - } - ] - ], "fields": [ [ "body", @@ -6086,123 +5794,27 @@ ] ], "request": { - "url": "https://example.com/", "output_format": "markdown" - }, - "source": "harness/web_fetch/providers/hosted_fetch.py", - "notes": { - "zh": "这是接口示例，不代表历史调用参数。渲染、代理、引擎、输出格式和时间预算以保存的配置及请求日志为准。保存完整原始响应，不补抓或补写正文；调用端日志不等于服务内部日志。", - "en": "This interface example does not establish historical call parameters. Rendering, proxy, engine, output format and time budget come from saved configuration and request logs. Preserve the complete response without supplemental retrieval or content repair; caller logs are not service-internal traces." } }, "capabilities": { - "scope": "POST https://fetch.browser-use.com/fetch", - "sources": [ - "hosted-fetch-adapter" - ], - "web": { - "status": { - "zh": "有限支持", - "en": "Limited support" - }, - "detail": { - "zh": "当前接入支持网页正文和结构任务；不代表实测通过率。", - "en": "The adapter supports webpage body and structure tasks; this is not a measured pass rate." - } - }, - "input": { - "zh": "已知网址；使用预先固定的原生接口参数。", - "en": "Known URL with native interface parameters fixed before calls." - }, - "structure": { - "zh": "保留原始响应，仅按声明字段读取正文；不补抓内容。", - "en": "Retain raw responses and read declared answer fields without supplemental fetching." - }, - "access": { - "zh": "需要服务账号及可用额度。", - "en": "Requires a service account and available quota." - }, - "notes": { - "zh": "这里登记的是当前接入范围，不推断服务商其他产品的能力。", - "en": "This describes the current adapter, not other products from the provider." - }, - "tested": { - "zh": "接入已通过离线模拟响应检查；不代表完成同范围正式评测。", - "en": "Adapter checked with offline simulated responses; no same-scope formal benchmark is implied." - }, - "document": { - "status": { - "zh": "未声明", - "en": "Undeclared" - }, - "detail": { - "zh": "当前接入未实现此类任务，不提供调用端补救。", - "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." - } - }, - "image": { - "status": { - "zh": "未声明", - "en": "Undeclared" - }, - "detail": { - "zh": "当前接入未实现此类任务，不提供调用端补救。", - "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." - } - }, - "av": { - "status": { - "zh": "未声明", - "en": "Undeclared" - }, - "detail": { - "zh": "当前接入未实现此类任务，不提供调用端补救。", - "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." - } - }, - "file": { - "status": { - "zh": "未声明", - "en": "Undeclared" - }, - "detail": { - "zh": "当前接入未实现此类任务，不提供调用端补救。", - "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." - } - }, - "js": { - "status": { - "zh": "未声明", - "en": "Undeclared" - }, - "detail": { - "zh": "渲染行为由原生服务及所选配置决定；未据离线检查确认页面执行能力。", - "en": "Rendering depends on the native service and selected configuration; offline checks do not establish page execution capability." - } - } + "scope": "POST https://fetch.browser-use.com/fetch" } }, "deepseek-fetch-api": { + "extends": "hosted_fetch", "label": { "zh": "DeepSeek Fetch API", "en": "DeepSeek Fetch API" }, "source_url": "https://platform.deepseek.com/api_keys", - "status": "implemented", - "kind": "adapter", "adapter_types": [ "deepseek_fetch" ], "access": { - "type": "hosted", "env_vars": [ "DEEPSEEK_API_KEY" ], - "required": true, - "priority": { - "zh": "B · 已知 URL 抓取", - "en": "B · Known-URL fetch" - }, "links": [ [ { @@ -6211,42 +5823,11 @@ }, "https://platform.deepseek.com/api_keys" ] - ], - "configuration": { - "zh": "在本地 .env 中配置所列变量。", - "en": "Set the listed variables in the local .env file." - }, - "account": { - "zh": "需要服务账号及可用额度。", - "en": "Requires a service account and available quota." - } - }, - "profile": { - "kind": { - "zh": "托管 API", - "en": "Managed API" - }, - "description": { - "zh": "通过原生接口获取网页文字，保存原始响应和调用日志。", - "en": "Retrieve webpage text through the native API, retaining raw responses and call logs." - } + ] }, "interface": { "entry": "POST https://api.deepseek.com/v1/tools/fetch", "auth": "Authorization: Bearer ", - "params": [ - [ - "url", - { - "zh": "string · 必填", - "en": "string · required" - }, - { - "zh": "本题提供的网址", - "en": "The task URL" - } - ] - ], "fields": [ [ "content; title", @@ -6256,168 +5837,39 @@ "en": "Native returned text; never completed from reference answers" } ] - ], - "request": { - "url": "https://example.com/" - }, - "source": "harness/web_fetch/providers/hosted_fetch.py", - "notes": { - "zh": "这是接口示例，不代表历史调用参数。渲染、代理、引擎、输出格式和时间预算以保存的配置及请求日志为准。保存完整原始响应，不补抓或补写正文；调用端日志不等于服务内部日志。", - "en": "This interface example does not establish historical call parameters. Rendering, proxy, engine, output format and time budget come from saved configuration and request logs. Preserve the complete response without supplemental retrieval or content repair; caller logs are not service-internal traces." - } + ] }, "capabilities": { - "scope": "POST https://api.deepseek.com/v1/tools/fetch", - "sources": [ - "hosted-fetch-adapter" - ], - "web": { - "status": { - "zh": "有限支持", - "en": "Limited support" - }, - "detail": { - "zh": "当前接入支持网页正文和结构任务；不代表实测通过率。", - "en": "The adapter supports webpage body and structure tasks; this is not a measured pass rate." - } - }, - "input": { - "zh": "已知网址；使用预先固定的原生接口参数。", - "en": "Known URL with native interface parameters fixed before calls." - }, - "structure": { - "zh": "保留原始响应，仅按声明字段读取正文；不补抓内容。", - "en": "Retain raw responses and read declared answer fields without supplemental fetching." - }, - "access": { - "zh": "需要服务账号及可用额度。", - "en": "Requires a service account and available quota." - }, - "notes": { - "zh": "这里登记的是当前接入范围，不推断服务商其他产品的能力。", - "en": "This describes the current adapter, not other products from the provider." - }, - "tested": { - "zh": "接入已通过离线模拟响应检查；不代表完成同范围正式评测。", - "en": "Adapter checked with offline simulated responses; no same-scope formal benchmark is implied." - }, - "document": { - "status": { - "zh": "未声明", - "en": "Undeclared" - }, - "detail": { - "zh": "当前接入未实现此类任务，不提供调用端补救。", - "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." - } - }, - "image": { - "status": { - "zh": "未声明", - "en": "Undeclared" - }, - "detail": { - "zh": "当前接入未实现此类任务，不提供调用端补救。", - "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." - } - }, - "av": { - "status": { - "zh": "未声明", - "en": "Undeclared" - }, - "detail": { - "zh": "当前接入未实现此类任务，不提供调用端补救。", - "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." - } - }, - "file": { - "status": { - "zh": "未声明", - "en": "Undeclared" - }, - "detail": { - "zh": "当前接入未实现此类任务，不提供调用端补救。", - "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." - } - }, - "js": { - "status": { - "zh": "未声明", - "en": "Undeclared" - }, - "detail": { - "zh": "渲染行为由原生服务及所选配置决定；未据离线检查确认页面执行能力。", - "en": "Rendering depends on the native service and selected configuration; offline checks do not establish page execution capability." - } - } + "scope": "POST https://api.deepseek.com/v1/tools/fetch" } }, "crawl4ai-cloud": { + "extends": "hosted_fetch", "label": { "zh": "Crawl4AI Cloud", - "en": "Crawl4AI Cloud" - }, - "source_url": "https://api.crawl4ai.com/docs/", - "status": "implemented", - "kind": "adapter", - "adapter_types": [ - "crawl4ai_cloud" - ], - "access": { - "type": "hosted", - "env_vars": [ - "CRAWL4AI_CLOUD_API_KEY" - ], - "required": true, - "priority": { - "zh": "B · 已知 URL 抓取", - "en": "B · Known-URL fetch" - }, - "links": [ - [ - { - "zh": "获取凭据", - "en": "Get credentials" - }, - "https://crawl4ai.com/dashboard/" - ] - ], - "configuration": { - "zh": "在本地 .env 中配置所列变量。", - "en": "Set the listed variables in the local .env file." - }, - "account": { - "zh": "需要服务账号及可用额度。", - "en": "Requires a service account and available quota." - } - }, - "profile": { - "kind": { - "zh": "托管 API", - "en": "Managed API" - }, - "description": { - "zh": "通过原生接口获取网页文字，保存原始响应和调用日志。", - "en": "Retrieve webpage text through the native API, retaining raw responses and call logs." - } + "en": "Crawl4AI Cloud" }, - "interface": { - "entry": "POST https://api.crawl4ai.com/scrape", - "auth": "Authorization: Bearer ", - "params": [ + "source_url": "https://api.crawl4ai.com/docs/", + "adapter_types": [ + "crawl4ai_cloud" + ], + "access": { + "env_vars": [ + "CRAWL4AI_CLOUD_API_KEY" + ], + "links": [ [ - "url", { - "zh": "string · 必填", - "en": "string · required" + "zh": "获取凭据", + "en": "Get credentials" }, - { - "zh": "本题提供的网址", - "en": "The task URL" - } + "https://crawl4ai.com/dashboard/" ] - ], + ] + }, + "interface": { + "entry": "POST https://api.crawl4ai.com/scrape", + "auth": "Authorization: Bearer ", "fields": [ [ "markdown", @@ -6429,100 +5881,11 @@ ] ], "request": { - "url": "https://example.com/", "format": "md" - }, - "source": "harness/web_fetch/providers/hosted_fetch.py", - "notes": { - "zh": "这是接口示例，不代表历史调用参数。渲染、代理、引擎、输出格式和时间预算以保存的配置及请求日志为准。保存完整原始响应，不补抓或补写正文；调用端日志不等于服务内部日志。", - "en": "This interface example does not establish historical call parameters. Rendering, proxy, engine, output format and time budget come from saved configuration and request logs. Preserve the complete response without supplemental retrieval or content repair; caller logs are not service-internal traces." } }, "capabilities": { - "scope": "POST https://api.crawl4ai.com/scrape", - "sources": [ - "hosted-fetch-adapter" - ], - "web": { - "status": { - "zh": "有限支持", - "en": "Limited support" - }, - "detail": { - "zh": "当前接入支持网页正文和结构任务；不代表实测通过率。", - "en": "The adapter supports webpage body and structure tasks; this is not a measured pass rate." - } - }, - "input": { - "zh": "已知网址；使用预先固定的原生接口参数。", - "en": "Known URL with native interface parameters fixed before calls." - }, - "structure": { - "zh": "保留原始响应，仅按声明字段读取正文；不补抓内容。", - "en": "Retain raw responses and read declared answer fields without supplemental fetching." - }, - "access": { - "zh": "需要服务账号及可用额度。", - "en": "Requires a service account and available quota." - }, - "notes": { - "zh": "这里登记的是当前接入范围，不推断服务商其他产品的能力。", - "en": "This describes the current adapter, not other products from the provider." - }, - "tested": { - "zh": "接入已通过离线模拟响应检查；不代表完成同范围正式评测。", - "en": "Adapter checked with offline simulated responses; no same-scope formal benchmark is implied." - }, - "document": { - "status": { - "zh": "未声明", - "en": "Undeclared" - }, - "detail": { - "zh": "当前接入未实现此类任务，不提供调用端补救。", - "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." - } - }, - "image": { - "status": { - "zh": "未声明", - "en": "Undeclared" - }, - "detail": { - "zh": "当前接入未实现此类任务，不提供调用端补救。", - "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." - } - }, - "av": { - "status": { - "zh": "未声明", - "en": "Undeclared" - }, - "detail": { - "zh": "当前接入未实现此类任务，不提供调用端补救。", - "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." - } - }, - "file": { - "status": { - "zh": "未声明", - "en": "Undeclared" - }, - "detail": { - "zh": "当前接入未实现此类任务，不提供调用端补救。", - "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." - } - }, - "js": { - "status": { - "zh": "未声明", - "en": "Undeclared" - }, - "detail": { - "zh": "渲染行为由原生服务及所选配置决定；未据离线检查确认页面执行能力。", - "en": "Rendering depends on the native service and selected configuration; offline checks do not establish page execution capability." - } - } + "scope": "POST https://api.crawl4ai.com/scrape" } } }, @@ -6964,6 +6327,144 @@ "en": "The browser handles access and native output retrieval. Embedded media and text use native Markdown; the shared scorer only checks target resource URLs. See the interpretation interface table for the full scope and boundaries." } } + }, + "hosted_fetch": { + "status": "implemented", + "access": { + "type": "hosted", + "account": { + "zh": "需要服务账号及可用额度。", + "en": "Requires a service account and available quota." + }, + "configuration": { + "zh": "在本地 .env 中配置所列变量。", + "en": "Set the listed variables in the local .env file." + }, + "required": true, + "priority": { + "zh": "B · 已知 URL 抓取", + "en": "B · Known-URL fetch" + } + }, + "kind": "adapter", + "profile": { + "kind": { + "zh": "托管 API", + "en": "Managed API" + }, + "description": { + "zh": "通过原生接口获取网页文字，保存原始响应和调用日志。", + "en": "Retrieve webpage text through the native API, retaining raw responses and call logs." + } + }, + "interface": { + "params": [ + [ + "url", + { + "zh": "string · 必填", + "en": "string · required" + }, + { + "zh": "本题提供的网址", + "en": "The task URL" + } + ] + ], + "request": { + "url": "https://example.com/" + }, + "source": "harness/web_fetch/providers/hosted_fetch.py", + "notes": { + "zh": "这是接口示例，不代表历史调用参数。渲染、代理、引擎、输出格式和时间预算以保存的配置及请求日志为准。保存完整原始响应，不补抓或补写正文；调用端日志不等于服务内部日志。", + "en": "This interface example does not establish historical call parameters. Rendering, proxy, engine, output format and time budget come from saved configuration and request logs. Preserve the complete response without supplemental retrieval or content repair; caller logs are not service-internal traces." + } + }, + "capabilities": { + "sources": [ + "hosted-fetch-adapter" + ], + "web": { + "status": { + "zh": "有限支持", + "en": "Limited support" + }, + "detail": { + "zh": "当前接入支持网页正文和结构任务；不代表实测通过率。", + "en": "The adapter supports webpage body and structure tasks; this is not a measured pass rate." + } + }, + "input": { + "zh": "已知网址；使用预先固定的原生接口参数。", + "en": "Known URL with native interface parameters fixed before calls." + }, + "structure": { + "zh": "保留原始响应，仅按声明字段读取正文；不补抓内容。", + "en": "Retain raw responses and read declared answer fields without supplemental fetching." + }, + "access": { + "zh": "需要服务账号及可用额度。", + "en": "Requires a service account and available quota." + }, + "notes": { + "zh": "这里登记的是当前接入范围，不推断服务商其他产品的能力。", + "en": "This describes the current adapter, not other products from the provider." + }, + "tested": { + "zh": "接入已通过离线模拟响应检查；不代表完成同范围正式评测。", + "en": "Adapter checked with offline simulated responses; no same-scope formal benchmark is implied." + }, + "document": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "image": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "av": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "file": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "当前接入未实现此类任务，不提供调用端补救。", + "en": "This task family is not implemented by the current adapter; no caller fallback is supplied." + } + }, + "js": { + "status": { + "zh": "未声明", + "en": "Undeclared" + }, + "detail": { + "zh": "渲染行为由原生服务及所选配置决定；未据离线检查确认页面执行能力。", + "en": "Rendering depends on the native service and selected configuration; offline checks do not establish page execution capability." + } + } + } } }, "unregistered_access": { From 06ecea14262efe2952bbd8737b708f6ba7869af7 Mon Sep 17 00:00:00 2001 From: Maisiqi <324751762+Maisiqi@users.noreply.github.com> Date: Fri, 23 Oct 2026 19:50:26 +0800 Subject: [PATCH 04/18] Share hosted provider checks across all five adapters --- tests/test_hosted_fetch.py | 70 ++++++++++++++------------------------ 1 file changed, 26 insertions(+), 44 deletions(-) diff --git a/tests/test_hosted_fetch.py b/tests/test_hosted_fetch.py index c16ef4f60..976415b5c 100644 --- a/tests/test_hosted_fetch.py +++ b/tests/test_hosted_fetch.py @@ -13,11 +13,13 @@ from lexbench_webfetch.models import EvalCase from harness.web_fetch.providers import build_provider from harness.web_fetch.providers.http_client import HttpResponse +from harness.web_fetch.providers.hosted_fetch import ENDPOINTS from harness.web_fetch.runner import run_evaluation from lexbench_webfetch.metrics import task_passed -TYPES = ('brightdata_unlocker', 'browser_use_fetch', 'browserbase_fetch') +TYPES = ('brightdata_unlocker', 'browser_use_fetch', 'browserbase_fetch', + 'deepseek_fetch', 'crawl4ai_cloud') class HostedFetchTest(unittest.TestCase): @@ -29,13 +31,14 @@ def provider(self, kind, **config): def response(self, kind, text='Required article text', status=200): payload = ({'body': text, 'status_code': status, 'final_url': self.case.url} - if kind == 'browser_use_fetch' else {'content': text, 'statusCode': status}) + if kind == 'browser_use_fetch' else {'markdown': text, 'origin_status': status} + if kind == 'crawl4ai_cloud' else {'content': text, 'title': 'Title', 'statusCode': status}) body = text.encode() if kind == 'brightdata_unlocker' else json.dumps(payload).encode() return HttpResponse(200, 'https://service.example/', {}, body) @patch('harness.web_fetch.providers.hosted_fetch.request') - def test_crawl4ai_cloud_contract_and_projection(self, call): + def test_crawl4ai_cloud_errors_and_projection(self, call): from harness.web_fetch.native_projection import require_native_answer_projection provider = self.provider('crawl4ai_cloud') self.assertEqual(provider.validate(), []) @@ -55,9 +58,6 @@ def test_crawl4ai_cloud_contract_and_projection(self, call): self.assertEqual(result.error_type,expected) self.assertEqual(base64.b64decode(result.metadata['_raw_response_b64']),raw) call.assert_called_once() - self.assertEqual(call.call_args.kwargs['json_body'], - {'url':self.case.url,'format':'md','engine':'browser','proxy':'none'}) - self.assertEqual(call.call_args.kwargs['timeout'],45) require_native_answer_projection(result.content,result.metadata,raw, title=result.title,final_url=result.final_url,request_url=self.case.url) for option in ({'engine':'invalid'},{'proxy':'invalid'},{'output_format':'html'}): @@ -67,11 +67,9 @@ def test_crawl4ai_cloud_contract_and_projection(self, call): def test_latest_article_track_preserves_native_answer_and_rejects_tampering(self, call): from harness.web_fetch.native_projection import require_native_answer_projection case = EvalCase('article-new', 'Article', 'https://example.com/article', 'article', [], track='articles') - for kind in (*TYPES, 'deepseek_fetch'): + for kind in TYPES: with self.subTest(kind=kind): - response = (HttpResponse(200, 'https://service.example/', {}, - b'{"title":"Title","content":"Required article text"}') - if kind == 'deepseek_fetch' else self.response(kind)) + response = self.response(kind) call.return_value = response result = self.provider(kind).fetch(case, 45) self.assertTrue(result.ok) @@ -86,6 +84,7 @@ def test_latest_article_track_preserves_native_answer_and_rejects_tampering(self def test_routes_and_preserved_evidence(self, call): for kind in TYPES: with self.subTest(kind=kind): + call.reset_mock() response = self.response(kind) call.return_value = response result = self.provider(kind).fetch(self.case, 45) @@ -94,6 +93,7 @@ def test_routes_and_preserved_evidence(self, call): self.assertEqual(base64.b64decode(result.metadata['_raw_response_b64']), response.body) self.assertNotIn('private-test-key', json.dumps(result.metadata)) self.assertNotEqual(method_info(kind)['contract'], '未知') + call.assert_called_once() body = call.call_args.kwargs['json_body'] self.assertEqual(body['url'], self.case.url) self.assertEqual(call.call_args.kwargs['timeout'], 45) @@ -112,10 +112,17 @@ def test_routes_and_preserved_evidence(self, call): session = body['session_id'] self.provider(kind).fetch(self.case, 45) self.assertNotEqual(session, call.call_args.kwargs['json_body']['session_id']) - else: + elif kind == 'browserbase_fetch': self.assertTrue(body['allowRedirects']) self.assertTrue(body['proxies']) self.assertNotIn('projectId', body) + elif kind == 'deepseek_fetch': + self.assertEqual(body, {'url': self.case.url}) + self.assertEqual(result.title, 'Title') + self.assertIsNone(result.status_code) + self.assertEqual(call.call_args.kwargs['headers'], {'Authorization': 'Bearer private-test-key'}) + else: + self.assertEqual(body, {'url': self.case.url, 'format': 'md', 'engine': 'browser', 'proxy': 'none'}) @patch('harness.web_fetch.providers.hosted_fetch.request') def test_brightdata_native_render_option_is_explicit(self, call): @@ -167,7 +174,7 @@ def test_http_200_is_not_enough(self, call): result = self.provider(kind).fetch(self.case, 45) self.assertFalse(result.ok) self.assertEqual(result.error_type, 'empty_content') - for kind in TYPES[1:]: + for kind in TYPES[1:3]: call.return_value = self.response(kind, 'Required article text', 403) self.assertEqual(self.provider(kind).fetch(self.case, 45).error_type, 'http_error') for raw in (b'not json', b'{"message":"Required article text"}', @@ -185,11 +192,13 @@ def test_http_200_is_not_enough(self, call): def test_api_errors_are_separate_from_content(self, call): for kind in TYPES: for status, label in ((401,'authentication'), (402,'quota'), (429,'rate_limit'), (500,'api_error')): - call.return_value = HttpResponse(status, 'https://service.example/', {}, b'{"error":"Required article text"}') + call.return_value = HttpResponse(status, 'https://service.example/', {}, b'{"error":{"message":"actual error"}}') result = self.provider(kind).fetch(self.case, 45) self.assertFalse(result.ok) self.assertEqual(result.content, '') self.assertEqual(result.error_type, label) + if kind == 'deepseek_fetch': + self.assertEqual(result.metadata['service_error']['message'], 'actual error') @patch('harness.web_fetch.providers.hosted_fetch.request') def test_unsupported_delivery_never_calls_service(self, call): @@ -218,15 +227,9 @@ def test_single_request_without_dns_fallback_or_task_assistance(self, lookup, ca lookup.assert_not_called() @patch('harness.web_fetch.providers.http_client.urllib.request.build_opener') - def test_runner_scores_and_saves_all_three_methods(self, call): + def test_runner_scores_and_saves_hosted_methods(self, call): def respond(request, **kwargs): - url = request.full_url - if 'brightdata' in url: - kind = TYPES[0] - elif 'browser-use' in url: - kind = TYPES[1] - else: - kind = TYPES[2] + kind = next(kind for kind in TYPES if ENDPOINTS[kind] == request.full_url) response = self.response(kind) return Mock(status=response.status, headers={'x-brd-debug':'req_id=test-native-id; render=true'}, read=Mock(return_value=response.body), geturl=Mock(return_value=response.url)) @@ -252,7 +255,7 @@ def respond(request, **kwargs): records, _ = run_evaluation(output_dir=root/'run', dataset_meta=dataset, dataset_path=dataset_path, cases=[self.case], providers=[self.provider(k, log_level='debug') for k in TYPES], provider_config_path=config_path, runs=1, timeout=45, delay=0, provider_concurrency=1,max_failure_retries=0) - self.assertEqual(len(records),3) + self.assertEqual(len(records), len(TYPES)) self.assertTrue(all(task_passed(r['ok'],r['metrics'],'body') for r in records)) for record in records: self.assertTrue((root/'run'/record['provider_metadata']['raw_response_path']).is_file()) @@ -269,7 +272,7 @@ def respond(request, **kwargs): from harness.web_fetch.report_build import build_report report = build_report(root/'run', root/'run/report', diagnostic_preview=True, language='en') self.assertTrue(report.is_file()) - self.assertEqual(call.call_count, 3) + self.assertEqual(call.call_count, len(TYPES)) def test_diagnostics_redact_new_auth_headers(self): with tempfile.TemporaryDirectory() as directory: @@ -280,27 +283,6 @@ def test_diagnostics_redact_new_auth_headers(self): self.assertNotIn('secret-bb',path.read_text()) self.assertNotIn('secret-bu',path.read_text()) - @patch('harness.web_fetch.providers.hosted_fetch.request') - def test_deepseek_url_only_and_exact_response(self, call): - raw = b'{"title":"Separate title","content":"Unchanged body"}' - call.return_value = HttpResponse(200, 'https://service.example/', {}, raw) - provider = self.provider('deepseek_fetch', log_level='debug') - result = provider.fetch(self.case, 45) - self.assertTrue(result.ok) - self.assertEqual(result.title, 'Separate title') - self.assertEqual(result.content, 'Unchanged body') - self.assertIsNone(result.status_code) - self.assertEqual(base64.b64decode(result.metadata['_raw_response_b64']), raw) - self.assertEqual(call.call_args.kwargs['json_body'], {'url':self.case.url}) - self.assertEqual(call.call_args.kwargs['timeout'], 45) - self.assertEqual(call.call_args.kwargs['headers'], {'Authorization':'Bearer private-test-key'}) - self.assertEqual(call.call_count, 1) - call.return_value = HttpResponse(429, 'https://service.example/', {}, b'{"error":{"code":"rate_limit","message":"actual limit error"}}') - failed = provider.fetch(self.case, 45) - self.assertFalse(failed.ok) - self.assertEqual(failed.error_type,'rate_limit') - self.assertEqual(failed.metadata['service_error']['message'],'actual limit error') - @patch('harness.web_fetch.runner.score_attempt', side_effect=AssertionError('Scorer must not run')) @patch('harness.web_fetch.providers.http_client.urllib.request.build_opener') def test_collection_retains_actual_http_errors_and_exception_trace_without_scoring(self, opener, scorer): From 9fb179584bf80ccf1838eaa5da0cca3c1b965f37 Mon Sep 17 00:00:00 2001 From: Maisiqi <324751762+Maisiqi@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:18:23 +0800 Subject: [PATCH 05/18] Narrow hosted fetch scope and retain existing diagnostic workflow --- configs/providers.example.json | 55 +++---------- docs/provider-contract.md | 86 ++++++++------------- harness/web_fetch/cli.py | 6 -- harness/web_fetch/providers/hosted_fetch.py | 16 +--- harness/web_fetch/runner.py | 35 +++------ tests/test_hosted_fetch.py | 48 +++++------- 6 files changed, 76 insertions(+), 170 deletions(-) diff --git a/configs/providers.example.json b/configs/providers.example.json index 1c2b5e9ab..190d4bd0f 100644 --- a/configs/providers.example.json +++ b/configs/providers.example.json @@ -73,9 +73,7 @@ }, "fixture": { "type": "fixture", - "fail_cases": [ - "javascript-spa" - ] + "fail_cases": ["javascript-spa"] }, "http-local": { "type": "http", @@ -121,11 +119,7 @@ "type": "firecrawl", "base_url": "https://api.firecrawl.dev", "api_key": "${FIRECRAWL_API_KEY:-}", - "formats": [ - "markdown", - "links", - "images" - ], + "formats": ["markdown", "links", "images"], "max_age": 0, "max_bytes": 20000000, "min_interval_seconds": 6.1, @@ -193,15 +187,8 @@ "type": "subprocess_jsonl", "method_type": "openclaw_builtin", "repo_path": "${OPENCLAW_REPO}", - "command": [ - "${OPENCLAW_NODE:-node}", - "--import", - "tsx", - "{harness_root}/harness/web_fetch/workers/openclaw_webfetch_worker.mts" - ], - "env": { - "OPENCLAW_REPO": "${OPENCLAW_REPO}" - }, + "command": ["${OPENCLAW_NODE:-node}", "--import", "tsx", "{harness_root}/harness/web_fetch/workers/openclaw_webfetch_worker.mts"], + "env": {"OPENCLAW_REPO": "${OPENCLAW_REPO}"}, "expected_repo_revision": "12c4bdf7482f24c85b011e9d858fdec3c245101c", "expected_node_version": "v24.15.0", "package_manager_command": "${OPENCLAW_PNPM:-~/.browser-eval/toolchains/openclaw-node/node_modules/.bin/pnpm}", @@ -216,12 +203,7 @@ "type": "subprocess_jsonl", "method_type": "deepseek_harness_builtin", "repo_path": "${DEEPSEEK_HARNESS_REPO}", - "command": [ - "${OPENCLAW_NODE:-node}", - "--import", - "tsx", - "{repo_path}/node_modules/.pnpm/.browser-harness-dsh-worker.mts" - ], + "command": ["${OPENCLAW_NODE:-node}", "--import", "tsx", "{repo_path}/node_modules/.pnpm/.browser-harness-dsh-worker.mts"], "expected_repo_revision": "dd6322d604e00eec1ba5e0c8541159906a21094a", "expected_node_version": "v24.15.0", "required_paths": [ @@ -244,34 +226,19 @@ "type": "subprocess_jsonl", "method_type": "langchain_webbase", "repo_path": ".", - "command": [ - "${OSS_PYTHON:-.venv/bin/python}", - "{harness_root}/harness/web_fetch/workers/langchain_webbase_worker.py" - ], + "command": ["${OSS_PYTHON:-.venv/bin/python}", "{harness_root}/harness/web_fetch/workers/langchain_webbase_worker.py"], "expected_python_version": "Python 3.11.12", - "expected_library_versions": { - "langchain-community": "0.3.30" - }, - "required_paths": [ - "{harness_root}/harness/web_fetch/workers/langchain_webbase_worker.py" - ] + "expected_library_versions": {"langchain-community": "0.3.30"}, + "required_paths": ["{harness_root}/harness/web_fetch/workers/langchain_webbase_worker.py"] }, "crawl4ai": { "type": "subprocess_jsonl", "method_type": "crawl4ai", "repo_path": ".", - "command": [ - "${OSS_PYTHON:-.venv/bin/python}", - "{harness_root}/harness/web_fetch/workers/crawl4ai_worker.py" - ], + "command": ["${OSS_PYTHON:-.venv/bin/python}", "{harness_root}/harness/web_fetch/workers/crawl4ai_worker.py"], "expected_python_version": "Python 3.11.12", - "expected_library_versions": { - "crawl4ai": "0.9.3" - }, - "required_paths": [ - "{harness_root}/harness/web_fetch/workers/crawl4ai_worker.py", - "{harness_root}/harness/web_fetch/network_dns.py" - ] + "expected_library_versions": {"crawl4ai": "0.9.3"}, + "required_paths": ["{harness_root}/harness/web_fetch/workers/crawl4ai_worker.py", "{harness_root}/harness/web_fetch/network_dns.py"] } } } diff --git a/docs/provider-contract.md b/docs/provider-contract.md index 560932ee1..4206f3609 100644 --- a/docs/provider-contract.md +++ b/docs/provider-contract.md @@ -43,8 +43,7 @@ These adapters implement webpage body/structure tasks. Unsupported document, media and file tasks fail before transport, without helper downloads or a substitute browser/Agent. Rendering and internal retries remain service-owned; offline adapter checks do not establish JavaScript capability or task pass rates. -Alternative structured formats require a separately declared output contract; -never derive schemas or parameters from expected answers. +Only Markdown output is supported by these adapters. Retain complete native responses separately from score-bearing answer fields. An API HTTP 200 does not prove target retrieval succeeded: explicit service or @@ -58,10 +57,6 @@ to the service. No automatic DNS fallback is added. Browser Use receives the cal budget through `timeout_ms`; the other services retain their native defaults within the outer deadline. Historical runs keep their recorded configuration. -For diagnostics without scoring, use `run --collect-only --runs 1 ---max-failure-retries 0`. It preserves responses and logs, but cannot produce a -scored report without explicit rescoring and does not certify a cohort. - API references: [Bright Data](https://docs.brightdata.com/api-reference/rest-api/unlocker/unlock-website), [fetch-use](https://browser-use.com/posts/fetch-use), [Browserbase Fetch](https://docs.browserbase.com/platform/fetch/overview), @@ -134,52 +129,36 @@ dynamic text route. Combining it with resource discovery is rejected before a call. Direct file delivery also rejects DOM completion: it cannot claim a rendered-page condition was satisfied while fetching only response bytes. -## Crawl4AI Cloud and the local library - -`crawl4ai-cloud` and local `crawl4ai` are separate methods: old local results -cannot be relabeled as Cloud results. The local worker uses headless Chrome, -cache bypass, `domcontentloaded`, overlay removal, iframe processing and -`raw_markdown`. Cloud returns service-cleaned Markdown with no exact counterpart -for every local option. Browser versions, extraction and site access may differ. -Retain Cloud engine/cache/usage fields in the raw response. A Cloud 403 can mean -target access denial, not only invalid credentials; native browser traces are -not promised. - -### Selected fixed-native comparison policy - -The example configuration selects fixed native parameters for `moli-webfetch`, -`firecrawl`, `cloudflare-markdown`, `tavily`, `exa`, `brightdata`, -`browser-use-fetch`, `browserbase`, `deepseek-fetch-api`, and `crawl4ai-cloud`. -Requests do not consult scoring requirements. Per-case retrieval hints are -rejected before transport; dynamic cases must declare the native retrieval mode. -Exa uniformly requests up to 100 native rich links. Cloudflare receives the URL -directly, including document URLs; the harness does not download a file for it. -Unsupported native routes remain unsupported rather than receiving caller help. -The full 30-second deadline covers the provider response and required decoding; -scorer input preparation, scoring and evidence persistence follow afterward. -A failure retry receives another 30 seconds. The final call supplies the result -and primary latency, while all physical answers and cumulative retry cost remain -recorded. A still-running timed-out call is quarantined instead of overlapped. - -### Reusable selected cohort - -`configs/providers.example.json` declares the ten retained methods in +## Selected native cohort + +`configs/providers.example.json` owns the ten retained methods in `default_providers`. Use `--providers configured --provider-config -configs/providers.example.json` with the existing `run`, `validate`, -`run_benchmark.py` or `run_certified_benchmark.py` entry point. Selection is -resolved before execution and frozen as explicit method names in run evidence. -The shared defaults are 30 seconds per physical call and one failure retry; -explicit CLI overrides remain available and are recorded. The no-argument -example still uses the offline fixture. Other adapters remain available by name. - -The selected HTTP adapters retain a 20,000,000-byte response ceiling, and Moli -retains its output ceiling. This is a caller evidence limit, not a native page -or subresource download setting. Moli receives no website-specific private -network exception or caller-specified download size cap. Every selected method -enables caller debug logs. Credentials remain in the repository `.env`; Moli's -local path must resolve to the version and hash pinned in the configuration. - -For a diagnostic collection, supply a frozen dataset and a new external output: +configs/providers.example.json` with `run`, `validate`, `run_benchmark.py` or +`run_certified_benchmark.py`. Selection is frozen as explicit names in evidence; +other adapters remain selectable, and the no-argument CLI still uses a fixture. + +- The default is 30 seconds per physical call and one failure retry. The final + response supplies the result and primary latency; every call and accumulated + retry cost is retained. CLI overrides remain explicit and recorded. +- Stop timing after the response and required decoding; preparation for scoring, + scoring and evidence persistence follow. Discard late answers and quarantine + still-running workers rather than overlapping abandoned calls. +- Reject per-case hints and never derive requests from grading requirements. + Dynamic cases declare native retrieval mode. Exa uniformly requests 100 rich + links; Cloudflare receives URLs directly, without helper document downloads. +- Keep the 20,000,000-byte HTTP response/Moli output ceiling and debug logs. + Moli retains its pinned binary identity and private-network guard without + per-host exceptions or a caller-imposed subresource download cap. Non-public + DNS answers must be resolved as an environment issue before a public cohort. +- `crawl4ai-cloud` is distinct from local `crawl4ai`: browser versions and + extraction differ, and old local results must not be relabeled as Cloud. + Cloud uses service-cleaned Markdown; retain its engine/cache/usage fields. + Native proxy options are supported but `none` remains the configured default; + choose any paid proxy policy before calls, never in response to scores. + +For diagnostics, supply frozen tasks and a new external output. `--measure-only` +retains scored evidence without building a report; it does not certify a cohort. +Historical unscored collections remain ineligible for scored reports. ```sh uv run --frozen python scripts/run_benchmark.py \ @@ -189,6 +168,5 @@ uv run --frozen python scripts/run_benchmark.py \ --output /path/to/new-run ``` -Publishable cohorts use `scripts/run_certified_benchmark.py` with the same -provider selection and timing arguments, plus its required case scope, work -root and output. Report-instance profiles and measured data remain outside Git. +Publishable cohorts use `scripts/run_certified_benchmark.py` with its required +scope, work root and output. Report profiles and measured data stay outside Git. diff --git a/harness/web_fetch/cli.py b/harness/web_fetch/cli.py index d07204ed0..695d299a4 100644 --- a/harness/web_fetch/cli.py +++ b/harness/web_fetch/cli.py @@ -51,11 +51,7 @@ def command_run(args: argparse.Namespace) -> int: delay=args.delay, provider_concurrency=args.provider_concurrency, max_failure_retries=args.max_failure_retries, - collect_only=getattr(args, "collect_only", False), ) - if getattr(args, "collect_only", False): - print(f"Collected responses and logs: {args.output.resolve()}") - return 0 report_path = build_report(args.output, args.output / 'report', diagnostic_preview=True, language=args.language) print(f"Report: {report_path.resolve()}") @@ -126,8 +122,6 @@ def build_parser() -> argparse.ArgumentParser: help="Select one task kind; intersects with --case-ids") run.add_argument("--runs", type=int, default=1) run.add_argument("--timeout", type=float, default=DEFAULT_CALL_TIMEOUT_SECONDS) - run.add_argument("--collect-only", action="store_true", - help="Collect one call per case without scoring, retries or a scored report") run.add_argument("--delay", type=float, default=0.5) run.add_argument( "--provider-concurrency", diff --git a/harness/web_fetch/providers/hosted_fetch.py b/harness/web_fetch/providers/hosted_fetch.py index a5a62f2bf..936038ec1 100644 --- a/harness/web_fetch/providers/hosted_fetch.py +++ b/harness/web_fetch/providers/hosted_fetch.py @@ -3,7 +3,6 @@ import base64 import hashlib -import json import uuid from lexbench_webfetch.models import EvalCase @@ -66,18 +65,15 @@ def validate(self) -> list[str]: if self.config.get("proxy", "none") not in {"none", "isp", "residential"}: errors.append("unsupported proxy") output_format = self.config.get("output_format", "markdown") - formats = {"crawl4ai_cloud": {"markdown"}, "browserbase_fetch": {"markdown", "json"}, - "browser_use_fetch": {"markdown", "structured"}, - "brightdata_unlocker": {"markdown"}, "deepseek_fetch": {"markdown"}} - if not isinstance(output_format, str) or output_format not in formats[self.type]: - errors.append("unsupported output_format") - if self.type == "browserbase_fetch" and output_format == "json" and not isinstance(self.config.get("schema"), dict): - errors.append("JSON output requires schema") + if output_format != "markdown": + errors.append("unsupported output_format; this adapter requires markdown") return errors def native_http_arguments(self, case, timeout): budget = inner_timeout_budget(timeout, native_default=True) output_format = self.config.get("output_format", "markdown") + if output_format != "markdown": + raise ValueError("This adapter requires markdown output") key = str(self.config.get("api_key", "")) body: dict = {"url": case.url} if self.type == "brightdata_unlocker": @@ -93,8 +89,6 @@ def native_http_arguments(self, case, timeout): headers = {"X-BB-API-Key": key} body.update(format=output_format, allowRedirects=True, proxies=self.config.get("proxies", True)) - if output_format == "json": - body["schema"] = self.config["schema"] elif self.type == "crawl4ai_cloud": headers = {"Authorization": f"Bearer {key}"} body.update(format="md", engine=self.config.get("engine", "browser"), @@ -156,8 +150,6 @@ def fetch(self, case: EvalCase, timeout: float, *, native_route_ledger=None) -> else: if self.type == "browserbase_fetch": content = payload.get("content") - if output_format == "json" and isinstance(content, (dict, list)): - content = json.dumps(content, ensure_ascii=False, separators=(",", ":")) status = _http_status(payload.get("statusCode", payload.get("status_code"))) service_error = payload.get("encoding") == "base64" or bool(payload.get("error")) elif self.type == "crawl4ai_cloud": diff --git a/harness/web_fetch/runner.py b/harness/web_fetch/runner.py index 700642821..e72e18379 100644 --- a/harness/web_fetch/runner.py +++ b/harness/web_fetch/runner.py @@ -499,7 +499,6 @@ def run_evaluation( reference_policy: str = "fresh", case_admission: Any | None = None, call_order: str = "provider", - collect_only: bool = False, ) -> tuple[list[dict[str, Any]], dict[str, Any]]: output_dir = output_dir.resolve() for name, value, minimum in ( @@ -512,8 +511,6 @@ def run_evaluation( if (not isinstance(value, (int, float)) or isinstance(value, bool) or not math.isfinite(value) or value < minimum or (name == 'timeout' and value == 0)): raise ValueError(f'{name} must be finite and {"positive" if name == "timeout" else "nonnegative"}') - if collect_only and (runs != 1 or max_failure_retries != 0): - raise ValueError('Collection-only diagnostics require one run and zero caller retries') if call_order not in {"provider", "case-alternating"}: raise ValueError("Unsupported call order") if call_order == "case-alternating" and provider_concurrency != 1: @@ -549,9 +546,7 @@ def run_evaluation( contained_path(output_dir, filename) if reference_policy not in {"fresh", "frozen_existing"}: raise ValueError("unknown reference policy") - if collect_only: - reference_checks = None - elif reference_policy == "frozen_existing": + if reference_policy == "frozen_existing": # Explicit local measurement of existing rules; no live-source claim. from .reference_lifecycle import validate_resource_source_evidence for case in cases: @@ -582,7 +577,6 @@ def run_evaluation( manifest = { "schema_version": 1, - "collection_only": collect_only, "runtime_source_sha256": hashlib.sha256(SCORING_SOURCE_SHA256.encode()+b'\0'+b'\0'.join( p.relative_to(Path(__file__).parent).as_posix().encode()+b'\0'+p.read_bytes() for p in sorted(Path(__file__).parent.rglob('*.py')))).hexdigest(), @@ -685,9 +679,7 @@ def run_provider(provider: Provider, selected_cases=None, selected_runs=None) -> binding = reference_checks["cases"].get(case.id) if reference_checks else None # Outside provider exception handling: expired references # stop measurement, never create a supplier failure record. - if collect_only: - pass - elif reference_policy == "frozen_existing": + if reference_policy == "frozen_existing": if hashlib.sha256(read_dataset_bytes(dataset_path)).hexdigest() != frozen_dataset_sha256: raise ReferenceLifecycleError("Frozen rules changed during the run", "reference_changed") else: @@ -768,19 +760,18 @@ def run_provider(provider: Provider, selected_cases=None, selected_runs=None) -> result.metadata['native_answer_required'] = True artifact = persist_result_artifacts(output_dir, artifact_rel, result, request_url=case.url) artifact_bytes = result.content.encode('utf-8', errors='replace') - measured = ({'content_sha256': hashlib.sha256(artifact_bytes).hexdigest(), - 'scoring_available': False} if collect_only else score_attempt( + measured = score_attempt( case, artifact_bytes, ok=result.ok, metadata=result.metadata, title=result.title, final_url=result.final_url, list_reference=load_list_reference(case, dataset_path.parent), status_code=result.status_code, prepared_answer=result.prepared_answer, - )) + ) measured['deadline_overrun'] = physical_elapsed_ms > timeout * 1000 measured['deadline_budget_ms'] = timeout * 1000 if binding is not None: measured['reference_sha256'] = binding['reference_sha256'] - physical_passed = None if collect_only else task_passed(result.ok, measured, case.task_kind) + physical_passed = task_passed(result.ok, measured, case.task_kind) physical_completed = utc_now() call_log = persist_public_call_log( output_dir, provider=provider, case=case, attempt_id=attempt_id, @@ -879,18 +870,12 @@ def run_provider(provider: Provider, selected_cases=None, selected_runs=None) -> "client_throttle_wait_ms": throttle_wait_ms, }, ).to_dict() - if not collect_only: - apply_request_status(record) - else: - record['scoring'] = {'mode': 'not_scored', 'reason': 'collection_only_diagnostic'} + apply_request_status(record) record["completed_at"] = finished from .models import execution_id record['run_id'] = execution_id(record) - if collect_only: - state = "RECV" if record["ok"] else "ERR" - else: - passed = task_passed(record['ok'],record['metrics'],case.task_kind) - state = "PASS" if passed else "FAIL" + passed = task_passed(record['ok'],record['metrics'],case.task_kind) + state = "PASS" if passed else "FAIL" with result_lock: records.append(record) results_file.write(json.dumps(record, ensure_ascii=True) + "\n") @@ -958,9 +943,7 @@ def run_provider(provider: Provider, selected_cases=None, selected_runs=None) -> encoding="utf-8", ) - summary = ({'mode': 'collection_only', 'scoring_state': 'not_scored', - 'attempt_count': len(records), 'responses_ok': sum(bool(r['ok']) for r in records), - 'task_pass_rate': None} if collect_only else summarize(records)) + summary = summarize(records) manifest["completed_at"] = utc_now() manifest["attempt_count"] = len(records) manifest["attempt_ids"] = [r['attempt_id'] for r in records] diff --git a/tests/test_hosted_fetch.py b/tests/test_hosted_fetch.py index 976415b5c..87b014700 100644 --- a/tests/test_hosted_fetch.py +++ b/tests/test_hosted_fetch.py @@ -150,22 +150,13 @@ def test_only_brightdata_receives_documented_native_debug_flag(self, call): self.assertIn('debug must be boolean', self.provider(TYPES[0], debug='true').validate()) @patch('harness.web_fetch.providers.hosted_fetch.request') - def test_native_structured_formats_preserve_only_returned_answers(self, call): - schema = {'type':'object','properties':{'records':{'type':'array','items':{'type':'object'}}}} - answer = {'records':[{'text':'Returned by service'}]} - call.return_value = HttpResponse(200,'https://service.example/',{},json.dumps({'content':answer,'statusCode':200}).encode()) - result = self.provider('browserbase_fetch', output_format='json', schema=schema).fetch(self.case,45) - self.assertTrue(result.ok) - self.assertEqual(json.loads(result.content),answer) - self.assertEqual(call.call_args.kwargs['json_body']['schema'],schema) - self.assertEqual(result.metadata['output_contract'],'json') - call.return_value = self.response('browser_use_fetch', json.dumps(answer)) - result = self.provider('browser_use_fetch', output_format='structured').fetch(self.case,45) - self.assertTrue(result.ok) - self.assertEqual(json.loads(result.content),answer) - self.assertEqual(call.call_args.kwargs['json_body']['output_format'],'structured') - self.assertTrue(self.provider('browserbase_fetch', output_format='json').validate()) - self.assertTrue(self.provider('browser_use_fetch', output_format='invented').validate()) + def test_unsupported_output_formats_are_rejected_before_transport(self, call): + for kind, output in (('browserbase_fetch', 'json'), ('browser_use_fetch', 'structured')): + provider = self.provider(kind, output_format=output) + self.assertTrue(provider.validate()) + with self.assertRaisesRegex(ValueError, 'requires markdown'): + provider.fetch(self.case, 30) + call.assert_not_called() @patch('harness.web_fetch.providers.hosted_fetch.request') def test_http_200_is_not_enough(self, call): @@ -283,9 +274,8 @@ def test_diagnostics_redact_new_auth_headers(self): self.assertNotIn('secret-bb',path.read_text()) self.assertNotIn('secret-bu',path.read_text()) - @patch('harness.web_fetch.runner.score_attempt', side_effect=AssertionError('Scorer must not run')) @patch('harness.web_fetch.providers.http_client.urllib.request.build_opener') - def test_collection_retains_actual_http_errors_and_exception_trace_without_scoring(self, opener, scorer): + def test_runner_retains_http_errors_and_exception_trace(self, opener): responses = [Mock(status=200, headers={}, read=Mock(return_value=b'{"title":"Title","content":"Body"}'), geturl=Mock(return_value='https://service.example/')), Mock(status=500, headers={}, read=Mock(return_value=b'{"error":{"message":"native failure"}}'), @@ -295,15 +285,13 @@ def test_collection_retains_actual_http_errors_and_exception_trace_without_scori cases = [EvalCase(str(i),'Control',self.case.url,'static',[]) for i in range(3)] with tempfile.TemporaryDirectory() as directory: root=Path(directory) - rows,summary=run_evaluation(output_dir=root/'run',dataset_meta={'cases':[vars(c) for c in cases]}, + rows,_=run_evaluation(output_dir=root/'run',dataset_meta={'cases':[vars(c) for c in cases]}, dataset_path=root/'dataset.json', cases=cases, providers=[self.provider('deepseek_fetch',log_level='debug')],provider_config_path=root/'providers.json', - runs=1,timeout=45,delay=0,collect_only=True,max_failure_retries=0) + runs=1,timeout=45,delay=0,max_failure_retries=0) self.assertEqual([r['ok'] for r in rows],[True,False,False]) - self.assertIsNone(summary['task_pass_rate']) self.assertEqual(opener.return_value.open.call_count,3) for row in rows: - self.assertNotIn('content_pass',row['metrics']) log=(root/'run'/row['provider_metadata']['diagnostic_log_path']).read_text() self.assertNotIn('private-test-key',log) self.assertIn('http_request_started',log) @@ -312,13 +300,17 @@ def test_collection_retains_actual_http_errors_and_exception_trace_without_scori self.assertIn('Traceback',failure_log) server_log=(root/'run'/rows[1]['provider_metadata']['diagnostic_log_path']).read_text() self.assertIn('native failure',server_log) - from harness.web_fetch.report_build import build_report - with self.assertRaisesRegex(ValueError,'Collection-only evidence'): - build_report(root/'run',root/'report') - scorer.assert_not_called() + + def test_historical_unscored_collection_cannot_build_report(self): + from harness.web_fetch.report_build import build_report + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root/'manifest.json').write_text('{"collection_only":true}') + with self.assertRaisesRegex(ValueError, 'Collection-only evidence'): + build_report(root, root/'report') @patch('harness.web_fetch.providers.http_client.urllib.request.build_opener') - def test_collection_timeout_has_actual_deadline_event_and_no_retry(self, opener): + def test_runner_timeout_has_actual_deadline_event_and_no_retry(self, opener): release=threading.Event() def blocked(*args,**kwargs): release.wait(1) @@ -330,7 +322,7 @@ def blocked(*args,**kwargs): rows,_=run_evaluation(output_dir=root/'run',dataset_meta={'cases':[vars(self.case)]}, dataset_path=root/'dataset.json',cases=[self.case], providers=[self.provider('deepseek_fetch',log_level='debug')],provider_config_path=root/'providers.json', - runs=1,timeout=0.02,delay=0,collect_only=True,max_failure_retries=0) + runs=1,timeout=0.02,delay=0,max_failure_retries=0) self.assertEqual(rows[0]['error_type'],'timeout') log=(root/'run'/rows[0]['provider_metadata']['diagnostic_log_path']).read_text() self.assertIn('answer_timeout',log) From de397602b859feac53f13811ad9b2dd27d9ee917 Mon Sep 17 00:00:00 2001 From: Maisiqi <324751762+Maisiqi@users.noreply.github.com> Date: Sun, 25 Oct 2026 17:19:08 +0800 Subject: [PATCH 06/18] Enable native residential proxy for selected Crawl4AI Cloud configuration --- configs/providers.example.json | 2 +- docs/provider-contract.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/providers.example.json b/configs/providers.example.json index 190d4bd0f..c376a87d8 100644 --- a/configs/providers.example.json +++ b/configs/providers.example.json @@ -17,7 +17,7 @@ "type": "crawl4ai_cloud", "api_key": "${CRAWL4AI_CLOUD_API_KEY}", "engine": "browser", - "proxy": "none", + "proxy": "residential", "output_format": "markdown", "dns_mode": "system", "log_level": "debug", diff --git a/docs/provider-contract.md b/docs/provider-contract.md index 4206f3609..ce815679d 100644 --- a/docs/provider-contract.md +++ b/docs/provider-contract.md @@ -37,7 +37,7 @@ runner and scorer. Their credentials and fixed options are declared in | Browser Use Fetch | `BROWSER_USE_API_KEY` | `/fetch`, Markdown, fresh session, US proxy; no Agent | | Browserbase Fetch | `BROWSERBASE_API_KEY` | `/v1/fetch`, Markdown, redirects and native proxy enabled; no Project ID | | DeepSeek Fetch API | `DEEPSEEK_API_KEY` | `/v1/tools/fetch`, URL-only request; retain content and title | -| Crawl4AI Cloud | `CRAWL4AI_CLOUD_API_KEY` | `/scrape`, `format=md`, `engine=browser`, `proxy=none` | +| Crawl4AI Cloud | `CRAWL4AI_CLOUD_API_KEY` | `/scrape`, `format=md`, `engine=browser`, `proxy=residential` | These adapters implement webpage body/structure tasks. Unsupported document, media and file tasks fail before transport, without helper downloads or a @@ -153,8 +153,8 @@ other adapters remain selectable, and the no-argument CLI still uses a fixture. - `crawl4ai-cloud` is distinct from local `crawl4ai`: browser versions and extraction differ, and old local results must not be relabeled as Cloud. Cloud uses service-cleaned Markdown; retain its engine/cache/usage fields. - Native proxy options are supported but `none` remains the configured default; - choose any paid proxy policy before calls, never in response to scores. + The selected configuration uniformly enables the service's native residential + proxy, which adds provider charges; never switch proxy policy in response to scores. For diagnostics, supply frozen tasks and a new external output. `--measure-only` retains scored evidence without building a report; it does not certify a cohort. From 3a0e0c607058d83889b7860ff02f1d6d79be0c41 Mon Sep 17 00:00:00 2001 From: Maisiqi <324751762+Maisiqi@users.noreply.github.com> Date: Sun, 25 Oct 2026 22:09:12 +0800 Subject: [PATCH 07/18] Deduplicate hosted adapter metadata and remove unused projection route --- README.md | 2 +- docs/README.zh-CN.md | 2 +- harness/web_fetch/data/report-methods.json | 61 ++++++---------------- harness/web_fetch/methods.py | 5 +- harness/web_fetch/native_projection.py | 4 +- 5 files changed, 22 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index 996ec7959..321b9107a 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Open the report path printed in the terminal. The example has one expected pass Next, [try HTTP, Moli or Chromium on a local article](examples/web_fetch/README.md). Optional adapters, service credentials and live evaluation commands are covered in the track guides above. -The reusable ten-method Web Fetch cohort is selected with `--providers configured --provider-config configs/providers.example.json`; shared defaults are 30 seconds per call and one failure retry. See the [provider contract](docs/provider-contract.md#reusable-selected-cohort). +The reusable ten-method Web Fetch cohort is selected with `--providers configured --provider-config configs/providers.example.json`; shared defaults are 30 seconds per call and one failure retry. See the [provider contract](docs/provider-contract.md#selected-native-cohort). ## Reports diff --git a/docs/README.zh-CN.md b/docs/README.zh-CN.md index 509593b09..c5bdd9c38 100644 --- a/docs/README.zh-CN.md +++ b/docs/README.zh-CN.md @@ -44,7 +44,7 @@ uv run --frozen browser-harness-eval run --delay 0 接下来可以[用 HTTP、Moli 或 Chromium 读取本地文章](../examples/web_fetch/README.md)。可选适配器、服务凭据和在线评测命令见上面的各流程指南。 -使用 `--providers configured --provider-config configs/providers.example.json` 选择可复用的十种 Web Fetch 方法；默认每次调用 30 秒，失败允许重试一次。详见[适配器规则](provider-contract.md#reusable-selected-cohort)。 +使用 `--providers configured --provider-config configs/providers.example.json` 选择可复用的十种 Web Fetch 方法；默认每次调用 30 秒，失败允许重试一次。详见[适配器规则](provider-contract.md#selected-native-cohort)。 ## 报告 diff --git a/harness/web_fetch/data/report-methods.json b/harness/web_fetch/data/report-methods.json index b04bcaac2..7d7bc070c 100644 --- a/harness/web_fetch/data/report-methods.json +++ b/harness/web_fetch/data/report-methods.json @@ -406,88 +406,48 @@ } }, "brightdata_unlocker": { + "extends": "hosted_fetch_adapter", "endpoint": "POST https://api.brightdata.com/request", "interface_method": "brightdata", "contract": { "zh": "已知网址交给 Web Unlocker，返回原生 Markdown。", "en": "Send a known URL to Web Unlocker and retain native Markdown." - }, - "output": "Markdown", - "freshness": { - "zh": "抓取、缓存及内部重试由服务负责；实际参数和耗时以调用记录为准；测试程序的失败重试次数以冻结的运行配置为准。", - "en": "Retrieval, caching and internal retries are service-owned; actual parameters and timing come from call evidence; caller failure retries follow the frozen run configuration." - }, - "leaderboard": { - "zh": "网页文字提取", - "en": "Webpage text retrieval" } }, "browserbase_fetch": { + "extends": "hosted_fetch_adapter", "endpoint": "POST https://api.browserbase.com/v1/fetch", "interface_method": "browserbase", "contract": { "zh": "通过 Fetch 接口获取网页文字，不替换为浏览器会话。", "en": "Retrieve webpage text through Fetch without substituting a browser session." - }, - "output": "Markdown", - "freshness": { - "zh": "抓取、缓存及内部重试由服务负责；实际参数和耗时以调用记录为准；测试程序的失败重试次数以冻结的运行配置为准。", - "en": "Retrieval, caching and internal retries are service-owned; actual parameters and timing come from call evidence; caller failure retries follow the frozen run configuration." - }, - "leaderboard": { - "zh": "网页文字提取", - "en": "Webpage text retrieval" } }, "browser_use_fetch": { + "extends": "hosted_fetch_adapter", "endpoint": "POST https://fetch.browser-use.com/fetch", "interface_method": "browser-use-fetch", "contract": { "zh": "通过 fetch-use 获取网页文字，不启动 Agent。", "en": "Retrieve webpage text through fetch-use without an Agent." - }, - "output": "Markdown", - "freshness": { - "zh": "抓取、缓存及内部重试由服务负责；实际参数和耗时以调用记录为准；测试程序的失败重试次数以冻结的运行配置为准。", - "en": "Retrieval, caching and internal retries are service-owned; actual parameters and timing come from call evidence; caller failure retries follow the frozen run configuration." - }, - "leaderboard": { - "zh": "网页文字提取", - "en": "Webpage text retrieval" } }, "deepseek_fetch": { + "extends": "hosted_fetch_adapter", "endpoint": "POST https://api.deepseek.com/v1/tools/fetch", "interface_method": "deepseek-fetch-api", "contract": { "zh": "只发送网址；正文与标题分别保留。", "en": "Send only the URL; retain content and title separately." - }, - "output": "Markdown", - "freshness": { - "zh": "抓取、缓存及内部重试由服务负责；实际参数和耗时以调用记录为准；测试程序的失败重试次数以冻结的运行配置为准。", - "en": "Retrieval, caching and internal retries are service-owned; actual parameters and timing come from call evidence; caller failure retries follow the frozen run configuration." - }, - "leaderboard": { - "zh": "网页文字提取", - "en": "Webpage text retrieval" } }, "crawl4ai_cloud": { + "extends": "hosted_fetch_adapter", "endpoint": "POST https://api.crawl4ai.com/scrape", "interface_method": "crawl4ai-cloud", "contract": { "zh": "调用 Cloud 抓取接口，与本机开源库分别登记。", "en": "Call the Cloud scrape API, registered separately from the local library." - }, - "output": "Markdown", - "freshness": { - "zh": "抓取、缓存及内部重试由服务负责；实际参数和耗时以调用记录为准；测试程序的失败重试次数以冻结的运行配置为准。", - "en": "Retrieval, caching and internal retries are service-owned; actual parameters and timing come from call evidence; caller failure retries follow the frozen run configuration." - }, - "leaderboard": { - "zh": "网页文字提取", - "en": "Webpage text retrieval" } } }, @@ -6465,6 +6425,17 @@ } } } + }, + "hosted_fetch_adapter": { + "output": "Markdown", + "freshness": { + "zh": "抓取、缓存及内部重试由服务负责；实际参数和耗时以调用记录为准；测试程序的失败重试次数以冻结的运行配置为准。", + "en": "Retrieval, caching and internal retries are service-owned; actual parameters and timing come from call evidence; caller failure retries follow the frozen run configuration." + }, + "leaderboard": { + "zh": "网页文字提取", + "en": "Webpage text retrieval" + } } }, "unregistered_access": { diff --git a/harness/web_fetch/methods.py b/harness/web_fetch/methods.py index fe6a00e05..e73d2fd82 100644 --- a/harness/web_fetch/methods.py +++ b/harness/web_fetch/methods.py @@ -14,7 +14,7 @@ def _resolve_catalog_templates(data): - """Expand named defaults once; methods override dictionaries and replace lists.""" + """Expand named defaults once; catalog entries override dictionaries and replace lists.""" def merge(base, override): result = deepcopy(base) for key, value in override.items(): @@ -35,7 +35,8 @@ def resolve(row, chain=()): raise ValueError('Cyclic method template: ' + parent) return merge(resolve(templates[parent], (*chain, parent)), row) - return {**data, 'methods': {name: resolve(row) for name, row in data['methods'].items()}} + return {**data, **{section: {name: resolve(row) for name, row in data[section].items()} + for section in ('methods', 'adapters')}} @lru_cache(maxsize=1) diff --git a/harness/web_fetch/native_projection.py b/harness/web_fetch/native_projection.py index d232899e2..521dda3d8 100644 --- a/harness/web_fetch/native_projection.py +++ b/harness/web_fetch/native_projection.py @@ -54,9 +54,7 @@ def native_answer_projection(provider_type, case, payload, status_code=None, *, field = ('body' if provider_type == 'browser_use_fetch' else 'markdown' if provider_type == 'crawl4ai_cloud' else 'content') value = payload.get(field) - fmt = ('text' if isinstance(value, str) else - 'json' if provider_type == 'browserbase_fetch' and interface == 'json' - and isinstance(value, (dict, list)) else 'empty') + fmt = 'text' if isinstance(value, str) else 'empty' declaration = {'schema_version': 2, 'content_pointer': [field], 'content_format': fmt if successful_http else 'empty', 'optional_content': True, 'result_fields': {}} From fd126854ff8c1831fdeef849129907e44c5a6d88 Mon Sep 17 00:00:00 2001 From: Maisiqi <324751762+Maisiqi@users.noreply.github.com> Date: Sun, 25 Oct 2026 22:20:26 +0800 Subject: [PATCH 08/18] Preserve partial catalog input and refresh verified projection fingerprints --- harness/web_fetch/methods.py | 2 +- tests/fixtures/native-route-baseline.json | 36 +++++++++++------------ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/harness/web_fetch/methods.py b/harness/web_fetch/methods.py index e73d2fd82..697a266d5 100644 --- a/harness/web_fetch/methods.py +++ b/harness/web_fetch/methods.py @@ -36,7 +36,7 @@ def resolve(row, chain=()): return merge(resolve(templates[parent], (*chain, parent)), row) return {**data, **{section: {name: resolve(row) for name, row in data[section].items()} - for section in ('methods', 'adapters')}} + for section in ('methods', 'adapters') if section in data}} @lru_cache(maxsize=1) diff --git a/tests/fixtures/native-route-baseline.json b/tests/fixtures/native-route-baseline.json index 081ae9367..e3dd208ed 100644 --- a/tests/fixtures/native-route-baseline.json +++ b/tests/fixtures/native-route-baseline.json @@ -1,74 +1,74 @@ { "cloudflare_content:configured": { "cells": 192, - "sha256": "f7c345477ff7ced84b450586cb2b83720fac64edf22dd5bc7725c012cda89beb" + "sha256": "e3bdf3b4fe8018fe4a1a5df5bf13e87b8e33684c7336481f5807d82d97e60ea3" }, "cloudflare_content:defaults": { "cells": 192, - "sha256": "ac5cabdd2f59cb366241c54fa6ac0d71d87b8e268f8b94c76e602264004f981c" + "sha256": "e7890002343058cb61322c3525d06eee314273fde454027c226fba890fa9c576" }, "cloudflare_markdown:configured": { "cells": 192, - "sha256": "c56aef9437389a1325df9b915ff84dff97d6699e00a54ad02c554cc25e0647d0" + "sha256": "a5f0791fdccaa79f8b466a08eaf5f40324e7f140bc26bd08d80d4e00584d8964" }, "cloudflare_markdown:defaults": { "cells": 192, - "sha256": "c86a678efcd9710c92eb5a731d485865ef1f87d4971e1873452536093871a6bb" + "sha256": "a7e2bc24318ef7d241cdb251393d333bea84bf8a19130bac5a84ff806904d903" }, "cowagent_builtin:configured": { "cells": 192, - "sha256": "cdb320a627bdca00381b7ecf42b504606c197128584dfe666dff20273da3d39c" + "sha256": "a98a3409674e91d9cebd86ace1760512906187a7f6644601bf76b39012f48b7b" }, "cowagent_builtin:defaults": { "cells": 192, - "sha256": "f0ccf993d06ba6d37a4d3b194a49257a4423da733f6e5a480a8e9744e8e8be60" + "sha256": "4c5ce7848d804995077fd79e8e62c0e8bbe7a3065350a326973b0f70c272b5d4" }, "exa_contents:configured": { "cells": 192, - "sha256": "9f45c203c8c619df4f49d55cfa2979a439b7e166a474ed03020856cbcb817719" + "sha256": "0e8e67119323a9f89ec9c55bd8a103bf9edd0c798b868a88fba7798ad1582e14" }, "exa_contents:defaults": { "cells": 192, - "sha256": "f54d453a98061790439f2286147832a4703f6d63f9e2d9e0564478d3a57b7efc" + "sha256": "d187f3cdc436823cdc1cd9adfe2ca62642a44ee94c86405c1377c4b2eda6b84b" }, "firecrawl:configured": { "cells": 192, - "sha256": "0b75cdbdec79a64649f320ab9c10344033930a405fece521e232e5f541d43198" + "sha256": "66e4ec865ebbe411a7889c93558dd14109888c421b619e7b994d644ce412e686" }, "firecrawl:defaults": { "cells": 192, - "sha256": "9e42b2a3667bb2bc31709c80bf23102d81aff562d66747778dda6b56050cc960" + "sha256": "7919ff51ba35dad5a45353cdd4f72eb9b992d10f11b741c06f1ff63278b1289d" }, "fixture:configured": { "cells": 192, - "sha256": "d72000f903b1751542e71463ab158c2789db593bc58f67ed6b84b50fa61c649f" + "sha256": "65e72a46085748150d84f40fb1c1714a83b7e8be04a766ae3ec84f4a0843d6c7" }, "fixture:defaults": { "cells": 192, - "sha256": "47324b12cd3951ae1ddd5711704c375632cc5be68272d2c4bfd128ac4f0bca82" + "sha256": "69519d867e35b2c1a3fc6607a3d08a7fc5058c31c0e8b621d3c3180e9ec1712d" }, "moli_webfetch:configured": { "cells": 192, - "sha256": "ae9c2025dadbc73057b0be857032e5b27f9f9a487eee524add19e40eae543676" + "sha256": "8d0a49e5f81e835e24cff1a7d1312539dcb3be71670fd86c0ee00174bcb8dbf1" }, "moli_webfetch:defaults": { "cells": 192, - "sha256": "8e9297fa0e7f40fc73c559fb801d34ce3d6ece280359ac8da9c008698d187091" + "sha256": "3d216930373cdd0d09cbd7b21b3e16d6c00d40fab70790796429ed22a0de7432" }, "subprocess_jsonl:configured": { "cells": 192, - "sha256": "088970f3a05691f115e10ed915187f808c80a058e648da5f16dc76edb19c5d37" + "sha256": "c9afc5dc6b1dbaac1d49b3fce1cac6d961d5d9a85e0cd1612a3b6e4572e78174" }, "subprocess_jsonl:defaults": { "cells": 192, - "sha256": "0491ed2a04e5cb1fd263a86dade57de3e8988ef62d4ff99198bdafecde2d64c3" + "sha256": "335a4542cf955d1332695d53aebcbf343aaba12321f8b269b1ca9e8bd957d798" }, "tavily_extract:configured": { "cells": 192, - "sha256": "69de47e62c3abb20c3349e077a37774851023c5c9614bbb5996473cde0822548" + "sha256": "856cdf4b46072373512bc2c0393d69f7e46f0ae3748d6d9a349ca1c749ece86c" }, "tavily_extract:defaults": { "cells": 192, - "sha256": "8b172f5af0cfcfd99e82972ffa8a0f8c1cf59a45f142cc56860267b7ccf3cf61" + "sha256": "601b88ea41144e4b2891cb5d2ff5ac624d8ef2420813227d96b8ea9395fe5b78" } } From 1c2f5d310700db97968d5f75e9a89c96f9505840 Mon Sep 17 00:00:00 2001 From: Maisiqi <324751762+Maisiqi@users.noreply.github.com> Date: Sun, 25 Oct 2026 23:44:40 +0800 Subject: [PATCH 09/18] Leave hosted service internal timeouts at native defaults --- docs/provider-contract.md | 11 +++++------ harness/web_fetch/providers/hosted_fetch.py | 2 +- tests/test_hosted_fetch.py | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/provider-contract.md b/docs/provider-contract.md index ce815679d..9f775201e 100644 --- a/docs/provider-contract.md +++ b/docs/provider-contract.md @@ -43,7 +43,6 @@ These adapters implement webpage body/structure tasks. Unsupported document, media and file tasks fail before transport, without helper downloads or a substitute browser/Agent. Rendering and internal retries remain service-owned; offline adapter checks do not establish JavaScript capability or task pass rates. -Only Markdown output is supported by these adapters. Retain complete native responses separately from score-bearing answer fields. An API HTTP 200 does not prove target retrieval succeeded: explicit service or @@ -53,9 +52,9 @@ requests, response headers/body and errors; it cannot expose undisclosed service internals. Only Bright Data receives the native debug option. `dns_mode: system` uses the local resolver for API hosts; target-site DNS belongs -to the service. No automatic DNS fallback is added. Browser Use receives the call -budget through `timeout_ms`; the other services retain their native defaults -within the outer deadline. Historical runs keep their recorded configuration. +to the service. No automatic DNS fallback is added. All five services retain their +native internal timeout defaults within the caller's deadline. Historical runs +keep their recorded configuration. API references: [Bright Data](https://docs.brightdata.com/api-reference/rest-api/unlocker/unlock-website), [fetch-use](https://browser-use.com/posts/fetch-use), @@ -153,8 +152,8 @@ other adapters remain selectable, and the no-argument CLI still uses a fixture. - `crawl4ai-cloud` is distinct from local `crawl4ai`: browser versions and extraction differ, and old local results must not be relabeled as Cloud. Cloud uses service-cleaned Markdown; retain its engine/cache/usage fields. - The selected configuration uniformly enables the service's native residential - proxy, which adds provider charges; never switch proxy policy in response to scores. + Its configured native proxy adds provider charges; never switch proxy policy + in response to scores. For diagnostics, supply frozen tasks and a new external output. `--measure-only` retains scored evidence without building a report; it does not certify a cohort. diff --git a/harness/web_fetch/providers/hosted_fetch.py b/harness/web_fetch/providers/hosted_fetch.py index 936038ec1..6ea8f7bb1 100644 --- a/harness/web_fetch/providers/hosted_fetch.py +++ b/harness/web_fetch/providers/hosted_fetch.py @@ -97,7 +97,7 @@ def native_http_arguments(self, case, timeout): headers = {"Authorization": f"Bearer {key}"} else: headers = {"X-Browser-Use-API-Key": key} - body.update(method="GET", output_format=output_format, timeout_ms=min(120000, int(budget * 1000)), + body.update(method="GET", output_format=output_format, follow_redirects=True, max_redirects=10, proxy_country=self.config.get("proxy_country", "US"), session_id=self._session_id) diff --git a/tests/test_hosted_fetch.py b/tests/test_hosted_fetch.py index 87b014700..3b0a0bdab 100644 --- a/tests/test_hosted_fetch.py +++ b/tests/test_hosted_fetch.py @@ -107,7 +107,7 @@ def test_routes_and_preserved_evidence(self, call): self.assertIsNone(result.status_code) elif kind == 'browser_use_fetch': self.assertNotIn('retry', body) - self.assertEqual(body['timeout_ms'], 45000) + self.assertNotIn('timeout_ms', body) self.assertEqual(body['output_format'], 'markdown') session = body['session_id'] self.provider(kind).fetch(self.case, 45) From d62590108be8ce470e6a2e2f2e31be6150b3af04 Mon Sep 17 00:00:00 2001 From: Maisiqi <324751762+Maisiqi@users.noreply.github.com> Date: Sun, 25 Oct 2026 23:45:03 +0800 Subject: [PATCH 10/18] Pin hosted adapter capability source to reviewed implementation --- harness/web_fetch/data/report-methods.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/harness/web_fetch/data/report-methods.json b/harness/web_fetch/data/report-methods.json index 7d7bc070c..540363855 100644 --- a/harness/web_fetch/data/report-methods.json +++ b/harness/web_fetch/data/report-methods.json @@ -6158,9 +6158,9 @@ "zh": "托管 Fetch 接入代码（能力范围）", "en": "Hosted Fetch adapter (implemented scope)" }, - "url": "https://github.com/lexmount/browser-eval/blob/38d224ec/harness/web_fetch/providers/hosted_fetch.py", - "checked_at": "2026-10-23", - "checked_version": "38d224ec" + "url": "https://github.com/lexmount/browser-eval/blob/1c2f5d310700db97968d5f75e9a89c96f9505840/harness/web_fetch/providers/hosted_fetch.py", + "checked_at": "2026-10-25", + "checked_version": "1c2f5d310700db97968d5f75e9a89c96f9505840" } }, "output_contract_labels": { From d4115273e7118bc657985331a037a0ec981fbffe Mon Sep 17 00:00:00 2001 From: Maisiqi <324751762+Maisiqi@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:48:29 +0800 Subject: [PATCH 11/18] fix(webfetch): distinguish native timeouts from caller deadlines --- harness/web_fetch/call_boundary.py | 7 +++- harness/web_fetch/runner.py | 2 +- tests/test_public_call_logs.py | 55 ++++++++++++++++++++++++++++-- 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/harness/web_fetch/call_boundary.py b/harness/web_fetch/call_boundary.py index 07750bf55..55d510c21 100644 --- a/harness/web_fetch/call_boundary.py +++ b/harness/web_fetch/call_boundary.py @@ -55,13 +55,16 @@ def work(): output.put((time.perf_counter(), value)) worker = threading.Thread(target=work, daemon=True, name='webfetch-answer') worker.start() + caller_deadline_exceeded = False try: completed, value = output.get(timeout=max(0, timeout - (time.perf_counter() - begin))) except queue.Empty: + caller_deadline_exceeded = True completed, value = time.perf_counter(), TimeoutError('End-to-end answer deadline exceeded') if completed - begin > timeout: + caller_deadline_exceeded = True value = TimeoutError('End-to-end answer deadline exceeded') - if isinstance(value, TimeoutError): + if caller_deadline_exceeded: sink = getattr(provider, "_call_diagnostics", None) if sink: sink.emit("caller", "answer_timeout", type=type(value).__name__, message=str(value), @@ -87,6 +90,7 @@ def work(): if case.task_kind == 'file': metadata['file_evaluation'] = evaluate_files(vars(case), []) metadata['_answer_elapsed_ms']=answer_elapsed_ms + metadata['caller_deadline_exceeded'] = True return ProviderResult( False, '', @@ -96,6 +100,7 @@ def work(): ), worker if isinstance(value, Exception): raise value + value.metadata['caller_deadline_exceeded'] = False if response_only: # Stop the user-observed timer before scorer input preparation/storage. value.metadata['_answer_elapsed_ms'] = round((completed - begin) * 1000, 2) diff --git a/harness/web_fetch/runner.py b/harness/web_fetch/runner.py index bf244da14..c2f4eeaef 100644 --- a/harness/web_fetch/runner.py +++ b/harness/web_fetch/runner.py @@ -341,7 +341,7 @@ def persist_public_call_log( {"stage": "call_started", "at": started_at, "status_code": None, "error_type": None}, {"stage": ("exception_raised" if exception is not None else - "answer_deadline_exceeded" if result.error_type == "timeout" else "response_received"), + "answer_deadline_exceeded" if metadata.get("caller_deadline_exceeded") else "response_received"), "at": completed_at, "status_code": result.status_code, "error_type": _public_text(result.error_type, limit=256)}, {"stage": "call_completed", "at": completed_at, diff --git a/tests/test_public_call_logs.py b/tests/test_public_call_logs.py index e3df72116..b454b260e 100644 --- a/tests/test_public_call_logs.py +++ b/tests/test_public_call_logs.py @@ -2,6 +2,7 @@ import hashlib import json import tempfile +import threading import unittest from pathlib import Path @@ -20,11 +21,11 @@ ) -def run(root: Path, provider: Provider, *, retry: int = 0): +def run(root: Path, provider: Provider, *, retry: int = 0, timeout: float = 2): return run_evaluation( output_dir=root, dataset_meta={"id": "call-log-test"}, dataset_path=Path("dataset.json"), cases=[CASE], providers=[provider], - provider_config_path=Path("providers.json"), runs=1, timeout=2, delay=0, + provider_config_path=Path("providers.json"), runs=1, timeout=timeout, delay=0, max_failure_retries=retry, )[0][0] @@ -43,6 +44,56 @@ def fetch(self, case, timeout): class PublicCallLogTests(unittest.TestCase): + def test_native_timeout_is_not_a_caller_deadline(self): + result = ProviderResult(False, "", error_type="timeout", + error_message="Native readiness wait ended") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + record = run(root, ResultProvider(result), timeout=30) + log = load_log(root, record["provider_metadata"]["call_log"]) + self.assertEqual(log["events"][1]["stage"], "response_received") + self.assertEqual(log["error"]["type"], "timeout") + self.assertFalse(log["timing"]["deadline_overrun"]) + self.assertFalse(record["provider_metadata"]["caller_deadline_exceeded"]) + + def test_outer_deadline_is_recorded_and_late_answer_discarded(self): + class Waiting(Provider): + def __init__(self): + super().__init__("waiting", {"type": "fixture"}) + self.release = threading.Event() + + @property + def cancellation_cleanup_seconds(self): + return 1 + + def cancel(self): + self.release.set() + + def fetch(self, case, timeout): + self.release.wait(5) + return ProviderResult(True, "Expected") + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + record = run(root, Waiting(), timeout=.02) + log = load_log(root, record["provider_metadata"]["call_log"]) + self.assertEqual(log["events"][1]["stage"], "answer_deadline_exceeded") + self.assertTrue(record["provider_metadata"]["caller_deadline_exceeded"]) + self.assertTrue(log["output"]["empty"]) + + def test_native_timeout_exception_keeps_its_origin(self): + class Broken(Provider): + def fetch(self, case, timeout): + raise TimeoutError("Native transport timed out") + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + record = run(root, Broken("broken", {"type": "fixture"}), timeout=30) + log = load_log(root, record["provider_metadata"]["call_log"]) + self.assertEqual(log["events"][1]["stage"], "exception_raised") + self.assertEqual(log["error"]["message"], "Native transport timed out") + self.assertFalse(log["timing"]["deadline_overrun"]) + def test_common_sensitive_fields_are_redacted_from_public_log(self): secret_lines = "\n".join([ 'refresh_token=refresh-secret', From 958a7903347d642a264ed75398433cacbf737049 Mon Sep 17 00:00:00 2001 From: Maisiqi <324751762+Maisiqi@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:56:58 +0800 Subject: [PATCH 12/18] fix(webfetch): retain malformed native text responses --- harness/web_fetch/native_projection.py | 6 +++- harness/web_fetch/providers/hosted_fetch.py | 5 ++- tests/fixtures/native-route-baseline.json | 36 ++++++++++----------- tests/test_hosted_fetch.py | 25 ++++++++++++++ 4 files changed, 52 insertions(+), 20 deletions(-) diff --git a/harness/web_fetch/native_projection.py b/harness/web_fetch/native_projection.py index 521dda3d8..3aece649a 100644 --- a/harness/web_fetch/native_projection.py +++ b/harness/web_fetch/native_projection.py @@ -337,7 +337,11 @@ def project_native_answer(payload, declaration, *, historical_scoring_source=Non if not isinstance(payload, bytes): raise ValueError('Native byte projection requires bytes') if kind == 'utf8': - return payload.decode('utf-8'), {} + try: + return payload.decode('utf-8'), {} + except UnicodeDecodeError: + # Invalid text remains raw evidence, never repaired answer text. + return '', {} if kind == 'empty': return '', {} raise ValueError('Unknown native byte projection') diff --git a/harness/web_fetch/providers/hosted_fetch.py b/harness/web_fetch/providers/hosted_fetch.py index 6ea8f7bb1..be012065b 100644 --- a/harness/web_fetch/providers/hosted_fetch.py +++ b/harness/web_fetch/providers/hosted_fetch.py @@ -136,7 +136,10 @@ def fetch(self, case: EvalCase, timeout: float, *, native_route_ledger=None) -> malformed = False service_error = False if self.type == "brightdata_unlocker": - content = response.body.decode("utf-8") if api_ok else "" + try: + content = response.body.decode("utf-8") if api_ok else "" + except UnicodeDecodeError: + malformed = True metadata["native_answer_projection"] = native_answer_projection( self.type, case, {}, response.status) metadata["upstream_http_status_available"] = False diff --git a/tests/fixtures/native-route-baseline.json b/tests/fixtures/native-route-baseline.json index e3dd208ed..ec02b0a0a 100644 --- a/tests/fixtures/native-route-baseline.json +++ b/tests/fixtures/native-route-baseline.json @@ -1,74 +1,74 @@ { "cloudflare_content:configured": { "cells": 192, - "sha256": "e3bdf3b4fe8018fe4a1a5df5bf13e87b8e33684c7336481f5807d82d97e60ea3" + "sha256": "a0d8995b7d325c226faff471e6b35bc119af467d28835768589caadc46390c13" }, "cloudflare_content:defaults": { "cells": 192, - "sha256": "e7890002343058cb61322c3525d06eee314273fde454027c226fba890fa9c576" + "sha256": "43d960a8a99e25c3c047f8fc6555b191371075274f4fc580ad9aaeb045957b51" }, "cloudflare_markdown:configured": { "cells": 192, - "sha256": "a5f0791fdccaa79f8b466a08eaf5f40324e7f140bc26bd08d80d4e00584d8964" + "sha256": "e050cc1bd52a9fa53a4998e86afa566f714e8c232fbe3424b3e47b746b0bc16f" }, "cloudflare_markdown:defaults": { "cells": 192, - "sha256": "a7e2bc24318ef7d241cdb251393d333bea84bf8a19130bac5a84ff806904d903" + "sha256": "9de2caab90c313bcd0807041dac3d7f042e543d30ad9af3e2ff9891e780c54ee" }, "cowagent_builtin:configured": { "cells": 192, - "sha256": "a98a3409674e91d9cebd86ace1760512906187a7f6644601bf76b39012f48b7b" + "sha256": "bdb0ede8be99af3458171f07185f762413b53723cbce9ce6fe8c75f1c03acc4b" }, "cowagent_builtin:defaults": { "cells": 192, - "sha256": "4c5ce7848d804995077fd79e8e62c0e8bbe7a3065350a326973b0f70c272b5d4" + "sha256": "f3171e1141469ebf39b1b1309a9ca2930122a600ef9b4d9823e9314eb3cd0386" }, "exa_contents:configured": { "cells": 192, - "sha256": "0e8e67119323a9f89ec9c55bd8a103bf9edd0c798b868a88fba7798ad1582e14" + "sha256": "caef107833355c2c9612d99182c47cb5c01a05debb9fc6e98137a80a43b9c4c3" }, "exa_contents:defaults": { "cells": 192, - "sha256": "d187f3cdc436823cdc1cd9adfe2ca62642a44ee94c86405c1377c4b2eda6b84b" + "sha256": "4b8b3fda3f0ddbcf7d367dfc42c81808b3d6e94747de6343f271c1b35e7446fb" }, "firecrawl:configured": { "cells": 192, - "sha256": "66e4ec865ebbe411a7889c93558dd14109888c421b619e7b994d644ce412e686" + "sha256": "da8306591a81fc59cac64f29174b5820da8831cff65f2594a5cd33a2d096de99" }, "firecrawl:defaults": { "cells": 192, - "sha256": "7919ff51ba35dad5a45353cdd4f72eb9b992d10f11b741c06f1ff63278b1289d" + "sha256": "212c4be7330a2c980412ac406074a46dfae359b52bc5e686879a0950cf482b91" }, "fixture:configured": { "cells": 192, - "sha256": "65e72a46085748150d84f40fb1c1714a83b7e8be04a766ae3ec84f4a0843d6c7" + "sha256": "0485c7e335a4b460efcd767df0fe17cc24dbed74a9948b45ab2d9099e33fd030" }, "fixture:defaults": { "cells": 192, - "sha256": "69519d867e35b2c1a3fc6607a3d08a7fc5058c31c0e8b621d3c3180e9ec1712d" + "sha256": "c86a9290a227a05dc49c544c8b180706fb39b047907fabf85c8f460dcfea793f" }, "moli_webfetch:configured": { "cells": 192, - "sha256": "8d0a49e5f81e835e24cff1a7d1312539dcb3be71670fd86c0ee00174bcb8dbf1" + "sha256": "6f8a8cb9a90a1f004d58c53cfc3fee4b84b0e91225d64b7357019102446be9be" }, "moli_webfetch:defaults": { "cells": 192, - "sha256": "3d216930373cdd0d09cbd7b21b3e16d6c00d40fab70790796429ed22a0de7432" + "sha256": "ff35523cc0072b55ac88b154a20bf1d35e05721d4f90f56853a969ca991f718e" }, "subprocess_jsonl:configured": { "cells": 192, - "sha256": "c9afc5dc6b1dbaac1d49b3fce1cac6d961d5d9a85e0cd1612a3b6e4572e78174" + "sha256": "fcc48fc3c6cb862dd2159c00c1d29ae923f4b8c4ad9cfd142522392994ece57a" }, "subprocess_jsonl:defaults": { "cells": 192, - "sha256": "335a4542cf955d1332695d53aebcbf343aaba12321f8b269b1ca9e8bd957d798" + "sha256": "eaa72e13f213b649e3da872e27b521ff50c84c9a6410f4406312ba4760147d7a" }, "tavily_extract:configured": { "cells": 192, - "sha256": "856cdf4b46072373512bc2c0393d69f7e46f0ae3748d6d9a349ca1c749ece86c" + "sha256": "8f16fbeaadc6ad0075ab3f9dcade71541a612a476fade02b2f25ebad2f77bbfb" }, "tavily_extract:defaults": { "cells": 192, - "sha256": "601b88ea41144e4b2891cb5d2ff5ac624d8ef2420813227d96b8ea9395fe5b78" + "sha256": "f3ab7c42705222a4d59e4d8269bf8e776924040c956537fee06a13e32d3eafa0" } } diff --git a/tests/test_hosted_fetch.py b/tests/test_hosted_fetch.py index 3b0a0bdab..a003a8276 100644 --- a/tests/test_hosted_fetch.py +++ b/tests/test_hosted_fetch.py @@ -37,6 +37,31 @@ def response(self, kind, text='Required article text', status=200): return HttpResponse(200, 'https://service.example/', {}, body) + @patch('harness.web_fetch.providers.hosted_fetch.request') + def test_brightdata_invalid_utf8_retains_response_evidence(self, call): + self.case.source_benchmark['input_media_kind'] = 'html_page' + raw = b'Invalid UTF-8: \xff' + call.return_value = HttpResponse(200, ENDPOINTS['brightdata_unlocker'], {}, raw) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + dataset = {'id': 'invalid-utf8', 'cases': [vars(self.case)]} + dataset_path = root / 'input.json' + dataset_path.write_text(json.dumps(dataset)) + records, _ = run_evaluation( + output_dir=root, dataset_meta=dataset, + dataset_path=dataset_path, cases=[self.case], + providers=[self.provider('brightdata_unlocker', uniform_native_parameters=True)], + provider_config_path=Path('providers.json'), runs=1, + timeout=30, delay=0, max_failure_retries=0) + record = records[0] + self.assertFalse(record['ok']) + self.assertEqual(record['error_type'], 'invalid_response') + self.assertEqual((root / record['artifact_path']).read_bytes(), b'') + metadata = record['provider_metadata'] + self.assertEqual((root / metadata['raw_response_path']).read_bytes(), raw) + self.assertEqual(metadata['raw_response_sha256'], hashlib.sha256(raw).hexdigest()) + call.assert_called_once() + @patch('harness.web_fetch.providers.hosted_fetch.request') def test_crawl4ai_cloud_errors_and_projection(self, call): from harness.web_fetch.native_projection import require_native_answer_projection From 7b0348910bf450cb77edd26ea590e5cc79c89ef3 Mon Sep 17 00:00:00 2001 From: Maisiqi <324751762+Maisiqi@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:01:01 +0800 Subject: [PATCH 13/18] fix(webfetch): validate malformed Crawl4AI option types --- harness/web_fetch/providers/hosted_fetch.py | 4 ++-- tests/test_hosted_fetch.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/harness/web_fetch/providers/hosted_fetch.py b/harness/web_fetch/providers/hosted_fetch.py index be012065b..8dd06affb 100644 --- a/harness/web_fetch/providers/hosted_fetch.py +++ b/harness/web_fetch/providers/hosted_fetch.py @@ -60,9 +60,9 @@ def validate(self) -> list[str]: if self.type == "browserbase_fetch" and not isinstance(self.config.get("proxies", False), bool): errors.append("proxies must be boolean") if self.type == "crawl4ai_cloud": - if self.config.get("engine", "browser") not in {"auto", "browser-light", "browser"}: + if self.config.get("engine", "browser") not in ("auto", "browser-light", "browser"): errors.append("unsupported engine") - if self.config.get("proxy", "none") not in {"none", "isp", "residential"}: + if self.config.get("proxy", "none") not in ("none", "isp", "residential"): errors.append("unsupported proxy") output_format = self.config.get("output_format", "markdown") if output_format != "markdown": diff --git a/tests/test_hosted_fetch.py b/tests/test_hosted_fetch.py index a003a8276..cba34f562 100644 --- a/tests/test_hosted_fetch.py +++ b/tests/test_hosted_fetch.py @@ -87,6 +87,11 @@ def test_crawl4ai_cloud_errors_and_projection(self, call): title=result.title,final_url=result.final_url,request_url=self.case.url) for option in ({'engine':'invalid'},{'proxy':'invalid'},{'output_format':'html'}): self.assertTrue(self.provider('crawl4ai_cloud',**option).validate()) + for field in ('engine', 'proxy'): + for value in ([], {}, None, True, 1): + with self.subTest(field=field, value=value): + self.assertIn('unsupported ' + field, + self.provider('crawl4ai_cloud', **{field: value}).validate()) @patch('harness.web_fetch.providers.hosted_fetch.request') def test_latest_article_track_preserves_native_answer_and_rejects_tampering(self, call): From 2e45c593047f7aaee07094b4163bdf1f75aeaec0 Mon Sep 17 00:00:00 2001 From: Maisiqi <324751762+Maisiqi@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:05:37 +0800 Subject: [PATCH 14/18] fix(webfetch): preserve explicit service failure classifications --- harness/web_fetch/providers/hosted_fetch.py | 4 ++-- tests/test_hosted_fetch.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/harness/web_fetch/providers/hosted_fetch.py b/harness/web_fetch/providers/hosted_fetch.py index 8dd06affb..7145e0a84 100644 --- a/harness/web_fetch/providers/hosted_fetch.py +++ b/harness/web_fetch/providers/hosted_fetch.py @@ -187,10 +187,10 @@ def fetch(self, case: EvalCase, timeout: float, *, native_route_ledger=None) -> "rate_limit" if response.status == 429 else "quota" if response.status == 402 else "api_error") message = f"Fetch API returned HTTP {response.status}; inspect retained response" - elif malformed: - error_type, message = "invalid_response", "Fetch API returned an invalid text response envelope" elif service_error: error_type, message = "api_error", "Fetch service reported an error or binary output; inspect retained response" + elif malformed: + error_type, message = "invalid_response", "Fetch API returned an invalid text response envelope" elif status is not None and not 200 <= status < 300: error_type, message = "http_error", f"Target page returned HTTP {status}" elif not content.strip(): diff --git a/tests/test_hosted_fetch.py b/tests/test_hosted_fetch.py index cba34f562..f1f25825c 100644 --- a/tests/test_hosted_fetch.py +++ b/tests/test_hosted_fetch.py @@ -37,6 +37,19 @@ def response(self, kind, text='Required article text', status=200): return HttpResponse(200, 'https://service.example/', {}, body) + @patch('harness.web_fetch.providers.hosted_fetch.request') + def test_explicit_service_errors_do_not_require_success_fields(self, call): + for kind in TYPES[1:]: + for payload, expected in (({'error': 'blocked'}, 'api_error'), + ({}, 'invalid_response')): + with self.subTest(kind=kind, payload=payload): + raw = json.dumps(payload).encode() + call.return_value = HttpResponse(200, ENDPOINTS[kind], {}, raw) + result = self.provider(kind).fetch(self.case, 30) + self.assertEqual(result.error_type, expected) + self.assertEqual(result.content, '') + self.assertEqual(base64.b64decode(result.metadata['_raw_response_b64']), raw) + @patch('harness.web_fetch.providers.hosted_fetch.request') def test_brightdata_invalid_utf8_retains_response_evidence(self, call): self.case.source_benchmark['input_media_kind'] = 'html_page' @@ -75,6 +88,8 @@ def test_crawl4ai_cloud_errors_and_projection(self, call): ({'markdown':''}, 200, 'empty_content'), ({'markdown':'body','success':False}, 200, 'api_error'), ({'markdown':'body','ok':False}, 200, 'api_error'), + ({'success':False,'error':'blocked'}, 200, 'api_error'), + ({'ok':False}, 200, 'api_error'), ]: call.reset_mock() raw=json.dumps(payload).encode() From b357e1227510bc593427aaf5570391081d44c97c Mon Sep 17 00:00:00 2001 From: Maisiqi <324751762+Maisiqi@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:17:09 +0800 Subject: [PATCH 15/18] fix(webfetch): normalize disclosed origin failures before scoring --- docs/provider-contract.md | 7 +++ harness/web_fetch/call_boundary.py | 10 ++++ harness/web_fetch/providers/cloudflare.py | 4 ++ harness/web_fetch/providers/exa.py | 2 + harness/web_fetch/providers/firecrawl.py | 1 + harness/web_fetch/providers/hosted_fetch.py | 1 + harness/web_fetch/providers/moli.py | 2 + harness/web_fetch/providers/tavily.py | 2 + tests/test_selected_native_policy.py | 57 +++++++++++++++++++++ 9 files changed, 86 insertions(+) diff --git a/docs/provider-contract.md b/docs/provider-contract.md index 9f775201e..892cc84a1 100644 --- a/docs/provider-contract.md +++ b/docs/provider-contract.md @@ -14,6 +14,13 @@ fetch(case: EvalCase, timeout: float) -> ProviderResult - provider-specific request ID, engine, server elapsed time and billing data; - a stable error type and actionable error message. +Disclose service API status as `metadata.api_http_status` and the final target-page +status as `metadata.upstream_status`; use `None` when unavailable. Do not copy an +API status into the target status. Before scoring and retry selection, the shared +answer boundary marks a known non-2xx target response as a failed call, preserving +its returned content and existing failure details. Content acceptance and call +success remain separate requirements; historical results are not rewritten. + Credentials are expanded from environment placeholders in the provider config. They must never be added to `metadata`, content, manifest or logs. diff --git a/harness/web_fetch/call_boundary.py b/harness/web_fetch/call_boundary.py index 55d510c21..4f3f59bd1 100644 --- a/harness/web_fetch/call_boundary.py +++ b/harness/web_fetch/call_boundary.py @@ -11,6 +11,16 @@ def prepare_answer(result, case): + # A successful service request is not proof that its target page succeeded. + # Adapters disclose origin status separately; never infer it from API status. + origin = result.metadata.get('upstream_status') + origin = origin if type(origin) is int and 100 <= origin <= 599 else None + result.metadata['upstream_status'] = origin + result.metadata['upstream_http_status_available'] = origin is not None + if result.ok and origin is not None and not 200 <= origin < 300: + result.ok = False + result.error_type = 'http_error' + result.error_message = f'Target page returned HTTP {origin}' # Prepare scorer input; fixed-native calls have already stopped their timer. result.prepared_answer = answer_input(result.content.encode('utf-8', errors='replace'), result.metadata, ok=result.ok, status_code=result.status_code) diff --git a/harness/web_fetch/providers/cloudflare.py b/harness/web_fetch/providers/cloudflare.py index d516a2310..26fca6179 100644 --- a/harness/web_fetch/providers/cloudflare.py +++ b/harness/web_fetch/providers/cloudflare.py @@ -453,6 +453,7 @@ def _fetch_payload(self, case: EvalCase, timeout: float, request_body: dict | No "_raw_response_b64": base64.b64encode(response.body).decode("ascii"), "raw_response_sha256": hashlib.sha256(response.body).hexdigest(), "raw_response_bytes": len(response.body), + "api_http_status": response.status, **({'native_answer_projection': native_answer_projection('cloudflare_markdown', case, payload, response.status, interface='/markdown')} if (envelope_success and isinstance(payload.get('result', ''), str)) or meta else {}), "route": route, @@ -532,6 +533,7 @@ def _fetch_media_resources( "_raw_response_b64": base64.b64encode(response.body).decode("ascii"), "raw_response_sha256": hashlib.sha256(response.body).hexdigest(), "raw_response_bytes": len(response.body), + "api_http_status": response.status, "route": "browser_rendering_scrape_resources", **({'native_answer_projection': (native_answer_projection('cloudflare_markdown', case, payload, response.status))} if envelope_success or meta else {}), @@ -710,6 +712,7 @@ def _fetch_direct_document( "_raw_response_b64": base64.b64encode(response.body).decode("ascii"), "raw_response_sha256": hashlib.sha256(response.body).hexdigest(), "raw_response_bytes": len(response.body), + "api_http_status": response.status, **({'native_answer_projection': {'schema_version': 2, 'content_pointer': ['result'], 'content_format': 'text', 'optional_content': True, 'result_fields': {'final_url': {'format': 'transport_response_url'}}}} @@ -777,6 +780,7 @@ def fetch(self, case: EvalCase, timeout: float, *, native_route_ledger=None) -> "_raw_response_b64": base64.b64encode(response.body).decode("ascii"), "raw_response_sha256": hashlib.sha256(response.body).hexdigest(), "raw_response_bytes": len(response.body), + "api_http_status": response.status, **({'native_answer_projection': native_answer_projection('cloudflare_content', case, payload, response.status)} if (envelope_success and isinstance(payload.get('result', ''), str)) or meta else {}), "output_contract": "rendered_html", diff --git a/harness/web_fetch/providers/exa.py b/harness/web_fetch/providers/exa.py index 6a6b057df..2de2f8cf0 100644 --- a/harness/web_fetch/providers/exa.py +++ b/harness/web_fetch/providers/exa.py @@ -143,6 +143,8 @@ def fetch(self, case: EvalCase, timeout: float, *, native_route_ledger=None) -> "_raw_response_b64": base64.b64encode(response.body).decode("ascii"), "raw_response_sha256": hashlib.sha256(response.body).hexdigest(), "raw_response_bytes": len(response.body), + "api_http_status": response.status, + "upstream_status": None, "empty_content": 200 <= response.status < 300 and not content, "request_id": payload.get("requestId"), "url_status": public_status, diff --git a/harness/web_fetch/providers/firecrawl.py b/harness/web_fetch/providers/firecrawl.py index 541e77455..f1565de07 100644 --- a/harness/web_fetch/providers/firecrawl.py +++ b/harness/web_fetch/providers/firecrawl.py @@ -400,6 +400,7 @@ def fetch(self, case: EvalCase, timeout: float, *, native_route_ledger=None) -> "_raw_response_b64": base64.b64encode(response.body).decode("ascii"), "raw_response_sha256": hashlib.sha256(response.body).hexdigest(), "raw_response_bytes": len(response.body), + "api_http_status": response.status, "upstream_status": ( metadata.get("statusCode") if isinstance(metadata.get("statusCode"), int) else None diff --git a/harness/web_fetch/providers/hosted_fetch.py b/harness/web_fetch/providers/hosted_fetch.py index 7145e0a84..ff950d14c 100644 --- a/harness/web_fetch/providers/hosted_fetch.py +++ b/harness/web_fetch/providers/hosted_fetch.py @@ -209,6 +209,7 @@ def fetch(self, case: EvalCase, timeout: float, *, native_route_ledger=None) -> metadata["render"] = self.config.get("render") if self.type == "browser_use_fetch": metadata["proxy_country"] = body["proxy_country"] + metadata['upstream_status'] = status return ProviderResult( ok=error_type is None, content=content, status_code=status, final_url=final_url, title=title if isinstance(title, str) else None, diff --git a/harness/web_fetch/providers/moli.py b/harness/web_fetch/providers/moli.py index fdda5dbb9..a68dcd2fc 100644 --- a/harness/web_fetch/providers/moli.py +++ b/harness/web_fetch/providers/moli.py @@ -775,6 +775,8 @@ def fetch(self, case: EvalCase, timeout: float, *, native_route_ledger=None) -> ) self._retain_source(metadata, raw, case) native_status, readiness_timed_out = _native_fetch_diagnostics(diagnostic_raw, case.url) + metadata['api_http_status'] = None + metadata['upstream_status'] = native_status http_context = (replay_native_http_error_context(raw, diagnostic_raw, case.url, process.returncode, {'schema_version': 1, 'format': 'fetch_log_http_error'}) if process.returncode == 0 and native_status is not None and 400 <= native_status < 600 else None) diff --git a/harness/web_fetch/providers/tavily.py b/harness/web_fetch/providers/tavily.py index 8e1429028..da2f36e35 100644 --- a/harness/web_fetch/providers/tavily.py +++ b/harness/web_fetch/providers/tavily.py @@ -92,6 +92,8 @@ def fetch(self, case: EvalCase, timeout: float, *, native_route_ledger=None) -> "_raw_response_b64": base64.b64encode(response.body).decode("ascii"), "raw_response_sha256": hashlib.sha256(response.body).hexdigest(), "raw_response_bytes": len(response.body), + "api_http_status": response.status, + "upstream_status": None, "empty_content": 200 <= response.status < 300 and not content, "response_time": ( payload.get("response_time") diff --git a/tests/test_selected_native_policy.py b/tests/test_selected_native_policy.py index d3a781eb7..8ba5e6b0e 100644 --- a/tests/test_selected_native_policy.py +++ b/tests/test_selected_native_policy.py @@ -169,3 +169,60 @@ def test_configured_cli_selection_reaches_runner(tmp_path): assert [p.name for p in run.call_args.kwargs['providers']] == ['fixture'] assert run.call_args.kwargs['timeout'] == 30 assert run.call_args.kwargs['max_failure_retries'] == 1 + + +@pytest.mark.parametrize('name', SELECTED) +@pytest.mark.parametrize('origin_status', [200, 404, None]) +def test_selected_origin_status_policy_controls_acceptance_and_retry(tmp_path, name, origin_status): + import hashlib + import sys + from lexbench_webfetch.metrics import task_passed + p, c = provider(name), case() + exposed = name in {'moli-webfetch', 'firecrawl', 'cloudflare-markdown', + 'browser-use-fetch', 'browserbase', 'crawl4ai-cloud'} + if name == 'moli-webfetch': + binary = tmp_path / 'moli-fixture' + event = (f'DEBUG fetch request complete method=GET url={c.url} ' + f'final_url={c.url} status={origin_status}') if origin_status is not None else '' + binary.write_text(f'#!{sys.executable}\nimport sys\n' + 'if sys.argv[1:] == ["--version"]:\n print("moli fixture"); sys.exit(0)\n' + f'print("Expected")\nprint({event!r}, file=sys.stderr)\n') + binary.chmod(0o700) + p = build_provider(name, {**p.config, 'binary': str(binary), + 'expected_reported_version': 'moli fixture', + 'expected_binary_sha256': hashlib.sha256(binary.read_bytes()).hexdigest()}) + payloads = { + 'firecrawl': {'success': True, 'data': {'markdown': 'Expected', 'metadata': {'statusCode': origin_status}}}, + 'cloudflare-markdown': {'success': True, 'result': 'Expected', 'meta': {'status': origin_status}}, + 'tavily': {'results': [{'url': c.url, 'raw_content': 'Expected'}]}, + 'exa': {'results': [{'url': c.url, 'text': 'Expected'}]}, + 'browser-use-fetch': {'body': 'Expected', 'status_code': origin_status, 'final_url': c.url}, + 'browserbase': {'content': 'Expected', 'statusCode': origin_status}, + 'deepseek-fetch-api': {'content': 'Expected', 'title': 'Title'}, + 'crawl4ai-cloud': {'markdown': 'Expected', 'origin_status': origin_status}, + } + # The selected Browser Use and Browserbase schemas require status on success; + # missing status is malformed rather than an invented origin error. + malformed = origin_status is None and name in {'browser-use-fetch', 'browserbase'} + raw = b'Expected' if name == 'brightdata' else json.dumps(payloads.get(name, {})).encode() + def respond(request, **kwargs): + return Mock(status=200, headers={}, read=Mock(return_value=raw), + geturl=Mock(return_value=request.full_url)) + with patch('harness.web_fetch.network_preflight.socket.getaddrinfo', + return_value=[(2, 1, 6, '', ('93.184.216.34', 443))]), \ + patch('harness.web_fetch.providers.http_client.urllib.request.build_opener') as opener: + opener.return_value.open.side_effect = respond + rows, _ = run_evaluation(output_dir=tmp_path / 'run', + dataset_meta={'id': 'origin-control', 'cases': [asdict(c)]}, + dataset_path=tmp_path / 'dataset.json', cases=[c], providers=[p], + provider_config_path=tmp_path / 'providers.json', runs=1, + timeout=30, delay=0, max_failure_retries=1) + row = rows[0] + expected_pass = not ((exposed and origin_status == 404) or malformed) + assert task_passed(row['ok'], row['metrics'], c.task_kind) == expected_pass + metadata = row['provider_metadata'] + assert metadata['upstream_status'] == (origin_status if exposed else None) + assert metadata['api_http_status'] == (None if name == 'moli-webfetch' else 200) + assert metadata['upstream_http_status_available'] == (exposed and origin_status is not None) + assert len(list((tmp_path / 'run' / 'call-logs').rglob('*.json'))) == (1 if expected_pass else 2) + assert 'Expected' in (tmp_path / 'run' / row['artifact_path']).read_text() From cc35b96078c394e4f3d2cfbb037a333fbcfab2ad Mon Sep 17 00:00:00 2001 From: Maisiqi <324751762+Maisiqi@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:29:06 +0800 Subject: [PATCH 16/18] fix(webfetch): preserve malformed hosted JSON response evidence --- harness/web_fetch/providers/hosted_fetch.py | 3 +- tests/test_hosted_fetch.py | 48 +++++++++++---------- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/harness/web_fetch/providers/hosted_fetch.py b/harness/web_fetch/providers/hosted_fetch.py index 7f24f7c2f..9993d559a 100644 --- a/harness/web_fetch/providers/hosted_fetch.py +++ b/harness/web_fetch/providers/hosted_fetch.py @@ -3,6 +3,7 @@ import base64 import hashlib +import json import uuid from lexbench_webfetch.models import EvalCase @@ -145,7 +146,7 @@ def fetch(self, case: EvalCase, timeout: float, *, native_route_ledger=None) -> metadata["upstream_http_status_available"] = False else: try: - payload = response.json() + payload = json.loads(response.body.decode("utf-8")) except ValueError: payload = None if not isinstance(payload, dict): diff --git a/tests/test_hosted_fetch.py b/tests/test_hosted_fetch.py index f1f25825c..8176ad24b 100644 --- a/tests/test_hosted_fetch.py +++ b/tests/test_hosted_fetch.py @@ -51,29 +51,33 @@ def test_explicit_service_errors_do_not_require_success_fields(self, call): self.assertEqual(base64.b64decode(result.metadata['_raw_response_b64']), raw) @patch('harness.web_fetch.providers.hosted_fetch.request') - def test_brightdata_invalid_utf8_retains_response_evidence(self, call): + def test_hosted_invalid_utf8_retains_response_evidence(self, call): self.case.source_benchmark['input_media_kind'] = 'html_page' - raw = b'Invalid UTF-8: \xff' - call.return_value = HttpResponse(200, ENDPOINTS['brightdata_unlocker'], {}, raw) - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - dataset = {'id': 'invalid-utf8', 'cases': [vars(self.case)]} - dataset_path = root / 'input.json' - dataset_path.write_text(json.dumps(dataset)) - records, _ = run_evaluation( - output_dir=root, dataset_meta=dataset, - dataset_path=dataset_path, cases=[self.case], - providers=[self.provider('brightdata_unlocker', uniform_native_parameters=True)], - provider_config_path=Path('providers.json'), runs=1, - timeout=30, delay=0, max_failure_retries=0) - record = records[0] - self.assertFalse(record['ok']) - self.assertEqual(record['error_type'], 'invalid_response') - self.assertEqual((root / record['artifact_path']).read_bytes(), b'') - metadata = record['provider_metadata'] - self.assertEqual((root / metadata['raw_response_path']).read_bytes(), raw) - self.assertEqual(metadata['raw_response_sha256'], hashlib.sha256(raw).hexdigest()) - call.assert_called_once() + for kind in TYPES: + with self.subTest(kind=kind): + call.reset_mock() + raw = (b'Invalid UTF-8: \xff' if kind == 'brightdata_unlocker' else + self.response(kind, text='BROKEN').body.replace(b'BROKEN', b'\xff')) + call.return_value = HttpResponse(200, ENDPOINTS[kind], {}, raw) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + dataset = {'id': 'invalid-utf8', 'cases': [vars(self.case)]} + dataset_path = root / 'input.json' + dataset_path.write_text(json.dumps(dataset)) + records, _ = run_evaluation( + output_dir=root, dataset_meta=dataset, + dataset_path=dataset_path, cases=[self.case], + providers=[self.provider(kind, uniform_native_parameters=True)], + provider_config_path=Path('providers.json'), runs=1, + timeout=30, delay=0, max_failure_retries=0) + record = records[0] + self.assertFalse(record['ok']) + self.assertEqual(record['error_type'], 'invalid_response') + self.assertEqual((root / record['artifact_path']).read_bytes(), b'') + metadata = record['provider_metadata'] + self.assertEqual((root / metadata['raw_response_path']).read_bytes(), raw) + self.assertEqual(metadata['raw_response_sha256'], hashlib.sha256(raw).hexdigest()) + call.assert_called_once() @patch('harness.web_fetch.providers.hosted_fetch.request') def test_crawl4ai_cloud_errors_and_projection(self, call): From a9c7429128025dd483704cb303e6b9fafb38f5b7 Mon Sep 17 00:00:00 2001 From: Maisiqi <324751762+Maisiqi@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:55:34 +0800 Subject: [PATCH 17/18] fix(webfetch): align all native routes and evidence decoding --- harness/web_fetch/native_routes.py | 6 ++++- harness/web_fetch/providers/cloudflare.py | 3 ++- harness/web_fetch/providers/hosted_fetch.py | 5 ++-- harness/web_fetch/providers/http_client.py | 5 ++++ harness/web_fetch/providers/moli.py | 10 ++++---- tests/test_hosted_fetch.py | 12 +++++++-- tests/test_selected_native_policy.py | 28 +++++++++++++++++---- 7 files changed, 52 insertions(+), 17 deletions(-) diff --git a/harness/web_fetch/native_routes.py b/harness/web_fetch/native_routes.py index c66353911..021577b91 100644 --- a/harness/web_fetch/native_routes.py +++ b/harness/web_fetch/native_routes.py @@ -23,6 +23,7 @@ require_native_projection_schema, ) from harness.web_fetch.publication_privacy import read_evidence_binding +from harness.web_fetch.providers.http_client import parse_utf8_json from harness.web_fetch.retrieval_hints import native_default_case INPUT_MEDIA_KINDS = frozenset({ @@ -494,7 +495,10 @@ def _expected_native_declaration(root, record, raw, cases_by_id): if case is None or case.get('url') != record.get('url'): raise ValueError('Native answer selection lacks its frozen request identity') try: - payload = json.loads(raw) if record['provider_type'] not in {'moli_webfetch', 'brightdata_unlocker'} else {} + if record['provider_type'] in {'browser_use_fetch', 'browserbase_fetch', 'deepseek_fetch', 'crawl4ai_cloud'}: + payload = parse_utf8_json(raw) + else: + payload = json.loads(raw) if record['provider_type'] not in {'moli_webfetch', 'brightdata_unlocker'} else {} except (ValueError, UnicodeDecodeError): payload = None if record['provider_type'] in {'browser_use_fetch', 'browserbase_fetch', 'deepseek_fetch', 'crawl4ai_cloud'}: diff --git a/harness/web_fetch/providers/cloudflare.py b/harness/web_fetch/providers/cloudflare.py index 6b9046ffe..cba8d96c5 100644 --- a/harness/web_fetch/providers/cloudflare.py +++ b/harness/web_fetch/providers/cloudflare.py @@ -524,7 +524,7 @@ def _fetch_media_resources( error_type, error_message = _interface_error( response_status=response.status, envelope_success=envelope_success, - origin_status=None, + origin_status=_origin_status(meta.get('status')), ) errors = payload.get("errors") errors = errors if isinstance(errors, list) else [] @@ -542,6 +542,7 @@ def _fetch_media_resources( "raw_response_bytes": len(response.body), "api_http_status": response.status, "route": "browser_rendering_scrape_resources", + "upstream_status": _origin_status(meta.get('status')), **({'native_answer_projection': (native_answer_projection('cloudflare_markdown', case, payload, response.status))} if envelope_success or meta else {}), "browser": self.config.get("browser"), diff --git a/harness/web_fetch/providers/hosted_fetch.py b/harness/web_fetch/providers/hosted_fetch.py index 9993d559a..191e0814e 100644 --- a/harness/web_fetch/providers/hosted_fetch.py +++ b/harness/web_fetch/providers/hosted_fetch.py @@ -3,7 +3,6 @@ import base64 import hashlib -import json import uuid from lexbench_webfetch.models import EvalCase @@ -12,7 +11,7 @@ from harness.web_fetch.native_projection import native_answer_projection from harness.web_fetch.network_dns import public_dns_context, validate_dns_mode from harness.web_fetch.providers.base import Provider -from harness.web_fetch.providers.http_client import inner_timeout_budget, request +from harness.web_fetch.providers.http_client import inner_timeout_budget, parse_utf8_json, request ENDPOINTS = { @@ -146,7 +145,7 @@ def fetch(self, case: EvalCase, timeout: float, *, native_route_ledger=None) -> metadata["upstream_http_status_available"] = False else: try: - payload = json.loads(response.body.decode("utf-8")) + payload = parse_utf8_json(response.body) except ValueError: payload = None if not isinstance(payload, dict): diff --git a/harness/web_fetch/providers/http_client.py b/harness/web_fetch/providers/http_client.py index 7fe1132b1..196ffbf2d 100644 --- a/harness/web_fetch/providers/http_client.py +++ b/harness/web_fetch/providers/http_client.py @@ -13,6 +13,11 @@ from harness.web_fetch.diagnostics import emit +def parse_utf8_json(body: bytes) -> Any: + """Use the same strict wire encoding for acquisition and evidence replay.""" + return json.loads(body.decode('utf-8')) + + @dataclass class HttpResponse: status: int diff --git a/harness/web_fetch/providers/moli.py b/harness/web_fetch/providers/moli.py index 7fa3df79d..037093a63 100644 --- a/harness/web_fetch/providers/moli.py +++ b/harness/web_fetch/providers/moli.py @@ -752,6 +752,11 @@ def fetch(self, case: EvalCase, timeout: float, *, native_route_ledger=None) -> error_message="Moli fetch exceeded time budget", metadata=metadata, ) + native_status, readiness_timed_out = _native_fetch_diagnostics(diagnostic_raw, case.url) + metadata['api_http_status'] = None + metadata['upstream_status'] = native_status + if native_status is not None: + metadata['http_status_source'] = 'same_call_native_document_event' if raw_task: self._retain_source(metadata, raw, case) if route == "native_document_bytes": @@ -776,14 +781,9 @@ def fetch(self, case: EvalCase, timeout: float, *, native_route_ledger=None) -> metadata=metadata, ) self._retain_source(metadata, raw, case) - native_status, readiness_timed_out = _native_fetch_diagnostics(diagnostic_raw, case.url) - metadata['api_http_status'] = None - metadata['upstream_status'] = native_status http_context = (replay_native_http_error_context(raw, diagnostic_raw, case.url, process.returncode, {'schema_version': 1, 'format': 'fetch_log_http_error'}) if process.returncode == 0 and native_status is not None and 400 <= native_status < 600 else None) - if native_status is not None: - metadata["http_status_source"] = "same_call_native_document_event" if readiness_timed_out: metadata["native_readiness_timed_out"] = True if process.returncode == 0 and not content.strip() and native_status not in range(400, 600): diff --git a/tests/test_hosted_fetch.py b/tests/test_hosted_fetch.py index 8176ad24b..241e7ced8 100644 --- a/tests/test_hosted_fetch.py +++ b/tests/test_hosted_fetch.py @@ -53,11 +53,17 @@ def test_explicit_service_errors_do_not_require_success_fields(self, call): @patch('harness.web_fetch.providers.hosted_fetch.request') def test_hosted_invalid_utf8_retains_response_evidence(self, call): self.case.source_benchmark['input_media_kind'] = 'html_page' - for kind in TYPES: - with self.subTest(kind=kind): + variants = [(kind, encoding) for kind in TYPES for encoding in + (['invalid'] if kind == 'brightdata_unlocker' else ['invalid', 'bom', 'utf16'])] + for kind, encoding in variants: + with self.subTest(kind=kind, encoding=encoding): call.reset_mock() raw = (b'Invalid UTF-8: \xff' if kind == 'brightdata_unlocker' else self.response(kind, text='BROKEN').body.replace(b'BROKEN', b'\xff')) + if encoding == 'bom': + raw = b'\xef\xbb\xbf' + self.response(kind).body + elif encoding == 'utf16': + raw = self.response(kind).body.decode().encode('utf-16') call.return_value = HttpResponse(200, ENDPOINTS[kind], {}, raw) with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -77,6 +83,8 @@ def test_hosted_invalid_utf8_retains_response_evidence(self, call): metadata = record['provider_metadata'] self.assertEqual((root / metadata['raw_response_path']).read_bytes(), raw) self.assertEqual(metadata['raw_response_sha256'], hashlib.sha256(raw).hexdigest()) + from harness.web_fetch.artifacts import verify_record_artifacts + verify_record_artifacts(root, record) call.assert_called_once() @patch('harness.web_fetch.providers.hosted_fetch.request') diff --git a/tests/test_selected_native_policy.py b/tests/test_selected_native_policy.py index 8ba5e6b0e..6b6c6abb4 100644 --- a/tests/test_selected_native_policy.py +++ b/tests/test_selected_native_policy.py @@ -171,13 +171,24 @@ def test_configured_cli_selection_reaches_runner(tmp_path): assert run.call_args.kwargs['max_failure_retries'] == 1 -@pytest.mark.parametrize('name', SELECTED) +@pytest.mark.parametrize('name,route', [(name, 'body') for name in SELECTED] + + [('cloudflare-markdown', 'image'), ('cloudflare-markdown', 'video'), + ('moli-webfetch', 'file')]) @pytest.mark.parametrize('origin_status', [200, 404, None]) -def test_selected_origin_status_policy_controls_acceptance_and_retry(tmp_path, name, origin_status): +def test_selected_origin_status_policy_controls_acceptance_and_retry(tmp_path, name, route, origin_status): import hashlib import sys from lexbench_webfetch.metrics import task_passed p, c = provider(name), case() + file_bytes = b'\x89PNG\r\n\x1a\n' + b'fixture bytes' * 8 + if route in {'image', 'video'}: + c = replace(c, track=route, task_kind='resource', expected_any=[], + expected_resources=[{'urls': ['https://example.test/asset.png' if route == 'image' + else 'https://example.test/asset.mp4'], 'kind': route}]) + elif route == 'file': + c = replace(c, task_kind='file', expected_any=[], + source_benchmark={'input_media_kind': 'direct_image'}, + source_verification={'mime': 'image/png', 'file_sha256': hashlib.sha256(file_bytes).hexdigest()}) exposed = name in {'moli-webfetch', 'firecrawl', 'cloudflare-markdown', 'browser-use-fetch', 'browserbase', 'crawl4ai-cloud'} if name == 'moli-webfetch': @@ -185,8 +196,9 @@ def test_selected_origin_status_policy_controls_acceptance_and_retry(tmp_path, n event = (f'DEBUG fetch request complete method=GET url={c.url} ' f'final_url={c.url} status={origin_status}') if origin_status is not None else '' binary.write_text(f'#!{sys.executable}\nimport sys\n' - 'if sys.argv[1:] == ["--version"]:\n print("moli fixture"); sys.exit(0)\n' - f'print("Expected")\nprint({event!r}, file=sys.stderr)\n') + 'if sys.argv[1:] == ["--version"]:\n print("moli fixture"); sys.exit(0)\n' + + (f'sys.stdout.buffer.write({file_bytes!r})\n' if route == 'file' else 'print("Expected")\n') + + f'print({event!r}, file=sys.stderr)\n') binary.chmod(0o700) p = build_provider(name, {**p.config, 'binary': str(binary), 'expected_reported_version': 'moli fixture', @@ -201,6 +213,9 @@ def test_selected_origin_status_policy_controls_acceptance_and_retry(tmp_path, n 'deepseek-fetch-api': {'content': 'Expected', 'title': 'Title'}, 'crawl4ai-cloud': {'markdown': 'Expected', 'origin_status': origin_status}, } + if route in {'image', 'video'}: + payloads[name]['result'] = [{'results': [{'attributes': [ + {'name': 'src', 'value': c.expected_resources[0]['urls'][0]}]}]}] # The selected Browser Use and Browserbase schemas require status on success; # missing status is malformed rather than an invented origin error. malformed = origin_status is None and name in {'browser-use-fetch', 'browserbase'} @@ -225,4 +240,7 @@ def respond(request, **kwargs): assert metadata['api_http_status'] == (None if name == 'moli-webfetch' else 200) assert metadata['upstream_http_status_available'] == (exposed and origin_status is not None) assert len(list((tmp_path / 'run' / 'call-logs').rglob('*.json'))) == (1 if expected_pass else 2) - assert 'Expected' in (tmp_path / 'run' / row['artifact_path']).read_text() + from harness.web_fetch.artifacts import verify_record_artifacts + verify_record_artifacts(tmp_path / 'run', row) + if route == 'body': + assert 'Expected' in (tmp_path / 'run' / row['artifact_path']).read_text() From 28a12de88d0fe523affa66b3de39d992c536694a Mon Sep 17 00:00:00 2001 From: Maisiqi <324751762+Maisiqi@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:58:23 +0800 Subject: [PATCH 18/18] fix(webfetch): keep evidence decoding independent of provider imports --- harness/record_io.py | 5 +++++ harness/web_fetch/native_routes.py | 3 +-- harness/web_fetch/providers/hosted_fetch.py | 3 ++- harness/web_fetch/providers/http_client.py | 5 ----- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/harness/record_io.py b/harness/record_io.py index bbde73baf..fbe341d82 100644 --- a/harness/record_io.py +++ b/harness/record_io.py @@ -4,6 +4,11 @@ from pathlib import Path +def parse_utf8_json(body: bytes): + """Use the same strict wire encoding for acquisition and evidence replay.""" + return json.loads(body.decode('utf-8')) + + def file_sha256(path): """Hash file bytes without loading the entire artifact into memory. diff --git a/harness/web_fetch/native_routes.py b/harness/web_fetch/native_routes.py index 021577b91..ddee6e8b1 100644 --- a/harness/web_fetch/native_routes.py +++ b/harness/web_fetch/native_routes.py @@ -12,7 +12,7 @@ from lexbench_webfetch.file_acceptance import evaluate_files from lexbench_webfetch.paths import contained_path -from harness.record_io import canonical_json_sha256, file_sha256 +from harness.record_io import canonical_json_sha256, file_sha256, parse_utf8_json from harness.web_fetch.config import DEFAULT_CALL_TIMEOUT_SECONDS from harness.web_fetch.native_projection import ( SCORE_ANSWER_FIELDS, @@ -23,7 +23,6 @@ require_native_projection_schema, ) from harness.web_fetch.publication_privacy import read_evidence_binding -from harness.web_fetch.providers.http_client import parse_utf8_json from harness.web_fetch.retrieval_hints import native_default_case INPUT_MEDIA_KINDS = frozenset({ diff --git a/harness/web_fetch/providers/hosted_fetch.py b/harness/web_fetch/providers/hosted_fetch.py index 191e0814e..cc0aa6eef 100644 --- a/harness/web_fetch/providers/hosted_fetch.py +++ b/harness/web_fetch/providers/hosted_fetch.py @@ -6,12 +6,13 @@ import uuid from lexbench_webfetch.models import EvalCase +from harness.record_io import parse_utf8_json from harness.web_fetch.models import ProviderResult from harness.web_fetch.native_routes import NativeRoute from harness.web_fetch.native_projection import native_answer_projection from harness.web_fetch.network_dns import public_dns_context, validate_dns_mode from harness.web_fetch.providers.base import Provider -from harness.web_fetch.providers.http_client import inner_timeout_budget, parse_utf8_json, request +from harness.web_fetch.providers.http_client import inner_timeout_budget, request ENDPOINTS = { diff --git a/harness/web_fetch/providers/http_client.py b/harness/web_fetch/providers/http_client.py index 196ffbf2d..7fe1132b1 100644 --- a/harness/web_fetch/providers/http_client.py +++ b/harness/web_fetch/providers/http_client.py @@ -13,11 +13,6 @@ from harness.web_fetch.diagnostics import emit -def parse_utf8_json(body: bytes) -> Any: - """Use the same strict wire encoding for acquisition and evidence replay.""" - return json.loads(body.decode('utf-8')) - - @dataclass class HttpResponse: status: int