diff --git a/mcp_server/README.md b/mcp_server/README.md index 3ab9d90..d9f315a 100644 --- a/mcp_server/README.md +++ b/mcp_server/README.md @@ -21,10 +21,18 @@ 成功返回 JSON（Pydantic）：`markdown_path`、`pages_tree_path`、`content_preview`（默认 2000 字）、`message` 等。 调用方应以返回的 `output_dir` 为准；服务不会复用或删除已有目录。 +本地 PDF 固定通过 `trigger_file` 直接 multipart 上传，不使用 TOS 或自动回退。MCP 触发时固定发送 +`token=None` 和 `server_generated_token=True`；同步请求使用 `(60, 1860)`，异步请求使用 `(60, 60)`， +只有服务端确认的 token 才能进入结果轮询。 + > 出于安全原因，根目录、HOME、当前工作目录及 Git 元数据目录不能作为首选输出目录。 健康检查、版本查询、按 token 手动恢复请使用 CLI：`uniparser health`、`uniparser version`、`uniparser fetch`。 +失败返回的 `error.code` 可能为 `CONFIG_ERROR`、`INPUT_ERROR`、`UPLOAD_ERROR`、`PARSE_ERROR` 或 +`TOKEN_NOT_FOUND`。触发失败时可能保留标准 `token` 供诊断，但 MCP 不会使用它或进入恢复流程；只有 +成功触发返回的 token 才会被轮询。`undefined` 最多检查三次，不会持续等待 1800 秒。 + ## 环境变量 | 变量 | 说明 | 默认 | @@ -81,5 +89,5 @@ uv run pytest tests/ -v `UNIPARSER_BASE_URL` 可省略（默认云服务）；本地自托管时设置为 `http://127.0.0.1:40001` 等。 -同步解析使用 `UniParserClient.sync_request_timeout`，轮询和结果获取使用 -`request_timeout`；二者均为客户端 HTTP 超时，与服务端解析预算相互独立。 +本地 PDF 同步直传显式使用 `(60, 1860)`，异步直传显式使用 `(60, 60)`；轮询和结果获取使用 +`UniParserClient.request_timeout`。这些都是客户端 HTTP 超时，与服务端解析预算相互独立。 diff --git a/mcp_server/tests/test_pipeline_parse.py b/mcp_server/tests/test_pipeline_parse.py index a88fdaf..76b7b13 100644 --- a/mcp_server/tests/test_pipeline_parse.py +++ b/mcp_server/tests/test_pipeline_parse.py @@ -1,4 +1,5 @@ import asyncio +import json from pathlib import Path from unittest.mock import MagicMock @@ -6,34 +7,29 @@ from uniparser_mcp.schemas import ParseRequest -def test_duplicate_triggers_internal_complete(tmp_path: Path): +def test_failed_trigger_does_not_attempt_token_recovery(tmp_path: Path): pdf = tmp_path / "paper.pdf" pdf.write_bytes(b"%PDF-1.4") client = MagicMock() - client.to_token.return_value = "tok123" client.trigger_file.return_value = { "status": "error", - "message": "Token is duplicated", - "token": "tok123", - } - client.get_result.side_effect = [ - {"status": "success"}, - {"status": "success", "pages_tree": True}, - ] - client.get_formatted.return_value = { - "status": "success", - "content": "# Hello", + "token": "diagnostic-token", + "message": "trigger failed", } - req = ParseRequest(file_path=str(pdf), output_dir=str(tmp_path / "out")) + out = tmp_path / "out" + req = ParseRequest(file_path=str(pdf), output_dir=str(out)) result = asyncio.run(run_parse(client, req)) - assert result.ok is True - assert result.token == "tok123" - assert result.trigger_meta_path is None - assert Path(result.markdown_path).is_file() + assert result.ok is False + assert result.error.code == "PARSE_ERROR" + assert result.error.token == "diagnostic-token" + saved_error = json.loads((out / "trigger_error.json").read_text(encoding="utf-8")) + assert saved_error["token"] == "diagnostic-token" client.trigger_file.assert_called_once() + client.get_result.assert_not_called() + client.get_formatted.assert_not_called() def test_success_parse_writes_trigger_meta(tmp_path: Path): @@ -58,6 +54,60 @@ def test_success_parse_writes_trigger_meta(tmp_path: Path): assert result.ok is True assert result.trigger_meta_path is not None assert Path(result.trigger_meta_path).is_file() + trigger_args, trigger_kwargs = client.trigger_file.call_args + assert trigger_args == (str(pdf.resolve()),) + assert trigger_kwargs["token"] is None + assert trigger_kwargs["server_generated_token"] is True + assert trigger_kwargs["http_timeout"] == (60.0, 1860.0) + + +def test_async_local_pdf_uses_direct_upload_timeout(tmp_path: Path): + pdf = tmp_path / "paper.pdf" + pdf.write_bytes(b"%PDF-1.4") + client = MagicMock() + client.trigger_file.return_value = {"status": "error", "description": "stop"} + + req = ParseRequest(file_path=str(pdf), output_dir=str(tmp_path / "out"), async_mode=True) + result = asyncio.run(run_parse(client, req)) + + assert result.ok is False + _, trigger_kwargs = client.trigger_file.call_args + assert trigger_kwargs["token"] is None + assert trigger_kwargs["server_generated_token"] is True + assert trigger_kwargs["http_timeout"] == (60.0, 60.0) + assert trigger_kwargs["sync"] is False + + +def test_direct_upload_transport_failure_uses_upload_error(tmp_path: Path): + pdf = tmp_path / "paper.pdf" + pdf.write_bytes(b"%PDF-1.4") + client = MagicMock() + client.trigger_file.return_value = { + "status": "error", + "description": "The write operation timed out", + "error_type": "WriteTimeout", + } + + req = ParseRequest(file_path=str(pdf), output_dir=str(tmp_path / "out")) + result = asyncio.run(run_parse(client, req)) + + assert result.ok is False + assert result.error.code == "UPLOAD_ERROR" + assert result.error.stage == "trigger_file" + assert result.error.token is None + + +def test_url_trigger_uses_server_generated_token(tmp_path: Path): + client = MagicMock() + client.trigger_url.return_value = {"status": "error", "description": "stop"} + + req = ParseRequest(pdf_url="https://example.com/paper.pdf", output_dir=str(tmp_path / "out")) + result = asyncio.run(run_parse(client, req)) + + assert result.ok is False + _, trigger_kwargs = client.trigger_url.call_args + assert trigger_kwargs["token"] is None + assert trigger_kwargs["server_generated_token"] is True def test_existing_output_is_preserved_and_result_uses_suffixed_sibling(tmp_path: Path): diff --git a/mcp_server/tests/test_pipeline_poll.py b/mcp_server/tests/test_pipeline_poll.py new file mode 100644 index 0000000..8105df3 --- /dev/null +++ b/mcp_server/tests/test_pipeline_poll.py @@ -0,0 +1,39 @@ +import asyncio +from unittest.mock import MagicMock + +import pytest +from uniparser_mcp.pipeline.poll import poll_until_success + + +def test_undefined_token_stops_after_bounded_checks(monkeypatch: pytest.MonkeyPatch): + async def no_sleep(_seconds: float) -> None: + return None + + monkeypatch.setattr("uniparser_mcp.pipeline.poll.asyncio.sleep", no_sleep) + client = MagicMock() + client.get_result.return_value = {"status": "undefined"} + + result = asyncio.run(poll_until_success(client, "missing-token")) + + assert result.ok is False + assert result.error.code == "TOKEN_NOT_FOUND" + assert result.error.token == "missing-token" + assert client.get_result.call_count == 3 + + +def test_pending_statuses_still_reach_success(monkeypatch: pytest.MonkeyPatch): + async def no_sleep(_seconds: float) -> None: + return None + + monkeypatch.setattr("uniparser_mcp.pipeline.poll.asyncio.sleep", no_sleep) + client = MagicMock() + client.get_result.side_effect = [ + {"status": "waiting"}, + {"status": "processing"}, + {"status": "success"}, + ] + + result = asyncio.run(poll_until_success(client, "valid-token")) + + assert result == {"status": "success"} + assert client.get_result.call_count == 3 diff --git a/mcp_server/uniparser_mcp/defaults.py b/mcp_server/uniparser_mcp/defaults.py index d4f254d..7f180fe 100644 --- a/mcp_server/uniparser_mcp/defaults.py +++ b/mcp_server/uniparser_mcp/defaults.py @@ -2,8 +2,11 @@ POLL_INTERVAL_SEC = 3 POLL_TIMEOUT_SEC = 1800 +UNDEFINED_MAX_POLLS = 3 +DIRECT_UPLOAD_REQUEST_TIMEOUT = (60.0, 60.0) +DIRECT_SYNC_UPLOAD_REQUEST_TIMEOUT = (60.0, 1860.0) -PENDING_STATUSES = frozenset({"undefined", "waiting", "processing"}) +PENDING_STATUSES = frozenset({"waiting", "processing"}) IMAGE_SUFFIXES = frozenset({".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".tif", ".tiff"}) diff --git a/mcp_server/uniparser_mcp/errors.py b/mcp_server/uniparser_mcp/errors.py index 0b0333e..69dcec5 100644 --- a/mcp_server/uniparser_mcp/errors.py +++ b/mcp_server/uniparser_mcp/errors.py @@ -13,12 +13,31 @@ def input_error(message: str) -> ErrorResult: return ErrorResult(error=ErrorDetail(code="INPUT_ERROR", message=message)) -def parse_error(stage: str, result: dict) -> ErrorResult: +def _operation_error(code: str, stage: str, result: dict) -> ErrorResult: return ErrorResult( error=ErrorDetail( - code="PARSE_ERROR", + code=code, message=result.get("description") or result.get("message") or str(result), stage=stage, token=result.get("token"), ) ) + + +def parse_error(stage: str, result: dict) -> ErrorResult: + return _operation_error("PARSE_ERROR", stage, result) + + +def upload_error(stage: str, result: dict) -> ErrorResult: + return _operation_error("UPLOAD_ERROR", stage, result) + + +def token_not_found_error(token: str, *, attempts: int) -> ErrorResult: + return ErrorResult( + error=ErrorDetail( + code="TOKEN_NOT_FOUND", + message=f"The service did not recognize this token after {attempts} checks.", + stage="get_result_poll", + token=token, + ), + ) diff --git a/mcp_server/uniparser_mcp/instructions.py b/mcp_server/uniparser_mcp/instructions.py index 0e3765a..b24c6c5 100644 --- a/mcp_server/uniparser_mcp/instructions.py +++ b/mcp_server/uniparser_mcp/instructions.py @@ -21,5 +21,8 @@ ## After calling uniparser_parse -Always include the `message` field from the tool response in your reply to the user. +On success, always include the `message` field from the tool response in your reply to the user. +On failure, report `error.message`. Never poll a token from a failed trigger; it is diagnostic only. Only a +token returned with `status=success` may be polled. Do not attempt recovery from a failed trigger. +Stop retrying when the error code is `TOKEN_NOT_FOUND`. """ diff --git a/mcp_server/uniparser_mcp/pipeline/parse.py b/mcp_server/uniparser_mcp/pipeline/parse.py index 76e617b..1d9bdfb 100644 --- a/mcp_server/uniparser_mcp/pipeline/parse.py +++ b/mcp_server/uniparser_mcp/pipeline/parse.py @@ -7,7 +7,11 @@ from mcp.server.fastmcp import Context -from uniparser_mcp.errors import input_error, parse_error +from uniparser_mcp.defaults import ( + DIRECT_SYNC_UPLOAD_REQUEST_TIMEOUT, + DIRECT_UPLOAD_REQUEST_TIMEOUT, +) +from uniparser_mcp.errors import input_error, parse_error, upload_error from uniparser_mcp.input import InputKind, ResolvedInput, display_label, resolve_request from uniparser_mcp.parse_options import resolve_trigger_kwargs, serialize_trigger_kwargs from uniparser_mcp.pipeline.output import ( @@ -20,14 +24,7 @@ from uniparser_mcp.schemas import ErrorResult, ParseRequest, ParseResult from uniparser_mcp.tools.response import build_parse_success from uniparser_tools.api.clients import UniParserClient -from uniparser_tools.common.constant import ErrorFlag, FormatFlag - - -def _is_token_duplicated(trigger: dict[str, Any]) -> bool: - if trigger.get("status") == "success": - return False - message = str(trigger.get("message") or trigger.get("description") or "") - return message == ErrorFlag.Token_Duplicated or "Token is duplicated" in message +from uniparser_tools.common.constant import FormatFlag async def _ctx_info(ctx: Context | None, message: str) -> None: @@ -48,14 +45,26 @@ async def _trigger_input( client.trigger_file, str(resolved.path), token=None, + server_generated_token=True, + http_timeout=( + DIRECT_SYNC_UPLOAD_REQUEST_TIMEOUT + if trigger_kwargs.get("sync", True) + else DIRECT_UPLOAD_REQUEST_TIMEOUT + ), **trigger_kwargs, ) + if not isinstance(trigger, dict): + trigger = { + "status": "error", + "message": "Direct upload returned an invalid response", + } return trigger, "trigger_file" if resolved.kind is InputKind.IMAGE: trigger = await asyncio.to_thread( client.trigger_snip, str(resolved.path), token=None, + server_generated_token=True, **trigger_kwargs, ) return trigger, "trigger_snip" @@ -63,6 +72,7 @@ async def _trigger_input( client.trigger_url, resolved.raw, token=None, + server_generated_token=True, **trigger_kwargs, ) return trigger, "trigger_url" @@ -156,19 +166,10 @@ async def run_parse(client: UniParserClient, req: ParseRequest, ctx: Context | N await _ctx_info(ctx, f"Parsing {display_label(resolved)}") trigger, stage = await _trigger_input(client, resolved, trigger_kwargs=trigger_kwargs) - if _is_token_duplicated(trigger): - token = trigger.get("token") or client.to_token(resolved.token_seed) - return await _complete_by_token( - client, - token, - resolved=resolved, - out_dir=out_dir, - ctx=ctx, - write_trigger_meta_file=False, - ) - if trigger.get("status") != "success": save_stage_error(out_dir, "trigger_error.json", trigger) + if stage == "trigger_file" and trigger.get("error_type"): + return upload_error(stage, trigger) return parse_error(stage, trigger) token = trigger.get("token") diff --git a/mcp_server/uniparser_mcp/pipeline/poll.py b/mcp_server/uniparser_mcp/pipeline/poll.py index 4bccf81..3478d8f 100644 --- a/mcp_server/uniparser_mcp/pipeline/poll.py +++ b/mcp_server/uniparser_mcp/pipeline/poll.py @@ -6,17 +6,18 @@ import time from typing import Any -from uniparser_mcp.defaults import PENDING_STATUSES, POLL_INTERVAL_SEC, POLL_TIMEOUT_SEC -from uniparser_mcp.errors import parse_error +from uniparser_mcp.defaults import PENDING_STATUSES, POLL_INTERVAL_SEC, POLL_TIMEOUT_SEC, UNDEFINED_MAX_POLLS +from uniparser_mcp.errors import parse_error, token_not_found_error from uniparser_mcp.schemas import ErrorResult from uniparser_tools.api.clients import UniParserClient async def poll_until_success(client: UniParserClient, token: str) -> dict[str, Any] | ErrorResult: - deadline = time.time() + POLL_TIMEOUT_SEC + deadline = time.monotonic() + POLL_TIMEOUT_SEC last: dict[str, Any] = {} + undefined_polls = 0 - while time.time() < deadline: + while time.monotonic() < deadline: last = await asyncio.to_thread( client.get_result, token, @@ -30,7 +31,14 @@ async def poll_until_success(client: UniParserClient, token: str) -> dict[str, A return last if status == "error": return parse_error("get_result_poll", last) - if status in PENDING_STATUSES or status is None: + if status == "undefined": + undefined_polls += 1 + if undefined_polls >= UNDEFINED_MAX_POLLS: + return token_not_found_error(token, attempts=undefined_polls) + await asyncio.sleep(POLL_INTERVAL_SEC) + continue + if status in PENDING_STATUSES: + undefined_polls = 0 await asyncio.sleep(POLL_INTERVAL_SEC) continue return parse_error("get_result_poll", last) diff --git a/skills/UniParser-Tools/SKILL.md b/skills/UniParser-Tools/SKILL.md index 0dd4b99..8e60a64 100644 --- a/skills/UniParser-Tools/SKILL.md +++ b/skills/UniParser-Tools/SKILL.md @@ -77,7 +77,7 @@ $env:UNIPARSER_API_KEY="your-api-key" ### CLI reference -**Pipeline:** `submit → poll get_result until success → fetch pages_tree + Markdown → save`. Default `sync=true` (trigger blocks until server done; CLI still polls and fetches). `--async` uses `sync=false` on submit only. +**Pipeline:** every trigger sends `token=None` with `server_generated_token=True`. Local PDFs use direct multipart upload with a 60-second upload window. Images use `trigger_snip`; public URLs use `trigger_url`. The CLI then polls `get_result` only with the token from a successful trigger, fetches `pages_tree` + Markdown, and saves them. Default `sync=true`; `--async` uses `sync=false` on the trigger only. **Input:** one positional `INPUT` per run—the CLI detects type by path suffix or `http(s)://` URL: @@ -104,7 +104,7 @@ Recovery (existing server job—see **Common issues**): uniparser fetch --token "TASK_TOKEN_FROM_PRIOR_RUN" ``` -Token sources: stdout JSON from a prior `uniparser --json parse …`, `trigger_meta.json` under the output directory, or the `token` field in a failed parse stderr JSON. +Valid recovery tokens come only from successful `uniparser --json parse …` stdout or `trigger_meta.json` written after a successful trigger. Never use a token from a failed trigger: it is diagnostic only and cannot be resumed with `fetch`. **Default output for** `fetch` (when `-o` / `--output-dir` is omitted): `~/Uni-Parser-Skill/token_/`, where `` is the first 8 characters of the token (e.g. `~/Uni-Parser-Skill/token_a1b2c3d4/token_a1b2c3d4.md`). Passing a prior `parse` directory with `-o` treats it only as the preferred path; because that directory already exists, `fetch` writes to an available sibling such as `paper_1`. Always use the returned `output_dir`. @@ -184,7 +184,9 @@ uniparser parse paper.pdf --json # wrong If the preferred output directory already exists, the CLI creates an available sibling such as `results_1` or `results_2`. Existing paths are never reused or deleted; use the returned `output_dir`. -**Common error codes** (stderr JSON): `CONFIG_ERROR`, `INPUT_ERROR`, `PARSE_ERROR`. +Failure JSON may carry the standard `token` field for diagnostics. Never pass that value to `fetch`; use `fetch` only with a token from successful parse output or `trigger_meta.json`. + +**Common error codes** (stderr JSON): `CONFIG_ERROR`, `INPUT_ERROR`, `UPLOAD_ERROR`, `PARSE_ERROR`, `TOKEN_NOT_FOUND`. ## Common issues @@ -194,13 +196,14 @@ On failure, show stderr JSON `error.message`. Do not substitute vision-only read | Problem | Cause | Solution | | ------------------------------------------------------------------------ | ------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `CONFIG_ERROR` | No API key or `uniparser` not installed | **Configuration** + `pip install "git+https://github.com/dptech-corp/UniParser-Tools.git"`; `uniparser auth --verify` | -| `Token is duplicated` | Job for this API key + exact input already exists | Do **not** re-run `uniparser parse`. Read `token` from stderr JSON or `trigger_meta.json`; run `uniparser fetch --token TOKEN` | +| `UPLOAD_ERROR` / write timeout | Direct file upload failed; any token in the error is diagnostic only | Re-run `uniparser parse`; the failed trigger cannot be resumed with `fetch` | +| `TOKEN_NOT_FOUND` / `status: undefined` | The service did not recognize the token after three checks | Stop fetching; verify that the token came from successful output or `trigger_meta.json` and is still within the retention period | | Job not done / long wait / CLI interrupted / `processing` / poll timeout | Sync or poll still running; or local process stopped while server job continues | Wait; do **not** start a second `uniparser parse` for the same input. Use saved `token` with `uniparser fetch --token TOKEN`; files appear only after exit 0 | | `502 Bad Gateway` on URL input | Server failed fetching or processing remote PDF | Retry `uniparser parse "same url"` once; or download and `uniparser parse local.pdf`; or `uniparser fetch --token TOKEN` if a prior job exists | | `PARSE_ERROR` | Server `status: error` at trigger / poll / fetch | Read `error.message` and `stage`; match rows above; check `trigger_error.json` / `pages_tree_error.json` / `formatted_error.json` under output dir if present | -**Limits:** large PDFs may take 10–20+ minutes; public service ≤5 concurrent requests; PDF URLs must be publicly accessible. Parsing can be inaccurate, so verify critical content against the source. Results are retained for only 24 hours—fetch and save them promptly. See [Important notes](./references/notes.md). Save `token` from success JSON or `trigger_meta.json` for recovery after interrupt or duplicate-token errors. +**Limits:** large PDFs may take 10–20+ minutes; public service ≤5 concurrent requests; PDF URLs must be publicly accessible. Parsing can be inaccurate, so verify critical content against the source. Results are retained for only 24 hours—fetch and save them promptly. See [Important notes](./references/notes.md). Save `token` only from success JSON or `trigger_meta.json`. ## Advanced @@ -209,6 +212,8 @@ For callbacks, custom `ParseMode`, or SDK examples, see [Common patterns](./refe Full CLI reference (flags, examples, JSON details): [CLI README](../../uniparser_tools/cli/README.md) in this repository. Optional MCP server setup is in the [UniParser-Tools GitHub repo](https://github.com/dptech-corp/UniParser-Tools); it is separate from this CLI workflow. +Its `uniparser_parse` tool uses the same direct local-PDF upload, server-generated token, successful-trigger-only +token use, and three-check `undefined` limit as the CLI. ## Reference documents diff --git a/skills/UniParser-Tools/references/api-reference.md b/skills/UniParser-Tools/references/api-reference.md index 8423634..3bc3eb0 100644 --- a/skills/UniParser-Tools/references/api-reference.md +++ b/skills/UniParser-Tools/references/api-reference.md @@ -89,6 +89,9 @@ trigger endpoints, including `trigger_url`. ### TOS presigned upload +This is an advanced SDK-only path. The CLI and MCP local-PDF workflows use direct `trigger_file` uploads and do not +call TOS or perform automatic upload fallback. + ```python uploaded = parser.upload_files_to_tos(["./document.pdf"]) source_url = uploaded["files"][0]["source_url"] diff --git a/skills/UniParser-Tools/references/notes.md b/skills/UniParser-Tools/references/notes.md index 8210df5..c2916ff 100644 --- a/skills/UniParser-Tools/references/notes.md +++ b/skills/UniParser-Tools/references/notes.md @@ -55,7 +55,7 @@ All API methods return a dict with consistent structure: ... } -# Error +# SDK error { "status": "error", "token": "abc123...", @@ -64,6 +64,11 @@ All API methods return a dict with consistent structure: } ``` +The low-level SDK retains its deterministic-token behavior for backward compatibility and may include that local value +in an error response. This is not part of the Agent Skill workflow. CLI and MCP always send `token=None` with +`server_generated_token=True` and only persist the `token` from a successful trigger response; never use an SDK error +token with CLI `fetch`. + ## Common Error Messages CLI workflow errors (config, duplicate token, 502, etc.) are documented in SKILL.md **Common issues**. This table covers additional SDK/API messages when calling the client directly: diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 2482963..b3751e5 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -73,6 +73,7 @@ def test_parse_with_config_file_api_key( result = runner.invoke(cli, ["parse", str(pdf), "-o", str(out)]) assert result.exit_code == 1 mock_client.trigger_file.assert_called_once() + mock_client.trigger_url.assert_not_called() def test_fetch_without_api_key(self, runner: CliRunner, env_without_api_key: None, no_config_file: None) -> None: result = runner.invoke(cli, ["fetch", "--token", "abc123"]) @@ -148,6 +149,7 @@ def test_parse_success_writes_trigger_meta_and_json_token( mock_client.get_formatted.return_value = {"status": "success", "content": "# Hi"} monkeypatch.setattr("uniparser_tools.cli.commands.parse.make_client", lambda ctx: (mock_client, None)) + monkeypatch.setattr("uniparser_tools.cli.core.pipeline.time.sleep", lambda _: None) out = tmp_path / "out" out.mkdir() @@ -171,6 +173,12 @@ def test_parse_success_writes_trigger_meta_and_json_token( assert meta["trigger_kwargs"]["sync"] is True assert "preset" not in meta assert (actual_out / "paper.md").is_file() + _, trigger_kwargs = mock_client.trigger_file.call_args + assert trigger_kwargs["file_path"] == str(pdf.resolve()) + assert trigger_kwargs["token"] is None + assert trigger_kwargs["server_generated_token"] is True + assert trigger_kwargs["http_timeout"] == (60.0, 1860.0) + mock_client.trigger_url.assert_not_called() def test_parse_default_trigger_kwargs( self, @@ -194,6 +202,9 @@ def test_parse_default_trigger_kwargs( ) _, call_kwargs = mock_client.trigger_file.call_args + assert call_kwargs["token"] is None + assert call_kwargs["server_generated_token"] is True + assert call_kwargs["http_timeout"] == (60.0, 1860.0) assert call_kwargs["sync"] is True assert call_kwargs["textual"] is ParseModeTextual.OCRHighQuality assert call_kwargs["equation"] is ParseMode.OCRHighQuality @@ -228,6 +239,75 @@ def test_parse_molecule_disable_override( assert call_kwargs["molecule"] is ParseMode.Disable assert call_kwargs["table"] is ParseMode.OCRHighQuality + def test_parse_transport_failure_does_not_recover_error_token( + self, + runner: CliRunner, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + monkeypatch.setenv("UNIPARSER_API_KEY", "test-key") + pdf = tmp_path / "paper.pdf" + pdf.write_bytes(b"%PDF-1.4") + mock_client = MagicMock() + mock_client.trigger_file.return_value = { + "status": "error", + "token": "diagnostic-token", + "description": "The write operation timed out", + "error_type": "WriteTimeout", + } + monkeypatch.setattr("uniparser_tools.cli.commands.parse.make_client", lambda ctx: (mock_client, None)) + + out = tmp_path / "out" + result = runner.invoke( + cli, + ["--json", "parse", str(pdf), "-o", str(out)], + env={**os.environ, "UNIPARSER_API_KEY": "test-key"}, + ) + + assert result.exit_code == 1 + payload = json.loads(result.stderr.strip().splitlines()[-1]) + assert payload["error"]["code"] == "UPLOAD_ERROR" + assert payload["error"]["stage"] == "trigger_file" + assert payload["token"] == "diagnostic-token" + assert not (out / "trigger_meta.json").exists() + saved_error = json.loads((out / "trigger_error.json").read_text(encoding="utf-8")) + assert saved_error["error_type"] == "WriteTimeout" + assert saved_error["token"] == "diagnostic-token" + mock_client.get_result.assert_not_called() + + def test_parse_direct_upload_failure_uses_upload_error( + self, + runner: CliRunner, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + monkeypatch.setenv("UNIPARSER_API_KEY", "test-key") + pdf = tmp_path / "paper.pdf" + pdf.write_bytes(b"%PDF-1.4") + mock_client = MagicMock() + mock_client.trigger_file.return_value = { + "status": "error", + "description": "Direct upload failed", + "error_type": "ConnectTimeout", + } + monkeypatch.setattr("uniparser_tools.cli.commands.parse.make_client", lambda ctx: (mock_client, None)) + + result = runner.invoke( + cli, + ["--json", "parse", str(pdf), "-o", str(tmp_path / "out")], + env={**os.environ, "UNIPARSER_API_KEY": "test-key"}, + ) + + assert result.exit_code == 1 + payload = json.loads(result.stderr.strip().splitlines()[-1]) + assert payload["error"]["code"] == "UPLOAD_ERROR" + assert payload["error"]["stage"] == "trigger_file" + mock_client.trigger_url.assert_not_called() + _, direct_kwargs = mock_client.trigger_file.call_args + assert direct_kwargs["token"] is None + assert direct_kwargs["server_generated_token"] is True + assert direct_kwargs["http_timeout"] == (60.0, 1860.0) + def test_parse_invalid_equation_mode_rejected( self, runner: CliRunner, @@ -339,6 +419,7 @@ def test_parse_help(self, runner: CliRunner) -> None: result = runner.invoke(cli, ["parse", "--help"]) assert result.exit_code == 0 assert "--output-dir" in result.stdout + assert "--upload-mode" not in result.stdout assert "--async" in result.stdout assert "--textual" in result.stdout assert "--molecule" in result.stdout diff --git a/tests/unit/test_cli_pipeline.py b/tests/unit/test_cli_pipeline.py new file mode 100644 index 0000000..ccc1e53 --- /dev/null +++ b/tests/unit/test_cli_pipeline.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import json +from pathlib import Path +from unittest.mock import MagicMock + +from uniparser_tools.cli.core.input import InputKind, ResolvedInput +from uniparser_tools.cli.core.pipeline import poll_until_success, trigger_input + + +def _local_pdf(path: Path) -> ResolvedInput: + return ResolvedInput( + kind=InputKind.FILE, + source_stem=path.stem, + raw=str(path), + path=path, + ) + + +def test_local_pdf_uses_direct_upload_with_server_token(tmp_path: Path) -> None: + pdf = tmp_path / "paper.pdf" + pdf.write_bytes(b"%PDF-1.4") + client = MagicMock() + client.trigger_file.return_value = {"status": "success", "token": "server-token"} + + result, stage = trigger_input(client, _local_pdf(pdf), trigger_kwargs={"sync": True}) + + assert result == {"status": "success", "token": "server-token"} + assert stage == "trigger_file" + client.trigger_file.assert_called_once_with( + file_path=str(pdf), + token=None, + server_generated_token=True, + http_timeout=(60.0, 1860.0), + sync=True, + ) + client.trigger_url.assert_not_called() + + +def test_async_local_pdf_uses_direct_upload_timeout(tmp_path: Path) -> None: + pdf = tmp_path / "paper.pdf" + pdf.write_bytes(b"%PDF-1.4") + client = MagicMock() + client.trigger_file.return_value = {"status": "success", "token": "server-token"} + + result, stage = trigger_input(client, _local_pdf(pdf), trigger_kwargs={"sync": False}) + + assert result == {"status": "success", "token": "server-token"} + assert stage == "trigger_file" + client.trigger_file.assert_called_once_with( + file_path=str(pdf), + token=None, + server_generated_token=True, + http_timeout=(60.0, 60.0), + sync=False, + ) + client.trigger_url.assert_not_called() + + +def test_undefined_token_stops_after_bounded_checks(monkeypatch, capsys) -> None: + client = MagicMock() + client.get_result.return_value = {"status": "undefined"} + monkeypatch.setattr("uniparser_tools.cli.core.pipeline.time.sleep", lambda _: None) + + result = poll_until_success(client, "missing-token") + + assert result == 1 + assert client.get_result.call_count == 3 + payload = json.loads(capsys.readouterr().err) + assert payload["error"]["code"] == "TOKEN_NOT_FOUND" + assert payload["token"] == "missing-token" + + +def test_pending_statuses_still_reach_success(monkeypatch) -> None: + client = MagicMock() + client.get_result.side_effect = [ + {"status": "waiting"}, + {"status": "processing"}, + {"status": "success"}, + ] + monkeypatch.setattr("uniparser_tools.cli.core.pipeline.time.sleep", lambda _: None) + + result = poll_until_success(client, "valid-token") + + assert result == {"status": "success"} + assert client.get_result.call_count == 3 diff --git a/tests/unit/test_notebook_hygiene.py b/tests/unit/test_notebook_hygiene.py index e3173ca..4523d70 100644 --- a/tests/unit/test_notebook_hygiene.py +++ b/tests/unit/test_notebook_hygiene.py @@ -53,3 +53,19 @@ def test_customer_docs_include_result_retention_notice() -> None: assert "24 小时" in readme assert "retained for only 24 hours" in skill assert "retained for only **24 hours**" in notes + + +def test_skill_only_uses_tokens_from_successful_triggers() -> None: + skill = (REPO_ROOT / "skills" / "UniParser-Tools" / "SKILL.md").read_text(encoding="utf-8") + notes = (REPO_ROOT / "skills" / "UniParser-Tools" / "references" / "notes.md").read_text(encoding="utf-8") + + assert "Never use a token from a failed trigger" in skill + assert "Save `token` only from success JSON or `trigger_meta.json`" in skill + assert "candidate_token" not in skill + assert "recoverable_token" not in skill + assert "the `token` field in a failed parse stderr JSON" not in skill + assert "--upload-mode" not in skill + assert "TOS" not in skill + assert "only persist the `token` from a successful trigger response" in notes + assert "candidate_token" not in notes + assert "recoverable_token" not in notes diff --git a/uniparser_tools/cli/README.md b/uniparser_tools/cli/README.md index 0c2df24..76c1333 100644 --- a/uniparser_tools/cli/README.md +++ b/uniparser_tools/cli/README.md @@ -47,6 +47,8 @@ uniparser auth uniparser parse /path/to/report.pdf ``` +本地 PDF 使用 multipart 直传，上传窗口为 60 秒，并要求服务端生成 token。只有服务端成功接收文件并创建任务后，CLI 才会记录可恢复 token。 + 默认把结果保存到： ```text @@ -341,7 +343,9 @@ uniparser parse paper.pdf -o ./out/paper uniparser fetch --token YOUR_TOKEN ``` -token 可在上次 `parse` 输出目录的 `trigger_meta.json` 中找到。 +CLI 触发时固定发送 `token=None` 和 `server_generated_token=True`。可用于恢复的 token 只能从成功 +`parse` 的 JSON 输出或 `trigger_meta.json` 中获取。失败 JSON 可能保留标准 `token` 字段用于诊断，但不得 +将它传给 `fetch`。 | 选项 | 说明 | |------|------| @@ -433,7 +437,8 @@ Parsing... report.pdf uniparser --json parse paper.pdf 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin)['markdown_path'])" ``` -失败时仍为 **stderr 单行 JSON**（`ok: false`），见上文 parse「常见错误」；exit code 为 1。 +失败时仍为 **stderr 单行 JSON**（`ok: false`），见下方「常见问题」；exit code 为 1。失败 JSON 中的 +`token` 仅用于诊断；只有成功 `parse` 输出或 `trigger_meta.json` 中的 token 可用于 `fetch`。 --- @@ -459,6 +464,14 @@ uniparser --json parse paper.pdf 2>/dev/null | python3 -c "import sys,json; prin 属正常现象；请等待 `Parsing...` 出现后保持终端不要关闭。若中断，用 `trigger_meta.json` 中的 token 执行 `uniparser fetch`。 +**提示 `UPLOAD_ERROR` / write timeout** + +本次直传失败。请重新运行 `parse`；错误结果中的 token 即使存在也仅供诊断，不能用于 `fetch`。 + +**提示 `TOKEN_NOT_FOUND` 或持续 `status: undefined`** + +CLI 会在三次检查后停止，不会继续等待 1800 秒。请确认 token 来自成功输出或 `trigger_meta.json`，并检查任务结果是否仍在 24 小时保留期内。 + --- ## 推荐使用流程 diff --git a/uniparser_tools/cli/core/defaults.py b/uniparser_tools/cli/core/defaults.py index 986e8bd..ff2061e 100644 --- a/uniparser_tools/cli/core/defaults.py +++ b/uniparser_tools/cli/core/defaults.py @@ -2,7 +2,10 @@ POLL_INTERVAL_SEC = 3 POLL_TIMEOUT_SEC = 1800 +UNDEFINED_MAX_POLLS = 3 +DIRECT_UPLOAD_REQUEST_TIMEOUT = (60.0, 60.0) +DIRECT_SYNC_UPLOAD_REQUEST_TIMEOUT = (60.0, 1860.0) -PENDING_STATUSES = frozenset({"undefined", "waiting", "processing"}) +PENDING_STATUSES = frozenset({"waiting", "processing"}) IMAGE_SUFFIXES = frozenset({".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".tif", ".tiff"}) diff --git a/uniparser_tools/cli/core/errors.py b/uniparser_tools/cli/core/errors.py index 571fc0a..2c5a3b3 100644 --- a/uniparser_tools/cli/core/errors.py +++ b/uniparser_tools/cli/core/errors.py @@ -31,16 +31,38 @@ def missing_token_error() -> int: return 1 +def _operation_error(code: str, stage: str, result: dict) -> int: + payload = { + "ok": False, + "token": result.get("token"), + "error": { + "code": code, + "message": result.get("description") or result.get("message") or str(result), + "stage": stage, + }, + } + emit_json_stderr(payload) + return 1 + + def parse_error(stage: str, result: dict) -> int: + return _operation_error("PARSE_ERROR", stage, result) + + +def upload_error(stage: str, result: dict) -> int: + return _operation_error("UPLOAD_ERROR", stage, result) + + +def token_not_found_error(token: str, *, attempts: int) -> int: emit_json_stderr( { "ok": False, + "token": token, "error": { - "code": "PARSE_ERROR", - "message": result.get("description") or result.get("message") or str(result), - "stage": stage, + "code": "TOKEN_NOT_FOUND", + "message": f"The service did not recognize this token after {attempts} checks.", + "stage": "get_result_poll", }, - "token": result.get("token"), } ) return 1 diff --git a/uniparser_tools/cli/core/pipeline.py b/uniparser_tools/cli/core/pipeline.py index 91d95a9..227a647 100644 --- a/uniparser_tools/cli/core/pipeline.py +++ b/uniparser_tools/cli/core/pipeline.py @@ -5,8 +5,15 @@ from pathlib import Path from typing import Any -from uniparser_tools.cli.core.defaults import PENDING_STATUSES, POLL_INTERVAL_SEC, POLL_TIMEOUT_SEC -from uniparser_tools.cli.core.errors import parse_error +from uniparser_tools.cli.core.defaults import ( + DIRECT_SYNC_UPLOAD_REQUEST_TIMEOUT, + DIRECT_UPLOAD_REQUEST_TIMEOUT, + PENDING_STATUSES, + POLL_INTERVAL_SEC, + POLL_TIMEOUT_SEC, + UNDEFINED_MAX_POLLS, +) +from uniparser_tools.cli.core.errors import parse_error, token_not_found_error, upload_error from uniparser_tools.cli.core.input import InputKind, ResolvedInput, display_label_for_input from uniparser_tools.cli.core.output import print_parsing_status, save_parse_results, write_trigger_meta from uniparser_tools.cli.core.parse_options import resolve_trigger_kwargs, serialize_trigger_kwargs @@ -16,23 +23,52 @@ def scientific_paper_trigger_kwargs(*, sync: bool = True) -> dict: return resolve_trigger_kwargs(sync=sync, overrides={}) -def trigger_input(client, resolved: ResolvedInput, *, trigger_kwargs: dict) -> tuple[dict, str]: +def trigger_input( + client, + resolved: ResolvedInput, + *, + trigger_kwargs: dict, +) -> tuple[dict, str]: kwargs = trigger_kwargs if resolved.kind is InputKind.FILE: - trigger = client.trigger_file(file_path=str(resolved.path), **kwargs) + trigger = client.trigger_file( + file_path=str(resolved.path), + token=None, + server_generated_token=True, + http_timeout=( + DIRECT_SYNC_UPLOAD_REQUEST_TIMEOUT if kwargs.get("sync", True) else DIRECT_UPLOAD_REQUEST_TIMEOUT + ), + **kwargs, + ) + if not isinstance(trigger, dict): + trigger = { + "status": "error", + "message": "Direct upload returned an invalid response", + } return trigger, "trigger_file" if resolved.kind is InputKind.IMAGE: - trigger = client.trigger_snip(snip_path=str(resolved.path), **kwargs) + trigger = client.trigger_snip( + snip_path=str(resolved.path), + token=None, + server_generated_token=True, + **kwargs, + ) return trigger, "trigger_snip" - trigger = client.trigger_url(pdf_url=resolved.raw, **kwargs) + trigger = client.trigger_url( + pdf_url=resolved.raw, + token=None, + server_generated_token=True, + **kwargs, + ) return trigger, "trigger_url" def poll_until_success(client, token: str) -> dict | int: - deadline = time.time() + POLL_TIMEOUT_SEC + deadline = time.monotonic() + POLL_TIMEOUT_SEC last: dict[str, Any] = {} + undefined_polls = 0 - while time.time() < deadline: + while time.monotonic() < deadline: last = client.get_result( token, content=False, @@ -45,7 +81,14 @@ def poll_until_success(client, token: str) -> dict | int: return last if status == "error": return parse_error("get_result_poll", last) - if status in PENDING_STATUSES or status is None: + if status == "undefined": + undefined_polls += 1 + if undefined_polls >= UNDEFINED_MAX_POLLS: + return token_not_found_error(token, attempts=undefined_polls) + time.sleep(POLL_INTERVAL_SEC) + continue + if status in PENDING_STATUSES: + undefined_polls = 0 time.sleep(POLL_INTERVAL_SEC) continue return parse_error("get_result_poll", last) @@ -124,6 +167,8 @@ def run_parse( trigger, stage = trigger_input(client, resolved, trigger_kwargs=trigger_kwargs) if trigger.get("status") != "success": save_stage_error(out_dir, "trigger_error.json", trigger) + if stage == "trigger_file" and trigger.get("error_type"): + return upload_error(stage, trigger) return parse_error(stage, trigger) token = trigger.get("token")