From afcac238c9d7d66375fbfb2f8b249ea6cc7f2683 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 5 Aug 2026 23:21:10 +0000 Subject: [PATCH 1/2] Support http(s) URLs in vlmrun gw chat Allow inline URL arguments for images, documents, and videos alongside local file paths. URLs are passed directly to the gateway as remote content parts (image_url, document_url, video_url) instead of being base64-encoded. Co-authored-by: Sudeep Pillai --- tests/test_gateway.py | 76 ++++++++++++++++++++++- vlmrun/cli/_cli/gateway.py | 122 ++++++++++++++++++++++++++++--------- 2 files changed, 167 insertions(+), 31 deletions(-) diff --git a/tests/test_gateway.py b/tests/test_gateway.py index 15c7d77..cf82f3c 100644 --- a/tests/test_gateway.py +++ b/tests/test_gateway.py @@ -364,9 +364,35 @@ def test_content_part_type(self, tmp_path): assert gw._content_part_type(tmp_path / "a.docx") == "document_url" assert gw._content_part_type(tmp_path / "a.png") == "image_url" assert gw._content_part_type(tmp_path / "a.jpg") == "image_url" + assert gw._content_part_type(tmp_path / "a.mp4") == "video_url" # Unidentifiable content still falls back to file_url. assert gw._content_part_type(tmp_path / "a.bin") == "file_url" + def test_content_part_type_from_url(self): + assert ( + gw._content_part_type_from_url("https://example.com/doc.pdf") + == "document_url" + ) + assert ( + gw._content_part_type_from_url("https://example.com/img.png") == "image_url" + ) + assert ( + gw._content_part_type_from_url("https://example.com/clip.mp4?token=1") + == "video_url" + ) + + def test_encode_url_part(self): + part = gw._encode_url_part("https://example.com/scan.jpg") + assert part == { + "type": "image_url", + "image_url": {"url": "https://example.com/scan.jpg"}, + } + + def test_encode_url_part_document(self): + part = gw._encode_url_part("https://example.com/report.pdf") + assert part["type"] == "document_url" + assert part["document_url"]["url"] == "https://example.com/report.pdf" + def test_encode_document_part(self, tmp_path): f = tmp_path / "doc.pdf" f.write_bytes(b"%PDF-1.7 fake") @@ -421,7 +447,7 @@ def test_sniff_mime_signatures(self): def test_build_messages_with_prompt(self, tmp_path): f = tmp_path / "img.png" f.write_bytes(PNG_BYTES) - messages = gw._build_messages([f], "describe") + messages = gw._build_messages([str(f)], "describe") assert len(messages) == 1 content = messages[0]["content"] assert content[0]["type"] == "image_url" @@ -432,9 +458,27 @@ def test_build_messages_mixed_files(self, tmp_path): doc = tmp_path / "doc.pdf" img.write_bytes(PNG_BYTES) doc.write_bytes(b"%PDF fake") - content = gw._build_messages([img, doc], None)[0]["content"] + content = gw._build_messages([str(img), str(doc)], None)[0]["content"] assert [p["type"] for p in content] == ["image_url", "document_url"] + def test_build_messages_with_url(self): + content = gw._build_messages(["https://example.com/scan.jpg"], None)[0][ + "content" + ] + assert content[0]["type"] == "image_url" + assert content[0]["image_url"]["url"] == "https://example.com/scan.jpg" + + def test_build_messages_mixed_file_and_url(self, tmp_path): + img = tmp_path / "img.png" + img.write_bytes(PNG_BYTES) + content = gw._build_messages([str(img), "https://example.com/doc.pdf"], None)[ + 0 + ]["content"] + assert content[0]["type"] == "image_url" + assert content[0]["image_url"]["url"].startswith("data:image/png;base64,") + assert content[1]["type"] == "document_url" + assert content[1]["document_url"]["url"] == "https://example.com/doc.pdf" + def test_parse_extra_json_and_string(self): parsed = gw._parse_extra(["temperature=0", "max_tokens=4096", "label=hello"]) assert parsed == {"temperature": 0, "max_tokens": 4096, "label": "hello"} @@ -840,7 +884,33 @@ def test_transcribe_bad_format(self, runner, patched_cli, tmp_path): def test_chat_requires_file(self, runner, patched_cli): result = runner.invoke(app, ["gw", "chat", "-m", "glm-ocr"]) assert result.exit_code == 1 - assert "at least one input file" in result.stdout.lower() + assert "at least one input" in result.stdout.lower() + + def test_chat_rejects_missing_file(self, runner, patched_cli): + result = runner.invoke( + app, ["gw", "chat", "missing.pdf", "-m", "glm-ocr", "--no-stream"] + ) + assert result.exit_code == 1 + assert "not a file" in result.stdout.lower() + + def test_chat_with_url(self, runner, patched_cli): + url = "https://example.com/scan.jpg" + result = runner.invoke( + app, + ["gw", "chat", url, "-m", "paddle-ocrv6", "--no-stream"], + ) + assert result.exit_code == 0, result.stdout + call = patched_cli["client"].gateway.completions.calls[-1] + content = call["messages"][0]["content"] + assert content[0]["type"] == "image_url" + assert content[0]["image_url"]["url"] == url + assert call["stream"] is False + + def test_chat_document_url_streams(self, runner, patched_cli): + url = "https://example.com/report.pdf" + result = runner.invoke(app, ["gw", "chat", url, "-m", "glm-ocr"]) + assert result.exit_code == 0, result.stdout + assert patched_cli["client"].gateway.completions.calls[-1]["stream"] is True def test_chat_with_file_json(self, runner, patched_cli, tmp_path): f = tmp_path / "doc.pdf" diff --git a/vlmrun/cli/_cli/gateway.py b/vlmrun/cli/_cli/gateway.py index 4c6fb3c..bf0ba50 100644 --- a/vlmrun/cli/_cli/gateway.py +++ b/vlmrun/cli/_cli/gateway.py @@ -26,6 +26,7 @@ from functools import lru_cache from pathlib import Path from typing import Any, Dict, List, Optional, Tuple +from urllib.parse import urlparse import typer from rich.console import Console @@ -43,7 +44,10 @@ format_file_size, handle_api_errors, ) -from vlmrun.constants import SUPPORTED_DOCUMENT_FILETYPES +from vlmrun.constants import ( + SUPPORTED_DOCUMENT_FILETYPES, + SUPPORTED_VIDEO_FILETYPES, +) console = Console() @@ -55,6 +59,8 @@ vlmrun gw chat a.pdf b.pdf -m paddleocr/pp-ocrv6 vlmrun gw chat img.jpg -m paddleocr/pp-ocrv6 vlmrun gw chat img.jpg -p "describe this image" -m qwen/qwen3.5-0.8b + vlmrun gw chat https://example.com/scan.jpg -m paddleocr/pp-ocrv6 + vlmrun gw chat https://example.com/report.pdf -m zai-org/glm-ocr vlmrun gw chat doc.pdf -m zai-org/glm-ocr -e temperature=0 -e max_tokens=4096 \b @@ -69,8 +75,9 @@ NOTES: Model ids are the full `/` shown by `vlmrun gw models`; short aliases (e.g. `glm-ocr`) also work. - Most gateway models (e.g. OCR models) require at least one input file and do - not accept text-only prompts. Use -p only for models that support it. + Inputs are local file paths or http(s) URLs (image, document or video). + Most gateway models (e.g. OCR models) require at least one input and do not + accept text-only prompts. Use -p only for models that support it. """ GATEWAY_HELP = """OCR, VLM, embedding and transcription models on the VLM Run gateway. @@ -136,6 +143,30 @@ def _guess_mime(path: Path, data: Optional[bytes] = None) -> str: return mime or "application/octet-stream" +def _is_http_url(value: str) -> bool: + """Return True if ``value`` looks like an http(s) URL.""" + return value.startswith(("http://", "https://")) + + +def _suffix_from_url(url: str) -> str: + """File extension from a URL path, ignoring query strings.""" + return Path(urlparse(url).path).suffix.lower() + + +def _content_part_type_for_suffix(suffix: str, mime: Optional[str] = None) -> str: + """Content-part type from a file extension and optional MIME type.""" + if suffix in SUPPORTED_DOCUMENT_FILETYPES: + return "document_url" + mime = mime or mimetypes.guess_type(f"name{suffix}")[0] or "" + if mime == "application/pdf": + return "document_url" + if mime.startswith("video/") or suffix in SUPPORTED_VIDEO_FILETYPES: + return "video_url" + if mime.startswith("image/"): + return "image_url" + return "file_url" + + def _content_part_type(path: Path, mime: Optional[str] = None) -> str: """Content-part type for a file. @@ -144,14 +175,14 @@ def _content_part_type(path: Path, mime: Optional[str] = None) -> str: identify: the gateway routes it through its document/PDF path, which fails outright on a plain image. """ - if path.suffix.lower() in SUPPORTED_DOCUMENT_FILETYPES: - return "document_url" - mime = mime or _guess_mime(path) - if mime == "application/pdf": - return "document_url" - if mime.startswith("image/"): - return "image_url" - return "file_url" + return _content_part_type_for_suffix(path.suffix.lower(), mime or _guess_mime(path)) + + +def _content_part_type_from_url(url: str) -> str: + """Content-part type for a remote http(s) URL.""" + suffix = _suffix_from_url(url) + mime, _ = mimetypes.guess_type(urlparse(url).path) + return _content_part_type_for_suffix(suffix, mime) def _encode_file_part(path: Path) -> Dict[str, Any]: @@ -171,6 +202,36 @@ def _encode_file_part(path: Path) -> Dict[str, Any]: } +def _encode_url_part(url: str) -> Dict[str, Any]: + """Build a gateway content part that references a remote http(s) URL.""" + key = _content_part_type_from_url(url) + return { + "type": key, + key: {"url": url}, + } + + +def _encode_chat_input(raw: str) -> Dict[str, Any]: + """Encode one chat input — a local file path or http(s) URL.""" + if _is_http_url(raw): + return _encode_url_part(raw) + path = Path(raw).expanduser() + return _encode_file_part(path) + + +def _validate_chat_input(raw: str) -> None: + """Ensure a non-URL chat input refers to a readable local file.""" + if _is_http_url(raw): + return + path = Path(raw).expanduser() + if not path.is_file(): + console.print( + f"[red]Error:[/] Input '{raw}' is not a file. " + "Provide a local path or an http(s) URL." + ) + raise typer.Exit(1) + + def _parse_response_format(value: str) -> Dict[str, Any]: """Parse ``--response-format`` into an OpenAI ``response_format`` object. @@ -206,9 +267,9 @@ def _parse_response_format(value: str) -> Dict[str, Any]: raise typer.Exit(1) -def _build_messages(files: List[Path], prompt: Optional[str]) -> List[Dict[str, Any]]: - """Build a single OpenAI-style user message from files + optional prompt.""" - content: List[Dict[str, Any]] = [_encode_file_part(f) for f in files] +def _build_messages(inputs: List[str], prompt: Optional[str]) -> List[Dict[str, Any]]: + """Build a single OpenAI-style user message from file paths/URLs + prompt.""" + content: List[Dict[str, Any]] = [_encode_chat_input(raw) for raw in inputs] if prompt: content.append({"type": "text", "text": prompt}) return [{"role": "user", "content": content}] @@ -533,11 +594,9 @@ def models( @app.command(help=CHAT_HELP, context_settings={"max_content_width": 120}) def chat( ctx: typer.Context, - files: List[Path] = typer.Argument( + inputs: List[str] = typer.Argument( None, - help="Input document/image file(s) to process. Repeatable.", - exists=True, - readable=True, + help="Input file path(s) or http(s) URL(s) (image, document or video). Repeatable.", ), model: str = typer.Option( ..., @@ -589,14 +648,17 @@ def chat( """Run a gateway model over one or more documents/images.""" client: VLMRun = ctx.obj - if not files and not prompt: + inputs = list(inputs or []) + if not inputs and not prompt: console.print( - "[red]Error:[/] Provide at least one input file. " + "[red]Error:[/] Provide at least one input file or URL. " "Most gateway models do not accept text-only input." ) raise typer.Exit(1) - files = files or [] + for raw in inputs: + _validate_chat_input(raw) + create_kwargs, extra_body = _split_create_kwargs(_parse_extra(extra)) if timeout is not None: create_kwargs["timeout"] = timeout @@ -621,22 +683,26 @@ def chat( if extra_body: create_kwargs["extra_body"] = extra_body - # Show the files being processed. - if files and not output_json: + # Show the inputs being processed. + if inputs and not output_json: tree = Tree("", guide_style="dim", hide_root=True) - for f in files: - size_str = format_file_size(f.stat().st_size) - tree.add(f"{f.name} [dim]({size_str})[/dim]") + for raw in inputs: + if _is_http_url(raw): + tree.add(raw) + else: + path = Path(raw).expanduser() + size_str = format_file_size(path.stat().st_size) + tree.add(f"{path.name} [dim]({size_str})[/dim]") console.print( Panel( tree, - title=f"Processing {len(files)} file(s) [dim]({model})[/dim]", + title=f"Processing {len(inputs)} input(s) [dim]({model})[/dim]", title_align="left", border_style="dim", ) ) - messages = _build_messages(files, prompt) + messages = _build_messages(inputs, prompt) start_time = time.time() status_msg = f"Processing ([bold]{model}[/bold])..." From 0ac6cdf0516de07c40afe72db8203caba93b1dd5 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 5 Aug 2026 23:21:43 +0000 Subject: [PATCH 2/2] Bump version to 0.7.2 Co-authored-by: Sudeep Pillai --- vlmrun/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vlmrun/version.py b/vlmrun/version.py index a5f830a..bc8c296 100644 --- a/vlmrun/version.py +++ b/vlmrun/version.py @@ -1 +1 @@ -__version__ = "0.7.1" +__version__ = "0.7.2"