Support http(s) URLs in vlmrun gw chat

tests/test_gateway.py

@@ -364,9 +364,35 @@ def test_content_part_type(self, tmp_path):
         assert gw._content_part_type(tmp_path / "a.docx") == "document_url"
         assert gw._content_part_type(tmp_path / "a.png") == "image_url"
         assert gw._content_part_type(tmp_path / "a.jpg") == "image_url"
+        assert gw._content_part_type(tmp_path / "a.mp4") == "video_url"
         # Unidentifiable content still falls back to file_url.
         assert gw._content_part_type(tmp_path / "a.bin") == "file_url"
 
+    def test_content_part_type_from_url(self):
+        assert (
+            gw._content_part_type_from_url("https://example.com/doc.pdf")
+            == "document_url"
+        )
+        assert (
+            gw._content_part_type_from_url("https://example.com/img.png") == "image_url"
+        )
+        assert (
+            gw._content_part_type_from_url("https://example.com/clip.mp4?token=1")
+            == "video_url"
+        )
+
+    def test_encode_url_part(self):
+        part = gw._encode_url_part("https://example.com/scan.jpg")
+        assert part == {
+            "type": "image_url",
+            "image_url": {"url": "https://example.com/scan.jpg"},
+        }
+
+    def test_encode_url_part_document(self):
+        part = gw._encode_url_part("https://example.com/report.pdf")
+        assert part["type"] == "document_url"
+        assert part["document_url"]["url"] == "https://example.com/report.pdf"
+
     def test_encode_document_part(self, tmp_path):
         f = tmp_path / "doc.pdf"
         f.write_bytes(b"%PDF-1.7 fake")
@@ -421,7 +447,7 @@ def test_sniff_mime_signatures(self):
     def test_build_messages_with_prompt(self, tmp_path):
         f = tmp_path / "img.png"
         f.write_bytes(PNG_BYTES)
-        messages = gw._build_messages([f], "describe")
+        messages = gw._build_messages([str(f)], "describe")
         assert len(messages) == 1
         content = messages[0]["content"]
         assert content[0]["type"] == "image_url"
@@ -432,9 +458,27 @@ def test_build_messages_mixed_files(self, tmp_path):
         doc = tmp_path / "doc.pdf"
         img.write_bytes(PNG_BYTES)
         doc.write_bytes(b"%PDF fake")
-        content = gw._build_messages([img, doc], None)[0]["content"]
+        content = gw._build_messages([str(img), str(doc)], None)[0]["content"]
         assert [p["type"] for p in content] == ["image_url", "document_url"]
 
+    def test_build_messages_with_url(self):
+        content = gw._build_messages(["https://example.com/scan.jpg"], None)[0][
+            "content"
+        ]
+        assert content[0]["type"] == "image_url"
+        assert content[0]["image_url"]["url"] == "https://example.com/scan.jpg"
+
+    def test_build_messages_mixed_file_and_url(self, tmp_path):
+        img = tmp_path / "img.png"
+        img.write_bytes(PNG_BYTES)
+        content = gw._build_messages([str(img), "https://example.com/doc.pdf"], None)[
+            0
+        ]["content"]
+        assert content[0]["type"] == "image_url"
+        assert content[0]["image_url"]["url"].startswith("data:image/png;base64,")
+        assert content[1]["type"] == "document_url"
+        assert content[1]["document_url"]["url"] == "https://example.com/doc.pdf"
+
     def test_parse_extra_json_and_string(self):
         parsed = gw._parse_extra(["temperature=0", "max_tokens=4096", "label=hello"])
         assert parsed == {"temperature": 0, "max_tokens": 4096, "label": "hello"}
@@ -840,7 +884,33 @@ def test_transcribe_bad_format(self, runner, patched_cli, tmp_path):
     def test_chat_requires_file(self, runner, patched_cli):
         result = runner.invoke(app, ["gw", "chat", "-m", "glm-ocr"])
         assert result.exit_code == 1
-        assert "at least one input file" in result.stdout.lower()
+        assert "at least one input" in result.stdout.lower()
+
+    def test_chat_rejects_missing_file(self, runner, patched_cli):
+        result = runner.invoke(
+            app, ["gw", "chat", "missing.pdf", "-m", "glm-ocr", "--no-stream"]
+        )
+        assert result.exit_code == 1
+        assert "not a file" in result.stdout.lower()
+
+    def test_chat_with_url(self, runner, patched_cli):
+        url = "https://example.com/scan.jpg"
+        result = runner.invoke(
+            app,
+            ["gw", "chat", url, "-m", "paddle-ocrv6", "--no-stream"],
+        )
+        assert result.exit_code == 0, result.stdout
+        call = patched_cli["client"].gateway.completions.calls[-1]
+        content = call["messages"][0]["content"]
+        assert content[0]["type"] == "image_url"
+        assert content[0]["image_url"]["url"] == url
+        assert call["stream"] is False
+
+    def test_chat_document_url_streams(self, runner, patched_cli):
+        url = "https://example.com/report.pdf"
+        result = runner.invoke(app, ["gw", "chat", url, "-m", "glm-ocr"])
+        assert result.exit_code == 0, result.stdout
+        assert patched_cli["client"].gateway.completions.calls[-1]["stream"] is True
 
     def test_chat_with_file_json(self, runner, patched_cli, tmp_path):
         f = tmp_path / "doc.pdf"

vlmrun/cli/_cli/gateway.py

@@ -26,6 +26,7 @@
 from functools import lru_cache
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
+from urllib.parse import urlparse
 
 import typer
 from rich.console import Console
@@ -43,7 +44,10 @@
     format_file_size,
     handle_api_errors,
 )
-from vlmrun.constants import SUPPORTED_DOCUMENT_FILETYPES
+from vlmrun.constants import (
+    SUPPORTED_DOCUMENT_FILETYPES,
+    SUPPORTED_VIDEO_FILETYPES,
+)
 
 console = Console()
 
@@ -55,6 +59,8 @@
   vlmrun gw chat a.pdf b.pdf -m paddleocr/pp-ocrv6
   vlmrun gw chat img.jpg -m paddleocr/pp-ocrv6
   vlmrun gw chat img.jpg -p "describe this image" -m qwen/qwen3.5-0.8b
+  vlmrun gw chat https://example.com/scan.jpg -m paddleocr/pp-ocrv6
+  vlmrun gw chat https://example.com/report.pdf -m zai-org/glm-ocr
   vlmrun gw chat doc.pdf -m zai-org/glm-ocr -e temperature=0 -e max_tokens=4096
 
 \b
@@ -69,8 +75,9 @@
 NOTES:
   Model ids are the full `<org>/<name>` shown by `vlmrun gw models`; short
   aliases (e.g. `glm-ocr`) also work.
-  Most gateway models (e.g. OCR models) require at least one input file and do
-  not accept text-only prompts. Use -p only for models that support it.
+  Inputs are local file paths or http(s) URLs (image, document or video).
+  Most gateway models (e.g. OCR models) require at least one input and do not
+  accept text-only prompts. Use -p only for models that support it.
 """
 
 GATEWAY_HELP = """OCR, VLM, embedding and transcription models on the VLM Run gateway.
@@ -136,6 +143,30 @@ def _guess_mime(path: Path, data: Optional[bytes] = None) -> str:
     return mime or "application/octet-stream"
 
 
+def _is_http_url(value: str) -> bool:
+    """Return True if ``value`` looks like an http(s) URL."""
+    return value.startswith(("http://", "https://"))
+
+
+def _suffix_from_url(url: str) -> str:
+    """File extension from a URL path, ignoring query strings."""
+    return Path(urlparse(url).path).suffix.lower()
+
+
+def _content_part_type_for_suffix(suffix: str, mime: Optional[str] = None) -> str:
+    """Content-part type from a file extension and optional MIME type."""
+    if suffix in SUPPORTED_DOCUMENT_FILETYPES:
+        return "document_url"
+    mime = mime or mimetypes.guess_type(f"name{suffix}")[0] or ""
+    if mime == "application/pdf":
+        return "document_url"
+    if mime.startswith("video/") or suffix in SUPPORTED_VIDEO_FILETYPES:
+        return "video_url"
+    if mime.startswith("image/"):
+        return "image_url"
+    return "file_url"
+
+
 def _content_part_type(path: Path, mime: Optional[str] = None) -> str:
     """Content-part type for a file.
 
@@ -144,14 +175,14 @@ def _content_part_type(path: Path, mime: Optional[str] = None) -> str:
     identify: the gateway routes it through its document/PDF path, which fails
     outright on a plain image.
     """
-    if path.suffix.lower() in SUPPORTED_DOCUMENT_FILETYPES:
-        return "document_url"
-    mime = mime or _guess_mime(path)
-    if mime == "application/pdf":
-        return "document_url"
-    if mime.startswith("image/"):
-        return "image_url"
-    return "file_url"
+    return _content_part_type_for_suffix(path.suffix.lower(), mime or _guess_mime(path))
+
+
+def _content_part_type_from_url(url: str) -> str:
+    """Content-part type for a remote http(s) URL."""
+    suffix = _suffix_from_url(url)
+    mime, _ = mimetypes.guess_type(urlparse(url).path)
+    return _content_part_type_for_suffix(suffix, mime)
 
 
 def _encode_file_part(path: Path) -> Dict[str, Any]:
@@ -171,6 +202,36 @@ def _encode_file_part(path: Path) -> Dict[str, Any]:
     }
 
 
+def _encode_url_part(url: str) -> Dict[str, Any]:
+    """Build a gateway content part that references a remote http(s) URL."""
+    key = _content_part_type_from_url(url)
+    return {
+        "type": key,
+        key: {"url": url},
+    }
+
+
+def _encode_chat_input(raw: str) -> Dict[str, Any]:
+    """Encode one chat input — a local file path or http(s) URL."""
+    if _is_http_url(raw):
+        return _encode_url_part(raw)
+    path = Path(raw).expanduser()
+    return _encode_file_part(path)
+
+
+def _validate_chat_input(raw: str) -> None:
+    """Ensure a non-URL chat input refers to a readable local file."""
+    if _is_http_url(raw):
+        return
+    path = Path(raw).expanduser()
+    if not path.is_file():
+        console.print(
+            f"[red]Error:[/] Input '{raw}' is not a file. "
+            "Provide a local path or an http(s) URL."
+        )
+        raise typer.Exit(1)
+
+
 def _parse_response_format(value: str) -> Dict[str, Any]:
     """Parse ``--response-format`` into an OpenAI ``response_format`` object.
 
@@ -206,9 +267,9 @@ def _parse_response_format(value: str) -> Dict[str, Any]:
     raise typer.Exit(1)
 
 
-def _build_messages(files: List[Path], prompt: Optional[str]) -> List[Dict[str, Any]]:
-    """Build a single OpenAI-style user message from files + optional prompt."""
-    content: List[Dict[str, Any]] = [_encode_file_part(f) for f in files]
+def _build_messages(inputs: List[str], prompt: Optional[str]) -> List[Dict[str, Any]]:
+    """Build a single OpenAI-style user message from file paths/URLs + prompt."""
+    content: List[Dict[str, Any]] = [_encode_chat_input(raw) for raw in inputs]
     if prompt:
         content.append({"type": "text", "text": prompt})
     return [{"role": "user", "content": content}]
@@ -533,11 +594,9 @@ def models(
 @app.command(help=CHAT_HELP, context_settings={"max_content_width": 120})
 def chat(
     ctx: typer.Context,
-    files: List[Path] = typer.Argument(
+    inputs: List[str] = typer.Argument(
         None,
-        help="Input document/image file(s) to process. Repeatable.",
-        exists=True,
-        readable=True,
+        help="Input file path(s) or http(s) URL(s) (image, document or video). Repeatable.",
     ),
     model: str = typer.Option(
         ...,
@@ -589,14 +648,17 @@ def chat(
     """Run a gateway model over one or more documents/images."""
     client: VLMRun = ctx.obj
 
-    if not files and not prompt:
+    inputs = list(inputs or [])
+    if not inputs and not prompt:
         console.print(
-            "[red]Error:[/] Provide at least one input file. "
+            "[red]Error:[/] Provide at least one input file or URL. "
             "Most gateway models do not accept text-only input."
         )
         raise typer.Exit(1)
 
-    files = files or []
+    for raw in inputs:
+        _validate_chat_input(raw)
+
     create_kwargs, extra_body = _split_create_kwargs(_parse_extra(extra))
     if timeout is not None:
         create_kwargs["timeout"] = timeout
@@ -621,22 +683,26 @@ def chat(
     if extra_body:
         create_kwargs["extra_body"] = extra_body
 
-    # Show the files being processed.
-    if files and not output_json:
+    # Show the inputs being processed.
+    if inputs and not output_json:
         tree = Tree("", guide_style="dim", hide_root=True)
-        for f in files:
-            size_str = format_file_size(f.stat().st_size)
-            tree.add(f"{f.name} [dim]({size_str})[/dim]")
+        for raw in inputs:
+            if _is_http_url(raw):
+                tree.add(raw)
+            else:
+                path = Path(raw).expanduser()
+                size_str = format_file_size(path.stat().st_size)
+                tree.add(f"{path.name} [dim]({size_str})[/dim]")
         console.print(
             Panel(
                 tree,
-                title=f"Processing {len(files)} file(s) [dim]({model})[/dim]",
+                title=f"Processing {len(inputs)} input(s) [dim]({model})[/dim]",
                 title_align="left",
                 border_style="dim",
             )
         )
 
-    messages = _build_messages(files, prompt)
+    messages = _build_messages(inputs, prompt)
     start_time = time.time()
     status_msg = f"Processing ([bold]{model}[/bold])..."
 

vlmrun/version.py

@@ -1 +1 @@
-__version__ = "0.7.1"
+__version__ = "0.7.2"