feat: depend on docling-slim and convert sources locally or with a Docling service

.github/workflows/checks.yml

@@ -48,6 +48,26 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./coverage.xml
 
+  run-slim-tests:
+    # The base install depends on docling-slim only: no PyTorch, no local models.
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.12']
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install uv and set the python version
+        uses: astral-sh/setup-uv@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          enable-cache: true
+      - name: Install dependencies without extras
+        run: uv sync --frozen
+      - name: Check that the local model stack is not installed
+        run: uv run --no-sync python -c "import importlib.util as u, sys; sys.exit(any(u.find_spec(m) for m in ('torch', 'docling_ibm_models')))"
+      - name: Run tests
+        run: uv run --no-sync pytest -m "not slow" tests
+
   build-package:
     runs-on: ubuntu-latest
     strategy:

README.md

@@ -36,6 +36,27 @@ pip install docling-agent
 
 Requires Python 3.11 or higher.
 
+The base install builds on [`docling-slim`](https://pypi.org/project/docling-slim/) and needs no PyTorch or local models. The agents work on `DoclingDocument`s, and source files (PDF, DOCX, images, ...) are converted by a [Docling Serve](https://github.com/docling-project/docling-serve) endpoint, such as Docling for IBM watsonx. Extras add capabilities that run locally:
+
+| Extra | Install | Adds |
+|-------|---------|------|
+| `local` | `pip install 'docling-agent[local]'` | Convert source files in-process with the Docling models |
+| `vlm` | `pip install 'docling-agent[vlm]'` | `DoclingExtractingAgent`, which runs a vision-language model locally (includes `local`) |
+| `evaluation` | `pip install 'docling-agent[evaluation]'` | Dependencies of the evaluation scripts in [`perfs/`](perfs/) |
+
+### Converting Source Files
+
+Tasks that ingest files, such as the `add` mode of the `docling-agent` CLI, pick the converter from these environment variables (also read from a `.env` file):
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `DOCLING_AGENT_CONVERSION_MODE` | `auto` | `remote`, `local`, or `auto`: remote when a service URL is set, local otherwise |
+| `DOCLING_AGENT_SERVICE_URL` | | Base URL of the Docling Serve endpoint |
+| `DOCLING_AGENT_SERVICE_API_KEY` | | API key for the endpoint |
+| `DOCLING_AGENT_SERVICE_TIMEOUT` | `300` | Timeout in seconds for a remote conversion |
+
+The `fast`, `standard`, and `expensive` conversion presets apply the same pipeline options locally and remotely.
+
 ## Getting Started
 
 Each snippet shows how to initialise an agent, run a task, and save the result.
@@ -84,7 +105,7 @@ updated.save_as_html("./scratch/updated_table.html")
 
 ### Extract Structured Data with a Schema
 
-Define a simple schema and provide a list of files (PDFs/images); the agent produces an HTML report with extracted fields ([example](examples/example_03_extract_schema.py)).
+Define a simple schema and provide a list of files (PDFs/images); the agent produces an HTML report with extracted fields ([example](examples/example_03_extract_schema.py)). This agent runs a vision-language model locally and needs the `vlm` extra.
 
 ```python
 from pathlib import Path
@@ -175,7 +196,7 @@ Typical defaults:
 
 ## Examples
 
-Explore the [`examples/`](examples/) folder for end-to-end scripts covering document writing, editing, extraction, enrichment, RAG querying, and more.
+Explore the [`examples/`](examples/) folder for end-to-end scripts covering document writing, editing, extraction, enrichment, RAG querying, and more. The examples that convert files with a local `DocumentConverter` (06 and 07) need the `local` extra, and the extraction example (03) needs the `vlm` extra.
 
 ## Technical Report
 

docling_agent/agent/extractor.py

@@ -2,8 +2,6 @@
 from pathlib import Path
 from typing import Any, ClassVar
 
-from docling.datamodel.base_models import InputFormat
-from docling.document_extractor import DocumentExtractor
 from docling_core.types.doc import (
     CodeLanguageLabel,
     DoclingDocument,
@@ -49,6 +47,18 @@ def __init__(
             backend=backend or self.default_backend(),
             tools=tools,
         )
+        # Imported here because DocumentExtractor pulls in the VLM stack (PyTorch, transformers),
+        # which importing docling_agent should not require when this agent is not used.
+        from docling.datamodel.base_models import InputFormat
+
+        try:
+            from docling.document_extractor import DocumentExtractor
+        except ImportError as exc:
+            raise ImportError(
+                "DoclingExtractingAgent runs a vision-language model locally. "
+                "Install the vlm extra with: pip install 'docling-agent[vlm]'"
+            ) from exc
+
         self._extractor: DocumentExtractor = DocumentExtractor(allowed_formats=[InputFormat.IMAGE, InputFormat.PDF])
         self._last_results: dict[str, list[Any]] = {}
 

docling_agent/agent/orchestrator.py

@@ -5,10 +5,9 @@
 import time
 from contextvars import ContextVar
 from pathlib import Path
-from typing import Literal, cast
+from typing import cast
 
 from docling.datamodel.base_models import ConversionStatus, FormatToMimeType, InputFormat
-from docling.document_converter import DocumentConverter
 from docling_core.transforms.serializer.markdown import MarkdownDocSerializer
 from docling_core.types.doc.document import (
     DocItemLabel,
@@ -29,6 +28,7 @@
 from docling_agent.agent.library import DocLibraryEntry, DoclingLibrary
 from docling_agent.agent.rag import DoclingRAGAgent
 from docling_agent.agent.writer import DoclingWritingAgent
+from docling_agent.conversion import ConversionPreset, SourceConverter, create_source_converter
 from docling_agent.logging import log_error, log_info, log_warning
 from docling_agent.task_model import (
     AddTask,
@@ -256,30 +256,26 @@ def _resolve_sources(self, task: AgentTask, library: DoclingLibrary) -> list[_So
 
         # Batch-convert all uncached files
         if raw_to_convert:
-            converter = self._build_source_converter(conversion)
-            for p in raw_to_convert:
-                source_key = str(p.resolve())
-                try:
-                    conv = converter.convert(p)
-                    if conv.status == ConversionStatus.SUCCESS:
-                        doc = conv.document
-                        entry = library.store(
-                            doc,
-                            source_key,
-                            project_id=task.project_id,
-                            original_mimetype=self._mimetype_for_input_format(conv.input.format),
-                            conversion_pipeline=self._pipeline_name_for_input_format(
-                                converter=converter,
-                                input_format=conv.input.format,
-                                conversion=conversion,
-                            ),
-                        )
-                        results.append((doc, entry.doc_id))
-                        log_info(f"Converted and cached: {p.name} → {entry.doc_id}")
-                    else:
-                        log_warning(f"Conversion failed for {p.name}: {conv.status}")
-                except Exception as exc:
-                    log_error(f"Error converting {p}: {exc}")
+            with self._build_source_converter(conversion) as converter:
+                for p in raw_to_convert:
+                    source_key = str(p.resolve())
+                    try:
+                        conv = converter.convert(p)
+                        if conv.status == ConversionStatus.SUCCESS:
+                            doc = conv.document
+                            entry = library.store(
+                                doc,
+                                source_key,
+                                project_id=task.project_id,
+                                original_mimetype=self._mimetype_for_input_format(conv.input.format),
+                                conversion_pipeline=converter.pipeline_name(conv.input.format),
+                            )
+                            results.append((doc, entry.doc_id))
+                            log_info(f"Converted and cached: {p.name} → {entry.doc_id}")
+                        else:
+                            log_warning(f"Conversion failed for {p.name}: {conv.status}")
+                    except Exception as exc:
+                        log_error(f"Error converting {p}: {exc}")
 
         log_info(f"Resolved {len(results)} document(s)")
         return results
@@ -756,46 +752,19 @@ def _mimetype_for_input_format(self, input_format: InputFormat) -> str | None:
         mimetypes = FormatToMimeType.get(input_format, [])
         return mimetypes[0] if mimetypes else None
 
-    def _pipeline_name_for_input_format(
-        self,
-        *,
-        converter: DocumentConverter,
-        input_format: InputFormat,
-        conversion: Literal["fast", "standard", "expensive"],
-    ) -> str:
-        format_option = converter.format_to_options.get(input_format)
-        if format_option is None:
-            return f"unknown:{conversion}"
-        return f"{format_option.pipeline_cls.__name__}:{conversion}"
-
-    def _source_conversion_preset(self, task: AgentTask) -> Literal["fast", "standard", "expensive"]:
+    def _source_conversion_preset(self, task: AgentTask) -> ConversionPreset:
         if isinstance(task, AddTask):
             return task.conversion
         return "standard"
 
-    def _build_source_converter(self, conversion: Literal["fast", "standard", "expensive"]) -> DocumentConverter:
-        converter = DocumentConverter()
-        for format_option in converter.format_to_options.values():
-            pipeline_options = format_option.pipeline_options
-            if pipeline_options is None:
-                continue
-            if hasattr(pipeline_options, "do_ocr"):
-                pipeline_options.do_ocr = conversion != "fast"
-            if hasattr(pipeline_options, "do_table_structure"):
-                pipeline_options.do_table_structure = conversion != "fast"
-            if hasattr(pipeline_options, "do_picture_classification"):
-                pipeline_options.do_picture_classification = True
-            if hasattr(pipeline_options, "do_chart_extraction"):
-                pipeline_options.do_chart_extraction = conversion == "expensive"
-            if hasattr(pipeline_options, "generate_page_images"):
-                pipeline_options.generate_page_images = True
-        return converter
+    def _build_source_converter(self, conversion: ConversionPreset) -> SourceConverter:
+        return create_source_converter(conversion)
 
     def _cache_entry_matches_conversion(
         self,
         entry: DocLibraryEntry,
         *,
-        conversion: Literal["fast", "standard", "expensive"],
+        conversion: ConversionPreset,
     ) -> bool:
         for run in entry.status.pipelines:
             if run.name.endswith(f":{conversion}"):