From b49679f5b21fff75ca696422edc54aabdb8678c9 Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Fri, 4 Sep 2026 14:52:50 +0100 Subject: [PATCH] docs: document llmman as a local model server llmman (https://github.com/llmmanorg/llmman) serves the Ollama API on port 17434, so it already works through any-llm's ollama/ provider by pointing api_base at it. Ollama only appears in docs and docstrings, so mirror those touchpoints: models.md, the AgentConfig.api_base docstring, and the Local Agent cookbook (.md and .ipynb). --- docs/agents/models.md | 13 ++++++++++++- docs/cookbook/agent-with-local-llm.md | 3 +++ docs/cookbook/agent_with_local_llm.ipynb | 3 +++ src/any_agent/config.py | 4 ++-- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/docs/agents/models.md b/docs/agents/models.md index 776bab4c..b0ee841b 100644 --- a/docs/agents/models.md +++ b/docs/agents/models.md @@ -14,10 +14,21 @@ The `model_args` parameter allows you to pass additional arguments to the model, The `api_base` parameter allows you to specify a custom API endpoint. This is useful when: -- Using a local model server (e.g., Ollama, llama.cpp, llamafile) +- Using a local model server (e.g., Ollama, llmman, llama.cpp, llamafile) - Routing through a proxy - Using a self-hosted model endpoint +For example, [llmman](https://github.com/llmmanorg/llmman) serves the Ollama API on port `17434` instead of `11434`, so it can be used through the `ollama/` provider by pointing `api_base` at it: + +```python +from any_agent import AgentConfig + +config = AgentConfig( + model_id="ollama/gemma4", + api_base="http://localhost:17434", +) +``` + The `api_key` parameter allows you to explicitly specify an API key for authentication. By default, `any-llm` will automatically search for common environment variables (like `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, etc.). See the [AnyLLM Provider Documentation](https://docs.mozilla.ai/any-llm/providers/) for the complete list of supported providers. diff --git a/docs/cookbook/agent-with-local-llm.md b/docs/cookbook/agent-with-local-llm.md index c197f1b8..d755ca42 100644 --- a/docs/cookbook/agent-with-local-llm.md +++ b/docs/cookbook/agent-with-local-llm.md @@ -37,6 +37,9 @@ we will use [Ollama](https://ollama.com/) to run our LLM locally! First, install Ollama by following their instructions: https://ollama.com/download +>**Note**: [llmman](https://github.com/llmmanorg/llmman) serves the same Ollama API on port `17434`. +> To use it instead, keep the `ollama/` prefix in `model_id` and add `api_base="http://localhost:17434"` to your `AgentConfig`. + ### Picking an LLM Pick a model that you can run locally based on your hardware and download it from your terminal. For example: diff --git a/docs/cookbook/agent_with_local_llm.ipynb b/docs/cookbook/agent_with_local_llm.ipynb index 6bebcf2a..41ff4204 100644 --- a/docs/cookbook/agent_with_local_llm.ipynb +++ b/docs/cookbook/agent_with_local_llm.ipynb @@ -64,6 +64,9 @@ "\n", "First, install Ollama by following their instructions: https://ollama.com/download\n", "\n", + ">**Note**: [llmman](https://github.com/llmmanorg/llmman) serves the same Ollama API on port `17434`.\n", + "> To use it instead, keep the `ollama/` prefix in `model_id` and add `api_base=\"http://localhost:17434\"` to your `AgentConfig`.\n", + "\n", "### Picking an LLM\n", "\n", "Pick a model that you can run locally based on your hardware and download it from your terminal. For example:\n", diff --git a/src/any_agent/config.py b/src/any_agent/config.py index 351dc9bd..18567909 100644 --- a/src/any_agent/config.py +++ b/src/any_agent/config.py @@ -86,8 +86,8 @@ class AgentConfig(BaseModel): api_base: str | None = None """Custom API endpoint URL for the model provider. - Use this to specify custom endpoints for local models (Ollama, llama.cpp, etc.) or proxy services. - For example: `http://localhost:11434/v1` for Ollama. + Use this to specify custom endpoints for local models (Ollama, llmman, llama.cpp, etc.) or proxy services. + For example: `http://localhost:11434/v1` for Ollama, or `http://localhost:17434` for llmman. """ api_key: str | None = None