diff --git a/docs/agents/models.md b/docs/agents/models.md index 776bab4c..b0ee841b 100644 --- a/docs/agents/models.md +++ b/docs/agents/models.md @@ -14,10 +14,21 @@ The `model_args` parameter allows you to pass additional arguments to the model, The `api_base` parameter allows you to specify a custom API endpoint. This is useful when: -- Using a local model server (e.g., Ollama, llama.cpp, llamafile) +- Using a local model server (e.g., Ollama, llmman, llama.cpp, llamafile) - Routing through a proxy - Using a self-hosted model endpoint +For example, [llmman](https://github.com/llmmanorg/llmman) serves the Ollama API on port `17434` instead of `11434`, so it can be used through the `ollama/` provider by pointing `api_base` at it: + +```python +from any_agent import AgentConfig + +config = AgentConfig( + model_id="ollama/gemma4", + api_base="http://localhost:17434", +) +``` + The `api_key` parameter allows you to explicitly specify an API key for authentication. By default, `any-llm` will automatically search for common environment variables (like `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, etc.). See the [AnyLLM Provider Documentation](https://docs.mozilla.ai/any-llm/providers/) for the complete list of supported providers. diff --git a/docs/cookbook/agent-with-local-llm.md b/docs/cookbook/agent-with-local-llm.md index c197f1b8..d755ca42 100644 --- a/docs/cookbook/agent-with-local-llm.md +++ b/docs/cookbook/agent-with-local-llm.md @@ -37,6 +37,9 @@ we will use [Ollama](https://ollama.com/) to run our LLM locally! First, install Ollama by following their instructions: https://ollama.com/download +>**Note**: [llmman](https://github.com/llmmanorg/llmman) serves the same Ollama API on port `17434`. +> To use it instead, keep the `ollama/` prefix in `model_id` and add `api_base="http://localhost:17434"` to your `AgentConfig`. + ### Picking an LLM Pick a model that you can run locally based on your hardware and download it from your terminal. For example: diff --git a/docs/cookbook/agent_with_local_llm.ipynb b/docs/cookbook/agent_with_local_llm.ipynb index 6bebcf2a..41ff4204 100644 --- a/docs/cookbook/agent_with_local_llm.ipynb +++ b/docs/cookbook/agent_with_local_llm.ipynb @@ -64,6 +64,9 @@ "\n", "First, install Ollama by following their instructions: https://ollama.com/download\n", "\n", + ">**Note**: [llmman](https://github.com/llmmanorg/llmman) serves the same Ollama API on port `17434`.\n", + "> To use it instead, keep the `ollama/` prefix in `model_id` and add `api_base=\"http://localhost:17434\"` to your `AgentConfig`.\n", + "\n", "### Picking an LLM\n", "\n", "Pick a model that you can run locally based on your hardware and download it from your terminal. For example:\n", diff --git a/src/any_agent/config.py b/src/any_agent/config.py index 351dc9bd..18567909 100644 --- a/src/any_agent/config.py +++ b/src/any_agent/config.py @@ -86,8 +86,8 @@ class AgentConfig(BaseModel): api_base: str | None = None """Custom API endpoint URL for the model provider. - Use this to specify custom endpoints for local models (Ollama, llama.cpp, etc.) or proxy services. - For example: `http://localhost:11434/v1` for Ollama. + Use this to specify custom endpoints for local models (Ollama, llmman, llama.cpp, etc.) or proxy services. + For example: `http://localhost:11434/v1` for Ollama, or `http://localhost:17434` for llmman. """ api_key: str | None = None