feat: add support for LoRA adapters

CMakeLists.txt

@@ -154,6 +154,18 @@ if(AGENT_CPP_BUILD_TESTS)
     target_link_libraries(test_grammar PRIVATE model ${LLAMA_COMMON_TARGET} llama)
     target_compile_features(test_grammar PRIVATE cxx_std_17)
 
+    add_executable(test_lora tests/test_lora.cpp)
+    target_include_directories(test_lora PRIVATE
+        src
+        tests
+        ${LLAMA_SOURCE_DIR}/common
+        ${LLAMA_SOURCE_DIR}/ggml/include
+        ${LLAMA_SOURCE_DIR}/include
+        ${LLAMA_SOURCE_DIR}/vendor
+    )
+    target_link_libraries(test_lora PRIVATE model ${LLAMA_COMMON_TARGET} llama)
+    target_compile_features(test_lora PRIVATE cxx_std_17)
+
     add_executable(test_callbacks tests/test_callbacks.cpp)
     target_include_directories(test_callbacks PRIVATE
         src
@@ -185,6 +197,7 @@ if(AGENT_CPP_BUILD_TESTS)
     add_test(NAME ToolTests COMMAND test_tool)
     add_test(NAME CallbacksTests COMMAND test_callbacks)
     add_test(NAME GrammarTests COMMAND test_grammar)
+    add_test(NAME LoraTests COMMAND test_lora)
     add_test(NAME ChatParserTests COMMAND test_chat_parser)
 
     if(AGENT_CPP_BUILD_MCP)
@@ -203,14 +216,14 @@ if(AGENT_CPP_BUILD_TESTS)
     # On Windows, DLLs are placed in the bin/ directory by llama.cpp
     # We need to add this directory to PATH so tests can find the DLLs
     # A hung test must fail rather than hold the runner until the job limit
-    set_tests_properties(ToolTests CallbacksTests GrammarTests ChatParserTests
+    set_tests_properties(ToolTests CallbacksTests GrammarTests LoraTests ChatParserTests
         PROPERTIES TIMEOUT 120
     )
 
     if(WIN32)
         # Multi-config generators (Visual Studio) place DLLs in bin/<config>,
         # single-config ones in bin/ - both are on PATH so tests can load them
-        set_tests_properties(ToolTests CallbacksTests GrammarTests ChatParserTests PROPERTIES
+        set_tests_properties(ToolTests CallbacksTests GrammarTests LoraTests ChatParserTests PROPERTIES
             ENVIRONMENT "PATH=${CMAKE_BINARY_DIR}/bin/${CMAKE_BUILD_TYPE}\;${CMAKE_BINARY_DIR}/bin/Release\;${CMAKE_BINARY_DIR}/bin\;$ENV{PATH}"
         )
     endif()
@@ -284,6 +297,19 @@ if(AGENT_CPP_BUILD_EXAMPLES)
     target_link_libraries(grammar-example PRIVATE agent model ${LLAMA_COMMON_TARGET} llama)
     target_compile_features(grammar-example PRIVATE cxx_std_17)
 
+    # LoRA example
+    add_executable(lora-example examples/lora/lora.cpp)
+    target_include_directories(lora-example PRIVATE
+        ${CMAKE_CURRENT_SOURCE_DIR}/src
+        ${CMAKE_CURRENT_SOURCE_DIR}/examples/shared
+        ${LLAMA_SOURCE_DIR}/common
+        ${LLAMA_SOURCE_DIR}/ggml/include
+        ${LLAMA_SOURCE_DIR}/include
+        ${LLAMA_SOURCE_DIR}/vendor
+    )
+    target_link_libraries(lora-example PRIVATE agent model ${LLAMA_COMMON_TARGET} llama)
+    target_compile_features(lora-example PRIVATE cxx_std_17)
+
     # MCP client example (requires MCP support)
     if(AGENT_CPP_BUILD_MCP)
         add_executable(mcp-example examples/mcp/mcp.cpp)

README.md

@@ -11,6 +11,8 @@ Building blocks for **local** agents in C++.
 
 - **[Grammar](./examples/grammar/README.md)** - Constrain model output to a GBNF grammar so every response matches a fixed structure.
 
+- **[LoRA](./examples/lora/README.md)** - Apply a GGUF LoRA adapter to a model with a configurable scale.
+
 - **[Memory](./examples/memory/README.md)** - Use tools that allow an agent to store and retrieve relevant information across conversations.
 
 - **[Multi-Agent](./examples/multi-agent/README.md)** - Build a multi-agent system with weight sharing where a main agent delegates to specialized sub-agents.
@@ -86,6 +88,21 @@ A grammar constrains every response, including the ones the agent would use to c
 
 See the [Grammar example](./examples/grammar/README.md) for a full working demo.
 
+### LoRA adapters
+
+Each `Model` can load its own [LoRA adapters](https://github.com/ggml-org/llama.cpp/tree/master/tools/completion#lora-low-rank-adaptation-adapters) on top of the shared base weights, useful for giving specialized agents in a [multi-agent](./examples/multi-agent/README.md) setup their own fine-tuned behavior without duplicating the base model in memory:
+
+```cpp
+ModelConfig config;
+config.loras = {
+    { "path/to/adapter.gguf", 1.0F },  // path, scale (defaults to 1.0)
+};
+
+auto model = Model::create_with_weights(shared_weights, config);
+```
+
+Multiple adapters may be stacked by adding more entries; each is scaled independently. See the [LoRA example](./examples/lora/README.md) for instructions on converting adapters from common training formats to GGUF.
+
 ## Tools
 
 Tools extend the agent's capabilities beyond text generation. Each tool defines:

examples/README.md

@@ -6,6 +6,10 @@ This directory contains example applications demonstrating agent.cpp capabilitie
 
 The [grammar](./grammar) example demonstrates constraining model output to a GBNF grammar, so every response is guaranteed to match a fixed structure. You can also point it at a custom grammar file and root rule.
 
+## LoRA
+
+The [lora](./lora) example demonstrates loading a GGUF LoRA adapter onto a model with a configurable scale.
+
 ## Shared Utilities
 
 The [shared](./shared) directory contains reusable helper components used across multiple examples. These are **not part of the public API** but can be useful as reference implementations.

examples/lora/CMakeLists.txt

@@ -0,0 +1,24 @@
+cmake_minimum_required(VERSION 3.14)
+project(lora-example VERSION 0.1.0)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../.. ${CMAKE_CURRENT_BINARY_DIR}/agent-cpp)
+
+add_executable(lora-example lora.cpp)
+
+target_include_directories(lora-example PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}/../../src
+    ${CMAKE_CURRENT_SOURCE_DIR}/../shared
+    ${LLAMA_SOURCE_DIR}/common
+    ${LLAMA_SOURCE_DIR}/ggml/include
+    ${LLAMA_SOURCE_DIR}/include
+    ${LLAMA_SOURCE_DIR}/vendor
+)
+
+target_link_libraries(lora-example PRIVATE agent-cpp::agent ${LLAMA_COMMON_TARGET} llama)
+target_compile_features(lora-example PRIVATE cxx_std_17)
+
+message(STATUS "LoRA example configured.")

examples/lora/README.md

@@ -0,0 +1,117 @@
+# LoRA Example
+
+This example loads a GGUF LoRA adapter onto a base model and runs an interactive chat. The adapter is applied to the model context with the requested scale, while the base model remains in the normal GGUF file.
+
+## Building
+
+> [!IMPORTANT]
+> Check the [llama.cpp build documentation](https://github.com/ggml-org/llama.cpp/blob/master/docs/build.md) to find
+> CMake flags you might want to pass depending on your available hardware.
+
+```bash
+cd examples/lora
+
+git -C ../.. submodule update --init --recursive
+
+cmake -B build
+cmake --build build -j$(nproc)
+```
+
+### Using a custom llama.cpp
+
+If you have llama.cpp already downloaded:
+
+```bash
+cmake -B build -DLLAMA_CPP_DIR=/path/to/your/llama.cpp
+cmake --build build -j$(nproc)
+```
+
+## Requirements
+
+The adapter must be GGUF and compatible with the base model. The architecture must match, and the same base model must be used for conversion and inference.
+
+Some adapter families also ship an aLoRA (Activated LoRA) variant alongside the standard one, in a separate folder such as `alora/`. This example only supports standard LoRA adapters — use the `lora/` variant, not `alora/`, if both are offered.
+
+## Preparing an Adapter
+
+If the adapter is already GGUF, skip to [Usage](#usage). Otherwise, convert it with llama.cpp's [`convert_lora_to_gguf.py`](https://github.com/ggml-org/llama.cpp/blob/master/convert_lora_to_gguf.py). You can download models from any registry; for Hugging Face, `huggingface-cli` is one option. The conversion input must contain the adapter configuration and weights. Use the same compatible base model for conversion and inference.
+
+### Worked Example: Granite Query Rewriting
+
+This example uses uv for dependency and environment management. The example below assumes you are in a uv-managed environment for the llama.cpp dependency setup. Run from the repository root:
+
+```bash
+wget https://huggingface.co/ibm-granite/granite-4.0-micro-GGUF/resolve/main/granite-4.0-micro-Q8_0.gguf
+
+cd deps/llama.cpp
+uv venv
+source .venv/bin/activate
+uv pip install -r requirements.txt
+```
+
+If your environment is configured with a package index that exposes an older `requests` build first, retry with:
+
+```bash
+uv pip install --index-strategy unsafe-best-match -r requirements.txt
+```
+
+```bash
+hf download ibm-granite/granitelib-rag-r1.0 \
+  "query_rewrite/granite-4.0-micro/lora/adapter_config.json" \
+  "query_rewrite/granite-4.0-micro/lora/adapter_model.safetensors" \
+  --local-dir ./granite-rag-adapters
+
+python convert_lora_to_gguf.py \
+  granite-rag-adapters/query_rewrite/granite-4.0-micro/lora \
+  --base-model-id ibm-granite/granite-4.0-micro \
+  --outfile ../../query_rewrite_lora.gguf
+```
+
+Return to the repository root and run the adapter with the binary built above:
+
+```bash
+cd ../..
+
+./examples/lora/build/lora-example \
+  -m ./granite-4.0-micro-Q8_0.gguf \
+  -l ./query_rewrite_lora.gguf \
+  -s 0.25
+```
+
+## Usage
+
+```bash
+./build/lora-example \
+  -m /path/to/model.gguf \
+  -l /path/to/adapter.gguf \
+  -s 0.25
+```
+
+`-s` is the adapter scale; it defaults to `1.0` if omitted. A scale of `0` disables the adapter for the context without unloading it. Multiple adapters can be stacked through `ModelConfig::loras`; this example keeps the command line focused on one adapter.
+
+## Using the Adapter's Prompt Template
+
+A task-specific adapter also needs its documented prompt format. The base model's chat template handles the conversation, and the adapter task tells the model what to do with the latest user turn. Provide the LLM with the task below, and it should follow those instructions.
+
+Use the exact prompt format documented for your adapter, and do not rely on a generic chat prompt. Loading successfully does not guarantee useful output unless the adapter-specific task instructions and output schema are supplied to the model. If you send a plain question with no task framing and no conversation history, the adapter has nothing to rewrite and will just echo the input back unchanged — this is the most common first-run surprise.
+
+## Example
+
+```console
+$ ./examples/lora/build/lora-example \
+  -m ./granite-4.0-micro-Q8_0.gguf \
+  -l ./query_rewrite_lora.gguf \
+  -s 0.25
+
+> Task: Rewrite the latest question as a self-contained question.
+
+Here is the latest question rewritten as a self-contained question:
+
+What is the capital city of France?
+
+> What about Germany?
+What is the capital city of Germany?
+>
+```
+
+This is the pattern to follow for a task-specific adapter: provide the task, the latest user question, and the expected rewritten output.

examples/lora/lora.cpp

@@ -0,0 +1,106 @@
+#include "agent.h"
+#include "chat_loop.h"
+#include "error.h"
+#include "model.h"
+#include <cstdio>
+#include <cstring>
+#include <memory>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+static constexpr float DEFAULT_SCALE = 1.0F;
+
+static void
+print_usage(int /*unused*/, char** argv)
+{
+    printf("\nexample usage:\n");
+    printf("\n    %s -m model.gguf -l adapter.gguf\n", argv[0]);
+    printf("\n");
+    printf("options:\n");
+    printf("  -m <path>       Path to the base GGUF model file (required)\n");
+    printf("  -l <path>       Path to the GGUF LoRA adapter (required)\n");
+    printf("  -s <scale>      Adapter scale (default: %.1f)\n", DEFAULT_SCALE);
+    printf("\n");
+}
+
+int
+main(int argc, char** argv)
+{
+    std::string model_path;
+    std::string adapter_path;
+    float scale = DEFAULT_SCALE;
+
+    for (int i = 1; i < argc; i++) {
+        if (strcmp(argv[i], "-m") == 0 || strcmp(argv[i], "-l") == 0 ||
+            strcmp(argv[i], "-s") == 0) {
+            if (i + 1 >= argc) {
+                print_usage(argc, argv);
+                return 1;
+            }
+            const char* value = argv[++i];
+            if (strcmp(argv[i - 1], "-m") == 0) {
+                model_path = value;
+            } else if (strcmp(argv[i - 1], "-l") == 0) {
+                adapter_path = value;
+            } else {
+                try {
+                    size_t parsed = 0;
+                    scale = std::stof(value, &parsed);
+                    if (parsed != strlen(value)) {
+                        throw std::invalid_argument("trailing characters");
+                    }
+                } catch (const std::exception&) {
+                    fprintf(stderr, "error: -s must be a number\n");
+                    return 1;
+                }
+            }
+        } else {
+            print_usage(argc, argv);
+            return 1;
+        }
+    }
+
+    if (model_path.empty() || adapter_path.empty()) {
+        print_usage(argc, argv);
+        return 1;
+    }
+
+    auto model_config = agent_cpp::ModelConfig{};
+    model_config.n_ctx = 4096;
+    model_config.temp = 0.0F;
+    model_config.loras.push_back({ adapter_path, scale });
+
+    printf("Loading base model '%s'...\n", model_path.c_str());
+    printf("Loading LoRA adapter '%s' at scale %.3f...\n",
+           adapter_path.c_str(),
+           scale);
+
+    std::shared_ptr<agent_cpp::Model> model;
+    try {
+        model = agent_cpp::Model::create(model_path, model_config);
+    } catch (const agent_cpp::ModelError& e) {
+        fprintf(stderr, "error: %s\n", e.what());
+        return 1;
+    }
+    printf("Model and LoRA adapter loaded successfully\n");
+
+    std::vector<std::unique_ptr<agent_cpp::Tool>> tools;
+    const std::string instructions =
+      "Follow the task prompt template expected by the loaded adapter. "
+      "The adapter may require a task-specific instruction, conversation "
+      "context, and output format rather than a general chat response.";
+
+    agent_cpp::Agent agent(
+      std::move(model), std::move(tools), {}, instructions);
+
+    printf("\nLoRA Demo ready!\n");
+    printf("   The adapter is applied for every response at scale %.3f.\n",
+           scale);
+    printf(
+      "   Use the adapter's documented task prompt template when chatting.\n");
+    printf("   Type an empty line to quit.\n\n");
+
+    run_chat_loop(agent);
+    return 0;
+}