diff --git a/CMakeLists.txt b/CMakeLists.txt index d0a8c12..64799b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -154,6 +154,18 @@ if(AGENT_CPP_BUILD_TESTS) target_link_libraries(test_grammar PRIVATE model ${LLAMA_COMMON_TARGET} llama) target_compile_features(test_grammar PRIVATE cxx_std_17) + add_executable(test_lora tests/test_lora.cpp) + target_include_directories(test_lora PRIVATE + src + tests + ${LLAMA_SOURCE_DIR}/common + ${LLAMA_SOURCE_DIR}/ggml/include + ${LLAMA_SOURCE_DIR}/include + ${LLAMA_SOURCE_DIR}/vendor + ) + target_link_libraries(test_lora PRIVATE model ${LLAMA_COMMON_TARGET} llama) + target_compile_features(test_lora PRIVATE cxx_std_17) + add_executable(test_callbacks tests/test_callbacks.cpp) target_include_directories(test_callbacks PRIVATE src @@ -185,6 +197,7 @@ if(AGENT_CPP_BUILD_TESTS) add_test(NAME ToolTests COMMAND test_tool) add_test(NAME CallbacksTests COMMAND test_callbacks) add_test(NAME GrammarTests COMMAND test_grammar) + add_test(NAME LoraTests COMMAND test_lora) add_test(NAME ChatParserTests COMMAND test_chat_parser) if(AGENT_CPP_BUILD_MCP) @@ -203,14 +216,14 @@ if(AGENT_CPP_BUILD_TESTS) # On Windows, DLLs are placed in the bin/ directory by llama.cpp # We need to add this directory to PATH so tests can find the DLLs # A hung test must fail rather than hold the runner until the job limit - set_tests_properties(ToolTests CallbacksTests GrammarTests ChatParserTests + set_tests_properties(ToolTests CallbacksTests GrammarTests LoraTests ChatParserTests PROPERTIES TIMEOUT 120 ) if(WIN32) # Multi-config generators (Visual Studio) place DLLs in bin/, # single-config ones in bin/ - both are on PATH so tests can load them - set_tests_properties(ToolTests CallbacksTests GrammarTests ChatParserTests PROPERTIES + set_tests_properties(ToolTests CallbacksTests GrammarTests LoraTests ChatParserTests PROPERTIES ENVIRONMENT "PATH=${CMAKE_BINARY_DIR}/bin/${CMAKE_BUILD_TYPE}\;${CMAKE_BINARY_DIR}/bin/Release\;${CMAKE_BINARY_DIR}/bin\;$ENV{PATH}" ) endif() @@ -284,6 +297,19 @@ if(AGENT_CPP_BUILD_EXAMPLES) target_link_libraries(grammar-example PRIVATE agent model ${LLAMA_COMMON_TARGET} llama) target_compile_features(grammar-example PRIVATE cxx_std_17) + # LoRA example + add_executable(lora-example examples/lora/lora.cpp) + target_include_directories(lora-example PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src + ${CMAKE_CURRENT_SOURCE_DIR}/examples/shared + ${LLAMA_SOURCE_DIR}/common + ${LLAMA_SOURCE_DIR}/ggml/include + ${LLAMA_SOURCE_DIR}/include + ${LLAMA_SOURCE_DIR}/vendor + ) + target_link_libraries(lora-example PRIVATE agent model ${LLAMA_COMMON_TARGET} llama) + target_compile_features(lora-example PRIVATE cxx_std_17) + # MCP client example (requires MCP support) if(AGENT_CPP_BUILD_MCP) add_executable(mcp-example examples/mcp/mcp.cpp) diff --git a/README.md b/README.md index b1957f4..620f6d0 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,8 @@ Building blocks for **local** agents in C++. - **[Grammar](./examples/grammar/README.md)** - Constrain model output to a GBNF grammar so every response matches a fixed structure. +- **[LoRA](./examples/lora/README.md)** - Apply a GGUF LoRA adapter to a model with a configurable scale. + - **[Memory](./examples/memory/README.md)** - Use tools that allow an agent to store and retrieve relevant information across conversations. - **[Multi-Agent](./examples/multi-agent/README.md)** - Build a multi-agent system with weight sharing where a main agent delegates to specialized sub-agents. @@ -86,6 +88,21 @@ A grammar constrains every response, including the ones the agent would use to c See the [Grammar example](./examples/grammar/README.md) for a full working demo. +### LoRA adapters + +Each `Model` can load its own [LoRA adapters](https://github.com/ggml-org/llama.cpp/tree/master/tools/completion#lora-low-rank-adaptation-adapters) on top of the shared base weights, useful for giving specialized agents in a [multi-agent](./examples/multi-agent/README.md) setup their own fine-tuned behavior without duplicating the base model in memory: + +```cpp +ModelConfig config; +config.loras = { + { "path/to/adapter.gguf", 1.0F }, // path, scale (defaults to 1.0) +}; + +auto model = Model::create_with_weights(shared_weights, config); +``` + +Multiple adapters may be stacked by adding more entries; each is scaled independently. See the [LoRA example](./examples/lora/README.md) for instructions on converting adapters from common training formats to GGUF. + ## Tools Tools extend the agent's capabilities beyond text generation. Each tool defines: diff --git a/examples/README.md b/examples/README.md index a060ab7..6dcc114 100644 --- a/examples/README.md +++ b/examples/README.md @@ -6,6 +6,10 @@ This directory contains example applications demonstrating agent.cpp capabilitie The [grammar](./grammar) example demonstrates constraining model output to a GBNF grammar, so every response is guaranteed to match a fixed structure. You can also point it at a custom grammar file and root rule. +## LoRA + +The [lora](./lora) example demonstrates loading a GGUF LoRA adapter onto a model with a configurable scale. + ## Shared Utilities The [shared](./shared) directory contains reusable helper components used across multiple examples. These are **not part of the public API** but can be useful as reference implementations. diff --git a/examples/lora/CMakeLists.txt b/examples/lora/CMakeLists.txt new file mode 100644 index 0000000..7a5768b --- /dev/null +++ b/examples/lora/CMakeLists.txt @@ -0,0 +1,24 @@ +cmake_minimum_required(VERSION 3.14) +project(lora-example VERSION 0.1.0) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../.. ${CMAKE_CURRENT_BINARY_DIR}/agent-cpp) + +add_executable(lora-example lora.cpp) + +target_include_directories(lora-example PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/../../src + ${CMAKE_CURRENT_SOURCE_DIR}/../shared + ${LLAMA_SOURCE_DIR}/common + ${LLAMA_SOURCE_DIR}/ggml/include + ${LLAMA_SOURCE_DIR}/include + ${LLAMA_SOURCE_DIR}/vendor +) + +target_link_libraries(lora-example PRIVATE agent-cpp::agent ${LLAMA_COMMON_TARGET} llama) +target_compile_features(lora-example PRIVATE cxx_std_17) + +message(STATUS "LoRA example configured.") diff --git a/examples/lora/README.md b/examples/lora/README.md new file mode 100644 index 0000000..99bb65a --- /dev/null +++ b/examples/lora/README.md @@ -0,0 +1,117 @@ +# LoRA Example + +This example loads a GGUF LoRA adapter onto a base model and runs an interactive chat. The adapter is applied to the model context with the requested scale, while the base model remains in the normal GGUF file. + +## Building + +> [!IMPORTANT] +> Check the [llama.cpp build documentation](https://github.com/ggml-org/llama.cpp/blob/master/docs/build.md) to find +> CMake flags you might want to pass depending on your available hardware. + +```bash +cd examples/lora + +git -C ../.. submodule update --init --recursive + +cmake -B build +cmake --build build -j$(nproc) +``` + +### Using a custom llama.cpp + +If you have llama.cpp already downloaded: + +```bash +cmake -B build -DLLAMA_CPP_DIR=/path/to/your/llama.cpp +cmake --build build -j$(nproc) +``` + +## Requirements + +The adapter must be GGUF and compatible with the base model. The architecture must match, and the same base model must be used for conversion and inference. + +Some adapter families also ship an aLoRA (Activated LoRA) variant alongside the standard one, in a separate folder such as `alora/`. This example only supports standard LoRA adapters ā€” use the `lora/` variant, not `alora/`, if both are offered. + +## Preparing an Adapter + +If the adapter is already GGUF, skip to [Usage](#usage). Otherwise, convert it with llama.cpp's [`convert_lora_to_gguf.py`](https://github.com/ggml-org/llama.cpp/blob/master/convert_lora_to_gguf.py). You can download models from any registry; for Hugging Face, `huggingface-cli` is one option. The conversion input must contain the adapter configuration and weights. Use the same compatible base model for conversion and inference. + +### Worked Example: Granite Query Rewriting + +This example uses uv for dependency and environment management. The example below assumes you are in a uv-managed environment for the llama.cpp dependency setup. Run from the repository root: + +```bash +wget https://huggingface.co/ibm-granite/granite-4.0-micro-GGUF/resolve/main/granite-4.0-micro-Q8_0.gguf + +cd deps/llama.cpp +uv venv +source .venv/bin/activate +uv pip install -r requirements.txt +``` + +If your environment is configured with a package index that exposes an older `requests` build first, retry with: + +```bash +uv pip install --index-strategy unsafe-best-match -r requirements.txt +``` + +```bash +hf download ibm-granite/granitelib-rag-r1.0 \ + "query_rewrite/granite-4.0-micro/lora/adapter_config.json" \ + "query_rewrite/granite-4.0-micro/lora/adapter_model.safetensors" \ + --local-dir ./granite-rag-adapters + +python convert_lora_to_gguf.py \ + granite-rag-adapters/query_rewrite/granite-4.0-micro/lora \ + --base-model-id ibm-granite/granite-4.0-micro \ + --outfile ../../query_rewrite_lora.gguf +``` + +Return to the repository root and run the adapter with the binary built above: + +```bash +cd ../.. + +./examples/lora/build/lora-example \ + -m ./granite-4.0-micro-Q8_0.gguf \ + -l ./query_rewrite_lora.gguf \ + -s 0.25 +``` + +## Usage + +```bash +./build/lora-example \ + -m /path/to/model.gguf \ + -l /path/to/adapter.gguf \ + -s 0.25 +``` + +`-s` is the adapter scale; it defaults to `1.0` if omitted. A scale of `0` disables the adapter for the context without unloading it. Multiple adapters can be stacked through `ModelConfig::loras`; this example keeps the command line focused on one adapter. + +## Using the Adapter's Prompt Template + +A task-specific adapter also needs its documented prompt format. The base model's chat template handles the conversation, and the adapter task tells the model what to do with the latest user turn. Provide the LLM with the task below, and it should follow those instructions. + +Use the exact prompt format documented for your adapter, and do not rely on a generic chat prompt. Loading successfully does not guarantee useful output unless the adapter-specific task instructions and output schema are supplied to the model. If you send a plain question with no task framing and no conversation history, the adapter has nothing to rewrite and will just echo the input back unchanged ā€” this is the most common first-run surprise. + +## Example + +```console +$ ./examples/lora/build/lora-example \ + -m ./granite-4.0-micro-Q8_0.gguf \ + -l ./query_rewrite_lora.gguf \ + -s 0.25 + +> Task: Rewrite the latest question as a self-contained question. + +Here is the latest question rewritten as a self-contained question: + +What is the capital city of France? + +> What about Germany? +What is the capital city of Germany? +> +``` + +This is the pattern to follow for a task-specific adapter: provide the task, the latest user question, and the expected rewritten output. diff --git a/examples/lora/lora.cpp b/examples/lora/lora.cpp new file mode 100644 index 0000000..ee0f2d1 --- /dev/null +++ b/examples/lora/lora.cpp @@ -0,0 +1,106 @@ +#include "agent.h" +#include "chat_loop.h" +#include "error.h" +#include "model.h" +#include +#include +#include +#include +#include +#include + +static constexpr float DEFAULT_SCALE = 1.0F; + +static void +print_usage(int /*unused*/, char** argv) +{ + printf("\nexample usage:\n"); + printf("\n %s -m model.gguf -l adapter.gguf\n", argv[0]); + printf("\n"); + printf("options:\n"); + printf(" -m Path to the base GGUF model file (required)\n"); + printf(" -l Path to the GGUF LoRA adapter (required)\n"); + printf(" -s Adapter scale (default: %.1f)\n", DEFAULT_SCALE); + printf("\n"); +} + +int +main(int argc, char** argv) +{ + std::string model_path; + std::string adapter_path; + float scale = DEFAULT_SCALE; + + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-m") == 0 || strcmp(argv[i], "-l") == 0 || + strcmp(argv[i], "-s") == 0) { + if (i + 1 >= argc) { + print_usage(argc, argv); + return 1; + } + const char* value = argv[++i]; + if (strcmp(argv[i - 1], "-m") == 0) { + model_path = value; + } else if (strcmp(argv[i - 1], "-l") == 0) { + adapter_path = value; + } else { + try { + size_t parsed = 0; + scale = std::stof(value, &parsed); + if (parsed != strlen(value)) { + throw std::invalid_argument("trailing characters"); + } + } catch (const std::exception&) { + fprintf(stderr, "error: -s must be a number\n"); + return 1; + } + } + } else { + print_usage(argc, argv); + return 1; + } + } + + if (model_path.empty() || adapter_path.empty()) { + print_usage(argc, argv); + return 1; + } + + auto model_config = agent_cpp::ModelConfig{}; + model_config.n_ctx = 4096; + model_config.temp = 0.0F; + model_config.loras.push_back({ adapter_path, scale }); + + printf("Loading base model '%s'...\n", model_path.c_str()); + printf("Loading LoRA adapter '%s' at scale %.3f...\n", + adapter_path.c_str(), + scale); + + std::shared_ptr model; + try { + model = agent_cpp::Model::create(model_path, model_config); + } catch (const agent_cpp::ModelError& e) { + fprintf(stderr, "error: %s\n", e.what()); + return 1; + } + printf("Model and LoRA adapter loaded successfully\n"); + + std::vector> tools; + const std::string instructions = + "Follow the task prompt template expected by the loaded adapter. " + "The adapter may require a task-specific instruction, conversation " + "context, and output format rather than a general chat response."; + + agent_cpp::Agent agent( + std::move(model), std::move(tools), {}, instructions); + + printf("\nLoRA Demo ready!\n"); + printf(" The adapter is applied for every response at scale %.3f.\n", + scale); + printf( + " Use the adapter's documented task prompt template when chatting.\n"); + printf(" Type an empty line to quit.\n\n"); + + run_chat_loop(agent); + return 0; +} diff --git a/src/model.cpp b/src/model.cpp index 97f86b3..c37db05 100644 --- a/src/model.cpp +++ b/src/model.cpp @@ -83,6 +83,9 @@ Model::~Model() if (ctx_ != nullptr) { llama_free(ctx_); } + for (llama_adapter_lora* lora : loras_) { + llama_adapter_lora_free(lora); + } // weights_ is automatically released when ref count drops to zero } @@ -91,6 +94,7 @@ Model::Model(Model&& other) noexcept , ctx_(other.ctx_) , sampler_(other.sampler_) , grammar_sampler_(other.grammar_sampler_) + , loras_(std::move(other.loras_)) , processed_tokens_(std::move(other.processed_tokens_)) , n_past_(other.n_past_) , config_(other.config_) @@ -111,11 +115,15 @@ Model::operator=(Model&& other) noexcept if (ctx_ != nullptr) { llama_free(ctx_); } + for (llama_adapter_lora* lora : loras_) { + llama_adapter_lora_free(lora); + } weights_ = std::move(other.weights_); ctx_ = other.ctx_; sampler_ = other.sampler_; grammar_sampler_ = other.grammar_sampler_; + loras_ = std::move(other.loras_); processed_tokens_ = std::move(other.processed_tokens_); n_past_ = other.n_past_; config_ = other.config_; @@ -146,6 +154,25 @@ Model::initialize_context(const ModelConfig& model_config) throw ModelError("failed to create llama context"); } + if (!model_config.loras.empty()) { + std::vector scales; + scales.reserve(model_config.loras.size()); + for (const LoraAdapterConfig& lora_config : model_config.loras) { + llama_adapter_lora* lora = llama_adapter_lora_init( + weights_->get_model(), lora_config.path.c_str()); + if (lora == nullptr) { + throw ModelError("failed to load LoRA adapter '" + + lora_config.path + "'"); + } + loras_.push_back(lora); + scales.push_back(lora_config.scale); + } + if (llama_set_adapters_lora( + ctx_, loras_.data(), loras_.size(), scales.data()) != 0) { + throw ModelError("failed to apply LoRA adapters to context"); + } + } + sampler_ = llama_sampler_chain_init(llama_sampler_chain_default_params()); if (!model_config.grammar.empty()) { diff --git a/src/model.h b/src/model.h index 91c11ac..b0d6437 100644 --- a/src/model.h +++ b/src/model.h @@ -8,12 +8,20 @@ #include #include #include +#include namespace agent_cpp { // Callback for streaming response chunks using ResponseCallback = std::function; +// LoRA adapter configuration +struct LoraAdapterConfig +{ + std::string path; + float scale = 1.0F; +}; + // Model configuration with sensible defaults struct ModelConfig { @@ -36,6 +44,11 @@ struct ModelConfig // Optional GBNF grammar and root rule name std::string grammar; std::string grammar_root = "root"; + // Optional LoRA adapters applied to this Model's context. Multiple + // adapters may be stacked; each is scaled independently. Adapters are + // loaded from the shared ModelWeights' base model, so different Model + // instances sharing the same weights can each carry their own set. + std::vector loras; }; /// Reads a GBNF file into a string for ModelConfig::grammar @@ -184,6 +197,12 @@ class Model return weights_; } + // Get the loaded LoRA adapters + [[nodiscard]] const std::vector& get_loras() const + { + return loras_; + } + // Save the current KV cache state (processed_tokens) to a file // Returns true on success, false on failure bool save_cache(const std::string& cache_path); @@ -210,6 +229,10 @@ class Model // Non-owning pointer to the grammar sampler in sampler_'s chain // Reset this one between turns without resetting the rest of the chain llama_sampler* grammar_sampler_ = nullptr; + // Owning handles for this Model's LoRA adapters, in ModelConfig::loras + // order. Set on this Model's context only, so different Model instances + // sharing the same ModelWeights can carry independent adapters. + std::vector loras_; std::vector processed_tokens_; // Track tokens in KV cache int n_past_ = 0; // Track position in KV cache ModelConfig config_; diff --git a/tests/test_lora.cpp b/tests/test_lora.cpp new file mode 100644 index 0000000..a033f19 --- /dev/null +++ b/tests/test_lora.cpp @@ -0,0 +1,50 @@ +#include "error.h" +#include "model.h" +#include "test_utils.h" + +TEST(test_model_config_loras_defaults) +{ + agent_cpp::ModelConfig config; + + ASSERT_TRUE(config.loras.empty()); +} + +TEST(test_lora_adapter_config_default_scale) +{ + agent_cpp::LoraAdapterConfig lora_config; + lora_config.path = "adapter.gguf"; + + ASSERT_EQ(lora_config.scale, 1.0F); +} + +TEST(test_model_config_loras_accepts_multiple_adapters) +{ + agent_cpp::ModelConfig config; + config.loras.push_back({ "math_lora.gguf", 1.0F }); + config.loras.push_back({ "style_lora.gguf", 0.5F }); + + ASSERT_EQ(config.loras.size(), (size_t)2); + ASSERT_EQ(config.loras[0].path, "math_lora.gguf"); + ASSERT_EQ(config.loras[1].scale, 0.5F); +} + +// Full adapter loading requires a loaded GGUF model and context. Verify +// manually in examples/lora. + +int +main() +{ + std::cout << "\n=== Running LoRA Unit Tests ===\n" << std::endl; + + try { + RUN_TEST(test_model_config_loras_defaults); + RUN_TEST(test_lora_adapter_config_default_scale); + RUN_TEST(test_model_config_loras_accepts_multiple_adapters); + + std::cout << "\n=== All tests passed! āœ“ ===\n" << std::endl; + return 0; + } catch (const std::exception& e) { + std::cerr << "\nāœ— TEST FAILED: " << e.what() << std::endl; + return 1; + } +}