Rag private network

docs/ramalama-rag.1.md

@@ -25,8 +25,11 @@ The pipeline:
 5. Embeddings are stored in a Qdrant on-disk collection.
 6. The Qdrant database is packaged into a `FROM scratch` OCI image.
 
-Two containers work together: a llama.cpp container serves the AI models,
+Multiple containers work together: llama.cpp containers serve the AI models,
 and a lightweight RAG container runs the document processing pipeline.
+RamaLama places them on a private container network and they reach each other
+by container name; the model server ports are not exposed on the host network.
+The network is created when the command starts and removed when it finishes.
 
 NOTE: this command requires a container engine (podman or docker).
 
@@ -82,6 +85,14 @@ Show this help message and exit
 OCI container image to use for the llama.cpp inference servers.
 Defaults to the accelerator-appropriate ramalama image.
 
+#### **--network**, **--net**=*network*
+Join the pipeline containers to an existing container network instead of
+creating a temporary private one.  Use this to reuse a pre-existing network;
+the containers still reach each other by name, so the network must provide
+name-based DNS (any user-defined podman/docker network does).  A network
+supplied this way is left in place when the command finishes; only the
+temporary network RamaLama creates by default is removed.
+
 #### **--ngl**=*value*
 Number of layers to store in VRAM: a number, `auto`, or `all`.
 When omitted, llama-server defaults to `auto`.
@@ -90,6 +101,12 @@ When omitted, llama-server defaults to `auto`.
 OCI container image for the RAG processing container.
 Defaults to the accelerator-appropriate ramalama-rag image.
 
+#### **--skip-cleanup**
+Leave the llama.cpp servers and their private network running after the command
+finishes instead of tearing them down.  Useful for debugging a failed run:
+inspect the servers with `podman logs <container>` and remove them manually
+afterwards (the command prints the exact cleanup command).
+
 #### **--threads**, **-t**=*integer*
 Number of CPU threads to use for llama.cpp inference.
 Defaults to half the available cores.

ramalama/cli.py

@@ -1090,10 +1090,9 @@ def _rag_args(args):
     rag_args = copy.copy(args)
     rag_args.MODEL = args.rag
     rag_args.image = args.rag_image
-    if args.engine == "podman":
-        rag_args.model_host = "host.containers.internal"
-    else:
-        rag_args.model_host = f"host.{args.engine}.internal"
+    # model_host is set later by _setup_rag_network to the model server's container
+    # name once the shared private network is created.
+    rag_args.model_host = None
     # If --name was specified, use it for the RAG proxy
     args.name = None
     # If --port was specified, use it for the RAG proxy, and

ramalama/common.py

@@ -351,8 +351,8 @@ def verify_checksum(filename: str) -> bool:
     return sha256_hash.hexdigest() == expected_checksum
 
 
-def genname():
-    return "ramalama-" + "".join(random.choices(string.ascii_letters + string.digits, k=10))
+def genname(prefix="ramalama-"):
+    return prefix + "".join(random.choices(string.ascii_letters + string.digits, k=10))
 
 
 @lru_cache

ramalama/engine.py

@@ -15,7 +15,7 @@
 # Live reference for checking global vars
 import ramalama.common
 from ramalama.arg_types import BaseEngineArgsType
-from ramalama.common import check_nvidia, engine_cmd, exec_cmd, get_accel_env_vars, host_path, perror, run_cmd
+from ramalama.common import check_nvidia, engine_cmd, exec_cmd, genname, get_accel_env_vars, host_path, perror, run_cmd
 from ramalama.compat import NamedTemporaryFile
 from ramalama.config import ActiveConfig
 from ramalama.host_utils import (
@@ -479,6 +479,50 @@ def stop_container(args, name: str, remove: bool = False):
                 raise
 
 
+def create_network(args) -> str:
+    """Create a private, user-defined container network and return its name.
+
+    Helper containers and their consumer join this network so they can reach
+    each other by container name (podman/docker provide name-based DNS on
+    user-defined networks), with nothing published to the network. On a dry
+    run the network is not created, but a name is still returned so the
+    generated command reflects it.
+    """
+    conman = str(args.engine) if args.engine is not None else None
+    if conman == "" or conman is None:
+        raise ValueError("no container manager (Podman, Docker) found")
+
+    name = genname("ramalama-net-")
+    if not getattr(args, "dryrun", False):
+        run_cmd([*engine_cmd(conman), "network", "create", name])
+    return name
+
+
+def remove_network(args, name: str) -> None:
+    """Remove a network created by create_network. Best effort; errors ignored.
+
+    On a dry run nothing was created, so nothing is removed.
+    """
+    if not name:
+        return
+    conman = str(args.engine) if args.engine is not None else None
+    if conman == "" or conman is None:
+        return
+    if getattr(args, "dryrun", False):
+        return
+
+    # `podman network rm -f` disconnects any lingering containers; docker has no
+    # such flag but the attached containers are removed before this is called.
+    conman_args = [*engine_cmd(conman), "network", "rm"]
+    if conman == "podman":
+        conman_args += ["-f"]
+    conman_args += [name]
+    try:
+        run_cmd(conman_args, ignore_all=True)
+    except Exception as e:  # Cleanup is best effort.
+        logger.debug(f"Failed to remove network {name}: {e}")
+
+
 def add_labels(args, add_label: Callable[[str], None]):
     label_map = {
         "MODEL": "ai.ramalama.model",

ramalama/plugins/runtimes/inference/llama_cpp.py

@@ -9,7 +9,7 @@
 import subprocess
 import sys
 import tempfile
-from collections.abc import Mapping
+from collections.abc import Callable, Mapping
 from dataclasses import asdict, dataclass, field
 from datetime import datetime, timezone
 from http.client import HTTPConnection
@@ -655,27 +655,36 @@ def _run_rag(self, args: argparse.Namespace, model: Any) -> None:
         args = _rag_args(args)
         model = RagTransport(model, assemble_command(args.model_args), args)
         model.ensure_model_exists(args)
-        embed_serve_args, embed_proc = self._start_rag_embedding_server(args)
-        try:
-            model.run(args, assemble_command(args))
-        finally:
-            from ramalama.plugins.runtimes.inference.rag.handler import _cleanup_servers
-
-            _cleanup_servers(args, [embed_serve_args], [embed_proc])
+        self._serve_rag_pipeline(args, lambda: model.run(args, assemble_command(args)))
 
     def _serve_rag(self, args: argparse.Namespace, model: Any) -> None:
         if not args.container:
             raise ValueError("ramalama serve --rag cannot be run with the --nocontainer option.")
         args = _rag_args(args)
         model = RagTransport(model, assemble_command(args.model_args), args)
         model.ensure_model_exists(args)
-        embed_serve_args, embed_proc = self._start_rag_embedding_server(args)
+        self._serve_rag_pipeline(args, lambda: model.serve(args, assemble_command(args)))
+
+    def _serve_rag_pipeline(self, args: argparse.Namespace, dispatch: Callable[[], None]) -> None:
+        """Start the RAG helper servers on a private network, then run ``dispatch``.
+
+        The model server, embedding server, and RAG proxy all join a shared
+        private network so they reach each other by container name without
+        publishing the helper servers to the host.
+        """
+        from ramalama.engine import remove_network
+        from ramalama.plugins.runtimes.inference.rag.handler import _cleanup_servers, _setup_rag_network
+
+        network_created = _setup_rag_network(args)
         try:
-            model.serve(args, assemble_command(args))
+            embed_serve_args, embed_proc = self._start_rag_embedding_server(args)
+            try:
+                dispatch()
+            finally:
+                _cleanup_servers(args, [embed_serve_args], [embed_proc])
         finally:
-            from ramalama.plugins.runtimes.inference.rag.handler import _cleanup_servers
-
-            _cleanup_servers(args, [embed_serve_args], [embed_proc])
+            if network_created:
+                remove_network(args, args.network)
 
     def _start_rag_embedding_server(self, args):
         """Start a llama.cpp embedding server for RAG inference and set embed_url on args."""
@@ -697,17 +706,14 @@ def _start_rag_embedding_server(self, args):
         embed_transport.ensure_model_exists(embed_serve_args)
 
         embed_cmd = assemble_command(embed_serve_args)
-        embed_proc = embed_transport.serve_nonblocking(embed_serve_args, embed_cmd, expose_to_containers=True)
+        embed_proc = embed_transport.serve_nonblocking(embed_serve_args, embed_cmd)
 
         if not args.dryrun:
             _wait_for_server(self, embed_serve_args, embed_transport.model_alias)
 
-        if args.model_args.engine == "podman":
-            embed_host = "host.containers.internal"
-        else:
-            embed_host = f"host.{args.model_args.engine}.internal"
-
-        args.embed_url = f"http://{embed_host}:{embed_port}"
+        # Reach the embedding server by its container name over the shared private
+        # network (serve_nonblocking assigned the name above).
+        args.embed_url = f"http://{embed_serve_args.name}:{embed_port}"
         return embed_serve_args, embed_proc
 
     def _register_run_subcommand(self, subparsers: "argparse._SubParsersAction") -> "argparse.ArgumentParser":

ramalama/plugins/runtimes/inference/rag/cli.py

@@ -103,6 +103,24 @@ def register_rag_subcommand(plugin, subparsers):
         help=f"number of CPU threads to use (default: {rt_config.threads})",
         completer=suppressCompleter,
     )
+    parser.add_argument(
+        "--network",
+        "--net",
+        dest="network",
+        type=str,
+        default=None,
+        help="join the pipeline containers to an existing container network "
+        "instead of creating a temporary private one; a user-supplied network "
+        "is left in place on exit",
+        completer=suppressCompleter,
+    )
+    parser.add_argument(
+        "--skip-cleanup",
+        dest="skip_cleanup",
+        action="store_true",
+        help="leave the llama.cpp servers and their network running after the "
+        "command for debugging (inspect with `podman logs`)",
+    )
     parser.set_defaults(func=lambda args: _rag_dispatch(plugin, args))