diff --git a/docs/options/mtmd.md b/docs/options/mtmd.md new file mode 100644 index 000000000..eb69e5ac0 --- /dev/null +++ b/docs/options/mtmd.md @@ -0,0 +1,9 @@ +####> This option file is used in: +####> ramalama run, ramalama sandbox goose, ramalama sandbox opencode, ramalama sandbox pi, ramalama serve +####> If this file is edited, make sure the changes +####> are applicable to all of those. +#### **--mtmd**=*BOOL* +Enable or disable multimodal support. +If enabled and a multimodal projector file is detected, `--mmproj` is +added to the llama-server command-line. +When omitted, multimodal support is automatically detected. diff --git a/docs/ramalama-run.1.md b/docs/ramalama-run.1.md index 5ed45f709..bcd9918ab 100644 --- a/docs/ramalama-run.1.md +++ b/docs/ramalama-run.1.md @@ -275,6 +275,16 @@ Make sure the sampling parameters like top_k on the web UI are set correctly. [//]: # (END included file options/model-draft.md) +[//]: # (BEGIN included file options/mtmd.md) +#### **--mtmd**=*BOOL* +Enable or disable multimodal support. +If enabled and a multimodal projector file is detected, `--mmproj` is +added to the llama-server command-line. +When omitted, multimodal support is automatically detected. + +[//]: # (END included file options/mtmd.md) + + [//]: # (BEGIN included file options/name.md) #### **--name**, **-n** Name of the container to run the Model in. diff --git a/docs/ramalama-run.1.md.in b/docs/ramalama-run.1.md.in index a6870076f..51c40024e 100644 --- a/docs/ramalama-run.1.md.in +++ b/docs/ramalama-run.1.md.in @@ -50,6 +50,8 @@ the execution. URL support means if a model is on a web site or even on your loc @@option model-draft +@@option mtmd + @@option name @@option ncmoe diff --git a/docs/ramalama-sandbox-goose.1.md b/docs/ramalama-sandbox-goose.1.md index 35c1fd3af..574b570f8 100644 --- a/docs/ramalama-sandbox-goose.1.md +++ b/docs/ramalama-sandbox-goose.1.md @@ -263,6 +263,16 @@ Only used when invoked in router mode (zero or multiple models). [//]: # (END included file options/models-max.md) +[//]: # (BEGIN included file options/mtmd.md) +#### **--mtmd**=*BOOL* +Enable or disable multimodal support. +If enabled and a multimodal projector file is detected, `--mmproj` is +added to the llama-server command-line. +When omitted, multimodal support is automatically detected. + +[//]: # (END included file options/mtmd.md) + + [//]: # (BEGIN included file options/name.md) #### **--name**, **-n** Name of the container to run the Model in. diff --git a/docs/ramalama-sandbox-goose.1.md.in b/docs/ramalama-sandbox-goose.1.md.in index d1dc81925..9145f499f 100644 --- a/docs/ramalama-sandbox-goose.1.md.in +++ b/docs/ramalama-sandbox-goose.1.md.in @@ -63,6 +63,8 @@ Otherwise only the agent container starts using the given url for the openai com @@option models-max +@@option mtmd + @@option name @@option ncmoe diff --git a/docs/ramalama-sandbox-opencode.1.md b/docs/ramalama-sandbox-opencode.1.md index 5af6bf194..adf8bd784 100644 --- a/docs/ramalama-sandbox-opencode.1.md +++ b/docs/ramalama-sandbox-opencode.1.md @@ -256,6 +256,16 @@ Only used when invoked in router mode (zero or multiple models). [//]: # (END included file options/models-max.md) +[//]: # (BEGIN included file options/mtmd.md) +#### **--mtmd**=*BOOL* +Enable or disable multimodal support. +If enabled and a multimodal projector file is detected, `--mmproj` is +added to the llama-server command-line. +When omitted, multimodal support is automatically detected. + +[//]: # (END included file options/mtmd.md) + + [//]: # (BEGIN included file options/name.md) #### **--name**, **-n** Name of the container to run the Model in. diff --git a/docs/ramalama-sandbox-opencode.1.md.in b/docs/ramalama-sandbox-opencode.1.md.in index 4568279e7..bf54ccf3c 100644 --- a/docs/ramalama-sandbox-opencode.1.md.in +++ b/docs/ramalama-sandbox-opencode.1.md.in @@ -61,6 +61,8 @@ Otherwise only the agent container starts using the given url for the openai com @@option models-max +@@option mtmd + @@option name @@option ncmoe diff --git a/docs/ramalama-sandbox-pi.1.md b/docs/ramalama-sandbox-pi.1.md index 868af0d3e..d0d4aa303 100644 --- a/docs/ramalama-sandbox-pi.1.md +++ b/docs/ramalama-sandbox-pi.1.md @@ -260,6 +260,16 @@ Only used when invoked in router mode (zero or multiple models). [//]: # (END included file options/models-max.md) +[//]: # (BEGIN included file options/mtmd.md) +#### **--mtmd**=*BOOL* +Enable or disable multimodal support. +If enabled and a multimodal projector file is detected, `--mmproj` is +added to the llama-server command-line. +When omitted, multimodal support is automatically detected. + +[//]: # (END included file options/mtmd.md) + + [//]: # (BEGIN included file options/name.md) #### **--name**, **-n** Name of the container to run the Model in. diff --git a/docs/ramalama-sandbox-pi.1.md.in b/docs/ramalama-sandbox-pi.1.md.in index 01b3bae38..d73aa4d89 100644 --- a/docs/ramalama-sandbox-pi.1.md.in +++ b/docs/ramalama-sandbox-pi.1.md.in @@ -65,6 +65,8 @@ Otherwise only the agent container starts using the given url for the openai com @@option models-max +@@option mtmd + @@option name @@option ncmoe diff --git a/docs/ramalama-serve.1.md b/docs/ramalama-serve.1.md index 83400ba02..2ea206ce8 100644 --- a/docs/ramalama-serve.1.md +++ b/docs/ramalama-serve.1.md @@ -312,6 +312,16 @@ Only used when invoked in router mode (zero or multiple models). [//]: # (END included file options/models-max.md) +[//]: # (BEGIN included file options/mtmd.md) +#### **--mtmd**=*BOOL* +Enable or disable multimodal support. +If enabled and a multimodal projector file is detected, `--mmproj` is +added to the llama-server command-line. +When omitted, multimodal support is automatically detected. + +[//]: # (END included file options/mtmd.md) + + [//]: # (BEGIN included file options/name.md) #### **--name**, **-n** Name of the container to run the Model in. diff --git a/docs/ramalama-serve.1.md.in b/docs/ramalama-serve.1.md.in index f862fcc0a..c065971e0 100644 --- a/docs/ramalama-serve.1.md.in +++ b/docs/ramalama-serve.1.md.in @@ -98,6 +98,8 @@ appending the path to the type, e.g. `--generate kube:/etc/containers/systemd`. @@option models-max +@@option mtmd + @@option name @@option ncmoe diff --git a/ramalama/plugins/runtimes/inference/llama_cpp.py b/ramalama/plugins/runtimes/inference/llama_cpp.py index 0a4a537e6..9eb7da09b 100644 --- a/ramalama/plugins/runtimes/inference/llama_cpp.py +++ b/ramalama/plugins/runtimes/inference/llama_cpp.py @@ -87,6 +87,7 @@ class LlamaCppConfig: backend: Literal["auto", "vulkan", "rocm", "cuda", "sycl", "openvino", "cann", "musa"] = "auto" cache_reuse: Optional[int] = None gguf_quantization_mode: GGUF_QUANTIZATION_MODES = DEFAULT_GGUF_QUANTIZATION_MODE # type: ignore[assignment] + mtmd: Optional[bool] = None ngl: Optional[str] = None ncmoe: Optional[int] = None spec_type: Optional[str] = None @@ -116,6 +117,8 @@ def __post_init__(self): self.threads = int(self.threads) if self.thinking is not None: self.thinking = coerce_to_bool(self.thinking) + if self.mtmd is not None: + self.mtmd = coerce_to_bool(self.mtmd) def _positive_int(value: str) -> int: @@ -506,6 +509,12 @@ def _add_inference_args(self, parser: "argparse.ArgumentParser", command: str) - help="min speculative decoding probability (default: 0.0)", completer=suppressCompleter, ) + parser.add_argument( + "--mtmd", + default=None, + help="enable/disable multimodal mode (default: on)", + action=CoerceToBool, + ) self._add_threads_arg(parser) if command == "serve": parser.add_argument( diff --git a/ramalama/plugins/runtimes/inference/llama_cpp_commands.py b/ramalama/plugins/runtimes/inference/llama_cpp_commands.py index cad0b4e3a..bc9e5d2c7 100644 --- a/ramalama/plugins/runtimes/inference/llama_cpp_commands.py +++ b/ramalama/plugins/runtimes/inference/llama_cpp_commands.py @@ -72,7 +72,8 @@ def _cmd_run(self, args: argparse.Namespace) -> list[str]: model_path = model._get_entry_model_path(is_container, should_generate, dry_run) cmd += ["--model", model_path] - mmproj_path = model._get_mmproj_path(is_container, should_generate, dry_run) + mtmd = getattr(args, 'mtmd', None) + mmproj_path = model._get_mmproj_path(is_container, should_generate, dry_run) if mtmd is not False else None if mmproj_path: cmd += ["--mmproj", str(mmproj_path)] diff --git a/test/unit/test_inference_engine_plugins.py b/test/unit/test_inference_engine_plugins.py index 0a5198a54..33897edd0 100644 --- a/test/unit/test_inference_engine_plugins.py +++ b/test/unit/test_inference_engine_plugins.py @@ -51,6 +51,7 @@ def make_ns( spec_draft_n_max=None, spec_draft_n_min=None, spec_draft_p_min=None, + mtmd=None, runtime_args=None, engine_args=None, gguf=None, @@ -80,6 +81,7 @@ def make_ns( spec_draft_n_max=spec_draft_n_max, spec_draft_n_min=spec_draft_n_min, spec_draft_p_min=spec_draft_p_min, + mtmd=mtmd, runtime_args=runtime_args or [], engine_args=engine_args or [], gguf=gguf, @@ -159,6 +161,7 @@ def test_defaults(self): assert config.temp == 0.8 assert config.thinking is None assert config.threads > 0 + assert config.mtmd is None def test_coerces_string_values(self): config = LlamaCppConfig(ngl="4", ncmoe="128", cache_reuse="512", temp="0.5", threads="8", thinking="false") @@ -338,6 +341,50 @@ def test_serve_thinking_default(self, mock_colorize): assert "--reasoning" not in cmd + @patch("ramalama.plugins.runtimes.inference.llama_cpp_commands.New") + @patch("ramalama.plugins.runtimes.inference.llama_cpp_commands.should_colorize", return_value=False) + def test_serve_mtmd_disabled(self, mock_colorize, mock_new): + mock_model = make_transport_model(mmproj_path="/mnt/models/mmproj.file") + mock_new.return_value = mock_model + + ns = make_ns(MODEL="ollama://mymodel", mtmd=False) + cmd = self.plugin.handle_subcommand("serve", ns) + + assert "--mmproj" not in cmd + + @patch("ramalama.plugins.runtimes.inference.llama_cpp_commands.New") + @patch("ramalama.plugins.runtimes.inference.llama_cpp_commands.should_colorize", return_value=False) + def test_serve_mtmd_enabled(self, mock_colorize, mock_new): + mock_model = make_transport_model(mmproj_path="/mnt/models/mmproj.file") + mock_new.return_value = mock_model + + ns = make_ns(MODEL="ollama://mymodel", mtmd=True) + cmd = self.plugin.handle_subcommand("serve", ns) + + assert "--mmproj" in cmd + + @patch("ramalama.plugins.runtimes.inference.llama_cpp_commands.New") + @patch("ramalama.plugins.runtimes.inference.llama_cpp_commands.should_colorize", return_value=False) + def test_serve_mtmd_enabled_without_mmproj(self, mock_colorize, mock_new): + mock_model = make_transport_model(mmproj_path=None) + mock_new.return_value = mock_model + + ns = make_ns(MODEL="ollama://mymodel", mtmd=True) + cmd = self.plugin.handle_subcommand("serve", ns) + + assert "--mmproj" not in cmd + + @patch("ramalama.plugins.runtimes.inference.llama_cpp_commands.New") + @patch("ramalama.plugins.runtimes.inference.llama_cpp_commands.should_colorize", return_value=False) + def test_serve_mtmd_default(self, mock_colorize, mock_new): + mock_model = make_transport_model(mmproj_path="/mnt/models/mmproj.file") + mock_new.return_value = mock_model + + ns = make_ns(MODEL="ollama://mymodel", mtmd=None) + cmd = self.plugin.handle_subcommand("serve", ns) + + assert "--mmproj" in cmd + @patch("ramalama.plugins.runtimes.inference.llama_cpp_commands.should_colorize", return_value=False) def test_serve_ctx_size(self, mock_colorize): ns = make_ns(ctx_size=4096)