API: Align with llama.cpp output so coding agents work out of the box

endpoints/OAI/router.py

@@ -123,6 +123,11 @@ async def chat_completion_request(
     raw_json = await request.json()
     xlogger.debug("[ENDPOINT] /v1/chat/completions", {"raw": raw_json})
 
+    # Normalize "developer" (e.g. newer OpenAI-style tooling) to "system"
+    for message in data.messages:
+        if message.role == "developer":
+            message.role = "system"
+
     async with load_lock:
         if data.model:
             await load_inline_model(data.model, request)
@@ -170,6 +175,32 @@ async def chat_completion_request(
         raise HTTPException(422, "/v1/chat/completions request cancelled by user.") from ex
 
 
+# Apply template endpoint
+@router.post("/apply-template", dependencies=[Depends(check_api_key)])
+@router.post("/v1/apply-template", dependencies=[Depends(check_api_key)])
+async def apply_template_request(request: Request, data: ChatCompletionRequest):
+    """
+    Renders the chat template for the given messages without generating and
+    returns the templated prompt. Used by clients to probe template
+    capabilities (e.g. whether the model supports a thinking toggle).
+    """
+
+    await check_model_container()
+
+    if model.container.prompt_template is None:
+        raise HTTPException(
+            422, "Cannot apply template because a prompt template is not set."
+        )
+
+    # Normalize the "developer" role, same as chat completions.
+    for message in data.messages:
+        if message.role == "developer":
+            message.role = "system"
+
+    prompt, _ = await apply_chat_template(data)
+    return {"prompt": prompt}
+
+
 # Embeddings endpoint
 @router.post(
     "/v1/embeddings",

endpoints/OAI/utils/chat_completion.py

@@ -525,7 +525,11 @@ async def apply_chat_template(data: ChatCompletionRequest):
 
         raise HTTPException(400, error_message) from exc
     except TemplateError as exc:
-        error_message = handle_request_error(f"TemplateError: {str(exc)}").error.message
+        # A TemplateError is the template rejecting client input (e.g. an
+        # unsupported reasoning_effort), not a server fault, so skip the trace.
+        error_message = handle_request_error(
+            f"TemplateError: {str(exc)}", exc_info=False
+        ).error.message
 
         raise HTTPException(400, error_message) from exc
 

endpoints/core/router.py

@@ -30,6 +30,7 @@
     ModelList,
     ModelLoadRequest,
     ModelLoadResponse,
+    ModelPropsModalities,
     ModelPropsResponse,
 )
 from endpoints.core.types.health import HealthCheckResponse
@@ -46,6 +47,7 @@
 )
 from endpoints.core.utils.lora import get_active_loras, get_lora_list
 from endpoints.core.utils.model import (
+    apply_llama_compat,
     get_current_model,
     get_current_model_list,
     get_dummy_models,
@@ -120,7 +122,7 @@ async def list_models(request: Request) -> ModelList:
     if config.model.use_dummy_models:
         models.data[:0] = get_dummy_models()
 
-    return models
+    return apply_llama_compat(models)
 
 
 # Currently loaded model endpoint
@@ -145,9 +147,13 @@ async def model_props() -> ModelPropsResponse:
     current_model_card = get_current_model()
     resp = ModelPropsResponse(
         total_slots=current_model_card.parameters.max_batch_size,
+        model_path=str(model.container.model_dir),
         default_generation_settings=ModelDefaultGenerationSettings(
             n_ctx=current_model_card.parameters.max_seq_len,
         ),
+        modalities=ModelPropsModalities(
+            vision=bool(current_model_card.parameters.use_vision),
+        ),
     )
 
     if current_model_card.parameters.prompt_template_content:

endpoints/core/types/model.py

@@ -28,6 +28,19 @@ class ModelCardParameters(BaseModel):
     draft: Optional["ModelCard"] = None
 
 
+class ModelCardMeta(BaseModel):
+    """llama.cpp-style metadata block for a model card."""
+
+    vocab_type: int = 2
+    n_vocab: int = 0
+    n_ctx: int = 0
+    n_ctx_train: int = 0
+    n_embd: int = 0
+    n_params: int = 0
+    size: int = 0
+    ftype: str = "unknown"
+
+
 class ModelCard(BaseModel):
     """Represents a single model card."""
 
@@ -38,13 +51,34 @@ class ModelCard(BaseModel):
     logging: Optional[LoggingConfig] = None
     parameters: Optional[ModelCardParameters] = None
 
+    # llama.cpp compatibility fields
+    aliases: Optional[List[str]] = None
+    meta: Optional[ModelCardMeta] = None
+
+
+class ModelEntry(BaseModel):
+    """Ollama-style entry for the `models` array of a model list."""
+
+    name: str
+    model: str
+    modified_at: str = ""
+    size: str = ""
+    digest: str = ""
+    type: str = "model"
+    description: str = ""
+    tags: List[str] = Field(default_factory=lambda: [""])
+    capabilities: List[str] = Field(default_factory=lambda: ["completion"])
+
 
 class ModelList(BaseModel):
     """Represents a list of model cards."""
 
     object: str = "list"
     data: List[ModelCard] = Field(default_factory=list)
 
+    # llama.cpp / Ollama compatibility: mirror of `data` in Ollama's shape
+    models: Optional[List[ModelEntry]] = None
+
 
 class DraftModelLoadRequest(BaseModel):
     """Represents a draft model load request."""
@@ -143,9 +177,17 @@ class ModelDefaultGenerationSettings(BaseModel):
     n_ctx: int
 
 
+class ModelPropsModalities(BaseModel):
+    """Modality support flags for model props (llama.cpp compat)."""
+
+    vision: bool = False
+
+
 class ModelPropsResponse(BaseModel):
     """Represents a model props response."""
 
     total_slots: int = 1
+    model_path: str = ""
     chat_template: str = ""
     default_generation_settings: ModelDefaultGenerationSettings
+    modalities: ModelPropsModalities = Field(default_factory=ModelPropsModalities)

endpoints/core/utils/model.py

@@ -1,4 +1,5 @@
 import asyncio
+import json
 import pathlib
 from asyncio import CancelledError
 from typing import Optional
@@ -8,12 +9,37 @@
 from common.tabby_config import config
 from endpoints.core.types.model import (
     ModelCard,
+    ModelCardMeta,
+    ModelCardParameters,
+    ModelEntry,
     ModelList,
     ModelLoadRequest,
     ModelLoadResponse,
 )
 
 
+def read_model_meta(model_dir: pathlib.Path, n_ctx: Optional[int] = None) -> ModelCardMeta:
+    """llama.cpp-style `meta` block, read from the model's config.json."""
+    try:
+        cfg = json.loads((model_dir / "config.json").read_text(encoding="utf8"))
+    except Exception:
+        cfg = {}
+    # Multimodal configs nest text settings under text_config; let those win.
+    cfg = {**cfg, **cfg.get("text_config", {})}
+    train = cfg.get("max_position_embeddings") or 0
+    try:
+        size = sum(f.stat().st_size for f in model_dir.glob("*.safetensors"))
+    except Exception:
+        size = 0
+    return ModelCardMeta(
+        n_vocab=cfg.get("vocab_size") or 0,
+        n_ctx=n_ctx or train,
+        n_ctx_train=train,
+        n_embd=cfg.get("hidden_size") or 0,
+        size=size,
+    )
+
+
 def get_model_list(model_path: pathlib.Path, draft_model_path: Optional[str] = None):
     """Get the list of models from the provided path."""
 
@@ -26,7 +52,10 @@ def get_model_list(model_path: pathlib.Path, draft_model_path: Optional[str] = N
     for path in model_path.iterdir():
         # Don't include the draft models path
         if path.is_dir() and path != draft_model_path:
-            model_card = ModelCard(id=path.name)
+            meta = read_model_meta(path)
+            model_card = ModelCard(id=path.name, meta=meta)
+            if meta.n_ctx_train:
+                model_card.parameters = ModelCardParameters(max_seq_len=meta.n_ctx_train)
             model_card_list.data.append(model_card)  # pylint: disable=no-member
 
     return model_card_list
@@ -41,12 +70,14 @@ async def get_current_model_list(model_type: str = "model"):
 
     current_models = []
     model_path = None
+    context_length = None
 
     # Make sure the model container exists
     match model_type:
         case "model":
             if model.container:
                 model_path = model.container.model_dir
+                context_length = model.container.max_seq_len
         case "draft":
             if model.container:
                 model_path = model.container.draft_model_dir
@@ -55,7 +86,10 @@ async def get_current_model_list(model_type: str = "model"):
                 model_path = model.embeddings_container.model_dir
 
     if model_path:
-        current_models.append(ModelCard(id=model_path.name))
+        model_card = ModelCard(id=model_path.name, meta=read_model_meta(model_path, context_length))
+        if context_length is not None:
+            model_card.parameters = ModelCardParameters(max_seq_len=context_length)
+        current_models.append(model_card)
 
     return ModelList(data=current_models)
 
@@ -68,6 +102,25 @@ def get_current_model():
     return model_card
 
 
+def apply_llama_compat(model_list: ModelList) -> ModelList:
+    """Mirror `data` into an Ollama-style `models` array (llama.cpp compat)."""
+
+    model_list.models = []
+    for card in model_list.data:
+        card.aliases = [card.id]
+        vision = bool(card.parameters and card.parameters.use_vision)
+        model_list.models.append(
+            ModelEntry(
+                name=card.id,
+                model=card.id,
+                size=str(card.meta.size) if card.meta else "",
+                capabilities=["completion"] + (["multimodal"] if vision else []),
+            )
+        )
+
+    return model_list
+
+
 def get_dummy_models():
     if config.model.dummy_model_names:
         return [ModelCard(id=dummy_id) for dummy_id in config.model.dummy_model_names]

endpoints/server.py

@@ -1,18 +1,27 @@
 import asyncio
 import uvicorn
-from fastapi import FastAPI
+from fastapi import Depends, FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from loguru import logger
 from typing import Optional
 
 from common import signals
+from common.auth import check_api_key
+from common.debug_requests import log_chat_completion_request
 from common.logger import UVICORN_LOG_CONFIG
 from common.errors import ContextLengthHTTPException, context_length_exception_handler
+from common.model import check_embeddings_container
 from common.networking import get_global_depends
 from common.tabby_config import config
 from endpoints.Kobold import router as KoboldRouter
 from endpoints.OAI import router as OAIRouter
+from endpoints.OAI.router import (
+    chat_completion_request,
+    completion_request,
+    embeddings,
+)
 from endpoints.core.router import router as CoreRouter
+from endpoints.core.router import list_models
 
 
 def setup_app(host: Optional[str] = None, port: Optional[int] = None):