diff --git a/ai_agents/.env.example b/ai_agents/.env.example index 3fcf728744..6e67f58259 100644 --- a/ai_agents/.env.example +++ b/ai_agents/.env.example @@ -55,6 +55,13 @@ GROK_PROXY_URL= # Gemini API key GEMINI_API_KEY= +# Extension: anthropic_llm2_python +# Anthropic API key +ANTHROPIC_BASE_URL=https://api.anthropic.com +ANTHROPIC_API_KEY= +ANTHROPIC_MODEL=claude-opus-5 +ANTHROPIC_PROXY_URL= + # Set this to Azure if you are using Azure OpenAI OPENAI_VENDOR= OPENAI_AZURE_ENDPOINT= diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/manifest-lock.json b/ai_agents/agents/examples/voice-assistant/tenapp/manifest-lock.json index 56dbf3c5f3..25e1b7ce14 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/manifest-lock.json +++ b/ai_agents/agents/examples/voice-assistant/tenapp/manifest-lock.json @@ -440,6 +440,23 @@ ], "path": "../../../ten_packages/extension/openai_llm2_python" }, + { + "type": "extension", + "name": "anthropic_llm2_python", + "version": "0.1.0", + "hash": "06de4fc1b5e65f521fa5e0211c2d38f012f22bb3278e475c3ad5098ab3211e0d", + "dependencies": [ + { + "type": "system", + "name": "ten_runtime_python" + }, + { + "type": "system", + "name": "ten_ai_base" + } + ], + "path": "../../../ten_packages/extension/anthropic_llm2_python" + }, { "type": "extension", "name": "azure_tts_python", @@ -952,4 +969,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/manifest.json b/ai_agents/agents/examples/voice-assistant/tenapp/manifest.json index bc24bd14d1..03f83ddad0 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/manifest.json +++ b/ai_agents/agents/examples/voice-assistant/tenapp/manifest.json @@ -87,6 +87,9 @@ { "path": "../../../ten_packages/extension/openai_llm2_python" }, + { + "path": "../../../ten_packages/extension/anthropic_llm2_python" + }, { "path": "../../../ten_packages/extension/azure_tts_python" }, diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/property.json b/ai_agents/agents/examples/voice-assistant/tenapp/property.json index b9cc5b8ee1..3b6fe0c15f 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/property.json +++ b/ai_agents/agents/examples/voice-assistant/tenapp/property.json @@ -1578,6 +1578,207 @@ } ] } + }, + { + "name": "voice_assistant_anthropic", + "auto_start": false, + "graph": { + "nodes": [ + { + "type": "extension", + "name": "agora_rtc", + "addon": "agora_rtc", + "extension_group": "default", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "app_certificate": "${env:AGORA_APP_CERTIFICATE|}", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": false + } + }, + { + "type": "extension", + "name": "stt", + "addon": "deepgram_asr_python", + "extension_group": "stt", + "property": { + "params": { + "api_key": "${env:DEEPGRAM_API_KEY}", + "language": "en-US", + "model": "nova-3" + } + } + }, + { + "type": "extension", + "name": "llm", + "addon": "anthropic_llm2_python", + "extension_group": "chatgpt", + "property": { + "api_key": "${env:ANTHROPIC_API_KEY}", + "base_url": "${env:ANTHROPIC_BASE_URL|}", + "model": "${env:ANTHROPIC_MODEL|claude-opus-5}", + "proxy_url": "${env:ANTHROPIC_PROXY_URL|}", + "max_tokens": 2048, + "prompt": "", + "effort": "low", + "thinking_display": "summarized", + "refusal_fallback": true, + "greeting": "TEN Agent connected. How can I help you today?" + } + }, + { + "type": "extension", + "name": "tts", + "addon": "elevenlabs_tts2_python", + "extension_group": "tts", + "property": { + "dump": false, + "dump_path": "./", + "params": { + "key": "${env:ELEVENLABS_TTS_KEY}", + "model_id": "eleven_multilingual_v2", + "voice_id": "pNInz6obpgDQGcFmaJgB", + "output_format": "pcm_16000" + } + } + }, + { + "type": "extension", + "name": "main_control", + "addon": "main_python", + "extension_group": "control", + "property": { + "greeting": "TEN Agent connected. How can I help you today?" + } + }, + { + "type": "extension", + "name": "message_collector", + "addon": "message_collector2", + "extension_group": "transcriber", + "property": {} + }, + { + "type": "extension", + "name": "weatherapi_tool_python", + "addon": "weatherapi_tool_python", + "extension_group": "default", + "property": { + "api_key": "${env:WEATHERAPI_API_KEY|}" + } + }, + { + "type": "extension", + "name": "streamid_adapter", + "addon": "streamid_adapter", + "property": {} + } + ], + "connections": [ + { + "extension": "main_control", + "cmd": [ + { + "names": [ + "on_user_joined", + "on_user_left" + ], + "source": [ + { + "extension": "agora_rtc" + } + ] + }, + { + "names": [ + "tool_register" + ], + "source": [ + { + "extension": "weatherapi_tool_python" + } + ] + } + ], + "data": [ + { + "name": "asr_result", + "source": [ + { + "extension": "stt" + } + ] + }, + { + "name": "tts_audio_start", + "source": [ + { + "extension": "tts" + } + ] + }, + { + "name": "tts_audio_end", + "source": [ + { + "extension": "tts" + } + ] + } + ] + }, + { + "extension": "agora_rtc", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension": "streamid_adapter" + } + ] + }, + { + "name": "pcm_frame", + "source": [ + { + "extension": "tts" + } + ] + } + ], + "data": [ + { + "name": "data", + "source": [ + { + "extension": "message_collector" + } + ] + } + ] + }, + { + "extension": "streamid_adapter", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension": "stt" + } + ] + } + ] + } + ] + } } ], "log": { diff --git a/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/README.md b/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/README.md new file mode 100644 index 0000000000..58ac093a6b --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/README.md @@ -0,0 +1,196 @@ +# anthropic_llm2_python + +An Anthropic Claude LLM2 extension for the TEN framework, talking to the +Anthropic Messages API directly. + +No extension previously targeted Claude. `bedrock_llm_python` is an Amazon +Nova extension that happens to use Bedrock's model-agnostic Converse API, so +overriding its `model` property could reach Claude, but that path is +undocumented and cannot express adaptive thinking or effort. + +## Features + +- Native Anthropic Messages API integration (no OpenAI-compatible shim) +- Full compatibility with the TEN LLM2 interface +- Streaming and non-streaming responses +- Tool calling support +- Reasoning surfaced through the LLM2 reasoning events, from Claude's native + `thinking` blocks +- Vision support for both remote image URLs and inline base64 data URIs +- Effort-based cost/latency control +- Server-side fallback on safety refusals, so a voice agent never goes silent + +## API + +Refer to the `api` definition in [manifest.json](manifest.json) and default +values in [property.json](property.json). + +| **Property** | **Type** | **Description** | +|---|---|---| +| `api_key` | `string` | API key for authenticating with Anthropic | +| `base_url` | `string` | Override the API base URL. Leave empty for the default | +| `model` | `string` | Model identifier (default `claude-opus-5`) | +| `max_tokens` | `int` | Maximum output tokens. Caps thinking **and** response text together | +| `prompt` | `string` | System prompt for the model | +| `proxy_url` | `string` | Optional HTTP proxy | +| `effort` | `string` | `low`, `medium`, `high`, `xhigh`, or `max`. Default `low` | +| `thinking_display` | `string` | `summarized` or `omitted`. Default `summarized` | +| `refusal_fallback` | `bool` | Route safety refusals to a fallback model. Default `true` | +| `refusal_message` | `string` | Spoken when a request is refused and no fallback succeeds | +| `custom_headers` | `object` | Extra HTTP headers. Scalar values only | + +There is deliberately no `temperature`, `top_p`, `top_k`, `presence_penalty`, +`frequency_penalty`, or `seed`. Claude Opus 5 rejects the first three with a +400 and does not accept the rest; `effort` is the tuning knob in their place. + +## Configuration + +Set the `ANTHROPIC_API_KEY` environment variable with your Anthropic API key: + +```bash +export ANTHROPIC_API_KEY=your_api_key +``` + +Optionally override the model or route through a proxy: + +```bash +export ANTHROPIC_MODEL=claude-sonnet-5 +export ANTHROPIC_PROXY_URL=http://127.0.0.1:7890 +``` + +## Usage + +Point a graph's LLM node at this addon. In any example's +`tenapp/property.json`, change the `addon` field of the LLM node: + +```json +{ + "type": "extension", + "name": "llm", + "addon": "anthropic_llm2_python", + "property": { + "model": "claude-opus-5", + "max_tokens": 2048, + "effort": "low" + } +} +``` + +No other change is needed. The extension implements the same +`llm-interface.json` contract as `openai_llm2_python`, so the surrounding +connections keep working unchanged. + +## Model compatibility + +Set `model` to any Claude model Anthropic currently serves. The extension +adjusts the request to what each one accepts, so switching models is a +one-line property change. + +| Model | Reasoning events | `effort` | Refusal fallback | +|---|---|---|---| +| `claude-opus-5` (default) | yes | `low`–`max` | yes | +| `claude-fable-5` | yes | `low`–`max` | yes | +| `claude-sonnet-5` | yes | `low`–`max` | no | +| `claude-opus-4-8`, `claude-opus-4-7` | yes | `low`–`max` | no | +| `claude-opus-4-6`, `claude-sonnet-4-6` | yes | `xhigh`, `max` → `high` | no | +| `claude-haiku-4-5` | no | ignored | no | +| `claude-sonnet-4-5`, `claude-opus-4-5`, `claude-opus-4-1` | no | ignored | no | + +Three separate capability gates are applied, because their support windows do +not line up: + +- **Adaptive thinking and `effort`** exist on the current generation only. + Older served models — Haiku 4.5, Sonnet 4.5, Opus 4.5 and earlier — reject + them, so those requests are sent plain. Reasoning events are unavailable + there; everything else, tools included, works normally. +- **Refusal fallback** is narrower still: only Opus 5 and Fable 5 accept the + `fallbacks` parameter. Sonnet 5, Opus 4.8 and Opus 4.6 return + `'' does not support the 'fallbacks' parameter`, so it is withheld + and refusals surface as `refusal_message` instead. +- **`effort` above `high`** arrived with Opus 4.7. On Opus 4.6 and Sonnet 4.6 + both `xhigh` and `max` are clamped to `high` rather than raising, so a + misconfigured graph keeps talking. + +The `model` property is the only thing consulted. A caller that sets +`LLMRequest.model` — `vision_analyze_tool_python` hardcodes `gpt-4o` — is +ignored rather than having that id forwarded to Anthropic, matching +`openai_llm2_python`. + +A model newer than this extension gets the current-generation path by default, +so new releases work without a code change. Fallback is the exception — it is +withheld for unrecognised models, since sending it where it is unsupported +fails the whole request. + +## Notes on the defaults + +### `effort` defaults to `low` + +The Anthropic API defaults to `high`. These graphs are realtime voice +assistants, where the latency of a single turn dominates the experience, and +`high` noticeably delays the first token. Claude Opus 5 performs unusually +well at `low`. + +Raise it for text or agentic graphs — `xhigh` is the recommended setting for +coding and agentic work. + +### `thinking_display` defaults to `summarized` + +Claude emits reasoning as structured `thinking` blocks, which this extension +maps onto `MESSAGE_REASONING_DELTA` / `MESSAGE_REASONING_DONE`. The API default +for `display` is `omitted`, which still streams thinking blocks but with empty +text — so a reasoning UI would render blank with no error. Setting +`summarized` keeps it populated. + +Set `omitted` to save tokens if you do not display reasoning. + +### `max_tokens` defaults to 2048 + +On Claude Opus 5, `max_tokens` caps thinking plus response text together, and +thinking is on by default. The 512 used by some other LLM extensions would +truncate mid-answer. + +### Refusals + +Anthropic's safety classifiers can decline a request with HTTP 200, +`stop_reason: "refusal"`, and an empty content list. For a voice agent that is +silence, and the user cannot distinguish it from a network failure. + +With `refusal_fallback` enabled the request is retried server-side on a +fallback model within the same call. If the whole chain still refuses, the +extension emits `refusal_message` so the agent says something. Set +`refusal_fallback` to `false` to stay off the beta endpoint entirely. + +## Development + +From the `ai_agents` directory: + +```bash +task format +task check +task lint-extension EXTENSION=anthropic_llm2_python +``` + +## Tests + +`tests/` runs the extension inside the TEN runtime rather than calling the +adapter directly, so it covers addon registration, the `llm-interface.json` +import, property injection, and the `chat_completion` cmd dispatch: + +```bash +task test-extension \ + EXTENSION=agents/ten_packages/extension/anthropic_llm2_python +``` + +`test_extension_loads_and_dispatches` needs no credentials — it uses an +invalid key and asserts the failure comes back from the API, which can only +happen if the whole path worked. The remaining tests call Anthropic for +real and are skipped unless `ANTHROPIC_API_KEY` is set; they cover +streaming, reasoning events, the `thinking_display` toggle, a two-turn tool +call round trip, and the legacy-model path. + +Set `ANTHROPIC_MODEL` to run them against a different model. The +reasoning tests skip themselves on models that cannot think. + +Code style follows the framework conventions: 80-character lines, Black +formatting, type hints on all parameters and return types, and explicit +exception logging via `ten_env.log_*`. diff --git a/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/__init__.py b/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/__init__.py new file mode 100644 index 0000000000..72593ab225 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/__init__.py @@ -0,0 +1,6 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from . import addon diff --git a/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/addon.py b/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/addon.py new file mode 100644 index 0000000000..a567a6a98a --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/addon.py @@ -0,0 +1,20 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from ten_runtime import ( + Addon, + register_addon_as_extension, + TenEnv, +) + + +@register_addon_as_extension("anthropic_llm2_python") +class AnthropicLLM2ExtensionAddon(Addon): + + def on_create_instance(self, ten_env: TenEnv, name: str, context) -> None: + from .extension import AnthropicLLM2Extension + + ten_env.log_info("AnthropicLLM2ExtensionAddon on_create_instance") + ten_env.on_create_instance_done(AnthropicLLM2Extension(name), context) diff --git a/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/anthropic_llm.py b/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/anthropic_llm.py new file mode 100644 index 0000000000..9a386a0ce2 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/anthropic_llm.py @@ -0,0 +1,650 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +import json +import re +import time +from typing import Any, AsyncGenerator, Literal + +import httpx +from anthropic import AsyncAnthropic +from pydantic import BaseModel, Field + +from ten_ai_base.struct import ( + ImageContent, + LLMMessageContent, + LLMMessageFunctionCall, + LLMMessageFunctionCallOutput, + LLMRequest, + LLMResponse, + LLMResponseMessageDelta, + LLMResponseMessageDone, + LLMResponseReasoningDelta, + LLMResponseReasoningDone, + LLMResponseToolCall, + TextContent, +) +from ten_ai_base.types import LLMToolMetadata +from ten_runtime.async_ten_env import AsyncTenEnv + +# Server-side refusal fallback. This beta flag pairs with the scalar +# `fallbacks: "default"` form; the older array form uses a different flag and +# mixing the two returns a 400. +FALLBACK_BETA = "server-side-fallback-2026-07-01" + +# Passed explicitly whenever `base_url` is unset. Leaving it to the SDK means +# it falls back to the ANTHROPIC_BASE_URL environment variable, and a variable +# that is exported but empty is not absent -- it becomes a base URL of "" and +# fails every request with a bare "Connection error". `property.json` maps the +# property to `${env:ANTHROPIC_BASE_URL|}`, so any deployment that exports the +# name without a value lands there. +DEFAULT_BASE_URL = "https://api.anthropic.com" + +# data:image/png;base64,iVBOR... +DATA_URI_RE = re.compile( + r"^data:(?P[^;,]+);base64,(?P.*)$", re.DOTALL +) + +# Adaptive thinking, `output_config.effort`, and server-side refusal fallback +# only exist on the current generation. Sending them to one of the older models +# Anthropic still serves returns a 400 -- notably `effort` on Claude Haiku 4.5, +# which is otherwise a natural pick for latency-sensitive voice graphs. +# +# The list is intentionally a closed set of older served models rather than an +# allow-list of new ones, so a model released after this extension was written +# gets the current-generation path by default and stays correct. Each entry is +# also the prefix of that model's dated id, so `claude-haiku-4-5-20251001` and +# the `claude-haiku-4-5` alias both match. Claude Opus 4 and Sonnet 4 are +# absent: they reached end-of-life and now return not_found either way. +LEGACY_MODEL_PREFIXES = ( + "claude-3-", + "claude-haiku-4-5", + "claude-opus-4-1", + "claude-opus-4-5", + "claude-sonnet-4-5", +) + +# Server-side refusal fallback is gated far more narrowly than adaptive +# thinking: only models with a published fallback target accept it. Sonnet 5, +# Opus 4.8, and Opus 4.6 all reject it with +# "'' does not support the `fallbacks` parameter", so this one is an +# allow-list -- an unfamiliar model goes without the fallback rather than +# having every request rejected. +FALLBACK_MODEL_PREFIXES = ( + "claude-fable-", + "claude-mythos-", + "claude-opus-5", +) + +# The `xhigh` effort level arrived with Opus 4.7; Opus 4.6 and Sonnet 4.6 +# reject it. Clamp rather than raise -- degrading one level keeps a +# misconfigured graph talking, where an error would drop the whole turn. +NO_XHIGH_MODEL_PREFIXES = ("claude-opus-4-6", "claude-sonnet-4-6") + +# Effort levels above `high`, in the order the API ranks them. +ABOVE_HIGH_EFFORTS = ("xhigh", "max") + +# Sampling controls the current generation rejects once adaptive thinking is +# on: `temperature` may only be 1, and top_p/top_k are not accepted at all. +# Graphs set them for whichever vendor they were written against -- +# main_python sends `temperature: 0.7` on every turn -- so drop them on the +# thinking path rather than fail the request. Older models still accept them. +THINKING_INCOMPATIBLE_PARAMS = ("temperature", "top_p", "top_k") + + +class AnthropicLLM2Config(BaseModel): + api_key: str = "" + base_url: str = "" + model: str = "claude-opus-5" + max_tokens: int = 2048 + prompt: str = "You are a helpful assistant." + proxy_url: str = "" + + # Depth of reasoning and overall token spend. `low` is the default because + # these graphs are realtime voice assistants where turn latency dominates + # the experience; raise it for text or agentic graphs. + effort: Literal["low", "medium", "high", "xhigh", "max"] = "low" + + # Claude surfaces reasoning as `thinking` blocks, which map onto the + # MESSAGE_REASONING_* events. The API default is "omitted", which streams + # those blocks with empty text -- so the playground's reasoning pane stays + # blank unless this is "summarized". + thinking_display: Literal["summarized", "omitted"] = "summarized" + + # Safety classifiers can decline a request with HTTP 200 and no content. + # For a voice agent that is silence, so route refusals to a fallback model + # server-side and, failing that, speak `refusal_message`. + refusal_fallback: bool = True + refusal_message: str = "Sorry, I'm not able to help with that request." + + custom_headers: dict[str, Any] = Field(default_factory=dict) + black_list_params: list[str] = Field( + default_factory=lambda: [ + "messages", + "tools", + "stream", + "model", + "system", + "max_tokens", + ] + ) + + def is_black_list_params(self, key: str) -> bool: + return key in self.black_list_params + + +class AnthropicLLM: + """Adapter between the ten_ai_base LLM contract and the Anthropic Messages + API. + + Translation runs in both directions: LLMRequest -> Messages request on the + way in, and Anthropic stream events -> LLMResponse events on the way out. + Neither side is ours to change, so every quirk is absorbed here. + """ + + def __init__(self, ten_env: AsyncTenEnv, config: AnthropicLLM2Config): + self.ten_env = ten_env + self.config = config + + self.http_client = None + if config.proxy_url: + ten_env.log_info(f"Setting httpx proxy: {config.proxy_url}") + self.http_client = httpx.AsyncClient(proxy=config.proxy_url) + + default_headers: dict[str, str] = {} + for key, value in config.custom_headers.items(): + if isinstance(value, (dict, list)): + ten_env.log_warn( + f"Skipping custom header '{key}':" + f" value must be a scalar, got {type(value).__name__}" + ) + continue + default_headers[str(key)] = str(value) + + self.client = AsyncAnthropic( + api_key=config.api_key, + base_url=config.base_url or DEFAULT_BASE_URL, + default_headers=default_headers or None, + http_client=self.http_client, + ) + + async def aclose(self) -> None: + """Release the connection pools. + + A worker starts and stops graphs repeatedly, so without this each + cycle leaks a pool -- and with a proxy configured, entries in the + proxy's connection table too. + """ + try: + await self.client.close() + except Exception as err: + self.ten_env.log_warn( + f"Failed to close the Anthropic client: {err}" + ) + if self.http_client is not None: + try: + await self.http_client.aclose() + except Exception as err: + self.ten_env.log_warn(f"Failed to close the HTTP client: {err}") + + # ------------------------------------------------------------------ + # Request translation + # ------------------------------------------------------------------ + + def _loads_arguments(self, raw: str) -> dict[str, Any]: + """ten_ai_base carries tool arguments as a JSON string; Anthropic wants + a parsed object.""" + if not raw: + return {} + try: + parsed = json.loads(raw) + except json.JSONDecodeError as err: + self.ten_env.log_warn(f"Malformed tool arguments {raw!r}: {err}") + return {} + return parsed if isinstance(parsed, dict) else {} + + def _image_block(self, url: str) -> dict[str, Any]: + """ImageURL.url carries both remote URLs and inline data URIs.""" + match = DATA_URI_RE.match(url) + if match: + return { + "type": "image", + "source": { + "type": "base64", + "media_type": match.group("media_type"), + "data": match.group("data"), + }, + } + return {"type": "image", "source": {"type": "url", "url": url}} + + def _content_blocks(self, content: Any) -> Any: + if isinstance(content, str): + return content + + blocks: list[dict[str, Any]] = [] + for item in content or []: + match item: + case TextContent(): + blocks.append({"type": "text", "text": item.text}) + case ImageContent(): + # ImageURL.detail has no Anthropic equivalent. + blocks.append(self._image_block(item.image_url.url)) + return blocks + + @staticmethod + def _is_block_turn( + message: dict[str, Any], role: str, block_type: str + ) -> bool: + """True when `message` is a turn made up only of `block_type` blocks. + + Used to append onto the turn instead of starting a new one, so parallel + tool use round-trips as a single assistant turn and a single user turn + rather than one message per call. + """ + content = message.get("content") + return ( + message.get("role") == role + and isinstance(content, list) + and bool(content) + and all(block.get("type") == block_type for block in content) + ) + + def _parse_messages( + self, messages: list[Any], base_prompt: str + ) -> tuple[str, list[dict[str, Any]]]: + """Returns (system_prompt, messages). + + Anthropic takes the system prompt as a top-level parameter rather than + a message, so in-history system turns are folded onto it. + """ + system_parts = [base_prompt] if base_prompt else [] + parsed: list[dict[str, Any]] = [] + + for message in messages: + match message: + case LLMMessageContent(): + if message.role == "system": + content = message.content + system_parts.append( + content + if isinstance(content, str) + else "\n".join( + item.text + for item in content or [] + if isinstance(item, TextContent) + ) + ) + continue + parsed.append( + { + "role": message.role, + "content": self._content_blocks(message.content), + } + ) + case LLMMessageFunctionCall(): + block = { + "type": "tool_use", + "id": message.call_id, + "name": message.name, + "input": self._loads_arguments(message.arguments), + } + # Parallel calls arrive as separate messages but came from + # one assistant turn, so rebuild that turn. + if parsed and self._is_block_turn( + parsed[-1], "assistant", "tool_use" + ): + parsed[-1]["content"].append(block) + else: + parsed.append({"role": "assistant", "content": [block]}) + case LLMMessageFunctionCallOutput(): + block = { + "type": "tool_result", + "tool_use_id": message.call_id, + "content": message.output, + } + # Anthropic wants every result from one assistant turn in a + # single user message. Splitting them is accepted but + # discourages future parallel tool use, so merge instead. + if parsed and self._is_block_turn( + parsed[-1], "user", "tool_result" + ): + parsed[-1]["content"].append(block) + else: + parsed.append({"role": "user", "content": [block]}) + + return "\n\n".join(part for part in system_parts if part), parsed + + def _convert_tool(self, tool: LLMToolMetadata) -> dict[str, Any]: + properties: dict[str, Any] = {} + required: list[str] = [] + + for param in tool.parameters: + schema: dict[str, Any] = { + "type": param.type, + "description": param.description, + } + if param.type == "array" and getattr(param, "items", None): + schema["items"] = param.items + properties[param.name] = schema + if param.required: + required.append(param.name) + + return { + "name": tool.name, + "description": tool.description, + "input_schema": { + "type": "object", + "properties": properties, + "required": required, + }, + } + + def _effort_for(self, model: str) -> str: + """Effort level, narrowed to what `model` accepts.""" + effort = self.config.effort + # `max` sits above `xhigh`, so a model that predates `xhigh` cannot + # accept it either. Clamp both to the highest level it does know. + if effort in ABOVE_HIGH_EFFORTS and model.startswith( + NO_XHIGH_MODEL_PREFIXES + ): + self.ten_env.log_info( + f"{model} does not support effort '{effort}'; using 'high'" + ) + return "high" + return effort + + def _build_request(self, request_input: LLMRequest) -> dict[str, Any]: + system_prompt, messages = self._parse_messages( + request_input.messages, request_input.prompt or self.config.prompt + ) + + if not messages: + raise RuntimeError( + "Anthropic requires at least one message; the translated " + "history is empty" + ) + if messages[0]["role"] != "user": + raise RuntimeError( + "Anthropic requires the first message to have role 'user'; " + f"got '{messages[0]['role']}'" + ) + + # `request_input.model` is deliberately ignored, matching + # openai_llm2_python. Callers set it for whichever vendor they were + # written against -- vision_analyze_tool_python hardcodes "gpt-4o" -- + # and honouring it would send that straight to Anthropic as a 404. + model = self.config.model + if request_input.model and request_input.model != model: + self.ten_env.log_debug( + f"ignoring caller-supplied model '{request_input.model}'; " + f"using the configured '{model}'" + ) + + req: dict[str, Any] = { + "model": model, + "max_tokens": self.config.max_tokens, + "messages": messages, + } + + if system_prompt: + req["system"] = system_prompt + + tools = [ + self._convert_tool(tool) for tool in (request_input.tools or []) + ] + if tools: + req["tools"] = tools + + if model.startswith(LEGACY_MODEL_PREFIXES): + # Older served models reject these outright, so send a plain + # request instead. Reasoning events are unavailable there. + self.ten_env.log_info( + f"{model} predates adaptive thinking, effort, and refusal " + "fallback; omitting them from the request" + ) + else: + req["thinking"] = { + "type": "adaptive", + "display": self.config.thinking_display, + } + req["output_config"] = {"effort": self._effort_for(model)} + + if self.config.refusal_fallback: + if model.startswith(FALLBACK_MODEL_PREFIXES): + req["betas"] = [FALLBACK_BETA] + # Passed as raw body rather than a typed kwarg so the + # extension keeps working on SDK releases that predate the + # parameter. + req["extra_body"] = {"fallbacks": "default"} + else: + self.ten_env.log_info( + f"{model} does not support the fallbacks parameter; " + "refusals will surface as refusal_message instead" + ) + + thinking_enabled = "thinking" in req + for key, value in (request_input.parameters or {}).items(): + if self.config.is_black_list_params(key): + continue + if thinking_enabled and key in THINKING_INCOMPATIBLE_PARAMS: + self.ten_env.log_debug( + f"dropping '{key}': adaptive thinking does not accept it" + ) + continue + self.ten_env.log_debug(f"set anthropic param: {key} = {value}") + req[key] = value + + return req + + # ------------------------------------------------------------------ + # Response translation + # ------------------------------------------------------------------ + + def _messages(self, req: dict[str, Any]) -> Any: + """The messages resource this request needs. + + Only refusal fallback requires the beta surface, so a graph that + turns it off stays on /v1/messages -- which is what an operator + disabling it for a gateway or compliance reason is asking for. + """ + if "betas" in req: + return self.client.beta.messages + return self.client.messages + + def _log_refusal(self, message: Any) -> None: + details = getattr(message, "stop_details", None) + category = getattr(details, "category", None) if details else None + self.ten_env.log_info( + f"Request refused by safety classifiers, category={category}" + ) + + async def get_chat_completions( + self, request_input: LLMRequest + ) -> AsyncGenerator[LLMResponse, None]: + req = self._build_request(request_input) + + self.ten_env.log_info( + f"get_chat_completions: {len(req['messages'])} messages, " + f"streaming: {request_input.streaming}, model: {req['model']}" + ) + + if request_input.streaming: + async for response in self._stream(req): + yield response + else: + async for response in self._complete(req): + yield response + + async def _stream( + self, req: dict[str, Any] + ) -> AsyncGenerator[LLMResponse, None]: + created = int(time.time()) + response_id = "" + text_content = "" + reasoning_content = "" + block_types: dict[int, str] = {} + tool_blocks: dict[int, dict[str, str]] = {} + + try: + async with self._messages(req).stream(**req) as stream: + async for event in stream: + match event.type: + case "message_start": + response_id = event.message.id + + case "content_block_start": + block = event.content_block + block_types[event.index] = block.type + if block.type == "tool_use": + tool_blocks[event.index] = { + "id": block.id, + "name": block.name, + "json": "", + } + + case "content_block_delta": + delta = event.delta + match delta.type: + case "text_delta": + text_content += delta.text + yield LLMResponseMessageDelta( + response_id=response_id, + role="assistant", + content=text_content, + delta=delta.text, + created=created, + ) + case "thinking_delta": + reasoning_content += delta.thinking + yield LLMResponseReasoningDelta( + response_id=response_id, + role="assistant", + content=reasoning_content, + delta=delta.thinking, + created=created, + ) + case "input_json_delta": + buffered = tool_blocks.get(event.index) + if buffered is not None: + buffered["json"] += delta.partial_json + + case "content_block_stop": + block_type = block_types.pop(event.index, None) + if block_type == "tool_use": + buffered = tool_blocks.pop(event.index, None) + if buffered is not None: + yield LLMResponseToolCall( + response_id=response_id, + id=response_id, + tool_call_id=buffered["id"], + name=buffered["name"], + arguments=self._loads_arguments( + buffered["json"] + ), + created=created, + ) + elif block_type == "thinking" and reasoning_content: + yield LLMResponseReasoningDone( + response_id=response_id, + role="assistant", + content=reasoning_content, + created=created, + ) + reasoning_content = "" + + final_message = await stream.get_final_message() + except Exception as err: + raise RuntimeError(f"CreateMessage failed, err: {err}") from err + + # Check stop_reason before trusting content: a pre-output refusal + # returns an empty content list. + if final_message.stop_reason == "refusal": + self._log_refusal(final_message) + yield LLMResponseMessageDone( + response_id=response_id or final_message.id, + role="assistant", + content=self.config.refusal_message, + created=created, + ) + return + + yield LLMResponseMessageDone( + response_id=response_id or final_message.id, + role="assistant", + content=text_content, + created=created, + ) + + async def _complete( + self, req: dict[str, Any] + ) -> AsyncGenerator[LLMResponse, None]: + created = int(time.time()) + + try: + message = await self._messages(req).create(**req) + except Exception as err: + raise RuntimeError(f"CreateMessage failed, err: {err}") from err + + if message.stop_reason == "refusal": + self._log_refusal(message) + yield LLMResponseMessageDone( + response_id=message.id, + role="assistant", + content=self.config.refusal_message, + created=created, + ) + return + + reasoning_content = "".join( + getattr(block, "thinking", "") or "" + for block in message.content + if block.type == "thinking" + ) + if reasoning_content: + yield LLMResponseReasoningDelta( + response_id=message.id, + role="assistant", + content=reasoning_content, + delta=reasoning_content, + created=created, + ) + yield LLMResponseReasoningDone( + response_id=message.id, + role="assistant", + content=reasoning_content, + created=created, + ) + + text_content = "".join( + block.text for block in message.content if block.type == "text" + ) + if text_content: + yield LLMResponseMessageDelta( + response_id=message.id, + role="assistant", + content=text_content, + delta=text_content, + created=created, + ) + + for block in message.content: + if block.type == "tool_use": + yield LLMResponseToolCall( + response_id=message.id, + id=message.id, + tool_call_id=block.id, + name=block.name, + arguments=( + block.input if isinstance(block.input, dict) else {} + ), + created=created, + ) + + yield LLMResponseMessageDone( + response_id=message.id, + role="assistant", + content=text_content, + created=created, + ) diff --git a/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/extension.py b/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/extension.py new file mode 100644 index 0000000000..2ef59af530 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/extension.py @@ -0,0 +1,128 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +import asyncio +from typing import AsyncGenerator + +from ten_ai_base.llm2 import AsyncLLM2BaseExtension +from ten_ai_base.struct import ( + LLMRequest, + LLMRequestRetrievePrompt, + LLMResponse, + LLMResponseRetrievePrompt, +) +from ten_runtime.async_ten_env import AsyncTenEnv + +from .anthropic_llm import AnthropicLLM, AnthropicLLM2Config + +# How long a request that arrives during startup waits for on_start to +# finish before giving up. Building the HTTP client takes a few tens of +# milliseconds, and the runtime can deliver a cmd inside that window. +STARTUP_TIMEOUT_SECONDS = 10.0 + + +class AnthropicLLM2Extension(AsyncLLM2BaseExtension): + def __init__(self, name: str): + super().__init__(name) + self.config: AnthropicLLM2Config | None = None + self.client: AnthropicLLM | None = None + self._started = asyncio.Event() + + async def on_init(self, ten_env: AsyncTenEnv) -> None: + ten_env.log_info("on_init") + await super().on_init(ten_env) + + async def on_start(self, async_ten_env: AsyncTenEnv) -> None: + async_ten_env.log_info("on_start") + await super().on_start(async_ten_env) + + try: + # An exception escaping on_start reaches the runtime's + # _exit_on_exception, which calls os._exit and takes the whole + # worker down. One bad property -- an effort level that is not in + # the Literal, say -- must fail this node, not the app. + try: + config_json, _ = await self.ten_env.get_property_to_json("") + self.config = AnthropicLLM2Config.model_validate_json( + config_json + ) + except Exception as err: + async_ten_env.log_error(f"Invalid configuration: {err}") + return + + if not self.config.api_key: + async_ten_env.log_error("API key is missing, exiting on_start") + return + + try: + self.client = AnthropicLLM(async_ten_env, self.config) + async_ten_env.log_info( + f"initialized with model: {self.config.model}, " + f"max_tokens: {self.config.max_tokens}, " + f"effort: {self.config.effort}" + ) + except Exception as err: + async_ten_env.log_error( + f"Failed to initialize AnthropicLLM: {err}" + ) + finally: + # Unblocks any chat_completion that raced startup, including the + # failure paths above -- they must report the real reason rather + # than time out. + self._started.set() + + async def on_stop(self, async_ten_env: AsyncTenEnv) -> None: + async_ten_env.log_info("on_stop") + # Base class first, so in-flight streams are cancelled before their + # connection pool goes away. + await super().on_stop(async_ten_env) + if self.client is not None: + await self.client.aclose() + self.client = None + + async def on_deinit(self, async_ten_env: AsyncTenEnv) -> None: + async_ten_env.log_info("on_deinit") + await super().on_deinit(async_ten_env) + + async def on_retrieve_prompt( + self, async_ten_env: AsyncTenEnv, request: LLMRequestRetrievePrompt + ) -> LLMResponseRetrievePrompt: + """Retrieve the current prompt from config.""" + prompt = self.config.prompt if self.config else "" + async_ten_env.log_info( + f"Retrieved prompt for request_id: {request.request_id}" + ) + return LLMResponseRetrievePrompt(prompt=prompt) + + def on_call_chat_completion( + self, async_ten_env: AsyncTenEnv, request_input: LLMRequest + ) -> AsyncGenerator[LLMResponse, None]: + return self._chat_completion(async_ten_env, request_input) + + async def _chat_completion( + self, async_ten_env: AsyncTenEnv, request_input: LLMRequest + ) -> AsyncGenerator[LLMResponse, None]: + # The runtime can deliver a cmd before on_start has built the client, + # so wait for startup instead of failing a request that is merely + # early. + if not self._started.is_set(): + async_ten_env.log_info("Request arrived during startup; waiting") + try: + await asyncio.wait_for( + self._started.wait(), STARTUP_TIMEOUT_SECONDS + ) + except asyncio.TimeoutError as err: + raise RuntimeError( + "AnthropicLLM did not finish starting within " + f"{STARTUP_TIMEOUT_SECONDS}s" + ) from err + + if self.client is None: + raise RuntimeError( + "AnthropicLLM is not initialized; check that api_key is set" + ) + + async for response in self.client.get_chat_completions(request_input): + yield response diff --git a/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/manifest.json b/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/manifest.json new file mode 100644 index 0000000000..ca1122f0ac --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/manifest.json @@ -0,0 +1,72 @@ +{ + "type": "extension", + "name": "anthropic_llm2_python", + "version": "0.1.0", + "dependencies": [ + { + "type": "system", + "name": "ten_runtime_python", + "version": "0.11" + }, + { + "type": "system", + "name": "ten_ai_base", + "version": "0.7" + } + ], + "package": { + "include": [ + "manifest.json", + "property.json", + "**.py", + "README.md", + "pyproject.toml", + "requirements.txt" + ] + }, + "api": { + "interface": [ + { + "import_uri": "../../system/ten_ai_base/api/llm-interface.json" + } + ], + "property": { + "properties": { + "api_key": { + "type": "string" + }, + "base_url": { + "type": "string" + }, + "model": { + "type": "string" + }, + "max_tokens": { + "type": "int32" + }, + "prompt": { + "type": "string" + }, + "proxy_url": { + "type": "string" + }, + "effort": { + "type": "string" + }, + "thinking_display": { + "type": "string" + }, + "refusal_fallback": { + "type": "bool" + }, + "refusal_message": { + "type": "string" + }, + "custom_headers": { + "type": "object", + "properties": {} + } + } + } + } +} diff --git a/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/property.json b/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/property.json new file mode 100644 index 0000000000..23cbb4e05a --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/property.json @@ -0,0 +1,11 @@ +{ + "api_key": "${env:ANTHROPIC_API_KEY}", + "base_url": "${env:ANTHROPIC_BASE_URL|}", + "model": "${env:ANTHROPIC_MODEL|claude-opus-5}", + "proxy_url": "${env:ANTHROPIC_PROXY_URL|}", + "max_tokens": 2048, + "prompt": "", + "effort": "low", + "thinking_display": "summarized", + "refusal_fallback": true +} diff --git a/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/pyproject.toml b/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/pyproject.toml new file mode 100644 index 0000000000..ae016b94f3 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/pyproject.toml @@ -0,0 +1,7 @@ +[project] +name = "anthropic-llm2-python" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = [ + "anthropic>=0.116.0", +] diff --git a/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/requirements.txt b/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/requirements.txt new file mode 100644 index 0000000000..9523674770 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/requirements.txt @@ -0,0 +1 @@ +anthropic>=0.116.0 diff --git a/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/tests/__init__.py b/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/tests/bin/start b/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/tests/bin/start new file mode 100755 index 0000000000..962e155407 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/tests/bin/start @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +cd "$(dirname "${BASH_SOURCE[0]}")/../.." + +export PYTHONPATH="$(pwd)/..":.ten/app:.ten/app/ten_packages/system/ten_runtime_python/lib:.ten/app/ten_packages/system/ten_runtime_python/interface:.ten/app/ten_packages/system/ten_ai_base/interface:$PYTHONPATH + +# If the Python app imports some modules that are compiled with a different +# version of libstdc++ (ex: PyTorch), the Python app may encounter confusing +# errors. To solve this problem, we can preload the correct version of +# libstdc++. +# +# export LD_PRELOAD=/lib/x86_64-linux-gnu/libstdc++.so.6 +# +# Another solution is to make sure the module 'ten_runtime_python' is imported +# _after_ the module that requires another version of libstdc++ is imported. +# +# Refer to https://github.com/pytorch/pytorch/issues/102360?from_wecom=1#issuecomment-1708989096 + +pytest -s tests/ "$@" diff --git a/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/tests/conftest.py b/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/tests/conftest.py new file mode 100644 index 0000000000..99eaefff1f --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/tests/conftest.py @@ -0,0 +1,100 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +import json +import threading + +import pytest +from typing_extensions import override + +from ten_runtime import ( + App, + TenEnv, +) + + +class FakeApp(App): + def __init__(self): + super().__init__() + self.event: threading.Event | None = None + + # In the case of a fake app, we use `on_init` to allow the blocked testing + # fixture to continue execution, rather than using `on_configure`. The + # reason is that in the TEN runtime C core, the relationship between the + # addon manager and the (fake) app is bound after `on_configure_done` is + # called. So we only need to let the testing fixture continue execution + # after this action in the TEN runtime C core, and at the upper layer + # timing, the earliest point is within the `on_init()` function of the + # upper TEN app. Therefore, we release the testing fixture lock within the + # user layer's `on_init()` of the TEN app. + def on_init(self, ten_env: TenEnv) -> None: + assert self.event + self.event.set() + + ten_env.on_init_done() + + @override + def on_configure(self, ten_env: TenEnv) -> None: + ten_env.init_property_from_json( + json.dumps( + { + "ten": { + "log": { + "handlers": [ + { + "matchers": [{"level": "info"}], + "formatter": { + "type": "plain", + "colored": True, + }, + "emitter": { + "type": "console", + "config": {"stream": "stdout"}, + }, + } + ] + } + } + } + ), + ) + + ten_env.on_configure_done() + + +class FakeAppCtx: + def __init__(self, event: threading.Event): + self.fake_app: FakeApp | None = None + self.event = event + + +def run_fake_app(fake_app_ctx: FakeAppCtx): + app = FakeApp() + app.event = fake_app_ctx.event + fake_app_ctx.fake_app = app + app.run(False) + + +@pytest.fixture(scope="session", autouse=True) +def global_setup_and_teardown(): + event = threading.Event() + fake_app_ctx = FakeAppCtx(event) + + fake_app_thread = threading.Thread( + target=run_fake_app, args=(fake_app_ctx,) + ) + fake_app_thread.start() + + event.wait() + + assert fake_app_ctx.fake_app is not None + + # Yield control to the test; after the test execution is complete, continue + # with the teardown process. + yield + + # Teardown part. + fake_app_ctx.fake_app.close() + fake_app_thread.join() diff --git a/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/tests/test_chat_completion.py b/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/tests/test_chat_completion.py new file mode 100644 index 0000000000..aafc553b81 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/anthropic_llm2_python/tests/test_chat_completion.py @@ -0,0 +1,393 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +"""End-to-end tests that drive the extension through the TEN runtime. + +A direct unit test of `AnthropicLLM` can only prove the translation is +right. These load the addon the way a graph does -- registration through +`addon.py`, the `llm-interface.json` import in `manifest.json`, property +injection, and the `chat_completion` cmd dispatched by +`AsyncLLM2BaseExtension` -- and assert on the streamed `CmdResult`s exactly +as `main_cascade_python` consumes them. + +Only `test_extension_loads_and_dispatches` runs without credentials. The +rest talk to the real Anthropic API and are skipped unless a key is set: + + export ANTHROPIC_API_KEY=sk-ant-... + task test-extension \ + EXTENSION=agents/ten_packages/extension/anthropic_llm2_python +""" + +import asyncio +import json +import os + +import pytest + +from anthropic_llm2_python.anthropic_llm import LEGACY_MODEL_PREFIXES +from ten_ai_base.struct import ( + LLMMessageContent, + LLMMessageFunctionCall, + LLMMessageFunctionCallOutput, + LLMRequest, + LLMResponse, + LLMResponseMessageDelta, + LLMResponseMessageDone, + LLMResponseReasoningDelta, + LLMResponseReasoningDone, + LLMResponseToolCall, + parse_llm_response, +) +from ten_ai_base.types import LLMToolMetadata, LLMToolMetadataParameter +from ten_runtime import ( + AsyncExtensionTester, + AsyncTenEnvTester, + Cmd, + StatusCode, +) + +ADDON_NAME = "anthropic_llm2_python" +API_KEY = os.environ.get("ANTHROPIC_API_KEY", "") +MODEL = os.environ.get("ANTHROPIC_MODEL") or "claude-opus-5" +LEGACY_MODEL = "claude-haiku-4-5" + +requires_api_key = pytest.mark.skipif( + not API_KEY, reason="ANTHROPIC_API_KEY is not set" +) + +# Adaptive thinking only engages when the task needs it, so a question the +# model can answer from memory yields no reasoning events at any effort. +THINKING_PROMPT = ( + "Compute 17 * 23 * 31 step by step, then tell me the sum of the digits " + "of the result." +) + +WEATHER_TOOL = LLMToolMetadata( + name="get_weather", + description="Get the current temperature for a city.", + parameters=[ + LLMToolMetadataParameter( + name="city", + type="string", + description="City name", + required=True, + ) + ], +) + + +def user(text: str) -> LLMMessageContent: + return LLMMessageContent(role="user", content=text) + + +def properties(**overrides) -> str: + """The same shape `property.json` resolves to, minus the env lookups.""" + props = { + "api_key": API_KEY, + "model": MODEL, + "max_tokens": 2048, + "prompt": "", + "effort": "low", + "thinking_display": "summarized", + "refusal_fallback": True, + } + props.update(overrides) + return json.dumps(props) + + +class ChatCompletionTester(AsyncExtensionTester): + """Sends one `chat_completion` and collects the streamed responses. + + Mirrors `main_cascade_python.agent.llm_exec`: results that are not yet + completed carry an `LLMResponse` payload, and the completed one carries + only the status. + """ + + def __init__(self, request: LLMRequest, timeout: float = 90.0) -> None: + super().__init__() + self.request = request + self.timeout = timeout + self.responses: list[LLMResponse] = [] + self.final_status: StatusCode | None = None + self.error: str | None = None + self._watchdog: asyncio.Task | None = None + + async def on_start(self, ten_env: AsyncTenEnvTester) -> None: + # Held in an attribute: the loop keeps only a weak reference to + # pending tasks, so a bare create_task can be collected mid-sleep and + # a hung request would block instead of failing. + self._watchdog = asyncio.create_task(self._abort_on_timeout(ten_env)) + + cmd = Cmd.create("chat_completion") + cmd.set_property_from_json(None, self.request.model_dump_json()) + + async for result, err in ten_env.send_cmd_ex(cmd): + if err is not None: + self.error = f"send_cmd_ex failed: {err.error_message()}" + break + if result is None: + continue + if result.is_completed(): + self.final_status = result.get_status_code() + continue + + payload, payload_err = result.get_property_to_json(None) + if payload_err is not None: + self.error = f"unreadable result payload: {payload_err}" + break + self.responses.append(parse_llm_response(payload)) + + if self._watchdog is not None: + self._watchdog.cancel() + ten_env.stop_test() + + async def _abort_on_timeout(self, ten_env: AsyncTenEnvTester) -> None: + await asyncio.sleep(self.timeout) + self.error = f"timed out after {self.timeout}s" + ten_env.stop_test() + + # -- accessors ----------------------------------------------------- + + def deltas(self) -> list[LLMResponseMessageDelta]: + return [ + r for r in self.responses if isinstance(r, LLMResponseMessageDelta) + ] + + def done(self) -> LLMResponseMessageDone | None: + for response in self.responses: + if isinstance(response, LLMResponseMessageDone): + return response + return None + + def reasoning(self) -> list[LLMResponse]: + return [ + r + for r in self.responses + if isinstance( + r, (LLMResponseReasoningDelta, LLMResponseReasoningDone) + ) + ] + + def tool_calls(self) -> list[LLMResponseToolCall]: + return [r for r in self.responses if isinstance(r, LLMResponseToolCall)] + + +def run_tester(tester: ChatCompletionTester, **overrides) -> None: + tester.set_test_mode_single(ADDON_NAME, properties(**overrides)) + err = tester.run() + assert err is None, err.error_message() + assert tester.error is None, tester.error + + +def test_extension_loads_and_dispatches(): + """Runs without credentials. + + Covers what a unit test of the adapter cannot reach: the addon + registers under its manifest name, the `llm-interface.json` import + resolves, the single-extension graph builds, properties are injected, + and `chat_completion` reaches `on_call_chat_completion`. The key is + deliberately invalid, so the only way to get a completed ERROR result + is for all of that to have worked and the API call to have failed. + """ + tester = ChatCompletionTester( + LLMRequest(request_id="load-check", messages=[user("hi")]), + timeout=45.0, + ) + run_tester(tester, api_key="sk-ant-invalid-key-used-by-the-load-check") + + assert tester.final_status == StatusCode.ERROR, ( + "expected the invalid key to be rejected inside the extension; got " + f"status {tester.final_status} after {len(tester.responses)} responses" + ) + + +@requires_api_key +def test_streaming_text(): + tester = ChatCompletionTester( + LLMRequest( + request_id="stream-text", + messages=[user("Name one planet, in one short sentence.")], + ) + ) + run_tester(tester) + + assert tester.final_status == StatusCode.OK + deltas = tester.deltas() + assert len(deltas) > 1, f"expected a stream, got {len(deltas)} delta(s)" + + done = tester.done() + assert done is not None, "no message_content_done event" + assert done.content and done.content.strip() + # The adapter accumulates, so every delta carries the text so far. + assert deltas[-1].content == done.content + + +@requires_api_key +def test_graph_supplied_sampling_parameters(): + """`main_python` sends `parameters={"temperature": 0.7}` on every turn. + + Forwarding it verbatim is a 400 -- adaptive thinking allows only + `temperature: 1` -- which would fail every turn of a real voice graph. + The adapter drops the conflicting keys on the thinking path. + """ + tester = ChatCompletionTester( + LLMRequest( + request_id="graph-parameters", + messages=[user("Name one planet, in one short sentence.")], + parameters={"temperature": 0.7, "top_p": 0.9}, + ) + ) + run_tester(tester) + + assert tester.final_status == StatusCode.OK, ( + "sampling parameters set by the graph were forwarded to a model that " + "rejects them" + ) + done = tester.done() + assert done is not None and done.content + + +@requires_api_key +def test_legacy_model_keeps_sampling_parameters(): + """The same keys are valid without thinking, so they must survive.""" + tester = ChatCompletionTester( + LLMRequest( + request_id="legacy-parameters", + messages=[user("Name one planet, in one short sentence.")], + parameters={"temperature": 0.2}, + ) + ) + run_tester(tester, model=LEGACY_MODEL) + + assert tester.final_status == StatusCode.OK + assert tester.done() is not None + + +@requires_api_key +@pytest.mark.skipif( + MODEL.startswith(LEGACY_MODEL_PREFIXES), + reason=f"{MODEL} does not support adaptive thinking", +) +def test_reasoning_reaches_the_graph(): + tester = ChatCompletionTester( + LLMRequest(request_id="reasoning", messages=[user(THINKING_PROMPT)]), + timeout=120.0, + ) + run_tester(tester, max_tokens=3000) + + assert tester.final_status == StatusCode.OK + assert ( + tester.reasoning() + ), "no reasoning events -- thinking blocks are not reaching the graph" + assert tester.done() is not None + + +@requires_api_key +@pytest.mark.skipif( + MODEL.startswith(LEGACY_MODEL_PREFIXES), + reason=f"{MODEL} does not support adaptive thinking", +) +def test_thinking_display_omitted_suppresses_reasoning(): + tester = ChatCompletionTester( + LLMRequest( + request_id="reasoning-off", messages=[user(THINKING_PROMPT)] + ), + timeout=120.0, + ) + run_tester(tester, max_tokens=3000, thinking_display="omitted") + + assert tester.final_status == StatusCode.OK + assert not tester.reasoning() + done = tester.done() + assert done is not None and done.content + + +@requires_api_key +def test_tool_call_round_trip(): + """Two turns through the graph. + + The second one is the highest-risk translation in the adapter: the two + tool calls and their two results each collapse into a single Anthropic + turn, and only the real API can confirm the merged shape is accepted. + """ + question = "What is the weather in Taipei and in Tokyo?" + first = ChatCompletionTester( + LLMRequest( + request_id="tool-call", + messages=[user(question)], + tools=[WEATHER_TOOL], + ) + ) + run_tester( + first, prompt="Use get_weather for every city you are asked about." + ) + + calls = first.tool_calls() + assert ( + calls + ), f"no tool call in {[type(r).__name__ for r in first.responses]}" + assert all(isinstance(c.arguments, dict) and c.arguments for c in calls) + + history: list = [user(question)] + for call in calls: + history.append( + LLMMessageFunctionCall( + type="function_call", + id=call.tool_call_id, + call_id=call.tool_call_id, + name=call.name, + arguments=json.dumps(call.arguments), + ) + ) + for call in calls: + city = str(call.arguments.get("city", "")) + history.append( + LLMMessageFunctionCallOutput( + type="function_call_output", + call_id=call.tool_call_id, + output="28C" if "Taipei" in city else "19C", + ) + ) + + second = ChatCompletionTester( + LLMRequest( + request_id="tool-result", + messages=history, + tools=[WEATHER_TOOL], + ) + ) + run_tester( + second, prompt="Use get_weather for every city you are asked about." + ) + + assert second.final_status == StatusCode.OK + done = second.done() + assert done is not None and done.content and done.content.strip() + + +@requires_api_key +def test_legacy_model_path(): + """A currently-served older model must take the degraded path. + + No thinking, no effort, no fallback -- but text and tools still work. + Sending the modern parameters to it is a 400, so this is the assertion + that the capability gating actually holds inside the runtime. + """ + tester = ChatCompletionTester( + LLMRequest( + request_id="legacy", + messages=[user("What is the weather in Taipei?")], + tools=[WEATHER_TOOL], + ) + ) + run_tester(tester, model=LEGACY_MODEL, prompt="Be brief.") + + assert ( + tester.final_status == StatusCode.OK + ), "the legacy model rejected the request -- capability gating failed" + assert tester.tool_calls() or tester.done() is not None + assert ( + not tester.reasoning() + ), "legacy models cannot think; reasoning events must not appear" diff --git a/docs/ai/L1/03_code_map.md b/docs/ai/L1/03_code_map.md index 0f6549e1c4..ab83a51aea 100644 --- a/docs/ai/L1/03_code_map.md +++ b/docs/ai/L1/03_code_map.md @@ -49,7 +49,7 @@ Other repo-root directories: `core/` (C runtime), `packages/` (example/core exte | --------- | ----- | ----------------------------------------------------------- | | ASR | 10+ | `deepgram_asr_python`, `azure_asr_python`, `aws_asr_python` | | TTS | 15+ | `deepgram_tts`, `elevenlabs_tts2_python`, `cartesia_tts` | -| LLM | 8+ | `openai_llm2_python`, `gemini_llm2_python`, `bedrock_llm_python` | +| LLM | 9+ | `openai_llm2_python`, `anthropic_llm2_python`, `gemini_llm2_python`, `bedrock_llm_python` | | Avatar | 5+ | `heygen_avatar_python`, `anam_avatar_python` | | Tools | 8+ | `bingsearch_tool_python`, `vision_tool_python` | | Transport | 3+ | `agora_rtc`, `websocket_server`, `http_server_python` |