feat: add smallest ai asr and tts extensions

ai_agents/agents/examples/voice-assistant/tenapp/manifest.json

@@ -167,9 +167,15 @@
     },
     {
       "path": "../../../ten_packages/extension/murf_tts_python"
+    },
+    {
+      "path": "../../../ten_packages/extension/smallest_asr_python"
+    },
+    {
+      "path": "../../../ten_packages/extension/smallest_tts_python"
     }
   ],
   "scripts": {
     "start": "scripts/start.sh"
   }
-}
+}

ai_agents/agents/examples/voice-assistant/tenapp/property.json

@@ -1379,6 +1379,205 @@
             }
           ]
         }
+      },
+      {
+        "name": "voice_assistant_smallest",
+        "auto_start": false,
+        "graph": {
+          "nodes": [
+            {
+              "type": "extension",
+              "name": "agora_rtc",
+              "addon": "agora_rtc",
+              "extension_group": "default",
+              "property": {
+                "app_id": "${env:AGORA_APP_ID}",
+                "app_certificate": "${env:AGORA_APP_CERTIFICATE|}",
+                "channel": "ten_agent_test",
+                "stream_id": 1234,
+                "remote_stream_id": 123,
+                "subscribe_audio": true,
+                "publish_audio": true,
+                "publish_data": true,
+                "enable_agora_asr": false
+              }
+            },
+            {
+              "type": "extension",
+              "name": "stt",
+              "addon": "smallest_asr_python",
+              "extension_group": "stt",
+              "property": {
+                "params": {
+                  "api_key": "${env:SMALLEST_API_KEY}",
+                  "model": "pulse",
+                  "language": "en",
+                  "sample_rate": 16000
+                }
+              }
+            },
+            {
+              "type": "extension",
+              "name": "llm",
+              "addon": "openai_llm2_python",
+              "extension_group": "chatgpt",
+              "property": {
+                "base_url": "https://api.openai.com/v1",
+                "api_key": "${env:OPENAI_API_KEY}",
+                "frequency_penalty": 0.9,
+                "model": "${env:OPENAI_MODEL}",
+                "max_tokens": 512,
+                "prompt": "",
+                "proxy_url": "${env:OPENAI_PROXY_URL|}",
+                "greeting": "TEN Agent connected. How can I help you today?",
+                "max_memory_length": 10
+              }
+            },
+            {
+              "type": "extension",
+              "name": "tts",
+              "addon": "smallest_tts_python",
+              "extension_group": "tts",
+              "property": {
+                "params": {
+                  "api_key": "${env:SMALLEST_API_KEY}",
+                  "model": "lightning_v3.1",
+                  "voice_id": "magnus",
+                  "sample_rate": 24000
+                }
+              }
+            },
+            {
+              "type": "extension",
+              "name": "main_control",
+              "addon": "main_python",
+              "extension_group": "control",
+              "property": {
+                "greeting": "TEN Agent connected. How can I help you today?"
+              }
+            },
+            {
+              "type": "extension",
+              "name": "message_collector",
+              "addon": "message_collector2",
+              "extension_group": "transcriber",
+              "property": {}
+            },
+            {
+              "type": "extension",
+              "name": "weatherapi_tool_python",
+              "addon": "weatherapi_tool_python",
+              "extension_group": "default",
+              "property": {
+                "api_key": "${env:WEATHERAPI_API_KEY|}"
+              }
+            },
+            {
+              "type": "extension",
+              "name": "streamid_adapter",
+              "addon": "streamid_adapter",
+              "property": {}
+            }
+          ],
+          "connections": [
+            {
+              "extension": "main_control",
+              "cmd": [
+                {
+                  "names": [
+                    "on_user_joined",
+                    "on_user_left"
+                  ],
+                  "source": [
+                    {
+                      "extension": "agora_rtc"
+                    }
+                  ]
+                },
+                {
+                  "names": [
+                    "tool_register"
+                  ],
+                  "source": [
+                    {
+                      "extension": "weatherapi_tool_python"
+                    }
+                  ]
+                }
+              ],
+              "data": [
+                {
+                  "name": "asr_result",
+                  "source": [
+                    {
+                      "extension": "stt"
+                    }
+                  ]
+                },
+                {
+                  "name": "tts_audio_start",
+                  "source": [
+                    {
+                      "extension": "tts"
+                    }
+                  ]
+                },
+                {
+                  "name": "tts_audio_end",
+                  "source": [
+                    {
+                      "extension": "tts"
+                    }
+                  ]
+                }
+              ]
+            },
+            {
+              "extension": "agora_rtc",
+              "audio_frame": [
+                {
+                  "name": "pcm_frame",
+                  "dest": [
+                    {
+                      "extension": "streamid_adapter"
+                    }
+                  ]
+                },
+                {
+                  "name": "pcm_frame",
+                  "source": [
+                    {
+                      "extension": "tts"
+                    }
+                  ]
+                }
+              ],
+              "data": [
+                {
+                  "name": "data",
+                  "source": [
+                    {
+                      "extension": "message_collector"
+                    }
+                  ]
+                }
+              ]
+            },
+            {
+              "extension": "streamid_adapter",
+              "audio_frame": [
+                {
+                  "name": "pcm_frame",
+                  "dest": [
+                    {
+                      "extension": "stt"
+                    }
+                  ]
+                }
+              ]
+            }
+          ]
+        }
       }
     ],
     "log": {

ai_agents/agents/integration_tests/asr_guarder/tests/test_multi_language.py

@@ -36,7 +36,9 @@
 MULTI_LANGUAGE_EXPECTED_LANGUAGE_ES = "es-ES"
 
 # Extensions that do not support Chinese and use Spanish as the second language
-_EXTENSIONS_USE_SPANISH = {"oracle_asr_python"}
+# (smallest: Chinese is region-gated on the vendor side; Spanish is
+# universally enabled)
+_EXTENSIONS_USE_SPANISH = {"oracle_asr_python", "smallest_asr_python"}
 
 
 RESULT_WAIT_TIMEOUT_SECS = 30

ai_agents/agents/ten_packages/extension/smallest_asr_python/README.md

@@ -0,0 +1,60 @@
+# Smallest AI ASR Extension (Pulse)
+
+Real-time speech-to-text extension for TEN Framework using
+[Smallest AI Pulse](https://docs.smallest.ai/waves/documentation/speech-to-text-pulse/overview)
+over the live WebSocket API (64 ms time-to-first-transcript, 38 languages
+with auto-detect).
+
+## Configuration
+
+All settings live under `params` in `property.json`:
+
+```json
+{
+  "params": {
+    "api_key": "${env:SMALLEST_API_KEY|}",
+    "model": "pulse",
+    "language": "en",
+    "sample_rate": 16000
+  }
+}
+```
+
+| Param | Default | Description |
+| --- | --- | --- |
+| `api_key` | — | Smallest AI API key (get one at [app.smallest.ai](https://app.smallest.ai)). Required. |
+| `url` | `wss://api.smallest.ai/waves/v1/stt/live` | WebSocket endpoint. |
+| `model` | `pulse` | Streaming model. Only `pulse` supports the live endpoint. |
+| `language` | `en` | ISO language code (e.g. `en`, `hi`, `es`). |
+| `sample_rate` | `16000` | Input PCM sample rate in Hz. |
+
+Any additional keys in `params` (e.g. `word_timestamps`, `eou_timeout`,
+`punctuate`) are forwarded verbatim as query parameters to the live
+endpoint — see the
+[feature docs](https://docs.smallest.ai/waves/documentation/speech-to-text-pulse/overview)
+for the full list.
+
+## Environment Variables
+
+- `SMALLEST_API_KEY` — Smallest AI API key.
+
+## Behavior
+
+- Audio is streamed as raw PCM16 binary frames.
+- Interim transcripts are sent with `final=false`; Pulse finalizes segments
+  automatically and on explicit `asr_finalize` signals (mapped to Pulse's
+  `{"type": "finalize"}` control message).
+- Transient WebSocket failures are retried with exponential backoff
+  (5 attempts) before a fatal error is reported.
+
+## Testing
+
+```bash
+task asr-guarder-test EXTENSION=smallest_asr_python
+```
+
+Extension-local tests (mocked vendor, no API key needed):
+
+```bash
+cd tests && ./bin/start
+```

ai_agents/agents/ten_packages/extension/smallest_asr_python/__init__.py

@@ -0,0 +1 @@
+from . import addon

ai_agents/agents/ten_packages/extension/smallest_asr_python/addon.py

@@ -0,0 +1,15 @@
+from ten_runtime import (
+    Addon,
+    register_addon_as_extension,
+    TenEnv,
+)
+
+from .extension import SmallestASRExtension
+
+
+@register_addon_as_extension("smallest_asr_python")
+class SmallestASRExtensionAddon(Addon):
+    def on_create_instance(self, ten: TenEnv, addon_name: str, context) -> None:
+
+        ten.log_info("on_create_instance")
+        ten.on_create_instance_done(SmallestASRExtension(addon_name), context)