diff --git a/CHANGES.md b/CHANGES.md
index 086214f..9cdba87 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,25 @@
# Changelog
+## [2026-09-11]
+
+### Changed
+- **Asynchronous LLM Client & Non-Blocking Scheduling (`src/rag/llm.py`)**:
+ - Removed in-process `threading.Lock()` and blocking `time.sleep()` in `LLMClient` to eliminate request serialization bottleneck across worker threads.
+ - Added `AsyncOpenAI` client for non-blocking asynchronous model invocations.
+ - Added `StreamJsonExtractor` state machine to parse and extract JSON response text deltas progressively in real time.
+ - Added `complete_async()` and `complete_stream()` methods with non-blocking rate limiting and async fallback cascades.
+- **Asynchronous Orchestrator Turn Pipeline (`src/agent/orchestrator.py`)**:
+ - Added `process_turn_async` running CPU/IO retrieval non-blockingly via `asyncio.to_thread`.
+ - Added `process_turn_stream` async generator emitting structured SSE events (`start`, `delta`, `replace`, `citations`, `done`) with safety guardrail evaluation and post-stream citation validation.
+- **Server-Sent Events (SSE) Streaming API (`backend/main.py`)**:
+ - Added `POST /api/chat/stream` endpoint returning `StreamingResponse(..., media_type="text/event-stream")`.
+ - Preserved synchronous `POST /api/chat` calling `orchestrator.process_turn` for backward compatibility with synchronous callers and test suites.
+- **Frontend Real-Time Token Streaming & Animated Feedback (`frontend/src/api.js`, `frontend/src/App.jsx`, `frontend/src/App.css`)**:
+ - Added `streamMessage` using `ReadableStreamDefaultReader` supporting CRLF/LF packet parsing and trailing buffer flushes.
+ - Fixed React 18 state batching race in `App.jsx` using atomic message existence checks.
+ - Added real-time pipeline status telemetry (`Searching OrbitMesh documentation...` -> `Generating diagnostic response...`) with an animated 3-dot pulse indicator and streaming cursor.
+
+
## [2026-09-04]
### Fixed
diff --git a/backend/main.py b/backend/main.py
index 5ea5258..74ceed2 100644
--- a/backend/main.py
+++ b/backend/main.py
@@ -4,7 +4,9 @@
from pathlib import Path
from typing import List, Optional
+import json
from fastapi import Depends, FastAPI, HTTPException, Security, status
+from fastapi.responses import StreamingResponse
from fastapi.middleware.cors import CORSMiddleware
from fastapi.security.api_key import APIKeyHeader
from pydantic import BaseModel, Field
@@ -144,6 +146,34 @@ def process_chat(request: ChatRequest, _: Optional[str] = Depends(verify_api_key
)
+@app.post("/api/chat/stream")
+async def process_chat_stream(request: ChatRequest, _: Optional[str] = Depends(verify_api_key)):
+ session_id = request.session_id.strip() if request.session_id else None
+ if not session_id:
+ session_id = f"web-{uuid.uuid4().hex[:8]}"
+
+ async def event_generator():
+ try:
+ async for event_packet in orchestrator.process_turn_stream(session_id, request.message):
+ event_name = event_packet.get("event", "message")
+ data_payload = json.dumps(event_packet.get("data", {}))
+ yield f"event: {event_name}\ndata: {data_payload}\n\n"
+ except Exception as e:
+ logger.error(f"Stream generation failed for session '{session_id}': {e}", exc_info=True)
+ err_payload = json.dumps({"error": "Stream generation failed"})
+ yield f"event: error\ndata: {err_payload}\n\n"
+
+ return StreamingResponse(
+ event_generator(),
+ media_type="text/event-stream",
+ headers={
+ "Cache-Control": "no-cache",
+ "Connection": "keep-alive",
+ "X-Accel-Buffering": "no",
+ }
+ )
+
+
if __name__ == "__main__":
import uvicorn
diff --git a/frontend/src/App.css b/frontend/src/App.css
index 85a529c..d021bde 100755
--- a/frontend/src/App.css
+++ b/frontend/src/App.css
@@ -565,3 +565,65 @@
padding-right: 16px;
}
}
+
+/* Streaming token cursor */
+.streaming-cursor {
+ display: inline-block;
+ width: 2px;
+ height: 1.1em;
+ background-color: #3182ce;
+ margin-left: 2px;
+ vertical-align: text-bottom;
+ animation: cursor-blink 0.8s infinite;
+}
+
+@keyframes cursor-blink {
+ 0%, 100% { opacity: 1; }
+ 50% { opacity: 0; }
+}
+
+/* Loading status and animated typing indicator */
+.loading-status-wrap {
+ display: flex;
+ align-items: center;
+ gap: 10px;
+}
+
+.typing-indicator {
+ display: inline-flex;
+ align-items: center;
+ gap: 4px;
+}
+
+.typing-indicator .dot {
+ width: 6px;
+ height: 6px;
+ background-color: #3182ce;
+ border-radius: 50%;
+ animation: dot-pulse 1.4s infinite ease-in-out both;
+}
+
+.typing-indicator .dot:nth-child(1) {
+ animation-delay: -0.32s;
+}
+
+.typing-indicator .dot:nth-child(2) {
+ animation-delay: -0.16s;
+}
+
+@keyframes dot-pulse {
+ 0%, 80%, 100% {
+ transform: scale(0.6);
+ opacity: 0.4;
+ }
+ 40% {
+ transform: scale(1);
+ opacity: 1;
+ }
+}
+
+.loading-text {
+ font-size: 0.88rem;
+ color: #4a5568;
+ font-style: italic;
+}
diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx
index 9051fdf..98f0730 100755
--- a/frontend/src/App.jsx
+++ b/frontend/src/App.jsx
@@ -1,5 +1,5 @@
import { useState, useRef, useEffect } from 'react';
-import { sendMessage } from './api';
+import { sendMessage, streamMessage } from './api';
import './App.css';
function generateSessionId() {
@@ -91,6 +91,7 @@ export default function App() {
const [isSidebarOpen, setIsSidebarOpen] = useState(true);
const [input, setInput] = useState('');
const [loading, setLoading] = useState(false);
+ const [agentStatus, setAgentStatus] = useState('Assistant is analyzing...');
const [error, setError] = useState(null);
const messagesEndRef = useRef(null);
@@ -184,54 +185,124 @@ export default function App() {
setInput('');
setLoading(true);
+ setAgentStatus('Assistant is analyzing...');
setError(null);
+ const assistantMsgId = 'assistant-' + Date.now();
+
try {
- const data = await sendMessage(activeSessionId, textToSend);
-
- const assistantMessage = {
- id: 'assistant-' + Date.now(),
- sender: 'assistant',
- text: data.response || 'No response returned.',
- citations: data.citations || [],
- action: data.action || 'instruct',
- timestamp: new Date().toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }),
- };
-
- setSessions((prev) =>
- prev.map((s) => {
- if (s.id === activeSessionId) {
- return {
- ...s,
- messages: [...s.messages, assistantMessage],
- };
- }
- return s;
- })
- );
- } catch (err) {
- setError(err.message || 'Failed to send message');
- setSessions((prev) =>
- prev.map((s) => {
- if (s.id === activeSessionId) {
- return {
- ...s,
- messages: [
- ...s.messages,
- {
- id: 'error-' + Date.now(),
+ await streamMessage(activeSessionId, textToSend, {
+ onStatus: (statusText) => {
+ setAgentStatus(statusText);
+ },
+ onChunk: (delta, isReplace) => {
+ setLoading(false);
+ setSessions((prev) =>
+ prev.map((s) => {
+ if (s.id !== activeSessionId) return s;
+ const exists = s.messages.some((m) => m.id === assistantMsgId);
+ if (!exists) {
+ const newMsg = {
+ id: assistantMsgId,
sender: 'assistant',
- text: 'Error: Unable to connect to backend server. Ensure backend is running.',
+ text: delta,
citations: [],
- action: 'error',
+ action: null,
+ timestamp: new Date().toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }),
+ isStreaming: true,
+ };
+ return { ...s, messages: [...s.messages, newMsg] };
+ }
+ return {
+ ...s,
+ messages: s.messages.map((m) => {
+ if (m.id !== assistantMsgId) return m;
+ return {
+ ...m,
+ text: isReplace ? delta : m.text + delta,
+ };
+ }),
+ };
+ })
+ );
+ },
+ onCitations: (citations) => {
+ setSessions((prev) =>
+ prev.map((s) => {
+ if (s.id !== activeSessionId) return s;
+ return {
+ ...s,
+ messages: s.messages.map((m) => {
+ if (m.id !== assistantMsgId) return m;
+ return { ...m, citations: citations || [] };
+ }),
+ };
+ })
+ );
+ },
+ onDone: (doneData) => {
+ setLoading(false);
+ setSessions((prev) =>
+ prev.map((s) => {
+ if (s.id !== activeSessionId) return s;
+ const exists = s.messages.some((m) => m.id === assistantMsgId);
+ if (!exists) {
+ const newMsg = {
+ id: assistantMsgId,
+ sender: 'assistant',
+ text: doneData.response || '',
+ citations: doneData.citations || [],
+ action: doneData.action || 'instruct',
timestamp: new Date().toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }),
- },
- ],
- };
- }
- return s;
- })
- );
+ isStreaming: false,
+ };
+ return { ...s, messages: [...s.messages, newMsg] };
+ }
+ return {
+ ...s,
+ messages: s.messages.map((m) => {
+ if (m.id !== assistantMsgId) return m;
+ return {
+ ...m,
+ text: doneData.response || m.text,
+ action: doneData.action || 'instruct',
+ citations: (m.citations && m.citations.length > 0) ? m.citations : (doneData.citations || []),
+ isStreaming: false,
+ };
+ }),
+ };
+ })
+ );
+ },
+ onError: (err) => {
+ setError(err.message || 'Stream connection error');
+ setLoading(false);
+ setSessions((prev) =>
+ prev.map((s) => {
+ if (s.id !== activeSessionId) return s;
+ const exists = s.messages.some((m) => m.id === assistantMsgId);
+ if (exists) return s;
+ return {
+ ...s,
+ messages: [
+ ...s.messages,
+ {
+ id: 'error-' + Date.now(),
+ sender: 'assistant',
+ text: 'Error: Unable to connect to backend server. Ensure backend is running.',
+ citations: [],
+ action: 'error',
+ timestamp: new Date().toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }),
+ },
+ ],
+ };
+ })
+ );
+ },
+ });
+ } catch (err) {
+ setError(err.message || 'Failed to send message');
+ setLoading(false);
} finally {
setLoading(false);
}
@@ -364,7 +435,10 @@ export default function App() {
{msg.timestamp}
-
@@ -393,7 +467,14 @@ export default function App() {
{loading && (
-
Assistant is analyzing...
+
+
+
+
+
+
+ {agentStatus}
+
)}
diff --git a/frontend/src/api.js b/frontend/src/api.js
index 2622108..2639c0c 100755
--- a/frontend/src/api.js
+++ b/frontend/src/api.js
@@ -27,3 +27,92 @@ export async function sendMessage(sessionId, message) {
return await response.json();
}
+
+function processPacket(packet, callbacks) {
+ if (!packet || !packet.trim()) return;
+ const { onChunk, onCitations, onDone, onError, onStatus } = callbacks;
+ let event = 'message';
+ let data = '';
+
+ for (const line of packet.split(/\r?\n/)) {
+ if (line.startsWith('event: ')) {
+ event = line.substring(7).trim();
+ } else if (line.startsWith('data: ')) {
+ data = line.substring(6).trim();
+ }
+ }
+
+ if (!data) return;
+ try {
+ const parsed = JSON.parse(data);
+ if (event === 'status' && onStatus) {
+ onStatus(parsed.status);
+ } else if (event === 'delta' && onChunk) {
+ onChunk(parsed.delta);
+ } else if (event === 'replace' && onChunk) {
+ onChunk(parsed.response, true);
+ } else if (event === 'citations' && onCitations) {
+ onCitations(parsed.citations);
+ } else if (event === 'done' && onDone) {
+ onDone(parsed);
+ } else if (event === 'error' && onError) {
+ onError(new Error(parsed.error || 'Stream error'));
+ }
+ } catch (e) {
+ console.warn('Failed to parse SSE packet:', data, e);
+ }
+}
+
+export async function streamMessage(sessionId, message, callbacks) {
+ const url = `${API_BASE_URL}/api/chat/stream`;
+ const headers = {
+ 'Content-Type': 'application/json',
+ };
+
+ if (API_KEY) {
+ headers['X-API-Key'] = API_KEY;
+ }
+
+ try {
+ const response = await fetch(url, {
+ method: 'POST',
+ headers,
+ body: JSON.stringify({
+ session_id: sessionId,
+ message: message,
+ }),
+ });
+
+ if (!response.ok) {
+ const errorText = await response.text();
+ throw new Error(`API error (${response.status}): ${errorText || response.statusText}`);
+ }
+
+ const reader = response.body.getReader();
+ const decoder = new TextDecoder('utf-8');
+ let buffer = '';
+
+ while (true) {
+ const { done, value } = await reader.read();
+ if (done) break;
+
+ buffer += decoder.decode(value, { stream: true });
+ const packets = buffer.split(/\r?\n\r?\n/);
+ buffer = packets.pop() || '';
+
+ for (const packet of packets) {
+ processPacket(packet, callbacks);
+ }
+ }
+
+ if (buffer.trim()) {
+ processPacket(buffer, callbacks);
+ }
+ } catch (err) {
+ if (callbacks.onError) {
+ callbacks.onError(err);
+ } else {
+ throw err;
+ }
+ }
+}
diff --git a/src/agent/orchestrator.py b/src/agent/orchestrator.py
index 65d3ab5..27abce3 100755
--- a/src/agent/orchestrator.py
+++ b/src/agent/orchestrator.py
@@ -1,7 +1,8 @@
from __future__ import annotations
import re
import threading
-from typing import Optional
+import asyncio
+from typing import Optional, AsyncIterator
from src.core.models import ResponseEnvelope, ActionEnum, Citation
from src.guardrails.input_guard import InputGuardrail
@@ -174,4 +175,311 @@ def process_turn(self, session_id: str, user_message: str) -> ResponseEnvelope:
session.is_escalated = True
self.session_manager.record_turn(session, clean_msg, final_envelope.response, step_executed=step_executed)
- return final_envelope
\ No newline at end of file
+ return final_envelope
+
+ async def process_turn_async(self, session_id: str, user_message: str) -> ResponseEnvelope:
+ self.last_retrieved_chunks = []
+ self.last_raw_envelope = None
+
+ session = self.session_manager.get_or_create(session_id)
+ session.turns_count += 1
+
+ isolated_msg, is_safe, clean_msg = InputGuardrail.sanitize_and_inspect(user_message)
+ msg_lower = clean_msg.lower().strip()
+
+ hazard_envelope = OutputGuardrail.check_hardware_safety(clean_msg, "")
+ if hazard_envelope:
+ session.is_escalated = True
+ self.session_manager.record_turn(session, clean_msg, hazard_envelope.response, step_executed="hazard_escalation")
+ return hazard_envelope
+
+ if not is_safe:
+ injection_envelope = ResponseEnvelope(
+ response="I can only assist with official OrbitMesh device troubleshooting and network configuration. How can I help with your OrbitMesh system?",
+ citations=[],
+ action=ActionEnum.ASK
+ )
+ self.session_manager.record_turn(session, clean_msg, injection_envelope.response)
+ return injection_envelope
+
+ if re.search(r"\b(r5\s*pro|n5\s*pro|pro\s+gateway|pro\s+node|orbitmesh\s+pro)\b", msg_lower):
+ session.identified_model = "OrbitMesh Pro"
+ elif re.search(r"\b(r1|r1\s*router|main\s*router)\b", msg_lower):
+ session.identified_model = "OrbitMesh R1"
+ elif re.search(r"\b(n1|n1\s*node|satellite\s*node|satellite)\b", msg_lower):
+ session.identified_model = "OrbitMesh N1"
+
+ is_yes = bool(re.search(r"\b(yes|yep|yeah|proceed|confirm|confirmed|sure|ok|okay|go\s+ahead|do\s+it)\b", msg_lower))
+ is_no = bool(re.search(r"\b(no|cancel|stop|abort|don'?t|do\s+not|nevermind|never\s+mind|skip)\b", msg_lower))
+
+ if session.pending_confirmation == "factory_reset":
+ if is_no and not is_yes:
+ session.pending_confirmation = None
+ restart_text = OutputGuardrail.get_section_text("reset-recovery-guide", "Restart ? no configuration loss")
+ first_sent = restart_text.split(". ")[0] if restart_text else "Disconnect the unit's power cable, wait 10 seconds, and reconnect it."
+ alt_resp = f"Understood, skipping factory reset. As a non-destructive alternative: {first_sent} Would you like to try this?"
+ citation = Citation(source_id="reset-recovery-guide", locator="Restart ? no configuration loss")
+ envelope = ResponseEnvelope(response=alt_resp, citations=[citation], action=ActionEnum.ASK)
+ self.session_manager.record_turn(session, clean_msg, envelope.response)
+ return envelope
+ elif is_yes and not is_no:
+ session.pending_confirmation = None
+ session.confirmed_facts["factory_reset_confirmed"] = True
+ model = session.identified_model or "N1"
+ if "pro" in model.lower():
+ citation = Citation(source_id="pro-quick-start-guide", locator="Factory reset")
+ pro_reset = OutputGuardrail.get_section_text("pro-quick-start-guide", "Factory reset")
+ step = pro_reset if pro_reset else "Hold the recessed reset pin for 10 seconds until the LED flashes blue, then release. The node returns to an unclaimed state."
+ else:
+ citation = Citation(source_id="reset-recovery-guide", locator="Factory reset ? erases configuration")
+ std_reset = OutputGuardrail.get_section_text("reset-recovery-guide", "Factory reset ? erases configuration")
+ paras = [p for p in std_reset.split("\n\n") if "only after confirmation" in p.lower()]
+ step = paras[0] if paras else "With the unit powered, hold reset for at least 15 seconds until the LED flashes red, then release. Keep power connected while it recovers."
+ envelope = ResponseEnvelope(response=step, citations=[citation], action=ActionEnum.INSTRUCT)
+ self.session_manager.record_turn(session, clean_msg, envelope.response, step_executed="factory_reset")
+ return envelope
+ elif any(w in msg_lower for w in ["anything else", "try before", "alternative", "before wiping", "before resetting"]):
+ session.pending_confirmation = None
+ alt_resp = "Before performing a factory reset, try power cycling the device: disconnect power for 10 seconds and reconnect. If it is an N1 node, you may also attempt a pairing reset by holding the reset button for 5?7 seconds until the LED pulses blue."
+ citation = Citation(source_id="reset-recovery-guide", locator="Restart ? no configuration loss")
+ envelope = ResponseEnvelope(response=alt_resp, citations=[citation], action=ActionEnum.ASK)
+ self.session_manager.record_turn(session, clean_msg, envelope.response)
+ return envelope
+ else:
+ session.pending_confirmation = None
+
+ model_str = session.identified_model or ""
+ product_line_filter = "Pro" if "pro" in model_str.lower() else ("Standard" if any(x in model_str.lower() for x in ["r1", "n1"]) else None)
+
+ self.last_retrieved_chunks = []
+ retrieved_chunks = await asyncio.to_thread(
+ self.retriever.retrieve,
+ query=clean_msg,
+ top_k=4,
+ product_line=product_line_filter,
+ include_archived=("archive" in msg_lower or "superseded" in msg_lower)
+ )
+ self.last_retrieved_chunks = retrieved_chunks
+
+ proposed_envelope = await self.llm.complete_async(clean_msg, session, retrieved_chunks)
+ self.last_raw_envelope = proposed_envelope
+
+ hardware_check = OutputGuardrail.check_hardware_safety(clean_msg, proposed_envelope.response)
+ if hardware_check:
+ final_envelope = hardware_check
+ else:
+ confirmed_reset = session.confirmed_facts.get("factory_reset_confirmed", False)
+ final_envelope = OutputGuardrail.check_factory_reset_safety(clean_msg, proposed_envelope, confirmed_reset)
+ if final_envelope.response.startswith("Warning: A factory reset") or (
+ any(w in clean_msg.lower() for w in ["factory reset", "full reset", "reset everything"]) and final_envelope.action == ActionEnum.ASK
+ ):
+ session.pending_confirmation = "factory_reset"
+ final_envelope = OutputGuardrail.check_sensitive_info_solicitation(final_envelope)
+
+ final_envelope.citations = OutputGuardrail.validate_and_repair_citations(
+ final_envelope.citations,
+ retrieved_chunks
+ )
+
+ step_executed = None
+ if final_envelope.action == ActionEnum.INSTRUCT:
+ resp_l = final_envelope.response.lower()
+ if "factory reset" in resp_l or "hold the reset" in resp_l or "reset pin" in resp_l:
+ step_executed = "factory_reset"
+ elif "ethernet" in resp_l or "cable" in resp_l:
+ step_executed = "cable_checked"
+ elif "power cycle" in resp_l or "unplug" in resp_l or "restart" in resp_l:
+ step_executed = "power_cycled"
+ elif "distance" in resp_l or "closer" in resp_l:
+ step_executed = "distance_checked"
+ elif "channel" in resp_l or "app" in resp_l:
+ step_executed = "channel_optimized"
+ else:
+ step_executed = "instruction_step"
+
+ if final_envelope.action == ActionEnum.RESOLVED:
+ session.is_resolved = True
+ elif final_envelope.action == ActionEnum.ESCALATE:
+ session.is_escalated = True
+
+ self.session_manager.record_turn(session, clean_msg, final_envelope.response, step_executed=step_executed)
+ return final_envelope
+
+ async def process_turn_stream(
+ self, session_id: str, user_message: str
+ ) -> AsyncIterator[dict]:
+ self.last_retrieved_chunks = []
+ self.last_raw_envelope = None
+
+ session = self.session_manager.get_or_create(session_id)
+ session.turns_count += 1
+ yield {"event": "start", "data": {"session_id": session_id}}
+
+ isolated_msg, is_safe, clean_msg = InputGuardrail.sanitize_and_inspect(user_message)
+ msg_lower = clean_msg.lower().strip()
+
+ hazard_envelope = OutputGuardrail.check_hardware_safety(clean_msg, "")
+ if hazard_envelope:
+ session.is_escalated = True
+ self.session_manager.record_turn(session, clean_msg, hazard_envelope.response, step_executed="hazard_escalation")
+ yield {"event": "delta", "data": {"delta": hazard_envelope.response}}
+ yield {"event": "citations", "data": {"citations": [{"source_id": c.source_id, "locator": c.locator} for c in hazard_envelope.citations]}}
+ action_str = hazard_envelope.action.value if hasattr(hazard_envelope.action, "value") else str(hazard_envelope.action)
+ yield {"event": "done", "data": {"session_id": session_id, "action": action_str, "response": hazard_envelope.response}}
+ return
+
+ if not is_safe:
+ injection_envelope = ResponseEnvelope(
+ response="I can only assist with official OrbitMesh device troubleshooting and network configuration. How can I help with your OrbitMesh system?",
+ citations=[],
+ action=ActionEnum.ASK
+ )
+ self.session_manager.record_turn(session, clean_msg, injection_envelope.response)
+ yield {"event": "delta", "data": {"delta": injection_envelope.response}}
+ yield {"event": "citations", "data": {"citations": []}}
+ yield {"event": "done", "data": {"session_id": session_id, "action": "ask", "response": injection_envelope.response}}
+ return
+
+ if re.search(r"\b(r5\s*pro|n5\s*pro|pro\s+gateway|pro\s+node|orbitmesh\s+pro)\b", msg_lower):
+ session.identified_model = "OrbitMesh Pro"
+ elif re.search(r"\b(r1|r1\s*router|main\s*router)\b", msg_lower):
+ session.identified_model = "OrbitMesh R1"
+ elif re.search(r"\b(n1|n1\s*node|satellite\s*node|satellite)\b", msg_lower):
+ session.identified_model = "OrbitMesh N1"
+
+ is_yes = bool(re.search(r"\b(yes|yep|yeah|proceed|confirm|confirmed|sure|ok|okay|go\s+ahead|do\s+it)\b", msg_lower))
+ is_no = bool(re.search(r"\b(no|cancel|stop|abort|don'?t|do\s+not|nevermind|never\s+mind|skip)\b", msg_lower))
+
+ if session.pending_confirmation == "factory_reset":
+ if is_no and not is_yes:
+ session.pending_confirmation = None
+ restart_text = OutputGuardrail.get_section_text("reset-recovery-guide", "Restart ? no configuration loss")
+ first_sent = restart_text.split(". ")[0] if restart_text else "Disconnect the unit's power cable, wait 10 seconds, and reconnect it."
+ alt_resp = f"Understood, skipping factory reset. As a non-destructive alternative: {first_sent} Would you like to try this?"
+ citation = Citation(source_id="reset-recovery-guide", locator="Restart ? no configuration loss")
+ envelope = ResponseEnvelope(response=alt_resp, citations=[citation], action=ActionEnum.ASK)
+ self.session_manager.record_turn(session, clean_msg, envelope.response)
+ yield {"event": "delta", "data": {"delta": alt_resp}}
+ yield {"event": "citations", "data": {"citations": [{"source_id": citation.source_id, "locator": citation.locator}]}}
+ yield {"event": "done", "data": {"session_id": session_id, "action": "ask", "response": alt_resp}}
+ return
+ elif is_yes and not is_no:
+ session.pending_confirmation = None
+ session.confirmed_facts["factory_reset_confirmed"] = True
+ model = session.identified_model or "N1"
+ if "pro" in model.lower():
+ citation = Citation(source_id="pro-quick-start-guide", locator="Factory reset")
+ pro_reset = OutputGuardrail.get_section_text("pro-quick-start-guide", "Factory reset")
+ step = pro_reset if pro_reset else "Hold the recessed reset pin for 10 seconds until the LED flashes blue, then release. The node returns to an unclaimed state."
+ else:
+ citation = Citation(source_id="reset-recovery-guide", locator="Factory reset ? erases configuration")
+ std_reset = OutputGuardrail.get_section_text("reset-recovery-guide", "Factory reset ? erases configuration")
+ paras = [p for p in std_reset.split("\n\n") if "only after confirmation" in p.lower()]
+ step = paras[0] if paras else "With the unit powered, hold reset for at least 15 seconds until the LED flashes red, then release. Keep power connected while it recovers."
+ envelope = ResponseEnvelope(response=step, citations=[citation], action=ActionEnum.INSTRUCT)
+ self.session_manager.record_turn(session, clean_msg, envelope.response, step_executed="factory_reset")
+ yield {"event": "delta", "data": {"delta": step}}
+ yield {"event": "citations", "data": {"citations": [{"source_id": citation.source_id, "locator": citation.locator}]}}
+ yield {"event": "done", "data": {"session_id": session_id, "action": "instruct", "response": step}}
+ return
+ elif any(w in msg_lower for w in ["anything else", "try before", "alternative", "before wiping", "before resetting"]):
+ session.pending_confirmation = None
+ alt_resp = "Before performing a factory reset, try power cycling the device: disconnect power for 10 seconds and reconnect. If it is an N1 node, you may also attempt a pairing reset by holding the reset button for 5?7 seconds until the LED pulses blue."
+ citation = Citation(source_id="reset-recovery-guide", locator="Restart ? no configuration loss")
+ envelope = ResponseEnvelope(response=alt_resp, citations=[citation], action=ActionEnum.ASK)
+ self.session_manager.record_turn(session, clean_msg, envelope.response)
+ yield {"event": "delta", "data": {"delta": alt_resp}}
+ yield {"event": "citations", "data": {"citations": [{"source_id": citation.source_id, "locator": citation.locator}]}}
+ yield {"event": "done", "data": {"session_id": session_id, "action": "ask", "response": alt_resp}}
+ return
+ else:
+ session.pending_confirmation = None
+
+ model_str = session.identified_model or ""
+ product_line_filter = "Pro" if "pro" in model_str.lower() else ("Standard" if any(x in model_str.lower() for x in ["r1", "n1"]) else None)
+
+ yield {"event": "status", "data": {"status": "Searching OrbitMesh documentation..."}}
+
+ self.last_retrieved_chunks = []
+ retrieved_chunks = await asyncio.to_thread(
+ self.retriever.retrieve,
+ query=clean_msg,
+ top_k=4,
+ product_line=product_line_filter,
+ include_archived=("archive" in msg_lower or "superseded" in msg_lower)
+ )
+ self.last_retrieved_chunks = retrieved_chunks
+
+ yield {"event": "status", "data": {"status": "Generating diagnostic response..."}}
+
+ streamed_pieces = []
+ final_envelope_candidate = None
+ async for delta, env in self.llm.complete_stream(clean_msg, session, retrieved_chunks):
+ if delta:
+ streamed_pieces.append(delta)
+ yield {"event": "delta", "data": {"delta": delta}}
+ if env:
+ final_envelope_candidate = env
+
+ if final_envelope_candidate is None:
+ final_envelope_candidate = ResponseEnvelope(
+ response="".join(streamed_pieces).strip(),
+ citations=[],
+ action=ActionEnum.ASK
+ )
+
+ self.last_raw_envelope = final_envelope_candidate
+
+ hardware_check = OutputGuardrail.check_hardware_safety(clean_msg, final_envelope_candidate.response)
+ if hardware_check:
+ final_envelope = hardware_check
+ else:
+ confirmed_reset = session.confirmed_facts.get("factory_reset_confirmed", False)
+ final_envelope = OutputGuardrail.check_factory_reset_safety(clean_msg, final_envelope_candidate, confirmed_reset)
+ if final_envelope.response.startswith("Warning: A factory reset") or (
+ any(w in clean_msg.lower() for w in ["factory reset", "full reset", "reset everything"]) and final_envelope.action == ActionEnum.ASK
+ ):
+ session.pending_confirmation = "factory_reset"
+ final_envelope = OutputGuardrail.check_sensitive_info_solicitation(final_envelope)
+
+ final_envelope.citations = OutputGuardrail.validate_and_repair_citations(
+ final_envelope.citations,
+ retrieved_chunks
+ )
+
+ streamed_full_text = "".join(streamed_pieces).strip()
+ if final_envelope.response.strip() != streamed_full_text and streamed_full_text:
+ yield {"event": "replace", "data": {"response": final_envelope.response}}
+
+ step_executed = None
+ if final_envelope.action == ActionEnum.INSTRUCT:
+ resp_l = final_envelope.response.lower()
+ if "factory reset" in resp_l or "hold the reset" in resp_l or "reset pin" in resp_l:
+ step_executed = "factory_reset"
+ elif "ethernet" in resp_l or "cable" in resp_l:
+ step_executed = "cable_checked"
+ elif "power cycle" in resp_l or "unplug" in resp_l or "restart" in resp_l:
+ step_executed = "power_cycled"
+ elif "distance" in resp_l or "closer" in resp_l:
+ step_executed = "distance_checked"
+ elif "channel" in resp_l or "app" in resp_l:
+ step_executed = "channel_optimized"
+ else:
+ step_executed = "instruction_step"
+
+ if final_envelope.action == ActionEnum.RESOLVED:
+ session.is_resolved = True
+ elif final_envelope.action == ActionEnum.ESCALATE:
+ session.is_escalated = True
+
+ self.session_manager.record_turn(session, clean_msg, final_envelope.response, step_executed=step_executed)
+
+ citations_list = [{"source_id": c.source_id, "locator": c.locator} for c in final_envelope.citations]
+ yield {"event": "citations", "data": {"citations": citations_list}}
+
+ action_str = final_envelope.action.value if hasattr(final_envelope.action, "value") else str(final_envelope.action)
+ yield {"event": "done", "data": {
+ "session_id": session_id,
+ "action": action_str,
+ "response": final_envelope.response
+ }}
diff --git a/src/core/config.py b/src/core/config.py
index 176a564..da9bcd7 100755
--- a/src/core/config.py
+++ b/src/core/config.py
@@ -43,8 +43,7 @@
OPENROUTER_MODEL = os.getenv('OPENROUTER_MODEL', 'nvidia/nemotron-3.5-lightning:free')
OPENROUTER_FALLBACK_MODELS = [
OPENROUTER_MODEL,
- 'liquid/lfm-2.5-2.6b:free',
- 'z-ai/glm-5.2:free',
+ 'nvidia/nemotron-3-super-120b-a12b:free',
'nvidia/nemotron-3-ultra-550b-a55b:free',
'nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free',
diff --git a/src/rag/llm.py b/src/rag/llm.py
index 07b96fa..628780f 100755
--- a/src/rag/llm.py
+++ b/src/rag/llm.py
@@ -5,7 +5,9 @@
import threading
import time
-from openai import OpenAI
+import asyncio
+from typing import AsyncIterator
+from openai import OpenAI, AsyncOpenAI
from src.core.config import (
OPENROUTER_API_KEY,
OPENROUTER_FALLBACK_MODELS,
@@ -192,12 +194,68 @@ def generate_response(
)
+
+class StreamJsonExtractor:
+ """Progressively extracts the 'response' string from an LLM stream returning JSON."""
+ def __init__(self):
+ self.buffer = ""
+ self.in_response = False
+ self.done_response = False
+
+ def feed(self, chunk: str) -> str:
+ if self.done_response:
+ return ""
+ self.buffer += chunk
+ if not self.in_response:
+ match = re.search(r'"response"\s*:\s*"', self.buffer)
+ if match:
+ self.in_response = True
+ self.buffer = self.buffer[match.end():]
+ else:
+ return ""
+
+ out = []
+ i = 0
+ n = len(self.buffer)
+ while i < n:
+ c = self.buffer[i]
+ if c == "\\":
+ if i + 1 < n:
+ nxt = self.buffer[i + 1]
+ if nxt == "n":
+ out.append("\n")
+ elif nxt == '"':
+ out.append('"')
+ elif nxt == "\\":
+ out.append("\\")
+ elif nxt == "t":
+ out.append("\t")
+ elif nxt == "r":
+ out.append("\r")
+ else:
+ out.append(nxt)
+ i += 2
+ continue
+ else:
+ break
+ elif c == '"':
+ self.done_response = True
+ self.buffer = self.buffer[i + 1:]
+ break
+ else:
+ out.append(c)
+ i += 1
+
+ if not self.done_response:
+ self.buffer = self.buffer[i:]
+ return "".join(out)
+
class LLMClient:
def __init__(self):
# "openrouter" is the legacy config name for live mode.
self.mode = "live" if LLM_MODE == "openrouter" else LLM_MODE
self.client: OpenAI | None = None
- self._rate_limit_lock = threading.Lock()
+ self.async_client: AsyncOpenAI | None = None
self.last_call_time = 0.0
self.rate_limit_delay = LLM_RATE_LIMIT_DELAY
self.fallback_models = OPENROUTER_FALLBACK_MODELS
@@ -217,7 +275,12 @@ def __init__(self):
base_url="https://openrouter.ai/api/v1",
timeout=self.request_timeout
)
- logger.info("Initialized OpenRouter LLM client.")
+ self.async_client = AsyncOpenAI(
+ api_key=OPENROUTER_API_KEY,
+ base_url="https://openrouter.ai/api/v1",
+ timeout=self.request_timeout
+ )
+ logger.info("Initialized OpenRouter sync and async clients.")
except Exception as e:
logger.error(
f"Failed to initialize OpenRouter client: {e}. "
@@ -305,18 +368,16 @@ def _call_with_fallback(self, messages: list[dict[str, str]]) -> tuple[ResponseE
logger.error("No OpenRouter client available for live call.")
return None, ""
- with self._rate_limit_lock:
- elapsed = time.time() - self.last_call_time
- if elapsed < self.rate_limit_delay:
- sleep_duration = self.rate_limit_delay - elapsed
- logger.debug(f"Rate limit buffer: sleeping for {sleep_duration:.2f}s before LLM call")
- time.sleep(sleep_duration)
- self.last_call_time = time.time()
+ elapsed = time.time() - self.last_call_time
+ if elapsed < self.rate_limit_delay:
+ sleep_duration = self.rate_limit_delay - elapsed
+ logger.debug(f"Rate limit buffer: sleeping for {sleep_duration:.2f}s before LLM call")
+ time.sleep(sleep_duration)
+ self.last_call_time = time.time()
for model_candidate in self.fallback_models:
try:
- with self._rate_limit_lock:
- self.last_call_time = time.time()
+ self.last_call_time = time.time()
resp = self.client.chat.completions.create(
model=model_candidate,
messages=messages,
@@ -382,3 +443,187 @@ def complete(
if self.mode == "record":
replay.save_fixture(messages, envelope, model_id)
return envelope
+
+ async def _call_with_fallback_async(self, messages: list[dict[str, str]]) -> tuple[ResponseEnvelope | None, str]:
+ if self.async_client is None:
+ logger.error("No AsyncOpenAI client available for live call.")
+ return None, ""
+
+ elapsed = time.time() - self.last_call_time
+ if elapsed < self.rate_limit_delay:
+ await asyncio.sleep(self.rate_limit_delay - elapsed)
+ self.last_call_time = time.time()
+
+ for model_candidate in self.fallback_models:
+ try:
+ self.last_call_time = time.time()
+ resp = await self.async_client.chat.completions.create(
+ model=model_candidate,
+ messages=messages,
+ response_format={"type": "json_object"},
+ temperature=0.0,
+ max_tokens=self.max_tokens,
+ timeout=self.request_timeout
+ )
+ if not resp.choices or len(resp.choices) == 0:
+ continue
+ raw_text = resp.choices[0].message.content or ""
+ if not raw_text.strip():
+ continue
+ data = self._parse_llm_response(raw_text)
+ if not data:
+ continue
+ citations = [Citation(**c) for c in data.get("citations", []) if isinstance(c, dict)]
+ action_raw = data.get("action")
+ try:
+ action = ActionEnum(action_raw)
+ except (ValueError, TypeError, KeyError):
+ action = ActionEnum.ASK
+ envelope = ResponseEnvelope(
+ response=str(data.get("response", raw_text.strip())),
+ citations=citations,
+ action=action
+ )
+ return envelope, model_candidate
+ except Exception as e:
+ logger.warning(f"Async model candidate '{model_candidate}' failed ({e}). Trying next fallback...")
+ await asyncio.sleep(0.5)
+ return None, ""
+
+ async def _call_stream_with_fallback(
+ self, messages: list[dict[str, str]]
+ ) -> AsyncIterator[tuple[str, ResponseEnvelope | None]]:
+ if self.async_client is None:
+ logger.error("No AsyncOpenAI client available for live streaming call.")
+ yield "", None
+ return
+
+ for model_candidate in self.fallback_models:
+ elapsed = time.time() - self.last_call_time
+ if elapsed < self.rate_limit_delay:
+ await asyncio.sleep(self.rate_limit_delay - elapsed)
+ self.last_call_time = time.time()
+
+ try:
+ stream = await self.async_client.chat.completions.create(
+ model=model_candidate,
+ messages=messages,
+ response_format={"type": "json_object"},
+ temperature=0.0,
+ max_tokens=self.max_tokens,
+ timeout=self.request_timeout,
+ stream=True
+ )
+ extractor = StreamJsonExtractor()
+ full_raw_text = []
+ yielded_any = False
+
+ async for chunk in stream:
+ if not chunk.choices or len(chunk.choices) == 0:
+ continue
+ delta = chunk.choices[0].delta.content or ""
+ if not delta:
+ continue
+ full_raw_text.append(delta)
+ token = extractor.feed(delta)
+ if token:
+ yielded_any = True
+ yield token, None
+
+ raw_text = "".join(full_raw_text).strip()
+ if not raw_text:
+ if not yielded_any:
+ logger.warning(f"Empty stream from model candidate '{model_candidate}'. Trying next...")
+ continue
+ break
+
+ data = self._parse_llm_response(raw_text)
+ citations = []
+ action = ActionEnum.INSTRUCT
+ resp_text = raw_text
+ if data:
+ citations = [Citation(**c) for c in data.get("citations", []) if isinstance(c, dict)]
+ action_raw = data.get("action")
+ try:
+ action = ActionEnum(action_raw)
+ except (ValueError, TypeError, KeyError):
+ action = ActionEnum.ASK
+ resp_text = str(data.get("response", raw_text))
+
+ if not yielded_any and resp_text:
+ yield resp_text, None
+
+ envelope = ResponseEnvelope(
+ response=resp_text,
+ citations=citations,
+ action=action
+ )
+ yield "", envelope
+ return
+ except Exception as e:
+ logger.warning(f"Streaming model candidate '{model_candidate}' failed ({e}). Trying next...")
+ await asyncio.sleep(0.5)
+
+ yield "", None
+
+ async def complete_async(
+ self,
+ user_message: str,
+ session: SessionState,
+ retrieved_chunks: list[DocumentChunk]
+ ) -> ResponseEnvelope:
+ if self.mode == "mock":
+ return MockLLM.generate_response(user_message, session, retrieved_chunks)
+
+ messages = self._build_messages(user_message, session, retrieved_chunks)
+
+ if self.mode == "replay":
+ return replay.load_fixture(messages)
+
+ envelope, model_id = await self._call_with_fallback_async(messages)
+ if envelope is None:
+ logger.error("All model candidates failed asynchronously. Returning escalation envelope.")
+ return build_escalation_envelope()
+
+ if self.mode == "record":
+ replay.save_fixture(messages, envelope, model_id)
+ return envelope
+
+ async def complete_stream(
+ self,
+ user_message: str,
+ session: SessionState,
+ retrieved_chunks: list[DocumentChunk]
+ ) -> AsyncIterator[tuple[str, ResponseEnvelope | None]]:
+ if self.mode == "mock":
+ full_env = MockLLM.generate_response(user_message, session, retrieved_chunks)
+ words = full_env.response.split(" ")
+ for i, word in enumerate(words):
+ suffix = " " if i < len(words) - 1 else ""
+ yield word + suffix, None
+ await asyncio.sleep(0.01)
+ yield "", full_env
+ return
+
+ messages = self._build_messages(user_message, session, retrieved_chunks)
+
+ if self.mode == "replay":
+ env = replay.load_fixture(messages)
+ yield "", env
+ return
+
+ final_env: ResponseEnvelope | None = None
+ async for delta, env in self._call_stream_with_fallback(messages):
+ if delta:
+ yield delta, None
+ if env:
+ final_env = env
+
+ if final_env is None:
+ logger.error("All streaming model candidates failed. Returning escalation envelope.")
+ final_env = build_escalation_envelope()
+
+ if self.mode == "record":
+ replay.save_fixture(messages, final_env, "streamed")
+
+ yield "", final_env