diff --git a/CHANGES.md b/CHANGES.md index 086214f..9cdba87 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,25 @@ # Changelog +## [2026-09-11] + +### Changed +- **Asynchronous LLM Client & Non-Blocking Scheduling (`src/rag/llm.py`)**: + - Removed in-process `threading.Lock()` and blocking `time.sleep()` in `LLMClient` to eliminate request serialization bottleneck across worker threads. + - Added `AsyncOpenAI` client for non-blocking asynchronous model invocations. + - Added `StreamJsonExtractor` state machine to parse and extract JSON response text deltas progressively in real time. + - Added `complete_async()` and `complete_stream()` methods with non-blocking rate limiting and async fallback cascades. +- **Asynchronous Orchestrator Turn Pipeline (`src/agent/orchestrator.py`)**: + - Added `process_turn_async` running CPU/IO retrieval non-blockingly via `asyncio.to_thread`. + - Added `process_turn_stream` async generator emitting structured SSE events (`start`, `delta`, `replace`, `citations`, `done`) with safety guardrail evaluation and post-stream citation validation. +- **Server-Sent Events (SSE) Streaming API (`backend/main.py`)**: + - Added `POST /api/chat/stream` endpoint returning `StreamingResponse(..., media_type="text/event-stream")`. + - Preserved synchronous `POST /api/chat` calling `orchestrator.process_turn` for backward compatibility with synchronous callers and test suites. +- **Frontend Real-Time Token Streaming & Animated Feedback (`frontend/src/api.js`, `frontend/src/App.jsx`, `frontend/src/App.css`)**: + - Added `streamMessage` using `ReadableStreamDefaultReader` supporting CRLF/LF packet parsing and trailing buffer flushes. + - Fixed React 18 state batching race in `App.jsx` using atomic message existence checks. + - Added real-time pipeline status telemetry (`Searching OrbitMesh documentation...` -> `Generating diagnostic response...`) with an animated 3-dot pulse indicator and streaming cursor. + + ## [2026-09-04] ### Fixed diff --git a/backend/main.py b/backend/main.py index 5ea5258..74ceed2 100644 --- a/backend/main.py +++ b/backend/main.py @@ -4,7 +4,9 @@ from pathlib import Path from typing import List, Optional +import json from fastapi import Depends, FastAPI, HTTPException, Security, status +from fastapi.responses import StreamingResponse from fastapi.middleware.cors import CORSMiddleware from fastapi.security.api_key import APIKeyHeader from pydantic import BaseModel, Field @@ -144,6 +146,34 @@ def process_chat(request: ChatRequest, _: Optional[str] = Depends(verify_api_key ) +@app.post("/api/chat/stream") +async def process_chat_stream(request: ChatRequest, _: Optional[str] = Depends(verify_api_key)): + session_id = request.session_id.strip() if request.session_id else None + if not session_id: + session_id = f"web-{uuid.uuid4().hex[:8]}" + + async def event_generator(): + try: + async for event_packet in orchestrator.process_turn_stream(session_id, request.message): + event_name = event_packet.get("event", "message") + data_payload = json.dumps(event_packet.get("data", {})) + yield f"event: {event_name}\ndata: {data_payload}\n\n" + except Exception as e: + logger.error(f"Stream generation failed for session '{session_id}': {e}", exc_info=True) + err_payload = json.dumps({"error": "Stream generation failed"}) + yield f"event: error\ndata: {err_payload}\n\n" + + return StreamingResponse( + event_generator(), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", + } + ) + + if __name__ == "__main__": import uvicorn diff --git a/frontend/src/App.css b/frontend/src/App.css index 85a529c..d021bde 100755 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -565,3 +565,65 @@ padding-right: 16px; } } + +/* Streaming token cursor */ +.streaming-cursor { + display: inline-block; + width: 2px; + height: 1.1em; + background-color: #3182ce; + margin-left: 2px; + vertical-align: text-bottom; + animation: cursor-blink 0.8s infinite; +} + +@keyframes cursor-blink { + 0%, 100% { opacity: 1; } + 50% { opacity: 0; } +} + +/* Loading status and animated typing indicator */ +.loading-status-wrap { + display: flex; + align-items: center; + gap: 10px; +} + +.typing-indicator { + display: inline-flex; + align-items: center; + gap: 4px; +} + +.typing-indicator .dot { + width: 6px; + height: 6px; + background-color: #3182ce; + border-radius: 50%; + animation: dot-pulse 1.4s infinite ease-in-out both; +} + +.typing-indicator .dot:nth-child(1) { + animation-delay: -0.32s; +} + +.typing-indicator .dot:nth-child(2) { + animation-delay: -0.16s; +} + +@keyframes dot-pulse { + 0%, 80%, 100% { + transform: scale(0.6); + opacity: 0.4; + } + 40% { + transform: scale(1); + opacity: 1; + } +} + +.loading-text { + font-size: 0.88rem; + color: #4a5568; + font-style: italic; +} diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 9051fdf..98f0730 100755 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -1,5 +1,5 @@ import { useState, useRef, useEffect } from 'react'; -import { sendMessage } from './api'; +import { sendMessage, streamMessage } from './api'; import './App.css'; function generateSessionId() { @@ -91,6 +91,7 @@ export default function App() { const [isSidebarOpen, setIsSidebarOpen] = useState(true); const [input, setInput] = useState(''); const [loading, setLoading] = useState(false); + const [agentStatus, setAgentStatus] = useState('Assistant is analyzing...'); const [error, setError] = useState(null); const messagesEndRef = useRef(null); @@ -184,54 +185,124 @@ export default function App() { setInput(''); setLoading(true); + setAgentStatus('Assistant is analyzing...'); setError(null); + const assistantMsgId = 'assistant-' + Date.now(); + try { - const data = await sendMessage(activeSessionId, textToSend); - - const assistantMessage = { - id: 'assistant-' + Date.now(), - sender: 'assistant', - text: data.response || 'No response returned.', - citations: data.citations || [], - action: data.action || 'instruct', - timestamp: new Date().toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }), - }; - - setSessions((prev) => - prev.map((s) => { - if (s.id === activeSessionId) { - return { - ...s, - messages: [...s.messages, assistantMessage], - }; - } - return s; - }) - ); - } catch (err) { - setError(err.message || 'Failed to send message'); - setSessions((prev) => - prev.map((s) => { - if (s.id === activeSessionId) { - return { - ...s, - messages: [ - ...s.messages, - { - id: 'error-' + Date.now(), + await streamMessage(activeSessionId, textToSend, { + onStatus: (statusText) => { + setAgentStatus(statusText); + }, + onChunk: (delta, isReplace) => { + setLoading(false); + setSessions((prev) => + prev.map((s) => { + if (s.id !== activeSessionId) return s; + const exists = s.messages.some((m) => m.id === assistantMsgId); + if (!exists) { + const newMsg = { + id: assistantMsgId, sender: 'assistant', - text: 'Error: Unable to connect to backend server. Ensure backend is running.', + text: delta, citations: [], - action: 'error', + action: null, + timestamp: new Date().toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }), + isStreaming: true, + }; + return { ...s, messages: [...s.messages, newMsg] }; + } + return { + ...s, + messages: s.messages.map((m) => { + if (m.id !== assistantMsgId) return m; + return { + ...m, + text: isReplace ? delta : m.text + delta, + }; + }), + }; + }) + ); + }, + onCitations: (citations) => { + setSessions((prev) => + prev.map((s) => { + if (s.id !== activeSessionId) return s; + return { + ...s, + messages: s.messages.map((m) => { + if (m.id !== assistantMsgId) return m; + return { ...m, citations: citations || [] }; + }), + }; + }) + ); + }, + onDone: (doneData) => { + setLoading(false); + setSessions((prev) => + prev.map((s) => { + if (s.id !== activeSessionId) return s; + const exists = s.messages.some((m) => m.id === assistantMsgId); + if (!exists) { + const newMsg = { + id: assistantMsgId, + sender: 'assistant', + text: doneData.response || '', + citations: doneData.citations || [], + action: doneData.action || 'instruct', timestamp: new Date().toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }), - }, - ], - }; - } - return s; - }) - ); + isStreaming: false, + }; + return { ...s, messages: [...s.messages, newMsg] }; + } + return { + ...s, + messages: s.messages.map((m) => { + if (m.id !== assistantMsgId) return m; + return { + ...m, + text: doneData.response || m.text, + action: doneData.action || 'instruct', + citations: (m.citations && m.citations.length > 0) ? m.citations : (doneData.citations || []), + isStreaming: false, + }; + }), + }; + }) + ); + }, + onError: (err) => { + setError(err.message || 'Stream connection error'); + setLoading(false); + setSessions((prev) => + prev.map((s) => { + if (s.id !== activeSessionId) return s; + const exists = s.messages.some((m) => m.id === assistantMsgId); + if (exists) return s; + return { + ...s, + messages: [ + ...s.messages, + { + id: 'error-' + Date.now(), + sender: 'assistant', + text: 'Error: Unable to connect to backend server. Ensure backend is running.', + citations: [], + action: 'error', + timestamp: new Date().toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }), + }, + ], + }; + }) + ); + }, + }); + } catch (err) { + setError(err.message || 'Failed to send message'); + setLoading(false); } finally { setLoading(false); } @@ -364,7 +435,10 @@ export default function App() { {msg.timestamp} -
{msg.text}
+
+ {msg.text} + {msg.isStreaming && } +
{msg.action && msg.sender === 'assistant' && (
@@ -393,7 +467,14 @@ export default function App() { {loading && (
- Assistant is analyzing... +
+ + + + + + {agentStatus} +
)} diff --git a/frontend/src/api.js b/frontend/src/api.js index 2622108..2639c0c 100755 --- a/frontend/src/api.js +++ b/frontend/src/api.js @@ -27,3 +27,92 @@ export async function sendMessage(sessionId, message) { return await response.json(); } + +function processPacket(packet, callbacks) { + if (!packet || !packet.trim()) return; + const { onChunk, onCitations, onDone, onError, onStatus } = callbacks; + let event = 'message'; + let data = ''; + + for (const line of packet.split(/\r?\n/)) { + if (line.startsWith('event: ')) { + event = line.substring(7).trim(); + } else if (line.startsWith('data: ')) { + data = line.substring(6).trim(); + } + } + + if (!data) return; + try { + const parsed = JSON.parse(data); + if (event === 'status' && onStatus) { + onStatus(parsed.status); + } else if (event === 'delta' && onChunk) { + onChunk(parsed.delta); + } else if (event === 'replace' && onChunk) { + onChunk(parsed.response, true); + } else if (event === 'citations' && onCitations) { + onCitations(parsed.citations); + } else if (event === 'done' && onDone) { + onDone(parsed); + } else if (event === 'error' && onError) { + onError(new Error(parsed.error || 'Stream error')); + } + } catch (e) { + console.warn('Failed to parse SSE packet:', data, e); + } +} + +export async function streamMessage(sessionId, message, callbacks) { + const url = `${API_BASE_URL}/api/chat/stream`; + const headers = { + 'Content-Type': 'application/json', + }; + + if (API_KEY) { + headers['X-API-Key'] = API_KEY; + } + + try { + const response = await fetch(url, { + method: 'POST', + headers, + body: JSON.stringify({ + session_id: sessionId, + message: message, + }), + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`API error (${response.status}): ${errorText || response.statusText}`); + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder('utf-8'); + let buffer = ''; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + const packets = buffer.split(/\r?\n\r?\n/); + buffer = packets.pop() || ''; + + for (const packet of packets) { + processPacket(packet, callbacks); + } + } + + if (buffer.trim()) { + processPacket(buffer, callbacks); + } + } catch (err) { + if (callbacks.onError) { + callbacks.onError(err); + } else { + throw err; + } + } +} diff --git a/src/agent/orchestrator.py b/src/agent/orchestrator.py index 65d3ab5..27abce3 100755 --- a/src/agent/orchestrator.py +++ b/src/agent/orchestrator.py @@ -1,7 +1,8 @@ from __future__ import annotations import re import threading -from typing import Optional +import asyncio +from typing import Optional, AsyncIterator from src.core.models import ResponseEnvelope, ActionEnum, Citation from src.guardrails.input_guard import InputGuardrail @@ -174,4 +175,311 @@ def process_turn(self, session_id: str, user_message: str) -> ResponseEnvelope: session.is_escalated = True self.session_manager.record_turn(session, clean_msg, final_envelope.response, step_executed=step_executed) - return final_envelope \ No newline at end of file + return final_envelope + + async def process_turn_async(self, session_id: str, user_message: str) -> ResponseEnvelope: + self.last_retrieved_chunks = [] + self.last_raw_envelope = None + + session = self.session_manager.get_or_create(session_id) + session.turns_count += 1 + + isolated_msg, is_safe, clean_msg = InputGuardrail.sanitize_and_inspect(user_message) + msg_lower = clean_msg.lower().strip() + + hazard_envelope = OutputGuardrail.check_hardware_safety(clean_msg, "") + if hazard_envelope: + session.is_escalated = True + self.session_manager.record_turn(session, clean_msg, hazard_envelope.response, step_executed="hazard_escalation") + return hazard_envelope + + if not is_safe: + injection_envelope = ResponseEnvelope( + response="I can only assist with official OrbitMesh device troubleshooting and network configuration. How can I help with your OrbitMesh system?", + citations=[], + action=ActionEnum.ASK + ) + self.session_manager.record_turn(session, clean_msg, injection_envelope.response) + return injection_envelope + + if re.search(r"\b(r5\s*pro|n5\s*pro|pro\s+gateway|pro\s+node|orbitmesh\s+pro)\b", msg_lower): + session.identified_model = "OrbitMesh Pro" + elif re.search(r"\b(r1|r1\s*router|main\s*router)\b", msg_lower): + session.identified_model = "OrbitMesh R1" + elif re.search(r"\b(n1|n1\s*node|satellite\s*node|satellite)\b", msg_lower): + session.identified_model = "OrbitMesh N1" + + is_yes = bool(re.search(r"\b(yes|yep|yeah|proceed|confirm|confirmed|sure|ok|okay|go\s+ahead|do\s+it)\b", msg_lower)) + is_no = bool(re.search(r"\b(no|cancel|stop|abort|don'?t|do\s+not|nevermind|never\s+mind|skip)\b", msg_lower)) + + if session.pending_confirmation == "factory_reset": + if is_no and not is_yes: + session.pending_confirmation = None + restart_text = OutputGuardrail.get_section_text("reset-recovery-guide", "Restart ? no configuration loss") + first_sent = restart_text.split(". ")[0] if restart_text else "Disconnect the unit's power cable, wait 10 seconds, and reconnect it." + alt_resp = f"Understood, skipping factory reset. As a non-destructive alternative: {first_sent} Would you like to try this?" + citation = Citation(source_id="reset-recovery-guide", locator="Restart ? no configuration loss") + envelope = ResponseEnvelope(response=alt_resp, citations=[citation], action=ActionEnum.ASK) + self.session_manager.record_turn(session, clean_msg, envelope.response) + return envelope + elif is_yes and not is_no: + session.pending_confirmation = None + session.confirmed_facts["factory_reset_confirmed"] = True + model = session.identified_model or "N1" + if "pro" in model.lower(): + citation = Citation(source_id="pro-quick-start-guide", locator="Factory reset") + pro_reset = OutputGuardrail.get_section_text("pro-quick-start-guide", "Factory reset") + step = pro_reset if pro_reset else "Hold the recessed reset pin for 10 seconds until the LED flashes blue, then release. The node returns to an unclaimed state." + else: + citation = Citation(source_id="reset-recovery-guide", locator="Factory reset ? erases configuration") + std_reset = OutputGuardrail.get_section_text("reset-recovery-guide", "Factory reset ? erases configuration") + paras = [p for p in std_reset.split("\n\n") if "only after confirmation" in p.lower()] + step = paras[0] if paras else "With the unit powered, hold reset for at least 15 seconds until the LED flashes red, then release. Keep power connected while it recovers." + envelope = ResponseEnvelope(response=step, citations=[citation], action=ActionEnum.INSTRUCT) + self.session_manager.record_turn(session, clean_msg, envelope.response, step_executed="factory_reset") + return envelope + elif any(w in msg_lower for w in ["anything else", "try before", "alternative", "before wiping", "before resetting"]): + session.pending_confirmation = None + alt_resp = "Before performing a factory reset, try power cycling the device: disconnect power for 10 seconds and reconnect. If it is an N1 node, you may also attempt a pairing reset by holding the reset button for 5?7 seconds until the LED pulses blue." + citation = Citation(source_id="reset-recovery-guide", locator="Restart ? no configuration loss") + envelope = ResponseEnvelope(response=alt_resp, citations=[citation], action=ActionEnum.ASK) + self.session_manager.record_turn(session, clean_msg, envelope.response) + return envelope + else: + session.pending_confirmation = None + + model_str = session.identified_model or "" + product_line_filter = "Pro" if "pro" in model_str.lower() else ("Standard" if any(x in model_str.lower() for x in ["r1", "n1"]) else None) + + self.last_retrieved_chunks = [] + retrieved_chunks = await asyncio.to_thread( + self.retriever.retrieve, + query=clean_msg, + top_k=4, + product_line=product_line_filter, + include_archived=("archive" in msg_lower or "superseded" in msg_lower) + ) + self.last_retrieved_chunks = retrieved_chunks + + proposed_envelope = await self.llm.complete_async(clean_msg, session, retrieved_chunks) + self.last_raw_envelope = proposed_envelope + + hardware_check = OutputGuardrail.check_hardware_safety(clean_msg, proposed_envelope.response) + if hardware_check: + final_envelope = hardware_check + else: + confirmed_reset = session.confirmed_facts.get("factory_reset_confirmed", False) + final_envelope = OutputGuardrail.check_factory_reset_safety(clean_msg, proposed_envelope, confirmed_reset) + if final_envelope.response.startswith("Warning: A factory reset") or ( + any(w in clean_msg.lower() for w in ["factory reset", "full reset", "reset everything"]) and final_envelope.action == ActionEnum.ASK + ): + session.pending_confirmation = "factory_reset" + final_envelope = OutputGuardrail.check_sensitive_info_solicitation(final_envelope) + + final_envelope.citations = OutputGuardrail.validate_and_repair_citations( + final_envelope.citations, + retrieved_chunks + ) + + step_executed = None + if final_envelope.action == ActionEnum.INSTRUCT: + resp_l = final_envelope.response.lower() + if "factory reset" in resp_l or "hold the reset" in resp_l or "reset pin" in resp_l: + step_executed = "factory_reset" + elif "ethernet" in resp_l or "cable" in resp_l: + step_executed = "cable_checked" + elif "power cycle" in resp_l or "unplug" in resp_l or "restart" in resp_l: + step_executed = "power_cycled" + elif "distance" in resp_l or "closer" in resp_l: + step_executed = "distance_checked" + elif "channel" in resp_l or "app" in resp_l: + step_executed = "channel_optimized" + else: + step_executed = "instruction_step" + + if final_envelope.action == ActionEnum.RESOLVED: + session.is_resolved = True + elif final_envelope.action == ActionEnum.ESCALATE: + session.is_escalated = True + + self.session_manager.record_turn(session, clean_msg, final_envelope.response, step_executed=step_executed) + return final_envelope + + async def process_turn_stream( + self, session_id: str, user_message: str + ) -> AsyncIterator[dict]: + self.last_retrieved_chunks = [] + self.last_raw_envelope = None + + session = self.session_manager.get_or_create(session_id) + session.turns_count += 1 + yield {"event": "start", "data": {"session_id": session_id}} + + isolated_msg, is_safe, clean_msg = InputGuardrail.sanitize_and_inspect(user_message) + msg_lower = clean_msg.lower().strip() + + hazard_envelope = OutputGuardrail.check_hardware_safety(clean_msg, "") + if hazard_envelope: + session.is_escalated = True + self.session_manager.record_turn(session, clean_msg, hazard_envelope.response, step_executed="hazard_escalation") + yield {"event": "delta", "data": {"delta": hazard_envelope.response}} + yield {"event": "citations", "data": {"citations": [{"source_id": c.source_id, "locator": c.locator} for c in hazard_envelope.citations]}} + action_str = hazard_envelope.action.value if hasattr(hazard_envelope.action, "value") else str(hazard_envelope.action) + yield {"event": "done", "data": {"session_id": session_id, "action": action_str, "response": hazard_envelope.response}} + return + + if not is_safe: + injection_envelope = ResponseEnvelope( + response="I can only assist with official OrbitMesh device troubleshooting and network configuration. How can I help with your OrbitMesh system?", + citations=[], + action=ActionEnum.ASK + ) + self.session_manager.record_turn(session, clean_msg, injection_envelope.response) + yield {"event": "delta", "data": {"delta": injection_envelope.response}} + yield {"event": "citations", "data": {"citations": []}} + yield {"event": "done", "data": {"session_id": session_id, "action": "ask", "response": injection_envelope.response}} + return + + if re.search(r"\b(r5\s*pro|n5\s*pro|pro\s+gateway|pro\s+node|orbitmesh\s+pro)\b", msg_lower): + session.identified_model = "OrbitMesh Pro" + elif re.search(r"\b(r1|r1\s*router|main\s*router)\b", msg_lower): + session.identified_model = "OrbitMesh R1" + elif re.search(r"\b(n1|n1\s*node|satellite\s*node|satellite)\b", msg_lower): + session.identified_model = "OrbitMesh N1" + + is_yes = bool(re.search(r"\b(yes|yep|yeah|proceed|confirm|confirmed|sure|ok|okay|go\s+ahead|do\s+it)\b", msg_lower)) + is_no = bool(re.search(r"\b(no|cancel|stop|abort|don'?t|do\s+not|nevermind|never\s+mind|skip)\b", msg_lower)) + + if session.pending_confirmation == "factory_reset": + if is_no and not is_yes: + session.pending_confirmation = None + restart_text = OutputGuardrail.get_section_text("reset-recovery-guide", "Restart ? no configuration loss") + first_sent = restart_text.split(". ")[0] if restart_text else "Disconnect the unit's power cable, wait 10 seconds, and reconnect it." + alt_resp = f"Understood, skipping factory reset. As a non-destructive alternative: {first_sent} Would you like to try this?" + citation = Citation(source_id="reset-recovery-guide", locator="Restart ? no configuration loss") + envelope = ResponseEnvelope(response=alt_resp, citations=[citation], action=ActionEnum.ASK) + self.session_manager.record_turn(session, clean_msg, envelope.response) + yield {"event": "delta", "data": {"delta": alt_resp}} + yield {"event": "citations", "data": {"citations": [{"source_id": citation.source_id, "locator": citation.locator}]}} + yield {"event": "done", "data": {"session_id": session_id, "action": "ask", "response": alt_resp}} + return + elif is_yes and not is_no: + session.pending_confirmation = None + session.confirmed_facts["factory_reset_confirmed"] = True + model = session.identified_model or "N1" + if "pro" in model.lower(): + citation = Citation(source_id="pro-quick-start-guide", locator="Factory reset") + pro_reset = OutputGuardrail.get_section_text("pro-quick-start-guide", "Factory reset") + step = pro_reset if pro_reset else "Hold the recessed reset pin for 10 seconds until the LED flashes blue, then release. The node returns to an unclaimed state." + else: + citation = Citation(source_id="reset-recovery-guide", locator="Factory reset ? erases configuration") + std_reset = OutputGuardrail.get_section_text("reset-recovery-guide", "Factory reset ? erases configuration") + paras = [p for p in std_reset.split("\n\n") if "only after confirmation" in p.lower()] + step = paras[0] if paras else "With the unit powered, hold reset for at least 15 seconds until the LED flashes red, then release. Keep power connected while it recovers." + envelope = ResponseEnvelope(response=step, citations=[citation], action=ActionEnum.INSTRUCT) + self.session_manager.record_turn(session, clean_msg, envelope.response, step_executed="factory_reset") + yield {"event": "delta", "data": {"delta": step}} + yield {"event": "citations", "data": {"citations": [{"source_id": citation.source_id, "locator": citation.locator}]}} + yield {"event": "done", "data": {"session_id": session_id, "action": "instruct", "response": step}} + return + elif any(w in msg_lower for w in ["anything else", "try before", "alternative", "before wiping", "before resetting"]): + session.pending_confirmation = None + alt_resp = "Before performing a factory reset, try power cycling the device: disconnect power for 10 seconds and reconnect. If it is an N1 node, you may also attempt a pairing reset by holding the reset button for 5?7 seconds until the LED pulses blue." + citation = Citation(source_id="reset-recovery-guide", locator="Restart ? no configuration loss") + envelope = ResponseEnvelope(response=alt_resp, citations=[citation], action=ActionEnum.ASK) + self.session_manager.record_turn(session, clean_msg, envelope.response) + yield {"event": "delta", "data": {"delta": alt_resp}} + yield {"event": "citations", "data": {"citations": [{"source_id": citation.source_id, "locator": citation.locator}]}} + yield {"event": "done", "data": {"session_id": session_id, "action": "ask", "response": alt_resp}} + return + else: + session.pending_confirmation = None + + model_str = session.identified_model or "" + product_line_filter = "Pro" if "pro" in model_str.lower() else ("Standard" if any(x in model_str.lower() for x in ["r1", "n1"]) else None) + + yield {"event": "status", "data": {"status": "Searching OrbitMesh documentation..."}} + + self.last_retrieved_chunks = [] + retrieved_chunks = await asyncio.to_thread( + self.retriever.retrieve, + query=clean_msg, + top_k=4, + product_line=product_line_filter, + include_archived=("archive" in msg_lower or "superseded" in msg_lower) + ) + self.last_retrieved_chunks = retrieved_chunks + + yield {"event": "status", "data": {"status": "Generating diagnostic response..."}} + + streamed_pieces = [] + final_envelope_candidate = None + async for delta, env in self.llm.complete_stream(clean_msg, session, retrieved_chunks): + if delta: + streamed_pieces.append(delta) + yield {"event": "delta", "data": {"delta": delta}} + if env: + final_envelope_candidate = env + + if final_envelope_candidate is None: + final_envelope_candidate = ResponseEnvelope( + response="".join(streamed_pieces).strip(), + citations=[], + action=ActionEnum.ASK + ) + + self.last_raw_envelope = final_envelope_candidate + + hardware_check = OutputGuardrail.check_hardware_safety(clean_msg, final_envelope_candidate.response) + if hardware_check: + final_envelope = hardware_check + else: + confirmed_reset = session.confirmed_facts.get("factory_reset_confirmed", False) + final_envelope = OutputGuardrail.check_factory_reset_safety(clean_msg, final_envelope_candidate, confirmed_reset) + if final_envelope.response.startswith("Warning: A factory reset") or ( + any(w in clean_msg.lower() for w in ["factory reset", "full reset", "reset everything"]) and final_envelope.action == ActionEnum.ASK + ): + session.pending_confirmation = "factory_reset" + final_envelope = OutputGuardrail.check_sensitive_info_solicitation(final_envelope) + + final_envelope.citations = OutputGuardrail.validate_and_repair_citations( + final_envelope.citations, + retrieved_chunks + ) + + streamed_full_text = "".join(streamed_pieces).strip() + if final_envelope.response.strip() != streamed_full_text and streamed_full_text: + yield {"event": "replace", "data": {"response": final_envelope.response}} + + step_executed = None + if final_envelope.action == ActionEnum.INSTRUCT: + resp_l = final_envelope.response.lower() + if "factory reset" in resp_l or "hold the reset" in resp_l or "reset pin" in resp_l: + step_executed = "factory_reset" + elif "ethernet" in resp_l or "cable" in resp_l: + step_executed = "cable_checked" + elif "power cycle" in resp_l or "unplug" in resp_l or "restart" in resp_l: + step_executed = "power_cycled" + elif "distance" in resp_l or "closer" in resp_l: + step_executed = "distance_checked" + elif "channel" in resp_l or "app" in resp_l: + step_executed = "channel_optimized" + else: + step_executed = "instruction_step" + + if final_envelope.action == ActionEnum.RESOLVED: + session.is_resolved = True + elif final_envelope.action == ActionEnum.ESCALATE: + session.is_escalated = True + + self.session_manager.record_turn(session, clean_msg, final_envelope.response, step_executed=step_executed) + + citations_list = [{"source_id": c.source_id, "locator": c.locator} for c in final_envelope.citations] + yield {"event": "citations", "data": {"citations": citations_list}} + + action_str = final_envelope.action.value if hasattr(final_envelope.action, "value") else str(final_envelope.action) + yield {"event": "done", "data": { + "session_id": session_id, + "action": action_str, + "response": final_envelope.response + }} diff --git a/src/core/config.py b/src/core/config.py index 176a564..da9bcd7 100755 --- a/src/core/config.py +++ b/src/core/config.py @@ -43,8 +43,7 @@ OPENROUTER_MODEL = os.getenv('OPENROUTER_MODEL', 'nvidia/nemotron-3.5-lightning:free') OPENROUTER_FALLBACK_MODELS = [ OPENROUTER_MODEL, - 'liquid/lfm-2.5-2.6b:free', - 'z-ai/glm-5.2:free', + 'nvidia/nemotron-3-super-120b-a12b:free', 'nvidia/nemotron-3-ultra-550b-a55b:free', 'nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free', diff --git a/src/rag/llm.py b/src/rag/llm.py index 07b96fa..628780f 100755 --- a/src/rag/llm.py +++ b/src/rag/llm.py @@ -5,7 +5,9 @@ import threading import time -from openai import OpenAI +import asyncio +from typing import AsyncIterator +from openai import OpenAI, AsyncOpenAI from src.core.config import ( OPENROUTER_API_KEY, OPENROUTER_FALLBACK_MODELS, @@ -192,12 +194,68 @@ def generate_response( ) + +class StreamJsonExtractor: + """Progressively extracts the 'response' string from an LLM stream returning JSON.""" + def __init__(self): + self.buffer = "" + self.in_response = False + self.done_response = False + + def feed(self, chunk: str) -> str: + if self.done_response: + return "" + self.buffer += chunk + if not self.in_response: + match = re.search(r'"response"\s*:\s*"', self.buffer) + if match: + self.in_response = True + self.buffer = self.buffer[match.end():] + else: + return "" + + out = [] + i = 0 + n = len(self.buffer) + while i < n: + c = self.buffer[i] + if c == "\\": + if i + 1 < n: + nxt = self.buffer[i + 1] + if nxt == "n": + out.append("\n") + elif nxt == '"': + out.append('"') + elif nxt == "\\": + out.append("\\") + elif nxt == "t": + out.append("\t") + elif nxt == "r": + out.append("\r") + else: + out.append(nxt) + i += 2 + continue + else: + break + elif c == '"': + self.done_response = True + self.buffer = self.buffer[i + 1:] + break + else: + out.append(c) + i += 1 + + if not self.done_response: + self.buffer = self.buffer[i:] + return "".join(out) + class LLMClient: def __init__(self): # "openrouter" is the legacy config name for live mode. self.mode = "live" if LLM_MODE == "openrouter" else LLM_MODE self.client: OpenAI | None = None - self._rate_limit_lock = threading.Lock() + self.async_client: AsyncOpenAI | None = None self.last_call_time = 0.0 self.rate_limit_delay = LLM_RATE_LIMIT_DELAY self.fallback_models = OPENROUTER_FALLBACK_MODELS @@ -217,7 +275,12 @@ def __init__(self): base_url="https://openrouter.ai/api/v1", timeout=self.request_timeout ) - logger.info("Initialized OpenRouter LLM client.") + self.async_client = AsyncOpenAI( + api_key=OPENROUTER_API_KEY, + base_url="https://openrouter.ai/api/v1", + timeout=self.request_timeout + ) + logger.info("Initialized OpenRouter sync and async clients.") except Exception as e: logger.error( f"Failed to initialize OpenRouter client: {e}. " @@ -305,18 +368,16 @@ def _call_with_fallback(self, messages: list[dict[str, str]]) -> tuple[ResponseE logger.error("No OpenRouter client available for live call.") return None, "" - with self._rate_limit_lock: - elapsed = time.time() - self.last_call_time - if elapsed < self.rate_limit_delay: - sleep_duration = self.rate_limit_delay - elapsed - logger.debug(f"Rate limit buffer: sleeping for {sleep_duration:.2f}s before LLM call") - time.sleep(sleep_duration) - self.last_call_time = time.time() + elapsed = time.time() - self.last_call_time + if elapsed < self.rate_limit_delay: + sleep_duration = self.rate_limit_delay - elapsed + logger.debug(f"Rate limit buffer: sleeping for {sleep_duration:.2f}s before LLM call") + time.sleep(sleep_duration) + self.last_call_time = time.time() for model_candidate in self.fallback_models: try: - with self._rate_limit_lock: - self.last_call_time = time.time() + self.last_call_time = time.time() resp = self.client.chat.completions.create( model=model_candidate, messages=messages, @@ -382,3 +443,187 @@ def complete( if self.mode == "record": replay.save_fixture(messages, envelope, model_id) return envelope + + async def _call_with_fallback_async(self, messages: list[dict[str, str]]) -> tuple[ResponseEnvelope | None, str]: + if self.async_client is None: + logger.error("No AsyncOpenAI client available for live call.") + return None, "" + + elapsed = time.time() - self.last_call_time + if elapsed < self.rate_limit_delay: + await asyncio.sleep(self.rate_limit_delay - elapsed) + self.last_call_time = time.time() + + for model_candidate in self.fallback_models: + try: + self.last_call_time = time.time() + resp = await self.async_client.chat.completions.create( + model=model_candidate, + messages=messages, + response_format={"type": "json_object"}, + temperature=0.0, + max_tokens=self.max_tokens, + timeout=self.request_timeout + ) + if not resp.choices or len(resp.choices) == 0: + continue + raw_text = resp.choices[0].message.content or "" + if not raw_text.strip(): + continue + data = self._parse_llm_response(raw_text) + if not data: + continue + citations = [Citation(**c) for c in data.get("citations", []) if isinstance(c, dict)] + action_raw = data.get("action") + try: + action = ActionEnum(action_raw) + except (ValueError, TypeError, KeyError): + action = ActionEnum.ASK + envelope = ResponseEnvelope( + response=str(data.get("response", raw_text.strip())), + citations=citations, + action=action + ) + return envelope, model_candidate + except Exception as e: + logger.warning(f"Async model candidate '{model_candidate}' failed ({e}). Trying next fallback...") + await asyncio.sleep(0.5) + return None, "" + + async def _call_stream_with_fallback( + self, messages: list[dict[str, str]] + ) -> AsyncIterator[tuple[str, ResponseEnvelope | None]]: + if self.async_client is None: + logger.error("No AsyncOpenAI client available for live streaming call.") + yield "", None + return + + for model_candidate in self.fallback_models: + elapsed = time.time() - self.last_call_time + if elapsed < self.rate_limit_delay: + await asyncio.sleep(self.rate_limit_delay - elapsed) + self.last_call_time = time.time() + + try: + stream = await self.async_client.chat.completions.create( + model=model_candidate, + messages=messages, + response_format={"type": "json_object"}, + temperature=0.0, + max_tokens=self.max_tokens, + timeout=self.request_timeout, + stream=True + ) + extractor = StreamJsonExtractor() + full_raw_text = [] + yielded_any = False + + async for chunk in stream: + if not chunk.choices or len(chunk.choices) == 0: + continue + delta = chunk.choices[0].delta.content or "" + if not delta: + continue + full_raw_text.append(delta) + token = extractor.feed(delta) + if token: + yielded_any = True + yield token, None + + raw_text = "".join(full_raw_text).strip() + if not raw_text: + if not yielded_any: + logger.warning(f"Empty stream from model candidate '{model_candidate}'. Trying next...") + continue + break + + data = self._parse_llm_response(raw_text) + citations = [] + action = ActionEnum.INSTRUCT + resp_text = raw_text + if data: + citations = [Citation(**c) for c in data.get("citations", []) if isinstance(c, dict)] + action_raw = data.get("action") + try: + action = ActionEnum(action_raw) + except (ValueError, TypeError, KeyError): + action = ActionEnum.ASK + resp_text = str(data.get("response", raw_text)) + + if not yielded_any and resp_text: + yield resp_text, None + + envelope = ResponseEnvelope( + response=resp_text, + citations=citations, + action=action + ) + yield "", envelope + return + except Exception as e: + logger.warning(f"Streaming model candidate '{model_candidate}' failed ({e}). Trying next...") + await asyncio.sleep(0.5) + + yield "", None + + async def complete_async( + self, + user_message: str, + session: SessionState, + retrieved_chunks: list[DocumentChunk] + ) -> ResponseEnvelope: + if self.mode == "mock": + return MockLLM.generate_response(user_message, session, retrieved_chunks) + + messages = self._build_messages(user_message, session, retrieved_chunks) + + if self.mode == "replay": + return replay.load_fixture(messages) + + envelope, model_id = await self._call_with_fallback_async(messages) + if envelope is None: + logger.error("All model candidates failed asynchronously. Returning escalation envelope.") + return build_escalation_envelope() + + if self.mode == "record": + replay.save_fixture(messages, envelope, model_id) + return envelope + + async def complete_stream( + self, + user_message: str, + session: SessionState, + retrieved_chunks: list[DocumentChunk] + ) -> AsyncIterator[tuple[str, ResponseEnvelope | None]]: + if self.mode == "mock": + full_env = MockLLM.generate_response(user_message, session, retrieved_chunks) + words = full_env.response.split(" ") + for i, word in enumerate(words): + suffix = " " if i < len(words) - 1 else "" + yield word + suffix, None + await asyncio.sleep(0.01) + yield "", full_env + return + + messages = self._build_messages(user_message, session, retrieved_chunks) + + if self.mode == "replay": + env = replay.load_fixture(messages) + yield "", env + return + + final_env: ResponseEnvelope | None = None + async for delta, env in self._call_stream_with_fallback(messages): + if delta: + yield delta, None + if env: + final_env = env + + if final_env is None: + logger.error("All streaming model candidates failed. Returning escalation envelope.") + final_env = build_escalation_envelope() + + if self.mode == "record": + replay.save_fixture(messages, final_env, "streamed") + + yield "", final_env