From d592d46226eec378c27fe3fc6d36b0e407c08507 Mon Sep 17 00:00:00 2001 From: Tan Jun Xian Date: Fri, 11 Sep 2026 13:52:27 +0800 Subject: [PATCH] feat(api): implement health/ pings to check database and qdrant status --- CHANGES.md | 6 +++ backend/main.py | 87 ++++++++++++++++++++++++++++++++++++- tests/test_api_health.py | 94 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 tests/test_api_health.py diff --git a/CHANGES.md b/CHANGES.md index 9cdba87..4b11081 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,12 @@ - **Server-Sent Events (SSE) Streaming API (`backend/main.py`)**: - Added `POST /api/chat/stream` endpoint returning `StreamingResponse(..., media_type="text/event-stream")`. - Preserved synchronous `POST /api/chat` calling `orchestrator.process_turn` for backward compatibility with synchronous callers and test suites. +- **Production Health & Readiness Probes (`backend/main.py`)**: + - Implemented `check_database_health()` probing active PostgreSQL connection (`SELECT 1;`) with automatic SQLite fallback verification. + - Implemented `check_qdrant_health()` executing shallow vector store connectivity checks with a 2-second timeout. + - Added `GET /api/health/live` returning HTTP 200 for process liveness monitoring. + - Added `GET /api/health/ready` deep readiness probe returning HTTP 200 or HTTP 503 based on critical dependency availability. + - Enriched `GET /api/health` with dependency telemetry while preserving backward compatibility with existing test suites. - **Frontend Real-Time Token Streaming & Animated Feedback (`frontend/src/api.js`, `frontend/src/App.jsx`, `frontend/src/App.css`)**: - Added `streamMessage` using `ReadableStreamDefaultReader` supporting CRLF/LF packet parsing and trailing buffer flushes. - Fixed React 18 state batching race in `App.jsx` using atomic message existence checks. diff --git a/backend/main.py b/backend/main.py index 74ceed2..f18ed9e 100644 --- a/backend/main.py +++ b/backend/main.py @@ -5,7 +5,10 @@ from typing import List, Optional import json -from fastapi import Depends, FastAPI, HTTPException, Security, status +import sqlite3 +from contextlib import closing +import urllib.request +from fastapi import Depends, FastAPI, HTTPException, Response, Security, status from fastapi.responses import StreamingResponse from fastapi.middleware.cors import CORSMiddleware from fastapi.security.api_key import APIKeyHeader @@ -17,8 +20,9 @@ sys.path.insert(0, str(PROJECT_ROOT)) from src.agent.orchestrator import OrbitMeshOrchestrator -from src.core.config import ensure_dirs +from src.core.config import ensure_dirs, DB_BACKEND, QDRANT_URL, QDRANT_PATH from src.core.logging import logger +from src.state.session import SessionStateManager ensure_dirs() @@ -107,12 +111,91 @@ class ChatResponse(BaseModel): action: str +def check_database_health() -> tuple[bool, dict]: + """Verify database connection viability.""" + try: + if SessionStateManager.is_postgres(): + with SessionStateManager._get_pg_conn() as conn: + with conn.cursor() as cur: + cur.execute("SELECT 1;") + cur.fetchone() + return True, {"backend": "postgres", "status": "connected"} + else: + db_path = SessionStateManager._db_path + db_path.parent.mkdir(parents=True, exist_ok=True) + with closing(sqlite3.connect(str(db_path), timeout=5.0)) as conn: + with conn: + conn.execute("SELECT 1;") + return True, {"backend": "sqlite", "status": "connected"} + except Exception as e: + logger.warning(f"Database health check probe failed: {e}") + return False, {"backend": DB_BACKEND, "status": "unhealthy", "error": str(e)} + + +def check_qdrant_health() -> tuple[bool, dict]: + """Verify Qdrant vector database accessibility.""" + try: + if QDRANT_URL: + health_url = f"{QDRANT_URL.rstrip('/')}/healthz" + req = urllib.request.Request(health_url, headers={"User-Agent": "OrbitMesh-HealthCheck"}) + with urllib.request.urlopen(req, timeout=2.0) as resp: + if resp.status == 200: + return True, {"mode": "server", "url": QDRANT_URL, "status": "connected"} + return False, {"mode": "server", "url": QDRANT_URL, "status": f"unexpected_status_{resp.status}"} + else: + if QDRANT_PATH.exists(): + return True, {"mode": "embedded", "path": str(QDRANT_PATH), "status": "ready"} + return True, {"mode": "embedded", "path": str(QDRANT_PATH), "status": "uninitialized"} + except Exception as e: + logger.warning(f"Qdrant health check probe failed: {e}") + return False, {"mode": "server" if QDRANT_URL else "embedded", "status": "unhealthy", "error": str(e)} + + @app.get("/api/health") def health_check(): + db_ok, db_info = check_database_health() + qdrant_ok, qdrant_info = check_qdrant_health() + overall = "ok" if (db_ok and qdrant_ok) else ("degraded" if db_ok else "unhealthy") return { "status": "ok", + "health": overall, "service": "orbitmesh-backend", "version": "0.0.1", + "dependencies": { + "database": db_info, + "vector_store": qdrant_info, + }, + } + + +@app.get("/api/health/live") +def liveness_check(): + """Liveness probe: verifies the process is running.""" + return {"status": "alive"} + + +@app.get("/api/health/ready") +def readiness_check(response: Response): + """Readiness probe: verifies critical dependencies before accepting traffic.""" + db_ok, db_info = check_database_health() + qdrant_ok, qdrant_info = check_qdrant_health() + + if not db_ok or not qdrant_ok: + response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE + return { + "status": "not_ready", + "dependencies": { + "database": db_info, + "vector_store": qdrant_info, + }, + } + + return { + "status": "ready", + "dependencies": { + "database": db_info, + "vector_store": qdrant_info, + }, } diff --git a/tests/test_api_health.py b/tests/test_api_health.py new file mode 100644 index 0000000..839117b --- /dev/null +++ b/tests/test_api_health.py @@ -0,0 +1,94 @@ +"""Test suite for FastAPI health, liveness, and readiness probes. + +Can be run via pytest: + pytest tests/test_api_health.py + +Or executed directly as a standalone test script: + python tests/test_api_health.py +""" +import sys +from pathlib import Path +from unittest.mock import patch +import pytest +from fastapi.testclient import TestClient + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from backend.main import app + + +@pytest.fixture +def client(): + return TestClient(app) + + +def test_health_endpoint_metadata(client): + """Verify GET /api/health returns 200 OK, service metadata, and dependency reports.""" + resp = client.get("/api/health") + assert resp.status_code == 200 + data = resp.json() + assert data["status"] == "ok" + assert data["service"] == "orbitmesh-backend" + assert "version" in data + assert "dependencies" in data + assert "database" in data["dependencies"] + assert "vector_store" in data["dependencies"] + + +def test_liveness_probe_returns_alive(client): + """Verify GET /api/health/live returns HTTP 200 with alive status.""" + resp = client.get("/api/health/live") + assert resp.status_code == 200 + data = resp.json() + assert data["status"] == "alive" + + +def test_readiness_probe_success(client): + """Verify GET /api/health/ready returns HTTP 200 when all dependencies are healthy.""" + with patch("backend.main.check_database_health", return_value=(True, {"status": "connected"})): + with patch("backend.main.check_qdrant_health", return_value=(True, {"status": "connected"})): + resp = client.get("/api/health/ready") + assert resp.status_code == 200 + data = resp.json() + assert data["status"] == "ready" + assert data["dependencies"]["database"]["status"] == "connected" + assert data["dependencies"]["vector_store"]["status"] == "connected" + + +def test_readiness_probe_database_failure(client): + """Verify GET /api/health/ready returns HTTP 503 when database is unhealthy.""" + with patch("backend.main.check_database_health", return_value=(False, {"status": "unhealthy", "error": "Connection refused"})): + with patch("backend.main.check_qdrant_health", return_value=(True, {"status": "connected"})): + resp = client.get("/api/health/ready") + assert resp.status_code == 503 + data = resp.json() + assert data["status"] == "not_ready" + assert data["dependencies"]["database"]["status"] == "unhealthy" + + +def test_readiness_probe_vector_store_failure(client): + """Verify GET /api/health/ready returns HTTP 503 when Qdrant is unreachable.""" + with patch("backend.main.check_database_health", return_value=(True, {"status": "connected"})): + with patch("backend.main.check_qdrant_health", return_value=(False, {"status": "unhealthy", "error": "Timeout"})): + resp = client.get("/api/health/ready") + assert resp.status_code == 503 + data = resp.json() + assert data["status"] == "not_ready" + assert data["dependencies"]["vector_store"]["status"] == "unhealthy" + + +if __name__ == "__main__": + c = TestClient(app) + print("Testing GET /api/health...") + test_health_endpoint_metadata(c) + print("Testing GET /api/health/live...") + test_liveness_probe_returns_alive(c) + print("Testing GET /api/health/ready (healthy)...") + test_readiness_probe_success(c) + print("Testing GET /api/health/ready (db failure -> 503)...") + test_readiness_probe_database_failure(c) + print("Testing GET /api/health/ready (qdrant failure -> 503)...") + test_readiness_probe_vector_store_failure(c) + print("All health and readiness probe checks passed successfully.")