feat(api): implement health/ pings to check database and qdrant status

CHANGES.md

@@ -14,6 +14,12 @@
 - **Server-Sent Events (SSE) Streaming API (`backend/main.py`)**:
   - Added `POST /api/chat/stream` endpoint returning `StreamingResponse(..., media_type="text/event-stream")`.
   - Preserved synchronous `POST /api/chat` calling `orchestrator.process_turn` for backward compatibility with synchronous callers and test suites.
+- **Production Health & Readiness Probes (`backend/main.py`)**:
+  - Implemented `check_database_health()` probing active PostgreSQL connection (`SELECT 1;`) with automatic SQLite fallback verification.
+  - Implemented `check_qdrant_health()` executing shallow vector store connectivity checks with a 2-second timeout.
+  - Added `GET /api/health/live` returning HTTP 200 for process liveness monitoring.
+  - Added `GET /api/health/ready` deep readiness probe returning HTTP 200 or HTTP 503 based on critical dependency availability.
+  - Enriched `GET /api/health` with dependency telemetry while preserving backward compatibility with existing test suites.
 - **Frontend Real-Time Token Streaming & Animated Feedback (`frontend/src/api.js`, `frontend/src/App.jsx`, `frontend/src/App.css`)**:
   - Added `streamMessage` using `ReadableStreamDefaultReader` supporting CRLF/LF packet parsing and trailing buffer flushes.
   - Fixed React 18 state batching race in `App.jsx` using atomic message existence checks.

backend/main.py

@@ -5,7 +5,10 @@
 from typing import List, Optional
 
 import json
-from fastapi import Depends, FastAPI, HTTPException, Security, status
+import sqlite3
+from contextlib import closing
+import urllib.request
+from fastapi import Depends, FastAPI, HTTPException, Response, Security, status
 from fastapi.responses import StreamingResponse
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.security.api_key import APIKeyHeader
@@ -17,8 +20,9 @@
     sys.path.insert(0, str(PROJECT_ROOT))
 
 from src.agent.orchestrator import OrbitMeshOrchestrator
-from src.core.config import ensure_dirs
+from src.core.config import ensure_dirs, DB_BACKEND, QDRANT_URL, QDRANT_PATH
 from src.core.logging import logger
+from src.state.session import SessionStateManager
 
 ensure_dirs()
 
@@ -107,12 +111,91 @@ class ChatResponse(BaseModel):
     action: str
 
 
+def check_database_health() -> tuple[bool, dict]:
+    """Verify database connection viability."""
+    try:
+        if SessionStateManager.is_postgres():
+            with SessionStateManager._get_pg_conn() as conn:
+                with conn.cursor() as cur:
+                    cur.execute("SELECT 1;")
+                    cur.fetchone()
+            return True, {"backend": "postgres", "status": "connected"}
+        else:
+            db_path = SessionStateManager._db_path
+            db_path.parent.mkdir(parents=True, exist_ok=True)
+            with closing(sqlite3.connect(str(db_path), timeout=5.0)) as conn:
+                with conn:
+                    conn.execute("SELECT 1;")
+            return True, {"backend": "sqlite", "status": "connected"}
+    except Exception as e:
+        logger.warning(f"Database health check probe failed: {e}")
+        return False, {"backend": DB_BACKEND, "status": "unhealthy", "error": str(e)}
+
+
+def check_qdrant_health() -> tuple[bool, dict]:
+    """Verify Qdrant vector database accessibility."""
+    try:
+        if QDRANT_URL:
+            health_url = f"{QDRANT_URL.rstrip('/')}/healthz"
+            req = urllib.request.Request(health_url, headers={"User-Agent": "OrbitMesh-HealthCheck"})
+            with urllib.request.urlopen(req, timeout=2.0) as resp:
+                if resp.status == 200:
+                    return True, {"mode": "server", "url": QDRANT_URL, "status": "connected"}
+                return False, {"mode": "server", "url": QDRANT_URL, "status": f"unexpected_status_{resp.status}"}
+        else:
+            if QDRANT_PATH.exists():
+                return True, {"mode": "embedded", "path": str(QDRANT_PATH), "status": "ready"}
+            return True, {"mode": "embedded", "path": str(QDRANT_PATH), "status": "uninitialized"}
+    except Exception as e:
+        logger.warning(f"Qdrant health check probe failed: {e}")
+        return False, {"mode": "server" if QDRANT_URL else "embedded", "status": "unhealthy", "error": str(e)}
+
+
 @app.get("/api/health")
 def health_check():
+    db_ok, db_info = check_database_health()
+    qdrant_ok, qdrant_info = check_qdrant_health()
+    overall = "ok" if (db_ok and qdrant_ok) else ("degraded" if db_ok else "unhealthy")
     return {
         "status": "ok",
+        "health": overall,
         "service": "orbitmesh-backend",
         "version": "0.0.1",
+        "dependencies": {
+            "database": db_info,
+            "vector_store": qdrant_info,
+        },
+    }
+
+
+@app.get("/api/health/live")
+def liveness_check():
+    """Liveness probe: verifies the process is running."""
+    return {"status": "alive"}
+
+
+@app.get("/api/health/ready")
+def readiness_check(response: Response):
+    """Readiness probe: verifies critical dependencies before accepting traffic."""
+    db_ok, db_info = check_database_health()
+    qdrant_ok, qdrant_info = check_qdrant_health()
+
+    if not db_ok or not qdrant_ok:
+        response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
+        return {
+            "status": "not_ready",
+            "dependencies": {
+                "database": db_info,
+                "vector_store": qdrant_info,
+            },
+        }
+
+    return {
+        "status": "ready",
+        "dependencies": {
+            "database": db_info,
+            "vector_store": qdrant_info,
+        },
     }
 
 

tests/test_api_health.py

@@ -0,0 +1,94 @@
+"""Test suite for FastAPI health, liveness, and readiness probes.
+
+Can be run via pytest:
+    pytest tests/test_api_health.py
+
+Or executed directly as a standalone test script:
+    python tests/test_api_health.py
+"""
+import sys
+from pathlib import Path
+from unittest.mock import patch
+import pytest
+from fastapi.testclient import TestClient
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from backend.main import app
+
+
+@pytest.fixture
+def client():
+    return TestClient(app)
+
+
+def test_health_endpoint_metadata(client):
+    """Verify GET /api/health returns 200 OK, service metadata, and dependency reports."""
+    resp = client.get("/api/health")
+    assert resp.status_code == 200
+    data = resp.json()
+    assert data["status"] == "ok"
+    assert data["service"] == "orbitmesh-backend"
+    assert "version" in data
+    assert "dependencies" in data
+    assert "database" in data["dependencies"]
+    assert "vector_store" in data["dependencies"]
+
+
+def test_liveness_probe_returns_alive(client):
+    """Verify GET /api/health/live returns HTTP 200 with alive status."""
+    resp = client.get("/api/health/live")
+    assert resp.status_code == 200
+    data = resp.json()
+    assert data["status"] == "alive"
+
+
+def test_readiness_probe_success(client):
+    """Verify GET /api/health/ready returns HTTP 200 when all dependencies are healthy."""
+    with patch("backend.main.check_database_health", return_value=(True, {"status": "connected"})):
+        with patch("backend.main.check_qdrant_health", return_value=(True, {"status": "connected"})):
+            resp = client.get("/api/health/ready")
+            assert resp.status_code == 200
+            data = resp.json()
+            assert data["status"] == "ready"
+            assert data["dependencies"]["database"]["status"] == "connected"
+            assert data["dependencies"]["vector_store"]["status"] == "connected"
+
+
+def test_readiness_probe_database_failure(client):
+    """Verify GET /api/health/ready returns HTTP 503 when database is unhealthy."""
+    with patch("backend.main.check_database_health", return_value=(False, {"status": "unhealthy", "error": "Connection refused"})):
+        with patch("backend.main.check_qdrant_health", return_value=(True, {"status": "connected"})):
+            resp = client.get("/api/health/ready")
+            assert resp.status_code == 503
+            data = resp.json()
+            assert data["status"] == "not_ready"
+            assert data["dependencies"]["database"]["status"] == "unhealthy"
+
+
+def test_readiness_probe_vector_store_failure(client):
+    """Verify GET /api/health/ready returns HTTP 503 when Qdrant is unreachable."""
+    with patch("backend.main.check_database_health", return_value=(True, {"status": "connected"})):
+        with patch("backend.main.check_qdrant_health", return_value=(False, {"status": "unhealthy", "error": "Timeout"})):
+            resp = client.get("/api/health/ready")
+            assert resp.status_code == 503
+            data = resp.json()
+            assert data["status"] == "not_ready"
+            assert data["dependencies"]["vector_store"]["status"] == "unhealthy"
+
+
+if __name__ == "__main__":
+    c = TestClient(app)
+    print("Testing GET /api/health...")
+    test_health_endpoint_metadata(c)
+    print("Testing GET /api/health/live...")
+    test_liveness_probe_returns_alive(c)
+    print("Testing GET /api/health/ready (healthy)...")
+    test_readiness_probe_success(c)
+    print("Testing GET /api/health/ready (db failure -> 503)...")
+    test_readiness_probe_database_failure(c)
+    print("Testing GET /api/health/ready (qdrant failure -> 503)...")
+    test_readiness_probe_vector_store_failure(c)
+    print("All health and readiness probe checks passed successfully.")