diff --git a/workspace/backend/app/main.py b/workspace/backend/app/main.py index db002e0e0..851c2a803 100644 --- a/workspace/backend/app/main.py +++ b/workspace/backend/app/main.py @@ -415,25 +415,27 @@ async def lifespan(app: FastAPI): except Exception as e: logger.error("LIFESPAN: database import failed: %s", e) - logger.info("LIFESPAN: creating timer task") - timer_task = asyncio.create_task(_timer_loop()) - - # One-off browser sweep at boot: sessions leaked before a restart (or - # closes that failed mid-deploy) get released now instead of waiting - # for the first periodic maintenance cycle. - from app.browser_maintenance import sweep_browser_tabs - startup_browser_sweep = asyncio.create_task(sweep_browser_tabs()) # noqa: F841 — keep ref so it isn't GC'd + background_tasks = [] + # Additional HTTP replicas can share the database/cache without adding + # schedulers. At least one deployment must retain background jobs. + if os.environ.get("BACKGROUND_JOBS_ENABLED", "true").strip().lower() in ("1", "true", "yes"): + logger.info("LIFESPAN: creating background tasks") + background_tasks.append(asyncio.create_task(_timer_loop())) + from app.browser_maintenance import sweep_browser_tabs + background_tasks.append(asyncio.create_task(sweep_browser_tabs())) + else: + logger.info("LIFESPAN: background jobs disabled on this replica") logger.info("LIFESPAN: yielding (startup complete)") - yield - timer_task.cancel() try: - await timer_task - except asyncio.CancelledError: - pass - # Shutdown: close Playwright browser - from app.browser import BrowserManager - await BrowserManager.get().shutdown() + yield + finally: + for task in background_tasks: + task.cancel() + await asyncio.gather(*background_tasks, return_exceptions=True) + # Shutdown: close Playwright browser + from app.browser import BrowserManager + await BrowserManager.get().shutdown() app = FastAPI( diff --git a/workspace/backend/tests/test_replica_lifespan.py b/workspace/backend/tests/test_replica_lifespan.py new file mode 100644 index 000000000..3f0c9e3ca --- /dev/null +++ b/workspace/backend/tests/test_replica_lifespan.py @@ -0,0 +1,43 @@ +"""HTTP replicas retain initialization without starting duplicate schedulers.""" +import asyncio +from unittest.mock import AsyncMock, MagicMock + +import pytest +import app.main as main +import app.database as database +import app.browser as browser +import app.browser_maintenance as maintenance + + +@pytest.mark.parametrize('enabled', [None, 'false', 'true']) +def test_replica_lifespan(monkeypatch, enabled): + if enabled is None: + monkeypatch.delenv('BACKGROUND_JOBS_ENABLED', raising=False) + else: + monkeypatch.setenv('BACKGROUND_JOBS_ENABLED', enabled) + monkeypatch.setenv('THREADPOOL_TOKENS', '6') + monkeypatch.setattr(database, '_is_sqlite', False) + started, cancelled = [], [] + + async def job(name): + started.append(name) + try: + await asyncio.Event().wait() + finally: + cancelled.append(name) + + monkeypatch.setattr(main, '_timer_loop', lambda: job('timer')) + monkeypatch.setattr(maintenance, 'sweep_browser_tabs', lambda: job('browser')) + manager = MagicMock(shutdown=AsyncMock()) + monkeypatch.setattr(browser.BrowserManager, 'get', lambda: manager) + + async def run(): + import anyio.to_thread + async with main.lifespan(main.app): + await asyncio.sleep(0) + assert anyio.to_thread.current_default_thread_limiter().total_tokens == 6 + assert sorted(started) == ([] if enabled == 'false' else ['browser', 'timer']) + assert sorted(cancelled) == sorted(started) + manager.shutdown.assert_awaited_once() + + asyncio.run(run()) diff --git a/workspace/operations/digitalocean/multi-node.md b/workspace/operations/digitalocean/multi-node.md new file mode 100644 index 000000000..79ea71c95 --- /dev/null +++ b/workspace/operations/digitalocean/multi-node.md @@ -0,0 +1,37 @@ +# Two backend nodes in SFO3 + +Both HTTP nodes run the same release image and share PostgreSQL, S3 file storage, +Valkey (TLS with the provider CA), and signing/authentication configuration. +Each node runs 8 workers with THREADPOOL_TOKENS=6, DB_POOL_SIZE=6, and +DB_MAX_OVERFLOW=2. The two web containers can use at most 128 PostgreSQL +connections together; account separately for deployment overlap and other clients +against the current 200-connection database ceiling. + +Set BACKGROUND_JOBS_ENABLED=true on the first node and false on the second. +The flag defaults to true for existing installations. Disabling it suppresses the +timer/maintenance loop and startup browser sweep while retaining request thread +limits, readiness, and browser shutdown. This keeps the additional HTTP node +from starting duplicate scheduled jobs. It does not fix existing per-worker +scheduler duplication on the first node. Scheduled jobs do not automatically +fail over if the first node goes down; moving to a dedicated, coordinated +scheduler is a separate change. Never disable jobs everywhere. + +Use a public Regional HTTP load balancer in the same VPC: port 80 to port 80, +HTTP /ready health checks every 10 seconds with a 5-second timeout and 3 healthy +or unhealthy observations, backend keepalive enabled, idle timeout 300 seconds, +sticky sessions and PROXY protocol disabled. Retain existing Cloudflare TLS +behavior when changing the backend DNS record. Validate streaming through the +full path; avoid abruptly stopping a backend with established streams. + +Before enrolling a second backend: verify TLS to Valkey, shared keys and pub/sub +between nodes, database readiness, authenticated polling and file reads. Keep +only one active Redis namespace during cutover; old and new Redis services do not +share cache invalidations or event notifications. Drain/remove old services before +splitting new requests across both nodes. Store credentials in root-readable env +files outside Git and mount the Valkey CA read-only. Configure the same Valkey URL +on every serving node. A single-node managed Valkey cluster is not highly available. + +After rollout: verify both load-balancer targets, public health/readiness, +authenticated requests, live traffic latency, CPU, database connections, and errors +on each node. Keep the former image/configuration and DNS record for rollback. +Rollback nodes together to the same cache endpoint; do not silently split pub/sub.