feat(workspace): support HTTP replicas without background jobs

workspace/backend/app/main.py

@@ -415,25 +415,27 @@ async def lifespan(app: FastAPI):
     except Exception as e:
         logger.error("LIFESPAN: database import failed: %s", e)
 
-    logger.info("LIFESPAN: creating timer task")
-    timer_task = asyncio.create_task(_timer_loop())
-
-    # One-off browser sweep at boot: sessions leaked before a restart (or
-    # closes that failed mid-deploy) get released now instead of waiting
-    # for the first periodic maintenance cycle.
-    from app.browser_maintenance import sweep_browser_tabs
-    startup_browser_sweep = asyncio.create_task(sweep_browser_tabs())  # noqa: F841 — keep ref so it isn't GC'd
+    background_tasks = []
+    # Additional HTTP replicas can share the database/cache without adding
+    # schedulers. At least one deployment must retain background jobs.
+    if os.environ.get("BACKGROUND_JOBS_ENABLED", "true").strip().lower() in ("1", "true", "yes"):
+        logger.info("LIFESPAN: creating background tasks")
+        background_tasks.append(asyncio.create_task(_timer_loop()))
+        from app.browser_maintenance import sweep_browser_tabs
+        background_tasks.append(asyncio.create_task(sweep_browser_tabs()))
+    else:
+        logger.info("LIFESPAN: background jobs disabled on this replica")
 
     logger.info("LIFESPAN: yielding (startup complete)")
-    yield
-    timer_task.cancel()
     try:
-        await timer_task
-    except asyncio.CancelledError:
-        pass
-    # Shutdown: close Playwright browser
-    from app.browser import BrowserManager
-    await BrowserManager.get().shutdown()
+        yield
+    finally:
+        for task in background_tasks:
+            task.cancel()
+        await asyncio.gather(*background_tasks, return_exceptions=True)
+        # Shutdown: close Playwright browser
+        from app.browser import BrowserManager
+        await BrowserManager.get().shutdown()
 
 
 app = FastAPI(

workspace/backend/tests/test_replica_lifespan.py

@@ -0,0 +1,43 @@
+"""HTTP replicas retain initialization without starting duplicate schedulers."""
+import asyncio
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+import app.main as main
+import app.database as database
+import app.browser as browser
+import app.browser_maintenance as maintenance
+
+
+@pytest.mark.parametrize('enabled', [None, 'false', 'true'])
+def test_replica_lifespan(monkeypatch, enabled):
+    if enabled is None:
+        monkeypatch.delenv('BACKGROUND_JOBS_ENABLED', raising=False)
+    else:
+        monkeypatch.setenv('BACKGROUND_JOBS_ENABLED', enabled)
+    monkeypatch.setenv('THREADPOOL_TOKENS', '6')
+    monkeypatch.setattr(database, '_is_sqlite', False)
+    started, cancelled = [], []
+
+    async def job(name):
+        started.append(name)
+        try:
+            await asyncio.Event().wait()
+        finally:
+            cancelled.append(name)
+
+    monkeypatch.setattr(main, '_timer_loop', lambda: job('timer'))
+    monkeypatch.setattr(maintenance, 'sweep_browser_tabs', lambda: job('browser'))
+    manager = MagicMock(shutdown=AsyncMock())
+    monkeypatch.setattr(browser.BrowserManager, 'get', lambda: manager)
+
+    async def run():
+        import anyio.to_thread
+        async with main.lifespan(main.app):
+            await asyncio.sleep(0)
+            assert anyio.to_thread.current_default_thread_limiter().total_tokens == 6
+            assert sorted(started) == ([] if enabled == 'false' else ['browser', 'timer'])
+        assert sorted(cancelled) == sorted(started)
+        manager.shutdown.assert_awaited_once()
+
+    asyncio.run(run())

workspace/operations/digitalocean/multi-node.md

@@ -0,0 +1,37 @@
+# Two backend nodes in SFO3
+
+Both HTTP nodes run the same release image and share PostgreSQL, S3 file storage,
+Valkey (TLS with the provider CA), and signing/authentication configuration.
+Each node runs 8 workers with THREADPOOL_TOKENS=6, DB_POOL_SIZE=6, and
+DB_MAX_OVERFLOW=2. The two web containers can use at most 128 PostgreSQL
+connections together; account separately for deployment overlap and other clients
+against the current 200-connection database ceiling.
+
+Set BACKGROUND_JOBS_ENABLED=true on the first node and false on the second.
+The flag defaults to true for existing installations. Disabling it suppresses the
+timer/maintenance loop and startup browser sweep while retaining request thread
+limits, readiness, and browser shutdown. This keeps the additional HTTP node
+from starting duplicate scheduled jobs. It does not fix existing per-worker
+scheduler duplication on the first node. Scheduled jobs do not automatically
+fail over if the first node goes down; moving to a dedicated, coordinated
+scheduler is a separate change. Never disable jobs everywhere.
+
+Use a public Regional HTTP load balancer in the same VPC: port 80 to port 80,
+HTTP /ready health checks every 10 seconds with a 5-second timeout and 3 healthy
+or unhealthy observations, backend keepalive enabled, idle timeout 300 seconds,
+sticky sessions and PROXY protocol disabled. Retain existing Cloudflare TLS
+behavior when changing the backend DNS record. Validate streaming through the
+full path; avoid abruptly stopping a backend with established streams.
+
+Before enrolling a second backend: verify TLS to Valkey, shared keys and pub/sub
+between nodes, database readiness, authenticated polling and file reads. Keep
+only one active Redis namespace during cutover; old and new Redis services do not
+share cache invalidations or event notifications. Drain/remove old services before
+splitting new requests across both nodes. Store credentials in root-readable env
+files outside Git and mount the Valkey CA read-only. Configure the same Valkey URL
+on every serving node. A single-node managed Valkey cluster is not highly available.
+
+After rollout: verify both load-balancer targets, public health/readiness,
+authenticated requests, live traffic latency, CPU, database connections, and errors
+on each node. Keep the former image/configuration and DNS record for rollback.
+Rollback nodes together to the same cache endpoint; do not silently split pub/sub.