From 3005c305dc08283078f49a5cb57dc02a780c65b9 Mon Sep 17 00:00:00 2001
From: QuangNguyen2609 <quangnguyentechno@gmail.com>
Date: Sat, 17 Oct 2026 11:30:13 +1030
Subject: [PATCH 01/13] feat(api): make the ingress admission wait configurable

SQL_FS_MAX_INGRESS_WAIT_MS sets how long a request or MCP listener waits
for an ingress permit before a 503 EWORKER_BUSY. The default rises from
250 ms to 1000 ms: a refused request is told to retry after 1 s, so
refusing a burst sooner only sends the client to the back of the line.
The worker queue wait keeps its 250 ms default.
---
 .changeset/ingress-wait.md                   |  5 +++
 src/api/ingress-admission.ts                 |  6 ++-
 src/api/tests/unit/ingress-admission.test.ts | 10 +++++
 src/api/tests/unit/ingress-wait.test.ts      | 39 +++++++++++++++++++-
 src/api/tests/unit/sandboxes.test.ts         |  1 +
 5 files changed, 57 insertions(+), 4 deletions(-)
 create mode 100644 .changeset/ingress-wait.md

diff --git a/.changeset/ingress-wait.md b/.changeset/ingress-wait.md
new file mode 100644
index 00000000..83b3ff08
--- /dev/null
+++ b/.changeset/ingress-wait.md
@@ -0,0 +1,5 @@
+---
+"sql-fs-api": patch
+---
+
+Add `SQL_FS_MAX_INGRESS_WAIT_MS`, the time a request or MCP listener waits for ingress capacity before it gets a 503 `EWORKER_BUSY`. The default ingress wait rises from 250 ms to 1000 ms, so one replica can absorb a short burst instead of refusing it. `SQL_FS_WORKER_QUEUE_WAIT_MS` keeps its 250 ms default. An invalid value stops startup.
diff --git a/src/api/ingress-admission.ts b/src/api/ingress-admission.ts
index 61b5b61b..438c43e2 100644
--- a/src/api/ingress-admission.ts
+++ b/src/api/ingress-admission.ts
@@ -32,6 +32,7 @@ export interface IngressOptions {
 	readonly maxBytes: number;
 	readonly maxTenantBytes: number;
 	readonly maxBodyBytes: number;
+	readonly waitMs: number;
 }
 
 export function loadIngressOptions(env: NodeJS.ProcessEnv = process.env): IngressOptions {
@@ -67,6 +68,7 @@ export function loadIngressOptions(env: NodeJS.ProcessEnv = process.env): Ingres
 		maxBytes,
 		maxTenantBytes,
 		maxBodyBytes: read("MAX_REQUEST_BODY_BYTES", 256 * 1024 * 1024),
+		waitMs: read("SQL_FS_MAX_INGRESS_WAIT_MS", 250),
 	};
 }
 
@@ -89,7 +91,7 @@ export class IngressAdmission {
 			maxQueuedPerTenant: options.maxTenantRequests,
 			maxQueuedBytes: 0,
 			maxQueuedBytesPerTenant: 0,
-			waitMs: 250,
+			waitMs: options.waitMs,
 		});
 		this.mcpListenerAdmission = new FairAdmission({
 			capacity: options.maxMcpListeners ?? options.maxRequests,
@@ -98,7 +100,7 @@ export class IngressAdmission {
 			maxQueuedPerTenant: options.maxTenantMcpListeners ?? options.maxTenantRequests,
 			maxQueuedBytes: 0,
 			maxQueuedBytesPerTenant: 0,
-			waitMs: 250,
+			waitMs: options.waitMs,
 		});
 	}
 
diff --git a/src/api/tests/unit/ingress-admission.test.ts b/src/api/tests/unit/ingress-admission.test.ts
index ee1979c8..6b49be33 100644
--- a/src/api/tests/unit/ingress-admission.test.ts
+++ b/src/api/tests/unit/ingress-admission.test.ts
@@ -30,6 +30,7 @@ function setup(maxRequests = 8, maxBytes = 128, maxBodyBytes = 256) {
 		maxBytes,
 		maxTenantBytes: Math.max(1, maxBytes - 1),
 		maxBodyBytes,
+		waitMs: 250,
 	});
 	const app = new Hono<{ Variables: AuthVariables }>();
 	let tenantSequence = 0;
@@ -59,6 +60,11 @@ afterEach(() => {
 });
 
 describe("ingress configuration", () => {
+	it("waits 250 ms at ingress by default, independent of the worker queue wait", () => {
+		expect(loadIngressOptions({ SQL_FS_WORKER_QUEUE_WAIT_MS: "900" }).waitMs).toBe(250);
+		expect(loadIngressOptions({ SQL_FS_MAX_INGRESS_WAIT_MS: "40" }).waitMs).toBe(40);
+	});
+
 	it("reserves explicit raw-body headroom with bounded concurrent requests", () => {
 		expect(loadIngressOptions({})).toEqual({
 			maxRequests: 8,
@@ -68,6 +74,7 @@ describe("ingress configuration", () => {
 			maxBytes: 128 * 1024 * 1024,
 			maxTenantBytes: 96 * 1024 * 1024,
 			maxBodyBytes: 256 * 1024 * 1024,
+			waitMs: 250,
 		});
 	});
 
@@ -79,6 +86,7 @@ describe("ingress configuration", () => {
 		"SQL_FS_MAX_INGRESS_BYTES",
 		"SQL_FS_MAX_INGRESS_TENANT_BYTES",
 		"MAX_REQUEST_BODY_BYTES",
+		"SQL_FS_MAX_INGRESS_WAIT_MS",
 	])("rejects invalid %s instead of disabling its budget", (name) => {
 		for (const value of ["0", "-1", "1.5", "NaN", "Infinity", "9007199254740992"]) {
 			expect(() => loadIngressOptions({ [name]: value })).toThrow(name);
@@ -264,6 +272,7 @@ describe("HTTP body admission", () => {
 			maxBytes: 100,
 			maxTenantBytes: 99,
 			maxBodyBytes: 256,
+			waitMs: 250,
 		});
 		const app = new Hono<{ Variables: AuthVariables }>();
 		app.onError((_error, c) => c.text("body rejected", 400));
@@ -575,6 +584,7 @@ describe("HTTP body admission", () => {
 			maxBytes: 150,
 			maxTenantBytes: 149,
 			maxBodyBytes: 256,
+			waitMs: 250,
 		});
 		const app = new Hono<{ Variables: AuthVariables }>();
 		app.use("/v1/*", authMiddleware);
diff --git a/src/api/tests/unit/ingress-wait.test.ts b/src/api/tests/unit/ingress-wait.test.ts
index 90aa42ab..044a9aae 100644
--- a/src/api/tests/unit/ingress-wait.test.ts
+++ b/src/api/tests/unit/ingress-wait.test.ts
@@ -11,11 +11,12 @@ function deferred() {
 	return { promise, resolve };
 }
 
-function setup(maxRequests = 8, maxTenantRequests = 4) {
+function setup(maxRequests = 8, maxTenantRequests = 4, env: NodeJS.ProcessEnv = {}) {
 	const ingress = new IngressAdmission(
 		loadIngressOptions({
 			SQL_FS_MAX_INGRESS_REQUESTS: String(maxRequests),
 			SQL_FS_MAX_INGRESS_TENANT_REQUESTS: String(maxTenantRequests),
+			...env,
 		}),
 	);
 	const app = new Hono<{ Variables: AuthVariables }>();
@@ -23,7 +24,7 @@ function setup(maxRequests = 8, maxTenantRequests = 4) {
 	app.use("*", (c, next) => {
 		if (c.req.path.startsWith("/bootstrap/")) return ingress.bootstrapMiddleware(c, next);
 		c.set("tenant", c.req.header("X-Tenant") ?? "observer");
-		return ingress.middleware(c, next);
+		return c.req.path.startsWith("/mcp") ? ingress.mcpMiddleware(c, next) : ingress.middleware(c, next);
 	});
 	app.post("/work", async (c) => c.text(await c.req.text()));
 	return { ingress, app };
@@ -196,6 +197,40 @@ describe("bounded ingress wait", () => {
 		}
 	});
 
+	it.each([
+		{ gate: "request", method: "POST", path: "/hold" },
+		{ gate: "MCP listener", method: "GET", path: "/mcp/listen" },
+	])("refuses a queued $gate only after SQL_FS_MAX_INGRESS_WAIT_MS", async ({ method, path }) => {
+		const { app, ingress } = setup(2, 1, { SQL_FS_MAX_INGRESS_WAIT_MS: "40" });
+		const hold = deferred();
+		app.on(["GET", "POST"], path, async (c) => {
+			await hold.promise;
+			return c.text("held");
+		});
+		const active = app.request(path, { method });
+		let waiting: Promise<Response> | undefined;
+		try {
+			await flush();
+			waiting = Promise.resolve(app.request(path, { method }));
+			await flush();
+			await vi.advanceTimersByTimeAsync(39);
+			const gate = () =>
+				method === "GET"
+					? { queued: ingress.metrics.queuedMcpListeners, timeouts: ingress.metrics.mcpListenerQueueTimeouts }
+					: { queued: ingress.metrics.queuedRequests, timeouts: ingress.metrics.queueTimeouts };
+			expect(gate()).toEqual({ queued: 1, timeouts: 0 });
+			await vi.advanceTimersByTimeAsync(1);
+			expect(gate()).toEqual({ queued: 0, timeouts: 1 });
+			const response = await waiting;
+			expect(response.status).toBe(503);
+			expect(response.headers.get("Retry-After")).toBe("1");
+			expect(await response.json()).toMatchObject({ code: "EWORKER_BUSY", retryable: true });
+		} finally {
+			hold.resolve();
+			await Promise.allSettled([active, ...(waiting ? [waiting] : [])].map(consume));
+		}
+	});
+
 	it("cancels a queued request without pulling its body and reuses its pending slot", async () => {
 		const { app, ingress } = setup(2, 1);
 		const hold = deferred();
diff --git a/src/api/tests/unit/sandboxes.test.ts b/src/api/tests/unit/sandboxes.test.ts
index eae9988c..938b1181 100644
--- a/src/api/tests/unit/sandboxes.test.ts
+++ b/src/api/tests/unit/sandboxes.test.ts
@@ -398,6 +398,7 @@ describe("sandbox creation body admission", () => {
 			maxBytes: 128,
 			maxTenantBytes: 96,
 			maxBodyBytes: 64,
+			waitMs: 250,
 		});
 		const app = new Hono<{ Variables: AuthVariables }>();
 		app.onError((_error, c) => c.json({ error: "body rejected" }, 500));

From 7dd026948a179c28b47ca2aebaceb274824a2877 Mon Sep 17 00:00:00 2001
From: QuangNguyen2609 <quangnguyentechno@gmail.com>
Date: Sat, 17 Oct 2026 11:40:47 +1030
Subject: [PATCH 02/13] docs: add an autoscaling guide for probes, metrics and
 replica count

AUTOSCALING.md documents /healthz, /readyz and /metrics, every metric
the server emits, the signals to scale on in priority order, what more
replicas do not fix, measured single-replica capacity, shutdown
draining, the two wait limits, and an Azure Container Apps scale rule.
CONFIGURATION.md gains SQL_FS_MAX_INGRESS_WAIT_MS and points its
autoscaling notes at the new guide.
---
 .changeset/ingress-wait.md |   2 +-
 AGENTS.md                  |   2 +-
 README.md                  |   2 +-
 docs/AUTOSCALING.md        | 269 +++++++++++++++++++++++++++++++++++++
 docs/CONFIGURATION.md      |  15 +--
 5 files changed, 278 insertions(+), 12 deletions(-)
 create mode 100644 docs/AUTOSCALING.md

diff --git a/.changeset/ingress-wait.md b/.changeset/ingress-wait.md
index 83b3ff08..9dcadaed 100644
--- a/.changeset/ingress-wait.md
+++ b/.changeset/ingress-wait.md
@@ -2,4 +2,4 @@
 "sql-fs-api": patch
 ---
 
-Add `SQL_FS_MAX_INGRESS_WAIT_MS`, the time a request or MCP listener waits for ingress capacity before it gets a 503 `EWORKER_BUSY`. The default ingress wait rises from 250 ms to 1000 ms, so one replica can absorb a short burst instead of refusing it. `SQL_FS_WORKER_QUEUE_WAIT_MS` keeps its 250 ms default. An invalid value stops startup.
+Add `SQL_FS_MAX_INGRESS_WAIT_MS`, the time a request or MCP listener waits for ingress capacity before it gets a 503 `EWORKER_BUSY`. The default stays 250 ms, the previous fixed value. An invalid value stops startup.
diff --git a/AGENTS.md b/AGENTS.md
index e53c00f2..c3efe609 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -19,7 +19,7 @@ pnpm verify:architecture # also worker failure, stream finalization, input budge
 
 Use `pnpm check:typescript` or `pnpm check:python` for SDK-only changes. Run a targeted core test with `pnpm exec vitest run src/sql-fs/tests/unit/sql-fs.cache.test.ts`. A nonexistent test selection must fail. `pnpm test:integration` is the lower-level command for an already migrated disposable database with `DATABASE_URL` and `REDIS_URL`; it can skip without those variables. Prefer `pnpm verify:local` for completion evidence.
 
-See [READINESS.md](docs/READINESS.md) for verified scope and hosted follow-up. Read [CONTRIBUTING.md](CONTRIBUTING.md) for setup and contribution steps, [DEVELOPER.md](docs/DEVELOPER.md) before changing consistency or session ownership, and [README.md](README.md) for API behavior, and [CONFIGURATION.md](docs/CONFIGURATION.md) for runtime settings.
+See [READINESS.md](docs/READINESS.md) for verified scope and hosted follow-up. Read [CONTRIBUTING.md](CONTRIBUTING.md) for setup and contribution steps, [DEVELOPER.md](docs/DEVELOPER.md) before changing consistency or session ownership, [README.md](README.md) for API behavior, [CONFIGURATION.md](docs/CONFIGURATION.md) for runtime settings, and [AUTOSCALING.md](docs/AUTOSCALING.md) for probes, metrics and replica scaling.
 
 Python type checking targets the oldest supported interpreter. `pnpm check:python:types` uses uv to provision Python 3.9 in `clients/python/.venv-py39`, separate from the Python 3.12 development environment. This keeps the dependency syntax aligned with the declared target. Mypy stays below version 2 while Python 3.9 is supported.
 
diff --git a/README.md b/README.md
index bfc6d3c8..1c57bf7c 100644
--- a/README.md
+++ b/README.md
@@ -133,7 +133,7 @@ docker run -p 8080:8080 \
 	sql-fs-api
 ```
 
-To run several replicas, point them all at one Postgres database and one control Redis in `REDIS_URL`. Put the blob cache on a separate Redis in `REDIS_DATA_URL` with `allkeys-lru`, and keep the control Redis on `noeviction`. [Redis eviction policy](docs/CONFIGURATION.md#redis-eviction-policy) explains why.
+To run several replicas, point them all at one Postgres database and one control Redis in `REDIS_URL`. Put the blob cache on a separate Redis in `REDIS_DATA_URL` with `allkeys-lru`, and keep the control Redis on `noeviction`. [Redis eviction policy](docs/CONFIGURATION.md#redis-eviction-policy) explains why. [AUTOSCALING.md](docs/AUTOSCALING.md) covers health probes, metrics and scaling the replica count.
 
 ## Develop
 
diff --git a/docs/AUTOSCALING.md b/docs/AUTOSCALING.md
new file mode 100644
index 00000000..a2488380
--- /dev/null
+++ b/docs/AUTOSCALING.md
@@ -0,0 +1,269 @@
+# Autoscaling
+
+This guide is for operators who run sql-fs in containers and want the replica count to follow load. It covers the probe and metrics endpoints, every metric that `GET /metrics` emits, which signals to scale on, measured capacity for one replica, and an example for Azure Container Apps.
+
+Each replica has fixed capacity. By default it runs 2 worker threads and admits 8 requests at a time. A replica never adds workers by itself, so more load needs more replicas. [CONFIGURATION.md](CONFIGURATION.md) lists every setting named here.
+
+## Endpoints
+
+| Endpoint | Auth | Returns | Use it for |
+|---|---|---|---|
+| `GET /healthz` | None | Always `200 {"status":"ok"}` while the process serves HTTP. | Liveness. It does not check Postgres or Redis. |
+| `GET /readyz` | None | `200 {"status":"ok"}` with an `eventLoop` lag summary. If Redis is configured and does not answer `PING` within 2 s, it returns `503 {"status":"degraded"}`. | Readiness and startup. It does not check Postgres. |
+| `GET /metrics` | Bearer JWT | Prometheus text format 0.0.4. | Scraping. |
+
+`/readyz` keeps returning 200 while the server shuts down. The platform must stop routing to a terminating replica by itself. Kubernetes and Azure Container Apps both do so.
+
+`/metrics` accepts the same bearer JWT as the REST API: an HS256 token signed with `AUTH_SECRET`, with a `sub` claim and an optional `tenant` claim that names a configured tenant. Without a valid token it returns 401. The metrics have no tenant labels, so a token for any tenant returns the same numbers. Give the scraper a token with a dedicated `sub`, minted through `POST /v1/auth/admin` or signed with `AUTH_SECRET` directly. In Prometheus, put the token in a file and point the scrape job at it:
+
+```yaml
+scrape_configs:
+  - job_name: sql-fs
+    metrics_path: /metrics
+    authorization:
+      credentials_file: /etc/prometheus/sql-fs-token
+    static_configs:
+      - targets: ["sql-fs-api:8080"]
+```
+
+The server also serves `GET /openapi.json` and `GET /docs` for the API schema. They are not operational endpoints.
+
+## Metric reference
+
+Every metric belongs to one replica. A gauge is the replica's current value. A counter counts from process start and resets when the process restarts, so read counters through `rate()` or `increase()`. Use `sum()` across replicas for fleet totals and `max()` to find the hot replica.
+
+The names below come from `src/api/worker-owner/metrics.ts` and from a scrape of a server started with the default settings.
+
+### Feature flags
+
+| Metric | Type | Meaning |
+|---|---|---|
+| `sql_fs_workers_enabled` | gauge | 1 when `SQL_FS_WORKERS_ENABLED` is `true`. When it is 0, `/metrics` emits only this metric and `sql_fs_ingress_enabled`. |
+| `sql_fs_ingress_enabled` | gauge | 1 when ingress admission is on. It is on exactly when workers are on. |
+
+### Worker pool
+
+| Metric | Type | Meaning |
+|---|---|---|
+| `sql_fs_worker_capacity` | gauge | Configured worker count, `SQL_FS_WORKER_COUNT`. |
+| `sql_fs_worker_ready` | gauge | Workers that are started and able to take work. |
+| `sql_fs_worker_busy` | gauge | Workers that run an operation now. |
+| `sql_fs_worker_resident_sessions` | gauge | Workers that hold a sandbox loaded. A worker keeps the last sandbox it served until another sandbox needs it. |
+| `sql_fs_worker_running_operations` | gauge | Admitted operations that run now. |
+| `sql_fs_worker_lock_waiting` | gauge | Admitted operations that wait for the sandbox's distributed lock. |
+| `sql_fs_worker_recovering_operations` | gauge | Admitted operations that recover a sandbox after a lost worker or lease. |
+| `sql_fs_worker_restarts_total` | counter | Workers replaced after a crash or after they passed a memory limit. |
+| `sql_fs_worker_unknown_outcomes_total` | counter | Execs that failed with `EEXEC_OUTCOME_UNKNOWN`. The script may have run. |
+| `sql_fs_worker_python_active` | gauge | Python executions that hold one of the `MAX_CONCURRENT_PYTHON` slots. |
+| `sql_fs_worker_python_queued` | gauge | Python executions that wait for a slot. |
+| `sql_fs_worker_javascript_active` | gauge | JavaScript executions that hold one of the `MAX_CONCURRENT_JS` slots. |
+| `sql_fs_worker_javascript_queued` | gauge | JavaScript executions that wait for a slot. |
+
+### Worker admission
+
+| Metric | Type | Meaning |
+|---|---|---|
+| `sql_fs_worker_queued` | gauge | Operations that wait for a worker. It is the sum of the next two gauges. |
+| `sql_fs_worker_runnable_queued` | gauge | Queued operations that a free worker could start. |
+| `sql_fs_worker_sandbox_waiting` | gauge | Queued operations that wait behind another operation on the same sandbox. |
+| `sql_fs_worker_queued_bytes` | gauge | Bytes that queued operations hold, counted against `SQL_FS_WORKER_QUEUE_BYTES`. |
+| `sql_fs_worker_offered_total` | counter | Operations that asked for a worker. |
+| `sql_fs_worker_admitted_total` | counter | Operations that got a worker, at once or after a wait. |
+| `sql_fs_worker_refused_total` | counter | Operations that failed with `EWORKER_BUSY` (503). It counts a full queue and a wait that ran out. |
+| `sql_fs_worker_queue_timeouts_total` | counter | The part of `sql_fs_worker_refused_total` whose wait passed `SQL_FS_WORKER_QUEUE_WAIT_MS`. |
+| `sql_fs_worker_cancelled_total` | counter | Operations that left the queue because the client disconnected or the server shut down. |
+| `sql_fs_worker_completed_total` | counter | Operations that released their worker. |
+| `sql_fs_worker_queue_wait_milliseconds` | histogram | Time from the request for a worker to admission. Buckets: 1, 5, 25, 100, 250 and 1000 ms. |
+| `sql_fs_worker_queue_wait_milliseconds_total` | counter | Same value as the histogram's `_sum`. |
+
+The wait histogram counts admitted operations only. An operation admitted at once records 0. A refused operation records nothing, so the histogram never goes above `SQL_FS_WORKER_QUEUE_WAIT_MS` and hides the requests that hurt most. Read it together with the refusal counters.
+
+### Ingress admission
+
+Ingress admission runs before the worker queue. It bounds the requests that one replica holds, including their request bodies.
+
+| Metric | Type | Meaning |
+|---|---|---|
+| `sql_fs_ingress_active_requests` | gauge | Requests that hold one of the `SQL_FS_MAX_INGRESS_REQUESTS` permits. A permit stays held until the response finishes. |
+| `sql_fs_ingress_queued_requests` | gauge | Requests that wait for a permit. |
+| `sql_fs_ingress_refused_total` | counter | Requests that failed with `EWORKER_BUSY` (503) at ingress. It counts a full queue, a wait that ran out and a body that did not fit the byte budget. It does not count MCP listeners. |
+| `sql_fs_ingress_queue_timeouts_total` | counter | The part of `sql_fs_ingress_refused_total` whose wait passed `SQL_FS_MAX_INGRESS_WAIT_MS`. |
+| `sql_fs_ingress_retained_raw_bytes` | gauge | Request-body bytes that admitted requests hold, counted against `SQL_FS_MAX_INGRESS_BYTES`. |
+| `sql_fs_ingress_payload_too_large_total` | counter | Requests that failed with 413 because the body passed its limit. More replicas do not help. |
+| `sql_fs_ingress_aborted_total` | counter | Requests whose client disconnected before the server finished. |
+| `sql_fs_ingress_mcp_listeners_active` | gauge | Open `GET /mcp` streams. |
+| `sql_fs_ingress_mcp_listeners_queued` | gauge | `GET /mcp` streams that wait for a listener permit. |
+| `sql_fs_ingress_mcp_listeners_refused_total` | counter | `GET /mcp` streams that failed with 503. |
+| `sql_fs_ingress_mcp_listeners_queue_timeouts_total` | counter | The part of `sql_fs_ingress_mcp_listeners_refused_total` whose wait ran out. |
+
+Ingress has no wait histogram. Use `sql_fs_ingress_queued_requests` and the refusal counters.
+
+### Memory
+
+| Metric | Type | Meaning |
+|---|---|---|
+| `sql_fs_worker_process_rss_bytes` | gauge | Resident memory of the whole server process, workers included. Compare it with the container memory limit. |
+| `sql_fs_worker_external_bytes` | gauge | Memory outside the V8 heaps that the workers report, such as buffers. |
+
+sql-fs does not export CPU. Read CPU from the platform, for example `container_cpu_usage_seconds_total` from cAdvisor.
+
+## What to scale on
+
+The signals below are in priority order. The queries use a 2 minute window. Shorten it if your scrape interval allows.
+
+### 1. Refusals and timeouts
+
+A 503 `EWORKER_BUSY` is what clients feel. It means a replica had no room for the request within its wait. Scale out when either gate refuses work for more than a short burst.
+
+```promql
+# Refusals per second, both gates, whole fleet
+sum(rate(sql_fs_ingress_refused_total[2m])) + sum(rate(sql_fs_worker_refused_total[2m]))
+
+# Share of worker requests refused
+sum(rate(sql_fs_worker_refused_total[2m])) / sum(rate(sql_fs_worker_offered_total[2m]))
+
+# MCP listener refusals per second
+sum(rate(sql_fs_ingress_mcp_listeners_refused_total[2m]))
+```
+
+Leave `sql_fs_ingress_payload_too_large_total` out of these queries. A body above its limit fails on every replica.
+
+### 2. Queue depth and wait
+
+Queues and waits grow before refusals start, so they give early warning. With the default 250 ms worker wait, a p95 worker wait above about 100 ms means requests are close to refusal.
+
+```promql
+# Worker wait p95, whole fleet. Keep "le" in the grouping.
+histogram_quantile(0.95, sum by (le) (rate(sql_fs_worker_queue_wait_milliseconds_bucket[2m])))
+
+# Worker wait p95 per replica
+histogram_quantile(0.95, sum by (le, instance) (rate(sql_fs_worker_queue_wait_milliseconds_bucket[2m])))
+
+# Work that more workers could start, per replica
+max(sql_fs_worker_runnable_queued)
+
+# Ingress queue, per replica
+max(sql_fs_ingress_queued_requests)
+```
+
+The p95 cannot pass `SQL_FS_WORKER_QUEUE_WAIT_MS`, because a request that waits longer is refused and not recorded. A p95 at the limit means the histogram is saturated, not that the wait is bounded. The top finite bucket is 1000 ms. If you raise `SQL_FS_WORKER_QUEUE_WAIT_MS` above 1000 ms, quantiles above 1000 ms report 1000.
+
+Scale on `sql_fs_worker_runnable_queued`, not `sql_fs_worker_queued`. The difference is `sql_fs_worker_sandbox_waiting`, which more replicas do not reduce.
+
+### 3. CPU and memory
+
+CPU and RSS are backstops. They catch load that the admission gates do not see, such as a few very heavy scripts.
+
+```promql
+# Highest RSS as a share of a 2 GiB limit
+max(sql_fs_worker_process_rss_bytes) / (2 * 1024 * 1024 * 1024)
+```
+
+As a starting point, scale out above about 70% CPU or 70% of the memory limit. Memory is a poor scaling trigger on its own. One large exec can push a replica from idle to near its limit in one request, faster than any autoscaler reacts. The [measured capacity](#measured-capacity) section shows the worst case.
+
+## What scaling does not fix
+
+Some waits stay the same with more replicas. Watch these gauges, but do not scale on them:
+
+- `sql_fs_worker_sandbox_waiting` counts operations queued behind another operation on the same sandbox. One sandbox runs one writer at a time on any replica.
+- `sql_fs_worker_lock_waiting` counts operations that wait for a sandbox's distributed lock, which another replica holds.
+- `sql_fs_worker_recovering_operations` counts recoveries after a lost worker or lease.
+
+One tenant can run at most `SQL_FS_WORKER_ACTIVE_TENANT_MAX` execs at a time on each replica. The default is half the worker count, which is 1 with the default 2 workers. A single heavy tenant therefore needs more replicas, or more workers on each replica, even when the replicas are otherwise idle. Its concurrent execs must stay at or below `replicas * SQL_FS_WORKER_ACTIVE_TENANT_MAX`, and reach that only if the load balancer spreads its requests evenly.
+
+Response bytes have no admission budget. Two large exec outputs on one replica can use most of its memory, and more replicas only help if the load balancer spreads those requests. [#267](https://github.com/Hazzng/sql-fs/issues/267) tracks a per-replica output budget.
+
+## Measured capacity
+
+These numbers come from one-off probes on 2026-10-16 against `main` at `c3fd9e0`. Each probe used one replica with 2 GiB of memory, 2 CPUs and the default settings. The numbers show the shape of the limits. Measure your own workload before you rely on them.
+
+| Load | Result |
+|---|---|
+| 100 agents, separate sandboxes, 5 s think time (about 20 execs/s) | 0 errors, p99 under 50 ms |
+| About 100 execs/s offered by one tenant | Saturates at about 71 execs/s, 23% of requests get 503 |
+| About 100 execs/s offered by ten tenants | 97 execs/s, 0 errors |
+| Small execs at saturation, 2 sandboxes | About 600 execs/s |
+| Small execs at saturation, 100 distinct sandboxes | 65 execs/s with one tenant, 115 execs/s with ten tenants |
+| Two concurrent 64 MiB outputs that are mostly newlines | Peak memory about 1.45 GiB |
+
+One tenant saturates early because it can hold only one of the two workers. Many distinct sandboxes are slower than a few, because each worker keeps one sandbox loaded and switching to another sandbox costs about 9 ms.
+
+To estimate the replica count:
+
+1. Divide the peak execs per second by the per-replica rate. Use about 70 when one tenant sends most of the load, and about 100 when many tenants share it. If each exec touches a different sandbox, use the lower distinct-sandbox rates from the table.
+2. Add about 50% headroom and round up.
+3. Check the per-tenant limit. The busiest tenant's concurrent execs must be at or below `replicas * SQL_FS_WORKER_ACTIVE_TENANT_MAX`.
+
+For example, a peak of 300 execs per second from many tenants needs 300 / 100 = 3 replicas, and 5 with headroom. If one tenant runs 6 execs at a time, 5 replicas with the default limit of 1 is too few. That tenant needs 6 replicas, or 3 or more replicas that each run 4 workers with a limit of 2. A replica with 4 workers needs its memory sized again.
+
+## Doing it well
+
+- Run at least 2 replicas. A single replica drops all traffic while it restarts or drains.
+- Configure Redis before you run more than one replica. `REDIS_URL` carries the sandbox locks. The [README](../README.md#deploy-with-docker) describes the multi-replica setup.
+- Scale out fast and scale in slowly. A short burst of refusals should add replicas within a minute. Removing replicas should wait for several minutes of low load, so that the count does not swing.
+- Let a replica drain before it stops. A script that is cut off mid-run has an unknown outcome, and the client gets no result.
+- Make clients retry. Retry a 503 with `retryable: true` after `Retry-After`, with jitter. Also retry connection errors on large uploads, because a client can see a refused upload as a socket error (`UND_ERR_SOCKET`) instead of the 503. Never retry `EEXEC_OUTCOME_UNKNOWN` automatically.
+- Size the caps for each box size and keep them fixed. Do not change worker counts, output caps or ingress limits at runtime to absorb load. Add replicas instead. The memory measurements in [CONFIGURATION.md](CONFIGURATION.md#sizing) hold only for the caps they were taken with.
+
+### Shutdown and draining
+
+On `SIGTERM` or `SIGINT`, the server does the following:
+
+1. It closes ingress admission. Queued requests, and new requests on open connections, get 503 `ESHUTTINGDOWN` with `Retry-After: 1`. Admitted requests continue.
+2. It stops accepting connections and waits for open requests to finish.
+3. It closes MCP sessions, then lets running worker operations finish for up to 30 s, then closes Postgres and Redis. It exits with code 0.
+4. If shutdown has not finished 60 s after the signal, it logs `shutdown_force_exit` and exits with code 1.
+
+Neither the 30 s nor the 60 s limit is configurable. A script can run for up to `SQL_FS_WORKER_EXECUTION_MS`, 120 s by default, so a long script can still be cut off. Set the platform's termination grace period above 60 s so that the platform does not kill the process before the server's own limit. To avoid cut-off scripts, keep `SQL_FS_WORKER_EXECUTION_MS` below about 60 s.
+
+### Wait limits
+
+A request can wait twice before it gets a 503. It first waits for an ingress permit, up to `SQL_FS_MAX_INGRESS_WAIT_MS`. An exec then waits for a worker, up to `SQL_FS_WORKER_QUEUE_WAIT_MS`. Both default to 250 ms, so an exec can queue for up to 0.5 s before it fails.
+
+A longer ingress wait does not absorb bursts. The ingress queue holds at most as many requests as ingress admits at once: `SQL_FS_MAX_INGRESS_REQUESTS` for the replica and `SQL_FS_MAX_INGRESS_TENANT_REQUESTS` for each tenant, 8 and 4 by default. In a burst of 100 requests, 8 run, 8 wait and the rest are refused at once, whatever the wait. The wait only decides how long those few queued requests hold their places. [#269](https://github.com/Hazzng/sql-fs/issues/269) tracks a deeper ingress queue.
+
+These runs used the same 2 GiB replica as [Measured capacity](#measured-capacity), with clients that waited 1 s after each 503, as `Retry-After: 1` asks. They ran on 2026-10-17.
+
+| Load | `SQL_FS_MAX_INGRESS_WAIT_MS=250` | `SQL_FS_MAX_INGRESS_WAIT_MS=1000` |
+|---|---|---|
+| 100 agents start at once, 10 tenants | 83 refusals | 83 refusals |
+| One tenant, about 100 execs/s offered, three 30 s runs: refusals | 317, 385, 395 | 403, 479, 414 |
+| The same runs: successful execs/s | 74.5, 69.8, 68.5 | 67.8, 63.4, 67.2 |
+| The same runs: p99 latency | 123, 151, 174 ms | 246, 283, 173 ms |
+
+The longer wait refused as many requests in the burst, and slightly more under sustained overload, with higher tail latency. Keep the default. Raise the wait only if your clients cannot retry, and add replicas for any load that refuses requests for more than a burst.
+
+## Example: Azure Container Apps
+
+`deploy/azure/aca.yaml` deploys sql-fs to Azure Container Apps. Its scale rule targets 50 concurrent requests for each replica. One replica admits only 8 requests by default and runs 2 operations at a time, so that rule starts adding replicas long after requests begin to fail. The example below shows only the fields to change. Merge them into the `template` block of `aca.yaml`, which keeps the image, environment and probes. It is a starting point, not a tested configuration.
+
+```yaml
+template:
+  terminationGracePeriodSeconds: 90
+  containers:
+    - name: api
+      resources:
+        cpu: 2.0
+        memory: 4Gi
+  scale:
+    minReplicas: 2
+    maxReplicas: 10
+    rules:
+      - name: concurrent-requests
+        http:
+          metadata:
+            concurrentRequests: "4"
+      - name: worker-refusals
+        custom:
+          type: prometheus
+          metadata:
+            serverAddress: http://prometheus.example.internal:9090
+            query: sum(rate(sql_fs_ingress_refused_total[2m])) + sum(rate(sql_fs_worker_refused_total[2m]))
+            threshold: "1"
+```
+
+The HTTP rule needs no Prometheus. It scales on concurrent requests, which track the ingress and worker gates. Set `concurrentRequests` to about the worker count plus the queue you accept. Use 4 for short execs with 2 workers. Use a value closer to the worker count when scripts run for seconds.
+
+The `prometheus` rule is a KEDA scaler. It needs a Prometheus server that scrapes every replica with the token from [Endpoints](#endpoints). It asks for one replica for each refusal per second. Container Apps uses the highest replica count that any rule asks for. Once refusals stop, this rule asks for none, so the HTTP rule holds the steady count.
+
+On the Consumption plan, CPU and memory come in fixed pairs, and 2 CPUs come with 4 GiB. The measured capacity above used 2 CPUs and 2 GiB.
diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md
index dbcc1e31..e66c2b83 100644
--- a/docs/CONFIGURATION.md
+++ b/docs/CONFIGURATION.md
@@ -171,7 +171,9 @@ To upgrade replicas that run with workers off, roll out in two stages. A rolling
 1. Deploy the new version with `SQL_FS_WORKERS_ENABLED=false` on every replica.
 2. When no replica of the older version remains, remove the setting and roll out again.
 
-**Admission.** Each tenant has a queue, and the server takes requests from the queues in turn. On one sandbox, a waiting writer goes before new readers. With the defaults, one tenant can hold one of the two workers and four of the eight ingress permits. Long-lived `GET /mcp` streams count against a separate listener quota. If a request waits more than 250 ms, it fails with `EWORKER_BUSY` (503) and `Retry-After: 1`.
+**Admission.** Each tenant has a queue, and the server takes requests from the queues in turn. On one sandbox, a waiting writer goes before new readers. With the defaults, one tenant can hold one of the two workers and four of the eight ingress permits. Long-lived `GET /mcp` streams count against a separate listener quota. A request that waits longer than `SQL_FS_MAX_INGRESS_WAIT_MS` for an ingress permit, or an exec that waits longer than `SQL_FS_WORKER_QUEUE_WAIT_MS` for a worker, fails with `EWORKER_BUSY` (503) and `Retry-After: 1`. With the defaults, an exec can queue for up to 0.5 s before it fails.
+
+A longer ingress wait does not absorb bursts, because the ingress queue holds no more requests than ingress admits at once. [Wait limits](AUTOSCALING.md#wait-limits) has the measurements.
 
 **Unknown outcomes.** If a script loses its worker, lease or response after it starts, the request fails with `EEXEC_OUTCOME_UNKNOWN` (503, `retryable: false`). The server and the SDKs do not replay it. That includes read-only scripts, because they can still have network side effects. Read the sandbox state before the next action. Earlier items in a batch can already be committed.
 
@@ -199,6 +201,7 @@ Every worker setting must be a positive integer. If a global limit is above 1, i
 | `SQL_FS_MAX_INGRESS_TENANT_BYTES` | Three quarters of `SQL_FS_MAX_INGRESS_BYTES`, at least 1 |
 | `SQL_FS_MAX_INGRESS_MCP_LISTENERS` | 8 listeners |
 | `SQL_FS_MAX_INGRESS_TENANT_MCP_LISTENERS` | 4 listeners |
+| `SQL_FS_MAX_INGRESS_WAIT_MS` | 250 ms |
 | `SQL_FS_WORKER_MAX_RSS_BYTES` | 1.5 GiB |
 | `SQL_FS_WORKER_MAX_EXTERNAL_BYTES` | 512 MiB |
 | `SQL_FS_WORKER_SOURCE_BYTES` | 1 MiB |
@@ -212,7 +215,7 @@ Every worker setting must be a positive integer. If a global limit is above 1, i
 | `SQL_FS_WORKER_SCRIPT_BUFFER_OPS` | 50000 changes |
 | `SQL_FS_WORKER_SCRIPT_BUFFER_BYTES` | 32 MiB |
 
-A replica does not add workers by itself. Scale the number of replicas on CPU and RSS, together with the worker queue, admission delay and refusal metrics from `GET /metrics`. Waits on one sandbox, on distributed locks and on recovery have their own gauges. More replicas do not shorten those waits.
+A replica does not add workers by itself. To handle more load, add replicas. [AUTOSCALING.md](AUTOSCALING.md) describes the metrics to scale on and the waits that more replicas do not shorten.
 
 ### Sizing
 
@@ -232,13 +235,7 @@ Ten concurrent 64 MiB `PUT` uploads, with `MAX_FILE_WRITE_BYTES` raised to 64 Mi
 
 One replica served 100 agents with separate sandboxes and 5 s think time, about 20 execs per second, with 0 errors and p99 latency under 50 ms. At about 100 execs per second, one tenant saturated at about 71 per second, because its active limit is half the worker count, 1 per replica. Ten tenants reached 97 per second with 0 errors.
 
-Add replicas instead of raising caps, and keep the caps fixed for each box size. Autoscale on these signals:
-
-- Queue wait, from the `sql_fs_worker_queue_wait_milliseconds` histogram.
-- Queue depth, from `sql_fs_worker_queued` and `sql_fs_ingress_queued_requests`.
-- Refusals and timeouts, from `sql_fs_worker_refused_total`, `sql_fs_worker_queue_timeouts_total`, `sql_fs_ingress_refused_total` and `sql_fs_ingress_queue_timeouts_total`.
-- Memory, from `sql_fs_worker_process_rss_bytes`.
-- CPU.
+Add replicas instead of raising caps, and keep the caps fixed for each box size. [AUTOSCALING.md](AUTOSCALING.md) lists the metrics to scale on, a replica-count estimate and an example scale rule.
 
 ### Large files
 

From 3f174095d5a57fb3fe3bc7e75ffc4cfd6e89da01 Mon Sep 17 00:00:00 2001
From: QuangNguyen2609 <quangnguyentechno@gmail.com>
Date: Sat, 17 Oct 2026 12:50:51 +1030
Subject: [PATCH 03/13] fix(api): end MCP listener streams before shutdown
 waits for connections

server.close() waited on every open connection before shutdownMcp() ran, so a
client holding a GET /mcp listener stream blocked the worker drain until the
60 s force exit. Shutdown now ends standalone listener streams first and leaves
in-flight tool calls to finish.
---
 .changeset/mcp-shutdown.md  |  5 +++
 scripts/verify-local.sh     |  1 +
 scripts/verify/shutdown.mjs | 61 +++++++++++++++++++++++++++++++++++++
 src/api/mcp/server.ts       | 14 +++++++++
 src/api/server.ts           |  3 +-
 5 files changed, 83 insertions(+), 1 deletion(-)
 create mode 100644 .changeset/mcp-shutdown.md
 create mode 100644 scripts/verify/shutdown.mjs

diff --git a/.changeset/mcp-shutdown.md b/.changeset/mcp-shutdown.md
new file mode 100644
index 00000000..37b8ec36
--- /dev/null
+++ b/.changeset/mcp-shutdown.md
@@ -0,0 +1,5 @@
+---
+"sql-fs-api": patch
+---
+
+Shut down promptly when MCP clients hold a `GET /mcp` listener stream open. The server waited for those streams before it drained workers, so every such shutdown ran into the 60 s force exit and exited with code 1. It now ends listener streams first and lets in-flight MCP tool calls finish.
diff --git a/scripts/verify-local.sh b/scripts/verify-local.sh
index 0d54935d..f28f283e 100644
--- a/scripts/verify-local.sh
+++ b/scripts/verify-local.sh
@@ -50,6 +50,7 @@ pnpm build
 pnpm --dir clients/typescript build
 "${verify_env[@]}" node scripts/verify/contracts.mjs
 "${verify_env[@]}" node scripts/verify/output-cap.mjs
+"${verify_env[@]}" node scripts/verify/shutdown.mjs
 "${verify_env[@]}" pnpm test:integration --reporter=default --reporter=json --outputFile="$evidence/integration.json"
 node --input-type=module - "$evidence/integration.json" <<'JS'
 import assert from "node:assert/strict";
diff --git a/scripts/verify/shutdown.mjs b/scripts/verify/shutdown.mjs
new file mode 100644
index 00000000..485af0f2
--- /dev/null
+++ b/scripts/verify/shutdown.mjs
@@ -0,0 +1,61 @@
+#!/usr/bin/env node
+import { once } from "node:events";
+import { performance } from "node:perf_hooks";
+import { LocalApiRun } from "./local-api.mjs";
+
+const run = new LocalApiRun("shutdown");
+let failure;
+try {
+	const server = await run.start({ SQL_FS_WORKERS_ENABLED: "true" });
+	await run.start({ SQL_FS_WORKERS_ENABLED: "true" });
+	const owner = await run.owner();
+	const headers = {
+		Authorization: `Bearer ${owner.token}`,
+		Accept: "application/json, text/event-stream",
+		"Content-Type": "application/json",
+	};
+	const initialize = await fetch(`${server.base}/mcp`, {
+		method: "POST",
+		headers,
+		body: JSON.stringify({
+			jsonrpc: "2.0",
+			id: 1,
+			method: "initialize",
+			params: { protocolVersion: "2025-03-26", capabilities: {}, clientInfo: { name: "shutdown-verifier", version: "1" } },
+		}),
+	});
+	await initialize.text();
+	const sessionId = initialize.headers.get("mcp-session-id");
+	run.equal(typeof sessionId === "string" && sessionId.length > 0, true, "MCP session opened");
+	const sessionHeaders = { ...headers, "mcp-session-id": sessionId };
+	const initialized = await fetch(`${server.base}/mcp`, {
+		method: "POST",
+		headers: sessionHeaders,
+		body: JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized" }),
+	});
+	await initialized.text();
+	const listener = await fetch(`${server.base}/mcp`, {
+		method: "GET",
+		headers: { ...sessionHeaders, Accept: "text/event-stream" },
+	});
+	run.equal(listener.status, 200, "MCP listener stream is open before SIGTERM");
+	const drained = listener.body
+		.pipeTo(new WritableStream())
+		.then(() => "ended")
+		.catch(() => "aborted");
+
+	const started = performance.now();
+	const exited = once(server.child, "exit");
+	server.child.kill("SIGTERM");
+	const [code] = await exited;
+	const seconds = (performance.now() - started) / 1000;
+	run.record({ case: "sigterm-with-mcp-listener", exitCode: code, seconds, listener: await drained });
+	run.equal(code, 0, "SIGTERM with an open MCP listener exits cleanly");
+	run.equal(seconds < 10, true, "SIGTERM with an open MCP listener does not wait for the force-exit timer");
+	run.equal(server.log.includes('"event":"shutdown_complete"'), true, "Shutdown logs shutdown_complete");
+	run.equal(server.log.includes('"event":"shutdown_force_exit"'), false, "Shutdown does not force-exit");
+} catch (error) {
+	failure = error;
+} finally {
+	await run.finish(failure);
+}
diff --git a/src/api/mcp/server.ts b/src/api/mcp/server.ts
index 7ab8b101..33e30c82 100644
--- a/src/api/mcp/server.ts
+++ b/src/api/mcp/server.ts
@@ -78,6 +78,20 @@ export function startMcpSessionSweeper(intervalMs: number = MCP_SWEEP_INTERVAL_M
 	if (typeof sweeperTimer.unref === "function") sweeperTimer.unref();
 }
 
+/**
+ * End every session's standalone `GET /mcp` stream, leaving in-flight tool calls to finish. Shutdown calls this
+ * before `server.close()`, which otherwise waits on those long-lived connections until the force-exit timer.
+ */
+export function closeMcpListeners(): void {
+	for (const entry of sessions.values()) {
+		try {
+			entry.transport.closeStandaloneSSEStream();
+		} catch {
+			// best-effort
+		}
+	}
+}
+
 /** Stop the sweeper and close every active MCP transport. Used during shutdown. */
 export async function shutdownMcp(): Promise<void> {
 	if (sweeperTimer !== undefined) {
diff --git a/src/api/server.ts b/src/api/server.ts
index a82c9790..d6c90df6 100644
--- a/src/api/server.ts
+++ b/src/api/server.ts
@@ -29,7 +29,7 @@ import {
 import { loadExecLockOptions } from "./exec-lock-config.js";
 import { IngressAdmission, loadIngressOptions } from "./ingress-admission.js";
 import { configuredBoolean, loadExecOutputBytes } from "./lib/env.js";
-import { handleMcpRequest, shutdownMcp, startMcpSessionSweeper } from "./mcp/server.js";
+import { closeMcpListeners, handleMcpRequest, shutdownMcp, startMcpSessionSweeper } from "./mcp/server.js";
 import { mcpOptionsResponse, withMcpCors } from "./mcp-cors.js";
 import { runMigrations } from "./migrations.js";
 import { openapiSpec } from "./openapi-spec.js";
@@ -376,6 +376,7 @@ if (isMain) {
 			shuttingDown = true;
 			console.log(JSON.stringify({ event: "shutdown_begin" }));
 			ingressAdmission?.close();
+			closeMcpListeners();
 			stopEventLoopMonitor();
 			// Force-exit guard so a hung Postgres or Redis cleanup cannot keep
 			// the process alive past the orchestrator's grace period.

From 23cf6c8ea8cbd5aa56fffb6e53fc41df86196bab Mon Sep 17 00:00:00 2001
From: QuangNguyen2609 <quangnguyentechno@gmail.com>
Date: Sat, 17 Oct 2026 12:50:54 +1030
Subject: [PATCH 04/13] fix(api): reject wait limits above Node's timer maximum

---
 .changeset/ingress-wait.md                        |  2 +-
 src/api/ingress-admission.ts                      |  3 ++-
 src/api/lib/env.ts                                |  8 ++++++++
 src/api/tests/unit/ingress-admission.test.ts      |  7 +++++++
 src/api/tests/unit/worker-resource-policy.test.ts |  6 ++++++
 src/api/worker-owner/pool.ts                      | 10 ++++++++--
 6 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/.changeset/ingress-wait.md b/.changeset/ingress-wait.md
index 9dcadaed..5605ffa0 100644
--- a/.changeset/ingress-wait.md
+++ b/.changeset/ingress-wait.md
@@ -2,4 +2,4 @@
 "sql-fs-api": patch
 ---
 
-Add `SQL_FS_MAX_INGRESS_WAIT_MS`, the time a request or MCP listener waits for ingress capacity before it gets a 503 `EWORKER_BUSY`. The default stays 250 ms, the previous fixed value. An invalid value stops startup.
+Add `SQL_FS_MAX_INGRESS_WAIT_MS`, the time a request or MCP listener waits for ingress capacity before it gets a 503 `EWORKER_BUSY`. The default stays 250 ms, the previous fixed value. Like the other ingress settings, an empty value uses the default. Any other invalid value, including a wait above 2147483647 ms, stops startup. `SQL_FS_WORKER_QUEUE_WAIT_MS` now rejects values above 2147483647 ms too, because Node would run such a timer after 1 ms.
diff --git a/src/api/ingress-admission.ts b/src/api/ingress-admission.ts
index 438c43e2..96cb9899 100644
--- a/src/api/ingress-admission.ts
+++ b/src/api/ingress-admission.ts
@@ -2,6 +2,7 @@ import { AsyncLocalStorage } from "node:async_hooks";
 import { ServerResponse } from "node:http";
 import { Http2ServerResponse } from "node:http2";
 import type { Context, MiddlewareHandler, Next } from "hono";
+import { assertTimerMs } from "./lib/env.js";
 import { FairAdmission } from "./worker-owner/admission.js";
 import { WorkerBusyError } from "./worker-owner/protocol.js";
 
@@ -68,7 +69,7 @@ export function loadIngressOptions(env: NodeJS.ProcessEnv = process.env): Ingres
 		maxBytes,
 		maxTenantBytes,
 		maxBodyBytes: read("MAX_REQUEST_BODY_BYTES", 256 * 1024 * 1024),
-		waitMs: read("SQL_FS_MAX_INGRESS_WAIT_MS", 250),
+		waitMs: assertTimerMs("SQL_FS_MAX_INGRESS_WAIT_MS", read("SQL_FS_MAX_INGRESS_WAIT_MS", 250)),
 	};
 }
 
diff --git a/src/api/lib/env.ts b/src/api/lib/env.ts
index f1f757f4..9d685574 100644
--- a/src/api/lib/env.ts
+++ b/src/api/lib/env.ts
@@ -30,6 +30,14 @@ export function configuredInteger(env: NodeJS.ProcessEnv, name: string, fallback
 	return parsed;
 }
 
+/** Node runs any longer `setTimeout` delay after 1 ms, so a larger wait would refuse requests at once. */
+export const MAX_TIMER_MS = 2_147_483_647;
+
+export function assertTimerMs(name: string, value: number): number {
+	if (value > MAX_TIMER_MS) throw new Error(`${name} must be at most ${MAX_TIMER_MS}`);
+	return value;
+}
+
 export function configuredBoolean(env: NodeJS.ProcessEnv, name: string, fallback: boolean): boolean {
 	const value = env[name];
 	if (value === undefined) return fallback;
diff --git a/src/api/tests/unit/ingress-admission.test.ts b/src/api/tests/unit/ingress-admission.test.ts
index 6b49be33..58c7ddab 100644
--- a/src/api/tests/unit/ingress-admission.test.ts
+++ b/src/api/tests/unit/ingress-admission.test.ts
@@ -65,6 +65,13 @@ describe("ingress configuration", () => {
 		expect(loadIngressOptions({ SQL_FS_MAX_INGRESS_WAIT_MS: "40" }).waitMs).toBe(40);
 	});
 
+	it("rejects an ingress wait that Node's timer would cut to 1 ms", () => {
+		expect(loadIngressOptions({ SQL_FS_MAX_INGRESS_WAIT_MS: "2147483647" }).waitMs).toBe(2147483647);
+		expect(() => loadIngressOptions({ SQL_FS_MAX_INGRESS_WAIT_MS: "2147483648" })).toThrow(
+			"SQL_FS_MAX_INGRESS_WAIT_MS must be at most 2147483647",
+		);
+	});
+
 	it("reserves explicit raw-body headroom with bounded concurrent requests", () => {
 		expect(loadIngressOptions({})).toEqual({
 			maxRequests: 8,
diff --git a/src/api/tests/unit/worker-resource-policy.test.ts b/src/api/tests/unit/worker-resource-policy.test.ts
index a1e9c515..be104e72 100644
--- a/src/api/tests/unit/worker-resource-policy.test.ts
+++ b/src/api/tests/unit/worker-resource-policy.test.ts
@@ -130,6 +130,12 @@ describe("worker resource policy", () => {
 		).toBe(65 * 1024 * 1024);
 	});
 
+	it("rejects a worker queue wait that Node's timer would cut to 1 ms", () => {
+		expect(() => loadWorkerPoolOptions({ SQL_FS_WORKER_QUEUE_WAIT_MS: "2147483648" })).toThrow(
+			"SQL_FS_WORKER_QUEUE_WAIT_MS must be at most 2147483647",
+		);
+	});
+
 	it("keeps an explicit SQL_FS_WORKER_MESSAGE_BYTES over the output-derived default", () => {
 		expect(loadWorkerPoolOptions({ SQL_FS_WORKER_MESSAGE_BYTES: "4096" }).executionLimits.maxWorkerMessageBytes).toBe(
 			4096,
diff --git a/src/api/worker-owner/pool.ts b/src/api/worker-owner/pool.ts
index 1e1831fe..fcba45f3 100644
--- a/src/api/worker-owner/pool.ts
+++ b/src/api/worker-owner/pool.ts
@@ -4,7 +4,13 @@ import { availableParallelism } from "node:os";
 import { Worker } from "node:worker_threads";
 import { retirePostgresSandboxGeneration } from "../../sql-fs/index.js";
 import type { ScriptTxBufferConfig } from "../../sql-fs/script-tx-buffer.js";
-import { configuredBoolean, configuredInteger, loadExecOutputBytes, positiveIntEnv } from "../lib/env.js";
+import {
+	assertTimerMs,
+	configuredBoolean,
+	configuredInteger,
+	loadExecOutputBytes,
+	positiveIntEnv,
+} from "../lib/env.js";
 import { readOnlyContext } from "../read-only-context.js";
 import { FairAdmission } from "./admission.js";
 import { assertWorkerMessageBudget } from "./message-budget.js";
@@ -117,7 +123,7 @@ export function loadWorkerPoolOptions(
 			"SQL_FS_WORKER_QUEUE_TENANT_BYTES",
 			Math.max(1, Math.floor(maxQueuedBytes / 2)),
 		),
-		waitMs: configuredInteger(env, "SQL_FS_WORKER_QUEUE_WAIT_MS", 250),
+		waitMs: assertTimerMs("SQL_FS_WORKER_QUEUE_WAIT_MS", configuredInteger(env, "SQL_FS_WORKER_QUEUE_WAIT_MS", 250)),
 		cancelGraceMs: configuredInteger(env, "SQL_FS_WORKER_CANCEL_GRACE_MS", 500),
 		startupMs: configuredInteger(env, "SQL_FS_WORKER_STARTUP_MS", 10_000),
 		maxRssBytes: configuredInteger(env, "SQL_FS_WORKER_MAX_RSS_BYTES", 1536 * 1024 * 1024),

From a0ac855ee24235946285f9966aad80f132836f2f Mon Sep 17 00:00:00 2001
From: QuangNguyen2609 <quangnguyentechno@gmail.com>
Date: Sat, 17 Oct 2026 12:50:58 +1030
Subject: [PATCH 05/13] docs: scrape every replica, state per-tenant queue
 limits and the new shutdown order

---
 docs/AUTOSCALING.md | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/docs/AUTOSCALING.md b/docs/AUTOSCALING.md
index a2488380..3d38b131 100644
--- a/docs/AUTOSCALING.md
+++ b/docs/AUTOSCALING.md
@@ -16,16 +16,27 @@ Each replica has fixed capacity. By default it runs 2 worker threads and admits
 
 `/metrics` accepts the same bearer JWT as the REST API: an HS256 token signed with `AUTH_SECRET`, with a `sub` claim and an optional `tenant` claim that names a configured tenant. Without a valid token it returns 401. The metrics have no tenant labels, so a token for any tenant returns the same numbers. Give the scraper a token with a dedicated `sub`, minted through `POST /v1/auth/admin` or signed with `AUTH_SECRET` directly. In Prometheus, put the token in a file and point the scrape job at it:
 
+Scrape every replica. A load-balanced service address reaches one replica per scrape, so fleet sums and per-replica queries would describe a random replica. On Kubernetes, discover the pods:
+
 ```yaml
 scrape_configs:
   - job_name: sql-fs
     metrics_path: /metrics
     authorization:
       credentials_file: /etc/prometheus/sql-fs-token
-    static_configs:
-      - targets: ["sql-fs-api:8080"]
+    kubernetes_sd_configs:
+      - role: pod
+    relabel_configs:
+      - source_labels: [__meta_kubernetes_pod_label_app]
+        regex: sql-fs-api
+        action: keep
+      - source_labels: [__meta_kubernetes_pod_ip]
+        target_label: __address__
+        replacement: "$1:8080"
 ```
 
+Change the `app` label to match your deployment. On a platform that does not expose each replica's address, use its own per-replica metrics collection instead of one scrape target.
+
 The server also serves `GET /openapi.json` and `GET /docs` for the API schema. They are not operational endpoints.
 
 ## Metric reference
@@ -210,9 +221,10 @@ For example, a peak of 300 execs per second from many tenants needs 300 / 100 =
 On `SIGTERM` or `SIGINT`, the server does the following:
 
 1. It closes ingress admission. Queued requests, and new requests on open connections, get 503 `ESHUTTINGDOWN` with `Retry-After: 1`. Admitted requests continue.
-2. It stops accepting connections and waits for open requests to finish.
-3. It closes MCP sessions, then lets running worker operations finish for up to 30 s, then closes Postgres and Redis. It exits with code 0.
-4. If shutdown has not finished 60 s after the signal, it logs `shutdown_force_exit` and exits with code 1.
+2. It ends every MCP session's `GET /mcp` listener stream. In-flight MCP tool calls continue.
+3. It stops accepting connections and waits for open requests to finish.
+4. It closes MCP sessions, then lets running worker operations finish for up to 30 s, then closes Postgres and Redis. It exits with code 0.
+5. If shutdown has not finished 60 s after the signal, it logs `shutdown_force_exit` and exits with code 1.
 
 Neither the 30 s nor the 60 s limit is configurable. A script can run for up to `SQL_FS_WORKER_EXECUTION_MS`, 120 s by default, so a long script can still be cut off. Set the platform's termination grace period above 60 s so that the platform does not kill the process before the server's own limit. To avoid cut-off scripts, keep `SQL_FS_WORKER_EXECUTION_MS` below about 60 s.
 
@@ -220,7 +232,7 @@ Neither the 30 s nor the 60 s limit is configurable. A script can run for up to
 
 A request can wait twice before it gets a 503. It first waits for an ingress permit, up to `SQL_FS_MAX_INGRESS_WAIT_MS`. An exec then waits for a worker, up to `SQL_FS_WORKER_QUEUE_WAIT_MS`. Both default to 250 ms, so an exec can queue for up to 0.5 s before it fails.
 
-A longer ingress wait does not absorb bursts. The ingress queue holds at most as many requests as ingress admits at once: `SQL_FS_MAX_INGRESS_REQUESTS` for the replica and `SQL_FS_MAX_INGRESS_TENANT_REQUESTS` for each tenant, 8 and 4 by default. In a burst of 100 requests, 8 run, 8 wait and the rest are refused at once, whatever the wait. The wait only decides how long those few queued requests hold their places. [#269](https://github.com/Hazzng/sql-fs/issues/269) tracks a deeper ingress queue.
+A longer ingress wait does not absorb bursts. The ingress queue holds at most as many requests as ingress admits at once: `SQL_FS_MAX_INGRESS_REQUESTS` for the replica and `SQL_FS_MAX_INGRESS_TENANT_REQUESTS` for each tenant, 8 and 4 by default. In a burst of 100 requests spread across tenants, 8 run, 8 wait and the rest are refused at once, whatever the wait. One tenant on its own gets 4 running and 4 waiting. The wait only decides how long those few queued requests hold their places. [#269](https://github.com/Hazzng/sql-fs/issues/269) tracks a deeper ingress queue.
 
 These runs used the same 2 GiB replica as [Measured capacity](#measured-capacity), with clients that waited 1 s after each 503, as `Retry-After: 1` asks. They ran on 2026-10-17.
 

From 21aab70eb9fb632c07518d04e13aae8e6d368807 Mon Sep 17 00:00:00 2001
From: QuangNguyen2609 <quangnguyentechno@gmail.com>
Date: Sat, 17 Oct 2026 12:58:47 +1030
Subject: [PATCH 06/13] test(verify): start one API in the shutdown check

---
 scripts/verify/shutdown.mjs | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/scripts/verify/shutdown.mjs b/scripts/verify/shutdown.mjs
index 485af0f2..0b1848a1 100644
--- a/scripts/verify/shutdown.mjs
+++ b/scripts/verify/shutdown.mjs
@@ -1,16 +1,19 @@
 #!/usr/bin/env node
 import { once } from "node:events";
+import { join } from "node:path";
 import { performance } from "node:perf_hooks";
-import { LocalApiRun } from "./local-api.mjs";
+import { LocalApiRun, repo } from "./local-api.mjs";
 
 const run = new LocalApiRun("shutdown");
 let failure;
 try {
 	const server = await run.start({ SQL_FS_WORKERS_ENABLED: "true" });
-	await run.start({ SQL_FS_WORKERS_ENABLED: "true" });
-	const owner = await run.owner();
+	// This check creates no sandboxes, so it signs its own token rather than registering an owner for cleanup,
+	// which would need a second live server after the first one exits.
+	const { signToken } = await import(join(repo, "dist/api/lib/jwt.js"));
+	const token = await signToken({ sub: `${run.runId}-mcp`, expiresIn: "1h", secret: process.env.AUTH_SECRET });
 	const headers = {
-		Authorization: `Bearer ${owner.token}`,
+		Authorization: `Bearer ${token}`,
 		Accept: "application/json, text/event-stream",
 		"Content-Type": "application/json",
 	};

From 224e877fed84b7b24e8a7233ac3027b226d99669 Mon Sep 17 00:00:00 2001
From: QuangNguyen2609 <quangnguyentechno@gmail.com>
Date: Sat, 17 Oct 2026 12:58:59 +1030
Subject: [PATCH 07/13] test(verify): shorten the shutdown check comment

---
 scripts/verify/shutdown.mjs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scripts/verify/shutdown.mjs b/scripts/verify/shutdown.mjs
index 0b1848a1..15fff905 100644
--- a/scripts/verify/shutdown.mjs
+++ b/scripts/verify/shutdown.mjs
@@ -8,8 +8,7 @@ const run = new LocalApiRun("shutdown");
 let failure;
 try {
 	const server = await run.start({ SQL_FS_WORKERS_ENABLED: "true" });
-	// This check creates no sandboxes, so it signs its own token rather than registering an owner for cleanup,
-	// which would need a second live server after the first one exits.
+	// No run.owner(): its cleanup needs a live server after this one exits.
 	const { signToken } = await import(join(repo, "dist/api/lib/jwt.js"));
 	const token = await signToken({ sub: `${run.runId}-mcp`, expiresIn: "1h", secret: process.env.AUTH_SECRET });
 	const headers = {

From 289a54b6033fceebb192d343a3054e12e88ea8d0 Mon Sep 17 00:00:00 2001
From: QuangNguyen2609 <quangnguyentechno@gmail.com>
Date: Sat, 17 Oct 2026 13:01:37 +1030
Subject: [PATCH 08/13] docs, test(verify): correct worker_busy and ingress
 shutdown scope; wait for child close

---
 docs/AUTOSCALING.md         | 4 ++--
 scripts/verify/shutdown.mjs | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/AUTOSCALING.md b/docs/AUTOSCALING.md
index 3d38b131..9f185a20 100644
--- a/docs/AUTOSCALING.md
+++ b/docs/AUTOSCALING.md
@@ -58,7 +58,7 @@ The names below come from `src/api/worker-owner/metrics.ts` and from a scrape of
 |---|---|---|
 | `sql_fs_worker_capacity` | gauge | Configured worker count, `SQL_FS_WORKER_COUNT`. |
 | `sql_fs_worker_ready` | gauge | Workers that are started and able to take work. |
-| `sql_fs_worker_busy` | gauge | Workers that run an operation now. |
+| `sql_fs_worker_busy` | gauge | Worker slots held by an admitted operation, including operations that wait for a lock or recovery. For work that executes now, use `sql_fs_worker_running_operations`. |
 | `sql_fs_worker_resident_sessions` | gauge | Workers that hold a sandbox loaded. A worker keeps the last sandbox it served until another sandbox needs it. |
 | `sql_fs_worker_running_operations` | gauge | Admitted operations that run now. |
 | `sql_fs_worker_lock_waiting` | gauge | Admitted operations that wait for the sandbox's distributed lock. |
@@ -220,7 +220,7 @@ For example, a peak of 300 execs per second from many tenants needs 300 / 100 =
 
 On `SIGTERM` or `SIGINT`, the server does the following:
 
-1. It closes ingress admission. Queued requests, and new requests on open connections, get 503 `ESHUTTINGDOWN` with `Retry-After: 1`. Admitted requests continue.
+1. It closes ingress admission. Queued and new `/v1` and `/mcp` requests get 503 `ESHUTTINGDOWN` with `Retry-After: 1`. Admitted requests continue. `/healthz`, `/readyz` and `/metrics` bypass ingress and keep answering.
 2. It ends every MCP session's `GET /mcp` listener stream. In-flight MCP tool calls continue.
 3. It stops accepting connections and waits for open requests to finish.
 4. It closes MCP sessions, then lets running worker operations finish for up to 30 s, then closes Postgres and Redis. It exits with code 0.
diff --git a/scripts/verify/shutdown.mjs b/scripts/verify/shutdown.mjs
index 15fff905..b74e3bbc 100644
--- a/scripts/verify/shutdown.mjs
+++ b/scripts/verify/shutdown.mjs
@@ -47,7 +47,7 @@ try {
 		.catch(() => "aborted");
 
 	const started = performance.now();
-	const exited = once(server.child, "exit");
+	const exited = once(server.child, "close");
 	server.child.kill("SIGTERM");
 	const [code] = await exited;
 	const seconds = (performance.now() - started) / 1000;

From 64fe69c24bcd72a7f531a453b81f7efa6cd011dd Mon Sep 17 00:00:00 2001
From: QuangNguyen2609 <quangnguyentechno@gmail.com>
Date: Sat, 17 Oct 2026 13:03:40 +1030
Subject: [PATCH 09/13] test(verify), docs: assert a clean listener end;
 qualify startup validation and wait p95

---
 .changeset/ingress-wait.md  | 2 +-
 docs/AUTOSCALING.md         | 2 +-
 scripts/verify/shutdown.mjs | 4 +++-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/.changeset/ingress-wait.md b/.changeset/ingress-wait.md
index 5605ffa0..20cc5569 100644
--- a/.changeset/ingress-wait.md
+++ b/.changeset/ingress-wait.md
@@ -2,4 +2,4 @@
 "sql-fs-api": patch
 ---
 
-Add `SQL_FS_MAX_INGRESS_WAIT_MS`, the time a request or MCP listener waits for ingress capacity before it gets a 503 `EWORKER_BUSY`. The default stays 250 ms, the previous fixed value. Like the other ingress settings, an empty value uses the default. Any other invalid value, including a wait above 2147483647 ms, stops startup. `SQL_FS_WORKER_QUEUE_WAIT_MS` now rejects values above 2147483647 ms too, because Node would run such a timer after 1 ms.
+Add `SQL_FS_MAX_INGRESS_WAIT_MS`, the time a request or MCP listener waits for ingress capacity before it gets a 503 `EWORKER_BUSY`. The default stays 250 ms, the previous fixed value. Like the other ingress settings, an empty value uses the default. With workers on, any other invalid value, including a wait above 2147483647 ms, stops startup. `SQL_FS_WORKER_QUEUE_WAIT_MS` now rejects values above 2147483647 ms too, because Node would run such a timer after 1 ms.
diff --git a/docs/AUTOSCALING.md b/docs/AUTOSCALING.md
index 9f185a20..c2a1dec8 100644
--- a/docs/AUTOSCALING.md
+++ b/docs/AUTOSCALING.md
@@ -157,7 +157,7 @@ max(sql_fs_worker_runnable_queued)
 max(sql_fs_ingress_queued_requests)
 ```
 
-The p95 cannot pass `SQL_FS_WORKER_QUEUE_WAIT_MS`, because a request that waits longer is refused and not recorded. A p95 at the limit means the histogram is saturated, not that the wait is bounded. The top finite bucket is 1000 ms. If you raise `SQL_FS_WORKER_QUEUE_WAIT_MS` above 1000 ms, quantiles above 1000 ms report 1000.
+The p95 normally stays below `SQL_FS_WORKER_QUEUE_WAIT_MS`, because a request that waits longer is refused and not recorded. An event-loop stall can delay the refusal and let a slightly later admission through. A p95 at the limit means the histogram is saturated, not that the wait is bounded. The top finite bucket is 1000 ms. If you raise `SQL_FS_WORKER_QUEUE_WAIT_MS` above 1000 ms, quantiles above 1000 ms report 1000.
 
 Scale on `sql_fs_worker_runnable_queued`, not `sql_fs_worker_queued`. The difference is `sql_fs_worker_sandbox_waiting`, which more replicas do not reduce.
 
diff --git a/scripts/verify/shutdown.mjs b/scripts/verify/shutdown.mjs
index b74e3bbc..dd63dacb 100644
--- a/scripts/verify/shutdown.mjs
+++ b/scripts/verify/shutdown.mjs
@@ -51,7 +51,9 @@ try {
 	server.child.kill("SIGTERM");
 	const [code] = await exited;
 	const seconds = (performance.now() - started) / 1000;
-	run.record({ case: "sigterm-with-mcp-listener", exitCode: code, seconds, listener: await drained });
+	const listenerEnd = await drained;
+	run.record({ case: "sigterm-with-mcp-listener", exitCode: code, seconds, listener: listenerEnd });
+	run.equal(listenerEnd, "ended", "Shutdown ends the MCP listener stream cleanly");
 	run.equal(code, 0, "SIGTERM with an open MCP listener exits cleanly");
 	run.equal(seconds < 10, true, "SIGTERM with an open MCP listener does not wait for the force-exit timer");
 	run.equal(server.log.includes('"event":"shutdown_complete"'), true, "Shutdown logs shutdown_complete");

From 7a2a274ddfcec24df5f313e63e56ed905a454791 Mon Sep 17 00:00:00 2001
From: QuangNguyen2609 <quangnguyentechno@gmail.com>
Date: Sat, 17 Oct 2026 13:04:23 +1030
Subject: [PATCH 10/13] docs: list the settings a 4-worker replica needs

---
 docs/AUTOSCALING.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/AUTOSCALING.md b/docs/AUTOSCALING.md
index c2a1dec8..bc9230fc 100644
--- a/docs/AUTOSCALING.md
+++ b/docs/AUTOSCALING.md
@@ -205,7 +205,7 @@ To estimate the replica count:
 2. Add about 50% headroom and round up.
 3. Check the per-tenant limit. The busiest tenant's concurrent execs must be at or below `replicas * SQL_FS_WORKER_ACTIVE_TENANT_MAX`.
 
-For example, a peak of 300 execs per second from many tenants needs 300 / 100 = 3 replicas, and 5 with headroom. If one tenant runs 6 execs at a time, 5 replicas with the default limit of 1 is too few. That tenant needs 6 replicas, or 3 or more replicas that each run 4 workers with a limit of 2. A replica with 4 workers needs its memory sized again.
+For example, a peak of 300 execs per second from many tenants needs 300 / 100 = 3 replicas, and 5 with headroom. If one tenant runs 6 execs at a time, 5 replicas with the default limit of 1 is too few. That tenant needs 6 replicas, or 3 or more replicas that each run 4 workers with a limit of 2. Startup rejects 4 workers with the default settings. Also set `SQL_FS_WORKER_MAX_PG_CONNECTIONS=8`, and either lower `SQL_FS_WORKER_HEAP_MB` (for example to 256) or raise `SQL_FS_WORKER_MAX_RSS_BYTES`, then size the replica's memory again.
 
 ## Doing it well
 

From 8bad586a670eb3ff03958496a7ff4cca42304da9 Mon Sep 17 00:00:00 2001
From: QuangNguyen2609 <quangnguyentechno@gmail.com>
Date: Sat, 17 Oct 2026 13:05:34 +1030
Subject: [PATCH 11/13] docs: describe refused connections and the effective 60
 s request deadline at shutdown

---
 docs/AUTOSCALING.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/AUTOSCALING.md b/docs/AUTOSCALING.md
index bc9230fc..8cf76dfe 100644
--- a/docs/AUTOSCALING.md
+++ b/docs/AUTOSCALING.md
@@ -12,7 +12,7 @@ Each replica has fixed capacity. By default it runs 2 worker threads and admits
 | `GET /readyz` | None | `200 {"status":"ok"}` with an `eventLoop` lag summary. If Redis is configured and does not answer `PING` within 2 s, it returns `503 {"status":"degraded"}`. | Readiness and startup. It does not check Postgres. |
 | `GET /metrics` | Bearer JWT | Prometheus text format 0.0.4. | Scraping. |
 
-`/readyz` keeps returning 200 while the server shuts down. The platform must stop routing to a terminating replica by itself. Kubernetes and Azure Container Apps both do so.
+`/readyz` does not report shutdown. A terminating replica refuses new connections, so probes on new connections fail to connect. The platform must stop routing to a terminating replica by itself. Kubernetes and Azure Container Apps both do so.
 
 `/metrics` accepts the same bearer JWT as the REST API: an HS256 token signed with `AUTH_SECRET`, with a `sub` claim and an optional `tenant` claim that names a configured tenant. Without a valid token it returns 401. The metrics have no tenant labels, so a token for any tenant returns the same numbers. Give the scraper a token with a dedicated `sub`, minted through `POST /v1/auth/admin` or signed with `AUTH_SECRET` directly. In Prometheus, put the token in a file and point the scrape job at it:
 
@@ -220,13 +220,13 @@ For example, a peak of 300 execs per second from many tenants needs 300 / 100 =
 
 On `SIGTERM` or `SIGINT`, the server does the following:
 
-1. It closes ingress admission. Queued and new `/v1` and `/mcp` requests get 503 `ESHUTTINGDOWN` with `Retry-After: 1`. Admitted requests continue. `/healthz`, `/readyz` and `/metrics` bypass ingress and keep answering.
+1. It closes ingress admission. Queued `/v1` and `/mcp` requests, and new ones on already-open connections, get 503 `ESHUTTINGDOWN` with `Retry-After: 1`. Admitted requests continue.
 2. It ends every MCP session's `GET /mcp` listener stream. In-flight MCP tool calls continue.
-3. It stops accepting connections and waits for open requests to finish.
+3. It stops accepting connections, so new requests fail to connect, and waits for open requests to finish.
 4. It closes MCP sessions, then lets running worker operations finish for up to 30 s, then closes Postgres and Redis. It exits with code 0.
 5. If shutdown has not finished 60 s after the signal, it logs `shutdown_force_exit` and exits with code 1.
 
-Neither the 30 s nor the 60 s limit is configurable. A script can run for up to `SQL_FS_WORKER_EXECUTION_MS`, 120 s by default, so a long script can still be cut off. Set the platform's termination grace period above 60 s so that the platform does not kill the process before the server's own limit. To avoid cut-off scripts, keep `SQL_FS_WORKER_EXECUTION_MS` below about 60 s.
+The 30 s drain starts only after open requests finish, so an in-flight request effectively has until the 60 s force exit. Neither limit is configurable. A script can run for up to `SQL_FS_WORKER_EXECUTION_MS`, 120 s by default, so a long script can still be cut off. Set the platform's termination grace period above 60 s so that the platform does not kill the process before the server's own limit. To avoid cut-off scripts, keep `SQL_FS_WORKER_EXECUTION_MS` below about 60 s.
 
 ### Wait limits
 

From fe50d5dea2955fc10938e0672354004b0788717b Mon Sep 17 00:00:00 2001
From: QuangNguyen2609 <quangnguyentechno@gmail.com>
Date: Sat, 17 Oct 2026 13:08:32 +1030
Subject: [PATCH 12/13] ci: run CI and Python SDK CI on push to main only

PR branches matching the push patterns ran each workflow twice per push, once
for push and once for pull_request. The pull_request run still reports every
required check.
---
 .github/workflows/ci.yml            | 2 +-
 .github/workflows/python-sdk-ci.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e829a49a..1b7aa2ee 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -2,7 +2,7 @@ name: CI
 
 on:
   push:
-    branches: [main, "ralph/**", "feature/**"]
+    branches: [main]
   pull_request:
 
 concurrency:
diff --git a/.github/workflows/python-sdk-ci.yml b/.github/workflows/python-sdk-ci.yml
index affa1360..924a8ed1 100644
--- a/.github/workflows/python-sdk-ci.yml
+++ b/.github/workflows/python-sdk-ci.yml
@@ -2,7 +2,7 @@ name: Python SDK CI
 
 on:
   push:
-    branches: [main, "ci/**", "feat/**", "fix/**", "feature/**", "ralph/**"]
+    branches: [main]
   pull_request:
 
 permissions:

From 85c1fce8e111251f8b9b627a1e8f1ef841509e15 Mon Sep 17 00:00:00 2001
From: QuangNguyen2609 <quangnguyentechno@gmail.com>
Date: Sat, 17 Oct 2026 13:12:07 +1030
Subject: [PATCH 13/13] docs: say new connections, not new requests, fail at
 shutdown

---
 docs/AUTOSCALING.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/AUTOSCALING.md b/docs/AUTOSCALING.md
index 8cf76dfe..061bc061 100644
--- a/docs/AUTOSCALING.md
+++ b/docs/AUTOSCALING.md
@@ -222,7 +222,7 @@ On `SIGTERM` or `SIGINT`, the server does the following:
 
 1. It closes ingress admission. Queued `/v1` and `/mcp` requests, and new ones on already-open connections, get 503 `ESHUTTINGDOWN` with `Retry-After: 1`. Admitted requests continue.
 2. It ends every MCP session's `GET /mcp` listener stream. In-flight MCP tool calls continue.
-3. It stops accepting connections, so new requests fail to connect, and waits for open requests to finish.
+3. It stops accepting connections, so new connections fail, and waits for open requests to finish.
 4. It closes MCP sessions, then lets running worker operations finish for up to 30 s, then closes Postgres and Redis. It exits with code 0.
 5. If shutdown has not finished 60 s after the signal, it logs `shutdown_force_exit` and exits with code 1.