Load-balance envd HTTP/2 pools by in-flight streams; keep HTTP version a transport-only argument

.changeset/envd-pool-growth.md

@@ -0,0 +1,5 @@
+---
+'@e2b/python-sdk': minor
+---
+
+The Python SDK now spreads sandbox traffic across HTTP/2 connection pools by load instead of hashing each sandbox to one of four fixed pools: every request goes to the pool with the fewest requests in flight, and a further pool is opened only once every open pool carries `E2B_ENVD_POOL_STREAMS` (default 90) requests — below the 100 concurrent streams the sandbox host allows per connection — up to `E2B_ENVD_POOL_SHARDS` (default raised from 4 to 16) pools. A process running a single sandbox keeps one connection; one with hundreds of long-running commands is no longer queued behind a saturated connection, whether they belong to many sandboxes or to a few.

packages/js-sdk/src/api/http2.ts

@@ -11,19 +11,19 @@ const DEFAULT_API_CONNECTION_LIMIT = 100
 // Override via env if your workload needs different.
 const DEFAULT_API_INFLIGHT_LIMIT = 1000
 
-// Fetchers are cached per proxy so requests without a proxy keep sharing a
-// single dispatcher while each distinct proxy URL gets its own.
+// Fetchers are cached per proxy and HTTP version so requests without a proxy
+// keep sharing a single dispatcher while each distinct proxy URL gets its own.
 const apiFetchers = new Map<string, typeof fetch>()
 
-export function createApiFetch(proxy?: string): typeof fetch {
-  const key = proxy ?? ''
+export function createApiFetch(proxy?: string, http2 = true): typeof fetch {
+  const key = `${http2 ? 'h2' : 'h1'}:${proxy ?? ''}`
 
   const cached = apiFetchers.get(key)
   if (cached) {
     return cached
   }
 
-  const apiFetch = createApiFetchForRuntime(runtime, { proxy })
+  const apiFetch = createApiFetchForRuntime(runtime, { proxy, http2 })
   apiFetchers.set(key, apiFetch)
 
   return apiFetch
@@ -35,6 +35,7 @@ export function createApiFetchForRuntime(
     connectionLimit?: number
     inflightLimit?: number
     proxy?: string
+    http2?: boolean
     loadUndici?: () => Promise<UndiciModule | undefined>
   } = {}
 ): typeof fetch {
@@ -45,6 +46,7 @@ export function createApiFetchForRuntime(
       connections: options.connectionLimit ?? getApiConnectionLimit(),
       inflightLimit: options.inflightLimit ?? getApiInflightLimit(),
       proxy: options.proxy,
+      http2: options.http2,
       loadUndici: options.loadUndici,
     })
   )

packages/js-sdk/src/envd/http2.ts

@@ -10,11 +10,12 @@ type EnvdFetchOptions = {
   connectionLimit?: number
   inflightLimit?: number
   proxy?: string
+  http2?: boolean
   loadUndici?: () => Promise<UndiciModule | undefined>
 }
 
-// Fetchers are cached per proxy so requests without a proxy keep sharing a
-// single dispatcher while each distinct proxy URL gets its own.
+// Fetchers are cached per proxy and HTTP version so requests without a proxy
+// keep sharing a single dispatcher while each distinct proxy URL gets its own.
 const envdFetchers = new Map<string, typeof fetch>()
 const envdRpcFetchers = new Map<string, typeof fetch>()
 const DEFAULT_ENVD_CONNECTION_LIMIT = 10
@@ -31,13 +32,18 @@ export function createEnvdFetchForRuntime(
       connections: options.connectionLimit ?? DEFAULT_ENVD_CONNECTION_LIMIT,
       inflightLimit: options.inflightLimit ?? 0,
       proxy: options.proxy,
+      http2: options.http2,
       loadUndici: options.loadUndici,
     })
   )
 }
 
-export function createEnvdFetch(proxy?: string): typeof fetch {
-  const key = proxy ?? ''
+function fetcherKey(proxy: string | undefined, http2: boolean): string {
+  return `${http2 ? 'h2' : 'h1'}:${proxy ?? ''}`
+}
+
+export function createEnvdFetch(proxy?: string, http2 = true): typeof fetch {
+  const key = fetcherKey(proxy, http2)
 
   const cached = envdFetchers.get(key)
   if (cached) {
@@ -49,14 +55,15 @@ export function createEnvdFetch(proxy?: string): typeof fetch {
   const envdFetch = createEnvdFetchForRuntime(runtime, {
     inflightLimit: getEnvdInflightLimit(),
     proxy,
+    http2,
   })
   envdFetchers.set(key, envdFetch)
 
   return envdFetch
 }
 
-export function createEnvdRpcFetch(proxy?: string): typeof fetch {
-  const key = proxy ?? ''
+export function createEnvdRpcFetch(proxy?: string, http2 = true): typeof fetch {
+  const key = fetcherKey(proxy, http2)
 
   const cached = envdRpcFetchers.get(key)
   if (cached) {
@@ -67,6 +74,7 @@ export function createEnvdRpcFetch(proxy?: string): typeof fetch {
     connectionLimit: getEnvdRpcConnectionLimit(),
     inflightLimit: getEnvdRpcInflightLimit(),
     proxy,
+    http2,
   })
   envdRpcFetchers.set(key, envdRpcFetch)
 

packages/js-sdk/src/undici.ts

@@ -10,10 +10,10 @@ type UndiciRequestInit = RequestInit & {
 }
 
 export type UndiciModule = {
-  Agent: new (options: { allowH2: true; connections?: number }) => unknown
+  Agent: new (options: { allowH2: boolean; connections?: number }) => unknown
   ProxyAgent: new (options: {
     uri: string
-    allowH2: true
+    allowH2: boolean
     connections?: number
     proxyTunnel: true
   }) => unknown
@@ -87,15 +87,17 @@ export function createRuntimeFetch(
 }
 
 /**
- * Build a fetch bound to a bounded undici dispatcher (HTTP/2 enabled,
- * `connections` origin connections, optional proxy tunnel), capped at
- * `inflightLimit` in-flight requests (`0` disables the cap). Falls back to
- * the global fetch — still capped — when undici cannot be loaded.
+ * Build a fetch bound to a bounded undici dispatcher (HTTP/2 enabled unless
+ * `http2` is `false`, `connections` origin connections, optional proxy
+ * tunnel), capped at `inflightLimit` in-flight requests (`0` disables the
+ * cap). Falls back to the global fetch — still capped — when undici cannot be
+ * loaded.
  */
 export async function buildDispatchedFetch(options: {
   connections: number
   inflightLimit: number
   proxy?: string
+  http2?: boolean
   loadUndici?: () => Promise<UndiciModule | undefined>
 }): Promise<typeof fetch> {
   const undici = await (options.loadUndici ?? loadUndici)()
@@ -105,15 +107,16 @@ export async function buildDispatchedFetch(options: {
   }
 
   const { Agent, ProxyAgent, fetch: undiciFetch } = undici
+  const allowH2 = options.http2 ?? true
   const dispatcher = options.proxy
     ? new ProxyAgent({
         uri: options.proxy,
-        allowH2: true,
+        allowH2,
         connections: options.connections,
         proxyTunnel: true,
       })
     : new Agent({
-        allowH2: true,
+        allowH2,
         connections: options.connections,
       })
   const fetchWithDispatcher = undiciFetch as unknown as (

packages/js-sdk/tests/api/http2.test.ts

@@ -144,17 +144,44 @@ test('late-binds the global fetch fallback when undici cannot be loaded', async
   }
 })
 
-test('caches API fetchers per proxy', async () => {
+test('pins the API dispatcher to HTTP/1.1 when http2 is false', async () => {
+  const agents: Array<{ allowH2?: boolean; connections?: number }> = []
+
+  class Agent {
+    constructor(options: { allowH2?: boolean; connections?: number }) {
+      agents.push(options)
+    }
+  }
+
+  const undiciFetch = vi.fn(() => Promise.resolve(new Response('ok')))
+
+  const { createApiFetchForRuntime } = await import('../../src/api/http2')
+
+  const fetcher = createApiFetchForRuntime('node', {
+    connectionLimit: 3,
+    http2: false,
+    loadUndici: () => Promise.resolve({ Agent, fetch: undiciFetch }),
+  })
+  await fetcher('https://api.e2b.app/sandboxes')
+
+  expect(agents).toEqual([{ allowH2: false, connections: 3 }])
+})
+
+test('caches API fetchers per proxy and HTTP version', async () => {
   const { createApiFetch } = await import('../../src/api/http2')
 
   const noProxy = createApiFetch()
   const proxyA = createApiFetch('http://127.0.0.1:8080')
   const proxyB = createApiFetch('http://127.0.0.1:9090')
+  const noProxyH1 = createApiFetch(undefined, false)
 
   expect(createApiFetch()).toBe(noProxy)
+  expect(createApiFetch(undefined, true)).toBe(noProxy)
   expect(createApiFetch('http://127.0.0.1:8080')).toBe(proxyA)
+  expect(createApiFetch(undefined, false)).toBe(noProxyH1)
   expect(proxyA).not.toBe(noProxy)
   expect(proxyA).not.toBe(proxyB)
+  expect(noProxyH1).not.toBe(noProxy)
 })
 
 test('getApiConnectionLimit throws on a malformed env value', async () => {

packages/js-sdk/tests/envd/http2.test.ts

@@ -89,23 +89,78 @@ test('uses a ProxyAgent dispatcher when a proxy is configured', async () => {
   expect(requests[0].init?.dispatcher).toBeInstanceOf(ProxyAgent)
 })
 
-test('caches envd fetchers per proxy', async () => {
+test('pins the dispatcher to HTTP/1.1 when http2 is false', async () => {
+  const agents: Array<{ allowH2?: boolean; connections?: number }> = []
+  const proxyAgents: Array<{ uri?: string; allowH2?: boolean }> = []
+
+  class Agent {
+    constructor(options: { allowH2?: boolean; connections?: number }) {
+      agents.push(options)
+    }
+  }
+
+  class ProxyAgent {
+    constructor(options: { uri?: string; allowH2?: boolean }) {
+      proxyAgents.push(options)
+    }
+  }
+
+  const undiciFetch = vi.fn(() => Promise.resolve(new Response('ok')))
+
+  const { createEnvdFetchForRuntime } = await import('../../src/envd/http2')
+
+  const direct = createEnvdFetchForRuntime('node', {
+    connectionLimit: 1,
+    http2: false,
+    loadUndici: () =>
+      Promise.resolve({ Agent, ProxyAgent, fetch: undiciFetch }),
+  })
+  await direct('https://example.com/status')
+
+  const proxied = createEnvdFetchForRuntime('node', {
+    connectionLimit: 1,
+    http2: false,
+    proxy: 'http://127.0.0.1:8080',
+    loadUndici: () =>
+      Promise.resolve({ Agent, ProxyAgent, fetch: undiciFetch }),
+  })
+  await proxied('https://example.com/status')
+
+  expect(agents).toEqual([{ allowH2: false, connections: 1 }])
+  expect(proxyAgents).toEqual([
+    {
+      uri: 'http://127.0.0.1:8080',
+      allowH2: false,
+      connections: 1,
+      proxyTunnel: true,
+    },
+  ])
+})
+
+test('caches envd fetchers per proxy and HTTP version', async () => {
   const { createEnvdFetch, createEnvdRpcFetch } =
     await import('../../src/envd/http2')
 
   const noProxy = createEnvdFetch()
   const proxyA = createEnvdFetch('http://127.0.0.1:8080')
+  const noProxyH1 = createEnvdFetch(undefined, false)
 
   expect(createEnvdFetch()).toBe(noProxy)
+  expect(createEnvdFetch(undefined, true)).toBe(noProxy)
   expect(createEnvdFetch('http://127.0.0.1:8080')).toBe(proxyA)
+  expect(createEnvdFetch(undefined, false)).toBe(noProxyH1)
   expect(proxyA).not.toBe(noProxy)
+  expect(noProxyH1).not.toBe(noProxy)
 
   const rpcNoProxy = createEnvdRpcFetch()
   const rpcProxyA = createEnvdRpcFetch('http://127.0.0.1:8080')
+  const rpcNoProxyH1 = createEnvdRpcFetch(undefined, false)
 
   expect(createEnvdRpcFetch()).toBe(rpcNoProxy)
   expect(createEnvdRpcFetch('http://127.0.0.1:8080')).toBe(rpcProxyA)
+  expect(createEnvdRpcFetch(undefined, false)).toBe(rpcNoProxyH1)
   expect(rpcProxyA).not.toBe(rpcNoProxy)
+  expect(rpcNoProxyH1).not.toBe(rpcNoProxy)
 })
 
 test('passes Request objects to undici as URL plus init', async () => {

packages/python-sdk/README.md

@@ -68,12 +68,12 @@ Per-call params still take precedence over the client's params, and clients are
 
 ### High-concurrency sandbox streams
 
-The Python SDK spreads sandbox `commands` and `files` traffic across four HTTP/2 connection pools by default. This prevents long-running streams in one process from all contending for a single connection's concurrent-stream limit.
+The sandbox host allows 100 concurrent streams per HTTP/2 connection, and a long-running command holds its stream for its whole lifetime. The Python SDK therefore spreads sandbox `commands`, `files` and `pty` traffic across HTTP/2 connection pools by load: each request goes to the pool with the fewest requests in flight, and a further pool (connection) is opened only once every open pool carries `E2B_ENVD_POOL_STREAMS` requests (default `90`), up to `E2B_ENVD_POOL_SHARDS` pools (default `16`). A process running a few sandboxes keeps a single connection; one running hundreds of concurrent commands is never queued behind a saturated connection.
 
-If one process needs more capacity, set `E2B_ENVD_POOL_SHARDS` before importing `e2b`. Each additional shard can open another connection to the sandbox host, so increase it only as needed:
+If one process needs more than `16 × 90` concurrent sandbox streams, raise the pool bound before importing `e2b`:
 
 ```sh
-E2B_ENVD_POOL_SHARDS=8 python eval.py
+E2B_ENVD_POOL_SHARDS=32 python eval.py
 ```
 
 ### 5. Code execution with Code Interpreter