fix: increase HTTP backend connect timeout from 5s to 30s and make configurable (#3782)

## Problem

When an HTTP MCP backend is slow to initialize (e.g. cold-starting
LiteLLM proxy taking 21s), the gateway's hardcoded **5-second
per-transport connect timeout** causes all three SDK transport attempts
to fail. The tool registration silently drops the backend's tools and
the agent starts without them.

From the issue logs:
```
[2026-04-13T16:37:36Z] [DEBUG] GetOrLaunch called for server: opslevel
[2026-04-13T16:37:57Z] [INFO]  Successfully registered tools from opslevel (took 21.213743169s)
```

The 21s is consumed by the transport fallback chain: streamable HTTP (5s
timeout) → SSE (5s timeout) → plain JSON (~11s). If the backend only
supports streamable HTTP, all three transports fail and tools are
silently dropped.

## Fix

### 1. Configurable per-transport connect timeout (default 30s → was 5s)

New `connect_timeout` field on server config:

**TOML:**
```toml
[servers.opslevel]
type = "http"
url = "http://opslevel-proxy:8080/mcp"
connect_timeout = 60  # seconds per transport attempt
```

**JSON stdin:**
```json
{
  "mcpServers": {
    "opslevel": {
      "type": "http",
      "url": "http://opslevel-proxy:8080/mcp",
      "connect_timeout": 60
    }
  }
}
```

### 2. Better error visibility

- Failed backend tool registration promoted from **WARN → ERROR**
- Summary error log lists which backends failed and impact on agents:
`Tool registration incomplete: 1 of 3 backends failed: [opslevel] —
agents will not see tools from these servers`

### 3. Reconnect uses stored timeout

The connect timeout is stored on the Connection struct so
`reconnectSDKTransport` reuses the same value instead of a hardcoded
10s.

## Changes

| File | Change |
|------|--------|
| `config_core.go` | Add `ConnectTimeout` field, `HTTPConnectTimeout()`
helper, `DefaultConnectTimeout` constant |
| `config_stdin.go` | Add `ConnectTimeout` to `StdinServerConfig`, map
it during conversion |
| `connection.go` | Add `connectTimeout` to `Connection`, apply default
of 30s when ≤ 0 |
| `http_transport.go` | Pass configurable timeout through
`trySDKTransport` and `newHTTPConnection` |
| `launcher.go` | Pass `serverCfg.HTTPConnectTimeout()` to
`NewHTTPConnection` |
| `tool_registry.go` | Promote failed backends to ERROR, add summary log
|
| Tests | Updated all `NewHTTPConnection` call sites with new parameter
|

Fixes #3718

Author: lpcox

internal/config/config_core.go

@@ -41,6 +41,7 @@ const (
 	DefaultStartupTimeout    = 30   // seconds (per spec §4.1.3)
 	DefaultToolTimeout       = 60   // seconds (per spec §4.1.3)
 	DefaultKeepaliveInterval = 1500 // seconds (25 minutes) — keeps HTTP backend sessions alive
+	DefaultConnectTimeout    = 30   // seconds — per-transport timeout for HTTP backend connect
 )
 
 // Config represents the internal gateway configuration.
@@ -150,6 +151,15 @@ func (c *Config) GetAPIKey() string {
 	return c.Gateway.APIKey
 }
 
+// HTTPConnectTimeout returns the per-transport connect timeout as a Duration.
+// Returns DefaultConnectTimeout when the field is zero or negative.
+func (s *ServerConfig) HTTPConnectTimeout() time.Duration {
+	if s == nil || s.ConnectTimeout <= 0 {
+		return time.Duration(DefaultConnectTimeout) * time.Second
+	}
+	return time.Duration(s.ConnectTimeout) * time.Second
+}
+
 // AuthConfig configures upstream authentication for HTTP MCP servers.
 type AuthConfig struct {
 	// Type is the authentication type. Currently only "github-oidc" is supported.
@@ -198,6 +208,13 @@ type ServerConfig struct {
 
 	// Guard is the name of the guard to use for this server (requires DIFC)
 	Guard string `toml:"guard" json:"guard,omitempty"`
+
+	// ConnectTimeout is the timeout (in seconds) used for SDK-managed HTTP transport connect attempts.
+	// The gateway tries multiple transports in sequence (streamable HTTP → SSE → plain JSON-RPC).
+	// This timeout applies to the streamable HTTP and SSE connection attempts; the plain JSON-RPC
+	// fallback uses the HTTP client's request timeout instead. Increase this for backends that are
+	// slow to initialize. Only applies to HTTP server types. Default: 30 seconds.
+	ConnectTimeout int `toml:"connect_timeout" json:"connect_timeout,omitempty"`
 }
 
 // GuardConfig represents a guard configuration for DIFC enforcement.

internal/config/config_core_test.go

@@ -114,6 +114,24 @@ func TestLoadFromFile_HTTPServerValid(t *testing.T) {
 	assert.Equal(t, "http://localhost:9090/mcp", server.URL)
 }
 
+// TestLoadFromFile_HTTPServerWithConnectTimeout verifies that connect_timeout
+// is parsed from TOML and returned correctly via HTTPConnectTimeout().
+func TestLoadFromFile_HTTPServerWithConnectTimeout(t *testing.T) {
+	path := writeTempTOML(t, `
+[servers.slowservice]
+type = "http"
+url = "http://localhost:9090/mcp"
+connect_timeout = 60
+`)
+	cfg, err := LoadFromFile(path)
+	require.NoError(t, err)
+	require.NotNil(t, cfg)
+	server, ok := cfg.Servers["slowservice"]
+	require.True(t, ok)
+	assert.Equal(t, 60, server.ConnectTimeout)
+	assert.Equal(t, 60*time.Second, server.HTTPConnectTimeout())
+}
+
 // TestLoadFromFile_AppliesGatewayDefaults verifies that when no [gateway] section
 // is present, default values are applied for port, startup timeout, and tool timeout.
 func TestLoadFromFile_AppliesGatewayDefaults(t *testing.T) {
@@ -471,6 +489,48 @@ func TestHTTPKeepaliveInterval(t *testing.T) {
 	}
 }
 
+// TestHTTPConnectTimeout tests all branches of the ServerConfig.HTTPConnectTimeout method.
+func TestHTTPConnectTimeout(t *testing.T) {
+	tests := []struct {
+		name     string
+		server   *ServerConfig
+		expected time.Duration
+	}{
+		{
+			name:     "nil receiver returns default",
+			server:   nil,
+			expected: time.Duration(DefaultConnectTimeout) * time.Second,
+		},
+		{
+			name:     "zero value returns default",
+			server:   &ServerConfig{},
+			expected: time.Duration(DefaultConnectTimeout) * time.Second,
+		},
+		{
+			name:     "negative value returns default",
+			server:   &ServerConfig{ConnectTimeout: -5},
+			expected: time.Duration(DefaultConnectTimeout) * time.Second,
+		},
+		{
+			name:     "positive value returns correct duration",
+			server:   &ServerConfig{ConnectTimeout: 60},
+			expected: 60 * time.Second,
+		},
+		{
+			name:     "default value returns 30 seconds",
+			server:   &ServerConfig{ConnectTimeout: DefaultConnectTimeout},
+			expected: time.Duration(DefaultConnectTimeout) * time.Second,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tt.server.HTTPConnectTimeout()
+			assert.Equal(t, tt.expected, got)
+		})
+	}
+}
+
 // TestIsDynamicTOMLPath verifies the branching logic of isDynamicTOMLPath,
 // which guards the unknown-field check by exempting map-valued sections
 // whose keys are not known at decode time.

internal/config/config_stdin.go

@@ -123,6 +123,10 @@ type StdinServerConfig struct {
 	// Auth configures upstream authentication for HTTP MCP servers.
 	Auth *AuthConfig `json:"auth,omitempty"`
 
+	// ConnectTimeout is the per-transport timeout (in seconds) for connecting to HTTP backends.
+	// Only applies to HTTP server types. Default: 30 seconds.
+	ConnectTimeout *int `json:"connect_timeout,omitempty"`
+
 	// AdditionalProperties stores any extra fields for custom server types
 	// This allows custom schemas to define their own fields beyond the standard ones
 	AdditionalProperties map[string]interface{} `json:"-"`
@@ -151,20 +155,21 @@ func (s *StdinServerConfig) UnmarshalJSON(data []byte) error {
 
 	// Known fields in the struct
 	knownFields := map[string]bool{
-		"type":           true,
-		"container":      true,
-		"entrypoint":     true,
-		"entrypointArgs": true,
-		"args":           true,
-		"mounts":         true,
-		"env":            true,
-		"url":            true,
-		"headers":        true,
-		"tools":          true,
-		"registry":       true,
-		"guard-policies": true,
-		"guard":          true,
-		"auth":           true,
+		"type":            true,
+		"container":       true,
+		"entrypoint":      true,
+		"entrypointArgs":  true,
+		"args":            true,
+		"mounts":          true,
+		"env":             true,
+		"url":             true,
+		"headers":         true,
+		"tools":           true,
+		"registry":        true,
+		"guard-policies":  true,
+		"guard":           true,
+		"auth":            true,
+		"connect_timeout": true,
 	}
 
 	// Store additional properties (fields not in the struct)
@@ -413,6 +418,9 @@ func convertStdinServerConfig(name string, server *StdinServerConfig, customSche
 			GuardPolicies: server.GuardPolicies,
 			Guard:         server.Guard,
 		}
+		if server.ConnectTimeout != nil {
+			serverCfg.ConnectTimeout = *server.ConnectTimeout
+		}
 		if server.Auth != nil {
 			serverCfg.Auth = &AuthConfig{
 				Type:     server.Auth.Type,

internal/launcher/launcher.go

@@ -135,7 +135,7 @@ func GetOrLaunch(l *Launcher, serverID string) (*mcp.Connection, error) {
 			}
 
 			// Create an HTTP connection
-			conn, err := mcp.NewHTTPConnection(l.ctx, serverID, serverCfg.URL, serverCfg.Headers, oidcProvider, oidcAudience, l.config.Gateway.HTTPKeepaliveInterval())
+			conn, err := mcp.NewHTTPConnection(l.ctx, serverID, serverCfg.URL, serverCfg.Headers, oidcProvider, oidcAudience, l.config.Gateway.HTTPKeepaliveInterval(), serverCfg.HTTPConnectTimeout())
 			if err != nil {
 				log.Printf("FAILED to create HTTP connection for server %q: %v", serverID, err)
 				logger.LogErrorWithServer(serverID, "backend", "Failed to create HTTP connection: %s, error=%v", serverID, err)

internal/mcp/connection.go

@@ -7,6 +7,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"os/exec"
 	"strings"
@@ -70,6 +71,7 @@ type Connection struct {
 	httpSessionID     string            // Session ID returned by the HTTP backend
 	httpTransportType HTTPTransportType // Type of HTTP transport in use
 	keepAliveInterval time.Duration     // Keepalive interval for SDK transports (0 = disabled)
+	connectTimeout    time.Duration     // Per-transport connect timeout for SDK transports
 	// sessionMu protects the mutable session fields: httpSessionID, session, and client.
 	// Always use getHTTPSessionID() or getSDKSession() to read these fields; the
 	// reconnect functions (reconnectPlainJSON, reconnectSDKTransport) hold the full Lock.
@@ -194,17 +196,29 @@ func NewConnection(ctx context.Context, serverID, command string, args []string,
 // Authorization header from the headers map.
 //
 // This ensures compatibility with all types of HTTP MCP servers.
-func NewHTTPConnection(ctx context.Context, serverID, url string, headers map[string]string, oidcProvider *oidc.Provider, oidcAudience string, keepAlive time.Duration) (*Connection, error) {
-	logger.LogInfo("backend", "Creating HTTP MCP connection with transport fallback, url=%s", url)
+func NewHTTPConnection(ctx context.Context, serverID, url string, headers map[string]string, oidcProvider *oidc.Provider, oidcAudience string, keepAlive time.Duration, connectTimeout time.Duration) (*Connection, error) {
+	// Apply default connect timeout when not specified
+	if connectTimeout <= 0 {
+		connectTimeout = 30 * time.Second
+	}
+	logger.LogInfo("backend", "Creating HTTP MCP connection with transport fallback, url=%s, connectTimeout=%v", url, connectTimeout)
 	ctx, cancel := context.WithCancel(ctx)
 
-	// Create an HTTP client with appropriate timeouts
+	// Create an HTTP client with appropriate timeouts.
+	// Keep the existing overall request timeout, but also apply connectTimeout to
+	// the underlying HTTP transport so plain JSON-RPC fallback attempts honor the
+	// configured per-attempt connection timeout instead of waiting for the full
+	// client timeout.
 	httpClient := &http.Client{
 		Timeout: 120 * time.Second, // Overall request timeout
 		Transport: &http.Transport{
-			MaxIdleConns:        10,
-			IdleConnTimeout:     90 * time.Second,
-			TLSHandshakeTimeout: 10 * time.Second,
+			DialContext: (&net.Dialer{
+				Timeout: connectTimeout,
+			}).DialContext,
+			MaxIdleConns:          10,
+			IdleConnTimeout:       90 * time.Second,
+			TLSHandshakeTimeout:   10 * time.Second,
+			ResponseHeaderTimeout: connectTimeout,
 		},
 	}
 
@@ -232,7 +246,7 @@ func NewHTTPConnection(ctx context.Context, serverID, url string, headers map[st
 
 	// Try 1: Streamable HTTP (2025-03-26 spec)
 	logConn.Printf("Attempting streamable HTTP transport for %s", url)
-	conn, err := tryStreamableHTTPTransport(ctx, cancel, serverID, url, headers, headerClient, keepAlive)
+	conn, err := tryStreamableHTTPTransport(ctx, cancel, serverID, url, headers, headerClient, keepAlive, connectTimeout)
 	if err == nil {
 		logger.LogInfo("backend", "Successfully connected using streamable HTTP transport, url=%s", url)
 		return conn, nil
@@ -241,7 +255,7 @@ func NewHTTPConnection(ctx context.Context, serverID, url string, headers map[st
 
 	// Try 2: SSE (2024-11-05 spec)
 	logConn.Printf("Attempting SSE transport for %s", url)
-	conn, err = trySSETransport(ctx, cancel, serverID, url, headers, headerClient, keepAlive)
+	conn, err = trySSETransport(ctx, cancel, serverID, url, headers, headerClient, keepAlive, connectTimeout)
 	if err == nil {
 		logger.LogWarn("backend", "⚠️  MCP over SSE (2024-11-05 spec) is DEPRECATED for url=%s. Please migrate to streamable HTTP transport (2025-03-26 spec).", url)
 		logger.LogInfo("backend", "Configured HTTP MCP server with SSE transport: %s", url)
@@ -366,7 +380,11 @@ func (c *Connection) reconnectSDKTransport() error {
 		return fmt.Errorf("cannot reconnect: unsupported transport type %s", c.httpTransportType)
 	}
 
-	connectCtx, cancel := context.WithTimeout(c.ctx, 10*time.Second)
+	timeout := c.connectTimeout
+	if timeout == 0 {
+		timeout = 30 * time.Second
+	}
+	connectCtx, cancel := context.WithTimeout(c.ctx, timeout)
 	defer cancel()
 
 	session, err := client.Connect(connectCtx, transport, nil)

internal/mcp/connection_arguments_test.go

@@ -159,7 +159,7 @@ func TestCallTool_ArgumentsPassed(t *testing.T) {
 			// Create connection
 			conn, err := NewHTTPConnection(context.Background(), "test-server", testServer.URL, map[string]string{
 				"Authorization": "test-token",
-			}, nil, "", 0)
+			}, nil, "", 0, 0)
 			require.NoError(t, err, "Failed to create HTTP connection")
 			defer conn.Close()
 
@@ -224,7 +224,7 @@ func TestCallTool_MissingArguments(t *testing.T) {
 
 	conn, err := NewHTTPConnection(context.Background(), "test-server", testServer.URL, map[string]string{
 		"Authorization": "test-token",
-	}, nil, "", 0)
+	}, nil, "", 0, 0)
 	require.NoError(t, err)
 	defer conn.Close()
 

internal/mcp/connection_stderr_test.go

@@ -37,7 +37,7 @@ func TestConnection_SendRequest(t *testing.T) {
 
 	conn, err := NewHTTPConnection(context.Background(), "test-server", srv.URL, map[string]string{
 		"Authorization": "test-token",
-	}, nil, "", 0)
+	}, nil, "", 0, 0)
 	require.NoError(t, err)
 	defer conn.Close()
 

internal/mcp/connection_test.go

@@ -43,7 +43,7 @@ func TestHTTPRequest_SessionIDHeader(t *testing.T) {
 	// Create an HTTP connection
 	conn, err := NewHTTPConnection(context.Background(), "test-server", testServer.URL, map[string]string{
 		"Authorization": "test-auth-token",
-	}, nil, "", 0)
+	}, nil, "", 0, 0)
 	require.NoError(t, err, "Failed to create HTTP connection")
 
 	// Create a context with session ID
@@ -80,7 +80,7 @@ func TestHTTPRequest_NoSessionID(t *testing.T) {
 	// Create an HTTP connection
 	conn, err := NewHTTPConnection(context.Background(), "test-server", testServer.URL, map[string]string{
 		"Authorization": "test-auth-token",
-	}, nil, "", 0)
+	}, nil, "", 0, 0)
 	require.NoError(t, err, "Failed to create HTTP connection")
 
 	// Send a request without session ID in context
@@ -118,7 +118,7 @@ func TestHTTPRequest_ConfiguredHeaders(t *testing.T) {
 	authToken := "configured-auth-token"
 	conn, err := NewHTTPConnection(context.Background(), "test-server", testServer.URL, map[string]string{
 		"Authorization": authToken,
-	}, nil, "", 0)
+	}, nil, "", 0, 0)
 	require.NoError(t, err, "Failed to create HTTP connection")
 
 	// Create a context with session ID
@@ -377,7 +377,7 @@ func TestHTTPRequest_ErrorResponses(t *testing.T) {
 			// Create connection with custom headers to use plain JSON transport
 			conn, err := NewHTTPConnection(context.Background(), "test-server", testServer.URL, map[string]string{
 				"Authorization": "test-token",
-			}, nil, "", 0)
+			}, nil, "", 0, 0)
 			if err != nil {
 				require.True(t, tt.expectError, "Unexpected error creating connection: %v", err)
 				if tt.errorSubstring != "" {
@@ -423,7 +423,7 @@ func TestConnection_IsHTTP(t *testing.T) {
 		"X-Custom":      "custom-value",
 	}
 
-	conn, err := NewHTTPConnection(context.Background(), "test-server", testServer.URL, headers, nil, "", 0)
+	conn, err := NewHTTPConnection(context.Background(), "test-server", testServer.URL, headers, nil, "", 0, 0)
 	require.NoError(t, err, "Failed to create HTTP connection")
 	defer conn.Close()
 
@@ -466,7 +466,7 @@ func TestHTTPConnection_InvalidURL(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			_, err := NewHTTPConnection(context.Background(), "test-server", tt.url, tt.headers, nil, "", 0)
+			_, err := NewHTTPConnection(context.Background(), "test-server", tt.url, tt.headers, nil, "", 0, 0)
 
 			if tt.expectError {
 				require.Error(t, err, "Expected an error but got none")
@@ -523,7 +523,7 @@ func TestNewHTTPConnectionStoresKeepalive(t *testing.T) {
 	headers := map[string]string{}
 	httpClient := &http.Client{}
 
-	conn := newHTTPConnection(ctx, cancel, client, nil, url, headers, httpClient, HTTPTransportStreamable, "test-server", keepAlive)
+	conn := newHTTPConnection(ctx, cancel, client, nil, url, headers, httpClient, HTTPTransportStreamable, "test-server", keepAlive, 0)
 
 	require.NotNil(t, conn)
 	assert.Equal(t, keepAlive, conn.keepAliveInterval,
@@ -610,7 +610,7 @@ func TestNewHTTPConnection(t *testing.T) {
 	headers := map[string]string{"Authorization": "test"}
 	httpClient := &http.Client{}
 
-	conn := newHTTPConnection(ctx, cancel, client, nil, url, headers, httpClient, HTTPTransportStreamable, "test-server", 0)
+	conn := newHTTPConnection(ctx, cancel, client, nil, url, headers, httpClient, HTTPTransportStreamable, "test-server", 0, 0)
 
 	require.NotNil(t, conn, "Connection should not be nil")
 	assert.Equal(t, client, conn.client, "Client should match")