From 954838eedaf46666eaecbfdcc5a0e41cb0fef756 Mon Sep 17 00:00:00 2001 From: ergdevops <115262164+ergdevops@users.noreply.github.com> Date: Mon, 31 Aug 2026 15:28:42 +0530 Subject: [PATCH 1/4] feat(http): add max connection age for inbound HTTP/2 server connections Pooled proxy-to-proxy HTTP/2 connections have no idle timeout or age bound: peers keep them alive indefinitely (keepalives prevent closure; MAX_IDLE_CONNS_PER_ENDPOINT only caps the count), and each held connection retains its receive-buffer high-water mark - up to the connection window - for its entire lifetime. In a mesh where a large caller fleet held ~300 connections into each server pod, we measured ~700KB retained per connection after ~300 rps load bursts (~200MB of proxy RSS per pod), released only when peer pods restarted. This adds LINKERD2_PROXY_INBOUND_SERVER_HTTP2_MAX_CONNECTION_AGE (duration; unset preserves current behavior). When set, the inbound HTTP/2 serve loop arms a timer as an additional select branch alongside the existing drain/teardown triggers and reuses the same graceful shutdown (GOAWAY) path: in-flight streams complete and clients reconnect transparently. A deterministic per-connection jitter of up to +10% (derived from the client's ephemeral port, avoiding a new dependency) prevents synchronized shutdown storms. Shutdowns are logged at INFO with the client address. Signed-off-by: ergdevops <115262164+ergdevops@users.noreply.github.com> --- linkerd/app/src/env/http2.rs | 7 +++ linkerd/http/h2/src/lib.rs | 5 ++ linkerd/proxy/http/src/server.rs | 34 +++++++++++-- linkerd/proxy/http/src/server/tests.rs | 70 ++++++++++++++++++++++++++ 4 files changed, 113 insertions(+), 3 deletions(-) diff --git a/linkerd/app/src/env/http2.rs b/linkerd/app/src/env/http2.rs index 42c5806ca6..ca02622826 100644 --- a/linkerd/app/src/env/http2.rs +++ b/linkerd/app/src/env/http2.rs @@ -9,6 +9,11 @@ pub(super) fn parse_server( Ok(ServerParams { flow_control: Some(parse_flow_control(strings, base)?), keep_alive: parse_keep_alive(strings, &format!("{base}_KEEP_ALIVE"))?, + max_connection_age: parse( + strings, + &format!("{base}_MAX_CONNECTION_AGE"), + parse_duration, + )?, max_concurrent_streams: parse( strings, &format!("{base}_MAX_CONCURRENT_STREAMS"), @@ -107,6 +112,7 @@ mod tests { env.insert("TEST_KEEP_ALIVE_INTERVAL", "2s"); env.insert("TEST_INITIAL_STREAM_WINDOW_SIZE", "1"); env.insert("TEST_INITIAL_CONNECTION_WINDOW_SIZE", "2"); + env.insert("TEST_MAX_CONNECTION_AGE", "900s"); let expected = h2::ServerParams { flow_control: Some(h2::FlowControl::Fixed { initial_stream_window_size: 1, @@ -116,6 +122,7 @@ mod tests { interval: Duration::from_secs(2), timeout: Duration::from_secs(1), }), + max_connection_age: Some(Duration::from_secs(900)), max_concurrent_streams: Some(3), max_frame_size: Some(4), max_header_list_size: Some(5), diff --git a/linkerd/http/h2/src/lib.rs b/linkerd/http/h2/src/lib.rs index ad1b109595..7867a78e5c 100644 --- a/linkerd/http/h2/src/lib.rs +++ b/linkerd/http/h2/src/lib.rs @@ -5,6 +5,11 @@ pub struct ServerParams { pub flow_control: Option, pub keep_alive: Option, pub max_concurrent_streams: Option, + /// If set, server connections are gracefully shut down (GOAWAY) after this + /// age, bounding how long a pooled peer connection (and the buffer + /// high-water memory it retains) can live. A per-connection jitter of up + /// to +10% is applied to avoid synchronized shutdowns. + pub max_connection_age: Option, // Internals pub max_frame_size: Option, diff --git a/linkerd/proxy/http/src/server.rs b/linkerd/proxy/http/src/server.rs index 5e4fcdc2d0..bad079bda7 100644 --- a/linkerd/proxy/http/src/server.rs +++ b/linkerd/proxy/http/src/server.rs @@ -8,9 +8,10 @@ use std::{ future::Future, pin::Pin, task::{Context, Poll}, + time::Duration, }; use tower::Service; -use tracing::{debug, Instrument}; +use tracing::{debug, info, Instrument}; #[cfg(test)] mod tests; @@ -36,6 +37,7 @@ pub struct ServeHttp { version: Variant, http1: hyper::server::conn::http1::Builder, http2: hyper::server::conn::http2::Builder, + max_connection_age: Option, inner: N, drain: drain::Watch, } @@ -70,6 +72,7 @@ where keep_alive, flow_control, max_concurrent_streams, + max_connection_age, max_frame_size, max_header_list_size, max_send_buf_size, @@ -124,6 +127,7 @@ where drain, http1, http2, + max_connection_age, } } } @@ -154,17 +158,18 @@ where let drain = self.drain.clone(); let http1 = self.http1.clone(); let http2 = self.http2.clone(); + let max_connection_age = self.max_connection_age; let res = io.peer_addr().map(|pa| { let (handle, closed) = ClientHandle::new(pa); let svc = self.inner.new_service(handle.clone()); let svc = SetClientHandle::new(handle, svc); - (svc, closed) + (svc, closed, pa) }); Box::pin( async move { - let (svc, closed) = res?; + let (svc, closed, peer) = res?; debug!(?version, "Handling as HTTP"); match version { Variant::Http1 => { @@ -201,6 +206,24 @@ where let io = hyper_util::rt::TokioIo::new(io); let mut conn = http2.serve_connection(io, svc); + // Bound the connection's lifetime if a max age is + // configured, so long-lived pooled peer connections + // (and the buffer high-water memory they retain) are + // periodically recycled. A deterministic per-connection + // jitter of up to +10%, derived from the client's + // ephemeral port, avoids synchronized shutdowns. + let max_age = async move { + match max_connection_age { + Some(age) => { + let jitter = + age.mul_f64(f64::from(peer.port() % 128) / 1280.0); + tokio::time::sleep(age + jitter).await; + } + None => std::future::pending().await, + } + }; + tokio::pin!(max_age); + tokio::select! { res = &mut conn => { debug!(?res, "The client is shutting down the connection"); @@ -216,6 +239,11 @@ where Pin::new(&mut conn).graceful_shutdown(); conn.await?; } + () = &mut max_age => { + info!(client.addr = %peer, "Max connection age reached; gracefully shutting down the connection"); + Pin::new(&mut conn).graceful_shutdown(); + conn.await?; + } } } } diff --git a/linkerd/proxy/http/src/server/tests.rs b/linkerd/proxy/http/src/server/tests.rs index a7b806144b..28ee40d429 100644 --- a/linkerd/proxy/http/src/server/tests.rs +++ b/linkerd/proxy/http/src/server/tests.rs @@ -159,6 +159,76 @@ async fn h2_stream_window_exhaustion() { */ } +/// Tests that a server connection is gracefully shut down after its max +/// connection age elapses: in-flight streams complete, but the connection is +/// then closed so the client must reconnect. +#[tokio::test(flavor = "current_thread", start_paused = true)] +async fn h2_max_connection_age() { + let _trace = linkerd_tracing::test::with_default_filter(LOG_LEVEL); + + const MAX_AGE: time::Duration = time::Duration::from_secs(10); + // The effective age includes a deterministic per-connection jitter of up + // to +10%; sleep comfortably past it. + const PAST_AGE: time::Duration = time::Duration::from_secs(12); + + let mut server = TestServer::connect_h2( + h2::ServerParams { + max_connection_age: Some(MAX_AGE), + ..Default::default() + }, + hyper::client::conn::http2::Builder::new(TokioExecutor::new()) + .timer(hyper_util::rt::TokioTimer::new()), + ) + .await; + + let bytes = Bytes::from_static(b"hello"); + + tracing::info!("Before the age elapses, requests are served normally"); + let rx = timeout(server.respond(bytes.clone())) + .await + .expect("timed out"); + let body = timeout(rx.collect()) + .await + .expect("response timed out") + .expect("response"); + assert_eq!(body.to_bytes(), bytes); + + tracing::info!("A stream that is in flight when the age elapses still completes"); + let (mut tx, mut body) = timeout(server.get()).await.expect("timed out"); + time::sleep(PAST_AGE).await; + tx.send_data(bytes.clone()) + .await + .expect("in-flight stream remains writable after the age elapses"); + drop(tx); + let data = body + .frame() + .await + .expect("yields a result") + .expect("yields a frame") + .into_data() + .expect("yields data"); + assert_eq!(data, bytes); + // Drain the body to its end; only an empty end-of-stream frame may remain. + while let Some(frame) = timeout(body.frame()).await.expect("body ends") { + let frame = frame.expect("body yields frames until end"); + if let Ok(data) = frame.into_data() { + assert!(data.is_empty(), "no more data expected"); + } + } + + tracing::info!("Once in-flight streams complete, the connection is closed"); + timeout(async { + loop { + if server.client.ready().await.is_err() { + return; + } + time::sleep(time::Duration::from_millis(100)).await; + } + }) + .await + .expect("client should observe the connection closing"); +} + // === Utilities === const LOG_LEVEL: &str = "h2::proto=trace,hyper=trace,linkerd=trace,info"; From a75823381175d7fe265634f4188e9c3849a2d559 Mon Sep 17 00:00:00 2001 From: ergdevops <115262164+ergdevops@users.noreply.github.com> Date: Mon, 31 Aug 2026 15:33:53 +0530 Subject: [PATCH 2/4] chore(http): log max-age connection shutdowns at debug At scale this fires once per connection per age period, which is too chatty for INFO; operators who want per-shutdown visibility can enable it via the proxy log filter. Signed-off-by: ergdevops <115262164+ergdevops@users.noreply.github.com> --- linkerd/proxy/http/src/server.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/linkerd/proxy/http/src/server.rs b/linkerd/proxy/http/src/server.rs index bad079bda7..9e9dc783f7 100644 --- a/linkerd/proxy/http/src/server.rs +++ b/linkerd/proxy/http/src/server.rs @@ -11,7 +11,7 @@ use std::{ time::Duration, }; use tower::Service; -use tracing::{debug, info, Instrument}; +use tracing::{debug, Instrument}; #[cfg(test)] mod tests; @@ -240,7 +240,7 @@ where conn.await?; } () = &mut max_age => { - info!(client.addr = %peer, "Max connection age reached; gracefully shutting down the connection"); + debug!(client.addr = %peer, "Max connection age reached; gracefully shutting down the connection"); Pin::new(&mut conn).graceful_shutdown(); conn.await?; } From 2bac29a7ca011dceb8458117c933e7a7e2c3e1d0 Mon Sep 17 00:00:00 2001 From: ergdevops <115262164+ergdevops@users.noreply.github.com> Date: Thu, 3 Sep 2026 11:16:43 +0530 Subject: [PATCH 3/4] refactor(http): address review feedback - Drop the doc comment on ServerParams::max_connection_age to match the struct's style. - Rename pa/peer to peer_addr for clarity and consistency. - Deduplicate the H2 serve loop's shutdown handling with a Teardown enum classifying the completion branches (ClientClosed / Drain / Graceful), as suggested in review. Signed-off-by: ergdevops <115262164+ergdevops@users.noreply.github.com> --- linkerd/http/h2/src/lib.rs | 4 --- linkerd/proxy/http/src/server.rs | 47 ++++++++++++++++++++++++-------- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/linkerd/http/h2/src/lib.rs b/linkerd/http/h2/src/lib.rs index 7867a78e5c..5bd844c2e4 100644 --- a/linkerd/http/h2/src/lib.rs +++ b/linkerd/http/h2/src/lib.rs @@ -5,10 +5,6 @@ pub struct ServerParams { pub flow_control: Option, pub keep_alive: Option, pub max_concurrent_streams: Option, - /// If set, server connections are gracefully shut down (GOAWAY) after this - /// age, bounding how long a pooled peer connection (and the buffer - /// high-water memory it retains) can live. A per-connection jitter of up - /// to +10% is applied to avoid synchronized shutdowns. pub max_connection_age: Option, // Internals diff --git a/linkerd/proxy/http/src/server.rs b/linkerd/proxy/http/src/server.rs index 9e9dc783f7..6823985c8d 100644 --- a/linkerd/proxy/http/src/server.rs +++ b/linkerd/proxy/http/src/server.rs @@ -31,6 +31,19 @@ pub struct NewServeHttp { params: X, } +/// How a served connection is being torn down, as decided by the serve loop's +/// completion branches. +enum Teardown { + /// The client closed the connection; nothing left to do. + ClientClosed, + /// The process is draining: shut down gracefully, holding the drain + /// release guard until the connection closes. + Drain(drain::ReleaseShutdown), + /// Stack teardown or max connection age: shut down gracefully, then wait + /// for the connection to close. + Graceful, +} + /// Serves HTTP connections with an inner service. #[derive(Clone, Debug)] pub struct ServeHttp { @@ -160,16 +173,16 @@ where let http2 = self.http2.clone(); let max_connection_age = self.max_connection_age; - let res = io.peer_addr().map(|pa| { - let (handle, closed) = ClientHandle::new(pa); + let res = io.peer_addr().map(|peer_addr| { + let (handle, closed) = ClientHandle::new(peer_addr); let svc = self.inner.new_service(handle.clone()); let svc = SetClientHandle::new(handle, svc); - (svc, closed, pa) + (svc, closed, peer_addr) }); Box::pin( async move { - let (svc, closed, peer) = res?; + let (svc, closed, peer_addr) = res?; debug!(?version, "Handling as HTTP"); match version { Variant::Http1 => { @@ -216,7 +229,7 @@ where match max_connection_age { Some(age) => { let jitter = - age.mul_f64(f64::from(peer.port() % 128) / 1280.0); + age.mul_f64(f64::from(peer_addr.port() % 128) / 1280.0); tokio::time::sleep(age + jitter).await; } None => std::future::pending().await, @@ -224,23 +237,33 @@ where }; tokio::pin!(max_age); - tokio::select! { + let teardown = tokio::select! { res = &mut conn => { debug!(?res, "The client is shutting down the connection"); - res? + res?; + Teardown::ClientClosed } shutdown = drain.signaled() => { debug!("The process is shutting down the connection"); - Pin::new(&mut conn).graceful_shutdown(); - shutdown.release_after(conn).await?; + Teardown::Drain(shutdown) } () = closed => { debug!("The stack is tearing down the connection"); - Pin::new(&mut conn).graceful_shutdown(); - conn.await?; + Teardown::Graceful } () = &mut max_age => { - debug!(client.addr = %peer, "Max connection age reached; gracefully shutting down the connection"); + debug!(client.addr = %peer_addr, "Max connection age reached; gracefully shutting down the connection"); + Teardown::Graceful + } + }; + + match teardown { + Teardown::ClientClosed => {} + Teardown::Drain(shutdown) => { + Pin::new(&mut conn).graceful_shutdown(); + shutdown.release_after(conn).await?; + } + Teardown::Graceful => { Pin::new(&mut conn).graceful_shutdown(); conn.await?; } From 8011af9c16b1fe361b26bc8abd56dee7baa10f53 Mon Sep 17 00:00:00 2001 From: ergdevops <115262164+ergdevops@users.noreply.github.com> Date: Fri, 4 Sep 2026 11:29:45 +0530 Subject: [PATCH 4/4] test(http): cover max connection age under both server prefixes Like every other SERVER_HTTP2 setting, max connection age parses under both the inbound and outbound server prefixes via the shared parse_server; make that symmetry explicit with a test exercising the real variable names. Signed-off-by: ergdevops <115262164+ergdevops@users.noreply.github.com> --- linkerd/app/src/env/http2.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/linkerd/app/src/env/http2.rs b/linkerd/app/src/env/http2.rs index ca02622826..18393606e0 100644 --- a/linkerd/app/src/env/http2.rs +++ b/linkerd/app/src/env/http2.rs @@ -154,4 +154,31 @@ mod tests { } ); } + + /// Like every other SERVER_HTTP2 setting, max connection age is parsed + /// under both the inbound and outbound server prefixes. + #[test] + fn max_connection_age_parses_for_both_servers() { + let mut env = HashMap::default(); + env.insert( + "LINKERD2_PROXY_INBOUND_SERVER_HTTP2_MAX_CONNECTION_AGE", + "15m", + ); + env.insert( + "LINKERD2_PROXY_OUTBOUND_SERVER_HTTP2_MAX_CONNECTION_AGE", + "1h", + ); + + let inbound = parse_server(&env, "LINKERD2_PROXY_INBOUND_SERVER_HTTP2").unwrap(); + assert_eq!( + inbound.max_connection_age, + Some(Duration::from_secs(15 * 60)) + ); + + let outbound = parse_server(&env, "LINKERD2_PROXY_OUTBOUND_SERVER_HTTP2").unwrap(); + assert_eq!( + outbound.max_connection_age, + Some(Duration::from_secs(60 * 60)) + ); + } }