From 31e264a055fda04025100e9d901e69d88e859dd7 Mon Sep 17 00:00:00 2001 From: Steve Hu Date: Thu, 29 Oct 2026 14:05:56 -0400 Subject: [PATCH 01/15] Preserve chain exception boundaries across asynchronous continuations --- README.md | 17 ++- .../networknt/exception/ExceptionHandler.java | 8 +- .../handler/HandlerContinuation.java | 48 ++++++++ .../RequestInterceptorInjectionHandler.java | 4 +- .../handler/HandlerContinuationTest.java | 58 +++++++++ ingress-proxy/pom.xml | 1 + .../proxy/ExternalServiceHandler.java | 6 +- .../ExternalServiceWorkerDispatchTest.java | 115 ++++++++++++++++-- 8 files changed, 228 insertions(+), 29 deletions(-) create mode 100644 handler/src/main/java/com/networknt/handler/HandlerContinuation.java create mode 100644 handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java diff --git a/README.md b/README.md index 2b42bea00b..836f8e7728 100644 --- a/README.md +++ b/README.md @@ -253,16 +253,13 @@ does not re-enter routing or write shared timeout fields. The streaming ExternalServiceWorkerDispatchTest forces read-listener continuation and verifies that an unrelated request progresses while the upstream is blocked; it also checks body preservation, one rewrite, direct GETs, and unchanged unmatched routing. -The worker continuation currently constructs a new default ExceptionHandler per -dispatched request. It returns the default structured error response (ERR10011 / -HTTP 400 for an uncaught IOException) after the original exception-handler frame -has returned. This does not reuse the chain's configured handler or its enabled -flag: the dispatched path applies default exception handling even when the chain -intentionally disables it. Other worker-entry paths still use their surrounding -chain policy. This is an accepted limitation of this fix, tracked in -[issue #2820](https://github.com/networknt/light-4j/issues/2820) for separate -exception-module continuation support and review. The current implementation -retains ingress-proxy's compile dependency on the exception module. +HandlerContinuation in the handler module captures the active chain exception +boundary before asynchronous body reads and matched-request worker dispatch. +ExceptionHandler supplies its existing policy without rebuilding middleware or +changing its next handler. Named handlers are retained; when exception handling +is disabled and omitted from the chain, failures remain under Undertow's default +policy. Nested boundaries restore their enclosing scope, and state is isolated +per exchange. ingress-proxy uses exception only for tests. Regressions compare worker-entry and streamed-continuation failures, cover 405 rejection and metrics injection, and check that concurrent initialization shares one HTTP client. diff --git a/exception/src/main/java/com/networknt/exception/ExceptionHandler.java b/exception/src/main/java/com/networknt/exception/ExceptionHandler.java index 955e3791d1..7a5342836c 100644 --- a/exception/src/main/java/com/networknt/exception/ExceptionHandler.java +++ b/exception/src/main/java/com/networknt/exception/ExceptionHandler.java @@ -17,6 +17,7 @@ package com.networknt.exception; import com.networknt.handler.Handler; +import com.networknt.handler.HandlerContinuation; import com.networknt.handler.MiddlewareHandler; import io.undertow.Handlers; import io.undertow.server.HttpHandler; @@ -79,13 +80,18 @@ public void handleRequest(final HttpServerExchange exchange) throws Exception { return; } + HandlerContinuation.execute(exchange, this::handleContinuation, + ex -> Handler.next(ex, next)); + } + + private void handleContinuation(HttpServerExchange exchange, HttpHandler continuation) throws Exception { // These two lines of code should be placed here to avoid the log being printed twice // when switching from the IO thread to the worker thread. String path = exchange.getRequestPath(); if(logger.isDebugEnabled()) logger.debug("ExceptionHandler starting - path: {}", path); try { - Handler.next(exchange, next); + continuation.handleRequest(exchange); } catch (Throwable e) { logger.error("ExceptionHandler error processing - path: {}", path, e); if(exchange.isResponseChannelAvailable()) { diff --git a/handler/src/main/java/com/networknt/handler/HandlerContinuation.java b/handler/src/main/java/com/networknt/handler/HandlerContinuation.java new file mode 100644 index 0000000000..0852d4b978 --- /dev/null +++ b/handler/src/main/java/com/networknt/handler/HandlerContinuation.java @@ -0,0 +1,48 @@ +package com.networknt.handler; + +import io.undertow.server.HttpHandler; +import io.undertow.server.HttpServerExchange; +import io.undertow.util.AttachmentKey; + +/** + * Carries the active exception boundary across an asynchronous hand-off. Boundary + * state belongs to an exchange, never a thread or a shared middleware instance. + * Capture before returning from the current chain frame; invoke the captured + * handler through Undertow's dispatch or root-handler machinery. + */ +public final class HandlerContinuation { + private static final AttachmentKey ACTIVE = AttachmentKey.create(Boundary.class); + + private HandlerContinuation() { } + + /** A chain-owned exception policy around a continuation, without changing its next handler. */ + @FunctionalInterface + public interface Boundary { + void handle(HttpServerExchange exchange, HttpHandler continuation) throws Exception; + } + + /** + * Runs inside the supplied boundary and restores the enclosing boundary on + * return. Nested boundaries and continuations can safely capture their scope. + */ + public static void execute(HttpServerExchange exchange, Boundary boundary, + HttpHandler continuation) throws Exception { + Boundary previous = exchange.putAttachment(ACTIVE, boundary); + try { + if (boundary == null) continuation.handleRequest(exchange); + else boundary.handle(exchange, continuation); + } finally { + if (previous == null) exchange.removeAttachment(ACTIVE); + else exchange.putAttachment(ACTIVE, previous); + } + } + + /** + * Captures the selected boundary, including its deliberate absence. No default + * exception policy is installed when the chain has none. + */ + public static HttpHandler capture(HttpServerExchange exchange, HttpHandler continuation) { + Boundary boundary = exchange.getAttachment(ACTIVE); + return resumed -> execute(resumed, boundary, continuation); + } +} diff --git a/handler/src/main/java/com/networknt/handler/RequestInterceptorInjectionHandler.java b/handler/src/main/java/com/networknt/handler/RequestInterceptorInjectionHandler.java index d772b72cc5..b3b970885b 100644 --- a/handler/src/main/java/com/networknt/handler/RequestInterceptorInjectionHandler.java +++ b/handler/src/main/java/com/networknt/handler/RequestInterceptorInjectionHandler.java @@ -178,7 +178,7 @@ private final class RequestBodyReadState implements ChannelListener Handler.next(ex, continuation)); this.channel = exchange.getRequestChannel(); this.bufferedData = new PooledByteBuffer[config.getMaxBuffers()]; this.exactLimit = config.getMaxBodyBytes() > 0; @@ -274,7 +274,7 @@ public void handleEvent(StreamSourceChannel ignored) { if (outcome == ReadOutcome.COMPLETE) { saveBufferAndResetUndertowConnector(exchange, bufferedData); if (!exchange.isResponseStarted()) - Connectors.executeRootHandler(ex -> Handler.next(ex, continuation), exchange); + Connectors.executeRootHandler(continuation, exchange); } } catch (Throwable e) { LOG.error(e.getMessage(), e); diff --git a/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java b/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java new file mode 100644 index 0000000000..2063075836 --- /dev/null +++ b/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java @@ -0,0 +1,58 @@ +package com.networknt.handler; + +import io.undertow.server.HttpHandler; +import io.undertow.server.HttpServerExchange; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.*; + +class HandlerContinuationTest { + @Test + void nestedCaptureRestoresEnclosingBoundaryAndRetainsSelectedInstances() throws Exception { + HttpServerExchange exchange = new HttpServerExchange(null); + List seen = new ArrayList<>(); + HandlerContinuation.Boundary outer = (ex, task) -> { + seen.add("outer"); + task.handleRequest(ex); + }; + HandlerContinuation.Boundary inner = (ex, task) -> { + seen.add("inner"); + task.handleRequest(ex); + }; + AtomicReference nested = new AtomicReference<>(); + AtomicReference enclosing = new AtomicReference<>(); + HandlerContinuation.execute(exchange, outer, ex -> { + HandlerContinuation.execute(ex, inner, innerExchange -> + nested.set(HandlerContinuation.capture(innerExchange, ignored -> seen.add("nested")))); + enclosing.set(HandlerContinuation.capture(ex, ignored -> seen.add("enclosing"))); + }); + seen.clear(); + nested.get().handleRequest(exchange); + enclosing.get().handleRequest(exchange); + assertEquals(List.of("inner", "nested", "outer", "enclosing"), seen); + seen.clear(); + HandlerContinuation.capture(exchange, ignored -> seen.add("unwrapped")).handleRequest(exchange); + assertEquals(List.of("unwrapped"), seen, "Neither frame may leave a boundary behind"); + } + + @Test + void absentBoundaryStaysAbsentAndFailuresRestoreScope() throws Exception { + HttpServerExchange exchange = new HttpServerExchange(null); + IOException failure = new IOException("failure"); + HttpHandler unwrapped = HandlerContinuation.capture(exchange, ignored -> { throw failure; }); + HandlerContinuation.Boundary selected = (ex, task) -> { + try { task.handleRequest(ex); } + catch (IOException e) { fail("A captured absent boundary must hide the active boundary", e); } + }; + HandlerContinuation.execute(exchange, selected, ex -> + assertSame(failure, assertThrows(IOException.class, () -> unwrapped.handleRequest(ex)))); + assertSame(failure, assertThrows(IOException.class, () -> + HandlerContinuation.execute(exchange, (ex, task) -> { throw failure; }, ignored -> {}))); + assertSame(failure, assertThrows(IOException.class, () -> unwrapped.handleRequest(exchange))); + } +} diff --git a/ingress-proxy/pom.xml b/ingress-proxy/pom.xml index 5de587361a..1677ea9c23 100644 --- a/ingress-proxy/pom.xml +++ b/ingress-proxy/pom.xml @@ -19,6 +19,7 @@ com.networknt exception + test com.networknt diff --git a/ingress-proxy/src/main/java/com/networknt/proxy/ExternalServiceHandler.java b/ingress-proxy/src/main/java/com/networknt/proxy/ExternalServiceHandler.java index 61729db2f6..ba1539a46c 100644 --- a/ingress-proxy/src/main/java/com/networknt/proxy/ExternalServiceHandler.java +++ b/ingress-proxy/src/main/java/com/networknt/proxy/ExternalServiceHandler.java @@ -4,7 +4,6 @@ import com.networknt.client.Http2Client; import com.networknt.client.ssl.TLSConfig; import com.networknt.config.Config; -import com.networknt.exception.ExceptionHandler; import com.networknt.handler.*; import com.networknt.handler.config.UrlRewriteRule; import com.networknt.httpstring.AttachmentConstants; @@ -133,9 +132,8 @@ public void handleRequest(HttpServerExchange exchange) throws Exception { requestHost, currentTimeout, connectTimeout, startTime); // Capture the match once: asynchronous body reads can resume on an I/O thread. if (exchange.isInIoThread()) { - exchange.dispatch(workerExchange -> new ExceptionHandler() - .setNext(forwardExchange -> forwardMatchedRequest(forwardExchange, context)) - .handleRequest(workerExchange)); + exchange.dispatch(HandlerContinuation.capture(exchange, + forwardExchange -> forwardMatchedRequest(forwardExchange, context))); } else { forwardMatchedRequest(exchange, context); } diff --git a/ingress-proxy/src/test/java/com/networknt/proxy/ExternalServiceWorkerDispatchTest.java b/ingress-proxy/src/test/java/com/networknt/proxy/ExternalServiceWorkerDispatchTest.java index aef4e2addf..d8e5c283e2 100644 --- a/ingress-proxy/src/test/java/com/networknt/proxy/ExternalServiceWorkerDispatchTest.java +++ b/ingress-proxy/src/test/java/com/networknt/proxy/ExternalServiceWorkerDispatchTest.java @@ -22,6 +22,11 @@ import org.junit.jupiter.api.Timeout; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.Arguments; +import io.undertow.util.HttpString; +import java.util.stream.Stream; import java.io.ByteArrayOutputStream; import java.net.InetSocketAddress; @@ -55,9 +60,44 @@ class ExternalServiceWorkerDispatchTest { private RequestInterceptor[] previousInterceptors; private String previousTlsProperty; private HttpHandler previousMetrics; + private Map previousExceptionConfig; + private static final String NAMED_EXCEPTION = "external-worker-named-exception-test"; + + enum Policy { ENABLED, NAMED, DISABLED, NAMED_DISABLED } + + static Stream streamedPolicies() { + return Stream.of(Policy.values()).flatMap(policy -> + Stream.of(false, true).map(failure -> Arguments.of(failure, policy))); + } + + private HttpHandler withPolicy(Policy policy, HttpHandler next) { + boolean named = policy == Policy.NAMED || policy == Policy.NAMED_DISABLED; + boolean enabled = policy == Policy.ENABLED || policy == Policy.NAMED; + // The default deliberately disagrees with the selected named configuration. + Config.getInstance().putInConfigCache("exception", Map.of("enabled", named ? !enabled : enabled)); + Config.getInstance().putInConfigCache(NAMED_EXCEPTION, Map.of("enabled", enabled)); + ExceptionHandler selected = new ExceptionHandler(named ? NAMED_EXCEPTION : "exception") { + @Override + public void setExchangeStatus(HttpServerExchange exchange, String code, Object... args) { + exchange.getResponseHeaders().put(new HttpString("X-Exception-Policy"), policy.name()); + super.setExchangeStatus(exchange, code, args); + } + }; + selected.setNext(next); + // Match handler-chain materialization: disabled middleware is omitted. + return selected.isEnabled() ? selected : next; + } + + private static void assertFailure(Policy policy, int status, String body, String selected) { + boolean enabled = policy == Policy.ENABLED || policy == Policy.NAMED; + assertEquals(enabled ? 400 : 500, status, body); + assertEquals(enabled ? policy.name() : "", selected, "Use the selected instance only"); + assertEquals(enabled, body.contains("ERR10011"), body); + } @BeforeEach void rememberSharedState() { + previousExceptionConfig = Config.getInstance().getJsonMapConfig("exception"); previousInterceptors = SingletonServiceFactory.getBeans(RequestInterceptor.class); previousTlsProperty = System.getProperty(TLS_PROPERTY); previousMetrics = Handler.getHandlers().get(MetricsConfig.CONFIG_NAME); @@ -68,6 +108,8 @@ void restoreSharedState() { SingletonServiceFactory.setBean(RequestInterceptor.class.getName(), previousInterceptors); if (previousMetrics == null) Handler.getHandlers().remove(MetricsConfig.CONFIG_NAME); else Handler.getHandlers().put(MetricsConfig.CONFIG_NAME, previousMetrics); + Config.getInstance().putInConfigCache("exception", previousExceptionConfig); + Config.getInstance().clearConfigCache(NAMED_EXCEPTION); Config.getInstance().clearConfigCache(EXTERNAL_CONFIG); Config.getInstance().clearConfigCache(INJECTION_CONFIG); if (previousTlsProperty == null) System.clearProperty(TLS_PROPERTY); @@ -75,8 +117,8 @@ void restoreSharedState() { } @ParameterizedTest - @ValueSource(booleans = {false, true}) - void streamedBodyContinuationCannotBlockTheFrontEndIoThread(boolean upstreamFailure) throws Exception { + @MethodSource("streamedPolicies") + void streamedBodyContinuationCannotBlockTheFrontEndIoThread(boolean upstreamFailure, Policy policy) throws Exception { BodyCaptureInterceptor interceptor = new BodyCaptureInterceptor(); SingletonServiceFactory.setBean(RequestInterceptor.class.getName(), new RequestInterceptor[]{interceptor}); @@ -89,7 +131,7 @@ void streamedBodyContinuationCannotBlockTheFrontEndIoThread(boolean upstreamFail RequestInterceptorInjectionHandler reader = new RequestInterceptorInjectionHandler(INJECTION_CONFIG); reader.setNext(injection); - return new ExceptionHandler().setNext(exchange -> { + return withPolicy(policy, exchange -> { reader.handleRequest(exchange); initialReadReturned.countDown(); }); @@ -116,9 +158,7 @@ void streamedBodyContinuationCannotBlockTheFrontEndIoThread(boolean upstreamFail fixture.releaseUpstream.countDown(); String response = new String(socket.getInputStream().readAllBytes(), StandardCharsets.UTF_8); if (upstreamFailure) { - assertTrue(response.startsWith("HTTP/1.1 400"), response); - assertTrue(response.contains("application/json"), response); - assertTrue(response.contains("ERR10011"), "Use framework structured errors: " + response); + assertRawFailure(policy, response); } else { assertTrue(response.startsWith("HTTP/1.1 200"), response); assertTrue(response.endsWith("accepted"), response); @@ -171,18 +211,69 @@ void unmatchedRequestRetainsItsOriginalIoThread() throws Exception { } } - @Test - void workerThreadFailureUsesTheSameFrameworkError() throws Exception { - try (ProxyFixture fixture = new ProxyFixture(handler -> new ExceptionHandler().setNext(handler)); + @ParameterizedTest + @EnumSource(Policy.class) + void workerThreadFailureUsesTheSelectedPolicy(Policy policy) throws Exception { + try (ProxyFixture fixture = new ProxyFixture(handler -> new BlockingHandler(withPolicy(policy, handler))); HttpClient client = HttpClient.newHttpClient()) { fixture.failUpstream.set(true); fixture.releaseUpstream.countDown(); HttpResponse response = client.send( HttpRequest.newBuilder(fixture.uri("/proxy/failure")) .timeout(Duration.ofSeconds(5)).build(), HttpResponse.BodyHandlers.ofString()); - assertEquals(400, response.statusCode(), "Preserve the framework ERR10011 status"); - assertTrue(response.headers().firstValue("Content-Type").orElse("").contains("application/json")); - assertTrue(response.body().contains("ERR10011"), response.body()); + assertFailure(policy, response.statusCode(), response.body(), + response.headers().firstValue("X-Exception-Policy").orElse("")); + } + } + + private static void assertRawFailure(Policy policy, String response) { + int status = Integer.parseInt(response.split(" ", 3)[1]); + String selected = response.lines().filter(line -> line.toLowerCase().startsWith("x-exception-policy:")) + .map(line -> line.substring(line.indexOf(':') + 1).trim()).findFirst().orElse(""); + assertFailure(policy, status, response, selected); + if (status == 400) assertTrue(response.contains("application/json"), response); + } + + @Test + void concurrentStreamedRequestsKeepTheirOwnBoundary() throws Exception { + BodyCaptureInterceptor interceptor = new BodyCaptureInterceptor(); + SingletonServiceFactory.setBean(RequestInterceptor.class.getName(), new RequestInterceptor[]{interceptor}); + Config.getInstance().putInConfigCache(INJECTION_CONFIG, Map.of( + "enabled", true, "maxBuffers", 4, "maxBodyBytes", 1024, + "appliedBodyInjectionPathPrefixes", List.of("/proxy"))); + int count = 8; + CountDownLatch initialReads = new CountDownLatch(count); + try (ProxyFixture fixture = new ProxyFixture(handler -> { + var chains = new java.util.EnumMap(Policy.class); + for (Policy policy : Policy.values()) { + RequestInterceptorInjectionHandler reader = new RequestInterceptorInjectionHandler(INJECTION_CONFIG); + reader.setNext(handler); + chains.put(policy, withPolicy(policy, exchange -> { + reader.handleRequest(exchange); + initialReads.countDown(); + })); + } + return exchange -> chains.get(Policy.valueOf(exchange.getRequestHeaders().getFirst("X-Policy"))) + .handleRequest(exchange); + }); var executor = Executors.newFixedThreadPool(count)) { + fixture.failUpstream.set(true); + fixture.releaseUpstream.countDown(); + var futures = IntStream.range(0, count).mapToObj(i -> executor.submit(() -> { + Policy policy = Policy.values()[i % Policy.values().length]; + try (Socket socket = new Socket("localhost", fixture.port())) { + socket.setSoTimeout(10000); + write(socket, "POST /proxy/concurrent HTTP/1.1\r\nHost: localhost\r\n" + + "X-Policy: " + policy + "\r\nContent-Type: application/json\r\n" + + "Transfer-Encoding: chunked\r\nConnection: close\r\n\r\n1\r\n{\r\n"); + assertTrue(initialReads.await(5, TimeUnit.SECONDS), "All original frames must return before resuming"); + write(socket, "1\r\n}\r\n0\r\n\r\n"); + assertRawFailure(policy, new String(socket.getInputStream().readAllBytes(), StandardCharsets.UTF_8)); + } + return null; + })).toList(); + for (var future : futures) future.get(15, TimeUnit.SECONDS); + assertEquals(count, fixture.handlerEntries.get()); + assertTrue(interceptor.onIoThread.get()); } } From 8f9d1ce94e499abfe5ba4b192036fdbf02c26719 Mon Sep 17 00:00:00 2001 From: Steve Hu Date: Thu, 29 Oct 2026 14:24:13 -0400 Subject: [PATCH 02/15] Carry exception boundaries through framework async hand-offs --- README.md | 11 ++- .../java/com/networknt/body/BodyHandler.java | 3 +- .../networknt/body/BodyContinuationTest.java | 38 +++++++++ .../java/com/networknt/dump/DumpHandler.java | 2 +- .../networknt/dump/DumpContinuationTest.java | 39 +++++++++ .../networknt/exception/ExceptionHandler.java | 11 ++- .../exception/ExceptionContinuationTest.java | 70 ++++++++++++++++ .../handler/HandlerContinuation.java | 67 ++++++++++++++-- .../RequestInterceptorInjectionHandler.java | 11 ++- .../handler/HandlerContinuationTest.java | 21 +++++ .../RequestInjectionContinuationTest.java | 79 +++++++++++++++++++ .../java/com/networknt/hmac/HmacHandler.java | 6 +- .../hmac/HmacReplayAdminHandler.java | 6 +- .../hmac/HmacHandlerIntegrationTest.java | 25 +++++- .../proxy/ExternalServiceHandler.java | 5 +- .../proxy/ExternalServiceInterruptTest.java | 66 ++++++++++++++++ .../ExternalServiceWorkerDispatchTest.java | 23 +++--- .../java/com/networknt/mcp/McpHandler.java | 2 +- .../networknt/mcp/McpContinuationTest.java | 39 +++++++++ .../com/networknt/handler/ProxyHandler.java | 15 +++- .../handler/ProxyContinuationTest.java | 48 +++++++++++ 21 files changed, 547 insertions(+), 40 deletions(-) create mode 100644 body/src/test/java/com/networknt/body/BodyContinuationTest.java create mode 100644 dump/src/test/java/com/networknt/dump/DumpContinuationTest.java create mode 100644 exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java create mode 100644 handler/src/test/java/com/networknt/handler/RequestInjectionContinuationTest.java create mode 100644 ingress-proxy/src/test/java/com/networknt/proxy/ExternalServiceInterruptTest.java create mode 100644 mcp-router/src/test/java/com/networknt/mcp/McpContinuationTest.java create mode 100644 proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java diff --git a/README.md b/README.md index 836f8e7728..d2f71ab0a9 100644 --- a/README.md +++ b/README.md @@ -259,7 +259,16 @@ ExceptionHandler supplies its existing policy without rebuilding middleware or changing its next handler. Named handlers are retained; when exception handling is disabled and omitted from the chain, failures remain under Undertow's default policy. Nested boundaries restore their enclosing scope, and state is isolated -per exchange. ingress-proxy uses exception only for tests. +per exchange. The request MDC is captured with each continuation and the resuming +thread's previous MDC is restored afterward. BodyHandler redispatch, HMAC replay +and administration completion, DumpHandler/McpHandler redispatch, and +ProxyHandler connection/action dispatch also +carry the selected boundary. Proxy callbacks retain their pending-I/O lifecycle. +Custom asynchronous handlers must explicitly capture before registering their +callbacks; this contract cannot intercept arbitrary third-party hand-offs. +Body-reading requests capture lazily only when a read suspends. ingress-proxy uses +exception only for tests. Interrupted upstream sends restore the interrupt flag +and stop retrying immediately. Regressions compare worker-entry and streamed-continuation failures, cover 405 rejection and metrics injection, and check that concurrent initialization shares one HTTP client. diff --git a/body/src/main/java/com/networknt/body/BodyHandler.java b/body/src/main/java/com/networknt/body/BodyHandler.java index 2246c91e21..77dc9ec2d8 100644 --- a/body/src/main/java/com/networknt/body/BodyHandler.java +++ b/body/src/main/java/com/networknt/body/BodyHandler.java @@ -19,6 +19,7 @@ import tools.jackson.core.type.TypeReference; import com.networknt.config.Config; import com.networknt.handler.Handler; +import com.networknt.handler.HandlerContinuation; import com.networknt.handler.MiddlewareHandler; import com.networknt.httpstring.AttachmentConstants; import com.networknt.utility.StringUtils; @@ -105,7 +106,7 @@ public void handleRequest(final HttpServerExchange exchange) throws Exception { String contentType = exchange.getRequestHeaders().getFirst(Headers.CONTENT_TYPE); if (contentType != null) { if (exchange.isInIoThread()) { - exchange.dispatch(this); + exchange.dispatch(HandlerContinuation.capture(exchange, this)); return; } exchange.startBlocking(); diff --git a/body/src/test/java/com/networknt/body/BodyContinuationTest.java b/body/src/test/java/com/networknt/body/BodyContinuationTest.java new file mode 100644 index 0000000000..48919fb462 --- /dev/null +++ b/body/src/test/java/com/networknt/body/BodyContinuationTest.java @@ -0,0 +1,38 @@ +package com.networknt.body; + +import com.networknt.handler.HandlerContinuation; +import io.undertow.Undertow; +import io.undertow.server.HttpHandler; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.http.*; +import java.time.Duration; +import static org.junit.jupiter.api.Assertions.*; + +class BodyContinuationTest { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void redispatchRetainsBoundary(boolean enabled) throws Exception { + BodyHandler body = new BodyHandler(); + body.setNext(exchange -> { + assertFalse(exchange.isInIoThread()); + throw new IllegalStateException("downstream failure"); + }); + HttpHandler root = enabled ? exchange -> HandlerContinuation.execute(exchange, (ex, task) -> { + try { task.handleRequest(ex); } + catch (IllegalStateException e) { ex.setStatusCode(409).getResponseSender().send("selected"); } + }, body) : body; + Undertow server = Undertow.builder().addHttpListener(0, "127.0.0.1").setHandler(root).build(); + try (HttpClient client = HttpClient.newHttpClient()) { + server.start(); + int port = ((InetSocketAddress) server.getListenerInfo().getFirst().getAddress()).getPort(); + var response = client.send(HttpRequest.newBuilder(URI.create("http://127.0.0.1:" + port + "/")) + .timeout(Duration.ofSeconds(5)).header("Content-Type", "text/plain") + .POST(HttpRequest.BodyPublishers.ofString("body")).build(), HttpResponse.BodyHandlers.ofString()); + assertEquals(enabled ? 409 : 500, response.statusCode(), response.body()); + assertEquals(enabled, response.body().contains("selected")); + } finally { server.stop(); } + } +} diff --git a/dump/src/main/java/com/networknt/dump/DumpHandler.java b/dump/src/main/java/com/networknt/dump/DumpHandler.java index 8a1d696ce5..bedaa3199f 100644 --- a/dump/src/main/java/com/networknt/dump/DumpHandler.java +++ b/dump/src/main/java/com/networknt/dump/DumpHandler.java @@ -77,7 +77,7 @@ public boolean isEnabled() { @Override public void handleRequest(final HttpServerExchange exchange) throws Exception { if (exchange.isInIoThread()) { - exchange.dispatch(this); + exchange.dispatch(com.networknt.handler.HandlerContinuation.capture(exchange, this)); return; } if(isEnabled()) { diff --git a/dump/src/test/java/com/networknt/dump/DumpContinuationTest.java b/dump/src/test/java/com/networknt/dump/DumpContinuationTest.java new file mode 100644 index 0000000000..a117e27626 --- /dev/null +++ b/dump/src/test/java/com/networknt/dump/DumpContinuationTest.java @@ -0,0 +1,39 @@ +package com.networknt.dump; + +import com.networknt.handler.HandlerContinuation; +import io.undertow.Undertow; +import io.undertow.server.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.http.*; +import java.time.Duration; +import static org.junit.jupiter.api.Assertions.*; + +class DumpContinuationTest { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void redispatchRetainsSelectedBoundary(boolean enabled) throws Exception { + DumpHandler handler = new DumpHandler() { + @Override public void handleRequest(HttpServerExchange exchange) throws Exception { + if (!exchange.isInIoThread()) throw new IllegalStateException("resumed failure"); + super.handleRequest(exchange); + } + }; + HttpHandler root = enabled ? exchange -> HandlerContinuation.execute(exchange, (ex, task) -> { + try { task.handleRequest(ex); } + catch (IllegalStateException e) { ex.setStatusCode(409).getResponseSender().send("selected"); } + }, handler) : handler; + Undertow server = Undertow.builder().addHttpListener(0, "127.0.0.1").setHandler(root).build(); + try (HttpClient client = HttpClient.newHttpClient()) { + server.start(); + int port = ((InetSocketAddress) server.getListenerInfo().getFirst().getAddress()).getPort(); + var response = client.send(HttpRequest.newBuilder(URI.create("http://127.0.0.1:" + port + "/mcp")) + .timeout(Duration.ofSeconds(5)).POST(HttpRequest.BodyPublishers.ofString("{}")) + .build(), HttpResponse.BodyHandlers.ofString()); + assertEquals(enabled ? 409 : 500, response.statusCode(), response.body()); + assertEquals(enabled, response.body().contains("selected")); + } finally { server.stop(); } + } +} diff --git a/exception/src/main/java/com/networknt/exception/ExceptionHandler.java b/exception/src/main/java/com/networknt/exception/ExceptionHandler.java index 7a5342836c..b84dc406e0 100644 --- a/exception/src/main/java/com/networknt/exception/ExceptionHandler.java +++ b/exception/src/main/java/com/networknt/exception/ExceptionHandler.java @@ -85,10 +85,10 @@ public void handleRequest(final HttpServerExchange exchange) throws Exception { } private void handleContinuation(HttpServerExchange exchange, HttpHandler continuation) throws Exception { - // These two lines of code should be placed here to avoid the log being printed twice - // when switching from the IO thread to the worker thread. + // Log each executing frame, including asynchronous continuations. The + // captured continuation restores the resuming thread's MDC after this frame. String path = exchange.getRequestPath(); - if(logger.isDebugEnabled()) logger.debug("ExceptionHandler starting - path: {}", path); + if(logger.isDebugEnabled()) logger.debug("ExceptionHandler frame starting - path: {}", path); try { continuation.handleRequest(exchange); @@ -130,9 +130,8 @@ private void handleContinuation(HttpServerExchange exchange, HttpHandler continu } } } finally { - if(logger.isDebugEnabled()) logger.debug("ExceptionHandler completed - path: {}", path); - // at last, clean the MDC. Most likely, correlationId in side. - //logger.debug("Clear MDC"); + if(logger.isDebugEnabled()) logger.debug("ExceptionHandler frame completed - path: {}", path); + // Clear this request frame; a captured continuation restores its caller's MDC. MDC.clear(); } } diff --git a/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java b/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java new file mode 100644 index 0000000000..805c2c10e5 --- /dev/null +++ b/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java @@ -0,0 +1,70 @@ +package com.networknt.exception; + +import com.networknt.config.Config; +import com.networknt.handler.HandlerContinuation; +import io.undertow.Undertow; +import io.undertow.server.HttpHandler; +import io.undertow.server.HttpServerExchange; +import io.undertow.util.HttpString; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.api.Timeout; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.http.*; +import java.time.Duration; +import java.util.Map; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.*; + +@Timeout(20) +class ExceptionContinuationTest { + static Stream cases() { + return Stream.of("exception", "continuation-named", "continuation-disabled") + .flatMap(name -> Stream.of(false, true).flatMap(async -> + Stream.of(false, true).map(runtime -> Arguments.of(name, async, runtime)))); + } + + @ParameterizedTest + @MethodSource("cases") + void selectedPolicyHandlesInitialAndResumedFailures(String name, boolean async, boolean runtime) throws Exception { + boolean enabled = !name.endsWith("disabled"); + Map previous = Config.getInstance().getJsonMapConfig("exception"); + Config.getInstance().putInConfigCache(name, Map.of("enabled", enabled)); + ExceptionHandler selected = new ExceptionHandler(name) { + @Override public void setExchangeStatus(HttpServerExchange exchange, String code, Object... args) { + exchange.getResponseHeaders().put(new HttpString("X-Selected"), name); + super.setExchangeStatus(exchange, code, args); + } + }; + HttpHandler failing = exchange -> { + if (runtime) throw new IllegalStateException("runtime failure"); + throw new IOException("checked failure"); + }; + HttpHandler work = exchange -> { + if (async) exchange.dispatch(HandlerContinuation.capture(exchange, failing)); + else failing.handleRequest(exchange); + }; + selected.setNext(work); + Undertow server = Undertow.builder().addHttpListener(0, "127.0.0.1") + .setHandler(selected.isEnabled() ? selected : work).build(); + try (HttpClient client = HttpClient.newHttpClient()) { + server.start(); + int port = ((InetSocketAddress) server.getListenerInfo().getFirst().getAddress()).getPort(); + HttpResponse response = client.send(HttpRequest.newBuilder( + URI.create("http://127.0.0.1:" + port + "/failure")).timeout(Duration.ofSeconds(5)).build(), + HttpResponse.BodyHandlers.ofString()); + assertEquals(enabled ? (runtime ? 500 : 400) : 500, response.statusCode(), response.body()); + assertEquals(enabled ? name : "", response.headers().firstValue("X-Selected").orElse("")); + assertEquals(enabled, response.body().contains(runtime ? "ERR10010" : "ERR10011"), response.body()); + } finally { + server.stop(); + if (name.equals("exception")) Config.getInstance().putInConfigCache(name, previous); + else Config.getInstance().clearConfigCache(name); + } + } +} diff --git a/handler/src/main/java/com/networknt/handler/HandlerContinuation.java b/handler/src/main/java/com/networknt/handler/HandlerContinuation.java index 0852d4b978..43c08c7381 100644 --- a/handler/src/main/java/com/networknt/handler/HandlerContinuation.java +++ b/handler/src/main/java/com/networknt/handler/HandlerContinuation.java @@ -1,8 +1,14 @@ package com.networknt.handler; import io.undertow.server.HttpHandler; +import io.undertow.server.Connectors; import io.undertow.server.HttpServerExchange; import io.undertow.util.AttachmentKey; +import org.slf4j.MDC; + +import java.util.Map; +import java.util.Objects; +import java.util.function.UnaryOperator; /** * Carries the active exception boundary across an asynchronous hand-off. Boundary @@ -11,11 +17,16 @@ * handler through Undertow's dispatch or root-handler machinery. */ public final class HandlerContinuation { + private static final Boundary UNHANDLED = (exchange, continuation) -> continuation.handleRequest(exchange); private static final AttachmentKey ACTIVE = AttachmentKey.create(Boundary.class); private HandlerContinuation() { } - /** A chain-owned exception policy around a continuation, without changing its next handler. */ + /** + * A chain-owned exception policy, without changing its next handler. Invoke + * the supplied continuation exactly once; handle or rethrow its failure. + * Implementations must be safe to reuse concurrently across exchanges. + */ @FunctionalInterface public interface Boundary { void handle(HttpServerExchange exchange, HttpHandler continuation) throws Exception; @@ -27,10 +38,9 @@ public interface Boundary { */ public static void execute(HttpServerExchange exchange, Boundary boundary, HttpHandler continuation) throws Exception { - Boundary previous = exchange.putAttachment(ACTIVE, boundary); + Boundary previous = exchange.putAttachment(ACTIVE, Objects.requireNonNull(boundary)); try { - if (boundary == null) continuation.handleRequest(exchange); - else boundary.handle(exchange, continuation); + boundary.handle(exchange, continuation); } finally { if (previous == null) exchange.removeAttachment(ACTIVE); else exchange.putAttachment(ACTIVE, previous); @@ -42,7 +52,52 @@ public static void execute(HttpServerExchange exchange, Boundary boundary, * exception policy is installed when the chain has none. */ public static HttpHandler capture(HttpServerExchange exchange, HttpHandler continuation) { - Boundary boundary = exchange.getAttachment(ACTIVE); - return resumed -> execute(resumed, boundary, continuation); + return capture(exchange).apply(continuation); + } + + /** + * Captures before registering a callback whose result determines its eventual + * work. Apply the returned wrapper inside that callback. Framework hand-offs + * must opt in; arbitrary third-party callbacks cannot be intercepted here. + * The request MDC is installed only for the resumed frame and the executing + * thread's previous MDC is restored, including on failure. + */ + public static UnaryOperator capture(HttpServerExchange exchange) { + Boundary active = exchange.getAttachment(ACTIVE); + Boundary boundary = active == null ? UNHANDLED : active; + Map context = MDC.getCopyOfContextMap(); + return continuation -> resumed -> { + Map previous = MDC.getCopyOfContextMap(); + restoreMdc(context); + try { + execute(resumed, boundary, continuation); + } finally { + restoreMdc(previous); + } + }; + } + + /** + * Adapts an already captured handler for a Runnable dispatch. Unlike a root + * handler, a successful callback may leave upstream I/O pending; do not end + * that exchange. Unhandled failures still enter Undertow's error path. + */ + public static Runnable task(HttpServerExchange exchange, HttpHandler continuation) { + return () -> { + try { + continuation.handleRequest(exchange); + } catch (Throwable failure) { + Connectors.executeRootHandler(ex -> { + if (failure instanceof Error error) throw error; + if (failure instanceof Exception exception) throw exception; + throw new IllegalStateException(failure); + }, exchange); + } + }; + } + + private static void restoreMdc(Map context) { + if (context == null) MDC.clear(); + else MDC.setContextMap(context); } } diff --git a/handler/src/main/java/com/networknt/handler/RequestInterceptorInjectionHandler.java b/handler/src/main/java/com/networknt/handler/RequestInterceptorInjectionHandler.java index b3b970885b..1f4d8fe180 100644 --- a/handler/src/main/java/com/networknt/handler/RequestInterceptorInjectionHandler.java +++ b/handler/src/main/java/com/networknt/handler/RequestInterceptorInjectionHandler.java @@ -167,7 +167,8 @@ private enum ReadOutcome { private final class RequestBodyReadState implements ChannelListener { private final HttpServerExchange exchange; private final RequestInjectionConfig config; - private final HttpHandler continuation; + private final HttpHandler nextHandler; + private HttpHandler resumedHandler; private final StreamSourceChannel channel; private final PooledByteBuffer[] bufferedData; private final boolean exactLimit; @@ -175,10 +176,10 @@ private final class RequestBodyReadState implements ChannelListener Handler.next(ex, continuation)); + this.nextHandler = nextHandler; this.channel = exchange.getRequestChannel(); this.bufferedData = new PooledByteBuffer[config.getMaxBuffers()]; this.exactLimit = config.getMaxBodyBytes() > 0; @@ -241,6 +242,8 @@ private ReadOutcome probeForEndOfStream() throws Exception { } private ReadOutcome awaitMoreData() { + if (resumedHandler == null) + resumedHandler = HandlerContinuation.capture(exchange, ex -> Handler.next(ex, nextHandler)); channel.getReadSetter().set(this); channel.resumeReads(); return ReadOutcome.PENDING; @@ -274,7 +277,7 @@ public void handleEvent(StreamSourceChannel ignored) { if (outcome == ReadOutcome.COMPLETE) { saveBufferAndResetUndertowConnector(exchange, bufferedData); if (!exchange.isResponseStarted()) - Connectors.executeRootHandler(continuation, exchange); + Connectors.executeRootHandler(resumedHandler, exchange); } } catch (Throwable e) { LOG.error(e.getMessage(), e); diff --git a/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java b/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java index 2063075836..b6c352d7f4 100644 --- a/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java +++ b/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java @@ -12,6 +12,27 @@ import static org.junit.jupiter.api.Assertions.*; class HandlerContinuationTest { + @Test + void resumedFrameCarriesRequestMdcAndRestoresCallingThreadEvenOnFailure() throws Exception { + var original = org.slf4j.MDC.getCopyOfContextMap(); + try { + HttpServerExchange exchange = new HttpServerExchange(null); + org.slf4j.MDC.put("request", "captured"); + HttpHandler resumed = HandlerContinuation.capture(exchange, ex -> { + assertEquals("captured", org.slf4j.MDC.get("request")); + org.slf4j.MDC.clear(); + throw new IOException("failure"); + }); + org.slf4j.MDC.clear(); + org.slf4j.MDC.put("worker", "unrelated"); + assertThrows(IOException.class, () -> resumed.handleRequest(exchange)); + assertEquals(java.util.Map.of("worker", "unrelated"), org.slf4j.MDC.getCopyOfContextMap()); + } finally { + if (original == null) org.slf4j.MDC.clear(); + else org.slf4j.MDC.setContextMap(original); + } + } + @Test void nestedCaptureRestoresEnclosingBoundaryAndRetainsSelectedInstances() throws Exception { HttpServerExchange exchange = new HttpServerExchange(null); diff --git a/handler/src/test/java/com/networknt/handler/RequestInjectionContinuationTest.java b/handler/src/test/java/com/networknt/handler/RequestInjectionContinuationTest.java new file mode 100644 index 0000000000..90f01b0646 --- /dev/null +++ b/handler/src/test/java/com/networknt/handler/RequestInjectionContinuationTest.java @@ -0,0 +1,79 @@ +package com.networknt.handler; + +import com.networknt.config.Config; +import com.networknt.service.SingletonServiceFactory; +import io.undertow.Undertow; +import io.undertow.server.HttpHandler; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.junit.jupiter.api.Timeout; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.jupiter.api.Assertions.*; + +@Timeout(20) +class RequestInjectionContinuationTest { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void partialBodyRetainsBoundaryOrItsAbsence(boolean enabled) throws Exception { + String config = "request-continuation-module-test"; + RequestInterceptor[] previous = SingletonServiceFactory.getBeans(RequestInterceptor.class); + RequestInterceptor interceptor = new RequestInterceptor() { + @Override public boolean isRequiredContent() { return true; } + @Override public boolean isEnabled() { return true; } + @Override public HttpHandler getNext() { return null; } + @Override public MiddlewareHandler setNext(HttpHandler next) { return this; } + @Override public void handleRequest(io.undertow.server.HttpServerExchange exchange) { } + }; + SingletonServiceFactory.setBean(RequestInterceptor.class.getName(), new RequestInterceptor[]{interceptor}); + Config.getInstance().putInConfigCache(config, Map.of("enabled", true, "maxBuffers", 4, + "maxBodyBytes", 1024, "appliedBodyInjectionPathPrefixes", List.of("/"))); + RequestInterceptorInjectionHandler reader = new RequestInterceptorInjectionHandler(config); + AtomicBoolean onIo = new AtomicBoolean(); + reader.setNext(exchange -> { + onIo.set(exchange.isInIoThread()); + throw new IOException("downstream"); + }); + CountDownLatch suspended = new CountDownLatch(1); + HandlerContinuation.Boundary boundary = (exchange, task) -> { + try { task.handleRequest(exchange); } + catch (IOException e) { exchange.setStatusCode(409).getResponseSender().send("selected boundary"); } + }; + Undertow server = Undertow.builder().addHttpListener(0, "127.0.0.1").setIoThreads(1) + .setHandler(exchange -> { + if (enabled) HandlerContinuation.execute(exchange, boundary, reader); + else reader.handleRequest(exchange); + suspended.countDown(); + }).build(); + try { + server.start(); + int port = ((InetSocketAddress) server.getListenerInfo().getFirst().getAddress()).getPort(); + try (Socket socket = new Socket("127.0.0.1", port)) { + socket.setSoTimeout(5000); + socket.getOutputStream().write(("POST / HTTP/1.1\r\nHost: localhost\r\nContent-Type: application/json\r\n" + + "Transfer-Encoding: chunked\r\nConnection: close\r\n\r\n1\r\n{\r\n").getBytes(StandardCharsets.UTF_8)); + socket.getOutputStream().flush(); + assertTrue(suspended.await(5, TimeUnit.SECONDS)); + socket.getOutputStream().write("1\r\n}\r\n0\r\n\r\n".getBytes(StandardCharsets.UTF_8)); + socket.getOutputStream().flush(); + String response = new String(socket.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + assertTrue(response.startsWith(enabled ? "HTTP/1.1 409" : "HTTP/1.1 500"), response); + assertEquals(enabled, response.contains("selected boundary"), response); + assertTrue(onIo.get(), "Exercise the actual channel-listener continuation"); + } + } finally { + server.stop(); + SingletonServiceFactory.setBean(RequestInterceptor.class.getName(), previous); + Config.getInstance().clearConfigCache(config); + } + } +} diff --git a/hmac/src/main/java/com/networknt/hmac/HmacHandler.java b/hmac/src/main/java/com/networknt/hmac/HmacHandler.java index 4d250974ab..5cd8f1387a 100644 --- a/hmac/src/main/java/com/networknt/hmac/HmacHandler.java +++ b/hmac/src/main/java/com/networknt/hmac/HmacHandler.java @@ -18,6 +18,7 @@ import com.networknt.config.ConfigException; import com.networknt.handler.Handler; +import com.networknt.handler.HandlerContinuation; import com.networknt.handler.HandlerChainValidator; import com.networknt.handler.MiddlewareHandler; import com.networknt.handler.RequestInterceptor; @@ -129,8 +130,9 @@ public void handleRequest(HttpServerExchange exchange) throws Exception { return; } + var continuation = HandlerContinuation.capture(exchange); exchange.dispatch(); - stage.whenComplete((outcome, failure) -> exchange.dispatch(completed -> { + stage.whenComplete((outcome, failure) -> exchange.dispatch(continuation.apply(completed -> { if (failure != null || outcome == null) { unavailable(completed, rule, store, "reserve_failed"); } else if (outcome instanceof ReserveOutcome.Duplicate) { @@ -156,7 +158,7 @@ public void handleRequest(HttpServerExchange exchange) throws Exception { } else { unavailable(completed, rule, store, "unknown_reserve_outcome"); } - })); + }))); } private void addReservationCompletionListener(HttpServerExchange exchange, diff --git a/hmac/src/main/java/com/networknt/hmac/HmacReplayAdminHandler.java b/hmac/src/main/java/com/networknt/hmac/HmacReplayAdminHandler.java index 0612d02343..0c28d50792 100644 --- a/hmac/src/main/java/com/networknt/hmac/HmacReplayAdminHandler.java +++ b/hmac/src/main/java/com/networknt/hmac/HmacReplayAdminHandler.java @@ -20,6 +20,7 @@ import com.networknt.config.Config; import com.networknt.handler.AuditAttachmentUtil; import com.networknt.handler.LightHttpHandler; +import com.networknt.handler.HandlerContinuation; import com.networknt.hmac.config.HmacConfig; import com.networknt.httpstring.AttachmentConstants; import com.networknt.security.UnifiedSecurityConfig; @@ -102,8 +103,9 @@ public void handleRequest(HttpServerExchange exchange) { setExchangeStatus(exchange, HmacVerifier.HMAC_UNAVAILABLE); return; } + var continuation = HandlerContinuation.capture(exchange); exchange.dispatch(); - removalStage.whenComplete((removed, failure) -> exchange.dispatch(completed -> { + removalStage.whenComplete((removed, failure) -> exchange.dispatch(continuation.apply(completed -> { if (failure != null) { audit(completed, "unavailable", store); setExchangeStatus(completed, HmacVerifier.HMAC_UNAVAILABLE); @@ -120,7 +122,7 @@ public void handleRequest(HttpServerExchange exchange) { } catch (Exception e) { setExchangeStatus(completed, HmacVerifier.HMAC_UNAVAILABLE); } - })); + }))); } private Removal parse(Object requestBody) { diff --git a/hmac/src/test/java/com/networknt/hmac/HmacHandlerIntegrationTest.java b/hmac/src/test/java/com/networknt/hmac/HmacHandlerIntegrationTest.java index da53949dbb..b600b7219f 100644 --- a/hmac/src/test/java/com/networknt/hmac/HmacHandlerIntegrationTest.java +++ b/hmac/src/test/java/com/networknt/hmac/HmacHandlerIntegrationTest.java @@ -59,6 +59,29 @@ import java.util.concurrent.atomic.AtomicReference; class HmacHandlerIntegrationTest { + private java.util.function.UnaryOperator chainWrapper = handler -> handler; + + @org.junit.jupiter.params.ParameterizedTest + @org.junit.jupiter.params.provider.ValueSource(booleans = {true, false}) + void replayCompletionRetainsExceptionBoundary(boolean enabled) throws Exception { + Context context = context("continuation-" + enabled); + if (enabled) chainWrapper = handler -> exchange -> + com.networknt.handler.HandlerContinuation.execute(exchange, (ex, task) -> { + try { task.handleRequest(ex); } + catch (java.io.IOException e) { ex.setStatusCode(409).getResponseSender().send("selected"); } + }, handler); + try (RunningServer server = start(context, new RequestInterceptor[]{context.interceptor}, + exchange -> { throw new java.io.IOException("callee failed"); })) { + var response = server.post("{}".getBytes(StandardCharsets.UTF_8), "continuation"); + Assertions.assertTrue(context.store.reserveStarted.await(5, TimeUnit.SECONDS)); + Assertions.assertFalse(response.isDone()); + context.store.allowReservation.complete(null); + var completed = response.get(5, TimeUnit.SECONDS); + Assertions.assertEquals(enabled ? 409 : 500, completed.statusCode()); + Assertions.assertEquals(enabled, new String(completed.body(), StandardCharsets.UTF_8).contains("selected")); + } + } + private static final String SECRET = "phase-3-exact-forwarding-secret"; private static final String INJECTION_CONFIG = "hmac-phase3-request-injection"; private final List configNames = new java.util.ArrayList<>(); @@ -298,7 +321,7 @@ private RunningServer start(Context context, hmacHandler.setNext(callee); RequestInterceptorInjectionHandler injection = new RequestInterceptorInjectionHandler(INJECTION_CONFIG); injection.setNext(hmacHandler); - return new RunningServer(injection, context, version); + return new RunningServer(chainWrapper.apply(injection), context, version); } private Context context(String suffix) { diff --git a/ingress-proxy/src/main/java/com/networknt/proxy/ExternalServiceHandler.java b/ingress-proxy/src/main/java/com/networknt/proxy/ExternalServiceHandler.java index ba1539a46c..08e7865acf 100644 --- a/ingress-proxy/src/main/java/com/networknt/proxy/ExternalServiceHandler.java +++ b/ingress-proxy/src/main/java/com/networknt/proxy/ExternalServiceHandler.java @@ -227,7 +227,10 @@ private HttpResponse sendWithRetries(HttpClient forwardingClient, HttpRe HttpRequest finalRequest = attempt == 0 ? request : buildRetryRequest(request, attempt); try { return forwardingClient.send(finalRequest, HttpResponse.BodyHandlers.ofByteArray()); - } catch (IOException | InterruptedException e) { + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw e; + } catch (IOException e) { if (attempt >= maxRetries - 1) throw e; logger.warn("Attempt {} failed ({}). Retrying...", attempt + 1, e.getMessage()); } diff --git a/ingress-proxy/src/test/java/com/networknt/proxy/ExternalServiceInterruptTest.java b/ingress-proxy/src/test/java/com/networknt/proxy/ExternalServiceInterruptTest.java new file mode 100644 index 0000000000..906cf30cd8 --- /dev/null +++ b/ingress-proxy/src/test/java/com/networknt/proxy/ExternalServiceInterruptTest.java @@ -0,0 +1,66 @@ +package com.networknt.proxy; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.net.*; +import java.net.http.*; +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.*; +import javax.net.ssl.*; +import static org.junit.jupiter.api.Assertions.*; + +class ExternalServiceInterruptTest { + @ParameterizedTest + @CsvSource({"1,1", "3,1", "3,3"}) + void interruptionStopsRetriesAndPreservesFlag(int attempts, int interruptAt) throws Exception { + var supported = ExternalServiceHandler.class.getDeclaredField("connectionCloseHeaderSupported"); + var logged = ExternalServiceHandler.class.getDeclaredField("connectionCloseHeaderFallbackLogged"); + supported.setAccessible(true); + logged.setAccessible(true); + Object previousSupported = supported.get(null); + var loggedFlag = (java.util.concurrent.atomic.AtomicBoolean) logged.get(null); + boolean previousLogged = loggedFlag.get(); + InterruptingClient client = new InterruptingClient(interruptAt); + var method = ExternalServiceHandler.class.getDeclaredMethod("sendWithRetries", HttpClient.class, HttpRequest.class, int.class); + method.setAccessible(true); + try { + var thrown = assertThrows(InvocationTargetException.class, () -> method.invoke( + new ExternalServiceHandler(), client, HttpRequest.newBuilder(URI.create("http://localhost/")).build(), attempts)); + assertInstanceOf(InterruptedException.class, thrown.getCause()); + assertEquals(interruptAt, client.calls); + assertTrue(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); + supported.set(null, previousSupported); + loggedFlag.set(previousLogged); + } + } + + private static final class InterruptingClient extends HttpClient { + private final int interruptAt; + private int calls; + InterruptingClient(int interruptAt) { this.interruptAt = interruptAt; } + @Override public HttpResponse send(HttpRequest request, HttpResponse.BodyHandler handler) + throws IOException, InterruptedException { + if (++calls == interruptAt) throw new InterruptedException("cancelled"); + throw new IOException("retryable"); + } + @Override public Optional cookieHandler() { return Optional.empty(); } + @Override public Optional connectTimeout() { return Optional.empty(); } + @Override public Redirect followRedirects() { return Redirect.NEVER; } + @Override public Optional proxy() { return Optional.empty(); } + @Override public SSLContext sslContext() { throw new UnsupportedOperationException(); } + @Override public SSLParameters sslParameters() { return new SSLParameters(); } + @Override public Optional authenticator() { return Optional.empty(); } + @Override public Version version() { return Version.HTTP_1_1; } + @Override public Optional executor() { return Optional.empty(); } + @Override public CompletableFuture> sendAsync(HttpRequest request, HttpResponse.BodyHandler handler) { + throw new UnsupportedOperationException(); + } + @Override public CompletableFuture> sendAsync(HttpRequest request, HttpResponse.BodyHandler handler, + HttpResponse.PushPromiseHandler push) { throw new UnsupportedOperationException(); } + } +} diff --git a/ingress-proxy/src/test/java/com/networknt/proxy/ExternalServiceWorkerDispatchTest.java b/ingress-proxy/src/test/java/com/networknt/proxy/ExternalServiceWorkerDispatchTest.java index d8e5c283e2..bc7fed51ca 100644 --- a/ingress-proxy/src/test/java/com/networknt/proxy/ExternalServiceWorkerDispatchTest.java +++ b/ingress-proxy/src/test/java/com/networknt/proxy/ExternalServiceWorkerDispatchTest.java @@ -61,9 +61,12 @@ class ExternalServiceWorkerDispatchTest { private String previousTlsProperty; private HttpHandler previousMetrics; private Map previousExceptionConfig; - private static final String NAMED_EXCEPTION = "external-worker-named-exception-test"; + enum Policy { + ENABLED, NAMED, DISABLED, NAMED_DISABLED; - enum Policy { ENABLED, NAMED, DISABLED, NAMED_DISABLED } + String configName() { return this == ENABLED ? "exception" : "external-worker-exception-" + name(); } + boolean enabled() { return this == ENABLED || this == NAMED; } + } static Stream streamedPolicies() { return Stream.of(Policy.values()).flatMap(policy -> @@ -71,12 +74,7 @@ static Stream streamedPolicies() { } private HttpHandler withPolicy(Policy policy, HttpHandler next) { - boolean named = policy == Policy.NAMED || policy == Policy.NAMED_DISABLED; - boolean enabled = policy == Policy.ENABLED || policy == Policy.NAMED; - // The default deliberately disagrees with the selected named configuration. - Config.getInstance().putInConfigCache("exception", Map.of("enabled", named ? !enabled : enabled)); - Config.getInstance().putInConfigCache(NAMED_EXCEPTION, Map.of("enabled", enabled)); - ExceptionHandler selected = new ExceptionHandler(named ? NAMED_EXCEPTION : "exception") { + ExceptionHandler selected = new ExceptionHandler(policy.configName()) { @Override public void setExchangeStatus(HttpServerExchange exchange, String code, Object... args) { exchange.getResponseHeaders().put(new HttpString("X-Exception-Policy"), policy.name()); @@ -98,6 +96,8 @@ private static void assertFailure(Policy policy, int status, String body, String @BeforeEach void rememberSharedState() { previousExceptionConfig = Config.getInstance().getJsonMapConfig("exception"); + for (Policy policy : Policy.values()) + Config.getInstance().putInConfigCache(policy.configName(), Map.of("enabled", policy.enabled())); previousInterceptors = SingletonServiceFactory.getBeans(RequestInterceptor.class); previousTlsProperty = System.getProperty(TLS_PROPERTY); previousMetrics = Handler.getHandlers().get(MetricsConfig.CONFIG_NAME); @@ -109,7 +109,8 @@ void restoreSharedState() { if (previousMetrics == null) Handler.getHandlers().remove(MetricsConfig.CONFIG_NAME); else Handler.getHandlers().put(MetricsConfig.CONFIG_NAME, previousMetrics); Config.getInstance().putInConfigCache("exception", previousExceptionConfig); - Config.getInstance().clearConfigCache(NAMED_EXCEPTION); + for (Policy policy : Policy.values()) + if (policy != Policy.ENABLED) Config.getInstance().clearConfigCache(policy.configName()); Config.getInstance().clearConfigCache(EXTERNAL_CONFIG); Config.getInstance().clearConfigCache(INJECTION_CONFIG); if (previousTlsProperty == null) System.clearProperty(TLS_PROPERTY); @@ -227,7 +228,9 @@ void workerThreadFailureUsesTheSelectedPolicy(Policy policy) throws Exception { } private static void assertRawFailure(Policy policy, String response) { - int status = Integer.parseInt(response.split(" ", 3)[1]); + assertTrue(response.matches("(?s)HTTP/1\\.[01] [0-9]{3} .*"), + "Expected a complete HTTP status line, received: " + response); + int status = Integer.parseInt(response.substring(9, 12)); String selected = response.lines().filter(line -> line.toLowerCase().startsWith("x-exception-policy:")) .map(line -> line.substring(line.indexOf(':') + 1).trim()).findFirst().orElse(""); assertFailure(policy, status, response, selected); diff --git a/mcp-router/src/main/java/com/networknt/mcp/McpHandler.java b/mcp-router/src/main/java/com/networknt/mcp/McpHandler.java index 258423c41d..3fa78a7e2b 100644 --- a/mcp-router/src/main/java/com/networknt/mcp/McpHandler.java +++ b/mcp-router/src/main/java/com/networknt/mcp/McpHandler.java @@ -173,7 +173,7 @@ private void handleSse(HttpServerExchange exchange, McpConfig config) throws Exc private void handleMessage(HttpServerExchange exchange, McpConfig config) { if (exchange.isInIoThread()) { - exchange.dispatch(this); + exchange.dispatch(com.networknt.handler.HandlerContinuation.capture(exchange, this)); return; } diff --git a/mcp-router/src/test/java/com/networknt/mcp/McpContinuationTest.java b/mcp-router/src/test/java/com/networknt/mcp/McpContinuationTest.java new file mode 100644 index 0000000000..068d93beb0 --- /dev/null +++ b/mcp-router/src/test/java/com/networknt/mcp/McpContinuationTest.java @@ -0,0 +1,39 @@ +package com.networknt.mcp; + +import com.networknt.handler.HandlerContinuation; +import io.undertow.Undertow; +import io.undertow.server.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.http.*; +import java.time.Duration; +import static org.junit.jupiter.api.Assertions.*; + +class McpContinuationTest { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void redispatchRetainsSelectedBoundary(boolean enabled) throws Exception { + McpHandler handler = new McpHandler() { + @Override public void handleRequest(HttpServerExchange exchange) throws Exception { + if (!exchange.isInIoThread()) throw new IllegalStateException("resumed failure"); + super.handleRequest(exchange); + } + }; + HttpHandler root = enabled ? exchange -> HandlerContinuation.execute(exchange, (ex, task) -> { + try { task.handleRequest(ex); } + catch (IllegalStateException e) { ex.setStatusCode(409).getResponseSender().send("selected"); } + }, handler) : handler; + Undertow server = Undertow.builder().addHttpListener(0, "127.0.0.1").setHandler(root).build(); + try (HttpClient client = HttpClient.newHttpClient()) { + server.start(); + int port = ((InetSocketAddress) server.getListenerInfo().getFirst().getAddress()).getPort(); + var response = client.send(HttpRequest.newBuilder(URI.create("http://127.0.0.1:" + port + "/mcp")) + .timeout(Duration.ofSeconds(5)).POST(HttpRequest.BodyPublishers.ofString("{}")) + .build(), HttpResponse.BodyHandlers.ofString()); + assertEquals(enabled ? 409 : 500, response.statusCode(), response.body()); + assertEquals(enabled, response.body().contains("selected")); + } finally { server.stop(); } + } +} diff --git a/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java b/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java index 99aa28d8c0..4e6f986f61 100644 --- a/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java +++ b/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java @@ -26,6 +26,7 @@ import java.security.cert.Certificate; import java.util.*; import java.util.concurrent.TimeUnit; +import java.util.function.UnaryOperator; import java.util.concurrent.atomic.AtomicLong; import javax.net.ssl.SSLPeerUnverifiedException; import java.security.cert.CertificateEncodingException; @@ -209,7 +210,6 @@ public void handleRequest(final HttpServerExchange exchange) throws Exception { if (exchange.isInIoThread()) exchange.dispatch(lightThreadExecutor, clientHandler); - else exchange.dispatch(exchange.getIoThread(), clientHandler); } @@ -378,6 +378,7 @@ public String toString() { } private final class ProxyClientHandler implements ProxyCallback, Runnable { + private final UnaryOperator continuation; private int tries; private final long timeout; private final int maxRetryAttempts; @@ -388,6 +389,7 @@ private final class ProxyClientHandler implements ProxyCallback ProxyClientHandler(HttpServerExchange exchange, ProxyClient.ProxyTarget target, long timeout, int maxRetryAttempts, Predicate idempotentPredicate, boolean streamingRequest) { this.exchange = exchange; + this.continuation = HandlerContinuation.capture(exchange); this.timeout = timeout; this.maxRetryAttempts = maxRetryAttempts; this.target = target; @@ -397,14 +399,16 @@ private final class ProxyClientHandler implements ProxyCallback @Override public void run() { - proxyClient.getConnection(this.target, this.exchange, this, -1, TimeUnit.MILLISECONDS); + HandlerContinuation.task(exchange, continuation.apply(ex -> + proxyClient.getConnection(this.target, ex, this, -1, TimeUnit.MILLISECONDS))).run(); } @Override public void completed(final HttpServerExchange exchange, final ProxyConnection connection) { LOG.trace("ProxyCallback completed for exchange: {}", exchange); exchange.putAttachment(CONNECTION, connection); - exchange.dispatch(lightThreadExecutor, new ProxyAction(connection, exchange, requestHeaders, rewriteHostHeader, reuseXForwarded, exchange.isRequestComplete() ? this : null, idempotentPredicate, urlRewriteRules, methodRewriteRules, queryParamRewriteRules, headerRewriteRules, streamingRequest, streamResponseContentTypes, streamIdleTimeout, streamResponseHeaderOverwrite)); + exchange.dispatch(lightThreadExecutor, HandlerContinuation.task(exchange, continuation.apply(ex -> + new ProxyAction(connection, ex, requestHeaders, rewriteHostHeader, reuseXForwarded, ex.isRequestComplete() ? this : null, idempotentPredicate, urlRewriteRules, methodRewriteRules, queryParamRewriteRules, headerRewriteRules, streamingRequest, streamResponseContentTypes, streamIdleTimeout, streamResponseHeaderOverwrite).run()))); } @Override @@ -499,6 +503,7 @@ void cancel(final HttpServerExchange exchange) { } private static class ProxyAction implements Runnable { + private final UnaryOperator continuation; private final ProxyConnection clientConnection; private final HttpServerExchange exchange; private final LightThreadExecutor lightThreadExecutor; @@ -521,6 +526,7 @@ private static class ProxyAction implements Runnable { List urlRewriteRules, List methodRewriteRules, Map> queryParamRewriteRules, Map> headerRewriteRules, boolean streamingRequest, List streamResponseContentTypes, int streamIdleTimeout, List streamResponseHeaderOverwrite) { + this.continuation = HandlerContinuation.capture(exchange); this.clientConnection = clientConnection; this.exchange = exchange; this.requestHeaders = requestHeaders; @@ -988,7 +994,8 @@ private void handleServerPush(ClientExchange ce) { if (i > 0) path = path.substring(0, i); - exchange.dispatch(lightThreadExecutor, new ProxyAction(new ProxyConnection(pushedRequest.getConnection(), path), exchange, requestHeaders, rewriteHostHeader, reuseXForwarded, null, idempotentPredicate, urlRewriteRules, methodRewriteRules, queryParamRewriteRules, headerRewriteRules, false, streamResponseContentTypes, streamIdleTimeout, streamResponseHeaderOverwrite)); + String targetPath = path; + exchange.dispatch(lightThreadExecutor, HandlerContinuation.task(exchange, continuation.apply(ex -> new ProxyAction(new ProxyConnection(pushedRequest.getConnection(), targetPath), ex, requestHeaders, rewriteHostHeader, reuseXForwarded, null, idempotentPredicate, urlRewriteRules, methodRewriteRules, queryParamRewriteRules, headerRewriteRules, false, streamResponseContentTypes, streamIdleTimeout, streamResponseHeaderOverwrite).run()))); }); return true; }); diff --git a/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java b/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java new file mode 100644 index 0000000000..7d59d37b30 --- /dev/null +++ b/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java @@ -0,0 +1,48 @@ +package com.networknt.handler; + +import io.undertow.Undertow; +import io.undertow.server.*; +import io.undertow.server.handlers.proxy.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.http.*; +import java.time.Duration; +import java.util.concurrent.TimeUnit; +import static org.junit.jupiter.api.Assertions.*; + +class ProxyContinuationTest { + @ParameterizedTest + @CsvSource({"true,false", "false,false", "true,true", "false,true"}) + void connectionDispatchRetainsBoundary(boolean enabled, boolean delayedCompletion) throws Exception { + var completion = new java.util.concurrent.CompletableFuture(); + ProxyClient client = new ProxyClient() { + @Override public ProxyTarget findTarget(HttpServerExchange exchange) { return new ProxyTarget() {}; } + @Override public void getConnection(ProxyTarget target, HttpServerExchange exchange, + ProxyCallback callback, long timeout, TimeUnit unit) { + if (delayedCompletion) completion.complete(() -> callback.completed(exchange, new ProxyConnection(null, "/"))); + else throw new IllegalStateException("connection callback failed"); + } + }; + ProxyHandler proxy = ProxyHandler.builder().setProxyClient(client).build(); + HttpHandler root = enabled ? exchange -> HandlerContinuation.execute(exchange, (ex, task) -> { + try { task.handleRequest(ex); } + catch (RuntimeException e) { ex.setStatusCode(409).getResponseSender().send("selected"); } + }, proxy) : proxy; + Undertow server = Undertow.builder().addHttpListener(0, "127.0.0.1").setHandler(root).build(); + try (HttpClient http = HttpClient.newHttpClient()) { + server.start(); + int port = ((InetSocketAddress) server.getListenerInfo().getFirst().getAddress()).getPort(); + var pending = http.sendAsync(HttpRequest.newBuilder(URI.create("http://127.0.0.1:" + port + "/")) + .timeout(Duration.ofSeconds(5)).build(), HttpResponse.BodyHandlers.ofString()); + if (delayedCompletion) { + assertFalse(pending.isDone(), "Connection acquisition must not end a pending exchange"); + completion.get(5, TimeUnit.SECONDS).run(); + } + var response = pending.get(5, TimeUnit.SECONDS); + assertEquals(enabled ? 409 : 500, response.statusCode(), response.body()); + assertEquals(enabled, response.body().contains("selected")); + } finally { server.stop(); } + } +} From 03ef782357fdf8797112d6cd60083a37b8493d4b Mon Sep 17 00:00:00 2001 From: Steve Hu Date: Thu, 29 Oct 2026 14:41:53 -0400 Subject: [PATCH 03/15] Preserve proxy callback lifecycle and reuse exchange MDC context --- README.md | 17 +- .../java/com/networknt/dump/DumpHandler.java | 3 +- .../networknt/exception/ExceptionHandler.java | 21 +- .../exception/ExceptionContinuationTest.java | 23 +- .../handler/HandlerContinuation.java | 45 ++- .../handler/HandlerContinuationTest.java | 6 +- .../ExternalServiceWorkerDispatchTest.java | 3 +- .../java/com/networknt/mcp/McpHandler.java | 3 +- .../com/networknt/handler/ProxyHandler.java | 93 ++++-- .../handler/ProxyContinuationTest.java | 295 ++++++++++++++++-- 10 files changed, 410 insertions(+), 99 deletions(-) diff --git a/README.md b/README.md index d2f71ab0a9..2073af7831 100644 --- a/README.md +++ b/README.md @@ -259,11 +259,18 @@ ExceptionHandler supplies its existing policy without rebuilding middleware or changing its next handler. Named handlers are retained; when exception handling is disabled and omitted from the chain, failures remain under Undertow's default policy. Nested boundaries restore their enclosing scope, and state is isolated -per exchange. The request MDC is captured with each continuation and the resuming -thread's previous MDC is restored afterward. BodyHandler redispatch, HMAC replay -and administration completion, DumpHandler/McpHandler redispatch, and -ProxyHandler connection/action dispatch also -carry the selected boundary. Proxy callbacks retain their pending-I/O lifecycle. +per exchange. The continuation contract carries only exception policy; MDC stays +with the existing MDC_CONTEXT attachment and LightThreadExecutor. Capturing and +resuming a boundary does not copy or replace thread-local MDC. ExceptionHandler +clears MDC at initial chain exit, not inside each resumed frame. +BodyHandler, DumpHandler and McpHandler redispatch, HMAC replay/admin completion, +and ProxyHandler acquisition, retries, timeout cancellation and client send/response +callbacks carry the selected boundary. Proxy actions reuse the request's captured +policy and preserve pending upstream I/O on success. Callback failure closes its +upstream connection. Before response commitment an unhandled failure returns 500; +after commitment it closes the connection without appending an error or marking +a truncated stream complete. Late failures on completed exchanges do not rewrite +the response or close a reused downstream connection. Custom asynchronous handlers must explicitly capture before registering their callbacks; this contract cannot intercept arbitrary third-party hand-offs. Body-reading requests capture lazily only when a read suspends. ingress-proxy uses diff --git a/dump/src/main/java/com/networknt/dump/DumpHandler.java b/dump/src/main/java/com/networknt/dump/DumpHandler.java index bedaa3199f..f63cd932c4 100644 --- a/dump/src/main/java/com/networknt/dump/DumpHandler.java +++ b/dump/src/main/java/com/networknt/dump/DumpHandler.java @@ -17,6 +17,7 @@ package com.networknt.dump; import com.networknt.handler.Handler; +import com.networknt.handler.HandlerContinuation; import com.networknt.handler.MiddlewareHandler; import io.undertow.Handlers; import io.undertow.server.HttpHandler; @@ -77,7 +78,7 @@ public boolean isEnabled() { @Override public void handleRequest(final HttpServerExchange exchange) throws Exception { if (exchange.isInIoThread()) { - exchange.dispatch(com.networknt.handler.HandlerContinuation.capture(exchange, this)); + exchange.dispatch(HandlerContinuation.capture(exchange, this)); return; } if(isEnabled()) { diff --git a/exception/src/main/java/com/networknt/exception/ExceptionHandler.java b/exception/src/main/java/com/networknt/exception/ExceptionHandler.java index b84dc406e0..c5bd1d2b1a 100644 --- a/exception/src/main/java/com/networknt/exception/ExceptionHandler.java +++ b/exception/src/main/java/com/networknt/exception/ExceptionHandler.java @@ -26,6 +26,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; +import org.xnio.IoUtils; /** * This is a middleware handler that should be put in the beginning of request/response @@ -80,13 +81,18 @@ public void handleRequest(final HttpServerExchange exchange) throws Exception { return; } - HandlerContinuation.execute(exchange, this::handleContinuation, - ex -> Handler.next(ex, next)); + try { + HandlerContinuation.execute(exchange, this::handleContinuation, + ex -> Handler.next(ex, next)); + } finally { + // Initial chain execution owns request-thread cleanup. Resumed frames + // use their executor's MDC_CONTEXT and must not erase that context. + MDC.clear(); + } } private void handleContinuation(HttpServerExchange exchange, HttpHandler continuation) throws Exception { - // Log each executing frame, including asynchronous continuations. The - // captured continuation restores the resuming thread's MDC after this frame. + // Log each executing frame, including asynchronous continuations. String path = exchange.getRequestPath(); if(logger.isDebugEnabled()) logger.debug("ExceptionHandler frame starting - path: {}", path); @@ -94,6 +100,11 @@ private void handleContinuation(HttpServerExchange exchange, HttpHandler continu continuation.handleRequest(exchange); } catch (Throwable e) { logger.error("ExceptionHandler error processing - path: {}", path, e); + if (exchange.isComplete()) return; + if (exchange.isResponseStarted()) { + IoUtils.safeClose(exchange.getConnection()); + return; + } if(exchange.isResponseChannelAvailable()) { //handle exceptions if(e instanceof RuntimeException) { @@ -131,8 +142,6 @@ private void handleContinuation(HttpServerExchange exchange, HttpHandler continu } } finally { if(logger.isDebugEnabled()) logger.debug("ExceptionHandler frame completed - path: {}", path); - // Clear this request frame; a captured continuation restores its caller's MDC. - MDC.clear(); } } diff --git a/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java b/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java index 805c2c10e5..6eee050b73 100644 --- a/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java +++ b/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java @@ -45,9 +45,22 @@ void selectedPolicyHandlesInitialAndResumedFailures(String name, boolean async, if (runtime) throw new IllegalStateException("runtime failure"); throw new IOException("checked failure"); }; + var resumedMdc = new java.util.concurrent.atomic.AtomicReference(); + var resumedDone = new java.util.concurrent.CountDownLatch(1); HttpHandler work = exchange -> { - if (async) exchange.dispatch(HandlerContinuation.capture(exchange, failing)); - else failing.handleRequest(exchange); + if (async) { + HttpHandler resumed = HandlerContinuation.capture(exchange, failing); + exchange.dispatch(() -> { + org.slf4j.MDC.put("executor", "owns-context"); + try { + HandlerContinuation.task(exchange, resumed).run(); + resumedMdc.set(org.slf4j.MDC.get("executor")); + } finally { + org.slf4j.MDC.clear(); + resumedDone.countDown(); + } + }); + } else failing.handleRequest(exchange); }; selected.setNext(work); Undertow server = Undertow.builder().addHttpListener(0, "127.0.0.1") @@ -61,9 +74,13 @@ void selectedPolicyHandlesInitialAndResumedFailures(String name, boolean async, assertEquals(enabled ? (runtime ? 500 : 400) : 500, response.statusCode(), response.body()); assertEquals(enabled ? name : "", response.headers().firstValue("X-Selected").orElse("")); assertEquals(enabled, response.body().contains(runtime ? "ERR10010" : "ERR10011"), response.body()); + if (async) { + assertTrue(resumedDone.await(3, java.util.concurrent.TimeUnit.SECONDS)); + assertEquals("owns-context", resumedMdc.get(), "Resumed exception handling must leave executor MDC alone"); + } } finally { server.stop(); - if (name.equals("exception")) Config.getInstance().putInConfigCache(name, previous); + if (name.equals("exception") && previous != null) Config.getInstance().putInConfigCache(name, previous); else Config.getInstance().clearConfigCache(name); } } diff --git a/handler/src/main/java/com/networknt/handler/HandlerContinuation.java b/handler/src/main/java/com/networknt/handler/HandlerContinuation.java index 43c08c7381..dea9c268fb 100644 --- a/handler/src/main/java/com/networknt/handler/HandlerContinuation.java +++ b/handler/src/main/java/com/networknt/handler/HandlerContinuation.java @@ -1,12 +1,13 @@ package com.networknt.handler; import io.undertow.server.HttpHandler; -import io.undertow.server.Connectors; import io.undertow.server.HttpServerExchange; import io.undertow.util.AttachmentKey; -import org.slf4j.MDC; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.xnio.IoUtils; +import io.undertow.util.StatusCodes; -import java.util.Map; import java.util.Objects; import java.util.function.UnaryOperator; @@ -17,6 +18,7 @@ * handler through Undertow's dispatch or root-handler machinery. */ public final class HandlerContinuation { + private static final Logger LOG = LoggerFactory.getLogger(HandlerContinuation.class); private static final Boundary UNHANDLED = (exchange, continuation) -> continuation.handleRequest(exchange); private static final AttachmentKey ACTIVE = AttachmentKey.create(Boundary.class); @@ -59,45 +61,36 @@ public static HttpHandler capture(HttpServerExchange exchange, HttpHandler conti * Captures before registering a callback whose result determines its eventual * work. Apply the returned wrapper inside that callback. Framework hand-offs * must opt in; arbitrary third-party callbacks cannot be intercepted here. - * The request MDC is installed only for the resumed frame and the executing - * thread's previous MDC is restored, including on failure. + * Captures only the exception policy. MDC remains owned by the existing + * MDC_CONTEXT attachment and its executor; no thread-local maps are copied. */ public static UnaryOperator capture(HttpServerExchange exchange) { Boundary active = exchange.getAttachment(ACTIVE); Boundary boundary = active == null ? UNHANDLED : active; - Map context = MDC.getCopyOfContextMap(); - return continuation -> resumed -> { - Map previous = MDC.getCopyOfContextMap(); - restoreMdc(context); - try { - execute(resumed, boundary, continuation); - } finally { - restoreMdc(previous); - } - }; + return continuation -> resumed -> execute(resumed, boundary, continuation); } /** * Adapts an already captured handler for a Runnable dispatch. Unlike a root * handler, a successful callback may leave upstream I/O pending; do not end - * that exchange. Unhandled failures still enter Undertow's error path. + * that exchange. Unhandled failures return 500 before commitment, close a + * committed stream, or leave an already completed exchange alone. */ public static Runnable task(HttpServerExchange exchange, HttpHandler continuation) { return () -> { try { continuation.handleRequest(exchange); } catch (Throwable failure) { - Connectors.executeRootHandler(ex -> { - if (failure instanceof Error error) throw error; - if (failure instanceof Exception exception) throw exception; - throw new IllegalStateException(failure); - }, exchange); + LOG.error("Unhandled continuation failure for {}", exchange.getRequestPath(), failure); + if (exchange.isComplete()) return; + if (exchange.isResponseStarted()) { + // Never replace committed headers or append an error to a stream. + IoUtils.safeClose(exchange.getConnection()); + } else { + exchange.setStatusCode(StatusCodes.INTERNAL_SERVER_ERROR); + exchange.endExchange(); + } } }; } - - private static void restoreMdc(Map context) { - if (context == null) MDC.clear(); - else MDC.setContextMap(context); - } } diff --git a/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java b/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java index b6c352d7f4..f158c73e76 100644 --- a/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java +++ b/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java @@ -13,14 +13,14 @@ class HandlerContinuationTest { @Test - void resumedFrameCarriesRequestMdcAndRestoresCallingThreadEvenOnFailure() throws Exception { + void captureLeavesExecutorMdcUntouched() throws Exception { var original = org.slf4j.MDC.getCopyOfContextMap(); try { HttpServerExchange exchange = new HttpServerExchange(null); org.slf4j.MDC.put("request", "captured"); HttpHandler resumed = HandlerContinuation.capture(exchange, ex -> { - assertEquals("captured", org.slf4j.MDC.get("request")); - org.slf4j.MDC.clear(); + assertEquals("unrelated", org.slf4j.MDC.get("worker")); + assertNull(org.slf4j.MDC.get("request")); throw new IOException("failure"); }); org.slf4j.MDC.clear(); diff --git a/ingress-proxy/src/test/java/com/networknt/proxy/ExternalServiceWorkerDispatchTest.java b/ingress-proxy/src/test/java/com/networknt/proxy/ExternalServiceWorkerDispatchTest.java index bc7fed51ca..fac39a771b 100644 --- a/ingress-proxy/src/test/java/com/networknt/proxy/ExternalServiceWorkerDispatchTest.java +++ b/ingress-proxy/src/test/java/com/networknt/proxy/ExternalServiceWorkerDispatchTest.java @@ -108,7 +108,8 @@ void restoreSharedState() { SingletonServiceFactory.setBean(RequestInterceptor.class.getName(), previousInterceptors); if (previousMetrics == null) Handler.getHandlers().remove(MetricsConfig.CONFIG_NAME); else Handler.getHandlers().put(MetricsConfig.CONFIG_NAME, previousMetrics); - Config.getInstance().putInConfigCache("exception", previousExceptionConfig); + if (previousExceptionConfig == null) Config.getInstance().clearConfigCache("exception"); + else Config.getInstance().putInConfigCache("exception", previousExceptionConfig); for (Policy policy : Policy.values()) if (policy != Policy.ENABLED) Config.getInstance().clearConfigCache(policy.configName()); Config.getInstance().clearConfigCache(EXTERNAL_CONFIG); diff --git a/mcp-router/src/main/java/com/networknt/mcp/McpHandler.java b/mcp-router/src/main/java/com/networknt/mcp/McpHandler.java index 3fa78a7e2b..0c51c8472c 100644 --- a/mcp-router/src/main/java/com/networknt/mcp/McpHandler.java +++ b/mcp-router/src/main/java/com/networknt/mcp/McpHandler.java @@ -6,6 +6,7 @@ import com.networknt.config.Config; import com.networknt.config.JsonMapper; import com.networknt.handler.Handler; +import com.networknt.handler.HandlerContinuation; import com.networknt.handler.MiddlewareHandler; import com.networknt.httpstring.AttachmentConstants; import com.networknt.rule.RuleConstants; @@ -173,7 +174,7 @@ private void handleSse(HttpServerExchange exchange, McpConfig config) throws Exc private void handleMessage(HttpServerExchange exchange, McpConfig config) { if (exchange.isInIoThread()) { - exchange.dispatch(com.networknt.handler.HandlerContinuation.capture(exchange, this)); + exchange.dispatch(HandlerContinuation.capture(exchange, this)); return; } diff --git a/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java b/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java index 4e6f986f61..c786130fce 100644 --- a/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java +++ b/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java @@ -26,6 +26,7 @@ import java.security.cert.Certificate; import java.util.*; import java.util.concurrent.TimeUnit; +import java.util.concurrent.Executor; import java.util.function.UnaryOperator; import java.util.concurrent.atomic.AtomicLong; import javax.net.ssl.SSLPeerUnverifiedException; @@ -104,7 +105,7 @@ public class ProxyHandler implements HttpHandler { private static final Logger LOG = LoggerFactory.getLogger(ProxyHandler.class); - private LightThreadExecutor lightThreadExecutor; + private static final Executor INLINE_EXECUTOR = Runnable::run; public static final AttachmentKey CONNECTION = AttachmentKey.create(ProxyConnection.class); private static final AttachmentKey EXCHANGE = AttachmentKey.create(HttpServerExchange.class); @@ -165,7 +166,6 @@ private ProxyHandler(Builder builder) { } public void handleRequest(final HttpServerExchange exchange) throws Exception { - this.lightThreadExecutor = new LightThreadExecutor(exchange); final ProxyClient.ProxyTarget target = proxyClient.findTarget(exchange); if (target == null) { @@ -209,7 +209,7 @@ public void handleRequest(final HttpServerExchange exchange) throws Exception { } - if (exchange.isInIoThread()) exchange.dispatch(lightThreadExecutor, clientHandler); + if (exchange.isInIoThread()) exchange.dispatch(INLINE_EXECUTOR, clientHandler); else exchange.dispatch(exchange.getIoThread(), clientHandler); } @@ -377,8 +377,44 @@ public String toString() { } } + /** One immutable policy snapshot and the existing request-MDC executor, reused by all callbacks. */ + private record ProxyContinuation(UnaryOperator boundary, LightThreadExecutor executor) { } + + /** Keep callback setup and upstream-resource cleanup inside the selected boundary. */ + private static Runnable continuationTask(HttpServerExchange exchange, + ProxyContinuation continuation, + HttpHandler callback) { + Runnable task = HandlerContinuation.task(exchange, continuation.boundary().apply(ex -> { + try { + callback.handleRequest(ex); + } catch (Exception | Error failure) { + ProxyConnection upstream = ex.getAttachment(CONNECTION); + if (upstream != null) IoUtils.safeClose(upstream.getConnection()); + throw failure; + } + })); + return () -> continuation.executor().execute(task); + } + + private static ClientCallback boundaryCallback(HttpServerExchange exchange, + ProxyContinuation continuation, ClientCallback callback) { + return new ClientCallback<>() { + @Override public void completed(ClientExchange result) { + if (exchange.isComplete()) { + IoUtils.safeClose(result.getConnection()); + return; + } + continuationTask(exchange, continuation, ex -> callback.completed(result)).run(); + } + @Override public void failed(IOException failure) { + if (!exchange.isComplete()) + continuationTask(exchange, continuation, ex -> callback.failed(failure)).run(); + } + }; + } + private final class ProxyClientHandler implements ProxyCallback, Runnable { - private final UnaryOperator continuation; + private final ProxyContinuation continuation; private int tries; private final long timeout; private final int maxRetryAttempts; @@ -389,7 +425,7 @@ private final class ProxyClientHandler implements ProxyCallback ProxyClientHandler(HttpServerExchange exchange, ProxyClient.ProxyTarget target, long timeout, int maxRetryAttempts, Predicate idempotentPredicate, boolean streamingRequest) { this.exchange = exchange; - this.continuation = HandlerContinuation.capture(exchange); + this.continuation = new ProxyContinuation(HandlerContinuation.capture(exchange), new LightThreadExecutor(exchange)); this.timeout = timeout; this.maxRetryAttempts = maxRetryAttempts; this.target = target; @@ -399,20 +435,30 @@ private final class ProxyClientHandler implements ProxyCallback @Override public void run() { - HandlerContinuation.task(exchange, continuation.apply(ex -> - proxyClient.getConnection(this.target, ex, this, -1, TimeUnit.MILLISECONDS))).run(); + continuationTask(exchange, continuation, ex -> + proxyClient.getConnection(this.target, ex, this, -1, TimeUnit.MILLISECONDS)).run(); } @Override public void completed(final HttpServerExchange exchange, final ProxyConnection connection) { - LOG.trace("ProxyCallback completed for exchange: {}", exchange); - exchange.putAttachment(CONNECTION, connection); - exchange.dispatch(lightThreadExecutor, HandlerContinuation.task(exchange, continuation.apply(ex -> - new ProxyAction(connection, ex, requestHeaders, rewriteHostHeader, reuseXForwarded, ex.isRequestComplete() ? this : null, idempotentPredicate, urlRewriteRules, methodRewriteRules, queryParamRewriteRules, headerRewriteRules, streamingRequest, streamResponseContentTypes, streamIdleTimeout, streamResponseHeaderOverwrite).run()))); + continuationTask(exchange, continuation, ex -> { + if (ex.isComplete()) { + IoUtils.safeClose(connection.getConnection()); + return; + } + LOG.trace("ProxyCallback completed for exchange: {}", ex); + ex.putAttachment(CONNECTION, connection); + ex.dispatch(INLINE_EXECUTOR, continuationTask(ex, continuation, resumed -> + new ProxyAction(connection, resumed, requestHeaders, rewriteHostHeader, reuseXForwarded, resumed.isRequestComplete() ? this : null, idempotentPredicate, urlRewriteRules, methodRewriteRules, queryParamRewriteRules, headerRewriteRules, streamingRequest, streamResponseContentTypes, streamIdleTimeout, streamResponseHeaderOverwrite, continuation).run())); + }).run(); } @Override public void failed(final HttpServerExchange exchange) { + continuationTask(exchange, continuation, this::handleFailed).run(); + } + + private void handleFailed(final HttpServerExchange exchange) { LOG.debug("Failed calling backend with tries = {} maxRetryAttempts = {} for exchange: {}", this.tries, this.maxRetryAttempts, exchange); final long time = System.currentTimeMillis(); @@ -451,7 +497,10 @@ public void queuedRequestFailed(HttpServerExchange exchange) { @Override public void couldNotResolveBackend(HttpServerExchange ex) { + continuationTask(ex, continuation, this::unresolvedBackend).run(); + } + private void unresolvedBackend(HttpServerExchange ex) { if (ex.isResponseStarted()) IoUtils.safeClose(ex.getConnection()); @@ -471,7 +520,10 @@ public void couldNotResolveBackend(HttpServerExchange ex) { } void cancel(final HttpServerExchange exchange) { + continuationTask(exchange, continuation, this::cancelRequest).run(); + } + private void cancelRequest(final HttpServerExchange exchange) { //NOTE: this method is called only in context of timeouts. final ProxyConnection connectionAttachment = exchange.getAttachment(CONNECTION); @@ -503,10 +555,9 @@ void cancel(final HttpServerExchange exchange) { } private static class ProxyAction implements Runnable { - private final UnaryOperator continuation; + private final ProxyContinuation continuation; private final ProxyConnection clientConnection; private final HttpServerExchange exchange; - private final LightThreadExecutor lightThreadExecutor; private final Map requestHeaders; private final boolean rewriteHostHeader; private final boolean reuseXForwarded; @@ -525,8 +576,9 @@ private static class ProxyAction implements Runnable { boolean rewriteHostHeader, boolean reuseXForwarded, ProxyClientHandler proxyClientHandler, Predicate idempotentPredicate, List urlRewriteRules, List methodRewriteRules, Map> queryParamRewriteRules, Map> headerRewriteRules, - boolean streamingRequest, List streamResponseContentTypes, int streamIdleTimeout, List streamResponseHeaderOverwrite) { - this.continuation = HandlerContinuation.capture(exchange); + boolean streamingRequest, List streamResponseContentTypes, int streamIdleTimeout, List streamResponseHeaderOverwrite, + ProxyContinuation continuation) { + this.continuation = continuation; this.clientConnection = clientConnection; this.exchange = exchange; this.requestHeaders = requestHeaders; @@ -542,8 +594,7 @@ private static class ProxyAction implements Runnable { this.streamResponseContentTypes = streamResponseContentTypes; this.streamIdleTimeout = streamIdleTimeout; this.streamResponseHeaderOverwrite = streamResponseHeaderOverwrite; - this.lightThreadExecutor = new LightThreadExecutor(exchange); - } + } @Override public void run() { @@ -897,7 +948,7 @@ private void attachRemoteHost(ClientRequest r) { } private void sendWithCallback(ClientRequest r, String host) { - this.clientConnection.getConnection().sendRequest(r, new ClientCallback<>() { + this.clientConnection.getConnection().sendRequest(r, boundaryCallback(exchange, continuation, new ClientCallback<>() { @Override public void completed(final ClientExchange result) { @@ -914,7 +965,7 @@ public void completed(final ClientExchange result) { if (exchange.getConnection().isPushSupported() && result.getConnection().isPushSupported()) this.handleServerPush(result); - result.setResponseListener(new ResponseCallback(exchange, proxyClientHandler, idempotentPredicate, headerRewriteRules, streamingRequest, streamResponseContentTypes, streamIdleTimeout, streamResponseHeaderOverwrite)); + result.setResponseListener(boundaryCallback(exchange, continuation, new ResponseCallback(exchange, proxyClientHandler, idempotentPredicate, headerRewriteRules, streamingRequest, streamResponseContentTypes, streamIdleTimeout, streamResponseHeaderOverwrite))); final IoExceptionHandler handler = new IoExceptionHandler(exchange, clientConnection.getConnection()); @@ -995,13 +1046,13 @@ private void handleServerPush(ClientExchange ce) { path = path.substring(0, i); String targetPath = path; - exchange.dispatch(lightThreadExecutor, HandlerContinuation.task(exchange, continuation.apply(ex -> new ProxyAction(new ProxyConnection(pushedRequest.getConnection(), targetPath), ex, requestHeaders, rewriteHostHeader, reuseXForwarded, null, idempotentPredicate, urlRewriteRules, methodRewriteRules, queryParamRewriteRules, headerRewriteRules, false, streamResponseContentTypes, streamIdleTimeout, streamResponseHeaderOverwrite).run()))); + exchange.dispatch(INLINE_EXECUTOR, continuationTask(exchange, continuation, ex -> new ProxyAction(new ProxyConnection(pushedRequest.getConnection(), targetPath), ex, requestHeaders, rewriteHostHeader, reuseXForwarded, null, idempotentPredicate, urlRewriteRules, methodRewriteRules, queryParamRewriteRules, headerRewriteRules, false, streamResponseContentTypes, streamIdleTimeout, streamResponseHeaderOverwrite, continuation).run())); }); return true; }); } - }); + })); } private void attachSslInfo(ClientRequest r) { diff --git a/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java b/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java index 7d59d37b30..97ff8b42a7 100644 --- a/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java +++ b/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java @@ -1,48 +1,279 @@ package com.networknt.handler; +import com.networknt.httpstring.AttachmentConstants; +import com.networknt.metrics.AbstractMetricsHandler; +import com.networknt.metrics.MetricsConfig; +import com.networknt.metrics.TimeSeriesDbSender; import io.undertow.Undertow; +import io.undertow.client.*; import io.undertow.server.*; import io.undertow.server.handlers.proxy.*; +import io.undertow.util.Headers; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; -import java.net.InetSocketAddress; -import java.net.URI; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.junit.jupiter.params.provider.Arguments; +import org.xnio.channels.Channels; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Proxy; +import java.net.*; import java.net.http.*; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.time.Duration; -import java.util.concurrent.TimeUnit; +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.*; +@Timeout(20) class ProxyContinuationTest { + enum Phase { ACQUIRE, RETRY, SEND_CALLBACK, RESPONSE_CALLBACK, TIMEOUT } + private static final class ControlledFailure extends RuntimeException { + final Phase phase; + ControlledFailure(Phase phase) { super("Injected failure at " + phase); this.phase = phase; } + } + + static Stream failureCases() { + return Stream.of(Phase.values()).flatMap(phase -> Stream.of(false, true) + .map(enabled -> Arguments.of(phase, enabled))); + } + @ParameterizedTest - @CsvSource({"true,false", "false,false", "true,true", "false,true"}) - void connectionDispatchRetainsBoundary(boolean enabled, boolean delayedCompletion) throws Exception { - var completion = new java.util.concurrent.CompletableFuture(); - ProxyClient client = new ProxyClient() { - @Override public ProxyTarget findTarget(HttpServerExchange exchange) { return new ProxyTarget() {}; } - @Override public void getConnection(ProxyTarget target, HttpServerExchange exchange, - ProxyCallback callback, long timeout, TimeUnit unit) { - if (delayedCompletion) completion.complete(() -> callback.completed(exchange, new ProxyConnection(null, "/"))); - else throw new IllegalStateException("connection callback failed"); - } + @MethodSource("failureCases") + void asynchronousFailuresUseSelectedBoundaryAndCloseUpstream(Phase phase, boolean enabled) throws Exception { + var previousMdc = org.slf4j.MDC.getCopyOfContextMap(); + AtomicInteger injected = new AtomicInteger(); + AtomicInteger selected = new AtomicInteger(); + AtomicInteger acquisitions = new AtomicInteger(); + var upstream = new java.util.concurrent.atomic.AtomicReference(); + CountDownLatch release = new CountDownLatch(1); + CompletableFuture retry = new CompletableFuture<>(); + java.util.function.Supplier failure = () -> { + assertEquals("canonical-request", org.slf4j.MDC.get("request"), + "Callbacks must use the existing MDC_CONTEXT executor, not a stale thread snapshot"); + injected.incrementAndGet(); + return new ControlledFailure(phase); }; - ProxyHandler proxy = ProxyHandler.builder().setProxyClient(client).build(); - HttpHandler root = enabled ? exchange -> HandlerContinuation.execute(exchange, (ex, task) -> { - try { task.handleRequest(ex); } - catch (RuntimeException e) { ex.setStatusCode(409).getResponseSender().send("selected"); } - }, proxy) : proxy; - Undertow server = Undertow.builder().addHttpListener(0, "127.0.0.1").setHandler(root).build(); - try (HttpClient http = HttpClient.newHttpClient()) { - server.start(); - int port = ((InetSocketAddress) server.getListenerInfo().getFirst().getAddress()).getPort(); - var pending = http.sendAsync(HttpRequest.newBuilder(URI.create("http://127.0.0.1:" + port + "/")) - .timeout(Duration.ofSeconds(5)).build(), HttpResponse.BodyHandlers.ofString()); - if (delayedCompletion) { - assertFalse(pending.isDone(), "Connection acquisition must not end a pending exchange"); - completion.get(5, TimeUnit.SECONDS).run(); - } - var response = pending.get(5, TimeUnit.SECONDS); + try (Fixture fixture = new Fixture(); HttpClient http = HttpClient.newHttpClient()) { + URI backend = fixture.start(exchange -> exchange.dispatch(() -> { + try { + if (phase == Phase.TIMEOUT && !release.await(5, TimeUnit.SECONDS)) return; + exchange.getResponseSender().send("accepted"); + } catch (InterruptedException e) { Thread.currentThread().interrupt(); } + })); + LoadBalancingProxyClient real = new LoadBalancingProxyClient().addHost(backend); + ProxyClient client = new ProxyClient() { + @Override public ProxyTarget findTarget(HttpServerExchange exchange) { return real.findTarget(exchange); } + @Override public void getConnection(ProxyTarget target, HttpServerExchange exchange, + ProxyCallback callback, long timeout, TimeUnit unit) { + int attempt = acquisitions.incrementAndGet(); + if (phase == Phase.ACQUIRE || (phase == Phase.RETRY && attempt > 1)) throw failure.get(); + if (phase == Phase.RETRY) { + retry.complete(() -> callback.failed(exchange)); + return; + } + real.getConnection(target, exchange, new ProxyCallback<>() { + @Override public void completed(HttpServerExchange ex, ProxyConnection connection) { + upstream.set(connection.getConnection()); + callback.completed(ex, new ProxyConnection(decorateConnection(connection.getConnection(), phase, failure), + connection.getTargetPath())); + } + @Override public void failed(HttpServerExchange ex) { callback.failed(ex); } + @Override public void couldNotResolveBackend(HttpServerExchange ex) { callback.couldNotResolveBackend(ex); } + @Override public void queuedRequestFailed(HttpServerExchange ex) { callback.queuedRequestFailed(ex); } + }, timeout, unit); + } + }; + ProxyHandler proxy = ProxyHandler.builder().setProxyClient(client) + .setMaxConnectionRetries(phase == Phase.RETRY ? 1 : 0) + .setMaxRequestTime(phase == Phase.TIMEOUT ? 300 : 0).build(); + HttpHandler work = exchange -> { + exchange.putAttachment(AttachmentConstants.MDC_CONTEXT, Map.of("request", "canonical-request")); + if (phase == Phase.TIMEOUT) { + exchange.putAttachment(AttachmentConstants.METRICS_HANDLER, new ThrowingMetrics(failure)); + exchange.putAttachment(AttachmentConstants.DOWNSTREAM_METRICS_NAME, "test"); + exchange.putAttachment(AttachmentConstants.DOWNSTREAM_METRICS_START, System.nanoTime()); + } + proxy.handleRequest(exchange); + }; + URI front = fixture.start(enabled ? exchange -> HandlerContinuation.execute(exchange, (ex, task) -> { + try { task.handleRequest(ex); } + catch (ControlledFailure e) { + assertEquals(phase, e.phase); + selected.incrementAndGet(); + ex.setStatusCode(409).getResponseSender().send("selected"); + } + }, work) : work); + var pending = http.sendAsync(HttpRequest.newBuilder(front).timeout(Duration.ofSeconds(8)).build(), + HttpResponse.BodyHandlers.ofString()); + if (phase == Phase.RETRY) retry.get(5, TimeUnit.SECONDS).run(); + var response = pending.get(10, TimeUnit.SECONDS); assertEquals(enabled ? 409 : 500, response.statusCode(), response.body()); - assertEquals(enabled, response.body().contains("selected")); - } finally { server.stop(); } + assertEquals(1, injected.get(), "The intentional failure must execute exactly once"); + assertEquals(enabled ? 1 : 0, selected.get(), "Do not accept an incidental exception"); + assertEquals(phase == Phase.RETRY ? 2 : 1, acquisitions.get()); + if (phase != Phase.ACQUIRE && phase != Phase.RETRY) + assertFalse(upstream.get().isOpen(), "Close upstream resources on callback failure"); + release.countDown(); + } finally { + release.countDown(); + if (previousMdc == null) org.slf4j.MDC.clear(); + else org.slf4j.MDC.setContextMap(previousMdc); + } + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void successfulProxyActionRemainsPendingUntilUpstreamCompletes(boolean enabled) throws Exception { + CountDownLatch release = new CountDownLatch(1); + CountDownLatch complete = new CountDownLatch(1); + try (Fixture fixture = new Fixture(); HttpClient http = HttpClient.newHttpClient()) { + URI backend = fixture.start(exchange -> exchange.dispatch(() -> { + try { + exchange.getResponseHeaders().put(Headers.CONTENT_TYPE, "text/event-stream"); + var channel = exchange.getResponseChannel(); + Channels.writeBlocking(channel, ByteBuffer.wrap("first".getBytes(StandardCharsets.UTF_8))); + Channels.flushBlocking(channel); + assertTrue(release.await(5, TimeUnit.SECONDS)); + Channels.writeBlocking(channel, ByteBuffer.wrap("second".getBytes(StandardCharsets.UTF_8))); + channel.shutdownWrites(); + Channels.flushBlocking(channel); + } catch (Exception e) { throw new RuntimeException(e); } + })); + ProxyHandler proxy = ProxyHandler.builder().setProxyClient(new LoadBalancingProxyClient().addHost(backend)) + .setMaxRequestTime(0).build(); + HttpHandler work = exchange -> { + exchange.addExchangeCompleteListener((ex, next) -> { complete.countDown(); next.proceed(); }); + proxy.handleRequest(exchange); + }; + URI front = fixture.start(enabled ? exchange -> HandlerContinuation.execute(exchange, + (ex, task) -> task.handleRequest(ex), work) : work); + var response = http.send(HttpRequest.newBuilder(front).timeout(Duration.ofSeconds(8)).build(), + HttpResponse.BodyHandlers.ofInputStream()); + assertEquals(200, response.statusCode()); + try (var body = response.body()) { + assertEquals("first", new String(body.readNBytes(5), StandardCharsets.UTF_8)); + assertEquals(1, complete.getCount(), "A successful action must leave streaming I/O pending"); + release.countDown(); + assertEquals("second", new String(body.readAllBytes(), StandardCharsets.UTF_8)); + } + assertTrue(complete.await(3, TimeUnit.SECONDS)); + } finally { release.countDown(); } + } + + @Test + void unhandledTaskFailureAfterHeadersClosesStreamWithoutAppendingAnError() throws Exception { + try (Fixture fixture = new Fixture()) { + URI front = fixture.start(exchange -> exchange.dispatch(() -> { + try { + var channel = exchange.getResponseChannel(); + Channels.writeBlocking(channel, ByteBuffer.wrap("first".getBytes(StandardCharsets.UTF_8))); + Channels.flushBlocking(channel); + assertTrue(exchange.isResponseStarted()); + HandlerContinuation.task(exchange, ex -> { throw new ControlledFailure(Phase.RESPONSE_CALLBACK); }).run(); + } catch (IOException e) { throw new RuntimeException(e); } + })); + try (Socket socket = new Socket(front.getHost(), front.getPort())) { + socket.setSoTimeout(5000); + socket.getOutputStream().write("GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n" + .getBytes(StandardCharsets.UTF_8)); + String raw = new String(socket.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + assertTrue(raw.startsWith("HTTP/1.1 200"), raw); + assertTrue(raw.contains("first"), raw); + assertFalse(raw.contains("500"), raw); + assertFalse(raw.endsWith("0\r\n\r\n"), "Do not mark a failed stream as successfully complete"); + } + } + } + + @Test + void lateFailureDoesNotCloseACompletedKeepAliveConnection() throws Exception { + var completed = new java.util.concurrent.atomic.AtomicReference(); + CountDownLatch done = new CountDownLatch(1); + try (Fixture fixture = new Fixture(); HttpClient http = HttpClient.newHttpClient()) { + URI front = fixture.start(exchange -> { + exchange.addExchangeCompleteListener((ex, next) -> { + completed.set(ex); + done.countDown(); + next.proceed(); + }); + exchange.getResponseSender().send("accepted"); + }); + var request = HttpRequest.newBuilder(front).timeout(Duration.ofSeconds(5)).build(); + assertEquals("accepted", http.send(request, HttpResponse.BodyHandlers.ofString()).body()); + assertTrue(done.await(3, TimeUnit.SECONDS)); + assertTrue(completed.get().isComplete()); + assertTrue(completed.get().getConnection().isOpen()); + HandlerContinuation.task(completed.get(), ex -> { + throw new ControlledFailure(Phase.RESPONSE_CALLBACK); + }).run(); + assertTrue(completed.get().getConnection().isOpen(), "A late callback must not close a reusable connection"); + assertEquals("accepted", http.send(request, HttpResponse.BodyHandlers.ofString()).body()); + } + } + + private static ClientConnection decorateConnection(ClientConnection real, Phase phase, + java.util.function.Supplier failure) { + return (ClientConnection) Proxy.newProxyInstance(ClientConnection.class.getClassLoader(), + new Class[]{ClientConnection.class}, (proxy, method, args) -> { + if (method.getName().equals("sendRequest")) { + @SuppressWarnings("unchecked") var callback = (ClientCallback) args[1]; + real.sendRequest((ClientRequest) args[0], new ClientCallback<>() { + @Override public void completed(ClientExchange result) { callback.completed(decorateExchange(result, phase, failure)); } + @Override public void failed(IOException e) { callback.failed(e); } + }); + return null; + } + try { return method.invoke(real, args); } + catch (InvocationTargetException e) { throw e.getCause(); } + }); + } + + private static ClientExchange decorateExchange(ClientExchange real, Phase phase, + java.util.function.Supplier failure) { + return (ClientExchange) Proxy.newProxyInstance(ClientExchange.class.getClassLoader(), + new Class[]{ClientExchange.class}, (proxy, method, args) -> { + if (method.getName().equals("setResponseListener")) { + if (phase == Phase.SEND_CALLBACK) throw failure.get(); + @SuppressWarnings("unchecked") var callback = (ClientCallback) args[0]; + real.setResponseListener(new ClientCallback<>() { + @Override public void completed(ClientExchange result) { callback.completed(decorateExchange(result, phase, failure)); } + @Override public void failed(IOException e) { callback.failed(e); } + }); + return null; + } + if (method.getName().equals("getResponse") && phase == Phase.RESPONSE_CALLBACK) throw failure.get(); + try { return method.invoke(real, args); } + catch (InvocationTargetException e) { throw e.getCause(); } + }); + } + + private static final class ThrowingMetrics extends AbstractMetricsHandler { + private final java.util.function.Supplier failure; + ThrowingMetrics(java.util.function.Supplier failure) { this.failure = failure; } + @Override public void injectMetrics(HttpServerExchange exchange, long startTime, String name, String endpoint) { throw failure.get(); } + @Override protected void createMetricsReporter(TimeSeriesDbSender sender, MetricsConfig config) { } + @Override public void handleRequest(HttpServerExchange exchange) { } + @Override public HttpHandler getNext() { return null; } + @Override public MiddlewareHandler setNext(HttpHandler next) { return this; } + } + + private static final class Fixture implements AutoCloseable { + private final List servers = new ArrayList<>(); + URI start(HttpHandler handler) { + Undertow server = Undertow.builder().addHttpListener(0, "127.0.0.1").setIoThreads(1).setHandler(handler).build(); + server.start(); + servers.add(server); + return URI.create("http://127.0.0.1:" + ((InetSocketAddress) server.getListenerInfo().getFirst().getAddress()).getPort()); + } + @Override public void close() { + for (int i = servers.size() - 1; i >= 0; i--) servers.get(i).stop(); + } } } From 45cba7f7ae6682787280d69e9ba3d245848b67c5 Mon Sep 17 00:00:00 2001 From: Steve Hu Date: Thu, 29 Oct 2026 14:59:44 -0400 Subject: [PATCH 04/15] Scope continuation MDC and isolate proxy failures to request streams --- README.md | 28 ++-- .../networknt/exception/ExceptionHandler.java | 6 - .../exception/ExceptionContinuationTest.java | 81 +++++++++- .../handler/HandlerContinuation.java | 81 ++++++++-- .../RequestInterceptorInjectionHandler.java | 21 ++- .../handler/HandlerContinuationTest.java | 33 +++- .../RequestInjectionContinuationTest.java | 23 ++- .../com/networknt/handler/ProxyHandler.java | 147 ++++++++++------- .../handler/ProxyContinuationTest.java | 149 +++++++++++++++++- 9 files changed, 458 insertions(+), 111 deletions(-) diff --git a/README.md b/README.md index 2073af7831..b814d8410f 100644 --- a/README.md +++ b/README.md @@ -259,18 +259,26 @@ ExceptionHandler supplies its existing policy without rebuilding middleware or changing its next handler. Named handlers are retained; when exception handling is disabled and omitted from the chain, failures remain under Undertow's default policy. Nested boundaries restore their enclosing scope, and state is isolated -per exchange. The continuation contract carries only exception policy; MDC stays -with the existing MDC_CONTEXT attachment and LightThreadExecutor. Capturing and -resuming a boundary does not copy or replace thread-local MDC. ExceptionHandler -clears MDC at initial chain exit, not inside each resumed frame. +per exchange. Capturing a boundary does not snapshot thread-local MDC. At resume, +one scope installs the canonical MDC_CONTEXT attachment (or an empty context when +absent), then restores the executing thread's previous MDC on every exit. Nested +frames reuse that scope. Proxy callbacks use this scope rather than the unscoped +LightThreadExecutor. ExceptionHandler retains its original response-channel +availability policy after commitment; it does not forcibly close general-service +responses. Its initial worker frame still clears request MDC at chain exit. BodyHandler, DumpHandler and McpHandler redispatch, HMAC replay/admin completion, and ProxyHandler acquisition, retries, timeout cancellation and client send/response -callbacks carry the selected boundary. Proxy actions reuse the request's captured -policy and preserve pending upstream I/O on success. Callback failure closes its -upstream connection. Before response commitment an unhandled failure returns 500; -after commitment it closes the connection without appending an error or marking -a truncated stream complete. Late failures on completed exchanges do not rewrite -the response or close a reused downstream connection. +callbacks carry the selected boundary. The body-read callback scopes interceptor +execution as well as downstream handling, and preserves pending-read lifecycle. +Proxy actions reuse one captured policy; internal synchronous calls do not add +boundary frames or reset MDC again. Unexpected proxy failures cancel only the +upstream request's channels on multiplexed connections. Before response commitment +an unhandled failure returns 500; after commitment it closes the failed response +stream without appending an error. Undertow's HTTP/2 ServerConnection.close() +sends RST_STREAM for that stream, preserving healthy siblings. Late failures on +completed exchanges do not rewrite a response or close a reused connection. +The neutral Runnable adapter propagates Error; selected exception policies keep +their existing handling behavior. Custom asynchronous handlers must explicitly capture before registering their callbacks; this contract cannot intercept arbitrary third-party hand-offs. Body-reading requests capture lazily only when a read suspends. ingress-proxy uses diff --git a/exception/src/main/java/com/networknt/exception/ExceptionHandler.java b/exception/src/main/java/com/networknt/exception/ExceptionHandler.java index c5bd1d2b1a..9b77fb6e11 100644 --- a/exception/src/main/java/com/networknt/exception/ExceptionHandler.java +++ b/exception/src/main/java/com/networknt/exception/ExceptionHandler.java @@ -26,7 +26,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; -import org.xnio.IoUtils; /** * This is a middleware handler that should be put in the beginning of request/response @@ -100,11 +99,6 @@ private void handleContinuation(HttpServerExchange exchange, HttpHandler continu continuation.handleRequest(exchange); } catch (Throwable e) { logger.error("ExceptionHandler error processing - path: {}", path, e); - if (exchange.isComplete()) return; - if (exchange.isResponseStarted()) { - IoUtils.safeClose(exchange.getConnection()); - return; - } if(exchange.isResponseChannelAvailable()) { //handle exceptions if(e instanceof RuntimeException) { diff --git a/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java b/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java index 6eee050b73..d43e114033 100644 --- a/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java +++ b/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java @@ -26,12 +26,13 @@ class ExceptionContinuationTest { static Stream cases() { return Stream.of("exception", "continuation-named", "continuation-disabled") .flatMap(name -> Stream.of(false, true).flatMap(async -> - Stream.of(false, true).map(runtime -> Arguments.of(name, async, runtime)))); + Stream.of(false, true).flatMap(runtime -> Stream.of(false, true) + .map(mdc -> Arguments.of(name, async, runtime, mdc))))); } @ParameterizedTest @MethodSource("cases") - void selectedPolicyHandlesInitialAndResumedFailures(String name, boolean async, boolean runtime) throws Exception { + void selectedPolicyHandlesInitialAndResumedFailures(String name, boolean async, boolean runtime, boolean hasMdc) throws Exception { boolean enabled = !name.endsWith("disabled"); Map previous = Config.getInstance().getJsonMapConfig("exception"); Config.getInstance().putInConfigCache(name, Map.of("enabled", enabled)); @@ -42,12 +43,18 @@ void selectedPolicyHandlesInitialAndResumedFailures(String name, boolean async, } }; HttpHandler failing = exchange -> { + if (async) { + assertEquals(hasMdc ? "canonical" : null, org.slf4j.MDC.get("request")); + assertNull(org.slf4j.MDC.get("executor")); + } if (runtime) throw new IllegalStateException("runtime failure"); throw new IOException("checked failure"); }; var resumedMdc = new java.util.concurrent.atomic.AtomicReference(); var resumedDone = new java.util.concurrent.CountDownLatch(1); HttpHandler work = exchange -> { + if (hasMdc) exchange.putAttachment(com.networknt.httpstring.AttachmentConstants.MDC_CONTEXT, + Map.of("request", "canonical")); if (async) { HttpHandler resumed = HandlerContinuation.capture(exchange, failing); exchange.dispatch(() -> { @@ -84,4 +91,74 @@ void selectedPolicyHandlesInitialAndResumedFailures(String name, boolean async, else Config.getInstance().clearConfigCache(name); } } + @ParameterizedTest + @org.junit.jupiter.params.provider.ValueSource(booleans = {false, true}) + void committedResponseIsNotTruncatedByInitialOrResumedBoundary(boolean resumed) throws Exception { + var release = new java.util.concurrent.CountDownLatch(1); + ExceptionHandler selected = new ExceptionHandler(); + HttpHandler streaming = exchange -> { + exchange.startBlocking(); + var output = exchange.getOutputStream(); + output.write("first".getBytes(java.nio.charset.StandardCharsets.UTF_8)); + output.flush(); + exchange.dispatch(() -> { + try { + assertTrue(release.await(5, java.util.concurrent.TimeUnit.SECONDS)); + output.write("second".getBytes(java.nio.charset.StandardCharsets.UTF_8)); + output.close(); + } catch (Exception e) { throw new RuntimeException(e); } + }); + throw new IOException("failure after sending response headers"); + }; + selected.setNext(exchange -> { + if (resumed) exchange.dispatch(HandlerContinuation.capture(exchange, streaming)); + else streaming.handleRequest(exchange); + }); + Undertow server = Undertow.builder().addHttpListener(0, "127.0.0.1").setHandler(selected).build(); + try (HttpClient client = HttpClient.newHttpClient()) { + server.start(); + int port = ((InetSocketAddress) server.getListenerInfo().getFirst().getAddress()).getPort(); + var response = client.send(HttpRequest.newBuilder(URI.create("http://127.0.0.1:" + port)) + .timeout(Duration.ofSeconds(8)).build(), HttpResponse.BodyHandlers.ofInputStream()); + assertEquals(200, response.statusCode()); + try (var body = response.body()) { + assertEquals("first", new String(body.readNBytes(5), java.nio.charset.StandardCharsets.UTF_8)); + release.countDown(); + assertEquals("second", new String(body.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8)); + } + } finally { + release.countDown(); + server.stop(); + } + } + + @org.junit.jupiter.api.Test + void lateResumedFailureDoesNotRewriteCompletedResponseOrCloseKeepAliveConnection() throws Exception { + var saved = new java.util.concurrent.atomic.AtomicReference(); + var completed = new java.util.concurrent.atomic.AtomicReference(); + var done = new java.util.concurrent.CountDownLatch(1); + ExceptionHandler selected = new ExceptionHandler(); + selected.setNext(exchange -> { + saved.set(HandlerContinuation.capture(exchange, ex -> { throw new IOException("late failure"); })); + exchange.addExchangeCompleteListener((ex, next) -> { + completed.set(ex); + done.countDown(); + next.proceed(); + }); + exchange.getResponseSender().send("accepted"); + }); + Undertow server = Undertow.builder().addHttpListener(0, "127.0.0.1").setHandler(selected).build(); + try (HttpClient client = HttpClient.newHttpClient()) { + server.start(); + int port = ((InetSocketAddress) server.getListenerInfo().getFirst().getAddress()).getPort(); + var request = HttpRequest.newBuilder(URI.create("http://127.0.0.1:" + port)).timeout(Duration.ofSeconds(5)).build(); + assertEquals("accepted", client.send(request, HttpResponse.BodyHandlers.ofString()).body()); + assertTrue(done.await(3, java.util.concurrent.TimeUnit.SECONDS)); + assertTrue(completed.get().isComplete()); + saved.get().handleRequest(completed.get()); + assertTrue(completed.get().getConnection().isOpen()); + assertEquals("accepted", client.send(request, HttpResponse.BodyHandlers.ofString()).body()); + } finally { server.stop(); } + } + } diff --git a/handler/src/main/java/com/networknt/handler/HandlerContinuation.java b/handler/src/main/java/com/networknt/handler/HandlerContinuation.java index dea9c268fb..58eed4738d 100644 --- a/handler/src/main/java/com/networknt/handler/HandlerContinuation.java +++ b/handler/src/main/java/com/networknt/handler/HandlerContinuation.java @@ -1,13 +1,16 @@ package com.networknt.handler; +import com.networknt.httpstring.AttachmentConstants; import io.undertow.server.HttpHandler; import io.undertow.server.HttpServerExchange; import io.undertow.util.AttachmentKey; +import io.undertow.util.StatusCodes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.xnio.IoUtils; -import io.undertow.util.StatusCodes; +import org.slf4j.MDC; +import java.io.IOException; +import java.util.Map; import java.util.Objects; import java.util.function.UnaryOperator; @@ -21,6 +24,7 @@ public final class HandlerContinuation { private static final Logger LOG = LoggerFactory.getLogger(HandlerContinuation.class); private static final Boundary UNHANDLED = (exchange, continuation) -> continuation.handleRequest(exchange); private static final AttachmentKey ACTIVE = AttachmentKey.create(Boundary.class); + private static final ThreadLocal MDC_EXCHANGE = new ThreadLocal<>(); private HandlerContinuation() { } @@ -61,13 +65,35 @@ public static HttpHandler capture(HttpServerExchange exchange, HttpHandler conti * Captures before registering a callback whose result determines its eventual * work. Apply the returned wrapper inside that callback. Framework hand-offs * must opt in; arbitrary third-party callbacks cannot be intercepted here. - * Captures only the exception policy. MDC remains owned by the existing - * MDC_CONTEXT attachment and its executor; no thread-local maps are copied. + * Captures only the exception policy. At resume, MDC comes from the canonical + * MDC_CONTEXT attachment, not from a captured thread-local snapshot. The + * executing thread's previous MDC is restored even on failure. Nested frames + * for the same exchange reuse the active scope. */ public static UnaryOperator capture(HttpServerExchange exchange) { Boundary active = exchange.getAttachment(ACTIVE); Boundary boundary = active == null ? UNHANDLED : active; - return continuation -> resumed -> execute(resumed, boundary, continuation); + return continuation -> resumed -> withMdc(resumed, + ex -> execute(ex, boundary, continuation)); + } + + private static void withMdc(HttpServerExchange exchange, HttpHandler callback) throws Exception { + if (MDC_EXCHANGE.get() == exchange) { + callback.handleRequest(exchange); + return; + } + HttpServerExchange previousExchange = MDC_EXCHANGE.get(); + Map previousMdc = MDC.getCopyOfContextMap(); + Map requestMdc = exchange.getAttachment(AttachmentConstants.MDC_CONTEXT); + MDC_EXCHANGE.set(exchange); + try { + setMdc(requestMdc); + callback.handleRequest(exchange); + } finally { + setMdc(previousMdc); + if (previousExchange == null) MDC_EXCHANGE.remove(); + else MDC_EXCHANGE.set(previousExchange); + } } /** @@ -79,18 +105,41 @@ public static UnaryOperator capture(HttpServerExchange exchange) { public static Runnable task(HttpServerExchange exchange, HttpHandler continuation) { return () -> { try { - continuation.handleRequest(exchange); - } catch (Throwable failure) { - LOG.error("Unhandled continuation failure for {}", exchange.getRequestPath(), failure); - if (exchange.isComplete()) return; - if (exchange.isResponseStarted()) { - // Never replace committed headers or append an error to a stream. - IoUtils.safeClose(exchange.getConnection()); - } else { - exchange.setStatusCode(StatusCodes.INTERNAL_SERVER_ERROR); - exchange.endExchange(); - } + withMdc(exchange, ex -> { + try { + continuation.handleRequest(ex); + } catch (Exception failure) { + failTask(ex, failure); + } + }); + } catch (RuntimeException failure) { + throw failure; + } catch (Exception impossible) { + throw new IllegalStateException(impossible); } }; } + + private static void failTask(HttpServerExchange exchange, Exception failure) { + LOG.error("Unhandled continuation failure for {}", exchange.getRequestPath(), failure); + if (exchange.isComplete()) return; + if (exchange.isResponseStarted()) { + // Never replace committed headers or append an error to a stream. + try { + // HTTP/2 ServerConnection.close() resets this stream only. + exchange.getConnection().close(); + } catch (IOException closeFailure) { + failure.addSuppressed(closeFailure); + LOG.debug("Unable to close failed response stream", closeFailure); + } + } else { + exchange.setStatusCode(StatusCodes.INTERNAL_SERVER_ERROR); + exchange.endExchange(); + } + } + + private static void setMdc(Map context) { + if (context == null || context.isEmpty()) MDC.clear(); + else MDC.setContextMap(context); + } } diff --git a/handler/src/main/java/com/networknt/handler/RequestInterceptorInjectionHandler.java b/handler/src/main/java/com/networknt/handler/RequestInterceptorInjectionHandler.java index 1f4d8fe180..763eb65bb0 100644 --- a/handler/src/main/java/com/networknt/handler/RequestInterceptorInjectionHandler.java +++ b/handler/src/main/java/com/networknt/handler/RequestInterceptorInjectionHandler.java @@ -268,23 +268,28 @@ private void closeBuffers() { @Override public void handleEvent(StreamSourceChannel ignored) { + // A pending read must not be auto-ended by a root-handler wrapper. + HandlerContinuation.task(exchange, ex -> resumeRead()).run(); + } + + private void resumeRead() throws Exception { + boolean complete; try { ReadOutcome outcome = readAvailable(); - if (outcome == ReadOutcome.PENDING) - return; - + if (outcome == ReadOutcome.PENDING) return; stopReading(); - if (outcome == ReadOutcome.COMPLETE) { - saveBufferAndResetUndertowConnector(exchange, bufferedData); - if (!exchange.isResponseStarted()) - Connectors.executeRootHandler(resumedHandler, exchange); - } + complete = outcome == ReadOutcome.COMPLETE; + if (complete) saveBufferAndResetUndertowConnector(exchange, bufferedData); } catch (Throwable e) { LOG.error(e.getMessage(), e); stopReading(); closeBuffers(); setExchangeStatus(exchange, GENERIC_EXCEPTION, e.getMessage()); + return; } + // Downstream failures belong to the selected chain boundary, not body-read errors. + if (complete && !exchange.isResponseStarted()) + Connectors.executeRootHandler(resumedHandler, exchange); } } diff --git a/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java b/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java index f158c73e76..d7589f6eb6 100644 --- a/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java +++ b/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java @@ -12,15 +12,21 @@ import static org.junit.jupiter.api.Assertions.*; class HandlerContinuationTest { - @Test - void captureLeavesExecutorMdcUntouched() throws Exception { + @org.junit.jupiter.params.ParameterizedTest + @org.junit.jupiter.params.provider.ValueSource(booleans = {false, true}) + void resumeScopesCanonicalRequestMdcAndRestoresWorker(boolean present) throws Exception { var original = org.slf4j.MDC.getCopyOfContextMap(); try { HttpServerExchange exchange = new HttpServerExchange(null); - org.slf4j.MDC.put("request", "captured"); + if (present) exchange.putAttachment(com.networknt.httpstring.AttachmentConstants.MDC_CONTEXT, + java.util.Map.of("request", "canonical")); + org.slf4j.MDC.put("request", "stale-at-capture"); HttpHandler resumed = HandlerContinuation.capture(exchange, ex -> { - assertEquals("unrelated", org.slf4j.MDC.get("worker")); - assertNull(org.slf4j.MDC.get("request")); + assertEquals(present ? "canonical" : null, org.slf4j.MDC.get("request")); + assertNull(org.slf4j.MDC.get("worker"), "Do not borrow the executing thread's MDC"); + HandlerContinuation.capture(ex, nested -> { + assertEquals(present ? "canonical" : null, org.slf4j.MDC.get("request")); + }).handleRequest(ex); throw new IOException("failure"); }); org.slf4j.MDC.clear(); @@ -33,6 +39,23 @@ void captureLeavesExecutorMdcUntouched() throws Exception { } } + @Test + void runnableAdapterPropagatesFatalErrorsAndRestoresMdc() { + var original = org.slf4j.MDC.getCopyOfContextMap(); + try { + HttpServerExchange exchange = new HttpServerExchange(null); + org.slf4j.MDC.put("worker", "prior"); + Error error = new OutOfMemoryError("injected sentinel, no memory pressure"); + Runnable task = HandlerContinuation.task(exchange, + HandlerContinuation.capture(exchange, ex -> { throw error; })); + assertSame(error, assertThrows(OutOfMemoryError.class, task::run)); + assertEquals("prior", org.slf4j.MDC.get("worker")); + } finally { + if (original == null) org.slf4j.MDC.clear(); + else org.slf4j.MDC.setContextMap(original); + } + } + @Test void nestedCaptureRestoresEnclosingBoundaryAndRetainsSelectedInstances() throws Exception { HttpServerExchange exchange = new HttpServerExchange(null); diff --git a/handler/src/test/java/com/networknt/handler/RequestInjectionContinuationTest.java b/handler/src/test/java/com/networknt/handler/RequestInjectionContinuationTest.java index 90f01b0646..b03480cf8b 100644 --- a/handler/src/test/java/com/networknt/handler/RequestInjectionContinuationTest.java +++ b/handler/src/test/java/com/networknt/handler/RequestInjectionContinuationTest.java @@ -27,12 +27,16 @@ class RequestInjectionContinuationTest { void partialBodyRetainsBoundaryOrItsAbsence(boolean enabled) throws Exception { String config = "request-continuation-module-test"; RequestInterceptor[] previous = SingletonServiceFactory.getBeans(RequestInterceptor.class); + var interceptorMdc = new java.util.concurrent.atomic.AtomicReference(); + var downstreamMdc = new java.util.concurrent.atomic.AtomicReference(); RequestInterceptor interceptor = new RequestInterceptor() { @Override public boolean isRequiredContent() { return true; } @Override public boolean isEnabled() { return true; } @Override public HttpHandler getNext() { return null; } @Override public MiddlewareHandler setNext(HttpHandler next) { return this; } - @Override public void handleRequest(io.undertow.server.HttpServerExchange exchange) { } + @Override public void handleRequest(io.undertow.server.HttpServerExchange exchange) { + interceptorMdc.set(org.slf4j.MDC.get("request")); + } }; SingletonServiceFactory.setBean(RequestInterceptor.class.getName(), new RequestInterceptor[]{interceptor}); Config.getInstance().putInConfigCache(config, Map.of("enabled", true, "maxBuffers", 4, @@ -41,6 +45,7 @@ void partialBodyRetainsBoundaryOrItsAbsence(boolean enabled) throws Exception { AtomicBoolean onIo = new AtomicBoolean(); reader.setNext(exchange -> { onIo.set(exchange.isInIoThread()); + downstreamMdc.set(org.slf4j.MDC.get("request")); throw new IOException("downstream"); }); CountDownLatch suspended = new CountDownLatch(1); @@ -50,6 +55,14 @@ void partialBodyRetainsBoundaryOrItsAbsence(boolean enabled) throws Exception { }; Undertow server = Undertow.builder().addHttpListener(0, "127.0.0.1").setIoThreads(1) .setHandler(exchange -> { + if (exchange.getRequestPath().equals("/probe")) { + exchange.getResponseSender().send(org.slf4j.MDC.get("worker") + ":" + org.slf4j.MDC.get("request")); + org.slf4j.MDC.clear(); + return; + } + exchange.putAttachment(com.networknt.httpstring.AttachmentConstants.MDC_CONTEXT, + Map.of("request", "canonical")); + org.slf4j.MDC.put("worker", "prior-io"); if (enabled) HandlerContinuation.execute(exchange, boundary, reader); else reader.handleRequest(exchange); suspended.countDown(); @@ -69,6 +82,14 @@ void partialBodyRetainsBoundaryOrItsAbsence(boolean enabled) throws Exception { assertTrue(response.startsWith(enabled ? "HTTP/1.1 409" : "HTTP/1.1 500"), response); assertEquals(enabled, response.contains("selected boundary"), response); assertTrue(onIo.get(), "Exercise the actual channel-listener continuation"); + assertEquals("canonical", interceptorMdc.get()); + assertEquals("canonical", downstreamMdc.get()); + try (var client = java.net.http.HttpClient.newHttpClient()) { + var probe = client.send(java.net.http.HttpRequest.newBuilder( + java.net.URI.create("http://127.0.0.1:" + port + "/probe")).build(), + java.net.http.HttpResponse.BodyHandlers.ofString()); + assertEquals("prior-io:null", probe.body(), "Restore the I/O thread's context after the callback"); + } } } finally { server.stop(); diff --git a/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java b/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java index c786130fce..e2f59ee78f 100644 --- a/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java +++ b/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java @@ -17,25 +17,9 @@ */ package com.networknt.handler; -import java.io.Closeable; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.nio.ByteBuffer; -import java.nio.channels.Channel; -import java.nio.channels.FileChannel; -import java.security.cert.Certificate; -import java.util.*; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.Executor; -import java.util.function.UnaryOperator; -import java.util.concurrent.atomic.AtomicLong; -import javax.net.ssl.SSLPeerUnverifiedException; -import java.security.cert.CertificateEncodingException; - import com.networknt.handler.config.MethodRewriteRule; import com.networknt.handler.config.QueryHeaderRewriteRule; import com.networknt.handler.config.UrlRewriteRule; -import com.networknt.handler.thread.LightThreadExecutor; import com.networknt.httpstring.AttachmentConstants; import com.networknt.httpstring.HttpStringConstants; import com.networknt.metrics.AbstractMetricsHandler; @@ -43,11 +27,24 @@ import com.networknt.utility.StringUtils; import io.undertow.UndertowLogger; import io.undertow.UndertowMessages; +import io.undertow.attribute.ExchangeAttribute; +import io.undertow.client.ClientCallback; +import io.undertow.client.ClientConnection; +import io.undertow.client.ClientExchange; +import io.undertow.client.ClientRequest; +import io.undertow.client.ClientResponse; +import io.undertow.client.ProxiedRequestAttachments; +import io.undertow.io.IoCallback; +import io.undertow.io.Sender; +import io.undertow.predicate.IdempotentPredicate; +import io.undertow.predicate.Predicate; import io.undertow.server.*; import io.undertow.server.handlers.ResponseCodeHandler; import io.undertow.server.handlers.proxy.ProxyCallback; import io.undertow.server.handlers.proxy.ProxyClient; import io.undertow.server.handlers.proxy.ProxyConnection; +import io.undertow.server.protocol.http.HttpAttachments; +import io.undertow.server.protocol.http.HttpContinue; import io.undertow.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,26 +54,28 @@ import org.xnio.IoUtils; import org.xnio.Option; import org.xnio.StreamConnection; -import org.xnio.XnioIoThread; import org.xnio.XnioExecutor; +import org.xnio.XnioIoThread; import org.xnio.XnioWorker; import org.xnio.channels.StreamSinkChannel; import org.xnio.channels.StreamSourceChannel; -import io.undertow.attribute.ExchangeAttribute; -import io.undertow.client.ClientCallback; -import io.undertow.client.ClientConnection; -import io.undertow.client.ClientExchange; -import io.undertow.client.ClientRequest; -import io.undertow.client.ClientResponse; -import io.undertow.client.ProxiedRequestAttachments; -import io.undertow.io.IoCallback; -import io.undertow.io.Sender; -import io.undertow.predicate.IdempotentPredicate; -import io.undertow.predicate.Predicate; -import io.undertow.server.protocol.http.HttpAttachments; -import io.undertow.server.protocol.http.HttpContinue; +import java.io.Closeable; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.Channel; +import java.nio.channels.FileChannel; +import java.security.cert.Certificate; +import java.security.cert.CertificateEncodingException; +import java.util.*; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; +import java.util.function.UnaryOperator; import java.util.stream.Collectors; +import javax.net.ssl.SSLPeerUnverifiedException; /** * An HTTP handler which proxies content to a remote server. @@ -377,37 +376,73 @@ public String toString() { } } - /** One immutable policy snapshot and the existing request-MDC executor, reused by all callbacks. */ - private record ProxyContinuation(UnaryOperator boundary, LightThreadExecutor executor) { } + /** One immutable policy snapshot, reused by all callbacks. MDC is scoped by capture(). */ + private record ProxyContinuation(UnaryOperator boundary) { } + private record CallbackFrame(ProxyContinuation continuation, HttpServerExchange exchange) { } + private static final ThreadLocal ACTIVE_CALLBACK = new ThreadLocal<>(); + private static final AttachmentKey CLIENT_EXCHANGE = AttachmentKey.create(ClientExchange.class); + private static final AttachmentKey ABORTING = AttachmentKey.create(Boolean.class); - /** Keep callback setup and upstream-resource cleanup inside the selected boundary. */ + /** Each asynchronous entry owns one boundary/MDC scope; synchronous internal calls reuse it. */ private static Runnable continuationTask(HttpServerExchange exchange, ProxyContinuation continuation, - HttpHandler callback) { - Runnable task = HandlerContinuation.task(exchange, continuation.boundary().apply(ex -> { + Consumer callback) { + return () -> { + CallbackFrame previous = ACTIVE_CALLBACK.get(); + if (previous != null && previous.continuation() == continuation && previous.exchange() == exchange) { + callback.accept(exchange); + return; + } + ACTIVE_CALLBACK.set(new CallbackFrame(continuation, exchange)); try { - callback.handleRequest(ex); - } catch (Exception | Error failure) { - ProxyConnection upstream = ex.getAttachment(CONNECTION); - if (upstream != null) IoUtils.safeClose(upstream.getConnection()); - throw failure; + HandlerContinuation.task(exchange, continuation.boundary().apply(ex -> { + try { + callback.accept(ex); + } catch (Exception | Error failure) { + abortUpstream(ex); + throw failure; + } + })).run(); + } finally { + if (previous == null) ACTIVE_CALLBACK.remove(); + else ACTIVE_CALLBACK.set(previous); } - })); - return () -> continuation.executor().execute(task); + }; + } + + private static void cancelClientExchange(ClientExchange request) { + if (request.getConnection().isMultiplexingSupported()) { + // Close only this request's channels, never its shared HTTP/2 connection. + IoUtils.safeClose(request.getRequestChannel(), request.getResponseChannel()); + } else IoUtils.safeClose(request.getConnection()); + } + + private static void abortUpstream(HttpServerExchange exchange) { + exchange.putAttachment(ABORTING, Boolean.TRUE); + ClientExchange request = exchange.removeAttachment(CLIENT_EXCHANGE); + if (request != null) cancelClientExchange(request); + else { + ProxyConnection connection = exchange.getAttachment(CONNECTION); + if (connection != null && !connection.getConnection().isMultiplexingSupported()) + IoUtils.safeClose(connection.getConnection()); + } } private static ClientCallback boundaryCallback(HttpServerExchange exchange, ProxyContinuation continuation, ClientCallback callback) { return new ClientCallback<>() { @Override public void completed(ClientExchange result) { - if (exchange.isComplete()) { - IoUtils.safeClose(result.getConnection()); + if (exchange.isComplete() || Boolean.TRUE.equals(exchange.getAttachment(ABORTING))) { + cancelClientExchange(result); return; } - continuationTask(exchange, continuation, ex -> callback.completed(result)).run(); + continuationTask(exchange, continuation, ex -> { + ex.putAttachment(CLIENT_EXCHANGE, result); + callback.completed(result); + }).run(); } @Override public void failed(IOException failure) { - if (!exchange.isComplete()) + if (!exchange.isComplete() && !Boolean.TRUE.equals(exchange.getAttachment(ABORTING))) continuationTask(exchange, continuation, ex -> callback.failed(failure)).run(); } }; @@ -425,7 +460,7 @@ private final class ProxyClientHandler implements ProxyCallback ProxyClientHandler(HttpServerExchange exchange, ProxyClient.ProxyTarget target, long timeout, int maxRetryAttempts, Predicate idempotentPredicate, boolean streamingRequest) { this.exchange = exchange; - this.continuation = new ProxyContinuation(HandlerContinuation.capture(exchange), new LightThreadExecutor(exchange)); + this.continuation = new ProxyContinuation(HandlerContinuation.capture(exchange)); this.timeout = timeout; this.maxRetryAttempts = maxRetryAttempts; this.target = target; @@ -443,13 +478,13 @@ public void run() { public void completed(final HttpServerExchange exchange, final ProxyConnection connection) { continuationTask(exchange, continuation, ex -> { if (ex.isComplete()) { - IoUtils.safeClose(connection.getConnection()); + if (!connection.getConnection().isMultiplexingSupported()) IoUtils.safeClose(connection.getConnection()); return; } LOG.trace("ProxyCallback completed for exchange: {}", ex); ex.putAttachment(CONNECTION, connection); - ex.dispatch(INLINE_EXECUTOR, continuationTask(ex, continuation, resumed -> - new ProxyAction(connection, resumed, requestHeaders, rewriteHostHeader, reuseXForwarded, resumed.isRequestComplete() ? this : null, idempotentPredicate, urlRewriteRules, methodRewriteRules, queryParamRewriteRules, headerRewriteRules, streamingRequest, streamResponseContentTypes, streamIdleTimeout, streamResponseHeaderOverwrite, continuation).run())); + continuationTask(ex, continuation, resumed -> + new ProxyAction(connection, resumed, requestHeaders, rewriteHostHeader, reuseXForwarded, resumed.isRequestComplete() ? this : null, idempotentPredicate, urlRewriteRules, methodRewriteRules, queryParamRewriteRules, headerRewriteRules, streamingRequest, streamResponseContentTypes, streamIdleTimeout, streamResponseHeaderOverwrite, continuation).run()).run(); }).run(); } @@ -467,7 +502,7 @@ private void handleFailed(final HttpServerExchange exchange) { if (this.timeout > 0 && time > this.timeout) { LOG.trace("Current time = {} passes timeout {} for exchange: {}", time, this.timeout, exchange); - cancel(exchange); + cancelRequest(exchange); } else { this.target = proxyClient.findTarget(exchange); @@ -480,19 +515,19 @@ private void handleFailed(final HttpServerExchange exchange) { } else { LOG.trace("Target is null, cannot resolve the backend"); - couldNotResolveBackend(exchange); // The context was registered when we started, so return 503 + unresolvedBackend(exchange); // The context was registered when we started, so return 503 } } } else { LOG.trace("Max number fo retry attempts reached for exchange: {}", exchange); - couldNotResolveBackend(exchange); + unresolvedBackend(exchange); } } @Override public void queuedRequestFailed(HttpServerExchange exchange) { - failed(exchange); + continuationTask(exchange, continuation, this::handleFailed).run(); } @Override @@ -530,7 +565,7 @@ private void cancelRequest(final HttpServerExchange exchange) { if (connectionAttachment != null) { ClientConnection clientConnection = connectionAttachment.getConnection(); UndertowLogger.PROXY_REQUEST_LOGGER.timingOutRequest(clientConnection.getPeerAddress() + "" + exchange.getRequestURI()); - IoUtils.safeClose(clientConnection); + abortUpstream(exchange); } else UndertowLogger.PROXY_REQUEST_LOGGER.timingOutRequest(exchange.getRequestURI()); @@ -594,7 +629,7 @@ private static class ProxyAction implements Runnable { this.streamResponseContentTypes = streamResponseContentTypes; this.streamIdleTimeout = streamIdleTimeout; this.streamResponseHeaderOverwrite = streamResponseHeaderOverwrite; - } + } @Override public void run() { diff --git a/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java b/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java index 97ff8b42a7..e0a0c517fe 100644 --- a/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java +++ b/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java @@ -5,6 +5,7 @@ import com.networknt.metrics.MetricsConfig; import com.networknt.metrics.TimeSeriesDbSender; import io.undertow.Undertow; +import io.undertow.UndertowOptions; import io.undertow.client.*; import io.undertow.server.*; import io.undertow.server.handlers.proxy.*; @@ -41,21 +42,28 @@ private static final class ControlledFailure extends RuntimeException { static Stream failureCases() { return Stream.of(Phase.values()).flatMap(phase -> Stream.of(false, true) - .map(enabled -> Arguments.of(phase, enabled))); + .flatMap(enabled -> Stream.of(false, true).map(mdc -> Arguments.of(phase, enabled, mdc)))); } @ParameterizedTest @MethodSource("failureCases") - void asynchronousFailuresUseSelectedBoundaryAndCloseUpstream(Phase phase, boolean enabled) throws Exception { + void asynchronousFailuresUseSelectedBoundaryAndCloseUpstream(Phase phase, boolean enabled, boolean hasMdc) throws Exception { var previousMdc = org.slf4j.MDC.getCopyOfContextMap(); + var legacyMdcLogger = (ch.qos.logback.classic.Logger) org.slf4j.LoggerFactory.getLogger( + "com.networknt.handler.thread.LightThreadExecutor"); + var legacyMdcEvents = new ch.qos.logback.core.read.ListAppender(); + legacyMdcEvents.start(); + legacyMdcLogger.addAppender(legacyMdcEvents); AtomicInteger injected = new AtomicInteger(); AtomicInteger selected = new AtomicInteger(); + AtomicInteger depth = new AtomicInteger(); + AtomicInteger maxDepth = new AtomicInteger(); AtomicInteger acquisitions = new AtomicInteger(); var upstream = new java.util.concurrent.atomic.AtomicReference(); CountDownLatch release = new CountDownLatch(1); CompletableFuture retry = new CompletableFuture<>(); java.util.function.Supplier failure = () -> { - assertEquals("canonical-request", org.slf4j.MDC.get("request"), + assertEquals(hasMdc ? "canonical-request" : null, org.slf4j.MDC.get("request"), "Callbacks must use the existing MDC_CONTEXT executor, not a stale thread snapshot"); injected.incrementAndGet(); return new ControlledFailure(phase); @@ -94,7 +102,7 @@ void asynchronousFailuresUseSelectedBoundaryAndCloseUpstream(Phase phase, boolea .setMaxConnectionRetries(phase == Phase.RETRY ? 1 : 0) .setMaxRequestTime(phase == Phase.TIMEOUT ? 300 : 0).build(); HttpHandler work = exchange -> { - exchange.putAttachment(AttachmentConstants.MDC_CONTEXT, Map.of("request", "canonical-request")); + if (hasMdc) exchange.putAttachment(AttachmentConstants.MDC_CONTEXT, Map.of("request", "canonical-request")); if (phase == Phase.TIMEOUT) { exchange.putAttachment(AttachmentConstants.METRICS_HANDLER, new ThrowingMetrics(failure)); exchange.putAttachment(AttachmentConstants.DOWNSTREAM_METRICS_NAME, "test"); @@ -103,12 +111,13 @@ void asynchronousFailuresUseSelectedBoundaryAndCloseUpstream(Phase phase, boolea proxy.handleRequest(exchange); }; URI front = fixture.start(enabled ? exchange -> HandlerContinuation.execute(exchange, (ex, task) -> { + maxDepth.accumulateAndGet(depth.incrementAndGet(), Math::max); try { task.handleRequest(ex); } catch (ControlledFailure e) { assertEquals(phase, e.phase); selected.incrementAndGet(); ex.setStatusCode(409).getResponseSender().send("selected"); - } + } finally { depth.decrementAndGet(); } }, work) : work); var pending = http.sendAsync(HttpRequest.newBuilder(front).timeout(Duration.ofSeconds(8)).build(), HttpResponse.BodyHandlers.ofString()); @@ -117,12 +126,16 @@ void asynchronousFailuresUseSelectedBoundaryAndCloseUpstream(Phase phase, boolea assertEquals(enabled ? 409 : 500, response.statusCode(), response.body()); assertEquals(1, injected.get(), "The intentional failure must execute exactly once"); assertEquals(enabled ? 1 : 0, selected.get(), "Do not accept an incidental exception"); + if (enabled) assertEquals(1, maxDepth.get(), "Internal calls must not nest boundary frames"); assertEquals(phase == Phase.RETRY ? 2 : 1, acquisitions.get()); if (phase != Phase.ACQUIRE && phase != Phase.RETRY) assertFalse(upstream.get().isOpen(), "Close upstream resources on callback failure"); + assertTrue(legacyMdcEvents.list.isEmpty(), "Callbacks must not invoke the unscoped MDC executor"); release.countDown(); } finally { release.countDown(); + legacyMdcLogger.detachAppender(legacyMdcEvents); + legacyMdcEvents.stop(); if (previousMdc == null) org.slf4j.MDC.clear(); else org.slf4j.MDC.setContextMap(previousMdc); } @@ -218,6 +231,126 @@ void lateFailureDoesNotCloseACompletedKeepAliveConnection() throws Exception { } } + @Test + void upstreamHttp2FailureResetsOnlyTheFailedStream() throws Exception { + CountDownLatch healthyEntered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + Set backendPeers = ConcurrentHashMap.newKeySet(); + Set frontendPeers = ConcurrentHashMap.newKeySet(); + var shared = new java.util.concurrent.atomic.AtomicReference(); + AtomicInteger injected = new AtomicInteger(); + try (Fixture fixture = new Fixture(); HttpClient http = HttpClient.newBuilder().version(HttpClient.Version.HTTP_2).build()) { + URI backend = fixture.start(exchange -> { + assertEquals("HTTP/2.0", exchange.getProtocol().toString()); + backendPeers.add(exchange.getSourceAddress().getPort()); + if (exchange.getRequestPath().equals("/healthy")) exchange.dispatch(() -> { + healthyEntered.countDown(); + try { + assertTrue(release.await(8, TimeUnit.SECONDS)); + exchange.getResponseSender().send("healthy"); + } catch (InterruptedException e) { Thread.currentThread().interrupt(); } + }); + else exchange.getResponseSender().send("accepted"); + }, true); + URI upstream = URI.create("h2c-prior://" + backend.getHost() + ":" + backend.getPort()); + ProxyClient client = new ProxyClient() { + @Override public ProxyTarget findTarget(HttpServerExchange exchange) { return new ProxyTarget() {}; } + @Override public void getConnection(ProxyTarget target, HttpServerExchange exchange, + ProxyCallback callback, long timeout, TimeUnit unit) { + java.util.function.Consumer connected = connection -> { + ClientConnection selected = connection; + if (exchange.getRequestPath().equals("/fail")) selected = decorateConnection(connection, + Phase.RESPONSE_CALLBACK, () -> { + injected.incrementAndGet(); + return new ControlledFailure(Phase.RESPONSE_CALLBACK); + }); + callback.completed(exchange, new ProxyConnection(selected, "/")); + }; + if (shared.get() != null) connected.accept(shared.get()); + else UndertowClient.getInstance().connect(new ClientCallback<>() { + @Override public void completed(ClientConnection connection) { + shared.set(connection); + connected.accept(connection); + } + @Override public void failed(IOException e) { callback.failed(exchange); } + }, upstream, exchange.getIoThread(), exchange.getConnection().getByteBufferPool(), org.xnio.OptionMap.EMPTY); + } + }; + ProxyHandler proxy = ProxyHandler.builder().setProxyClient(client).setMaxRequestTime(0).build(); + URI front = fixture.start(exchange -> { + frontendPeers.add(exchange.getSourceAddress().getPort()); + HandlerContinuation.execute(exchange, (ex, task) -> { + try { task.handleRequest(ex); } + catch (ControlledFailure e) { ex.setStatusCode(409).getResponseSender().send("selected"); } + }, proxy); + }, true); + var warm = http.send(HttpRequest.newBuilder(front.resolve("/warm")).build(), HttpResponse.BodyHandlers.ofString()); + assertEquals(HttpClient.Version.HTTP_2, warm.version()); + assertEquals("accepted", warm.body()); + assertTrue(shared.get().isMultiplexingSupported()); + var healthy = http.sendAsync(HttpRequest.newBuilder(front.resolve("/healthy")).timeout(Duration.ofSeconds(10)).build(), + HttpResponse.BodyHandlers.ofString()); + assertTrue(healthyEntered.await(5, TimeUnit.SECONDS)); + var failed = http.send(HttpRequest.newBuilder(front.resolve("/fail")).timeout(Duration.ofSeconds(5)).build(), + HttpResponse.BodyHandlers.ofString()); + assertEquals(409, failed.statusCode(), failed.body()); + assertEquals(1, injected.get()); + assertTrue(shared.get().isOpen(), "Keep the shared upstream HTTP/2 connection open"); + assertFalse(healthy.isDone(), "Do not cancel the healthy sibling stream"); + release.countDown(); + assertEquals("healthy", healthy.get(5, TimeUnit.SECONDS).body()); + assertEquals("accepted", http.send(HttpRequest.newBuilder(front.resolve("/after")).build(), + HttpResponse.BodyHandlers.ofString()).body()); + assertEquals(1, backendPeers.size(), "All upstream requests must share one physical connection"); + assertEquals(1, frontendPeers.size(), "All downstream requests must share one physical connection"); + } finally { + release.countDown(); + if (shared.get() != null) shared.get().close(); + } + } + + @Test + void committedDownstreamHttp2FailureDoesNotCancelSiblingStream() throws Exception { + CountDownLatch healthyEntered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + Set peers = ConcurrentHashMap.newKeySet(); + try (Fixture fixture = new Fixture(); HttpClient http = HttpClient.newBuilder().version(HttpClient.Version.HTTP_2).build()) { + URI front = fixture.start(exchange -> { + peers.add(exchange.getSourceAddress().getPort()); + if (exchange.getRequestPath().equals("/healthy")) exchange.dispatch(() -> { + healthyEntered.countDown(); + try { + assertTrue(release.await(8, TimeUnit.SECONDS)); + exchange.getResponseSender().send("healthy"); + } catch (InterruptedException e) { Thread.currentThread().interrupt(); } + }); + else if (exchange.getRequestPath().equals("/fail")) exchange.dispatch(() -> { + try { + var channel = exchange.getResponseChannel(); + Channels.writeBlocking(channel, ByteBuffer.wrap("first".getBytes(StandardCharsets.UTF_8))); + Channels.flushBlocking(channel); + assertTrue(exchange.isResponseStarted()); + HandlerContinuation.task(exchange, ex -> { throw new ControlledFailure(Phase.RESPONSE_CALLBACK); }).run(); + } catch (IOException e) { throw new RuntimeException(e); } + }); + else exchange.getResponseSender().send("accepted"); + }, true); + assertEquals(HttpClient.Version.HTTP_2, http.send(HttpRequest.newBuilder(front.resolve("/warm")).build(), + HttpResponse.BodyHandlers.ofString()).version()); + var healthy = http.sendAsync(HttpRequest.newBuilder(front.resolve("/healthy")).timeout(Duration.ofSeconds(10)).build(), + HttpResponse.BodyHandlers.ofString()); + assertTrue(healthyEntered.await(5, TimeUnit.SECONDS)); + assertThrows(IOException.class, () -> http.send(HttpRequest.newBuilder(front.resolve("/fail")) + .timeout(Duration.ofSeconds(5)).build(), HttpResponse.BodyHandlers.ofString())); + assertFalse(healthy.isDone()); + release.countDown(); + assertEquals("healthy", healthy.get(5, TimeUnit.SECONDS).body()); + assertEquals("accepted", http.send(HttpRequest.newBuilder(front.resolve("/after")).build(), + HttpResponse.BodyHandlers.ofString()).body()); + assertEquals(1, peers.size(), "RST_STREAM must not kill the parent HTTP/2 connection"); + } finally { release.countDown(); } + } + private static ClientConnection decorateConnection(ClientConnection real, Phase phase, java.util.function.Supplier failure) { return (ClientConnection) Proxy.newProxyInstance(ClientConnection.class.getClassLoader(), @@ -266,8 +399,10 @@ private static final class ThrowingMetrics extends AbstractMetricsHandler { private static final class Fixture implements AutoCloseable { private final List servers = new ArrayList<>(); - URI start(HttpHandler handler) { - Undertow server = Undertow.builder().addHttpListener(0, "127.0.0.1").setIoThreads(1).setHandler(handler).build(); + URI start(HttpHandler handler) { return start(handler, false); } + URI start(HttpHandler handler, boolean http2) { + Undertow server = Undertow.builder().addHttpListener(0, "127.0.0.1").setIoThreads(1) + .setServerOption(UndertowOptions.ENABLE_HTTP2, http2).setHandler(handler).build(); server.start(); servers.add(server); return URI.create("http://127.0.0.1:" + ((InetSocketAddress) server.getListenerInfo().getFirst().getAddress()).getPort()); From 3efe64e37472e42c4f9ca519d531bffc7c7b4684 Mon Sep 17 00:00:00 2001 From: Steve Hu Date: Fri, 30 Oct 2026 09:41:04 -0400 Subject: [PATCH 05/15] Terminate fatal continuations and guard aborted HTTP2 callbacks --- README.md | 8 +- .../handler/HandlerContinuation.java | 73 ++++++++++------ .../networknt/handler/MiddlewareHandler.java | 14 +-- .../handler/HandlerContinuationTest.java | 43 +++++++++ .../com/networknt/handler/ProxyHandler.java | 87 +++++++++---------- .../handler/ProxyContinuationTest.java | 73 ++++++++++------ 6 files changed, 192 insertions(+), 106 deletions(-) diff --git a/README.md b/README.md index b814d8410f..7f51f638e5 100644 --- a/README.md +++ b/README.md @@ -262,7 +262,10 @@ policy. Nested boundaries restore their enclosing scope, and state is isolated per exchange. Capturing a boundary does not snapshot thread-local MDC. At resume, one scope installs the canonical MDC_CONTEXT attachment (or an empty context when absent), then restores the executing thread's previous MDC on every exit. Nested -frames reuse that scope. Proxy callbacks use this scope rather than the unscoped +frames reuse that scope. Policy identity and exchange identity share one frame +tracker; proxy callbacks do not maintain a second thread-local guard. Framework +MDC updates publish immutable snapshots through addHandlerMDCContext, so readers +do not traverse a concurrently mutated HashMap. Proxy callbacks use this scope rather than the unscoped LightThreadExecutor. ExceptionHandler retains its original response-channel availability policy after commitment; it does not forcibly close general-service responses. Its initial worker frame still clears request MDC at chain exit. @@ -277,7 +280,8 @@ an unhandled failure returns 500; after commitment it closes the failed response stream without appending an error. Undertow's HTTP/2 ServerConnection.close() sends RST_STREAM for that stream, preserving healthy siblings. Late failures on completed exchanges do not rewrite a response or close a reused connection. -The neutral Runnable adapter propagates Error; selected exception policies keep +The neutral Runnable adapter attempts stream/request cleanup before propagating +the original Error; selected exception policies keep their existing handling behavior. Custom asynchronous handlers must explicitly capture before registering their callbacks; this contract cannot intercept arbitrary third-party hand-offs. diff --git a/handler/src/main/java/com/networknt/handler/HandlerContinuation.java b/handler/src/main/java/com/networknt/handler/HandlerContinuation.java index 58eed4738d..65cd536388 100644 --- a/handler/src/main/java/com/networknt/handler/HandlerContinuation.java +++ b/handler/src/main/java/com/networknt/handler/HandlerContinuation.java @@ -24,7 +24,28 @@ public final class HandlerContinuation { private static final Logger LOG = LoggerFactory.getLogger(HandlerContinuation.class); private static final Boundary UNHANDLED = (exchange, continuation) -> continuation.handleRequest(exchange); private static final AttachmentKey ACTIVE = AttachmentKey.create(Boundary.class); - private static final ThreadLocal MDC_EXCHANGE = new ThreadLocal<>(); + private record Frame(HttpServerExchange exchange, CapturedBoundary boundary) { } + private static final ThreadLocal CURRENT_FRAME = new ThreadLocal<>(); + + @FunctionalInterface + private interface ScopedAction { void run() throws E; } + + private record CapturedBoundary(Boundary boundary) implements UnaryOperator { + @Override public HttpHandler apply(HttpHandler continuation) { + return exchange -> withMdc(exchange, () -> { + Frame previous = CURRENT_FRAME.get(); + CURRENT_FRAME.set(new Frame(exchange, this)); + try { execute(exchange, boundary, continuation); } + finally { CURRENT_FRAME.set(previous); } + }); + } + } + + // Package-private: proxy callbacks can reuse their exact active policy frame. + static boolean isActive(HttpServerExchange exchange, UnaryOperator boundary) { + Frame frame = CURRENT_FRAME.get(); + return frame != null && frame.exchange() == exchange && frame.boundary() == boundary; + } private HandlerContinuation() { } @@ -73,26 +94,28 @@ public static HttpHandler capture(HttpServerExchange exchange, HttpHandler conti public static UnaryOperator capture(HttpServerExchange exchange) { Boundary active = exchange.getAttachment(ACTIVE); Boundary boundary = active == null ? UNHANDLED : active; - return continuation -> resumed -> withMdc(resumed, - ex -> execute(ex, boundary, continuation)); + return new CapturedBoundary(boundary); } - private static void withMdc(HttpServerExchange exchange, HttpHandler callback) throws Exception { - if (MDC_EXCHANGE.get() == exchange) { - callback.handleRequest(exchange); + private static void withMdc(HttpServerExchange exchange, ScopedAction callback) throws E { + Frame previousFrame = CURRENT_FRAME.get(); + if (previousFrame != null && previousFrame.exchange() == exchange) { + callback.run(); return; } - HttpServerExchange previousExchange = MDC_EXCHANGE.get(); Map previousMdc = MDC.getCopyOfContextMap(); - Map requestMdc = exchange.getAttachment(AttachmentConstants.MDC_CONTEXT); - MDC_EXCHANGE.set(exchange); + Map requestMdc; + synchronized (exchange) { + requestMdc = exchange.getAttachment(AttachmentConstants.MDC_CONTEXT); + } + CURRENT_FRAME.set(new Frame(exchange, null)); try { setMdc(requestMdc); - callback.handleRequest(exchange); + callback.run(); } finally { setMdc(previousMdc); - if (previousExchange == null) MDC_EXCHANGE.remove(); - else MDC_EXCHANGE.set(previousExchange); + if (previousFrame == null) CURRENT_FRAME.remove(); + else CURRENT_FRAME.set(previousFrame); } } @@ -103,21 +126,21 @@ private static void withMdc(HttpServerExchange exchange, HttpHandler callback) t * committed stream, or leave an already completed exchange alone. */ public static Runnable task(HttpServerExchange exchange, HttpHandler continuation) { - return () -> { + return () -> withMdc(exchange, () -> { try { - withMdc(exchange, ex -> { - try { - continuation.handleRequest(ex); - } catch (Exception failure) { - failTask(ex, failure); - } - }); - } catch (RuntimeException failure) { - throw failure; - } catch (Exception impossible) { - throw new IllegalStateException(impossible); + continuation.handleRequest(exchange); + } catch (Error fatal) { + // Best-effort request/stream termination; preserve the original fatal error. + try { + if (!exchange.isComplete() && exchange.getConnection() != null) exchange.getConnection().close(); + } catch (Throwable cleanupFailure) { + if (cleanupFailure != fatal) fatal.addSuppressed(cleanupFailure); + } + throw fatal; + } catch (Exception failure) { + failTask(exchange, failure); } - }; + }); } private static void failTask(HttpServerExchange exchange, Exception failure) { diff --git a/handler/src/main/java/com/networknt/handler/MiddlewareHandler.java b/handler/src/main/java/com/networknt/handler/MiddlewareHandler.java index 7c5c370edd..5fadca5eb3 100644 --- a/handler/src/main/java/com/networknt/handler/MiddlewareHandler.java +++ b/handler/src/main/java/com/networknt/handler/MiddlewareHandler.java @@ -21,6 +21,7 @@ import io.undertow.server.HttpServerExchange; import java.util.HashMap; +import java.util.Collections; /** * A interface for middleware handlers. All middleware handlers must implement this interface @@ -67,13 +68,12 @@ public interface MiddlewareHandler extends LightHttpHandler { * @param value The value for the field. */ default void addHandlerMDCContext(final HttpServerExchange exchange, String field, String value) { - var context = exchange.getAttachment(AttachmentConstants.MDC_CONTEXT); - if (context != null) { - context.put(field, value); - } else { - var newContext = new HashMap(); - newContext.put(field, value); - exchange.putAttachment(AttachmentConstants.MDC_CONTEXT, newContext); + synchronized (exchange) { + var context = exchange.getAttachment(AttachmentConstants.MDC_CONTEXT); + var updated = context == null ? new HashMap() : new HashMap<>(context); + updated.put(field, value); + // Publish a stable snapshot; readers must not mutate the attachment directly. + exchange.putAttachment(AttachmentConstants.MDC_CONTEXT, Collections.unmodifiableMap(updated)); } } } diff --git a/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java b/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java index d7589f6eb6..bf3d912616 100644 --- a/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java +++ b/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java @@ -12,6 +12,49 @@ import static org.junit.jupiter.api.Assertions.*; class HandlerContinuationTest { + @Test + void activePolicyTracksIdentityAndExchangeAndRestoresAfterNestedFailure() throws Exception { + HttpServerExchange first = new HttpServerExchange(null); + HttpServerExchange second = new HttpServerExchange(null); + var policy = HandlerContinuation.capture(first); + var otherPolicy = HandlerContinuation.capture(first); + assertFalse(HandlerContinuation.isActive(first, policy)); + policy.apply(ex -> { + assertTrue(HandlerContinuation.isActive(first, policy)); + assertFalse(HandlerContinuation.isActive(first, otherPolicy)); + assertFalse(HandlerContinuation.isActive(second, policy)); + assertThrows(IOException.class, () -> policy.apply(nested -> { + assertTrue(HandlerContinuation.isActive(second, policy)); + assertFalse(HandlerContinuation.isActive(first, policy)); + throw new IOException("nested failure"); + }).handleRequest(second)); + assertTrue(HandlerContinuation.isActive(first, policy)); + }).handleRequest(first); + assertFalse(HandlerContinuation.isActive(first, policy)); + assertFalse(HandlerContinuation.isActive(second, policy)); + } + + @Test + void mdcPublicationRetainsImmutableSnapshotsAndConcurrentUpdates() throws Exception { + HttpServerExchange exchange = new HttpServerExchange(null); + MiddlewareHandler middleware = new MiddlewareHandler() { + @Override public boolean isEnabled() { return true; } + @Override public HttpHandler getNext() { return null; } + @Override public MiddlewareHandler setNext(HttpHandler next) { return this; } + @Override public void handleRequest(HttpServerExchange ex) { } + }; + middleware.addHandlerMDCContext(exchange, "first", "one"); + var first = exchange.getAttachment(com.networknt.httpstring.AttachmentConstants.MDC_CONTEXT); + assertThrows(UnsupportedOperationException.class, () -> first.put("bad", "mutation")); + try (var workers = java.util.concurrent.Executors.newFixedThreadPool(4)) { + var updates = java.util.stream.IntStream.range(0, 20).mapToObj(i -> workers.submit(() -> + middleware.addHandlerMDCContext(exchange, "field" + i, "value" + i))).toList(); + for (var update : updates) update.get(); + } + assertEquals(java.util.Map.of("first", "one"), first); + assertEquals(21, exchange.getAttachment(com.networknt.httpstring.AttachmentConstants.MDC_CONTEXT).size()); + } + @org.junit.jupiter.params.ParameterizedTest @org.junit.jupiter.params.provider.ValueSource(booleans = {false, true}) void resumeScopesCanonicalRequestMdcAndRestoresWorker(boolean present) throws Exception { diff --git a/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java b/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java index e2f59ee78f..7ecddda49b 100644 --- a/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java +++ b/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java @@ -17,6 +17,20 @@ */ package com.networknt.handler; +import java.io.Closeable; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.Channel; +import java.nio.channels.FileChannel; +import java.security.cert.Certificate; +import java.util.*; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import javax.net.ssl.SSLPeerUnverifiedException; +import java.security.cert.CertificateEncodingException; + import com.networknt.handler.config.MethodRewriteRule; import com.networknt.handler.config.QueryHeaderRewriteRule; import com.networknt.handler.config.UrlRewriteRule; @@ -27,24 +41,11 @@ import com.networknt.utility.StringUtils; import io.undertow.UndertowLogger; import io.undertow.UndertowMessages; -import io.undertow.attribute.ExchangeAttribute; -import io.undertow.client.ClientCallback; -import io.undertow.client.ClientConnection; -import io.undertow.client.ClientExchange; -import io.undertow.client.ClientRequest; -import io.undertow.client.ClientResponse; -import io.undertow.client.ProxiedRequestAttachments; -import io.undertow.io.IoCallback; -import io.undertow.io.Sender; -import io.undertow.predicate.IdempotentPredicate; -import io.undertow.predicate.Predicate; import io.undertow.server.*; import io.undertow.server.handlers.ResponseCodeHandler; import io.undertow.server.handlers.proxy.ProxyCallback; import io.undertow.server.handlers.proxy.ProxyClient; import io.undertow.server.handlers.proxy.ProxyConnection; -import io.undertow.server.protocol.http.HttpAttachments; -import io.undertow.server.protocol.http.HttpContinue; import io.undertow.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,28 +55,28 @@ import org.xnio.IoUtils; import org.xnio.Option; import org.xnio.StreamConnection; -import org.xnio.XnioExecutor; import org.xnio.XnioIoThread; +import org.xnio.XnioExecutor; import org.xnio.XnioWorker; import org.xnio.channels.StreamSinkChannel; import org.xnio.channels.StreamSourceChannel; +import io.undertow.attribute.ExchangeAttribute; +import io.undertow.client.ClientCallback; +import io.undertow.client.ClientConnection; +import io.undertow.client.ClientExchange; +import io.undertow.client.ClientRequest; +import io.undertow.client.ClientResponse; +import io.undertow.client.ProxiedRequestAttachments; +import io.undertow.io.IoCallback; +import io.undertow.io.Sender; +import io.undertow.predicate.IdempotentPredicate; +import io.undertow.predicate.Predicate; +import io.undertow.server.protocol.http.HttpAttachments; +import io.undertow.server.protocol.http.HttpContinue; -import java.io.Closeable; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.nio.ByteBuffer; -import java.nio.channels.Channel; -import java.nio.channels.FileChannel; -import java.security.cert.Certificate; -import java.security.cert.CertificateEncodingException; -import java.util.*; -import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import java.util.function.UnaryOperator; import java.util.stream.Collectors; -import javax.net.ssl.SSLPeerUnverifiedException; /** * An HTTP handler which proxies content to a remote server. @@ -378,8 +379,6 @@ public String toString() { /** One immutable policy snapshot, reused by all callbacks. MDC is scoped by capture(). */ private record ProxyContinuation(UnaryOperator boundary) { } - private record CallbackFrame(ProxyContinuation continuation, HttpServerExchange exchange) { } - private static final ThreadLocal ACTIVE_CALLBACK = new ThreadLocal<>(); private static final AttachmentKey CLIENT_EXCHANGE = AttachmentKey.create(ClientExchange.class); private static final AttachmentKey ABORTING = AttachmentKey.create(Boolean.class); @@ -388,25 +387,18 @@ private static Runnable continuationTask(HttpServerExchange exchange, ProxyContinuation continuation, Consumer callback) { return () -> { - CallbackFrame previous = ACTIVE_CALLBACK.get(); - if (previous != null && previous.continuation() == continuation && previous.exchange() == exchange) { + if (HandlerContinuation.isActive(exchange, continuation.boundary())) { callback.accept(exchange); return; } - ACTIVE_CALLBACK.set(new CallbackFrame(continuation, exchange)); - try { - HandlerContinuation.task(exchange, continuation.boundary().apply(ex -> { - try { - callback.accept(ex); - } catch (Exception | Error failure) { - abortUpstream(ex); - throw failure; - } - })).run(); - } finally { - if (previous == null) ACTIVE_CALLBACK.remove(); - else ACTIVE_CALLBACK.set(previous); - } + HandlerContinuation.task(exchange, continuation.boundary().apply(ex -> { + try { + callback.accept(ex); + } catch (Exception | Error failure) { + abortUpstream(ex); + throw failure; + } + })).run(); }; } @@ -477,7 +469,7 @@ public void run() { @Override public void completed(final HttpServerExchange exchange, final ProxyConnection connection) { continuationTask(exchange, continuation, ex -> { - if (ex.isComplete()) { + if (ex.isComplete() || ex.isResponseStarted() || Boolean.TRUE.equals(ex.getAttachment(ABORTING))) { if (!connection.getConnection().isMultiplexingSupported()) IoUtils.safeClose(connection.getConnection()); return; } @@ -494,6 +486,8 @@ public void failed(final HttpServerExchange exchange) { } private void handleFailed(final HttpServerExchange exchange) { + if (exchange.isComplete() || exchange.isResponseStarted() + || Boolean.TRUE.equals(exchange.getAttachment(ABORTING))) return; LOG.debug("Failed calling backend with tries = {} maxRetryAttempts = {} for exchange: {}", this.tries, this.maxRetryAttempts, exchange); final long time = System.currentTimeMillis(); @@ -559,6 +553,7 @@ void cancel(final HttpServerExchange exchange) { } private void cancelRequest(final HttpServerExchange exchange) { + exchange.putAttachment(ABORTING, Boolean.TRUE); //NOTE: this method is called only in context of timeouts. final ProxyConnection connectionAttachment = exchange.getAttachment(CONNECTION); diff --git a/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java b/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java index e0a0c517fe..cd51eba83b 100644 --- a/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java +++ b/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java @@ -34,7 +34,7 @@ @Timeout(20) class ProxyContinuationTest { - enum Phase { ACQUIRE, RETRY, SEND_CALLBACK, RESPONSE_CALLBACK, TIMEOUT } + enum Phase { ACQUIRE, RETRY, SEND_CALLBACK, RESPONSE_CALLBACK, TIMEOUT, BEFORE_SEND } private static final class ControlledFailure extends RuntimeException { final Phase phase; ControlledFailure(Phase phase) { super("Injected failure at " + phase); this.phase = phase; } @@ -49,11 +49,6 @@ static Stream failureCases() { @MethodSource("failureCases") void asynchronousFailuresUseSelectedBoundaryAndCloseUpstream(Phase phase, boolean enabled, boolean hasMdc) throws Exception { var previousMdc = org.slf4j.MDC.getCopyOfContextMap(); - var legacyMdcLogger = (ch.qos.logback.classic.Logger) org.slf4j.LoggerFactory.getLogger( - "com.networknt.handler.thread.LightThreadExecutor"); - var legacyMdcEvents = new ch.qos.logback.core.read.ListAppender(); - legacyMdcEvents.start(); - legacyMdcLogger.addAppender(legacyMdcEvents); AtomicInteger injected = new AtomicInteger(); AtomicInteger selected = new AtomicInteger(); AtomicInteger depth = new AtomicInteger(); @@ -64,7 +59,7 @@ void asynchronousFailuresUseSelectedBoundaryAndCloseUpstream(Phase phase, boolea CompletableFuture retry = new CompletableFuture<>(); java.util.function.Supplier failure = () -> { assertEquals(hasMdc ? "canonical-request" : null, org.slf4j.MDC.get("request"), - "Callbacks must use the existing MDC_CONTEXT executor, not a stale thread snapshot"); + "Callbacks must install canonical request MDC for this frame"); injected.incrementAndGet(); return new ControlledFailure(phase); }; @@ -130,12 +125,9 @@ void asynchronousFailuresUseSelectedBoundaryAndCloseUpstream(Phase phase, boolea assertEquals(phase == Phase.RETRY ? 2 : 1, acquisitions.get()); if (phase != Phase.ACQUIRE && phase != Phase.RETRY) assertFalse(upstream.get().isOpen(), "Close upstream resources on callback failure"); - assertTrue(legacyMdcEvents.list.isEmpty(), "Callbacks must not invoke the unscoped MDC executor"); release.countDown(); } finally { release.countDown(); - legacyMdcLogger.detachAppender(legacyMdcEvents); - legacyMdcEvents.stop(); if (previousMdc == null) org.slf4j.MDC.clear(); else org.slf4j.MDC.setContextMap(previousMdc); } @@ -231,8 +223,11 @@ void lateFailureDoesNotCloseACompletedKeepAliveConnection() throws Exception { } } - @Test - void upstreamHttp2FailureResetsOnlyTheFailedStream() throws Exception { + @ParameterizedTest + @ValueSource(strings = {"response", "timeout", "before-send", "late"}) + void upstreamHttp2FailureResetsOnlyTheFailedStream(String mode) throws Exception { + CompletableFuture late = new CompletableFuture<>(); + AtomicInteger failedAcquisitions = new AtomicInteger(); CountDownLatch healthyEntered = new CountDownLatch(1); CountDownLatch release = new CountDownLatch(1); Set backendPeers = ConcurrentHashMap.newKeySet(); @@ -243,8 +238,9 @@ void upstreamHttp2FailureResetsOnlyTheFailedStream() throws Exception { URI backend = fixture.start(exchange -> { assertEquals("HTTP/2.0", exchange.getProtocol().toString()); backendPeers.add(exchange.getSourceAddress().getPort()); - if (exchange.getRequestPath().equals("/healthy")) exchange.dispatch(() -> { - healthyEntered.countDown(); + if (exchange.getRequestPath().equals("/healthy") + || (mode.equals("timeout") && exchange.getRequestPath().equals("/fail"))) exchange.dispatch(() -> { + if (exchange.getRequestPath().equals("/healthy")) healthyEntered.countDown(); try { assertTrue(release.await(8, TimeUnit.SECONDS)); exchange.getResponseSender().send("healthy"); @@ -257,15 +253,24 @@ void upstreamHttp2FailureResetsOnlyTheFailedStream() throws Exception { @Override public ProxyTarget findTarget(HttpServerExchange exchange) { return new ProxyTarget() {}; } @Override public void getConnection(ProxyTarget target, HttpServerExchange exchange, ProxyCallback callback, long timeout, TimeUnit unit) { + if (exchange.getRequestPath().equals("/fail")) failedAcquisitions.incrementAndGet(); java.util.function.Consumer connected = connection -> { ClientConnection selected = connection; - if (exchange.getRequestPath().equals("/fail")) selected = decorateConnection(connection, - Phase.RESPONSE_CALLBACK, () -> { + if (exchange.getRequestPath().equals("/fail") && !mode.equals("timeout")) selected = decorateConnection(connection, + mode.equals("before-send") || mode.equals("late") ? Phase.BEFORE_SEND : Phase.RESPONSE_CALLBACK, () -> { injected.incrementAndGet(); return new ControlledFailure(Phase.RESPONSE_CALLBACK); }); callback.completed(exchange, new ProxyConnection(selected, "/")); }; + if (mode.equals("late") && exchange.getRequestPath().equals("/fail")) { + late.complete(() -> { + connected.accept(shared.get()); + callback.failed(exchange); + callback.queuedRequestFailed(exchange); + }); + return; + } if (shared.get() != null) connected.accept(shared.get()); else UndertowClient.getInstance().connect(new ClientCallback<>() { @Override public void completed(ClientConnection connection) { @@ -276,7 +281,8 @@ void upstreamHttp2FailureResetsOnlyTheFailedStream() throws Exception { }, upstream, exchange.getIoThread(), exchange.getConnection().getByteBufferPool(), org.xnio.OptionMap.EMPTY); } }; - ProxyHandler proxy = ProxyHandler.builder().setProxyClient(client).setMaxRequestTime(0).build(); + ProxyHandler proxy = ProxyHandler.builder().setProxyClient(client).setMaxRequestTime(0) + .setPathPrefixMaxRequestTime(Map.of("/fail", mode.equals("timeout") || mode.equals("late") ? 300 : 0)).build(); URI front = fixture.start(exchange -> { frontendPeers.add(exchange.getSourceAddress().getPort()); HandlerContinuation.execute(exchange, (ex, task) -> { @@ -293,8 +299,11 @@ void upstreamHttp2FailureResetsOnlyTheFailedStream() throws Exception { assertTrue(healthyEntered.await(5, TimeUnit.SECONDS)); var failed = http.send(HttpRequest.newBuilder(front.resolve("/fail")).timeout(Duration.ofSeconds(5)).build(), HttpResponse.BodyHandlers.ofString()); - assertEquals(409, failed.statusCode(), failed.body()); - assertEquals(1, injected.get()); + boolean timeout = mode.equals("timeout") || mode.equals("late"); + assertEquals(timeout ? 504 : 409, failed.statusCode(), failed.body()); + if (mode.equals("late")) late.get(3, TimeUnit.SECONDS).run(); + assertEquals(timeout ? 0 : 1, injected.get()); + assertEquals(1, failedAcquisitions.get(), "Late failure callbacks must not retry"); assertTrue(shared.get().isOpen(), "Keep the shared upstream HTTP/2 connection open"); assertFalse(healthy.isDone(), "Do not cancel the healthy sibling stream"); release.countDown(); @@ -309,8 +318,11 @@ void upstreamHttp2FailureResetsOnlyTheFailedStream() throws Exception { } } - @Test - void committedDownstreamHttp2FailureDoesNotCancelSiblingStream() throws Exception { + @ParameterizedTest + @org.junit.jupiter.params.provider.CsvSource({"false,true", "true,true", "true,false"}) + void downstreamHttp2FailureDoesNotCancelSiblingStream(boolean fatal, boolean committed) throws Exception { + CompletableFuture propagated = new CompletableFuture<>(); + Error sentinel = new StackOverflowError("injected fatal error"); CountDownLatch healthyEntered = new CountDownLatch(1); CountDownLatch release = new CountDownLatch(1); Set peers = ConcurrentHashMap.newKeySet(); @@ -326,11 +338,18 @@ void committedDownstreamHttp2FailureDoesNotCancelSiblingStream() throws Exceptio }); else if (exchange.getRequestPath().equals("/fail")) exchange.dispatch(() -> { try { - var channel = exchange.getResponseChannel(); - Channels.writeBlocking(channel, ByteBuffer.wrap("first".getBytes(StandardCharsets.UTF_8))); - Channels.flushBlocking(channel); - assertTrue(exchange.isResponseStarted()); - HandlerContinuation.task(exchange, ex -> { throw new ControlledFailure(Phase.RESPONSE_CALLBACK); }).run(); + if (committed) { + var channel = exchange.getResponseChannel(); + Channels.writeBlocking(channel, ByteBuffer.wrap("first".getBytes(StandardCharsets.UTF_8))); + Channels.flushBlocking(channel); + } + assertEquals(committed, exchange.isResponseStarted()); + try { + HandlerContinuation.task(exchange, ex -> { + if (fatal) throw sentinel; + throw new ControlledFailure(Phase.RESPONSE_CALLBACK); + }).run(); + } catch (Error error) { propagated.complete(error); } } catch (IOException e) { throw new RuntimeException(e); } }); else exchange.getResponseSender().send("accepted"); @@ -342,6 +361,7 @@ else if (exchange.getRequestPath().equals("/fail")) exchange.dispatch(() -> { assertTrue(healthyEntered.await(5, TimeUnit.SECONDS)); assertThrows(IOException.class, () -> http.send(HttpRequest.newBuilder(front.resolve("/fail")) .timeout(Duration.ofSeconds(5)).build(), HttpResponse.BodyHandlers.ofString())); + if (fatal) assertSame(sentinel, propagated.get(3, TimeUnit.SECONDS)); assertFalse(healthy.isDone()); release.countDown(); assertEquals("healthy", healthy.get(5, TimeUnit.SECONDS).body()); @@ -356,6 +376,7 @@ private static ClientConnection decorateConnection(ClientConnection real, Phase return (ClientConnection) Proxy.newProxyInstance(ClientConnection.class.getClassLoader(), new Class[]{ClientConnection.class}, (proxy, method, args) -> { if (method.getName().equals("sendRequest")) { + if (phase == Phase.BEFORE_SEND) throw failure.get(); @SuppressWarnings("unchecked") var callback = (ClientCallback) args[1]; real.sendRequest((ClientRequest) args[0], new ClientCallback<>() { @Override public void completed(ClientExchange result) { callback.completed(decorateExchange(result, phase, failure)); } From 79301b47e25252197bc9deda5074f38c5365cd3c Mon Sep 17 00:00:00 2001 From: Steve Hu Date: Fri, 30 Oct 2026 11:55:19 -0400 Subject: [PATCH 06/15] Publish request MDC through a stable atomic holder and guard fatal cleanup --- README.md | 15 ++- .../networknt/exception/ExceptionHandler.java | 2 + .../exception/ExceptionContinuationTest.java | 34 ++++++ handler/pom.xml | 5 + .../handler/HandlerContinuation.java | 101 ++++++++++++------ .../networknt/handler/MiddlewareHandler.java | 14 +-- .../networknt/handler/RequestMdcContext.java | 71 ++++++++++++ .../handler/HandlerContinuationTest.java | 83 ++++++++++++-- .../com/networknt/handler/ProxyHandler.java | 56 +++++----- .../handler/ProxyContinuationTest.java | 95 +++++++++++++++- 10 files changed, 391 insertions(+), 85 deletions(-) create mode 100644 handler/src/main/java/com/networknt/handler/RequestMdcContext.java diff --git a/README.md b/README.md index 7f51f638e5..97c635e90a 100644 --- a/README.md +++ b/README.md @@ -264,8 +264,14 @@ one scope installs the canonical MDC_CONTEXT attachment (or an empty context whe absent), then restores the executing thread's previous MDC on every exit. Nested frames reuse that scope. Policy identity and exchange identity share one frame tracker; proxy callbacks do not maintain a second thread-local guard. Framework -MDC updates publish immutable snapshots through addHandlerMDCContext, so readers -do not traverse a concurrently mutated HashMap. Proxy callbacks use this scope rather than the unscoped +MDC updates use one stable RequestMdcContext holder with an atomic immutable-map +reference. Capture and the initial ExceptionHandler frame initialize that holder +before any asynchronous hand-off; addHandlerMDCContext can also initialize it on +first serial use. Later updates replace only the holder's snapshot, never its +attachment. Custom asynchronous code must initialize/capture before hand-off. +The Map-typed attachment key remains compatible with read-only consumers; direct +attachment replacement or mutation is unsupported after initialization. +Proxy callbacks use this scope rather than the unscoped LightThreadExecutor. ExceptionHandler retains its original response-channel availability policy after commitment; it does not forcibly close general-service responses. Its initial worker frame still clears request MDC at chain exit. @@ -280,8 +286,9 @@ an unhandled failure returns 500; after commitment it closes the failed response stream without appending an error. Undertow's HTTP/2 ServerConnection.close() sends RST_STREAM for that stream, preserving healthy siblings. Late failures on completed exchanges do not rewrite a response or close a reused connection. -The neutral Runnable adapter attempts stream/request cleanup before propagating -the original Error; selected exception policies keep +The neutral Runnable adapter attempts a 500 before commitment or a stream reset +after commitment before propagating the original Error. Failed writes/resets are +suppressed on that Error and cannot replace it; selected exception policies keep their existing handling behavior. Custom asynchronous handlers must explicitly capture before registering their callbacks; this contract cannot intercept arbitrary third-party hand-offs. diff --git a/exception/src/main/java/com/networknt/exception/ExceptionHandler.java b/exception/src/main/java/com/networknt/exception/ExceptionHandler.java index 9b77fb6e11..24d4118be3 100644 --- a/exception/src/main/java/com/networknt/exception/ExceptionHandler.java +++ b/exception/src/main/java/com/networknt/exception/ExceptionHandler.java @@ -19,6 +19,7 @@ import com.networknt.handler.Handler; import com.networknt.handler.HandlerContinuation; import com.networknt.handler.MiddlewareHandler; +import com.networknt.handler.RequestMdcContext; import io.undertow.Handlers; import io.undertow.server.HttpHandler; import io.undertow.server.HttpServerExchange; @@ -71,6 +72,7 @@ public ExceptionHandler(String configName) { @Override public void handleRequest(final HttpServerExchange exchange) throws Exception { + RequestMdcContext.initialize(exchange); // dispatch here to make sure that all exceptions will be capture in this handler // otherwise, some of the exceptions will be captured in Connectors class in Undertow // As we've updated Server.java to redirect the logs to slf4j but still it make sense diff --git a/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java b/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java index d43e114033..353da3e8e0 100644 --- a/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java +++ b/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java @@ -161,4 +161,38 @@ void lateResumedFailureDoesNotRewriteCompletedResponseOrCloseKeepAliveConnection } finally { server.stop(); } } + @ParameterizedTest + @org.junit.jupiter.params.provider.ValueSource(booleans = {false, true}) + void installedBoundaryRetainsExistingFatalErrorMappingOnHttp11(boolean resumed) throws Exception { + var initializedOnIoThread = new java.util.concurrent.atomic.AtomicBoolean(); + var calls = new java.util.concurrent.atomic.AtomicInteger(); + Error sentinel = new StackOverflowError("installed boundary sentinel"); + ExceptionHandler selected = new ExceptionHandler(); + HttpHandler failing = exchange -> { calls.incrementAndGet(); throw sentinel; }; + selected.setNext(exchange -> { + if (exchange.getRequestPath().equals("/after")) exchange.getResponseSender().send("accepted"); + else if (resumed) exchange.dispatch(HandlerContinuation.capture(exchange, failing)); + else failing.handleRequest(exchange); + }); + Undertow server = Undertow.builder().addHttpListener(0, "127.0.0.1").setHandler(exchange -> { + selected.handleRequest(exchange); + initializedOnIoThread.set(exchange.isInIoThread() + && exchange.getAttachment(com.networknt.httpstring.AttachmentConstants.MDC_CONTEXT) + instanceof com.networknt.handler.RequestMdcContext); + }).build(); + try (HttpClient client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build()) { + server.start(); + int port = ((InetSocketAddress) server.getListenerInfo().getFirst().getAddress()).getPort(); + URI endpoint = URI.create("http://127.0.0.1:" + port); + var response = client.send(HttpRequest.newBuilder(endpoint).timeout(Duration.ofSeconds(5)).build(), + HttpResponse.BodyHandlers.ofString()); + assertEquals(400, response.statusCode(), response.body()); + assertTrue(response.body().contains("ERR10011"), response.body()); + assertEquals(1, calls.get()); + assertTrue(initializedOnIoThread.get(), "Publish the holder before initial worker dispatch"); + assertEquals("accepted", client.send(HttpRequest.newBuilder(endpoint.resolve("/after")).build(), + HttpResponse.BodyHandlers.ofString()).body()); + } finally { server.stop(); } + } + } diff --git a/handler/pom.xml b/handler/pom.xml index 6ef2321cb9..995f96422b 100644 --- a/handler/pom.xml +++ b/handler/pom.xml @@ -90,6 +90,11 @@ com.networknt common + + org.mockito + mockito-core + test + diff --git a/handler/src/main/java/com/networknt/handler/HandlerContinuation.java b/handler/src/main/java/com/networknt/handler/HandlerContinuation.java index 65cd536388..a292246cee 100644 --- a/handler/src/main/java/com/networknt/handler/HandlerContinuation.java +++ b/handler/src/main/java/com/networknt/handler/HandlerContinuation.java @@ -1,6 +1,5 @@ package com.networknt.handler; -import com.networknt.httpstring.AttachmentConstants; import io.undertow.server.HttpHandler; import io.undertow.server.HttpServerExchange; import io.undertow.util.AttachmentKey; @@ -32,17 +31,18 @@ private interface ScopedAction { void run() throws E; } private record CapturedBoundary(Boundary boundary) implements UnaryOperator { @Override public HttpHandler apply(HttpHandler continuation) { - return exchange -> withMdc(exchange, () -> { - Frame previous = CURRENT_FRAME.get(); - CURRENT_FRAME.set(new Frame(exchange, this)); - try { execute(exchange, boundary, continuation); } - finally { CURRENT_FRAME.set(previous); } - }); + return new CapturedHandler(this, continuation); } } - // Package-private: proxy callbacks can reuse their exact active policy frame. - static boolean isActive(HttpServerExchange exchange, UnaryOperator boundary) { + private record CapturedHandler(CapturedBoundary policy, HttpHandler continuation) implements HttpHandler { + @Override public void handleRequest(HttpServerExchange exchange) throws Exception { + withMdc(exchange, policy, () -> execute(exchange, policy.boundary(), continuation)); + } + } + + /** True only while this exact captured policy is executing for this exchange. */ + public static boolean isActive(HttpServerExchange exchange, UnaryOperator boundary) { Frame frame = CURRENT_FRAME.get(); return frame != null && frame.exchange() == exchange && frame.boundary() == boundary; } @@ -65,6 +65,7 @@ public interface Boundary { */ public static void execute(HttpServerExchange exchange, Boundary boundary, HttpHandler continuation) throws Exception { + RequestMdcContext.initialize(exchange); Boundary previous = exchange.putAttachment(ACTIVE, Objects.requireNonNull(boundary)); try { boundary.handle(exchange, continuation); @@ -92,25 +93,29 @@ public static HttpHandler capture(HttpServerExchange exchange, HttpHandler conti * for the same exchange reuse the active scope. */ public static UnaryOperator capture(HttpServerExchange exchange) { + RequestMdcContext.initialize(exchange); Boundary active = exchange.getAttachment(ACTIVE); Boundary boundary = active == null ? UNHANDLED : active; return new CapturedBoundary(boundary); } - private static void withMdc(HttpServerExchange exchange, ScopedAction callback) throws E { + private static void withMdc(HttpServerExchange exchange, CapturedBoundary policy, + ScopedAction callback) throws E { Frame previousFrame = CURRENT_FRAME.get(); if (previousFrame != null && previousFrame.exchange() == exchange) { - callback.run(); + if (policy == null || previousFrame.boundary() == policy) callback.run(); + else { + CURRENT_FRAME.set(new Frame(exchange, policy)); + try { callback.run(); } + finally { CURRENT_FRAME.set(previousFrame); } + } return; } Map previousMdc = MDC.getCopyOfContextMap(); - Map requestMdc; - synchronized (exchange) { - requestMdc = exchange.getAttachment(AttachmentConstants.MDC_CONTEXT); - } - CURRENT_FRAME.set(new Frame(exchange, null)); + Map requestMdc = RequestMdcContext.read(exchange); + CURRENT_FRAME.set(new Frame(exchange, policy)); try { - setMdc(requestMdc); + if (!requestMdc.equals(previousMdc == null ? Map.of() : previousMdc)) setMdc(requestMdc); callback.run(); } finally { setMdc(previousMdc); @@ -126,21 +131,53 @@ private static void withMdc(HttpServerExchange exchange, S * committed stream, or leave an already completed exchange alone. */ public static Runnable task(HttpServerExchange exchange, HttpHandler continuation) { - return () -> withMdc(exchange, () -> { - try { - continuation.handleRequest(exchange); - } catch (Error fatal) { - // Best-effort request/stream termination; preserve the original fatal error. - try { - if (!exchange.isComplete() && exchange.getConnection() != null) exchange.getConnection().close(); - } catch (Throwable cleanupFailure) { - if (cleanupFailure != fatal) fatal.addSuppressed(cleanupFailure); - } - throw fatal; - } catch (Exception failure) { - failTask(exchange, failure); - } - }); + if (continuation instanceof CapturedHandler captured) { + HttpHandler action = ex -> execute(ex, captured.policy().boundary(), captured.continuation()); + return () -> withMdc(exchange, captured.policy(), () -> invokeTask(exchange, action)); + } + return () -> withMdc(exchange, null, () -> invokeTask(exchange, continuation)); + } + + private static void invokeTask(HttpServerExchange exchange, HttpHandler continuation) { + try { + continuation.handleRequest(exchange); + } catch (Error fatal) { + failFatal(exchange, fatal); + throw fatal; + } catch (Exception failure) { + failTask(exchange, failure); + } + } + + private static void failFatal(HttpServerExchange exchange, Error fatal) { + if (exchange.isComplete()) return; + if (exchange.isResponseStarted()) { + closeAfterFatal(exchange, fatal); + return; + } + try { + exchange.setStatusCode(StatusCodes.INTERNAL_SERVER_ERROR); + exchange.endExchange(); + } catch (Throwable writeFailure) { + rememberCleanupFailure(fatal, writeFailure); + closeAfterFatal(exchange, fatal); + } + } + + private static void closeAfterFatal(HttpServerExchange exchange, Error fatal) { + try { + if (exchange.getConnection() != null) exchange.getConnection().close(); + } catch (Throwable closeFailure) { + rememberCleanupFailure(fatal, closeFailure); + } + } + + private static void rememberCleanupFailure(Error fatal, Throwable cleanupFailure) { + if (cleanupFailure == fatal) return; + try { fatal.addSuppressed(cleanupFailure); } + catch (Throwable ignored) { + // Preserve the original Error even if the VM cannot allocate suppression metadata. + } } private static void failTask(HttpServerExchange exchange, Exception failure) { diff --git a/handler/src/main/java/com/networknt/handler/MiddlewareHandler.java b/handler/src/main/java/com/networknt/handler/MiddlewareHandler.java index 5fadca5eb3..05b5175178 100644 --- a/handler/src/main/java/com/networknt/handler/MiddlewareHandler.java +++ b/handler/src/main/java/com/networknt/handler/MiddlewareHandler.java @@ -20,8 +20,6 @@ import io.undertow.server.HttpHandler; import io.undertow.server.HttpServerExchange; -import java.util.HashMap; -import java.util.Collections; /** * A interface for middleware handlers. All middleware handlers must implement this interface @@ -61,19 +59,15 @@ public interface MiddlewareHandler extends LightHttpHandler { boolean isEnabled(); /** - * Adds a field and value to the MDC context for the handler. + * Adds a field and value to the MDC context for the handler. First use must + * occur in the serial frame before asynchronous hand-off. Subsequent updates + * publish immutable snapshots through the stable holder. * * @param exchange HttpServerExchange to add the context to. * @param field The field name. * @param value The value for the field. */ default void addHandlerMDCContext(final HttpServerExchange exchange, String field, String value) { - synchronized (exchange) { - var context = exchange.getAttachment(AttachmentConstants.MDC_CONTEXT); - var updated = context == null ? new HashMap() : new HashMap<>(context); - updated.put(field, value); - // Publish a stable snapshot; readers must not mutate the attachment directly. - exchange.putAttachment(AttachmentConstants.MDC_CONTEXT, Collections.unmodifiableMap(updated)); - } + RequestMdcContext.initialize(exchange).putField(field, value); } } diff --git a/handler/src/main/java/com/networknt/handler/RequestMdcContext.java b/handler/src/main/java/com/networknt/handler/RequestMdcContext.java new file mode 100644 index 0000000000..a3d19ef310 --- /dev/null +++ b/handler/src/main/java/com/networknt/handler/RequestMdcContext.java @@ -0,0 +1,71 @@ +package com.networknt.handler; + +import com.networknt.httpstring.AttachmentConstants; +import io.undertow.server.HttpServerExchange; + +import java.util.AbstractMap; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; + +/** + * Stable, read-only Map view of request MDC. Initialize in the first serial + * request frame before handing the exchange to another thread. Subsequent + * publication changes only the atomic snapshot, never the attachment map. + */ +public final class RequestMdcContext extends AbstractMap { + private final AtomicReference> current; + + private RequestMdcContext(Map initial) { + current = new AtomicReference<>(initial == null || initial.isEmpty() + ? Map.of() : Collections.unmodifiableMap(new HashMap<>(initial))); + } + + /** + * First use must be serial. Capture and ExceptionHandler call this before + * dispatch; repeated calls after publication do not write any attachment. + * Legacy Map-valued context is migrated once in that serial frame. + */ + public static RequestMdcContext initialize(HttpServerExchange exchange) { + Map attached = exchange.getAttachment(AttachmentConstants.MDC_CONTEXT); + if (attached instanceof RequestMdcContext holder) return holder; + RequestMdcContext holder = new RequestMdcContext(attached); + exchange.putAttachment(AttachmentConstants.MDC_CONTEXT, holder); + return holder; + } + + static Map read(HttpServerExchange exchange) { + Map attached = exchange.getAttachment(AttachmentConstants.MDC_CONTEXT); + if (attached == null) return Map.of(); + if (attached instanceof RequestMdcContext holder) return holder.snapshot(); + throw new IllegalStateException("Initialize request MDC in the serial frame before asynchronous hand-off"); + } + + Map snapshot() { return current.get(); } + + void putField(String field, String value) { + current.updateAndGet(previous -> { + Map next = new HashMap<>(previous); + next.put(field, value); + return Collections.unmodifiableMap(next); + }); + } + + void putFields(Map fields) { + current.updateAndGet(previous -> { + Map next = new HashMap<>(previous); + next.putAll(fields); + return Collections.unmodifiableMap(next); + }); + } + + @Override public String get(Object key) { return current.get().get(key); } + @Override public int size() { return current.get().size(); } + @Override public Set> entrySet() { return current.get().entrySet(); } + @Override public void forEach(BiConsumer consumer) { + current.get().forEach(consumer); + } +} diff --git a/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java b/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java index bf3d912616..7b6c2ae0fd 100644 --- a/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java +++ b/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java @@ -36,15 +36,17 @@ void activePolicyTracksIdentityAndExchangeAndRestoresAfterNestedFailure() throws @Test void mdcPublicationRetainsImmutableSnapshotsAndConcurrentUpdates() throws Exception { - HttpServerExchange exchange = new HttpServerExchange(null); + HttpServerExchange exchange = org.mockito.Mockito.spy(new HttpServerExchange(null)); MiddlewareHandler middleware = new MiddlewareHandler() { @Override public boolean isEnabled() { return true; } @Override public HttpHandler getNext() { return null; } @Override public MiddlewareHandler setNext(HttpHandler next) { return this; } - @Override public void handleRequest(HttpServerExchange ex) { } + @Override public void handleRequest(HttpServerExchange ex) { /* Publication-only fixture. */ } }; middleware.addHandlerMDCContext(exchange, "first", "one"); - var first = exchange.getAttachment(com.networknt.httpstring.AttachmentConstants.MDC_CONTEXT); + var holder = assertInstanceOf(RequestMdcContext.class, + exchange.getAttachment(com.networknt.httpstring.AttachmentConstants.MDC_CONTEXT)); + var first = holder.snapshot(); assertThrows(UnsupportedOperationException.class, () -> first.put("bad", "mutation")); try (var workers = java.util.concurrent.Executors.newFixedThreadPool(4)) { var updates = java.util.stream.IntStream.range(0, 20).mapToObj(i -> workers.submit(() -> @@ -52,7 +54,73 @@ void mdcPublicationRetainsImmutableSnapshotsAndConcurrentUpdates() throws Except for (var update : updates) update.get(); } assertEquals(java.util.Map.of("first", "one"), first); - assertEquals(21, exchange.getAttachment(com.networknt.httpstring.AttachmentConstants.MDC_CONTEXT).size()); + assertEquals(21, holder.size()); + assertSame(holder, exchange.getAttachment(com.networknt.httpstring.AttachmentConstants.MDC_CONTEXT)); + org.mockito.Mockito.verify(exchange, org.mockito.Mockito.times(1)).putAttachment( + org.mockito.Mockito.eq(com.networknt.httpstring.AttachmentConstants.MDC_CONTEXT), org.mockito.Mockito.any()); + } + + @Test + void readersIterateStableSnapshotsWhileWritersPublishToInitializedHolder() throws Exception { + HttpServerExchange exchange = org.mockito.Mockito.spy(new HttpServerExchange(null)); + RequestMdcContext holder = RequestMdcContext.initialize(exchange); + holder.putFields(java.util.Map.of("epoch", "0", "mirror", "0")); + var retained = holder.snapshot(); + var barrier = new java.util.concurrent.CyclicBarrier(4); + var writersDone = new java.util.concurrent.CountDownLatch(2); + var reads = new java.util.concurrent.atomic.AtomicInteger(); + try (var workers = java.util.concurrent.Executors.newFixedThreadPool(4)) { + List> tasks = new ArrayList<>(); + for (int writer = 0; writer < 2; writer++) { + int id = writer; + tasks.add(workers.submit(() -> { + try { + barrier.await(5, java.util.concurrent.TimeUnit.SECONDS); + for (int i = 1; i <= 2000; i++) { + String epoch = id + ":" + i; + holder.putFields(java.util.Map.of("epoch", epoch, "mirror", epoch)); + } + } finally { writersDone.countDown(); } + return null; + })); + } + for (int reader = 0; reader < 2; reader++) tasks.add(workers.submit(() -> { + barrier.await(5, java.util.concurrent.TimeUnit.SECONDS); + do { + var snapshot = holder.snapshot(); + var copy = new java.util.HashMap(); + snapshot.forEach(copy::put); + assertEquals(copy.get("epoch"), copy.get("mirror"), "Never iterate a torn generation"); + assertThrows(UnsupportedOperationException.class, () -> snapshot.put("bad", "mutation")); + reads.incrementAndGet(); + } while (writersDone.getCount() != 0); + return null; + })); + for (var task : tasks) task.get(10, java.util.concurrent.TimeUnit.SECONDS); + } + assertTrue(reads.get() >= 2); + assertEquals(java.util.Map.of("epoch", "0", "mirror", "0"), retained, + "A mutable HashMap implementation must fail this retained-generation assertion"); + assertSame(holder, exchange.getAttachment(com.networknt.httpstring.AttachmentConstants.MDC_CONTEXT)); + org.mockito.Mockito.verify(exchange, org.mockito.Mockito.times(1)).putAttachment( + org.mockito.Mockito.eq(com.networknt.httpstring.AttachmentConstants.MDC_CONTEXT), org.mockito.Mockito.any()); + } + + @Test + void failedFatalStatusWriteCannotReplaceOriginalError() throws Exception { + Error fatal = new StackOverflowError("original"); + RuntimeException writeFailure = new IllegalStateException("status write failed"); + IOException closeFailure = new IOException("reset failed too"); + var connection = org.mockito.Mockito.mock(io.undertow.server.ServerConnection.class); + org.mockito.Mockito.doThrow(closeFailure).when(connection).close(); + HttpServerExchange exchange = org.mockito.Mockito.spy(new HttpServerExchange(connection)); + org.mockito.Mockito.doThrow(writeFailure).when(exchange).endExchange(); + Error observed = assertThrows(StackOverflowError.class, () -> + HandlerContinuation.task(exchange, ex -> { throw fatal; }).run()); + assertSame(fatal, observed); + assertEquals(500, exchange.getStatusCode()); + assertArrayEquals(new Throwable[]{writeFailure, closeFailure}, fatal.getSuppressed()); + org.mockito.Mockito.verify(connection).close(); } @org.junit.jupiter.params.ParameterizedTest @@ -86,13 +154,16 @@ void resumeScopesCanonicalRequestMdcAndRestoresWorker(boolean present) throws Ex void runnableAdapterPropagatesFatalErrorsAndRestoresMdc() { var original = org.slf4j.MDC.getCopyOfContextMap(); try { - HttpServerExchange exchange = new HttpServerExchange(null); + HttpServerExchange exchange = org.mockito.Mockito.spy(new HttpServerExchange(null)); + org.mockito.Mockito.doReturn(exchange).when(exchange).endExchange(); org.slf4j.MDC.put("worker", "prior"); Error error = new OutOfMemoryError("injected sentinel, no memory pressure"); Runnable task = HandlerContinuation.task(exchange, HandlerContinuation.capture(exchange, ex -> { throw error; })); assertSame(error, assertThrows(OutOfMemoryError.class, task::run)); assertEquals("prior", org.slf4j.MDC.get("worker")); + org.mockito.Mockito.verify(exchange).endExchange(); + assertEquals(500, exchange.getStatusCode()); } finally { if (original == null) org.slf4j.MDC.clear(); else org.slf4j.MDC.setContextMap(original); @@ -139,7 +210,7 @@ void absentBoundaryStaysAbsentAndFailuresRestoreScope() throws Exception { HandlerContinuation.execute(exchange, selected, ex -> assertSame(failure, assertThrows(IOException.class, () -> unwrapped.handleRequest(ex)))); assertSame(failure, assertThrows(IOException.class, () -> - HandlerContinuation.execute(exchange, (ex, task) -> { throw failure; }, ignored -> {}))); + HandlerContinuation.execute(exchange, (ex, task) -> { throw failure; }, ignored -> { /* The boundary throws before this continuation runs. */ }))); assertSame(failure, assertThrows(IOException.class, () -> unwrapped.handleRequest(exchange))); } } diff --git a/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java b/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java index 7ecddda49b..7ad944a225 100644 --- a/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java +++ b/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java @@ -440,6 +440,15 @@ private static ClientCallback boundaryCallback(HttpServerExchang }; } + private static boolean terminalCallback(HttpServerExchange exchange) { + if (exchange.isComplete() || Boolean.TRUE.equals(exchange.getAttachment(ABORTING))) return true; + if (!exchange.isResponseStarted()) return false; + LOG.warn("Proxy callback arrived after response commitment for {}; terminating the failed stream", exchange.getRequestPath()); + abortUpstream(exchange); + IoUtils.safeClose(exchange.getConnection()); + return true; + } + private final class ProxyClientHandler implements ProxyCallback, Runnable { private final ProxyContinuation continuation; private int tries; @@ -469,7 +478,7 @@ public void run() { @Override public void completed(final HttpServerExchange exchange, final ProxyConnection connection) { continuationTask(exchange, continuation, ex -> { - if (ex.isComplete() || ex.isResponseStarted() || Boolean.TRUE.equals(ex.getAttachment(ABORTING))) { + if (terminalCallback(ex)) { if (!connection.getConnection().isMultiplexingSupported()) IoUtils.safeClose(connection.getConnection()); return; } @@ -486,37 +495,24 @@ public void failed(final HttpServerExchange exchange) { } private void handleFailed(final HttpServerExchange exchange) { - if (exchange.isComplete() || exchange.isResponseStarted() - || Boolean.TRUE.equals(exchange.getAttachment(ABORTING))) return; - LOG.debug("Failed calling backend with tries = {} maxRetryAttempts = {} for exchange: {}", this.tries, this.maxRetryAttempts, exchange); - - final long time = System.currentTimeMillis(); - - if (this.tries++ < this.maxRetryAttempts) { - - if (this.timeout > 0 && time > this.timeout) { - LOG.trace("Current time = {} passes timeout {} for exchange: {}", time, this.timeout, exchange); - cancelRequest(exchange); - - } else { - this.target = proxyClient.findTarget(exchange); - LOG.trace("Retry target = {}", target); - - if (this.target != null) { - final long remaining = this.timeout > 0 ? this.timeout - time : -1; - LOG.trace("Retry with remaining = {} for exchange: {}", remaining, exchange); - proxyClient.getConnection(target, exchange, this, remaining, TimeUnit.MILLISECONDS); - - } else { - LOG.trace("Target is null, cannot resolve the backend"); - unresolvedBackend(exchange); // The context was registered when we started, so return 503 - } - } - - } else { - LOG.trace("Max number fo retry attempts reached for exchange: {}", exchange); + if (terminalCallback(exchange)) return; + LOG.debug("Failed calling backend with tries = {} maxRetryAttempts = {} for exchange: {}", tries, maxRetryAttempts, exchange); + if (tries++ >= maxRetryAttempts) { unresolvedBackend(exchange); + return; + } + long time = System.currentTimeMillis(); + if (timeout > 0 && time > timeout) { + cancelRequest(exchange); + return; + } + target = proxyClient.findTarget(exchange); + if (target == null) { + unresolvedBackend(exchange); + return; } + long remaining = timeout > 0 ? timeout - time : -1; + proxyClient.getConnection(target, exchange, this, remaining, TimeUnit.MILLISECONDS); } @Override diff --git a/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java b/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java index cd51eba83b..fb90ad3fa5 100644 --- a/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java +++ b/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java @@ -359,7 +359,11 @@ else if (exchange.getRequestPath().equals("/fail")) exchange.dispatch(() -> { var healthy = http.sendAsync(HttpRequest.newBuilder(front.resolve("/healthy")).timeout(Duration.ofSeconds(10)).build(), HttpResponse.BodyHandlers.ofString()); assertTrue(healthyEntered.await(5, TimeUnit.SECONDS)); - assertThrows(IOException.class, () -> http.send(HttpRequest.newBuilder(front.resolve("/fail")) + if (fatal && !committed) { + var failed = http.send(HttpRequest.newBuilder(front.resolve("/fail")).timeout(Duration.ofSeconds(5)).build(), + HttpResponse.BodyHandlers.ofString()); + assertEquals(500, failed.statusCode()); + } else assertThrows(IOException.class, () -> http.send(HttpRequest.newBuilder(front.resolve("/fail")) .timeout(Duration.ofSeconds(5)).build(), HttpResponse.BodyHandlers.ofString())); if (fatal) assertSame(sentinel, propagated.get(3, TimeUnit.SECONDS)); assertFalse(healthy.isDone()); @@ -371,6 +375,91 @@ else if (exchange.getRequestPath().equals("/fail")) exchange.dispatch(() -> { } finally { release.countDown(); } } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void fatalHttp11FailureTerminatesRequestAndPropagatesOriginal(boolean committed) throws Exception { + Error fatal = new StackOverflowError("HTTP/1.1 fatal sentinel"); + CompletableFuture propagated = new CompletableFuture<>(); + try (Fixture fixture = new Fixture(); HttpClient http = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build()) { + URI front = fixture.start(exchange -> { + if (exchange.getRequestPath().equals("/after")) { + exchange.getResponseSender().send("accepted"); + return; + } + exchange.dispatch(() -> { + try { + if (committed) { + var channel = exchange.getResponseChannel(); + Channels.writeBlocking(channel, ByteBuffer.wrap("first".getBytes(StandardCharsets.UTF_8))); + Channels.flushBlocking(channel); + } + HandlerContinuation.task(exchange, ex -> { throw fatal; }).run(); + } catch (Error error) { propagated.complete(error); } + catch (IOException e) { propagated.completeExceptionally(e); } + }); + }); + var request = HttpRequest.newBuilder(front).timeout(Duration.ofSeconds(5)).build(); + if (committed) assertThrows(IOException.class, () -> http.send(request, HttpResponse.BodyHandlers.ofString())); + else assertEquals(500, http.send(request, HttpResponse.BodyHandlers.ofString()).statusCode()); + assertSame(fatal, propagated.get(3, TimeUnit.SECONDS)); + assertEquals("accepted", http.send(HttpRequest.newBuilder(front.resolve("/after")).build(), + HttpResponse.BodyHandlers.ofString()).body()); + } + } + + @ParameterizedTest + @ValueSource(strings = {"completed", "failed", "queued"}) + void callbackAfterCommitTerminatesIncompleteStreamWithoutForwarding(String kind) throws Exception { + CompletableFuture late = new CompletableFuture<>(); + AtomicInteger acquisitions = new AtomicInteger(); + var closed = new java.util.concurrent.atomic.AtomicBoolean(); + ClientConnection unusedConnection = (ClientConnection) Proxy.newProxyInstance(ClientConnection.class.getClassLoader(), + new Class[]{ClientConnection.class}, (proxy, method, args) -> switch (method.getName()) { + case "isMultiplexingSupported" -> false; + case "close" -> { closed.set(true); yield null; } + case "isOpen" -> !closed.get(); + case "toString" -> "unused late connection"; + default -> throw new AssertionError("A late callback must not start a ProxyAction: " + method.getName()); + }); + ProxyClient client = new ProxyClient() { + @Override public ProxyTarget findTarget(HttpServerExchange exchange) { return new ProxyTarget() {}; } + @Override public void getConnection(ProxyTarget target, HttpServerExchange exchange, + ProxyCallback callback, long timeout, TimeUnit unit) { + acquisitions.incrementAndGet(); + late.complete(() -> { + assertTrue(exchange.isResponseStarted()); + assertFalse(exchange.isComplete()); + switch (kind) { + case "completed" -> callback.completed(exchange, new ProxyConnection(unusedConnection, "/")); + case "failed" -> callback.failed(exchange); + case "queued" -> callback.queuedRequestFailed(exchange); + default -> throw new AssertionError(kind); + } + }); + } + }; + ProxyHandler proxy = ProxyHandler.builder().setProxyClient(client).setMaxRequestTime(0).build(); + try (Fixture fixture = new Fixture()) { + URI front = fixture.start(exchange -> exchange.dispatch(worker -> { + proxy.handleRequest(worker); + worker.startBlocking(); + worker.getOutputStream().write("first".getBytes(StandardCharsets.UTF_8)); + worker.getOutputStream().flush(); + })); + try (Socket socket = new Socket(front.getHost(), front.getPort())) { + socket.setSoTimeout(5000); + socket.getOutputStream().write("GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n" + .getBytes(StandardCharsets.UTF_8)); + late.get(3, TimeUnit.SECONDS).run(); + String raw = new String(socket.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + assertTrue(raw.startsWith("HTTP/1.1 200"), raw); + assertTrue(raw.contains("first"), raw); + assertEquals(1, acquisitions.get()); + if (kind.equals("completed")) assertTrue(closed.get()); + } + } + } + private static ClientConnection decorateConnection(ClientConnection real, Phase phase, java.util.function.Supplier failure) { return (ClientConnection) Proxy.newProxyInstance(ClientConnection.class.getClassLoader(), @@ -412,8 +501,8 @@ private static final class ThrowingMetrics extends AbstractMetricsHandler { private final java.util.function.Supplier failure; ThrowingMetrics(java.util.function.Supplier failure) { this.failure = failure; } @Override public void injectMetrics(HttpServerExchange exchange, long startTime, String name, String endpoint) { throw failure.get(); } - @Override protected void createMetricsReporter(TimeSeriesDbSender sender, MetricsConfig config) { } - @Override public void handleRequest(HttpServerExchange exchange) { } + @Override protected void createMetricsReporter(TimeSeriesDbSender sender, MetricsConfig config) { /* No reporter in this fixture. */ } + @Override public void handleRequest(HttpServerExchange exchange) { /* Lookup-only metrics fixture. */ } @Override public HttpHandler getNext() { return null; } @Override public MiddlewareHandler setNext(HttpHandler next) { return this; } } From fca536f6df46a6b83413b029b246746bf485c2a7 Mon Sep 17 00:00:00 2001 From: Steve Hu Date: Fri, 30 Oct 2026 13:40:15 -0400 Subject: [PATCH 07/15] Isolate continuation state and guard asynchronous failure cleanup --- README.md | 21 +- .../networknt/exception/ExceptionHandler.java | 2 - .../exception/ExceptionContinuationTest.java | 69 ++--- handler/qualification/MdcAllocationProbe.java | 57 +++++ handler/qualification/README.md | 23 ++ handler/qualification/allocation.json | 115 +++++++++ handler/qualification/mutation-control.txt | 9 + handler/qualification/run-allocation.py | 33 +++ .../handler/HandlerContinuation.java | 161 ++++++------ .../networknt/handler/MiddlewareHandler.java | 1 - .../networknt/handler/RequestMdcContext.java | 27 +- .../handler/HandlerContinuationTest.java | 241 ++++++++++++------ .../com/networknt/handler/ProxyHandler.java | 155 ++++++----- .../handler/ProxyContinuationTest.java | 240 ++++++++++++++--- 14 files changed, 826 insertions(+), 328 deletions(-) create mode 100644 handler/qualification/MdcAllocationProbe.java create mode 100644 handler/qualification/README.md create mode 100644 handler/qualification/allocation.json create mode 100644 handler/qualification/mutation-control.txt create mode 100644 handler/qualification/run-allocation.py diff --git a/README.md b/README.md index 97c635e90a..d09b950b9c 100644 --- a/README.md +++ b/README.md @@ -260,15 +260,18 @@ changing its next handler. Named handlers are retained; when exception handling is disabled and omitted from the chain, failures remain under Undertow's default policy. Nested boundaries restore their enclosing scope, and state is isolated per exchange. Capturing a boundary does not snapshot thread-local MDC. At resume, -one scope installs the canonical MDC_CONTEXT attachment (or an empty context when +one scope reads the directly captured request holder (or an empty context when absent), then restores the executing thread's previous MDC on every exit. Nested frames reuse that scope. Policy identity and exchange identity share one frame tracker; proxy callbacks do not maintain a second thread-local guard. Framework MDC updates use one stable RequestMdcContext holder with an atomic immutable-map -reference. Capture and the initial ExceptionHandler frame initialize that holder -before any asynchronous hand-off; addHandlerMDCContext can also initialize it on +reference. The first boundary execution or capture initializes that holder +before its asynchronous hand-off; addHandlerMDCContext can also initialize it on first serial use. Later updates replace only the holder's snapshot, never its -attachment. Custom asynchronous code must initialize/capture before hand-off. +attachment. Captured frames and their MDC helper updates reuse the holder directly, +without attachment lookups. Active exception policy is thread-scoped and captured +by value; overlapping frames never rewrite a policy attachment. Custom asynchronous +code must initialize/capture before hand-off. The Map-typed attachment key remains compatible with read-only consumers; direct attachment replacement or mutation is unsupported after initialization. Proxy callbacks use this scope rather than the unscoped @@ -289,7 +292,15 @@ completed exchanges do not rewrite a response or close a reused connection. The neutral Runnable adapter attempts a 500 before commitment or a stream reset after commitment before propagating the original Error. Failed writes/resets are suppressed on that Error and cannot replace it; selected exception policies keep -their existing handling behavior. +their existing handling behavior. Ordinary exceptions use the same guarded cleanup. +Proxy callbacks serialize on private per-request state rather than mutating +connection, cancellation or timeout attachments. The legacy public `CONNECTION` +key remains declared but is no longer populated; external consumers of that +implementation attachment need compatibility review before upgrade. A proxy-generated terminal +response is marked before sending, so benign late callbacks leave it draining. +Unexpected committed-response callbacks reset only the affected HTTP/2 stream. +Allocation samples and a reproducible local probe are in +[handler/qualification](handler/qualification/README.md). Custom asynchronous handlers must explicitly capture before registering their callbacks; this contract cannot intercept arbitrary third-party hand-offs. Body-reading requests capture lazily only when a read suspends. ingress-proxy uses diff --git a/exception/src/main/java/com/networknt/exception/ExceptionHandler.java b/exception/src/main/java/com/networknt/exception/ExceptionHandler.java index 24d4118be3..9b77fb6e11 100644 --- a/exception/src/main/java/com/networknt/exception/ExceptionHandler.java +++ b/exception/src/main/java/com/networknt/exception/ExceptionHandler.java @@ -19,7 +19,6 @@ import com.networknt.handler.Handler; import com.networknt.handler.HandlerContinuation; import com.networknt.handler.MiddlewareHandler; -import com.networknt.handler.RequestMdcContext; import io.undertow.Handlers; import io.undertow.server.HttpHandler; import io.undertow.server.HttpServerExchange; @@ -72,7 +71,6 @@ public ExceptionHandler(String configName) { @Override public void handleRequest(final HttpServerExchange exchange) throws Exception { - RequestMdcContext.initialize(exchange); // dispatch here to make sure that all exceptions will be capture in this handler // otherwise, some of the exceptions will be captured in Connectors class in Undertow // As we've updated Server.java to redirect the logs to slf4j but still it make sense diff --git a/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java b/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java index 353da3e8e0..ca4696d2fb 100644 --- a/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java +++ b/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java @@ -1,22 +1,31 @@ package com.networknt.exception; + import com.networknt.config.Config; import com.networknt.handler.HandlerContinuation; import io.undertow.Undertow; import io.undertow.server.HttpHandler; import io.undertow.server.HttpServerExchange; import io.undertow.util.HttpString; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.slf4j.MDC; import java.io.IOException; import java.net.InetSocketAddress; import java.net.URI; import java.net.http.*; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.*; @@ -44,26 +53,25 @@ void selectedPolicyHandlesInitialAndResumedFailures(String name, boolean async, }; HttpHandler failing = exchange -> { if (async) { - assertEquals(hasMdc ? "canonical" : null, org.slf4j.MDC.get("request")); - assertNull(org.slf4j.MDC.get("executor")); + assertEquals(hasMdc ? "canonical" : null, MDC.get("request")); + assertNull(MDC.get("executor")); } if (runtime) throw new IllegalStateException("runtime failure"); throw new IOException("checked failure"); }; - var resumedMdc = new java.util.concurrent.atomic.AtomicReference(); - var resumedDone = new java.util.concurrent.CountDownLatch(1); + var resumedMdc = new AtomicReference(); + var resumedDone = new CountDownLatch(1); HttpHandler work = exchange -> { - if (hasMdc) exchange.putAttachment(com.networknt.httpstring.AttachmentConstants.MDC_CONTEXT, - Map.of("request", "canonical")); + if (hasMdc) selected.addHandlerMDCContext(exchange, "request", "canonical"); if (async) { HttpHandler resumed = HandlerContinuation.capture(exchange, failing); exchange.dispatch(() -> { - org.slf4j.MDC.put("executor", "owns-context"); + MDC.put("executor", "owns-context"); try { HandlerContinuation.task(exchange, resumed).run(); - resumedMdc.set(org.slf4j.MDC.get("executor")); + resumedMdc.set(MDC.get("executor")); } finally { - org.slf4j.MDC.clear(); + MDC.clear(); resumedDone.countDown(); } }); @@ -82,7 +90,7 @@ void selectedPolicyHandlesInitialAndResumedFailures(String name, boolean async, assertEquals(enabled ? name : "", response.headers().firstValue("X-Selected").orElse("")); assertEquals(enabled, response.body().contains(runtime ? "ERR10010" : "ERR10011"), response.body()); if (async) { - assertTrue(resumedDone.await(3, java.util.concurrent.TimeUnit.SECONDS)); + assertTrue(resumedDone.await(3, TimeUnit.SECONDS)); assertEquals("owns-context", resumedMdc.get(), "Resumed exception handling must leave executor MDC alone"); } } finally { @@ -92,19 +100,19 @@ void selectedPolicyHandlesInitialAndResumedFailures(String name, boolean async, } } @ParameterizedTest - @org.junit.jupiter.params.provider.ValueSource(booleans = {false, true}) + @ValueSource(booleans = {false, true}) void committedResponseIsNotTruncatedByInitialOrResumedBoundary(boolean resumed) throws Exception { - var release = new java.util.concurrent.CountDownLatch(1); + var release = new CountDownLatch(1); ExceptionHandler selected = new ExceptionHandler(); HttpHandler streaming = exchange -> { exchange.startBlocking(); var output = exchange.getOutputStream(); - output.write("first".getBytes(java.nio.charset.StandardCharsets.UTF_8)); + output.write("first".getBytes(StandardCharsets.UTF_8)); output.flush(); exchange.dispatch(() -> { try { - assertTrue(release.await(5, java.util.concurrent.TimeUnit.SECONDS)); - output.write("second".getBytes(java.nio.charset.StandardCharsets.UTF_8)); + assertTrue(release.await(5, TimeUnit.SECONDS)); + output.write("second".getBytes(StandardCharsets.UTF_8)); output.close(); } catch (Exception e) { throw new RuntimeException(e); } }); @@ -122,9 +130,9 @@ void committedResponseIsNotTruncatedByInitialOrResumedBoundary(boolean resumed) .timeout(Duration.ofSeconds(8)).build(), HttpResponse.BodyHandlers.ofInputStream()); assertEquals(200, response.statusCode()); try (var body = response.body()) { - assertEquals("first", new String(body.readNBytes(5), java.nio.charset.StandardCharsets.UTF_8)); + assertEquals("first", new String(body.readNBytes(5), StandardCharsets.UTF_8)); release.countDown(); - assertEquals("second", new String(body.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8)); + assertEquals("second", new String(body.readAllBytes(), StandardCharsets.UTF_8)); } } finally { release.countDown(); @@ -132,11 +140,11 @@ void committedResponseIsNotTruncatedByInitialOrResumedBoundary(boolean resumed) } } - @org.junit.jupiter.api.Test + @Test void lateResumedFailureDoesNotRewriteCompletedResponseOrCloseKeepAliveConnection() throws Exception { - var saved = new java.util.concurrent.atomic.AtomicReference(); - var completed = new java.util.concurrent.atomic.AtomicReference(); - var done = new java.util.concurrent.CountDownLatch(1); + var saved = new AtomicReference(); + var completed = new AtomicReference(); + var done = new CountDownLatch(1); ExceptionHandler selected = new ExceptionHandler(); selected.setNext(exchange -> { saved.set(HandlerContinuation.capture(exchange, ex -> { throw new IOException("late failure"); })); @@ -153,7 +161,7 @@ void lateResumedFailureDoesNotRewriteCompletedResponseOrCloseKeepAliveConnection int port = ((InetSocketAddress) server.getListenerInfo().getFirst().getAddress()).getPort(); var request = HttpRequest.newBuilder(URI.create("http://127.0.0.1:" + port)).timeout(Duration.ofSeconds(5)).build(); assertEquals("accepted", client.send(request, HttpResponse.BodyHandlers.ofString()).body()); - assertTrue(done.await(3, java.util.concurrent.TimeUnit.SECONDS)); + assertTrue(done.await(3, TimeUnit.SECONDS)); assertTrue(completed.get().isComplete()); saved.get().handleRequest(completed.get()); assertTrue(completed.get().getConnection().isOpen()); @@ -162,10 +170,9 @@ void lateResumedFailureDoesNotRewriteCompletedResponseOrCloseKeepAliveConnection } @ParameterizedTest - @org.junit.jupiter.params.provider.ValueSource(booleans = {false, true}) + @ValueSource(booleans = {false, true}) void installedBoundaryRetainsExistingFatalErrorMappingOnHttp11(boolean resumed) throws Exception { - var initializedOnIoThread = new java.util.concurrent.atomic.AtomicBoolean(); - var calls = new java.util.concurrent.atomic.AtomicInteger(); + var calls = new AtomicInteger(); Error sentinel = new StackOverflowError("installed boundary sentinel"); ExceptionHandler selected = new ExceptionHandler(); HttpHandler failing = exchange -> { calls.incrementAndGet(); throw sentinel; }; @@ -174,12 +181,7 @@ void installedBoundaryRetainsExistingFatalErrorMappingOnHttp11(boolean resumed) else if (resumed) exchange.dispatch(HandlerContinuation.capture(exchange, failing)); else failing.handleRequest(exchange); }); - Undertow server = Undertow.builder().addHttpListener(0, "127.0.0.1").setHandler(exchange -> { - selected.handleRequest(exchange); - initializedOnIoThread.set(exchange.isInIoThread() - && exchange.getAttachment(com.networknt.httpstring.AttachmentConstants.MDC_CONTEXT) - instanceof com.networknt.handler.RequestMdcContext); - }).build(); + Undertow server = Undertow.builder().addHttpListener(0, "127.0.0.1").setHandler(selected).build(); try (HttpClient client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build()) { server.start(); int port = ((InetSocketAddress) server.getListenerInfo().getFirst().getAddress()).getPort(); @@ -189,7 +191,6 @@ void installedBoundaryRetainsExistingFatalErrorMappingOnHttp11(boolean resumed) assertEquals(400, response.statusCode(), response.body()); assertTrue(response.body().contains("ERR10011"), response.body()); assertEquals(1, calls.get()); - assertTrue(initializedOnIoThread.get(), "Publish the holder before initial worker dispatch"); assertEquals("accepted", client.send(HttpRequest.newBuilder(endpoint.resolve("/after")).build(), HttpResponse.BodyHandlers.ofString()).body()); } finally { server.stop(); } diff --git a/handler/qualification/MdcAllocationProbe.java b/handler/qualification/MdcAllocationProbe.java new file mode 100644 index 0000000000..ceca65c366 --- /dev/null +++ b/handler/qualification/MdcAllocationProbe.java @@ -0,0 +1,57 @@ +import com.networknt.handler.HandlerContinuation; +import com.networknt.httpstring.AttachmentConstants; +import io.undertow.server.HttpHandler; +import io.undertow.server.HttpServerExchange; +import org.slf4j.MDC; +import java.lang.management.ManagementFactory; +import java.util.*; + +/** Warmed allocation-only microprobe. Excludes registration, transport, logging and gateway throughput. */ +public class MdcAllocationProbe { + private static volatile int sink; + private static final int WARMUP = 100000; + private static final int ITERATIONS = 100000; + private static final com.sun.management.ThreadMXBean BEAN = + (com.sun.management.ThreadMXBean) ManagementFactory.getThreadMXBean(); + private static final long THREAD = Thread.currentThread().threadId(); + private static final HttpHandler CALLBACK = exchange -> sink++; + + private static void measure(String name, Runnable operation) { + for (int i = 0; i < WARMUP; i++) operation.run(); + for (int sample = 0; sample < 3; sample++) { + long before = BEAN.getThreadAllocatedBytes(THREAD); + for (int i = 0; i < ITERATIONS; i++) operation.run(); + long bytes = BEAN.getThreadAllocatedBytes(THREAD) - before; + System.out.printf(Locale.ROOT, + "ALLOC:{\"case\":\"%s\",\"sample\":%d,\"bytes_per_frame\":%.4f}%n", + name, sample, (double) bytes / ITERATIONS); + } + } + + private static void frame(String name, Map request, Map caller) { + HttpServerExchange exchange = new HttpServerExchange(null); + exchange.putAttachment(AttachmentConstants.MDC_CONTEXT, request); + Runnable task = HandlerContinuation.task(exchange, HandlerContinuation.capture(exchange, CALLBACK)); + MDC.clear(); + if (!caller.isEmpty()) MDC.setContextMap(caller); + measure(name, task); + Map restored = MDC.getCopyOfContextMap(); + if (!Objects.equals(caller, restored == null ? Map.of() : restored)) + throw new AssertionError("Caller MDC was not restored"); + } + + public static void main(String[] args) { + if (!BEAN.isThreadAllocatedMemorySupported()) throw new IllegalStateException("Allocation counter unavailable"); + BEAN.setThreadAllocatedMemoryEnabled(true); + measure("callback_only", () -> sink++); + frame("empty_request_empty_caller", Map.of(), Map.of()); + Map three = Map.of("correlation", "a", "trace", "b", "client", "c"); + frame("three_fields_matching_caller", three, three); + frame("three_fields_different_caller", three, Map.of("worker", "outside")); + frame("three_fields_different_caller_same_size", three, Map.of("worker", "outside", "other", "prior", "tenant", "outside")); + Map twelve = new HashMap<>(); + for (int i = 0; i < 12; i++) twelve.put("field" + i, "value" + i); + frame("twelve_fields_different_caller", twelve, Map.of("worker", "outside")); + MDC.clear(); + } +} diff --git a/handler/qualification/README.md b/handler/qualification/README.md new file mode 100644 index 0000000000..af9085509b --- /dev/null +++ b/handler/qualification/README.md @@ -0,0 +1,23 @@ +# Continuation allocation evidence for issue #2820 + +Build the focused tests first, then run from the repository root: + +```sh +mvn -pl handler -am test -Dtest=HandlerContinuationTest -Dsurefire.failIfNoSpecifiedTests=false +python3 handler/qualification/run-allocation.py +``` + +`allocation.json` records raw samples, warmed per-resume medians, JDK details and +source hashes. The standalone probe reuses one captured runnable and increments a +volatile counter so the callback cannot disappear. It verifies caller MDC restoration. +It does not measure capture/registration, transport, logging, monitor contention, +or end-to-end gateway throughput. JIT and machine variation affect the figures. + +A resume still snapshots the caller MDC and restores it on exit, including when +entry already matches the request map: the callback can change MDC. That restoration +is required for shared I/O/worker threads. Nested calls for the same policy reuse +the scope. Proxy request state caches its captured handler and runnable so callbacks +do not recreate those adapters. One frame is allocated per outer resume. + +These local figures quantify remaining allocation; they do not close a gateway +performance qualification gate or establish a zero-allocation path. diff --git a/handler/qualification/allocation.json b/handler/qualification/allocation.json new file mode 100644 index 0000000000..de320ff1fc --- /dev/null +++ b/handler/qualification/allocation.json @@ -0,0 +1,115 @@ +{ + "method": "ThreadMXBean warmed allocation probe: 100000 warmup, 100000 iterations per sample, three samples", + "exclusions": [ + "registration/capture", + "network transport", + "logging", + "gateway throughput" + ], + "java": "openjdk version \"25.0.3\" 2026-04-21 LTS\nOpenJDK Runtime Environment Corretto-25.0.3.9.1 (build 25.0.3+9-LTS)\nOpenJDK 64-Bit Server VM Corretto-25.0.3.9.1 (build 25.0.3+9-LTS, mixed mode, sharing)\n", + "samples": [ + { + "case": "callback_only", + "sample": 0, + "bytes_per_frame": 0.0105 + }, + { + "case": "callback_only", + "sample": 1, + "bytes_per_frame": 0.0 + }, + { + "case": "callback_only", + "sample": 2, + "bytes_per_frame": 0.0 + }, + { + "case": "empty_request_empty_caller", + "sample": 0, + "bytes_per_frame": 74.9799 + }, + { + "case": "empty_request_empty_caller", + "sample": 1, + "bytes_per_frame": 56.0 + }, + { + "case": "empty_request_empty_caller", + "sample": 2, + "bytes_per_frame": 56.0 + }, + { + "case": "three_fields_matching_caller", + "sample": 0, + "bytes_per_frame": 848.0 + }, + { + "case": "three_fields_matching_caller", + "sample": 1, + "bytes_per_frame": 848.0 + }, + { + "case": "three_fields_matching_caller", + "sample": 2, + "bytes_per_frame": 848.0 + }, + { + "case": "three_fields_different_caller", + "sample": 0, + "bytes_per_frame": 904.0 + }, + { + "case": "three_fields_different_caller", + "sample": 1, + "bytes_per_frame": 904.0 + }, + { + "case": "three_fields_different_caller", + "sample": 2, + "bytes_per_frame": 904.0 + }, + { + "case": "three_fields_different_caller_same_size", + "sample": 0, + "bytes_per_frame": 1184.0 + }, + { + "case": "three_fields_different_caller_same_size", + "sample": 1, + "bytes_per_frame": 1184.0 + }, + { + "case": "three_fields_different_caller_same_size", + "sample": 2, + "bytes_per_frame": 1184.0 + }, + { + "case": "twelve_fields_different_caller", + "sample": 0, + "bytes_per_frame": 1384.0 + }, + { + "case": "twelve_fields_different_caller", + "sample": 1, + "bytes_per_frame": 1384.0 + }, + { + "case": "twelve_fields_different_caller", + "sample": 2, + "bytes_per_frame": 1384.0 + } + ], + "median_bytes_per_resume": { + "callback_only": 0.0, + "empty_request_empty_caller": 56.0, + "three_fields_matching_caller": 848.0, + "three_fields_different_caller": 904.0, + "three_fields_different_caller_same_size": 1184.0, + "twelve_fields_different_caller": 1384.0 + }, + "source_sha256": { + "handler/src/main/java/com/networknt/handler/HandlerContinuation.java": "d4facd6530d2fe2de3e10962cfc91db7e25f73cde2a663542bb2e18be159e1ed", + "handler/src/main/java/com/networknt/handler/RequestMdcContext.java": "e9788732232232700d33a63ea24eeb0a6a52899dc0a5aa9730f0bf5b7b6692c5", + "proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java": "db186cd98d2e2579f32f71b7594b9706344489745a1332d9264a03d2f8b1e50c" + } +} diff --git a/handler/qualification/mutation-control.txt b/handler/qualification/mutation-control.txt new file mode 100644 index 0000000000..63d0a30b53 --- /dev/null +++ b/handler/qualification/mutation-control.txt @@ -0,0 +1,9 @@ +Negative control for productionResumesReadStableSnapshotsWhileWritersPublish: +Temporarily replace immutable CAS publication with one mutable HashMap and return +an unmodifiable view over that same backing map. Run the production resume/helper +reader-writer test. It fails deterministically after the held reader observes the +writer's publication: + +A retained generation must not change during publication ==> expected: <0> but was: <2000> + +The original source was restored byte-for-byte before the final passing runs. diff --git a/handler/qualification/run-allocation.py b/handler/qualification/run-allocation.py new file mode 100644 index 0000000000..5be950f2fb --- /dev/null +++ b/handler/qualification/run-allocation.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""Run after HandlerContinuationTest has built this worktree; JDK allocation counters required.""" +from pathlib import Path +import hashlib +import json +import statistics +import subprocess +import tempfile +import xml.etree.ElementTree as ET + +folder = Path(__file__).resolve().parent +root = folder.parents[1] +report = ET.parse(root / "handler/target/surefire-reports/TEST-com.networknt.handler.HandlerContinuationTest.xml") +classpath = next(p.attrib["value"] for p in report.findall(".//property") if p.attrib["name"] == "java.class.path") +with tempfile.TemporaryDirectory(prefix="continuation-allocation-") as output: + subprocess.run(["javac", "-cp", classpath, "-d", output, str(folder / "MdcAllocationProbe.java")], check=True) + result = subprocess.run(["java", "-Xms64m", "-Xmx256m", "-cp", output + ":" + classpath, "MdcAllocationProbe"], + check=True, text=True, capture_output=True) +samples = [json.loads(line[6:]) for line in result.stdout.splitlines() if line.startswith("ALLOC:")] +sources = ["handler/src/main/java/com/networknt/handler/HandlerContinuation.java", + "handler/src/main/java/com/networknt/handler/RequestMdcContext.java", + "proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java"] +data = { + "method": "ThreadMXBean warmed allocation probe: 100000 warmup, 100000 iterations per sample, three samples", + "exclusions": ["registration/capture", "network transport", "logging", "gateway throughput"], + "java": subprocess.run(["java", "-version"], text=True, capture_output=True, check=True).stderr, + "samples": samples, + "median_bytes_per_resume": {name: statistics.median(s["bytes_per_frame"] for s in samples if s["case"] == name) + for name in dict.fromkeys(s["case"] for s in samples)}, + "source_sha256": {name: hashlib.sha256((root / name).read_bytes()).hexdigest() for name in sources}, +} +(folder / "allocation.json").write_text(json.dumps(data, indent=2) + "\n") +print(json.dumps(data["median_bytes_per_resume"], indent=2)) diff --git a/handler/src/main/java/com/networknt/handler/HandlerContinuation.java b/handler/src/main/java/com/networknt/handler/HandlerContinuation.java index a292246cee..dbec69b710 100644 --- a/handler/src/main/java/com/networknt/handler/HandlerContinuation.java +++ b/handler/src/main/java/com/networknt/handler/HandlerContinuation.java @@ -2,34 +2,31 @@ import io.undertow.server.HttpHandler; import io.undertow.server.HttpServerExchange; -import io.undertow.util.AttachmentKey; import io.undertow.util.StatusCodes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; -import java.io.IOException; import java.util.Map; import java.util.Objects; import java.util.function.UnaryOperator; /** * Carries the active exception boundary across an asynchronous hand-off. Boundary - * state belongs to an exchange, never a thread or a shared middleware instance. + * policy is captured from the executing frame, never stored on shared middleware. * Capture before returning from the current chain frame; invoke the captured * handler through Undertow's dispatch or root-handler machinery. */ public final class HandlerContinuation { private static final Logger LOG = LoggerFactory.getLogger(HandlerContinuation.class); private static final Boundary UNHANDLED = (exchange, continuation) -> continuation.handleRequest(exchange); - private static final AttachmentKey ACTIVE = AttachmentKey.create(Boundary.class); - private record Frame(HttpServerExchange exchange, CapturedBoundary boundary) { } + private record Frame(HttpServerExchange exchange, CapturedBoundary boundary, boolean mdcScoped) { } private static final ThreadLocal CURRENT_FRAME = new ThreadLocal<>(); @FunctionalInterface private interface ScopedAction { void run() throws E; } - private record CapturedBoundary(Boundary boundary) implements UnaryOperator { + private record CapturedBoundary(Boundary boundary, RequestMdcContext context) implements UnaryOperator { @Override public HttpHandler apply(HttpHandler continuation) { return new CapturedHandler(this, continuation); } @@ -37,7 +34,7 @@ private record CapturedBoundary(Boundary boundary) implements UnaryOperator execute(exchange, policy.boundary(), continuation)); + withMdc(exchange, policy, () -> policy.boundary().handle(exchange, continuation)); } } @@ -65,14 +62,13 @@ public interface Boundary { */ public static void execute(HttpServerExchange exchange, Boundary boundary, HttpHandler continuation) throws Exception { - RequestMdcContext.initialize(exchange); - Boundary previous = exchange.putAttachment(ACTIVE, Objects.requireNonNull(boundary)); - try { - boundary.handle(exchange, continuation); - } finally { - if (previous == null) exchange.removeAttachment(ACTIVE); - else exchange.putAttachment(ACTIVE, previous); - } + Objects.requireNonNull(boundary); + Frame previous = CURRENT_FRAME.get(); + RequestMdcContext context = context(exchange); + CURRENT_FRAME.set(new Frame(exchange, new CapturedBoundary(boundary, context), + previous != null && previous.exchange() == exchange && previous.mdcScoped())); + try { boundary.handle(exchange, continuation); } + finally { restoreFrame(previous); } } /** @@ -87,114 +83,99 @@ public static HttpHandler capture(HttpServerExchange exchange, HttpHandler conti * Captures before registering a callback whose result determines its eventual * work. Apply the returned wrapper inside that callback. Framework hand-offs * must opt in; arbitrary third-party callbacks cannot be intercepted here. - * Captures only the exception policy. At resume, MDC comes from the canonical - * MDC_CONTEXT attachment, not from a captured thread-local snapshot. The + * Captures the exception policy and the stable request MDC holder. At resume, + * MDC comes from its current immutable generation, not a thread-local snapshot. The * executing thread's previous MDC is restored even on failure. Nested frames * for the same exchange reuse the active scope. */ public static UnaryOperator capture(HttpServerExchange exchange) { - RequestMdcContext.initialize(exchange); - Boundary active = exchange.getAttachment(ACTIVE); - Boundary boundary = active == null ? UNHANDLED : active; - return new CapturedBoundary(boundary); + Frame frame = CURRENT_FRAME.get(); + Boundary boundary = frame != null && frame.exchange() == exchange ? frame.boundary().boundary() : UNHANDLED; + return new CapturedBoundary(boundary, context(exchange)); + } + + // Resume paths use the captured holder, never the exchange attachment map. + static RequestMdcContext activeContext(HttpServerExchange exchange) { + Frame frame = CURRENT_FRAME.get(); + return frame != null && frame.exchange() == exchange ? frame.boundary().context() : null; + } + + private static RequestMdcContext context(HttpServerExchange exchange) { + RequestMdcContext active = activeContext(exchange); + return active != null ? active : RequestMdcContext.initialize(exchange); } private static void withMdc(HttpServerExchange exchange, CapturedBoundary policy, ScopedAction callback) throws E { Frame previousFrame = CURRENT_FRAME.get(); - if (previousFrame != null && previousFrame.exchange() == exchange) { - if (policy == null || previousFrame.boundary() == policy) callback.run(); - else { - CURRENT_FRAME.set(new Frame(exchange, policy)); - try { callback.run(); } - finally { CURRENT_FRAME.set(previousFrame); } - } + boolean reuseMdc = previousFrame != null && previousFrame.exchange() == exchange && previousFrame.mdcScoped(); + if (reuseMdc && previousFrame.boundary() == policy) { + callback.run(); return; } - Map previousMdc = MDC.getCopyOfContextMap(); - Map requestMdc = RequestMdcContext.read(exchange); - CURRENT_FRAME.set(new Frame(exchange, policy)); + Map previousMdc = reuseMdc ? null : MDC.getCopyOfContextMap(); + CURRENT_FRAME.set(new Frame(exchange, policy, true)); try { - if (!requestMdc.equals(previousMdc == null ? Map.of() : previousMdc)) setMdc(requestMdc); + if (!reuseMdc) { + Map requestMdc = policy.context().snapshot(); + if (!requestMdc.equals(previousMdc == null ? Map.of() : previousMdc)) setMdc(requestMdc); + } callback.run(); } finally { - setMdc(previousMdc); - if (previousFrame == null) CURRENT_FRAME.remove(); - else CURRENT_FRAME.set(previousFrame); + if (!reuseMdc) setMdc(previousMdc); + restoreFrame(previousFrame); } } + private static void restoreFrame(Frame previous) { + if (previous == null) CURRENT_FRAME.remove(); + else CURRENT_FRAME.set(previous); + } + /** * Adapts an already captured handler for a Runnable dispatch. Unlike a root * handler, a successful callback may leave upstream I/O pending; do not end * that exchange. Unhandled failures return 500 before commitment, close a * committed stream, or leave an already completed exchange alone. */ + @SuppressWarnings("java:S1181") // Fatal errors require best-effort request cleanup before rethrow. public static Runnable task(HttpServerExchange exchange, HttpHandler continuation) { - if (continuation instanceof CapturedHandler captured) { - HttpHandler action = ex -> execute(ex, captured.policy().boundary(), captured.continuation()); - return () -> withMdc(exchange, captured.policy(), () -> invokeTask(exchange, action)); - } - return () -> withMdc(exchange, null, () -> invokeTask(exchange, continuation)); - } - - private static void invokeTask(HttpServerExchange exchange, HttpHandler continuation) { - try { - continuation.handleRequest(exchange); - } catch (Error fatal) { - failFatal(exchange, fatal); - throw fatal; - } catch (Exception failure) { - failTask(exchange, failure); - } + HttpHandler captured = continuation instanceof CapturedHandler ? continuation : capture(exchange, continuation); + return () -> { + try { captured.handleRequest(exchange); } + catch (Error fatal) { + finishFailedRequest(exchange, fatal); + throw fatal; + } catch (Exception failure) { + finishFailedRequest(exchange, failure); + LOG.error("Unhandled continuation failure for {}", exchange.getRequestPath(), failure); + } + }; } - private static void failFatal(HttpServerExchange exchange, Error fatal) { + /** Preserve the original failure even if the response or stream cleanup fails. */ + @SuppressWarnings("java:S1181") // Error cleanup must not mask the original VM/application Error. + private static void finishFailedRequest(HttpServerExchange exchange, Throwable failure) { if (exchange.isComplete()) return; - if (exchange.isResponseStarted()) { - closeAfterFatal(exchange, fatal); - return; - } - try { - exchange.setStatusCode(StatusCodes.INTERNAL_SERVER_ERROR); - exchange.endExchange(); - } catch (Throwable writeFailure) { - rememberCleanupFailure(fatal, writeFailure); - closeAfterFatal(exchange, fatal); + if (!exchange.isResponseStarted()) { + try { + exchange.setStatusCode(StatusCodes.INTERNAL_SERVER_ERROR); + exchange.endExchange(); + return; + } catch (Throwable writeFailure) { rememberCleanupFailure(failure, writeFailure); } } - } - - private static void closeAfterFatal(HttpServerExchange exchange, Error fatal) { try { + // HTTP/2 ServerConnection.close() resets only this stream. if (exchange.getConnection() != null) exchange.getConnection().close(); - } catch (Throwable closeFailure) { - rememberCleanupFailure(fatal, closeFailure); - } + } catch (Throwable closeFailure) { rememberCleanupFailure(failure, closeFailure); } } - private static void rememberCleanupFailure(Error fatal, Throwable cleanupFailure) { - if (cleanupFailure == fatal) return; - try { fatal.addSuppressed(cleanupFailure); } + @SuppressWarnings("java:S1181") // Suppression metadata can itself fail under memory pressure. + private static void rememberCleanupFailure(Throwable failure, Throwable cleanupFailure) { + if (cleanupFailure == failure) return; + try { failure.addSuppressed(cleanupFailure); } catch (Throwable ignored) { - // Preserve the original Error even if the VM cannot allocate suppression metadata. - } - } - - private static void failTask(HttpServerExchange exchange, Exception failure) { - LOG.error("Unhandled continuation failure for {}", exchange.getRequestPath(), failure); - if (exchange.isComplete()) return; - if (exchange.isResponseStarted()) { - // Never replace committed headers or append an error to a stream. - try { - // HTTP/2 ServerConnection.close() resets this stream only. - exchange.getConnection().close(); - } catch (IOException closeFailure) { - failure.addSuppressed(closeFailure); - LOG.debug("Unable to close failed response stream", closeFailure); - } - } else { - exchange.setStatusCode(StatusCodes.INTERNAL_SERVER_ERROR); - exchange.endExchange(); + // Preserve the original failure even when suppression metadata cannot be allocated. } } diff --git a/handler/src/main/java/com/networknt/handler/MiddlewareHandler.java b/handler/src/main/java/com/networknt/handler/MiddlewareHandler.java index 05b5175178..894a34826b 100644 --- a/handler/src/main/java/com/networknt/handler/MiddlewareHandler.java +++ b/handler/src/main/java/com/networknt/handler/MiddlewareHandler.java @@ -16,7 +16,6 @@ package com.networknt.handler; -import com.networknt.httpstring.AttachmentConstants; import io.undertow.server.HttpHandler; import io.undertow.server.HttpServerExchange; diff --git a/handler/src/main/java/com/networknt/handler/RequestMdcContext.java b/handler/src/main/java/com/networknt/handler/RequestMdcContext.java index a3d19ef310..7d5638054f 100644 --- a/handler/src/main/java/com/networknt/handler/RequestMdcContext.java +++ b/handler/src/main/java/com/networknt/handler/RequestMdcContext.java @@ -25,11 +25,13 @@ private RequestMdcContext(Map initial) { } /** - * First use must be serial. Capture and ExceptionHandler call this before - * dispatch; repeated calls after publication do not write any attachment. + * First use must be serial. Boundary execution and capture initialize before + * registering asynchronous work; repeated calls do not write any attachment. * Legacy Map-valued context is migrated once in that serial frame. */ public static RequestMdcContext initialize(HttpServerExchange exchange) { + RequestMdcContext active = HandlerContinuation.activeContext(exchange); + if (active != null) return active; Map attached = exchange.getAttachment(AttachmentConstants.MDC_CONTEXT); if (attached instanceof RequestMdcContext holder) return holder; RequestMdcContext holder = new RequestMdcContext(attached); @@ -37,14 +39,8 @@ public static RequestMdcContext initialize(HttpServerExchange exchange) { return holder; } - static Map read(HttpServerExchange exchange) { - Map attached = exchange.getAttachment(AttachmentConstants.MDC_CONTEXT); - if (attached == null) return Map.of(); - if (attached instanceof RequestMdcContext holder) return holder.snapshot(); - throw new IllegalStateException("Initialize request MDC in the serial frame before asynchronous hand-off"); - } - - Map snapshot() { return current.get(); } + /** One immutable generation suitable for coherent iteration by external readers. */ + public Map snapshot() { return current.get(); } void putField(String field, String value) { current.updateAndGet(previous -> { @@ -54,15 +50,10 @@ void putField(String field, String value) { }); } - void putFields(Map fields) { - current.updateAndGet(previous -> { - Map next = new HashMap<>(previous); - next.putAll(fields); - return Collections.unmodifiableMap(next); - }); - } - @Override public String get(Object key) { return current.get().get(key); } + @Override public boolean containsKey(Object key) { return current.get().containsKey(key); } + @Override public boolean equals(Object other) { return other == this || current.get().equals(other); } + @Override public int hashCode() { return current.get().hashCode(); } @Override public int size() { return current.get().size(); } @Override public Set> entrySet() { return current.get().entrySet(); } @Override public void forEach(BiConsumer consumer) { diff --git a/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java b/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java index 7b6c2ae0fd..a97dd67d23 100644 --- a/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java +++ b/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java @@ -1,15 +1,28 @@ package com.networknt.handler; + +import com.networknt.httpstring.AttachmentConstants; import io.undertow.server.HttpHandler; import io.undertow.server.HttpServerExchange; +import io.undertow.server.ServerConnection; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.slf4j.MDC; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; class HandlerContinuationTest { @Test @@ -36,7 +49,7 @@ void activePolicyTracksIdentityAndExchangeAndRestoresAfterNestedFailure() throws @Test void mdcPublicationRetainsImmutableSnapshotsAndConcurrentUpdates() throws Exception { - HttpServerExchange exchange = org.mockito.Mockito.spy(new HttpServerExchange(null)); + HttpServerExchange exchange = spy(new HttpServerExchange(null)); MiddlewareHandler middleware = new MiddlewareHandler() { @Override public boolean isEnabled() { return true; } @Override public HttpHandler getNext() { return null; } @@ -45,65 +58,139 @@ void mdcPublicationRetainsImmutableSnapshotsAndConcurrentUpdates() throws Except }; middleware.addHandlerMDCContext(exchange, "first", "one"); var holder = assertInstanceOf(RequestMdcContext.class, - exchange.getAttachment(com.networknt.httpstring.AttachmentConstants.MDC_CONTEXT)); + exchange.getAttachment(AttachmentConstants.MDC_CONTEXT)); var first = holder.snapshot(); assertThrows(UnsupportedOperationException.class, () -> first.put("bad", "mutation")); - try (var workers = java.util.concurrent.Executors.newFixedThreadPool(4)) { + try (var workers = Executors.newFixedThreadPool(4)) { var updates = java.util.stream.IntStream.range(0, 20).mapToObj(i -> workers.submit(() -> middleware.addHandlerMDCContext(exchange, "field" + i, "value" + i))).toList(); for (var update : updates) update.get(); } - assertEquals(java.util.Map.of("first", "one"), first); + assertEquals(Map.of("first", "one"), first); assertEquals(21, holder.size()); - assertSame(holder, exchange.getAttachment(com.networknt.httpstring.AttachmentConstants.MDC_CONTEXT)); - org.mockito.Mockito.verify(exchange, org.mockito.Mockito.times(1)).putAttachment( - org.mockito.Mockito.eq(com.networknt.httpstring.AttachmentConstants.MDC_CONTEXT), org.mockito.Mockito.any()); + assertSame(holder, exchange.getAttachment(AttachmentConstants.MDC_CONTEXT)); + verify(exchange, times(1)).putAttachment( + eq(AttachmentConstants.MDC_CONTEXT), any()); } @Test - void readersIterateStableSnapshotsWhileWritersPublishToInitializedHolder() throws Exception { - HttpServerExchange exchange = org.mockito.Mockito.spy(new HttpServerExchange(null)); - RequestMdcContext holder = RequestMdcContext.initialize(exchange); - holder.putFields(java.util.Map.of("epoch", "0", "mirror", "0")); - var retained = holder.snapshot(); - var barrier = new java.util.concurrent.CyclicBarrier(4); - var writersDone = new java.util.concurrent.CountDownLatch(2); - var reads = new java.util.concurrent.atomic.AtomicInteger(); - try (var workers = java.util.concurrent.Executors.newFixedThreadPool(4)) { - List> tasks = new ArrayList<>(); - for (int writer = 0; writer < 2; writer++) { - int id = writer; - tasks.add(workers.submit(() -> { - try { - barrier.await(5, java.util.concurrent.TimeUnit.SECONDS); - for (int i = 1; i <= 2000; i++) { - String epoch = id + ":" + i; - holder.putFields(java.util.Map.of("epoch", epoch, "mirror", epoch)); - } - } finally { writersDone.countDown(); } - return null; - })); - } - for (int reader = 0; reader < 2; reader++) tasks.add(workers.submit(() -> { - barrier.await(5, java.util.concurrent.TimeUnit.SECONDS); - do { - var snapshot = holder.snapshot(); - var copy = new java.util.HashMap(); - snapshot.forEach(copy::put); - assertEquals(copy.get("epoch"), copy.get("mirror"), "Never iterate a torn generation"); - assertThrows(UnsupportedOperationException.class, () -> snapshot.put("bad", "mutation")); - reads.incrementAndGet(); - } while (writersDone.getCount() != 0); - return null; + void productionResumesReadStableSnapshotsWhileWritersPublish() throws Exception { + HttpServerExchange exchange = spy(new HttpServerExchange(null)); + exchange.putAttachment(AttachmentConstants.MDC_CONTEXT, Map.of("epoch", "0", "legacy", "retained")); + MiddlewareHandler middleware = publicationMiddleware(); + AtomicReference context = new AtomicReference<>(); + AtomicReference writer = new AtomicReference<>(); + AtomicReference reader = new AtomicReference<>(); + CountDownLatch readerHoldingSnapshot = new CountDownLatch(1); + CountDownLatch updated = new CountDownLatch(1); + HandlerContinuation.execute(exchange, (ex, next) -> next.handleRequest(ex), ex -> { + context.set(RequestMdcContext.initialize(ex)); + reader.set(HandlerContinuation.capture(ex, resumed -> { + assertEquals("retained", MDC.get("legacy")); + var generation = context.get().snapshot(); + var iterator = generation.entrySet().iterator(); + assertTrue(iterator.hasNext()); + var first = iterator.next(); + readerHoldingSnapshot.countDown(); + assertTrue(updated.await(5, TimeUnit.SECONDS)); + var copy = new HashMap(); + copy.put(first.getKey(), first.getValue()); + iterator.forEachRemaining(entry -> copy.put(entry.getKey(), entry.getValue())); + assertEquals("0", copy.get("epoch"), "A retained generation must not change during publication"); + assertThrows(UnsupportedOperationException.class, () -> generation.put("bad", "mutation")); + })); + writer.set(HandlerContinuation.capture(ex, resumed -> { + assertTrue(readerHoldingSnapshot.await(5, TimeUnit.SECONDS)); + for (int i = 1; i <= 2000; i++) middleware.addHandlerMDCContext(resumed, "epoch", Integer.toString(i)); + updated.countDown(); })); - for (var task : tasks) task.get(10, java.util.concurrent.TimeUnit.SECONDS); + }); + clearInvocations(exchange); + try (var workers = Executors.newFixedThreadPool(2)) { + var reading = workers.submit(() -> { reader.get().handleRequest(exchange); return null; }); + var writing = workers.submit(() -> { writer.get().handleRequest(exchange); return null; }); + reading.get(10, TimeUnit.SECONDS); + writing.get(10, TimeUnit.SECONDS); + } + // Both production resumes and helper updates reuse the captured holder without attachment access. + verify(exchange, never()).getAttachment(AttachmentConstants.MDC_CONTEXT); + verify(exchange, never()).putAttachment(eq(AttachmentConstants.MDC_CONTEXT), any()); + reader.set(HandlerContinuation.capture(exchange, resumed -> assertEquals("2000", MDC.get("epoch")))); + reader.get().handleRequest(exchange); + assertEquals("retained", context.get().snapshot().get("legacy")); + } + + private static MiddlewareHandler publicationMiddleware() { + return new MiddlewareHandler() { + @Override public boolean isEnabled() { return true; } + @Override public HttpHandler getNext() { return null; } + @Override public MiddlewareHandler setNext(HttpHandler next) { return this; } + @Override public void handleRequest(HttpServerExchange ex) { /* Publication-only fixture. */ } + }; + } + + @Test + void legacyContextInRawRunnableIsMigratedBeforeItRuns() throws Exception { + HttpServerExchange exchange = spy(new HttpServerExchange(null)); + exchange.putAttachment(AttachmentConstants.MDC_CONTEXT, Map.of("legacy", "value")); + var callback = HandlerContinuation.task(exchange, ex -> assertEquals("value", MDC.get("legacy"))); + clearInvocations(exchange); + try (var workers = Executors.newSingleThreadExecutor()) { workers.submit(callback).get(5, TimeUnit.SECONDS); } + verify(exchange, never()).getAttachment(AttachmentConstants.MDC_CONTEXT); + verify(exchange, never()).putAttachment(eq(AttachmentConstants.MDC_CONTEXT), any()); + } + + @Test + void exceptionCleanupPreservesFailureWhenStatusWriteAndResetThrow() throws Exception { + IOException original = new IOException("callback failed"); + RuntimeException writeFailure = new IllegalStateException("write failed"); + RuntimeException closeFailure = new IllegalStateException("close failed"); + var connection = mock(ServerConnection.class); + doThrow(closeFailure).when(connection).close(); + var exchange = spy(new HttpServerExchange(connection)); + doThrow(writeFailure).when(exchange).setStatusCode(500); + assertDoesNotThrow(() -> HandlerContinuation.task(exchange, ex -> { throw original; }).run()); + assertArrayEquals(new Throwable[]{writeFailure, closeFailure}, original.getSuppressed()); + verify(connection).close(); + } + + @Test + void overlappingFramesKeepTheirOwnPolicyWithoutAttachmentAccess() throws Exception { + var exchange = spy(new HttpServerExchange(null)); + var aEntered = new CountDownLatch(1); + var bEntered = new CountDownLatch(1); + var aCaptured = new CountDownLatch(1); + var callsA = new AtomicInteger(); + var callsB = new AtomicInteger(); + var a = new AtomicReference(); + var b = new AtomicReference(); + HandlerContinuation.execute(exchange, (ex, next) -> { callsA.incrementAndGet(); next.handleRequest(ex); }, ex -> + a.set(HandlerContinuation.capture(ex, resumed -> { + aEntered.countDown(); + assertTrue(bEntered.await(5, TimeUnit.SECONDS)); + HandlerContinuation.capture(resumed, child -> { /* Exercise the captured policy. */ }).handleRequest(resumed); + aCaptured.countDown(); + }))); + HandlerContinuation.execute(exchange, (ex, next) -> { callsB.incrementAndGet(); next.handleRequest(ex); }, ex -> + b.set(HandlerContinuation.capture(ex, resumed -> { + assertTrue(aEntered.await(5, TimeUnit.SECONDS)); + bEntered.countDown(); + assertTrue(aCaptured.await(5, TimeUnit.SECONDS)); + }))); + callsA.set(0); + callsB.set(0); + clearInvocations(exchange); + try (var workers = Executors.newFixedThreadPool(2)) { + var first = workers.submit(() -> { a.get().handleRequest(exchange); return null; }); + var second = workers.submit(() -> { b.get().handleRequest(exchange); return null; }); + first.get(10, TimeUnit.SECONDS); + second.get(10, TimeUnit.SECONDS); } - assertTrue(reads.get() >= 2); - assertEquals(java.util.Map.of("epoch", "0", "mirror", "0"), retained, - "A mutable HashMap implementation must fail this retained-generation assertion"); - assertSame(holder, exchange.getAttachment(com.networknt.httpstring.AttachmentConstants.MDC_CONTEXT)); - org.mockito.Mockito.verify(exchange, org.mockito.Mockito.times(1)).putAttachment( - org.mockito.Mockito.eq(com.networknt.httpstring.AttachmentConstants.MDC_CONTEXT), org.mockito.Mockito.any()); + assertEquals(2, callsA.get()); + assertEquals(1, callsB.get()); + verify(exchange, never()).getAttachment(any()); + verify(exchange, never()).putAttachment(any(), any()); + verify(exchange, never()).removeAttachment(any()); } @Test @@ -111,62 +198,62 @@ void failedFatalStatusWriteCannotReplaceOriginalError() throws Exception { Error fatal = new StackOverflowError("original"); RuntimeException writeFailure = new IllegalStateException("status write failed"); IOException closeFailure = new IOException("reset failed too"); - var connection = org.mockito.Mockito.mock(io.undertow.server.ServerConnection.class); - org.mockito.Mockito.doThrow(closeFailure).when(connection).close(); - HttpServerExchange exchange = org.mockito.Mockito.spy(new HttpServerExchange(connection)); - org.mockito.Mockito.doThrow(writeFailure).when(exchange).endExchange(); - Error observed = assertThrows(StackOverflowError.class, () -> - HandlerContinuation.task(exchange, ex -> { throw fatal; }).run()); + var connection = mock(ServerConnection.class); + doThrow(closeFailure).when(connection).close(); + HttpServerExchange exchange = spy(new HttpServerExchange(connection)); + doThrow(writeFailure).when(exchange).endExchange(); + Runnable callback = HandlerContinuation.task(exchange, ex -> { throw fatal; }); + Error observed = assertThrows(StackOverflowError.class, callback::run); assertSame(fatal, observed); assertEquals(500, exchange.getStatusCode()); assertArrayEquals(new Throwable[]{writeFailure, closeFailure}, fatal.getSuppressed()); - org.mockito.Mockito.verify(connection).close(); + verify(connection).close(); } - @org.junit.jupiter.params.ParameterizedTest - @org.junit.jupiter.params.provider.ValueSource(booleans = {false, true}) + @ParameterizedTest + @ValueSource(booleans = {false, true}) void resumeScopesCanonicalRequestMdcAndRestoresWorker(boolean present) throws Exception { - var original = org.slf4j.MDC.getCopyOfContextMap(); + var original = MDC.getCopyOfContextMap(); try { HttpServerExchange exchange = new HttpServerExchange(null); - if (present) exchange.putAttachment(com.networknt.httpstring.AttachmentConstants.MDC_CONTEXT, - java.util.Map.of("request", "canonical")); - org.slf4j.MDC.put("request", "stale-at-capture"); + if (present) exchange.putAttachment(AttachmentConstants.MDC_CONTEXT, + Map.of("request", "canonical")); + MDC.put("request", "stale-at-capture"); HttpHandler resumed = HandlerContinuation.capture(exchange, ex -> { - assertEquals(present ? "canonical" : null, org.slf4j.MDC.get("request")); - assertNull(org.slf4j.MDC.get("worker"), "Do not borrow the executing thread's MDC"); + assertEquals(present ? "canonical" : null, MDC.get("request")); + assertNull(MDC.get("worker"), "Do not borrow the executing thread's MDC"); HandlerContinuation.capture(ex, nested -> { - assertEquals(present ? "canonical" : null, org.slf4j.MDC.get("request")); + assertEquals(present ? "canonical" : null, MDC.get("request")); }).handleRequest(ex); throw new IOException("failure"); }); - org.slf4j.MDC.clear(); - org.slf4j.MDC.put("worker", "unrelated"); + MDC.clear(); + MDC.put("worker", "unrelated"); assertThrows(IOException.class, () -> resumed.handleRequest(exchange)); - assertEquals(java.util.Map.of("worker", "unrelated"), org.slf4j.MDC.getCopyOfContextMap()); + assertEquals(Map.of("worker", "unrelated"), MDC.getCopyOfContextMap()); } finally { - if (original == null) org.slf4j.MDC.clear(); - else org.slf4j.MDC.setContextMap(original); + if (original == null) MDC.clear(); + else MDC.setContextMap(original); } } @Test void runnableAdapterPropagatesFatalErrorsAndRestoresMdc() { - var original = org.slf4j.MDC.getCopyOfContextMap(); + var original = MDC.getCopyOfContextMap(); try { - HttpServerExchange exchange = org.mockito.Mockito.spy(new HttpServerExchange(null)); - org.mockito.Mockito.doReturn(exchange).when(exchange).endExchange(); - org.slf4j.MDC.put("worker", "prior"); + HttpServerExchange exchange = spy(new HttpServerExchange(null)); + doReturn(exchange).when(exchange).endExchange(); + MDC.put("worker", "prior"); Error error = new OutOfMemoryError("injected sentinel, no memory pressure"); Runnable task = HandlerContinuation.task(exchange, HandlerContinuation.capture(exchange, ex -> { throw error; })); assertSame(error, assertThrows(OutOfMemoryError.class, task::run)); - assertEquals("prior", org.slf4j.MDC.get("worker")); - org.mockito.Mockito.verify(exchange).endExchange(); + assertEquals("prior", MDC.get("worker")); + verify(exchange).endExchange(); assertEquals(500, exchange.getStatusCode()); } finally { - if (original == null) org.slf4j.MDC.clear(); - else org.slf4j.MDC.setContextMap(original); + if (original == null) MDC.clear(); + else MDC.setContextMap(original); } } diff --git a/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java b/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java index 7ad944a225..5def153373 100644 --- a/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java +++ b/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java @@ -109,7 +109,6 @@ public class ProxyHandler implements HttpHandler { public static final AttachmentKey CONNECTION = AttachmentKey.create(ProxyConnection.class); private static final AttachmentKey EXCHANGE = AttachmentKey.create(HttpServerExchange.class); - private static final AttachmentKey TIMEOUT_KEY = AttachmentKey.create(XnioExecutor.Key.class); private final ProxyClient proxyClient; private int maxRequestTime; @@ -200,7 +199,7 @@ public void handleRequest(final HttpServerExchange exchange) throws Exception { if (timeout > 0) { LOG.trace("Adding timeout to exchange of {} ms to exchange: {}", effectiveMaxRequestTime, exchange); final XnioExecutor.Key key = WorkerUtils.executeAfter(exchange.getIoThread(), () -> clientHandler.cancel(exchange), effectiveMaxRequestTime, TimeUnit.MILLISECONDS); - exchange.putAttachment(TIMEOUT_KEY, key); + clientHandler.continuation.timeoutKey = key; exchange.addExchangeCompleteListener((exchange1, nextListener) -> { LOG.trace("Exchange completed for exchange removing timeout key: {}", exchange1); key.remove(); @@ -349,14 +348,6 @@ private static void removeHeaders(HeaderMap headers, List headerName } } - private static void cancelExchangeTimeout(HttpServerExchange exchange) { - XnioExecutor.Key key = exchange.getAttachment(TIMEOUT_KEY); - if (key != null) { - key.remove(); - exchange.removeAttachment(TIMEOUT_KEY); - } - } - @Override public String toString() { var proxyTargets = proxyClient.getAllTargets(); @@ -377,29 +368,48 @@ public String toString() { } } - /** One immutable policy snapshot, reused by all callbacks. MDC is scoped by capture(). */ - private record ProxyContinuation(UnaryOperator boundary) { } - private static final AttachmentKey CLIENT_EXCHANGE = AttachmentKey.create(ClientExchange.class); - private static final AttachmentKey ABORTING = AttachmentKey.create(Boolean.class); + /** State is created serially, captured directly, and serialized by its own monitor. */ + private static final class ProxyContinuation { + private final UnaryOperator boundary; + private final Runnable resumed; + private Consumer callback; + private ProxyConnection connection; + private ClientExchange request; + private boolean stopping; + private volatile XnioExecutor.Key timeoutKey; + + private ProxyContinuation(HttpServerExchange exchange, UnaryOperator boundary) { + this.boundary = boundary; + this.resumed = HandlerContinuation.task(exchange, boundary.apply(ex -> { + try { callback.accept(ex); } + catch (Exception | Error failure) { + abortUpstream(this); + throw failure; + } + })); + } - /** Each asynchronous entry owns one boundary/MDC scope; synchronous internal calls reuse it. */ - private static Runnable continuationTask(HttpServerExchange exchange, - ProxyContinuation continuation, - Consumer callback) { - return () -> { - if (HandlerContinuation.isActive(exchange, continuation.boundary())) { + private synchronized void run(HttpServerExchange exchange, Consumer callback) { + if (HandlerContinuation.isActive(exchange, boundary)) { callback.accept(exchange); return; } - HandlerContinuation.task(exchange, continuation.boundary().apply(ex -> { - try { - callback.accept(ex); - } catch (Exception | Error failure) { - abortUpstream(ex); - throw failure; - } - })).run(); - }; + this.callback = callback; + try { resumed.run(); } + finally { this.callback = null; } + } + + private void cancelTimeout() { + XnioExecutor.Key key = timeoutKey; + if (key != null) key.remove(); + } + } + + /** Each asynchronous entry owns one boundary/MDC scope; internal calls reuse it. */ + private static Runnable continuationTask(HttpServerExchange exchange, + ProxyContinuation continuation, + Consumer callback) { + return () -> continuation.run(exchange, callback); } private static void cancelClientExchange(ClientExchange request) { @@ -409,12 +419,13 @@ private static void cancelClientExchange(ClientExchange request) { } else IoUtils.safeClose(request.getConnection()); } - private static void abortUpstream(HttpServerExchange exchange) { - exchange.putAttachment(ABORTING, Boolean.TRUE); - ClientExchange request = exchange.removeAttachment(CLIENT_EXCHANGE); + private static void abortUpstream(ProxyContinuation state) { + state.stopping = true; + ClientExchange request = state.request; + state.request = null; if (request != null) cancelClientExchange(request); else { - ProxyConnection connection = exchange.getAttachment(CONNECTION); + ProxyConnection connection = state.connection; if (connection != null && !connection.getConnection().isMultiplexingSupported()) IoUtils.safeClose(connection.getConnection()); } @@ -424,27 +435,29 @@ private static ClientCallback boundaryCallback(HttpServerExchang ProxyContinuation continuation, ClientCallback callback) { return new ClientCallback<>() { @Override public void completed(ClientExchange result) { - if (exchange.isComplete() || Boolean.TRUE.equals(exchange.getAttachment(ABORTING))) { - cancelClientExchange(result); - return; - } continuationTask(exchange, continuation, ex -> { - ex.putAttachment(CLIENT_EXCHANGE, result); + if (ex.isComplete() || continuation.stopping) { + cancelClientExchange(result); + return; + } + continuation.request = result; callback.completed(result); }).run(); } @Override public void failed(IOException failure) { - if (!exchange.isComplete() && !Boolean.TRUE.equals(exchange.getAttachment(ABORTING))) - continuationTask(exchange, continuation, ex -> callback.failed(failure)).run(); + continuationTask(exchange, continuation, ex -> { + if (!ex.isComplete() && !continuation.stopping) callback.failed(failure); + }).run(); } }; } - private static boolean terminalCallback(HttpServerExchange exchange) { - if (exchange.isComplete() || Boolean.TRUE.equals(exchange.getAttachment(ABORTING))) return true; + /** Ignore callbacks owned by a terminal response; terminate only unexpected commitment. */ + private static boolean stopCallbackIfTerminal(HttpServerExchange exchange, ProxyContinuation state) { + if (exchange.isComplete() || state.stopping) return true; if (!exchange.isResponseStarted()) return false; - LOG.warn("Proxy callback arrived after response commitment for {}; terminating the failed stream", exchange.getRequestPath()); - abortUpstream(exchange); + LOG.warn("Unexpected proxy callback after response commitment for {}; terminating the stream", exchange.getRequestPath()); + abortUpstream(state); IoUtils.safeClose(exchange.getConnection()); return true; } @@ -461,7 +474,7 @@ private final class ProxyClientHandler implements ProxyCallback ProxyClientHandler(HttpServerExchange exchange, ProxyClient.ProxyTarget target, long timeout, int maxRetryAttempts, Predicate idempotentPredicate, boolean streamingRequest) { this.exchange = exchange; - this.continuation = new ProxyContinuation(HandlerContinuation.capture(exchange)); + this.continuation = new ProxyContinuation(exchange, HandlerContinuation.capture(exchange)); this.timeout = timeout; this.maxRetryAttempts = maxRetryAttempts; this.target = target; @@ -471,21 +484,22 @@ private final class ProxyClientHandler implements ProxyCallback @Override public void run() { - continuationTask(exchange, continuation, ex -> - proxyClient.getConnection(this.target, ex, this, -1, TimeUnit.MILLISECONDS)).run(); + continuationTask(exchange, continuation, ex -> { + if (!stopCallbackIfTerminal(ex, continuation)) + proxyClient.getConnection(this.target, ex, this, -1, TimeUnit.MILLISECONDS); + }).run(); } @Override public void completed(final HttpServerExchange exchange, final ProxyConnection connection) { continuationTask(exchange, continuation, ex -> { - if (terminalCallback(ex)) { + if (stopCallbackIfTerminal(ex, continuation)) { if (!connection.getConnection().isMultiplexingSupported()) IoUtils.safeClose(connection.getConnection()); return; } LOG.trace("ProxyCallback completed for exchange: {}", ex); - ex.putAttachment(CONNECTION, connection); - continuationTask(ex, continuation, resumed -> - new ProxyAction(connection, resumed, requestHeaders, rewriteHostHeader, reuseXForwarded, resumed.isRequestComplete() ? this : null, idempotentPredicate, urlRewriteRules, methodRewriteRules, queryParamRewriteRules, headerRewriteRules, streamingRequest, streamResponseContentTypes, streamIdleTimeout, streamResponseHeaderOverwrite, continuation).run()).run(); + continuation.connection = connection; + new ProxyAction(connection, ex, requestHeaders, rewriteHostHeader, reuseXForwarded, ex.isRequestComplete() ? this : null, idempotentPredicate, urlRewriteRules, methodRewriteRules, queryParamRewriteRules, headerRewriteRules, streamingRequest, streamResponseContentTypes, streamIdleTimeout, streamResponseHeaderOverwrite, continuation).run(); }).run(); } @@ -495,7 +509,7 @@ public void failed(final HttpServerExchange exchange) { } private void handleFailed(final HttpServerExchange exchange) { - if (terminalCallback(exchange)) return; + if (stopCallbackIfTerminal(exchange, continuation)) return; LOG.debug("Failed calling backend with tries = {} maxRetryAttempts = {} for exchange: {}", tries, maxRetryAttempts, exchange); if (tries++ >= maxRetryAttempts) { unresolvedBackend(exchange); @@ -526,6 +540,8 @@ public void couldNotResolveBackend(HttpServerExchange ex) { } private void unresolvedBackend(HttpServerExchange ex) { + if (ex.isComplete() || continuation.stopping) return; + abortUpstream(continuation); if (ex.isResponseStarted()) IoUtils.safeClose(ex.getConnection()); @@ -549,14 +565,14 @@ void cancel(final HttpServerExchange exchange) { } private void cancelRequest(final HttpServerExchange exchange) { - exchange.putAttachment(ABORTING, Boolean.TRUE); + if (exchange.isComplete() || continuation.stopping) return; + abortUpstream(continuation); //NOTE: this method is called only in context of timeouts. - final ProxyConnection connectionAttachment = exchange.getAttachment(CONNECTION); + final ProxyConnection connectionAttachment = continuation.connection; if (connectionAttachment != null) { ClientConnection clientConnection = connectionAttachment.getConnection(); UndertowLogger.PROXY_REQUEST_LOGGER.timingOutRequest(clientConnection.getPeerAddress() + "" + exchange.getRequestURI()); - abortUpstream(exchange); } else UndertowLogger.PROXY_REQUEST_LOGGER.timingOutRequest(exchange.getRequestURI()); @@ -991,7 +1007,7 @@ public void completed(final ClientExchange result) { if (exchange.getConnection().isPushSupported() && result.getConnection().isPushSupported()) this.handleServerPush(result); - result.setResponseListener(boundaryCallback(exchange, continuation, new ResponseCallback(exchange, proxyClientHandler, idempotentPredicate, headerRewriteRules, streamingRequest, streamResponseContentTypes, streamIdleTimeout, streamResponseHeaderOverwrite))); + result.setResponseListener(boundaryCallback(exchange, continuation, new ResponseCallback(exchange, continuation, proxyClientHandler, idempotentPredicate, headerRewriteRules, streamingRequest, streamResponseContentTypes, streamIdleTimeout, streamResponseHeaderOverwrite))); final IoExceptionHandler handler = new IoExceptionHandler(exchange, clientConnection.getConnection()); @@ -1000,7 +1016,7 @@ public void completed(final ClientExchange result) { return; } - HTTPTrailerChannelListener trailerListener = new HTTPTrailerChannelListener(exchange, result, exchange, proxyClientHandler, idempotentPredicate); + HTTPTrailerChannelListener trailerListener = new HTTPTrailerChannelListener(exchange, result, exchange, continuation, proxyClientHandler, idempotentPredicate); if (!exchange.isRequestComplete()) { @@ -1013,7 +1029,7 @@ public void completed(final ClientExchange result) { @Override public void failed(IOException e) { - handleFailure(exchange, proxyClientHandler, idempotentPredicate, e); + continuationTask(exchange, continuation, ex -> handleFailure(ex, continuation, proxyClientHandler, idempotentPredicate, e)).run(); } private void prepContinueHandler(ClientExchange result) { @@ -1045,7 +1061,7 @@ private void prepRequestChannelForContinue(ClientExchange ce, IoExceptionHandler exchange.getRequestChannel(), ce.getRequestChannel(), ChannelListeners.closingChannelListener(), - new HTTPTrailerChannelListener(exchange, ce, exchange, proxyClientHandler, idempotentPredicate), + new HTTPTrailerChannelListener(exchange, ce, exchange, continuation, proxyClientHandler, idempotentPredicate), io, io, exchange.getConnection().getByteBufferPool()), @@ -1102,7 +1118,7 @@ private void attachSslInfo(ClientRequest r) { } } - static void handleFailure(HttpServerExchange exchange, ProxyClientHandler proxyClientHandler, Predicate idempotentRequestPredicate, IOException e) { + private static void handleFailure(HttpServerExchange exchange, ProxyContinuation continuation, ProxyClientHandler proxyClientHandler, Predicate idempotentRequestPredicate, IOException e) { UndertowLogger.PROXY_REQUEST_LOGGER.proxyRequestFailed(exchange.getRequestURI(), e); if (exchange.isResponseStarted()) { @@ -1113,6 +1129,7 @@ static void handleFailure(HttpServerExchange exchange, ProxyClientHandler proxyC proxyClientHandler.failed(exchange); //this will attempt a retry if configured to do so else { + abortUpstream(continuation); exchange.setStatusCode(StatusCodes.SERVICE_UNAVAILABLE); exchange.endExchange(); } @@ -1121,6 +1138,7 @@ static void handleFailure(HttpServerExchange exchange, ProxyClientHandler proxyC private static final class ResponseCallback implements ClientCallback { private final HttpServerExchange exchange; + private final ProxyContinuation continuation; private final ProxyClientHandler proxyClientHandler; private final Predicate idempotentPredicate; private final Map> headerRewriteRules; @@ -1129,9 +1147,10 @@ private static final class ResponseCallback implements ClientCallback streamResponseHeaderOverwrite; - private ResponseCallback(HttpServerExchange exchange, ProxyClientHandler proxyClientHandler, Predicate idempotentPredicate, Map> headerRewriteRules, + private ResponseCallback(HttpServerExchange exchange, ProxyContinuation continuation, ProxyClientHandler proxyClientHandler, Predicate idempotentPredicate, Map> headerRewriteRules, boolean streamingRequest, List streamResponseContentTypes, int streamIdleTimeout, List streamResponseHeaderOverwrite) { this.exchange = exchange; + this.continuation = continuation; this.proxyClientHandler = proxyClientHandler; this.idempotentPredicate = idempotentPredicate; this.headerRewriteRules = headerRewriteRules; @@ -1154,7 +1173,7 @@ public void completed(final ClientExchange result) { boolean streamingResponse = isStreamingResponse(inbound, streamResponseContentTypes); if (streamingResponse) { - if (!streamingRequest) cancelExchangeTimeout(exchange); + if (!streamingRequest) continuation.cancelTimeout(); removeHeaders(outbound, streamResponseHeaderOverwrite); } @@ -1171,7 +1190,7 @@ public void completed(final ClientExchange result) { responseChannel = new IdleTimeoutStreamSourceChannel(responseChannel, exchange, result.getConnection(), streamIdleTimeout); } - Transfer.initiateTransfer(responseChannel, exchange.getResponseChannel(), ChannelListeners.closingChannelListener(), new HTTPTrailerChannelListener(result, exchange, exchange, proxyClientHandler, idempotentPredicate), handler, handler, exchange.getConnection().getByteBufferPool()); + Transfer.initiateTransfer(responseChannel, exchange.getResponseChannel(), ChannelListeners.closingChannelListener(), new HTTPTrailerChannelListener(result, exchange, exchange, continuation, proxyClientHandler, idempotentPredicate), handler, handler, exchange.getConnection().getByteBufferPool()); AbstractMetricsHandler metricsHandler = (AbstractMetricsHandler) exchange.getAttachment(AttachmentConstants.METRICS_HANDLER); if(metricsHandler != null) { @@ -1206,7 +1225,7 @@ private void handleUpgradeChannelOnComplete(ClientExchange result) { @Override public void failed(IOException e) { - handleFailure(exchange, proxyClientHandler, idempotentPredicate, e); + handleFailure(exchange, continuation, proxyClientHandler, idempotentPredicate, e); } } @@ -1386,13 +1405,15 @@ private static final class HTTPTrailerChannelListener implements ChannelListener private final Attachable source; private final Attachable target; private final HttpServerExchange exchange; + private final ProxyContinuation continuation; private final ProxyClientHandler proxyClientHandler; private final Predicate idempotentPredicate; - private HTTPTrailerChannelListener(final Attachable source, final Attachable target, HttpServerExchange exchange, ProxyClientHandler proxyClientHandler, Predicate idempotentPredicate) { + private HTTPTrailerChannelListener(final Attachable source, final Attachable target, HttpServerExchange exchange, ProxyContinuation continuation, ProxyClientHandler proxyClientHandler, Predicate idempotentPredicate) { this.source = source; this.target = target; this.exchange = exchange; + this.continuation = continuation; this.proxyClientHandler = proxyClientHandler; this.idempotentPredicate = idempotentPredicate; } @@ -1424,13 +1445,13 @@ public void handleEvent(final StreamSinkChannel channel) { LOG.error("IOException: ", e); - handleFailure(exchange, proxyClientHandler, idempotentPredicate, e); + continuationTask(exchange, continuation, ex -> handleFailure(ex, continuation, proxyClientHandler, idempotentPredicate, e)).run(); } catch (Exception e) { LOG.error("Exception: ", e); - handleFailure(exchange, proxyClientHandler, idempotentPredicate, new IOException(e)); + continuationTask(exchange, continuation, ex -> handleFailure(ex, continuation, proxyClientHandler, idempotentPredicate, new IOException(e))).run(); } } diff --git a/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java b/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java index fb90ad3fa5..370e0e69b7 100644 --- a/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java +++ b/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java @@ -1,5 +1,6 @@ package com.networknt.handler; + import com.networknt.httpstring.AttachmentConstants; import com.networknt.metrics.AbstractMetricsHandler; import com.networknt.metrics.MetricsConfig; @@ -13,12 +14,15 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; -import org.junit.jupiter.params.provider.Arguments; +import org.slf4j.MDC; import org.xnio.channels.Channels; import java.io.IOException; +import java.io.InputStream; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Proxy; import java.net.*; @@ -28,8 +32,13 @@ import java.time.Duration; import java.util.*; import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.function.Supplier; import java.util.stream.Stream; + import static org.junit.jupiter.api.Assertions.*; @Timeout(20) @@ -48,17 +57,17 @@ static Stream failureCases() { @ParameterizedTest @MethodSource("failureCases") void asynchronousFailuresUseSelectedBoundaryAndCloseUpstream(Phase phase, boolean enabled, boolean hasMdc) throws Exception { - var previousMdc = org.slf4j.MDC.getCopyOfContextMap(); + var previousMdc = MDC.getCopyOfContextMap(); AtomicInteger injected = new AtomicInteger(); AtomicInteger selected = new AtomicInteger(); - AtomicInteger depth = new AtomicInteger(); + ThreadLocal depth = ThreadLocal.withInitial(() -> 0); AtomicInteger maxDepth = new AtomicInteger(); AtomicInteger acquisitions = new AtomicInteger(); - var upstream = new java.util.concurrent.atomic.AtomicReference(); + var upstream = new AtomicReference(); CountDownLatch release = new CountDownLatch(1); CompletableFuture retry = new CompletableFuture<>(); - java.util.function.Supplier failure = () -> { - assertEquals(hasMdc ? "canonical-request" : null, org.slf4j.MDC.get("request"), + Supplier failure = () -> { + assertEquals(hasMdc ? "canonical-request" : null, MDC.get("request"), "Callbacks must install canonical request MDC for this frame"); injected.incrementAndGet(); return new ControlledFailure(phase); @@ -97,7 +106,7 @@ void asynchronousFailuresUseSelectedBoundaryAndCloseUpstream(Phase phase, boolea .setMaxConnectionRetries(phase == Phase.RETRY ? 1 : 0) .setMaxRequestTime(phase == Phase.TIMEOUT ? 300 : 0).build(); HttpHandler work = exchange -> { - if (hasMdc) exchange.putAttachment(AttachmentConstants.MDC_CONTEXT, Map.of("request", "canonical-request")); + if (hasMdc) RequestMdcContext.initialize(exchange).putField("request", "canonical-request"); if (phase == Phase.TIMEOUT) { exchange.putAttachment(AttachmentConstants.METRICS_HANDLER, new ThrowingMetrics(failure)); exchange.putAttachment(AttachmentConstants.DOWNSTREAM_METRICS_NAME, "test"); @@ -106,13 +115,14 @@ void asynchronousFailuresUseSelectedBoundaryAndCloseUpstream(Phase phase, boolea proxy.handleRequest(exchange); }; URI front = fixture.start(enabled ? exchange -> HandlerContinuation.execute(exchange, (ex, task) -> { - maxDepth.accumulateAndGet(depth.incrementAndGet(), Math::max); + depth.set(depth.get() + 1); + maxDepth.accumulateAndGet(depth.get(), Math::max); try { task.handleRequest(ex); } catch (ControlledFailure e) { assertEquals(phase, e.phase); selected.incrementAndGet(); ex.setStatusCode(409).getResponseSender().send("selected"); - } finally { depth.decrementAndGet(); } + } finally { if (depth.get() == 1) depth.remove(); else depth.set(depth.get() - 1); } }, work) : work); var pending = http.sendAsync(HttpRequest.newBuilder(front).timeout(Duration.ofSeconds(8)).build(), HttpResponse.BodyHandlers.ofString()); @@ -128,8 +138,8 @@ void asynchronousFailuresUseSelectedBoundaryAndCloseUpstream(Phase phase, boolea release.countDown(); } finally { release.countDown(); - if (previousMdc == null) org.slf4j.MDC.clear(); - else org.slf4j.MDC.setContextMap(previousMdc); + if (previousMdc == null) MDC.clear(); + else MDC.setContextMap(previousMdc); } } @@ -199,7 +209,7 @@ void unhandledTaskFailureAfterHeadersClosesStreamWithoutAppendingAnError() throw @Test void lateFailureDoesNotCloseACompletedKeepAliveConnection() throws Exception { - var completed = new java.util.concurrent.atomic.AtomicReference(); + var completed = new AtomicReference(); CountDownLatch done = new CountDownLatch(1); try (Fixture fixture = new Fixture(); HttpClient http = HttpClient.newHttpClient()) { URI front = fixture.start(exchange -> { @@ -232,7 +242,7 @@ void upstreamHttp2FailureResetsOnlyTheFailedStream(String mode) throws Exception CountDownLatch release = new CountDownLatch(1); Set backendPeers = ConcurrentHashMap.newKeySet(); Set frontendPeers = ConcurrentHashMap.newKeySet(); - var shared = new java.util.concurrent.atomic.AtomicReference(); + var shared = new AtomicReference(); AtomicInteger injected = new AtomicInteger(); try (Fixture fixture = new Fixture(); HttpClient http = HttpClient.newBuilder().version(HttpClient.Version.HTTP_2).build()) { URI backend = fixture.start(exchange -> { @@ -254,7 +264,7 @@ void upstreamHttp2FailureResetsOnlyTheFailedStream(String mode) throws Exception @Override public void getConnection(ProxyTarget target, HttpServerExchange exchange, ProxyCallback callback, long timeout, TimeUnit unit) { if (exchange.getRequestPath().equals("/fail")) failedAcquisitions.incrementAndGet(); - java.util.function.Consumer connected = connection -> { + Consumer connected = connection -> { ClientConnection selected = connection; if (exchange.getRequestPath().equals("/fail") && !mode.equals("timeout")) selected = decorateConnection(connection, mode.equals("before-send") || mode.equals("late") ? Phase.BEFORE_SEND : Phase.RESPONSE_CALLBACK, () -> { @@ -319,7 +329,7 @@ void upstreamHttp2FailureResetsOnlyTheFailedStream(String mode) throws Exception } @ParameterizedTest - @org.junit.jupiter.params.provider.CsvSource({"false,true", "true,true", "true,false"}) + @CsvSource({"false,true", "true,true", "true,false"}) void downstreamHttp2FailureDoesNotCancelSiblingStream(boolean fatal, boolean committed) throws Exception { CompletableFuture propagated = new CompletableFuture<>(); Error sentinel = new StackOverflowError("injected fatal error"); @@ -411,8 +421,10 @@ void fatalHttp11FailureTerminatesRequestAndPropagatesOriginal(boolean committed) @ValueSource(strings = {"completed", "failed", "queued"}) void callbackAfterCommitTerminatesIncompleteStreamWithoutForwarding(String kind) throws Exception { CompletableFuture late = new CompletableFuture<>(); + CountDownLatch committed = new CountDownLatch(1); + CountDownLatch delivered = new CountDownLatch(1); AtomicInteger acquisitions = new AtomicInteger(); - var closed = new java.util.concurrent.atomic.AtomicBoolean(); + var closed = new AtomicBoolean(); ClientConnection unusedConnection = (ClientConnection) Proxy.newProxyInstance(ClientConnection.class.getClassLoader(), new Class[]{ClientConnection.class}, (proxy, method, args) -> switch (method.getName()) { case "isMultiplexingSupported" -> false; @@ -427,25 +439,31 @@ void callbackAfterCommitTerminatesIncompleteStreamWithoutForwarding(String kind) ProxyCallback callback, long timeout, TimeUnit unit) { acquisitions.incrementAndGet(); late.complete(() -> { - assertTrue(exchange.isResponseStarted()); - assertFalse(exchange.isComplete()); - switch (kind) { - case "completed" -> callback.completed(exchange, new ProxyConnection(unusedConnection, "/")); - case "failed" -> callback.failed(exchange); - case "queued" -> callback.queuedRequestFailed(exchange); - default -> throw new AssertionError(kind); - } + try { + assertTrue(committed.await(5, TimeUnit.SECONDS)); + assertTrue(exchange.isResponseStarted()); + assertFalse(exchange.isComplete()); + switch (kind) { + case "completed" -> callback.completed(exchange, new ProxyConnection(unusedConnection, "/")); + case "failed" -> callback.failed(exchange); + case "queued" -> callback.queuedRequestFailed(exchange); + default -> throw new AssertionError(kind); + } + } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new AssertionError(e); } + finally { delivered.countDown(); } + }); + exchange.dispatch(worker -> { + worker.startBlocking(); + worker.getOutputStream().write("first".getBytes(StandardCharsets.UTF_8)); + worker.getOutputStream().flush(); + committed.countDown(); + assertTrue(delivered.await(5, TimeUnit.SECONDS)); }); } }; ProxyHandler proxy = ProxyHandler.builder().setProxyClient(client).setMaxRequestTime(0).build(); try (Fixture fixture = new Fixture()) { - URI front = fixture.start(exchange -> exchange.dispatch(worker -> { - proxy.handleRequest(worker); - worker.startBlocking(); - worker.getOutputStream().write("first".getBytes(StandardCharsets.UTF_8)); - worker.getOutputStream().flush(); - })); + URI front = fixture.start(proxy); try (Socket socket = new Socket(front.getHost(), front.getPort())) { socket.setSoTimeout(5000); socket.getOutputStream().write("GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n" @@ -460,8 +478,162 @@ void callbackAfterCommitTerminatesIncompleteStreamWithoutForwarding(String kind) } } + @ParameterizedTest + @ValueSource(strings = {"completed", "failed", "queued"}) + void unexpectedCommittedHttp2CallbackResetsOnlyItsStream(String kind) throws Exception { + CompletableFuture late = new CompletableFuture<>(); + CountDownLatch committed = new CountDownLatch(1); + CountDownLatch delivered = new CountDownLatch(1); + AtomicInteger acquisitions = new AtomicInteger(); + CountDownLatch healthyEntered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + Set peers = ConcurrentHashMap.newKeySet(); + ClientConnection reusable = (ClientConnection) Proxy.newProxyInstance(ClientConnection.class.getClassLoader(), + new Class[]{ClientConnection.class}, (proxy, method, args) -> switch (method.getName()) { + case "isMultiplexingSupported", "isOpen" -> true; + case "toString" -> "shared late HTTP/2 connection"; + default -> throw new AssertionError("Do not use or close the late shared connection: " + method.getName()); + }); + ProxyClient client = new ProxyClient() { + @Override public ProxyTarget findTarget(HttpServerExchange exchange) { return new ProxyTarget() {}; } + @Override public void getConnection(ProxyTarget target, HttpServerExchange exchange, + ProxyCallback callback, long timeout, TimeUnit unit) { + acquisitions.incrementAndGet(); + late.complete(() -> { + try { + assertTrue(committed.await(5, TimeUnit.SECONDS)); + assertTrue(exchange.isResponseStarted()); + assertFalse(exchange.isComplete()); + switch (kind) { + case "completed" -> callback.completed(exchange, new ProxyConnection(reusable, "/")); + case "failed" -> callback.failed(exchange); + case "queued" -> callback.queuedRequestFailed(exchange); + default -> throw new AssertionError(kind); + } + } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new AssertionError(e); } + finally { delivered.countDown(); } + }); + exchange.dispatch(worker -> { + worker.startBlocking(); + worker.getOutputStream().write("first".getBytes(StandardCharsets.UTF_8)); + worker.getOutputStream().flush(); + committed.countDown(); + assertTrue(delivered.await(5, TimeUnit.SECONDS)); + }); + } + }; + ProxyHandler proxy = ProxyHandler.builder().setProxyClient(client).setMaxRequestTime(0).build(); + try (Fixture fixture = new Fixture(); HttpClient http = HttpClient.newBuilder().version(HttpClient.Version.HTTP_2).build()) { + try { + URI front = fixture.start(exchange -> { + peers.add(exchange.getSourceAddress().getPort()); + if (exchange.getRequestPath().equals("/healthy")) exchange.dispatch(() -> { + healthyEntered.countDown(); + try { + assertTrue(release.await(8, TimeUnit.SECONDS)); + exchange.getResponseSender().send("healthy"); + } catch (InterruptedException e) { Thread.currentThread().interrupt(); } + }); + else if (exchange.getRequestPath().equals("/fail")) proxy.handleRequest(exchange); + else exchange.getResponseSender().send("accepted"); + }, true); + assertEquals(HttpClient.Version.HTTP_2, http.send(HttpRequest.newBuilder(front.resolve("/warm")).build(), + HttpResponse.BodyHandlers.ofString()).version()); + var healthy = http.sendAsync(HttpRequest.newBuilder(front.resolve("/healthy")).timeout(Duration.ofSeconds(10)).build(), + HttpResponse.BodyHandlers.ofString()); + assertTrue(healthyEntered.await(5, TimeUnit.SECONDS)); + var failed = http.send(HttpRequest.newBuilder(front.resolve("/fail")).timeout(Duration.ofSeconds(5)).build(), + HttpResponse.BodyHandlers.ofInputStream()); + assertEquals(200, failed.statusCode()); + try (var body = failed.body()) { + assertEquals("first", new String(body.readNBytes(5), StandardCharsets.UTF_8)); + late.get(3, TimeUnit.SECONDS).run(); + assertThrows(IOException.class, body::readAllBytes); + } + assertEquals(1, acquisitions.get()); + assertFalse(healthy.isDone()); + release.countDown(); + assertEquals("healthy", healthy.get(5, TimeUnit.SECONDS).body()); + assertEquals("accepted", http.send(HttpRequest.newBuilder(front.resolve("/after")).build(), + HttpResponse.BodyHandlers.ofString()).body()); + assertEquals(1, peers.size(), "Only the failed HTTP/2 stream may be reset"); + } finally { release.countDown(); http.shutdownNow(); } + } finally { release.countDown(); } + } + + @Test + void lateCallbacksAfterProxy503LeaveDrainingHttp2ResponseAlone() throws Exception { + CompletableFuture callbacksDone = new CompletableFuture<>(); + CountDownLatch releaseBody = new CountDownLatch(1); + CountDownLatch healthyEntered = new CountDownLatch(1); + CountDownLatch releaseHealthy = new CountDownLatch(1); + Set peers = ConcurrentHashMap.newKeySet(); + ClientConnection reusable = (ClientConnection) Proxy.newProxyInstance(ClientConnection.class.getClassLoader(), + new Class[]{ClientConnection.class}, (proxy, method, args) -> switch (method.getName()) { + case "isMultiplexingSupported", "isOpen" -> true; + default -> throw new AssertionError("Late callbacks must not use or close the shared upstream: " + method.getName()); + }); + ProxyClient client = new ProxyClient() { + @Override public ProxyTarget findTarget(HttpServerExchange exchange) { return new ProxyTarget() {}; } + @Override public void getConnection(ProxyTarget target, HttpServerExchange exchange, + ProxyCallback callback, long timeout, TimeUnit unit) { + try { + assertFalse(exchange.isRequestComplete()); + callback.couldNotResolveBackend(exchange); + var responseChannel = exchange.getResponseChannel(); + if (responseChannel != null) { responseChannel.shutdownWrites(); responseChannel.flush(); } + assertTrue(exchange.isResponseStarted()); + assertFalse(exchange.isComplete(), "The 503 is still draining the request body"); + callback.completed(exchange, new ProxyConnection(reusable, "/")); + callback.failed(exchange); + callback.queuedRequestFailed(exchange); + assertTrue(exchange.getConnection().isOpen(), "Keep the draining 503 stream open"); + callbacksDone.complete(null); + } catch (Throwable failure) { callbacksDone.completeExceptionally(failure); throw new AssertionError(failure); } + } + }; + ProxyHandler proxy = ProxyHandler.builder().setProxyClient(client).setMaxRequestTime(0).build(); + try (Fixture fixture = new Fixture(); HttpClient http = HttpClient.newBuilder().version(HttpClient.Version.HTTP_2).build()) { + URI front = fixture.start(exchange -> { + peers.add(exchange.getSourceAddress().getPort()); + if (exchange.getRequestPath().equals("/fail")) proxy.handleRequest(exchange); + else if (exchange.getRequestPath().equals("/healthy")) exchange.dispatch(() -> { + healthyEntered.countDown(); + try { + assertTrue(releaseHealthy.await(8, TimeUnit.SECONDS)); + exchange.getResponseSender().send("healthy"); + } catch (InterruptedException e) { Thread.currentThread().interrupt(); } + }); + else exchange.getResponseSender().send("accepted"); + }, true); + assertEquals(HttpClient.Version.HTTP_2, http.send(HttpRequest.newBuilder(front.resolve("/warm")).build(), + HttpResponse.BodyHandlers.ofString()).version()); + var healthy = http.sendAsync(HttpRequest.newBuilder(front.resolve("/healthy")).build(), HttpResponse.BodyHandlers.ofString()); + assertTrue(healthyEntered.await(5, TimeUnit.SECONDS)); + var body = HttpRequest.BodyPublishers.ofInputStream(() -> new InputStream() { + @Override public int read() throws IOException { + try { if (!releaseBody.await(8, TimeUnit.SECONDS)) throw new IOException("Body was not released"); } + catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new IOException(e); } + return -1; + } + }); + var failed = http.sendAsync(HttpRequest.newBuilder(front.resolve("/fail")).timeout(Duration.ofSeconds(10)).POST(body).build(), + HttpResponse.BodyHandlers.ofString()); + try { callbacksDone.get(5, TimeUnit.SECONDS); } + catch (Exception failure) { releaseBody.countDown(); releaseHealthy.countDown(); throw failure; } + assertFalse(healthy.isDone()); + releaseBody.countDown(); + assertEquals(503, failed.get(5, TimeUnit.SECONDS).statusCode()); + releaseHealthy.countDown(); + assertEquals("healthy", healthy.get(5, TimeUnit.SECONDS).body()); + assertEquals("accepted", http.send(HttpRequest.newBuilder(front.resolve("/after")).build(), + HttpResponse.BodyHandlers.ofString()).body()); + assertEquals(1, peers.size()); + } finally { releaseBody.countDown(); releaseHealthy.countDown(); } + } + private static ClientConnection decorateConnection(ClientConnection real, Phase phase, - java.util.function.Supplier failure) { + Supplier failure) { return (ClientConnection) Proxy.newProxyInstance(ClientConnection.class.getClassLoader(), new Class[]{ClientConnection.class}, (proxy, method, args) -> { if (method.getName().equals("sendRequest")) { @@ -479,7 +651,7 @@ private static ClientConnection decorateConnection(ClientConnection real, Phase } private static ClientExchange decorateExchange(ClientExchange real, Phase phase, - java.util.function.Supplier failure) { + Supplier failure) { return (ClientExchange) Proxy.newProxyInstance(ClientExchange.class.getClassLoader(), new Class[]{ClientExchange.class}, (proxy, method, args) -> { if (method.getName().equals("setResponseListener")) { @@ -498,8 +670,8 @@ private static ClientExchange decorateExchange(ClientExchange real, Phase phase, } private static final class ThrowingMetrics extends AbstractMetricsHandler { - private final java.util.function.Supplier failure; - ThrowingMetrics(java.util.function.Supplier failure) { this.failure = failure; } + private final Supplier failure; + ThrowingMetrics(Supplier failure) { this.failure = failure; } @Override public void injectMetrics(HttpServerExchange exchange, long startTime, String name, String endpoint) { throw failure.get(); } @Override protected void createMetricsReporter(TimeSeriesDbSender sender, MetricsConfig config) { /* No reporter in this fixture. */ } @Override public void handleRequest(HttpServerExchange exchange) { /* Lookup-only metrics fixture. */ } From 373fb38143a0faaca98691a32d5639ab593d23c6 Mon Sep 17 00:00:00 2001 From: Steve Hu Date: Fri, 30 Oct 2026 13:51:52 -0400 Subject: [PATCH 08/15] Resolve continuation Sonar findings and verify typed error responses --- .../networknt/body/BodyContinuationTest.java | 2 +- .../networknt/dump/DumpContinuationTest.java | 2 +- .../networknt/exception/ExceptionHandler.java | 63 ++++++++--------- .../exception/ExceptionContinuationTest.java | 40 +++++++++++ handler/qualification/allocation.json | 30 ++++----- .../handler/HandlerContinuation.java | 58 ++++++++-------- .../handler/HandlerContinuationTest.java | 4 +- .../RequestInjectionContinuationTest.java | 14 +++- .../hmac/HmacHandlerIntegrationTest.java | 2 +- .../networknt/mcp/McpContinuationTest.java | 2 +- .../com/networknt/handler/ProxyHandler.java | 67 +++++++++---------- .../handler/ProxyContinuationTest.java | 12 ++-- 12 files changed, 168 insertions(+), 128 deletions(-) diff --git a/body/src/test/java/com/networknt/body/BodyContinuationTest.java b/body/src/test/java/com/networknt/body/BodyContinuationTest.java index 48919fb462..bd6f623f36 100644 --- a/body/src/test/java/com/networknt/body/BodyContinuationTest.java +++ b/body/src/test/java/com/networknt/body/BodyContinuationTest.java @@ -22,7 +22,7 @@ void redispatchRetainsBoundary(boolean enabled) throws Exception { }); HttpHandler root = enabled ? exchange -> HandlerContinuation.execute(exchange, (ex, task) -> { try { task.handleRequest(ex); } - catch (IllegalStateException e) { ex.setStatusCode(409).getResponseSender().send("selected"); } + catch (IllegalStateException _) { ex.setStatusCode(409).getResponseSender().send("selected"); } }, body) : body; Undertow server = Undertow.builder().addHttpListener(0, "127.0.0.1").setHandler(root).build(); try (HttpClient client = HttpClient.newHttpClient()) { diff --git a/dump/src/test/java/com/networknt/dump/DumpContinuationTest.java b/dump/src/test/java/com/networknt/dump/DumpContinuationTest.java index a117e27626..4fbda15b7f 100644 --- a/dump/src/test/java/com/networknt/dump/DumpContinuationTest.java +++ b/dump/src/test/java/com/networknt/dump/DumpContinuationTest.java @@ -23,7 +23,7 @@ void redispatchRetainsSelectedBoundary(boolean enabled) throws Exception { }; HttpHandler root = enabled ? exchange -> HandlerContinuation.execute(exchange, (ex, task) -> { try { task.handleRequest(ex); } - catch (IllegalStateException e) { ex.setStatusCode(409).getResponseSender().send("selected"); } + catch (IllegalStateException _) { ex.setStatusCode(409).getResponseSender().send("selected"); } }, handler) : handler; Undertow server = Undertow.builder().addHttpListener(0, "127.0.0.1").setHandler(root).build(); try (HttpClient client = HttpClient.newHttpClient()) { diff --git a/exception/src/main/java/com/networknt/exception/ExceptionHandler.java b/exception/src/main/java/com/networknt/exception/ExceptionHandler.java index 9b77fb6e11..5ba2c007bd 100644 --- a/exception/src/main/java/com/networknt/exception/ExceptionHandler.java +++ b/exception/src/main/java/com/networknt/exception/ExceptionHandler.java @@ -19,6 +19,7 @@ import com.networknt.handler.Handler; import com.networknt.handler.HandlerContinuation; import com.networknt.handler.MiddlewareHandler; +import com.networknt.status.Status; import io.undertow.Handlers; import io.undertow.server.HttpHandler; import io.undertow.server.HttpServerExchange; @@ -99,46 +100,38 @@ private void handleContinuation(HttpServerExchange exchange, HttpHandler continu continuation.handleRequest(exchange); } catch (Throwable e) { logger.error("ExceptionHandler error processing - path: {}", path, e); - if(exchange.isResponseChannelAvailable()) { - //handle exceptions - if(e instanceof RuntimeException) { - // check if it is FrameworkException which is subclass of RuntimeException. - if(e instanceof FrameworkException) { - FrameworkException fe = (FrameworkException)e; - exchange.setStatusCode(fe.getStatus().getStatusCode()); - exchange.getResponseHeaders().put(Headers.CONTENT_TYPE, "application/json"); - exchange.getResponseSender().send(fe.getStatus().toString()); - logger.error(fe.getStatus().toString(), e); - } else { - setExchangeStatus(exchange, STATUS_RUNTIME_EXCEPTION); - } - } else { - if(e instanceof ApiException) { - ApiException ae = (ApiException)e; - exchange.setStatusCode(ae.getStatus().getStatusCode()); - exchange.getResponseHeaders().put(Headers.CONTENT_TYPE, "application/json"); - exchange.getResponseSender().send(ae.getStatus().toString()); - logger.error(ae.getStatus().toString(), e); - } else if(e instanceof ClientException){ - ClientException ce = (ClientException)e; - if(ce.getStatus().getStatusCode() == 0){ - setExchangeStatus(exchange, STATUS_UNCAUGHT_EXCEPTION); - } else { - exchange.setStatusCode(ce.getStatus().getStatusCode()); - exchange.getResponseHeaders().put(Headers.CONTENT_TYPE, "application/json"); - exchange.getResponseSender().send(ce.getStatus().toString()); - } - - } else { - setExchangeStatus(exchange, STATUS_UNCAUGHT_EXCEPTION); - } - } - } + if (exchange.isResponseChannelAvailable()) writeFailureResponse(exchange, e); } finally { if(logger.isDebugEnabled()) logger.debug("ExceptionHandler frame completed - path: {}", path); } } + private void writeFailureResponse(HttpServerExchange exchange, Throwable failure) { + if (failure instanceof RuntimeException) { + if (failure instanceof FrameworkException framework) { + sendStatus(exchange, framework.getStatus()); + logger.error(framework.getStatus().toString(), failure); + } else setExchangeStatus(exchange, STATUS_RUNTIME_EXCEPTION); + return; + } + if (failure instanceof ApiException api) { + sendStatus(exchange, api.getStatus()); + logger.error(api.getStatus().toString(), failure); + return; + } + if (failure instanceof ClientException && ClientException.getStatus().getStatusCode() != 0) { + sendStatus(exchange, ClientException.getStatus()); + return; + } + setExchangeStatus(exchange, STATUS_UNCAUGHT_EXCEPTION); + } + + private static void sendStatus(HttpServerExchange exchange, Status status) { + exchange.setStatusCode(status.getStatusCode()); + exchange.getResponseHeaders().put(Headers.CONTENT_TYPE, "application/json"); + exchange.getResponseSender().send(status.toString()); + } + @Override public HttpHandler getNext() { return next; diff --git a/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java b/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java index ca4696d2fb..45aebdc624 100644 --- a/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java +++ b/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java @@ -3,6 +3,7 @@ import com.networknt.config.Config; import com.networknt.handler.HandlerContinuation; +import com.networknt.status.Status; import io.undertow.Undertow; import io.undertow.server.HttpHandler; import io.undertow.server.HttpServerExchange; @@ -99,6 +100,45 @@ void selectedPolicyHandlesInitialAndResumedFailures(String name, boolean async, else Config.getInstance().clearConfigCache(name); } } + static Stream typedFailureCases() { + return Stream.of("framework", "api", "client", "client-without-status") + .flatMap(kind -> Stream.of(false, true).map(resumed -> Arguments.of(kind, resumed))); + } + + @ParameterizedTest + @MethodSource("typedFailureCases") + void typedFailuresRetainResponseStatusAndBody(String kind, boolean resumed) throws Exception { + Status previousClientStatus = ClientException.getStatus(); + Status status = new Status(429, "TEST_TYPED_FAILURE", "Typed failure", "Preserve the supplied status"); + Exception failure = switch (kind) { + case "framework" -> new FrameworkException(status); + case "api" -> new ApiException(status); + case "client" -> new ClientException(status); + case "client-without-status" -> new ClientException(new Status()); + default -> throw new IllegalArgumentException(kind); + }; + ExceptionHandler selected = new ExceptionHandler(); + HttpHandler failing = exchange -> { throw failure; }; + selected.setNext(exchange -> { + if (resumed) exchange.dispatch(HandlerContinuation.capture(exchange, failing)); + else failing.handleRequest(exchange); + }); + Undertow server = Undertow.builder().addHttpListener(0, "127.0.0.1").setHandler(selected).build(); + try (HttpClient client = HttpClient.newHttpClient()) { + server.start(); + int port = ((InetSocketAddress) server.getListenerInfo().getFirst().getAddress()).getPort(); + var response = client.send(HttpRequest.newBuilder(URI.create("http://127.0.0.1:" + port)) + .timeout(Duration.ofSeconds(5)).build(), HttpResponse.BodyHandlers.ofString()); + boolean missingStatus = kind.equals("client-without-status"); + assertEquals(missingStatus ? 400 : 429, response.statusCode(), response.body()); + assertTrue(response.body().contains(missingStatus ? "ERR10011" : "TEST_TYPED_FAILURE"), response.body()); + assertEquals("application/json", response.headers().firstValue("Content-Type").orElse("")); + } finally { + server.stop(); + ClientException.setStatus(previousClientStatus); + } + } + @ParameterizedTest @ValueSource(booleans = {false, true}) void committedResponseIsNotTruncatedByInitialOrResumedBoundary(boolean resumed) throws Exception { diff --git a/handler/qualification/allocation.json b/handler/qualification/allocation.json index de320ff1fc..3abecad2b6 100644 --- a/handler/qualification/allocation.json +++ b/handler/qualification/allocation.json @@ -26,7 +26,7 @@ { "case": "empty_request_empty_caller", "sample": 0, - "bytes_per_frame": 74.9799 + "bytes_per_frame": 60.7921 }, { "case": "empty_request_empty_caller", @@ -56,60 +56,60 @@ { "case": "three_fields_different_caller", "sample": 0, - "bytes_per_frame": 904.0 + "bytes_per_frame": 853.378 }, { "case": "three_fields_different_caller", "sample": 1, - "bytes_per_frame": 904.0 + "bytes_per_frame": 808.0 }, { "case": "three_fields_different_caller", "sample": 2, - "bytes_per_frame": 904.0 + "bytes_per_frame": 808.0 }, { "case": "three_fields_different_caller_same_size", "sample": 0, - "bytes_per_frame": 1184.0 + "bytes_per_frame": 1056.0 }, { "case": "three_fields_different_caller_same_size", "sample": 1, - "bytes_per_frame": 1184.0 + "bytes_per_frame": 1056.0 }, { "case": "three_fields_different_caller_same_size", "sample": 2, - "bytes_per_frame": 1184.0 + "bytes_per_frame": 1056.0 }, { "case": "twelve_fields_different_caller", "sample": 0, - "bytes_per_frame": 1384.0 + "bytes_per_frame": 1288.0 }, { "case": "twelve_fields_different_caller", "sample": 1, - "bytes_per_frame": 1384.0 + "bytes_per_frame": 1288.0 }, { "case": "twelve_fields_different_caller", "sample": 2, - "bytes_per_frame": 1384.0 + "bytes_per_frame": 1288.0 } ], "median_bytes_per_resume": { "callback_only": 0.0, "empty_request_empty_caller": 56.0, "three_fields_matching_caller": 848.0, - "three_fields_different_caller": 904.0, - "three_fields_different_caller_same_size": 1184.0, - "twelve_fields_different_caller": 1384.0 + "three_fields_different_caller": 808.0, + "three_fields_different_caller_same_size": 1056.0, + "twelve_fields_different_caller": 1288.0 }, "source_sha256": { - "handler/src/main/java/com/networknt/handler/HandlerContinuation.java": "d4facd6530d2fe2de3e10962cfc91db7e25f73cde2a663542bb2e18be159e1ed", + "handler/src/main/java/com/networknt/handler/HandlerContinuation.java": "b650d3da1471fe7c96cab45e9458dc3b80bd0cc8143767ed87acad1da748fc01", "handler/src/main/java/com/networknt/handler/RequestMdcContext.java": "e9788732232232700d33a63ea24eeb0a6a52899dc0a5aa9730f0bf5b7b6692c5", - "proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java": "db186cd98d2e2579f32f71b7594b9706344489745a1332d9264a03d2f8b1e50c" + "proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java": "88195df98d81d55ea70549127fa7258b5464983945e4eae68d9531bd1fa83927" } } diff --git a/handler/src/main/java/com/networknt/handler/HandlerContinuation.java b/handler/src/main/java/com/networknt/handler/HandlerContinuation.java index dbec69b710..257d5b8102 100644 --- a/handler/src/main/java/com/networknt/handler/HandlerContinuation.java +++ b/handler/src/main/java/com/networknt/handler/HandlerContinuation.java @@ -34,7 +34,33 @@ private record CapturedBoundary(Boundary boundary, RequestMdcContext context) im private record CapturedHandler(CapturedBoundary policy, HttpHandler continuation) implements HttpHandler { @Override public void handleRequest(HttpServerExchange exchange) throws Exception { - withMdc(exchange, policy, () -> policy.boundary().handle(exchange, continuation)); + withMdc(exchange, () -> policy.boundary().handle(exchange, continuation)); + } + + private void withMdc(HttpServerExchange exchange, ScopedAction callback) throws E { + Frame previousFrame = CURRENT_FRAME.get(); + boolean reuseMdc = previousFrame != null && previousFrame.exchange() == exchange && previousFrame.mdcScoped(); + if (reuseMdc && previousFrame.boundary() == policy) { + callback.run(); + return; + } + Map previousMdc = reuseMdc ? null : MDC.getCopyOfContextMap(); + CURRENT_FRAME.set(new Frame(exchange, policy, true)); + try { + if (!reuseMdc) { + Map requestMdc = policy.context().snapshot(); + if (!requestMdc.equals(previousMdc == null ? Map.of() : previousMdc)) setMdc(requestMdc); + } + callback.run(); + } finally { + if (!reuseMdc) setMdc(previousMdc); + restoreFrame(previousFrame); + } + } + + private static void setMdc(Map context) { + if (context == null || context.isEmpty()) MDC.clear(); + else MDC.setContextMap(context); } } @@ -52,6 +78,7 @@ private HandlerContinuation() { } * Implementations must be safe to reuse concurrently across exchanges. */ @FunctionalInterface + @SuppressWarnings("java:S112") // Matches Undertow HttpHandler; boundaries must preserve arbitrary checked failures. public interface Boundary { void handle(HttpServerExchange exchange, HttpHandler continuation) throws Exception; } @@ -105,28 +132,6 @@ private static RequestMdcContext context(HttpServerExchange exchange) { return active != null ? active : RequestMdcContext.initialize(exchange); } - private static void withMdc(HttpServerExchange exchange, CapturedBoundary policy, - ScopedAction callback) throws E { - Frame previousFrame = CURRENT_FRAME.get(); - boolean reuseMdc = previousFrame != null && previousFrame.exchange() == exchange && previousFrame.mdcScoped(); - if (reuseMdc && previousFrame.boundary() == policy) { - callback.run(); - return; - } - Map previousMdc = reuseMdc ? null : MDC.getCopyOfContextMap(); - CURRENT_FRAME.set(new Frame(exchange, policy, true)); - try { - if (!reuseMdc) { - Map requestMdc = policy.context().snapshot(); - if (!requestMdc.equals(previousMdc == null ? Map.of() : previousMdc)) setMdc(requestMdc); - } - callback.run(); - } finally { - if (!reuseMdc) setMdc(previousMdc); - restoreFrame(previousFrame); - } - } - private static void restoreFrame(Frame previous) { if (previous == null) CURRENT_FRAME.remove(); else CURRENT_FRAME.set(previous); @@ -174,13 +179,8 @@ private static void finishFailedRequest(HttpServerExchange exchange, Throwable f private static void rememberCleanupFailure(Throwable failure, Throwable cleanupFailure) { if (cleanupFailure == failure) return; try { failure.addSuppressed(cleanupFailure); } - catch (Throwable ignored) { + catch (Throwable _) { // Preserve the original failure even when suppression metadata cannot be allocated. } } - - private static void setMdc(Map context) { - if (context == null || context.isEmpty()) MDC.clear(); - else MDC.setContextMap(context); - } } diff --git a/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java b/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java index a97dd67d23..1d483913bf 100644 --- a/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java +++ b/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java @@ -291,8 +291,8 @@ void absentBoundaryStaysAbsentAndFailuresRestoreScope() throws Exception { IOException failure = new IOException("failure"); HttpHandler unwrapped = HandlerContinuation.capture(exchange, ignored -> { throw failure; }); HandlerContinuation.Boundary selected = (ex, task) -> { - try { task.handleRequest(ex); } - catch (IOException e) { fail("A captured absent boundary must hide the active boundary", e); } + assertDoesNotThrow(() -> task.handleRequest(ex), + "A captured absent boundary must hide the active boundary"); }; HandlerContinuation.execute(exchange, selected, ex -> assertSame(failure, assertThrows(IOException.class, () -> unwrapped.handleRequest(ex)))); diff --git a/handler/src/test/java/com/networknt/handler/RequestInjectionContinuationTest.java b/handler/src/test/java/com/networknt/handler/RequestInjectionContinuationTest.java index b03480cf8b..78aa9463f9 100644 --- a/handler/src/test/java/com/networknt/handler/RequestInjectionContinuationTest.java +++ b/handler/src/test/java/com/networknt/handler/RequestInjectionContinuationTest.java @@ -51,7 +51,7 @@ void partialBodyRetainsBoundaryOrItsAbsence(boolean enabled) throws Exception { CountDownLatch suspended = new CountDownLatch(1); HandlerContinuation.Boundary boundary = (exchange, task) -> { try { task.handleRequest(exchange); } - catch (IOException e) { exchange.setStatusCode(409).getResponseSender().send("selected boundary"); } + catch (IOException _) { exchange.setStatusCode(409).getResponseSender().send("selected boundary"); } }; Undertow server = Undertow.builder().addHttpListener(0, "127.0.0.1").setIoThreads(1) .setHandler(exchange -> { @@ -72,8 +72,16 @@ void partialBodyRetainsBoundaryOrItsAbsence(boolean enabled) throws Exception { int port = ((InetSocketAddress) server.getListenerInfo().getFirst().getAddress()).getPort(); try (Socket socket = new Socket("127.0.0.1", port)) { socket.setSoTimeout(5000); - socket.getOutputStream().write(("POST / HTTP/1.1\r\nHost: localhost\r\nContent-Type: application/json\r\n" - + "Transfer-Encoding: chunked\r\nConnection: close\r\n\r\n1\r\n{\r\n").getBytes(StandardCharsets.UTF_8)); + socket.getOutputStream().write((""" + POST / HTTP/1.1\r + Host: localhost\r + Content-Type: application/json\r + Transfer-Encoding: chunked\r + Connection: close\r + \r + 1\r + {\r + """).getBytes(StandardCharsets.UTF_8)); socket.getOutputStream().flush(); assertTrue(suspended.await(5, TimeUnit.SECONDS)); socket.getOutputStream().write("1\r\n}\r\n0\r\n\r\n".getBytes(StandardCharsets.UTF_8)); diff --git a/hmac/src/test/java/com/networknt/hmac/HmacHandlerIntegrationTest.java b/hmac/src/test/java/com/networknt/hmac/HmacHandlerIntegrationTest.java index b600b7219f..6bb128503b 100644 --- a/hmac/src/test/java/com/networknt/hmac/HmacHandlerIntegrationTest.java +++ b/hmac/src/test/java/com/networknt/hmac/HmacHandlerIntegrationTest.java @@ -68,7 +68,7 @@ void replayCompletionRetainsExceptionBoundary(boolean enabled) throws Exception if (enabled) chainWrapper = handler -> exchange -> com.networknt.handler.HandlerContinuation.execute(exchange, (ex, task) -> { try { task.handleRequest(ex); } - catch (java.io.IOException e) { ex.setStatusCode(409).getResponseSender().send("selected"); } + catch (java.io.IOException _) { ex.setStatusCode(409).getResponseSender().send("selected"); } }, handler); try (RunningServer server = start(context, new RequestInterceptor[]{context.interceptor}, exchange -> { throw new java.io.IOException("callee failed"); })) { diff --git a/mcp-router/src/test/java/com/networknt/mcp/McpContinuationTest.java b/mcp-router/src/test/java/com/networknt/mcp/McpContinuationTest.java index 068d93beb0..08ce32b05d 100644 --- a/mcp-router/src/test/java/com/networknt/mcp/McpContinuationTest.java +++ b/mcp-router/src/test/java/com/networknt/mcp/McpContinuationTest.java @@ -23,7 +23,7 @@ void redispatchRetainsSelectedBoundary(boolean enabled) throws Exception { }; HttpHandler root = enabled ? exchange -> HandlerContinuation.execute(exchange, (ex, task) -> { try { task.handleRequest(ex); } - catch (IllegalStateException e) { ex.setStatusCode(409).getResponseSender().send("selected"); } + catch (IllegalStateException _) { ex.setStatusCode(409).getResponseSender().send("selected"); } }, handler) : handler; Undertow server = Undertow.builder().addHttpListener(0, "127.0.0.1").setHandler(root).build(); try (HttpClient client = HttpClient.newHttpClient()) { diff --git a/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java b/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java index 5def153373..e7feeb1436 100644 --- a/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java +++ b/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java @@ -378,6 +378,7 @@ private static final class ProxyContinuation { private boolean stopping; private volatile XnioExecutor.Key timeoutKey; + @SuppressWarnings("java:S1181") // Release the upstream stream before rethrowing fatal Errors unchanged. private ProxyContinuation(HttpServerExchange exchange, UnaryOperator boundary) { this.boundary = boundary; this.resumed = HandlerContinuation.task(exchange, boundary.apply(ex -> { @@ -431,27 +432,6 @@ private static void abortUpstream(ProxyContinuation state) { } } - private static ClientCallback boundaryCallback(HttpServerExchange exchange, - ProxyContinuation continuation, ClientCallback callback) { - return new ClientCallback<>() { - @Override public void completed(ClientExchange result) { - continuationTask(exchange, continuation, ex -> { - if (ex.isComplete() || continuation.stopping) { - cancelClientExchange(result); - return; - } - continuation.request = result; - callback.completed(result); - }).run(); - } - @Override public void failed(IOException failure) { - continuationTask(exchange, continuation, ex -> { - if (!ex.isComplete() && !continuation.stopping) callback.failed(failure); - }).run(); - } - }; - } - /** Ignore callbacks owned by a terminal response; terminate only unexpected commitment. */ private static boolean stopCallbackIfTerminal(HttpServerExchange exchange, ProxyContinuation state) { if (exchange.isComplete() || state.stopping) return true; @@ -989,8 +969,28 @@ private void attachRemoteHost(ClientRequest r) { } } + private ClientCallback boundaryCallback(ClientCallback callback) { + return new ClientCallback<>() { + @Override public void completed(ClientExchange result) { + continuationTask(exchange, continuation, ex -> { + if (ex.isComplete() || continuation.stopping) { + cancelClientExchange(result); + return; + } + continuation.request = result; + callback.completed(result); + }).run(); + } + @Override public void failed(IOException failure) { + continuationTask(exchange, continuation, ex -> { + if (!ex.isComplete() && !continuation.stopping) callback.failed(failure); + }).run(); + } + }; + } + private void sendWithCallback(ClientRequest r, String host) { - this.clientConnection.getConnection().sendRequest(r, boundaryCallback(exchange, continuation, new ClientCallback<>() { + this.clientConnection.getConnection().sendRequest(r, boundaryCallback(new ClientCallback<>() { @Override public void completed(final ClientExchange result) { @@ -1007,7 +1007,7 @@ public void completed(final ClientExchange result) { if (exchange.getConnection().isPushSupported() && result.getConnection().isPushSupported()) this.handleServerPush(result); - result.setResponseListener(boundaryCallback(exchange, continuation, new ResponseCallback(exchange, continuation, proxyClientHandler, idempotentPredicate, headerRewriteRules, streamingRequest, streamResponseContentTypes, streamIdleTimeout, streamResponseHeaderOverwrite))); + result.setResponseListener(boundaryCallback(new ResponseCallback(ProxyAction.this))); final IoExceptionHandler handler = new IoExceptionHandler(exchange, clientConnection.getConnection()); @@ -1147,17 +1147,16 @@ private static final class ResponseCallback implements ClientCallback streamResponseHeaderOverwrite; - private ResponseCallback(HttpServerExchange exchange, ProxyContinuation continuation, ProxyClientHandler proxyClientHandler, Predicate idempotentPredicate, Map> headerRewriteRules, - boolean streamingRequest, List streamResponseContentTypes, int streamIdleTimeout, List streamResponseHeaderOverwrite) { - this.exchange = exchange; - this.continuation = continuation; - this.proxyClientHandler = proxyClientHandler; - this.idempotentPredicate = idempotentPredicate; - this.headerRewriteRules = headerRewriteRules; - this.streamingRequest = streamingRequest; - this.streamResponseContentTypes = streamResponseContentTypes; - this.streamIdleTimeout = streamIdleTimeout; - this.streamResponseHeaderOverwrite = streamResponseHeaderOverwrite; + private ResponseCallback(ProxyAction action) { + this.exchange = action.exchange; + this.continuation = action.continuation; + this.proxyClientHandler = action.proxyClientHandler; + this.idempotentPredicate = action.idempotentPredicate; + this.headerRewriteRules = action.headerRewriteRules; + this.streamingRequest = action.streamingRequest; + this.streamResponseContentTypes = action.streamResponseContentTypes; + this.streamIdleTimeout = action.streamIdleTimeout; + this.streamResponseHeaderOverwrite = action.streamResponseHeaderOverwrite; } @Override diff --git a/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java b/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java index 370e0e69b7..3e15d3bfc5 100644 --- a/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java +++ b/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java @@ -77,7 +77,7 @@ void asynchronousFailuresUseSelectedBoundaryAndCloseUpstream(Phase phase, boolea try { if (phase == Phase.TIMEOUT && !release.await(5, TimeUnit.SECONDS)) return; exchange.getResponseSender().send("accepted"); - } catch (InterruptedException e) { Thread.currentThread().interrupt(); } + } catch (InterruptedException _) { Thread.currentThread().interrupt(); } })); LoadBalancingProxyClient real = new LoadBalancingProxyClient().addHost(backend); ProxyClient client = new ProxyClient() { @@ -254,7 +254,7 @@ void upstreamHttp2FailureResetsOnlyTheFailedStream(String mode) throws Exception try { assertTrue(release.await(8, TimeUnit.SECONDS)); exchange.getResponseSender().send("healthy"); - } catch (InterruptedException e) { Thread.currentThread().interrupt(); } + } catch (InterruptedException _) { Thread.currentThread().interrupt(); } }); else exchange.getResponseSender().send("accepted"); }, true); @@ -297,7 +297,7 @@ void upstreamHttp2FailureResetsOnlyTheFailedStream(String mode) throws Exception frontendPeers.add(exchange.getSourceAddress().getPort()); HandlerContinuation.execute(exchange, (ex, task) -> { try { task.handleRequest(ex); } - catch (ControlledFailure e) { ex.setStatusCode(409).getResponseSender().send("selected"); } + catch (ControlledFailure _) { ex.setStatusCode(409).getResponseSender().send("selected"); } }, proxy); }, true); var warm = http.send(HttpRequest.newBuilder(front.resolve("/warm")).build(), HttpResponse.BodyHandlers.ofString()); @@ -344,7 +344,7 @@ void downstreamHttp2FailureDoesNotCancelSiblingStream(boolean fatal, boolean com try { assertTrue(release.await(8, TimeUnit.SECONDS)); exchange.getResponseSender().send("healthy"); - } catch (InterruptedException e) { Thread.currentThread().interrupt(); } + } catch (InterruptedException _) { Thread.currentThread().interrupt(); } }); else if (exchange.getRequestPath().equals("/fail")) exchange.dispatch(() -> { try { @@ -532,7 +532,7 @@ void unexpectedCommittedHttp2CallbackResetsOnlyItsStream(String kind) throws Exc try { assertTrue(release.await(8, TimeUnit.SECONDS)); exchange.getResponseSender().send("healthy"); - } catch (InterruptedException e) { Thread.currentThread().interrupt(); } + } catch (InterruptedException _) { Thread.currentThread().interrupt(); } }); else if (exchange.getRequestPath().equals("/fail")) proxy.handleRequest(exchange); else exchange.getResponseSender().send("accepted"); @@ -602,7 +602,7 @@ else if (exchange.getRequestPath().equals("/healthy")) exchange.dispatch(() -> { try { assertTrue(releaseHealthy.await(8, TimeUnit.SECONDS)); exchange.getResponseSender().send("healthy"); - } catch (InterruptedException e) { Thread.currentThread().interrupt(); } + } catch (InterruptedException _) { Thread.currentThread().interrupt(); } }); else exchange.getResponseSender().send("accepted"); }, true); From 4df4bf6a1bf4929dbe68b23f31e3c30a2b062082 Mon Sep 17 00:00:00 2001 From: Steve Hu Date: Fri, 30 Oct 2026 13:53:55 -0400 Subject: [PATCH 09/15] Defer typed exception status formatting in error logs --- .../java/com/networknt/exception/ExceptionHandler.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/exception/src/main/java/com/networknt/exception/ExceptionHandler.java b/exception/src/main/java/com/networknt/exception/ExceptionHandler.java index 5ba2c007bd..bceb6ab227 100644 --- a/exception/src/main/java/com/networknt/exception/ExceptionHandler.java +++ b/exception/src/main/java/com/networknt/exception/ExceptionHandler.java @@ -109,14 +109,16 @@ private void handleContinuation(HttpServerExchange exchange, HttpHandler continu private void writeFailureResponse(HttpServerExchange exchange, Throwable failure) { if (failure instanceof RuntimeException) { if (failure instanceof FrameworkException framework) { - sendStatus(exchange, framework.getStatus()); - logger.error(framework.getStatus().toString(), failure); + Status status = framework.getStatus(); + sendStatus(exchange, status); + logger.error("{}", status, failure); } else setExchangeStatus(exchange, STATUS_RUNTIME_EXCEPTION); return; } if (failure instanceof ApiException api) { - sendStatus(exchange, api.getStatus()); - logger.error(api.getStatus().toString(), failure); + Status status = api.getStatus(); + sendStatus(exchange, status); + logger.error("{}", status, failure); return; } if (failure instanceof ClientException && ClientException.getStatus().getStatusCode() != 0) { From 7325d702e294b366671f382c62e6d395cd437a1d Mon Sep 17 00:00:00 2001 From: Steve Hu Date: Fri, 30 Oct 2026 15:28:40 -0400 Subject: [PATCH 10/15] Retain exchange policy and confine proxy attempts to I/O ownership --- README.md | 29 +-- .../networknt/exception/ExceptionHandler.java | 65 +++--- .../exception/ExceptionContinuationTest.java | 14 ++ handler/qualification/MdcAllocationProbe.java | 57 ----- handler/qualification/README.md | 23 -- handler/qualification/allocation.json | 115 ---------- handler/qualification/mutation-control.txt | 9 - handler/qualification/run-allocation.py | 33 --- .../handler/HandlerContinuation.java | 72 +++++-- .../networknt/handler/RequestMdcContext.java | 11 +- .../com/networknt/handler/RequestState.java | 51 +++++ .../handler/HandlerContinuationTest.java | 75 ++++++- .../com/networknt/handler/ProxyHandler.java | 200 ++++++++++------- .../handler/ProxyContinuationTest.java | 203 ++++++++++++++++++ 14 files changed, 573 insertions(+), 384 deletions(-) delete mode 100644 handler/qualification/MdcAllocationProbe.java delete mode 100644 handler/qualification/README.md delete mode 100644 handler/qualification/allocation.json delete mode 100644 handler/qualification/mutation-control.txt delete mode 100644 handler/qualification/run-allocation.py create mode 100644 handler/src/main/java/com/networknt/handler/RequestState.java diff --git a/README.md b/README.md index d09b950b9c..5287d87e3b 100644 --- a/README.md +++ b/README.md @@ -268,10 +268,13 @@ MDC updates use one stable RequestMdcContext holder with an atomic immutable-map reference. The first boundary execution or capture initializes that holder before its asynchronous hand-off; addHandlerMDCContext can also initialize it on first serial use. Later updates replace only the holder's snapshot, never its -attachment. Captured frames and their MDC helper updates reuse the holder directly, -without attachment lookups. Active exception policy is thread-scoped and captured -by value; overlapping frames never rewrite a policy attachment. Custom asynchronous -code must initialize/capture before hand-off. +attachment. An exchange-owned RequestState retains the entry exception policy +and MDC holder. A synchronized weak-key/weak-value index supports off-frame lookups +without reading Undertow's mutable attachment map; the exchange attachment owns the +state's lifetime. The index lock covers lookup/initialization only, never handlers +or callbacks. Captured frames read their holder directly. Thread-local frames only +select nested policy overrides and scope MDC; off-frame capture retains the entry +policy. Custom asynchronous code must initialize/capture before hand-off. The Map-typed attachment key remains compatible with read-only consumers; direct attachment replacement or mutation is unsupported after initialization. Proxy callbacks use this scope rather than the unscoped @@ -293,14 +296,16 @@ The neutral Runnable adapter attempts a 500 before commitment or a stream reset after commitment before propagating the original Error. Failed writes/resets are suppressed on that Error and cannot replace it; selected exception policies keep their existing handling behavior. Ordinary exceptions use the same guarded cleanup. -Proxy callbacks serialize on private per-request state rather than mutating -connection, cancellation or timeout attachments. The legacy public `CONNECTION` -key remains declared but is no longer populated; external consumers of that -implementation attachment need compatibility review before upgrade. A proxy-generated terminal -response is marked before sending, so benign late callbacks leave it draining. -Unexpected committed-response callbacks reset only the affected HTTP/2 stream. -Allocation samples and a reproducible local probe are in -[handler/qualification](handler/qualification/README.md). +Proxy callbacks execute on the downstream exchange's I/O thread, without a +callback-wide monitor or shared mutable callback slot. The public `CONNECTION` +attachment is populated when a connection is accepted. Retry generations discard +late acquisition/send/response callbacks and release the previous attempt's resources +before accepting a new connection. A proxy-generated terminal response is marked +before sending, so benign late callbacks leave it draining. Pushed exchanges inherit +the selected policy and an independent MDC snapshot, with their own cancellation +state. Unexpected committed-response callbacks reset only the affected HTTP/2 stream. +Local allocation probes are separate qualification artifacts, not repository source +or CI qualification; gateway performance remains an explicit review gate. Custom asynchronous handlers must explicitly capture before registering their callbacks; this contract cannot intercept arbitrary third-party hand-offs. Body-reading requests capture lazily only when a read suspends. ingress-proxy uses diff --git a/exception/src/main/java/com/networknt/exception/ExceptionHandler.java b/exception/src/main/java/com/networknt/exception/ExceptionHandler.java index bceb6ab227..6526fd4931 100644 --- a/exception/src/main/java/com/networknt/exception/ExceptionHandler.java +++ b/exception/src/main/java/com/networknt/exception/ExceptionHandler.java @@ -86,11 +86,12 @@ public void handleRequest(final HttpServerExchange exchange) throws Exception { ex -> Handler.next(ex, next)); } finally { // Initial chain execution owns request-thread cleanup. Resumed frames - // use their executor's MDC_CONTEXT and must not erase that context. + // scope and restore the executing thread's MDC themselves. MDC.clear(); } } + @SuppressWarnings("java:S3776") // Preserve the established exception-policy branches while adapting asynchronous frames. private void handleContinuation(HttpServerExchange exchange, HttpHandler continuation) throws Exception { // Log each executing frame, including asynchronous continuations. String path = exchange.getRequestPath(); @@ -100,40 +101,44 @@ private void handleContinuation(HttpServerExchange exchange, HttpHandler continu continuation.handleRequest(exchange); } catch (Throwable e) { logger.error("ExceptionHandler error processing - path: {}", path, e); - if (exchange.isResponseChannelAvailable()) writeFailureResponse(exchange, e); + if(exchange.isResponseChannelAvailable()) { + //handle exceptions + if(e instanceof RuntimeException) { + // check if it is FrameworkException which is subclass of RuntimeException. + if(e instanceof FrameworkException fe) { + exchange.setStatusCode(fe.getStatus().getStatusCode()); + exchange.getResponseHeaders().put(Headers.CONTENT_TYPE, "application/json"); + exchange.getResponseSender().send(fe.getStatus().toString()); + if (logger.isErrorEnabled()) logger.error(fe.getStatus().toString(), e); + } else { + setExchangeStatus(exchange, STATUS_RUNTIME_EXCEPTION); + } + } else { + if(e instanceof ApiException ae) { + exchange.setStatusCode(ae.getStatus().getStatusCode()); + exchange.getResponseHeaders().put(Headers.CONTENT_TYPE, "application/json"); + exchange.getResponseSender().send(ae.getStatus().toString()); + if (logger.isErrorEnabled()) logger.error(ae.getStatus().toString(), e); + } else if(e instanceof ClientException){ + Status clientStatus = ClientException.getStatus(); + if(clientStatus.getStatusCode() == 0){ + setExchangeStatus(exchange, STATUS_UNCAUGHT_EXCEPTION); + } else { + exchange.setStatusCode(clientStatus.getStatusCode()); + exchange.getResponseHeaders().put(Headers.CONTENT_TYPE, "application/json"); + exchange.getResponseSender().send(clientStatus.toString()); + } + + } else { + setExchangeStatus(exchange, STATUS_UNCAUGHT_EXCEPTION); + } + } + } } finally { if(logger.isDebugEnabled()) logger.debug("ExceptionHandler frame completed - path: {}", path); } } - private void writeFailureResponse(HttpServerExchange exchange, Throwable failure) { - if (failure instanceof RuntimeException) { - if (failure instanceof FrameworkException framework) { - Status status = framework.getStatus(); - sendStatus(exchange, status); - logger.error("{}", status, failure); - } else setExchangeStatus(exchange, STATUS_RUNTIME_EXCEPTION); - return; - } - if (failure instanceof ApiException api) { - Status status = api.getStatus(); - sendStatus(exchange, status); - logger.error("{}", status, failure); - return; - } - if (failure instanceof ClientException && ClientException.getStatus().getStatusCode() != 0) { - sendStatus(exchange, ClientException.getStatus()); - return; - } - setExchangeStatus(exchange, STATUS_UNCAUGHT_EXCEPTION); - } - - private static void sendStatus(HttpServerExchange exchange, Status status) { - exchange.setStatusCode(status.getStatusCode()); - exchange.getResponseHeaders().put(Headers.CONTENT_TYPE, "application/json"); - exchange.getResponseSender().send(status.toString()); - } - @Override public HttpHandler getNext() { return next; diff --git a/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java b/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java index 45aebdc624..ed99c2e1fd 100644 --- a/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java +++ b/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java @@ -2,6 +2,9 @@ import com.networknt.config.Config; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; import com.networknt.handler.HandlerContinuation; import com.networknt.status.Status; import io.undertow.Undertow; @@ -117,6 +120,10 @@ void typedFailuresRetainResponseStatusAndBody(String kind, boolean resumed) thro case "client-without-status" -> new ClientException(new Status()); default -> throw new IllegalArgumentException(kind); }; + var events = new ListAppender(); + var log = (Logger) ExceptionHandler.logger; + events.start(); + log.addAppender(events); ExceptionHandler selected = new ExceptionHandler(); HttpHandler failing = exchange -> { throw failure; }; selected.setNext(exchange -> { @@ -133,9 +140,16 @@ void typedFailuresRetainResponseStatusAndBody(String kind, boolean resumed) thro assertEquals(missingStatus ? 400 : 429, response.statusCode(), response.body()); assertTrue(response.body().contains(missingStatus ? "ERR10011" : "TEST_TYPED_FAILURE"), response.body()); assertEquals("application/json", response.headers().firstValue("Content-Type").orElse("")); + if (kind.equals("framework") || kind.equals("api")) + assertTrue(events.list.stream().anyMatch(event -> status.toString().equals(event.getMessage()) + && event.getThrowableProxy() != null + && failure.getClass().getName().equals(event.getThrowableProxy().getClassName())), + "Keep the existing raw JSON log message and throwable"); } finally { server.stop(); ClientException.setStatus(previousClientStatus); + log.detachAppender(events); + events.stop(); } } diff --git a/handler/qualification/MdcAllocationProbe.java b/handler/qualification/MdcAllocationProbe.java deleted file mode 100644 index ceca65c366..0000000000 --- a/handler/qualification/MdcAllocationProbe.java +++ /dev/null @@ -1,57 +0,0 @@ -import com.networknt.handler.HandlerContinuation; -import com.networknt.httpstring.AttachmentConstants; -import io.undertow.server.HttpHandler; -import io.undertow.server.HttpServerExchange; -import org.slf4j.MDC; -import java.lang.management.ManagementFactory; -import java.util.*; - -/** Warmed allocation-only microprobe. Excludes registration, transport, logging and gateway throughput. */ -public class MdcAllocationProbe { - private static volatile int sink; - private static final int WARMUP = 100000; - private static final int ITERATIONS = 100000; - private static final com.sun.management.ThreadMXBean BEAN = - (com.sun.management.ThreadMXBean) ManagementFactory.getThreadMXBean(); - private static final long THREAD = Thread.currentThread().threadId(); - private static final HttpHandler CALLBACK = exchange -> sink++; - - private static void measure(String name, Runnable operation) { - for (int i = 0; i < WARMUP; i++) operation.run(); - for (int sample = 0; sample < 3; sample++) { - long before = BEAN.getThreadAllocatedBytes(THREAD); - for (int i = 0; i < ITERATIONS; i++) operation.run(); - long bytes = BEAN.getThreadAllocatedBytes(THREAD) - before; - System.out.printf(Locale.ROOT, - "ALLOC:{\"case\":\"%s\",\"sample\":%d,\"bytes_per_frame\":%.4f}%n", - name, sample, (double) bytes / ITERATIONS); - } - } - - private static void frame(String name, Map request, Map caller) { - HttpServerExchange exchange = new HttpServerExchange(null); - exchange.putAttachment(AttachmentConstants.MDC_CONTEXT, request); - Runnable task = HandlerContinuation.task(exchange, HandlerContinuation.capture(exchange, CALLBACK)); - MDC.clear(); - if (!caller.isEmpty()) MDC.setContextMap(caller); - measure(name, task); - Map restored = MDC.getCopyOfContextMap(); - if (!Objects.equals(caller, restored == null ? Map.of() : restored)) - throw new AssertionError("Caller MDC was not restored"); - } - - public static void main(String[] args) { - if (!BEAN.isThreadAllocatedMemorySupported()) throw new IllegalStateException("Allocation counter unavailable"); - BEAN.setThreadAllocatedMemoryEnabled(true); - measure("callback_only", () -> sink++); - frame("empty_request_empty_caller", Map.of(), Map.of()); - Map three = Map.of("correlation", "a", "trace", "b", "client", "c"); - frame("three_fields_matching_caller", three, three); - frame("three_fields_different_caller", three, Map.of("worker", "outside")); - frame("three_fields_different_caller_same_size", three, Map.of("worker", "outside", "other", "prior", "tenant", "outside")); - Map twelve = new HashMap<>(); - for (int i = 0; i < 12; i++) twelve.put("field" + i, "value" + i); - frame("twelve_fields_different_caller", twelve, Map.of("worker", "outside")); - MDC.clear(); - } -} diff --git a/handler/qualification/README.md b/handler/qualification/README.md deleted file mode 100644 index af9085509b..0000000000 --- a/handler/qualification/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# Continuation allocation evidence for issue #2820 - -Build the focused tests first, then run from the repository root: - -```sh -mvn -pl handler -am test -Dtest=HandlerContinuationTest -Dsurefire.failIfNoSpecifiedTests=false -python3 handler/qualification/run-allocation.py -``` - -`allocation.json` records raw samples, warmed per-resume medians, JDK details and -source hashes. The standalone probe reuses one captured runnable and increments a -volatile counter so the callback cannot disappear. It verifies caller MDC restoration. -It does not measure capture/registration, transport, logging, monitor contention, -or end-to-end gateway throughput. JIT and machine variation affect the figures. - -A resume still snapshots the caller MDC and restores it on exit, including when -entry already matches the request map: the callback can change MDC. That restoration -is required for shared I/O/worker threads. Nested calls for the same policy reuse -the scope. Proxy request state caches its captured handler and runnable so callbacks -do not recreate those adapters. One frame is allocated per outer resume. - -These local figures quantify remaining allocation; they do not close a gateway -performance qualification gate or establish a zero-allocation path. diff --git a/handler/qualification/allocation.json b/handler/qualification/allocation.json deleted file mode 100644 index 3abecad2b6..0000000000 --- a/handler/qualification/allocation.json +++ /dev/null @@ -1,115 +0,0 @@ -{ - "method": "ThreadMXBean warmed allocation probe: 100000 warmup, 100000 iterations per sample, three samples", - "exclusions": [ - "registration/capture", - "network transport", - "logging", - "gateway throughput" - ], - "java": "openjdk version \"25.0.3\" 2026-04-21 LTS\nOpenJDK Runtime Environment Corretto-25.0.3.9.1 (build 25.0.3+9-LTS)\nOpenJDK 64-Bit Server VM Corretto-25.0.3.9.1 (build 25.0.3+9-LTS, mixed mode, sharing)\n", - "samples": [ - { - "case": "callback_only", - "sample": 0, - "bytes_per_frame": 0.0105 - }, - { - "case": "callback_only", - "sample": 1, - "bytes_per_frame": 0.0 - }, - { - "case": "callback_only", - "sample": 2, - "bytes_per_frame": 0.0 - }, - { - "case": "empty_request_empty_caller", - "sample": 0, - "bytes_per_frame": 60.7921 - }, - { - "case": "empty_request_empty_caller", - "sample": 1, - "bytes_per_frame": 56.0 - }, - { - "case": "empty_request_empty_caller", - "sample": 2, - "bytes_per_frame": 56.0 - }, - { - "case": "three_fields_matching_caller", - "sample": 0, - "bytes_per_frame": 848.0 - }, - { - "case": "three_fields_matching_caller", - "sample": 1, - "bytes_per_frame": 848.0 - }, - { - "case": "three_fields_matching_caller", - "sample": 2, - "bytes_per_frame": 848.0 - }, - { - "case": "three_fields_different_caller", - "sample": 0, - "bytes_per_frame": 853.378 - }, - { - "case": "three_fields_different_caller", - "sample": 1, - "bytes_per_frame": 808.0 - }, - { - "case": "three_fields_different_caller", - "sample": 2, - "bytes_per_frame": 808.0 - }, - { - "case": "three_fields_different_caller_same_size", - "sample": 0, - "bytes_per_frame": 1056.0 - }, - { - "case": "three_fields_different_caller_same_size", - "sample": 1, - "bytes_per_frame": 1056.0 - }, - { - "case": "three_fields_different_caller_same_size", - "sample": 2, - "bytes_per_frame": 1056.0 - }, - { - "case": "twelve_fields_different_caller", - "sample": 0, - "bytes_per_frame": 1288.0 - }, - { - "case": "twelve_fields_different_caller", - "sample": 1, - "bytes_per_frame": 1288.0 - }, - { - "case": "twelve_fields_different_caller", - "sample": 2, - "bytes_per_frame": 1288.0 - } - ], - "median_bytes_per_resume": { - "callback_only": 0.0, - "empty_request_empty_caller": 56.0, - "three_fields_matching_caller": 848.0, - "three_fields_different_caller": 808.0, - "three_fields_different_caller_same_size": 1056.0, - "twelve_fields_different_caller": 1288.0 - }, - "source_sha256": { - "handler/src/main/java/com/networknt/handler/HandlerContinuation.java": "b650d3da1471fe7c96cab45e9458dc3b80bd0cc8143767ed87acad1da748fc01", - "handler/src/main/java/com/networknt/handler/RequestMdcContext.java": "e9788732232232700d33a63ea24eeb0a6a52899dc0a5aa9730f0bf5b7b6692c5", - "proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java": "88195df98d81d55ea70549127fa7258b5464983945e4eae68d9531bd1fa83927" - } -} diff --git a/handler/qualification/mutation-control.txt b/handler/qualification/mutation-control.txt deleted file mode 100644 index 63d0a30b53..0000000000 --- a/handler/qualification/mutation-control.txt +++ /dev/null @@ -1,9 +0,0 @@ -Negative control for productionResumesReadStableSnapshotsWhileWritersPublish: -Temporarily replace immutable CAS publication with one mutable HashMap and return -an unmodifiable view over that same backing map. Run the production resume/helper -reader-writer test. It fails deterministically after the held reader observes the -writer's publication: - -A retained generation must not change during publication ==> expected: <0> but was: <2000> - -The original source was restored byte-for-byte before the final passing runs. diff --git a/handler/qualification/run-allocation.py b/handler/qualification/run-allocation.py deleted file mode 100644 index 5be950f2fb..0000000000 --- a/handler/qualification/run-allocation.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python3 -"""Run after HandlerContinuationTest has built this worktree; JDK allocation counters required.""" -from pathlib import Path -import hashlib -import json -import statistics -import subprocess -import tempfile -import xml.etree.ElementTree as ET - -folder = Path(__file__).resolve().parent -root = folder.parents[1] -report = ET.parse(root / "handler/target/surefire-reports/TEST-com.networknt.handler.HandlerContinuationTest.xml") -classpath = next(p.attrib["value"] for p in report.findall(".//property") if p.attrib["name"] == "java.class.path") -with tempfile.TemporaryDirectory(prefix="continuation-allocation-") as output: - subprocess.run(["javac", "-cp", classpath, "-d", output, str(folder / "MdcAllocationProbe.java")], check=True) - result = subprocess.run(["java", "-Xms64m", "-Xmx256m", "-cp", output + ":" + classpath, "MdcAllocationProbe"], - check=True, text=True, capture_output=True) -samples = [json.loads(line[6:]) for line in result.stdout.splitlines() if line.startswith("ALLOC:")] -sources = ["handler/src/main/java/com/networknt/handler/HandlerContinuation.java", - "handler/src/main/java/com/networknt/handler/RequestMdcContext.java", - "proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java"] -data = { - "method": "ThreadMXBean warmed allocation probe: 100000 warmup, 100000 iterations per sample, three samples", - "exclusions": ["registration/capture", "network transport", "logging", "gateway throughput"], - "java": subprocess.run(["java", "-version"], text=True, capture_output=True, check=True).stderr, - "samples": samples, - "median_bytes_per_resume": {name: statistics.median(s["bytes_per_frame"] for s in samples if s["case"] == name) - for name in dict.fromkeys(s["case"] for s in samples)}, - "source_sha256": {name: hashlib.sha256((root / name).read_bytes()).hexdigest() for name in sources}, -} -(folder / "allocation.json").write_text(json.dumps(data, indent=2) + "\n") -print(json.dumps(data["median_bytes_per_resume"], indent=2)) diff --git a/handler/src/main/java/com/networknt/handler/HandlerContinuation.java b/handler/src/main/java/com/networknt/handler/HandlerContinuation.java index 257d5b8102..05806fdbfc 100644 --- a/handler/src/main/java/com/networknt/handler/HandlerContinuation.java +++ b/handler/src/main/java/com/networknt/handler/HandlerContinuation.java @@ -13,28 +13,40 @@ /** * Carries the active exception boundary across an asynchronous hand-off. Boundary - * policy is captured from the executing frame, never stored on shared middleware. - * Capture before returning from the current chain frame; invoke the captured - * handler through Undertow's dispatch or root-handler machinery. + * policy belongs to the exchange. A frame only selects nested policy overrides. + * Initialize before the first hand-off; subsequent captures can occur off-frame. + * Invoke captured handlers through Undertow's dispatch or root-handler machinery. */ public final class HandlerContinuation { private static final Logger LOG = LoggerFactory.getLogger(HandlerContinuation.class); - private static final Boundary UNHANDLED = (exchange, continuation) -> continuation.handleRequest(exchange); private record Frame(HttpServerExchange exchange, CapturedBoundary boundary, boolean mdcScoped) { } private static final ThreadLocal CURRENT_FRAME = new ThreadLocal<>(); @FunctionalInterface private interface ScopedAction { void run() throws E; } - private record CapturedBoundary(Boundary boundary, RequestMdcContext context) implements UnaryOperator { + private record CapturedBoundary(UnaryOperator boundary, RequestState state) implements UnaryOperator { @Override public HttpHandler apply(HttpHandler continuation) { return new CapturedHandler(this, continuation); } } - private record CapturedHandler(CapturedBoundary policy, HttpHandler continuation) implements HttpHandler { + private static final class CapturedHandler implements HttpHandler { + private final CapturedBoundary policy; + private final HttpHandler handler; + + private CapturedHandler(CapturedBoundary policy, HttpHandler continuation) { + this(policy, continuation, true); + } + + private CapturedHandler(CapturedBoundary policy, HttpHandler continuation, boolean enterBoundary) { + this.policy = policy; + this.handler = enterBoundary + ? policy.boundary().apply(ex -> withMdc(ex, () -> continuation.handleRequest(ex))) : continuation; + } + @Override public void handleRequest(HttpServerExchange exchange) throws Exception { - withMdc(exchange, () -> policy.boundary().handle(exchange, continuation)); + withMdc(exchange, () -> handler.handleRequest(exchange)); } private void withMdc(HttpServerExchange exchange, ScopedAction callback) throws E { @@ -48,7 +60,7 @@ private void withMdc(HttpServerExchange exchange, ScopedAc CURRENT_FRAME.set(new Frame(exchange, policy, true)); try { if (!reuseMdc) { - Map requestMdc = policy.context().snapshot(); + Map requestMdc = policy.state().mdc.snapshot(); if (!requestMdc.equals(previousMdc == null ? Map.of() : previousMdc)) setMdc(requestMdc); } callback.run(); @@ -91,8 +103,10 @@ public static void execute(HttpServerExchange exchange, Boundary boundary, HttpHandler continuation) throws Exception { Objects.requireNonNull(boundary); Frame previous = CURRENT_FRAME.get(); - RequestMdcContext context = context(exchange); - CURRENT_FRAME.set(new Frame(exchange, new CapturedBoundary(boundary, context), + RequestState state = RequestState.initialize(exchange); + UnaryOperator wrapper = next -> ex -> boundary.handle(ex, next); + state.rememberBoundary(wrapper); + CURRENT_FRAME.set(new Frame(exchange, new CapturedBoundary(wrapper, state), previous != null && previous.exchange() == exchange && previous.mdcScoped())); try { boundary.handle(exchange, continuation); } finally { restoreFrame(previous); } @@ -107,7 +121,8 @@ public static HttpHandler capture(HttpServerExchange exchange, HttpHandler conti } /** - * Captures before registering a callback whose result determines its eventual + * Captures the entry policy even outside an executing frame; an active nested + * frame overrides that policy. Capture before registering a callback whose result determines its eventual * work. Apply the returned wrapper inside that callback. Framework hand-offs * must opt in; arbitrary third-party callbacks cannot be intercepted here. * Captures the exception policy and the stable request MDC holder. At resume, @@ -116,20 +131,20 @@ public static HttpHandler capture(HttpServerExchange exchange, HttpHandler conti * for the same exchange reuse the active scope. */ public static UnaryOperator capture(HttpServerExchange exchange) { + RequestState state = RequestState.initialize(exchange); Frame frame = CURRENT_FRAME.get(); - Boundary boundary = frame != null && frame.exchange() == exchange ? frame.boundary().boundary() : UNHANDLED; - return new CapturedBoundary(boundary, context(exchange)); - } - - // Resume paths use the captured holder, never the exchange attachment map. - static RequestMdcContext activeContext(HttpServerExchange exchange) { - Frame frame = CURRENT_FRAME.get(); - return frame != null && frame.exchange() == exchange ? frame.boundary().context() : null; + UnaryOperator boundary = frame != null && frame.exchange() == exchange + ? frame.boundary().boundary() : state.boundary(); + return new CapturedBoundary(boundary, state); } - private static RequestMdcContext context(HttpServerExchange exchange) { - RequestMdcContext active = activeContext(exchange); - return active != null ? active : RequestMdcContext.initialize(exchange); + /** Give a child exchange its own state while retaining the captured policy and request MDC. */ + public static UnaryOperator inherit(HttpServerExchange child, UnaryOperator captured) { + if (!(captured instanceof CapturedBoundary parent)) + throw new IllegalArgumentException("Expected a captured continuation policy"); + RequestState state = RequestState.initialize(child, parent.state().mdc.snapshot()); + state.rememberBoundary(parent.boundary()); + return new CapturedBoundary(parent.boundary(), state); } private static void restoreFrame(Frame previous) { @@ -158,6 +173,19 @@ public static Runnable task(HttpServerExchange exchange, HttpHandler continuatio }; } + /** + * Schedule work already accepted by a boundary onto its required transport + * thread. Restore that captured MDC/scope without invoking the policy twice. + * Unhandled failures use task() cleanup; an asynchronous boundary owns any + * custom cross-thread error handling of its own. + */ + public static Runnable acceptedTask(HttpServerExchange exchange, UnaryOperator captured, + HttpHandler continuation) { + if (!(captured instanceof CapturedBoundary policy)) + throw new IllegalArgumentException("Expected a captured continuation policy"); + return task(exchange, new CapturedHandler(policy, continuation, false)); + } + /** Preserve the original failure even if the response or stream cleanup fails. */ @SuppressWarnings("java:S1181") // Error cleanup must not mask the original VM/application Error. private static void finishFailedRequest(HttpServerExchange exchange, Throwable failure) { diff --git a/handler/src/main/java/com/networknt/handler/RequestMdcContext.java b/handler/src/main/java/com/networknt/handler/RequestMdcContext.java index 7d5638054f..7f1321b8cd 100644 --- a/handler/src/main/java/com/networknt/handler/RequestMdcContext.java +++ b/handler/src/main/java/com/networknt/handler/RequestMdcContext.java @@ -1,6 +1,5 @@ package com.networknt.handler; -import com.networknt.httpstring.AttachmentConstants; import io.undertow.server.HttpServerExchange; import java.util.AbstractMap; @@ -19,7 +18,7 @@ public final class RequestMdcContext extends AbstractMap { private final AtomicReference> current; - private RequestMdcContext(Map initial) { + RequestMdcContext(Map initial) { current = new AtomicReference<>(initial == null || initial.isEmpty() ? Map.of() : Collections.unmodifiableMap(new HashMap<>(initial))); } @@ -30,13 +29,7 @@ private RequestMdcContext(Map initial) { * Legacy Map-valued context is migrated once in that serial frame. */ public static RequestMdcContext initialize(HttpServerExchange exchange) { - RequestMdcContext active = HandlerContinuation.activeContext(exchange); - if (active != null) return active; - Map attached = exchange.getAttachment(AttachmentConstants.MDC_CONTEXT); - if (attached instanceof RequestMdcContext holder) return holder; - RequestMdcContext holder = new RequestMdcContext(attached); - exchange.putAttachment(AttachmentConstants.MDC_CONTEXT, holder); - return holder; + return RequestState.initialize(exchange).mdc; } /** One immutable generation suitable for coherent iteration by external readers. */ diff --git a/handler/src/main/java/com/networknt/handler/RequestState.java b/handler/src/main/java/com/networknt/handler/RequestState.java new file mode 100644 index 0000000000..8560f8a0ab --- /dev/null +++ b/handler/src/main/java/com/networknt/handler/RequestState.java @@ -0,0 +1,51 @@ +package com.networknt.handler; + +import com.networknt.httpstring.AttachmentConstants; +import io.undertow.server.HttpHandler; +import io.undertow.server.HttpServerExchange; +import io.undertow.util.AttachmentKey; + +import java.lang.ref.WeakReference; +import java.util.Map; +import java.util.WeakHashMap; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.UnaryOperator; + +/** Exchange-owned state with an independent lookup for callbacks on other threads. */ +final class RequestState { + private static final AttachmentKey KEY = AttachmentKey.create(RequestState.class); + // Weak keys AND values: a user boundary may itself reference its exchange. + // The exchange attachment, rather than this index, owns the state's lifetime. + private static final Map> INDEX = new WeakHashMap<>(); + final RequestMdcContext mdc; + private final AtomicReference> entryBoundary = new AtomicReference<>(); + + private RequestState(RequestMdcContext mdc) { this.mdc = mdc; } + + static RequestState initialize(HttpServerExchange exchange) { return initialize(exchange, Map.of()); } + + static RequestState initialize(HttpServerExchange exchange, Map inheritedMdc) { + synchronized (INDEX) { + WeakReference reference = INDEX.get(exchange); + RequestState existing = reference == null ? null : reference.get(); + if (existing != null) return existing; + // First use is serial, before hand-off. Later lookups never touch the + // unsynchronized attachment map, including off-frame helper calls. + Map attached = exchange.getAttachment(AttachmentConstants.MDC_CONTEXT); + RequestMdcContext mdc = attached instanceof RequestMdcContext holder ? holder + : new RequestMdcContext(attached == null ? inheritedMdc : attached); + RequestState state = new RequestState(mdc); + exchange.putAttachment(AttachmentConstants.MDC_CONTEXT, mdc); + exchange.putAttachment(KEY, state); + INDEX.put(exchange, new WeakReference<>(state)); + return state; + } + } + + void rememberBoundary(UnaryOperator boundary) { entryBoundary.compareAndSet(null, boundary); } + + UnaryOperator boundary() { + UnaryOperator boundary = entryBoundary.get(); + return boundary == null ? UnaryOperator.identity() : boundary; + } +} diff --git a/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java b/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java index 1d483913bf..55b74de469 100644 --- a/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java +++ b/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java @@ -25,6 +25,79 @@ import static org.mockito.Mockito.*; class HandlerContinuationTest { + @Test + void offFrameCaptureRetainsEntryPolicyAndHolderOnAnotherThread() throws Exception { + var exchange = spy(new HttpServerExchange(null)); + var selected = new AtomicInteger(); + HandlerContinuation.execute(exchange, (ex, next) -> { + try { next.handleRequest(ex); } + catch (IOException _) { selected.incrementAndGet(); } + }, ex -> publicationMiddleware().addHandlerMDCContext(ex, "request", "entry")); + var holder = RequestMdcContext.initialize(exchange); + clearInvocations(exchange); + try (var worker = Executors.newSingleThreadExecutor()) { + worker.submit(() -> { + assertSame(holder, RequestMdcContext.initialize(exchange)); + HandlerContinuation.capture(exchange, ex -> { + assertEquals("entry", MDC.get("request")); + throw new IOException("off-frame failure"); + }).handleRequest(exchange); + return null; + }).get(5, TimeUnit.SECONDS); + } + assertEquals(1, selected.get()); + verify(exchange, never()).getAttachment(any()); + verify(exchange, never()).putAttachment(any(), any()); + } + + @Test + void deferredBoundaryRetainsItsContinuationAndRestoresWorkerMdc() throws Exception { + var exchange = new HttpServerExchange(null); + var pending = new AtomicReference(); + HandlerContinuation.execute(exchange, (ex, next) -> pending.set(next), ex -> { + publicationMiddleware().addHandlerMDCContext(ex, "request", "deferred"); + }); + pending.getAndSet(null).handleRequest(exchange); + var calls = new AtomicInteger(); + HandlerContinuation.task(exchange, HandlerContinuation.capture(exchange, ex -> { + assertEquals("deferred", MDC.get("request")); + calls.incrementAndGet(); + })).run(); + assertEquals(0, calls.get()); + try (var worker = Executors.newSingleThreadExecutor()) { + worker.submit(() -> { + MDC.put("worker", "prior"); + try { + pending.getAndSet(null).handleRequest(exchange); + assertEquals("prior", MDC.get("worker")); + } finally { MDC.clear(); } + return null; + }).get(5, TimeUnit.SECONDS); + } + assertEquals(1, calls.get()); + } + + @Test + void inheritedExchangeOwnsItsMdcAndUsesTheSelectedPolicy() throws Exception { + var parent = new HttpServerExchange(null); + var child = new HttpServerExchange(null); + var policy = new AtomicReference>(); + var seen = new ArrayList(); + HandlerContinuation.execute(parent, (ex, next) -> { seen.add(ex); next.handleRequest(ex); }, ex -> { + publicationMiddleware().addHandlerMDCContext(ex, "request", "parent"); + policy.set(HandlerContinuation.capture(ex)); + }); + seen.clear(); + HandlerContinuation.inherit(child, policy.get()).apply(ex -> { + assertSame(child, ex); + assertEquals("parent", MDC.get("request")); + publicationMiddleware().addHandlerMDCContext(ex, "request", "child"); + }).handleRequest(child); + assertEquals(List.of(child), seen); + assertEquals("parent", RequestMdcContext.initialize(parent).snapshot().get("request")); + assertEquals("child", RequestMdcContext.initialize(child).snapshot().get("request")); + } + @Test void activePolicyTracksIdentityAndExchangeAndRestoresAfterNestedFailure() throws Exception { HttpServerExchange first = new HttpServerExchange(null); @@ -282,7 +355,7 @@ void nestedCaptureRestoresEnclosingBoundaryAndRetainsSelectedInstances() throws assertEquals(List.of("inner", "nested", "outer", "enclosing"), seen); seen.clear(); HandlerContinuation.capture(exchange, ignored -> seen.add("unwrapped")).handleRequest(exchange); - assertEquals(List.of("unwrapped"), seen, "Neither frame may leave a boundary behind"); + assertEquals(List.of("outer", "unwrapped"), seen, "Off-frame capture retains the exchange entry boundary"); } @Test diff --git a/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java b/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java index e7feeb1436..4229e83507 100644 --- a/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java +++ b/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java @@ -368,36 +368,43 @@ public String toString() { } } - /** State is created serially, captured directly, and serialized by its own monitor. */ + /** Mutable transport state is confined to the downstream exchange's I/O thread. */ private static final class ProxyContinuation { private final UnaryOperator boundary; - private final Runnable resumed; - private Consumer callback; private ProxyConnection connection; private ClientExchange request; private boolean stopping; private volatile XnioExecutor.Key timeoutKey; + private long attempt; - @SuppressWarnings("java:S1181") // Release the upstream stream before rethrowing fatal Errors unchanged. - private ProxyContinuation(HttpServerExchange exchange, UnaryOperator boundary) { - this.boundary = boundary; - this.resumed = HandlerContinuation.task(exchange, boundary.apply(ex -> { - try { callback.accept(ex); } - catch (Exception | Error failure) { - abortUpstream(this); - throw failure; - } - })); - } + private ProxyContinuation(UnaryOperator boundary) { this.boundary = boundary; } - private synchronized void run(HttpServerExchange exchange, Consumer callback) { + private void run(HttpServerExchange exchange, Consumer callback) { + if (!exchange.isInIoThread()) { + exchange.getIoThread().execute(() -> run(exchange, callback)); + return; + } if (HandlerContinuation.isActive(exchange, boundary)) { - callback.accept(exchange); + acceptedCallback(exchange, callback); return; } - this.callback = callback; - try { resumed.run(); } - finally { this.callback = null; } + HandlerContinuation.task(exchange, boundary.apply(ex -> acceptedCallback(ex, callback))).run(); + } + + @SuppressWarnings("java:S1181") // Release the upstream stream before rethrowing fatal Errors unchanged. + private void acceptedCallback(HttpServerExchange exchange, Consumer callback) { + // A user boundary may defer its continuation onto a worker. Restore + // transport ownership without entering that asynchronous policy again. + if (!exchange.isInIoThread()) { + exchange.getIoThread().execute(HandlerContinuation.acceptedTask(exchange, boundary, + ex -> acceptedCallback(ex, callback))); + return; + } + try { callback.accept(exchange); } + catch (Exception | Error failure) { + abortUpstream(this); + throw failure; + } } private void cancelTimeout() { @@ -420,16 +427,21 @@ private static void cancelClientExchange(ClientExchange request) { } else IoUtils.safeClose(request.getConnection()); } - private static void abortUpstream(ProxyContinuation state) { - state.stopping = true; + private static void releaseAttempt(ProxyContinuation state) { ClientExchange request = state.request; + ProxyConnection connection = state.connection; state.request = null; + state.connection = null; if (request != null) cancelClientExchange(request); - else { - ProxyConnection connection = state.connection; - if (connection != null && !connection.getConnection().isMultiplexingSupported()) - IoUtils.safeClose(connection.getConnection()); - } + // A stream can belong to the preceding pooled connection. Always release + // a distinct current HTTP/1 connection as well; never close an HTTP/2 parent. + if (connection != null && (request == null || request.getConnection() != connection.getConnection()) + && !connection.getConnection().isMultiplexingSupported()) IoUtils.safeClose(connection.getConnection()); + } + + private static void abortUpstream(ProxyContinuation state) { + state.stopping = true; + releaseAttempt(state); } /** Ignore callbacks owned by a terminal response; terminate only unexpected commitment. */ @@ -454,7 +466,7 @@ private final class ProxyClientHandler implements ProxyCallback ProxyClientHandler(HttpServerExchange exchange, ProxyClient.ProxyTarget target, long timeout, int maxRetryAttempts, Predicate idempotentPredicate, boolean streamingRequest) { this.exchange = exchange; - this.continuation = new ProxyContinuation(exchange, HandlerContinuation.capture(exchange)); + this.continuation = new ProxyContinuation(HandlerContinuation.capture(exchange)); this.timeout = timeout; this.maxRetryAttempts = maxRetryAttempts; this.target = target; @@ -466,7 +478,36 @@ private final class ProxyClientHandler implements ProxyCallback public void run() { continuationTask(exchange, continuation, ex -> { if (!stopCallbackIfTerminal(ex, continuation)) - proxyClient.getConnection(this.target, ex, this, -1, TimeUnit.MILLISECONDS); + acquire(this.target, -1); + }).run(); + } + + private void acquire(ProxyClient.ProxyTarget target, long remaining) { + // A retry owns new resources. Drop and cancel the previous stream before + // accepting any resources from this attempt. + long attempt = ++continuation.attempt; + releaseAttempt(continuation); + proxyClient.getConnection(target, exchange, new ProxyCallback<>() { + @Override public void completed(HttpServerExchange ex, ProxyConnection connection) { + continuationTask(ex, continuation, resumed -> { + if (attempt != continuation.attempt) { + if (!connection.getConnection().isMultiplexingSupported()) IoUtils.safeClose(connection.getConnection()); + } else ProxyClientHandler.this.completed(resumed, connection); + }).run(); + } + @Override public void failed(HttpServerExchange ex) { failedAttempt(ex, attempt); } + @Override public void queuedRequestFailed(HttpServerExchange ex) { failedAttempt(ex, attempt); } + @Override public void couldNotResolveBackend(HttpServerExchange ex) { + continuationTask(ex, continuation, resumed -> { + if (attempt == continuation.attempt) unresolvedBackend(resumed); + }).run(); + } + }, remaining, TimeUnit.MILLISECONDS); + } + + private void failedAttempt(HttpServerExchange ex, long attempt) { + continuationTask(ex, continuation, resumed -> { + if (attempt == continuation.attempt) handleFailed(resumed); }).run(); } @@ -479,6 +520,7 @@ public void completed(final HttpServerExchange exchange, final ProxyConnection c } LOG.trace("ProxyCallback completed for exchange: {}", ex); continuation.connection = connection; + ex.putAttachment(CONNECTION, connection); new ProxyAction(connection, ex, requestHeaders, rewriteHostHeader, reuseXForwarded, ex.isRequestComplete() ? this : null, idempotentPredicate, urlRewriteRules, methodRewriteRules, queryParamRewriteRules, headerRewriteRules, streamingRequest, streamResponseContentTypes, streamIdleTimeout, streamResponseHeaderOverwrite, continuation).run(); }).run(); } @@ -506,7 +548,7 @@ private void handleFailed(final HttpServerExchange exchange) { return; } long remaining = timeout > 0 ? timeout - time : -1; - proxyClient.getConnection(target, exchange, this, remaining, TimeUnit.MILLISECONDS); + acquire(target, remaining); } @Override @@ -546,9 +588,9 @@ void cancel(final HttpServerExchange exchange) { private void cancelRequest(final HttpServerExchange exchange) { if (exchange.isComplete() || continuation.stopping) return; + final ProxyConnection connectionAttachment = continuation.connection; abortUpstream(continuation); //NOTE: this method is called only in context of timeouts. - final ProxyConnection connectionAttachment = continuation.connection; if (connectionAttachment != null) { ClientConnection clientConnection = connectionAttachment.getConnection(); @@ -579,6 +621,7 @@ private void cancelRequest(final HttpServerExchange exchange) { private static class ProxyAction implements Runnable { private final ProxyContinuation continuation; private final ProxyConnection clientConnection; + private final long attempt; private final HttpServerExchange exchange; private final Map requestHeaders; private final boolean rewriteHostHeader; @@ -601,6 +644,7 @@ private static class ProxyAction implements Runnable { boolean streamingRequest, List streamResponseContentTypes, int streamIdleTimeout, List streamResponseHeaderOverwrite, ProxyContinuation continuation) { this.continuation = continuation; + this.attempt = continuation.attempt; this.clientConnection = clientConnection; this.exchange = exchange; this.requestHeaders = requestHeaders; @@ -983,7 +1027,7 @@ private ClientCallback boundaryCallback(ClientCallback { - if (!ex.isComplete() && !continuation.stopping) callback.failed(failure); + if (!ex.isComplete() && !continuation.stopping && attempt == continuation.attempt) callback.failed(failure); }).run(); } }; @@ -1009,7 +1053,7 @@ public void completed(final ClientExchange result) { result.setResponseListener(boundaryCallback(new ResponseCallback(ProxyAction.this))); - final IoExceptionHandler handler = new IoExceptionHandler(exchange, clientConnection.getConnection()); + final IoExceptionHandler handler = new IoExceptionHandler(exchange, continuation, attempt); if (requiresContinueResponse) { this.prepRequestChannelForContinue(result, handler); @@ -1075,25 +1119,30 @@ private void prepRequestChannelForContinue(ClientExchange ce, IoExceptionHandler private void handleServerPush(ClientExchange ce) { ce.setPushHandler((originalRequest, pushedRequest) -> { - - LOG.debug("Sending push request {} received from {} to target {} for exchange {}", pushedRequest.getRequest(), r, host, exchange); - - final ClientRequest req = pushedRequest.getRequest(); - - exchange.getConnection().pushResource(req.getPath(), req.getMethod(), req.getRequestHeaders(), exchange -> { - String path = req.getPath(); - int i = path.indexOf("?"); - - if (i > 0) - path = path.substring(0, i); - - String targetPath = path; - exchange.dispatch(INLINE_EXECUTOR, continuationTask(exchange, continuation, ex -> new ProxyAction(new ProxyConnection(pushedRequest.getConnection(), targetPath), ex, requestHeaders, rewriteHostHeader, reuseXForwarded, null, idempotentPredicate, urlRewriteRules, methodRewriteRules, queryParamRewriteRules, headerRewriteRules, false, streamResponseContentTypes, streamIdleTimeout, streamResponseHeaderOverwrite, continuation).run())); - }); + continuationTask(exchange, continuation, parent -> { + if (parent.isComplete() || continuation.stopping) { + cancelClientExchange(pushedRequest); + return; + } + LOG.debug("Sending push request {} received from {} to target {} for exchange {}", pushedRequest.getRequest(), r, host, parent); + final ClientRequest req = pushedRequest.getRequest(); + boolean accepted = parent.getConnection().pushResource(req.getPath(), req.getMethod(), req.getRequestHeaders(), child -> { + String path = req.getPath(); + int i = path.indexOf("?"); + if (i > 0) path = path.substring(0, i); + String targetPath = path; + ProxyContinuation pushed = new ProxyContinuation(HandlerContinuation.inherit(child, continuation.boundary)); + ProxyConnection pushedConnection = new ProxyConnection(pushedRequest.getConnection(), targetPath); + child.putAttachment(CONNECTION, pushedConnection); + pushed.connection = pushedConnection; + child.dispatch(INLINE_EXECUTOR, continuationTask(child, pushed, ex -> new ProxyAction(pushedConnection, ex, requestHeaders, rewriteHostHeader, reuseXForwarded, null, idempotentPredicate, urlRewriteRules, methodRewriteRules, queryParamRewriteRules, headerRewriteRules, false, streamResponseContentTypes, streamIdleTimeout, streamResponseHeaderOverwrite, pushed).run())); + }); + if (!accepted) cancelClientExchange(pushedRequest); + }).run(); return true; }); - } + })); } @@ -1139,6 +1188,7 @@ private static final class ResponseCallback implements ClientCallback> headerRewriteRules; @@ -1150,6 +1200,7 @@ private static final class ResponseCallback implements ClientCallback 0) { @@ -1405,6 +1456,7 @@ private static final class HTTPTrailerChannelListener implements ChannelListener private final Attachable target; private final HttpServerExchange exchange; private final ProxyContinuation continuation; + private final long attempt; private final ProxyClientHandler proxyClientHandler; private final Predicate idempotentPredicate; @@ -1413,6 +1465,7 @@ private HTTPTrailerChannelListener(final Attachable source, final Attachable tar this.target = target; this.exchange = exchange; this.continuation = continuation; + this.attempt = continuation.attempt; this.proxyClientHandler = proxyClientHandler; this.idempotentPredicate = idempotentPredicate; } @@ -1444,47 +1497,48 @@ public void handleEvent(final StreamSinkChannel channel) { LOG.error("IOException: ", e); - continuationTask(exchange, continuation, ex -> handleFailure(ex, continuation, proxyClientHandler, idempotentPredicate, e)).run(); + continuationTask(exchange, continuation, ex -> { + if (attempt == continuation.attempt && !continuation.stopping) + handleFailure(ex, continuation, proxyClientHandler, idempotentPredicate, e); + }).run(); } catch (Exception e) { LOG.error("Exception: ", e); - continuationTask(exchange, continuation, ex -> handleFailure(ex, continuation, proxyClientHandler, idempotentPredicate, new IOException(e))).run(); + continuationTask(exchange, continuation, ex -> { + if (attempt == continuation.attempt && !continuation.stopping) + handleFailure(ex, continuation, proxyClientHandler, idempotentPredicate, new IOException(e)); + }).run(); } } } private static final class IoExceptionHandler implements ChannelExceptionHandler { - private final HttpServerExchange exchange; - private final ClientConnection clientConnection; + private final ProxyContinuation continuation; + private final long attempt; - private IoExceptionHandler(HttpServerExchange exchange, ClientConnection clientConnection) { + private IoExceptionHandler(HttpServerExchange exchange, ProxyContinuation continuation, long attempt) { this.exchange = exchange; - this.clientConnection = clientConnection; + this.continuation = continuation; + this.attempt = attempt; } @Override - public void handleException(Channel channel, IOException e) { - IoUtils.safeClose(channel); - IoUtils.safeClose(clientConnection); - - if (exchange.isResponseStarted()) { - UndertowLogger.REQUEST_IO_LOGGER.debug("Exception reading from target server", e); - - if (!exchange.isResponseStarted()) { - exchange.setStatusCode(StatusCodes.INTERNAL_SERVER_ERROR); - exchange.endExchange(); - - } else IoUtils.safeClose(exchange.getConnection()); - - } else { - UndertowLogger.REQUEST_IO_LOGGER.ioException(e); - exchange.setStatusCode(StatusCodes.INTERNAL_SERVER_ERROR); - exchange.endExchange(); - } + public void handleException(Channel channel, IOException failure) { + continuationTask(exchange, continuation, ex -> { + IoUtils.safeClose(channel); + if (attempt != continuation.attempt || continuation.stopping || ex.isComplete()) return; + abortUpstream(continuation); + UndertowLogger.REQUEST_IO_LOGGER.ioException(failure); + if (ex.isResponseStarted()) IoUtils.safeClose(ex.getConnection()); + else { + ex.setStatusCode(StatusCodes.INTERNAL_SERVER_ERROR); + ex.endExchange(); + } + }).run(); } } diff --git a/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java b/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java index 3e15d3bfc5..0e85a0c51e 100644 --- a/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java +++ b/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java @@ -6,6 +6,10 @@ import com.networknt.metrics.MetricsConfig; import com.networknt.metrics.TimeSeriesDbSender; import io.undertow.Undertow; +import io.undertow.attribute.ExchangeAttribute; +import io.undertow.util.HttpString; +import io.undertow.util.HeaderMap; +import io.undertow.util.Methods; import io.undertow.UndertowOptions; import io.undertow.client.*; import io.undertow.server.*; @@ -449,6 +453,7 @@ void callbackAfterCommitTerminatesIncompleteStreamWithoutForwarding(String kind) case "queued" -> callback.queuedRequestFailed(exchange); default -> throw new AssertionError(kind); } + awaitIoCallbacks(exchange); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new AssertionError(e); } finally { delivered.countDown(); } }); @@ -510,6 +515,7 @@ void unexpectedCommittedHttp2CallbackResetsOnlyItsStream(String kind) throws Exc case "queued" -> callback.queuedRequestFailed(exchange); default -> throw new AssertionError(kind); } + awaitIoCallbacks(exchange); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new AssertionError(e); } finally { delivered.countDown(); } }); @@ -632,6 +638,203 @@ else if (exchange.getRequestPath().equals("/healthy")) exchange.dispatch(() -> { } finally { releaseBody.countDown(); releaseHealthy.countDown(); } } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void http2PushUsesItsOwnExchangeAndSelectedBoundary(boolean rejectPush) throws Exception { + var pushedResponses = new ConcurrentHashMap>>(); + var pushObserved = new CountDownLatch(1); + HttpResponse.PushPromiseHandler acceptPush = (request, pushed, acceptor) -> { + pushedResponses.put(pushed, acceptor.apply(HttpResponse.BodyHandlers.ofString())); + pushObserved.countDown(); + }; + var parentExchange = new AtomicReference(); + var childExchange = new AtomicReference(); + var failures = new CompletableFuture(); + Set peers = ConcurrentHashMap.newKeySet(); + try (Fixture fixture = new Fixture(); HttpClient http = HttpClient.newBuilder().version(HttpClient.Version.HTTP_2).build()) { + try { + URI backend = fixture.start(exchange -> { + if (exchange.getRequestPath().equals("/parent")) { + assertTrue(exchange.getConnection().pushResource("/pushed", Methods.GET, + new HeaderMap(), child -> child.getResponseSender().send("pushed"))); + exchange.getResponseSender().send("parent"); + } else exchange.getResponseSender().send("pushed"); + }, true); + URI h2 = URI.create("h2c-prior://" + backend.getHost() + ":" + backend.getPort()); + ProxyHandler proxy = ProxyHandler.builder().setProxyClient(new LoadBalancingProxyClient().addHost(h2)) + .setMaxRequestTime(0).addRequestHeader(new HttpString("X-Push-Test"), new ExchangeAttribute() { + @Override public String readAttribute(HttpServerExchange ex) { + if (rejectPush && ex.getRequestPath().equals("/pushed")) + throw new ControlledFailure(Phase.BEFORE_SEND); + return "value"; + } + @Override public void writeAttribute(HttpServerExchange ex, String value) { + throw new UnsupportedOperationException("Read-only test attribute"); + } + }).build(); + URI front = fixture.start(exchange -> { + peers.add(exchange.getSourceAddress().getPort()); + if (exchange.getRequestPath().equals("/warm")) exchange.getResponseSender().send("warm"); + else HandlerContinuation.execute(exchange, (ex, next) -> { + if (ex.getRequestPath().equals("/pushed")) childExchange.set(ex); + else parentExchange.set(ex); + try { next.handleRequest(ex); } + catch (ControlledFailure failure) { + assertEquals("/pushed", ex.getRequestPath()); + assertEquals(Phase.BEFORE_SEND, failure.phase); + ex.setStatusCode(409).getResponseSender().send("selected child"); + } catch (Exception | Error failure) { failures.complete(failure); throw failure; } + }, proxy); + }, true); + assertEquals(HttpClient.Version.HTTP_2, http.sendAsync(HttpRequest.newBuilder(front.resolve("/warm")).build(), + HttpResponse.BodyHandlers.ofString(), acceptPush).get(5, TimeUnit.SECONDS).version()); + var parent = http.sendAsync(HttpRequest.newBuilder(front.resolve("/parent")).timeout(Duration.ofSeconds(6)).build(), + HttpResponse.BodyHandlers.ofString(), acceptPush); + assertEquals("parent", parent.get(8, TimeUnit.SECONDS).body()); + assertTrue(pushObserved.await(5, TimeUnit.SECONDS), "Must receive a real HTTP/2 push promise"); + assertEquals(1, pushedResponses.size(), "Must exercise actual HTTP/2 push"); + var pushed = pushedResponses.values().iterator().next().get(5, TimeUnit.SECONDS); + assertEquals(rejectPush ? 409 : 200, pushed.statusCode(), pushed.body()); + assertEquals(rejectPush ? "selected child" : "pushed", pushed.body()); + assertNotNull(parentExchange.get()); + assertNotNull(childExchange.get()); + assertNotSame(parentExchange.get(), childExchange.get()); + assertFalse(failures.isDone(), "Push must not corrupt the parent exchange"); + assertEquals("pushed", http.sendAsync(HttpRequest.newBuilder(front.resolve("/after")).build(), + HttpResponse.BodyHandlers.ofString(), acceptPush).get(5, TimeUnit.SECONDS).body()); + assertEquals(1, peers.size(), "The parent connection must survive its pushed response"); + } finally { http.shutdownNow(); } + } + } + + @Test + void deferredBoundaryHasNoCallbackSlotAndPreservesPublicConnectionAttachment() throws Exception { + var frontendExchange = new AtomicReference(); + var acquired = new AtomicReference(); + var boundaryCalls = new AtomicInteger(); + var callbackFailure = new CompletableFuture(); + try (Fixture fixture = new Fixture(); HttpClient http = HttpClient.newHttpClient()) { + URI backend = fixture.start(exchange -> exchange.getResponseSender().send("accepted")); + var real = new LoadBalancingProxyClient().addHost(backend); + ProxyClient client = new ProxyClient() { + @Override public ProxyTarget findTarget(HttpServerExchange ex) { return real.findTarget(ex); } + @Override public void getConnection(ProxyTarget target, HttpServerExchange ex, + ProxyCallback callback, long timeout, TimeUnit unit) { + assertTrue(ex.isInIoThread(), "No callback-wide monitor: transport work belongs to the I/O thread"); + real.getConnection(target, ex, new ProxyCallback<>() { + @Override public void completed(HttpServerExchange exchange, ProxyConnection connection) { + acquired.set(connection); + callback.completed(exchange, connection); + } + @Override public void failed(HttpServerExchange exchange) { callback.failed(exchange); } + @Override public void couldNotResolveBackend(HttpServerExchange exchange) { callback.couldNotResolveBackend(exchange); } + @Override public void queuedRequestFailed(HttpServerExchange exchange) { callback.queuedRequestFailed(exchange); } + }, timeout, unit); + } + }; + var proxy = ProxyHandler.builder().setProxyClient(client).setMaxRequestTime(0).build(); + URI front = fixture.start(exchange -> { + frontendExchange.set(exchange); + HandlerContinuation.execute(exchange, (ex, next) -> { + if (boundaryCalls.getAndIncrement() == 0) next.handleRequest(ex); + else ex.dispatch(ex.getConnection().getWorker(), () -> { + try { next.handleRequest(ex); } + catch (Throwable failure) { callbackFailure.complete(failure); } + }); + }, proxy); + }); + var response = http.send(HttpRequest.newBuilder(front).timeout(Duration.ofSeconds(8)).build(), HttpResponse.BodyHandlers.ofString()); + assertEquals("accepted", response.body()); + assertTrue(boundaryCalls.get() > 1); + assertFalse(callbackFailure.isDone()); + awaitIoCallbacks(frontendExchange.get()); + assertSame(acquired.get(), frontendExchange.get().getAttachment(ProxyHandler.CONNECTION)); + } + } + + @Test + void retryTimeoutReleasesCurrentConnectionAndIgnoresOldAcquisitionFailure() throws Exception { + var firstResponseFailure = new CompletableFuture(); + var secondSend = new CountDownLatch(1); + var connections = new CopyOnWriteArrayList(); + var acquired = new CopyOnWriteArrayList(); + var callbacks = new CopyOnWriteArrayList>(); + var frontend = new AtomicReference(); + var attempts = new AtomicInteger(); + try (Fixture fixture = new Fixture(); HttpClient http = HttpClient.newHttpClient()) { + URI backend = fixture.start(HttpServerExchange::dispatch); + ProxyClient client = new ProxyClient() { + @Override public ProxyTarget findTarget(HttpServerExchange ex) { return new ProxyTarget() {}; } + @Override public void getConnection(ProxyTarget target, HttpServerExchange exchange, + ProxyCallback callback, long timeout, TimeUnit unit) { + assertTrue(exchange.isInIoThread()); + callbacks.add(callback); + int attempt = attempts.incrementAndGet(); + UndertowClient.getInstance().connect(new ClientCallback<>() { + @Override public void completed(ClientConnection connection) { + connections.add(connection); + ClientConnection decorated = (ClientConnection) Proxy.newProxyInstance(ClientConnection.class.getClassLoader(), + new Class[]{ClientConnection.class}, (proxy, method, args) -> { + if (method.getName().equals("sendRequest")) { + if (attempt == 2) { secondSend.countDown(); return null; } + @SuppressWarnings("unchecked") var send = (ClientCallback) args[1]; + connection.sendRequest((ClientRequest) args[0], new ClientCallback<>() { + @Override public void completed(ClientExchange request) { + ClientExchange decoratedRequest = (ClientExchange) Proxy.newProxyInstance(ClientExchange.class.getClassLoader(), + new Class[]{ClientExchange.class}, (proxy, method, args) -> { + if (method.getName().equals("setResponseListener")) { + @SuppressWarnings("unchecked") var listener = (ClientCallback) args[0]; + firstResponseFailure.complete(() -> listener.failed(new IOException("first response failed"))); + return null; + } + try { return method.invoke(request, args); } + catch (InvocationTargetException e) { throw e.getCause(); } + }); + send.completed(decoratedRequest); + } + @Override public void failed(IOException e) { send.failed(e); } + }); + return null; + } + try { return method.invoke(connection, args); } + catch (InvocationTargetException e) { throw e.getCause(); } + }); + var selected = new ProxyConnection(decorated, "/"); + acquired.add(selected); + callback.completed(exchange, selected); + } + @Override public void failed(IOException e) { callback.failed(exchange); } + }, backend, exchange.getIoThread(), exchange.getConnection().getByteBufferPool(), org.xnio.OptionMap.EMPTY); + } + }; + var proxy = ProxyHandler.builder().setProxyClient(client).setMaxConnectionRetries(1).setMaxRequestTime(1500).build(); + URI front = fixture.start(exchange -> { frontend.set(exchange); proxy.handleRequest(exchange); }); + var response = http.sendAsync(HttpRequest.newBuilder(front).timeout(Duration.ofSeconds(6)).build(), HttpResponse.BodyHandlers.ofString()); + firstResponseFailure.get(5, TimeUnit.SECONDS).run(); + assertTrue(secondSend.await(5, TimeUnit.SECONDS)); + callbacks.getFirst().failed(frontend.get()); + callbacks.getFirst().queuedRequestFailed(frontend.get()); + callbacks.getFirst().couldNotResolveBackend(frontend.get()); + callbacks.getFirst().completed(frontend.get(), acquired.getFirst()); + awaitIoCallbacks(frontend.get()); + assertSame(acquired.getLast(), frontend.get().getAttachment(ProxyHandler.CONNECTION)); + assertEquals(2, attempts.get(), "An old acquisition callback cannot retry the current attempt"); + assertEquals(504, response.get(5, TimeUnit.SECONDS).statusCode()); + assertEquals(2, connections.size()); + assertFalse(connections.getFirst().isOpen()); + assertFalse(connections.getLast().isOpen(), "Timeout must release the current attempt's connection"); + } + } + + private static void awaitIoCallbacks(HttpServerExchange exchange) { + assertFalse(exchange.isInIoThread(), "Only the test thread waits for I/O ownership"); + var drained = new CompletableFuture(); + exchange.getIoThread().execute(() -> drained.complete(null)); + try { drained.get(5, TimeUnit.SECONDS); } + catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new AssertionError(e); } + catch (ExecutionException | TimeoutException e) { throw new AssertionError(e); } + } + private static ClientConnection decorateConnection(ClientConnection real, Phase phase, Supplier failure) { return (ClientConnection) Proxy.newProxyInstance(ClientConnection.class.getClassLoader(), From 2b2788d0354f63fef06b4c2c512d982caa5680f9 Mon Sep 17 00:00:00 2001 From: Steve Hu Date: Fri, 30 Oct 2026 15:31:58 -0400 Subject: [PATCH 11/15] Clarify inherited policy and MDC initialization expressions --- .../java/com/networknt/handler/HandlerContinuation.java | 8 ++++---- .../src/main/java/com/networknt/handler/RequestState.java | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/handler/src/main/java/com/networknt/handler/HandlerContinuation.java b/handler/src/main/java/com/networknt/handler/HandlerContinuation.java index 05806fdbfc..542c955c61 100644 --- a/handler/src/main/java/com/networknt/handler/HandlerContinuation.java +++ b/handler/src/main/java/com/networknt/handler/HandlerContinuation.java @@ -140,11 +140,11 @@ public static UnaryOperator capture(HttpServerExchange exchange) { /** Give a child exchange its own state while retaining the captured policy and request MDC. */ public static UnaryOperator inherit(HttpServerExchange child, UnaryOperator captured) { - if (!(captured instanceof CapturedBoundary parent)) + if (!(captured instanceof CapturedBoundary(var boundary, var parentState))) throw new IllegalArgumentException("Expected a captured continuation policy"); - RequestState state = RequestState.initialize(child, parent.state().mdc.snapshot()); - state.rememberBoundary(parent.boundary()); - return new CapturedBoundary(parent.boundary(), state); + RequestState state = RequestState.initialize(child, parentState.mdc.snapshot()); + state.rememberBoundary(boundary); + return new CapturedBoundary(boundary, state); } private static void restoreFrame(Frame previous) { diff --git a/handler/src/main/java/com/networknt/handler/RequestState.java b/handler/src/main/java/com/networknt/handler/RequestState.java index 8560f8a0ab..e80047d4d3 100644 --- a/handler/src/main/java/com/networknt/handler/RequestState.java +++ b/handler/src/main/java/com/networknt/handler/RequestState.java @@ -32,8 +32,9 @@ static RequestState initialize(HttpServerExchange exchange, Map // First use is serial, before hand-off. Later lookups never touch the // unsynchronized attachment map, including off-frame helper calls. Map attached = exchange.getAttachment(AttachmentConstants.MDC_CONTEXT); + Map initialMdc = attached == null ? inheritedMdc : attached; RequestMdcContext mdc = attached instanceof RequestMdcContext holder ? holder - : new RequestMdcContext(attached == null ? inheritedMdc : attached); + : new RequestMdcContext(initialMdc); RequestState state = new RequestState(mdc); exchange.putAttachment(AttachmentConstants.MDC_CONTEXT, mdc); exchange.putAttachment(KEY, state); From 07adade8b13154f4a1e7f31145786b41d7236f37 Mon Sep 17 00:00:00 2001 From: Steve Hu Date: Fri, 30 Oct 2026 16:23:08 -0400 Subject: [PATCH 12/15] Carry typed lexical scopes through ordered proxy callbacks --- README.md | 23 +- .../networknt/body/BodyContinuationTest.java | 2 +- .../networknt/dump/DumpContinuationTest.java | 2 +- .../networknt/exception/ExceptionHandler.java | 66 ++- .../exception/ExceptionContinuationTest.java | 67 ++- .../handler/HandlerContinuation.java | 383 +++++++++++------- .../com/networknt/handler/RequestState.java | 98 +++-- .../handler/HandlerContinuationTest.java | 234 +++++++++-- .../RequestInjectionContinuationTest.java | 3 +- .../java/com/networknt/hmac/HmacHandler.java | 2 +- .../hmac/HmacReplayAdminHandler.java | 2 +- .../hmac/HmacHandlerIntegrationTest.java | 2 +- .../networknt/mcp/McpContinuationTest.java | 2 +- .../com/networknt/handler/ProxyHandler.java | 138 +++---- .../handler/ProxyContinuationTest.java | 50 ++- 15 files changed, 698 insertions(+), 376 deletions(-) diff --git a/README.md b/README.md index 5287d87e3b..78798a2749 100644 --- a/README.md +++ b/README.md @@ -268,13 +268,14 @@ MDC updates use one stable RequestMdcContext holder with an atomic immutable-map reference. The first boundary execution or capture initializes that holder before its asynchronous hand-off; addHandlerMDCContext can also initialize it on first serial use. Later updates replace only the holder's snapshot, never its -attachment. An exchange-owned RequestState retains the entry exception policy -and MDC holder. A synchronized weak-key/weak-value index supports off-frame lookups -without reading Undertow's mutable attachment map; the exchange attachment owns the -state's lifetime. The index lock covers lookup/initialization only, never handlers -or callbacks. Captured frames read their holder directly. Thread-local frames only -select nested policy overrides and scope MDC; off-frame capture retains the entry -policy. Custom asynchronous code must initialize/capture before hand-off. +attachment. RequestState stores only MDC in a concurrent weak-identity index; it never stores +an exception policy. Completion removes the entry immediately, with weak-key cleanup +for abandoned exchanges. Captured scopes read the holder directly, avoiding index +lookups on callback resumes. An explicitly retained HandlerContinuation.Scope carries +the lexical policy across hand-offs; a new capture outside its frame has no policy. +This prevents an earlier sub-pipeline boundary from shadowing a later one. Native +Undertow HandlerWrapper policies are applied at execution, not capture time. +Custom asynchronous code must initialize/capture before hand-off. The Map-typed attachment key remains compatible with read-only consumers; direct attachment replacement or mutation is unsupported after initialization. Proxy callbacks use this scope rather than the unscoped @@ -293,11 +294,13 @@ stream without appending an error. Undertow's HTTP/2 ServerConnection.close() sends RST_STREAM for that stream, preserving healthy siblings. Late failures on completed exchanges do not rewrite a response or close a reused connection. The neutral Runnable adapter attempts a 500 before commitment or a stream reset -after commitment before propagating the original Error. Failed writes/resets are -suppressed on that Error and cannot replace it; selected exception policies keep +after commitment before propagating the original Error. Failed writes/resets cannot replace the original Error; selected exception policies keep their existing handling behavior. Ordinary exceptions use the same guarded cleanup. Proxy callbacks execute on the downstream exchange's I/O thread, without a -callback-wide monitor or shared mutable callback slot. The public `CONNECTION` +callback-wide monitor or shared mutable callback slot. A per-request FIFO queues +both foreign-thread and owner-thread arrivals, preserving submission order. Executor +rejection retires pending resources and sends the failure through the selected policy; +accepted work that fails replays only its failure through that policy. The public `CONNECTION` attachment is populated when a connection is accepted. Retry generations discard late acquisition/send/response callbacks and release the previous attempt's resources before accepting a new connection. A proxy-generated terminal response is marked diff --git a/body/src/test/java/com/networknt/body/BodyContinuationTest.java b/body/src/test/java/com/networknt/body/BodyContinuationTest.java index bd6f623f36..ba2c1f4d0c 100644 --- a/body/src/test/java/com/networknt/body/BodyContinuationTest.java +++ b/body/src/test/java/com/networknt/body/BodyContinuationTest.java @@ -20,7 +20,7 @@ void redispatchRetainsBoundary(boolean enabled) throws Exception { assertFalse(exchange.isInIoThread()); throw new IllegalStateException("downstream failure"); }); - HttpHandler root = enabled ? exchange -> HandlerContinuation.execute(exchange, (ex, task) -> { + HttpHandler root = enabled ? exchange -> HandlerContinuation.execute(exchange, task -> ex -> { try { task.handleRequest(ex); } catch (IllegalStateException _) { ex.setStatusCode(409).getResponseSender().send("selected"); } }, body) : body; diff --git a/dump/src/test/java/com/networknt/dump/DumpContinuationTest.java b/dump/src/test/java/com/networknt/dump/DumpContinuationTest.java index 4fbda15b7f..74e6c0e53e 100644 --- a/dump/src/test/java/com/networknt/dump/DumpContinuationTest.java +++ b/dump/src/test/java/com/networknt/dump/DumpContinuationTest.java @@ -21,7 +21,7 @@ void redispatchRetainsSelectedBoundary(boolean enabled) throws Exception { super.handleRequest(exchange); } }; - HttpHandler root = enabled ? exchange -> HandlerContinuation.execute(exchange, (ex, task) -> { + HttpHandler root = enabled ? exchange -> HandlerContinuation.execute(exchange, task -> ex -> { try { task.handleRequest(ex); } catch (IllegalStateException _) { ex.setStatusCode(409).getResponseSender().send("selected"); } }, handler) : handler; diff --git a/exception/src/main/java/com/networknt/exception/ExceptionHandler.java b/exception/src/main/java/com/networknt/exception/ExceptionHandler.java index 6526fd4931..1b845abc27 100644 --- a/exception/src/main/java/com/networknt/exception/ExceptionHandler.java +++ b/exception/src/main/java/com/networknt/exception/ExceptionHandler.java @@ -22,6 +22,7 @@ import com.networknt.status.Status; import io.undertow.Handlers; import io.undertow.server.HttpHandler; +import io.undertow.server.HandlerWrapper; import io.undertow.server.HttpServerExchange; import io.undertow.util.Headers; import org.slf4j.Logger; @@ -42,7 +43,7 @@ * * @author Steve Hu */ -public class ExceptionHandler implements MiddlewareHandler { +public class ExceptionHandler implements MiddlewareHandler, HandlerWrapper { static final Logger logger = LoggerFactory.getLogger(ExceptionHandler.class); private String configName = ExceptionConfig.CONFIG_NAME; @@ -82,7 +83,7 @@ public void handleRequest(final HttpServerExchange exchange) throws Exception { } try { - HandlerContinuation.execute(exchange, this::handleContinuation, + HandlerContinuation.execute(exchange, this, ex -> Handler.next(ex, next)); } finally { // Initial chain execution owns request-thread cleanup. Resumed frames @@ -91,7 +92,6 @@ public void handleRequest(final HttpServerExchange exchange) throws Exception { } } - @SuppressWarnings("java:S3776") // Preserve the established exception-policy branches while adapting asynchronous frames. private void handleContinuation(HttpServerExchange exchange, HttpHandler continuation) throws Exception { // Log each executing frame, including asynchronous continuations. String path = exchange.getRequestPath(); @@ -101,44 +101,38 @@ private void handleContinuation(HttpServerExchange exchange, HttpHandler continu continuation.handleRequest(exchange); } catch (Throwable e) { logger.error("ExceptionHandler error processing - path: {}", path, e); - if(exchange.isResponseChannelAvailable()) { - //handle exceptions - if(e instanceof RuntimeException) { - // check if it is FrameworkException which is subclass of RuntimeException. - if(e instanceof FrameworkException fe) { - exchange.setStatusCode(fe.getStatus().getStatusCode()); - exchange.getResponseHeaders().put(Headers.CONTENT_TYPE, "application/json"); - exchange.getResponseSender().send(fe.getStatus().toString()); - if (logger.isErrorEnabled()) logger.error(fe.getStatus().toString(), e); - } else { - setExchangeStatus(exchange, STATUS_RUNTIME_EXCEPTION); - } - } else { - if(e instanceof ApiException ae) { - exchange.setStatusCode(ae.getStatus().getStatusCode()); - exchange.getResponseHeaders().put(Headers.CONTENT_TYPE, "application/json"); - exchange.getResponseSender().send(ae.getStatus().toString()); - if (logger.isErrorEnabled()) logger.error(ae.getStatus().toString(), e); - } else if(e instanceof ClientException){ - Status clientStatus = ClientException.getStatus(); - if(clientStatus.getStatusCode() == 0){ - setExchangeStatus(exchange, STATUS_UNCAUGHT_EXCEPTION); - } else { - exchange.setStatusCode(clientStatus.getStatusCode()); - exchange.getResponseHeaders().put(Headers.CONTENT_TYPE, "application/json"); - exchange.getResponseSender().send(clientStatus.toString()); - } - - } else { - setExchangeStatus(exchange, STATUS_UNCAUGHT_EXCEPTION); - } - } - } + if (exchange.isResponseChannelAvailable()) writeFailure(exchange, e); } finally { if(logger.isDebugEnabled()) logger.debug("ExceptionHandler frame completed - path: {}", path); } } + @Override + public HttpHandler wrap(HttpHandler continuation) { + return exchange -> handleContinuation(exchange, continuation); + } + + private void writeFailure(HttpServerExchange exchange, Throwable failure) { + if (failure instanceof FrameworkException framework) { + sendTypedStatus(exchange, framework.getStatus(), failure, true); + } else if (failure instanceof RuntimeException) { + setExchangeStatus(exchange, STATUS_RUNTIME_EXCEPTION); + } else if (failure instanceof ApiException api) { + sendTypedStatus(exchange, api.getStatus(), failure, true); + } else if (failure instanceof ClientException) { + Status status = ClientException.getStatus(); + if (status.getStatusCode() == 0) setExchangeStatus(exchange, STATUS_UNCAUGHT_EXCEPTION); + else sendTypedStatus(exchange, status, failure, false); + } else setExchangeStatus(exchange, STATUS_UNCAUGHT_EXCEPTION); + } + + private static void sendTypedStatus(HttpServerExchange exchange, Status status, Throwable failure, boolean log) { + exchange.setStatusCode(status.getStatusCode()); + exchange.getResponseHeaders().put(Headers.CONTENT_TYPE, "application/json"); + exchange.getResponseSender().send(status.toString()); + if (log && logger.isErrorEnabled()) logger.error(status.toString(), failure); + } + @Override public HttpHandler getNext() { return next; diff --git a/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java b/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java index ed99c2e1fd..84e1dada97 100644 --- a/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java +++ b/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java @@ -1,10 +1,9 @@ package com.networknt.exception; - import com.networknt.config.Config; import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.spi.ILoggingEvent; -import ch.qos.logback.core.read.ListAppender; +import ch.qos.logback.core.AppenderBase; import com.networknt.handler.HandlerContinuation; import com.networknt.status.Status; import io.undertow.Undertow; @@ -26,8 +25,10 @@ import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.Map; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; @@ -103,6 +104,50 @@ void selectedPolicyHandlesInitialAndResumedFailures(String name, boolean async, else Config.getInstance().clearConfigCache(name); } } + @ParameterizedTest + @ValueSource(strings = {"rejected", "accepted-io", "accepted-error"}) + void dispatcherFailuresRetainFrameworkPolicy(String mode) throws Exception { + String name = "continuation-dispatch-failure"; + Config.getInstance().putInConfigCache(name, Map.of("enabled", true)); + var calls = new AtomicInteger(); + ExceptionHandler selected = new ExceptionHandler(name) { + @Override public void setExchangeStatus(HttpServerExchange exchange, String code, Object... args) { + exchange.getResponseHeaders().put(new HttpString("X-Selected"), name); + super.setExchangeStatus(exchange, code, args); + } + }; + selected.setNext(exchange -> { + var scope = HandlerContinuation.capture(exchange); + exchange.dispatch(() -> { + if (mode.equals("rejected")) { + scope.dispatcher(task -> { + throw new RejectedExecutionException("injected shutdown"); + }, () -> false, () -> { /* No transport resources in this fixture. */ }) + .execute(ex -> fail("Rejected callback ran")); + } else scope.resume(ex -> { + calls.incrementAndGet(); + if (mode.equals("accepted-error")) throw new StackOverflowError("injected fatal sentinel"); + throw new IOException("accepted I/O failure"); + }); + }); + }); + var server = Undertow.builder().addHttpListener(0, "127.0.0.1").setHandler(selected).build(); + try (var client = HttpClient.newHttpClient()) { + server.start(); + int port = ((InetSocketAddress) server.getListenerInfo().getFirst().getAddress()).getPort(); + var response = client.send(HttpRequest.newBuilder(URI.create("http://127.0.0.1:" + port + "/dispatch")) + .timeout(Duration.ofSeconds(5)).build(), HttpResponse.BodyHandlers.ofString()); + boolean rejected = mode.equals("rejected"); + assertEquals(rejected ? 500 : 400, response.statusCode(), response.body()); + assertTrue(response.body().contains(rejected ? "ERR10010" : "ERR10011"), response.body()); + assertEquals(name, response.headers().firstValue("X-Selected").orElseThrow()); + assertEquals(rejected ? 0 : 1, calls.get()); + } finally { + server.stop(); + Config.getInstance().clearConfigCache(name); + } + } + static Stream typedFailureCases() { return Stream.of("framework", "api", "client", "client-without-status") .flatMap(kind -> Stream.of(false, true).map(resumed -> Arguments.of(kind, resumed))); @@ -120,7 +165,17 @@ void typedFailuresRetainResponseStatusAndBody(String kind, boolean resumed) thro case "client-without-status" -> new ClientException(new Status()); default -> throw new IllegalArgumentException(kind); }; - var events = new ListAppender(); + var logged = new CompletableFuture(); + String expectedMessage = status.toString(); + var events = new AppenderBase() { + @Override protected void append(ILoggingEvent event) { + if (expectedMessage.equals(event.getMessage()) && event.getThrowableProxy() != null + && failure.getClass().getName().equals(event.getThrowableProxy().getClassName())) { + event.prepareForDeferredProcessing(); + logged.complete(event); + } + } + }; var log = (Logger) ExceptionHandler.logger; events.start(); log.addAppender(events); @@ -141,10 +196,8 @@ void typedFailuresRetainResponseStatusAndBody(String kind, boolean resumed) thro assertTrue(response.body().contains(missingStatus ? "ERR10011" : "TEST_TYPED_FAILURE"), response.body()); assertEquals("application/json", response.headers().firstValue("Content-Type").orElse("")); if (kind.equals("framework") || kind.equals("api")) - assertTrue(events.list.stream().anyMatch(event -> status.toString().equals(event.getMessage()) - && event.getThrowableProxy() != null - && failure.getClass().getName().equals(event.getThrowableProxy().getClassName())), - "Keep the existing raw JSON log message and throwable"); + assertEquals(expectedMessage, logged.get(5, TimeUnit.SECONDS).getMessage(), + "Await the server's typed log event, independently of response delivery"); } finally { server.stop(); ClientException.setStatus(previousClientStatus); diff --git a/handler/src/main/java/com/networknt/handler/HandlerContinuation.java b/handler/src/main/java/com/networknt/handler/HandlerContinuation.java index 542c955c61..1a35f8c0b8 100644 --- a/handler/src/main/java/com/networknt/handler/HandlerContinuation.java +++ b/handler/src/main/java/com/networknt/handler/HandlerContinuation.java @@ -1,5 +1,6 @@ package com.networknt.handler; +import io.undertow.server.HandlerWrapper; import io.undertow.server.HttpHandler; import io.undertow.server.HttpServerExchange; import io.undertow.util.StatusCodes; @@ -9,206 +10,286 @@ import java.util.Map; import java.util.Objects; -import java.util.function.UnaryOperator; - -/** - * Carries the active exception boundary across an asynchronous hand-off. Boundary - * policy belongs to the exchange. A frame only selects nested policy overrides. - * Initialize before the first hand-off; subsequent captures can occur off-frame. - * Invoke captured handlers through Undertow's dispatch or root-handler machinery. - */ +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BooleanSupplier; + +/** Lexical exception policy, carried explicitly across asynchronous hand-offs. */ public final class HandlerContinuation { private static final Logger LOG = LoggerFactory.getLogger(HandlerContinuation.class); - private record Frame(HttpServerExchange exchange, CapturedBoundary boundary, boolean mdcScoped) { } - private static final ThreadLocal CURRENT_FRAME = new ThreadLocal<>(); + private record Frame(HttpServerExchange exchange, HandlerWrapper boundary, RequestState state, + Scope scope, boolean mdcScoped) { } + private static final ThreadLocal CURRENT = new ThreadLocal<>(); + private static final Runnable NO_CLEANUP = () -> { }; + + private HandlerContinuation() { } - @FunctionalInterface - private interface ScopedAction { void run() throws E; } + /** Execute within a native Undertow wrapper; restore the enclosing lexical scope on exit. */ + public static void execute(HttpServerExchange exchange, HandlerWrapper boundary, HttpHandler continuation) throws Exception { + Objects.requireNonNull(boundary); + Frame previous = CURRENT.get(); + boolean sameExchange = previous != null && previous.exchange() == exchange; + RequestState state = sameExchange ? previous.state() : RequestState.initialize(exchange); + Scope scope = new Scope(exchange, boundary, state); + Frame entry = new Frame(exchange, boundary, state, scope, sameExchange && previous.mdcScoped()); + CURRENT.set(entry); + try { boundary.wrap(new EntryContinuation(scope, continuation, entry)).handleRequest(exchange); } + finally { restore(previous); } + } - private record CapturedBoundary(UnaryOperator boundary, RequestState state) implements UnaryOperator { - @Override public HttpHandler apply(HttpHandler continuation) { - return new CapturedHandler(this, continuation); + /** + * Capture the current lexical policy. Outside a frame the policy is absent: + * an exchange cannot identify a caller's position in the chain. Off-thread + * callers must retain the Scope captured before their hand-off. + */ + public static Scope capture(HttpServerExchange exchange) { + Frame frame = CURRENT.get(); + boolean sameExchange = frame != null && frame.exchange() == exchange; + return sameExchange ? frame.scope() : new Scope(exchange, null, RequestState.initialize(exchange)); + } + + public static HttpHandler capture(HttpServerExchange exchange, HttpHandler continuation) { + return capture(exchange).wrap(continuation); + } + + public static Runnable task(HttpServerExchange exchange, HttpHandler continuation) { + if (continuation instanceof BoundHandler bound) { + if (exchange != bound.scope.exchange) throw new IllegalArgumentException("Fork the captured scope for a different exchange"); + return bound.scope.task(bound.continuation); } + return capture(exchange).task(continuation); } - private static final class CapturedHandler implements HttpHandler { - private final CapturedBoundary policy; - private final HttpHandler handler; + /** A reusable, typed request scope. It never guesses a policy from exchange history. */ + public static final class Scope implements HandlerWrapper { + private final HttpServerExchange exchange; + private final HandlerWrapper boundary; + private final RequestState state; + private final Frame resumedFrame; - private CapturedHandler(CapturedBoundary policy, HttpHandler continuation) { - this(policy, continuation, true); + private Scope(HttpServerExchange exchange, HandlerWrapper boundary, RequestState state) { + this.exchange = exchange; + this.boundary = boundary; + this.state = state; + this.resumedFrame = new Frame(exchange, boundary, state, this, true); } - private CapturedHandler(CapturedBoundary policy, HttpHandler continuation, boolean enterBoundary) { - this.policy = policy; - this.handler = enterBoundary - ? policy.boundary().apply(ex -> withMdc(ex, () -> continuation.handleRequest(ex))) : continuation; + public boolean isActive() { + Frame frame = CURRENT.get(); + return frame != null && frame.exchange() == exchange && frame.scope() == this && frame.mdcScoped(); } - @Override public void handleRequest(HttpServerExchange exchange) throws Exception { - withMdc(exchange, () -> handler.handleRequest(exchange)); + public boolean isActive(HttpServerExchange expected) { return exchange == expected && isActive(); } + + public Scope fork(HttpServerExchange child) { + return new Scope(child, boundary, RequestState.initialize(child, state.mdc.snapshot())); } - private void withMdc(HttpServerExchange exchange, ScopedAction callback) throws E { - Frame previousFrame = CURRENT_FRAME.get(); - boolean reuseMdc = previousFrame != null && previousFrame.exchange() == exchange && previousFrame.mdcScoped(); - if (reuseMdc && previousFrame.boundary() == policy) { - callback.run(); - return; - } - Map previousMdc = reuseMdc ? null : MDC.getCopyOfContextMap(); - CURRENT_FRAME.set(new Frame(exchange, policy, true)); + @Override public HttpHandler wrap(HttpHandler continuation) { return new BoundHandler(this, continuation); } + + public Runnable task(HttpHandler continuation) { return () -> execute(continuation); } + + /** Selected policy handles ordinary failures; unhandled Errors are cleaned up then propagated. */ + public void execute(HttpHandler continuation) { + boolean returned = false; try { - if (!reuseMdc) { - Map requestMdc = policy.state().mdc.snapshot(); - if (!requestMdc.equals(previousMdc == null ? Map.of() : previousMdc)) setMdc(requestMdc); - } - callback.run(); + inContext(continuation, true); + returned = true; + } catch (Exception failure) { + finishFailedRequest(exchange); + LOG.error("Unhandled continuation failure for {}", exchange.getRequestPath(), failure); + returned = true; } finally { - if (!reuseMdc) setMdc(previousMdc); - restoreFrame(previousFrame); + if (!returned) finishFailedRequest(exchange); } } - private static void setMdc(Map context) { - if (context == null || context.isEmpty()) MDC.clear(); - else MDC.setContextMap(context); + /** Resume accepted work; replay only a failure through the selected policy, never the work itself. */ + public void resume(HttpHandler continuation) { + try { inContext(continuation, false); } + catch (Exception | Error failure) { execute(new FailureHandler(failure)); } } - } - /** True only while this exact captured policy is executing for this exchange. */ - public static boolean isActive(HttpServerExchange exchange, UnaryOperator boundary) { - Frame frame = CURRENT_FRAME.get(); - return frame != null && frame.exchange() == exchange && frame.boundary() == boundary; + public void fail(Exception failure) { execute(new FailureHandler(failure)); } + + public Dispatcher ioDispatcher(Runnable onFailure) { + return dispatcher(exchange.getIoThread(), exchange::isInIoThread, onFailure); + } + + public Dispatcher dispatcher(Executor executor, BooleanSupplier inExecutor, Runnable onFailure) { + return new Dispatcher(this, executor, inExecutor, onFailure); + } + + private void inContext(HttpHandler continuation, boolean enterBoundary) throws Exception { + Frame previous = CURRENT.get(); + boolean reuse = previous != null && previous.exchange() == exchange && previous.mdcScoped(); + boolean same = reuse && previous.scope() == this; + Map before = reuse ? null : MDC.getCopyOfContextMap(); + if (!same) CURRENT.set(resumedFrame); + try { + if (!reuse) { + Map request = state.mdc.snapshot(); + if (!request.equals(before == null ? Map.of() : before)) setMdc(request); + } + if (enterBoundary && boundary != null) { + HttpHandler contextual = continuation instanceof ScopedCallback callback && callback.scope() == this + ? continuation : new ContextualHandler(this, continuation); + boundary.wrap(contextual).handleRequest(exchange); + } else continuation.handleRequest(exchange); + } finally { + if (!reuse) setMdc(before); + if (!same) restore(previous); + } + } } - private HandlerContinuation() { } + private interface ScopedCallback extends HttpHandler { Scope scope(); } - /** - * A chain-owned exception policy, without changing its next handler. Invoke - * the supplied continuation exactly once; handle or rethrow its failure. - * Implementations must be safe to reuse concurrently across exchanges. - */ - @FunctionalInterface - @SuppressWarnings("java:S112") // Matches Undertow HttpHandler; boundaries must preserve arbitrary checked failures. - public interface Boundary { - void handle(HttpServerExchange exchange, HttpHandler continuation) throws Exception; + private record EntryContinuation(Scope scope, HttpHandler continuation, Frame entry) implements HttpHandler { + @Override public void handleRequest(HttpServerExchange exchange) throws Exception { + if (CURRENT.get() == entry) continuation.handleRequest(exchange); + else scope.inContext(continuation, false); + } } - /** - * Runs inside the supplied boundary and restores the enclosing boundary on - * return. Nested boundaries and continuations can safely capture their scope. - */ - public static void execute(HttpServerExchange exchange, Boundary boundary, - HttpHandler continuation) throws Exception { - Objects.requireNonNull(boundary); - Frame previous = CURRENT_FRAME.get(); - RequestState state = RequestState.initialize(exchange); - UnaryOperator wrapper = next -> ex -> boundary.handle(ex, next); - state.rememberBoundary(wrapper); - CURRENT_FRAME.set(new Frame(exchange, new CapturedBoundary(wrapper, state), - previous != null && previous.exchange() == exchange && previous.mdcScoped())); - try { boundary.handle(exchange, continuation); } - finally { restoreFrame(previous); } + private record BoundHandler(Scope scope, HttpHandler continuation, ContextualHandler contextual) implements HttpHandler { + private BoundHandler(Scope scope, HttpHandler continuation) { + this(scope, continuation, new ContextualHandler(scope, continuation)); + } + @Override public void handleRequest(HttpServerExchange exchange) throws Exception { + if (exchange != scope.exchange) throw new IllegalArgumentException("Fork the captured scope for a different exchange"); + scope.inContext(contextual, true); + } } - /** - * Captures the selected boundary, including its deliberate absence. No default - * exception policy is installed when the chain has none. - */ - public static HttpHandler capture(HttpServerExchange exchange, HttpHandler continuation) { - return capture(exchange).apply(continuation); + private record ContextualHandler(Scope scope, HttpHandler continuation) implements ScopedCallback { + @Override public void handleRequest(HttpServerExchange exchange) throws Exception { + if (exchange != scope.exchange) throw new IllegalArgumentException("Fork the captured scope for a different exchange"); + scope.inContext(continuation, false); + } } - /** - * Captures the entry policy even outside an executing frame; an active nested - * frame overrides that policy. Capture before registering a callback whose result determines its eventual - * work. Apply the returned wrapper inside that callback. Framework hand-offs - * must opt in; arbitrary third-party callbacks cannot be intercepted here. - * Captures the exception policy and the stable request MDC holder. At resume, - * MDC comes from its current immutable generation, not a thread-local snapshot. The - * executing thread's previous MDC is restored even on failure. Nested frames - * for the same exchange reuse the active scope. - */ - public static UnaryOperator capture(HttpServerExchange exchange) { - RequestState state = RequestState.initialize(exchange); - Frame frame = CURRENT_FRAME.get(); - UnaryOperator boundary = frame != null && frame.exchange() == exchange - ? frame.boundary().boundary() : state.boundary(); - return new CapturedBoundary(boundary, state); + private record FailureHandler(Throwable failure) implements HttpHandler { + @Override public void handleRequest(HttpServerExchange exchange) throws Exception { + if (failure instanceof Exception exception) throw exception; + if (failure instanceof Error error) throw error; + throw new IllegalArgumentException("Unsupported failure", failure); + } } - /** Give a child exchange its own state while retaining the captured policy and request MDC. */ - public static UnaryOperator inherit(HttpServerExchange child, UnaryOperator captured) { - if (!(captured instanceof CapturedBoundary(var boundary, var parentState))) - throw new IllegalArgumentException("Expected a captured continuation policy"); - RequestState state = RequestState.initialize(child, parentState.mdc.snapshot()); - state.rememberBoundary(boundary); - return new CapturedBoundary(boundary, state); + /** FIFO callbacks on an owning executor, including arrivals from that executor itself. */ + public static final class Dispatcher { + private record Event(Scope scope, HttpHandler callback, Runnable rejectedCleanup, boolean accepted) implements ScopedCallback { + @Override public void handleRequest(HttpServerExchange exchange) throws Exception { scope.inContext(callback, false); } + } + private final Scope scope; + private final Executor executor; + private final BooleanSupplier inExecutor; + private final Runnable onFailure; + private final ConcurrentLinkedQueue queue = new ConcurrentLinkedQueue<>(); + private final AtomicInteger work = new AtomicInteger(); + private final AtomicReference rejected = new AtomicReference<>(); + private final AtomicBoolean stopped = new AtomicBoolean(); + private final AtomicBoolean failureReported = new AtomicBoolean(); + private final Runnable drainTask = this::drain; + + private Dispatcher(Scope scope, Executor executor, BooleanSupplier inExecutor, Runnable onFailure) { + this.scope = scope; + this.executor = executor; + this.inExecutor = inExecutor; + this.onFailure = onFailure; + } + + public void execute(HttpHandler callback) { execute(callback, NO_CLEANUP); } + + public void execute(HttpHandler callback, Runnable rejectedCleanup) { submit(callback, rejectedCleanup, false); } + + public void resume(HttpHandler callback, Runnable rejectedCleanup) { submit(callback, rejectedCleanup, true); } + + private void submit(HttpHandler callback, Runnable rejectedCleanup, boolean accepted) { + queue.add(new Event(scope, callback, rejectedCleanup, accepted)); + if (stopped.get()) { rejectQueued(); return; } + if (work.getAndIncrement() != 0) return; + if (inExecutor.getAsBoolean()) drain(); + else { + try { executor.execute(drainTask); } + catch (RejectedExecutionException failure) { + rejected.compareAndSet(null, failure); + stopped.set(true); + rejectQueued(); + } + } + } + + private void drain() { + int missed = 1; + boolean returned = false; + try { + do { + Event event; + while ((event = queue.poll()) != null) { + if (event.accepted()) scope.resume(event); + else scope.execute(event); + } + missed = work.addAndGet(-missed); + } while (missed != 0); + returned = true; + } finally { + if (!returned) { + // Scope.execute already preserves/rethrows the fatal Error. + // Retire queued resources without running their transport work. + stopped.set(true); + rejectQueued(); + } + } + } + + private void rejectQueued() { + Event event; + while ((event = queue.poll()) != null) cleanup(event.rejectedCleanup()); + if (failureReported.compareAndSet(false, true)) { + cleanup(onFailure); + RejectedExecutionException failure = rejected.get(); + if (failure != null) scope.fail(failure); + } + } } - private static void restoreFrame(Frame previous) { - if (previous == null) CURRENT_FRAME.remove(); - else CURRENT_FRAME.set(previous); + private static void cleanup(Runnable cleanup) { + try { cleanup.run(); } + catch (Throwable ignored) { /* Retain the callback failure and retire the remaining resources. */ } } - /** - * Adapts an already captured handler for a Runnable dispatch. Unlike a root - * handler, a successful callback may leave upstream I/O pending; do not end - * that exchange. Unhandled failures return 500 before commitment, close a - * committed stream, or leave an already completed exchange alone. - */ - @SuppressWarnings("java:S1181") // Fatal errors require best-effort request cleanup before rethrow. - public static Runnable task(HttpServerExchange exchange, HttpHandler continuation) { - HttpHandler captured = continuation instanceof CapturedHandler ? continuation : capture(exchange, continuation); - return () -> { - try { captured.handleRequest(exchange); } - catch (Error fatal) { - finishFailedRequest(exchange, fatal); - throw fatal; - } catch (Exception failure) { - finishFailedRequest(exchange, failure); - LOG.error("Unhandled continuation failure for {}", exchange.getRequestPath(), failure); - } - }; + private static void restore(Frame frame) { + if (frame == null) CURRENT.remove(); + else CURRENT.set(frame); } - /** - * Schedule work already accepted by a boundary onto its required transport - * thread. Restore that captured MDC/scope without invoking the policy twice. - * Unhandled failures use task() cleanup; an asynchronous boundary owns any - * custom cross-thread error handling of its own. - */ - public static Runnable acceptedTask(HttpServerExchange exchange, UnaryOperator captured, - HttpHandler continuation) { - if (!(captured instanceof CapturedBoundary policy)) - throw new IllegalArgumentException("Expected a captured continuation policy"); - return task(exchange, new CapturedHandler(policy, continuation, false)); + private static void setMdc(Map context) { + if (context == null || context.isEmpty()) MDC.clear(); + else MDC.setContextMap(context); } - /** Preserve the original failure even if the response or stream cleanup fails. */ - @SuppressWarnings("java:S1181") // Error cleanup must not mask the original VM/application Error. - private static void finishFailedRequest(HttpServerExchange exchange, Throwable failure) { + /** Best-effort cleanup must never mask an already propagating failure. */ + private static void finishFailedRequest(HttpServerExchange exchange) { if (exchange.isComplete()) return; if (!exchange.isResponseStarted()) { try { exchange.setStatusCode(StatusCodes.INTERNAL_SERVER_ERROR); exchange.endExchange(); return; - } catch (Throwable writeFailure) { rememberCleanupFailure(failure, writeFailure); } + } catch (Throwable ignored) { + // The primary failure is retained by the caller; try stream cleanup. + } } try { - // HTTP/2 ServerConnection.close() resets only this stream. if (exchange.getConnection() != null) exchange.getConnection().close(); - } catch (Throwable closeFailure) { rememberCleanupFailure(failure, closeFailure); } - } - - @SuppressWarnings("java:S1181") // Suppression metadata can itself fail under memory pressure. - private static void rememberCleanupFailure(Throwable failure, Throwable cleanupFailure) { - if (cleanupFailure == failure) return; - try { failure.addSuppressed(cleanupFailure); } - catch (Throwable _) { - // Preserve the original failure even when suppression metadata cannot be allocated. + } catch (Throwable ignored) { + // Preserve the primary failure, including VM Errors, if close also fails. } } } diff --git a/handler/src/main/java/com/networknt/handler/RequestState.java b/handler/src/main/java/com/networknt/handler/RequestState.java index e80047d4d3..ecc51a7e7c 100644 --- a/handler/src/main/java/com/networknt/handler/RequestState.java +++ b/handler/src/main/java/com/networknt/handler/RequestState.java @@ -1,52 +1,88 @@ package com.networknt.handler; import com.networknt.httpstring.AttachmentConstants; -import io.undertow.server.HttpHandler; import io.undertow.server.HttpServerExchange; -import io.undertow.util.AttachmentKey; +import java.lang.ref.Reference; +import java.lang.ref.ReferenceQueue; import java.lang.ref.WeakReference; import java.util.Map; -import java.util.WeakHashMap; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.UnaryOperator; +import java.util.concurrent.ConcurrentHashMap; -/** Exchange-owned state with an independent lookup for callbacks on other threads. */ +/** MDC only: exception policy is carried explicitly by captured scopes. */ final class RequestState { - private static final AttachmentKey KEY = AttachmentKey.create(RequestState.class); - // Weak keys AND values: a user boundary may itself reference its exchange. - // The exchange attachment, rather than this index, owns the state's lifetime. - private static final Map> INDEX = new WeakHashMap<>(); + private interface Key { HttpServerExchange get(); } + + private static final class WeakKey extends WeakReference implements Key { + private final int hash; + private WeakKey(HttpServerExchange exchange) { + super(exchange, CLEARED); + hash = System.identityHashCode(exchange); + } + @Override public int hashCode() { return hash; } + @Override public boolean equals(Object other) { + HttpServerExchange exchange = get(); + return this == other || exchange != null && other instanceof Key key && exchange == key.get(); + } + } + + /** Reusable lookup key; never inserted and never retains an exchange after get(). */ + private static final class LookupKey implements Key { + private HttpServerExchange exchange; + @Override public HttpServerExchange get() { return exchange; } + @Override public int hashCode() { return System.identityHashCode(exchange); } + @Override public boolean equals(Object other) { + return other instanceof Key key && exchange == key.get(); + } + } + + private static final ReferenceQueue CLEARED = new ReferenceQueue<>(); + private static final ConcurrentHashMap INDEX = new ConcurrentHashMap<>(); + private static final ThreadLocal LOOKUP = ThreadLocal.withInitial(LookupKey::new); + // Values contain strings/MDC only, never a boundary closure or an exchange. final RequestMdcContext mdc; - private final AtomicReference> entryBoundary = new AtomicReference<>(); private RequestState(RequestMdcContext mdc) { this.mdc = mdc; } + static RequestState find(HttpServerExchange exchange) { + LookupKey key = LOOKUP.get(); + key.exchange = exchange; + try { return INDEX.get(key); } + finally { key.exchange = null; } + } + static RequestState initialize(HttpServerExchange exchange) { return initialize(exchange, Map.of()); } static RequestState initialize(HttpServerExchange exchange, Map inheritedMdc) { - synchronized (INDEX) { - WeakReference reference = INDEX.get(exchange); - RequestState existing = reference == null ? null : reference.get(); - if (existing != null) return existing; - // First use is serial, before hand-off. Later lookups never touch the - // unsynchronized attachment map, including off-frame helper calls. - Map attached = exchange.getAttachment(AttachmentConstants.MDC_CONTEXT); - Map initialMdc = attached == null ? inheritedMdc : attached; - RequestMdcContext mdc = attached instanceof RequestMdcContext holder ? holder - : new RequestMdcContext(initialMdc); - RequestState state = new RequestState(mdc); - exchange.putAttachment(AttachmentConstants.MDC_CONTEXT, mdc); - exchange.putAttachment(KEY, state); - INDEX.put(exchange, new WeakReference<>(state)); - return state; + RequestState existing = find(exchange); + if (existing != null) return existing; + if (exchange.isComplete()) return detached(exchange, inheritedMdc); + // Bound cleanup work; ordinary completion removes its entry immediately. + for (int count = 0; count < 16; count++) { + Reference key = CLEARED.poll(); + if (key == null) break; + INDEX.remove(key); } + WeakKey key = new WeakKey(exchange); + try { + RequestState state = INDEX.computeIfAbsent(key, ignored -> { + RequestState created = detached(exchange, inheritedMdc); + exchange.putAttachment(AttachmentConstants.MDC_CONTEXT, created.mdc); + exchange.addExchangeCompleteListener((completed, next) -> { + INDEX.remove(key); + next.proceed(); + }); + return created; + }); + if (exchange.isComplete()) INDEX.remove(key, state); + return state; + } finally { Reference.reachabilityFence(exchange); } } - void rememberBoundary(UnaryOperator boundary) { entryBoundary.compareAndSet(null, boundary); } - - UnaryOperator boundary() { - UnaryOperator boundary = entryBoundary.get(); - return boundary == null ? UnaryOperator.identity() : boundary; + private static RequestState detached(HttpServerExchange exchange, Map inheritedMdc) { + Map attached = exchange.getAttachment(AttachmentConstants.MDC_CONTEXT); + Map initial = attached == null ? inheritedMdc : attached; + RequestMdcContext context = attached instanceof RequestMdcContext holder ? holder : new RequestMdcContext(initial); + return new RequestState(context); } } diff --git a/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java b/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java index 55b74de469..9eb9f6093c 100644 --- a/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java +++ b/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java @@ -1,8 +1,16 @@ package com.networknt.handler; - import com.networknt.httpstring.AttachmentConstants; +import java.net.URI; +import java.net.InetSocketAddress; +import java.net.http.HttpResponse; +import java.net.http.HttpRequest; +import java.net.http.HttpClient; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.LinkedBlockingQueue; +import io.undertow.Undertow; import io.undertow.server.HttpHandler; +import io.undertow.server.HandlerWrapper; import io.undertow.server.HttpServerExchange; import io.undertow.server.ServerConnection; import org.junit.jupiter.api.Test; @@ -29,19 +37,26 @@ class HandlerContinuationTest { void offFrameCaptureRetainsEntryPolicyAndHolderOnAnotherThread() throws Exception { var exchange = spy(new HttpServerExchange(null)); var selected = new AtomicInteger(); - HandlerContinuation.execute(exchange, (ex, next) -> { + var saved = new AtomicReference(); + HandlerContinuation.execute(exchange, next -> ex -> { try { next.handleRequest(ex); } catch (IOException _) { selected.incrementAndGet(); } - }, ex -> publicationMiddleware().addHandlerMDCContext(ex, "request", "entry")); + }, ex -> { + publicationMiddleware().addHandlerMDCContext(ex, "request", "entry"); + saved.set(HandlerContinuation.capture(ex)); + }); var holder = RequestMdcContext.initialize(exchange); clearInvocations(exchange); try (var worker = Executors.newSingleThreadExecutor()) { worker.submit(() -> { assertSame(holder, RequestMdcContext.initialize(exchange)); - HandlerContinuation.capture(exchange, ex -> { + HttpHandler failing = ex -> { assertEquals("entry", MDC.get("request")); throw new IOException("off-frame failure"); - }).handleRequest(exchange); + }; + assertThrows(IOException.class, () -> HandlerContinuation.capture(exchange, failing).handleRequest(exchange), + "An exchange alone cannot infer lexical policy outside its frame"); + saved.get().wrap(failing).handleRequest(exchange); return null; }).get(5, TimeUnit.SECONDS); } @@ -54,15 +69,17 @@ void offFrameCaptureRetainsEntryPolicyAndHolderOnAnotherThread() throws Exceptio void deferredBoundaryRetainsItsContinuationAndRestoresWorkerMdc() throws Exception { var exchange = new HttpServerExchange(null); var pending = new AtomicReference(); - HandlerContinuation.execute(exchange, (ex, next) -> pending.set(next), ex -> { + var saved = new AtomicReference(); + HandlerContinuation.execute(exchange, next -> ex -> pending.set(next), ex -> { publicationMiddleware().addHandlerMDCContext(ex, "request", "deferred"); + saved.set(HandlerContinuation.capture(ex)); }); pending.getAndSet(null).handleRequest(exchange); var calls = new AtomicInteger(); - HandlerContinuation.task(exchange, HandlerContinuation.capture(exchange, ex -> { + saved.get().task(ex -> { assertEquals("deferred", MDC.get("request")); calls.incrementAndGet(); - })).run(); + }).run(); assertEquals(0, calls.get()); try (var worker = Executors.newSingleThreadExecutor()) { worker.submit(() -> { @@ -81,14 +98,14 @@ void deferredBoundaryRetainsItsContinuationAndRestoresWorkerMdc() throws Excepti void inheritedExchangeOwnsItsMdcAndUsesTheSelectedPolicy() throws Exception { var parent = new HttpServerExchange(null); var child = new HttpServerExchange(null); - var policy = new AtomicReference>(); + var policy = new AtomicReference(); var seen = new ArrayList(); - HandlerContinuation.execute(parent, (ex, next) -> { seen.add(ex); next.handleRequest(ex); }, ex -> { + HandlerContinuation.execute(parent, next -> ex -> { seen.add(ex); next.handleRequest(ex); }, ex -> { publicationMiddleware().addHandlerMDCContext(ex, "request", "parent"); policy.set(HandlerContinuation.capture(ex)); }); seen.clear(); - HandlerContinuation.inherit(child, policy.get()).apply(ex -> { + policy.get().fork(child).wrap(ex -> { assertSame(child, ex); assertEquals("parent", MDC.get("request")); publicationMiddleware().addHandlerMDCContext(ex, "request", "child"); @@ -104,20 +121,21 @@ void activePolicyTracksIdentityAndExchangeAndRestoresAfterNestedFailure() throws HttpServerExchange second = new HttpServerExchange(null); var policy = HandlerContinuation.capture(first); var otherPolicy = HandlerContinuation.capture(first); - assertFalse(HandlerContinuation.isActive(first, policy)); - policy.apply(ex -> { - assertTrue(HandlerContinuation.isActive(first, policy)); - assertFalse(HandlerContinuation.isActive(first, otherPolicy)); - assertFalse(HandlerContinuation.isActive(second, policy)); - assertThrows(IOException.class, () -> policy.apply(nested -> { - assertTrue(HandlerContinuation.isActive(second, policy)); - assertFalse(HandlerContinuation.isActive(first, policy)); + assertFalse(policy.isActive(first)); + policy.wrap(ex -> { + assertTrue(policy.isActive(first)); + assertFalse(otherPolicy.isActive(first)); + assertFalse(policy.isActive(second)); + var child = policy.fork(second); + assertThrows(IOException.class, () -> child.wrap(nested -> { + assertTrue(child.isActive(second)); + assertFalse(policy.isActive(first)); throw new IOException("nested failure"); }).handleRequest(second)); - assertTrue(HandlerContinuation.isActive(first, policy)); + assertTrue(policy.isActive(first)); }).handleRequest(first); - assertFalse(HandlerContinuation.isActive(first, policy)); - assertFalse(HandlerContinuation.isActive(second, policy)); + assertFalse(policy.isActive(first)); + assertFalse(policy.isActive(second)); } @Test @@ -156,7 +174,7 @@ void productionResumesReadStableSnapshotsWhileWritersPublish() throws Exception AtomicReference reader = new AtomicReference<>(); CountDownLatch readerHoldingSnapshot = new CountDownLatch(1); CountDownLatch updated = new CountDownLatch(1); - HandlerContinuation.execute(exchange, (ex, next) -> next.handleRequest(ex), ex -> { + HandlerContinuation.execute(exchange, next -> ex -> next.handleRequest(ex), ex -> { context.set(RequestMdcContext.initialize(ex)); reader.set(HandlerContinuation.capture(ex, resumed -> { assertEquals("retained", MDC.get("legacy")); @@ -193,6 +211,158 @@ void productionResumesReadStableSnapshotsWhileWritersPublish() throws Exception assertEquals("retained", context.get().snapshot().get("legacy")); } + @Test + void earlierNarrowPolicyCannotShadowLaterPolicyOrOffFrameCapture() throws Exception { + var exchange = new HttpServerExchange(null); + var narrow = new AtomicReference(); + var later = new AtomicReference(); + var seen = new ArrayList(); + HandlerWrapper first = next -> ex -> { + try { next.handleRequest(ex); } + catch (IOException _) { seen.add("narrow"); } + }; + HandlerWrapper second = next -> ex -> { + try { next.handleRequest(ex); } + catch (IOException _) { seen.add("later"); } + }; + HandlerContinuation.execute(exchange, first, ex -> narrow.set(HandlerContinuation.capture(ex))); + var outside = HandlerContinuation.capture(exchange); + HandlerContinuation.execute(exchange, second, ex -> later.set(HandlerContinuation.capture(ex))); + HttpHandler fail = ex -> { throw new IOException("selected failure"); }; + assertThrows(IOException.class, () -> outside.wrap(fail).handleRequest(exchange)); + narrow.get().wrap(fail).handleRequest(exchange); + later.get().wrap(fail).handleRequest(exchange); + assertEquals(List.of("narrow", "later"), seen); + } + + @Test + void dispatchPreservesForeignArrivalBeforeOwnerArrival() throws Exception { + var exchange = new HttpServerExchange(null); + var scheduled = new LinkedBlockingQueue(); + Thread owner = Thread.currentThread(); + var dispatcher = HandlerContinuation.capture(exchange).dispatcher(scheduled::add, + () -> Thread.currentThread() == owner, () -> fail("Unexpected dispatch rejection")); + var seen = new ArrayList(); + try (var worker = Executors.newSingleThreadExecutor()) { + worker.submit(() -> dispatcher.execute(ex -> seen.add("foreign"))).get(5, TimeUnit.SECONDS); + } + dispatcher.execute(ex -> seen.add("owner")); + assertTrue(seen.isEmpty(), "Owner arrival must not overtake an already queued foreign arrival"); + scheduled.take().run(); + assertEquals(List.of("foreign", "owner"), seen); + assertTrue(scheduled.isEmpty()); + } + + @Test + void rejectedDispatchUsesSelectedPolicyAndRetiresEachResource() throws Exception { + var exchange = new HttpServerExchange(null); + var scope = new AtomicReference(); + var failures = new AtomicInteger(); + var cleanup = new AtomicInteger(); + var abort = new AtomicInteger(); + HandlerContinuation.execute(exchange, next -> ex -> { + try { next.handleRequest(ex); } + catch (RejectedExecutionException _) { failures.incrementAndGet(); } + }, ex -> scope.set(HandlerContinuation.capture(ex))); + var dispatcher = scope.get().dispatcher(task -> { + throw new RejectedExecutionException("executor stopped"); + }, () -> false, abort::incrementAndGet); + dispatcher.execute(ex -> fail("Rejected callback must not run"), cleanup::incrementAndGet); + dispatcher.execute(ex -> fail("Stopped dispatcher must not run"), cleanup::incrementAndGet); + assertEquals(1, failures.get()); + assertEquals(1, abort.get()); + assertEquals(2, cleanup.get()); + } + + @Test + void acceptedWorkRunsOnceAndItsFailureUsesSelectedPolicy() throws Exception { + var exchange = new HttpServerExchange(null); + var scope = new AtomicReference(); + var failure = new IOException("accepted failure"); + var observed = new AtomicReference(); + var calls = new AtomicInteger(); + HandlerContinuation.execute(exchange, next -> ex -> { + try { next.handleRequest(ex); } + catch (IOException caught) { observed.set(caught); } + }, ex -> scope.set(HandlerContinuation.capture(ex))); + scope.get().resume(ex -> { calls.incrementAndGet(); throw failure; }); + assertSame(failure, observed.get()); + assertEquals(1, calls.get()); + } + + @Test + void captureDoesNotApplyWrapperUntilExecution() throws Exception { + var exchange = new HttpServerExchange(null); + var applications = new AtomicInteger(); + var captured = new AtomicReference(); + HandlerWrapper boundary = next -> { + applications.incrementAndGet(); + return next; + }; + HandlerContinuation.execute(exchange, boundary, ex -> { + captured.set(HandlerContinuation.capture(ex, resumed -> assertSame(exchange, resumed))); + assertEquals(1, applications.get()); + }); + captured.get().handleRequest(exchange); + assertEquals(2, applications.get()); + } + + @Test + void completionRemovesIndexEntryWhileRetainedScopeKeepsItsMdc() throws Exception { + var completed = new CountDownLatch(1); + var captured = new AtomicReference(); + var request = new AtomicReference(); + var holder = new AtomicReference(); + var server = Undertow.builder().addHttpListener(0, "127.0.0.1").setHandler(exchange -> { + publicationMiddleware().addHandlerMDCContext(exchange, "request", "completed"); + request.set(exchange); + holder.set(RequestMdcContext.initialize(exchange)); + captured.set(HandlerContinuation.capture(exchange)); + assertNotNull(RequestState.find(exchange)); + exchange.addExchangeCompleteListener((ex, next) -> { + next.proceed(); + completed.countDown(); + }); + exchange.getResponseSender().send("done"); + }).build(); + try (var client = HttpClient.newHttpClient()) { + server.start(); + int port = ((InetSocketAddress) server.getListenerInfo().getFirst().getAddress()).getPort(); + var response = client.send(HttpRequest.newBuilder( + URI.create("http://127.0.0.1:" + port + "/lifetime")).build(), + HttpResponse.BodyHandlers.ofString()); + assertEquals("done", response.body()); + assertTrue(completed.await(5, TimeUnit.SECONDS)); + assertNull(RequestState.find(request.get())); + assertSame(holder.get(), RequestMdcContext.initialize(request.get())); + captured.get().wrap(ex -> assertEquals("completed", MDC.get("request"))).handleRequest(request.get()); + assertNull(RequestState.find(request.get()), "Late MDC access must not reinsert a completed request"); + } finally { server.stop(); } + } + + @Test + void fatalDispatchRetiresQueuedAndFutureCallbacksWithoutMaskingError() { + var exchange = spy(new HttpServerExchange(null)); + doReturn(exchange).when(exchange).endExchange(); + var retired = new AtomicInteger(); + var aborted = new AtomicInteger(); + var fatal = new StackOverflowError("injected callback sentinel"); + var dispatcher = HandlerContinuation.capture(exchange).dispatcher(Runnable::run, () -> true, + aborted::incrementAndGet); + Error observed = assertThrows(StackOverflowError.class, () -> dispatcher.execute(ex -> { + dispatcher.execute(queued -> fail("Queued callback survived fatal failure"), () -> { + retired.incrementAndGet(); + throw new IllegalStateException("cleanup failed too"); + }); + throw fatal; + })); + assertSame(fatal, observed); + dispatcher.execute(ex -> fail("Future callback survived fatal failure"), retired::incrementAndGet); + assertEquals(2, retired.get()); + assertEquals(1, aborted.get()); + verify(exchange).endExchange(); + } + private static MiddlewareHandler publicationMiddleware() { return new MiddlewareHandler() { @Override public boolean isEnabled() { return true; } @@ -223,7 +393,7 @@ void exceptionCleanupPreservesFailureWhenStatusWriteAndResetThrow() throws Excep var exchange = spy(new HttpServerExchange(connection)); doThrow(writeFailure).when(exchange).setStatusCode(500); assertDoesNotThrow(() -> HandlerContinuation.task(exchange, ex -> { throw original; }).run()); - assertArrayEquals(new Throwable[]{writeFailure, closeFailure}, original.getSuppressed()); + verify(exchange).setStatusCode(500); verify(connection).close(); } @@ -237,14 +407,14 @@ void overlappingFramesKeepTheirOwnPolicyWithoutAttachmentAccess() throws Excepti var callsB = new AtomicInteger(); var a = new AtomicReference(); var b = new AtomicReference(); - HandlerContinuation.execute(exchange, (ex, next) -> { callsA.incrementAndGet(); next.handleRequest(ex); }, ex -> + HandlerContinuation.execute(exchange, next -> ex -> { callsA.incrementAndGet(); next.handleRequest(ex); }, ex -> a.set(HandlerContinuation.capture(ex, resumed -> { aEntered.countDown(); assertTrue(bEntered.await(5, TimeUnit.SECONDS)); HandlerContinuation.capture(resumed, child -> { /* Exercise the captured policy. */ }).handleRequest(resumed); aCaptured.countDown(); }))); - HandlerContinuation.execute(exchange, (ex, next) -> { callsB.incrementAndGet(); next.handleRequest(ex); }, ex -> + HandlerContinuation.execute(exchange, next -> ex -> { callsB.incrementAndGet(); next.handleRequest(ex); }, ex -> b.set(HandlerContinuation.capture(ex, resumed -> { assertTrue(aEntered.await(5, TimeUnit.SECONDS)); bEntered.countDown(); @@ -279,7 +449,7 @@ void failedFatalStatusWriteCannotReplaceOriginalError() throws Exception { Error observed = assertThrows(StackOverflowError.class, callback::run); assertSame(fatal, observed); assertEquals(500, exchange.getStatusCode()); - assertArrayEquals(new Throwable[]{writeFailure, closeFailure}, fatal.getSuppressed()); + verify(exchange).endExchange(); verify(connection).close(); } @@ -334,11 +504,11 @@ void runnableAdapterPropagatesFatalErrorsAndRestoresMdc() { void nestedCaptureRestoresEnclosingBoundaryAndRetainsSelectedInstances() throws Exception { HttpServerExchange exchange = new HttpServerExchange(null); List seen = new ArrayList<>(); - HandlerContinuation.Boundary outer = (ex, task) -> { + HandlerWrapper outer = task -> ex -> { seen.add("outer"); task.handleRequest(ex); }; - HandlerContinuation.Boundary inner = (ex, task) -> { + HandlerWrapper inner = task -> ex -> { seen.add("inner"); task.handleRequest(ex); }; @@ -355,7 +525,7 @@ void nestedCaptureRestoresEnclosingBoundaryAndRetainsSelectedInstances() throws assertEquals(List.of("inner", "nested", "outer", "enclosing"), seen); seen.clear(); HandlerContinuation.capture(exchange, ignored -> seen.add("unwrapped")).handleRequest(exchange); - assertEquals(List.of("outer", "unwrapped"), seen, "Off-frame capture retains the exchange entry boundary"); + assertEquals(List.of("unwrapped"), seen, "Lexical policy ends with its frame; retain a Scope across hand-off"); } @Test @@ -363,14 +533,14 @@ void absentBoundaryStaysAbsentAndFailuresRestoreScope() throws Exception { HttpServerExchange exchange = new HttpServerExchange(null); IOException failure = new IOException("failure"); HttpHandler unwrapped = HandlerContinuation.capture(exchange, ignored -> { throw failure; }); - HandlerContinuation.Boundary selected = (ex, task) -> { + HandlerWrapper selected = task -> ex -> { assertDoesNotThrow(() -> task.handleRequest(ex), "A captured absent boundary must hide the active boundary"); }; HandlerContinuation.execute(exchange, selected, ex -> assertSame(failure, assertThrows(IOException.class, () -> unwrapped.handleRequest(ex)))); assertSame(failure, assertThrows(IOException.class, () -> - HandlerContinuation.execute(exchange, (ex, task) -> { throw failure; }, ignored -> { /* The boundary throws before this continuation runs. */ }))); + HandlerContinuation.execute(exchange, task -> ex -> { throw failure; }, ignored -> { /* The boundary throws before this continuation runs. */ }))); assertSame(failure, assertThrows(IOException.class, () -> unwrapped.handleRequest(exchange))); } } diff --git a/handler/src/test/java/com/networknt/handler/RequestInjectionContinuationTest.java b/handler/src/test/java/com/networknt/handler/RequestInjectionContinuationTest.java index 78aa9463f9..d6bf634d6f 100644 --- a/handler/src/test/java/com/networknt/handler/RequestInjectionContinuationTest.java +++ b/handler/src/test/java/com/networknt/handler/RequestInjectionContinuationTest.java @@ -4,6 +4,7 @@ import com.networknt.service.SingletonServiceFactory; import io.undertow.Undertow; import io.undertow.server.HttpHandler; +import io.undertow.server.HandlerWrapper; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import org.junit.jupiter.api.Timeout; @@ -49,7 +50,7 @@ void partialBodyRetainsBoundaryOrItsAbsence(boolean enabled) throws Exception { throw new IOException("downstream"); }); CountDownLatch suspended = new CountDownLatch(1); - HandlerContinuation.Boundary boundary = (exchange, task) -> { + HandlerWrapper boundary = task -> exchange -> { try { task.handleRequest(exchange); } catch (IOException _) { exchange.setStatusCode(409).getResponseSender().send("selected boundary"); } }; diff --git a/hmac/src/main/java/com/networknt/hmac/HmacHandler.java b/hmac/src/main/java/com/networknt/hmac/HmacHandler.java index 5cd8f1387a..c919a4c477 100644 --- a/hmac/src/main/java/com/networknt/hmac/HmacHandler.java +++ b/hmac/src/main/java/com/networknt/hmac/HmacHandler.java @@ -132,7 +132,7 @@ public void handleRequest(HttpServerExchange exchange) throws Exception { var continuation = HandlerContinuation.capture(exchange); exchange.dispatch(); - stage.whenComplete((outcome, failure) -> exchange.dispatch(continuation.apply(completed -> { + stage.whenComplete((outcome, failure) -> exchange.dispatch(continuation.wrap(completed -> { if (failure != null || outcome == null) { unavailable(completed, rule, store, "reserve_failed"); } else if (outcome instanceof ReserveOutcome.Duplicate) { diff --git a/hmac/src/main/java/com/networknt/hmac/HmacReplayAdminHandler.java b/hmac/src/main/java/com/networknt/hmac/HmacReplayAdminHandler.java index 0c28d50792..105f480e35 100644 --- a/hmac/src/main/java/com/networknt/hmac/HmacReplayAdminHandler.java +++ b/hmac/src/main/java/com/networknt/hmac/HmacReplayAdminHandler.java @@ -105,7 +105,7 @@ public void handleRequest(HttpServerExchange exchange) { } var continuation = HandlerContinuation.capture(exchange); exchange.dispatch(); - removalStage.whenComplete((removed, failure) -> exchange.dispatch(continuation.apply(completed -> { + removalStage.whenComplete((removed, failure) -> exchange.dispatch(continuation.wrap(completed -> { if (failure != null) { audit(completed, "unavailable", store); setExchangeStatus(completed, HmacVerifier.HMAC_UNAVAILABLE); diff --git a/hmac/src/test/java/com/networknt/hmac/HmacHandlerIntegrationTest.java b/hmac/src/test/java/com/networknt/hmac/HmacHandlerIntegrationTest.java index 6bb128503b..c943b07218 100644 --- a/hmac/src/test/java/com/networknt/hmac/HmacHandlerIntegrationTest.java +++ b/hmac/src/test/java/com/networknt/hmac/HmacHandlerIntegrationTest.java @@ -66,7 +66,7 @@ class HmacHandlerIntegrationTest { void replayCompletionRetainsExceptionBoundary(boolean enabled) throws Exception { Context context = context("continuation-" + enabled); if (enabled) chainWrapper = handler -> exchange -> - com.networknt.handler.HandlerContinuation.execute(exchange, (ex, task) -> { + com.networknt.handler.HandlerContinuation.execute(exchange, task -> ex -> { try { task.handleRequest(ex); } catch (java.io.IOException _) { ex.setStatusCode(409).getResponseSender().send("selected"); } }, handler); diff --git a/mcp-router/src/test/java/com/networknt/mcp/McpContinuationTest.java b/mcp-router/src/test/java/com/networknt/mcp/McpContinuationTest.java index 08ce32b05d..568aa53d60 100644 --- a/mcp-router/src/test/java/com/networknt/mcp/McpContinuationTest.java +++ b/mcp-router/src/test/java/com/networknt/mcp/McpContinuationTest.java @@ -21,7 +21,7 @@ void redispatchRetainsSelectedBoundary(boolean enabled) throws Exception { super.handleRequest(exchange); } }; - HttpHandler root = enabled ? exchange -> HandlerContinuation.execute(exchange, (ex, task) -> { + HttpHandler root = enabled ? exchange -> HandlerContinuation.execute(exchange, task -> ex -> { try { task.handleRequest(ex); } catch (IllegalStateException _) { ex.setStatusCode(409).getResponseSender().send("selected"); } }, handler) : handler; diff --git a/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java b/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java index 4229e83507..8ddfafdc17 100644 --- a/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java +++ b/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java @@ -75,7 +75,6 @@ import io.undertow.server.protocol.http.HttpContinue; import java.util.function.Consumer; -import java.util.function.UnaryOperator; import java.util.stream.Collectors; /** @@ -370,41 +369,35 @@ public String toString() { /** Mutable transport state is confined to the downstream exchange's I/O thread. */ private static final class ProxyContinuation { - private final UnaryOperator boundary; + private static final Runnable NO_REJECTED_RESOURCE = () -> { }; + private final HandlerContinuation.Scope scope; + private final HandlerContinuation.Dispatcher dispatcher; private ProxyConnection connection; private ClientExchange request; private boolean stopping; private volatile XnioExecutor.Key timeoutKey; private long attempt; - private ProxyContinuation(UnaryOperator boundary) { this.boundary = boundary; } + private ProxyContinuation(HandlerContinuation.Scope scope) { + this.scope = scope; + this.dispatcher = scope.ioDispatcher(() -> abortUpstream(this)); + } - private void run(HttpServerExchange exchange, Consumer callback) { - if (!exchange.isInIoThread()) { - exchange.getIoThread().execute(() -> run(exchange, callback)); - return; - } - if (HandlerContinuation.isActive(exchange, boundary)) { - acceptedCallback(exchange, callback); - return; - } - HandlerContinuation.task(exchange, boundary.apply(ex -> acceptedCallback(ex, callback))).run(); + private void run(Consumer callback) { run(callback, NO_REJECTED_RESOURCE); } + + private void run(Consumer callback, Runnable rejectedCleanup) { + dispatcher.execute(ex -> acceptedCallback(ex, callback, rejectedCleanup), rejectedCleanup); } - @SuppressWarnings("java:S1181") // Release the upstream stream before rethrowing fatal Errors unchanged. - private void acceptedCallback(HttpServerExchange exchange, Consumer callback) { - // A user boundary may defer its continuation onto a worker. Restore - // transport ownership without entering that asynchronous policy again. + private void acceptedCallback(HttpServerExchange exchange, Consumer callback, + Runnable rejectedCleanup) { if (!exchange.isInIoThread()) { - exchange.getIoThread().execute(HandlerContinuation.acceptedTask(exchange, boundary, - ex -> acceptedCallback(ex, callback))); + dispatcher.resume(ex -> acceptedCallback(ex, callback, rejectedCleanup), rejectedCleanup); return; } - try { callback.accept(exchange); } - catch (Exception | Error failure) { - abortUpstream(this); - throw failure; - } + boolean returned = false; + try { callback.accept(exchange); returned = true; } + finally { if (!returned) abortUpstream(this); } } private void cancelTimeout() { @@ -413,13 +406,6 @@ private void cancelTimeout() { } } - /** Each asynchronous entry owns one boundary/MDC scope; internal calls reuse it. */ - private static Runnable continuationTask(HttpServerExchange exchange, - ProxyContinuation continuation, - Consumer callback) { - return () -> continuation.run(exchange, callback); - } - private static void cancelClientExchange(ClientExchange request) { if (request.getConnection().isMultiplexingSupported()) { // Close only this request's channels, never its shared HTTP/2 connection. @@ -454,7 +440,7 @@ private static boolean stopCallbackIfTerminal(HttpServerExchange exchange, Proxy return true; } - private final class ProxyClientHandler implements ProxyCallback, Runnable { + private final class ProxyClientHandler implements Runnable { private final ProxyContinuation continuation; private int tries; private final long timeout; @@ -476,10 +462,10 @@ private final class ProxyClientHandler implements ProxyCallback @Override public void run() { - continuationTask(exchange, continuation, ex -> { + continuation.run(ex -> { if (!stopCallbackIfTerminal(ex, continuation)) acquire(this.target, -1); - }).run(); + }); } private void acquire(ProxyClient.ProxyTarget target, long remaining) { @@ -489,45 +475,41 @@ private void acquire(ProxyClient.ProxyTarget target, long remaining) { releaseAttempt(continuation); proxyClient.getConnection(target, exchange, new ProxyCallback<>() { @Override public void completed(HttpServerExchange ex, ProxyConnection connection) { - continuationTask(ex, continuation, resumed -> { + continuation.run(resumed -> { if (attempt != continuation.attempt) { if (!connection.getConnection().isMultiplexingSupported()) IoUtils.safeClose(connection.getConnection()); - } else ProxyClientHandler.this.completed(resumed, connection); - }).run(); + } else ProxyClientHandler.this.acceptConnection(resumed, connection); + }, () -> { if (!connection.getConnection().isMultiplexingSupported()) IoUtils.safeClose(connection.getConnection()); }); } @Override public void failed(HttpServerExchange ex) { failedAttempt(ex, attempt); } @Override public void queuedRequestFailed(HttpServerExchange ex) { failedAttempt(ex, attempt); } @Override public void couldNotResolveBackend(HttpServerExchange ex) { - continuationTask(ex, continuation, resumed -> { + continuation.run(resumed -> { if (attempt == continuation.attempt) unresolvedBackend(resumed); - }).run(); + }); } }, remaining, TimeUnit.MILLISECONDS); } private void failedAttempt(HttpServerExchange ex, long attempt) { - continuationTask(ex, continuation, resumed -> { + continuation.run(resumed -> { if (attempt == continuation.attempt) handleFailed(resumed); - }).run(); + }); } - @Override - public void completed(final HttpServerExchange exchange, final ProxyConnection connection) { - continuationTask(exchange, continuation, ex -> { - if (stopCallbackIfTerminal(ex, continuation)) { - if (!connection.getConnection().isMultiplexingSupported()) IoUtils.safeClose(connection.getConnection()); - return; - } - LOG.trace("ProxyCallback completed for exchange: {}", ex); - continuation.connection = connection; - ex.putAttachment(CONNECTION, connection); - new ProxyAction(connection, ex, requestHeaders, rewriteHostHeader, reuseXForwarded, ex.isRequestComplete() ? this : null, idempotentPredicate, urlRewriteRules, methodRewriteRules, queryParamRewriteRules, headerRewriteRules, streamingRequest, streamResponseContentTypes, streamIdleTimeout, streamResponseHeaderOverwrite, continuation).run(); - }).run(); + private void acceptConnection(final HttpServerExchange ex, final ProxyConnection connection) { + if (stopCallbackIfTerminal(ex, continuation)) { + if (!connection.getConnection().isMultiplexingSupported()) IoUtils.safeClose(connection.getConnection()); + return; + } + LOG.trace("ProxyCallback completed for exchange: {}", ex); + continuation.connection = connection; + ex.putAttachment(CONNECTION, connection); + new ProxyAction(connection, ex, requestHeaders, rewriteHostHeader, reuseXForwarded, ex.isRequestComplete() ? this : null, idempotentPredicate, urlRewriteRules, methodRewriteRules, queryParamRewriteRules, headerRewriteRules, streamingRequest, streamResponseContentTypes, streamIdleTimeout, streamResponseHeaderOverwrite, continuation).run(); } - @Override - public void failed(final HttpServerExchange exchange) { - continuationTask(exchange, continuation, this::handleFailed).run(); + private void failed(final HttpServerExchange exchange) { + continuation.run(this::handleFailed); } private void handleFailed(final HttpServerExchange exchange) { @@ -551,16 +533,6 @@ private void handleFailed(final HttpServerExchange exchange) { acquire(target, remaining); } - @Override - public void queuedRequestFailed(HttpServerExchange exchange) { - continuationTask(exchange, continuation, this::handleFailed).run(); - } - - @Override - public void couldNotResolveBackend(HttpServerExchange ex) { - continuationTask(ex, continuation, this::unresolvedBackend).run(); - } - private void unresolvedBackend(HttpServerExchange ex) { if (ex.isComplete() || continuation.stopping) return; abortUpstream(continuation); @@ -583,7 +555,7 @@ private void unresolvedBackend(HttpServerExchange ex) { } void cancel(final HttpServerExchange exchange) { - continuationTask(exchange, continuation, this::cancelRequest).run(); + continuation.run(this::cancelRequest); } private void cancelRequest(final HttpServerExchange exchange) { @@ -1016,19 +988,19 @@ private void attachRemoteHost(ClientRequest r) { private ClientCallback boundaryCallback(ClientCallback callback) { return new ClientCallback<>() { @Override public void completed(ClientExchange result) { - continuationTask(exchange, continuation, ex -> { - if (ex.isComplete() || continuation.stopping) { + continuation.run(ex -> { + if (ex.isComplete() || continuation.stopping || attempt != continuation.attempt) { cancelClientExchange(result); return; } continuation.request = result; callback.completed(result); - }).run(); + }, () -> cancelClientExchange(result)); } @Override public void failed(IOException failure) { - continuationTask(exchange, continuation, ex -> { + continuation.run(ex -> { if (!ex.isComplete() && !continuation.stopping && attempt == continuation.attempt) callback.failed(failure); - }).run(); + }); } }; } @@ -1073,7 +1045,7 @@ public void completed(final ClientExchange result) { @Override public void failed(IOException e) { - continuationTask(exchange, continuation, ex -> handleFailure(ex, continuation, proxyClientHandler, idempotentPredicate, e)).run(); + continuation.run(ex -> handleFailure(ex, continuation, proxyClientHandler, idempotentPredicate, e)); } private void prepContinueHandler(ClientExchange result) { @@ -1119,7 +1091,7 @@ private void prepRequestChannelForContinue(ClientExchange ce, IoExceptionHandler private void handleServerPush(ClientExchange ce) { ce.setPushHandler((originalRequest, pushedRequest) -> { - continuationTask(exchange, continuation, parent -> { + continuation.run(parent -> { if (parent.isComplete() || continuation.stopping) { cancelClientExchange(pushedRequest); return; @@ -1131,14 +1103,14 @@ private void handleServerPush(ClientExchange ce) { int i = path.indexOf("?"); if (i > 0) path = path.substring(0, i); String targetPath = path; - ProxyContinuation pushed = new ProxyContinuation(HandlerContinuation.inherit(child, continuation.boundary)); + ProxyContinuation pushed = new ProxyContinuation(continuation.scope.fork(child)); ProxyConnection pushedConnection = new ProxyConnection(pushedRequest.getConnection(), targetPath); child.putAttachment(CONNECTION, pushedConnection); pushed.connection = pushedConnection; - child.dispatch(INLINE_EXECUTOR, continuationTask(child, pushed, ex -> new ProxyAction(pushedConnection, ex, requestHeaders, rewriteHostHeader, reuseXForwarded, null, idempotentPredicate, urlRewriteRules, methodRewriteRules, queryParamRewriteRules, headerRewriteRules, false, streamResponseContentTypes, streamIdleTimeout, streamResponseHeaderOverwrite, pushed).run())); + child.dispatch(INLINE_EXECUTOR, () -> pushed.run(ex -> new ProxyAction(pushedConnection, ex, requestHeaders, rewriteHostHeader, reuseXForwarded, null, idempotentPredicate, urlRewriteRules, methodRewriteRules, queryParamRewriteRules, headerRewriteRules, false, streamResponseContentTypes, streamIdleTimeout, streamResponseHeaderOverwrite, pushed).run())); }); if (!accepted) cancelClientExchange(pushedRequest); - }).run(); + }, () -> cancelClientExchange(pushedRequest)); return true; }); } @@ -1497,19 +1469,19 @@ public void handleEvent(final StreamSinkChannel channel) { LOG.error("IOException: ", e); - continuationTask(exchange, continuation, ex -> { + continuation.run(ex -> { if (attempt == continuation.attempt && !continuation.stopping) handleFailure(ex, continuation, proxyClientHandler, idempotentPredicate, e); - }).run(); + }); } catch (Exception e) { LOG.error("Exception: ", e); - continuationTask(exchange, continuation, ex -> { + continuation.run(ex -> { if (attempt == continuation.attempt && !continuation.stopping) handleFailure(ex, continuation, proxyClientHandler, idempotentPredicate, new IOException(e)); - }).run(); + }); } } @@ -1528,7 +1500,7 @@ private IoExceptionHandler(HttpServerExchange exchange, ProxyContinuation contin @Override public void handleException(Channel channel, IOException failure) { - continuationTask(exchange, continuation, ex -> { + continuation.run(ex -> { IoUtils.safeClose(channel); if (attempt != continuation.attempt || continuation.stopping || ex.isComplete()) return; abortUpstream(continuation); @@ -1538,7 +1510,7 @@ public void handleException(Channel channel, IOException failure) { ex.setStatusCode(StatusCodes.INTERNAL_SERVER_ERROR); ex.endExchange(); } - }).run(); + }); } } diff --git a/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java b/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java index 0e85a0c51e..5fe49d2ec4 100644 --- a/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java +++ b/proxy-handler/src/test/java/com/networknt/handler/ProxyContinuationTest.java @@ -118,7 +118,7 @@ void asynchronousFailuresUseSelectedBoundaryAndCloseUpstream(Phase phase, boolea } proxy.handleRequest(exchange); }; - URI front = fixture.start(enabled ? exchange -> HandlerContinuation.execute(exchange, (ex, task) -> { + URI front = fixture.start(enabled ? exchange -> HandlerContinuation.execute(exchange, task -> ex -> { depth.set(depth.get() + 1); maxDepth.accumulateAndGet(depth.get(), Math::max); try { task.handleRequest(ex); } @@ -172,7 +172,7 @@ void successfulProxyActionRemainsPendingUntilUpstreamCompletes(boolean enabled) proxy.handleRequest(exchange); }; URI front = fixture.start(enabled ? exchange -> HandlerContinuation.execute(exchange, - (ex, task) -> task.handleRequest(ex), work) : work); + task -> ex -> task.handleRequest(ex), work) : work); var response = http.send(HttpRequest.newBuilder(front).timeout(Duration.ofSeconds(8)).build(), HttpResponse.BodyHandlers.ofInputStream()); assertEquals(200, response.statusCode()); @@ -299,7 +299,7 @@ void upstreamHttp2FailureResetsOnlyTheFailedStream(String mode) throws Exception .setPathPrefixMaxRequestTime(Map.of("/fail", mode.equals("timeout") || mode.equals("late") ? 300 : 0)).build(); URI front = fixture.start(exchange -> { frontendPeers.add(exchange.getSourceAddress().getPort()); - HandlerContinuation.execute(exchange, (ex, task) -> { + HandlerContinuation.execute(exchange, task -> ex -> { try { task.handleRequest(ex); } catch (ControlledFailure _) { ex.setStatusCode(409).getResponseSender().send("selected"); } }, proxy); @@ -583,23 +583,28 @@ void lateCallbacksAfterProxy503LeaveDrainingHttp2ResponseAlone() throws Exceptio @Override public ProxyTarget findTarget(HttpServerExchange exchange) { return new ProxyTarget() {}; } @Override public void getConnection(ProxyTarget target, HttpServerExchange exchange, ProxyCallback callback, long timeout, TimeUnit unit) { - try { - assertFalse(exchange.isRequestComplete()); - callback.couldNotResolveBackend(exchange); - var responseChannel = exchange.getResponseChannel(); - if (responseChannel != null) { responseChannel.shutdownWrites(); responseChannel.flush(); } - assertTrue(exchange.isResponseStarted()); - assertFalse(exchange.isComplete(), "The 503 is still draining the request body"); - callback.completed(exchange, new ProxyConnection(reusable, "/")); - callback.failed(exchange); - callback.queuedRequestFailed(exchange); - assertTrue(exchange.getConnection().isOpen(), "Keep the draining 503 stream open"); - callbacksDone.complete(null); - } catch (Throwable failure) { callbacksDone.completeExceptionally(failure); throw new AssertionError(failure); } + assertFalse(exchange.isRequestComplete()); + callback.couldNotResolveBackend(exchange); + exchange.getIoThread().execute(() -> { + try { + // The dispatcher processes the queued terminal callback + // before this I/O task commits and probes its response. + var responseChannel = exchange.getResponseChannel(); + if (responseChannel != null) { responseChannel.shutdownWrites(); responseChannel.flush(); } + assertTrue(exchange.isResponseStarted()); + assertFalse(exchange.isComplete(), "The 503 is still draining the request body"); + callback.completed(exchange, new ProxyConnection(reusable, "/")); + callback.failed(exchange); + callback.queuedRequestFailed(exchange); + assertTrue(exchange.getConnection().isOpen(), "Keep the draining 503 stream open"); + callbacksDone.complete(null); + } catch (Exception | Error failure) { callbacksDone.completeExceptionally(failure); } + }); } }; ProxyHandler proxy = ProxyHandler.builder().setProxyClient(client).setMaxRequestTime(0).build(); try (Fixture fixture = new Fixture(); HttpClient http = HttpClient.newBuilder().version(HttpClient.Version.HTTP_2).build()) { + try { URI front = fixture.start(exchange -> { peers.add(exchange.getSourceAddress().getPort()); if (exchange.getRequestPath().equals("/fail")) proxy.handleRequest(exchange); @@ -635,7 +640,8 @@ else if (exchange.getRequestPath().equals("/healthy")) exchange.dispatch(() -> { assertEquals("accepted", http.send(HttpRequest.newBuilder(front.resolve("/after")).build(), HttpResponse.BodyHandlers.ofString()).body()); assertEquals(1, peers.size()); - } finally { releaseBody.countDown(); releaseHealthy.countDown(); } + } finally { releaseBody.countDown(); releaseHealthy.countDown(); http.shutdownNow(); } + } } @ParameterizedTest @@ -675,7 +681,7 @@ void http2PushUsesItsOwnExchangeAndSelectedBoundary(boolean rejectPush) throws E URI front = fixture.start(exchange -> { peers.add(exchange.getSourceAddress().getPort()); if (exchange.getRequestPath().equals("/warm")) exchange.getResponseSender().send("warm"); - else HandlerContinuation.execute(exchange, (ex, next) -> { + else HandlerContinuation.execute(exchange, next -> ex -> { if (ex.getRequestPath().equals("/pushed")) childExchange.set(ex); else parentExchange.set(ex); try { next.handleRequest(ex); } @@ -735,7 +741,7 @@ void deferredBoundaryHasNoCallbackSlotAndPreservesPublicConnectionAttachment() t var proxy = ProxyHandler.builder().setProxyClient(client).setMaxRequestTime(0).build(); URI front = fixture.start(exchange -> { frontendExchange.set(exchange); - HandlerContinuation.execute(exchange, (ex, next) -> { + HandlerContinuation.execute(exchange, next -> ex -> { if (boundaryCalls.getAndIncrement() == 0) next.handleRequest(ex); else ex.dispatch(ex.getConnection().getWorker(), () -> { try { next.handleRequest(ex); } @@ -755,6 +761,8 @@ void deferredBoundaryHasNoCallbackSlotAndPreservesPublicConnectionAttachment() t @Test void retryTimeoutReleasesCurrentConnectionAndIgnoresOldAcquisitionFailure() throws Exception { var firstResponseFailure = new CompletableFuture(); + var staleSendCompletion = new CompletableFuture(); + var staleResponseCompletion = new CompletableFuture(); var secondSend = new CountDownLatch(1); var connections = new CopyOnWriteArrayList(); var acquired = new CopyOnWriteArrayList(); @@ -785,11 +793,13 @@ void retryTimeoutReleasesCurrentConnectionAndIgnoresOldAcquisitionFailure() thro if (method.getName().equals("setResponseListener")) { @SuppressWarnings("unchecked") var listener = (ClientCallback) args[0]; firstResponseFailure.complete(() -> listener.failed(new IOException("first response failed"))); + staleResponseCompletion.complete(() -> listener.completed(request)); return null; } try { return method.invoke(request, args); } catch (InvocationTargetException e) { throw e.getCause(); } }); + staleSendCompletion.complete(() -> send.completed(decoratedRequest)); send.completed(decoratedRequest); } @Override public void failed(IOException e) { send.failed(e); } @@ -812,6 +822,8 @@ void retryTimeoutReleasesCurrentConnectionAndIgnoresOldAcquisitionFailure() thro var response = http.sendAsync(HttpRequest.newBuilder(front).timeout(Duration.ofSeconds(6)).build(), HttpResponse.BodyHandlers.ofString()); firstResponseFailure.get(5, TimeUnit.SECONDS).run(); assertTrue(secondSend.await(5, TimeUnit.SECONDS)); + staleSendCompletion.get(5, TimeUnit.SECONDS).run(); + staleResponseCompletion.get(5, TimeUnit.SECONDS).run(); callbacks.getFirst().failed(frontend.get()); callbacks.getFirst().queuedRequestFailed(frontend.get()); callbacks.getFirst().couldNotResolveBackend(frontend.get()); From 05445b43d0404ce46d79cf77a5392d5925463b47 Mon Sep 17 00:00:00 2001 From: Steve Hu Date: Fri, 30 Oct 2026 16:27:24 -0400 Subject: [PATCH 13/15] Simplify continuation context and retire unused callback parameters --- .../handler/HandlerContinuation.java | 74 ++++++++++--------- .../com/networknt/handler/RequestState.java | 2 +- .../com/networknt/handler/ProxyHandler.java | 14 ++-- 3 files changed, 49 insertions(+), 41 deletions(-) diff --git a/handler/src/main/java/com/networknt/handler/HandlerContinuation.java b/handler/src/main/java/com/networknt/handler/HandlerContinuation.java index 1a35f8c0b8..6ac200616c 100644 --- a/handler/src/main/java/com/networknt/handler/HandlerContinuation.java +++ b/handler/src/main/java/com/networknt/handler/HandlerContinuation.java @@ -100,11 +100,11 @@ public void execute(HttpHandler continuation) { inContext(continuation, true); returned = true; } catch (Exception failure) { - finishFailedRequest(exchange); + finishFailedRequest(); LOG.error("Unhandled continuation failure for {}", exchange.getRequestPath(), failure); returned = true; } finally { - if (!returned) finishFailedRequest(exchange); + if (!returned) finishFailedRequest(); } } @@ -135,16 +135,46 @@ private void inContext(HttpHandler continuation, boolean enterBoundary) throws E Map request = state.mdc.snapshot(); if (!request.equals(before == null ? Map.of() : before)) setMdc(request); } - if (enterBoundary && boundary != null) { - HttpHandler contextual = continuation instanceof ScopedCallback callback && callback.scope() == this - ? continuation : new ContextualHandler(this, continuation); - boundary.wrap(contextual).handleRequest(exchange); - } else continuation.handleRequest(exchange); + invoke(continuation, enterBoundary); } finally { if (!reuse) setMdc(before); if (!same) restore(previous); } } + + private void invoke(HttpHandler continuation, boolean enterBoundary) throws Exception { + if (!enterBoundary || boundary == null) { + continuation.handleRequest(exchange); + return; + } + HttpHandler contextual = continuation instanceof ScopedCallback callback && callback.scope() == this + ? continuation : new ContextualHandler(this, continuation); + boundary.wrap(contextual).handleRequest(exchange); + } + + private static void setMdc(Map context) { + if (context == null || context.isEmpty()) MDC.clear(); + else MDC.setContextMap(context); + } + + /** Best-effort cleanup must never mask an already propagating failure. */ + private void finishFailedRequest() { + if (exchange.isComplete()) return; + if (!exchange.isResponseStarted()) { + try { + exchange.setStatusCode(StatusCodes.INTERNAL_SERVER_ERROR); + exchange.endExchange(); + return; + } catch (Throwable _) { + // The primary failure is retained by the caller; try stream cleanup. + } + } + try { + if (exchange.getConnection() != null) exchange.getConnection().close(); + } catch (Throwable _) { + // Preserve the primary failure, including VM Errors, if close also fails. + } + } } private interface ScopedCallback extends HttpHandler { Scope scope(); } @@ -257,11 +287,11 @@ private void rejectQueued() { if (failure != null) scope.fail(failure); } } - } + private static void cleanup(Runnable cleanup) { + try { cleanup.run(); } + catch (Throwable _) { /* Retain the callback failure and retire the remaining resources. */ } + } - private static void cleanup(Runnable cleanup) { - try { cleanup.run(); } - catch (Throwable ignored) { /* Retain the callback failure and retire the remaining resources. */ } } private static void restore(Frame frame) { @@ -269,27 +299,5 @@ private static void restore(Frame frame) { else CURRENT.set(frame); } - private static void setMdc(Map context) { - if (context == null || context.isEmpty()) MDC.clear(); - else MDC.setContextMap(context); - } - /** Best-effort cleanup must never mask an already propagating failure. */ - private static void finishFailedRequest(HttpServerExchange exchange) { - if (exchange.isComplete()) return; - if (!exchange.isResponseStarted()) { - try { - exchange.setStatusCode(StatusCodes.INTERNAL_SERVER_ERROR); - exchange.endExchange(); - return; - } catch (Throwable ignored) { - // The primary failure is retained by the caller; try stream cleanup. - } - } - try { - if (exchange.getConnection() != null) exchange.getConnection().close(); - } catch (Throwable ignored) { - // Preserve the primary failure, including VM Errors, if close also fails. - } - } } diff --git a/handler/src/main/java/com/networknt/handler/RequestState.java b/handler/src/main/java/com/networknt/handler/RequestState.java index ecc51a7e7c..bb29c20a33 100644 --- a/handler/src/main/java/com/networknt/handler/RequestState.java +++ b/handler/src/main/java/com/networknt/handler/RequestState.java @@ -59,7 +59,7 @@ static RequestState initialize(HttpServerExchange exchange, Map if (exchange.isComplete()) return detached(exchange, inheritedMdc); // Bound cleanup work; ordinary completion removes its entry immediately. for (int count = 0; count < 16; count++) { - Reference key = CLEARED.poll(); + var key = CLEARED.poll(); if (key == null) break; INDEX.remove(key); } diff --git a/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java b/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java index 8ddfafdc17..75596d32bb 100644 --- a/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java +++ b/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java @@ -197,7 +197,7 @@ public void handleRequest(final HttpServerExchange exchange) throws Exception { if (timeout > 0) { LOG.trace("Adding timeout to exchange of {} ms to exchange: {}", effectiveMaxRequestTime, exchange); - final XnioExecutor.Key key = WorkerUtils.executeAfter(exchange.getIoThread(), () -> clientHandler.cancel(exchange), effectiveMaxRequestTime, TimeUnit.MILLISECONDS); + final XnioExecutor.Key key = WorkerUtils.executeAfter(exchange.getIoThread(), () -> clientHandler.cancel(), effectiveMaxRequestTime, TimeUnit.MILLISECONDS); clientHandler.continuation.timeoutKey = key; exchange.addExchangeCompleteListener((exchange1, nextListener) -> { LOG.trace("Exchange completed for exchange removing timeout key: {}", exchange1); @@ -481,8 +481,8 @@ private void acquire(ProxyClient.ProxyTarget target, long remaining) { } else ProxyClientHandler.this.acceptConnection(resumed, connection); }, () -> { if (!connection.getConnection().isMultiplexingSupported()) IoUtils.safeClose(connection.getConnection()); }); } - @Override public void failed(HttpServerExchange ex) { failedAttempt(ex, attempt); } - @Override public void queuedRequestFailed(HttpServerExchange ex) { failedAttempt(ex, attempt); } + @Override public void failed(HttpServerExchange ex) { failedAttempt(attempt); } + @Override public void queuedRequestFailed(HttpServerExchange ex) { failedAttempt(attempt); } @Override public void couldNotResolveBackend(HttpServerExchange ex) { continuation.run(resumed -> { if (attempt == continuation.attempt) unresolvedBackend(resumed); @@ -491,7 +491,7 @@ private void acquire(ProxyClient.ProxyTarget target, long remaining) { }, remaining, TimeUnit.MILLISECONDS); } - private void failedAttempt(HttpServerExchange ex, long attempt) { + private void failedAttempt(long attempt) { continuation.run(resumed -> { if (attempt == continuation.attempt) handleFailed(resumed); }); @@ -508,7 +508,7 @@ private void acceptConnection(final HttpServerExchange ex, final ProxyConnection new ProxyAction(connection, ex, requestHeaders, rewriteHostHeader, reuseXForwarded, ex.isRequestComplete() ? this : null, idempotentPredicate, urlRewriteRules, methodRewriteRules, queryParamRewriteRules, headerRewriteRules, streamingRequest, streamResponseContentTypes, streamIdleTimeout, streamResponseHeaderOverwrite, continuation).run(); } - private void failed(final HttpServerExchange exchange) { + private void failed() { continuation.run(this::handleFailed); } @@ -554,7 +554,7 @@ private void unresolvedBackend(HttpServerExchange ex) { } } - void cancel(final HttpServerExchange exchange) { + void cancel() { continuation.run(this::cancelRequest); } @@ -1147,7 +1147,7 @@ private static void handleFailure(HttpServerExchange exchange, ProxyContinuation IoUtils.safeClose(exchange.getConnection()); } else if (idempotentRequestPredicate.resolve(exchange) && proxyClientHandler != null) - proxyClientHandler.failed(exchange); //this will attempt a retry if configured to do so + proxyClientHandler.failed(); //this will attempt a retry if configured to do so else { abortUpstream(continuation); From 8ce60fadd70d470ec71c7a213f06b244c3eb35ab Mon Sep 17 00:00:00 2001 From: Steve Hu Date: Fri, 30 Oct 2026 16:57:12 -0400 Subject: [PATCH 14/15] Confine proxy terminal cleanup and preserve suspended listener policy --- README.md | 29 ++- .../networknt/exception/ExceptionHandler.java | 10 +- .../exception/ExceptionContinuationTest.java | 8 +- .../handler/HandlerContinuation.java | 142 +++----------- .../RequestInterceptorInjectionHandler.java | 10 +- .../com/networknt/handler/RequestState.java | 7 +- .../handler/HandlerContinuationTest.java | 149 +++++---------- .../RequestInjectionContinuationTest.java | 61 ++++++ .../handler/ProxyCallbackDispatcher.java | 103 +++++++++++ .../com/networknt/handler/ProxyHandler.java | 33 +++- .../handler/ProxyCallbackDispatcherTest.java | 175 ++++++++++++++++++ 11 files changed, 464 insertions(+), 263 deletions(-) create mode 100644 proxy-handler/src/main/java/com/networknt/handler/ProxyCallbackDispatcher.java create mode 100644 proxy-handler/src/test/java/com/networknt/handler/ProxyCallbackDispatcherTest.java diff --git a/README.md b/README.md index 78798a2749..5c35cb6079 100644 --- a/README.md +++ b/README.md @@ -269,8 +269,9 @@ reference. The first boundary execution or capture initializes that holder before its asynchronous hand-off; addHandlerMDCContext can also initialize it on first serial use. Later updates replace only the holder's snapshot, never its attachment. RequestState stores only MDC in a concurrent weak-identity index; it never stores -an exception policy. Completion removes the entry immediately, with weak-key cleanup -for abandoned exchanges. Captured scopes read the holder directly, avoiding index +an exception policy. Entries are removed through bounded ReferenceQueue cleanup +after their exchange is collected; initialization never registers a completion listener. +Captured scopes read the holder directly, avoiding index lookups on callback resumes. An explicitly retained HandlerContinuation.Scope carries the lexical policy across hand-offs; a new capture outside its frame has no policy. This prevents an earlier sub-pipeline boundary from shadowing a later one. Native @@ -284,7 +285,8 @@ availability policy after commitment; it does not forcibly close general-service responses. Its initial worker frame still clears request MDC at chain exit. BodyHandler, DumpHandler and McpHandler redispatch, HMAC replay/admin completion, and ProxyHandler acquisition, retries, timeout cancellation and client send/response -callbacks carry the selected boundary. The body-read callback scopes interceptor +callbacks carry the selected boundary. The body-read listener itself uses its +pre-suspension scope and scopes interceptor execution as well as downstream handling, and preserves pending-read lifecycle. Proxy actions reuse one captured policy; internal synchronous calls do not add boundary frames or reset MDC again. Unexpected proxy failures cancel only the @@ -294,13 +296,20 @@ stream without appending an error. Undertow's HTTP/2 ServerConnection.close() sends RST_STREAM for that stream, preserving healthy siblings. Late failures on completed exchanges do not rewrite a response or close a reused connection. The neutral Runnable adapter attempts a 500 before commitment or a stream reset -after commitment before propagating the original Error. Failed writes/resets cannot replace the original Error; selected exception policies keep +after commitment before propagating the original Error. Failed writes/resets are +attached as suppressed failures and logged without replacing the original Error; +selected exception policies keep their existing handling behavior. Ordinary exceptions use the same guarded cleanup. Proxy callbacks execute on the downstream exchange's I/O thread, without a callback-wide monitor or shared mutable callback slot. A per-request FIFO queues -both foreign-thread and owner-thread arrivals, preserving submission order. Executor -rejection retires pending resources and sends the failure through the selected policy; -accepted work that fails replays only its failure through that policy. The public `CONNECTION` +both foreign-thread and owner-thread arrivals, preserving submission order. The queue +is private to proxy-handler. Fatal cleanup runs on its I/O owner before +the queue stops. Executor rejection closes an immutable resource snapshot +published through a volatile field +and reports the failure through the selected policy once; it never reads or mutates +the owner's request/connection/attempt/stopping fields. Later stopped submissions +only close their own incoming resources. Accepted work that fails replays only its +failure through that policy. The public `CONNECTION` attachment is populated when a connection is accepted. Retry generations discard late acquisition/send/response callbacks and release the previous attempt's resources before accepting a new connection. A proxy-generated terminal response is marked @@ -309,6 +318,12 @@ the selected policy and an independent MDC snapshot, with their own cancellation state. Unexpected committed-response callbacks reset only the affected HTTP/2 stream. Local allocation probes are separate qualification artifacts, not repository source or CI qualification; gateway performance remains an explicit review gate. +The review-11 ThreadMXBean probe includes request setup and four callbacks: both +branches measured 1.25-1.31 KB with empty MDC and 2.97-3.04 KB with three fields +(20k warm-up, three 20k samples). A separate real HTTP/1.1 loopback probe includes +ExceptionHandler, ProxyHandler, upstream transport and the Java HTTP client; its +129-132 KB/request totals are not attributable solely to continuation overhead. +Neither probe is a production throughput qualification. Custom asynchronous handlers must explicitly capture before registering their callbacks; this contract cannot intercept arbitrary third-party hand-offs. Body-reading requests capture lazily only when a read suspends. ingress-proxy uses diff --git a/exception/src/main/java/com/networknt/exception/ExceptionHandler.java b/exception/src/main/java/com/networknt/exception/ExceptionHandler.java index 1b845abc27..2351104b3b 100644 --- a/exception/src/main/java/com/networknt/exception/ExceptionHandler.java +++ b/exception/src/main/java/com/networknt/exception/ExceptionHandler.java @@ -22,7 +22,6 @@ import com.networknt.status.Status; import io.undertow.Handlers; import io.undertow.server.HttpHandler; -import io.undertow.server.HandlerWrapper; import io.undertow.server.HttpServerExchange; import io.undertow.util.Headers; import org.slf4j.Logger; @@ -43,7 +42,7 @@ * * @author Steve Hu */ -public class ExceptionHandler implements MiddlewareHandler, HandlerWrapper { +public class ExceptionHandler implements MiddlewareHandler { static final Logger logger = LoggerFactory.getLogger(ExceptionHandler.class); private String configName = ExceptionConfig.CONFIG_NAME; @@ -83,7 +82,7 @@ public void handleRequest(final HttpServerExchange exchange) throws Exception { } try { - HandlerContinuation.execute(exchange, this, + HandlerContinuation.execute(exchange, continuation -> ex -> handleContinuation(ex, continuation), ex -> Handler.next(ex, next)); } finally { // Initial chain execution owns request-thread cleanup. Resumed frames @@ -107,11 +106,6 @@ private void handleContinuation(HttpServerExchange exchange, HttpHandler continu } } - @Override - public HttpHandler wrap(HttpHandler continuation) { - return exchange -> handleContinuation(exchange, continuation); - } - private void writeFailure(HttpServerExchange exchange, Throwable failure) { if (failure instanceof FrameworkException framework) { sendTypedStatus(exchange, framework.getStatus(), failure, true); diff --git a/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java b/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java index 84e1dada97..57d7c72df3 100644 --- a/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java +++ b/exception/src/test/java/com/networknt/exception/ExceptionContinuationTest.java @@ -119,12 +119,8 @@ void dispatcherFailuresRetainFrameworkPolicy(String mode) throws Exception { selected.setNext(exchange -> { var scope = HandlerContinuation.capture(exchange); exchange.dispatch(() -> { - if (mode.equals("rejected")) { - scope.dispatcher(task -> { - throw new RejectedExecutionException("injected shutdown"); - }, () -> false, () -> { /* No transport resources in this fixture. */ }) - .execute(ex -> fail("Rejected callback ran")); - } else scope.resume(ex -> { + scope.resume(ex -> { + if (mode.equals("rejected")) throw new RejectedExecutionException("executor rejected transport"); calls.incrementAndGet(); if (mode.equals("accepted-error")) throw new StackOverflowError("injected fatal sentinel"); throw new IOException("accepted I/O failure"); diff --git a/handler/src/main/java/com/networknt/handler/HandlerContinuation.java b/handler/src/main/java/com/networknt/handler/HandlerContinuation.java index 6ac200616c..5d8f29d7ef 100644 --- a/handler/src/main/java/com/networknt/handler/HandlerContinuation.java +++ b/handler/src/main/java/com/networknt/handler/HandlerContinuation.java @@ -10,21 +10,13 @@ import java.util.Map; import java.util.Objects; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.Executor; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.BooleanSupplier; /** Lexical exception policy, carried explicitly across asynchronous hand-offs. */ public final class HandlerContinuation { private static final Logger LOG = LoggerFactory.getLogger(HandlerContinuation.class); - private record Frame(HttpServerExchange exchange, HandlerWrapper boundary, RequestState state, + private record Frame(HttpServerExchange exchange, RequestState state, Scope scope, boolean mdcScoped) { } private static final ThreadLocal CURRENT = new ThreadLocal<>(); - private static final Runnable NO_CLEANUP = () -> { }; private HandlerContinuation() { } @@ -35,7 +27,7 @@ public static void execute(HttpServerExchange exchange, HandlerWrapper boundary, boolean sameExchange = previous != null && previous.exchange() == exchange; RequestState state = sameExchange ? previous.state() : RequestState.initialize(exchange); Scope scope = new Scope(exchange, boundary, state); - Frame entry = new Frame(exchange, boundary, state, scope, sameExchange && previous.mdcScoped()); + Frame entry = new Frame(exchange, state, scope, sameExchange && previous.mdcScoped()); CURRENT.set(entry); try { boundary.wrap(new EntryContinuation(scope, continuation, entry)).handleRequest(exchange); } finally { restore(previous); } @@ -75,16 +67,9 @@ private Scope(HttpServerExchange exchange, HandlerWrapper boundary, RequestState this.exchange = exchange; this.boundary = boundary; this.state = state; - this.resumedFrame = new Frame(exchange, boundary, state, this, true); + this.resumedFrame = new Frame(exchange, state, this, true); } - public boolean isActive() { - Frame frame = CURRENT.get(); - return frame != null && frame.exchange() == exchange && frame.scope() == this && frame.mdcScoped(); - } - - public boolean isActive(HttpServerExchange expected) { return exchange == expected && isActive(); } - public Scope fork(HttpServerExchange child) { return new Scope(child, boundary, RequestState.initialize(child, state.mdc.snapshot())); } @@ -95,16 +80,14 @@ public Scope fork(HttpServerExchange child) { /** Selected policy handles ordinary failures; unhandled Errors are cleaned up then propagated. */ public void execute(HttpHandler continuation) { - boolean returned = false; try { inContext(continuation, true); - returned = true; } catch (Exception failure) { - finishFailedRequest(); + finishFailedRequest(failure); LOG.error("Unhandled continuation failure for {}", exchange.getRequestPath(), failure); - returned = true; - } finally { - if (!returned) finishFailedRequest(); + } catch (Error failure) { + finishFailedRequest(failure); + throw failure; } } @@ -114,16 +97,6 @@ public void resume(HttpHandler continuation) { catch (Exception | Error failure) { execute(new FailureHandler(failure)); } } - public void fail(Exception failure) { execute(new FailureHandler(failure)); } - - public Dispatcher ioDispatcher(Runnable onFailure) { - return dispatcher(exchange.getIoThread(), exchange::isInIoThread, onFailure); - } - - public Dispatcher dispatcher(Executor executor, BooleanSupplier inExecutor, Runnable onFailure) { - return new Dispatcher(this, executor, inExecutor, onFailure); - } - private void inContext(HttpHandler continuation, boolean enterBoundary) throws Exception { Frame previous = CURRENT.get(); boolean reuse = previous != null && previous.exchange() == exchange && previous.mdcScoped(); @@ -158,23 +131,33 @@ private static void setMdc(Map context) { } /** Best-effort cleanup must never mask an already propagating failure. */ - private void finishFailedRequest() { + private void finishFailedRequest(Throwable primary) { if (exchange.isComplete()) return; if (!exchange.isResponseStarted()) { try { exchange.setStatusCode(StatusCodes.INTERNAL_SERVER_ERROR); exchange.endExchange(); return; - } catch (Throwable _) { - // The primary failure is retained by the caller; try stream cleanup. + } catch (Throwable cleanupFailure) { + rememberCleanupFailure(primary, cleanupFailure); } } try { if (exchange.getConnection() != null) exchange.getConnection().close(); + } catch (Throwable cleanupFailure) { + rememberCleanupFailure(primary, cleanupFailure); + } + } + private static void rememberCleanupFailure(Throwable primary, Throwable cleanupFailure) { + if (primary == cleanupFailure) return; + try { + primary.addSuppressed(cleanupFailure); + LOG.error("Continuation cleanup failed; retaining the original failure", cleanupFailure); } catch (Throwable _) { - // Preserve the primary failure, including VM Errors, if close also fails. + // Diagnostics can fail under memory pressure; never replace the primary failure. } } + } private interface ScopedCallback extends HttpHandler { Scope scope(); } @@ -211,89 +194,6 @@ private record FailureHandler(Throwable failure) implements HttpHandler { } } - /** FIFO callbacks on an owning executor, including arrivals from that executor itself. */ - public static final class Dispatcher { - private record Event(Scope scope, HttpHandler callback, Runnable rejectedCleanup, boolean accepted) implements ScopedCallback { - @Override public void handleRequest(HttpServerExchange exchange) throws Exception { scope.inContext(callback, false); } - } - private final Scope scope; - private final Executor executor; - private final BooleanSupplier inExecutor; - private final Runnable onFailure; - private final ConcurrentLinkedQueue queue = new ConcurrentLinkedQueue<>(); - private final AtomicInteger work = new AtomicInteger(); - private final AtomicReference rejected = new AtomicReference<>(); - private final AtomicBoolean stopped = new AtomicBoolean(); - private final AtomicBoolean failureReported = new AtomicBoolean(); - private final Runnable drainTask = this::drain; - - private Dispatcher(Scope scope, Executor executor, BooleanSupplier inExecutor, Runnable onFailure) { - this.scope = scope; - this.executor = executor; - this.inExecutor = inExecutor; - this.onFailure = onFailure; - } - - public void execute(HttpHandler callback) { execute(callback, NO_CLEANUP); } - - public void execute(HttpHandler callback, Runnable rejectedCleanup) { submit(callback, rejectedCleanup, false); } - - public void resume(HttpHandler callback, Runnable rejectedCleanup) { submit(callback, rejectedCleanup, true); } - - private void submit(HttpHandler callback, Runnable rejectedCleanup, boolean accepted) { - queue.add(new Event(scope, callback, rejectedCleanup, accepted)); - if (stopped.get()) { rejectQueued(); return; } - if (work.getAndIncrement() != 0) return; - if (inExecutor.getAsBoolean()) drain(); - else { - try { executor.execute(drainTask); } - catch (RejectedExecutionException failure) { - rejected.compareAndSet(null, failure); - stopped.set(true); - rejectQueued(); - } - } - } - - private void drain() { - int missed = 1; - boolean returned = false; - try { - do { - Event event; - while ((event = queue.poll()) != null) { - if (event.accepted()) scope.resume(event); - else scope.execute(event); - } - missed = work.addAndGet(-missed); - } while (missed != 0); - returned = true; - } finally { - if (!returned) { - // Scope.execute already preserves/rethrows the fatal Error. - // Retire queued resources without running their transport work. - stopped.set(true); - rejectQueued(); - } - } - } - - private void rejectQueued() { - Event event; - while ((event = queue.poll()) != null) cleanup(event.rejectedCleanup()); - if (failureReported.compareAndSet(false, true)) { - cleanup(onFailure); - RejectedExecutionException failure = rejected.get(); - if (failure != null) scope.fail(failure); - } - } - private static void cleanup(Runnable cleanup) { - try { cleanup.run(); } - catch (Throwable _) { /* Retain the callback failure and retire the remaining resources. */ } - } - - } - private static void restore(Frame frame) { if (frame == null) CURRENT.remove(); else CURRENT.set(frame); diff --git a/handler/src/main/java/com/networknt/handler/RequestInterceptorInjectionHandler.java b/handler/src/main/java/com/networknt/handler/RequestInterceptorInjectionHandler.java index 763eb65bb0..9d65086992 100644 --- a/handler/src/main/java/com/networknt/handler/RequestInterceptorInjectionHandler.java +++ b/handler/src/main/java/com/networknt/handler/RequestInterceptorInjectionHandler.java @@ -169,6 +169,7 @@ private final class RequestBodyReadState implements ChannelListener Handler.next(ex, nextHandler)); + if (resumedHandler == null) { + HandlerContinuation.Scope scope = HandlerContinuation.capture(exchange); + resumedHandler = scope.wrap(ex -> Handler.next(ex, nextHandler)); + readContinuation = scope.task(ex -> resumeRead()); + } channel.getReadSetter().set(this); channel.resumeReads(); return ReadOutcome.PENDING; @@ -269,7 +273,7 @@ private void closeBuffers() { @Override public void handleEvent(StreamSourceChannel ignored) { // A pending read must not be auto-ended by a root-handler wrapper. - HandlerContinuation.task(exchange, ex -> resumeRead()).run(); + readContinuation.run(); } private void resumeRead() throws Exception { diff --git a/handler/src/main/java/com/networknt/handler/RequestState.java b/handler/src/main/java/com/networknt/handler/RequestState.java index bb29c20a33..c0ced87d69 100644 --- a/handler/src/main/java/com/networknt/handler/RequestState.java +++ b/handler/src/main/java/com/networknt/handler/RequestState.java @@ -57,7 +57,7 @@ static RequestState initialize(HttpServerExchange exchange, Map RequestState existing = find(exchange); if (existing != null) return existing; if (exchange.isComplete()) return detached(exchange, inheritedMdc); - // Bound cleanup work; ordinary completion removes its entry immediately. + // Bounded weak-key cleanup; no exchange-completion listener is registered. for (int count = 0; count < 16; count++) { var key = CLEARED.poll(); if (key == null) break; @@ -68,13 +68,8 @@ static RequestState initialize(HttpServerExchange exchange, Map RequestState state = INDEX.computeIfAbsent(key, ignored -> { RequestState created = detached(exchange, inheritedMdc); exchange.putAttachment(AttachmentConstants.MDC_CONTEXT, created.mdc); - exchange.addExchangeCompleteListener((completed, next) -> { - INDEX.remove(key); - next.proceed(); - }); return created; }); - if (exchange.isComplete()) INDEX.remove(key, state); return state; } finally { Reference.reachabilityFence(exchange); } } diff --git a/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java b/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java index 9eb9f6093c..b937f03fbd 100644 --- a/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java +++ b/handler/src/test/java/com/networknt/handler/HandlerContinuationTest.java @@ -1,14 +1,6 @@ package com.networknt.handler; import com.networknt.httpstring.AttachmentConstants; -import java.net.URI; -import java.net.InetSocketAddress; -import java.net.http.HttpResponse; -import java.net.http.HttpRequest; -import java.net.http.HttpClient; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.LinkedBlockingQueue; -import io.undertow.Undertow; import io.undertow.server.HttpHandler; import io.undertow.server.HandlerWrapper; import io.undertow.server.HttpServerExchange; @@ -19,6 +11,7 @@ import org.slf4j.MDC; import java.io.IOException; +import java.lang.ref.WeakReference; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -121,21 +114,21 @@ void activePolicyTracksIdentityAndExchangeAndRestoresAfterNestedFailure() throws HttpServerExchange second = new HttpServerExchange(null); var policy = HandlerContinuation.capture(first); var otherPolicy = HandlerContinuation.capture(first); - assertFalse(policy.isActive(first)); + assertNotSame(policy, HandlerContinuation.capture(first)); policy.wrap(ex -> { - assertTrue(policy.isActive(first)); - assertFalse(otherPolicy.isActive(first)); - assertFalse(policy.isActive(second)); + assertSame(policy, HandlerContinuation.capture(first)); + assertNotSame(otherPolicy, HandlerContinuation.capture(first)); + assertNotSame(policy, HandlerContinuation.capture(second)); var child = policy.fork(second); assertThrows(IOException.class, () -> child.wrap(nested -> { - assertTrue(child.isActive(second)); - assertFalse(policy.isActive(first)); + assertSame(child, HandlerContinuation.capture(second)); + assertNotSame(policy, HandlerContinuation.capture(first)); throw new IOException("nested failure"); }).handleRequest(second)); - assertTrue(policy.isActive(first)); + assertSame(policy, HandlerContinuation.capture(first)); }).handleRequest(first); - assertFalse(policy.isActive(first)); - assertFalse(policy.isActive(second)); + assertNotSame(policy, HandlerContinuation.capture(first)); + assertNotSame(policy, HandlerContinuation.capture(second)); } @Test @@ -235,45 +228,6 @@ void earlierNarrowPolicyCannotShadowLaterPolicyOrOffFrameCapture() throws Except assertEquals(List.of("narrow", "later"), seen); } - @Test - void dispatchPreservesForeignArrivalBeforeOwnerArrival() throws Exception { - var exchange = new HttpServerExchange(null); - var scheduled = new LinkedBlockingQueue(); - Thread owner = Thread.currentThread(); - var dispatcher = HandlerContinuation.capture(exchange).dispatcher(scheduled::add, - () -> Thread.currentThread() == owner, () -> fail("Unexpected dispatch rejection")); - var seen = new ArrayList(); - try (var worker = Executors.newSingleThreadExecutor()) { - worker.submit(() -> dispatcher.execute(ex -> seen.add("foreign"))).get(5, TimeUnit.SECONDS); - } - dispatcher.execute(ex -> seen.add("owner")); - assertTrue(seen.isEmpty(), "Owner arrival must not overtake an already queued foreign arrival"); - scheduled.take().run(); - assertEquals(List.of("foreign", "owner"), seen); - assertTrue(scheduled.isEmpty()); - } - - @Test - void rejectedDispatchUsesSelectedPolicyAndRetiresEachResource() throws Exception { - var exchange = new HttpServerExchange(null); - var scope = new AtomicReference(); - var failures = new AtomicInteger(); - var cleanup = new AtomicInteger(); - var abort = new AtomicInteger(); - HandlerContinuation.execute(exchange, next -> ex -> { - try { next.handleRequest(ex); } - catch (RejectedExecutionException _) { failures.incrementAndGet(); } - }, ex -> scope.set(HandlerContinuation.capture(ex))); - var dispatcher = scope.get().dispatcher(task -> { - throw new RejectedExecutionException("executor stopped"); - }, () -> false, abort::incrementAndGet); - dispatcher.execute(ex -> fail("Rejected callback must not run"), cleanup::incrementAndGet); - dispatcher.execute(ex -> fail("Stopped dispatcher must not run"), cleanup::incrementAndGet); - assertEquals(1, failures.get()); - assertEquals(1, abort.get()); - assertEquals(2, cleanup.get()); - } - @Test void acceptedWorkRunsOnceAndItsFailureUsesSelectedPolicy() throws Exception { var exchange = new HttpServerExchange(null); @@ -307,60 +261,39 @@ void captureDoesNotApplyWrapperUntilExecution() throws Exception { assertEquals(2, applications.get()); } - @Test - void completionRemovesIndexEntryWhileRetainedScopeKeepsItsMdc() throws Exception { - var completed = new CountDownLatch(1); - var captured = new AtomicReference(); - var request = new AtomicReference(); - var holder = new AtomicReference(); - var server = Undertow.builder().addHttpListener(0, "127.0.0.1").setHandler(exchange -> { - publicationMiddleware().addHandlerMDCContext(exchange, "request", "completed"); - request.set(exchange); - holder.set(RequestMdcContext.initialize(exchange)); - captured.set(HandlerContinuation.capture(exchange)); - assertNotNull(RequestState.find(exchange)); - exchange.addExchangeCompleteListener((ex, next) -> { - next.proceed(); - completed.countDown(); - }); - exchange.getResponseSender().send("done"); - }).build(); - try (var client = HttpClient.newHttpClient()) { - server.start(); - int port = ((InetSocketAddress) server.getListenerInfo().getFirst().getAddress()).getPort(); - var response = client.send(HttpRequest.newBuilder( - URI.create("http://127.0.0.1:" + port + "/lifetime")).build(), - HttpResponse.BodyHandlers.ofString()); - assertEquals("done", response.body()); - assertTrue(completed.await(5, TimeUnit.SECONDS)); - assertNull(RequestState.find(request.get())); - assertSame(holder.get(), RequestMdcContext.initialize(request.get())); - captured.get().wrap(ex -> assertEquals("completed", MDC.get("request"))).handleRequest(request.get()); - assertNull(RequestState.find(request.get()), "Late MDC access must not reinsert a completed request"); - } finally { server.stop(); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void initializationNeverRegistersACompletionListener(boolean alreadyComplete) { + var exchange = spy(new HttpServerExchange(null)); + doReturn(alreadyComplete, true).when(exchange).isComplete(); + doThrow(new IllegalStateException("exchange completed during registration")) + .when(exchange).addExchangeCompleteListener(any()); + var state = assertDoesNotThrow(() -> RequestState.initialize(exchange)); + assertNotNull(state.mdc); + assertTrue(exchange.isComplete()); + verify(exchange, never()).addExchangeCompleteListener(any()); + if (alreadyComplete) verify(exchange, never()).putAttachment(any(), any()); + else assertSame(state, RequestState.initialize(exchange)); } @Test - void fatalDispatchRetiresQueuedAndFutureCallbacksWithoutMaskingError() { - var exchange = spy(new HttpServerExchange(null)); - doReturn(exchange).when(exchange).endExchange(); - var retired = new AtomicInteger(); - var aborted = new AtomicInteger(); - var fatal = new StackOverflowError("injected callback sentinel"); - var dispatcher = HandlerContinuation.capture(exchange).dispatcher(Runnable::run, () -> true, - aborted::incrementAndGet); - Error observed = assertThrows(StackOverflowError.class, () -> dispatcher.execute(ex -> { - dispatcher.execute(queued -> fail("Queued callback survived fatal failure"), () -> { - retired.incrementAndGet(); - throw new IllegalStateException("cleanup failed too"); - }); - throw fatal; - })); - assertSame(fatal, observed); - dispatcher.execute(ex -> fail("Future callback survived fatal failure"), retired::incrementAndGet); - assertEquals(2, retired.get()); - assertEquals(1, aborted.get()); - verify(exchange).endExchange(); + void weakIndexReleasesAbandonedExchangeAndMdcAfterReferenceQueueCleanup() throws Exception { + var abandoned = abandonedRequest(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (abandoned.get(1).get() != null && System.nanoTime() < deadline) { + System.gc(); + // Each new registration performs bounded ReferenceQueue expunging. + RequestState.initialize(new HttpServerExchange(null)); + Thread.sleep(10); + } + assertNull(abandoned.get(0).get(), "The index must not retain the exchange"); + assertNull(abandoned.get(1).get(), "A queued weak key must be removed, releasing its MDC state"); + } + + private static List> abandonedRequest() { + var exchange = new HttpServerExchange(null); + var state = RequestState.initialize(exchange); + return List.of(new WeakReference<>(exchange), new WeakReference<>(state)); } private static MiddlewareHandler publicationMiddleware() { @@ -393,6 +326,7 @@ void exceptionCleanupPreservesFailureWhenStatusWriteAndResetThrow() throws Excep var exchange = spy(new HttpServerExchange(connection)); doThrow(writeFailure).when(exchange).setStatusCode(500); assertDoesNotThrow(() -> HandlerContinuation.task(exchange, ex -> { throw original; }).run()); + assertArrayEquals(new Throwable[]{writeFailure, closeFailure}, original.getSuppressed()); verify(exchange).setStatusCode(500); verify(connection).close(); } @@ -448,6 +382,7 @@ void failedFatalStatusWriteCannotReplaceOriginalError() throws Exception { Runnable callback = HandlerContinuation.task(exchange, ex -> { throw fatal; }); Error observed = assertThrows(StackOverflowError.class, callback::run); assertSame(fatal, observed); + assertArrayEquals(new Throwable[]{writeFailure, closeFailure}, fatal.getSuppressed()); assertEquals(500, exchange.getStatusCode()); verify(exchange).endExchange(); verify(connection).close(); diff --git a/handler/src/test/java/com/networknt/handler/RequestInjectionContinuationTest.java b/handler/src/test/java/com/networknt/handler/RequestInjectionContinuationTest.java index d6bf634d6f..65ca5f57f6 100644 --- a/handler/src/test/java/com/networknt/handler/RequestInjectionContinuationTest.java +++ b/handler/src/test/java/com/networknt/handler/RequestInjectionContinuationTest.java @@ -3,6 +3,14 @@ import com.networknt.config.Config; import com.networknt.service.SingletonServiceFactory; import io.undertow.Undertow; +import io.undertow.server.DefaultByteBufferPool; +import io.undertow.server.HttpServerExchange; +import io.undertow.server.ServerConnection; +import io.undertow.util.HeaderMap; +import io.undertow.util.Methods; +import org.junit.jupiter.api.Test; +import org.xnio.ChannelListener; +import org.xnio.channels.StreamSourceChannel; import io.undertow.server.HttpHandler; import io.undertow.server.HandlerWrapper; import org.junit.jupiter.params.ParameterizedTest; @@ -18,8 +26,11 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; @Timeout(20) class RequestInjectionContinuationTest { @@ -106,4 +117,54 @@ void partialBodyRetainsBoundaryOrItsAbsence(boolean enabled) throws Exception { Config.getInstance().clearConfigCache(config); } } + @Test + void listenerFailureOutsideBodyReadTryUsesCapturedPolicy() throws Exception { + String config = "request-listener-failure"; + RequestInterceptor[] previous = SingletonServiceFactory.getBeans(RequestInterceptor.class); + var interceptor = mock(RequestInterceptor.class); + when(interceptor.isRequiredContent()).thenReturn(true); + SingletonServiceFactory.setBean(RequestInterceptor.class.getName(), new RequestInterceptor[]{interceptor}); + Config.getInstance().putInConfigCache(config, Map.of("enabled", true, "maxBuffers", 4, + "maxBodyBytes", 1024, "appliedBodyInjectionPathPrefixes", List.of("/"))); + var exchange = mock(HttpServerExchange.class); + var connection = mock(ServerConnection.class); + var channel = mock(StreamSourceChannel.class); + ChannelListener.Setter setter = new ChannelListener.SimpleSetter<>(); + doReturn(setter).when(channel).getReadSetter(); + when(channel.read(any(java.nio.ByteBuffer.class))).thenReturn(0, -1); + var headers = new HeaderMap(); + when(exchange.getRequestHeaders()).thenReturn(headers); + when(exchange.getRequestMethod()).thenReturn(Methods.POST); + when(exchange.getRequestPath()).thenReturn("/"); + when(exchange.getRequestContentLength()).thenReturn(-1L); + when(exchange.getRequestChannel()).thenReturn(channel); + when(exchange.getConnection()).thenReturn(connection); + var failure = new IllegalStateException("outside resumeRead's body-read catch"); + var responseChecks = new AtomicInteger(); + when(exchange.isResponseStarted()).thenAnswer(invocation -> { + if (responseChecks.getAndIncrement() == 1) throw failure; // outside the body-read catch + return false; + }); + var selected = new AtomicReference(); + try (var pool = new DefaultByteBufferPool(false, 16)) { + when(connection.getByteBufferPool()).thenReturn(pool); + var reader = new RequestInterceptorInjectionHandler(config); + reader.setNext(ex -> fail("Failure must occur before downstream execution")); + HandlerContinuation.execute(exchange, next -> ex -> { + try { next.handleRequest(ex); } + catch (IllegalStateException caught) { selected.set(caught); } + }, reader); + ChannelListener listener = + ((ChannelListener.SimpleSetter) setter).get(); + assertNotNull(listener); + listener.handleEvent(channel); + assertSame(failure, selected.get(), "The listener itself must retain the lexical policy"); + verify(channel).suspendReads(); + verify(exchange, never()).setStatusCode(500); + } finally { + SingletonServiceFactory.setBean(RequestInterceptor.class.getName(), previous); + Config.getInstance().clearConfigCache(config); + } + } + } diff --git a/proxy-handler/src/main/java/com/networknt/handler/ProxyCallbackDispatcher.java b/proxy-handler/src/main/java/com/networknt/handler/ProxyCallbackDispatcher.java new file mode 100644 index 0000000000..c812a1ac04 --- /dev/null +++ b/proxy-handler/src/main/java/com/networknt/handler/ProxyCallbackDispatcher.java @@ -0,0 +1,103 @@ +package com.networknt.handler; + +import io.undertow.server.HttpHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BooleanSupplier; +import java.util.function.Consumer; + +/** Proxy-local FIFO. Transport callbacks and ownerFailure run only on the owning I/O thread. */ +final class ProxyCallbackDispatcher { + private static final Logger LOG = LoggerFactory.getLogger(ProxyCallbackDispatcher.class); + private static final Runnable NO_RESOURCE = () -> { }; + private record Event(HttpHandler callback, Runnable discardedResource, boolean accepted) { } + + private final HandlerContinuation.Scope scope; + private final Executor executor; + private final BooleanSupplier inOwner; + private final Runnable ownerFailure; + private final Consumer rejectedExecutor; + private final ConcurrentLinkedQueue queue = new ConcurrentLinkedQueue<>(); + private final AtomicInteger work = new AtomicInteger(); + private final AtomicReference terminal = new AtomicReference<>(); + private final Runnable drainTask = this::drain; + + ProxyCallbackDispatcher(HandlerContinuation.Scope scope, Executor executor, BooleanSupplier inOwner, + Runnable ownerFailure, Consumer rejectedExecutor) { + this.scope = scope; + this.executor = executor; + this.inOwner = inOwner; + this.ownerFailure = ownerFailure; + this.rejectedExecutor = rejectedExecutor; + } + + void execute(HttpHandler callback) { execute(callback, NO_RESOURCE); } + + /** discardedResource may run on any submitting thread: close only this incoming event's resource. */ + void execute(HttpHandler callback, Runnable discardedResource) { submit(callback, discardedResource, false); } + + void resume(HttpHandler callback, Runnable discardedResource) { submit(callback, discardedResource, true); } + + private void submit(HttpHandler callback, Runnable discardedResource, boolean accepted) { + queue.add(new Event(callback, discardedResource, accepted)); + Throwable failure = terminal.get(); + if (failure != null) { discardQueued(failure); return; } + if (work.getAndIncrement() != 0) return; + if (inOwner.getAsBoolean()) drain(); + else { + try { executor.execute(drainTask); } + catch (RejectedExecutionException rejected) { + if (terminal.compareAndSet(null, rejected)) { + discardQueued(rejected); + // Exactly the rejecting submission owns this terminal notification. + // It must use a published resource snapshot, never mutable proxy fields. + rejectedExecutor.accept(rejected); + } + } + } + } + + private void drain() { + int missed = 1; + try { + do { + Event event; + while ((event = queue.poll()) != null) { + if (event.accepted()) scope.resume(event.callback()); + else scope.execute(event.callback()); + } + missed = work.addAndGet(-missed); + } while (missed != 0); + } catch (Error failure) { + // This path is still on the owner. Stop before retiring queued resources. + cleanup(ownerFailure, failure); + terminal.compareAndSet(null, failure); + discardQueued(failure); + throw failure; + } + } + + private void discardQueued(Throwable failure) { + Event event; + while ((event = queue.poll()) != null) cleanup(event.discardedResource(), failure); + } + + static void cleanup(Runnable cleanup, Throwable primary) { + try { cleanup.run(); } + catch (Throwable failure) { + if (primary == failure) return; + try { + primary.addSuppressed(failure); + LOG.error("Proxy callback resource cleanup failed", failure); + } catch (Throwable _) { + // Best-effort diagnostics must not mask the original terminal failure. + } + } + } +} diff --git a/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java b/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java index 75596d32bb..2c5e2db65c 100644 --- a/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java +++ b/proxy-handler/src/main/java/com/networknt/handler/ProxyHandler.java @@ -371,16 +371,21 @@ public String toString() { private static final class ProxyContinuation { private static final Runnable NO_REJECTED_RESOURCE = () -> { }; private final HandlerContinuation.Scope scope; - private final HandlerContinuation.Dispatcher dispatcher; + private final ProxyCallbackDispatcher dispatcher; private ProxyConnection connection; private ClientExchange request; private boolean stopping; private volatile XnioExecutor.Key timeoutKey; private long attempt; + private volatile UpstreamResources publishedResources = UpstreamResources.EMPTY; - private ProxyContinuation(HandlerContinuation.Scope scope) { + private ProxyContinuation(HttpServerExchange exchange, HandlerContinuation.Scope scope) { this.scope = scope; - this.dispatcher = scope.ioDispatcher(() -> abortUpstream(this)); + this.dispatcher = new ProxyCallbackDispatcher(scope, exchange.getIoThread(), + () -> exchange.isInIoThread(), () -> abortUpstream(this), failure -> { + ProxyCallbackDispatcher.cleanup(publishedResources::close, failure); + scope.execute(ex -> { throw failure; }); + }); } private void run(Consumer callback) { run(callback, NO_REJECTED_RESOURCE); } @@ -400,12 +405,26 @@ private void acceptedCallback(HttpServerExchange exchange, Consumer boundaryCallback(ClientCallback cancelClientExchange(result)); } @@ -1103,10 +1125,11 @@ private void handleServerPush(ClientExchange ce) { int i = path.indexOf("?"); if (i > 0) path = path.substring(0, i); String targetPath = path; - ProxyContinuation pushed = new ProxyContinuation(continuation.scope.fork(child)); + ProxyContinuation pushed = new ProxyContinuation(child, continuation.scope.fork(child)); ProxyConnection pushedConnection = new ProxyConnection(pushedRequest.getConnection(), targetPath); child.putAttachment(CONNECTION, pushedConnection); pushed.connection = pushedConnection; + pushed.publishResources(); child.dispatch(INLINE_EXECUTOR, () -> pushed.run(ex -> new ProxyAction(pushedConnection, ex, requestHeaders, rewriteHostHeader, reuseXForwarded, null, idempotentPredicate, urlRewriteRules, methodRewriteRules, queryParamRewriteRules, headerRewriteRules, false, streamResponseContentTypes, streamIdleTimeout, streamResponseHeaderOverwrite, pushed).run())); }); if (!accepted) cancelClientExchange(pushedRequest); diff --git a/proxy-handler/src/test/java/com/networknt/handler/ProxyCallbackDispatcherTest.java b/proxy-handler/src/test/java/com/networknt/handler/ProxyCallbackDispatcherTest.java new file mode 100644 index 0000000000..195915206e --- /dev/null +++ b/proxy-handler/src/test/java/com/networknt/handler/ProxyCallbackDispatcherTest.java @@ -0,0 +1,175 @@ +package com.networknt.handler; + +import io.undertow.Undertow; +import io.undertow.server.HttpServerExchange; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.*; + +@Timeout(30) +class ProxyCallbackDispatcherTest { + @Test + void foreignArrivalCannotBeOvertakenByOwnerArrival() throws Exception { + var exchange = new HttpServerExchange(null); + var scheduled = new LinkedBlockingQueue(); + Thread owner = Thread.currentThread(); + var queue = new ProxyCallbackDispatcher(HandlerContinuation.capture(exchange), scheduled::add, + () -> Thread.currentThread() == owner, () -> fail("Unexpected owner failure"), + failure -> fail("Unexpected rejection", failure)); + var seen = new ArrayList(); + try (var worker = Executors.newSingleThreadExecutor()) { + worker.submit(() -> queue.execute(ex -> seen.add("foreign"))).get(5, TimeUnit.SECONDS); + } + queue.execute(ex -> seen.add("owner")); + assertTrue(seen.isEmpty()); + scheduled.take().run(); + assertEquals(List.of("foreign", "owner"), seen); + assertTrue(scheduled.isEmpty()); + } + + @Test + void onlyRejectingSubmissionNotifiesPolicyAndNeverCallsOwnerCleanup() throws Exception { + var exchange = new HttpServerExchange(null); + var selected = new AtomicReference(); + var policyCalls = new AtomicInteger(); + HandlerContinuation.execute(exchange, next -> ex -> { + try { next.handleRequest(ex); } + catch (RejectedExecutionException _) { policyCalls.incrementAndGet(); } + }, ex -> selected.set(HandlerContinuation.capture(ex))); + var rejectThread = new AtomicReference(); + var retired = new AtomicInteger(); + var cleanupFailure = new IllegalStateException("isolated resource cleanup"); + var rejection = new RejectedExecutionException("owner executor stopped"); + var queue = new ProxyCallbackDispatcher(selected.get(), task -> { throw rejection; }, + () -> false, () -> fail("Mutable owner cleanup ran on a rejecting submitter"), failure -> { + assertTrue(rejectThread.compareAndSet(null, Thread.currentThread())); + selected.get().execute(ex -> { throw failure; }); + }); + try (var workers = Executors.newFixedThreadPool(4)) { + Thread first = workers.submit(() -> { + queue.execute(ex -> fail("Rejected work ran"), () -> { + retired.incrementAndGet(); + throw cleanupFailure; + }); + return Thread.currentThread(); + }).get(5, TimeUnit.SECONDS); + var futureSubmissions = new ArrayList>(); + for (int n = 0; n < 100; n++) + futureSubmissions.add(workers.submit(() -> queue.execute(ex -> fail("Stopped work ran"), + retired::incrementAndGet))); + for (var future : futureSubmissions) future.get(5, TimeUnit.SECONDS); + assertSame(first, rejectThread.get()); + } + assertEquals(1, policyCalls.get()); + assertEquals(101, retired.get()); + assertArrayEquals(new Throwable[]{cleanupFailure}, rejection.getSuppressed()); + } + + @Test + void concurrentProducersPreserveTheirOrderAndDoNotLoseIdleWakeups() throws Exception { + var exchange = new HttpServerExchange(null); + var owner = new AtomicReference(); + var problem = new AtomicReference(); + try (var executor = Executors.newSingleThreadExecutor(task -> new Thread(() -> { + owner.set(Thread.currentThread()); + task.run(); + })); var producers = Executors.newFixedThreadPool(4)) { + var queue = new ProxyCallbackDispatcher(HandlerContinuation.capture(exchange), executor, + () -> Thread.currentThread() == owner.get(), + () -> problem.compareAndSet(null, "owner failure"), + failure -> problem.compareAndSet(null, "unexpected rejection")); + int[] expected = new int[4]; + for (int round = 0; round < 40; round++) { + var submitted = new ArrayList>(); + int base = round * 200; + for (int producer = 0; producer < 4; producer++) { + int id = producer; + submitted.add(producers.submit(() -> { + for (int n = base; n < base + 200; n++) { + int sequence = n; + queue.execute(ex -> { + if (Thread.currentThread() != owner.get()) + problem.compareAndSet(null, "callback escaped owner"); + if (expected[id]++ != sequence) + problem.compareAndSet(null, "producer order changed"); + }); + if ((n & 7) == 0) Thread.yield(); + } + })); + } + for (var future : submitted) future.get(5, TimeUnit.SECONDS); + var drained = new CompletableFuture(); + queue.execute(ex -> drained.complete(null)); + drained.get(5, TimeUnit.SECONDS); + } + assertArrayEquals(new int[]{8000, 8000, 8000, 8000}, expected); + assertNull(problem.get(), problem.get()); + } + } + + @Test + void fatalOwnerFailureRetiresQueuedAndForeignEventsAndRetainsDiagnostics() throws Exception { + var fatal = new StackOverflowError("injected fatal callback"); + var cleanupFailure = new IllegalStateException("queued resource cleanup"); + var observed = new CompletableFuture(); + var dispatcher = new CompletableFuture(); + var ownerThread = new AtomicReference(); + var cleanupThread = new AtomicReference(); + var ownerAborts = new AtomicInteger(); + var retired = new AtomicInteger(); + var server = Undertow.builder().addHttpListener(0, "127.0.0.1").setIoThreads(1).setHandler(exchange -> { + ownerThread.set(Thread.currentThread()); + var queue = new ProxyCallbackDispatcher(HandlerContinuation.capture(exchange), exchange.getIoThread(), + exchange::isInIoThread, () -> { + cleanupThread.set(Thread.currentThread()); + ownerAborts.incrementAndGet(); + }, failure -> observed.completeExceptionally(failure)); + dispatcher.complete(queue); + try { + queue.execute(ex -> { + queue.execute(queued -> fail("Queued transport work survived fatal error"), () -> { + retired.incrementAndGet(); + throw cleanupFailure; + }); + throw fatal; + }); + } catch (Error failure) { observed.complete(failure); } + }).build(); + try (var client = HttpClient.newHttpClient()) { + server.start(); + int port = ((InetSocketAddress) server.getListenerInfo().getFirst().getAddress()).getPort(); + var response = client.send(HttpRequest.newBuilder(URI.create("http://127.0.0.1:" + port + "/fatal")) + .timeout(Duration.ofSeconds(5)).build(), HttpResponse.BodyHandlers.ofString()); + assertEquals(500, response.statusCode()); + assertSame(fatal, observed.get(5, TimeUnit.SECONDS)); + try (var producers = Executors.newFixedThreadPool(4)) { + var futures = new ArrayList>(); + for (int n = 0; n < 100; n++) + futures.add(producers.submit(() -> dispatcher.join().execute(ex -> fail("Stopped transport ran"), + retired::incrementAndGet))); + for (var future : futures) future.get(5, TimeUnit.SECONDS); + } + assertSame(ownerThread.get(), cleanupThread.get()); + assertEquals(1, ownerAborts.get()); + assertEquals(101, retired.get()); + assertArrayEquals(new Throwable[]{cleanupFailure}, fatal.getSuppressed()); + } finally { server.stop(); } + } +} From a3bec2c1a192d5a257258eb9f4d1dd75cea00e71 Mon Sep 17 00:00:00 2001 From: Steve Hu Date: Fri, 30 Oct 2026 17:03:21 -0400 Subject: [PATCH 15/15] Document observed allocation probe variation --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5c35cb6079..83a955ec86 100644 --- a/README.md +++ b/README.md @@ -319,10 +319,10 @@ state. Unexpected committed-response callbacks reset only the affected HTTP/2 st Local allocation probes are separate qualification artifacts, not repository source or CI qualification; gateway performance remains an explicit review gate. The review-11 ThreadMXBean probe includes request setup and four callbacks: both -branches measured 1.25-1.31 KB with empty MDC and 2.97-3.04 KB with three fields +branches measured 1.25-1.31 KB with empty MDC and 2.97-3.40 KB with three fields (20k warm-up, three 20k samples). A separate real HTTP/1.1 loopback probe includes ExceptionHandler, ProxyHandler, upstream transport and the Java HTTP client; its -129-132 KB/request totals are not attributable solely to continuation overhead. +129-133 KB/request totals are not attributable solely to continuation overhead. Neither probe is a production throughput qualification. Custom asynchronous handlers must explicitly capture before registering their callbacks; this contract cannot intercept arbitrary third-party hand-offs.