From 69777c43dc5c97b739dbce4142fb09284fe4d45a Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Thu, 1 Oct 2026 16:39:43 -0700 Subject: [PATCH 1/7] fix(core): surface activity processing timeouts When ProcessActivityTimeout elapses, BotApplication.ProcessAsync now throws BotHandlerException wrapping a TimeoutException (with the activity attached) instead of returning normally. HTTP returns 500, Socket Mode replies/acks 500, and the BotBuilder adapter invokes OnTurnError, rather than reporting success. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/design-decouple-cancellation-token.md | 23 +++++++-- src/Microsoft.Teams.Core/BotApplication.cs | 12 ++++- .../Hosting/BotApplicationOptions.cs | 3 ++ .../CompatAdapterTests.cs | 45 ++++++++++++++++- .../SocketMode/SocketModeHostingTests.cs | 17 +++++++ .../BotApplicationTests.cs | 50 ++++++++++++++++++- 6 files changed, 139 insertions(+), 11 deletions(-) diff --git a/docs/design-decouple-cancellation-token.md b/docs/design-decouple-cancellation-token.md index 00fc2e2c7..a64fb021a 100644 --- a/docs/design-decouple-cancellation-token.md +++ b/docs/design-decouple-cancellation-token.md @@ -57,19 +57,32 @@ await MiddleWare.RunPipelineAsync(this, activity, this.OnActivity, 0, token); The HTTP request's `cancellationToken` is no longer forwarded to the handler pipeline. -#### 3. Graceful timeout handling +#### 3. Timeout handling -A new catch clause handles the timeout without crashing: +A dedicated catch clause records the timeout (log, `HandlerErrors` metric, and an error status on the turn span) and then surfaces it as a failure: ```csharp catch (OperationCanceledException) when (cts.IsCancellationRequested) { - _logger.LogWarning("Activity processing timed out after {Timeout}: Id={Id}", - _processActivityTimeout, activity.Id); + _logger.ActivityTimedOut(_processActivityTimeout, activity.Id); + Telemetry.HandlerErrors.Add(1, activityTypeTag); + span?.SetStatus(ActivityStatusCode.Error, "timeout"); + throw new BotHandlerException( + "Activity processing timed out", + new TimeoutException($"Activity processing exceeded the configured ProcessActivityTimeout of {_processActivityTimeout}."), + activity); } ``` -This prevents `BotHandlerException` from being thrown when the timeout fires, which is a recoverable situation (the handler simply took too long). +The timeout surfaces as a `BotHandlerException` wrapping a `TimeoutException` (with the offending activity attached), so every transport reports the turn as failed instead of successful: + +- **HTTP**: the exception propagates to the endpoint like any other handler failure, producing a 500 if the response has not started. +- **Socket Mode**: `SocketModeTransport` converts the `BotHandlerException` into a 500 reply (or 500 ack for non-invoke activities) and invokes the error hook. +- **BotBuilder compat**: `TeamsBotFrameworkHttpAdapter` invokes `OnTurnError`. + +Callers can distinguish a framework timeout from other handler failures by checking `BotHandlerException.InnerException is TimeoutException`. + +> **History:** this clause originally swallowed the timeout and returned normally, treating it as recoverable. That caused timed-out turns to be acknowledged as successful (HTTP 200, Socket Mode 200), hiding failures from the sender, so timeouts are now surfaced. ## Design Decisions diff --git a/src/Microsoft.Teams.Core/BotApplication.cs b/src/Microsoft.Teams.Core/BotApplication.cs index d4c0adb81..77d2979cb 100644 --- a/src/Microsoft.Teams.Core/BotApplication.cs +++ b/src/Microsoft.Teams.Core/BotApplication.cs @@ -185,7 +185,9 @@ public BotApplication(ConversationClient conversationClient, UserTokenClient use /// A task that represents the asynchronous activity processing operation. /// Thrown if the request body cannot be deserialized into a valid activity. /// Thrown if the activity's service URL does not match the serviceurl claim of the authenticated caller. - /// Thrown if an error occurs while processing the activity, wrapping the original exception and the offending . + /// Thrown if an error occurs while processing the activity, wrapping the original exception and the offending . + /// Also thrown when processing exceeds , in which case + /// is a . public virtual async Task ProcessAsync(HttpContext httpContext, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(httpContext); @@ -225,7 +227,9 @@ await ProcessAsync( /// Reserved for the caller's cancellation. Note: a dedicated timeout governs activity processing. /// A task that represents the asynchronous activity processing operation. /// Thrown if the activity's service URL does not match the serviceurl claim of . - /// Thrown if an error occurs while processing the activity, wrapping the original exception and the offending . + /// Thrown if an error occurs while processing the activity, wrapping the original exception and the offending . + /// Also thrown when processing exceeds , in which case + /// is a . public virtual async Task ProcessAsync(CoreActivity activity, ClaimsPrincipal? user, string? correlationVector, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(activity); @@ -278,6 +282,10 @@ public virtual async Task ProcessAsync(CoreActivity activity, ClaimsPrincipal? u _logger.ActivityTimedOut(_processActivityTimeout, activity.Id); Telemetry.HandlerErrors.Add(1, activityTypeTag); span?.SetStatus(ActivityStatusCode.Error, "timeout"); + throw new BotHandlerException( + "Activity processing timed out", + new TimeoutException($"Activity processing exceeded the configured ProcessActivityTimeout of {_processActivityTimeout}."), + activity); } catch (Exception ex) { diff --git a/src/Microsoft.Teams.Core/Hosting/BotApplicationOptions.cs b/src/Microsoft.Teams.Core/Hosting/BotApplicationOptions.cs index 0b14d46df..2b07fab29 100644 --- a/src/Microsoft.Teams.Core/Hosting/BotApplicationOptions.cs +++ b/src/Microsoft.Teams.Core/Hosting/BotApplicationOptions.cs @@ -18,6 +18,9 @@ public class BotApplicationOptions /// This timeout replaces the HTTP request's cancellation token so that handlers /// (especially streaming handlers) are not canceled when the incoming HTTP connection closes. /// Defaults to 5 minutes. Set to to disable the timeout. + /// When the timeout elapses, processing fails with a whose + /// is a , so the inbound + /// transport reports the turn as failed rather than successful. /// public TimeSpan ProcessActivityTimeout { get; set; } = TimeSpan.FromMinutes(5); } diff --git a/test/Microsoft.Teams.Apps.BotBuilder.UnitTests/CompatAdapterTests.cs b/test/Microsoft.Teams.Apps.BotBuilder.UnitTests/CompatAdapterTests.cs index 94025c4db..b88f0b46e 100644 --- a/test/Microsoft.Teams.Apps.BotBuilder.UnitTests/CompatAdapterTests.cs +++ b/test/Microsoft.Teams.Apps.BotBuilder.UnitTests/CompatAdapterTests.cs @@ -8,6 +8,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Teams.Core; +using Microsoft.Teams.Core.Hosting; using Microsoft.Teams.Core.Http; using Microsoft.Teams.Core.Schema; using Moq; @@ -168,7 +169,46 @@ public async Task ProcessAsync_OnTurnError_ReceivesTurnContextWithTurnState() Assert.Equal("customTurnStateValue", capturedCustomTurnState); } - private static TeamsBotFrameworkHttpAdapter CreateCompatAdapter(UserTokenClient? userTokenClient = null) + [Fact] + public async Task ProcessAsync_Timeout_InvokesOnTurnErrorWithTimeout() + { + // Arrange + TeamsBotFrameworkHttpAdapter adapter = CreateCompatAdapter(processActivityTimeout: TimeSpan.FromMilliseconds(50)); + + Exception? capturedException = null; + adapter.OnTurnError = (_, exception) => + { + capturedException = exception; + return Task.CompletedTask; + }; + + Mock mockBot = new(); + mockBot + .Setup(b => b.OnTurnAsync(It.IsAny(), It.IsAny())) + .Returns((_, ct) => Task.Delay(Timeout.Infinite, ct)); + + CoreActivity activity = new() + { + Type = ActivityType.Message, + Id = "act123", + ServiceUrl = new Uri("https://smba.trafficmanager.net/teams/"), + Conversation = new Conversation("conv123"), + From = new Teams.Core.Schema.ChannelAccount { Id = "user123" } + }; + + DefaultHttpContext httpContext = new(); + httpContext.Request.Body = new MemoryStream(Encoding.UTF8.GetBytes(activity.ToJson())); + httpContext.Request.ContentType = "application/json"; + + // Act + await adapter.ProcessAsync(httpContext.Request, httpContext.Response, mockBot.Object, CancellationToken.None); + + // Assert + BotHandlerException botHandlerException = Assert.IsType(capturedException); + Assert.IsType(botHandlerException.InnerException); + } + + private static TeamsBotFrameworkHttpAdapter CreateCompatAdapter(UserTokenClient? userTokenClient = null, TimeSpan? processActivityTimeout = null) { HttpClient httpClient = new(); ConversationClient conversationClient = new(httpClient, NullLogger.Instance); @@ -176,7 +216,8 @@ private static TeamsBotFrameworkHttpAdapter CreateCompatAdapter(UserTokenClient? BotApplication botApplication = new( conversationClient, userTokenClient ?? CreateMockUserTokenClient().Object, - NullLogger.Instance); + NullLogger.Instance, + processActivityTimeout is null ? null : new BotApplicationOptions { ProcessActivityTimeout = processActivityTimeout.Value }); TeamsBotFrameworkHttpAdapter compatAdapter = new( botApplication, diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeHostingTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeHostingTests.cs index 4cb149e39..95fb683ff 100644 --- a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeHostingTests.cs +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeHostingTests.cs @@ -216,6 +216,23 @@ public async Task DispatchAsync_NonInvokeReturnsOkWithoutBody() Assert.Equal(new SocketDispatchResult(200), result); } + [Fact] + public async Task DispatchAsync_ProcessingTimeout_ThrowsBotHandlerException() + { + ServiceCollection services = CreateServices(); + services.AddTeamsBotApplication(options => options.ProcessActivityTimeout = TimeSpan.FromMilliseconds(50)); + await using ServiceProvider provider = services.BuildServiceProvider(); + TeamsBotApplication app = provider.GetRequiredService(); + app.OnMessage((_, ct) => Task.Delay(Timeout.Infinite, ct)); + + Core.BotHandlerException exception = await Assert.ThrowsAsync(() => + SocketModeServiceRegistration.DispatchAsync( + app, + new Core.Schema.CoreActivity { Type = TeamsActivityTypes.Message })); + + Assert.IsType(exception.InnerException); + } + [Fact] public async Task UseTeamsBotApplication_WithSocketMode_ThrowsAndMapsNothing() { diff --git a/test/Microsoft.Teams.Core.UnitTests/BotApplicationTests.cs b/test/Microsoft.Teams.Core.UnitTests/BotApplicationTests.cs index 1712549f2..7bba3a4d4 100644 --- a/test/Microsoft.Teams.Core.UnitTests/BotApplicationTests.cs +++ b/test/Microsoft.Teams.Core.UnitTests/BotApplicationTests.cs @@ -411,11 +411,57 @@ public override Task ProcessAsync(CoreActivity activity, ClaimsPrincipal? user, } } + [Fact] + public async Task ProcessAsync_CoreActivity_Timeout_ThrowsBotHandlerExceptionWithTimeoutInner() + { + BotApplication botApp = CreateBotApplication(TimeSpan.FromMilliseconds(50)); + botApp.OnActivity = (_, ct) => Task.Delay(Timeout.Infinite, ct); + CoreActivity activity = new(ActivityType.Message) { Id = "act123" }; + + BotHandlerException exception = await Assert.ThrowsAsync(() => + botApp.ProcessAsync(activity, user: null, correlationVector: null)); + + Assert.IsType(exception.InnerException); + Assert.Same(activity, exception.Activity); + } + + [Fact] + public async Task ProcessAsync_HttpContext_Timeout_ThrowsBotHandlerException() + { + BotApplication botApp = CreateBotApplication(TimeSpan.FromMilliseconds(50)); + botApp.OnActivity = (_, ct) => Task.Delay(Timeout.Infinite, ct); + CoreActivity activity = new(ActivityType.Message) { Id = "act123" }; + DefaultHttpContext httpContext = CreateHttpContextWithActivity(activity); + + BotHandlerException exception = await Assert.ThrowsAsync(() => + botApp.ProcessAsync(httpContext)); + + Assert.IsType(exception.InnerException); + Assert.Equal("act123", exception.Activity?.Id); + } + + [Fact] + public async Task ProcessAsync_HandlerThrowsTimeoutException_ThrowsBotHandlerException() + { + BotApplication botApp = CreateBotApplication(); + TimeoutException handlerException = new("handler timeout"); + botApp.OnActivity = (_, _) => throw handlerException; + CoreActivity activity = new(ActivityType.Message) { Id = "act123" }; + + BotHandlerException exception = await Assert.ThrowsAsync(() => + botApp.ProcessAsync(activity, user: null, correlationVector: null)); + + Assert.Same(handlerException, exception.InnerException); + Assert.Equal("Error processing activity", exception.Message); + Assert.Same(activity, exception.Activity); + } + private static BotApplicationOptions CreateOptions(string appId) => new() { AppId = appId }; - private static BotApplication CreateBotApplication() => - new(CreateMockConversationClient(), CreateMockUserTokenClient(), NullLogger.Instance); + private static BotApplication CreateBotApplication(TimeSpan? processActivityTimeout = null) => + new(CreateMockConversationClient(), CreateMockUserTokenClient(), NullLogger.Instance, + processActivityTimeout is null ? null : new BotApplicationOptions { ProcessActivityTimeout = processActivityTimeout.Value }); private static ConversationClient CreateMockConversationClient() { From dcfeaf7dd9044cad7d5f5f97761d21120e3c2982 Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Fri, 2 Oct 2026 09:18:43 -0700 Subject: [PATCH 2/7] docs: revert design doc changes Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/design-decouple-cancellation-token.md | 23 +++++----------------- 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/docs/design-decouple-cancellation-token.md b/docs/design-decouple-cancellation-token.md index a64fb021a..00fc2e2c7 100644 --- a/docs/design-decouple-cancellation-token.md +++ b/docs/design-decouple-cancellation-token.md @@ -57,32 +57,19 @@ await MiddleWare.RunPipelineAsync(this, activity, this.OnActivity, 0, token); The HTTP request's `cancellationToken` is no longer forwarded to the handler pipeline. -#### 3. Timeout handling +#### 3. Graceful timeout handling -A dedicated catch clause records the timeout (log, `HandlerErrors` metric, and an error status on the turn span) and then surfaces it as a failure: +A new catch clause handles the timeout without crashing: ```csharp catch (OperationCanceledException) when (cts.IsCancellationRequested) { - _logger.ActivityTimedOut(_processActivityTimeout, activity.Id); - Telemetry.HandlerErrors.Add(1, activityTypeTag); - span?.SetStatus(ActivityStatusCode.Error, "timeout"); - throw new BotHandlerException( - "Activity processing timed out", - new TimeoutException($"Activity processing exceeded the configured ProcessActivityTimeout of {_processActivityTimeout}."), - activity); + _logger.LogWarning("Activity processing timed out after {Timeout}: Id={Id}", + _processActivityTimeout, activity.Id); } ``` -The timeout surfaces as a `BotHandlerException` wrapping a `TimeoutException` (with the offending activity attached), so every transport reports the turn as failed instead of successful: - -- **HTTP**: the exception propagates to the endpoint like any other handler failure, producing a 500 if the response has not started. -- **Socket Mode**: `SocketModeTransport` converts the `BotHandlerException` into a 500 reply (or 500 ack for non-invoke activities) and invokes the error hook. -- **BotBuilder compat**: `TeamsBotFrameworkHttpAdapter` invokes `OnTurnError`. - -Callers can distinguish a framework timeout from other handler failures by checking `BotHandlerException.InnerException is TimeoutException`. - -> **History:** this clause originally swallowed the timeout and returned normally, treating it as recoverable. That caused timed-out turns to be acknowledged as successful (HTTP 200, Socket Mode 200), hiding failures from the sender, so timeouts are now surfaced. +This prevents `BotHandlerException` from being thrown when the timeout fires, which is a recoverable situation (the handler simply took too long). ## Design Decisions From 39c4bf92fdad62f710495ef6af7bdaaeddb15c97 Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Fri, 2 Oct 2026 11:08:07 -0700 Subject: [PATCH 3/7] xml doc and test hang fix Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/Microsoft.Teams.Core/BotApplication.cs | 12 ++++++++---- .../Hosting/BotApplicationOptions.cs | 8 +++++--- .../CompatAdapterTests.cs | 3 ++- .../SocketMode/SocketModeHostingTests.cs | 3 ++- .../BotApplicationTests.cs | 6 ++++-- 5 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/Microsoft.Teams.Core/BotApplication.cs b/src/Microsoft.Teams.Core/BotApplication.cs index 77d2979cb..55f0ea0c4 100644 --- a/src/Microsoft.Teams.Core/BotApplication.cs +++ b/src/Microsoft.Teams.Core/BotApplication.cs @@ -186,8 +186,10 @@ public BotApplication(ConversationClient conversationClient, UserTokenClient use /// Thrown if the request body cannot be deserialized into a valid activity. /// Thrown if the activity's service URL does not match the serviceurl claim of the authenticated caller. /// Thrown if an error occurs while processing the activity, wrapping the original exception and the offending . - /// Also thrown when processing exceeds , in which case - /// is a . + /// Also thrown when processing exceeds and the pipeline observes + /// the resulting cancellation, in which case is a . + /// The timeout is cooperative: a handler that ignores its cancellation token or performs blocking I/O keeps running past + /// the timeout and is not surfaced this way. The timeout is disabled when a debugger is attached. public virtual async Task ProcessAsync(HttpContext httpContext, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(httpContext); @@ -228,8 +230,10 @@ await ProcessAsync( /// A task that represents the asynchronous activity processing operation. /// Thrown if the activity's service URL does not match the serviceurl claim of . /// Thrown if an error occurs while processing the activity, wrapping the original exception and the offending . - /// Also thrown when processing exceeds , in which case - /// is a . + /// Also thrown when processing exceeds and the pipeline observes + /// the resulting cancellation, in which case is a . + /// The timeout is cooperative: a handler that ignores its cancellation token or performs blocking I/O keeps running past + /// the timeout and is not surfaced this way. The timeout is disabled when a debugger is attached. public virtual async Task ProcessAsync(CoreActivity activity, ClaimsPrincipal? user, string? correlationVector, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(activity); diff --git a/src/Microsoft.Teams.Core/Hosting/BotApplicationOptions.cs b/src/Microsoft.Teams.Core/Hosting/BotApplicationOptions.cs index 2b07fab29..23803af07 100644 --- a/src/Microsoft.Teams.Core/Hosting/BotApplicationOptions.cs +++ b/src/Microsoft.Teams.Core/Hosting/BotApplicationOptions.cs @@ -18,9 +18,11 @@ public class BotApplicationOptions /// This timeout replaces the HTTP request's cancellation token so that handlers /// (especially streaming handlers) are not canceled when the incoming HTTP connection closes. /// Defaults to 5 minutes. Set to to disable the timeout. - /// When the timeout elapses, processing fails with a whose - /// is a , so the inbound - /// transport reports the turn as failed rather than successful. + /// The timeout is cooperative: it cancels the token passed to middleware and handlers. When the pipeline + /// observes that cancellation, processing fails with a whose + /// is a , so the inbound transport + /// reports the turn as failed. Handlers that ignore the token or perform blocking I/O are not interrupted. + /// The timeout is disabled when a debugger is attached. /// public TimeSpan ProcessActivityTimeout { get; set; } = TimeSpan.FromMinutes(5); } diff --git a/test/Microsoft.Teams.Apps.BotBuilder.UnitTests/CompatAdapterTests.cs b/test/Microsoft.Teams.Apps.BotBuilder.UnitTests/CompatAdapterTests.cs index b88f0b46e..6b1b0fc5f 100644 --- a/test/Microsoft.Teams.Apps.BotBuilder.UnitTests/CompatAdapterTests.cs +++ b/test/Microsoft.Teams.Apps.BotBuilder.UnitTests/CompatAdapterTests.cs @@ -183,9 +183,10 @@ public async Task ProcessAsync_Timeout_InvokesOnTurnErrorWithTimeout() }; Mock mockBot = new(); + // Bounded rather than infinite: with a debugger attached the processing timeout is disabled, so this fails instead of hanging. mockBot .Setup(b => b.OnTurnAsync(It.IsAny(), It.IsAny())) - .Returns((_, ct) => Task.Delay(Timeout.Infinite, ct)); + .Returns((_, ct) => Task.Delay(TimeSpan.FromSeconds(10), ct)); CoreActivity activity = new() { diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeHostingTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeHostingTests.cs index 95fb683ff..dc147aae1 100644 --- a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeHostingTests.cs +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeHostingTests.cs @@ -223,7 +223,8 @@ public async Task DispatchAsync_ProcessingTimeout_ThrowsBotHandlerException() services.AddTeamsBotApplication(options => options.ProcessActivityTimeout = TimeSpan.FromMilliseconds(50)); await using ServiceProvider provider = services.BuildServiceProvider(); TeamsBotApplication app = provider.GetRequiredService(); - app.OnMessage((_, ct) => Task.Delay(Timeout.Infinite, ct)); + // Bounded rather than infinite: with a debugger attached the processing timeout is disabled, so this fails instead of hanging. + app.OnMessage((_, ct) => Task.Delay(TimeSpan.FromSeconds(10), ct)); Core.BotHandlerException exception = await Assert.ThrowsAsync(() => SocketModeServiceRegistration.DispatchAsync( diff --git a/test/Microsoft.Teams.Core.UnitTests/BotApplicationTests.cs b/test/Microsoft.Teams.Core.UnitTests/BotApplicationTests.cs index 7bba3a4d4..4e7a3ea2f 100644 --- a/test/Microsoft.Teams.Core.UnitTests/BotApplicationTests.cs +++ b/test/Microsoft.Teams.Core.UnitTests/BotApplicationTests.cs @@ -415,7 +415,8 @@ public override Task ProcessAsync(CoreActivity activity, ClaimsPrincipal? user, public async Task ProcessAsync_CoreActivity_Timeout_ThrowsBotHandlerExceptionWithTimeoutInner() { BotApplication botApp = CreateBotApplication(TimeSpan.FromMilliseconds(50)); - botApp.OnActivity = (_, ct) => Task.Delay(Timeout.Infinite, ct); + // Bounded rather than infinite: with a debugger attached the processing timeout is disabled, so this fails instead of hanging. + botApp.OnActivity = (_, ct) => Task.Delay(TimeSpan.FromSeconds(10), ct); CoreActivity activity = new(ActivityType.Message) { Id = "act123" }; BotHandlerException exception = await Assert.ThrowsAsync(() => @@ -429,7 +430,8 @@ public async Task ProcessAsync_CoreActivity_Timeout_ThrowsBotHandlerExceptionWit public async Task ProcessAsync_HttpContext_Timeout_ThrowsBotHandlerException() { BotApplication botApp = CreateBotApplication(TimeSpan.FromMilliseconds(50)); - botApp.OnActivity = (_, ct) => Task.Delay(Timeout.Infinite, ct); + // Bounded rather than infinite: with a debugger attached the processing timeout is disabled, so this fails instead of hanging. + botApp.OnActivity = (_, ct) => Task.Delay(TimeSpan.FromSeconds(10), ct); CoreActivity activity = new(ActivityType.Message) { Id = "act123" }; DefaultHttpContext httpContext = CreateHttpContextWithActivity(activity); From 1bacd9869780206611c734d822f882caf11c4f38 Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Fri, 2 Oct 2026 11:20:40 -0700 Subject: [PATCH 4/7] changed warning to error and updated xml for inner exception Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/Microsoft.Teams.Core/BotApplication.cs | 4 ++++ src/Microsoft.Teams.Core/Log.cs | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Teams.Core/BotApplication.cs b/src/Microsoft.Teams.Core/BotApplication.cs index 55f0ea0c4..49f75cff6 100644 --- a/src/Microsoft.Teams.Core/BotApplication.cs +++ b/src/Microsoft.Teams.Core/BotApplication.cs @@ -188,6 +188,8 @@ public BotApplication(ConversationClient conversationClient, UserTokenClient use /// Thrown if an error occurs while processing the activity, wrapping the original exception and the offending . /// Also thrown when processing exceeds and the pipeline observes /// the resulting cancellation, in which case is a . + /// A thrown by a handler is wrapped the same way, so an inner + /// does not by itself indicate that elapsed. /// The timeout is cooperative: a handler that ignores its cancellation token or performs blocking I/O keeps running past /// the timeout and is not surfaced this way. The timeout is disabled when a debugger is attached. public virtual async Task ProcessAsync(HttpContext httpContext, CancellationToken cancellationToken = default) @@ -232,6 +234,8 @@ await ProcessAsync( /// Thrown if an error occurs while processing the activity, wrapping the original exception and the offending . /// Also thrown when processing exceeds and the pipeline observes /// the resulting cancellation, in which case is a . + /// A thrown by a handler is wrapped the same way, so an inner + /// does not by itself indicate that elapsed. /// The timeout is cooperative: a handler that ignores its cancellation token or performs blocking I/O keeps running past /// the timeout and is not surfaced this way. The timeout is disabled when a debugger is attached. public virtual async Task ProcessAsync(CoreActivity activity, ClaimsPrincipal? user, string? correlationVector, CancellationToken cancellationToken = default) diff --git a/src/Microsoft.Teams.Core/Log.cs b/src/Microsoft.Teams.Core/Log.cs index a8e9e3e92..23fd0b8dc 100644 --- a/src/Microsoft.Teams.Core/Log.cs +++ b/src/Microsoft.Teams.Core/Log.cs @@ -31,7 +31,7 @@ internal static partial class Log [LoggerMessage(EventId = 4, Level = LogLevel.Trace, Message = "Received activity: \n {Activity}")] public static partial void ReceivedActivityJson(this ILogger logger, string activity); - [LoggerMessage(EventId = 5, Level = LogLevel.Warning, Message = "Activity processing timed out after {Timeout}: Id={Id}")] + [LoggerMessage(EventId = 5, Level = LogLevel.Error, Message = "Activity processing timed out after {Timeout}: Id={Id}")] public static partial void ActivityTimedOut(this ILogger logger, TimeSpan timeout, string? id); [LoggerMessage(EventId = 6, Level = LogLevel.Error, Message = "Error processing activity: Id={Id}")] From a0a2e24c12f14d1c4ff839f7790708d0493dcdd4 Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Fri, 2 Oct 2026 11:26:33 -0700 Subject: [PATCH 5/7] docs: remove stale cancellation token design doc Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/design-decouple-cancellation-token.md | 104 --------------------- 1 file changed, 104 deletions(-) delete mode 100644 docs/design-decouple-cancellation-token.md diff --git a/docs/design-decouple-cancellation-token.md b/docs/design-decouple-cancellation-token.md deleted file mode 100644 index 00fc2e2c7..000000000 --- a/docs/design-decouple-cancellation-token.md +++ /dev/null @@ -1,104 +0,0 @@ -# Design: Decouple CancellationToken from Incoming HTTP Request - -## Problem - -When a bot handler performs long-running work — most notably streaming LLM responses back to Teams — the `CancellationToken` passed into the handler is tied to the lifetime of the **incoming HTTP request** (`HttpContext.RequestAborted`). Teams closes that connection once it receives the initial HTTP response (typically within ~15 seconds), which fires the cancellation token and aborts any in-flight outbound calls the handler is still making. - -### Observed behavior - -``` -dbug: HTTP POST .../v3/conversations/.../activities/... Response Status 202 -fail: Error processing activity: Id=... - System.Threading.Tasks.TaskCanceledException: The operation was canceled. - ---> System.IO.IOException: Unable to read data from the transport connection: - The I/O operation has been aborted because of either a thread exit or an application request. -``` - -The exception propagates through the OpenAI streaming pipeline, through `BotApplication.ProcessAsync`, and surfaces as a 500 to the ASP.NET middleware — even though the bot was functioning correctly. - -### Why this matters - -Streaming bots send responses via the **Bot Connector API** (`ConversationClient.SendActivityAsync`), not through the original HTTP response body. The handler legitimately outlives the HTTP request, so cancellation of that request should **not** cancel the handler's work. - -## Solution - -Replace the HTTP-bound `CancellationToken` with a **configurable timeout-based token** inside `BotApplication.ProcessAsync`. - -### Changes - -#### 1. `BotApplicationOptions.ProcessActivityTimeout` - -A new property on `BotApplicationOptions`: - -```csharp -public TimeSpan ProcessActivityTimeout { get; set; } = TimeSpan.FromMinutes(5); -``` - -- **Default: 5 minutes** — long enough for streaming LLM responses, short enough to prevent runaway handlers. -- Set to `Timeout.InfiniteTimeSpan` to disable the timeout entirely. -- Configurable per application instance via DI / builder options. - -#### 2. `BotApplication.ProcessAsync` — token replacement - -Before this change: - -```csharp -CancellationToken token = Debugger.IsAttached ? CancellationToken.None : cancellationToken; -await MiddleWare.RunPipelineAsync(this, activity, this.OnActivity, 0, token); -``` - -After: - -```csharp -using var cts = new CancellationTokenSource(_processActivityTimeout); -CancellationToken token = Debugger.IsAttached ? CancellationToken.None : cts.Token; -await MiddleWare.RunPipelineAsync(this, activity, this.OnActivity, 0, token); -``` - -The HTTP request's `cancellationToken` is no longer forwarded to the handler pipeline. - -#### 3. Graceful timeout handling - -A new catch clause handles the timeout without crashing: - -```csharp -catch (OperationCanceledException) when (cts.IsCancellationRequested) -{ - _logger.LogWarning("Activity processing timed out after {Timeout}: Id={Id}", - _processActivityTimeout, activity.Id); -} -``` - -This prevents `BotHandlerException` from being thrown when the timeout fires, which is a recoverable situation (the handler simply took too long). - -## Design Decisions - -### Why not keep the HTTP token as a linked source? - -Using `CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)` would still propagate HTTP disconnection to the handler — defeating the purpose. The HTTP request completing is an **expected** event for streaming handlers, not an error signal. - -### Why a timeout instead of `CancellationToken.None`? - -Unbounded processing is a resource leak risk. A timeout provides a safety net: -- Prevents handlers from running indefinitely if the LLM or external service hangs. -- Gives operators a tuning knob via `ProcessActivityTimeout`. -- Preserves the existing `Debugger.IsAttached` → `CancellationToken.None` escape hatch for debugging. - -### Why handle this at the framework level? - -- Every streaming bot would need the same workaround in user code. -- The framework owns the token plumbing and is the right place to define its semantics. -- Non-streaming bots are unaffected — 5 minutes is generous for synchronous handlers and can be reduced via options. - -### Impact on non-streaming bots - -Non-streaming handlers that complete within the HTTP request lifetime are unaffected. The 5-minute default is well above typical synchronous handler durations. Apps that want tighter timeouts can set `ProcessActivityTimeout` to a lower value. - -## Alternatives Considered - -| Alternative | Drawback | -|---|---| -| Catch `TaskCanceledException` in each sample/handler | Pushes framework responsibility to every consumer; easy to forget | -| Use `CancellationToken.None` unconditionally | No timeout safety net; runaway handlers can leak resources | -| Expose a `bool IsStreaming` flag to switch behavior | Over-engineered; all handlers benefit from decoupling | -| Let ASP.NET Core's `RequestTimeout` middleware handle it | That controls the *HTTP* timeout, not the *handler processing* timeout — different concerns | From e5bee1095aeaa7eb66abdb228ae2d4ca7263ca82 Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Fri, 2 Oct 2026 13:25:45 -0700 Subject: [PATCH 6/7] added RecordException to activity timeout Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/Microsoft.Teams.Core/BotApplication.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.Teams.Core/BotApplication.cs b/src/Microsoft.Teams.Core/BotApplication.cs index 49f75cff6..afc5f2a89 100644 --- a/src/Microsoft.Teams.Core/BotApplication.cs +++ b/src/Microsoft.Teams.Core/BotApplication.cs @@ -289,11 +289,11 @@ public virtual async Task ProcessAsync(CoreActivity activity, ClaimsPrincipal? u { _logger.ActivityTimedOut(_processActivityTimeout, activity.Id); Telemetry.HandlerErrors.Add(1, activityTypeTag); + TimeoutException timeoutException = new($"Activity processing exceeded the configured ProcessActivityTimeout of {_processActivityTimeout}."); + // RecordException sets the status from the exception message; set "timeout" afterward so it wins. + span.RecordException(timeoutException); span?.SetStatus(ActivityStatusCode.Error, "timeout"); - throw new BotHandlerException( - "Activity processing timed out", - new TimeoutException($"Activity processing exceeded the configured ProcessActivityTimeout of {_processActivityTimeout}."), - activity); + throw new BotHandlerException("Activity processing timed out", timeoutException, activity); } catch (Exception ex) { From 69b5393fbaf2671d873a60966c098d82cda7de3b Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Fri, 2 Oct 2026 14:10:19 -0700 Subject: [PATCH 7/7] readded and updated design doc Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/design-decouple-cancellation-token.md | 116 +++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 docs/design-decouple-cancellation-token.md diff --git a/docs/design-decouple-cancellation-token.md b/docs/design-decouple-cancellation-token.md new file mode 100644 index 000000000..41dca51c0 --- /dev/null +++ b/docs/design-decouple-cancellation-token.md @@ -0,0 +1,116 @@ +# Design: Decouple CancellationToken from Incoming HTTP Request + +## Problem + +When a bot handler performs long-running work — most notably streaming LLM responses back to Teams — the `CancellationToken` passed into the handler is tied to the lifetime of the **incoming HTTP request** (`HttpContext.RequestAborted`). Teams closes that connection once it receives the initial HTTP response (typically within ~15 seconds), which fires the cancellation token and aborts any in-flight outbound calls the handler is still making. + +### Observed behavior + +``` +dbug: HTTP POST .../v3/conversations/.../activities/... Response Status 202 +fail: Error processing activity: Id=... + System.Threading.Tasks.TaskCanceledException: The operation was canceled. + ---> System.IO.IOException: Unable to read data from the transport connection: + The I/O operation has been aborted because of either a thread exit or an application request. +``` + +The exception propagates through the OpenAI streaming pipeline, through `BotApplication.ProcessAsync`, and surfaces as a 500 to the ASP.NET middleware — even though the bot was functioning correctly. + +### Why this matters + +Streaming bots send responses via the **Bot Connector API** (`ConversationClient.SendActivityAsync`), not through the original HTTP response body. The handler legitimately outlives the HTTP request, so cancellation of that request should **not** cancel the handler's work. + +## Solution + +Replace the HTTP-bound `CancellationToken` with a **configurable timeout-based token** inside `BotApplication.ProcessAsync`. + +### Changes + +#### 1. `BotApplicationOptions.ProcessActivityTimeout` + +A new property on `BotApplicationOptions`: + +```csharp +public TimeSpan ProcessActivityTimeout { get; set; } = TimeSpan.FromMinutes(5); +``` + +- **Default: 5 minutes** — long enough for streaming LLM responses, short enough to prevent runaway handlers. +- Set to `Timeout.InfiniteTimeSpan` to disable the timeout entirely. +- Configurable per application instance via DI / builder options. + +#### 2. `BotApplication.ProcessAsync` — token replacement + +Before this change: + +```csharp +CancellationToken token = Debugger.IsAttached ? CancellationToken.None : cancellationToken; +await MiddleWare.RunPipelineAsync(this, activity, this.OnActivity, 0, token); +``` + +After: + +```csharp +using var cts = new CancellationTokenSource(_processActivityTimeout); +CancellationToken token = Debugger.IsAttached ? CancellationToken.None : cts.Token; +await MiddleWare.RunPipelineAsync(this, activity, this.OnActivity, 0, token); +``` + +The HTTP request's `cancellationToken` is no longer forwarded to the handler pipeline. + +#### 3. Timeout handling + +A dedicated catch clause records the timeout and then surfaces it as a turn failure: + +```csharp +catch (OperationCanceledException) when (cts.IsCancellationRequested) +{ + _logger.ActivityTimedOut(_processActivityTimeout, activity.Id); // LogLevel.Error + Telemetry.HandlerErrors.Add(1, activityTypeTag); + TimeoutException timeoutException = new($"Activity processing exceeded the configured ProcessActivityTimeout of {_processActivityTimeout}."); + span.RecordException(timeoutException); + span?.SetStatus(ActivityStatusCode.Error, "timeout"); + throw new BotHandlerException("Activity processing timed out", timeoutException, activity); +} +``` + +Each transport then reports the turn as failed, the same way it handles any other `BotHandlerException`: + +- **HTTP**: the exception propagates to the endpoint, producing a 500 if the response has not started. +- **Socket Mode**: `SocketModeTransport` replies with a 500 (or a 500 ack for non-invoke activities) and invokes the error hook. +- **BotBuilder compat**: `TeamsBotFrameworkHttpAdapter` invokes `OnTurnError`. + +The timeout is cooperative. It only takes effect when the handler (or the I/O it performs) observes the cancellation token. A handler that ignores the token or does blocking I/O keeps running past the timeout and is not surfaced this way. The timeout is also disabled when a debugger is attached. + +> **History:** this catch originally logged a warning and returned normally, treating a timeout as recoverable. That caused timed-out turns to be acknowledged as successful (HTTP 200, Socket Mode 200), hiding the failure from the sender, so timeouts are now surfaced as `BotHandlerException`. + +## Design Decisions + +### Why not keep the HTTP token as a linked source? + +Using `CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)` would still propagate HTTP disconnection to the handler — defeating the purpose. The HTTP request completing is an **expected** event for streaming handlers, not an error signal. + +### Why a timeout instead of `CancellationToken.None`? + +Unbounded processing is a resource leak risk. A timeout provides a safety net: +- Prevents handlers from running indefinitely if the LLM or external service hangs. +- Gives operators a tuning knob via `ProcessActivityTimeout`. +- Preserves the existing `Debugger.IsAttached` → `CancellationToken.None` escape hatch for debugging. + +### Why handle this at the framework level? + +- Every streaming bot would need the same workaround in user code. +- The framework owns the token plumbing and is the right place to define its semantics. +- Non-streaming bots are unaffected — 5 minutes is generous for synchronous handlers and can be reduced via options. + +### Impact on non-streaming bots + +Non-streaming handlers that complete within the HTTP request lifetime are unaffected. The 5-minute default is well above typical synchronous handler durations. Apps that want tighter timeouts can set `ProcessActivityTimeout` to a lower value. A turn that exceeds the timeout (including a long-running streaming turn) fails with a `BotHandlerException` and the transport reports an error; apps that legitimately need longer turns can raise `ProcessActivityTimeout` or set it to `Timeout.InfiniteTimeSpan`. + +## Alternatives Considered + +| Alternative | Drawback | +|---|---| +| Catch `TaskCanceledException` in each sample/handler | Pushes framework responsibility to every consumer; easy to forget | +| Use `CancellationToken.None` unconditionally | No timeout safety net; runaway handlers can leak resources | +| Expose a `bool IsStreaming` flag to switch behavior | Over-engineered; all handlers benefit from decoupling | +| Let ASP.NET Core's `RequestTimeout` middleware handle it | That controls the *HTTP* timeout, not the *handler processing* timeout — different concerns |