From 820cf7c6a2bcbcffadf509a1a7362bfb1ab9990e Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Tue, 22 Sep 2026 11:21:42 -0700 Subject: [PATCH 01/31] feat(apps): add Socket Mode protocol models Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SocketMode/SocketModeJson.cs | 22 +++ .../SocketMode/SocketModeProtocol.cs | 11 ++ .../SocketMode/SocketModeProtocolModels.cs | 81 +++++++++ .../SocketMode/SocketModeProtocolTests.cs | 163 ++++++++++++++++++ 4 files changed, 277 insertions(+) create mode 100644 src/Microsoft.Teams.Apps/SocketMode/SocketModeJson.cs create mode 100644 src/Microsoft.Teams.Apps/SocketMode/SocketModeProtocol.cs create mode 100644 src/Microsoft.Teams.Apps/SocketMode/SocketModeProtocolModels.cs create mode 100644 test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeProtocolTests.cs diff --git a/src/Microsoft.Teams.Apps/SocketMode/SocketModeJson.cs b/src/Microsoft.Teams.Apps/SocketMode/SocketModeJson.cs new file mode 100644 index 00000000..095832a5 --- /dev/null +++ b/src/Microsoft.Teams.Apps/SocketMode/SocketModeJson.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Microsoft.Teams.Apps.SocketMode; + +internal static class SocketModeJson +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + PropertyNameCaseInsensitive = true, + }; + + internal static T? Deserialize(string json) + => JsonSerializer.Deserialize(json, SerializerOptions); + + internal static string Serialize(T value) + => JsonSerializer.Serialize(value, SerializerOptions); +} diff --git a/src/Microsoft.Teams.Apps/SocketMode/SocketModeProtocol.cs b/src/Microsoft.Teams.Apps/SocketMode/SocketModeProtocol.cs new file mode 100644 index 00000000..08c79bc4 --- /dev/null +++ b/src/Microsoft.Teams.Apps/SocketMode/SocketModeProtocol.cs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.Teams.Apps.SocketMode; + +internal static class SocketModeProtocol +{ + internal const int CurrentVersion = 1; + internal const string DefaultNegotiateBaseUrl = "https://botapi.skype.com"; + internal const string NegotiatePath = "/v3/websockets/connect"; +} diff --git a/src/Microsoft.Teams.Apps/SocketMode/SocketModeProtocolModels.cs b/src/Microsoft.Teams.Apps/SocketMode/SocketModeProtocolModels.cs new file mode 100644 index 00000000..000088b5 --- /dev/null +++ b/src/Microsoft.Teams.Apps/SocketMode/SocketModeProtocolModels.cs @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Microsoft.Teams.Apps.SocketMode; + +internal sealed class SocketModeNegotiateResponse +{ + [JsonPropertyName("url")] + public string? Url { get; init; } + + [JsonPropertyName("accessToken")] + public string? AccessToken { get; init; } + + [JsonPropertyName("expiresIn")] + public int ExpiresIn { get; init; } +} + +internal sealed class SocketReadyFrame +{ + [JsonPropertyName("botKey")] + public string? BotKey { get; init; } + + [JsonPropertyName("connectionId")] + public string? ConnectionId { get; init; } +} + +internal sealed class SocketActivityEnvelope +{ + [JsonPropertyName("protocolVersion")] + public int? ProtocolVersion { get; init; } + + [JsonPropertyName("envelopeId")] + public string? EnvelopeId { get; init; } + + [JsonPropertyName("type")] + public string? Type { get; init; } + + [JsonPropertyName("ackRequired")] + public bool? AckRequired { get; init; } + + [JsonPropertyName("payload")] + public JsonElement? Payload { get; init; } + + [JsonPropertyName("activity")] + public JsonElement? Activity { get; init; } + + [JsonPropertyName("cv")] + public string? CorrelationVector { get; init; } +} + +internal sealed class SocketReplyFrame +{ + [JsonPropertyName("protocolVersion")] + public int ProtocolVersion { get; init; } = SocketModeProtocol.CurrentVersion; + + [JsonPropertyName("envelopeId")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? EnvelopeId { get; init; } + + [JsonPropertyName("botKey")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? BotKey { get; init; } + + [JsonPropertyName("status")] + public int Status { get; init; } + + [JsonPropertyName("body")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public object? Body { get; init; } + + [JsonPropertyName("recvAt")] + public long ReceivedAtUnixMilliseconds { get; init; } + + [JsonPropertyName("ts")] + public long TimestampUnixMilliseconds { get; init; } +} + +internal sealed record SocketDispatchResult(int Status, object? Body = null); diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeProtocolTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeProtocolTests.cs new file mode 100644 index 00000000..c477d7f4 --- /dev/null +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeProtocolTests.cs @@ -0,0 +1,163 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json; +using Microsoft.Teams.Apps.SocketMode; + +namespace Microsoft.Teams.Apps.UnitTests.SocketMode; + +public class SocketModeProtocolTests +{ + [Theory] + [InlineData(""" + { + "url": "https://signalr.example.test/client", + "accessToken": "secret", + "expiresIn": 3600 + } + """)] + [InlineData(""" + { + "Url": "https://signalr.example.test/client", + "AccessToken": "secret", + "ExpiresIn": 3600 + } + """)] + public void NegotiateResponse_DeserializesPropertyNamesCaseInsensitively(string json) + { + SocketModeNegotiateResponse? response = SocketModeJson.Deserialize(json); + + Assert.NotNull(response); + Assert.Equal("https://signalr.example.test/client", response.Url); + Assert.Equal("secret", response.AccessToken); + Assert.Equal(3600, response.ExpiresIn); + } + + [Theory] + [InlineData(""" + { + "protocolVersion": 1, + "envelopeId": "env-1", + "type": "message", + "ackRequired": true, + "payload": { "type": "message", "text": "hello" }, + "cv": "cv-value" + } + """)] + [InlineData(""" + { + "ProtocolVersion": 1, + "EnvelopeId": "env-1", + "Type": "message", + "AckRequired": true, + "Payload": { "type": "message", "text": "hello" }, + "Cv": "cv-value" + } + """)] + public void ActivityEnvelope_DeserializesPropertyNamesCaseInsensitively(string json) + { + SocketActivityEnvelope? envelope = SocketModeJson.Deserialize(json); + + Assert.NotNull(envelope); + Assert.Equal(1, envelope.ProtocolVersion); + Assert.Equal("env-1", envelope.EnvelopeId); + Assert.Equal("message", envelope.Type); + Assert.True(envelope.AckRequired); + Assert.Equal("cv-value", envelope.CorrelationVector); + Assert.Equal(JsonValueKind.Object, envelope.Payload?.ValueKind); + Assert.Equal("message", envelope.Payload?.GetProperty("type").GetString()); + } + + [Fact] + public void ActivityEnvelope_PreservesPayloadAndActivityAliases() + { + const string json = """ + { + "payload": { "type": "message" }, + "activity": { "type": "invoke" } + } + """; + + SocketActivityEnvelope? envelope = SocketModeJson.Deserialize(json); + + Assert.NotNull(envelope); + Assert.Equal("message", envelope.Payload?.GetProperty("type").GetString()); + Assert.Equal("invoke", envelope.Activity?.GetProperty("type").GetString()); + } + + [Fact] + public void ActivityEnvelope_IgnoresUnknownProperties() + { + const string json = """ + { + "protocolVersion": 1, + "futureField": "ignored" + } + """; + + SocketActivityEnvelope? envelope = SocketModeJson.Deserialize(json); + + Assert.NotNull(envelope); + Assert.Equal(1, envelope.ProtocolVersion); + } + + [Fact] + public void ReadyFrame_DeserializesPropertyNamesCaseInsensitively() + { + const string json = """ + { + "BotKey": "bot-id", + "ConnectionId": "connection-id" + } + """; + + SocketReadyFrame? frame = SocketModeJson.Deserialize(json); + + Assert.NotNull(frame); + Assert.Equal("bot-id", frame.BotKey); + Assert.Equal("connection-id", frame.ConnectionId); + } + + [Fact] + public void ReplyFrame_SerializesExactWireShape() + { + SocketReplyFrame frame = new() + { + EnvelopeId = "env-1", + BotKey = "bot-id", + Status = 202, + Body = new { result = "accepted" }, + ReceivedAtUnixMilliseconds = 1_789_999_999_000, + TimestampUnixMilliseconds = 1_790_000_000_000, + }; + + using JsonDocument document = JsonDocument.Parse(SocketModeJson.Serialize(frame)); + JsonElement root = document.RootElement; + + Assert.Equal(1, root.GetProperty("protocolVersion").GetInt32()); + Assert.Equal("env-1", root.GetProperty("envelopeId").GetString()); + Assert.Equal("bot-id", root.GetProperty("botKey").GetString()); + Assert.Equal(202, root.GetProperty("status").GetInt32()); + Assert.Equal("accepted", root.GetProperty("body").GetProperty("result").GetString()); + Assert.Equal(1_789_999_999_000, root.GetProperty("recvAt").GetInt64()); + Assert.Equal(1_790_000_000_000, root.GetProperty("ts").GetInt64()); + } + + [Fact] + public void ReplyFrame_OmitsNullableProperties() + { + SocketReplyFrame frame = new() + { + Status = 200, + ReceivedAtUnixMilliseconds = 1, + TimestampUnixMilliseconds = 2, + }; + + using JsonDocument document = JsonDocument.Parse(SocketModeJson.Serialize(frame)); + JsonElement root = document.RootElement; + + Assert.False(root.TryGetProperty("envelopeId", out _)); + Assert.False(root.TryGetProperty("botKey", out _)); + Assert.False(root.TryGetProperty("body", out _)); + } +} From f641476b99a679df36128943ad7437b35c02ac6c Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Tue, 22 Sep 2026 11:44:26 -0700 Subject: [PATCH 02/31] feat(apps): parse Socket Mode envelopes and replies Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SocketMode/SocketModeEnvelope.cs | 80 ++++++++ .../SocketMode/SocketModeEnvelopeTests.cs | 179 ++++++++++++++++++ 2 files changed, 259 insertions(+) create mode 100644 src/Microsoft.Teams.Apps/SocketMode/SocketModeEnvelope.cs create mode 100644 test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeEnvelopeTests.cs diff --git a/src/Microsoft.Teams.Apps/SocketMode/SocketModeEnvelope.cs b/src/Microsoft.Teams.Apps/SocketMode/SocketModeEnvelope.cs new file mode 100644 index 00000000..0f998790 --- /dev/null +++ b/src/Microsoft.Teams.Apps/SocketMode/SocketModeEnvelope.cs @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json; +using Microsoft.Teams.Core.Schema; + +namespace Microsoft.Teams.Apps.SocketMode; + +internal static class SocketModeEnvelope +{ + internal static bool TryReadActivity(SocketActivityEnvelope envelope, out CoreActivity? activity) + { + ArgumentNullException.ThrowIfNull(envelope); + + return TryReadActivityCandidate(envelope.Payload, out activity) + || TryReadActivityCandidate(envelope.Activity, out activity); + } + + internal static SocketReplyFrame CreateAcknowledgement( + SocketActivityEnvelope envelope, + string? botKey, + long receivedAtUnixMilliseconds, + long timestampUnixMilliseconds, + int status = 200) + { + ArgumentNullException.ThrowIfNull(envelope); + + return new SocketReplyFrame + { + EnvelopeId = envelope.EnvelopeId, + BotKey = botKey, + Status = status, + ReceivedAtUnixMilliseconds = receivedAtUnixMilliseconds, + TimestampUnixMilliseconds = timestampUnixMilliseconds, + }; + } + + internal static SocketReplyFrame CreateInvokeReply( + SocketActivityEnvelope envelope, + string? botKey, + SocketDispatchResult result, + long receivedAtUnixMilliseconds, + long timestampUnixMilliseconds) + { + ArgumentNullException.ThrowIfNull(envelope); + ArgumentNullException.ThrowIfNull(result); + + return new SocketReplyFrame + { + EnvelopeId = envelope.EnvelopeId, + BotKey = botKey, + Status = result.Status, + Body = result.Body, + ReceivedAtUnixMilliseconds = receivedAtUnixMilliseconds, + TimestampUnixMilliseconds = timestampUnixMilliseconds, + }; + } + + private static bool TryReadActivityCandidate(JsonElement? candidate, out CoreActivity? activity) + { + activity = null; + + if (candidate is not { ValueKind: JsonValueKind.Object } element + || !element.TryGetProperty("type", out JsonElement type) + || type.ValueKind != JsonValueKind.String) + { + return false; + } + + try + { + activity = CoreActivity.FromJsonString(element.GetRawText()); + return true; + } + catch (JsonException) + { + return false; + } + } +} diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeEnvelopeTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeEnvelopeTests.cs new file mode 100644 index 00000000..1ffee40c --- /dev/null +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeEnvelopeTests.cs @@ -0,0 +1,179 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json; +using Microsoft.Teams.Apps.SocketMode; +using Microsoft.Teams.Core.Schema; + +namespace Microsoft.Teams.Apps.UnitTests.SocketMode; + +public class SocketModeEnvelopeTests +{ + [Fact] + public void TryReadActivity_ReadsPayload() + { + SocketActivityEnvelope envelope = DeserializeEnvelope(""" + { + "payload": { + "type": "message", + "id": "activity-1", + "serviceUrl": "https://smba.trafficmanager.net/teams/", + "text": "hello" + } + } + """); + + bool found = SocketModeEnvelope.TryReadActivity(envelope, out CoreActivity? activity); + + Assert.True(found); + Assert.NotNull(activity); + Assert.Equal("message", activity.Type); + Assert.Equal("activity-1", activity.Id); + Assert.Equal(new Uri("https://smba.trafficmanager.net/teams/"), activity.ServiceUrl); + Assert.Equal("hello", activity.Properties.Get("text")); + } + + [Fact] + public void TryReadActivity_ReadsActivityAlias() + { + SocketActivityEnvelope envelope = DeserializeEnvelope(""" + { + "activity": { + "type": "invoke", + "id": "activity-1" + } + } + """); + + bool found = SocketModeEnvelope.TryReadActivity(envelope, out CoreActivity? activity); + + Assert.True(found); + Assert.NotNull(activity); + Assert.Equal("invoke", activity.Type); + Assert.Equal("activity-1", activity.Id); + } + + [Fact] + public void TryReadActivity_PrefersValidPayload() + { + SocketActivityEnvelope envelope = DeserializeEnvelope(""" + { + "payload": { + "type": "message", + "id": "payload-activity" + }, + "activity": { + "type": "invoke", + "id": "activity-alias" + } + } + """); + + bool found = SocketModeEnvelope.TryReadActivity(envelope, out CoreActivity? activity); + + Assert.True(found); + Assert.Equal("payload-activity", activity?.Id); + } + + [Theory] + [InlineData("\"not-an-object\"")] + [InlineData("[]")] + [InlineData("""{ "id": "missing-type" }""")] + [InlineData("""{ "type": 42 }""")] + public void TryReadActivity_FallsBackWhenPayloadIsMalformed(string malformedPayload) + { + SocketActivityEnvelope envelope = DeserializeEnvelope($$""" + { + "payload": {{malformedPayload}}, + "activity": { + "type": "message", + "id": "fallback-activity" + } + } + """); + + bool found = SocketModeEnvelope.TryReadActivity(envelope, out CoreActivity? activity); + + Assert.True(found); + Assert.Equal("fallback-activity", activity?.Id); + } + + [Fact] + public void TryReadActivity_ReturnsFalseWhenNeitherCandidateIsValid() + { + SocketActivityEnvelope envelope = DeserializeEnvelope(""" + { + "payload": { "id": "missing-type" }, + "activity": null + } + """); + + bool found = SocketModeEnvelope.TryReadActivity(envelope, out CoreActivity? activity); + + Assert.False(found); + Assert.Null(activity); + } + + [Fact] + public void CreateAcknowledgement_UsesEnvelopeIdentityAndHasNoBody() + { + SocketActivityEnvelope envelope = new() { EnvelopeId = "env-1" }; + + SocketReplyFrame reply = SocketModeEnvelope.CreateAcknowledgement( + envelope, + "bot-id", + receivedAtUnixMilliseconds: 100, + timestampUnixMilliseconds: 200, + status: 202); + + Assert.Equal(SocketModeProtocol.CurrentVersion, reply.ProtocolVersion); + Assert.Equal("env-1", reply.EnvelopeId); + Assert.Equal("bot-id", reply.BotKey); + Assert.Equal(202, reply.Status); + Assert.Null(reply.Body); + Assert.Equal(100, reply.ReceivedAtUnixMilliseconds); + Assert.Equal(200, reply.TimestampUnixMilliseconds); + } + + [Fact] + public void CreateInvokeReply_UsesDispatchResult() + { + SocketActivityEnvelope envelope = new() { EnvelopeId = "env-1" }; + SocketDispatchResult result = new(201, new { result = "created" }); + + SocketReplyFrame reply = SocketModeEnvelope.CreateInvokeReply( + envelope, + "bot-id", + result, + receivedAtUnixMilliseconds: 100, + timestampUnixMilliseconds: 200); + + Assert.Equal(SocketModeProtocol.CurrentVersion, reply.ProtocolVersion); + Assert.Equal("env-1", reply.EnvelopeId); + Assert.Equal("bot-id", reply.BotKey); + Assert.Equal(201, reply.Status); + Assert.Same(result.Body, reply.Body); + Assert.Equal(100, reply.ReceivedAtUnixMilliseconds); + Assert.Equal(200, reply.TimestampUnixMilliseconds); + } + + [Fact] + public void CreateInvokeReply_AllowsNoBody() + { + SocketActivityEnvelope envelope = new() { EnvelopeId = "env-1" }; + + SocketReplyFrame reply = SocketModeEnvelope.CreateInvokeReply( + envelope, + botKey: null, + new SocketDispatchResult(204), + receivedAtUnixMilliseconds: 100, + timestampUnixMilliseconds: 200); + + Assert.Equal(204, reply.Status); + Assert.Null(reply.Body); + } + + private static SocketActivityEnvelope DeserializeEnvelope(string json) + => SocketModeJson.Deserialize(json) + ?? throw new InvalidOperationException("Expected a Socket Mode envelope."); +} From bd665ead1cd7de063c4d8f46a57880f6720a852b Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Tue, 22 Sep 2026 12:13:29 -0700 Subject: [PATCH 03/31] feat(apps): add Socket Mode negotiation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SocketMode/SocketModeNegotiator.cs | 196 ++++++++++ .../SocketMode/SocketModeNegotiatorTests.cs | 346 ++++++++++++++++++ 2 files changed, 542 insertions(+) create mode 100644 src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs create mode 100644 test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeNegotiatorTests.cs diff --git a/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs b/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs new file mode 100644 index 00000000..613a76c2 --- /dev/null +++ b/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs @@ -0,0 +1,196 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Diagnostics.CodeAnalysis; +using System.Net; +using System.Net.Http.Headers; + +namespace Microsoft.Teams.Apps.SocketMode; + +internal sealed class SocketModeNegotiator +{ + private static readonly TimeSpan DefaultTimeout = TimeSpan.FromSeconds(15); + + private readonly HttpClient _httpClient; + private readonly Func> _getBotToken; + private readonly TimeSpan _timeout; + private readonly TimeProvider _timeProvider; + + internal SocketModeNegotiator( + HttpClient httpClient, + Func> getBotToken, + TimeSpan? timeout = null, + TimeProvider? timeProvider = null) + { + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + _getBotToken = getBotToken ?? throw new ArgumentNullException(nameof(getBotToken)); + _timeout = timeout ?? DefaultTimeout; + _timeProvider = timeProvider ?? TimeProvider.System; + + if (_timeout <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(timeout), "The negotiate timeout must be greater than zero."); + } + } + + internal async Task NegotiateAsync( + Uri negotiateUri, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(negotiateUri); + EnsureSecureNegotiateUri(negotiateUri); + + string? token = await _getBotToken(cancellationToken).ConfigureAwait(false); + if (string.IsNullOrWhiteSpace(token)) + { + throw new InvalidOperationException( + "Socket Mode negotiate could not acquire a Bot Framework app token."); + } + + using HttpRequestMessage request = new(HttpMethod.Post, negotiateUri); + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token); + + using CancellationTokenSource timeoutSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutSource.CancelAfter(_timeout); + + try + { + using HttpResponseMessage response = await _httpClient + .SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeoutSource.Token) + .ConfigureAwait(false); + + if (!response.IsSuccessStatusCode) + { + throw new SocketModeNegotiateException( + response.StatusCode, + GetRetryAfter(response)); + } + + string json = await response.Content + .ReadAsStringAsync(timeoutSource.Token) + .ConfigureAwait(false); + + SocketModeNegotiateResponse negotiateResponse; + try + { + negotiateResponse = SocketModeJson.Deserialize(json) + ?? throw new InvalidDataException("Socket Mode negotiate returned an empty response."); + } + catch (System.Text.Json.JsonException exception) + { + throw new InvalidDataException( + "Socket Mode negotiate returned invalid JSON.", + exception); + } + + if (string.IsNullOrWhiteSpace(negotiateResponse.Url) + || string.IsNullOrWhiteSpace(negotiateResponse.AccessToken)) + { + throw new InvalidDataException( + "Socket Mode negotiate response is missing url or accessToken."); + } + + if (!Uri.TryCreate(negotiateResponse.Url, UriKind.Absolute, out Uri? signalRUri)) + { + throw new InvalidDataException( + "Socket Mode negotiate response contains an invalid SignalR URL."); + } + + EnsureSecureSignalRUri(signalRUri); + return negotiateResponse; + } + catch (OperationCanceledException exception) when ( + !cancellationToken.IsCancellationRequested + && timeoutSource.IsCancellationRequested) + { + throw new TimeoutException( + $"Socket Mode negotiate timed out after {_timeout}.", + exception); + } + } + + private static void EnsureSecureNegotiateUri(Uri uri) + { + if (!uri.IsAbsoluteUri) + { + throw new ArgumentException( + "Socket Mode negotiate URI must be absolute.", + nameof(uri)); + } + + if (IsSecureOrLoopback(uri)) + { + return; + } + + throw new ArgumentException( + "Socket Mode negotiate URI must use HTTPS unless it targets loopback.", + nameof(uri)); + } + + private static void EnsureSecureSignalRUri(Uri uri) + { + if (IsSecureOrLoopback(uri)) + { + return; + } + + throw new InvalidDataException( + "Socket Mode negotiate response SignalR URL must use HTTPS unless it targets loopback."); + } + + private static bool IsSecureOrLoopback(Uri uri) + { + if (uri.Scheme == Uri.UriSchemeHttps) + { + return true; + } + + string host = uri.Host.Trim('[', ']'); + bool isLoopback = + host.Equals("localhost", StringComparison.OrdinalIgnoreCase) + || host.Equals("127.0.0.1", StringComparison.Ordinal) + || host.Equals("::1", StringComparison.Ordinal); + + return uri.Scheme == Uri.UriSchemeHttp && isLoopback; + } + + private TimeSpan? GetRetryAfter(HttpResponseMessage response) + { + RetryConditionHeaderValue? retryAfter = response.Headers.RetryAfter; + TimeSpan? delay = retryAfter?.Delta; + + if (delay is null && retryAfter?.Date is DateTimeOffset retryDate) + { + delay = retryDate - _timeProvider.GetUtcNow(); + } + + return delay is null || delay >= TimeSpan.Zero + ? delay + : TimeSpan.Zero; + } +} + +[SuppressMessage( + "Design", + "CA1032:Implement standard exception constructors", + Justification = "This internal transport exception is constructed only from an HTTP status and Retry-After value.")] +[SuppressMessage( + "Design", + "CA1064:Exceptions should be public", + Justification = "Socket Mode transport errors are internal implementation details, not public SDK contracts.")] +internal sealed class SocketModeNegotiateException : Exception +{ + internal SocketModeNegotiateException( + HttpStatusCode statusCode, + TimeSpan? retryAfter) + : base($"Socket Mode negotiate failed with HTTP {(int)statusCode}.") + { + StatusCode = statusCode; + RetryAfter = retryAfter; + } + + internal HttpStatusCode StatusCode { get; } + + internal TimeSpan? RetryAfter { get; } +} diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeNegotiatorTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeNegotiatorTests.cs new file mode 100644 index 00000000..bcfa27e3 --- /dev/null +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeNegotiatorTests.cs @@ -0,0 +1,346 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using Microsoft.Teams.Apps.SocketMode; + +namespace Microsoft.Teams.Apps.UnitTests.SocketMode; + +public class SocketModeNegotiatorTests +{ + private const string BotToken = "bot-token"; + private static readonly Uri NegotiateUri = new("https://botapi.skype.com/amer/v3/websockets/connect"); + + [Theory] + [InlineData("https://botapi.skype.com/v3/websockets/connect")] + [InlineData("http://localhost:5000/v3/websockets/connect")] + [InlineData("http://127.0.0.1:5000/v3/websockets/connect")] + [InlineData("http://[::1]:5000/v3/websockets/connect")] + public async Task NegotiateAsync_AllowsSecureAndLoopbackUris(string uri) + { + RecordingHandler handler = SuccessHandler(); + SocketModeNegotiator negotiator = CreateNegotiator(handler); + + SocketModeNegotiateResponse response = await negotiator.NegotiateAsync(new Uri(uri)); + + Assert.Equal("https://signalr.example.test/client", response.Url); + Assert.Equal(1, handler.SendCount); + } + + [Theory] + [InlineData("http://botapi.skype.com/v3/websockets/connect")] + [InlineData("http://127.0.0.2/v3/websockets/connect")] + [InlineData("ftp://botapi.skype.com/v3/websockets/connect")] + public async Task NegotiateAsync_RejectsInsecureRemoteUris(string uri) + { + RecordingHandler handler = SuccessHandler(); + SocketModeNegotiator negotiator = CreateNegotiator(handler); + + await Assert.ThrowsAsync( + () => negotiator.NegotiateAsync(new Uri(uri))); + + Assert.Equal(0, handler.SendCount); + } + + [Fact] + public async Task NegotiateAsync_RejectsRelativeUri() + { + RecordingHandler handler = SuccessHandler(); + SocketModeNegotiator negotiator = CreateNegotiator(handler); + + await Assert.ThrowsAsync( + () => negotiator.NegotiateAsync(new Uri("/v3/websockets/connect", UriKind.Relative))); + + Assert.Equal(0, handler.SendCount); + } + + [Fact] + public async Task NegotiateAsync_SendsAuthenticatedPostWithoutBody() + { + RecordingHandler handler = SuccessHandler(); + SocketModeNegotiator negotiator = CreateNegotiator(handler); + + await negotiator.NegotiateAsync(NegotiateUri); + + Assert.Equal(HttpMethod.Post, handler.Method); + Assert.Equal(NegotiateUri, handler.RequestUri); + Assert.Equal("Bearer", handler.Authorization?.Scheme); + Assert.Equal(BotToken, handler.Authorization?.Parameter); + Assert.False(handler.HadContent); + } + + [Theory] + [InlineData(""" + { + "url": "https://signalr.example.test/client", + "accessToken": "signalr-token", + "expiresIn": 3600 + } + """)] + [InlineData(""" + { + "Url": "https://signalr.example.test/client", + "AccessToken": "signalr-token", + "ExpiresIn": 3600 + } + """)] + public async Task NegotiateAsync_DeserializesSuccessfulResponse(string json) + { + RecordingHandler handler = JsonHandler(HttpStatusCode.OK, json); + SocketModeNegotiator negotiator = CreateNegotiator(handler); + + SocketModeNegotiateResponse response = await negotiator.NegotiateAsync(NegotiateUri); + + Assert.Equal("https://signalr.example.test/client", response.Url); + Assert.Equal("signalr-token", response.AccessToken); + Assert.Equal(3600, response.ExpiresIn); + } + + [Theory] + [InlineData("""{ "accessToken": "signalr-token" }""")] + [InlineData("""{ "url": "https://signalr.example.test/client" }""")] + [InlineData("""{ "url": "", "accessToken": "signalr-token" }""")] + [InlineData("""{ "url": "https://signalr.example.test/client", "accessToken": "" }""")] + public async Task NegotiateAsync_RejectsMissingResponseFields(string json) + { + RecordingHandler handler = JsonHandler(HttpStatusCode.OK, json); + SocketModeNegotiator negotiator = CreateNegotiator(handler); + + await Assert.ThrowsAsync( + () => negotiator.NegotiateAsync(NegotiateUri)); + } + + [Theory] + [InlineData("not-json")] + [InlineData("")] + public async Task NegotiateAsync_RejectsInvalidResponseJson(string json) + { + RecordingHandler handler = JsonHandler(HttpStatusCode.OK, json); + SocketModeNegotiator negotiator = CreateNegotiator(handler); + + await Assert.ThrowsAsync( + () => negotiator.NegotiateAsync(NegotiateUri)); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task NegotiateAsync_RejectsMissingBotTokenWithoutSendingRequest(string? token) + { + RecordingHandler handler = SuccessHandler(); + SocketModeNegotiator negotiator = CreateNegotiator(handler, _ => Task.FromResult(token)); + + await Assert.ThrowsAsync( + () => negotiator.NegotiateAsync(NegotiateUri)); + + Assert.Equal(0, handler.SendCount); + } + + [Fact] + public async Task NegotiateAsync_ThrowsTypedExceptionWithDeltaRetryAfter() + { + RecordingHandler handler = new((_, _) => + { + HttpResponseMessage response = new(HttpStatusCode.TooManyRequests); + response.Headers.RetryAfter = new RetryConditionHeaderValue(TimeSpan.FromSeconds(12)); + return Task.FromResult(response); + }); + SocketModeNegotiator negotiator = CreateNegotiator(handler); + + SocketModeNegotiateException exception = + await Assert.ThrowsAsync( + () => negotiator.NegotiateAsync(NegotiateUri)); + + Assert.Equal(HttpStatusCode.TooManyRequests, exception.StatusCode); + Assert.Equal(TimeSpan.FromSeconds(12), exception.RetryAfter); + } + + [Fact] + public async Task NegotiateAsync_ParsesDateRetryAfter() + { + DateTimeOffset now = new(2026, 9, 22, 12, 0, 0, TimeSpan.Zero); + RecordingHandler handler = new((_, _) => + { + HttpResponseMessage response = new(HttpStatusCode.ServiceUnavailable); + response.Headers.RetryAfter = new RetryConditionHeaderValue(now.AddSeconds(30)); + return Task.FromResult(response); + }); + SocketModeNegotiator negotiator = CreateNegotiator( + handler, + timeProvider: new FixedTimeProvider(now)); + + SocketModeNegotiateException exception = + await Assert.ThrowsAsync( + () => negotiator.NegotiateAsync(NegotiateUri)); + + Assert.Equal(TimeSpan.FromSeconds(30), exception.RetryAfter); + } + + [Fact] + public async Task NegotiateAsync_ClampsPastRetryAfterDateToZero() + { + DateTimeOffset now = new(2026, 9, 22, 12, 0, 0, TimeSpan.Zero); + RecordingHandler handler = new((_, _) => + { + HttpResponseMessage response = new(HttpStatusCode.ServiceUnavailable); + response.Headers.RetryAfter = new RetryConditionHeaderValue(now.AddSeconds(-30)); + return Task.FromResult(response); + }); + SocketModeNegotiator negotiator = CreateNegotiator( + handler, + timeProvider: new FixedTimeProvider(now)); + + SocketModeNegotiateException exception = + await Assert.ThrowsAsync( + () => negotiator.NegotiateAsync(NegotiateUri)); + + Assert.Equal(TimeSpan.Zero, exception.RetryAfter); + } + + [Fact] + public async Task NegotiateAsync_LeavesRetryAfterNullWhenAbsent() + { + RecordingHandler handler = new((_, _) => + Task.FromResult(new HttpResponseMessage(HttpStatusCode.ServiceUnavailable))); + SocketModeNegotiator negotiator = CreateNegotiator(handler); + + SocketModeNegotiateException exception = + await Assert.ThrowsAsync( + () => negotiator.NegotiateAsync(NegotiateUri)); + + Assert.Null(exception.RetryAfter); + } + + [Fact] + public async Task NegotiateAsync_HonorsCallerCancellation() + { + RecordingHandler handler = HangingHandler(); + SocketModeNegotiator negotiator = CreateNegotiator(handler); + using CancellationTokenSource cancellationSource = new(); + + cancellationSource.Cancel(); + + await Assert.ThrowsAnyAsync( + () => negotiator.NegotiateAsync(NegotiateUri, cancellationSource.Token)); + } + + [Fact] + public async Task NegotiateAsync_TimesOutHangingRequest() + { + RecordingHandler handler = HangingHandler(); + SocketModeNegotiator negotiator = CreateNegotiator( + handler, + timeout: TimeSpan.FromMilliseconds(50)); + + TimeoutException exception = await Assert.ThrowsAsync( + () => negotiator.NegotiateAsync(NegotiateUri)); + + Assert.Contains("timed out", exception.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task NegotiateAsync_DoesNotExposeSecretsInServiceFailure() + { + const string signalRToken = "signalr-secret"; + RecordingHandler handler = JsonHandler( + HttpStatusCode.BadGateway, + $$"""{ "accessToken": "{{signalRToken}}" }"""); + SocketModeNegotiator negotiator = CreateNegotiator(handler); + + SocketModeNegotiateException exception = + await Assert.ThrowsAsync( + () => negotiator.NegotiateAsync(NegotiateUri)); + + Assert.DoesNotContain(BotToken, exception.ToString(), StringComparison.Ordinal); + Assert.DoesNotContain(signalRToken, exception.ToString(), StringComparison.Ordinal); + } + + [Theory] + [InlineData("http://signalr.example.test/client")] + [InlineData("ftp://signalr.example.test/client")] + [InlineData("/relative/client")] + public async Task NegotiateAsync_RejectsInvalidSignalRUrl(string signalRUrl) + { + RecordingHandler handler = JsonHandler( + HttpStatusCode.OK, + $$""" + { + "url": "{{signalRUrl}}", + "accessToken": "signalr-token" + } + """); + SocketModeNegotiator negotiator = CreateNegotiator(handler); + + await Assert.ThrowsAsync( + () => negotiator.NegotiateAsync(NegotiateUri)); + } + + private static SocketModeNegotiator CreateNegotiator( + RecordingHandler handler, + Func>? getBotToken = null, + TimeSpan? timeout = null, + TimeProvider? timeProvider = null) + { + HttpClient httpClient = new(handler); + return new SocketModeNegotiator( + httpClient, + getBotToken ?? (_ => Task.FromResult(BotToken)), + timeout, + timeProvider); + } + + private static RecordingHandler SuccessHandler() + => JsonHandler( + HttpStatusCode.OK, + """ + { + "url": "https://signalr.example.test/client", + "accessToken": "signalr-token", + "expiresIn": 3600 + } + """); + + private static RecordingHandler JsonHandler(HttpStatusCode statusCode, string json) + => new((_, _) => Task.FromResult(new HttpResponseMessage(statusCode) + { + Content = new StringContent(json, Encoding.UTF8, "application/json"), + })); + + private static RecordingHandler HangingHandler() + => new(async (_, cancellationToken) => + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return new HttpResponseMessage(HttpStatusCode.OK); + }); + + private sealed class RecordingHandler( + Func> send) + : HttpMessageHandler + { + internal int SendCount { get; private set; } + internal HttpMethod? Method { get; private set; } + internal Uri? RequestUri { get; private set; } + internal AuthenticationHeaderValue? Authorization { get; private set; } + internal bool HadContent { get; private set; } + + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + SendCount++; + Method = request.Method; + RequestUri = request.RequestUri; + Authorization = request.Headers.Authorization; + HadContent = request.Content is not null; + return send(request, cancellationToken); + } + } + + private sealed class FixedTimeProvider(DateTimeOffset now) : TimeProvider + { + public override DateTimeOffset GetUtcNow() => now; + } +} From 059705c03c7b820cef19d12c71d8e1b8e345d648 Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Tue, 22 Sep 2026 14:46:22 -0700 Subject: [PATCH 04/31] feat(apps): define Socket Mode connection abstraction Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SocketMode/SocketModeConnection.cs | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 src/Microsoft.Teams.Apps/SocketMode/SocketModeConnection.cs diff --git a/src/Microsoft.Teams.Apps/SocketMode/SocketModeConnection.cs b/src/Microsoft.Teams.Apps/SocketMode/SocketModeConnection.cs new file mode 100644 index 00000000..dcf0596a --- /dev/null +++ b/src/Microsoft.Teams.Apps/SocketMode/SocketModeConnection.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.Teams.Apps.SocketMode; + +internal interface ISocketConnection : IAsyncDisposable +{ + TimeSpan? TokenLifetime { get; } + + Task StartAsync(CancellationToken cancellationToken); + + Task StopAsync(CancellationToken cancellationToken = default); +} + +internal interface ISocketConnectionFactory +{ + ISocketConnection Create( + Uri negotiateUri, + SocketConnectionHandlers handlers); +} + +internal sealed class SocketConnectionHandlers +{ + internal SocketConnectionHandlers( + Func> onActivity, + Action onReady, + Action onClosed) + { + OnActivity = onActivity ?? throw new ArgumentNullException(nameof(onActivity)); + OnReady = onReady ?? throw new ArgumentNullException(nameof(onReady)); + OnClosed = onClosed ?? throw new ArgumentNullException(nameof(onClosed)); + } + + internal Func> OnActivity { get; } + + internal Action OnReady { get; } + + internal Action OnClosed { get; } +} From 3c0e90213af4964ed07ca9b9815e5e83f88b9e29 Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Wed, 23 Sep 2026 11:05:50 -0700 Subject: [PATCH 05/31] docs(apps): document Socket Mode transport Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SocketMode/SocketModeConnection.cs | 41 ++++++++++ .../SocketMode/SocketModeEnvelope.cs | 30 ++++++++ .../SocketMode/SocketModeJson.cs | 15 ++++ .../SocketMode/SocketModeNegotiator.cs | 51 ++++++++++++- .../SocketMode/SocketModeProtocol.cs | 14 ++++ .../SocketMode/SocketModeProtocolModels.cs | 74 +++++++++++++++++++ 6 files changed, 222 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.Teams.Apps/SocketMode/SocketModeConnection.cs b/src/Microsoft.Teams.Apps/SocketMode/SocketModeConnection.cs index dcf0596a..65f4feb5 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/SocketModeConnection.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/SocketModeConnection.cs @@ -3,24 +3,56 @@ namespace Microsoft.Teams.Apps.SocketMode; +/// +/// Represents a live Socket Mode transport connection. +/// internal interface ISocketConnection : IAsyncDisposable { + /// + /// Gets the remaining lifetime of the connection token, when available. + /// TimeSpan? TokenLifetime { get; } + /// + /// Starts the connection. + /// + /// A token for cancelling the operation. Task StartAsync(CancellationToken cancellationToken); + /// + /// Stops the connection. + /// + /// A token for cancelling the operation. Task StopAsync(CancellationToken cancellationToken = default); } +/// +/// Creates Socket Mode transport connections. +/// internal interface ISocketConnectionFactory { + /// + /// Creates a connection for the specified negotiate endpoint and handlers. + /// + /// The endpoint used to negotiate the connection. + /// Callbacks for frames and connection closure. + /// A new Socket Mode connection. ISocketConnection Create( Uri negotiateUri, SocketConnectionHandlers handlers); } +/// +/// Defines callbacks used by a Socket Mode connection. +/// internal sealed class SocketConnectionHandlers { + /// + /// Initializes the connection callbacks. + /// + /// Handles an incoming activity envelope. + /// Handles the ready frame. + /// Handles connection closure. internal SocketConnectionHandlers( Func> onActivity, Action onReady, @@ -31,9 +63,18 @@ internal SocketConnectionHandlers( OnClosed = onClosed ?? throw new ArgumentNullException(nameof(onClosed)); } + /// + /// Gets the incoming activity callback. + /// internal Func> OnActivity { get; } + /// + /// Gets the ready-frame callback. + /// internal Action OnReady { get; } + /// + /// Gets the connection-closed callback. + /// internal Action OnClosed { get; } } diff --git a/src/Microsoft.Teams.Apps/SocketMode/SocketModeEnvelope.cs b/src/Microsoft.Teams.Apps/SocketMode/SocketModeEnvelope.cs index 0f998790..4b0dd43b 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/SocketModeEnvelope.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/SocketModeEnvelope.cs @@ -6,8 +6,17 @@ namespace Microsoft.Teams.Apps.SocketMode; +/// +/// Parses Socket Mode envelopes and creates reply frames. +/// internal static class SocketModeEnvelope { + /// + /// Attempts to read a Teams activity from an envelope. + /// + /// The envelope to inspect. + /// The parsed activity, when successful. + /// true when the envelope contains a valid activity; otherwise, false. internal static bool TryReadActivity(SocketActivityEnvelope envelope, out CoreActivity? activity) { ArgumentNullException.ThrowIfNull(envelope); @@ -16,6 +25,15 @@ internal static bool TryReadActivity(SocketActivityEnvelope envelope, out CoreAc || TryReadActivityCandidate(envelope.Activity, out activity); } + /// + /// Creates an acknowledgement for an activity envelope. + /// + /// The envelope being acknowledged. + /// The key identifying the bot connection. + /// The time the envelope was received. + /// The time the reply was created. + /// The acknowledgement status code. + /// The acknowledgement frame. internal static SocketReplyFrame CreateAcknowledgement( SocketActivityEnvelope envelope, string? botKey, @@ -35,6 +53,15 @@ internal static SocketReplyFrame CreateAcknowledgement( }; } + /// + /// Creates a reply for an invoke activity envelope. + /// + /// The envelope being answered. + /// The key identifying the bot connection. + /// The dispatch result to return. + /// The time the envelope was received. + /// The time the reply was created. + /// The invoke reply frame. internal static SocketReplyFrame CreateInvokeReply( SocketActivityEnvelope envelope, string? botKey, @@ -56,6 +83,9 @@ internal static SocketReplyFrame CreateInvokeReply( }; } + /// + /// Attempts to deserialize an activity candidate from an envelope. + /// private static bool TryReadActivityCandidate(JsonElement? candidate, out CoreActivity? activity) { activity = null; diff --git a/src/Microsoft.Teams.Apps/SocketMode/SocketModeJson.cs b/src/Microsoft.Teams.Apps/SocketMode/SocketModeJson.cs index 095832a5..93cb14f4 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/SocketModeJson.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/SocketModeJson.cs @@ -6,6 +6,9 @@ namespace Microsoft.Teams.Apps.SocketMode; +/// +/// Provides the shared JSON configuration for Socket Mode frames. +/// internal static class SocketModeJson { private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) @@ -14,9 +17,21 @@ internal static class SocketModeJson PropertyNameCaseInsensitive = true, }; + /// + /// Deserializes a Socket Mode JSON payload. + /// + /// The payload type. + /// The JSON payload. + /// The deserialized value, or null for a JSON null value. internal static T? Deserialize(string json) => JsonSerializer.Deserialize(json, SerializerOptions); + /// + /// Serializes a Socket Mode payload. + /// + /// The payload type. + /// The value to serialize. + /// The serialized JSON payload. internal static string Serialize(T value) => JsonSerializer.Serialize(value, SerializerOptions); } diff --git a/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs b/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs index 613a76c2..4e5f62a6 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs @@ -7,6 +7,9 @@ namespace Microsoft.Teams.Apps.SocketMode; +/// +/// Negotiates connection details for a Socket Mode transport. +/// internal sealed class SocketModeNegotiator { private static readonly TimeSpan DefaultTimeout = TimeSpan.FromSeconds(15); @@ -16,6 +19,14 @@ internal sealed class SocketModeNegotiator private readonly TimeSpan _timeout; private readonly TimeProvider _timeProvider; + /// + /// Initializes a Socket Mode negotiator. + /// + /// The client used to call the negotiate endpoint. + /// Acquires a Bot Framework app token. + /// The negotiate request timeout. + /// The time provider used to evaluate retry delays. + /// Thrown when is not positive. internal SocketModeNegotiator( HttpClient httpClient, Func> getBotToken, @@ -33,6 +44,14 @@ internal SocketModeNegotiator( } } + /// + /// Negotiates the endpoint and access token for a Socket Mode connection. + /// + /// The Socket Mode negotiate endpoint. + /// A token for cancelling the operation. + /// The negotiated connection details. + /// Thrown when the endpoint returns an unsuccessful status code. + /// Thrown when negotiation exceeds the configured timeout. internal async Task NegotiateAsync( Uri negotiateUri, CancellationToken cancellationToken = default) @@ -109,6 +128,9 @@ internal async Task NegotiateAsync( } } + /// + /// Ensures that a negotiate URI uses an allowed scheme and host. + /// private static void EnsureSecureNegotiateUri(Uri uri) { if (!uri.IsAbsoluteUri) @@ -128,6 +150,9 @@ private static void EnsureSecureNegotiateUri(Uri uri) nameof(uri)); } + /// + /// Ensures that a negotiated SignalR URI uses an allowed scheme and host. + /// private static void EnsureSecureSignalRUri(Uri uri) { if (IsSecureOrLoopback(uri)) @@ -139,6 +164,9 @@ private static void EnsureSecureSignalRUri(Uri uri) "Socket Mode negotiate response SignalR URL must use HTTPS unless it targets loopback."); } + /// + /// Determines whether a URI is secure or an HTTP loopback endpoint. + /// private static bool IsSecureOrLoopback(Uri uri) { if (uri.Scheme == Uri.UriSchemeHttps) @@ -155,6 +183,9 @@ private static bool IsSecureOrLoopback(Uri uri) return uri.Scheme == Uri.UriSchemeHttp && isLoopback; } + /// + /// Gets the server-provided retry delay. + /// private TimeSpan? GetRetryAfter(HttpResponseMessage response) { RetryConditionHeaderValue? retryAfter = response.Headers.RetryAfter; @@ -171,16 +202,24 @@ private static bool IsSecureOrLoopback(Uri uri) } } +/// +/// Represents an unsuccessful Socket Mode negotiate response. +/// [SuppressMessage( - "Design", - "CA1032:Implement standard exception constructors", - Justification = "This internal transport exception is constructed only from an HTTP status and Retry-After value.")] + "Design", + "CA1032:Implement standard exception constructors", + Justification = "This internal transport exception is constructed only from an HTTP status and Retry-After value.")] [SuppressMessage( "Design", "CA1064:Exceptions should be public", Justification = "Socket Mode transport errors are internal implementation details, not public SDK contracts.")] internal sealed class SocketModeNegotiateException : Exception { + /// + /// Initializes a Socket Mode negotiation exception. + /// + /// The HTTP status returned by the negotiate endpoint. + /// The server-provided retry delay, when available. internal SocketModeNegotiateException( HttpStatusCode statusCode, TimeSpan? retryAfter) @@ -190,7 +229,13 @@ internal SocketModeNegotiateException( RetryAfter = retryAfter; } + /// + /// Gets the HTTP status returned by the negotiate endpoint. + /// internal HttpStatusCode StatusCode { get; } + /// + /// Gets the server-provided retry delay, when available. + /// internal TimeSpan? RetryAfter { get; } } diff --git a/src/Microsoft.Teams.Apps/SocketMode/SocketModeProtocol.cs b/src/Microsoft.Teams.Apps/SocketMode/SocketModeProtocol.cs index 08c79bc4..1b336b6e 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/SocketModeProtocol.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/SocketModeProtocol.cs @@ -3,9 +3,23 @@ namespace Microsoft.Teams.Apps.SocketMode; +/// +/// Defines constants used by the Socket Mode protocol. +/// internal static class SocketModeProtocol { + /// + /// The protocol version emitted by this SDK. + /// internal const int CurrentVersion = 1; + + /// + /// The default base URL for Socket Mode negotiation. + /// internal const string DefaultNegotiateBaseUrl = "https://botapi.skype.com"; + + /// + /// The path for the Socket Mode negotiate endpoint. + /// internal const string NegotiatePath = "/v3/websockets/connect"; } diff --git a/src/Microsoft.Teams.Apps/SocketMode/SocketModeProtocolModels.cs b/src/Microsoft.Teams.Apps/SocketMode/SocketModeProtocolModels.cs index 000088b5..babe1733 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/SocketModeProtocolModels.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/SocketModeProtocolModels.cs @@ -6,76 +6,150 @@ namespace Microsoft.Teams.Apps.SocketMode; +/// +/// Contains the connection details returned by Socket Mode negotiation. +/// internal sealed class SocketModeNegotiateResponse { + /// + /// Gets the negotiated SignalR endpoint. + /// [JsonPropertyName("url")] public string? Url { get; init; } + /// + /// Gets the access token for the negotiated endpoint. + /// [JsonPropertyName("accessToken")] public string? AccessToken { get; init; } + /// + /// Gets the access token lifetime in seconds. + /// [JsonPropertyName("expiresIn")] public int ExpiresIn { get; init; } } +/// +/// Identifies an established Socket Mode connection. +/// internal sealed class SocketReadyFrame { + /// + /// Gets the key identifying the bot connection. + /// [JsonPropertyName("botKey")] public string? BotKey { get; init; } + /// + /// Gets the server-assigned connection identifier. + /// [JsonPropertyName("connectionId")] public string? ConnectionId { get; init; } } +/// +/// Represents an activity delivered over Socket Mode. +/// internal sealed class SocketActivityEnvelope { + /// + /// Gets the protocol version used by the envelope. + /// [JsonPropertyName("protocolVersion")] public int? ProtocolVersion { get; init; } + /// + /// Gets the identifier used to correlate replies with the envelope. + /// [JsonPropertyName("envelopeId")] public string? EnvelopeId { get; init; } + /// + /// Gets the envelope type. + /// [JsonPropertyName("type")] public string? Type { get; init; } + /// + /// Gets whether the envelope requires a reply. + /// [JsonPropertyName("ackRequired")] public bool? AckRequired { get; init; } + /// + /// Gets the activity payload. + /// [JsonPropertyName("payload")] public JsonElement? Payload { get; init; } + /// + /// Gets the activity from the alternate activity field. + /// [JsonPropertyName("activity")] public JsonElement? Activity { get; init; } + /// + /// Gets the correlation vector associated with the envelope. + /// [JsonPropertyName("cv")] public string? CorrelationVector { get; init; } } +/// +/// Represents a reply sent for a Socket Mode activity envelope. +/// internal sealed class SocketReplyFrame { + /// + /// Gets the Socket Mode protocol version. + /// [JsonPropertyName("protocolVersion")] public int ProtocolVersion { get; init; } = SocketModeProtocol.CurrentVersion; + /// + /// Gets the identifier of the envelope being answered. + /// [JsonPropertyName("envelopeId")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? EnvelopeId { get; init; } + /// + /// Gets the key identifying the bot connection. + /// [JsonPropertyName("botKey")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? BotKey { get; init; } + /// + /// Gets the reply status code. + /// [JsonPropertyName("status")] public int Status { get; init; } + /// + /// Gets the reply body. + /// [JsonPropertyName("body")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public object? Body { get; init; } + /// + /// Gets the envelope receive time as Unix time in milliseconds. + /// [JsonPropertyName("recvAt")] public long ReceivedAtUnixMilliseconds { get; init; } + /// + /// Gets the reply creation time as Unix time in milliseconds. + /// [JsonPropertyName("ts")] public long TimestampUnixMilliseconds { get; init; } } +/// +/// Contains the status and optional body produced by activity dispatch. +/// +/// The dispatch status code. +/// The optional response body. internal sealed record SocketDispatchResult(int Status, object? Body = null); From ab6f062149547a1c377d03a3f46b6a57cb2dbd56 Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Wed, 23 Sep 2026 14:50:31 -0700 Subject: [PATCH 06/31] fix(apps): allow Socket Mode websocket URLs Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SocketMode/SocketModeNegotiator.cs | 17 +++++--- .../SocketMode/SocketModeNegotiatorTests.cs | 43 +++++++++++++++---- 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs b/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs index 4e5f62a6..b8dcb21d 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs @@ -140,7 +140,7 @@ private static void EnsureSecureNegotiateUri(Uri uri) nameof(uri)); } - if (IsSecureOrLoopback(uri)) + if (IsSecureOrLoopback(uri, allowWebSocketSchemes: false)) { return; } @@ -155,21 +155,22 @@ private static void EnsureSecureNegotiateUri(Uri uri) /// private static void EnsureSecureSignalRUri(Uri uri) { - if (IsSecureOrLoopback(uri)) + if (IsSecureOrLoopback(uri, allowWebSocketSchemes: true)) { return; } throw new InvalidDataException( - "Socket Mode negotiate response SignalR URL must use HTTPS unless it targets loopback."); + "Socket Mode negotiate response SignalR URL must use HTTPS or WSS unless it targets loopback."); } /// - /// Determines whether a URI is secure or an HTTP loopback endpoint. + /// Determines whether a URI uses an allowed secure scheme or its loopback equivalent. /// - private static bool IsSecureOrLoopback(Uri uri) + private static bool IsSecureOrLoopback(Uri uri, bool allowWebSocketSchemes) { - if (uri.Scheme == Uri.UriSchemeHttps) + if (uri.Scheme == Uri.UriSchemeHttps + || (allowWebSocketSchemes && uri.Scheme == "wss")) { return true; } @@ -180,7 +181,9 @@ private static bool IsSecureOrLoopback(Uri uri) || host.Equals("127.0.0.1", StringComparison.Ordinal) || host.Equals("::1", StringComparison.Ordinal); - return uri.Scheme == Uri.UriSchemeHttp && isLoopback; + return isLoopback + && (uri.Scheme == Uri.UriSchemeHttp + || (allowWebSocketSchemes && uri.Scheme == "ws")); } /// diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeNegotiatorTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeNegotiatorTests.cs index bcfa27e3..332f0faf 100644 --- a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeNegotiatorTests.cs +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeNegotiatorTests.cs @@ -25,15 +25,17 @@ public async Task NegotiateAsync_AllowsSecureAndLoopbackUris(string uri) SocketModeNegotiateResponse response = await negotiator.NegotiateAsync(new Uri(uri)); - Assert.Equal("https://signalr.example.test/client", response.Url); + Assert.Equal("wss://signalr.example.test/client", response.Url); Assert.Equal(1, handler.SendCount); } [Theory] [InlineData("http://botapi.skype.com/v3/websockets/connect")] [InlineData("http://127.0.0.2/v3/websockets/connect")] + [InlineData("wss://botapi.skype.com/v3/websockets/connect")] + [InlineData("ws://localhost:5000/v3/websockets/connect")] [InlineData("ftp://botapi.skype.com/v3/websockets/connect")] - public async Task NegotiateAsync_RejectsInsecureRemoteUris(string uri) + public async Task NegotiateAsync_RejectsUnsupportedNegotiateUris(string uri) { RecordingHandler handler = SuccessHandler(); SocketModeNegotiator negotiator = CreateNegotiator(handler); @@ -74,14 +76,14 @@ public async Task NegotiateAsync_SendsAuthenticatedPostWithoutBody() [Theory] [InlineData(""" { - "url": "https://signalr.example.test/client", + "url": "wss://signalr.example.test/client", "accessToken": "signalr-token", "expiresIn": 3600 } """)] [InlineData(""" { - "Url": "https://signalr.example.test/client", + "Url": "wss://signalr.example.test/client", "AccessToken": "signalr-token", "ExpiresIn": 3600 } @@ -93,16 +95,16 @@ public async Task NegotiateAsync_DeserializesSuccessfulResponse(string json) SocketModeNegotiateResponse response = await negotiator.NegotiateAsync(NegotiateUri); - Assert.Equal("https://signalr.example.test/client", response.Url); + Assert.Equal("wss://signalr.example.test/client", response.Url); Assert.Equal("signalr-token", response.AccessToken); Assert.Equal(3600, response.ExpiresIn); } [Theory] [InlineData("""{ "accessToken": "signalr-token" }""")] - [InlineData("""{ "url": "https://signalr.example.test/client" }""")] + [InlineData("""{ "url": "wss://signalr.example.test/client" }""")] [InlineData("""{ "url": "", "accessToken": "signalr-token" }""")] - [InlineData("""{ "url": "https://signalr.example.test/client", "accessToken": "" }""")] + [InlineData("""{ "url": "wss://signalr.example.test/client", "accessToken": "" }""")] public async Task NegotiateAsync_RejectsMissingResponseFields(string json) { RecordingHandler handler = JsonHandler(HttpStatusCode.OK, json); @@ -258,8 +260,33 @@ await Assert.ThrowsAsync( Assert.DoesNotContain(signalRToken, exception.ToString(), StringComparison.Ordinal); } + [Theory] + [InlineData("wss://signalr.example.test/client")] + [InlineData("https://signalr.example.test/client")] + [InlineData("ws://localhost:5000/client")] + [InlineData("ws://127.0.0.1:5000/client")] + [InlineData("ws://[::1]:5000/client")] + [InlineData("http://localhost:5000/client")] + public async Task NegotiateAsync_AllowsSecureAndLoopbackSignalRUrls(string signalRUrl) + { + RecordingHandler handler = JsonHandler( + HttpStatusCode.OK, + $$""" + { + "url": "{{signalRUrl}}", + "accessToken": "signalr-token" + } + """); + SocketModeNegotiator negotiator = CreateNegotiator(handler); + + SocketModeNegotiateResponse response = await negotiator.NegotiateAsync(NegotiateUri); + + Assert.Equal(signalRUrl, response.Url); + } + [Theory] [InlineData("http://signalr.example.test/client")] + [InlineData("ws://signalr.example.test/client")] [InlineData("ftp://signalr.example.test/client")] [InlineData("/relative/client")] public async Task NegotiateAsync_RejectsInvalidSignalRUrl(string signalRUrl) @@ -297,7 +324,7 @@ private static RecordingHandler SuccessHandler() HttpStatusCode.OK, """ { - "url": "https://signalr.example.test/client", + "url": "wss://signalr.example.test/client", "accessToken": "signalr-token", "expiresIn": 3600 } From 9eea9be28d1beb04de70d389849613b9a83d3c7d Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Wed, 23 Sep 2026 14:51:49 -0700 Subject: [PATCH 07/31] fix(apps): require HTTP SignalR negotiate URLs Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SocketMode/SocketModeNegotiator.cs | 17 ++++++------- .../SocketMode/SocketModeNegotiatorTests.cs | 24 +++++++++---------- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs b/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs index b8dcb21d..4e5f62a6 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs @@ -140,7 +140,7 @@ private static void EnsureSecureNegotiateUri(Uri uri) nameof(uri)); } - if (IsSecureOrLoopback(uri, allowWebSocketSchemes: false)) + if (IsSecureOrLoopback(uri)) { return; } @@ -155,22 +155,21 @@ private static void EnsureSecureNegotiateUri(Uri uri) /// private static void EnsureSecureSignalRUri(Uri uri) { - if (IsSecureOrLoopback(uri, allowWebSocketSchemes: true)) + if (IsSecureOrLoopback(uri)) { return; } throw new InvalidDataException( - "Socket Mode negotiate response SignalR URL must use HTTPS or WSS unless it targets loopback."); + "Socket Mode negotiate response SignalR URL must use HTTPS unless it targets loopback."); } /// - /// Determines whether a URI uses an allowed secure scheme or its loopback equivalent. + /// Determines whether a URI is secure or an HTTP loopback endpoint. /// - private static bool IsSecureOrLoopback(Uri uri, bool allowWebSocketSchemes) + private static bool IsSecureOrLoopback(Uri uri) { - if (uri.Scheme == Uri.UriSchemeHttps - || (allowWebSocketSchemes && uri.Scheme == "wss")) + if (uri.Scheme == Uri.UriSchemeHttps) { return true; } @@ -181,9 +180,7 @@ private static bool IsSecureOrLoopback(Uri uri, bool allowWebSocketSchemes) || host.Equals("127.0.0.1", StringComparison.Ordinal) || host.Equals("::1", StringComparison.Ordinal); - return isLoopback - && (uri.Scheme == Uri.UriSchemeHttp - || (allowWebSocketSchemes && uri.Scheme == "ws")); + return uri.Scheme == Uri.UriSchemeHttp && isLoopback; } /// diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeNegotiatorTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeNegotiatorTests.cs index 332f0faf..0aa68e67 100644 --- a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeNegotiatorTests.cs +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeNegotiatorTests.cs @@ -25,7 +25,7 @@ public async Task NegotiateAsync_AllowsSecureAndLoopbackUris(string uri) SocketModeNegotiateResponse response = await negotiator.NegotiateAsync(new Uri(uri)); - Assert.Equal("wss://signalr.example.test/client", response.Url); + Assert.Equal("https://signalr.example.test/client", response.Url); Assert.Equal(1, handler.SendCount); } @@ -76,14 +76,14 @@ public async Task NegotiateAsync_SendsAuthenticatedPostWithoutBody() [Theory] [InlineData(""" { - "url": "wss://signalr.example.test/client", + "url": "https://signalr.example.test/client", "accessToken": "signalr-token", "expiresIn": 3600 } """)] [InlineData(""" { - "Url": "wss://signalr.example.test/client", + "Url": "https://signalr.example.test/client", "AccessToken": "signalr-token", "ExpiresIn": 3600 } @@ -95,16 +95,16 @@ public async Task NegotiateAsync_DeserializesSuccessfulResponse(string json) SocketModeNegotiateResponse response = await negotiator.NegotiateAsync(NegotiateUri); - Assert.Equal("wss://signalr.example.test/client", response.Url); + Assert.Equal("https://signalr.example.test/client", response.Url); Assert.Equal("signalr-token", response.AccessToken); Assert.Equal(3600, response.ExpiresIn); } [Theory] [InlineData("""{ "accessToken": "signalr-token" }""")] - [InlineData("""{ "url": "wss://signalr.example.test/client" }""")] + [InlineData("""{ "url": "https://signalr.example.test/client" }""")] [InlineData("""{ "url": "", "accessToken": "signalr-token" }""")] - [InlineData("""{ "url": "wss://signalr.example.test/client", "accessToken": "" }""")] + [InlineData("""{ "url": "https://signalr.example.test/client", "accessToken": "" }""")] public async Task NegotiateAsync_RejectsMissingResponseFields(string json) { RecordingHandler handler = JsonHandler(HttpStatusCode.OK, json); @@ -261,13 +261,11 @@ await Assert.ThrowsAsync( } [Theory] - [InlineData("wss://signalr.example.test/client")] [InlineData("https://signalr.example.test/client")] - [InlineData("ws://localhost:5000/client")] - [InlineData("ws://127.0.0.1:5000/client")] - [InlineData("ws://[::1]:5000/client")] [InlineData("http://localhost:5000/client")] - public async Task NegotiateAsync_AllowsSecureAndLoopbackSignalRUrls(string signalRUrl) + [InlineData("http://127.0.0.1:5000/client")] + [InlineData("http://[::1]:5000/client")] + public async Task NegotiateAsync_AllowsHttpsAndLoopbackHttpSignalRUrls(string signalRUrl) { RecordingHandler handler = JsonHandler( HttpStatusCode.OK, @@ -286,7 +284,9 @@ public async Task NegotiateAsync_AllowsSecureAndLoopbackSignalRUrls(string signa [Theory] [InlineData("http://signalr.example.test/client")] + [InlineData("wss://signalr.example.test/client")] [InlineData("ws://signalr.example.test/client")] + [InlineData("ws://localhost:5000/client")] [InlineData("ftp://signalr.example.test/client")] [InlineData("/relative/client")] public async Task NegotiateAsync_RejectsInvalidSignalRUrl(string signalRUrl) @@ -324,7 +324,7 @@ private static RecordingHandler SuccessHandler() HttpStatusCode.OK, """ { - "url": "wss://signalr.example.test/client", + "url": "https://signalr.example.test/client", "accessToken": "signalr-token", "expiresIn": 3600 } From b994c3fbdca5962b361a3e18677751e621f1a4f6 Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Wed, 23 Sep 2026 14:58:08 -0700 Subject: [PATCH 08/31] fix(apps): include region in Socket Mode endpoint Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/Microsoft.Teams.Apps/SocketMode/SocketModeProtocol.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.Teams.Apps/SocketMode/SocketModeProtocol.cs b/src/Microsoft.Teams.Apps/SocketMode/SocketModeProtocol.cs index 1b336b6e..d9c048d5 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/SocketModeProtocol.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/SocketModeProtocol.cs @@ -21,5 +21,5 @@ internal static class SocketModeProtocol /// /// The path for the Socket Mode negotiate endpoint. /// - internal const string NegotiatePath = "/v3/websockets/connect"; + internal const string NegotiatePath = "/amer/v3/websockets/connect"; } From ac2a03100acc9d33fe231e58f7a2582dfbcbfbe9 Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Wed, 23 Sep 2026 15:00:51 -0700 Subject: [PATCH 09/31] fix(apps): time out Socket Mode token acquisition Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SocketMode/SocketModeNegotiator.cs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs b/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs index 4e5f62a6..06511acf 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs @@ -59,21 +59,21 @@ internal async Task NegotiateAsync( ArgumentNullException.ThrowIfNull(negotiateUri); EnsureSecureNegotiateUri(negotiateUri); - string? token = await _getBotToken(cancellationToken).ConfigureAwait(false); - if (string.IsNullOrWhiteSpace(token)) - { - throw new InvalidOperationException( - "Socket Mode negotiate could not acquire a Bot Framework app token."); - } - - using HttpRequestMessage request = new(HttpMethod.Post, negotiateUri); - request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token); - using CancellationTokenSource timeoutSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); timeoutSource.CancelAfter(_timeout); try { + string? token = await _getBotToken(timeoutSource.Token).ConfigureAwait(false); + if (string.IsNullOrWhiteSpace(token)) + { + throw new InvalidOperationException( + "Socket Mode negotiate could not acquire a Bot Framework app token."); + } + + using HttpRequestMessage request = new(HttpMethod.Post, negotiateUri); + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token); + using HttpResponseMessage response = await _httpClient .SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeoutSource.Token) .ConfigureAwait(false); From dd689420643c2d169c2b8ab366641ff124e99b6a Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Wed, 23 Sep 2026 15:01:41 -0700 Subject: [PATCH 10/31] test(apps): cover Socket Mode token timeout Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SocketMode/SocketModeNegotiatorTests.cs | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeNegotiatorTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeNegotiatorTests.cs index 0aa68e67..621acb0d 100644 --- a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeNegotiatorTests.cs +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeNegotiatorTests.cs @@ -243,6 +243,26 @@ public async Task NegotiateAsync_TimesOutHangingRequest() Assert.Contains("timed out", exception.Message, StringComparison.Ordinal); } + [Fact] + public async Task NegotiateAsync_TimesOutHangingBotTokenAcquisition() + { + RecordingHandler handler = SuccessHandler(); + SocketModeNegotiator negotiator = CreateNegotiator( + handler, + async cancellationToken => + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return BotToken; + }, + timeout: TimeSpan.FromMilliseconds(50)); + + TimeoutException exception = await Assert.ThrowsAsync( + () => negotiator.NegotiateAsync(NegotiateUri)); + + Assert.Contains("timed out", exception.Message, StringComparison.Ordinal); + Assert.Equal(0, handler.SendCount); + } + [Fact] public async Task NegotiateAsync_DoesNotExposeSecretsInServiceFailure() { From 2ebe3cfb9280c60f44c4caa033a3abc26b69fade Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Wed, 23 Sep 2026 15:10:01 -0700 Subject: [PATCH 11/31] fix(apps): tighten Socket Mode validation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs | 3 ++- .../SocketMode/SocketModeEnvelopeTests.cs | 1 + .../SocketMode/SocketModeNegotiatorTests.cs | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs b/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs index 06511acf..88e09348 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs @@ -169,7 +169,8 @@ private static void EnsureSecureSignalRUri(Uri uri) /// private static bool IsSecureOrLoopback(Uri uri) { - if (uri.Scheme == Uri.UriSchemeHttps) + if (uri.Scheme == Uri.UriSchemeHttps + && !string.IsNullOrEmpty(uri.Host)) { return true; } diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeEnvelopeTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeEnvelopeTests.cs index 1ffee40c..43f3355a 100644 --- a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeEnvelopeTests.cs +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeEnvelopeTests.cs @@ -80,6 +80,7 @@ public void TryReadActivity_PrefersValidPayload() [InlineData("[]")] [InlineData("""{ "id": "missing-type" }""")] [InlineData("""{ "type": 42 }""")] + [InlineData("""{ "type": "message", "id": 42 }""")] public void TryReadActivity_FallsBackWhenPayloadIsMalformed(string malformedPayload) { SocketActivityEnvelope envelope = DeserializeEnvelope($$""" diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeNegotiatorTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeNegotiatorTests.cs index 621acb0d..cd017376 100644 --- a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeNegotiatorTests.cs +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeNegotiatorTests.cs @@ -309,6 +309,7 @@ public async Task NegotiateAsync_AllowsHttpsAndLoopbackHttpSignalRUrls(string si [InlineData("ws://localhost:5000/client")] [InlineData("ftp://signalr.example.test/client")] [InlineData("/relative/client")] + [InlineData("https:foo")] public async Task NegotiateAsync_RejectsInvalidSignalRUrl(string signalRUrl) { RecordingHandler handler = JsonHandler( From 6802ad6682b4ff83978f5fc36c6248ef5d19f401 Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Thu, 24 Sep 2026 11:03:34 -0700 Subject: [PATCH 12/31] feat(apps): define default Socket Mode geographies Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SocketMode/SocketModeProtocol.cs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.Teams.Apps/SocketMode/SocketModeProtocol.cs b/src/Microsoft.Teams.Apps/SocketMode/SocketModeProtocol.cs index d9c048d5..60bef175 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/SocketModeProtocol.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/SocketModeProtocol.cs @@ -19,7 +19,13 @@ internal static class SocketModeProtocol internal const string DefaultNegotiateBaseUrl = "https://botapi.skype.com"; /// - /// The path for the Socket Mode negotiate endpoint. + /// The default geographies for Socket Mode connections. /// - internal const string NegotiatePath = "/amer/v3/websockets/connect"; + internal static IReadOnlyList DefaultGeos { get; } = + Array.AsReadOnly(["amer", "emea", "apac"]); + + /// + /// The negotiate path appended after the geography segment. + /// + internal const string NegotiatePath = "/v3/websockets/connect"; } From 99e8c6f075a96bef8b7a3e92ab6143e013639ded Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Tue, 22 Sep 2026 16:21:57 -0700 Subject: [PATCH 13/31] feat(apps): add SignalR Socket Mode connection Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Microsoft.Teams.Apps.csproj | 8 + .../SocketMode/SignalRClientConnection.cs | 179 ++++++++ .../SocketMode/SignalRSocketConnection.cs | 317 +++++++++++++ .../SocketMode/SocketModeNegotiator.cs | 20 +- .../SignalRSocketConnectionTests.cs | 433 ++++++++++++++++++ 5 files changed, 955 insertions(+), 2 deletions(-) create mode 100644 src/Microsoft.Teams.Apps/SocketMode/SignalRClientConnection.cs create mode 100644 src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs create mode 100644 test/Microsoft.Teams.Apps.UnitTests/SocketMode/SignalRSocketConnectionTests.cs diff --git a/src/Microsoft.Teams.Apps/Microsoft.Teams.Apps.csproj b/src/Microsoft.Teams.Apps/Microsoft.Teams.Apps.csproj index 5cc9d782..6c80dd5a 100644 --- a/src/Microsoft.Teams.Apps/Microsoft.Teams.Apps.csproj +++ b/src/Microsoft.Teams.Apps/Microsoft.Teams.Apps.csproj @@ -23,6 +23,14 @@ + + + + + + + + diff --git a/src/Microsoft.Teams.Apps/SocketMode/SignalRClientConnection.cs b/src/Microsoft.Teams.Apps/SocketMode/SignalRClientConnection.cs new file mode 100644 index 00000000..06c4b25c --- /dev/null +++ b/src/Microsoft.Teams.Apps/SocketMode/SignalRClientConnection.cs @@ -0,0 +1,179 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.AspNetCore.SignalR.Client; + +namespace Microsoft.Teams.Apps.SocketMode; + +/// +/// Creates a SignalR client connection for a negotiated Socket Mode endpoint. +/// +/// The negotiated SignalR endpoint. +/// The negotiated access token. +/// The interval between SignalR keep-alive messages. +/// The interval before the SignalR server is considered unavailable. +/// A SignalR client connection. +internal delegate ISignalRClientConnection SignalRConnectionBuilder( + Uri url, + string accessToken, + TimeSpan keepAliveInterval, + TimeSpan serverTimeout); + +/// +/// Provides the SignalR operations required by a Socket Mode connection. +/// +internal interface ISignalRClientConnection : IAsyncDisposable +{ + /// + /// Registers the handler for activity envelopes. + /// + /// The activity handler that returns an optional reply frame. + void OnActivity(Func> handler); + + /// + /// Registers the handler for the SocketReady frame. + /// + /// The ready frame handler. + void OnReady(Action handler); + + /// + /// Registers the handler for terminal connection closure. + /// + /// The closure handler. + void OnClosed(Action handler); + + /// + /// Starts the SignalR connection. + /// + /// A token for cancelling the operation. + Task StartAsync(CancellationToken cancellationToken); + + /// + /// Stops the SignalR connection. + /// + /// A token for cancelling the operation. + Task StopAsync(CancellationToken cancellationToken); +} + +/// +/// Adapts the official SignalR client to the Socket Mode transport. +/// +/// The underlying SignalR hub connection. +internal sealed class SignalRClientConnection(HubConnection connection) + : ISignalRClientConnection +{ + private readonly HubConnection _connection = + connection ?? throw new ArgumentNullException(nameof(connection)); + private readonly List _subscriptions = []; + private int _disposed; + + /// + /// Creates a SignalR client for a negotiated Socket Mode endpoint. + /// + /// The negotiated SignalR endpoint. + /// The negotiated access token. + /// The interval between keep-alive messages. + /// The interval before the server is considered unavailable. + /// A configured SignalR client connection. + internal static ISignalRClientConnection Create( + Uri url, + string accessToken, + TimeSpan keepAliveInterval, + TimeSpan serverTimeout) + { + ArgumentNullException.ThrowIfNull(url); + ArgumentException.ThrowIfNullOrWhiteSpace(accessToken); + + HubConnection connection = new HubConnectionBuilder() + .WithUrl(url, options => + { + options.AccessTokenProvider = + () => Task.FromResult(accessToken); + }) + .WithKeepAliveInterval(keepAliveInterval) + .WithServerTimeout(serverTimeout) + .Build(); + + return new SignalRClientConnection(connection); + } + + /// + public void OnActivity( + Func> handler) + { + ArgumentNullException.ThrowIfNull(handler); + ThrowIfDisposed(); + + _subscriptions.Add( + _connection.On( + "Activity", + handler)); + } + + /// + public void OnReady(Action handler) + { + ArgumentNullException.ThrowIfNull(handler); + ThrowIfDisposed(); + + _subscriptions.Add( + _connection.On( + "SocketReady", + handler)); + } + + /// + public void OnClosed(Action handler) + { + ArgumentNullException.ThrowIfNull(handler); + ThrowIfDisposed(); + + _connection.Closed += exception => + { + handler(exception); + return Task.CompletedTask; + }; + } + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + ThrowIfDisposed(); + return _connection.StartAsync(cancellationToken); + } + + /// + public Task StopAsync(CancellationToken cancellationToken) + { + if (Volatile.Read(ref _disposed) != 0) + { + return Task.CompletedTask; + } + + return _connection.StopAsync(cancellationToken); + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + foreach (IDisposable subscription in _subscriptions) + { + subscription.Dispose(); + } + + _subscriptions.Clear(); + await _connection.DisposeAsync().ConfigureAwait(false); + } + + private void ThrowIfDisposed() + { + ObjectDisposedException.ThrowIf( + Volatile.Read(ref _disposed) != 0, + this); + } +} diff --git a/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs b/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs new file mode 100644 index 00000000..83bd5c9d --- /dev/null +++ b/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs @@ -0,0 +1,317 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Teams.Apps.SocketMode; + +/// +/// Creates one-generation SignalR Socket Mode connections. +/// +internal sealed class SignalRSocketConnectionFactory : ISocketConnectionFactory +{ + private readonly ISocketModeNegotiator _negotiator; + private readonly SignalRConnectionBuilder _createSignalRConnection; + private readonly TimeSpan _readinessTimeout; + private readonly TimeSpan _keepAliveInterval; + private readonly TimeSpan _serverTimeout; + private readonly ILogger _logger; + + /// + /// Initializes a SignalR Socket Mode connection factory. + /// + /// Negotiates SignalR connection details. + /// The time allowed for the SocketReady frame. + /// The interval between SignalR keep-alive messages. + /// The interval before the SignalR server is considered unavailable. + /// The logger for connection lifecycle failures. + /// An optional SignalR client factory. + internal SignalRSocketConnectionFactory( + ISocketModeNegotiator negotiator, + TimeSpan readinessTimeout, + TimeSpan keepAliveInterval, + TimeSpan serverTimeout, + ILogger logger, + SignalRConnectionBuilder? createSignalRConnection = null) + { + _negotiator = negotiator ?? throw new ArgumentNullException(nameof(negotiator)); + _createSignalRConnection = + createSignalRConnection ?? SignalRClientConnection.Create; + _readinessTimeout = EnsurePositive(readinessTimeout, nameof(readinessTimeout)); + _keepAliveInterval = EnsurePositive(keepAliveInterval, nameof(keepAliveInterval)); + _serverTimeout = EnsurePositive(serverTimeout, nameof(serverTimeout)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + public ISocketConnection Create( + Uri negotiateUri, + SocketConnectionHandlers handlers) + { + ArgumentNullException.ThrowIfNull(negotiateUri); + ArgumentNullException.ThrowIfNull(handlers); + + return new SignalRSocketConnection( + negotiateUri, + handlers, + _negotiator, + _createSignalRConnection, + _readinessTimeout, + _keepAliveInterval, + _serverTimeout, + _logger); + } + + private static TimeSpan EnsurePositive(TimeSpan value, string parameterName) + => value > TimeSpan.Zero + ? value + : throw new ArgumentOutOfRangeException( + parameterName, + "Socket Mode connection timeouts and intervals must be greater than zero."); +} + +/// +/// Manages one generation of a SignalR Socket Mode connection. +/// +internal sealed class SignalRSocketConnection : ISocketConnection +{ + private readonly Uri _negotiateUri; + private readonly SocketConnectionHandlers _handlers; + private readonly ISocketModeNegotiator _negotiator; + private readonly SignalRConnectionBuilder _createSignalRConnection; + private readonly TimeSpan _readinessTimeout; + private readonly TimeSpan _keepAliveInterval; + private readonly TimeSpan _serverTimeout; + private readonly ILogger _logger; + private readonly CancellationTokenSource _lifetimeSource = new(); + private readonly TaskCompletionSource _readySource = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly object _stopLock = new(); + + private ISignalRClientConnection? _connection; + private Task? _stopTask; + private int _started; + private int _stopped; + private int _disposed; + private int _readySettled; + private int _closedReported; + + /// + /// Initializes one generation of a SignalR Socket Mode connection. + /// + /// The endpoint used to negotiate the connection. + /// Callbacks for frames and connection closure. + /// Negotiates SignalR connection details. + /// Creates the SignalR client connection. + /// The time allowed for the SocketReady frame. + /// The interval between SignalR keep-alive messages. + /// The interval before the SignalR server is considered unavailable. + /// The logger for connection lifecycle failures. + internal SignalRSocketConnection( + Uri negotiateUri, + SocketConnectionHandlers handlers, + ISocketModeNegotiator negotiator, + SignalRConnectionBuilder createSignalRConnection, + TimeSpan readinessTimeout, + TimeSpan keepAliveInterval, + TimeSpan serverTimeout, + ILogger logger) + { + _negotiateUri = negotiateUri ?? throw new ArgumentNullException(nameof(negotiateUri)); + _handlers = handlers ?? throw new ArgumentNullException(nameof(handlers)); + _negotiator = negotiator ?? throw new ArgumentNullException(nameof(negotiator)); + _createSignalRConnection = + createSignalRConnection + ?? throw new ArgumentNullException(nameof(createSignalRConnection)); + _readinessTimeout = readinessTimeout; + _keepAliveInterval = keepAliveInterval; + _serverTimeout = serverTimeout; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + public TimeSpan? TokenLifetime { get; private set; } + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + ThrowIfDisposed(); + + if (Interlocked.Exchange(ref _started, 1) != 0) + { + throw new InvalidOperationException( + "A Socket Mode connection generation can only be started once."); + } + + using CancellationTokenSource startSource = + CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + _lifetimeSource.Token); + + try + { + SocketModeNegotiateResponse negotiateResponse = + await _negotiator + .NegotiateAsync(_negotiateUri, startSource.Token) + .ConfigureAwait(false); + + TokenLifetime = negotiateResponse.ExpiresIn > 0 + ? TimeSpan.FromSeconds(negotiateResponse.ExpiresIn) + : null; + + ISignalRClientConnection connection = _createSignalRConnection( + new Uri(negotiateResponse.Url!, UriKind.Absolute), + negotiateResponse.AccessToken!, + _keepAliveInterval, + _serverTimeout); + _connection = connection; + + connection.OnActivity(_handlers.OnActivity); + connection.OnReady(HandleReady); + connection.OnClosed(HandleClosed); + + await connection + .StartAsync(startSource.Token) + .ConfigureAwait(false); + + try + { + await _readySource.Task + .WaitAsync(_readinessTimeout, startSource.Token) + .ConfigureAwait(false); + } + catch (TimeoutException exception) + { + throw new TimeoutException( + $"Socket Mode readiness timed out after {_readinessTimeout}.", + exception); + } + } + catch + { + await StopAfterFailedStartAsync().ConfigureAwait(false); + throw; + } + } + + /// + public Task StopAsync(CancellationToken cancellationToken = default) + { + lock (_stopLock) + { + return _stopTask ??= StopCoreAsync(cancellationToken); + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + try + { + await StopAsync(CancellationToken.None).ConfigureAwait(false); + } + finally + { + ISignalRClientConnection? connection = _connection; + _connection = null; + if (connection is not null) + { + await connection.DisposeAsync().ConfigureAwait(false); + } + + _lifetimeSource.Dispose(); + } + } + + [SuppressMessage( + "Design", + "CA1031:Do not catch general exception types", + Justification = "The readiness gate must remain settled even when an observer throws; the failure is logged.")] + private void HandleReady(SocketReadyFrame frame) + { + if (_lifetimeSource.IsCancellationRequested + || Interlocked.CompareExchange(ref _readySettled, 1, 0) != 0) + { + return; + } + + _readySource.TrySetResult(); + + try + { + _handlers.OnReady(frame); + } + catch (Exception exception) + { + _logger.LogWarning( + exception, + "Socket Mode ready observer failed after the connection became ready."); + } + } + + private void HandleClosed(Exception? error) + { + if (Interlocked.CompareExchange(ref _readySettled, 1, 0) == 0) + { + _readySource.TrySetException( + error ?? new IOException( + "Socket Mode connection closed before SocketReady.")); + } + + if (Interlocked.Exchange(ref _closedReported, 1) != 0) + { + return; + } + + _handlers.OnClosed(error); + } + + private async Task StopCoreAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _stopped, 1) != 0) + { + return; + } + + await _lifetimeSource.CancelAsync().ConfigureAwait(false); + + ISignalRClientConnection? connection = _connection; + if (connection is not null) + { + await connection + .StopAsync(cancellationToken) + .ConfigureAwait(false); + } + } + + [SuppressMessage( + "Design", + "CA1031:Do not catch general exception types", + Justification = "Cleanup failure is logged so the original startup exception remains the reported failure.")] + private async Task StopAfterFailedStartAsync() + { + try + { + await StopAsync(CancellationToken.None).ConfigureAwait(false); + } + catch (Exception exception) + { + _logger.LogWarning( + exception, + "Socket Mode connection cleanup failed after startup failure."); + } + } + + private void ThrowIfDisposed() + { + ObjectDisposedException.ThrowIf( + Volatile.Read(ref _disposed) != 0, + this); + } +} diff --git a/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs b/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs index 88e09348..20a19203 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs @@ -7,10 +7,26 @@ namespace Microsoft.Teams.Apps.SocketMode; +/// +/// Negotiates connection details for a Socket Mode connection. +/// +internal interface ISocketModeNegotiator +{ + /// + /// Negotiates the endpoint and access token for a Socket Mode connection. + /// + /// The Socket Mode negotiate endpoint. + /// A token for cancelling the operation. + /// The negotiated connection details. + Task NegotiateAsync( + Uri negotiateUri, + CancellationToken cancellationToken = default); +} + /// /// Negotiates connection details for a Socket Mode transport. /// -internal sealed class SocketModeNegotiator +internal sealed class SocketModeNegotiator : ISocketModeNegotiator { private static readonly TimeSpan DefaultTimeout = TimeSpan.FromSeconds(15); @@ -52,7 +68,7 @@ internal SocketModeNegotiator( /// The negotiated connection details. /// Thrown when the endpoint returns an unsuccessful status code. /// Thrown when negotiation exceeds the configured timeout. - internal async Task NegotiateAsync( + public async Task NegotiateAsync( Uri negotiateUri, CancellationToken cancellationToken = default) { diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SignalRSocketConnectionTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SignalRSocketConnectionTests.cs new file mode 100644 index 00000000..0a799c7c --- /dev/null +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SignalRSocketConnectionTests.cs @@ -0,0 +1,433 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Teams.Apps.SocketMode; + +namespace Microsoft.Teams.Apps.UnitTests.SocketMode; + +public class SignalRSocketConnectionTests +{ + private static readonly Uri NegotiateUri = + new("https://botapi.skype.com/amer/v3/websockets/connect"); + private static readonly Uri SignalRUri = + new("https://signalr.example.test/client"); + + [Fact] + public async Task StartAsync_NegotiatesCreatesAndStartsConnection() + { + TestHarness harness = CreateHarness(expiresIn: 3600); + + Task start = harness.Connection.StartAsync(CancellationToken.None); + await harness.SignalR.Started.Task; + harness.SignalR.Ready(new SocketReadyFrame + { + BotKey = "bot-id", + ConnectionId = "connection-id", + }); + await start; + + Assert.Equal(1, harness.Negotiator.CallCount); + Assert.Equal(NegotiateUri, harness.Negotiator.NegotiateUri); + Assert.Equal(SignalRUri, harness.SignalRBuilder.Url); + Assert.Equal("signalr-token", harness.SignalRBuilder.AccessToken); + Assert.Equal(TimeSpan.FromSeconds(15), harness.SignalRBuilder.KeepAliveInterval); + Assert.Equal(TimeSpan.FromSeconds(30), harness.SignalRBuilder.ServerTimeout); + Assert.Equal(TimeSpan.FromHours(1), harness.Connection.TokenLifetime); + Assert.Equal(1, harness.SignalR.StartCount); + Assert.Equal(1, harness.ReadyFrames.Count); + } + + [Fact] + public async Task StartAsync_DoesNotCompleteBeforeSocketReady() + { + TestHarness harness = CreateHarness(); + + Task start = harness.Connection.StartAsync(CancellationToken.None); + await harness.SignalR.Started.Task; + + Assert.False(start.IsCompleted); + + harness.SignalR.Ready(new SocketReadyFrame()); + await start; + } + + [Fact] + public async Task StartAsync_IgnoresDuplicateSocketReady() + { + TestHarness harness = CreateHarness(); + + Task start = harness.Connection.StartAsync(CancellationToken.None); + await harness.SignalR.Started.Task; + harness.SignalR.Ready(new SocketReadyFrame { ConnectionId = "first" }); + harness.SignalR.Ready(new SocketReadyFrame { ConnectionId = "second" }); + await start; + + SocketReadyFrame ready = Assert.Single(harness.ReadyFrames); + Assert.Equal("first", ready.ConnectionId); + } + + [Fact] + public async Task StartAsync_SettlesBeforeReadyObserverThrows() + { + FakeNegotiator negotiator = new(SuccessfulNegotiation()); + FakeSignalRClientConnection signalR = new(); + FakeSignalRConnectionBuilder signalRBuilder = new(signalR); + SocketConnectionHandlers handlers = new( + _ => Task.FromResult(null), + _ => throw new InvalidOperationException("observer failed"), + _ => { }); + SignalRSocketConnection connection = CreateConnection( + negotiator, + signalRBuilder, + handlers); + + Task start = connection.StartAsync(CancellationToken.None); + await signalR.Started.Task; + signalR.Ready(new SocketReadyFrame()); + await start; + + Assert.True(start.IsCompletedSuccessfully); + } + + [Fact] + public async Task ActivityHandler_ReturnsClientResult() + { + SocketReplyFrame expected = new() { Status = 202 }; + TestHarness harness = CreateHarness( + onActivity: _ => Task.FromResult(expected)); + + Task start = harness.Connection.StartAsync(CancellationToken.None); + await harness.SignalR.Started.Task; + harness.SignalR.Ready(new SocketReadyFrame()); + await start; + + SocketReplyFrame? actual = await harness.SignalR.Activity( + new SocketActivityEnvelope { EnvelopeId = "env-1" }); + + Assert.Same(expected, actual); + } + + [Fact] + public async Task ActivityHandler_AllowsNullClientResult() + { + TestHarness harness = CreateHarness(); + + Task start = harness.Connection.StartAsync(CancellationToken.None); + await harness.SignalR.Started.Task; + harness.SignalR.Ready(new SocketReadyFrame()); + await start; + + SocketReplyFrame? actual = await harness.SignalR.Activity( + new SocketActivityEnvelope()); + + Assert.Null(actual); + } + + [Fact] + public async Task CloseBeforeReady_FailsStartupAndNotifiesClosed() + { + TestHarness harness = CreateHarness(); + IOException expected = new("connection lost"); + + Task start = harness.Connection.StartAsync(CancellationToken.None); + await harness.SignalR.Started.Task; + harness.SignalR.Close(expected); + + IOException actual = await Assert.ThrowsAsync(() => start); + Assert.Same(expected, actual); + Assert.Same(expected, Assert.Single(harness.CloseErrors)); + Assert.Equal(1, harness.SignalR.StopCount); + } + + [Fact] + public async Task CloseAfterReady_NotifiesClosedOnce() + { + TestHarness harness = CreateHarness(); + IOException expected = new("connection lost"); + + Task start = harness.Connection.StartAsync(CancellationToken.None); + await harness.SignalR.Started.Task; + harness.SignalR.Ready(new SocketReadyFrame()); + await start; + + harness.SignalR.Close(expected); + harness.SignalR.Close(new IOException("duplicate")); + + Assert.Same(expected, Assert.Single(harness.CloseErrors)); + } + + [Fact] + public async Task ReadinessTimeout_StopsConnection() + { + TestHarness harness = CreateHarness( + readinessTimeout: TimeSpan.FromMilliseconds(50)); + + Task start = harness.Connection.StartAsync(CancellationToken.None); + await harness.SignalR.Started.Task; + + TimeoutException exception = + await Assert.ThrowsAsync(() => start); + + Assert.Contains("readiness timed out", exception.Message, StringComparison.Ordinal); + Assert.Equal(1, harness.SignalR.StopCount); + } + + [Fact] + public async Task CallerCancellation_StopsStartup() + { + TestHarness harness = CreateHarness(); + using CancellationTokenSource cancellationSource = new(); + + Task start = harness.Connection.StartAsync(cancellationSource.Token); + await harness.SignalR.Started.Task; + await cancellationSource.CancelAsync(); + + await Assert.ThrowsAnyAsync(() => start); + Assert.Equal(1, harness.SignalR.StopCount); + } + + [Fact] + public async Task StopAsync_InterruptsReadinessWaitAndIsIdempotent() + { + TestHarness harness = CreateHarness(); + + Task start = harness.Connection.StartAsync(CancellationToken.None); + await harness.SignalR.Started.Task; + + await harness.Connection.StopAsync(); + await harness.Connection.StopAsync(); + + await Assert.ThrowsAnyAsync(() => start); + Assert.Equal(1, harness.SignalR.StopCount); + } + + [Fact] + public async Task DisposeAsync_StopsAndDisposesUnderlyingConnectionOnce() + { + TestHarness harness = CreateHarness(); + + Task start = harness.Connection.StartAsync(CancellationToken.None); + await harness.SignalR.Started.Task; + harness.SignalR.Ready(new SocketReadyFrame()); + await start; + + await harness.Connection.DisposeAsync(); + await harness.Connection.DisposeAsync(); + + Assert.Equal(1, harness.SignalR.StopCount); + Assert.Equal(1, harness.SignalR.DisposeCount); + } + + [Fact] + public async Task StartAsync_CannotBeCalledTwice() + { + TestHarness harness = CreateHarness(); + + Task start = harness.Connection.StartAsync(CancellationToken.None); + await harness.SignalR.Started.Task; + harness.SignalR.Ready(new SocketReadyFrame()); + await start; + + await Assert.ThrowsAsync( + () => harness.Connection.StartAsync(CancellationToken.None)); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public async Task StartAsync_LeavesTokenLifetimeNullForNonpositiveExpiry( + int expiresIn) + { + TestHarness harness = CreateHarness(expiresIn); + + Task start = harness.Connection.StartAsync(CancellationToken.None); + await harness.SignalR.Started.Task; + harness.SignalR.Ready(new SocketReadyFrame()); + await start; + + Assert.Null(harness.Connection.TokenLifetime); + } + + [Fact] + public void Factory_CreatesDistinctConnectionGenerations() + { + FakeNegotiator negotiator = new(SuccessfulNegotiation()); + FakeSignalRConnectionBuilder signalRBuilder = + new(new FakeSignalRClientConnection()); + SignalRSocketConnectionFactory factory = new( + negotiator, + TimeSpan.FromSeconds(30), + TimeSpan.FromSeconds(15), + TimeSpan.FromSeconds(30), + NullLogger.Instance, + signalRBuilder.Create); + SocketConnectionHandlers handlers = DefaultHandlers(); + + ISocketConnection first = factory.Create(NegotiateUri, handlers); + ISocketConnection second = factory.Create(NegotiateUri, handlers); + + Assert.NotSame(first, second); + } + + private static TestHarness CreateHarness( + int expiresIn = 3600, + Func>? onActivity = null, + TimeSpan? readinessTimeout = null) + { + FakeNegotiator negotiator = new(SuccessfulNegotiation(expiresIn)); + FakeSignalRClientConnection signalR = new(); + FakeSignalRConnectionBuilder signalRBuilder = new(signalR); + List readyFrames = []; + List closeErrors = []; + SocketConnectionHandlers handlers = new( + onActivity ?? (_ => Task.FromResult(null)), + readyFrames.Add, + closeErrors.Add); + + return new TestHarness( + CreateConnection( + negotiator, + signalRBuilder, + handlers, + readinessTimeout), + negotiator, + signalRBuilder, + signalR, + readyFrames, + closeErrors); + } + + private static SignalRSocketConnection CreateConnection( + ISocketModeNegotiator negotiator, + FakeSignalRConnectionBuilder signalRBuilder, + SocketConnectionHandlers handlers, + TimeSpan? readinessTimeout = null) + => new( + NegotiateUri, + handlers, + negotiator, + signalRBuilder.Create, + readinessTimeout ?? TimeSpan.FromSeconds(30), + TimeSpan.FromSeconds(15), + TimeSpan.FromSeconds(30), + NullLogger.Instance); + + private static SocketModeNegotiateResponse SuccessfulNegotiation( + int expiresIn = 3600) + => new() + { + Url = SignalRUri.ToString(), + AccessToken = "signalr-token", + ExpiresIn = expiresIn, + }; + + private static SocketConnectionHandlers DefaultHandlers() + => new( + _ => Task.FromResult(null), + _ => { }, + _ => { }); + + private sealed record TestHarness( + SignalRSocketConnection Connection, + FakeNegotiator Negotiator, + FakeSignalRConnectionBuilder SignalRBuilder, + FakeSignalRClientConnection SignalR, + List ReadyFrames, + List CloseErrors); + + private sealed class FakeNegotiator(SocketModeNegotiateResponse response) + : ISocketModeNegotiator + { + internal int CallCount { get; private set; } + internal Uri? NegotiateUri { get; private set; } + + public Task NegotiateAsync( + Uri negotiateUri, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + CallCount++; + NegotiateUri = negotiateUri; + return Task.FromResult(response); + } + } + + private sealed class FakeSignalRConnectionBuilder( + FakeSignalRClientConnection connection) + { + internal Uri? Url { get; private set; } + internal string? AccessToken { get; private set; } + internal TimeSpan KeepAliveInterval { get; private set; } + internal TimeSpan ServerTimeout { get; private set; } + + internal ISignalRClientConnection Create( + Uri url, + string accessToken, + TimeSpan keepAliveInterval, + TimeSpan serverTimeout) + { + Url = url; + AccessToken = accessToken; + KeepAliveInterval = keepAliveInterval; + ServerTimeout = serverTimeout; + return connection; + } + } + + private sealed class FakeSignalRClientConnection + : ISignalRClientConnection + { + private Func>? + _onActivity; + private Action? _onReady; + private Action? _onClosed; + + internal TaskCompletionSource Started { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + internal int StartCount { get; private set; } + internal int StopCount { get; private set; } + internal int DisposeCount { get; private set; } + + public void OnActivity( + Func> handler) + => _onActivity = handler; + + public void OnReady(Action handler) + => _onReady = handler; + + public void OnClosed(Action handler) + => _onClosed = handler; + + public Task StartAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + StartCount++; + Started.TrySetResult(); + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + StopCount++; + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + DisposeCount++; + return ValueTask.CompletedTask; + } + + internal Task Activity( + SocketActivityEnvelope envelope) + => (_onActivity ?? throw new InvalidOperationException())(envelope); + + internal void Ready(SocketReadyFrame frame) + => (_onReady ?? throw new InvalidOperationException())(frame); + + internal void Close(Exception? error) + => (_onClosed ?? throw new InvalidOperationException())(error); + } +} From 66c82d5cd0f91bb61aed2d618da302a4dc3c0e06 Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Wed, 23 Sep 2026 13:36:12 -0700 Subject: [PATCH 14/31] refactor(apps): clarify SignalR connection factory Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SocketMode/SignalRClientConnection.cs | 2 +- .../SocketMode/SignalRSocketConnection.cs | 26 ++++++++-------- .../SignalRSocketConnectionTests.cs | 30 +++++++++---------- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/Microsoft.Teams.Apps/SocketMode/SignalRClientConnection.cs b/src/Microsoft.Teams.Apps/SocketMode/SignalRClientConnection.cs index 06c4b25c..b852cd65 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/SignalRClientConnection.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/SignalRClientConnection.cs @@ -13,7 +13,7 @@ namespace Microsoft.Teams.Apps.SocketMode; /// The interval between SignalR keep-alive messages. /// The interval before the SignalR server is considered unavailable. /// A SignalR client connection. -internal delegate ISignalRClientConnection SignalRConnectionBuilder( +internal delegate ISignalRClientConnection CreateSignalRClientConnection( Uri url, string accessToken, TimeSpan keepAliveInterval, diff --git a/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs b/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs index 83bd5c9d..3615ac67 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs @@ -12,7 +12,7 @@ namespace Microsoft.Teams.Apps.SocketMode; internal sealed class SignalRSocketConnectionFactory : ISocketConnectionFactory { private readonly ISocketModeNegotiator _negotiator; - private readonly SignalRConnectionBuilder _createSignalRConnection; + private readonly CreateSignalRClientConnection _createSignalRClientConnection; private readonly TimeSpan _readinessTimeout; private readonly TimeSpan _keepAliveInterval; private readonly TimeSpan _serverTimeout; @@ -26,18 +26,18 @@ internal sealed class SignalRSocketConnectionFactory : ISocketConnectionFactory /// The interval between SignalR keep-alive messages. /// The interval before the SignalR server is considered unavailable. /// The logger for connection lifecycle failures. - /// An optional SignalR client factory. + /// An optional SignalR client factory. internal SignalRSocketConnectionFactory( ISocketModeNegotiator negotiator, TimeSpan readinessTimeout, TimeSpan keepAliveInterval, TimeSpan serverTimeout, ILogger logger, - SignalRConnectionBuilder? createSignalRConnection = null) + CreateSignalRClientConnection? createSignalRClientConnection = null) { _negotiator = negotiator ?? throw new ArgumentNullException(nameof(negotiator)); - _createSignalRConnection = - createSignalRConnection ?? SignalRClientConnection.Create; + _createSignalRClientConnection = + createSignalRClientConnection ?? SignalRClientConnection.Create; _readinessTimeout = EnsurePositive(readinessTimeout, nameof(readinessTimeout)); _keepAliveInterval = EnsurePositive(keepAliveInterval, nameof(keepAliveInterval)); _serverTimeout = EnsurePositive(serverTimeout, nameof(serverTimeout)); @@ -56,7 +56,7 @@ public ISocketConnection Create( negotiateUri, handlers, _negotiator, - _createSignalRConnection, + _createSignalRClientConnection, _readinessTimeout, _keepAliveInterval, _serverTimeout, @@ -79,7 +79,7 @@ internal sealed class SignalRSocketConnection : ISocketConnection private readonly Uri _negotiateUri; private readonly SocketConnectionHandlers _handlers; private readonly ISocketModeNegotiator _negotiator; - private readonly SignalRConnectionBuilder _createSignalRConnection; + private readonly CreateSignalRClientConnection _createSignalRClientConnection; private readonly TimeSpan _readinessTimeout; private readonly TimeSpan _keepAliveInterval; private readonly TimeSpan _serverTimeout; @@ -103,7 +103,7 @@ internal sealed class SignalRSocketConnection : ISocketConnection /// The endpoint used to negotiate the connection. /// Callbacks for frames and connection closure. /// Negotiates SignalR connection details. - /// Creates the SignalR client connection. + /// Creates the SignalR client connection. /// The time allowed for the SocketReady frame. /// The interval between SignalR keep-alive messages. /// The interval before the SignalR server is considered unavailable. @@ -112,7 +112,7 @@ internal SignalRSocketConnection( Uri negotiateUri, SocketConnectionHandlers handlers, ISocketModeNegotiator negotiator, - SignalRConnectionBuilder createSignalRConnection, + CreateSignalRClientConnection createSignalRClientConnection, TimeSpan readinessTimeout, TimeSpan keepAliveInterval, TimeSpan serverTimeout, @@ -121,9 +121,9 @@ internal SignalRSocketConnection( _negotiateUri = negotiateUri ?? throw new ArgumentNullException(nameof(negotiateUri)); _handlers = handlers ?? throw new ArgumentNullException(nameof(handlers)); _negotiator = negotiator ?? throw new ArgumentNullException(nameof(negotiator)); - _createSignalRConnection = - createSignalRConnection - ?? throw new ArgumentNullException(nameof(createSignalRConnection)); + _createSignalRClientConnection = + createSignalRClientConnection + ?? throw new ArgumentNullException(nameof(createSignalRClientConnection)); _readinessTimeout = readinessTimeout; _keepAliveInterval = keepAliveInterval; _serverTimeout = serverTimeout; @@ -160,7 +160,7 @@ await _negotiator ? TimeSpan.FromSeconds(negotiateResponse.ExpiresIn) : null; - ISignalRClientConnection connection = _createSignalRConnection( + ISignalRClientConnection connection = _createSignalRClientConnection( new Uri(negotiateResponse.Url!, UriKind.Absolute), negotiateResponse.AccessToken!, _keepAliveInterval, diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SignalRSocketConnectionTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SignalRSocketConnectionTests.cs index 0a799c7c..8c502c4f 100644 --- a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SignalRSocketConnectionTests.cs +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SignalRSocketConnectionTests.cs @@ -29,10 +29,10 @@ public async Task StartAsync_NegotiatesCreatesAndStartsConnection() Assert.Equal(1, harness.Negotiator.CallCount); Assert.Equal(NegotiateUri, harness.Negotiator.NegotiateUri); - Assert.Equal(SignalRUri, harness.SignalRBuilder.Url); - Assert.Equal("signalr-token", harness.SignalRBuilder.AccessToken); - Assert.Equal(TimeSpan.FromSeconds(15), harness.SignalRBuilder.KeepAliveInterval); - Assert.Equal(TimeSpan.FromSeconds(30), harness.SignalRBuilder.ServerTimeout); + Assert.Equal(SignalRUri, harness.SignalRFactory.Url); + Assert.Equal("signalr-token", harness.SignalRFactory.AccessToken); + Assert.Equal(TimeSpan.FromSeconds(15), harness.SignalRFactory.KeepAliveInterval); + Assert.Equal(TimeSpan.FromSeconds(30), harness.SignalRFactory.ServerTimeout); Assert.Equal(TimeSpan.FromHours(1), harness.Connection.TokenLifetime); Assert.Equal(1, harness.SignalR.StartCount); Assert.Equal(1, harness.ReadyFrames.Count); @@ -72,14 +72,14 @@ public async Task StartAsync_SettlesBeforeReadyObserverThrows() { FakeNegotiator negotiator = new(SuccessfulNegotiation()); FakeSignalRClientConnection signalR = new(); - FakeSignalRConnectionBuilder signalRBuilder = new(signalR); + FakeSignalRConnectionFactory signalRFactory = new(signalR); SocketConnectionHandlers handlers = new( _ => Task.FromResult(null), _ => throw new InvalidOperationException("observer failed"), _ => { }); SignalRSocketConnection connection = CreateConnection( negotiator, - signalRBuilder, + signalRFactory, handlers); Task start = connection.StartAsync(CancellationToken.None); @@ -253,7 +253,7 @@ public async Task StartAsync_LeavesTokenLifetimeNullForNonpositiveExpiry( public void Factory_CreatesDistinctConnectionGenerations() { FakeNegotiator negotiator = new(SuccessfulNegotiation()); - FakeSignalRConnectionBuilder signalRBuilder = + FakeSignalRConnectionFactory signalRFactory = new(new FakeSignalRClientConnection()); SignalRSocketConnectionFactory factory = new( negotiator, @@ -261,7 +261,7 @@ public void Factory_CreatesDistinctConnectionGenerations() TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30), NullLogger.Instance, - signalRBuilder.Create); + signalRFactory.Create); SocketConnectionHandlers handlers = DefaultHandlers(); ISocketConnection first = factory.Create(NegotiateUri, handlers); @@ -277,7 +277,7 @@ private static TestHarness CreateHarness( { FakeNegotiator negotiator = new(SuccessfulNegotiation(expiresIn)); FakeSignalRClientConnection signalR = new(); - FakeSignalRConnectionBuilder signalRBuilder = new(signalR); + FakeSignalRConnectionFactory signalRFactory = new(signalR); List readyFrames = []; List closeErrors = []; SocketConnectionHandlers handlers = new( @@ -288,11 +288,11 @@ private static TestHarness CreateHarness( return new TestHarness( CreateConnection( negotiator, - signalRBuilder, + signalRFactory, handlers, readinessTimeout), negotiator, - signalRBuilder, + signalRFactory, signalR, readyFrames, closeErrors); @@ -300,14 +300,14 @@ private static TestHarness CreateHarness( private static SignalRSocketConnection CreateConnection( ISocketModeNegotiator negotiator, - FakeSignalRConnectionBuilder signalRBuilder, + FakeSignalRConnectionFactory signalRFactory, SocketConnectionHandlers handlers, TimeSpan? readinessTimeout = null) => new( NegotiateUri, handlers, negotiator, - signalRBuilder.Create, + signalRFactory.Create, readinessTimeout ?? TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30), @@ -331,7 +331,7 @@ private static SocketConnectionHandlers DefaultHandlers() private sealed record TestHarness( SignalRSocketConnection Connection, FakeNegotiator Negotiator, - FakeSignalRConnectionBuilder SignalRBuilder, + FakeSignalRConnectionFactory SignalRFactory, FakeSignalRClientConnection SignalR, List ReadyFrames, List CloseErrors); @@ -353,7 +353,7 @@ public Task NegotiateAsync( } } - private sealed class FakeSignalRConnectionBuilder( + private sealed class FakeSignalRConnectionFactory( FakeSignalRClientConnection connection) { internal Uri? Url { get; private set; } From 55717e08ac0f7b07baf61ef36189559b4a9928f3 Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Wed, 23 Sep 2026 14:03:56 -0700 Subject: [PATCH 15/31] fix(apps): dispose late Socket Mode connection Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SocketMode/SignalRSocketConnection.cs | 30 ++++++++++++- .../SignalRSocketConnectionTests.cs | 43 +++++++++++++++++++ 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs b/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs index 3615ac67..acb95a38 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs @@ -165,7 +165,13 @@ await _negotiator negotiateResponse.AccessToken!, _keepAliveInterval, _serverTimeout); - _connection = connection; + if (!TryPublishConnection(connection)) + { + await connection.DisposeAsync().ConfigureAwait(false); + throw new OperationCanceledException( + "Socket Mode connection stopped before startup completed.", + startSource.Token); + } connection.OnActivity(_handlers.OnActivity); connection.OnReady(HandleReady); @@ -281,7 +287,12 @@ private async Task StopCoreAsync(CancellationToken cancellationToken) await _lifetimeSource.CancelAsync().ConfigureAwait(false); - ISignalRClientConnection? connection = _connection; + ISignalRClientConnection? connection; + lock (_stopLock) + { + connection = _connection; + } + if (connection is not null) { await connection @@ -290,6 +301,21 @@ await connection } } + private bool TryPublishConnection(ISignalRClientConnection connection) + { + lock (_stopLock) + { + if (Volatile.Read(ref _stopped) != 0 + || Volatile.Read(ref _disposed) != 0) + { + return false; + } + + _connection = connection; + return true; + } + } + [SuppressMessage( "Design", "CA1031:Do not catch general exception types", diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SignalRSocketConnectionTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SignalRSocketConnectionTests.cs index 8c502c4f..7f260fc7 100644 --- a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SignalRSocketConnectionTests.cs +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SignalRSocketConnectionTests.cs @@ -219,6 +219,29 @@ public async Task DisposeAsync_StopsAndDisposesUnderlyingConnectionOnce() Assert.Equal(1, harness.SignalR.DisposeCount); } + [Fact] + public async Task DisposeAsync_DisposesConnectionCreatedAfterShutdown() + { + DeferredNegotiator negotiator = new(); + FakeSignalRClientConnection signalR = new(); + FakeSignalRConnectionFactory signalRFactory = new(signalR); + SignalRSocketConnection connection = CreateConnection( + negotiator, + signalRFactory, + DefaultHandlers()); + + Task start = connection.StartAsync(CancellationToken.None); + await negotiator.Started.Task; + await connection.DisposeAsync(); + + negotiator.Complete(SuccessfulNegotiation()); + + await Assert.ThrowsAnyAsync(() => start); + Assert.Equal(0, signalR.StartCount); + Assert.Equal(0, signalR.StopCount); + Assert.Equal(1, signalR.DisposeCount); + } + [Fact] public async Task StartAsync_CannotBeCalledTwice() { @@ -353,6 +376,26 @@ public Task NegotiateAsync( } } + private sealed class DeferredNegotiator : ISocketModeNegotiator + { + private readonly TaskCompletionSource + _response = new(TaskCreationOptions.RunContinuationsAsynchronously); + + internal TaskCompletionSource Started { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task NegotiateAsync( + Uri negotiateUri, + CancellationToken cancellationToken = default) + { + Started.TrySetResult(); + return _response.Task; + } + + internal void Complete(SocketModeNegotiateResponse response) + => _response.TrySetResult(response); + } + private sealed class FakeSignalRConnectionFactory( FakeSignalRClientConnection connection) { From 8b4382608bbfa47f8c343d93b002cb62159c5c7b Mon Sep 17 00:00:00 2001 From: teddyam <113314164+teddyam@users.noreply.github.com> Date: Wed, 23 Sep 2026 14:10:03 -0700 Subject: [PATCH 16/31] Update OnActivity to await readiness Refactor OnActivity to await readiness before handling activity. Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../SocketMode/SignalRSocketConnection.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs b/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs index acb95a38..38736f1e 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs @@ -173,7 +173,13 @@ await _negotiator startSource.Token); } - connection.OnActivity(_handlers.OnActivity); + connection.OnActivity(async envelope => + { + await _readySource.Task + .WaitAsync(_lifetimeSource.Token) + .ConfigureAwait(false); + return await _handlers.OnActivity(envelope).ConfigureAwait(false); + }); connection.OnReady(HandleReady); connection.OnClosed(HandleClosed); From 99b445466b59bf35521dbb2122c749a9da957121 Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Wed, 23 Sep 2026 14:13:28 -0700 Subject: [PATCH 17/31] fix(apps): identify planned Socket Mode closure Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SocketMode/SignalRSocketConnection.cs | 7 ++++-- .../SocketMode/SocketModeConnection.cs | 6 ++--- .../SignalRSocketConnectionTests.cs | 24 ++++++++++++------- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs b/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs index 38736f1e..b6c619c9 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs @@ -269,7 +269,10 @@ private void HandleReady(SocketReadyFrame frame) private void HandleClosed(Exception? error) { - if (Interlocked.CompareExchange(ref _readySettled, 1, 0) == 0) + bool planned = Volatile.Read(ref _stopped) != 0; + + if (!planned + && Interlocked.CompareExchange(ref _readySettled, 1, 0) == 0) { _readySource.TrySetException( error ?? new IOException( @@ -281,7 +284,7 @@ private void HandleClosed(Exception? error) return; } - _handlers.OnClosed(error); + _handlers.OnClosed(error, planned); } private async Task StopCoreAsync(CancellationToken cancellationToken) diff --git a/src/Microsoft.Teams.Apps/SocketMode/SocketModeConnection.cs b/src/Microsoft.Teams.Apps/SocketMode/SocketModeConnection.cs index 65f4feb5..77ba42fe 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/SocketModeConnection.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/SocketModeConnection.cs @@ -52,11 +52,11 @@ internal sealed class SocketConnectionHandlers /// /// Handles an incoming activity envelope. /// Handles the ready frame. - /// Handles connection closure. + /// Handles connection closure and indicates whether it was planned. internal SocketConnectionHandlers( Func> onActivity, Action onReady, - Action onClosed) + Action onClosed) { OnActivity = onActivity ?? throw new ArgumentNullException(nameof(onActivity)); OnReady = onReady ?? throw new ArgumentNullException(nameof(onReady)); @@ -76,5 +76,5 @@ internal SocketConnectionHandlers( /// /// Gets the connection-closed callback. /// - internal Action OnClosed { get; } + internal Action OnClosed { get; } } diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SignalRSocketConnectionTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SignalRSocketConnectionTests.cs index 7f260fc7..ef0fd20d 100644 --- a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SignalRSocketConnectionTests.cs +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SignalRSocketConnectionTests.cs @@ -76,7 +76,7 @@ public async Task StartAsync_SettlesBeforeReadyObserverThrows() SocketConnectionHandlers handlers = new( _ => Task.FromResult(null), _ => throw new InvalidOperationException("observer failed"), - _ => { }); + (_, _) => { }); SignalRSocketConnection connection = CreateConnection( negotiator, signalRFactory, @@ -136,7 +136,9 @@ public async Task CloseBeforeReady_FailsStartupAndNotifiesClosed() IOException actual = await Assert.ThrowsAsync(() => start); Assert.Same(expected, actual); - Assert.Same(expected, Assert.Single(harness.CloseErrors)); + (Exception? error, bool planned) = Assert.Single(harness.CloseEvents); + Assert.Same(expected, error); + Assert.False(planned); Assert.Equal(1, harness.SignalR.StopCount); } @@ -154,7 +156,9 @@ public async Task CloseAfterReady_NotifiesClosedOnce() harness.SignalR.Close(expected); harness.SignalR.Close(new IOException("duplicate")); - Assert.Same(expected, Assert.Single(harness.CloseErrors)); + (Exception? error, bool planned) = Assert.Single(harness.CloseEvents); + Assert.Same(expected, error); + Assert.False(planned); } [Fact] @@ -185,6 +189,9 @@ public async Task CallerCancellation_StopsStartup() await Assert.ThrowsAnyAsync(() => start); Assert.Equal(1, harness.SignalR.StopCount); + (Exception? error, bool planned) = Assert.Single(harness.CloseEvents); + Assert.Null(error); + Assert.True(planned); } [Fact] @@ -302,11 +309,11 @@ private static TestHarness CreateHarness( FakeSignalRClientConnection signalR = new(); FakeSignalRConnectionFactory signalRFactory = new(signalR); List readyFrames = []; - List closeErrors = []; + List<(Exception? Error, bool Planned)> closeEvents = []; SocketConnectionHandlers handlers = new( onActivity ?? (_ => Task.FromResult(null)), readyFrames.Add, - closeErrors.Add); + (error, planned) => closeEvents.Add((error, planned))); return new TestHarness( CreateConnection( @@ -318,7 +325,7 @@ private static TestHarness CreateHarness( signalRFactory, signalR, readyFrames, - closeErrors); + closeEvents); } private static SignalRSocketConnection CreateConnection( @@ -349,7 +356,7 @@ private static SocketConnectionHandlers DefaultHandlers() => new( _ => Task.FromResult(null), _ => { }, - _ => { }); + (_, _) => { }); private sealed record TestHarness( SignalRSocketConnection Connection, @@ -357,7 +364,7 @@ private sealed record TestHarness( FakeSignalRConnectionFactory SignalRFactory, FakeSignalRClientConnection SignalR, List ReadyFrames, - List CloseErrors); + List<(Exception? Error, bool Planned)> CloseEvents); private sealed class FakeNegotiator(SocketModeNegotiateResponse response) : ISocketModeNegotiator @@ -454,6 +461,7 @@ public Task StopAsync(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); StopCount++; + _onClosed?.Invoke(null); return Task.CompletedTask; } From fc99cb5620b2fb04b35186a52c105f335d0c7be5 Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Wed, 23 Sep 2026 14:23:43 -0700 Subject: [PATCH 18/31] fix(apps): dispose failed Socket Mode startup Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SocketMode/SignalRSocketConnection.cs | 38 ++++++++++++++++--- .../SignalRSocketConnectionTests.cs | 8 +++- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs b/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs index b6c619c9..17147595 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs @@ -230,14 +230,14 @@ public async ValueTask DisposeAsync() } finally { - ISignalRClientConnection? connection = _connection; - _connection = null; - if (connection is not null) + try { - await connection.DisposeAsync().ConfigureAwait(false); + await DisposeConnectionAsync().ConfigureAwait(false); + } + finally + { + _lifetimeSource.Dispose(); } - - _lifetimeSource.Dispose(); } } @@ -325,6 +325,21 @@ private bool TryPublishConnection(ISignalRClientConnection connection) } } + private async ValueTask DisposeConnectionAsync() + { + ISignalRClientConnection? connection; + lock (_stopLock) + { + connection = _connection; + _connection = null; + } + + if (connection is not null) + { + await connection.DisposeAsync().ConfigureAwait(false); + } + } + [SuppressMessage( "Design", "CA1031:Do not catch general exception types", @@ -341,6 +356,17 @@ private async Task StopAfterFailedStartAsync() exception, "Socket Mode connection cleanup failed after startup failure."); } + + try + { + await DisposeConnectionAsync().ConfigureAwait(false); + } + catch (Exception exception) + { + _logger.LogWarning( + exception, + "Socket Mode connection disposal failed after startup failure."); + } } private void ThrowIfDisposed() diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SignalRSocketConnectionTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SignalRSocketConnectionTests.cs index ef0fd20d..9024a9a1 100644 --- a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SignalRSocketConnectionTests.cs +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SignalRSocketConnectionTests.cs @@ -35,7 +35,7 @@ public async Task StartAsync_NegotiatesCreatesAndStartsConnection() Assert.Equal(TimeSpan.FromSeconds(30), harness.SignalRFactory.ServerTimeout); Assert.Equal(TimeSpan.FromHours(1), harness.Connection.TokenLifetime); Assert.Equal(1, harness.SignalR.StartCount); - Assert.Equal(1, harness.ReadyFrames.Count); + Assert.Single(harness.ReadyFrames); } [Fact] @@ -140,6 +140,7 @@ public async Task CloseBeforeReady_FailsStartupAndNotifiesClosed() Assert.Same(expected, error); Assert.False(planned); Assert.Equal(1, harness.SignalR.StopCount); + Assert.Equal(1, harness.SignalR.DisposeCount); } [Fact] @@ -175,6 +176,10 @@ public async Task ReadinessTimeout_StopsConnection() Assert.Contains("readiness timed out", exception.Message, StringComparison.Ordinal); Assert.Equal(1, harness.SignalR.StopCount); + Assert.Equal(1, harness.SignalR.DisposeCount); + + await harness.Connection.DisposeAsync(); + Assert.Equal(1, harness.SignalR.DisposeCount); } [Fact] @@ -189,6 +194,7 @@ public async Task CallerCancellation_StopsStartup() await Assert.ThrowsAnyAsync(() => start); Assert.Equal(1, harness.SignalR.StopCount); + Assert.Equal(1, harness.SignalR.DisposeCount); (Exception? error, bool planned) = Assert.Single(harness.CloseEvents); Assert.Null(error); Assert.True(planned); From d9f5233b7f27cb9467c5c5aeab1380d6617fc0c4 Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Wed, 23 Sep 2026 14:37:25 -0700 Subject: [PATCH 19/31] fix(apps): detach SignalR close handlers Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SocketMode/SignalRClientConnection.cs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Teams.Apps/SocketMode/SignalRClientConnection.cs b/src/Microsoft.Teams.Apps/SocketMode/SignalRClientConnection.cs index b852cd65..adc5dd9b 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/SignalRClientConnection.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/SignalRClientConnection.cs @@ -65,6 +65,7 @@ internal sealed class SignalRClientConnection(HubConnection connection) private readonly HubConnection _connection = connection ?? throw new ArgumentNullException(nameof(connection)); private readonly List _subscriptions = []; + private readonly List> _closedHandlers = []; private int _disposed; /// @@ -128,11 +129,13 @@ public void OnClosed(Action handler) ArgumentNullException.ThrowIfNull(handler); ThrowIfDisposed(); - _connection.Closed += exception => + Func closedHandler = exception => { handler(exception); return Task.CompletedTask; }; + _connection.Closed += closedHandler; + _closedHandlers.Add(closedHandler); } /// @@ -161,6 +164,13 @@ public async ValueTask DisposeAsync() return; } + foreach (Func closedHandler in _closedHandlers) + { + _connection.Closed -= closedHandler; + } + + _closedHandlers.Clear(); + foreach (IDisposable subscription in _subscriptions) { subscription.Dispose(); From a629457c1d00d589193c4ba7d55f1a75147cfe5d Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Thu, 24 Sep 2026 11:06:28 -0700 Subject: [PATCH 20/31] fix(apps): isolate Socket Mode stop cancellation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SocketMode/SignalRSocketConnection.cs | 9 ++++-- .../SignalRSocketConnectionTests.cs | 30 ++++++++++++++++++- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs b/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs index 17147595..f2852f2d 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/SignalRSocketConnection.cs @@ -210,10 +210,13 @@ await _readySource.Task /// public Task StopAsync(CancellationToken cancellationToken = default) { + Task stopTask; lock (_stopLock) { - return _stopTask ??= StopCoreAsync(cancellationToken); + stopTask = _stopTask ??= StopCoreAsync(); } + + return stopTask.WaitAsync(cancellationToken); } /// @@ -287,7 +290,7 @@ private void HandleClosed(Exception? error) _handlers.OnClosed(error, planned); } - private async Task StopCoreAsync(CancellationToken cancellationToken) + private async Task StopCoreAsync() { if (Interlocked.Exchange(ref _stopped, 1) != 0) { @@ -305,7 +308,7 @@ private async Task StopCoreAsync(CancellationToken cancellationToken) if (connection is not null) { await connection - .StopAsync(cancellationToken) + .StopAsync(CancellationToken.None) .ConfigureAwait(false); } } diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SignalRSocketConnectionTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SignalRSocketConnectionTests.cs index 9024a9a1..f79fa122 100644 --- a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SignalRSocketConnectionTests.cs +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SignalRSocketConnectionTests.cs @@ -215,6 +215,33 @@ public async Task StopAsync_InterruptsReadinessWaitAndIsIdempotent() Assert.Equal(1, harness.SignalR.StopCount); } + [Fact] + public async Task StopAsync_CallerCancellationDoesNotPoisonDispose() + { + TestHarness harness = CreateHarness(); + Task start = harness.Connection.StartAsync(CancellationToken.None); + await harness.SignalR.Started.Task; + harness.SignalR.Ready(new SocketReadyFrame()); + await start; + + harness.SignalR.StopCompletion = + new(TaskCreationOptions.RunContinuationsAsynchronously); + using CancellationTokenSource cancellationSource = new(); + await cancellationSource.CancelAsync(); + + await Assert.ThrowsAnyAsync( + () => harness.Connection.StopAsync(cancellationSource.Token)); + + ValueTask dispose = harness.Connection.DisposeAsync(); + Assert.False(dispose.IsCompleted); + + harness.SignalR.StopCompletion.TrySetResult(); + await dispose; + + Assert.Equal(1, harness.SignalR.StopCount); + Assert.Equal(1, harness.SignalR.DisposeCount); + } + [Fact] public async Task DisposeAsync_StopsAndDisposesUnderlyingConnectionOnce() { @@ -444,6 +471,7 @@ private sealed class FakeSignalRClientConnection internal int StartCount { get; private set; } internal int StopCount { get; private set; } internal int DisposeCount { get; private set; } + internal TaskCompletionSource? StopCompletion { get; set; } public void OnActivity( Func> handler) @@ -468,7 +496,7 @@ public Task StopAsync(CancellationToken cancellationToken) cancellationToken.ThrowIfCancellationRequested(); StopCount++; _onClosed?.Invoke(null); - return Task.CompletedTask; + return StopCompletion?.Task ?? Task.CompletedTask; } public ValueTask DisposeAsync() From 92ac69744bdddb68089544ab41fe4505509495a4 Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Wed, 23 Sep 2026 22:00:51 -0700 Subject: [PATCH 21/31] feat(apps): add per-geo Socket Mode supervisor Adds an internal GeoSocket that keeps one geo connected: initial startup within a monotonic budget, Retry-After/backoff retries, reconnect after unexpected closure, and make-before-break token rotation with a retiring handoff window. Dispatch is fenced to the active or retiring generation, and every connection is stopped and disposed exactly once. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SocketMode/GeoSocket.cs | 629 +++++++++++++++++ .../SocketMode/GeoSocketTests.cs | 631 ++++++++++++++++++ 2 files changed, 1260 insertions(+) create mode 100644 src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs create mode 100644 test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs diff --git a/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs b/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs new file mode 100644 index 00000000..b7bb8314 --- /dev/null +++ b/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs @@ -0,0 +1,629 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Teams.Apps.SocketMode; + +/// +/// Provides the retry policy, dispatch, and lifecycle reporting a needs from its transport. +/// +internal interface IGeoSocketOwner +{ + /// + /// Gets the time allowed for the initial connection to become ready. Zero permits a single unbounded attempt. + /// + TimeSpan StartupTimeout { get; } + + /// + /// Gets how long before token expiry a replacement connection is established. + /// + TimeSpan TokenRefreshMargin { get; } + + /// + /// Gets how long a replaced connection keeps dispatching after its successor is ready. + /// + TimeSpan HandoffWindow { get; } + + /// + /// Gets the server-requested retry delay for a failure, when one was provided. + /// + /// The failure that caused the retry. + TimeSpan? GetRetryAfter(Exception? error); + + /// + /// Gets the backoff delay for a zero-based retry attempt. + /// + /// The zero-based retry attempt. + TimeSpan GetBackoffDelay(int attempt); + + /// + /// Dispatches an activity received on a live connection for a geo. + /// + /// The geo that received the activity. + /// The received activity envelope. + Task DispatchAsync(string geo, SocketActivityEnvelope envelope); + + /// + /// Reports that a connection generation for a geo received SocketReady. + /// + /// The geo whose connection became ready. + /// The ready frame. + void OnGeoReady(string geo, SocketReadyFrame frame); + + /// + /// Reports that inbound delivery for a geo was unexpectedly interrupted. + /// + /// The geo that disconnected. + /// The failure that closed the connection, when available. + void OnGeoDisconnected(string geo, Exception? error); + + /// + /// Reports that inbound delivery for a geo resumed after an unexpected interruption. + /// + /// The geo that reconnected. + void OnGeoReconnected(string geo); +} + +/// +/// Keeps one geo connected: establishes the initial connection within a startup budget, reconnects after +/// unexpected closure, and rotates tokens make-before-break. +/// +/// +/// Each connection attempt is a new generation. Activities are dispatched only from the active ready generation +/// or a predecessor that is explicitly retiring during a token rotation handoff. +/// +internal sealed class GeoSocket : IAsyncDisposable +{ + private static readonly TimeSpan MinimumRefreshDelay = TimeSpan.FromSeconds(1); + + private readonly IGeoSocketOwner _owner; + private readonly Uri _negotiateUri; + private readonly ISocketConnectionFactory _connectionFactory; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly CancellationTokenSource _stopSource = new(); + private readonly CancellationToken _stopToken; + private readonly object _sync = new(); + private readonly HashSet _owned = []; + private readonly HashSet _retiring = []; + private readonly HashSet _retirements = []; + + private long _generation; + private Generation? _active; + private ITimer? _refreshTimer; + private Task? _supervisor; + private Task? _stopTask; + private bool _disconnected; + private bool _stopping; + private int _started; + private int _disposed; + + /// + /// Initializes a supervisor for one geo. + /// + /// The transport that owns this geo. + /// The geo identifier. + /// The negotiate endpoint for the geo. + /// Creates one connection per generation. + /// The logger for lifecycle events. + /// The time provider for startup budgets, retries, rotation, and handoff. + internal GeoSocket( + IGeoSocketOwner owner, + string geo, + Uri negotiateUri, + ISocketConnectionFactory connectionFactory, + ILogger logger, + TimeProvider? timeProvider = null) + { + _owner = owner ?? throw new ArgumentNullException(nameof(owner)); + ArgumentException.ThrowIfNullOrWhiteSpace(geo); + Geo = geo; + _negotiateUri = negotiateUri ?? throw new ArgumentNullException(nameof(negotiateUri)); + _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; + _stopToken = _stopSource.Token; + } + + /// + /// Gets the geo identifier. + /// + internal string Geo { get; } + + /// + /// Establishes the initial ready connection, then supervises it in the background. + /// + /// A token for cancelling startup. + /// Thrown when no generation becomes ready within the startup budget. + internal async Task StartAsync(CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); + if (Interlocked.Exchange(ref _started, 1) != 0) + { + throw new InvalidOperationException("A GeoSocket can only be started once."); + } + + Generation initial = await ConnectInitialAsync(cancellationToken).ConfigureAwait(false); + + lock (_sync) + { + if (!_stopping) + { + _supervisor = SuperviseAsync(initial); + return; + } + } + + throw new OperationCanceledException("Socket Mode geo stopped before startup completed."); + } + + /// + /// Stops supervision and stops and disposes every connection this geo owns. + /// + internal Task StopAsync() + { + lock (_sync) + { + _stopping = true; + return _stopTask ??= StopCoreAsync(); + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + try + { + await StopAsync().ConfigureAwait(false); + } + finally + { + _stopSource.Dispose(); + } + } + + private async Task ConnectInitialAsync(CancellationToken cancellationToken) + { + using CancellationTokenSource startSource = + CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _stopToken); + TimeSpan budget = _owner.StartupTimeout; + long startedAt = _timeProvider.GetTimestamp(); + + for (int attempt = 0; ; attempt++) + { + using CancellationTokenSource deadlineSource = new(Timeout.InfiniteTimeSpan, _timeProvider); + if (budget > TimeSpan.Zero) + { + deadlineSource.CancelAfter(RemainingBudget(budget, startedAt)); + } + + using CancellationTokenSource attemptSource = + CancellationTokenSource.CreateLinkedTokenSource(startSource.Token, deadlineSource.Token); + + try + { + return await ConnectAsync(attemptSource.Token).ConfigureAwait(false); + } + catch (Exception exception) when (!startSource.IsCancellationRequested) + { + Exception error = deadlineSource.IsCancellationRequested + ? new TimeoutException( + $"Socket Mode geo '{Geo}' did not become ready within {budget}.", + exception) + : exception; + TimeSpan delay = GetRetryDelay(error, attempt); + + if (budget <= TimeSpan.Zero || delay >= RemainingBudget(budget, startedAt)) + { + if (ReferenceEquals(error, exception)) + { + throw; + } + + throw error; + } + + _logger.LogWarning( + exception, + "Socket Mode geo {Geo} initial connection failed; retrying in {Delay}.", + Geo, + delay); + await Task.Delay(delay, _timeProvider, startSource.Token).ConfigureAwait(false); + } + } + } + + private async Task SuperviseAsync(Generation current) + { + try + { + while (true) + { + CloseReason reason = await current.Closed.Task.WaitAsync(_stopToken).ConfigureAwait(false); + + if (reason.Planned) + { + _logger.LogInformation("Socket Mode geo {Geo} rotating connection before token expiry.", Geo); + Generation replacement = await ReconnectAsync(null, delayFirstAttempt: false).ConfigureAwait(false); + StartRetirement(current); + current = replacement; + } + else + { + await ReleaseAsync(current.Connection).ConfigureAwait(false); + current = await ReconnectAsync(reason.Error, delayFirstAttempt: true).ConfigureAwait(false); + } + + ReportReconnected(); + } + } + catch (OperationCanceledException) when (_stopToken.IsCancellationRequested) + { + } + catch (Exception exception) + { + _logger.LogError(exception, "Socket Mode geo {Geo} supervisor stopped unexpectedly.", Geo); + throw; + } + } + + private async Task ReconnectAsync(Exception? previousError, bool delayFirstAttempt) + { + Exception? error = previousError; + int retry = 0; + + for (int attempt = 1; ; attempt++) + { + if (delayFirstAttempt || attempt > 1) + { + await Task.Delay(GetRetryDelay(error, retry++), _timeProvider, _stopToken).ConfigureAwait(false); + } + + try + { + return await ConnectAsync(_stopToken).ConfigureAwait(false); + } + catch (Exception exception) when (!_stopToken.IsCancellationRequested) + { + error = exception; + _logger.LogWarning( + exception, + "Socket Mode geo {Geo} reconnect attempt {Attempt} failed.", + Geo, + attempt); + } + } + } + + private async Task ConnectAsync(CancellationToken cancellationToken) + { + Generation generation; + lock (_sync) + { + ThrowIfStopping(); + generation = new Generation(++_generation); + } + + generation.Connection = _connectionFactory.Create( + _negotiateUri, + new SocketConnectionHandlers( + envelope => DispatchAsync(generation, envelope), + frame => HandleReady(generation, frame), + (error, planned) => HandleClosed(generation, error, planned))); + + bool owned; + lock (_sync) + { + owned = !_stopping && _owned.Add(generation.Connection); + } + + if (!owned) + { + await StopAndDisposeAsync(generation.Connection).ConfigureAwait(false); + throw new OperationCanceledException("Socket Mode geo is stopping."); + } + + try + { + await generation.Connection.StartAsync(cancellationToken).ConfigureAwait(false); + + // StartAsync completes after SocketReady, but the ready callback may still be running. + if (!TryPromote(generation)) + { + lock (_sync) + { + ThrowIfStopping(); + } + + throw new IOException("Socket Mode connection closed before it became active."); + } + } + catch + { + await ReleaseAsync(generation.Connection).ConfigureAwait(false); + throw; + } + + ScheduleRefresh(generation); + return generation; + } + + private Task DispatchAsync(Generation generation, SocketActivityEnvelope envelope) + { + lock (_sync) + { + if (_stopping || (_active != generation && !_retiring.Contains(generation.Id))) + { + return Task.FromResult(null); + } + } + + return _owner.DispatchAsync(Geo, envelope); + } + + private void HandleReady(Generation generation, SocketReadyFrame frame) + { + if (TryPromote(generation)) + { + _owner.OnGeoReady(Geo, frame); + } + } + + private bool TryPromote(Generation generation) + { + lock (_sync) + { + if (_stopping || generation.IsClosed || generation.Id != _generation) + { + return false; + } + + if (_active is Generation previous && previous != generation) + { + _retiring.Add(previous.Id); + } + + _active = generation; + return true; + } + } + + private void HandleClosed(Generation generation, Exception? error, bool planned) + { + bool disconnected = false; + + lock (_sync) + { + generation.IsClosed = true; + _retiring.Remove(generation.Id); + + if (_active == generation) + { + _active = null; + _refreshTimer?.Dispose(); + _refreshTimer = null; + disconnected = !planned && !_stopping; + _disconnected |= disconnected; + } + + generation.Closed.TrySetResult(new CloseReason(error, planned)); + } + + if (disconnected) + { + _logger.LogWarning(error, "Socket Mode geo {Geo} disconnected; inbound delivery paused.", Geo); + _owner.OnGeoDisconnected(Geo, error); + } + } + + private void ReportReconnected() + { + lock (_sync) + { + if (!_disconnected || _stopping) + { + return; + } + + _disconnected = false; + } + + _logger.LogInformation("Socket Mode geo {Geo} reconnected; inbound delivery resumed.", Geo); + _owner.OnGeoReconnected(Geo); + } + + private void ScheduleRefresh(Generation generation) + { + if (generation.Connection.TokenLifetime is not TimeSpan lifetime || lifetime <= TimeSpan.Zero) + { + return; + } + + TimeSpan delay = lifetime - _owner.TokenRefreshMargin; + if (delay < MinimumRefreshDelay) + { + delay = MinimumRefreshDelay; + } + + lock (_sync) + { + if (_stopping || _active != generation) + { + return; + } + + _refreshTimer?.Dispose(); + _refreshTimer = _timeProvider.CreateTimer( + _ => RequestRotation(generation), + null, + delay, + Timeout.InfiniteTimeSpan); + } + } + + private void RequestRotation(Generation generation) + { + lock (_sync) + { + if (!_stopping && _active == generation) + { + generation.Closed.TrySetResult(new CloseReason(null, Planned: true)); + } + } + } + + private void StartRetirement(Generation previous) + { + Task retirement = RetireAsync(previous); + lock (_sync) + { + if (!retirement.IsCompleted) + { + _retirements.Add(retirement); + } + } + + _ = retirement.ContinueWith( + completed => + { + lock (_sync) + { + _retirements.Remove(completed); + } + }, + CancellationToken.None, + TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); + } + + [SuppressMessage( + "Design", + "CA1031:Do not catch general exception types", + Justification = "Retirement runs in the background; failures are logged so they cannot fault the supervisor.")] + private async Task RetireAsync(Generation previous) + { + try + { + TimeSpan handoff = _owner.HandoffWindow; + await Task.Delay( + handoff > TimeSpan.Zero ? handoff : TimeSpan.Zero, + _timeProvider, + _stopToken).ConfigureAwait(false); + + lock (_sync) + { + _retiring.Remove(previous.Id); + } + + await ReleaseAsync(previous.Connection).ConfigureAwait(false); + } + catch (OperationCanceledException) when (_stopToken.IsCancellationRequested) + { + } + catch (Exception exception) + { + _logger.LogWarning(exception, "Socket Mode geo {Geo} failed to retire a replaced connection.", Geo); + } + } + + private async Task StopCoreAsync() + { + // Leave the caller's lock before cancelling and stopping connections. + await Task.Yield(); + await _stopSource.CancelAsync().ConfigureAwait(false); + + ISocketConnection[] connections; + Task[] retirements; + Task? supervisor; + lock (_sync) + { + _active = null; + _retiring.Clear(); + _refreshTimer?.Dispose(); + _refreshTimer = null; + connections = [.. _owned]; + _owned.Clear(); + retirements = [.. _retirements]; + supervisor = _supervisor; + } + + try + { + await Task.WhenAll(connections.Select(StopAndDisposeAsync)).ConfigureAwait(false); + } + finally + { + await Task.WhenAll(retirements).ConfigureAwait(false); + if (supervisor is not null) + { + await supervisor.ConfigureAwait(false); + } + } + } + + private async Task ReleaseAsync(ISocketConnection connection) + { + lock (_sync) + { + if (!_owned.Remove(connection)) + { + return; + } + } + + await StopAndDisposeAsync(connection).ConfigureAwait(false); + } + + private static async Task StopAndDisposeAsync(ISocketConnection connection) + { + try + { + await connection.StopAsync(CancellationToken.None).ConfigureAwait(false); + } + finally + { + await connection.DisposeAsync().ConfigureAwait(false); + } + } + + private TimeSpan GetRetryDelay(Exception? error, int attempt) + { + TimeSpan delay = _owner.GetRetryAfter(error) ?? _owner.GetBackoffDelay(attempt); + return delay > TimeSpan.Zero ? delay : TimeSpan.Zero; + } + + private TimeSpan RemainingBudget(TimeSpan budget, long startedAt) + { + TimeSpan remaining = budget - _timeProvider.GetElapsedTime(startedAt); + return remaining > TimeSpan.Zero ? remaining : TimeSpan.Zero; + } + + private void ThrowIfStopping() + { + if (_stopping) + { + throw new OperationCanceledException("Socket Mode geo is stopping."); + } + } + + private sealed class Generation(long id) + { + internal long Id { get; } = id; + + internal ISocketConnection Connection { get; set; } = null!; + + internal TaskCompletionSource Closed { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + internal bool IsClosed { get; set; } + } + + private sealed record CloseReason(Exception? Error, bool Planned); +} diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs new file mode 100644 index 00000000..a5899e9e --- /dev/null +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs @@ -0,0 +1,631 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Diagnostics; +using System.Net; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Teams.Apps.SocketMode; + +namespace Microsoft.Teams.Apps.UnitTests.SocketMode; + +public class GeoSocketTests +{ + private static readonly Uri NegotiateUri = new("https://botapi.skype.com/amer/v3/websockets/connect"); + + [Fact] + public async Task StartAsync_CompletesOnlyAfterSocketReady() + { + Harness harness = new(); + FakeConnection connection = harness.Factory.Enqueue(); + + Task start = harness.Socket.StartAsync(); + await connection.Started.Task; + Assert.False(start.IsCompleted); + + connection.Ready("initial"); + await start; + + Assert.NotNull(await connection.Activity("a1")); + Assert.Equal(["initial"], harness.Owner.ReadyIds); + Assert.Equal(["a1"], harness.Owner.Dispatched); + Assert.Empty(harness.Owner.Disconnections); + } + + [Fact] + public async Task StartAsync_RetriesWithBackoffWithinBudget() + { + Harness harness = new() { Backoff = _ => TimeSpan.FromSeconds(2) }; + FakeConnection failed = harness.Factory.Enqueue(startError: new IOException("first")); + FakeConnection retry = harness.Factory.Enqueue(); + + Task start = harness.Socket.StartAsync(); + await harness.Time.WaitForTimerAsync(TimeSpan.FromSeconds(2)); + harness.Time.Advance(TimeSpan.FromSeconds(2)); + await retry.Started.Task; + retry.Ready("retry"); + await start; + + Assert.Equal(1, failed.StopCount); + Assert.Equal(1, failed.DisposeCount); + Assert.Null(await failed.Activity("stale")); + Assert.Equal(["retry"], harness.Owner.ReadyIds); + } + + [Fact] + public async Task StartAsync_PrefersRetryAfterOverBackoff() + { + Harness harness = new() { Backoff = _ => TimeSpan.FromSeconds(2) }; + harness.Factory.Enqueue(startError: new SocketModeNegotiateException( + HttpStatusCode.TooManyRequests, + TimeSpan.FromSeconds(7))); + FakeConnection retry = harness.Factory.Enqueue(); + + Task start = harness.Socket.StartAsync(); + await harness.Time.WaitForTimerAsync(TimeSpan.FromSeconds(7)); + Assert.False(harness.Time.HasTimerDueIn(TimeSpan.FromSeconds(2))); + + harness.Time.Advance(TimeSpan.FromSeconds(7)); + await retry.Started.Task; + retry.Ready("retry"); + await start; + } + + [Fact] + public async Task StartAsync_FailsWhenPendingAttemptExceedsBudget() + { + Harness harness = new() { StartupTimeout = TimeSpan.FromSeconds(5) }; + FakeConnection connection = harness.Factory.Enqueue(); + + Task start = harness.Socket.StartAsync(); + await connection.Started.Task; + await harness.Time.WaitForTimerAsync(TimeSpan.FromSeconds(5)); + harness.Time.Advance(TimeSpan.FromSeconds(5)); + + await Assert.ThrowsAsync(() => start); + Assert.Equal(1, harness.Factory.CreateCount); + Assert.Equal(1, connection.DisposeCount); + } + + [Fact] + public async Task StartAsync_ZeroBudgetMakesOneAttemptWithoutRetry() + { + Harness harness = new() { StartupTimeout = TimeSpan.Zero }; + harness.Factory.Enqueue(startError: new IOException("failed")); + + IOException error = await Assert.ThrowsAsync(() => harness.Socket.StartAsync()); + + Assert.Equal("failed", error.Message); + Assert.Equal(1, harness.Factory.CreateCount); + } + + [Fact] + public async Task UnexpectedClose_ReportsDisconnectedThenReconnected() + { + Harness harness = new() { Backoff = _ => TimeSpan.FromSeconds(3) }; + FakeConnection initial = harness.Factory.Enqueue(); + FakeConnection replacement = harness.Factory.Enqueue(); + await harness.StartReadyAsync(initial); + + IOException dropped = new("dropped"); + initial.Close(dropped); + + Assert.Same(dropped, Assert.Single(harness.Owner.Disconnections)); + Assert.Null(await initial.Activity("after-close")); + await harness.Time.WaitForTimerAsync(TimeSpan.FromSeconds(3)); + harness.Time.Advance(TimeSpan.FromSeconds(3)); + await replacement.Started.Task; + replacement.Ready("replacement"); + await WaitUntilAsync(() => harness.Owner.Reconnections == 1); + + Assert.Equal(1, initial.DisposeCount); + Assert.NotNull(await replacement.Activity("resumed")); + Assert.DoesNotContain("after-close", harness.Owner.Dispatched); + } + + [Fact] + public async Task Rotation_IsMakeBeforeBreakWithHandoff() + { + Harness harness = new(); + FakeConnection initial = harness.Factory.Enqueue(TimeSpan.FromSeconds(10)); + FakeConnection replacement = harness.Factory.Enqueue(); + await harness.StartReadyAsync(initial); + + harness.Time.Advance(TimeSpan.FromSeconds(5)); + await replacement.Started.Task; + Assert.NotNull(await initial.Activity("during-replacement-startup")); + Assert.Equal(0, initial.StopCount); + + replacement.Ready("replacement"); + await harness.Time.WaitForTimerAsync(harness.Owner.HandoffWindow); + Assert.NotNull(await initial.Activity("retiring")); + Assert.NotNull(await replacement.Activity("active")); + Assert.Equal(0, initial.StopCount); + + harness.Time.Advance(harness.Owner.HandoffWindow); + await initial.Disposed.Task; + + Assert.Equal(1, initial.StopCount); + Assert.Null(await initial.Activity("retired")); + Assert.DoesNotContain("retired", harness.Owner.Dispatched); + Assert.Empty(harness.Owner.Disconnections); + Assert.Equal(0, harness.Owner.Reconnections); + Assert.Equal(2, harness.Owner.ReadyIds.Count); + } + + [Fact] + public async Task Rotation_UsesOneSecondMinimumDelay() + { + Harness harness = new() { TokenRefreshMargin = TimeSpan.FromSeconds(30) }; + FakeConnection initial = harness.Factory.Enqueue(TimeSpan.FromSeconds(10)); + FakeConnection replacement = harness.Factory.Enqueue(); + await harness.StartReadyAsync(initial); + + Assert.True(harness.Time.HasTimerDueIn(TimeSpan.FromSeconds(1))); + harness.Time.Advance(TimeSpan.FromSeconds(1)); + await replacement.Started.Task; + } + + [Fact] + public async Task TokenLifetimeAbsent_DoesNotScheduleRotation() + { + Harness harness = new(); + FakeConnection connection = harness.Factory.Enqueue(tokenLifetime: null); + await harness.StartReadyAsync(connection); + + Assert.Equal(0, harness.Time.ScheduledTimerCount); + harness.Time.Advance(TimeSpan.FromDays(1)); + + Assert.Equal(1, harness.Factory.CreateCount); + } + + [Fact] + public async Task PredecessorDropDuringRotation_ReportsOutageUntilReplacementReady() + { + Harness harness = new(); + FakeConnection initial = harness.Factory.Enqueue(TimeSpan.FromSeconds(10)); + FakeConnection replacement = harness.Factory.Enqueue(); + await harness.StartReadyAsync(initial); + + harness.Time.Advance(TimeSpan.FromSeconds(5)); + await replacement.Started.Task; + initial.Close(new IOException("dropped")); + + Assert.Single(harness.Owner.Disconnections); + Assert.Null(await initial.Activity("dropped")); + + replacement.Ready("replacement"); + await WaitUntilAsync(() => harness.Owner.Reconnections == 1); + await harness.Time.WaitForTimerAsync(harness.Owner.HandoffWindow); + harness.Time.Advance(harness.Owner.HandoffWindow); + await initial.Disposed.Task; + + Assert.Equal(1, initial.DisposeCount); + } + + [Fact] + public async Task RepeatedRotation_ReleasesEveryPredecessorOnce() + { + Harness harness = new() { HandoffWindow = TimeSpan.Zero }; + FakeConnection[] connections = + [ + harness.Factory.Enqueue(TimeSpan.FromSeconds(10)), + harness.Factory.Enqueue(TimeSpan.FromSeconds(10)), + harness.Factory.Enqueue(TimeSpan.FromSeconds(10)), + harness.Factory.Enqueue(TimeSpan.FromSeconds(10)), + ]; + await harness.StartReadyAsync(connections[0]); + + for (int i = 1; i < connections.Length; i++) + { + await harness.Time.WaitForTimerAsync(TimeSpan.FromSeconds(5)); + harness.Time.Advance(TimeSpan.FromSeconds(5)); + await connections[i].Started.Task; + connections[i].Ready($"gen-{i}"); + await connections[i - 1].Disposed.Task; + } + + Assert.All(connections[..^1], connection => Assert.Equal(1, connection.DisposeCount)); + Assert.Equal(0, connections[^1].DisposeCount); + Assert.Empty(harness.Owner.Disconnections); + + await harness.Socket.StopAsync(); + Assert.All(connections, connection => Assert.Equal(1, connection.DisposeCount)); + } + + [Fact] + public async Task StopAsync_DuringStartup_CancelsAndDisposesOnce() + { + Harness harness = new(); + FakeConnection connection = harness.Factory.Enqueue(); + + Task start = harness.Socket.StartAsync(); + await connection.Started.Task; + await harness.Socket.StopAsync(); + + await Assert.ThrowsAnyAsync(() => start); + await harness.Socket.StopAsync(); + await harness.Socket.DisposeAsync(); + await harness.Socket.DisposeAsync(); + + Assert.Equal(1, connection.StopCount); + Assert.Equal(1, connection.DisposeCount); + Assert.Empty(harness.Owner.Disconnections); + } + + [Fact] + public async Task StartAsync_CallerCancellationInterruptsRetryDelay() + { + Harness harness = new() { Backoff = _ => TimeSpan.FromMinutes(1), StartupTimeout = TimeSpan.FromMinutes(5) }; + FakeConnection failed = harness.Factory.Enqueue(startError: new IOException("failed")); + using CancellationTokenSource cancellation = new(); + + Task start = harness.Socket.StartAsync(cancellation.Token); + await harness.Time.WaitForTimerAsync(TimeSpan.FromMinutes(1)); + await cancellation.CancelAsync(); + + await Assert.ThrowsAnyAsync(() => start); + Assert.Equal(1, harness.Factory.CreateCount); + Assert.Equal(1, failed.DisposeCount); + } + + [Fact] + public async Task StopAsync_DuringReconnectDelay_CancelsWithoutNewGeneration() + { + Harness harness = new() { Backoff = _ => TimeSpan.FromMinutes(1) }; + FakeConnection initial = harness.Factory.Enqueue(); + await harness.StartReadyAsync(initial); + + initial.Close(new IOException("dropped")); + await harness.Time.WaitForTimerAsync(TimeSpan.FromMinutes(1)); + await harness.Socket.StopAsync(); + harness.Time.Advance(TimeSpan.FromMinutes(1)); + + Assert.Equal(1, harness.Factory.CreateCount); + Assert.Equal(1, initial.DisposeCount); + Assert.Equal(0, harness.Owner.Reconnections); + } + + [Fact] + public async Task StopAsync_DuringRotation_DisposesActiveAndConnectingOnce() + { + Harness harness = new(); + FakeConnection active = harness.Factory.Enqueue(TimeSpan.FromSeconds(10)); + FakeConnection connecting = harness.Factory.Enqueue(); + await harness.StartReadyAsync(active); + + harness.Time.Advance(TimeSpan.FromSeconds(5)); + await connecting.Started.Task; + await harness.Socket.StopAsync(); + + Assert.Equal(1, active.StopCount); + Assert.Equal(1, active.DisposeCount); + Assert.Equal(1, connecting.StopCount); + Assert.Equal(1, connecting.DisposeCount); + Assert.Empty(harness.Owner.Disconnections); + Assert.Null(await active.Activity("stopped")); + } + + [Fact] + public async Task StopAsync_DuringHandoff_DisposesActiveAndRetiringOnce() + { + Harness harness = new() { HandoffWindow = TimeSpan.FromMinutes(1) }; + FakeConnection retiring = harness.Factory.Enqueue(TimeSpan.FromSeconds(10)); + FakeConnection active = harness.Factory.Enqueue(); + await harness.StartReadyAsync(retiring); + + harness.Time.Advance(TimeSpan.FromSeconds(5)); + await active.Started.Task; + active.Ready("replacement"); + await harness.Time.WaitForTimerAsync(TimeSpan.FromMinutes(1)); + await harness.Socket.StopAsync(); + harness.Time.Advance(TimeSpan.FromMinutes(1)); + + Assert.Equal(1, retiring.StopCount); + Assert.Equal(1, retiring.DisposeCount); + Assert.Equal(1, active.StopCount); + Assert.Equal(1, active.DisposeCount); + Assert.Empty(harness.Owner.Disconnections); + } + + private static async Task WaitUntilAsync(Func condition) + { + Stopwatch elapsed = Stopwatch.StartNew(); + while (!condition()) + { + if (elapsed.Elapsed > TimeSpan.FromSeconds(5)) + { + throw new TimeoutException("The asynchronous test condition was not met."); + } + + await Task.Yield(); + } + } + + private sealed class Harness + { + private GeoSocket? _socket; + + internal ManualTimeProvider Time { get; } = new(); + + internal FakeOwner Owner { get; } = new(); + + internal FakeConnectionFactory Factory { get; } = new(); + + internal TimeSpan StartupTimeout { init => Owner.StartupTimeout = value; } + + internal TimeSpan TokenRefreshMargin { init => Owner.TokenRefreshMargin = value; } + + internal TimeSpan HandoffWindow { init => Owner.HandoffWindow = value; } + + internal Func Backoff { init => Owner.Backoff = value; } + + internal GeoSocket Socket => + _socket ??= new GeoSocket(Owner, "amer", NegotiateUri, Factory, NullLogger.Instance, Time); + + internal async Task StartReadyAsync(FakeConnection connection) + { + Task start = Socket.StartAsync(); + await connection.Started.Task; + connection.Ready("initial"); + await start; + } + } + + private sealed class FakeOwner : IGeoSocketOwner + { + private readonly object _sync = new(); + private readonly List _dispatched = []; + private readonly List _readyIds = []; + private readonly List _disconnections = []; + private int _reconnections; + + public TimeSpan StartupTimeout { get; set; } = TimeSpan.FromSeconds(30); + + public TimeSpan TokenRefreshMargin { get; set; } = TimeSpan.FromSeconds(5); + + public TimeSpan HandoffWindow { get; set; } = TimeSpan.FromSeconds(5); + + internal Func Backoff { get; set; } = _ => TimeSpan.FromSeconds(1); + + internal List Dispatched { get { lock (_sync) { return [.. _dispatched]; } } } + + internal List ReadyIds { get { lock (_sync) { return [.. _readyIds]; } } } + + internal List Disconnections { get { lock (_sync) { return [.. _disconnections]; } } } + + internal int Reconnections => Volatile.Read(ref _reconnections); + + public TimeSpan? GetRetryAfter(Exception? error) + => (error as SocketModeNegotiateException)?.RetryAfter; + + public TimeSpan GetBackoffDelay(int attempt) => Backoff(attempt); + + public Task DispatchAsync(string geo, SocketActivityEnvelope envelope) + { + lock (_sync) + { + _dispatched.Add(envelope.EnvelopeId!); + } + + return Task.FromResult(new SocketReplyFrame { Status = 200 }); + } + + public void OnGeoReady(string geo, SocketReadyFrame frame) + { + lock (_sync) + { + _readyIds.Add(frame.ConnectionId!); + } + } + + public void OnGeoDisconnected(string geo, Exception? error) + { + lock (_sync) + { + _disconnections.Add(error); + } + } + + public void OnGeoReconnected(string geo) => Interlocked.Increment(ref _reconnections); + } + + private sealed class FakeConnectionFactory : ISocketConnectionFactory + { + private readonly Queue _pending = []; + private int _createCount; + + internal int CreateCount => Volatile.Read(ref _createCount); + + internal FakeConnection Enqueue(TimeSpan? tokenLifetime = null, Exception? startError = null) + { + FakeConnection connection = new(tokenLifetime, startError); + lock (_pending) + { + _pending.Enqueue(connection); + } + + return connection; + } + + public ISocketConnection Create(Uri negotiateUri, SocketConnectionHandlers handlers) + { + Interlocked.Increment(ref _createCount); + FakeConnection connection; + lock (_pending) + { + connection = _pending.Dequeue(); + } + + connection.Handlers = handlers; + return connection; + } + } + + private sealed class FakeConnection(TimeSpan? tokenLifetime, Exception? startError) : ISocketConnection + { + private readonly TaskCompletionSource _ready = new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _closed; + private int _stopCount; + private int _disposeCount; + + public TimeSpan? TokenLifetime { get; } = tokenLifetime; + + internal SocketConnectionHandlers Handlers { get; set; } = null!; + + internal TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + internal TaskCompletionSource Disposed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + internal int StopCount => Volatile.Read(ref _stopCount); + + internal int DisposeCount => Volatile.Read(ref _disposeCount); + + public async Task StartAsync(CancellationToken cancellationToken) + { + Started.TrySetResult(); + if (startError is not null) + { + throw startError; + } + + await _ready.Task.WaitAsync(cancellationToken); + } + + public Task StopAsync(CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref _stopCount); + _ready.TrySetCanceled(CancellationToken.None); + RaiseClosed(null, planned: true); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + Interlocked.Increment(ref _disposeCount); + Disposed.TrySetResult(); + return ValueTask.CompletedTask; + } + + internal void Ready(string connectionId) + { + Handlers.OnReady(new SocketReadyFrame { ConnectionId = connectionId }); + _ready.TrySetResult(); + } + + internal void Close(Exception error) => RaiseClosed(error, planned: false); + + internal Task Activity(string envelopeId) + => Handlers.OnActivity(new SocketActivityEnvelope { EnvelopeId = envelopeId }); + + private void RaiseClosed(Exception? error, bool planned) + { + if (Interlocked.Exchange(ref _closed, 1) == 0) + { + Handlers.OnClosed(error, planned); + } + } + } + + private sealed class ManualTimeProvider : TimeProvider + { + private readonly object _sync = new(); + private readonly List _timers = []; + private long _now; + + public override long TimestampFrequency => TimeSpan.TicksPerSecond; + + internal int ScheduledTimerCount + { + get + { + lock (_sync) + { + return _timers.Count(timer => timer.Due is not null); + } + } + } + + public override long GetTimestamp() + { + lock (_sync) + { + return _now; + } + } + + public override DateTimeOffset GetUtcNow() => DateTimeOffset.UnixEpoch.AddTicks(GetTimestamp()); + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + ManualTimer timer = new(this, callback, state); + timer.Change(dueTime, period); + lock (_sync) + { + _timers.Add(timer); + } + + return timer; + } + + internal bool HasTimerDueIn(TimeSpan dueIn) + { + lock (_sync) + { + return _timers.Any(timer => timer.Due == _now + dueIn.Ticks); + } + } + + internal Task WaitForTimerAsync(TimeSpan dueIn) => WaitUntilAsync(() => HasTimerDueIn(dueIn)); + + internal void Advance(TimeSpan amount) + { + List due; + lock (_sync) + { + _now += amount.Ticks; + due = [.. _timers.Where(timer => timer.Due <= _now)]; + foreach (ManualTimer timer in due) + { + timer.Due = null; + } + } + + foreach (ManualTimer timer in due) + { + timer.Fire(); + } + } + + private sealed class ManualTimer(ManualTimeProvider owner, TimerCallback callback, object? state) : ITimer + { + internal long? Due { get; set; } + + public bool Change(TimeSpan dueTime, TimeSpan period) + { + lock (owner._sync) + { + Due = dueTime == Timeout.InfiniteTimeSpan ? null : owner._now + dueTime.Ticks; + } + + return true; + } + + public void Dispose() + { + lock (owner._sync) + { + Due = null; + owner._timers.Remove(this); + } + } + + public ValueTask DisposeAsync() + { + Dispose(); + return ValueTask.CompletedTask; + } + + internal void Fire() => callback(state); + } + } +} From 5e4c28ffcec9f37ab2d64169d1731cf52af0b5a4 Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Wed, 23 Sep 2026 22:39:29 -0700 Subject: [PATCH 22/31] feat(apps): add multi-geo Socket Mode transport Resolve per-geo negotiate endpoints, start every geo with fail-fast startup, stop all geos, aggregate status internally, and dispatch envelopes with invoke/ack replies and 500 on handler failure. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SocketMode/GeoSocket.cs | 5 +- .../SocketMode/SocketModeTransport.cs | 476 +++++++++++++++++ .../SocketMode/SocketModeEndToEndTests.cs | 287 +++++++++++ .../SocketMode/SocketModeTransportTests.cs | 484 ++++++++++++++++++ 4 files changed, 1249 insertions(+), 3 deletions(-) create mode 100644 src/Microsoft.Teams.Apps/SocketMode/SocketModeTransport.cs create mode 100644 test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeEndToEndTests.cs create mode 100644 test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeTransportTests.cs diff --git a/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs b/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs index b7bb8314..0a4d343c 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs @@ -104,7 +104,7 @@ internal sealed class GeoSocket : IAsyncDisposable /// Initializes a supervisor for one geo. /// /// The transport that owns this geo. - /// The geo identifier. + /// The geo identifier. Empty when the negotiate URL has no geo segment. /// The negotiate endpoint for the geo. /// Creates one connection per generation. /// The logger for lifecycle events. @@ -118,8 +118,7 @@ internal GeoSocket( TimeProvider? timeProvider = null) { _owner = owner ?? throw new ArgumentNullException(nameof(owner)); - ArgumentException.ThrowIfNullOrWhiteSpace(geo); - Geo = geo; + Geo = geo ?? throw new ArgumentNullException(nameof(geo)); _negotiateUri = negotiateUri ?? throw new ArgumentNullException(nameof(negotiateUri)); _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); diff --git a/src/Microsoft.Teams.Apps/SocketMode/SocketModeTransport.cs b/src/Microsoft.Teams.Apps/SocketMode/SocketModeTransport.cs new file mode 100644 index 00000000..fc043c77 --- /dev/null +++ b/src/Microsoft.Teams.Apps/SocketMode/SocketModeTransport.cs @@ -0,0 +1,476 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Diagnostics.CodeAnalysis; +using System.Runtime.ExceptionServices; +using Microsoft.Extensions.Logging; +using Microsoft.Teams.Apps.Schema; +using Microsoft.Teams.Core.Schema; + +namespace Microsoft.Teams.Apps.SocketMode; + +/// +/// Describes the lifecycle state of a Socket Mode transport or one of its geos. +/// +internal enum SocketModeStatus +{ + /// + /// The transport has not started. + /// + Idle, + + /// + /// A connection is being established. + /// + Connecting, + + /// + /// Every connection is ready to receive activities. + /// + Ready, + + /// + /// A connection dropped unexpectedly and is recovering. + /// + Disconnected, + + /// + /// The transport has stopped. + /// + Stopped, +} + +/// +/// Configures a . +/// +internal sealed class SocketModeTransportOptions +{ + /// + /// Gets the base URL used to negotiate each geo connection. + /// + internal Uri NegotiateBaseUri { get; init; } = new(SocketModeProtocol.DefaultNegotiateBaseUrl); + + /// + /// Gets the geos to connect. An empty string connects to the base URL without a geo segment. + /// + internal IReadOnlyList Geos { get; init; } = SocketModeProtocol.DefaultGeos; + + /// + /// Gets the time each geo has to establish its initial connection. + /// + internal TimeSpan StartupTimeout { get; init; } = TimeSpan.FromSeconds(30); + + /// + /// Gets an explicit reconnect delay schedule. The last delay repeats. When omitted, capped exponential + /// backoff with full jitter is used. + /// + internal IReadOnlyList? ReconnectDelays { get; init; } +} + +/// +/// Connects one per configured geo and routes their activities to the app. +/// +/// +/// Startup is all-or-nothing: every geo must become ready or the transport stops and startup fails. Once +/// started, each geo is supervised independently. +/// +internal sealed class SocketModeTransport : IGeoSocketOwner, IAsyncDisposable +{ + private static readonly TimeSpan TokenRefreshMargin = TimeSpan.FromSeconds(60); + private static readonly TimeSpan HandoffWindow = TimeSpan.FromSeconds(5); + private static readonly TimeSpan ReconnectInitialDelay = TimeSpan.FromSeconds(1); + private static readonly TimeSpan ReconnectMaxDelay = TimeSpan.FromSeconds(15); + + private readonly SocketModeTransportOptions _options; + private readonly IReadOnlyList<(string Geo, Uri NegotiateUri)> _geos; + private readonly ISocketConnectionFactory _connectionFactory; + private readonly Func> _dispatch; + private readonly Func? _onError; + private readonly string? _botKey; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly Random _random; + private readonly object _sync = new(); + private readonly Dictionary _geoStatuses = []; + + private GeoSocket[] _geoSockets = []; + private SocketModeStatus _lifecycle = SocketModeStatus.Idle; + private Task? _stopTask; + + /// + /// Initializes a Socket Mode transport and resolves its geos. + /// + /// The transport options. + /// Creates one connection per geo generation. + /// Processes an inbound activity and returns its status and optional body. + /// The logger for transport lifecycle and dispatch failures. + /// The bot key echoed on reply frames, when available. + /// Observes activity processing failures before the failure reply is returned. + /// The time provider for supervision timing and reply timestamps. + /// The random source for reconnect jitter. + /// Thrown when the geos are empty, null, or duplicated. + /// Thrown when the startup timeout is negative. + internal SocketModeTransport( + SocketModeTransportOptions options, + ISocketConnectionFactory connectionFactory, + Func> dispatch, + ILogger logger, + string? botKey = null, + Func? onError = null, + TimeProvider? timeProvider = null, + Random? random = null) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); + _dispatch = dispatch ?? throw new ArgumentNullException(nameof(dispatch)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _botKey = botKey; + _onError = onError; + _timeProvider = timeProvider ?? TimeProvider.System; + _random = random ?? Random.Shared; + + ArgumentNullException.ThrowIfNull(options.NegotiateBaseUri, nameof(options)); + ArgumentOutOfRangeException.ThrowIfLessThan(options.StartupTimeout, TimeSpan.Zero, nameof(options)); + _geos = ResolveGeos(options.NegotiateBaseUri, options.Geos); + } + + /// + /// Gets the aggregate status: ready when every geo is ready, disconnected when any geo is recovering from a + /// drop, and connecting otherwise. + /// + internal SocketModeStatus Status + { + get + { + lock (_sync) + { + if (_lifecycle is SocketModeStatus.Idle or SocketModeStatus.Stopped) + { + return _lifecycle; + } + + if (_geoStatuses.Count > 0 && _geoStatuses.Values.All(status => status == SocketModeStatus.Ready)) + { + return SocketModeStatus.Ready; + } + + return _geoStatuses.ContainsValue(SocketModeStatus.Disconnected) + ? SocketModeStatus.Disconnected + : SocketModeStatus.Connecting; + } + } + } + + /// + /// Gets a snapshot of each geo's status. + /// + internal IReadOnlyDictionary GeoStatuses + { + get + { + lock (_sync) + { + return new Dictionary(_geoStatuses); + } + } + } + + /// + TimeSpan IGeoSocketOwner.StartupTimeout => _options.StartupTimeout; + + /// + TimeSpan IGeoSocketOwner.TokenRefreshMargin => TokenRefreshMargin; + + /// + TimeSpan IGeoSocketOwner.HandoffWindow => HandoffWindow; + + /// + /// Connects every geo. Completes once all are ready; otherwise stops the transport and throws the first + /// geo failure. + /// + /// A token for cancelling startup. + internal async Task StartAsync(CancellationToken cancellationToken = default) + { + GeoSocket[] geoSockets; + lock (_sync) + { + if (_lifecycle != SocketModeStatus.Idle) + { + throw new InvalidOperationException("A Socket Mode transport can only be started once."); + } + + _lifecycle = SocketModeStatus.Connecting; + geoSockets = [.. _geos.Select(geo => new GeoSocket( + this, + geo.Geo, + geo.NegotiateUri, + _connectionFactory, + _logger, + _timeProvider))]; + foreach (GeoSocket geoSocket in geoSockets) + { + _geoStatuses[geoSocket.Geo] = SocketModeStatus.Connecting; + } + + _geoSockets = geoSockets; + } + + _logger.LogInformation( + "Socket Mode connecting {Count} geo(s): {Geos}.", + geoSockets.Length, + string.Join(", ", geoSockets.Select(geoSocket => geoSocket.Geo))); + + using CancellationTokenSource startSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + Exception? firstFailure = null; + + async Task StartGeoAsync(GeoSocket geoSocket) + { + try + { + await geoSocket.StartAsync(startSource.Token).ConfigureAwait(false); + } + catch (Exception exception) + { + if (Interlocked.CompareExchange(ref firstFailure, exception, null) is null) + { + await startSource.CancelAsync().ConfigureAwait(false); + } + + throw; + } + } + + try + { + await Task.WhenAll(geoSockets.Select(StartGeoAsync)).ConfigureAwait(false); + } + catch (Exception exception) + { + bool stoppedExternally; + lock (_sync) + { + stoppedExternally = _stopTask is not null; + } + + await StopAsync().ConfigureAwait(false); + + if (stoppedExternally) + { + throw new OperationCanceledException("Socket Mode transport stopped before startup completed.", exception); + } + + ExceptionDispatchInfo.Throw(firstFailure ?? exception); + } + + lock (_sync) + { + if (_stopTask is not null) + { + throw new OperationCanceledException("Socket Mode transport stopped before startup completed."); + } + } + + _logger.LogInformation("Socket Mode ready across {Count} geo(s).", geoSockets.Length); + } + + /// + /// Stops every geo and disposes their connections. Idempotent. + /// + internal Task StopAsync() + { + lock (_sync) + { + if (_stopTask is not null) + { + return _stopTask; + } + + _lifecycle = SocketModeStatus.Stopped; + GeoSocket[] geoSockets = _geoSockets; + _stopTask = Task.WhenAll(geoSockets.Select(geoSocket => geoSocket.DisposeAsync().AsTask())); + return _stopTask; + } + } + + /// + public ValueTask DisposeAsync() => new(StopAsync()); + + /// + TimeSpan? IGeoSocketOwner.GetRetryAfter(Exception? error) + => (error as SocketModeNegotiateException)?.RetryAfter; + + /// + [SuppressMessage( + "Security", + "CA5394:Do not use insecure randomness", + Justification = "Jitter only spreads reconnect attempts; it has no security purpose.")] + TimeSpan IGeoSocketOwner.GetBackoffDelay(int attempt) + { + if (_options.ReconnectDelays is { Count: > 0 } schedule) + { + return schedule[Math.Min(attempt, schedule.Count - 1)]; + } + + double capSeconds = Math.Min( + ReconnectInitialDelay.TotalSeconds * Math.Pow(2, Math.Min(attempt, 30)), + ReconnectMaxDelay.TotalSeconds); + return TimeSpan.FromSeconds(capSeconds * _random.NextDouble()); + } + + /// + Task IGeoSocketOwner.DispatchAsync(string geo, SocketActivityEnvelope envelope) + => HandleEnvelopeAsync(envelope); + + /// + void IGeoSocketOwner.OnGeoReady(string geo, SocketReadyFrame frame) + { + SetGeoStatus(geo, SocketModeStatus.Ready); + _logger.LogDebug("Socket Mode geo {Geo} ready with connection {ConnectionId}.", geo, frame.ConnectionId); + } + + /// + void IGeoSocketOwner.OnGeoDisconnected(string geo, Exception? error) + => SetGeoStatus(geo, SocketModeStatus.Disconnected); + + /// + void IGeoSocketOwner.OnGeoReconnected(string geo) + => SetGeoStatus(geo, SocketModeStatus.Ready); + + [SuppressMessage( + "Design", + "CA1031:Do not catch general exception types", + Justification = "Any activity processing failure must be logged and answered with a 500 reply frame.")] + private async Task HandleEnvelopeAsync(SocketActivityEnvelope envelope) + { + long receivedAt = NowUnixMilliseconds(); + + if (envelope.ProtocolVersion > SocketModeProtocol.CurrentVersion) + { + _logger.LogWarning( + "Socket Mode rejecting envelope {EnvelopeId} with unsupported protocol version {Version}.", + envelope.EnvelopeId, + envelope.ProtocolVersion); + return SocketModeEnvelope.CreateInvokeReply( + envelope, + _botKey, + new SocketDispatchResult(400, new { error = $"unsupported protocolVersion {envelope.ProtocolVersion}" }), + receivedAt, + NowUnixMilliseconds()); + } + + if (!SocketModeEnvelope.TryReadActivity(envelope, out CoreActivity? activity) || activity is null) + { + _logger.LogWarning("Socket Mode envelope {EnvelopeId} had no activity payload; dropping.", envelope.EnvelopeId); + return null; + } + + bool invoke = IsInvoke(envelope, activity); + + try + { + SocketDispatchResult result = await _dispatch(activity).ConfigureAwait(false); + return invoke + ? SocketModeEnvelope.CreateInvokeReply(envelope, _botKey, result, receivedAt, NowUnixMilliseconds()) + : SocketModeEnvelope.CreateAcknowledgement( + envelope, + _botKey, + receivedAt, + NowUnixMilliseconds(), + result.Status); + } + catch (Exception exception) + { + _logger.LogError( + exception, + "Socket Mode failed to process activity {ActivityType} in envelope {EnvelopeId}.", + activity.Type, + envelope.EnvelopeId); + await ReportErrorAsync(exception).ConfigureAwait(false); + + return invoke + ? SocketModeEnvelope.CreateInvokeReply( + envelope, + _botKey, + new SocketDispatchResult(500, new { error = "bot handler error" }), + receivedAt, + NowUnixMilliseconds()) + : SocketModeEnvelope.CreateAcknowledgement(envelope, _botKey, receivedAt, NowUnixMilliseconds(), 500); + } + } + + [SuppressMessage( + "Design", + "CA1031:Do not catch general exception types", + Justification = "A failing error observer is logged so it cannot prevent the failure reply.")] + private async Task ReportErrorAsync(Exception exception) + { + if (_onError is null) + { + return; + } + + try + { + await _onError(exception).ConfigureAwait(false); + } + catch (Exception hookException) + { + _logger.LogWarning(hookException, "Socket Mode error observer failed."); + } + } + + private void SetGeoStatus(string geo, SocketModeStatus status) + { + lock (_sync) + { + if (_lifecycle != SocketModeStatus.Stopped) + { + _geoStatuses[geo] = status; + } + } + } + + private long NowUnixMilliseconds() => _timeProvider.GetUtcNow().ToUnixTimeMilliseconds(); + + // An envelope is an invoke when its type says so, falling back to the activity type when the envelope has none. + private static bool IsInvoke(SocketActivityEnvelope envelope, CoreActivity activity) + => string.Equals( + string.IsNullOrEmpty(envelope.Type) ? activity.Type : envelope.Type, + TeamsActivityTypes.Invoke, + StringComparison.OrdinalIgnoreCase); + + private static List<(string Geo, Uri NegotiateUri)> ResolveGeos(Uri baseUri, IReadOnlyList geos) + { + if (geos is null || geos.Count == 0) + { + throw new ArgumentException( + "Socket Mode requires at least one geo. Use an empty string to connect without a geo segment.", + nameof(geos)); + } + + string baseUrl = baseUri.AbsoluteUri.TrimEnd('/'); + HashSet seen = new(StringComparer.OrdinalIgnoreCase); + List<(string Geo, Uri NegotiateUri)> resolved = []; + + foreach (string geo in geos) + { + if (geo is null) + { + throw new ArgumentException("Socket Mode geos cannot contain null.", nameof(geos)); + } + + string segment = geo.Trim().Trim('/'); + if (!seen.Add(segment)) + { + throw new ArgumentException($"Socket Mode geo '{geo}' is listed more than once.", nameof(geos)); + } + + string path = segment.Length > 0 + ? $"/{Uri.EscapeDataString(segment)}{SocketModeProtocol.NegotiatePath}" + : SocketModeProtocol.NegotiatePath; + resolved.Add((segment, new Uri(baseUrl + path, UriKind.Absolute))); + } + + return resolved; + } +} diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeEndToEndTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeEndToEndTests.cs new file mode 100644 index 00000000..eff08935 --- /dev/null +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeEndToEndTests.cs @@ -0,0 +1,287 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Concurrent; +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Teams.Apps.SocketMode; +using Microsoft.Teams.Core.Schema; + +namespace Microsoft.Teams.Apps.UnitTests.SocketMode; + +/// +/// Runs Socket Mode end to end in process: transport, geo supervisors, negotiator and SignalR connection factory are real; only HTTP and the SignalR client are faked. +/// +public class SocketModeEndToEndTests +{ + [Fact] + public async Task Transport_ComposesNegotiatorAndSignalRFactoryAcrossGeos() + { + EndToEnd stack = new(); + + Task start = stack.Transport.StartAsync(); + FakeSignalRClient[] clients = await stack.SignalR.WaitForAsync(3); + foreach (FakeSignalRClient client in clients) + { + client.Ready(); + } + + await start; + + Assert.Equal(SocketModeStatus.Ready, stack.Transport.Status); + Assert.Equal( + [ + "https://botapi.skype.com/amer/v3/websockets/connect", + "https://botapi.skype.com/apac/v3/websockets/connect", + "https://botapi.skype.com/emea/v3/websockets/connect", + ], + stack.Http.Requests.Select(request => request.AbsoluteUri).Order(StringComparer.Ordinal)); + Assert.All(stack.Http.Authorizations, authorization => Assert.Equal("Bearer bot-token", authorization)); + Assert.Equal( + ["https://signalr.test/amer", "https://signalr.test/apac", "https://signalr.test/emea"], + clients.Select(client => client.Url.AbsoluteUri).Order(StringComparer.Ordinal)); + + FakeSignalRClient emea = clients.Single(client => client.Url.AbsolutePath == "/emea"); + SocketReplyFrame? reply = await emea.ActivityAsync(new SocketActivityEnvelope + { + ProtocolVersion = SocketModeProtocol.CurrentVersion, + EnvelopeId = "env-1", + Type = "invoke", + Payload = JsonDocument.Parse("""{"type":"invoke","id":"a1"}""").RootElement.Clone(), + }); + + Assert.NotNull(reply); + Assert.Equal("env-1", reply.EnvelopeId); + Assert.Equal("bot-id", reply.BotKey); + Assert.Equal(200, reply.Status); + Assert.Equal("a1", Assert.Single(stack.Dispatched).Id); + + await stack.Transport.StopAsync(); + + Assert.Equal(SocketModeStatus.Stopped, stack.Transport.Status); + Assert.All(clients, client => + { + Assert.Equal(1, client.StopCount); + Assert.Equal(1, client.DisposeCount); + }); + } + + [Fact] + public async Task Transport_HonorsNegotiatorRetryAfterDuringStartup() + { + EndToEnd stack = new(new SocketModeTransportOptions { Geos = ["amer"] }); + stack.Http.ThrottleOnce(TimeSpan.Zero); + + Task start = stack.Transport.StartAsync(); + FakeSignalRClient client = (await stack.SignalR.WaitForAsync(1))[0]; + client.Ready(); + await start; + + Assert.Equal(2, stack.Http.Requests.Count); + Assert.Equal(SocketModeStatus.Ready, stack.Transport.Status); + + await stack.Transport.DisposeAsync(); + } + + [Fact] + public async Task Transport_FailsStartupWithNegotiatorErrorWhenBudgetIsExhausted() + { + EndToEnd stack = new(new SocketModeTransportOptions { Geos = ["amer"], StartupTimeout = TimeSpan.Zero }); + stack.Http.ThrottleOnce(TimeSpan.Zero); + + SocketModeNegotiateException error = + await Assert.ThrowsAsync(() => stack.Transport.StartAsync()); + + Assert.Equal(HttpStatusCode.TooManyRequests, error.StatusCode); + Assert.Equal(SocketModeStatus.Stopped, stack.Transport.Status); + Assert.Empty(stack.SignalR.Clients); + } + + private sealed class EndToEnd + { + internal EndToEnd(SocketModeTransportOptions? options = null) + { + SocketModeNegotiator negotiator = new( + new HttpClient(Http), + _ => Task.FromResult("bot-token")); + SignalRSocketConnectionFactory factory = new( + negotiator, + TimeSpan.FromSeconds(30), + TimeSpan.FromSeconds(15), + TimeSpan.FromSeconds(30), + NullLogger.Instance, + SignalR.Create); + Transport = new SocketModeTransport( + options ?? new SocketModeTransportOptions(), + factory, + activity => + { + Dispatched.Enqueue(activity); + return Task.FromResult(new SocketDispatchResult(200, null)); + }, + NullLogger.Instance, + botKey: "bot-id"); + } + + internal NegotiateHandler Http { get; } = new(); + internal FakeSignalRClientFactory SignalR { get; } = new(); + internal ConcurrentQueue Dispatched { get; } = new(); + internal SocketModeTransport Transport { get; } + } + + private sealed class NegotiateHandler : HttpMessageHandler + { + private readonly object _gate = new(); + private TimeSpan? _throttle; + + internal ConcurrentQueue Requests { get; } = new(); + internal ConcurrentQueue Authorizations { get; } = new(); + + internal void ThrottleOnce(TimeSpan retryAfter) + { + lock (_gate) + { + _throttle = retryAfter; + } + } + + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + Uri uri = request.RequestUri ?? throw new InvalidOperationException("Missing request URI."); + Requests.Enqueue(uri); + Authorizations.Enqueue(request.Headers.Authorization?.ToString()); + + TimeSpan? throttle; + lock (_gate) + { + throttle = _throttle; + _throttle = null; + } + + if (throttle is TimeSpan retryAfter) + { + HttpResponseMessage throttled = new(HttpStatusCode.TooManyRequests); + throttled.Headers.RetryAfter = new RetryConditionHeaderValue(retryAfter); + return Task.FromResult(throttled); + } + + string geo = uri.Segments[1].TrimEnd('/'); + string json = $$"""{"url":"https://signalr.test/{{geo}}","accessToken":"token-{{geo}}","expiresIn":3600}"""; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(json, Encoding.UTF8, "application/json"), + }); + } + } + + private sealed class FakeSignalRClientFactory + { + private readonly object _gate = new(); + private TaskCompletionSource _changed = NewSignal(); + + internal List Clients { get; } = []; + + internal ISignalRClientConnection Create( + Uri url, + string accessToken, + TimeSpan keepAliveInterval, + TimeSpan serverTimeout) + { + FakeSignalRClient client = new(url, Changed); + lock (_gate) + { + Clients.Add(client); + } + + Changed(); + return client; + } + + internal async Task WaitForAsync(int count) + { + using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(10)); + while (true) + { + Task changed; + lock (_gate) + { + if (Clients.Count >= count && Clients.Take(count).All(client => client.StartCount > 0)) + { + return [.. Clients.Take(count)]; + } + + changed = _changed.Task; + } + + await changed.WaitAsync(timeout.Token); + } + } + + private void Changed() + { + TaskCompletionSource previous; + lock (_gate) + { + previous = _changed; + _changed = NewSignal(); + } + + previous.TrySetResult(); + } + + private static TaskCompletionSource NewSignal() => new(TaskCreationOptions.RunContinuationsAsynchronously); + } + + private sealed class FakeSignalRClient(Uri url, Action changed) : ISignalRClientConnection + { + private Func>? _onActivity; + private Action? _onReady; + private Action? _onClosed; + private int _startCount; + private int _stopCount; + private int _disposeCount; + + internal Uri Url { get; } = url; + internal int StartCount => Volatile.Read(ref _startCount); + internal int StopCount => Volatile.Read(ref _stopCount); + internal int DisposeCount => Volatile.Read(ref _disposeCount); + + public void OnActivity(Func> handler) => _onActivity = handler; + + public void OnReady(Action handler) => _onReady = handler; + + public void OnClosed(Action handler) => _onClosed = handler; + + public Task StartAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Interlocked.Increment(ref _startCount); + changed(); + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) + { + Interlocked.Increment(ref _stopCount); + _onClosed?.Invoke(null); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + Interlocked.Increment(ref _disposeCount); + return ValueTask.CompletedTask; + } + + internal void Ready() + => (_onReady ?? throw new InvalidOperationException())(new SocketReadyFrame { BotKey = "bot-id" }); + + internal Task ActivityAsync(SocketActivityEnvelope envelope) + => (_onActivity ?? throw new InvalidOperationException())(envelope); + } +} diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeTransportTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeTransportTests.cs new file mode 100644 index 00000000..1209653f --- /dev/null +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeTransportTests.cs @@ -0,0 +1,484 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Net; +using System.Text.Json; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Teams.Apps.SocketMode; +using Microsoft.Teams.Core.Schema; + +namespace Microsoft.Teams.Apps.UnitTests.SocketMode; + +public class SocketModeTransportTests +{ + private static readonly DateTimeOffset Now = DateTimeOffset.FromUnixTimeMilliseconds(1_700_000_000_000); + + [Fact] + public async Task StartAsync_ConnectsEveryDefaultGeo() + { + Harness harness = new(); + + Task start = harness.Transport.StartAsync(); + FakeConnection[] connections = await harness.Factory.WaitForAsync(3); + + Assert.Equal( + [ + "https://botapi.skype.com/amer/v3/websockets/connect", + "https://botapi.skype.com/apac/v3/websockets/connect", + "https://botapi.skype.com/emea/v3/websockets/connect", + ], + SortedUris(connections)); + Assert.False(start.IsCompleted); + Assert.Equal(SocketModeStatus.Connecting, harness.Transport.Status); + + foreach (FakeConnection connection in connections) + { + connection.Ready(); + } + + await start; + Assert.Equal(SocketModeStatus.Ready, harness.Transport.Status); + Assert.All(harness.Transport.GeoStatuses.Values, status => Assert.Equal(SocketModeStatus.Ready, status)); + } + + [Fact] + public async Task StartAsync_BuildsGeoUrisFromCustomBaseAndEmptyGeo() + { + Harness harness = new(new SocketModeTransportOptions + { + NegotiateBaseUri = new Uri("http://localhost:3978/"), + Geos = ["", " /eu/ "], + }); + + Task start = harness.Transport.StartAsync(); + FakeConnection[] connections = await harness.Factory.WaitForAsync(2); + + Assert.Equal( + [ + "http://localhost:3978/eu/v3/websockets/connect", + "http://localhost:3978/v3/websockets/connect", + ], + SortedUris(connections)); + foreach (FakeConnection connection in connections) + { + connection.Ready(); + } + + await start; + } + + [Theory] + [InlineData(new object[] { new string[0] })] + [InlineData(new object[] { new[] { "amer", "AMER" } })] + public void Constructor_RejectsInvalidGeos(string[] geos) + { + Assert.Throws(() => new Harness(new SocketModeTransportOptions { Geos = geos }).Transport); + } + + [Fact] + public void Constructor_RejectsNegativeStartupTimeout() + { + Assert.Throws(() => new Harness(new SocketModeTransportOptions + { + StartupTimeout = TimeSpan.FromSeconds(-1), + }).Transport); + } + + [Fact] + public async Task StartAsync_WhenAnyGeoFails_StopsEveryGeoAndThrowsThatFailure() + { + IOException failure = new("emea failed"); + Harness harness = new(new SocketModeTransportOptions { StartupTimeout = TimeSpan.Zero }); + harness.Factory.FailStart("emea", failure); + + IOException thrown = await Assert.ThrowsAsync(() => harness.Transport.StartAsync()); + + Assert.Same(failure, thrown); + Assert.Equal(SocketModeStatus.Stopped, harness.Transport.Status); + Assert.All(harness.Factory.Connections, connection => Assert.Equal(1, connection.DisposeCount)); + } + + [Fact] + public async Task StopAsync_DuringStartup_CancelsStartAndDisposesConnections() + { + Harness harness = new(); + Task start = harness.Transport.StartAsync(); + FakeConnection[] connections = await harness.Factory.WaitForAsync(3); + + await harness.Transport.StopAsync(); + + await Assert.ThrowsAnyAsync(() => start); + Assert.All(connections, connection => Assert.Equal(1, connection.DisposeCount)); + Assert.Equal(SocketModeStatus.Stopped, harness.Transport.Status); + } + + [Fact] + public async Task StopAsync_IsIdempotentAndDisposesEachConnectionOnce() + { + Harness harness = new(); + FakeConnection[] connections = await harness.StartReadyAsync(); + + await harness.Transport.StopAsync(); + await harness.Transport.StopAsync(); + await harness.Transport.DisposeAsync(); + + Assert.All(connections, connection => + { + Assert.Equal(1, connection.StopCount); + Assert.Equal(1, connection.DisposeCount); + }); + Assert.Equal(SocketModeStatus.Stopped, harness.Transport.Status); + await Assert.ThrowsAsync(() => harness.Transport.StartAsync()); + } + + [Fact] + public async Task Status_ReportsDisconnectedUntilDroppedGeoRecovers() + { + Harness harness = new(new SocketModeTransportOptions { ReconnectDelays = [TimeSpan.Zero] }); + FakeConnection[] connections = await harness.StartReadyAsync(); + FakeConnection amer = connections.Single(connection => connection.Geo == "amer"); + + amer.Close(new IOException("dropped")); + + Assert.Equal(SocketModeStatus.Disconnected, harness.Transport.Status); + Assert.Equal(SocketModeStatus.Disconnected, harness.Transport.GeoStatuses["amer"]); + Assert.Equal(SocketModeStatus.Ready, harness.Transport.GeoStatuses["emea"]); + + FakeConnection replacement = (await harness.Factory.WaitForAsync(4))[3]; + replacement.Ready(); + await WaitUntilAsync(() => harness.Transport.Status == SocketModeStatus.Ready); + } + + [Fact] + public async Task Dispatch_InvokeReturnsHandlerStatusAndBody() + { + Harness harness = new() { Dispatch = _ => Task.FromResult(new SocketDispatchResult(201, "created")) }; + FakeConnection connection = (await harness.StartReadyAsync())[0]; + + SocketReplyFrame? reply = await connection.ActivityAsync(Envelope("invoke", """{"type":"invoke","id":"a1"}""")); + + Assert.NotNull(reply); + Assert.Equal("env-1", reply.EnvelopeId); + Assert.Equal("bot-id", reply.BotKey); + Assert.Equal(201, reply.Status); + Assert.Equal("created", reply.Body); + Assert.Equal(Now.ToUnixTimeMilliseconds(), reply.ReceivedAtUnixMilliseconds); + Assert.Equal("a1", Assert.Single(harness.Dispatched).Id); + } + + [Fact] + public async Task Dispatch_MessageReturnsAcknowledgementWithHandlerStatus() + { + Harness harness = new() { Dispatch = _ => Task.FromResult(new SocketDispatchResult(202, "ignored")) }; + FakeConnection connection = (await harness.StartReadyAsync())[0]; + + SocketReplyFrame? reply = await connection.ActivityAsync(Envelope("message", """{"type":"message"}""")); + + Assert.NotNull(reply); + Assert.Equal(202, reply.Status); + Assert.Null(reply.Body); + } + + [Fact] + public async Task Dispatch_ClassifiesInvokeFromActivityWhenEnvelopeTypeIsAbsent() + { + Harness harness = new() { Dispatch = _ => Task.FromResult(new SocketDispatchResult(200, "result")) }; + FakeConnection connection = (await harness.StartReadyAsync())[0]; + + SocketReplyFrame? reply = await connection.ActivityAsync(Envelope(null, """{"type":"invoke"}""")); + + Assert.Equal("result", reply?.Body); + } + + [Theory] + [InlineData("invoke", """{"type":"invoke"}""", true)] + [InlineData("message", """{"type":"message"}""", false)] + public async Task Dispatch_HandlerFailureReturns500AndReportsError(string type, string payload, bool hasBody) + { + InvalidOperationException failure = new("handler failed"); + List reported = []; + Harness harness = new() + { + Dispatch = _ => throw failure, + OnError = exception => + { + reported.Add(exception); + return Task.CompletedTask; + }, + }; + FakeConnection connection = (await harness.StartReadyAsync())[0]; + + SocketReplyFrame? reply = await connection.ActivityAsync(Envelope(type, payload)); + + Assert.NotNull(reply); + Assert.Equal(500, reply.Status); + Assert.Equal(hasBody, reply.Body is not null); + Assert.Same(failure, Assert.Single(reported)); + } + + [Fact] + public async Task Dispatch_FailingErrorObserverStillReturns500() + { + Harness harness = new() + { + Dispatch = _ => Task.FromException(new InvalidOperationException("handler failed")), + OnError = _ => throw new InvalidOperationException("observer failed"), + }; + FakeConnection connection = (await harness.StartReadyAsync())[0]; + + SocketReplyFrame? reply = await connection.ActivityAsync(Envelope("invoke", """{"type":"invoke"}""")); + + Assert.Equal(500, reply?.Status); + } + + [Fact] + public async Task Dispatch_RejectsUnsupportedProtocolVersionWithoutDispatching() + { + Harness harness = new(); + FakeConnection connection = (await harness.StartReadyAsync())[0]; + + SocketReplyFrame? reply = await connection.ActivityAsync( + Envelope("message", """{"type":"message"}""", protocolVersion: SocketModeProtocol.CurrentVersion + 1)); + + Assert.Equal(400, reply?.Status); + Assert.Empty(harness.Dispatched); + } + + [Fact] + public async Task Dispatch_DropsEnvelopeWithoutActivity() + { + Harness harness = new(); + FakeConnection connection = (await harness.StartReadyAsync())[0]; + + SocketReplyFrame? reply = await connection.ActivityAsync(Envelope("message", """{"text":"no type"}""")); + + Assert.Null(reply); + Assert.Empty(harness.Dispatched); + } + + [Fact] + public void RetryPolicy_UsesScheduleThenRepeatsLastDelay() + { + IGeoSocketOwner owner = new Harness(new SocketModeTransportOptions + { + ReconnectDelays = [TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(4)], + }).Transport; + + Assert.Equal(TimeSpan.FromSeconds(1), owner.GetBackoffDelay(0)); + Assert.Equal(TimeSpan.FromSeconds(4), owner.GetBackoffDelay(1)); + Assert.Equal(TimeSpan.FromSeconds(4), owner.GetBackoffDelay(9)); + } + + [Fact] + public void RetryPolicy_JittersCappedExponentialBackoff() + { + IGeoSocketOwner owner = new Harness { Random = new FixedRandom(0.5) }.Transport; + + Assert.Equal(TimeSpan.FromSeconds(0.5), owner.GetBackoffDelay(0)); + Assert.Equal(TimeSpan.FromSeconds(2), owner.GetBackoffDelay(2)); + Assert.Equal(TimeSpan.FromSeconds(7.5), owner.GetBackoffDelay(100)); + } + + [Fact] + public void RetryPolicy_ReadsRetryAfterFromNegotiateFailure() + { + IGeoSocketOwner owner = new Harness().Transport; + + Assert.Equal( + TimeSpan.FromSeconds(7), + owner.GetRetryAfter(new SocketModeNegotiateException(HttpStatusCode.TooManyRequests, TimeSpan.FromSeconds(7)))); + Assert.Null(owner.GetRetryAfter(new IOException())); + Assert.Null(owner.GetRetryAfter(null)); + } + + private static SocketActivityEnvelope Envelope(string? type, string payload, int protocolVersion = 1) + => new() + { + ProtocolVersion = protocolVersion, + EnvelopeId = "env-1", + Type = type, + Payload = JsonDocument.Parse(payload).RootElement.Clone(), + }; + + private static string[] SortedUris(FakeConnection[] connections) + => [.. connections.Select(connection => connection.NegotiateUri.AbsoluteUri).Order(StringComparer.Ordinal)]; + + private static async Task WaitUntilAsync(Func condition) + { + using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(5)); + while (!condition()) + { + await Task.Delay(1, timeout.Token); + } + } + + private sealed class Harness + { + private readonly SocketModeTransportOptions _options; + private SocketModeTransport? _transport; + + internal Harness(SocketModeTransportOptions? options = null) + { + _options = options ?? new SocketModeTransportOptions(); + } + + internal FakeConnectionFactory Factory { get; } = new(); + + internal List Dispatched { get; } = []; + + internal Func> Dispatch { get; init; } = + _ => Task.FromResult(new SocketDispatchResult(200)); + + internal Func? OnError { get; init; } + + internal Random? Random { get; init; } + + internal SocketModeTransport Transport => _transport ??= new SocketModeTransport( + _options, + Factory, + activity => + { + lock (Dispatched) + { + Dispatched.Add(activity); + } + + return Dispatch(activity); + }, + NullLogger.Instance, + "bot-id", + OnError, + new FixedTimeProvider(), + Random); + + internal async Task StartReadyAsync() + { + Task start = Transport.StartAsync(); + FakeConnection[] connections = await Factory.WaitForAsync(_options.Geos.Count); + foreach (FakeConnection connection in connections) + { + connection.Ready(); + } + + await start; + return connections; + } + } + + private sealed class FakeConnectionFactory : ISocketConnectionFactory + { + private readonly Dictionary _startFailures = []; + private readonly List _connections = []; + + internal FakeConnection[] Connections + { + get + { + lock (_connections) + { + return [.. _connections]; + } + } + } + + internal void FailStart(string geo, Exception error) => _startFailures[geo] = error; + + internal async Task WaitForAsync(int count) + { + await WaitUntilAsync(() => Connections.Length >= count); + FakeConnection[] connections = Connections; + await Task.WhenAll(connections.Select(connection => connection.Started.Task)); + return connections; + } + + public ISocketConnection Create(Uri negotiateUri, SocketConnectionHandlers handlers) + { + string[] segments = negotiateUri.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries); + string geo = segments.Length > 3 ? segments[0] : string.Empty; + FakeConnection connection = new(negotiateUri, geo, handlers, _startFailures.GetValueOrDefault(geo)); + lock (_connections) + { + _connections.Add(connection); + } + + return connection; + } + } + + private sealed class FakeConnection( + Uri negotiateUri, + string geo, + SocketConnectionHandlers handlers, + Exception? startError) : ISocketConnection + { + private readonly TaskCompletionSource _ready = new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _closed; + private int _stopCount; + private int _disposeCount; + + public TimeSpan? TokenLifetime => null; + + internal Uri NegotiateUri { get; } = negotiateUri; + + internal string Geo { get; } = geo; + + internal TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + internal int StopCount => Volatile.Read(ref _stopCount); + + internal int DisposeCount => Volatile.Read(ref _disposeCount); + + public async Task StartAsync(CancellationToken cancellationToken) + { + Started.TrySetResult(); + if (startError is not null) + { + throw startError; + } + + await _ready.Task.WaitAsync(cancellationToken); + } + + public Task StopAsync(CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref _stopCount); + _ready.TrySetCanceled(CancellationToken.None); + RaiseClosed(null, planned: true); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + Interlocked.Increment(ref _disposeCount); + return ValueTask.CompletedTask; + } + + internal void Ready() + { + handlers.OnReady(new SocketReadyFrame { ConnectionId = NegotiateUri.AbsolutePath }); + _ready.TrySetResult(); + } + + internal void Close(Exception error) => RaiseClosed(error, planned: false); + + internal Task ActivityAsync(SocketActivityEnvelope envelope) => handlers.OnActivity(envelope); + + private void RaiseClosed(Exception? error, bool planned) + { + if (Interlocked.Exchange(ref _closed, 1) == 0) + { + handlers.OnClosed(error, planned); + } + } + } + + private sealed class FixedTimeProvider : TimeProvider + { + public override DateTimeOffset GetUtcNow() => Now; + } + + private sealed class FixedRandom(double value) : Random + { + public override double NextDouble() => value; + } +} From f73599e31fb628d818a2eae68ad3a1156d74ba5e Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Wed, 23 Sep 2026 22:00:51 -0700 Subject: [PATCH 23/31] feat(apps): add per-geo Socket Mode supervisor Adds an internal GeoSocket that keeps one geo connected: initial startup within a monotonic budget, Retry-After/backoff retries, reconnect after unexpected closure, and make-before-break token rotation with a retiring handoff window. Dispatch is fenced to the active or retiring generation, and every connection is stopped and disposed exactly once. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SocketMode/GeoSocket.cs | 629 +++++++++++++++++ .../SocketMode/GeoSocketTests.cs | 631 ++++++++++++++++++ 2 files changed, 1260 insertions(+) create mode 100644 src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs create mode 100644 test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs diff --git a/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs b/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs new file mode 100644 index 00000000..b7bb8314 --- /dev/null +++ b/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs @@ -0,0 +1,629 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Teams.Apps.SocketMode; + +/// +/// Provides the retry policy, dispatch, and lifecycle reporting a needs from its transport. +/// +internal interface IGeoSocketOwner +{ + /// + /// Gets the time allowed for the initial connection to become ready. Zero permits a single unbounded attempt. + /// + TimeSpan StartupTimeout { get; } + + /// + /// Gets how long before token expiry a replacement connection is established. + /// + TimeSpan TokenRefreshMargin { get; } + + /// + /// Gets how long a replaced connection keeps dispatching after its successor is ready. + /// + TimeSpan HandoffWindow { get; } + + /// + /// Gets the server-requested retry delay for a failure, when one was provided. + /// + /// The failure that caused the retry. + TimeSpan? GetRetryAfter(Exception? error); + + /// + /// Gets the backoff delay for a zero-based retry attempt. + /// + /// The zero-based retry attempt. + TimeSpan GetBackoffDelay(int attempt); + + /// + /// Dispatches an activity received on a live connection for a geo. + /// + /// The geo that received the activity. + /// The received activity envelope. + Task DispatchAsync(string geo, SocketActivityEnvelope envelope); + + /// + /// Reports that a connection generation for a geo received SocketReady. + /// + /// The geo whose connection became ready. + /// The ready frame. + void OnGeoReady(string geo, SocketReadyFrame frame); + + /// + /// Reports that inbound delivery for a geo was unexpectedly interrupted. + /// + /// The geo that disconnected. + /// The failure that closed the connection, when available. + void OnGeoDisconnected(string geo, Exception? error); + + /// + /// Reports that inbound delivery for a geo resumed after an unexpected interruption. + /// + /// The geo that reconnected. + void OnGeoReconnected(string geo); +} + +/// +/// Keeps one geo connected: establishes the initial connection within a startup budget, reconnects after +/// unexpected closure, and rotates tokens make-before-break. +/// +/// +/// Each connection attempt is a new generation. Activities are dispatched only from the active ready generation +/// or a predecessor that is explicitly retiring during a token rotation handoff. +/// +internal sealed class GeoSocket : IAsyncDisposable +{ + private static readonly TimeSpan MinimumRefreshDelay = TimeSpan.FromSeconds(1); + + private readonly IGeoSocketOwner _owner; + private readonly Uri _negotiateUri; + private readonly ISocketConnectionFactory _connectionFactory; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly CancellationTokenSource _stopSource = new(); + private readonly CancellationToken _stopToken; + private readonly object _sync = new(); + private readonly HashSet _owned = []; + private readonly HashSet _retiring = []; + private readonly HashSet _retirements = []; + + private long _generation; + private Generation? _active; + private ITimer? _refreshTimer; + private Task? _supervisor; + private Task? _stopTask; + private bool _disconnected; + private bool _stopping; + private int _started; + private int _disposed; + + /// + /// Initializes a supervisor for one geo. + /// + /// The transport that owns this geo. + /// The geo identifier. + /// The negotiate endpoint for the geo. + /// Creates one connection per generation. + /// The logger for lifecycle events. + /// The time provider for startup budgets, retries, rotation, and handoff. + internal GeoSocket( + IGeoSocketOwner owner, + string geo, + Uri negotiateUri, + ISocketConnectionFactory connectionFactory, + ILogger logger, + TimeProvider? timeProvider = null) + { + _owner = owner ?? throw new ArgumentNullException(nameof(owner)); + ArgumentException.ThrowIfNullOrWhiteSpace(geo); + Geo = geo; + _negotiateUri = negotiateUri ?? throw new ArgumentNullException(nameof(negotiateUri)); + _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; + _stopToken = _stopSource.Token; + } + + /// + /// Gets the geo identifier. + /// + internal string Geo { get; } + + /// + /// Establishes the initial ready connection, then supervises it in the background. + /// + /// A token for cancelling startup. + /// Thrown when no generation becomes ready within the startup budget. + internal async Task StartAsync(CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); + if (Interlocked.Exchange(ref _started, 1) != 0) + { + throw new InvalidOperationException("A GeoSocket can only be started once."); + } + + Generation initial = await ConnectInitialAsync(cancellationToken).ConfigureAwait(false); + + lock (_sync) + { + if (!_stopping) + { + _supervisor = SuperviseAsync(initial); + return; + } + } + + throw new OperationCanceledException("Socket Mode geo stopped before startup completed."); + } + + /// + /// Stops supervision and stops and disposes every connection this geo owns. + /// + internal Task StopAsync() + { + lock (_sync) + { + _stopping = true; + return _stopTask ??= StopCoreAsync(); + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + try + { + await StopAsync().ConfigureAwait(false); + } + finally + { + _stopSource.Dispose(); + } + } + + private async Task ConnectInitialAsync(CancellationToken cancellationToken) + { + using CancellationTokenSource startSource = + CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _stopToken); + TimeSpan budget = _owner.StartupTimeout; + long startedAt = _timeProvider.GetTimestamp(); + + for (int attempt = 0; ; attempt++) + { + using CancellationTokenSource deadlineSource = new(Timeout.InfiniteTimeSpan, _timeProvider); + if (budget > TimeSpan.Zero) + { + deadlineSource.CancelAfter(RemainingBudget(budget, startedAt)); + } + + using CancellationTokenSource attemptSource = + CancellationTokenSource.CreateLinkedTokenSource(startSource.Token, deadlineSource.Token); + + try + { + return await ConnectAsync(attemptSource.Token).ConfigureAwait(false); + } + catch (Exception exception) when (!startSource.IsCancellationRequested) + { + Exception error = deadlineSource.IsCancellationRequested + ? new TimeoutException( + $"Socket Mode geo '{Geo}' did not become ready within {budget}.", + exception) + : exception; + TimeSpan delay = GetRetryDelay(error, attempt); + + if (budget <= TimeSpan.Zero || delay >= RemainingBudget(budget, startedAt)) + { + if (ReferenceEquals(error, exception)) + { + throw; + } + + throw error; + } + + _logger.LogWarning( + exception, + "Socket Mode geo {Geo} initial connection failed; retrying in {Delay}.", + Geo, + delay); + await Task.Delay(delay, _timeProvider, startSource.Token).ConfigureAwait(false); + } + } + } + + private async Task SuperviseAsync(Generation current) + { + try + { + while (true) + { + CloseReason reason = await current.Closed.Task.WaitAsync(_stopToken).ConfigureAwait(false); + + if (reason.Planned) + { + _logger.LogInformation("Socket Mode geo {Geo} rotating connection before token expiry.", Geo); + Generation replacement = await ReconnectAsync(null, delayFirstAttempt: false).ConfigureAwait(false); + StartRetirement(current); + current = replacement; + } + else + { + await ReleaseAsync(current.Connection).ConfigureAwait(false); + current = await ReconnectAsync(reason.Error, delayFirstAttempt: true).ConfigureAwait(false); + } + + ReportReconnected(); + } + } + catch (OperationCanceledException) when (_stopToken.IsCancellationRequested) + { + } + catch (Exception exception) + { + _logger.LogError(exception, "Socket Mode geo {Geo} supervisor stopped unexpectedly.", Geo); + throw; + } + } + + private async Task ReconnectAsync(Exception? previousError, bool delayFirstAttempt) + { + Exception? error = previousError; + int retry = 0; + + for (int attempt = 1; ; attempt++) + { + if (delayFirstAttempt || attempt > 1) + { + await Task.Delay(GetRetryDelay(error, retry++), _timeProvider, _stopToken).ConfigureAwait(false); + } + + try + { + return await ConnectAsync(_stopToken).ConfigureAwait(false); + } + catch (Exception exception) when (!_stopToken.IsCancellationRequested) + { + error = exception; + _logger.LogWarning( + exception, + "Socket Mode geo {Geo} reconnect attempt {Attempt} failed.", + Geo, + attempt); + } + } + } + + private async Task ConnectAsync(CancellationToken cancellationToken) + { + Generation generation; + lock (_sync) + { + ThrowIfStopping(); + generation = new Generation(++_generation); + } + + generation.Connection = _connectionFactory.Create( + _negotiateUri, + new SocketConnectionHandlers( + envelope => DispatchAsync(generation, envelope), + frame => HandleReady(generation, frame), + (error, planned) => HandleClosed(generation, error, planned))); + + bool owned; + lock (_sync) + { + owned = !_stopping && _owned.Add(generation.Connection); + } + + if (!owned) + { + await StopAndDisposeAsync(generation.Connection).ConfigureAwait(false); + throw new OperationCanceledException("Socket Mode geo is stopping."); + } + + try + { + await generation.Connection.StartAsync(cancellationToken).ConfigureAwait(false); + + // StartAsync completes after SocketReady, but the ready callback may still be running. + if (!TryPromote(generation)) + { + lock (_sync) + { + ThrowIfStopping(); + } + + throw new IOException("Socket Mode connection closed before it became active."); + } + } + catch + { + await ReleaseAsync(generation.Connection).ConfigureAwait(false); + throw; + } + + ScheduleRefresh(generation); + return generation; + } + + private Task DispatchAsync(Generation generation, SocketActivityEnvelope envelope) + { + lock (_sync) + { + if (_stopping || (_active != generation && !_retiring.Contains(generation.Id))) + { + return Task.FromResult(null); + } + } + + return _owner.DispatchAsync(Geo, envelope); + } + + private void HandleReady(Generation generation, SocketReadyFrame frame) + { + if (TryPromote(generation)) + { + _owner.OnGeoReady(Geo, frame); + } + } + + private bool TryPromote(Generation generation) + { + lock (_sync) + { + if (_stopping || generation.IsClosed || generation.Id != _generation) + { + return false; + } + + if (_active is Generation previous && previous != generation) + { + _retiring.Add(previous.Id); + } + + _active = generation; + return true; + } + } + + private void HandleClosed(Generation generation, Exception? error, bool planned) + { + bool disconnected = false; + + lock (_sync) + { + generation.IsClosed = true; + _retiring.Remove(generation.Id); + + if (_active == generation) + { + _active = null; + _refreshTimer?.Dispose(); + _refreshTimer = null; + disconnected = !planned && !_stopping; + _disconnected |= disconnected; + } + + generation.Closed.TrySetResult(new CloseReason(error, planned)); + } + + if (disconnected) + { + _logger.LogWarning(error, "Socket Mode geo {Geo} disconnected; inbound delivery paused.", Geo); + _owner.OnGeoDisconnected(Geo, error); + } + } + + private void ReportReconnected() + { + lock (_sync) + { + if (!_disconnected || _stopping) + { + return; + } + + _disconnected = false; + } + + _logger.LogInformation("Socket Mode geo {Geo} reconnected; inbound delivery resumed.", Geo); + _owner.OnGeoReconnected(Geo); + } + + private void ScheduleRefresh(Generation generation) + { + if (generation.Connection.TokenLifetime is not TimeSpan lifetime || lifetime <= TimeSpan.Zero) + { + return; + } + + TimeSpan delay = lifetime - _owner.TokenRefreshMargin; + if (delay < MinimumRefreshDelay) + { + delay = MinimumRefreshDelay; + } + + lock (_sync) + { + if (_stopping || _active != generation) + { + return; + } + + _refreshTimer?.Dispose(); + _refreshTimer = _timeProvider.CreateTimer( + _ => RequestRotation(generation), + null, + delay, + Timeout.InfiniteTimeSpan); + } + } + + private void RequestRotation(Generation generation) + { + lock (_sync) + { + if (!_stopping && _active == generation) + { + generation.Closed.TrySetResult(new CloseReason(null, Planned: true)); + } + } + } + + private void StartRetirement(Generation previous) + { + Task retirement = RetireAsync(previous); + lock (_sync) + { + if (!retirement.IsCompleted) + { + _retirements.Add(retirement); + } + } + + _ = retirement.ContinueWith( + completed => + { + lock (_sync) + { + _retirements.Remove(completed); + } + }, + CancellationToken.None, + TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); + } + + [SuppressMessage( + "Design", + "CA1031:Do not catch general exception types", + Justification = "Retirement runs in the background; failures are logged so they cannot fault the supervisor.")] + private async Task RetireAsync(Generation previous) + { + try + { + TimeSpan handoff = _owner.HandoffWindow; + await Task.Delay( + handoff > TimeSpan.Zero ? handoff : TimeSpan.Zero, + _timeProvider, + _stopToken).ConfigureAwait(false); + + lock (_sync) + { + _retiring.Remove(previous.Id); + } + + await ReleaseAsync(previous.Connection).ConfigureAwait(false); + } + catch (OperationCanceledException) when (_stopToken.IsCancellationRequested) + { + } + catch (Exception exception) + { + _logger.LogWarning(exception, "Socket Mode geo {Geo} failed to retire a replaced connection.", Geo); + } + } + + private async Task StopCoreAsync() + { + // Leave the caller's lock before cancelling and stopping connections. + await Task.Yield(); + await _stopSource.CancelAsync().ConfigureAwait(false); + + ISocketConnection[] connections; + Task[] retirements; + Task? supervisor; + lock (_sync) + { + _active = null; + _retiring.Clear(); + _refreshTimer?.Dispose(); + _refreshTimer = null; + connections = [.. _owned]; + _owned.Clear(); + retirements = [.. _retirements]; + supervisor = _supervisor; + } + + try + { + await Task.WhenAll(connections.Select(StopAndDisposeAsync)).ConfigureAwait(false); + } + finally + { + await Task.WhenAll(retirements).ConfigureAwait(false); + if (supervisor is not null) + { + await supervisor.ConfigureAwait(false); + } + } + } + + private async Task ReleaseAsync(ISocketConnection connection) + { + lock (_sync) + { + if (!_owned.Remove(connection)) + { + return; + } + } + + await StopAndDisposeAsync(connection).ConfigureAwait(false); + } + + private static async Task StopAndDisposeAsync(ISocketConnection connection) + { + try + { + await connection.StopAsync(CancellationToken.None).ConfigureAwait(false); + } + finally + { + await connection.DisposeAsync().ConfigureAwait(false); + } + } + + private TimeSpan GetRetryDelay(Exception? error, int attempt) + { + TimeSpan delay = _owner.GetRetryAfter(error) ?? _owner.GetBackoffDelay(attempt); + return delay > TimeSpan.Zero ? delay : TimeSpan.Zero; + } + + private TimeSpan RemainingBudget(TimeSpan budget, long startedAt) + { + TimeSpan remaining = budget - _timeProvider.GetElapsedTime(startedAt); + return remaining > TimeSpan.Zero ? remaining : TimeSpan.Zero; + } + + private void ThrowIfStopping() + { + if (_stopping) + { + throw new OperationCanceledException("Socket Mode geo is stopping."); + } + } + + private sealed class Generation(long id) + { + internal long Id { get; } = id; + + internal ISocketConnection Connection { get; set; } = null!; + + internal TaskCompletionSource Closed { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + internal bool IsClosed { get; set; } + } + + private sealed record CloseReason(Exception? Error, bool Planned); +} diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs new file mode 100644 index 00000000..a5899e9e --- /dev/null +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs @@ -0,0 +1,631 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Diagnostics; +using System.Net; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Teams.Apps.SocketMode; + +namespace Microsoft.Teams.Apps.UnitTests.SocketMode; + +public class GeoSocketTests +{ + private static readonly Uri NegotiateUri = new("https://botapi.skype.com/amer/v3/websockets/connect"); + + [Fact] + public async Task StartAsync_CompletesOnlyAfterSocketReady() + { + Harness harness = new(); + FakeConnection connection = harness.Factory.Enqueue(); + + Task start = harness.Socket.StartAsync(); + await connection.Started.Task; + Assert.False(start.IsCompleted); + + connection.Ready("initial"); + await start; + + Assert.NotNull(await connection.Activity("a1")); + Assert.Equal(["initial"], harness.Owner.ReadyIds); + Assert.Equal(["a1"], harness.Owner.Dispatched); + Assert.Empty(harness.Owner.Disconnections); + } + + [Fact] + public async Task StartAsync_RetriesWithBackoffWithinBudget() + { + Harness harness = new() { Backoff = _ => TimeSpan.FromSeconds(2) }; + FakeConnection failed = harness.Factory.Enqueue(startError: new IOException("first")); + FakeConnection retry = harness.Factory.Enqueue(); + + Task start = harness.Socket.StartAsync(); + await harness.Time.WaitForTimerAsync(TimeSpan.FromSeconds(2)); + harness.Time.Advance(TimeSpan.FromSeconds(2)); + await retry.Started.Task; + retry.Ready("retry"); + await start; + + Assert.Equal(1, failed.StopCount); + Assert.Equal(1, failed.DisposeCount); + Assert.Null(await failed.Activity("stale")); + Assert.Equal(["retry"], harness.Owner.ReadyIds); + } + + [Fact] + public async Task StartAsync_PrefersRetryAfterOverBackoff() + { + Harness harness = new() { Backoff = _ => TimeSpan.FromSeconds(2) }; + harness.Factory.Enqueue(startError: new SocketModeNegotiateException( + HttpStatusCode.TooManyRequests, + TimeSpan.FromSeconds(7))); + FakeConnection retry = harness.Factory.Enqueue(); + + Task start = harness.Socket.StartAsync(); + await harness.Time.WaitForTimerAsync(TimeSpan.FromSeconds(7)); + Assert.False(harness.Time.HasTimerDueIn(TimeSpan.FromSeconds(2))); + + harness.Time.Advance(TimeSpan.FromSeconds(7)); + await retry.Started.Task; + retry.Ready("retry"); + await start; + } + + [Fact] + public async Task StartAsync_FailsWhenPendingAttemptExceedsBudget() + { + Harness harness = new() { StartupTimeout = TimeSpan.FromSeconds(5) }; + FakeConnection connection = harness.Factory.Enqueue(); + + Task start = harness.Socket.StartAsync(); + await connection.Started.Task; + await harness.Time.WaitForTimerAsync(TimeSpan.FromSeconds(5)); + harness.Time.Advance(TimeSpan.FromSeconds(5)); + + await Assert.ThrowsAsync(() => start); + Assert.Equal(1, harness.Factory.CreateCount); + Assert.Equal(1, connection.DisposeCount); + } + + [Fact] + public async Task StartAsync_ZeroBudgetMakesOneAttemptWithoutRetry() + { + Harness harness = new() { StartupTimeout = TimeSpan.Zero }; + harness.Factory.Enqueue(startError: new IOException("failed")); + + IOException error = await Assert.ThrowsAsync(() => harness.Socket.StartAsync()); + + Assert.Equal("failed", error.Message); + Assert.Equal(1, harness.Factory.CreateCount); + } + + [Fact] + public async Task UnexpectedClose_ReportsDisconnectedThenReconnected() + { + Harness harness = new() { Backoff = _ => TimeSpan.FromSeconds(3) }; + FakeConnection initial = harness.Factory.Enqueue(); + FakeConnection replacement = harness.Factory.Enqueue(); + await harness.StartReadyAsync(initial); + + IOException dropped = new("dropped"); + initial.Close(dropped); + + Assert.Same(dropped, Assert.Single(harness.Owner.Disconnections)); + Assert.Null(await initial.Activity("after-close")); + await harness.Time.WaitForTimerAsync(TimeSpan.FromSeconds(3)); + harness.Time.Advance(TimeSpan.FromSeconds(3)); + await replacement.Started.Task; + replacement.Ready("replacement"); + await WaitUntilAsync(() => harness.Owner.Reconnections == 1); + + Assert.Equal(1, initial.DisposeCount); + Assert.NotNull(await replacement.Activity("resumed")); + Assert.DoesNotContain("after-close", harness.Owner.Dispatched); + } + + [Fact] + public async Task Rotation_IsMakeBeforeBreakWithHandoff() + { + Harness harness = new(); + FakeConnection initial = harness.Factory.Enqueue(TimeSpan.FromSeconds(10)); + FakeConnection replacement = harness.Factory.Enqueue(); + await harness.StartReadyAsync(initial); + + harness.Time.Advance(TimeSpan.FromSeconds(5)); + await replacement.Started.Task; + Assert.NotNull(await initial.Activity("during-replacement-startup")); + Assert.Equal(0, initial.StopCount); + + replacement.Ready("replacement"); + await harness.Time.WaitForTimerAsync(harness.Owner.HandoffWindow); + Assert.NotNull(await initial.Activity("retiring")); + Assert.NotNull(await replacement.Activity("active")); + Assert.Equal(0, initial.StopCount); + + harness.Time.Advance(harness.Owner.HandoffWindow); + await initial.Disposed.Task; + + Assert.Equal(1, initial.StopCount); + Assert.Null(await initial.Activity("retired")); + Assert.DoesNotContain("retired", harness.Owner.Dispatched); + Assert.Empty(harness.Owner.Disconnections); + Assert.Equal(0, harness.Owner.Reconnections); + Assert.Equal(2, harness.Owner.ReadyIds.Count); + } + + [Fact] + public async Task Rotation_UsesOneSecondMinimumDelay() + { + Harness harness = new() { TokenRefreshMargin = TimeSpan.FromSeconds(30) }; + FakeConnection initial = harness.Factory.Enqueue(TimeSpan.FromSeconds(10)); + FakeConnection replacement = harness.Factory.Enqueue(); + await harness.StartReadyAsync(initial); + + Assert.True(harness.Time.HasTimerDueIn(TimeSpan.FromSeconds(1))); + harness.Time.Advance(TimeSpan.FromSeconds(1)); + await replacement.Started.Task; + } + + [Fact] + public async Task TokenLifetimeAbsent_DoesNotScheduleRotation() + { + Harness harness = new(); + FakeConnection connection = harness.Factory.Enqueue(tokenLifetime: null); + await harness.StartReadyAsync(connection); + + Assert.Equal(0, harness.Time.ScheduledTimerCount); + harness.Time.Advance(TimeSpan.FromDays(1)); + + Assert.Equal(1, harness.Factory.CreateCount); + } + + [Fact] + public async Task PredecessorDropDuringRotation_ReportsOutageUntilReplacementReady() + { + Harness harness = new(); + FakeConnection initial = harness.Factory.Enqueue(TimeSpan.FromSeconds(10)); + FakeConnection replacement = harness.Factory.Enqueue(); + await harness.StartReadyAsync(initial); + + harness.Time.Advance(TimeSpan.FromSeconds(5)); + await replacement.Started.Task; + initial.Close(new IOException("dropped")); + + Assert.Single(harness.Owner.Disconnections); + Assert.Null(await initial.Activity("dropped")); + + replacement.Ready("replacement"); + await WaitUntilAsync(() => harness.Owner.Reconnections == 1); + await harness.Time.WaitForTimerAsync(harness.Owner.HandoffWindow); + harness.Time.Advance(harness.Owner.HandoffWindow); + await initial.Disposed.Task; + + Assert.Equal(1, initial.DisposeCount); + } + + [Fact] + public async Task RepeatedRotation_ReleasesEveryPredecessorOnce() + { + Harness harness = new() { HandoffWindow = TimeSpan.Zero }; + FakeConnection[] connections = + [ + harness.Factory.Enqueue(TimeSpan.FromSeconds(10)), + harness.Factory.Enqueue(TimeSpan.FromSeconds(10)), + harness.Factory.Enqueue(TimeSpan.FromSeconds(10)), + harness.Factory.Enqueue(TimeSpan.FromSeconds(10)), + ]; + await harness.StartReadyAsync(connections[0]); + + for (int i = 1; i < connections.Length; i++) + { + await harness.Time.WaitForTimerAsync(TimeSpan.FromSeconds(5)); + harness.Time.Advance(TimeSpan.FromSeconds(5)); + await connections[i].Started.Task; + connections[i].Ready($"gen-{i}"); + await connections[i - 1].Disposed.Task; + } + + Assert.All(connections[..^1], connection => Assert.Equal(1, connection.DisposeCount)); + Assert.Equal(0, connections[^1].DisposeCount); + Assert.Empty(harness.Owner.Disconnections); + + await harness.Socket.StopAsync(); + Assert.All(connections, connection => Assert.Equal(1, connection.DisposeCount)); + } + + [Fact] + public async Task StopAsync_DuringStartup_CancelsAndDisposesOnce() + { + Harness harness = new(); + FakeConnection connection = harness.Factory.Enqueue(); + + Task start = harness.Socket.StartAsync(); + await connection.Started.Task; + await harness.Socket.StopAsync(); + + await Assert.ThrowsAnyAsync(() => start); + await harness.Socket.StopAsync(); + await harness.Socket.DisposeAsync(); + await harness.Socket.DisposeAsync(); + + Assert.Equal(1, connection.StopCount); + Assert.Equal(1, connection.DisposeCount); + Assert.Empty(harness.Owner.Disconnections); + } + + [Fact] + public async Task StartAsync_CallerCancellationInterruptsRetryDelay() + { + Harness harness = new() { Backoff = _ => TimeSpan.FromMinutes(1), StartupTimeout = TimeSpan.FromMinutes(5) }; + FakeConnection failed = harness.Factory.Enqueue(startError: new IOException("failed")); + using CancellationTokenSource cancellation = new(); + + Task start = harness.Socket.StartAsync(cancellation.Token); + await harness.Time.WaitForTimerAsync(TimeSpan.FromMinutes(1)); + await cancellation.CancelAsync(); + + await Assert.ThrowsAnyAsync(() => start); + Assert.Equal(1, harness.Factory.CreateCount); + Assert.Equal(1, failed.DisposeCount); + } + + [Fact] + public async Task StopAsync_DuringReconnectDelay_CancelsWithoutNewGeneration() + { + Harness harness = new() { Backoff = _ => TimeSpan.FromMinutes(1) }; + FakeConnection initial = harness.Factory.Enqueue(); + await harness.StartReadyAsync(initial); + + initial.Close(new IOException("dropped")); + await harness.Time.WaitForTimerAsync(TimeSpan.FromMinutes(1)); + await harness.Socket.StopAsync(); + harness.Time.Advance(TimeSpan.FromMinutes(1)); + + Assert.Equal(1, harness.Factory.CreateCount); + Assert.Equal(1, initial.DisposeCount); + Assert.Equal(0, harness.Owner.Reconnections); + } + + [Fact] + public async Task StopAsync_DuringRotation_DisposesActiveAndConnectingOnce() + { + Harness harness = new(); + FakeConnection active = harness.Factory.Enqueue(TimeSpan.FromSeconds(10)); + FakeConnection connecting = harness.Factory.Enqueue(); + await harness.StartReadyAsync(active); + + harness.Time.Advance(TimeSpan.FromSeconds(5)); + await connecting.Started.Task; + await harness.Socket.StopAsync(); + + Assert.Equal(1, active.StopCount); + Assert.Equal(1, active.DisposeCount); + Assert.Equal(1, connecting.StopCount); + Assert.Equal(1, connecting.DisposeCount); + Assert.Empty(harness.Owner.Disconnections); + Assert.Null(await active.Activity("stopped")); + } + + [Fact] + public async Task StopAsync_DuringHandoff_DisposesActiveAndRetiringOnce() + { + Harness harness = new() { HandoffWindow = TimeSpan.FromMinutes(1) }; + FakeConnection retiring = harness.Factory.Enqueue(TimeSpan.FromSeconds(10)); + FakeConnection active = harness.Factory.Enqueue(); + await harness.StartReadyAsync(retiring); + + harness.Time.Advance(TimeSpan.FromSeconds(5)); + await active.Started.Task; + active.Ready("replacement"); + await harness.Time.WaitForTimerAsync(TimeSpan.FromMinutes(1)); + await harness.Socket.StopAsync(); + harness.Time.Advance(TimeSpan.FromMinutes(1)); + + Assert.Equal(1, retiring.StopCount); + Assert.Equal(1, retiring.DisposeCount); + Assert.Equal(1, active.StopCount); + Assert.Equal(1, active.DisposeCount); + Assert.Empty(harness.Owner.Disconnections); + } + + private static async Task WaitUntilAsync(Func condition) + { + Stopwatch elapsed = Stopwatch.StartNew(); + while (!condition()) + { + if (elapsed.Elapsed > TimeSpan.FromSeconds(5)) + { + throw new TimeoutException("The asynchronous test condition was not met."); + } + + await Task.Yield(); + } + } + + private sealed class Harness + { + private GeoSocket? _socket; + + internal ManualTimeProvider Time { get; } = new(); + + internal FakeOwner Owner { get; } = new(); + + internal FakeConnectionFactory Factory { get; } = new(); + + internal TimeSpan StartupTimeout { init => Owner.StartupTimeout = value; } + + internal TimeSpan TokenRefreshMargin { init => Owner.TokenRefreshMargin = value; } + + internal TimeSpan HandoffWindow { init => Owner.HandoffWindow = value; } + + internal Func Backoff { init => Owner.Backoff = value; } + + internal GeoSocket Socket => + _socket ??= new GeoSocket(Owner, "amer", NegotiateUri, Factory, NullLogger.Instance, Time); + + internal async Task StartReadyAsync(FakeConnection connection) + { + Task start = Socket.StartAsync(); + await connection.Started.Task; + connection.Ready("initial"); + await start; + } + } + + private sealed class FakeOwner : IGeoSocketOwner + { + private readonly object _sync = new(); + private readonly List _dispatched = []; + private readonly List _readyIds = []; + private readonly List _disconnections = []; + private int _reconnections; + + public TimeSpan StartupTimeout { get; set; } = TimeSpan.FromSeconds(30); + + public TimeSpan TokenRefreshMargin { get; set; } = TimeSpan.FromSeconds(5); + + public TimeSpan HandoffWindow { get; set; } = TimeSpan.FromSeconds(5); + + internal Func Backoff { get; set; } = _ => TimeSpan.FromSeconds(1); + + internal List Dispatched { get { lock (_sync) { return [.. _dispatched]; } } } + + internal List ReadyIds { get { lock (_sync) { return [.. _readyIds]; } } } + + internal List Disconnections { get { lock (_sync) { return [.. _disconnections]; } } } + + internal int Reconnections => Volatile.Read(ref _reconnections); + + public TimeSpan? GetRetryAfter(Exception? error) + => (error as SocketModeNegotiateException)?.RetryAfter; + + public TimeSpan GetBackoffDelay(int attempt) => Backoff(attempt); + + public Task DispatchAsync(string geo, SocketActivityEnvelope envelope) + { + lock (_sync) + { + _dispatched.Add(envelope.EnvelopeId!); + } + + return Task.FromResult(new SocketReplyFrame { Status = 200 }); + } + + public void OnGeoReady(string geo, SocketReadyFrame frame) + { + lock (_sync) + { + _readyIds.Add(frame.ConnectionId!); + } + } + + public void OnGeoDisconnected(string geo, Exception? error) + { + lock (_sync) + { + _disconnections.Add(error); + } + } + + public void OnGeoReconnected(string geo) => Interlocked.Increment(ref _reconnections); + } + + private sealed class FakeConnectionFactory : ISocketConnectionFactory + { + private readonly Queue _pending = []; + private int _createCount; + + internal int CreateCount => Volatile.Read(ref _createCount); + + internal FakeConnection Enqueue(TimeSpan? tokenLifetime = null, Exception? startError = null) + { + FakeConnection connection = new(tokenLifetime, startError); + lock (_pending) + { + _pending.Enqueue(connection); + } + + return connection; + } + + public ISocketConnection Create(Uri negotiateUri, SocketConnectionHandlers handlers) + { + Interlocked.Increment(ref _createCount); + FakeConnection connection; + lock (_pending) + { + connection = _pending.Dequeue(); + } + + connection.Handlers = handlers; + return connection; + } + } + + private sealed class FakeConnection(TimeSpan? tokenLifetime, Exception? startError) : ISocketConnection + { + private readonly TaskCompletionSource _ready = new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _closed; + private int _stopCount; + private int _disposeCount; + + public TimeSpan? TokenLifetime { get; } = tokenLifetime; + + internal SocketConnectionHandlers Handlers { get; set; } = null!; + + internal TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + internal TaskCompletionSource Disposed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + internal int StopCount => Volatile.Read(ref _stopCount); + + internal int DisposeCount => Volatile.Read(ref _disposeCount); + + public async Task StartAsync(CancellationToken cancellationToken) + { + Started.TrySetResult(); + if (startError is not null) + { + throw startError; + } + + await _ready.Task.WaitAsync(cancellationToken); + } + + public Task StopAsync(CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref _stopCount); + _ready.TrySetCanceled(CancellationToken.None); + RaiseClosed(null, planned: true); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + Interlocked.Increment(ref _disposeCount); + Disposed.TrySetResult(); + return ValueTask.CompletedTask; + } + + internal void Ready(string connectionId) + { + Handlers.OnReady(new SocketReadyFrame { ConnectionId = connectionId }); + _ready.TrySetResult(); + } + + internal void Close(Exception error) => RaiseClosed(error, planned: false); + + internal Task Activity(string envelopeId) + => Handlers.OnActivity(new SocketActivityEnvelope { EnvelopeId = envelopeId }); + + private void RaiseClosed(Exception? error, bool planned) + { + if (Interlocked.Exchange(ref _closed, 1) == 0) + { + Handlers.OnClosed(error, planned); + } + } + } + + private sealed class ManualTimeProvider : TimeProvider + { + private readonly object _sync = new(); + private readonly List _timers = []; + private long _now; + + public override long TimestampFrequency => TimeSpan.TicksPerSecond; + + internal int ScheduledTimerCount + { + get + { + lock (_sync) + { + return _timers.Count(timer => timer.Due is not null); + } + } + } + + public override long GetTimestamp() + { + lock (_sync) + { + return _now; + } + } + + public override DateTimeOffset GetUtcNow() => DateTimeOffset.UnixEpoch.AddTicks(GetTimestamp()); + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + ManualTimer timer = new(this, callback, state); + timer.Change(dueTime, period); + lock (_sync) + { + _timers.Add(timer); + } + + return timer; + } + + internal bool HasTimerDueIn(TimeSpan dueIn) + { + lock (_sync) + { + return _timers.Any(timer => timer.Due == _now + dueIn.Ticks); + } + } + + internal Task WaitForTimerAsync(TimeSpan dueIn) => WaitUntilAsync(() => HasTimerDueIn(dueIn)); + + internal void Advance(TimeSpan amount) + { + List due; + lock (_sync) + { + _now += amount.Ticks; + due = [.. _timers.Where(timer => timer.Due <= _now)]; + foreach (ManualTimer timer in due) + { + timer.Due = null; + } + } + + foreach (ManualTimer timer in due) + { + timer.Fire(); + } + } + + private sealed class ManualTimer(ManualTimeProvider owner, TimerCallback callback, object? state) : ITimer + { + internal long? Due { get; set; } + + public bool Change(TimeSpan dueTime, TimeSpan period) + { + lock (owner._sync) + { + Due = dueTime == Timeout.InfiniteTimeSpan ? null : owner._now + dueTime.Ticks; + } + + return true; + } + + public void Dispose() + { + lock (owner._sync) + { + Due = null; + owner._timers.Remove(this); + } + } + + public ValueTask DisposeAsync() + { + Dispose(); + return ValueTask.CompletedTask; + } + + internal void Fire() => callback(state); + } + } +} From 8953d5e791aa7c905e6a1cef1e123bfecf086459 Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Wed, 23 Sep 2026 22:39:29 -0700 Subject: [PATCH 24/31] feat(apps): add multi-geo Socket Mode transport Resolve per-geo negotiate endpoints, start every geo with fail-fast startup, stop all geos, aggregate status internally, and dispatch envelopes with invoke/ack replies and 500 on handler failure. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SocketMode/GeoSocket.cs | 5 +- .../SocketMode/SocketModeTransport.cs | 476 +++++++++++++++++ .../SocketMode/SocketModeEndToEndTests.cs | 287 +++++++++++ .../SocketMode/SocketModeTransportTests.cs | 484 ++++++++++++++++++ 4 files changed, 1249 insertions(+), 3 deletions(-) create mode 100644 src/Microsoft.Teams.Apps/SocketMode/SocketModeTransport.cs create mode 100644 test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeEndToEndTests.cs create mode 100644 test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeTransportTests.cs diff --git a/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs b/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs index b7bb8314..0a4d343c 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs @@ -104,7 +104,7 @@ internal sealed class GeoSocket : IAsyncDisposable /// Initializes a supervisor for one geo. /// /// The transport that owns this geo. - /// The geo identifier. + /// The geo identifier. Empty when the negotiate URL has no geo segment. /// The negotiate endpoint for the geo. /// Creates one connection per generation. /// The logger for lifecycle events. @@ -118,8 +118,7 @@ internal GeoSocket( TimeProvider? timeProvider = null) { _owner = owner ?? throw new ArgumentNullException(nameof(owner)); - ArgumentException.ThrowIfNullOrWhiteSpace(geo); - Geo = geo; + Geo = geo ?? throw new ArgumentNullException(nameof(geo)); _negotiateUri = negotiateUri ?? throw new ArgumentNullException(nameof(negotiateUri)); _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); diff --git a/src/Microsoft.Teams.Apps/SocketMode/SocketModeTransport.cs b/src/Microsoft.Teams.Apps/SocketMode/SocketModeTransport.cs new file mode 100644 index 00000000..fc043c77 --- /dev/null +++ b/src/Microsoft.Teams.Apps/SocketMode/SocketModeTransport.cs @@ -0,0 +1,476 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Diagnostics.CodeAnalysis; +using System.Runtime.ExceptionServices; +using Microsoft.Extensions.Logging; +using Microsoft.Teams.Apps.Schema; +using Microsoft.Teams.Core.Schema; + +namespace Microsoft.Teams.Apps.SocketMode; + +/// +/// Describes the lifecycle state of a Socket Mode transport or one of its geos. +/// +internal enum SocketModeStatus +{ + /// + /// The transport has not started. + /// + Idle, + + /// + /// A connection is being established. + /// + Connecting, + + /// + /// Every connection is ready to receive activities. + /// + Ready, + + /// + /// A connection dropped unexpectedly and is recovering. + /// + Disconnected, + + /// + /// The transport has stopped. + /// + Stopped, +} + +/// +/// Configures a . +/// +internal sealed class SocketModeTransportOptions +{ + /// + /// Gets the base URL used to negotiate each geo connection. + /// + internal Uri NegotiateBaseUri { get; init; } = new(SocketModeProtocol.DefaultNegotiateBaseUrl); + + /// + /// Gets the geos to connect. An empty string connects to the base URL without a geo segment. + /// + internal IReadOnlyList Geos { get; init; } = SocketModeProtocol.DefaultGeos; + + /// + /// Gets the time each geo has to establish its initial connection. + /// + internal TimeSpan StartupTimeout { get; init; } = TimeSpan.FromSeconds(30); + + /// + /// Gets an explicit reconnect delay schedule. The last delay repeats. When omitted, capped exponential + /// backoff with full jitter is used. + /// + internal IReadOnlyList? ReconnectDelays { get; init; } +} + +/// +/// Connects one per configured geo and routes their activities to the app. +/// +/// +/// Startup is all-or-nothing: every geo must become ready or the transport stops and startup fails. Once +/// started, each geo is supervised independently. +/// +internal sealed class SocketModeTransport : IGeoSocketOwner, IAsyncDisposable +{ + private static readonly TimeSpan TokenRefreshMargin = TimeSpan.FromSeconds(60); + private static readonly TimeSpan HandoffWindow = TimeSpan.FromSeconds(5); + private static readonly TimeSpan ReconnectInitialDelay = TimeSpan.FromSeconds(1); + private static readonly TimeSpan ReconnectMaxDelay = TimeSpan.FromSeconds(15); + + private readonly SocketModeTransportOptions _options; + private readonly IReadOnlyList<(string Geo, Uri NegotiateUri)> _geos; + private readonly ISocketConnectionFactory _connectionFactory; + private readonly Func> _dispatch; + private readonly Func? _onError; + private readonly string? _botKey; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly Random _random; + private readonly object _sync = new(); + private readonly Dictionary _geoStatuses = []; + + private GeoSocket[] _geoSockets = []; + private SocketModeStatus _lifecycle = SocketModeStatus.Idle; + private Task? _stopTask; + + /// + /// Initializes a Socket Mode transport and resolves its geos. + /// + /// The transport options. + /// Creates one connection per geo generation. + /// Processes an inbound activity and returns its status and optional body. + /// The logger for transport lifecycle and dispatch failures. + /// The bot key echoed on reply frames, when available. + /// Observes activity processing failures before the failure reply is returned. + /// The time provider for supervision timing and reply timestamps. + /// The random source for reconnect jitter. + /// Thrown when the geos are empty, null, or duplicated. + /// Thrown when the startup timeout is negative. + internal SocketModeTransport( + SocketModeTransportOptions options, + ISocketConnectionFactory connectionFactory, + Func> dispatch, + ILogger logger, + string? botKey = null, + Func? onError = null, + TimeProvider? timeProvider = null, + Random? random = null) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); + _dispatch = dispatch ?? throw new ArgumentNullException(nameof(dispatch)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _botKey = botKey; + _onError = onError; + _timeProvider = timeProvider ?? TimeProvider.System; + _random = random ?? Random.Shared; + + ArgumentNullException.ThrowIfNull(options.NegotiateBaseUri, nameof(options)); + ArgumentOutOfRangeException.ThrowIfLessThan(options.StartupTimeout, TimeSpan.Zero, nameof(options)); + _geos = ResolveGeos(options.NegotiateBaseUri, options.Geos); + } + + /// + /// Gets the aggregate status: ready when every geo is ready, disconnected when any geo is recovering from a + /// drop, and connecting otherwise. + /// + internal SocketModeStatus Status + { + get + { + lock (_sync) + { + if (_lifecycle is SocketModeStatus.Idle or SocketModeStatus.Stopped) + { + return _lifecycle; + } + + if (_geoStatuses.Count > 0 && _geoStatuses.Values.All(status => status == SocketModeStatus.Ready)) + { + return SocketModeStatus.Ready; + } + + return _geoStatuses.ContainsValue(SocketModeStatus.Disconnected) + ? SocketModeStatus.Disconnected + : SocketModeStatus.Connecting; + } + } + } + + /// + /// Gets a snapshot of each geo's status. + /// + internal IReadOnlyDictionary GeoStatuses + { + get + { + lock (_sync) + { + return new Dictionary(_geoStatuses); + } + } + } + + /// + TimeSpan IGeoSocketOwner.StartupTimeout => _options.StartupTimeout; + + /// + TimeSpan IGeoSocketOwner.TokenRefreshMargin => TokenRefreshMargin; + + /// + TimeSpan IGeoSocketOwner.HandoffWindow => HandoffWindow; + + /// + /// Connects every geo. Completes once all are ready; otherwise stops the transport and throws the first + /// geo failure. + /// + /// A token for cancelling startup. + internal async Task StartAsync(CancellationToken cancellationToken = default) + { + GeoSocket[] geoSockets; + lock (_sync) + { + if (_lifecycle != SocketModeStatus.Idle) + { + throw new InvalidOperationException("A Socket Mode transport can only be started once."); + } + + _lifecycle = SocketModeStatus.Connecting; + geoSockets = [.. _geos.Select(geo => new GeoSocket( + this, + geo.Geo, + geo.NegotiateUri, + _connectionFactory, + _logger, + _timeProvider))]; + foreach (GeoSocket geoSocket in geoSockets) + { + _geoStatuses[geoSocket.Geo] = SocketModeStatus.Connecting; + } + + _geoSockets = geoSockets; + } + + _logger.LogInformation( + "Socket Mode connecting {Count} geo(s): {Geos}.", + geoSockets.Length, + string.Join(", ", geoSockets.Select(geoSocket => geoSocket.Geo))); + + using CancellationTokenSource startSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + Exception? firstFailure = null; + + async Task StartGeoAsync(GeoSocket geoSocket) + { + try + { + await geoSocket.StartAsync(startSource.Token).ConfigureAwait(false); + } + catch (Exception exception) + { + if (Interlocked.CompareExchange(ref firstFailure, exception, null) is null) + { + await startSource.CancelAsync().ConfigureAwait(false); + } + + throw; + } + } + + try + { + await Task.WhenAll(geoSockets.Select(StartGeoAsync)).ConfigureAwait(false); + } + catch (Exception exception) + { + bool stoppedExternally; + lock (_sync) + { + stoppedExternally = _stopTask is not null; + } + + await StopAsync().ConfigureAwait(false); + + if (stoppedExternally) + { + throw new OperationCanceledException("Socket Mode transport stopped before startup completed.", exception); + } + + ExceptionDispatchInfo.Throw(firstFailure ?? exception); + } + + lock (_sync) + { + if (_stopTask is not null) + { + throw new OperationCanceledException("Socket Mode transport stopped before startup completed."); + } + } + + _logger.LogInformation("Socket Mode ready across {Count} geo(s).", geoSockets.Length); + } + + /// + /// Stops every geo and disposes their connections. Idempotent. + /// + internal Task StopAsync() + { + lock (_sync) + { + if (_stopTask is not null) + { + return _stopTask; + } + + _lifecycle = SocketModeStatus.Stopped; + GeoSocket[] geoSockets = _geoSockets; + _stopTask = Task.WhenAll(geoSockets.Select(geoSocket => geoSocket.DisposeAsync().AsTask())); + return _stopTask; + } + } + + /// + public ValueTask DisposeAsync() => new(StopAsync()); + + /// + TimeSpan? IGeoSocketOwner.GetRetryAfter(Exception? error) + => (error as SocketModeNegotiateException)?.RetryAfter; + + /// + [SuppressMessage( + "Security", + "CA5394:Do not use insecure randomness", + Justification = "Jitter only spreads reconnect attempts; it has no security purpose.")] + TimeSpan IGeoSocketOwner.GetBackoffDelay(int attempt) + { + if (_options.ReconnectDelays is { Count: > 0 } schedule) + { + return schedule[Math.Min(attempt, schedule.Count - 1)]; + } + + double capSeconds = Math.Min( + ReconnectInitialDelay.TotalSeconds * Math.Pow(2, Math.Min(attempt, 30)), + ReconnectMaxDelay.TotalSeconds); + return TimeSpan.FromSeconds(capSeconds * _random.NextDouble()); + } + + /// + Task IGeoSocketOwner.DispatchAsync(string geo, SocketActivityEnvelope envelope) + => HandleEnvelopeAsync(envelope); + + /// + void IGeoSocketOwner.OnGeoReady(string geo, SocketReadyFrame frame) + { + SetGeoStatus(geo, SocketModeStatus.Ready); + _logger.LogDebug("Socket Mode geo {Geo} ready with connection {ConnectionId}.", geo, frame.ConnectionId); + } + + /// + void IGeoSocketOwner.OnGeoDisconnected(string geo, Exception? error) + => SetGeoStatus(geo, SocketModeStatus.Disconnected); + + /// + void IGeoSocketOwner.OnGeoReconnected(string geo) + => SetGeoStatus(geo, SocketModeStatus.Ready); + + [SuppressMessage( + "Design", + "CA1031:Do not catch general exception types", + Justification = "Any activity processing failure must be logged and answered with a 500 reply frame.")] + private async Task HandleEnvelopeAsync(SocketActivityEnvelope envelope) + { + long receivedAt = NowUnixMilliseconds(); + + if (envelope.ProtocolVersion > SocketModeProtocol.CurrentVersion) + { + _logger.LogWarning( + "Socket Mode rejecting envelope {EnvelopeId} with unsupported protocol version {Version}.", + envelope.EnvelopeId, + envelope.ProtocolVersion); + return SocketModeEnvelope.CreateInvokeReply( + envelope, + _botKey, + new SocketDispatchResult(400, new { error = $"unsupported protocolVersion {envelope.ProtocolVersion}" }), + receivedAt, + NowUnixMilliseconds()); + } + + if (!SocketModeEnvelope.TryReadActivity(envelope, out CoreActivity? activity) || activity is null) + { + _logger.LogWarning("Socket Mode envelope {EnvelopeId} had no activity payload; dropping.", envelope.EnvelopeId); + return null; + } + + bool invoke = IsInvoke(envelope, activity); + + try + { + SocketDispatchResult result = await _dispatch(activity).ConfigureAwait(false); + return invoke + ? SocketModeEnvelope.CreateInvokeReply(envelope, _botKey, result, receivedAt, NowUnixMilliseconds()) + : SocketModeEnvelope.CreateAcknowledgement( + envelope, + _botKey, + receivedAt, + NowUnixMilliseconds(), + result.Status); + } + catch (Exception exception) + { + _logger.LogError( + exception, + "Socket Mode failed to process activity {ActivityType} in envelope {EnvelopeId}.", + activity.Type, + envelope.EnvelopeId); + await ReportErrorAsync(exception).ConfigureAwait(false); + + return invoke + ? SocketModeEnvelope.CreateInvokeReply( + envelope, + _botKey, + new SocketDispatchResult(500, new { error = "bot handler error" }), + receivedAt, + NowUnixMilliseconds()) + : SocketModeEnvelope.CreateAcknowledgement(envelope, _botKey, receivedAt, NowUnixMilliseconds(), 500); + } + } + + [SuppressMessage( + "Design", + "CA1031:Do not catch general exception types", + Justification = "A failing error observer is logged so it cannot prevent the failure reply.")] + private async Task ReportErrorAsync(Exception exception) + { + if (_onError is null) + { + return; + } + + try + { + await _onError(exception).ConfigureAwait(false); + } + catch (Exception hookException) + { + _logger.LogWarning(hookException, "Socket Mode error observer failed."); + } + } + + private void SetGeoStatus(string geo, SocketModeStatus status) + { + lock (_sync) + { + if (_lifecycle != SocketModeStatus.Stopped) + { + _geoStatuses[geo] = status; + } + } + } + + private long NowUnixMilliseconds() => _timeProvider.GetUtcNow().ToUnixTimeMilliseconds(); + + // An envelope is an invoke when its type says so, falling back to the activity type when the envelope has none. + private static bool IsInvoke(SocketActivityEnvelope envelope, CoreActivity activity) + => string.Equals( + string.IsNullOrEmpty(envelope.Type) ? activity.Type : envelope.Type, + TeamsActivityTypes.Invoke, + StringComparison.OrdinalIgnoreCase); + + private static List<(string Geo, Uri NegotiateUri)> ResolveGeos(Uri baseUri, IReadOnlyList geos) + { + if (geos is null || geos.Count == 0) + { + throw new ArgumentException( + "Socket Mode requires at least one geo. Use an empty string to connect without a geo segment.", + nameof(geos)); + } + + string baseUrl = baseUri.AbsoluteUri.TrimEnd('/'); + HashSet seen = new(StringComparer.OrdinalIgnoreCase); + List<(string Geo, Uri NegotiateUri)> resolved = []; + + foreach (string geo in geos) + { + if (geo is null) + { + throw new ArgumentException("Socket Mode geos cannot contain null.", nameof(geos)); + } + + string segment = geo.Trim().Trim('/'); + if (!seen.Add(segment)) + { + throw new ArgumentException($"Socket Mode geo '{geo}' is listed more than once.", nameof(geos)); + } + + string path = segment.Length > 0 + ? $"/{Uri.EscapeDataString(segment)}{SocketModeProtocol.NegotiatePath}" + : SocketModeProtocol.NegotiatePath; + resolved.Add((segment, new Uri(baseUrl + path, UriKind.Absolute))); + } + + return resolved; + } +} diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeEndToEndTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeEndToEndTests.cs new file mode 100644 index 00000000..eff08935 --- /dev/null +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeEndToEndTests.cs @@ -0,0 +1,287 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Concurrent; +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Teams.Apps.SocketMode; +using Microsoft.Teams.Core.Schema; + +namespace Microsoft.Teams.Apps.UnitTests.SocketMode; + +/// +/// Runs Socket Mode end to end in process: transport, geo supervisors, negotiator and SignalR connection factory are real; only HTTP and the SignalR client are faked. +/// +public class SocketModeEndToEndTests +{ + [Fact] + public async Task Transport_ComposesNegotiatorAndSignalRFactoryAcrossGeos() + { + EndToEnd stack = new(); + + Task start = stack.Transport.StartAsync(); + FakeSignalRClient[] clients = await stack.SignalR.WaitForAsync(3); + foreach (FakeSignalRClient client in clients) + { + client.Ready(); + } + + await start; + + Assert.Equal(SocketModeStatus.Ready, stack.Transport.Status); + Assert.Equal( + [ + "https://botapi.skype.com/amer/v3/websockets/connect", + "https://botapi.skype.com/apac/v3/websockets/connect", + "https://botapi.skype.com/emea/v3/websockets/connect", + ], + stack.Http.Requests.Select(request => request.AbsoluteUri).Order(StringComparer.Ordinal)); + Assert.All(stack.Http.Authorizations, authorization => Assert.Equal("Bearer bot-token", authorization)); + Assert.Equal( + ["https://signalr.test/amer", "https://signalr.test/apac", "https://signalr.test/emea"], + clients.Select(client => client.Url.AbsoluteUri).Order(StringComparer.Ordinal)); + + FakeSignalRClient emea = clients.Single(client => client.Url.AbsolutePath == "/emea"); + SocketReplyFrame? reply = await emea.ActivityAsync(new SocketActivityEnvelope + { + ProtocolVersion = SocketModeProtocol.CurrentVersion, + EnvelopeId = "env-1", + Type = "invoke", + Payload = JsonDocument.Parse("""{"type":"invoke","id":"a1"}""").RootElement.Clone(), + }); + + Assert.NotNull(reply); + Assert.Equal("env-1", reply.EnvelopeId); + Assert.Equal("bot-id", reply.BotKey); + Assert.Equal(200, reply.Status); + Assert.Equal("a1", Assert.Single(stack.Dispatched).Id); + + await stack.Transport.StopAsync(); + + Assert.Equal(SocketModeStatus.Stopped, stack.Transport.Status); + Assert.All(clients, client => + { + Assert.Equal(1, client.StopCount); + Assert.Equal(1, client.DisposeCount); + }); + } + + [Fact] + public async Task Transport_HonorsNegotiatorRetryAfterDuringStartup() + { + EndToEnd stack = new(new SocketModeTransportOptions { Geos = ["amer"] }); + stack.Http.ThrottleOnce(TimeSpan.Zero); + + Task start = stack.Transport.StartAsync(); + FakeSignalRClient client = (await stack.SignalR.WaitForAsync(1))[0]; + client.Ready(); + await start; + + Assert.Equal(2, stack.Http.Requests.Count); + Assert.Equal(SocketModeStatus.Ready, stack.Transport.Status); + + await stack.Transport.DisposeAsync(); + } + + [Fact] + public async Task Transport_FailsStartupWithNegotiatorErrorWhenBudgetIsExhausted() + { + EndToEnd stack = new(new SocketModeTransportOptions { Geos = ["amer"], StartupTimeout = TimeSpan.Zero }); + stack.Http.ThrottleOnce(TimeSpan.Zero); + + SocketModeNegotiateException error = + await Assert.ThrowsAsync(() => stack.Transport.StartAsync()); + + Assert.Equal(HttpStatusCode.TooManyRequests, error.StatusCode); + Assert.Equal(SocketModeStatus.Stopped, stack.Transport.Status); + Assert.Empty(stack.SignalR.Clients); + } + + private sealed class EndToEnd + { + internal EndToEnd(SocketModeTransportOptions? options = null) + { + SocketModeNegotiator negotiator = new( + new HttpClient(Http), + _ => Task.FromResult("bot-token")); + SignalRSocketConnectionFactory factory = new( + negotiator, + TimeSpan.FromSeconds(30), + TimeSpan.FromSeconds(15), + TimeSpan.FromSeconds(30), + NullLogger.Instance, + SignalR.Create); + Transport = new SocketModeTransport( + options ?? new SocketModeTransportOptions(), + factory, + activity => + { + Dispatched.Enqueue(activity); + return Task.FromResult(new SocketDispatchResult(200, null)); + }, + NullLogger.Instance, + botKey: "bot-id"); + } + + internal NegotiateHandler Http { get; } = new(); + internal FakeSignalRClientFactory SignalR { get; } = new(); + internal ConcurrentQueue Dispatched { get; } = new(); + internal SocketModeTransport Transport { get; } + } + + private sealed class NegotiateHandler : HttpMessageHandler + { + private readonly object _gate = new(); + private TimeSpan? _throttle; + + internal ConcurrentQueue Requests { get; } = new(); + internal ConcurrentQueue Authorizations { get; } = new(); + + internal void ThrottleOnce(TimeSpan retryAfter) + { + lock (_gate) + { + _throttle = retryAfter; + } + } + + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + Uri uri = request.RequestUri ?? throw new InvalidOperationException("Missing request URI."); + Requests.Enqueue(uri); + Authorizations.Enqueue(request.Headers.Authorization?.ToString()); + + TimeSpan? throttle; + lock (_gate) + { + throttle = _throttle; + _throttle = null; + } + + if (throttle is TimeSpan retryAfter) + { + HttpResponseMessage throttled = new(HttpStatusCode.TooManyRequests); + throttled.Headers.RetryAfter = new RetryConditionHeaderValue(retryAfter); + return Task.FromResult(throttled); + } + + string geo = uri.Segments[1].TrimEnd('/'); + string json = $$"""{"url":"https://signalr.test/{{geo}}","accessToken":"token-{{geo}}","expiresIn":3600}"""; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(json, Encoding.UTF8, "application/json"), + }); + } + } + + private sealed class FakeSignalRClientFactory + { + private readonly object _gate = new(); + private TaskCompletionSource _changed = NewSignal(); + + internal List Clients { get; } = []; + + internal ISignalRClientConnection Create( + Uri url, + string accessToken, + TimeSpan keepAliveInterval, + TimeSpan serverTimeout) + { + FakeSignalRClient client = new(url, Changed); + lock (_gate) + { + Clients.Add(client); + } + + Changed(); + return client; + } + + internal async Task WaitForAsync(int count) + { + using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(10)); + while (true) + { + Task changed; + lock (_gate) + { + if (Clients.Count >= count && Clients.Take(count).All(client => client.StartCount > 0)) + { + return [.. Clients.Take(count)]; + } + + changed = _changed.Task; + } + + await changed.WaitAsync(timeout.Token); + } + } + + private void Changed() + { + TaskCompletionSource previous; + lock (_gate) + { + previous = _changed; + _changed = NewSignal(); + } + + previous.TrySetResult(); + } + + private static TaskCompletionSource NewSignal() => new(TaskCreationOptions.RunContinuationsAsynchronously); + } + + private sealed class FakeSignalRClient(Uri url, Action changed) : ISignalRClientConnection + { + private Func>? _onActivity; + private Action? _onReady; + private Action? _onClosed; + private int _startCount; + private int _stopCount; + private int _disposeCount; + + internal Uri Url { get; } = url; + internal int StartCount => Volatile.Read(ref _startCount); + internal int StopCount => Volatile.Read(ref _stopCount); + internal int DisposeCount => Volatile.Read(ref _disposeCount); + + public void OnActivity(Func> handler) => _onActivity = handler; + + public void OnReady(Action handler) => _onReady = handler; + + public void OnClosed(Action handler) => _onClosed = handler; + + public Task StartAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Interlocked.Increment(ref _startCount); + changed(); + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) + { + Interlocked.Increment(ref _stopCount); + _onClosed?.Invoke(null); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + Interlocked.Increment(ref _disposeCount); + return ValueTask.CompletedTask; + } + + internal void Ready() + => (_onReady ?? throw new InvalidOperationException())(new SocketReadyFrame { BotKey = "bot-id" }); + + internal Task ActivityAsync(SocketActivityEnvelope envelope) + => (_onActivity ?? throw new InvalidOperationException())(envelope); + } +} diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeTransportTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeTransportTests.cs new file mode 100644 index 00000000..1209653f --- /dev/null +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeTransportTests.cs @@ -0,0 +1,484 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Net; +using System.Text.Json; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Teams.Apps.SocketMode; +using Microsoft.Teams.Core.Schema; + +namespace Microsoft.Teams.Apps.UnitTests.SocketMode; + +public class SocketModeTransportTests +{ + private static readonly DateTimeOffset Now = DateTimeOffset.FromUnixTimeMilliseconds(1_700_000_000_000); + + [Fact] + public async Task StartAsync_ConnectsEveryDefaultGeo() + { + Harness harness = new(); + + Task start = harness.Transport.StartAsync(); + FakeConnection[] connections = await harness.Factory.WaitForAsync(3); + + Assert.Equal( + [ + "https://botapi.skype.com/amer/v3/websockets/connect", + "https://botapi.skype.com/apac/v3/websockets/connect", + "https://botapi.skype.com/emea/v3/websockets/connect", + ], + SortedUris(connections)); + Assert.False(start.IsCompleted); + Assert.Equal(SocketModeStatus.Connecting, harness.Transport.Status); + + foreach (FakeConnection connection in connections) + { + connection.Ready(); + } + + await start; + Assert.Equal(SocketModeStatus.Ready, harness.Transport.Status); + Assert.All(harness.Transport.GeoStatuses.Values, status => Assert.Equal(SocketModeStatus.Ready, status)); + } + + [Fact] + public async Task StartAsync_BuildsGeoUrisFromCustomBaseAndEmptyGeo() + { + Harness harness = new(new SocketModeTransportOptions + { + NegotiateBaseUri = new Uri("http://localhost:3978/"), + Geos = ["", " /eu/ "], + }); + + Task start = harness.Transport.StartAsync(); + FakeConnection[] connections = await harness.Factory.WaitForAsync(2); + + Assert.Equal( + [ + "http://localhost:3978/eu/v3/websockets/connect", + "http://localhost:3978/v3/websockets/connect", + ], + SortedUris(connections)); + foreach (FakeConnection connection in connections) + { + connection.Ready(); + } + + await start; + } + + [Theory] + [InlineData(new object[] { new string[0] })] + [InlineData(new object[] { new[] { "amer", "AMER" } })] + public void Constructor_RejectsInvalidGeos(string[] geos) + { + Assert.Throws(() => new Harness(new SocketModeTransportOptions { Geos = geos }).Transport); + } + + [Fact] + public void Constructor_RejectsNegativeStartupTimeout() + { + Assert.Throws(() => new Harness(new SocketModeTransportOptions + { + StartupTimeout = TimeSpan.FromSeconds(-1), + }).Transport); + } + + [Fact] + public async Task StartAsync_WhenAnyGeoFails_StopsEveryGeoAndThrowsThatFailure() + { + IOException failure = new("emea failed"); + Harness harness = new(new SocketModeTransportOptions { StartupTimeout = TimeSpan.Zero }); + harness.Factory.FailStart("emea", failure); + + IOException thrown = await Assert.ThrowsAsync(() => harness.Transport.StartAsync()); + + Assert.Same(failure, thrown); + Assert.Equal(SocketModeStatus.Stopped, harness.Transport.Status); + Assert.All(harness.Factory.Connections, connection => Assert.Equal(1, connection.DisposeCount)); + } + + [Fact] + public async Task StopAsync_DuringStartup_CancelsStartAndDisposesConnections() + { + Harness harness = new(); + Task start = harness.Transport.StartAsync(); + FakeConnection[] connections = await harness.Factory.WaitForAsync(3); + + await harness.Transport.StopAsync(); + + await Assert.ThrowsAnyAsync(() => start); + Assert.All(connections, connection => Assert.Equal(1, connection.DisposeCount)); + Assert.Equal(SocketModeStatus.Stopped, harness.Transport.Status); + } + + [Fact] + public async Task StopAsync_IsIdempotentAndDisposesEachConnectionOnce() + { + Harness harness = new(); + FakeConnection[] connections = await harness.StartReadyAsync(); + + await harness.Transport.StopAsync(); + await harness.Transport.StopAsync(); + await harness.Transport.DisposeAsync(); + + Assert.All(connections, connection => + { + Assert.Equal(1, connection.StopCount); + Assert.Equal(1, connection.DisposeCount); + }); + Assert.Equal(SocketModeStatus.Stopped, harness.Transport.Status); + await Assert.ThrowsAsync(() => harness.Transport.StartAsync()); + } + + [Fact] + public async Task Status_ReportsDisconnectedUntilDroppedGeoRecovers() + { + Harness harness = new(new SocketModeTransportOptions { ReconnectDelays = [TimeSpan.Zero] }); + FakeConnection[] connections = await harness.StartReadyAsync(); + FakeConnection amer = connections.Single(connection => connection.Geo == "amer"); + + amer.Close(new IOException("dropped")); + + Assert.Equal(SocketModeStatus.Disconnected, harness.Transport.Status); + Assert.Equal(SocketModeStatus.Disconnected, harness.Transport.GeoStatuses["amer"]); + Assert.Equal(SocketModeStatus.Ready, harness.Transport.GeoStatuses["emea"]); + + FakeConnection replacement = (await harness.Factory.WaitForAsync(4))[3]; + replacement.Ready(); + await WaitUntilAsync(() => harness.Transport.Status == SocketModeStatus.Ready); + } + + [Fact] + public async Task Dispatch_InvokeReturnsHandlerStatusAndBody() + { + Harness harness = new() { Dispatch = _ => Task.FromResult(new SocketDispatchResult(201, "created")) }; + FakeConnection connection = (await harness.StartReadyAsync())[0]; + + SocketReplyFrame? reply = await connection.ActivityAsync(Envelope("invoke", """{"type":"invoke","id":"a1"}""")); + + Assert.NotNull(reply); + Assert.Equal("env-1", reply.EnvelopeId); + Assert.Equal("bot-id", reply.BotKey); + Assert.Equal(201, reply.Status); + Assert.Equal("created", reply.Body); + Assert.Equal(Now.ToUnixTimeMilliseconds(), reply.ReceivedAtUnixMilliseconds); + Assert.Equal("a1", Assert.Single(harness.Dispatched).Id); + } + + [Fact] + public async Task Dispatch_MessageReturnsAcknowledgementWithHandlerStatus() + { + Harness harness = new() { Dispatch = _ => Task.FromResult(new SocketDispatchResult(202, "ignored")) }; + FakeConnection connection = (await harness.StartReadyAsync())[0]; + + SocketReplyFrame? reply = await connection.ActivityAsync(Envelope("message", """{"type":"message"}""")); + + Assert.NotNull(reply); + Assert.Equal(202, reply.Status); + Assert.Null(reply.Body); + } + + [Fact] + public async Task Dispatch_ClassifiesInvokeFromActivityWhenEnvelopeTypeIsAbsent() + { + Harness harness = new() { Dispatch = _ => Task.FromResult(new SocketDispatchResult(200, "result")) }; + FakeConnection connection = (await harness.StartReadyAsync())[0]; + + SocketReplyFrame? reply = await connection.ActivityAsync(Envelope(null, """{"type":"invoke"}""")); + + Assert.Equal("result", reply?.Body); + } + + [Theory] + [InlineData("invoke", """{"type":"invoke"}""", true)] + [InlineData("message", """{"type":"message"}""", false)] + public async Task Dispatch_HandlerFailureReturns500AndReportsError(string type, string payload, bool hasBody) + { + InvalidOperationException failure = new("handler failed"); + List reported = []; + Harness harness = new() + { + Dispatch = _ => throw failure, + OnError = exception => + { + reported.Add(exception); + return Task.CompletedTask; + }, + }; + FakeConnection connection = (await harness.StartReadyAsync())[0]; + + SocketReplyFrame? reply = await connection.ActivityAsync(Envelope(type, payload)); + + Assert.NotNull(reply); + Assert.Equal(500, reply.Status); + Assert.Equal(hasBody, reply.Body is not null); + Assert.Same(failure, Assert.Single(reported)); + } + + [Fact] + public async Task Dispatch_FailingErrorObserverStillReturns500() + { + Harness harness = new() + { + Dispatch = _ => Task.FromException(new InvalidOperationException("handler failed")), + OnError = _ => throw new InvalidOperationException("observer failed"), + }; + FakeConnection connection = (await harness.StartReadyAsync())[0]; + + SocketReplyFrame? reply = await connection.ActivityAsync(Envelope("invoke", """{"type":"invoke"}""")); + + Assert.Equal(500, reply?.Status); + } + + [Fact] + public async Task Dispatch_RejectsUnsupportedProtocolVersionWithoutDispatching() + { + Harness harness = new(); + FakeConnection connection = (await harness.StartReadyAsync())[0]; + + SocketReplyFrame? reply = await connection.ActivityAsync( + Envelope("message", """{"type":"message"}""", protocolVersion: SocketModeProtocol.CurrentVersion + 1)); + + Assert.Equal(400, reply?.Status); + Assert.Empty(harness.Dispatched); + } + + [Fact] + public async Task Dispatch_DropsEnvelopeWithoutActivity() + { + Harness harness = new(); + FakeConnection connection = (await harness.StartReadyAsync())[0]; + + SocketReplyFrame? reply = await connection.ActivityAsync(Envelope("message", """{"text":"no type"}""")); + + Assert.Null(reply); + Assert.Empty(harness.Dispatched); + } + + [Fact] + public void RetryPolicy_UsesScheduleThenRepeatsLastDelay() + { + IGeoSocketOwner owner = new Harness(new SocketModeTransportOptions + { + ReconnectDelays = [TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(4)], + }).Transport; + + Assert.Equal(TimeSpan.FromSeconds(1), owner.GetBackoffDelay(0)); + Assert.Equal(TimeSpan.FromSeconds(4), owner.GetBackoffDelay(1)); + Assert.Equal(TimeSpan.FromSeconds(4), owner.GetBackoffDelay(9)); + } + + [Fact] + public void RetryPolicy_JittersCappedExponentialBackoff() + { + IGeoSocketOwner owner = new Harness { Random = new FixedRandom(0.5) }.Transport; + + Assert.Equal(TimeSpan.FromSeconds(0.5), owner.GetBackoffDelay(0)); + Assert.Equal(TimeSpan.FromSeconds(2), owner.GetBackoffDelay(2)); + Assert.Equal(TimeSpan.FromSeconds(7.5), owner.GetBackoffDelay(100)); + } + + [Fact] + public void RetryPolicy_ReadsRetryAfterFromNegotiateFailure() + { + IGeoSocketOwner owner = new Harness().Transport; + + Assert.Equal( + TimeSpan.FromSeconds(7), + owner.GetRetryAfter(new SocketModeNegotiateException(HttpStatusCode.TooManyRequests, TimeSpan.FromSeconds(7)))); + Assert.Null(owner.GetRetryAfter(new IOException())); + Assert.Null(owner.GetRetryAfter(null)); + } + + private static SocketActivityEnvelope Envelope(string? type, string payload, int protocolVersion = 1) + => new() + { + ProtocolVersion = protocolVersion, + EnvelopeId = "env-1", + Type = type, + Payload = JsonDocument.Parse(payload).RootElement.Clone(), + }; + + private static string[] SortedUris(FakeConnection[] connections) + => [.. connections.Select(connection => connection.NegotiateUri.AbsoluteUri).Order(StringComparer.Ordinal)]; + + private static async Task WaitUntilAsync(Func condition) + { + using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(5)); + while (!condition()) + { + await Task.Delay(1, timeout.Token); + } + } + + private sealed class Harness + { + private readonly SocketModeTransportOptions _options; + private SocketModeTransport? _transport; + + internal Harness(SocketModeTransportOptions? options = null) + { + _options = options ?? new SocketModeTransportOptions(); + } + + internal FakeConnectionFactory Factory { get; } = new(); + + internal List Dispatched { get; } = []; + + internal Func> Dispatch { get; init; } = + _ => Task.FromResult(new SocketDispatchResult(200)); + + internal Func? OnError { get; init; } + + internal Random? Random { get; init; } + + internal SocketModeTransport Transport => _transport ??= new SocketModeTransport( + _options, + Factory, + activity => + { + lock (Dispatched) + { + Dispatched.Add(activity); + } + + return Dispatch(activity); + }, + NullLogger.Instance, + "bot-id", + OnError, + new FixedTimeProvider(), + Random); + + internal async Task StartReadyAsync() + { + Task start = Transport.StartAsync(); + FakeConnection[] connections = await Factory.WaitForAsync(_options.Geos.Count); + foreach (FakeConnection connection in connections) + { + connection.Ready(); + } + + await start; + return connections; + } + } + + private sealed class FakeConnectionFactory : ISocketConnectionFactory + { + private readonly Dictionary _startFailures = []; + private readonly List _connections = []; + + internal FakeConnection[] Connections + { + get + { + lock (_connections) + { + return [.. _connections]; + } + } + } + + internal void FailStart(string geo, Exception error) => _startFailures[geo] = error; + + internal async Task WaitForAsync(int count) + { + await WaitUntilAsync(() => Connections.Length >= count); + FakeConnection[] connections = Connections; + await Task.WhenAll(connections.Select(connection => connection.Started.Task)); + return connections; + } + + public ISocketConnection Create(Uri negotiateUri, SocketConnectionHandlers handlers) + { + string[] segments = negotiateUri.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries); + string geo = segments.Length > 3 ? segments[0] : string.Empty; + FakeConnection connection = new(negotiateUri, geo, handlers, _startFailures.GetValueOrDefault(geo)); + lock (_connections) + { + _connections.Add(connection); + } + + return connection; + } + } + + private sealed class FakeConnection( + Uri negotiateUri, + string geo, + SocketConnectionHandlers handlers, + Exception? startError) : ISocketConnection + { + private readonly TaskCompletionSource _ready = new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _closed; + private int _stopCount; + private int _disposeCount; + + public TimeSpan? TokenLifetime => null; + + internal Uri NegotiateUri { get; } = negotiateUri; + + internal string Geo { get; } = geo; + + internal TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + internal int StopCount => Volatile.Read(ref _stopCount); + + internal int DisposeCount => Volatile.Read(ref _disposeCount); + + public async Task StartAsync(CancellationToken cancellationToken) + { + Started.TrySetResult(); + if (startError is not null) + { + throw startError; + } + + await _ready.Task.WaitAsync(cancellationToken); + } + + public Task StopAsync(CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref _stopCount); + _ready.TrySetCanceled(CancellationToken.None); + RaiseClosed(null, planned: true); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + Interlocked.Increment(ref _disposeCount); + return ValueTask.CompletedTask; + } + + internal void Ready() + { + handlers.OnReady(new SocketReadyFrame { ConnectionId = NegotiateUri.AbsolutePath }); + _ready.TrySetResult(); + } + + internal void Close(Exception error) => RaiseClosed(error, planned: false); + + internal Task ActivityAsync(SocketActivityEnvelope envelope) => handlers.OnActivity(envelope); + + private void RaiseClosed(Exception? error, bool planned) + { + if (Interlocked.Exchange(ref _closed, 1) == 0) + { + handlers.OnClosed(error, planned); + } + } + } + + private sealed class FixedTimeProvider : TimeProvider + { + public override DateTimeOffset GetUtcNow() => Now; + } + + private sealed class FixedRandom(double value) : Random + { + public override double NextDouble() => value; + } +} From e5a0b3fa2c94a93e79217162021c838d53338946 Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Thu, 24 Sep 2026 13:23:16 -0700 Subject: [PATCH 25/31] marked each geoStatus as stopped in StopAsync --- src/Microsoft.Teams.Apps/SocketMode/SocketModeTransport.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.Teams.Apps/SocketMode/SocketModeTransport.cs b/src/Microsoft.Teams.Apps/SocketMode/SocketModeTransport.cs index fc043c77..c698d18f 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/SocketModeTransport.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/SocketModeTransport.cs @@ -286,8 +286,11 @@ internal Task StopAsync() } _lifecycle = SocketModeStatus.Stopped; - GeoSocket[] geoSockets = _geoSockets; - _stopTask = Task.WhenAll(geoSockets.Select(geoSocket => geoSocket.DisposeAsync().AsTask())); + foreach (string geo in _geoStatuses.Keys.ToArray()) + { + _geoStatuses[geo] = SocketModeStatus.Stopped; + } + _stopTask = Task.WhenAll(_geoSockets.Select(geoSocket => geoSocket.DisposeAsync().AsTask())); return _stopTask; } } From ab2a6ae462c5b3d57879c89f4a19df540615cd02 Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Thu, 24 Sep 2026 15:12:12 -0700 Subject: [PATCH 26/31] fix(apps): harden Socket Mode shutdown Log connection stop/dispose failures instead of throwing so they cannot mask the original startup failure or end a geo's supervisor, and mark every geo as Stopped when the transport stops. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SocketMode/GeoSocket.cs | 26 +++-- .../SocketMode/SocketModeTransport.cs | 9 +- .../SocketMode/GeoSocketTests.cs | 94 ++++++++++++++++++- .../SocketMode/SocketModeTransportTests.cs | 38 +++++++- 4 files changed, 147 insertions(+), 20 deletions(-) diff --git a/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs b/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs index 0a4d343c..abbb287f 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs @@ -160,6 +160,7 @@ internal async Task StartAsync(CancellationToken cancellationToken = default) /// /// Stops supervision and stops and disposes every connection this geo owns. + /// Connection cleanup failures are logged rather than thrown. /// internal Task StopAsync() { @@ -502,10 +503,6 @@ private void StartRetirement(Generation previous) TaskScheduler.Default); } - [SuppressMessage( - "Design", - "CA1031:Do not catch general exception types", - Justification = "Retirement runs in the background; failures are logged so they cannot fault the supervisor.")] private async Task RetireAsync(Generation previous) { try @@ -526,10 +523,6 @@ await Task.Delay( catch (OperationCanceledException) when (_stopToken.IsCancellationRequested) { } - catch (Exception exception) - { - _logger.LogWarning(exception, "Socket Mode geo {Geo} failed to retire a replaced connection.", Geo); - } } private async Task StopCoreAsync() @@ -580,16 +573,29 @@ private async Task ReleaseAsync(ISocketConnection connection) await StopAndDisposeAsync(connection).ConfigureAwait(false); } - private static async Task StopAndDisposeAsync(ISocketConnection connection) + [SuppressMessage( + "Design", + "CA1031:Do not catch general exception types", + Justification = "Connection cleanup is best-effort; failures are logged so they cannot mask the caller's outcome.")] + private async Task StopAndDisposeAsync(ISocketConnection connection) { try { await connection.StopAsync(CancellationToken.None).ConfigureAwait(false); } - finally + catch (Exception exception) + { + _logger.LogWarning(exception, "Socket Mode geo {Geo} failed to stop a connection.", Geo); + } + + try { await connection.DisposeAsync().ConfigureAwait(false); } + catch (Exception exception) + { + _logger.LogWarning(exception, "Socket Mode geo {Geo} failed to dispose a connection.", Geo); + } } private TimeSpan GetRetryDelay(Exception? error, int attempt) diff --git a/src/Microsoft.Teams.Apps/SocketMode/SocketModeTransport.cs b/src/Microsoft.Teams.Apps/SocketMode/SocketModeTransport.cs index fc043c77..518abf6b 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/SocketModeTransport.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/SocketModeTransport.cs @@ -274,7 +274,7 @@ async Task StartGeoAsync(GeoSocket geoSocket) } /// - /// Stops every geo and disposes their connections. Idempotent. + /// Stops every geo and disposes their connections. Idempotent; connection cleanup failures are logged rather than thrown. /// internal Task StopAsync() { @@ -286,8 +286,11 @@ internal Task StopAsync() } _lifecycle = SocketModeStatus.Stopped; - GeoSocket[] geoSockets = _geoSockets; - _stopTask = Task.WhenAll(geoSockets.Select(geoSocket => geoSocket.DisposeAsync().AsTask())); + foreach (string geo in _geoStatuses.Keys.ToArray()) + { + _geoStatuses[geo] = SocketModeStatus.Stopped; + } + _stopTask = Task.WhenAll(_geoSockets.Select(geoSocket => geoSocket.DisposeAsync().AsTask())); return _stopTask; } } diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs index a5899e9e..89fce337 100644 --- a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs @@ -3,7 +3,7 @@ using System.Diagnostics; using System.Net; -using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Logging; using Microsoft.Teams.Apps.SocketMode; namespace Microsoft.Teams.Apps.UnitTests.SocketMode; @@ -327,6 +327,49 @@ public async Task StopAsync_DuringHandoff_DisposesActiveAndRetiringOnce() Assert.Empty(harness.Owner.Disconnections); } + [Fact] + public async Task StopAsync_LogsConnectionCleanupFailuresAndStillDisposes() + { + Harness harness = new(); + FakeConnection connection = harness.Factory.Enqueue(); + connection.StopError = new IOException("stop failed"); + connection.DisposeError = new IOException("dispose failed"); + await harness.StartReadyAsync(connection); + + await harness.Socket.StopAsync(); + + Assert.Equal(1, connection.StopCount); + Assert.Equal(1, connection.DisposeCount); + Assert.Equal( + [connection.StopError, connection.DisposeError], + harness.Logger.Warnings.Select(warning => warning.Exception)); + } + + [Fact] + public async Task Retirement_LogsDisposeFailureAndKeepsReplacementActive() + { + Harness harness = new() { HandoffWindow = TimeSpan.FromSeconds(5) }; + FakeConnection retiring = harness.Factory.Enqueue(TimeSpan.FromSeconds(10)); + FakeConnection active = harness.Factory.Enqueue(); + retiring.DisposeError = new IOException("dispose failed"); + await harness.StartReadyAsync(retiring); + + harness.Time.Advance(TimeSpan.FromSeconds(5)); + await active.Started.Task; + active.Ready("replacement"); + await harness.Time.WaitForTimerAsync(TimeSpan.FromSeconds(5)); + harness.Time.Advance(TimeSpan.FromSeconds(5)); + await WaitUntilAsync(() => harness.Logger.Warnings.Length > 0); + + Assert.Same(retiring.DisposeError, Assert.Single(harness.Logger.Warnings).Exception); + Assert.NotNull(await active.Activity("after-retirement")); + + await harness.Socket.StopAsync(); + + Assert.Equal(1, retiring.DisposeCount); + Assert.Equal(1, active.DisposeCount); + } + private static async Task WaitUntilAsync(Func condition) { Stopwatch elapsed = Stopwatch.StartNew(); @@ -351,6 +394,8 @@ private sealed class Harness internal FakeConnectionFactory Factory { get; } = new(); + internal RecordingLogger Logger { get; } = new(); + internal TimeSpan StartupTimeout { init => Owner.StartupTimeout = value; } internal TimeSpan TokenRefreshMargin { init => Owner.TokenRefreshMargin = value; } @@ -360,7 +405,7 @@ private sealed class Harness internal Func Backoff { init => Owner.Backoff = value; } internal GeoSocket Socket => - _socket ??= new GeoSocket(Owner, "amer", NegotiateUri, Factory, NullLogger.Instance, Time); + _socket ??= new GeoSocket(Owner, "amer", NegotiateUri, Factory, Logger, Time); internal async Task StartReadyAsync(FakeConnection connection) { @@ -480,6 +525,10 @@ private sealed class FakeConnection(TimeSpan? tokenLifetime, Exception? startErr internal int DisposeCount => Volatile.Read(ref _disposeCount); + internal Exception? StopError { get; set; } + + internal Exception? DisposeError { get; set; } + public async Task StartAsync(CancellationToken cancellationToken) { Started.TrySetResult(); @@ -496,14 +545,14 @@ public Task StopAsync(CancellationToken cancellationToken = default) Interlocked.Increment(ref _stopCount); _ready.TrySetCanceled(CancellationToken.None); RaiseClosed(null, planned: true); - return Task.CompletedTask; + return StopError is null ? Task.CompletedTask : Task.FromException(StopError); } public ValueTask DisposeAsync() { Interlocked.Increment(ref _disposeCount); Disposed.TrySetResult(); - return ValueTask.CompletedTask; + return DisposeError is null ? ValueTask.CompletedTask : ValueTask.FromException(DisposeError); } internal void Ready(string connectionId) @@ -526,6 +575,43 @@ private void RaiseClosed(Exception? error, bool planned) } } + private sealed class RecordingLogger : ILogger + { + private readonly List<(string Message, Exception? Exception)> _warnings = []; + + internal (string Message, Exception? Exception)[] Warnings + { + get + { + lock (_warnings) + { + return [.. _warnings]; + } + } + } + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + if (logLevel == LogLevel.Warning) + { + lock (_warnings) + { + _warnings.Add((formatter(state, exception), exception)); + } + } + } + } + private sealed class ManualTimeProvider : TimeProvider { private readonly object _sync = new(); diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeTransportTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeTransportTests.cs index 1209653f..49b86699 100644 --- a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeTransportTests.cs +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeTransportTests.cs @@ -98,6 +98,27 @@ public async Task StartAsync_WhenAnyGeoFails_StopsEveryGeoAndThrowsThatFailure() Assert.All(harness.Factory.Connections, connection => Assert.Equal(1, connection.DisposeCount)); } + [Fact] + public async Task StartAsync_WhenCleanupFails_StillThrowsTheStartupFailure() + { + IOException failure = new("emea failed"); + Harness harness = new(new SocketModeTransportOptions { StartupTimeout = TimeSpan.Zero }); + harness.Factory.FailDispose("amer", new IOException("amer dispose failed")); + harness.Factory.FailDispose("apac", new IOException("apac dispose failed")); + + Task start = harness.Transport.StartAsync(); + FakeConnection[] connections = await harness.Factory.WaitForAsync(3); + connections.Single(connection => connection.Geo == "amer").Ready(); + connections.Single(connection => connection.Geo == "apac").Ready(); + connections.Single(connection => connection.Geo == "emea").Fail(failure); + + IOException thrown = await Assert.ThrowsAsync(() => start); + + Assert.Same(failure, thrown); + Assert.Equal(SocketModeStatus.Stopped, harness.Transport.Status); + Assert.All(harness.Factory.Connections, connection => Assert.Equal(1, connection.DisposeCount)); + } + [Fact] public async Task StopAsync_DuringStartup_CancelsStartAndDisposesConnections() { @@ -368,6 +389,7 @@ internal async Task StartReadyAsync() private sealed class FakeConnectionFactory : ISocketConnectionFactory { private readonly Dictionary _startFailures = []; + private readonly Dictionary _disposeFailures = []; private readonly List _connections = []; internal FakeConnection[] Connections @@ -383,6 +405,8 @@ internal FakeConnection[] Connections internal void FailStart(string geo, Exception error) => _startFailures[geo] = error; + internal void FailDispose(string geo, Exception error) => _disposeFailures[geo] = error; + internal async Task WaitForAsync(int count) { await WaitUntilAsync(() => Connections.Length >= count); @@ -395,7 +419,12 @@ public ISocketConnection Create(Uri negotiateUri, SocketConnectionHandlers handl { string[] segments = negotiateUri.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries); string geo = segments.Length > 3 ? segments[0] : string.Empty; - FakeConnection connection = new(negotiateUri, geo, handlers, _startFailures.GetValueOrDefault(geo)); + FakeConnection connection = new( + negotiateUri, + geo, + handlers, + _startFailures.GetValueOrDefault(geo), + _disposeFailures.GetValueOrDefault(geo)); lock (_connections) { _connections.Add(connection); @@ -409,7 +438,8 @@ private sealed class FakeConnection( Uri negotiateUri, string geo, SocketConnectionHandlers handlers, - Exception? startError) : ISocketConnection + Exception? startError, + Exception? disposeError) : ISocketConnection { private readonly TaskCompletionSource _ready = new(TaskCreationOptions.RunContinuationsAsynchronously); private int _closed; @@ -450,7 +480,7 @@ public Task StopAsync(CancellationToken cancellationToken = default) public ValueTask DisposeAsync() { Interlocked.Increment(ref _disposeCount); - return ValueTask.CompletedTask; + return disposeError is null ? ValueTask.CompletedTask : ValueTask.FromException(disposeError); } internal void Ready() @@ -459,6 +489,8 @@ internal void Ready() _ready.TrySetResult(); } + internal void Fail(Exception error) => _ready.TrySetException(error); + internal void Close(Exception error) => RaiseClosed(error, planned: false); internal Task ActivityAsync(SocketActivityEnvelope envelope) => handlers.OnActivity(envelope); From f760306c9d56ec2c3d3e9299bf6feea5ebf2c68d Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Thu, 24 Sep 2026 15:33:58 -0700 Subject: [PATCH 27/31] fix(apps): keep Socket Mode stop from rethrowing supervisor faults A faulted supervisor is already logged; rethrowing it from stop could mask the original startup failure in the transport. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SocketMode/GeoSocket.cs | 2 +- .../SocketMode/GeoSocketTests.cs | 31 ++++++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs b/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs index abbb287f..ccfd6d4d 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs @@ -555,7 +555,7 @@ private async Task StopCoreAsync() await Task.WhenAll(retirements).ConfigureAwait(false); if (supervisor is not null) { - await supervisor.ConfigureAwait(false); + await supervisor.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); } } } diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs index 89fce337..995f2d80 100644 --- a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs @@ -268,6 +268,26 @@ public async Task StartAsync_CallerCancellationInterruptsRetryDelay() Assert.Equal(1, failed.DisposeCount); } + [Fact] + public async Task StopAsync_CompletesWhenSupervisorHasFaulted() + { + Harness harness = new() { Backoff = _ => TimeSpan.Zero }; + FakeConnection initial = harness.Factory.Enqueue(); + FakeConnection replacement = harness.Factory.Enqueue(); + harness.Owner.ReconnectedError = new InvalidOperationException("observer failed"); + await harness.StartReadyAsync(initial); + + initial.Close(new IOException("dropped")); + await replacement.Started.Task; + replacement.Ready("replacement"); + await WaitUntilAsync(() => harness.Owner.Reconnections == 1); + + await harness.Socket.StopAsync(); + + Assert.Equal(1, replacement.StopCount); + Assert.Equal(1, replacement.DisposeCount); + } + [Fact] public async Task StopAsync_DuringReconnectDelay_CancelsWithoutNewGeneration() { @@ -471,7 +491,16 @@ public void OnGeoDisconnected(string geo, Exception? error) } } - public void OnGeoReconnected(string geo) => Interlocked.Increment(ref _reconnections); + internal Exception? ReconnectedError { get; set; } + + public void OnGeoReconnected(string geo) + { + Interlocked.Increment(ref _reconnections); + if (ReconnectedError is not null) + { + throw ReconnectedError; + } + } } private sealed class FakeConnectionFactory : ISocketConnectionFactory From 28ad9e2b146c4c996ca95d1cd5c0a0e836fa27db Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Fri, 25 Sep 2026 14:20:17 -0700 Subject: [PATCH 28/31] test --- src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs b/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs index ccfd6d4d..af3868f7 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs @@ -632,3 +632,4 @@ private sealed class Generation(long id) private sealed record CloseReason(Exception? Error, bool Planned); } + From 1f20c53d6848373a4e717ba99e57d527d3277127 Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Thu, 1 Oct 2026 10:24:18 -0700 Subject: [PATCH 29/31] fix(apps): stop retrying Socket Mode on negotiate 401/403 --- .../SocketMode/GeoSocket.cs | 51 ++++++++++-- .../SocketMode/SocketModeNegotiator.cs | 30 ++++++- .../SocketMode/GeoSocketTests.cs | 83 +++++++++++++++++++ .../SocketMode/SocketModeNegotiatorTests.cs | 30 +++++++ 4 files changed, 188 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs b/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs index af3868f7..2877467c 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs @@ -136,6 +136,7 @@ internal GeoSocket( /// /// A token for cancelling startup. /// Thrown when no generation becomes ready within the startup budget. + /// Thrown without retrying when negotiate rejects the bot with HTTP 401 or 403. internal async Task StartAsync(CancellationToken cancellationToken = default) { ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); @@ -213,6 +214,11 @@ private async Task ConnectInitialAsync(CancellationToken cancellatio } catch (Exception exception) when (!startSource.IsCancellationRequested) { + if (SocketModeNegotiateException.IsNonRetryable(exception)) + { + throw; + } + Exception error = deadlineSource.IsCancellationRequested ? new TimeoutException( $"Socket Mode geo '{Geo}' did not become ready within {budget}.", @@ -248,19 +254,29 @@ private async Task SuperviseAsync(Generation current) { CloseReason reason = await current.Closed.Task.WaitAsync(_stopToken).ConfigureAwait(false); + Generation? replacement; if (reason.Planned) { _logger.LogInformation("Socket Mode geo {Geo} rotating connection before token expiry.", Geo); - Generation replacement = await ReconnectAsync(null, delayFirstAttempt: false).ConfigureAwait(false); - StartRetirement(current); - current = replacement; + replacement = await ReconnectAsync(null, delayFirstAttempt: false).ConfigureAwait(false); + if (replacement is not null) + { + StartRetirement(current); + } } else { await ReleaseAsync(current.Connection).ConfigureAwait(false); - current = await ReconnectAsync(reason.Error, delayFirstAttempt: true).ConfigureAwait(false); + replacement = await ReconnectAsync(reason.Error, delayFirstAttempt: true).ConfigureAwait(false); } + if (replacement is null) + { + return; + } + + current = replacement; + ReportReconnected(); } } @@ -274,7 +290,8 @@ private async Task SuperviseAsync(Generation current) } } - private async Task ReconnectAsync(Exception? previousError, bool delayFirstAttempt) + /// The ready replacement, or when the geo gave up after a non-retryable failure. + private async Task ReconnectAsync(Exception? previousError, bool delayFirstAttempt) { Exception? error = previousError; int retry = 0; @@ -292,6 +309,12 @@ private async Task ReconnectAsync(Exception? previousError, bool del } catch (Exception exception) when (!_stopToken.IsCancellationRequested) { + if (SocketModeNegotiateException.IsNonRetryable(exception)) + { + StopAfterNonRetryable(exception); + return null; + } + error = exception; _logger.LogWarning( exception, @@ -302,6 +325,24 @@ private async Task ReconnectAsync(Exception? previousError, bool del } } + /// + /// Gives up on this geo: stops every connection it owns, including one still serving during a token rotation, + /// and reports it disconnected. Other geos are unaffected. + /// + /// The non-retryable failure. + private void StopAfterNonRetryable(Exception error) + { + _logger.LogError( + error, + "Socket Mode geo {Geo} reconnect was rejected; inbound delivery for this geo has stopped until the app is restarted.", + Geo); + + // Not awaited: stopping waits for this supervisor to finish. Marking the geo as stopping happens + // synchronously, so closing connections cannot report a second disconnect. + _ = StopAsync(); + _owner.OnGeoDisconnected(Geo, error); + } + private async Task ConnectAsync(CancellationToken cancellationToken) { Generation generation; diff --git a/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs b/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs index 20a19203..cb15f6ee 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/SocketModeNegotiator.cs @@ -240,7 +240,7 @@ internal sealed class SocketModeNegotiateException : Exception internal SocketModeNegotiateException( HttpStatusCode statusCode, TimeSpan? retryAfter) - : base($"Socket Mode negotiate failed with HTTP {(int)statusCode}.") + : base(CreateMessage(statusCode)) { StatusCode = statusCode; RetryAfter = retryAfter; @@ -255,4 +255,32 @@ internal SocketModeNegotiateException( /// Gets the server-provided retry delay, when available. /// internal TimeSpan? RetryAfter { get; } + + /// + /// Gets whether the service rejected the bot (HTTP 401 or 403), which retrying cannot fix. + /// + internal bool IsAuthError => StatusCode is HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden; + + /// + /// Determines whether a connection failure must not be retried. + /// + /// The connection failure. + /// when negotiate rejected the bot with HTTP 401 or 403. + internal static bool IsNonRetryable(Exception? error) + => error is SocketModeNegotiateException { IsAuthError: true }; + + private static string CreateMessage(HttpStatusCode statusCode) + { + string failure = $"Socket Mode negotiate failed with HTTP {(int)statusCode}."; + return statusCode switch + { + HttpStatusCode.Unauthorized => + $"{failure} The bot could not be authenticated: verify the bot credentials (client ID and secret, " + + "certificate, or managed identity) and restart the app after correcting them.", + HttpStatusCode.Forbidden => + $"{failure} This bot is not authorized to use Socket Mode: verify the bot registration and Socket Mode " + + "access for this environment, then restart the app.", + _ => failure, + }; + } } diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs index 995f2d80..2420d87e 100644 --- a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs @@ -98,6 +98,70 @@ public async Task StartAsync_ZeroBudgetMakesOneAttemptWithoutRetry() Assert.Equal(1, harness.Factory.CreateCount); } + [Theory] + [InlineData(HttpStatusCode.Unauthorized)] + [InlineData(HttpStatusCode.Forbidden)] + public async Task StartAsync_AuthRejectionFailsWithoutRetry(HttpStatusCode status) + { + Harness harness = new(); + SocketModeNegotiateException rejected = new(status, retryAfter: null); + FakeConnection connection = harness.Factory.Enqueue(startError: rejected); + + SocketModeNegotiateException error = + await Assert.ThrowsAsync(() => harness.Socket.StartAsync()); + + Assert.Same(rejected, error); + Assert.Equal(1, harness.Factory.CreateCount); + Assert.Equal(0, harness.Time.ScheduledTimerCount); + Assert.Equal(1, connection.DisposeCount); + Assert.Empty(harness.Logger.Warnings); + } + + [Fact] + public async Task Reconnect_AuthRejectionStopsTheGeo() + { + Harness harness = new() { Backoff = _ => TimeSpan.FromSeconds(3) }; + FakeConnection initial = harness.Factory.Enqueue(); + SocketModeNegotiateException rejected = new(HttpStatusCode.Unauthorized, retryAfter: null); + FakeConnection rejectedAttempt = harness.Factory.Enqueue(startError: rejected); + await harness.StartReadyAsync(initial); + + IOException dropped = new("dropped"); + initial.Close(dropped); + await harness.Time.WaitForTimerAsync(TimeSpan.FromSeconds(3)); + harness.Time.Advance(TimeSpan.FromSeconds(3)); + await WaitUntilAsync(() => harness.Owner.Disconnections.Count == 2); + await harness.Socket.StopAsync(); + + Assert.Equal([dropped, rejected], harness.Owner.Disconnections); + Assert.Same(rejected, Assert.Single(harness.Logger.Errors)); + Assert.Equal(2, harness.Factory.CreateCount); + Assert.Equal(0, harness.Time.ScheduledTimerCount); + Assert.Equal(1, rejectedAttempt.DisposeCount); + Assert.Equal(0, harness.Owner.Reconnections); + } + + [Fact] + public async Task Rotation_AuthRejectionStopsTheConnectionStillServing() + { + Harness harness = new(); + FakeConnection initial = harness.Factory.Enqueue(TimeSpan.FromSeconds(10)); + SocketModeNegotiateException rejected = new(HttpStatusCode.Forbidden, retryAfter: null); + harness.Factory.Enqueue(startError: rejected); + await harness.StartReadyAsync(initial); + + harness.Time.Advance(TimeSpan.FromSeconds(5)); + await initial.Disposed.Task; + await WaitUntilAsync(() => harness.Owner.Disconnections.Count == 1); + + Assert.Same(rejected, Assert.Single(harness.Owner.Disconnections)); + Assert.Equal(1, initial.StopCount); + Assert.Null(await initial.Activity("after-rejection")); + Assert.DoesNotContain("after-rejection", harness.Owner.Dispatched); + Assert.Equal(2, harness.Factory.CreateCount); + Assert.DoesNotContain(harness.Logger.Warnings, warning => warning.Message.Contains("paused", StringComparison.Ordinal)); + } + [Fact] public async Task UnexpectedClose_ReportsDisconnectedThenReconnected() { @@ -607,6 +671,18 @@ private void RaiseClosed(Exception? error, bool planned) private sealed class RecordingLogger : ILogger { private readonly List<(string Message, Exception? Exception)> _warnings = []; + private readonly List _errors = []; + + internal Exception?[] Errors + { + get + { + lock (_errors) + { + return [.. _errors]; + } + } + } internal (string Message, Exception? Exception)[] Warnings { @@ -638,6 +714,13 @@ public void Log( _warnings.Add((formatter(state, exception), exception)); } } + else if (logLevel == LogLevel.Error) + { + lock (_errors) + { + _errors.Add(exception); + } + } } } diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeNegotiatorTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeNegotiatorTests.cs index cd017376..5be6b3e3 100644 --- a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeNegotiatorTests.cs +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/SocketModeNegotiatorTests.cs @@ -216,6 +216,36 @@ await Assert.ThrowsAsync( Assert.Null(exception.RetryAfter); } + [Theory] + [InlineData(HttpStatusCode.Unauthorized, "verify the bot credentials")] + [InlineData(HttpStatusCode.Forbidden, "not authorized to use Socket Mode")] + public async Task NegotiateAsync_AuthFailureIsNonRetryableWithActionableMessage(HttpStatusCode status, string guidance) + { + RecordingHandler handler = new((_, _) => Task.FromResult(new HttpResponseMessage(status))); + SocketModeNegotiator negotiator = CreateNegotiator(handler); + + SocketModeNegotiateException exception = + await Assert.ThrowsAsync( + () => negotiator.NegotiateAsync(NegotiateUri)); + + Assert.True(exception.IsAuthError); + Assert.True(SocketModeNegotiateException.IsNonRetryable(exception)); + Assert.Contains($"HTTP {(int)status}", exception.Message, StringComparison.Ordinal); + Assert.Contains(guidance, exception.Message, StringComparison.Ordinal); + } + + [Theory] + [InlineData(HttpStatusCode.TooManyRequests)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + public void OtherNegotiateFailuresRemainRetryable(HttpStatusCode status) + { + SocketModeNegotiateException exception = new(status, retryAfter: null); + + Assert.False(exception.IsAuthError); + Assert.False(SocketModeNegotiateException.IsNonRetryable(exception)); + Assert.False(SocketModeNegotiateException.IsNonRetryable(new IOException("dropped"))); + } + [Fact] public async Task NegotiateAsync_HonorsCallerCancellation() { From 7e8f5fe93032c92d584e3e6c44b49a3f988865ab Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Thu, 1 Oct 2026 10:44:45 -0700 Subject: [PATCH 30/31] removed delayFirstAttempt --- .../SocketMode/GeoSocket.cs | 12 ++-- .../SocketMode/GeoSocketTests.cs | 60 ++++++++++++++++--- 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs b/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs index 2877467c..dc943bfa 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs @@ -258,7 +258,7 @@ private async Task SuperviseAsync(Generation current) if (reason.Planned) { _logger.LogInformation("Socket Mode geo {Geo} rotating connection before token expiry.", Geo); - replacement = await ReconnectAsync(null, delayFirstAttempt: false).ConfigureAwait(false); + replacement = await ReconnectAsync(null).ConfigureAwait(false); if (replacement is not null) { StartRetirement(current); @@ -267,7 +267,7 @@ private async Task SuperviseAsync(Generation current) else { await ReleaseAsync(current.Connection).ConfigureAwait(false); - replacement = await ReconnectAsync(reason.Error, delayFirstAttempt: true).ConfigureAwait(false); + replacement = await ReconnectAsync(reason.Error).ConfigureAwait(false); } if (replacement is null) @@ -291,17 +291,13 @@ private async Task SuperviseAsync(Generation current) } /// The ready replacement, or when the geo gave up after a non-retryable failure. - private async Task ReconnectAsync(Exception? previousError, bool delayFirstAttempt) + private async Task ReconnectAsync(Exception? previousError) { Exception? error = previousError; - int retry = 0; for (int attempt = 1; ; attempt++) { - if (delayFirstAttempt || attempt > 1) - { - await Task.Delay(GetRetryDelay(error, retry++), _timeProvider, _stopToken).ConfigureAwait(false); - } + await Task.Delay(GetRetryDelay(error, attempt - 1), _timeProvider, _stopToken).ConfigureAwait(false); try { diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs index 2420d87e..ce25644f 100644 --- a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs @@ -147,10 +147,11 @@ public async Task Rotation_AuthRejectionStopsTheConnectionStillServing() Harness harness = new(); FakeConnection initial = harness.Factory.Enqueue(TimeSpan.FromSeconds(10)); SocketModeNegotiateException rejected = new(HttpStatusCode.Forbidden, retryAfter: null); - harness.Factory.Enqueue(startError: rejected); + FakeConnection rejectedAttempt = harness.Factory.Enqueue(startError: rejected); await harness.StartReadyAsync(initial); harness.Time.Advance(TimeSpan.FromSeconds(5)); + await harness.AdvanceThroughBackoffAsync(rejectedAttempt); await initial.Disposed.Task; await WaitUntilAsync(() => harness.Owner.Disconnections.Count == 1); @@ -195,7 +196,7 @@ public async Task Rotation_IsMakeBeforeBreakWithHandoff() await harness.StartReadyAsync(initial); harness.Time.Advance(TimeSpan.FromSeconds(5)); - await replacement.Started.Task; + await harness.AdvanceThroughBackoffAsync(replacement); Assert.NotNull(await initial.Activity("during-replacement-startup")); Assert.Equal(0, initial.StopCount); @@ -226,7 +227,44 @@ public async Task Rotation_UsesOneSecondMinimumDelay() Assert.True(harness.Time.HasTimerDueIn(TimeSpan.FromSeconds(1))); harness.Time.Advance(TimeSpan.FromSeconds(1)); + await harness.AdvanceThroughBackoffAsync(replacement); + } + + [Fact] + public async Task Rotation_WaitsForFirstBackoffDelayBeforeConnectingReplacement() + { + List attempts = []; + Harness harness = new() + { + Backoff = attempt => + { + lock (attempts) + { + attempts.Add(attempt); + } + + return TimeSpan.FromMilliseconds(700); + }, + }; + FakeConnection initial = harness.Factory.Enqueue(TimeSpan.FromSeconds(10)); + FakeConnection replacement = harness.Factory.Enqueue(); + await harness.StartReadyAsync(initial); + + harness.Time.Advance(TimeSpan.FromSeconds(5)); + await harness.Time.WaitForTimerAsync(TimeSpan.FromMilliseconds(700)); + + Assert.Equal(1, harness.Factory.CreateCount); + lock (attempts) + { + Assert.Equal([0], attempts); + } + + Assert.NotNull(await initial.Activity("during-backoff")); + + harness.Time.Advance(TimeSpan.FromMilliseconds(700)); await replacement.Started.Task; + + Assert.Equal(2, harness.Factory.CreateCount); } [Fact] @@ -251,7 +289,7 @@ public async Task PredecessorDropDuringRotation_ReportsOutageUntilReplacementRea await harness.StartReadyAsync(initial); harness.Time.Advance(TimeSpan.FromSeconds(5)); - await replacement.Started.Task; + await harness.AdvanceThroughBackoffAsync(replacement); initial.Close(new IOException("dropped")); Assert.Single(harness.Owner.Disconnections); @@ -283,7 +321,7 @@ public async Task RepeatedRotation_ReleasesEveryPredecessorOnce() { await harness.Time.WaitForTimerAsync(TimeSpan.FromSeconds(5)); harness.Time.Advance(TimeSpan.FromSeconds(5)); - await connections[i].Started.Task; + await harness.AdvanceThroughBackoffAsync(connections[i]); connections[i].Ready($"gen-{i}"); await connections[i - 1].Disposed.Task; } @@ -378,7 +416,7 @@ public async Task StopAsync_DuringRotation_DisposesActiveAndConnectingOnce() await harness.StartReadyAsync(active); harness.Time.Advance(TimeSpan.FromSeconds(5)); - await connecting.Started.Task; + await harness.AdvanceThroughBackoffAsync(connecting); await harness.Socket.StopAsync(); Assert.Equal(1, active.StopCount); @@ -398,7 +436,7 @@ public async Task StopAsync_DuringHandoff_DisposesActiveAndRetiringOnce() await harness.StartReadyAsync(retiring); harness.Time.Advance(TimeSpan.FromSeconds(5)); - await active.Started.Task; + await harness.AdvanceThroughBackoffAsync(active); active.Ready("replacement"); await harness.Time.WaitForTimerAsync(TimeSpan.FromMinutes(1)); await harness.Socket.StopAsync(); @@ -439,7 +477,7 @@ public async Task Retirement_LogsDisposeFailureAndKeepsReplacementActive() await harness.StartReadyAsync(retiring); harness.Time.Advance(TimeSpan.FromSeconds(5)); - await active.Started.Task; + await harness.AdvanceThroughBackoffAsync(active); active.Ready("replacement"); await harness.Time.WaitForTimerAsync(TimeSpan.FromSeconds(5)); harness.Time.Advance(TimeSpan.FromSeconds(5)); @@ -498,6 +536,14 @@ internal async Task StartReadyAsync(FakeConnection connection) connection.Ready("initial"); await start; } + + internal async Task AdvanceThroughBackoffAsync(FakeConnection next) + { + TimeSpan delay = Owner.Backoff(0); + await Time.WaitForTimerAsync(delay); + Time.Advance(delay); + await next.Started.Task; + } } private sealed class FakeOwner : IGeoSocketOwner From 97e82489795335db90edbb9a099e0388f38057e1 Mon Sep 17 00:00:00 2001 From: Teddy Arida-Moody Date: Thu, 1 Oct 2026 11:19:26 -0700 Subject: [PATCH 31/31] lily comments --- .../SocketMode/GeoSocket.cs | 38 ++++++++-- .../SocketMode/GeoSocketTests.cs | 72 ++++++++++++++++++- 2 files changed, 102 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs b/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs index dc943bfa..45b7725c 100644 --- a/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs +++ b/src/Microsoft.Teams.Apps/SocketMode/GeoSocket.cs @@ -89,6 +89,7 @@ internal sealed class GeoSocket : IAsyncDisposable private readonly HashSet _owned = []; private readonly HashSet _retiring = []; private readonly HashSet _retirements = []; + private readonly HashSet _releases = []; private long _generation; private Generation? _active; @@ -449,14 +450,20 @@ private void HandleClosed(Generation generation, Exception? error, bool planned) disconnected = !planned && !_stopping; _disconnected |= disconnected; } - - generation.Closed.TrySetResult(new CloseReason(error, planned)); } - if (disconnected) + // Report the disconnect before waking the supervisor so a fast reconnect cannot be reported first. + try + { + if (disconnected) + { + _logger.LogWarning(error, "Socket Mode geo {Geo} disconnected; inbound delivery paused.", Geo); + _owner.OnGeoDisconnected(Geo, error); + } + } + finally { - _logger.LogWarning(error, "Socket Mode geo {Geo} disconnected; inbound delivery paused.", Geo); - _owner.OnGeoDisconnected(Geo, error); + generation.Closed.TrySetResult(new CloseReason(error, planned)); } } @@ -570,6 +577,7 @@ private async Task StopCoreAsync() ISocketConnection[] connections; Task[] retirements; + Task[] releases; Task? supervisor; lock (_sync) { @@ -580,6 +588,7 @@ private async Task StopCoreAsync() connections = [.. _owned]; _owned.Clear(); retirements = [.. _retirements]; + releases = [.. _releases]; supervisor = _supervisor; } @@ -590,6 +599,7 @@ private async Task StopCoreAsync() finally { await Task.WhenAll(retirements).ConfigureAwait(false); + await Task.WhenAll(releases).ConfigureAwait(false); if (supervisor is not null) { await supervisor.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); @@ -599,15 +609,31 @@ private async Task StopCoreAsync() private async Task ReleaseAsync(ISocketConnection connection) { + TaskCompletionSource released = new(TaskCreationOptions.RunContinuationsAsynchronously); lock (_sync) { if (!_owned.Remove(connection)) { return; } + + // Registered with the ownership claim so StopAsync can await cleanup it no longer owns. + _releases.Add(released.Task); } - await StopAndDisposeAsync(connection).ConfigureAwait(false); + try + { + await StopAndDisposeAsync(connection).ConfigureAwait(false); + } + finally + { + lock (_sync) + { + _releases.Remove(released.Task); + } + + released.TrySetResult(); + } } [SuppressMessage( diff --git a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs index ce25644f..b092dfe4 100644 --- a/test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs +++ b/test/Microsoft.Teams.Apps.UnitTests/SocketMode/GeoSocketTests.cs @@ -187,6 +187,38 @@ public async Task UnexpectedClose_ReportsDisconnectedThenReconnected() Assert.DoesNotContain("after-close", harness.Owner.Dispatched); } + [Fact] + public async Task UnexpectedClose_ReportsDisconnectedBeforeSupervisorReconnects() + { + Harness harness = new() { Backoff = _ => TimeSpan.Zero }; + FakeConnection initial = harness.Factory.Enqueue(); + FakeConnection replacement = harness.Factory.Enqueue(); + await harness.StartReadyAsync(initial); + + using ManualResetEventSlim entered = new(); + using ManualResetEventSlim release = new(); + harness.Owner.Disconnecting = () => + { + entered.Set(); + release.Wait(TimeSpan.FromSeconds(5)); + }; + + Task closing = Task.Run(() => initial.Close(new IOException("dropped"))); + Assert.True(entered.Wait(TimeSpan.FromSeconds(5))); + + Assert.False(SpinWait.SpinUntil(() => initial.DisposeCount > 0, TimeSpan.FromMilliseconds(200))); + Assert.Equal(1, harness.Factory.CreateCount); + + release.Set(); + await closing; + await replacement.Started.Task; + replacement.Ready("replacement"); + await WaitUntilAsync(() => harness.Owner.Reconnections == 1); + + Assert.Single(harness.Owner.Disconnections); + Assert.Equal(1, initial.DisposeCount); + } + [Fact] public async Task Rotation_IsMakeBeforeBreakWithHandoff() { @@ -449,6 +481,29 @@ public async Task StopAsync_DuringHandoff_DisposesActiveAndRetiringOnce() Assert.Empty(harness.Owner.Disconnections); } + [Fact] + public async Task StopAsync_WaitsForFailedStartupAttemptCleanup() + { + Harness harness = new(); + TaskCompletionSource disposeGate = new(TaskCreationOptions.RunContinuationsAsynchronously); + FakeConnection failed = harness.Factory.Enqueue(startError: new IOException("connect failed")); + failed.DisposeGate = disposeGate.Task; + + Task start = harness.Socket.StartAsync(); + await failed.Disposed.Task; + + Task stop = harness.Socket.StopAsync(); + Assert.NotSame(stop, await Task.WhenAny(stop, Task.Delay(TimeSpan.FromMilliseconds(200)))); + + disposeGate.SetResult(); + await stop; + await Assert.ThrowsAsync(() => start); + + Assert.Equal(1, failed.StopCount); + Assert.Equal(1, failed.DisposeCount); + Assert.Equal(1, harness.Factory.CreateCount); + } + [Fact] public async Task StopAsync_LogsConnectionCleanupFailuresAndStillDisposes() { @@ -593,8 +648,11 @@ public void OnGeoReady(string geo, SocketReadyFrame frame) } } + internal Action? Disconnecting { get; set; } + public void OnGeoDisconnected(string geo, Exception? error) { + Disconnecting?.Invoke(); lock (_sync) { _disconnections.Add(error); @@ -687,11 +745,21 @@ public Task StopAsync(CancellationToken cancellationToken = default) return StopError is null ? Task.CompletedTask : Task.FromException(StopError); } - public ValueTask DisposeAsync() + internal Task? DisposeGate { get; set; } + + public async ValueTask DisposeAsync() { Interlocked.Increment(ref _disposeCount); Disposed.TrySetResult(); - return DisposeError is null ? ValueTask.CompletedTask : ValueTask.FromException(DisposeError); + if (DisposeGate is not null) + { + await DisposeGate; + } + + if (DisposeError is not null) + { + throw DisposeError; + } } internal void Ready(string connectionId)