From 37b7b1ad5d929de5c084d5b7109ad1868754c610 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Mon, 21 Sep 2026 12:23:15 -0400 Subject: [PATCH 01/13] feat(cluster): preserve independent replication identity Signed-off-by: Yordis Prieto --- proto.lock | 5 + .../Cluster/MemberInfoTests.cs | 95 +++++++++++++++++++ src/EventStore.Core/Cluster/ClusterInfo.cs | 9 +- src/EventStore.Core/Cluster/MemberInfo.cs | 33 +++++-- .../Data/GossipAdvertiseInfo.cs | 7 +- src/EventStore.Core/Data/VNodeInfo.cs | 15 ++- src/Protos/Grpc/cluster.proto | 1 + 7 files changed, 147 insertions(+), 18 deletions(-) diff --git a/proto.lock b/proto.lock index 068485f54f..0534f18460 100644 --- a/proto.lock +++ b/proto.lock @@ -1988,6 +1988,11 @@ "id": 21, "name": "es_version", "type": "string" + }, + { + "id": 22, + "name": "replication_end_point", + "type": "EndPoint" } ] } diff --git a/src/EventStore.Core.Tests/Cluster/MemberInfoTests.cs b/src/EventStore.Core.Tests/Cluster/MemberInfoTests.cs index 14d94644ce..04bc15faac 100644 --- a/src/EventStore.Core.Tests/Cluster/MemberInfoTests.cs +++ b/src/EventStore.Core.Tests/Cluster/MemberInfoTests.cs @@ -1,5 +1,6 @@ using System; using System.Net; +using System.Reflection; using EventStore.Core.Data; using NUnit.Framework; @@ -8,6 +9,13 @@ namespace EventStore.Core.Tests.Cluster; [TestFixture] public class MemberInfoTests { + private static readonly DnsEndPoint InternalTcp = new("internal", 1112); + private static readonly DnsEndPoint InternalSecureTcp = new("internal-secure", 2112); + private static readonly DnsEndPoint ExternalTcp = new("external", 1113); + private static readonly DnsEndPoint ExternalSecureTcp = new("external-secure", 2113); + private static readonly DnsEndPoint Http = new("http", 2113); + private static readonly DnsEndPoint Replication = new("replication", 3113); + [Test] public void member_with_dns_endpoint_should_equal() { @@ -50,4 +58,91 @@ public void member_with_ip_endpoint_should_equal() Assert.True(memberWithDnsEndPoint.Is(ipEndPoint)); Assert.True(memberWithDnsEndPoint.Is(dnsEndPoint)); } + + [Test] + public void grpc_round_trip_preserves_tcp_and_replication_endpoints() + { + var member = EventStore.Core.Cluster.MemberInfo.Initial(Guid.NewGuid(), DateTime.UtcNow, + VNodeState.Unknown, true, + InternalTcp, null, null, ExternalSecureTcp, Http, + "client", 2113, 1113, 0, false, replicationEndPoint: Replication); + + var result = FromGrpcClusterInfo(ToGrpcClusterInfo( + new EventStore.Core.Cluster.ClusterInfo(member))).Members[0]; + + Assert.That(result.InternalTcpEndPoint, Is.EqualTo(InternalTcp)); + Assert.That(result.InternalSecureTcpEndPoint, Is.Null); + Assert.That(result.ExternalTcpEndPoint, Is.Null); + Assert.That(result.ExternalSecureTcpEndPoint, Is.EqualTo(ExternalSecureTcp)); + Assert.That(result.HttpEndPoint, Is.EqualTo(Http)); + Assert.That(result.ReplicationEndPoint, Is.EqualTo(Replication)); + } + + [Test] + public void explicit_replication_endpoint_is_recognized_without_replacing_tcp_endpoints() + { + var member = CreateMember(Replication); + var vnode = new VNodeInfo(Guid.NewGuid(), 0, + new IPEndPoint(IPAddress.Loopback, 1112), null, + new IPEndPoint(IPAddress.Loopback, 1113), null, + Http, false, Replication); + var advertise = new GossipAdvertiseInfo( + InternalTcp, InternalSecureTcp, ExternalTcp, ExternalSecureTcp, Http, + null, null, 0, null, 0, 0, Replication); + + Assert.That(member.Is(Replication), Is.True); + Assert.That(member.InternalTcpEndPoint, Is.EqualTo(InternalTcp)); + Assert.That(member.InternalSecureTcpEndPoint, Is.EqualTo(InternalSecureTcp)); + Assert.That(member.ExternalTcpEndPoint, Is.EqualTo(ExternalTcp)); + Assert.That(member.ExternalSecureTcpEndPoint, Is.EqualTo(ExternalSecureTcp)); + Assert.That(vnode.ReplicationEndPoint, Is.SameAs(Replication)); + Assert.That(advertise.ReplicationEndPoint, Is.SameAs(Replication)); + } + + [Test] + public void missing_replication_endpoint_falls_back_to_http_endpoint() + { + var member = CreateMember(); + var vnode = new VNodeInfo(Guid.NewGuid(), 0, + new IPEndPoint(IPAddress.Loopback, 1112), null, + new IPEndPoint(IPAddress.Loopback, 1113), null, + Http, false); + var advertise = new GossipAdvertiseInfo( + InternalTcp, InternalSecureTcp, ExternalTcp, ExternalSecureTcp, Http, + null, null, 0, null, 0, 0); + + Assert.That(member.ReplicationEndPoint, Is.SameAs(Http)); + Assert.That(vnode.ReplicationEndPoint, Is.SameAs(Http)); + Assert.That(advertise.ReplicationEndPoint, Is.SameAs(Http)); + } + + [Test] + public void grpc_member_without_replication_endpoint_falls_back_to_http_endpoint() + { + var grpcCluster = ToGrpcClusterInfo( + new EventStore.Core.Cluster.ClusterInfo(CreateMember(Replication))); + grpcCluster.Members[0].ReplicationEndPoint = null; + + var result = FromGrpcClusterInfo(grpcCluster).Members[0]; + + Assert.That(result.ReplicationEndPoint, Is.EqualTo(Http)); + } + + private static EventStore.Core.Cluster.MemberInfo CreateMember(DnsEndPoint replicationEndPoint = null) => + EventStore.Core.Cluster.MemberInfo.Initial(Guid.NewGuid(), DateTime.UtcNow, + VNodeState.Unknown, true, + InternalTcp, InternalSecureTcp, ExternalTcp, ExternalSecureTcp, Http, + "client", 2113, 1113, 0, false, replicationEndPoint: replicationEndPoint); + + private static EventStore.Cluster.ClusterInfo ToGrpcClusterInfo( + EventStore.Core.Cluster.ClusterInfo clusterInfo) => + (EventStore.Cluster.ClusterInfo)typeof(EventStore.Core.Cluster.ClusterInfo) + .GetMethod("ToGrpcClusterInfo", BindingFlags.NonPublic | BindingFlags.Static)! + .Invoke(null, [clusterInfo])!; + + private static EventStore.Core.Cluster.ClusterInfo FromGrpcClusterInfo( + EventStore.Cluster.ClusterInfo clusterInfo) => + (EventStore.Core.Cluster.ClusterInfo)typeof(EventStore.Core.Cluster.ClusterInfo) + .GetMethod("FromGrpcClusterInfo", BindingFlags.NonPublic | BindingFlags.Static)! + .Invoke(null, [clusterInfo, null])!; } diff --git a/src/EventStore.Core/Cluster/ClusterInfo.cs b/src/EventStore.Core/Cluster/ClusterInfo.cs index 42bf18fb0e..6276119ed3 100644 --- a/src/EventStore.Core/Cluster/ClusterInfo.cs +++ b/src/EventStore.Core/Cluster/ClusterInfo.cs @@ -83,7 +83,11 @@ internal static ClusterInfo FromGrpcClusterInfo(EventStore.Cluster.ClusterInfo g x.AdvertiseHostToClientAs, (int)x.AdvertiseHttpPortToClientAs, (int)x.AdvertiseTcpPortToClientAs, x.LastCommitPosition, x.WriterCheckpoint, x.ChaserCheckpoint, x.EpochPosition, x.EpochNumber, Uuid.FromDto(x.EpochId).ToGuid(), x.NodePriority, - x.IsReadOnlyReplica, x.EsVersion == String.Empty ? null : x.EsVersion + x.IsReadOnlyReplica, x.EsVersion == String.Empty ? null : x.EsVersion, + x.ReplicationEndPoint is null + ? null + : new DnsEndPoint(x.ReplicationEndPoint.Address, (int)x.ReplicationEndPoint.Port) + .WithClusterDns(clusterDns) )).ToArray(); return new ClusterInfo(receivedMembers); } @@ -99,6 +103,9 @@ internal static EventStore.Cluster.ClusterInfo ToGrpcClusterInfo(ClusterInfo clu HttpEndPoint = new EventStore.Cluster.EndPoint( x.HttpEndPoint.GetHost(), (uint)x.HttpEndPoint.GetPort()), + ReplicationEndPoint = new EventStore.Cluster.EndPoint( + x.ReplicationEndPoint.GetHost(), + (uint)x.ReplicationEndPoint.GetPort()), InternalTcp = x.InternalSecureTcpEndPoint != null ? new EventStore.Cluster.EndPoint( x.InternalSecureTcpEndPoint.GetHost(), diff --git a/src/EventStore.Core/Cluster/MemberInfo.cs b/src/EventStore.Core/Cluster/MemberInfo.cs index f006448cc1..fbcacf901c 100644 --- a/src/EventStore.Core/Cluster/MemberInfo.cs +++ b/src/EventStore.Core/Cluster/MemberInfo.cs @@ -20,6 +20,7 @@ public class MemberInfo : IEquatable public readonly EndPoint ExternalTcpEndPoint; public readonly EndPoint ExternalSecureTcpEndPoint; public readonly EndPoint HttpEndPoint; + public readonly EndPoint ReplicationEndPoint; public readonly string AdvertiseHostToClientAs; public readonly int AdvertiseHttpPortToClientAs; public readonly int AdvertiseTcpPortToClientAs; @@ -37,12 +38,13 @@ public class MemberInfo : IEquatable public readonly string ESVersion; public static MemberInfo ForManager(Guid instanceId, DateTime timeStamp, bool isAlive, - EndPoint httpEndPoint, string esVersion = VersionInfo.UnknownVersion) + EndPoint httpEndPoint, string esVersion = VersionInfo.UnknownVersion, + EndPoint replicationEndPoint = null) { return new MemberInfo(instanceId, timeStamp, VNodeState.Manager, isAlive, httpEndPoint, null, httpEndPoint, null, httpEndPoint, null, 0, 0, - -1, -1, -1, -1, -1, Guid.Empty, 0, false, esVersion); + -1, -1, -1, -1, -1, Guid.Empty, 0, false, esVersion, replicationEndPoint); } public static MemberInfo ForVNode(Guid instanceId, @@ -64,7 +66,8 @@ public static MemberInfo ForVNode(Guid instanceId, int epochNumber, Guid epochId, int nodePriority, - bool isReadOnlyReplica, string esVersion = VersionInfo.UnknownVersion) + bool isReadOnlyReplica, string esVersion = VersionInfo.UnknownVersion, + EndPoint replicationEndPoint = null) { if (state == VNodeState.Manager) { @@ -76,7 +79,8 @@ public static MemberInfo ForVNode(Guid instanceId, externalTcpEndPoint, externalSecureTcpEndPoint, httpEndPoint, advertiseHostToClientAs, advertiseHttpPortToClientAs, advertiseTcpPortToClientAs, lastCommitPosition, writerCheckpoint, chaserCheckpoint, - epochPosition, epochNumber, epochId, nodePriority, isReadOnlyReplica, esVersion); + epochPosition, epochNumber, epochId, nodePriority, isReadOnlyReplica, esVersion, + replicationEndPoint); } public static MemberInfo Initial(Guid instanceId, @@ -92,7 +96,8 @@ public static MemberInfo Initial(Guid instanceId, int advertiseHttpPortToClientAs, int advertiseTcpPortToClientAs, int nodePriority, - bool isReadOnlyReplica, string esVersion = VersionInfo.UnknownVersion) + bool isReadOnlyReplica, string esVersion = VersionInfo.UnknownVersion, + EndPoint replicationEndPoint = null) { if (state == VNodeState.Manager) { @@ -103,7 +108,8 @@ public static MemberInfo Initial(Guid instanceId, internalTcpEndPoint, internalSecureTcpEndPoint, externalTcpEndPoint, externalSecureTcpEndPoint, httpEndPoint, advertiseHostToClientAs, advertiseHttpPortToClientAs, advertiseTcpPortToClientAs, - -1, -1, -1, -1, -1, Guid.Empty, nodePriority, isReadOnlyReplica, esVersion); + -1, -1, -1, -1, -1, Guid.Empty, nodePriority, isReadOnlyReplica, esVersion, + replicationEndPoint); } internal MemberInfo(Guid instanceId, DateTime timeStamp, VNodeState state, bool isAlive, @@ -111,7 +117,8 @@ internal MemberInfo(Guid instanceId, DateTime timeStamp, VNodeState state, bool EndPoint externalTcpEndPoint, EndPoint externalSecureTcpEndPoint, EndPoint httpEndPoint, string advertiseHostToClientAs, int advertiseHttpPortToClientAs, int advertiseTcpPortToClientAs, long lastCommitPosition, long writerCheckpoint, long chaserCheckpoint, - long epochPosition, int epochNumber, Guid epochId, int nodePriority, bool isReadOnlyReplica, string esVersion = null) + long epochPosition, int epochNumber, Guid epochId, int nodePriority, bool isReadOnlyReplica, + string esVersion = null, EndPoint replicationEndPoint = null) { Ensure.Equal(false, internalTcpEndPoint == null && internalSecureTcpEndPoint == null, "Both internal TCP endpoints are null"); Ensure.NotNull(httpEndPoint, nameof(httpEndPoint)); @@ -127,6 +134,7 @@ internal MemberInfo(Guid instanceId, DateTime timeStamp, VNodeState state, bool ExternalTcpEndPoint = externalTcpEndPoint; ExternalSecureTcpEndPoint = externalSecureTcpEndPoint; HttpEndPoint = httpEndPoint; + ReplicationEndPoint = replicationEndPoint ?? httpEndPoint; AdvertiseHostToClientAs = advertiseHostToClientAs; AdvertiseHttpPortToClientAs = advertiseHttpPortToClientAs; AdvertiseTcpPortToClientAs = advertiseTcpPortToClientAs; @@ -160,6 +168,7 @@ internal MemberInfo(MemberInfoDto dto) ? new DnsEndPoint(dto.ExternalTcpIp, dto.ExternalSecureTcpPort) : null; HttpEndPoint = new DnsEndPoint(dto.HttpEndPointIp, dto.HttpEndPointPort); + ReplicationEndPoint = HttpEndPoint; AdvertiseHostToClientAs = dto.AdvertiseHostToClientAs; AdvertiseHttpPortToClientAs = dto.AdvertiseHttpPortToClientAs; AdvertiseTcpPortToClientAs = dto.AdvertiseTcpPortToClientAs; @@ -176,11 +185,12 @@ internal MemberInfo(MemberInfoDto dto) public bool Is(EndPoint endPoint) { return endPoint != null - && HttpEndPoint.EndPointEquals(endPoint) + && (HttpEndPoint.EndPointEquals(endPoint) + || ReplicationEndPoint.EndPointEquals(endPoint) || (InternalTcpEndPoint != null && InternalTcpEndPoint.EndPointEquals(endPoint)) || (InternalSecureTcpEndPoint != null && InternalSecureTcpEndPoint.EndPointEquals(endPoint)) || (ExternalTcpEndPoint != null && ExternalTcpEndPoint.EndPointEquals(endPoint)) - || (ExternalSecureTcpEndPoint != null && ExternalSecureTcpEndPoint.EndPointEquals(endPoint)); + || (ExternalSecureTcpEndPoint != null && ExternalSecureTcpEndPoint.EndPointEquals(endPoint))); } public MemberInfo Updated(DateTime utcNow, @@ -211,7 +221,7 @@ public MemberInfo Updated(DateTime utcNow, epoch != null ? epoch.EpochNumber : EpochNumber, epoch != null ? epoch.EpochId : EpochId, nodePriority ?? NodePriority, - IsReadOnlyReplica, esVersion ?? ESVersion); + IsReadOnlyReplica, esVersion ?? ESVersion, ReplicationEndPoint); } public override string ToString() @@ -228,6 +238,7 @@ public override string ToString() $"{(InternalSecureTcpEndPoint == null ? "n/a" : InternalSecureTcpEndPoint.ToString())}, " + $"{(ExternalTcpEndPoint == null ? "n/a" : ExternalTcpEndPoint.ToString())}, " + $"{(ExternalSecureTcpEndPoint == null ? "n/a" : ExternalSecureTcpEndPoint.ToString())}, " + + $"Replication:{ReplicationEndPoint}, " + $"{HttpEndPoint}, (ADVERTISED: HTTP:{AdvertiseHostToClientAs}:{AdvertiseHttpPortToClientAs}, TCP:{AdvertiseHostToClientAs}:{AdvertiseTcpPortToClientAs}), " + $"Version: {ESVersion}] " + $"{LastCommitPosition}/{WriterCheckpoint}/{ChaserCheckpoint}/E{EpochNumber}@{EpochPosition}:{EpochId:B} | {TimeStamp:yyyy-MM-dd HH:mm:ss.fff}"; @@ -254,6 +265,7 @@ public bool Equals(MemberInfo other) && Equals(other.ExternalTcpEndPoint, ExternalTcpEndPoint) && Equals(other.ExternalSecureTcpEndPoint, ExternalSecureTcpEndPoint) && Equals(other.HttpEndPoint, HttpEndPoint) + && Equals(other.ReplicationEndPoint, ReplicationEndPoint) && other.AdvertiseHostToClientAs == AdvertiseHostToClientAs && other.AdvertiseHttpPortToClientAs == AdvertiseHttpPortToClientAs && other.AdvertiseTcpPortToClientAs == AdvertiseTcpPortToClientAs @@ -299,6 +311,7 @@ public override int GetHashCode() result = (result * 397) ^ (ExternalSecureTcpEndPoint != null ? ExternalSecureTcpEndPoint.GetHashCode() : 0); result = (result * 397) ^ HttpEndPoint.GetHashCode(); + result = (result * 397) ^ ReplicationEndPoint.GetHashCode(); result = (result * 397) ^ (AdvertiseHostToClientAs != null ? AdvertiseHostToClientAs.GetHashCode() : 0); result = (result * 397) ^ AdvertiseHttpPortToClientAs.GetHashCode(); result = (result * 397) ^ AdvertiseTcpPortToClientAs.GetHashCode(); diff --git a/src/EventStore.Core/Data/GossipAdvertiseInfo.cs b/src/EventStore.Core/Data/GossipAdvertiseInfo.cs index 4c2c09cd25..4fcfcca51f 100644 --- a/src/EventStore.Core/Data/GossipAdvertiseInfo.cs +++ b/src/EventStore.Core/Data/GossipAdvertiseInfo.cs @@ -10,6 +10,7 @@ public class GossipAdvertiseInfo public DnsEndPoint ExternalTcp { get; } public DnsEndPoint ExternalSecureTcp { get; } public DnsEndPoint HttpEndPoint { get; } + public DnsEndPoint ReplicationEndPoint { get; } public string AdvertiseInternalHostAs { get; } public string AdvertiseExternalHostAs { get; } public int AdvertiseHttpPortAs { get; } @@ -21,7 +22,8 @@ public GossipAdvertiseInfo(DnsEndPoint internalTcp, DnsEndPoint internalSecureTc DnsEndPoint externalTcp, DnsEndPoint externalSecureTcp, DnsEndPoint httpEndPoint, string advertiseInternalHostAs, string advertiseExternalHostAs, int advertiseHttpPortAs, - string advertiseHostToClientAs, int advertiseHttpPortToClientAs, int advertiseTcpPortToClientAs) + string advertiseHostToClientAs, int advertiseHttpPortToClientAs, int advertiseTcpPortToClientAs, + DnsEndPoint replicationEndPoint = null) { Ensure.Equal(false, internalTcp == null && internalSecureTcp == null, "Both internal TCP endpoints are null"); @@ -30,6 +32,7 @@ public GossipAdvertiseInfo(DnsEndPoint internalTcp, DnsEndPoint internalSecureTc ExternalTcp = externalTcp; ExternalSecureTcp = externalSecureTcp; HttpEndPoint = httpEndPoint; + ReplicationEndPoint = replicationEndPoint ?? httpEndPoint; AdvertiseInternalHostAs = advertiseInternalHostAs; AdvertiseExternalHostAs = advertiseExternalHostAs; AdvertiseHttpPortAs = advertiseHttpPortAs; @@ -43,7 +46,7 @@ public override string ToString() return string.Format( $"IntTcp: {InternalTcp}, IntSecureTcp: {InternalSecureTcp}\n" + $"ExtTcp: {ExternalTcp}, ExtSecureTcp: {ExternalSecureTcp}\n" + - $"Http: {HttpEndPoint}, HttpAdvertiseAs: {AdvertiseExternalHostAs}:{AdvertiseHttpPortAs},\n" + + $"Http: {HttpEndPoint}, Replication: {ReplicationEndPoint}, HttpAdvertiseAs: {AdvertiseExternalHostAs}:{AdvertiseHttpPortAs},\n" + $"HttpAdvertiseToClientAs: {AdvertiseHostToClientAs}:{AdvertiseHttpPortToClientAs},\n" + $"TcpAdvertiseToClientAs: {AdvertiseHostToClientAs}:{AdvertiseTcpPortToClientAs}"); } diff --git a/src/EventStore.Core/Data/VNodeInfo.cs b/src/EventStore.Core/Data/VNodeInfo.cs index a217f5bbd5..d8aa86d500 100644 --- a/src/EventStore.Core/Data/VNodeInfo.cs +++ b/src/EventStore.Core/Data/VNodeInfo.cs @@ -13,13 +13,15 @@ public class VNodeInfo public readonly IPEndPoint ExternalTcp; public readonly IPEndPoint ExternalSecureTcp; public readonly EndPoint HttpEndPoint; + public readonly EndPoint ReplicationEndPoint; public readonly bool IsReadOnlyReplica; public VNodeInfo(Guid instanceId, int debugIndex, IPEndPoint internalTcp, IPEndPoint internalSecureTcp, IPEndPoint externalTcp, IPEndPoint externalSecureTcp, EndPoint httpEndPoint, - bool isReadOnlyReplica) + bool isReadOnlyReplica, + EndPoint replicationEndPoint = null) { Ensure.NotEmptyGuid(instanceId, "instanceId"); Ensure.Equal(false, internalTcp == null && internalSecureTcp == null, "Both internal TCP endpoints are null"); @@ -32,30 +34,33 @@ public VNodeInfo(Guid instanceId, int debugIndex, ExternalTcp = externalTcp; ExternalSecureTcp = externalSecureTcp; HttpEndPoint = httpEndPoint; + ReplicationEndPoint = replicationEndPoint ?? httpEndPoint; IsReadOnlyReplica = isReadOnlyReplica; } public bool Is(EndPoint endPoint) { return endPoint != null - && HttpEndPoint.Equals(endPoint) + && (HttpEndPoint.Equals(endPoint) + || ReplicationEndPoint.Equals(endPoint) || (InternalTcp != null && InternalTcp.Equals(endPoint)) || (InternalSecureTcp != null && InternalSecureTcp.Equals(endPoint)) || (ExternalTcp != null && ExternalTcp.Equals(endPoint)) - || (ExternalSecureTcp != null && ExternalSecureTcp.Equals(endPoint)); + || (ExternalSecureTcp != null && ExternalSecureTcp.Equals(endPoint))); } public override string ToString() { return string.Format("InstanceId: {0:B}, InternalTcp: {1}, InternalSecureTcp: {2}, " + - "ExternalTcp: {3}, ExternalSecureTcp: {4}, HttpEndPoint: {5}," + - "IsReadOnlyReplica: {6}", + "ExternalTcp: {3}, ExternalSecureTcp: {4}, HttpEndPoint: {5}, " + + "ReplicationEndPoint: {6}, IsReadOnlyReplica: {7}", InstanceId, InternalTcp, InternalSecureTcp, ExternalTcp, ExternalSecureTcp, HttpEndPoint, + ReplicationEndPoint, IsReadOnlyReplica); } } diff --git a/src/Protos/Grpc/cluster.proto b/src/Protos/Grpc/cluster.proto index e7e5cd25a1..5d1b4e9d1d 100644 --- a/src/Protos/Grpc/cluster.proto +++ b/src/Protos/Grpc/cluster.proto @@ -146,4 +146,5 @@ message MemberInfo { uint32 advertise_http_port_to_client_as = 19; uint32 advertise_tcp_port_to_client_as = 20; string es_version = 21; + EndPoint replication_end_point = 22; } From e322bb89578c8e104c2df8b7f6ecfc0d88bc1df3 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Mon, 21 Sep 2026 13:20:46 -0400 Subject: [PATCH 02/13] fix(cluster): preserve replication identity for monitoring Signed-off-by: Yordis Prieto --- src/EventStore.Core.Tests/Cluster/MemberInfoTests.cs | 10 ++++++++++ src/EventStore.Core/Cluster/ClientClusterInfo.cs | 6 +++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/EventStore.Core.Tests/Cluster/MemberInfoTests.cs b/src/EventStore.Core.Tests/Cluster/MemberInfoTests.cs index 04bc15faac..0db8ed8764 100644 --- a/src/EventStore.Core.Tests/Cluster/MemberInfoTests.cs +++ b/src/EventStore.Core.Tests/Cluster/MemberInfoTests.cs @@ -99,6 +99,16 @@ public void explicit_replication_endpoint_is_recognized_without_replacing_tcp_en Assert.That(advertise.ReplicationEndPoint, Is.SameAs(Replication)); } + [Test] + public void client_member_preserves_the_replication_endpoint() + { + var clientMember = new EventStore.Core.Cluster.ClientClusterInfo.ClientMemberInfo( + CreateMember(Replication)); + + Assert.That(clientMember.ReplicationEndPointIp, Is.EqualTo(Replication.Host)); + Assert.That(clientMember.ReplicationEndPointPort, Is.EqualTo(Replication.Port)); + } + [Test] public void missing_replication_endpoint_falls_back_to_http_endpoint() { diff --git a/src/EventStore.Core/Cluster/ClientClusterInfo.cs b/src/EventStore.Core/Cluster/ClientClusterInfo.cs index 442abee0d0..b1e516d8f8 100644 --- a/src/EventStore.Core/Cluster/ClientClusterInfo.cs +++ b/src/EventStore.Core/Cluster/ClientClusterInfo.cs @@ -47,6 +47,8 @@ public class ClientMemberInfo public string InternalHttpEndPointIp { get; set; } public int InternalHttpEndPointPort { get; set; } + public string ReplicationEndPointIp { get; set; } + public int ReplicationEndPointPort { get; set; } public string HttpEndPointIp { get; set; } public int HttpEndPointPort { get; set; } @@ -84,6 +86,8 @@ public ClientMemberInfo(MemberInfo member) InternalHttpEndPointIp = member.HttpEndPoint.GetHost(); InternalHttpEndPointPort = member.HttpEndPoint.GetPort(); + ReplicationEndPointIp = member.ReplicationEndPoint.GetHost(); + ReplicationEndPointPort = member.ReplicationEndPoint.GetPort(); HttpEndPointIp = string.IsNullOrEmpty(member.AdvertiseHostToClientAs) ? member.HttpEndPoint.GetHost() @@ -125,6 +129,7 @@ public override string ToString() $"InternalTcpIp: {InternalTcpIp}, InternalTcpPort: {InternalTcpPort}, InternalSecureTcpPort: {InternalSecureTcpPort}, " + $"ExternalTcpIp: {ExternalTcpIp}, ExternalTcpPort: {ExternalTcpPort}, ExternalSecureTcpPort: {ExternalSecureTcpPort}, " + $"InternalHttpEndPointIp: {InternalHttpEndPointIp}, InternalHttpEndPointPort: {InternalHttpEndPointPort}, " + + $"ReplicationEndPointIp: {ReplicationEndPointIp}, ReplicationEndPointPort: {ReplicationEndPointPort}, " + $"HttpEndPointIp: {HttpEndPointIp}, HttpEndPointPort: {HttpEndPointPort}, " + $"LastCommitPosition: {LastCommitPosition}, WriterCheckpoint: {WriterCheckpoint}, ChaserCheckpoint: {ChaserCheckpoint}, " + $"EpochPosition: {EpochPosition}, EpochNumber: {EpochNumber}, EpochId: {EpochId:B}, NodePriority: {NodePriority}, " + @@ -133,4 +138,3 @@ public override string ToString() } } } - From 9a1a9fcef7a9b010551569ec90a2c54745df298a Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Mon, 21 Sep 2026 16:11:54 -0400 Subject: [PATCH 03/13] fix(cluster): advertise the replication identity Signed-off-by: Yordis Prieto --- .../with_cluster_node_and_custom_settings.cs | 7 ++++++- src/EventStore.Core/ClusterVNode.cs | 14 +++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/EventStore.Core.XUnit.Tests/Configuration/ClusterNodeOptionsTests/when_building/with_cluster_node_and_custom_settings.cs b/src/EventStore.Core.XUnit.Tests/Configuration/ClusterNodeOptionsTests/when_building/with_cluster_node_and_custom_settings.cs index 4a38e0746b..e9eb8027df 100644 --- a/src/EventStore.Core.XUnit.Tests/Configuration/ClusterNodeOptionsTests/when_building/with_cluster_node_and_custom_settings.cs +++ b/src/EventStore.Core.XUnit.Tests/Configuration/ClusterNodeOptionsTests/when_building/with_cluster_node_and_custom_settings.cs @@ -151,7 +151,12 @@ options with [Test] public void should_advertise_the_configured_replication_port() { - Assert.AreEqual(3112, _node.GossipAdvertiseInfo.InternalSecureTcp.Port); + Assert.Multiple(() => + { + Assert.AreEqual(_options.Interface.ReplicationPort, _node.NodeInfo.ReplicationEndPoint.GetPort()); + Assert.AreEqual(3112, _node.GossipAdvertiseInfo.ReplicationEndPoint.Port); + Assert.AreEqual(3112, _node.GossipAdvertiseInfo.InternalSecureTcp.Port); + }); } } diff --git a/src/EventStore.Core/ClusterVNode.cs b/src/EventStore.Core/ClusterVNode.cs index 5ee76da1f9..0ac48b9d39 100644 --- a/src/EventStore.Core/ClusterVNode.cs +++ b/src/EventStore.Core/ClusterVNode.cs @@ -297,6 +297,9 @@ public ClusterVNode(ClusterVNodeOptions options, var enableExternalTcp = nodeTcpOptions.EnableExternalTcp; var httpEndPoint = new IPEndPoint(options.Interface.NodeIp, options.Interface.NodePort); + var replicationEndPoint = new IPEndPoint( + options.Interface.ReplicationIp, + options.Interface.ReplicationPort); var intTcp = disableInternalTcpTls ? new IPEndPoint(options.Interface.ReplicationIp, @@ -332,7 +335,7 @@ public ClusterVNode(ClusterVNodeOptions options, Log.Information("Quorum size set to {quorum}.", options.Cluster.QuorumSize); NodeInfo = new VNodeInfo(instanceId.Value, debugIndex, intTcp, intSecIp, extTcp, extSecIp, - httpEndPoint, options.Cluster.ReadOnlyReplica); + httpEndPoint, options.Cluster.ReadOnlyReplica, replicationEndPoint); var metricsConfiguration = MetricsConfiguration.Get(configuration); var trackers = new Trackers(); @@ -967,12 +970,16 @@ GossipAdvertiseInfo GetGossipAdvertiseInfo() options.Interface.NodePortAdvertiseAs > 0 ? options.Interface.NodePortAdvertiseAs : NodeInfo.HttpEndPoint.GetPort()); + var advertisedReplicationEndPoint = new DnsEndPoint(intHostToAdvertise, + replicationPortAdvertiseAs > 0 + ? replicationPortAdvertiseAs + : NodeInfo.ReplicationEndPoint.GetPort()); return new GossipAdvertiseInfo(intTcpEndPoint, intSecureTcpEndPoint, extTcpEndPoint, extSecureTcpEndPoint, httpEndPoint, options.Interface.ReplicationHostAdvertiseAs, options.Interface.NodeHostAdvertiseAs, options.Interface.NodePortAdvertiseAs, options.Interface.AdvertiseHostToClientAs, options.Interface.AdvertiseNodePortToClientAs, - nodeTcpOptions?.NodeTcpPortAdvertiseAs ?? 0); + nodeTcpOptions?.NodeTcpPortAdvertiseAs ?? 0, advertisedReplicationEndPoint); } _httpService = new KestrelHttpService(_mainQueue, NodeInfo.HttpEndPoint); @@ -1485,7 +1492,8 @@ GossipAdvertiseInfo GetGossipAdvertiseInfo() GossipAdvertiseInfo.AdvertiseHostToClientAs, GossipAdvertiseInfo.AdvertiseHttpPortToClientAs, GossipAdvertiseInfo.AdvertiseTcpPortToClientAs, - options.Cluster.NodePriority, options.Cluster.ReadOnlyReplica, VersionInfo.Version); + options.Cluster.NodePriority, options.Cluster.ReadOnlyReplica, VersionInfo.Version, + GossipAdvertiseInfo.ReplicationEndPoint); // ELECTIONS TRACKER _mainBus.Subscribe(trackers.ElectionCounterTracker); From 28f5203600a276377f8c1e9389b5fb4cd909cea2 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Mon, 21 Sep 2026 12:46:27 -0400 Subject: [PATCH 04/13] feat(cluster): isolate internal replication traffic Signed-off-by: Yordis Prieto --- .../Services/ReplicationEndpointPolicy.cs | 35 ++++++++++ src/EventStore.ClusterNode/Program.cs | 30 ++++++++- .../Helpers/MiniClusterNode.cs | 17 ++++- ...node_becomes_leader_with_unindexed_data.cs | 18 +++--- .../ReplicationEndpointPolicyTests.cs | 42 ++++++++++++ .../GrpcRequestForwardingServiceTests.cs | 5 +- ...RequestForwardingTransportSecurityTests.cs | 3 +- .../GrpcReplicaServiceFactoryTests.cs | 31 ++++++++- .../GrpcReplicaServiceSupervisorTests.cs | 9 +-- .../ClusterVNodeOptionsValidatorTests.cs | 64 +++++++++++++++++++ src/EventStore.Core/ClusterVNode.cs | 11 +++- .../Configuration/ClusterVNodeOptions.cs | 8 +-- .../ClusterVNodeOptionsExtensions.cs | 14 +++- .../ClusterVNodeOptionsValidator.cs | 24 +++++++ .../Services/ElectionsService.cs | 2 +- .../Services/Gossip/NodeGossipService.cs | 2 +- .../GrpcReplicaServiceSupervisor.cs | 5 +- .../Replication/ReplicationGrpcClient.cs | 31 +++++++-- .../GrpcRequestForwardingSupervisor.cs | 8 +-- 19 files changed, 314 insertions(+), 45 deletions(-) create mode 100644 src/EventStore.ClusterNode/Components/Services/ReplicationEndpointPolicy.cs create mode 100644 src/EventStore.Core.Tests/Regression/ReplicationEndpointPolicyTests.cs diff --git a/src/EventStore.ClusterNode/Components/Services/ReplicationEndpointPolicy.cs b/src/EventStore.ClusterNode/Components/Services/ReplicationEndpointPolicy.cs new file mode 100644 index 0000000000..a83b6f0494 --- /dev/null +++ b/src/EventStore.ClusterNode/Components/Services/ReplicationEndpointPolicy.cs @@ -0,0 +1,35 @@ +using System; +using System.Net; +using Microsoft.AspNetCore.Http; + +namespace EventStore.ClusterNode.Components.Services; + +public sealed class ReplicationEndpointPolicy(IPEndPoint listenEndPoint) +{ + private static readonly PathString ReplicationServicePath = "/event_store.replication.Replication"; + private static readonly PathString RequestForwardingServicePath = "/event_store.forwarding.RequestForwarding"; + + public bool Allows(HttpContext context) + { + var isReplicationConnection = Matches(context.Connection.LocalIpAddress, context.Connection.LocalPort); + var isInternalRequest = context.Request.Path.StartsWithSegments( + ReplicationServicePath, + StringComparison.Ordinal) || context.Request.Path.StartsWithSegments( + RequestForwardingServicePath, + StringComparison.Ordinal); + + return isReplicationConnection == isInternalRequest; + } + + private bool Matches(IPAddress localAddress, int localPort) + { + if (localPort != listenEndPoint.Port) + { + return false; + } + + return listenEndPoint.Address.Equals(IPAddress.Any) || + listenEndPoint.Address.Equals(IPAddress.IPv6Any) || + listenEndPoint.Address.Equals(localAddress); + } +} diff --git a/src/EventStore.ClusterNode/Program.cs b/src/EventStore.ClusterNode/Program.cs index a82abd59cb..0b8c1d31a7 100644 --- a/src/EventStore.ClusterNode/Program.cs +++ b/src/EventStore.ClusterNode/Program.cs @@ -25,6 +25,7 @@ using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.DataProtection; using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Server.Kestrel.Core; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -272,6 +273,8 @@ async Task Run(ClusterVNodeHostedService hostedService, ManualResetEventSlim sig { x.SuppressStatusMessages = true; }); + var replicationEndpointPolicy = new ReplicationEndpointPolicy( + new System.Net.IPEndPoint(options.Interface.ReplicationIp, options.Interface.ReplicationPort)); builder.WebHost.ConfigureKestrel(server => { @@ -283,6 +286,9 @@ async Task Run(ClusterVNodeHostedService hostedService, ManualResetEventSlim sig server.Listen(options.Interface.NodeIp, options.Interface.NodePort, listenOptions => ConfigureHttpOptions(listenOptions, hostedService, useHttps: !hostedService.Node.DisableHttps)); + server.Listen(options.Interface.ReplicationIp, options.Interface.ReplicationPort, listenOptions => + ConfigureHttpOptions(listenOptions, hostedService, + useHttps: !hostedService.Node.DisableHttps, http2Only: true)); if (hostedService.Node.EnableUnixSocket) { @@ -325,6 +331,16 @@ async Task Run(ClusterVNodeHostedService hostedService, ManualResetEventSlim sig builder.Services.AddSingleton(hostedService); var app = builder.Build(); + app.Use(async (context, next) => + { + if (!replicationEndpointPolicy.Allows(context)) + { + context.Response.StatusCode = StatusCodes.Status404NotFound; + return; + } + + await next(context); + }); app.UseMiddleware(); hostedService.Node.Startup.Configure(app); if (oauthEnabled) @@ -376,14 +392,22 @@ async Task Run(ClusterVNodeHostedService hostedService, ManualResetEventSlim sig } } - private static void ConfigureHttpOptions(ListenOptions listenOptions, ClusterVNodeHostedService hostedService, - bool useHttps) + private static void ConfigureHttpOptions( + ListenOptions listenOptions, + ClusterVNodeHostedService hostedService, + bool useHttps, + bool http2Only = false) { + if (http2Only) + { + listenOptions.Protocols = HttpProtocols.Http2; + } + if (useHttps) { listenOptions.UseHttps(CreateServerOptionsSelectionCallback(hostedService), null); } - else + else if (!http2Only) { listenOptions.Use(next => new ClearTextHttpMultiplexingMiddleware(next).OnConnectAsync); diff --git a/src/EventStore.Core.Tests/Helpers/MiniClusterNode.cs b/src/EventStore.Core.Tests/Helpers/MiniClusterNode.cs index 444ab592d2..7af8632d71 100644 --- a/src/EventStore.Core.Tests/Helpers/MiniClusterNode.cs +++ b/src/EventStore.Core.Tests/Helpers/MiniClusterNode.cs @@ -25,6 +25,7 @@ using EventStore.Plugins.Subsystems; using EventStore.TcpUnitTestPlugin; using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Server.Kestrel.Core; using Microsoft.AspNetCore.Server.Kestrel.Https; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; @@ -45,6 +46,7 @@ public class MiniClusterNode public IPEndPoint InternalTcpEndPoint { get; } public IPEndPoint ExternalTcpEndPoint { get; } public IPEndPoint HttpEndPoint { get; } + public IPEndPoint ReplicationEndPoint { get; } public readonly int DebugIndex; @@ -68,7 +70,8 @@ public MiniClusterNode(string pathname, int debugIndex, IPEndPoint internalTcp, bool disableFlushToDisk = false, bool readOnlyReplica = false, int nodePriority = 0, string intHostAdvertiseAs = null, IExpiryStrategy expiryStrategy = null, ArchiveOptions archiveOptions = null, bool archiver = false, - int clusterSize = 3, bool unsafeAllowSurplusNodes = false) + int clusterSize = 3, bool unsafeAllowSurplusNodes = false, + string replicationHostAdvertiseAs = null) { RunningTime.Start(); @@ -78,6 +81,7 @@ public MiniClusterNode(string pathname, int debugIndex, IPEndPoint internalTcp, InternalTcpEndPoint = internalTcp; ExternalTcpEndPoint = externalTcp; HttpEndPoint = httpEndPoint; + ReplicationEndPoint = internalTcp; _dbPath = Path.Combine( pathname, @@ -124,7 +128,7 @@ public MiniClusterNode(string pathname, int debugIndex, IPEndPoint internalTcp, ReplicationHeartbeatTimeout = 2_000, ReplicationHeartbeatInterval = 2_000, EnableTrustedAuth = enableTrustedAuth, - ReplicationHostAdvertiseAs = intHostAdvertiseAs + ReplicationHostAdvertiseAs = replicationHostAdvertiseAs ?? intHostAdvertiseAs }, Database = new() { @@ -215,7 +219,7 @@ public MiniClusterNode(string pathname, int debugIndex, IPEndPoint internalTcp, webHost .UseKestrel(o => { - o.Listen(HttpEndPoint, options => + void ConfigureHttps(ListenOptions options) { options.UseHttps(new HttpsConnectionAdapterOptions { @@ -233,6 +237,13 @@ public MiniClusterNode(string pathname, int debugIndex, IPEndPoint internalTcp, return isValid; } }); + } + + o.Listen(HttpEndPoint, ConfigureHttps); + o.Listen(ReplicationEndPoint, options => + { + options.Protocols = HttpProtocols.Http2; + ConfigureHttps(options); }); }) .UseStartup(Node.Startup); diff --git a/src/EventStore.Core.Tests/Integration/when_node_becomes_leader_with_unindexed_data.cs b/src/EventStore.Core.Tests/Integration/when_node_becomes_leader_with_unindexed_data.cs index f7e6400840..b43356b8e2 100644 --- a/src/EventStore.Core.Tests/Integration/when_node_becomes_leader_with_unindexed_data.cs +++ b/src/EventStore.Core.Tests/Integration/when_node_becomes_leader_with_unindexed_data.cs @@ -25,7 +25,7 @@ namespace EventStore.Core.Tests.Integration; [TestFixture(typeof(LogFormat.V2), typeof(string))] public class when_node_becomes_leader_with_unindexed_data : specification_with_cluster { - private const string FakeHostAdvertiseAs = "192.168.123.123"; + private const string FakeReplicationHostAdvertiseAs = "192.168.123.123"; private const string Username = "admin"; private const string Password = "changeit"; @@ -163,15 +163,15 @@ private async Task> ReadAllEvents(IPEndPoint endpoint } private MiniClusterNode CreateNode(int index, Endpoints endpoints, EndPoint[] gossipSeeds, - int nodePriority, string intHostAdvertiseAs) => new( + int nodePriority, string replicationHostAdvertiseAs) => new( PathName, index, endpoints.InternalTcp, endpoints.ExternalTcp, endpoints.HttpEndPoint, subsystems: Array.Empty(), gossipSeeds: gossipSeeds, - nodePriority: nodePriority, intHostAdvertiseAs: intHostAdvertiseAs); + nodePriority: nodePriority, replicationHostAdvertiseAs: replicationHostAdvertiseAs); - private Task StartNode(int i, int priority, string intHostAdvertiseAs = null) + private Task StartNode(int i, int priority, string replicationHostAdvertiseAs = null) { - _nodes[i] = CreateNode(i, _nodeEndpoints[i], _nodeGossipSeeds[i], priority, intHostAdvertiseAs); + _nodes[i] = CreateNode(i, _nodeEndpoints[i], _nodeGossipSeeds[i], priority, replicationHostAdvertiseAs); _nodes[i].Start(); return Task.CompletedTask; } @@ -296,9 +296,9 @@ public async Task new_events_should_have_correct_event_numbers(bool appendInitia await ShutdownAllNodes(keepDb: true); // make node 1 become the leader by setting its priority to 1 - // node 0 can't become a follower since it can't replicate over internal TCP due to the fake --int-host-advertise-as - await StartNode(0, priority: 0, intHostAdvertiseAs: FakeHostAdvertiseAs); - await StartNode(1, priority: 1, intHostAdvertiseAs: FakeHostAdvertiseAs); + // node 0 can't become a follower because the fake replication address prevents gRPC replication + await StartNode(0, priority: 0, replicationHostAdvertiseAs: FakeReplicationHostAdvertiseAs); + await StartNode(1, priority: 1, replicationHostAdvertiseAs: FakeReplicationHostAdvertiseAs); try { @@ -336,7 +336,7 @@ public async Task new_events_should_have_correct_event_numbers(bool appendInitia // shut down both nodes await ShutdownAllNodes(maxIdx: 2, keepDb: true); - // start both nodes again without the fake --int-host-advertise-as so that they can form a cluster + // start both nodes again without the fake replication address so that they can form a cluster await StartNode(0, priority: 0); await StartNode(1, priority: 1); try diff --git a/src/EventStore.Core.Tests/Regression/ReplicationEndpointPolicyTests.cs b/src/EventStore.Core.Tests/Regression/ReplicationEndpointPolicyTests.cs new file mode 100644 index 0000000000..63eac29563 --- /dev/null +++ b/src/EventStore.Core.Tests/Regression/ReplicationEndpointPolicyTests.cs @@ -0,0 +1,42 @@ +using System.Net; +using EventStore.ClusterNode.Components.Services; +using Microsoft.AspNetCore.Http; +using NUnit.Framework; + +namespace EventStore.Core.Tests.Regression; + +[TestFixture] +public class ReplicationEndpointPolicyTests +{ + [TestCase(2113, "/streams", true)] + [TestCase(2113, "/event_store.replication.Replication/Replicate", false)] + [TestCase(2113, "/event_store.forwarding.RequestForwarding/Forward", false)] + [TestCase(1112, "/streams", false)] + [TestCase(1112, "/event_store.replication.Replication/Replicate", true)] + [TestCase(1112, "/event_store.forwarding.RequestForwarding/Forward", true)] + public void internal_services_are_isolated_from_the_public_node_listener( + int localPort, + string path, + bool expected) + { + var policy = new ReplicationEndpointPolicy(new IPEndPoint(IPAddress.Loopback, 1112)); + var context = new DefaultHttpContext(); + context.Connection.LocalIpAddress = IPAddress.Loopback; + context.Connection.LocalPort = localPort; + context.Request.Path = path; + + Assert.That(policy.Allows(context), Is.EqualTo(expected)); + } + + [Test] + public void wildcard_replication_binding_matches_the_resolved_local_address() + { + var policy = new ReplicationEndpointPolicy(new IPEndPoint(IPAddress.Any, 1112)); + var context = new DefaultHttpContext(); + context.Connection.LocalIpAddress = IPAddress.Parse("192.0.2.1"); + context.Connection.LocalPort = 1112; + context.Request.Path = "/event_store.replication.Replication/Replicate"; + + Assert.That(policy.Allows(context), Is.True); + } +} diff --git a/src/EventStore.Core.Tests/Services/RequestForwarding/GrpcRequestForwardingServiceTests.cs b/src/EventStore.Core.Tests/Services/RequestForwarding/GrpcRequestForwardingServiceTests.cs index e861b74159..b333bd58be 100644 --- a/src/EventStore.Core.Tests/Services/RequestForwarding/GrpcRequestForwardingServiceTests.cs +++ b/src/EventStore.Core.Tests/Services/RequestForwarding/GrpcRequestForwardingServiceTests.cs @@ -571,7 +571,7 @@ public void pre_replica_connects_to_the_leader_http_endpoint() fixture.Supervisor.Handle(new SystemMessage.BecomePreReplica( Guid.NewGuid(), Guid.NewGuid(), fixture.Leader)); - Assert.That(fixture.Factory.EndPoints.Single(), Is.EqualTo(fixture.Leader.HttpEndPoint)); + Assert.That(fixture.Factory.EndPoints.Single(), Is.EqualTo(fixture.Leader.ReplicationEndPoint)); } [Test] @@ -806,7 +806,8 @@ private static MemberInfo CreateLeader(Guid? instanceId = null, int httpPort = 2 0, Guid.NewGuid(), 0, - false); + false, + replicationEndPoint: new DnsEndPoint("leader-forwarding.internal", 3112)); private static async Task WaitUntil(Func condition) { diff --git a/src/EventStore.Core.Tests/Services/RequestForwarding/GrpcRequestForwardingTransportSecurityTests.cs b/src/EventStore.Core.Tests/Services/RequestForwarding/GrpcRequestForwardingTransportSecurityTests.cs index 17fe60ab3f..479f678e6f 100644 --- a/src/EventStore.Core.Tests/Services/RequestForwarding/GrpcRequestForwardingTransportSecurityTests.cs +++ b/src/EventStore.Core.Tests/Services/RequestForwarding/GrpcRequestForwardingTransportSecurityTests.cs @@ -134,7 +134,8 @@ private static MemberInfo CreateLeader() => MemberInfo.ForVNode( 0, Guid.NewGuid(), 0, - false); + false, + replicationEndPoint: new DnsEndPoint("leader-forwarding.internal", 3112)); private sealed class RejectingServiceFactory : IGrpcRequestForwardingServiceFactory { diff --git a/src/EventStore.Core.Tests/Services/Transport/Grpc/Replication/GrpcReplicaServiceFactoryTests.cs b/src/EventStore.Core.Tests/Services/Transport/Grpc/Replication/GrpcReplicaServiceFactoryTests.cs index 19697515ac..91ba8c4a35 100644 --- a/src/EventStore.Core.Tests/Services/Transport/Grpc/Replication/GrpcReplicaServiceFactoryTests.cs +++ b/src/EventStore.Core.Tests/Services/Transport/Grpc/Replication/GrpcReplicaServiceFactoryTests.cs @@ -37,7 +37,30 @@ public void replication_client_uses_node_certificate_names_and_owns_its_http_cli Assert.That(nodeHttpClientFactory.AdditionalCertificateNames, Is.EqualTo(new[] { "cluster.internal" })); client.Dispose(); - Assert.That(nodeHttpClientFactory.Handler.Disposed, Is.True); + Assert.That(nodeHttpClientFactory.TransportHandler.Disposed, Is.True); + } + + [Test] + public void replication_client_normalizes_legacy_sub_second_heartbeat_values() + { + var nodeHttpClientFactory = new RecordingNodeHttpClientFactory(); + var factory = new ReplicationGrpcClientFactory( + Uri.UriSchemeHttps, + nodeHttpClientFactory, + TimeSpan.FromMilliseconds(700), + TimeSpan.FromMilliseconds(700)); + + using var client = factory.Create(new DnsEndPoint("leader.internal", 1112)); + + Assert.Multiple(() => + { + Assert.That(nodeHttpClientFactory.SocketsHandler.KeepAlivePingDelay, + Is.EqualTo(TimeSpan.FromSeconds(1))); + Assert.That(nodeHttpClientFactory.SocketsHandler.KeepAlivePingTimeout, + Is.EqualTo(TimeSpan.FromSeconds(1))); + Assert.That(nodeHttpClientFactory.SocketsHandler.KeepAlivePingPolicy, + Is.EqualTo(HttpKeepAlivePingPolicy.Always)); + }); } [Test] @@ -99,7 +122,8 @@ private sealed class TrackingReplicationGrpcClientFactory : IReplicationGrpcClie private sealed class RecordingNodeHttpClientFactory : INodeHttpClientFactory { - public RecordingHttpMessageHandler Handler { get; } = new(); + public RecordingHttpMessageHandler TransportHandler { get; } = new(); + public SocketsHttpHandler SocketsHandler { get; } = new(); public string[] AdditionalCertificateNames { get; private set; } public HttpClient CreateHttpClient( @@ -107,7 +131,8 @@ public HttpClient CreateHttpClient( Action configureSocketsHttpHandler = null) { AdditionalCertificateNames = additionalCertificateNames; - return new HttpClient(Handler); + configureSocketsHttpHandler?.Invoke(SocketsHandler); + return new HttpClient(TransportHandler); } } diff --git a/src/EventStore.Core.Tests/Services/Transport/Grpc/Replication/GrpcReplicaServiceSupervisorTests.cs b/src/EventStore.Core.Tests/Services/Transport/Grpc/Replication/GrpcReplicaServiceSupervisorTests.cs index a6eaedf281..024d596827 100644 --- a/src/EventStore.Core.Tests/Services/Transport/Grpc/Replication/GrpcReplicaServiceSupervisorTests.cs +++ b/src/EventStore.Core.Tests/Services/Transport/Grpc/Replication/GrpcReplicaServiceSupervisorTests.cs @@ -22,7 +22,7 @@ public class GrpcReplicaServiceSupervisorTests [TestCase(false)] [TestCase(true)] - public async Task pre_replica_state_starts_and_tracks_a_stream_for_the_advertised_http_endpoints( + public async Task pre_replica_state_starts_and_tracks_a_stream_for_the_advertised_replication_endpoints( bool readOnlyReplica) { var fixture = CreateFixture(); @@ -37,7 +37,7 @@ public async Task pre_replica_state_starts_and_tracks_a_stream_for_the_advertise var request = fixture.Factory.Requests.Single(); Assert.Multiple(() => { - Assert.That(request.Endpoints.LeaderEndPoint, Is.EqualTo(fixture.Leader.HttpEndPoint)); + Assert.That(request.Endpoints.LeaderEndPoint, Is.EqualTo(fixture.Leader.ReplicationEndPoint)); Assert.That(request.Endpoints.AdvertisedReplicaEndPoint, Is.EqualTo(fixture.AdvertisedEndPoint)); Assert.That(request.Service.StartCalls, Is.EqualTo(1)); Assert.That(fixture.TrackedTasks.Single(), Is.SameAs(request.Service.Task)); @@ -410,7 +410,7 @@ private static Fixture CreateFixture( startException, createException, beforeCreateReturns); - var advertisedEndPoint = new DnsEndPoint("replica.internal", 2113); + var advertisedEndPoint = new DnsEndPoint("replica.internal", 1112); var trackedTasks = new List(); var supervisor = new GrpcReplicaServiceSupervisor( publisher, @@ -447,7 +447,8 @@ private static MemberInfo CreateLeader() => MemberInfo.ForVNode( 0, Guid.NewGuid(), 0, - false); + false, + replicationEndPoint: new DnsEndPoint("leader.replication.internal", 1112)); private static SystemMessage.StateChangeMessage CreateReplicaState( VNodeState state, diff --git a/src/EventStore.Core.XUnit.Tests/Configuration/ClusterVNodeOptionsValidatorTests.cs b/src/EventStore.Core.XUnit.Tests/Configuration/ClusterVNodeOptionsValidatorTests.cs index 3caee89cc6..37a3703c9c 100644 --- a/src/EventStore.Core.XUnit.Tests/Configuration/ClusterVNodeOptionsValidatorTests.cs +++ b/src/EventStore.Core.XUnit.Tests/Configuration/ClusterVNodeOptionsValidatorTests.cs @@ -1,4 +1,5 @@ using System; +using System.Net; using EventStore.Common.Exceptions; using EventStore.Core.Authentication; using EventStore.Core.Services; @@ -9,6 +10,69 @@ namespace EventStore.Core.XUnit.Tests.Configuration; // Some other tests are in ClusterNodeOptionsTests/when_building public class ClusterVNodeOptionsValidatorTests { + [Theory] + [InlineData("127.0.0.1", "127.0.0.1")] + [InlineData("0.0.0.0", "127.0.0.1")] + [InlineData("127.0.0.1", "0.0.0.0")] + public void node_and_replication_listeners_cannot_overlap(string nodeIp, string replicationIp) + { + var options = new ClusterVNodeOptions + { + Interface = new() + { + NodeIp = IPAddress.Parse(nodeIp), + NodePort = 2113, + ReplicationIp = IPAddress.Parse(replicationIp), + ReplicationPort = 2113, + } + }; + + Assert.Throws(() => ClusterVNodeOptionsValidator.Validate(options)); + } + + [Fact] + public void node_and_replication_listeners_can_use_the_same_port_on_distinct_addresses() + { + var options = new ClusterVNodeOptions + { + Interface = new() + { + NodeIp = IPAddress.Parse("127.0.0.1"), + NodePort = 2113, + ReplicationIp = IPAddress.Parse("127.0.0.2"), + ReplicationPort = 2113, + } + }; + + ClusterVNodeOptionsValidator.Validate(options); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void replication_heartbeat_interval_must_be_positive(int interval) + { + var options = new ClusterVNodeOptions + { + Interface = new() { ReplicationHeartbeatInterval = interval } + }; + + Assert.Throws(() => ClusterVNodeOptionsValidator.Validate(options)); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void replication_heartbeat_timeout_must_be_positive(int timeout) + { + var options = new ClusterVNodeOptions + { + Interface = new() { ReplicationHeartbeatTimeout = timeout } + }; + + Assert.Throws(() => ClusterVNodeOptionsValidator.Validate(options)); + } + [Theory] [InlineData(false, false, true)] [InlineData(false, true, true)] diff --git a/src/EventStore.Core/ClusterVNode.cs b/src/EventStore.Core/ClusterVNode.cs index 0ac48b9d39..ecb61b7f31 100644 --- a/src/EventStore.Core/ClusterVNode.cs +++ b/src/EventStore.Core/ClusterVNode.cs @@ -1537,13 +1537,17 @@ GossipAdvertiseInfo GetGossipAdvertiseInfo() _grpcReplicaServiceSupervisor = new GrpcReplicaServiceSupervisor( _mainQueue, new GrpcReplicaServiceFactory( - new ReplicationGrpcClientFactory(uriScheme, _nodeHttpClientFactory), + new ReplicationGrpcClientFactory( + uriScheme, + _nodeHttpClientFactory, + TimeSpan.FromMilliseconds(options.Interface.ReplicationHeartbeatInterval), + TimeSpan.FromMilliseconds(options.Interface.ReplicationHeartbeatTimeout)), new ReplicaSubscriptionDataSource(Db, epochManager), NodeInfo.InstanceId, options.Cluster.ReadOnlyReplica ? ReplicaPromotability.NonPromotable : ReplicaPromotability.Promotable), - GossipAdvertiseInfo.HttpEndPoint, + GossipAdvertiseInfo.ReplicationEndPoint, AddTask); _mainBus.Subscribe(_grpcReplicaServiceSupervisor); _mainBus.Subscribe(_grpcReplicaServiceSupervisor); @@ -2272,5 +2276,6 @@ private void ReloadCertificates(ClusterVNodeOptions options) } public override string ToString() => - $"[{NodeInfo.InstanceId:B}, {NodeInfo.InternalTcp}, {NodeInfo.ExternalTcp}, {NodeInfo.HttpEndPoint}]"; + $"[{NodeInfo.InstanceId:B}, {NodeInfo.InternalTcp}, {NodeInfo.ExternalTcp}, " + + $"{NodeInfo.ReplicationEndPoint}, {NodeInfo.HttpEndPoint}]"; } diff --git a/src/EventStore.Core/Configuration/ClusterVNodeOptions.cs b/src/EventStore.Core/Configuration/ClusterVNodeOptions.cs index 620253566d..968310fdd9 100644 --- a/src/EventStore.Core/Configuration/ClusterVNodeOptions.cs +++ b/src/EventStore.Core/Configuration/ClusterVNodeOptions.cs @@ -620,7 +620,7 @@ public record GrpcOptions [Description("Interface Options")] public record InterfaceOptions { - [Description("The IP Address used by internal replication between nodes in the cluster.")] + [Description("The IP address used by the gRPC replication listener.")] public IPAddress ReplicationIp { get; init; } = IPAddress.Loopback; [Description("The IP Address for the node.")] @@ -629,7 +629,7 @@ public record InterfaceOptions [Description("The Port to run the HTTP server on.")] public int NodePort { get; init; } = 2113; - [Description("The TCP port used by internal replication between nodes in the cluster.")] + [Description("The port used by the gRPC replication listener.")] public int ReplicationPort { get; init; } = 1112; [Description("Advertise the Node's host name to other nodes and external clients as.")] @@ -661,11 +661,11 @@ public int GetReplicationPortAdvertiseAs() => ? ReplicationPortAdvertiseAs : ReplicationTcpPortAdvertiseAs; - [Description("Heartbeat timeout for Replication TCP sockets."), + [Description("Keepalive ping timeout for gRPC replication connections. Values below 1000 ms use the HTTP/2 minimum of 1000 ms."), Unit("ms")] public int ReplicationHeartbeatTimeout { get; init; } = 700; - [Description("Heartbeat interval for Replication TCP sockets."), + [Description("Keepalive ping interval for gRPC replication connections. Values below 1000 ms use the HTTP/2 minimum of 1000 ms."), Unit("ms")] public int ReplicationHeartbeatInterval { get; init; } = 700; diff --git a/src/EventStore.Core/Configuration/ClusterVNodeOptionsExtensions.cs b/src/EventStore.Core/Configuration/ClusterVNodeOptionsExtensions.cs index 562cbcd82a..052f6bb17a 100644 --- a/src/EventStore.Core/Configuration/ClusterVNodeOptionsExtensions.cs +++ b/src/EventStore.Core/Configuration/ClusterVNodeOptionsExtensions.cs @@ -99,7 +99,7 @@ public static ClusterVNodeOptions WithExternalTcpOn( options with { Interface = options.Interface with { NodeIp = endPoint.Address, } }; /// - /// Sets the internal tcp endpoint to the specified value + /// Sets the gRPC replication endpoint to the specified value /// /// The /// The internal endpoint to use @@ -148,6 +148,18 @@ options with } }; + public static ClusterVNodeOptions AdvertiseReplicationHostAs( + this ClusterVNodeOptions options, + EndPoint endPoint) => + options with + { + Interface = options.Interface with + { + ReplicationHostAdvertiseAs = endPoint.GetHost(), + ReplicationPortAdvertiseAs = endPoint.GetPort() + } + }; + /// /// /// The diff --git a/src/EventStore.Core/Configuration/ClusterVNodeOptionsValidator.cs b/src/EventStore.Core/Configuration/ClusterVNodeOptionsValidator.cs index 41c7b4b39f..35d4a1312a 100644 --- a/src/EventStore.Core/Configuration/ClusterVNodeOptionsValidator.cs +++ b/src/EventStore.Core/Configuration/ClusterVNodeOptionsValidator.cs @@ -32,6 +32,23 @@ public static void Validate(ClusterVNodeOptions options) throw new ArgumentNullException(nameof(options.Interface.ReplicationIp)); } + if (options.Interface.NodePort == options.Interface.ReplicationPort && + EndpointsOverlap(options.Interface.NodeIp, options.Interface.ReplicationIp)) + { + throw new ArgumentException( + $"{nameof(options.Interface.NodePort)} and {nameof(options.Interface.ReplicationPort)} cannot bind the same endpoint."); + } + + if (options.Interface.ReplicationHeartbeatInterval <= 0) + { + throw new ArgumentOutOfRangeException(nameof(options.Interface.ReplicationHeartbeatInterval)); + } + + if (options.Interface.ReplicationHeartbeatTimeout <= 0) + { + throw new ArgumentOutOfRangeException(nameof(options.Interface.ReplicationHeartbeatTimeout)); + } + if (options.Cluster.ClusterSize <= 0) { throw new ArgumentOutOfRangeException(nameof(options.Cluster.ClusterSize), options.Cluster.ClusterSize, @@ -174,4 +191,11 @@ public static bool ValidateForStartup(ClusterVNodeOptions options) return true; } + private static bool EndpointsOverlap(System.Net.IPAddress first, System.Net.IPAddress second) => + first.Equals(second) || + first.Equals(System.Net.IPAddress.Any) || + second.Equals(System.Net.IPAddress.Any) || + first.Equals(System.Net.IPAddress.IPv6Any) || + second.Equals(System.Net.IPAddress.IPv6Any); + } diff --git a/src/EventStore.Core/Services/ElectionsService.cs b/src/EventStore.Core/Services/ElectionsService.cs index d865b66e26..d0a6f9bed8 100644 --- a/src/EventStore.Core/Services/ElectionsService.cs +++ b/src/EventStore.Core/Services/ElectionsService.cs @@ -143,7 +143,7 @@ public ElectionsService(IPublisher publisher, memberInfo.AdvertiseHostToClientAs, memberInfo.AdvertiseHttpPortToClientAs, memberInfo.AdvertiseTcpPortToClientAs, ownInfo.LastCommitPosition, ownInfo.WriterCheckpoint, ownInfo.ChaserCheckpoint, ownInfo.EpochPosition, ownInfo.EpochNumber, ownInfo.EpochId, ownInfo.NodePriority, - memberInfo.IsReadOnlyReplica, VersionInfo.Version) + memberInfo.IsReadOnlyReplica, VersionInfo.Version, memberInfo.ReplicationEndPoint) }; } diff --git a/src/EventStore.Core/Services/Gossip/NodeGossipService.cs b/src/EventStore.Core/Services/Gossip/NodeGossipService.cs index 803e69d77d..3014089eaf 100644 --- a/src/EventStore.Core/Services/Gossip/NodeGossipService.cs +++ b/src/EventStore.Core/Services/Gossip/NodeGossipService.cs @@ -72,7 +72,7 @@ protected override MemberInfo GetInitialMe() lastEpoch == null ? -1 : lastEpoch.EpochNumber, lastEpoch == null ? Guid.Empty : lastEpoch.EpochId, _nodePriority, - _memberInfo.IsReadOnlyReplica, _memberInfo.ESVersion); + _memberInfo.IsReadOnlyReplica, _memberInfo.ESVersion, _memberInfo.ReplicationEndPoint); } protected override MemberInfo GetUpdatedMe(MemberInfo me) diff --git a/src/EventStore.Core/Services/Replication/GrpcReplicaServiceSupervisor.cs b/src/EventStore.Core/Services/Replication/GrpcReplicaServiceSupervisor.cs index 52d2f29e3d..d4b8bf3c17 100644 --- a/src/EventStore.Core/Services/Replication/GrpcReplicaServiceSupervisor.cs +++ b/src/EventStore.Core/Services/Replication/GrpcReplicaServiceSupervisor.cs @@ -277,7 +277,7 @@ private async ValueTask ReplaceActiveAsync( { active.Service = _factory.Create( new FencedPublisher(this, active), - new GrpcReplicaConnectionEndpoints(leader.HttpEndPoint, _advertisedReplicaEndPoint)); + new GrpcReplicaConnectionEndpoints(leader.ReplicationEndPoint, _advertisedReplicaEndPoint)); SetActive(active); var task = active.Service.Start(); _trackTask(task); @@ -296,7 +296,8 @@ private async ValueTask ReplaceActiveAsync( { ClearActive(active); await StopAsync(active.Service); - Log.Warning(exception, "Failed to start replication stream to [{leaderEndPoint}].", leader.HttpEndPoint); + Log.Warning(exception, "Failed to start replication stream to [{leaderEndPoint}].", + leader.ReplicationEndPoint); _publisher.Publish(new ReplicationMessage.LeaderConnectionFailed( leaderConnectionCorrelationId, leader)); } diff --git a/src/EventStore.Core/Services/Replication/ReplicationGrpcClient.cs b/src/EventStore.Core/Services/Replication/ReplicationGrpcClient.cs index 4c8f0ea9e9..463f63a119 100644 --- a/src/EventStore.Core/Services/Replication/ReplicationGrpcClient.cs +++ b/src/EventStore.Core/Services/Replication/ReplicationGrpcClient.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Net; +using System.Net.Http; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -32,24 +33,37 @@ public interface IReplicationGrpcCall : IDisposable public sealed class ReplicationGrpcClientFactory : IReplicationGrpcClientFactory { + private static readonly TimeSpan DefaultKeepAlivePingDelay = TimeSpan.FromMilliseconds(700); + private static readonly TimeSpan DefaultKeepAlivePingTimeout = TimeSpan.FromMilliseconds(700); + private static readonly TimeSpan MinimumKeepAlivePingValue = TimeSpan.FromSeconds(1); private readonly string _uriScheme; private readonly INodeHttpClientFactory _nodeHttpClientFactory; + private readonly TimeSpan _keepAlivePingDelay; + private readonly TimeSpan _keepAlivePingTimeout; public ReplicationGrpcClientFactory( string uriScheme, - INodeHttpClientFactory nodeHttpClientFactory) + INodeHttpClientFactory nodeHttpClientFactory, + TimeSpan? keepAlivePingDelay = null, + TimeSpan? keepAlivePingTimeout = null) { Ensure.NotNullOrEmpty(uriScheme, nameof(uriScheme)); Ensure.NotNull(nodeHttpClientFactory, nameof(nodeHttpClientFactory)); _uriScheme = uriScheme; _nodeHttpClientFactory = nodeHttpClientFactory; + _keepAlivePingDelay = NormalizeKeepAliveValue(keepAlivePingDelay ?? DefaultKeepAlivePingDelay); + _keepAlivePingTimeout = NormalizeKeepAliveValue(keepAlivePingTimeout ?? DefaultKeepAlivePingTimeout); } + private static TimeSpan NormalizeKeepAliveValue(TimeSpan value) => + value < MinimumKeepAlivePingValue ? MinimumKeepAlivePingValue : value; + public IReplicationGrpcClient Create(EndPoint leaderEndPoint) { Ensure.NotNull(leaderEndPoint, nameof(leaderEndPoint)); - return new ReplicationGrpcClient(_uriScheme, leaderEndPoint, _nodeHttpClientFactory); + return new ReplicationGrpcClient(_uriScheme, leaderEndPoint, _nodeHttpClientFactory, + _keepAlivePingDelay, _keepAlivePingTimeout); } } @@ -60,9 +74,18 @@ internal sealed class ReplicationGrpcClient : IReplicationGrpcClient public ReplicationGrpcClient( string uriScheme, EndPoint leaderEndPoint, - INodeHttpClientFactory nodeHttpClientFactory) + INodeHttpClientFactory nodeHttpClientFactory, + TimeSpan keepAlivePingDelay, + TimeSpan keepAlivePingTimeout) { - var httpClient = nodeHttpClientFactory.CreateHttpClient(leaderEndPoint.GetOtherNames()); + var httpClient = nodeHttpClientFactory.CreateHttpClient( + leaderEndPoint.GetOtherNames(), + handler => + { + handler.KeepAlivePingDelay = keepAlivePingDelay; + handler.KeepAlivePingTimeout = keepAlivePingTimeout; + handler.KeepAlivePingPolicy = HttpKeepAlivePingPolicy.Always; + }); httpClient.Timeout = Timeout.InfiniteTimeSpan; httpClient.DefaultRequestVersion = new Version(2, 0); diff --git a/src/EventStore.Core/Services/RequestForwarding/GrpcRequestForwardingSupervisor.cs b/src/EventStore.Core/Services/RequestForwarding/GrpcRequestForwardingSupervisor.cs index 0d91272c93..91c6792a28 100644 --- a/src/EventStore.Core/Services/RequestForwarding/GrpcRequestForwardingSupervisor.cs +++ b/src/EventStore.Core/Services/RequestForwarding/GrpcRequestForwardingSupervisor.cs @@ -214,11 +214,11 @@ private void ConnectToLeader(MemberInfo leader) IGrpcRequestForwardingService service = null; try { - var active = new ActiveStream(leader.InstanceId, leader.HttpEndPoint, connectionGeneration); + var active = new ActiveStream(leader.InstanceId, leader.ReplicationEndPoint, connectionGeneration); service = _factory.Create( message => TryPublishIfActive(active, message), _publisher.Publish, - leader.HttpEndPoint, + leader.ReplicationEndPoint, new ForwardingSessionGeneration(connectionGeneration)); active.Service = service; _active = active; @@ -237,7 +237,7 @@ private void ConnectToLeader(MemberInfo leader) service?.Stop(); Log.Warning(exception, "Failed to start request forwarding stream to [{leaderEndPoint}].", - leader.HttpEndPoint); + leader.ReplicationEndPoint); if (connectionGeneration == _connectionGeneration) { ScheduleReconnect(leader.InstanceId, connectionGeneration); @@ -280,7 +280,7 @@ private bool HasHealthyStreamTo(MemberInfo leader) => _active is not null && !_active.Service.Task.IsCompleted && _active.LeaderId == leader.InstanceId && - Equals(_active.LeaderEndPoint, leader.HttpEndPoint); + Equals(_active.LeaderEndPoint, leader.ReplicationEndPoint); private void PublishIfActive(ActiveStream active, Message message) { From 09439b018433fced39baeab1066061d7d40dc0a5 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Mon, 21 Sep 2026 13:24:00 -0400 Subject: [PATCH 05/13] fix(monitoring): match isolated replication members Signed-off-by: Yordis Prieto --- .../Services/ClusterStatusService.cs | 7 ++++- .../Regression/ClusterStatusServiceTests.cs | 27 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 src/EventStore.Core.Tests/Regression/ClusterStatusServiceTests.cs diff --git a/src/EventStore.ClusterNode/Components/Services/ClusterStatusService.cs b/src/EventStore.ClusterNode/Components/Services/ClusterStatusService.cs index 5c753987ec..4b05e97e54 100644 --- a/src/EventStore.ClusterNode/Components/Services/ClusterStatusService.cs +++ b/src/EventStore.ClusterNode/Components/Services/ClusterStatusService.cs @@ -211,7 +211,9 @@ private static ClientClusterInfo.ClientMemberInfo FindMemberByInternalEndpoint( string endpoint) { var cleaned = endpoint.Replace("Unspecified/", "", StringComparison.OrdinalIgnoreCase); - return members.FirstOrDefault(x => string.Equals(InternalTcpEndpoint(x), cleaned, StringComparison.OrdinalIgnoreCase)); + return members.FirstOrDefault(x => + string.Equals(ReplicationEndpoint(x), cleaned, StringComparison.OrdinalIgnoreCase) || + string.Equals(InternalTcpEndpoint(x), cleaned, StringComparison.OrdinalIgnoreCase)); } private static Uri BuildLeaderAddress( @@ -224,6 +226,9 @@ private static string InternalTcpEndpoint(ClientClusterInfo.ClientMemberInfo mem member.InternalTcpIp, member.InternalSecureTcpPort != 0 ? member.InternalSecureTcpPort : member.InternalTcpPort); + private static string ReplicationEndpoint(ClientClusterInfo.ClientMemberInfo member) => + Endpoint(member.ReplicationEndPointIp, member.ReplicationEndPointPort); + private static string HttpEndpoint(ClientClusterInfo.ClientMemberInfo member) => Endpoint(member.HttpEndPointIp, member.HttpEndPointPort); diff --git a/src/EventStore.Core.Tests/Regression/ClusterStatusServiceTests.cs b/src/EventStore.Core.Tests/Regression/ClusterStatusServiceTests.cs new file mode 100644 index 0000000000..32429f1ab1 --- /dev/null +++ b/src/EventStore.Core.Tests/Regression/ClusterStatusServiceTests.cs @@ -0,0 +1,27 @@ +using System.Collections.Generic; +using System.Reflection; +using EventStore.ClusterNode.Components.Services; +using EventStore.Core.Cluster; +using NUnit.Framework; + +namespace EventStore.Core.Tests.Regression; + +[TestFixture] +public class ClusterStatusServiceTests +{ + [Test] + public void replication_statistics_match_the_member_replication_endpoint() + { + var member = new ClientClusterInfo.ClientMemberInfo + { + ReplicationEndPointIp = "replica.internal", + ReplicationEndPointPort = 1112 + }; + + var result = (ClientClusterInfo.ClientMemberInfo)typeof(ClusterStatusService) + .GetMethod("FindMemberByInternalEndpoint", BindingFlags.NonPublic | BindingFlags.Static)! + .Invoke(null, [new List { member }, "replica.internal:1112"])!; + + Assert.That(result, Is.SameAs(member)); + } +} From bcefb7545c6a3e3d128f3d5e9b73b07938b9314e Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Mon, 21 Sep 2026 12:51:10 -0400 Subject: [PATCH 06/13] chore(transport): retire the internal TCP runtime Signed-off-by: Yordis Prieto --- proto.lock | 57 +- .../Components/Pages/Cluster.razor | 27 +- .../Services/ClusterStatusService.cs | 12 +- .../ClientAPI/Helpers/EventDataComparer.cs | 46 -- .../ClientAPI/Helpers/EventsStream.cs | 25 - .../ClientAPI/Helpers/TcpType.cs | 7 - .../ClientAPI/Helpers/TestConnection.cs | 72 -- .../Helpers/TestConnectionLifecycle.cs | 77 -- .../ClientAPI/Helpers/TestEvent.cs | 22 - .../ClientAPI/Helpers/Writer.cs | 98 --- .../ClientAPI/SpecificationWithMiniNode.cs | 102 --- .../Cluster/MemberInfoTests.cs | 159 ++-- src/EventStore.Core.Tests/DefaultData.cs | 2 - .../EventStore.Core.Tests.csproj | 1 - .../Helpers/ClientApiLoggerBridge.cs | 92 --- .../Helpers/MiniClusterNode.cs | 58 +- src/EventStore.Core.Tests/Helpers/MiniNode.cs | 57 +- .../when_archiving_and_restoring_a_cluster.cs | 65 +- .../Integration/specification_with_cluster.cs | 46 +- .../when_cluster_nodes_are_restarted.cs | 2 +- ...node_becomes_leader_with_unindexed_data.cs | 9 +- .../ClusterSettingsFactory.cs | 8 +- .../ElectionsService/ClusterVNodeSettings.cs | 12 +- .../ElectionsService/ElectionServiceUnit.cs | 22 +- .../ElectionsService/ElectionsServiceTests.cs | 46 +- .../LeaderNode/ElectionsServiceUnitTests.cs | 4 +- .../Randomized/RandomizedElectionsTestCase.cs | 5 +- .../Randomized/UpdateGossipProcessor.cs | 4 +- ...wn_when_started_and_set_full_imediately.cs | 4 +- ...nown_when_started_and_set_to_full_later.cs | 4 +- .../GossipService/NodeGossipServiceTests.cs | 25 +- .../LogReplication/LogReplicationFixture.cs | 5 - .../connecting_to_read_only_replica.cs | 102 ++- .../GrpcRequestForwardingServiceTests.cs | 10 +- ...RequestForwardingTransportSecurityTests.cs | 10 +- .../RequestForwardingServiceTests.cs | 6 +- .../when_writing_and_deposed_as_leader.cs | 6 +- ...sed_as_leader_and_replica_moves_forward.cs | 6 +- .../Forwarding/ForwardingGrpcCodecTests.cs | 28 +- .../GrpcReplicaServiceSupervisorTests.cs | 5 - .../Transport/Tcp/core_tcp_package.cs | 199 ----- .../Services/Transport/Tcp/ssl_connection.cs | 3 +- .../Tcp/ssl_connections_mutual_auth.cs | 3 +- .../InaugurationManagerTests.cs | 3 +- .../Services/VNode/ShutdownServiceTests.cs | 5 - .../Services/VNode/leader_info_provider.cs | 291 ++------ .../PublicTcpApiTestService.cs | 123 ---- .../TcpApiTestPlugin/TcpApiTestOptions.cs | 12 - .../TcpApiTestPlugin/TcpApiTestPlugin.cs | 38 - .../Truncation/when_truncating_database.cs | 17 +- .../when_building/with_default_settings.cs | 22 +- .../ClusterVNodeOptionsValidatorTests.cs | 26 - .../Metrics/ElectionsCounterTrackerTests.cs | 3 +- .../InMemory/GossipListenerServiceTests.cs | 5 - .../Telemetry/TelemetryServiceTests.cs | 5 - .../LegacyPolicySelectorFactory.cs | 1 - .../Cluster/ClientClusterInfo.cs | 28 - src/EventStore.Core/Cluster/ClusterInfo.cs | 35 +- src/EventStore.Core/Cluster/MemberInfo.cs | 109 +-- src/EventStore.Core/ClusterVNode.cs | 133 +--- .../Configuration/ClusterVNodeOptions.cs | 12 +- .../ClusterVNodeOptionsExtensions.cs | 42 +- .../Data/GossipAdvertiseInfo.cs | 37 +- src/EventStore.Core/Data/VNodeInfo.cs | 42 +- src/EventStore.Core/EventStore.Core.csproj | 7 - .../Helpers/IAsyncMessageFramer.cs | 16 + .../Helpers/LengthPrefixSuffixFramer.cs | 3 +- .../Helpers/MessageFramingException.cs | 10 + src/EventStore.Core/Messages/ClientMessage.cs | 19 +- .../Messages/ClusterInfoDto.cs | 32 - src/EventStore.Core/Messages/MemberInfoDto.cs | 95 --- .../Messages/TcpClientMessageDto.cs | 670 ----------------- src/EventStore.Core/Messages/TcpMessage.cs | 127 ---- src/EventStore.Core/Messaging/CoreMessage.cs | 1 - src/EventStore.Core/NodeTcpOptions.cs | 9 - .../Services/AuthorizationGateway.cs | 4 +- .../Services/ElectionsService.cs | 4 +- .../Services/Gossip/GossipServiceBase.cs | 8 +- .../Services/Gossip/NodeGossipService.cs | 8 +- .../Services/Monitoring/MonitoringService.cs | 92 +-- .../Services/Monitoring/SystemStatsHelper.cs | 14 - .../PersistentSubscriptionService.cs | 20 - .../Services/Replication/LogRecordFramer.cs | 2 +- .../Services/Replication/TransactionFramer.cs | 2 +- .../GrpcRequestForwardingSupervisor.cs | 10 +- .../Services/RequestForwardingService.cs | 6 +- .../Storage/ReaderIndex/EventFilter.cs | 67 +- .../Services/SubscriptionsService.cs | 36 - .../Services/TcpSendService.cs | 14 - .../Grpc/Forwarding/ForwardingGrpcCodec.cs | 10 +- .../Grpc/Forwarding/ForwardingService.cs | 2 +- .../Transport/Tcp/ClientTcpDispatcher.cs | 600 --------------- .../Transport/Tcp/ClientWriteTcpDispatcher.cs | 439 ----------- .../Services/Transport/Tcp/ITcpDispatcher.cs | 14 - .../Transport/Tcp/ProtobufExtensions.cs | 112 --- .../Transport/Tcp/SendOverTcpEnvelope.cs | 29 - .../Services/Transport/Tcp/TcpCommand.cs | 73 -- .../Transport/Tcp/TcpConnectionManager.cs | 682 ------------------ .../Services/Transport/Tcp/TcpDispatcher.cs | 134 ---- .../Services/Transport/Tcp/TcpPackage.cs | 236 ------ .../Services/Transport/Tcp/TcpService.cs | 160 ---- .../Tcp/UnableToAcquireStreamException.cs | 8 - .../Services/VNode/ClusterVNodeController.cs | 33 +- .../Services/VNode/LeaderInfoProvider.cs | 29 +- src/EventStore.Core/Settings/VNodeSettings.cs | 105 --- src/EventStore.Core/Util/Opts.cs | 9 - ...tion_with_standard_projections_runnning.cs | 22 +- .../projections_system/when_starting_up.cs | 6 +- src/Protos/ClientAPI/ClientMessageDtos.proto | 450 ------------ src/Protos/Grpc/cluster.proto | 13 +- src/Protos/Grpc/forwarding.proto | 4 +- 111 files changed, 532 insertions(+), 6456 deletions(-) delete mode 100644 src/EventStore.Core.Tests/ClientAPI/Helpers/EventDataComparer.cs delete mode 100644 src/EventStore.Core.Tests/ClientAPI/Helpers/EventsStream.cs delete mode 100644 src/EventStore.Core.Tests/ClientAPI/Helpers/TcpType.cs delete mode 100644 src/EventStore.Core.Tests/ClientAPI/Helpers/TestConnection.cs delete mode 100644 src/EventStore.Core.Tests/ClientAPI/Helpers/TestConnectionLifecycle.cs delete mode 100644 src/EventStore.Core.Tests/ClientAPI/Helpers/TestEvent.cs delete mode 100644 src/EventStore.Core.Tests/ClientAPI/Helpers/Writer.cs delete mode 100644 src/EventStore.Core.Tests/ClientAPI/SpecificationWithMiniNode.cs delete mode 100644 src/EventStore.Core.Tests/Helpers/ClientApiLoggerBridge.cs delete mode 100644 src/EventStore.Core.Tests/Services/Transport/Tcp/core_tcp_package.cs delete mode 100644 src/EventStore.Core.Tests/TcpApiTestPlugin/PublicTcpApiTestService.cs delete mode 100644 src/EventStore.Core.Tests/TcpApiTestPlugin/TcpApiTestOptions.cs delete mode 100644 src/EventStore.Core.Tests/TcpApiTestPlugin/TcpApiTestPlugin.cs create mode 100644 src/EventStore.Core/Helpers/IAsyncMessageFramer.cs create mode 100644 src/EventStore.Core/Helpers/MessageFramingException.cs delete mode 100644 src/EventStore.Core/Messages/ClusterInfoDto.cs delete mode 100644 src/EventStore.Core/Messages/MemberInfoDto.cs delete mode 100755 src/EventStore.Core/Messages/TcpClientMessageDto.cs delete mode 100644 src/EventStore.Core/Messages/TcpMessage.cs delete mode 100644 src/EventStore.Core/NodeTcpOptions.cs delete mode 100644 src/EventStore.Core/Services/TcpSendService.cs delete mode 100644 src/EventStore.Core/Services/Transport/Tcp/ClientTcpDispatcher.cs delete mode 100644 src/EventStore.Core/Services/Transport/Tcp/ClientWriteTcpDispatcher.cs delete mode 100644 src/EventStore.Core/Services/Transport/Tcp/ITcpDispatcher.cs delete mode 100644 src/EventStore.Core/Services/Transport/Tcp/ProtobufExtensions.cs delete mode 100644 src/EventStore.Core/Services/Transport/Tcp/SendOverTcpEnvelope.cs delete mode 100644 src/EventStore.Core/Services/Transport/Tcp/TcpCommand.cs delete mode 100644 src/EventStore.Core/Services/Transport/Tcp/TcpConnectionManager.cs delete mode 100644 src/EventStore.Core/Services/Transport/Tcp/TcpDispatcher.cs delete mode 100644 src/EventStore.Core/Services/Transport/Tcp/TcpPackage.cs delete mode 100644 src/EventStore.Core/Services/Transport/Tcp/TcpService.cs delete mode 100644 src/EventStore.Core/Services/Transport/Tcp/UnableToAcquireStreamException.cs delete mode 100644 src/EventStore.Core/Settings/VNodeSettings.cs delete mode 100755 src/Protos/ClientAPI/ClientMessageDtos.proto diff --git a/proto.lock b/proto.lock index 0534f18460..facf535cd7 100644 --- a/proto.lock +++ b/proto.lock @@ -1909,26 +1909,6 @@ "name": "http_end_point", "type": "EndPoint" }, - { - "id": 6, - "name": "internal_tcp", - "type": "EndPoint" - }, - { - "id": 7, - "name": "external_tcp", - "type": "EndPoint" - }, - { - "id": 8, - "name": "internal_tcp_uses_tls", - "type": "bool" - }, - { - "id": 9, - "name": "external_tcp_uses_tls", - "type": "bool" - }, { "id": 10, "name": "last_commit_position", @@ -1979,11 +1959,6 @@ "name": "advertise_http_port_to_client_as", "type": "uint32" }, - { - "id": 20, - "name": "advertise_tcp_port_to_client_as", - "type": "uint32" - }, { "id": 21, "name": "es_version", @@ -1994,6 +1969,20 @@ "name": "replication_end_point", "type": "EndPoint" } + ], + "reserved_ids": [ + 6, + 7, + 8, + 9, + 20 + ], + "reserved_names": [ + "internal_tcp", + "external_tcp", + "internal_tcp_uses_tls", + "external_tcp_uses_tls", + "advertise_tcp_port_to_client_as" ] } ], @@ -2789,21 +2778,19 @@ { "name": "LeaderInfo", "fields": [ - { - "id": 1, - "name": "external_tcp", - "type": "EndPoint" - }, - { - "id": 2, - "name": "is_secure", - "type": "bool" - }, { "id": 3, "name": "http", "type": "EndPoint" } + ], + "reserved_ids": [ + 1, + 2 + ], + "reserved_names": [ + "external_tcp", + "is_secure" ] }, { diff --git a/src/EventStore.ClusterNode/Components/Pages/Cluster.razor b/src/EventStore.ClusterNode/Components/Pages/Cluster.razor index 47d3c0279e..e72be103d8 100644 --- a/src/EventStore.ClusterNode/Components/Pages/Cluster.razor +++ b/src/EventStore.ClusterNode/Components/Pages/Cluster.razor @@ -43,8 +43,7 @@ Status Timestamp (UTC) Checkpoints - TCP - HTTP + HTTP / gRPC Actions @@ -52,7 +51,7 @@ @if (!ClusterMembers.Any()) { - @ClusterEmptyMessage + @ClusterEmptyMessage } else @@ -77,10 +76,6 @@

@EpochLabel(member)

} - -

Internal: @InternalTcpEndpoint(member)

-

External: @ExternalTcpEndpoint(member)

- @HttpEndpoint(member)
@@ -307,7 +302,7 @@
- +
@@ -370,9 +365,7 @@ .Append("Snapshot taken at ") .Append(TimestampLabel(ClusterReadAt ?? DateTime.UtcNow)) .AppendLine() - .Append(PadRight("Internal Tcp", 31)).Append(' ') - .Append(PadRight("External Tcp", 31)).Append(' ') - .Append(PadRight("Http", 23)).Append(' ') + .Append(PadRight("HTTP / gRPC", 23)).Append(' ') .Append(PadRight("Status", 11)).Append(' ') .Append(PadRight("State", 18)).Append(' ') .Append(PadRight("Timestamp (UTC)", 19)).Append(" Checkpoints"); @@ -380,8 +373,6 @@ foreach (var member in ClusterMembers) { builder.AppendLine() - .Append(PadRight(InternalTcpEndpoint(member), 31)).Append(' ') - .Append(PadRight(ExternalTcpEndpoint(member), 31)).Append(' ') .Append(PadRight(HttpEndpoint(member), 23)).Append(' ') .Append(PadRight(MemberStatus(member), 11)).Append(' ') .Append(PadRight(member.State.ToString(), 18)).Append(' ') @@ -500,16 +491,6 @@ private static string MemberStatus(ClientClusterInfo.ClientMemberInfo member) => member.IsAlive ? "Alive" : "Unreachable"; - private static string InternalTcpEndpoint(ClientClusterInfo.ClientMemberInfo member) => - Endpoint( - member.InternalTcpIp, - member.InternalSecureTcpPort != 0 ? member.InternalSecureTcpPort : member.InternalTcpPort); - - private static string ExternalTcpEndpoint(ClientClusterInfo.ClientMemberInfo member) => - Endpoint( - member.ExternalTcpIp, - member.ExternalSecureTcpPort != 0 ? member.ExternalSecureTcpPort : member.ExternalTcpPort); - private static string HttpEndpoint(ClientClusterInfo.ClientMemberInfo member) => Endpoint(member.HttpEndPointIp, member.HttpEndPointPort); diff --git a/src/EventStore.ClusterNode/Components/Services/ClusterStatusService.cs b/src/EventStore.ClusterNode/Components/Services/ClusterStatusService.cs index 4b05e97e54..6941244086 100644 --- a/src/EventStore.ClusterNode/Components/Services/ClusterStatusService.cs +++ b/src/EventStore.ClusterNode/Components/Services/ClusterStatusService.cs @@ -171,7 +171,7 @@ private ClusterReplicaRow ParseReplicaRow( : Guid.Empty; var totalBytesSent = row.TotalBytesSent; var previousRow = _previousReplicas.GetValueOrDefault(connectionId); - var replicaNode = FindMemberByInternalEndpoint(members, row.SubscriptionEndpoint); + var replicaNode = FindMemberByEndpoint(members, row.SubscriptionEndpoint); var isCatchingUp = replicaNode?.State == VNodeState.CatchingUp; var catchupStartTime = now; var catchupStartBytesSent = totalBytesSent; @@ -206,14 +206,14 @@ private ClusterReplicaRow ParseReplicaRow( private ClaimsPrincipal CurrentUser => httpContextAccessor.HttpContext?.User ?? new ClaimsPrincipal(new ClaimsIdentity()); - private static ClientClusterInfo.ClientMemberInfo FindMemberByInternalEndpoint( + private static ClientClusterInfo.ClientMemberInfo FindMemberByEndpoint( IReadOnlyList members, string endpoint) { var cleaned = endpoint.Replace("Unspecified/", "", StringComparison.OrdinalIgnoreCase); return members.FirstOrDefault(x => string.Equals(ReplicationEndpoint(x), cleaned, StringComparison.OrdinalIgnoreCase) || - string.Equals(InternalTcpEndpoint(x), cleaned, StringComparison.OrdinalIgnoreCase)); + string.Equals(HttpEndpoint(x), cleaned, StringComparison.OrdinalIgnoreCase)); } private static Uri BuildLeaderAddress( @@ -221,14 +221,8 @@ private static Uri BuildLeaderAddress( ClientClusterInfo.ClientMemberInfo leader) => new UriBuilder(request.Scheme, leader.HttpEndPointIp, leader.HttpEndPointPort).Uri; - private static string InternalTcpEndpoint(ClientClusterInfo.ClientMemberInfo member) => - Endpoint( - member.InternalTcpIp, - member.InternalSecureTcpPort != 0 ? member.InternalSecureTcpPort : member.InternalTcpPort); - private static string ReplicationEndpoint(ClientClusterInfo.ClientMemberInfo member) => Endpoint(member.ReplicationEndPointIp, member.ReplicationEndPointPort); - private static string HttpEndpoint(ClientClusterInfo.ClientMemberInfo member) => Endpoint(member.HttpEndPointIp, member.HttpEndPointPort); diff --git a/src/EventStore.Core.Tests/ClientAPI/Helpers/EventDataComparer.cs b/src/EventStore.Core.Tests/ClientAPI/Helpers/EventDataComparer.cs deleted file mode 100644 index 15a424be6c..0000000000 --- a/src/EventStore.Core.Tests/ClientAPI/Helpers/EventDataComparer.cs +++ /dev/null @@ -1,46 +0,0 @@ -using EventStore.ClientAPI; -using EventStore.Common.Utils; - -namespace EventStore.Core.Tests.ClientAPI.Helpers; - -internal static class EventDataComparer -{ - public static bool Equal(EventData expected, RecordedEvent actual) - { - if (expected.EventId != actual.EventId) - { - return false; - } - - if (expected.Type != actual.EventType) - { - return false; - } - - var expectedDataString = Helper.UTF8NoBom.GetString(expected.Data ?? new byte[0]); - var expectedMetadataString = Helper.UTF8NoBom.GetString(expected.Metadata ?? new byte[0]); - - var actualDataString = Helper.UTF8NoBom.GetString(actual.Data ?? new byte[0]); - var actualMetadataDataString = Helper.UTF8NoBom.GetString(actual.Metadata ?? new byte[0]); - - return expectedDataString == actualDataString && expectedMetadataString == actualMetadataDataString; - } - - public static bool Equal(EventData[] expected, RecordedEvent[] actual) - { - if (expected.Length != actual.Length) - { - return false; - } - - for (var i = 0; i < expected.Length; i++) - { - if (!Equal(expected[i], actual[i])) - { - return false; - } - } - - return true; - } -} diff --git a/src/EventStore.Core.Tests/ClientAPI/Helpers/EventsStream.cs b/src/EventStore.Core.Tests/ClientAPI/Helpers/EventsStream.cs deleted file mode 100644 index da4bb62ce2..0000000000 --- a/src/EventStore.Core.Tests/ClientAPI/Helpers/EventsStream.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System.Threading.Tasks; -using EventStore.ClientAPI; - -namespace EventStore.Core.Tests.ClientAPI.Helpers; - -internal class EventsStream -{ - private const int SliceSize = 10; - - public static async Task Count(IEventStoreConnection store, string stream) - { - var result = 0; - while (true) - { - var slice = await store.ReadStreamEventsForwardAsync(stream, result, SliceSize, false); - result += slice.Events.Length; - if (slice.IsEndOfStream) - { - break; - } - } - - return result; - } -} diff --git a/src/EventStore.Core.Tests/ClientAPI/Helpers/TcpType.cs b/src/EventStore.Core.Tests/ClientAPI/Helpers/TcpType.cs deleted file mode 100644 index b860eb63ac..0000000000 --- a/src/EventStore.Core.Tests/ClientAPI/Helpers/TcpType.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace EventStore.Core.Tests.ClientAPI.Helpers; - -public enum TcpType -{ - Normal, - Ssl -} diff --git a/src/EventStore.Core.Tests/ClientAPI/Helpers/TestConnection.cs b/src/EventStore.Core.Tests/ClientAPI/Helpers/TestConnection.cs deleted file mode 100644 index 674773889c..0000000000 --- a/src/EventStore.Core.Tests/ClientAPI/Helpers/TestConnection.cs +++ /dev/null @@ -1,72 +0,0 @@ -using System; -using System.Net; -using System.Threading; -using EventStore.ClientAPI; -using EventStore.ClientAPI.Internal; -using EventStore.ClientAPI.SystemData; -using EventStore.Core.Tests.Helpers; -using NUnit.Framework; - -namespace EventStore.Core.Tests.ClientAPI.Helpers; - -public static class TestConnection -{ - private static int _nextConnId = -1; - - public static IEventStoreConnection Create(IPEndPoint endPoint, TcpType tcpType = TcpType.Ssl, - UserCredentials userCredentials = null) - { - return EventStoreConnection.Create(Settings(tcpType, userCredentials), - endPoint.ToESTcpUri(), - $"ESC-{Interlocked.Increment(ref _nextConnId)}"); - } - - public static IEventStoreConnection CreateMiniNodeClient(IPEndPoint endPoint, TcpType tcpType = TcpType.Ssl, - UserCredentials userCredentials = null) - { - return EventStoreConnection.Create(Settings( - tcpType, - userCredentials, - limitAttemptsForOperationTo: 10, - reconnectionDelay: TimeSpan.FromMilliseconds(100)), - endPoint.ToESTcpUri(), - $"ESC-{Interlocked.Increment(ref _nextConnId)}"); - } - - public static IEventStoreConnection To(MiniNode miniNode, TcpType tcpType, - UserCredentials userCredentials = null) - { - return EventStoreConnection.Create(Settings(tcpType, userCredentials), - miniNode.TcpEndPoint.ToESTcpUri(), - $"ESC-{Interlocked.Increment(ref _nextConnId)}"); - } - - private static ConnectionSettingsBuilder Settings( - TcpType tcpType, - UserCredentials userCredentials, - int limitAttemptsForOperationTo = 1, - TimeSpan? reconnectionDelay = null) - { - var settings = ConnectionSettings.Create() - .SetDefaultUserCredentials(userCredentials) - .UseCustomLogger(ClientApiLoggerBridge.Default) - .EnableVerboseLogging() - .LimitReconnectionsTo(10) - .LimitAttemptsForOperationTo(limitAttemptsForOperationTo) - .SetTimeoutCheckPeriodTo(TimeSpan.FromMilliseconds(100)) - .SetReconnectionDelayTo(reconnectionDelay ?? TimeSpan.Zero) - .FailOnNoServerResponse() - //.SetOperationTimeoutTo(TimeSpan.FromDays(1)) - ; - if (tcpType == TcpType.Ssl) - { - settings.DisableServerCertificateValidation(); - } - else - { - settings.DisableTls(); - } - - return settings; - } -} diff --git a/src/EventStore.Core.Tests/ClientAPI/Helpers/TestConnectionLifecycle.cs b/src/EventStore.Core.Tests/ClientAPI/Helpers/TestConnectionLifecycle.cs deleted file mode 100644 index 9ea1365f78..0000000000 --- a/src/EventStore.Core.Tests/ClientAPI/Helpers/TestConnectionLifecycle.cs +++ /dev/null @@ -1,77 +0,0 @@ -using System; -using System.Threading.Tasks; -using EventStore.ClientAPI; - -namespace EventStore.Core.Tests.ClientAPI.Helpers; - -public static class TestConnectionLifecycle -{ - public static async Task ReconnectUntilReady( - Func createConnection, - Func readinessProbe, - TimeSpan timeout) - { - var deadline = DateTime.UtcNow + timeout; - - while (true) - { - IEventStoreConnection connection = null; - - try - { - connection = createConnection(); - await connection.ConnectAsync(); - await readinessProbe(connection); - return connection; - } - catch (Exception ex) - { - if (connection != null) - { - TryCloseConnection(connection); - } - - if (IsTransientConnectionFailure(ex) && DateTime.UtcNow < deadline) - { - await Task.Delay(250); - continue; - } - - throw; - } - } - } - - public static async Task CloseConnectionAndWait(IEventStoreConnection connection, TimeSpan timeout) - { - var closed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - connection.Closed += (_, _) => closed.TrySetResult(); - connection.Close(); - await closed.Task.WithTimeout(timeout); - } - - public static bool IsTransientConnectionFailure(Exception ex) => - ex.GetType().Name is "ConnectionClosedException" - or "RetriesLimitReachedException" - or "NotAuthenticatedException" - or "AccessDeniedException"; - - public static void TryCloseConnection(IEventStoreConnection connection) - { - try - { - connection.Close(); - } - catch - { - } - } - - public static void DisposeIfNeeded(object candidate) - { - if (candidate is IDisposable disposable) - { - disposable.Dispose(); - } - } -} diff --git a/src/EventStore.Core.Tests/ClientAPI/Helpers/TestEvent.cs b/src/EventStore.Core.Tests/ClientAPI/Helpers/TestEvent.cs deleted file mode 100644 index 8fb92bedf3..0000000000 --- a/src/EventStore.Core.Tests/ClientAPI/Helpers/TestEvent.cs +++ /dev/null @@ -1,22 +0,0 @@ -using System; -using System.Text; -using EventStore.ClientAPI; -using EventStore.Common.Utils; - -namespace EventStore.Core.Tests.ClientAPI.Helpers; - -public class TestEvent -{ - public static EventData NewTestEvent(string data = null, string metadata = null, string eventName = "TestEvent") - { - return NewTestEvent(Guid.NewGuid(), data, metadata, eventName); - } - - public static EventData NewTestEvent(Guid eventId, string data = null, string metadata = null, string eventName = "TestEvent") - { - var encodedData = Helper.UTF8NoBom.GetBytes(data ?? eventId.ToString()); - var encodedMetadata = Helper.UTF8NoBom.GetBytes(metadata ?? "metadata"); - - return new EventData(eventId, eventName, false, encodedData, encodedMetadata); - } -} diff --git a/src/EventStore.Core.Tests/ClientAPI/Helpers/Writer.cs b/src/EventStore.Core.Tests/ClientAPI/Helpers/Writer.cs deleted file mode 100644 index 236e47ccb6..0000000000 --- a/src/EventStore.Core.Tests/ClientAPI/Helpers/Writer.cs +++ /dev/null @@ -1,98 +0,0 @@ -using System.Threading.Tasks; -using EventStore.ClientAPI; -using NUnit.Framework; - -namespace EventStore.Core.Tests.ClientAPI.Helpers; - -internal class StreamWriter -{ - private readonly IEventStoreConnection _store; - private readonly string _stream; - private readonly long _version; - - public StreamWriter(IEventStoreConnection store, string stream, long version) - { - _store = store; - _stream = stream; - _version = version; - } - - public async Task Append(params EventData[] events) - { - for (var i = 0; i < events.Length; i++) - { - var expVer = _version == ExpectedVersion.Any ? ExpectedVersion.Any : _version + i; - var nextExpVer = (await _store.AppendToStreamAsync(_stream, expVer, new[] { events[i] })).NextExpectedVersion; - if (_version != ExpectedVersion.Any) - { - Assert.AreEqual(expVer + 1, nextExpVer); - } - } - - return new TailWriter(_store, _stream); - } -} - -internal class TailWriter -{ - private readonly IEventStoreConnection _store; - private readonly string _stream; - - public TailWriter(IEventStoreConnection store, string stream) - { - _store = store; - _stream = stream; - } - - public async Task Then(EventData @event, long expectedVersion) - { - await _store.AppendToStreamAsync(_stream, expectedVersion, new[] { @event }); - return this; - } -} - -internal class TransactionalWriter -{ - private readonly IEventStoreConnection _store; - private readonly string _stream; - - public TransactionalWriter(IEventStoreConnection store, string stream) - { - _store = store; - _stream = stream; - } - - public async Task StartTransaction(long expectedVersion) - { - return new OngoingTransaction(await _store.StartTransactionAsync(_stream, expectedVersion)); - } - - public OngoingTransaction ContinueTransaction(long transactionId) - { - return new OngoingTransaction(_store.ContinueTransaction(transactionId)); - } -} - -//TODO GFY this should be removed and merged with the public idea of a transaction. -internal class OngoingTransaction -{ - private readonly EventStoreTransaction _transaction; - - public long TransactionId => _transaction.TransactionId; - - public OngoingTransaction(EventStoreTransaction transaction) - { - _transaction = transaction; - } - - public async Task Write(params EventData[] events) - { - await _transaction.WriteAsync(events); - return this; - } - - public Task Commit() - { - return _transaction.CommitAsync(); - } -} diff --git a/src/EventStore.Core.Tests/ClientAPI/SpecificationWithMiniNode.cs b/src/EventStore.Core.Tests/ClientAPI/SpecificationWithMiniNode.cs deleted file mode 100644 index ddd01c25d0..0000000000 --- a/src/EventStore.Core.Tests/ClientAPI/SpecificationWithMiniNode.cs +++ /dev/null @@ -1,102 +0,0 @@ -using System; -using System.Threading.Tasks; -using EventStore.ClientAPI; -using EventStore.Core.Tests.ClientAPI.Helpers; -using EventStore.Core.Tests.Helpers; -using NUnit.Framework; - -namespace EventStore.Core.Tests.ClientAPI; - -public abstract class SpecificationWithMiniNode : SpecificationWithDirectoryPerTestFixture -{ - private readonly int _chunkSize; - protected MiniNode _node; - protected IEventStoreConnection _conn; - protected virtual TimeSpan Timeout { get; } = TimeSpan.FromMinutes(1); - protected virtual TimeSpan StartupTimeout => TimeSpan.FromMinutes(5); - - protected virtual Task Given() => Task.CompletedTask; - - protected abstract Task When(); - - protected virtual IEventStoreConnection BuildConnection(MiniNode node) - { - return TestConnection.CreateMiniNodeClient(node.TcpEndPoint, TcpType.Ssl); - } - - protected Task CloseConnectionAndWait(IEventStoreConnection connection) => - TestConnectionLifecycle.CloseConnectionAndWait(connection, Timeout); - - protected SpecificationWithMiniNode() : this(chunkSize: 1024 * 1024) { } - - protected SpecificationWithMiniNode(int chunkSize) - { - _chunkSize = chunkSize; - } - - [OneTimeSetUp] - public override async Task TestFixtureSetUp() - { - - MiniNodeLogging.Setup(); - - try - { - await base.TestFixtureSetUp(); - } - catch (Exception ex) - { - throw new Exception("TestFixtureSetUp Failed", ex); - } - - try - { - _node = new MiniNode(PathName, chunkSize: _chunkSize); - await _node.Start(StartupTimeout); - await _node.WaitForTcpEndPoint().WithTimeout(StartupTimeout); - _conn = await TestConnectionLifecycle.ReconnectUntilReady( - () => BuildConnection(_node), - connection => connection.ReadAllEventsForwardAsync(Position.Start, 1, false, DefaultData.AdminCredentials), - StartupTimeout); - } - catch (Exception ex) - { - MiniNodeLogging.WriteLogs(); - throw new Exception("MiniNodeSetUp Failed", ex); - } - - try - { - await Given().WithTimeout(Timeout); - } - catch (Exception ex) - { - MiniNodeLogging.WriteLogs(); - throw new Exception("Given Failed", ex); - } - - try - { - await When().WithTimeout(Timeout); - } - catch (Exception ex) - { - MiniNodeLogging.WriteLogs(); - throw new Exception("When Failed", ex); - } - } - - [OneTimeTearDown] - public override async Task TestFixtureTearDown() - { - if (_conn != null) - { - await TestConnectionLifecycle.CloseConnectionAndWait(_conn, Timeout); - } - - await _node.Shutdown(); - await base.TestFixtureTearDown(); - - MiniNodeLogging.Clear(); - } -} diff --git a/src/EventStore.Core.Tests/Cluster/MemberInfoTests.cs b/src/EventStore.Core.Tests/Cluster/MemberInfoTests.cs index 0db8ed8764..7851965069 100644 --- a/src/EventStore.Core.Tests/Cluster/MemberInfoTests.cs +++ b/src/EventStore.Core.Tests/Cluster/MemberInfoTests.cs @@ -9,13 +9,6 @@ namespace EventStore.Core.Tests.Cluster; [TestFixture] public class MemberInfoTests { - private static readonly DnsEndPoint InternalTcp = new("internal", 1112); - private static readonly DnsEndPoint InternalSecureTcp = new("internal-secure", 2112); - private static readonly DnsEndPoint ExternalTcp = new("external", 1113); - private static readonly DnsEndPoint ExternalSecureTcp = new("external-secure", 2113); - private static readonly DnsEndPoint Http = new("http", 2113); - private static readonly DnsEndPoint Replication = new("replication", 3113); - [Test] public void member_with_dns_endpoint_should_equal() { @@ -24,12 +17,7 @@ public void member_with_dns_endpoint_should_equal() var memberWithDnsEndPoint = EventStore.Core.Cluster.MemberInfo.Initial(Guid.Empty, DateTime.UtcNow, VNodeState.Unknown, true, new DnsEndPoint(ipAddress, port), - new DnsEndPoint(ipAddress, port), - new DnsEndPoint(ipAddress, port), - new DnsEndPoint(ipAddress, port), - new DnsEndPoint(ipAddress, port), - null, 0, 0, - 0, false); + null, 0, 0, false); var ipEndPoint = new IPEndPoint(IPAddress.Parse(ipAddress), port); var dnsEndPoint = new DnsEndPoint(ipAddress, port); @@ -46,11 +34,7 @@ public void member_with_ip_endpoint_should_equal() var memberWithDnsEndPoint = EventStore.Core.Cluster.MemberInfo.Initial(Guid.Empty, DateTime.UtcNow, VNodeState.Unknown, true, new IPEndPoint(IPAddress.Parse(ipAddress), port), - new IPEndPoint(IPAddress.Parse(ipAddress), port), - new IPEndPoint(IPAddress.Parse(ipAddress), port), - new IPEndPoint(IPAddress.Parse(ipAddress), port), - new IPEndPoint(IPAddress.Parse(ipAddress), port), - null, 0, 0, 0, false); + null, 0, 0, false); var ipEndPoint = new IPEndPoint(IPAddress.Parse(ipAddress), port); var dnsEndPoint = new DnsEndPoint(ipAddress, port); @@ -60,99 +44,80 @@ public void member_with_ip_endpoint_should_equal() } [Test] - public void grpc_round_trip_preserves_tcp_and_replication_endpoints() + public void internal_gossip_round_trip_preserves_the_grpc_replication_endpoint() { - var member = EventStore.Core.Cluster.MemberInfo.Initial(Guid.NewGuid(), DateTime.UtcNow, - VNodeState.Unknown, true, - InternalTcp, null, null, ExternalSecureTcp, Http, - "client", 2113, 1113, 0, false, replicationEndPoint: Replication); - - var result = FromGrpcClusterInfo(ToGrpcClusterInfo( - new EventStore.Core.Cluster.ClusterInfo(member))).Members[0]; - - Assert.That(result.InternalTcpEndPoint, Is.EqualTo(InternalTcp)); - Assert.That(result.InternalSecureTcpEndPoint, Is.Null); - Assert.That(result.ExternalTcpEndPoint, Is.Null); - Assert.That(result.ExternalSecureTcpEndPoint, Is.EqualTo(ExternalSecureTcp)); - Assert.That(result.HttpEndPoint, Is.EqualTo(Http)); - Assert.That(result.ReplicationEndPoint, Is.EqualTo(Replication)); + var replicationEndPoint = new DnsEndPoint("replication-node", 1112); + var httpEndPoint = new DnsEndPoint("public-node", 2113); + var member = EventStore.Core.Cluster.MemberInfo.Initial( + Guid.NewGuid(), + DateTime.UtcNow, + VNodeState.Unknown, + true, + httpEndPoint, + null, + 0, + 0, + false, + replicationEndPoint: replicationEndPoint); + + var grpc = ToGrpcClusterInfo(new EventStore.Core.Cluster.ClusterInfo(member)); + var roundTrip = FromGrpcClusterInfo(grpc); + + Assert.That(roundTrip.Members, Has.Length.EqualTo(1)); + Assert.That(roundTrip.Members[0].HttpEndPoint, Is.EqualTo(httpEndPoint)); + Assert.That(roundTrip.Members[0].ReplicationEndPoint, Is.EqualTo(replicationEndPoint)); } [Test] - public void explicit_replication_endpoint_is_recognized_without_replacing_tcp_endpoints() + public void member_without_a_replication_endpoint_uses_its_http_endpoint() { - var member = CreateMember(Replication); - var vnode = new VNodeInfo(Guid.NewGuid(), 0, - new IPEndPoint(IPAddress.Loopback, 1112), null, - new IPEndPoint(IPAddress.Loopback, 1113), null, - Http, false, Replication); - var advertise = new GossipAdvertiseInfo( - InternalTcp, InternalSecureTcp, ExternalTcp, ExternalSecureTcp, Http, - null, null, 0, null, 0, 0, Replication); - - Assert.That(member.Is(Replication), Is.True); - Assert.That(member.InternalTcpEndPoint, Is.EqualTo(InternalTcp)); - Assert.That(member.InternalSecureTcpEndPoint, Is.EqualTo(InternalSecureTcp)); - Assert.That(member.ExternalTcpEndPoint, Is.EqualTo(ExternalTcp)); - Assert.That(member.ExternalSecureTcpEndPoint, Is.EqualTo(ExternalSecureTcp)); - Assert.That(vnode.ReplicationEndPoint, Is.SameAs(Replication)); - Assert.That(advertise.ReplicationEndPoint, Is.SameAs(Replication)); + var httpEndPoint = new DnsEndPoint("mixed-version-node", 2113); + var member = EventStore.Core.Cluster.MemberInfo.Initial( + Guid.NewGuid(), + DateTime.UtcNow, + VNodeState.Unknown, + true, + httpEndPoint, + null, + 0, + 0, + false); + var grpc = ToGrpcClusterInfo(new EventStore.Core.Cluster.ClusterInfo(member)); + grpc.Members[0].ReplicationEndPoint = null; + + var roundTrip = FromGrpcClusterInfo(grpc); + + Assert.That(roundTrip.Members[0].ReplicationEndPoint, Is.EqualTo(httpEndPoint)); } [Test] public void client_member_preserves_the_replication_endpoint() { + var replicationEndPoint = new DnsEndPoint("replication-node", 1112); var clientMember = new EventStore.Core.Cluster.ClientClusterInfo.ClientMemberInfo( - CreateMember(Replication)); - - Assert.That(clientMember.ReplicationEndPointIp, Is.EqualTo(Replication.Host)); - Assert.That(clientMember.ReplicationEndPointPort, Is.EqualTo(Replication.Port)); + EventStore.Core.Cluster.MemberInfo.Initial( + Guid.NewGuid(), + DateTime.UtcNow, + VNodeState.Unknown, + true, + new DnsEndPoint("public-node", 2113), + null, + 0, + 0, + false, + replicationEndPoint: replicationEndPoint)); + + Assert.That(clientMember.ReplicationEndPointIp, Is.EqualTo(replicationEndPoint.Host)); + Assert.That(clientMember.ReplicationEndPointPort, Is.EqualTo(replicationEndPoint.Port)); } - [Test] - public void missing_replication_endpoint_falls_back_to_http_endpoint() - { - var member = CreateMember(); - var vnode = new VNodeInfo(Guid.NewGuid(), 0, - new IPEndPoint(IPAddress.Loopback, 1112), null, - new IPEndPoint(IPAddress.Loopback, 1113), null, - Http, false); - var advertise = new GossipAdvertiseInfo( - InternalTcp, InternalSecureTcp, ExternalTcp, ExternalSecureTcp, Http, - null, null, 0, null, 0, 0); - - Assert.That(member.ReplicationEndPoint, Is.SameAs(Http)); - Assert.That(vnode.ReplicationEndPoint, Is.SameAs(Http)); - Assert.That(advertise.ReplicationEndPoint, Is.SameAs(Http)); - } - - [Test] - public void grpc_member_without_replication_endpoint_falls_back_to_http_endpoint() - { - var grpcCluster = ToGrpcClusterInfo( - new EventStore.Core.Cluster.ClusterInfo(CreateMember(Replication))); - grpcCluster.Members[0].ReplicationEndPoint = null; - - var result = FromGrpcClusterInfo(grpcCluster).Members[0]; - - Assert.That(result.ReplicationEndPoint, Is.EqualTo(Http)); - } - - private static EventStore.Core.Cluster.MemberInfo CreateMember(DnsEndPoint replicationEndPoint = null) => - EventStore.Core.Cluster.MemberInfo.Initial(Guid.NewGuid(), DateTime.UtcNow, - VNodeState.Unknown, true, - InternalTcp, InternalSecureTcp, ExternalTcp, ExternalSecureTcp, Http, - "client", 2113, 1113, 0, false, replicationEndPoint: replicationEndPoint); - - private static EventStore.Cluster.ClusterInfo ToGrpcClusterInfo( - EventStore.Core.Cluster.ClusterInfo clusterInfo) => + private static EventStore.Cluster.ClusterInfo ToGrpcClusterInfo(EventStore.Core.Cluster.ClusterInfo clusterInfo) => (EventStore.Cluster.ClusterInfo)typeof(EventStore.Core.Cluster.ClusterInfo) - .GetMethod("ToGrpcClusterInfo", BindingFlags.NonPublic | BindingFlags.Static)! - .Invoke(null, [clusterInfo])!; + .GetMethod("ToGrpcClusterInfo", BindingFlags.Static | BindingFlags.NonPublic) + .Invoke(null, [clusterInfo]); - private static EventStore.Core.Cluster.ClusterInfo FromGrpcClusterInfo( - EventStore.Cluster.ClusterInfo clusterInfo) => + private static EventStore.Core.Cluster.ClusterInfo FromGrpcClusterInfo(EventStore.Cluster.ClusterInfo clusterInfo) => (EventStore.Core.Cluster.ClusterInfo)typeof(EventStore.Core.Cluster.ClusterInfo) - .GetMethod("FromGrpcClusterInfo", BindingFlags.NonPublic | BindingFlags.Static)! - .Invoke(null, [clusterInfo, null])!; + .GetMethod("FromGrpcClusterInfo", BindingFlags.Static | BindingFlags.NonPublic) + .Invoke(null, [clusterInfo, null]); } diff --git a/src/EventStore.Core.Tests/DefaultData.cs b/src/EventStore.Core.Tests/DefaultData.cs index 3c647b08fc..262f7312dc 100644 --- a/src/EventStore.Core.Tests/DefaultData.cs +++ b/src/EventStore.Core.Tests/DefaultData.cs @@ -1,5 +1,4 @@ using System.Net; -using EventStore.ClientAPI.SystemData; using EventStore.Core.Services; namespace EventStore.Core.Tests; @@ -8,7 +7,6 @@ public class DefaultData { public static string AdminUsername = SystemUsers.Admin; public static string AdminPassword = SystemUsers.DefaultAdminPassword; - public static UserCredentials AdminCredentials = new UserCredentials(AdminUsername, AdminPassword); public static NetworkCredential AdminNetworkCredentials = new NetworkCredential(AdminUsername, AdminPassword); public static ClusterVNodeOptions.DefaultUserOptions DefaultUserOptions = new ClusterVNodeOptions.DefaultUserOptions() { diff --git a/src/EventStore.Core.Tests/EventStore.Core.Tests.csproj b/src/EventStore.Core.Tests/EventStore.Core.Tests.csproj index 3be13ef772..4aee9b0985 100644 --- a/src/EventStore.Core.Tests/EventStore.Core.Tests.csproj +++ b/src/EventStore.Core.Tests/EventStore.Core.Tests.csproj @@ -4,7 +4,6 @@ true - diff --git a/src/EventStore.Core.Tests/Helpers/ClientApiLoggerBridge.cs b/src/EventStore.Core.Tests/Helpers/ClientApiLoggerBridge.cs deleted file mode 100644 index 862b03493f..0000000000 --- a/src/EventStore.Core.Tests/Helpers/ClientApiLoggerBridge.cs +++ /dev/null @@ -1,92 +0,0 @@ -using System; -using EventStore.Common.Utils; -using ILogger = Serilog.ILogger; - -namespace EventStore.Core.Tests.Helpers; - -public class ClientApiLoggerBridge : EventStore.ClientAPI.ILogger -{ - public static readonly ClientApiLoggerBridge Default = - new ClientApiLoggerBridge(Serilog.Log.ForContext(Serilog.Core.Constants.SourceContextPropertyName, - "client-api")); - - private readonly Serilog.ILogger _log; - - public ClientApiLoggerBridge(ILogger log) - { - Ensure.NotNull(log, "log"); - _log = log; - } - - public void Error(string format, params object[] args) - { - if (args.Length == 0) - { - _log.Error(format); - } - else - { - _log.Error(format, args); - } - } - - public void Error(Exception ex, string format, params object[] args) - { - if (args.Length == 0) - { - _log.Error(ex, format); - } - else - { - _log.Error(ex, format, args); - } - } - - public void Info(string format, params object[] args) - { - if (args.Length == 0) - { - _log.Information(format); - } - else - { - _log.Information(format, args); - } - } - - public void Info(Exception ex, string format, params object[] args) - { - if (args.Length == 0) - { - _log.Information(ex, format); - } - else - { - _log.Information(ex, format, args); - } - } - - public void Debug(string format, params object[] args) - { - if (args.Length == 0) - { - _log.Debug(format); - } - else - { - _log.Debug(format, args); - } - } - - public void Debug(Exception ex, string format, params object[] args) - { - if (args.Length == 0) - { - _log.Debug(ex, format); - } - else - { - _log.Debug(ex, format, args); - } - } -} diff --git a/src/EventStore.Core.Tests/Helpers/MiniClusterNode.cs b/src/EventStore.Core.Tests/Helpers/MiniClusterNode.cs index 7af8632d71..ca908fe9a1 100644 --- a/src/EventStore.Core.Tests/Helpers/MiniClusterNode.cs +++ b/src/EventStore.Core.Tests/Helpers/MiniClusterNode.cs @@ -20,10 +20,8 @@ using EventStore.Core.Services.Monitoring; using EventStore.Core.Services.PersistentSubscription.ConsumerStrategy; using EventStore.Core.Services.Storage.ReaderIndex; -using EventStore.Core.Tests.Services.Transport.Tcp; using EventStore.Core.TransactionLog.Chunks; using EventStore.Plugins.Subsystems; -using EventStore.TcpUnitTestPlugin; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Server.Kestrel.Core; using Microsoft.AspNetCore.Server.Kestrel.Https; @@ -43,8 +41,6 @@ public class MiniClusterNode private static readonly ILogger Log = Serilog.Log.ForContext>(); - public IPEndPoint InternalTcpEndPoint { get; } - public IPEndPoint ExternalTcpEndPoint { get; } public IPEndPoint HttpEndPoint { get; } public IPEndPoint ReplicationEndPoint { get; } @@ -64,34 +60,31 @@ public class MiniClusterNode public VNodeState NodeState = VNodeState.Unknown; private readonly IHost _host; - public MiniClusterNode(string pathname, int debugIndex, IPEndPoint internalTcp, IPEndPoint externalTcp, - IPEndPoint httpEndPoint, EndPoint[] gossipSeeds, ISubsystem[] subsystems = null, + public MiniClusterNode(string pathname, int debugIndex, IPEndPoint nodeEndPoint, IPEndPoint replicationEndPoint, + EndPoint[] gossipSeeds, ISubsystem[] subsystems = null, bool enableTrustedAuth = false, int memTableSize = 1000, bool disableFlushToDisk = false, bool readOnlyReplica = false, int nodePriority = 0, - string intHostAdvertiseAs = null, IExpiryStrategy expiryStrategy = null, + string replicationHostAdvertiseAs = null, + IExpiryStrategy expiryStrategy = null, ArchiveOptions archiveOptions = null, bool archiver = false, - int clusterSize = 3, bool unsafeAllowSurplusNodes = false, - string replicationHostAdvertiseAs = null) + int clusterSize = 3, bool unsafeAllowSurplusNodes = false) { RunningTime.Start(); RunCount += 1; DebugIndex = debugIndex; - InternalTcpEndPoint = internalTcp; - ExternalTcpEndPoint = externalTcp; - HttpEndPoint = httpEndPoint; - ReplicationEndPoint = internalTcp; + HttpEndPoint = nodeEndPoint; + ReplicationEndPoint = replicationEndPoint; _dbPath = Path.Combine( pathname, - $"mini-cluster-node-db-{externalTcp.Port}-{httpEndPoint.Port}"); + $"mini-cluster-node-db-{nodeEndPoint.Port}"); Directory.CreateDirectory(_dbPath); FileStreamExtensions.ConfigureFlush(disableFlushToDisk); subsystems ??= []; - subsystems = [.. subsystems, new TcpApiTestPlugin()]; var options = new ClusterVNodeOptions { @@ -121,14 +114,12 @@ public MiniClusterNode(string pathname, int debugIndex, IPEndPoint internalTcp, }, Interface = new() { - ReplicationIp = InternalTcpEndPoint.Address, - NodeIp = ExternalTcpEndPoint.Address, - ReplicationPort = InternalTcpEndPoint.Port, + NodeIp = HttpEndPoint.Address, NodePort = HttpEndPoint.Port, - ReplicationHeartbeatTimeout = 2_000, - ReplicationHeartbeatInterval = 2_000, - EnableTrustedAuth = enableTrustedAuth, - ReplicationHostAdvertiseAs = replicationHostAdvertiseAs ?? intHostAdvertiseAs + ReplicationIp = ReplicationEndPoint.Address, + ReplicationPort = ReplicationEndPoint.Port, + ReplicationHostAdvertiseAs = replicationHostAdvertiseAs, + EnableTrustedAuth = enableTrustedAuth }, Database = new() { @@ -153,14 +144,7 @@ public MiniClusterNode(string pathname, int debugIndex, IPEndPoint internalTcp, PlugableComponents = subsystems }; - var configuration = new List> { - new("EventStore:TcpPlugin:NodeTcpPort", externalTcp.Port.ToString()), - new("EventStore:TcpPlugin:EnableExternalTcp", "true"), - new("EventStore:TcpUnitTestPlugin:NodeTcpPort", externalTcp.Port.ToString()), - new("EventStore:TcpUnitTestPlugin:NodeHeartbeatInterval", "10000"), - new("EventStore:TcpUnitTestPlugin:NodeHeartbeatTimeout", "10000"), - new("EventStore:TcpUnitTestPlugin:Insecure", options.Application.Insecure.ToString()), - }; + var configuration = new List>(); if (archiveOptions is not null) { @@ -180,9 +164,9 @@ public MiniClusterNode(string pathname, int debugIndex, IPEndPoint internalTcp, var inMemConf = new ConfigurationBuilder() .AddInMemoryCollection(configuration) .Build(); - var serverCertificate = ssl_connections.GetServerCertificate(); + var serverCertificate = TestCertificates.GetServerCertificate(); var trustedRootCertificates = - new X509Certificate2Collection(ssl_connections.GetRootCertificate()); + new X509Certificate2Collection(TestCertificates.GetRootCertificate()); options = options.Secure(trustedRootCertificates, serverCertificate); _isReadOnlyReplica = readOnlyReplica; @@ -195,8 +179,8 @@ public MiniClusterNode(string pathname, int debugIndex, IPEndPoint internalTcp, RuntimeInformation.RuntimeMode, "GC:", GC.MaxGeneration == 0 ? "NON-GENERATION (PROBABLY BOEHM)" - : $"{GC.MaxGeneration + 1} GENERATIONS", "DBPATH:", _dbPath, "ExTCP ENDPOINT:", - ExternalTcpEndPoint, "ExHTTP ENDPOINT:", HttpEndPoint); + : $"{GC.MaxGeneration + 1} GENERATIONS", "DBPATH:", _dbPath, "NODE ENDPOINT:", + HttpEndPoint, "HTTP ENDPOINT:", HttpEndPoint); var logFormatFactory = LogFormatHelper.LogFormatFactory; Node = new ClusterVNode(options, logFormatFactory, new AuthenticationProviderFactory( @@ -240,11 +224,7 @@ void ConfigureHttps(ListenOptions options) } o.Listen(HttpEndPoint, ConfigureHttps); - o.Listen(ReplicationEndPoint, options => - { - options.Protocols = HttpProtocols.Http2; - ConfigureHttps(options); - }); + o.Listen(ReplicationEndPoint, ConfigureHttps); }) .UseStartup(Node.Startup); }) diff --git a/src/EventStore.Core.Tests/Helpers/MiniNode.cs b/src/EventStore.Core.Tests/Helpers/MiniNode.cs index 9a41dc772b..95dd4d7d5e 100644 --- a/src/EventStore.Core.Tests/Helpers/MiniNode.cs +++ b/src/EventStore.Core.Tests/Helpers/MiniNode.cs @@ -5,7 +5,6 @@ using System.Linq; using System.Net; using System.Net.Http; -using System.Net.Sockets; using System.Security.Cryptography.X509Certificates; using System.Threading.Tasks; using EventStore.Common.Utils; @@ -20,13 +19,11 @@ using EventStore.Core.Services.Monitoring; using EventStore.Core.Services.Storage.ReaderIndex; using EventStore.Core.Tests.Index.Hashers; -using EventStore.Core.Tests.Services.Transport.Tcp; using EventStore.Core.TransactionLog.Chunks; using EventStore.Plugins.Authentication; using EventStore.Plugins.Authorization; using EventStore.Plugins.Subsystems; using EventStore.Plugins.Transforms; -using EventStore.TcpUnitTestPlugin; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Server.Kestrel.Core; using Microsoft.AspNetCore.Server.Kestrel.Https; @@ -45,8 +42,6 @@ public class MiniNode public const int CachedChunkSize = ChunkSize + ChunkHeader.Size + ChunkFooter.Size; protected static readonly ILogger Log = Serilog.Log.ForContext(); - public IPEndPoint TcpEndPoint { get; protected set; } - public IPEndPoint IntTcpEndPoint { get; protected set; } public IPEndPoint HttpEndPoint { get; protected set; } } @@ -71,7 +66,7 @@ public class MiniNode : MiniNode, IAsyncDisposable public Task AdminUserCreated => _adminUserCreated.Task; public MiniNode(string pathname, - int? tcpPort = null, int? httpPort = null, + int? httpPort = null, ISubsystem[] subsystems = null, int chunkSize = ChunkSize, int cachedChunkSize = CachedChunkSize, bool enableTrustedAuth = false, int memTableSize = 1000, @@ -97,26 +92,21 @@ public MiniNode(string pathname, var ip = IPAddress.Loopback; - int extTcpPort = tcpPort ?? PortsHelper.GetAvailablePort(ip); int httpEndPointPort = httpPort ?? PortsHelper.GetAvailablePort(ip); - int intTcpPort = PortsHelper.GetAvailablePort(ip); if (string.IsNullOrEmpty(dbPath)) { DbPath = Path.Combine(pathname, - $"mini-node-db-{extTcpPort}-{httpEndPointPort}"); + $"mini-node-db-{httpEndPointPort}"); } else { DbPath = dbPath; } - TcpEndPoint = new IPEndPoint(ip, extTcpPort); - IntTcpEndPoint = new IPEndPoint(ip, intTcpPort); HttpEndPoint = new IPEndPoint(ip, httpEndPointPort); subsystems ??= []; - subsystems = [.. subsystems, new TcpApiTestPlugin()]; var options = new ClusterVNodeOptions { @@ -130,8 +120,6 @@ public MiniNode(string pathname, }, Interface = new() { - ReplicationHeartbeatInterval = 10_000, - ReplicationHeartbeatTimeout = 10_000, EnableTrustedAuth = enableTrustedAuth }, Cluster = new() @@ -162,21 +150,11 @@ public MiniNode(string pathname, LoadedOptions = ClusterVNodeOptions.GetLoadedOptions(new ConfigurationBuilder() .AddEventStoreDefaultValues() .Build()), - }.Secure(new X509Certificate2Collection(ssl_connections.GetRootCertificate()), - ssl_connections.GetServerCertificate()) - .WithReplicationEndpointOn(IntTcpEndPoint) - .WithExternalTcpOn(TcpEndPoint) + }.Secure(new X509Certificate2Collection(TestCertificates.GetRootCertificate()), + TestCertificates.GetServerCertificate()) .WithNodeEndpointOn(HttpEndPoint); - var inMemConf = new ConfigurationBuilder() - .AddInMemoryCollection(new KeyValuePair[] { - new("EventStore:TcpPlugin:NodeTcpPort", extTcpPort.ToString()), - new("EventStore:TcpPlugin:EnableExternalTcp", "true"), - new("EventStore:TcpUnitTestPlugin:NodeTcpPort", extTcpPort.ToString()), - new("EventStore:TcpUnitTestPlugin:NodeHeartbeatInterval", "10000"), - new("EventStore:TcpUnitTestPlugin:NodeHeartbeatTimeout", "10000"), - new("EventStore:TcpUnitTestPlugin:Insecure", options.Application.Insecure.ToString()), - }).Build(); + var inMemConf = new ConfigurationBuilder().Build(); if (advertisedExtHostAddress != null) { @@ -200,7 +178,7 @@ public MiniNode(string pathname, ? "NON-GENERATION (PROBABLY BOEHM)" : $"{GC.MaxGeneration + 1} GENERATIONS", "DBPATH:", DbPath, - "TCP ENDPOINT:", TcpEndPoint, + "NODE ENDPOINT:", HttpEndPoint, "HTTP ENDPOINT:", HttpEndPoint); var logFormatFactory = LogFormatHelper.LogFormatFactory @@ -244,12 +222,12 @@ public MiniNode(string pathname, { options.UseHttps(new HttpsConnectionAdapterOptions { - ServerCertificate = ssl_connections.GetServerCertificate(), + ServerCertificate = TestCertificates.GetServerCertificate(), ClientCertificateMode = ClientCertificateMode.AllowCertificate, ClientCertificateValidation = (certificate, chain, sslPolicyErrors) => { var (isValid, error) = - ClusterVNode.ValidateClientCertificate(certificate, chain, sslPolicyErrors, () => null, () => new X509Certificate2Collection(ssl_connections.GetRootCertificate())); + ClusterVNode.ValidateClientCertificate(certificate, chain, sslPolicyErrors, () => null, () => new X509Certificate2Collection(TestCertificates.GetRootCertificate())); if (!isValid && error != null) { Log.Error("Client certificate validation error: {e}", error); @@ -335,25 +313,6 @@ void WaitForAdminUser(StorageMessage.EventCommitted m) Log.Information("MiniNode successfully started!"); } - public async Task WaitForTcpEndPoint() - { - while (true) - { - using var client = new TcpClient(); - - try - { - await client.ConnectAsync(TcpEndPoint.Address, TcpEndPoint.Port) - .WaitAsync(TimeSpan.FromMilliseconds(250)); - return; - } - catch (Exception ex) when (ex is SocketException or TimeoutException) - { - await Task.Delay(100); - } - } - } - public async Task Shutdown(bool keepDb = false) { diff --git a/src/EventStore.Core.Tests/Integration/Archive/when_archiving_and_restoring_a_cluster.cs b/src/EventStore.Core.Tests/Integration/Archive/when_archiving_and_restoring_a_cluster.cs index 781da7f1ce..d8e41d20c9 100644 --- a/src/EventStore.Core.Tests/Integration/Archive/when_archiving_and_restoring_a_cluster.cs +++ b/src/EventStore.Core.Tests/Integration/Archive/when_archiving_and_restoring_a_cluster.cs @@ -2,11 +2,12 @@ using System.IO; using System.Linq; using System.Net; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Amazon.S3; using Amazon.S3.Model; -using EventStore.ClientAPI; +using EventStore.Client.Streams; using EventStore.Core.Data; using EventStore.Core.Messages; using EventStore.Core.Messaging; @@ -18,7 +19,11 @@ using EventStore.Core.Tests.Helpers; using EventStore.Core.TransactionLog.Chunks.TFChunk; using EventStore.Core.TransactionLog.FileNamingStrategy; +using Google.Protobuf; +using Grpc.Net.Client; using NUnit.Framework; +using GrpcMetadata = EventStore.Core.Services.Transport.Grpc.Constants.Metadata; +using StreamsClient = EventStore.Client.Streams.Streams.StreamsClient; namespace EventStore.Core.Tests.Integration.Archive; @@ -44,6 +49,8 @@ public class when_archiving_and_restoring_a_cluster private long _archivedCheckpoint; private int _restoredNodeIndex; private int _completedIterations; + private GrpcChannel _channel; + private StreamsClient _client; protected override int NodeCount => 4; protected override TimeSpan GivenTimeout => SoakTimeout; @@ -117,19 +124,13 @@ protected override MiniClusterNode CreateNode( new( PathName, index, - endpoints.InternalTcp, - endpoints.ExternalTcp, - endpoints.HttpEndPoint, + endpoints.NodeEndPoint, + endpoints.ReplicationEndPoint, gossipSeeds, readOnlyReplica: index == ArchiverNodeIndex, archiveOptions: _archiveOptions.Enabled ? _archiveOptions : null, archiver: index == ArchiverNodeIndex); - protected override IEventStoreConnection CreateConnection() => - EventStoreConnection.Create( - ConnectionSettings.Create().DisableServerCertificateValidation(), - GetLeader().ExternalTcpEndPoint); - protected override async Task Given() { var payload = new byte[256 * 1024]; @@ -141,10 +142,30 @@ protected override async Task Given() var leader = await ReconnectToLeader(); for (var eventNumber = 0; eventNumber < EventsPerIteration; eventNumber++) { - await _conn.AppendToStreamAsync( - Stream, - EventStore.ClientAPI.ExpectedVersion.Any, - new EventData(Guid.NewGuid(), "archive-event", isJson: false, payload, Array.Empty())); + using var call = _client.Append(); + await call.RequestStream.WriteAsync(new AppendReq + { + Options = new() + { + Any = new(), + StreamIdentifier = new() { StreamName = ByteString.CopyFromUtf8(Stream) } + } + }); + await call.RequestStream.WriteAsync(new AppendReq + { + ProposedMessage = new() + { + Id = Core.Services.Transport.Grpc.Uuid.NewUuid().ToDto(), + Data = ByteString.CopyFrom(payload), + CustomMetadata = ByteString.Empty, + Metadata = { + { GrpcMetadata.Type, "archive-event" }, + { GrpcMetadata.ContentType, GrpcMetadata.ContentTypes.ApplicationOctetStream } + } + } + }); + await call.RequestStream.CompleteAsync(); + await call.ResponseAsync; } AssertEx.IsOrBecomesTrue( @@ -175,11 +196,16 @@ await _conn.AppendToStreamAsync( private async Task> ReconnectToLeader() { var leader = GetLeader(); - _conn?.Close(); - _conn = EventStoreConnection.Create( - ConnectionSettings.Create().DisableServerCertificateValidation(), - leader.ExternalTcpEndPoint); - await _conn.ConnectAsync(); + _channel?.Dispose(); + _channel = GrpcChannel.ForAddress(new Uri($"https://{leader.HttpEndPoint}"), + new GrpcChannelOptions + { + HttpHandler = new SocketsHttpHandler + { + SslOptions = { RemoteCertificateValidationCallback = delegate { return true; } } + } + }); + _client = new StreamsClient(_channel); return leader; } @@ -235,7 +261,7 @@ private async Task RestoreNode(int nodeIndex, int coldChunkNumber) private EndPoint[] GossipSeedsFor(int nodeIndex) => _nodeEndpoints .Where((_, index) => index != nodeIndex) - .Select(x => (EndPoint)x.HttpEndPoint) + .Select(x => (EndPoint)x.NodeEndPoint) .ToArray(); private async Task WaitForArchiveCheckpoint(long minimum) @@ -258,6 +284,7 @@ private async Task WaitForArchiveCheckpoint(long minimum) [OneTimeTearDown] public override async Task TestFixtureTearDown() { + _channel?.Dispose(); await base.TestFixtureTearDown(); if (_s3Client is null) { diff --git a/src/EventStore.Core.Tests/Integration/specification_with_cluster.cs b/src/EventStore.Core.Tests/Integration/specification_with_cluster.cs index 6b8aa33884..b88b48a586 100644 --- a/src/EventStore.Core.Tests/Integration/specification_with_cluster.cs +++ b/src/EventStore.Core.Tests/Integration/specification_with_cluster.cs @@ -4,7 +4,6 @@ using System.Net; using System.Net.Sockets; using System.Threading.Tasks; -using EventStore.ClientAPI; using EventStore.Core.Data; using EventStore.Core.Tests.Helpers; using EventStore.Plugins.Subsystems; @@ -17,7 +16,6 @@ public abstract class specification_with_cluster : Specif { protected MiniClusterNode[] _nodes; protected Endpoints[] _nodeEndpoints; - protected IEventStoreConnection _conn; protected virtual TimeSpan GivenTimeout { get; } = TimeSpan.FromMinutes(2); protected virtual int NodeCount => 3; @@ -25,15 +23,13 @@ public abstract class specification_with_cluster : Specif protected class Endpoints { - public readonly IPEndPoint InternalTcp; - public readonly IPEndPoint ExternalTcp; - public readonly IPEndPoint HttpEndPoint; + public readonly IPEndPoint NodeEndPoint; + public readonly IPEndPoint ReplicationEndPoint; public IEnumerable Ports() { - yield return InternalTcp.Port; - yield return ExternalTcp.Port; - yield return HttpEndPoint.Port; + yield return NodeEndPoint.Port; + yield return ReplicationEndPoint.Port; } private readonly List _sockets; @@ -44,21 +40,15 @@ public Endpoints() var defaultLoopBack = new IPEndPoint(IPAddress.Loopback, 0); - var internalTcp = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); - internalTcp.Bind(defaultLoopBack); - _sockets.Add(internalTcp); + var nodeEndpoint = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + nodeEndpoint.Bind(defaultLoopBack); + _sockets.Add(nodeEndpoint); + var replicationEndpoint = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + replicationEndpoint.Bind(defaultLoopBack); + _sockets.Add(replicationEndpoint); - var externalTcp = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); - externalTcp.Bind(defaultLoopBack); - _sockets.Add(externalTcp); - - var httpEndPoint = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); - httpEndPoint.Bind(defaultLoopBack); - _sockets.Add(httpEndPoint); - - InternalTcp = CopyEndpoint((IPEndPoint)internalTcp.LocalEndPoint); - ExternalTcp = CopyEndpoint((IPEndPoint)externalTcp.LocalEndPoint); - HttpEndPoint = CopyEndpoint((IPEndPoint)httpEndPoint.LocalEndPoint); + NodeEndPoint = CopyEndpoint((IPEndPoint)nodeEndpoint.LocalEndPoint); + ReplicationEndPoint = CopyEndpoint((IPEndPoint)replicationEndpoint.LocalEndPoint); } public void DisposeSockets() @@ -102,7 +92,7 @@ public override async Task TestFixtureSetUp() nodeIndex, _nodeEndpoints[nodeIndex], _nodeEndpoints.Where((_, otherIndex) => otherIndex != nodeIndex) - .Select(x => (EndPoint)x.HttpEndPoint) + .Select(x => (EndPoint)x.NodeEndPoint) .ToArray(), wait)); _nodes[nodeIndex] = _nodeCreationFactory[nodeIndex](true); @@ -144,9 +134,6 @@ public override async Task TestFixtureSetUp() onFail: MiniNodeLogging.WriteLogs, msg: $"Waiting for followers timed out! States={string.Join(", ", _nodes.Select(n => n.NodeState))}"); - _conn = CreateConnection(); - await _conn.ConnectAsync(); - try { await Given().WithTimeout(GivenTimeout); @@ -158,9 +145,6 @@ public override async Task TestFixtureSetUp() } } - protected virtual IEventStoreConnection CreateConnection() => - EventStoreConnection.Create(_nodes[0].ExternalTcpEndPoint); - protected virtual void BeforeNodesStart() { } @@ -171,8 +155,7 @@ protected virtual void BeforeNodesStart() protected virtual MiniClusterNode CreateNode(int index, Endpoints endpoints, EndPoint[] gossipSeeds, bool wait = true) => new( - PathName, index, endpoints.InternalTcp, - endpoints.ExternalTcp, endpoints.HttpEndPoint, + PathName, index, endpoints.NodeEndPoint, endpoints.ReplicationEndPoint, subsystems: Array.Empty(), gossipSeeds: gossipSeeds); [TearDown] @@ -187,7 +170,6 @@ public void AfterEachTest() [OneTimeTearDown] public override async Task TestFixtureTearDown() { - _conn?.Close(); if (_nodes is not null) { await Task.WhenAll(_nodes.Where(node => node is not null).Select(node => node.Shutdown())); diff --git a/src/EventStore.Core.Tests/Integration/when_cluster_nodes_are_restarted.cs b/src/EventStore.Core.Tests/Integration/when_cluster_nodes_are_restarted.cs index 53791c1be1..ee99b6ff2d 100644 --- a/src/EventStore.Core.Tests/Integration/when_cluster_nodes_are_restarted.cs +++ b/src/EventStore.Core.Tests/Integration/when_cluster_nodes_are_restarted.cs @@ -102,7 +102,7 @@ private int SelectRestartNode(bool[] restartedNodes, bool restartLeader) private EndPoint[] GossipSeedsFor(int restartedNodeIndex) => _nodeEndpoints .Where((_, index) => index != restartedNodeIndex) - .Select(x => (EndPoint)x.HttpEndPoint) + .Select(x => (EndPoint)x.NodeEndPoint) .ToArray(); [Test] diff --git a/src/EventStore.Core.Tests/Integration/when_node_becomes_leader_with_unindexed_data.cs b/src/EventStore.Core.Tests/Integration/when_node_becomes_leader_with_unindexed_data.cs index b43356b8e2..0c83704b4d 100644 --- a/src/EventStore.Core.Tests/Integration/when_node_becomes_leader_with_unindexed_data.cs +++ b/src/EventStore.Core.Tests/Integration/when_node_becomes_leader_with_unindexed_data.cs @@ -49,9 +49,9 @@ private class WrongExpectedVersionException : Exception { } protected override async Task Given() { _nodeGossipSeeds = new[] { - new EndPoint[] {_nodeEndpoints[1].HttpEndPoint, _nodeEndpoints[2].HttpEndPoint}, - new EndPoint[] {_nodeEndpoints[0].HttpEndPoint, _nodeEndpoints[2].HttpEndPoint}, - new EndPoint[] {_nodeEndpoints[0].HttpEndPoint, _nodeEndpoints[1].HttpEndPoint} + new EndPoint[] {_nodeEndpoints[1].NodeEndPoint, _nodeEndpoints[2].NodeEndPoint}, + new EndPoint[] {_nodeEndpoints[0].NodeEndPoint, _nodeEndpoints[2].NodeEndPoint}, + new EndPoint[] {_nodeEndpoints[0].NodeEndPoint, _nodeEndpoints[1].NodeEndPoint} }; _httpClient = new HttpClient(new SocketsHttpHandler { @@ -164,8 +164,7 @@ private async Task> ReadAllEvents(IPEndPoint endpoint private MiniClusterNode CreateNode(int index, Endpoints endpoints, EndPoint[] gossipSeeds, int nodePriority, string replicationHostAdvertiseAs) => new( - PathName, index, endpoints.InternalTcp, - endpoints.ExternalTcp, endpoints.HttpEndPoint, + PathName, index, endpoints.NodeEndPoint, endpoints.ReplicationEndPoint, subsystems: Array.Empty(), gossipSeeds: gossipSeeds, nodePriority: nodePriority, replicationHostAdvertiseAs: replicationHostAdvertiseAs); diff --git a/src/EventStore.Core.Tests/Services/ElectionsService/ClusterSettingsFactory.cs b/src/EventStore.Core.Tests/Services/ElectionsService/ClusterSettingsFactory.cs index f63c8c9319..b072443420 100644 --- a/src/EventStore.Core.Tests/Services/ElectionsService/ClusterSettingsFactory.cs +++ b/src/EventStore.Core.Tests/Services/ElectionsService/ClusterSettingsFactory.cs @@ -1,9 +1,7 @@ using System; using System.Linq; using System.Net; -using System.Runtime.InteropServices; using EventStore.Core.Cluster.Settings; -using EventStore.Core.Tests.Services.Transport.Tcp; namespace EventStore.Core.Tests.Services.ElectionsService; @@ -14,13 +12,9 @@ public class ClusterSettingsFactory private static ClusterVNodeSettings CreateVNode(int nodeNumber, bool isReadOnlyReplica) { - int tcpIntPort = StartingPort + nodeNumber * 2, - tcpExtPort = tcpIntPort + 1, - httpPort = tcpIntPort + 11; + var httpPort = StartingPort + nodeNumber; return new ClusterVNodeSettings(Guid.NewGuid(), 0, - GetLoopbackForPort(tcpIntPort), null, - GetLoopbackForPort(tcpExtPort), null, GetLoopbackForPort(httpPort), 0, isReadOnlyReplica); } diff --git a/src/EventStore.Core.Tests/Services/ElectionsService/ClusterVNodeSettings.cs b/src/EventStore.Core.Tests/Services/ElectionsService/ClusterVNodeSettings.cs index e1fb14badc..b9c4a888d9 100644 --- a/src/EventStore.Core.Tests/Services/ElectionsService/ClusterVNodeSettings.cs +++ b/src/EventStore.Core.Tests/Services/ElectionsService/ClusterVNodeSettings.cs @@ -16,24 +16,14 @@ public class ClusterVNodeSettings public readonly bool ReadOnlyReplica; public ClusterVNodeSettings(Guid instanceId, int debugIndex, - IPEndPoint internalTcpEndPoint, - IPEndPoint internalSecureTcpEndPoint, - IPEndPoint externalTcpEndPoint, - IPEndPoint externalSecureTcpEndPoint, IPEndPoint httpEndPoint, int nodePriority, bool readOnlyReplica) { Ensure.NotEmptyGuid(instanceId, "instanceId"); - Ensure.Equal(false, internalTcpEndPoint == null && internalSecureTcpEndPoint == null, "Both internal TCP endpoints are null"); - Ensure.NotNull(httpEndPoint, nameof(httpEndPoint)); - NodeInfo = new VNodeInfo(instanceId, debugIndex, - internalTcpEndPoint, internalSecureTcpEndPoint, - externalTcpEndPoint, externalSecureTcpEndPoint, - httpEndPoint, - readOnlyReplica); + NodeInfo = new VNodeInfo(instanceId, debugIndex, httpEndPoint, readOnlyReplica); NodePriority = nodePriority; diff --git a/src/EventStore.Core.Tests/Services/ElectionsService/ElectionServiceUnit.cs b/src/EventStore.Core.Tests/Services/ElectionsService/ElectionServiceUnit.cs index b24a1e875e..a07c837474 100644 --- a/src/EventStore.Core.Tests/Services/ElectionsService/ElectionServiceUnit.cs +++ b/src/EventStore.Core.Tests/Services/ElectionsService/ElectionServiceUnit.cs @@ -47,11 +47,7 @@ public ElectionsServiceUnit(ClusterSettings clusterSettings) _bus = new(GetType().Name); var memberInfo = MemberInfo.Initial(clusterSettings.Self.NodeInfo.InstanceId, InitialDate, VNodeState.Unknown, true, - clusterSettings.Self.NodeInfo.InternalTcp, - clusterSettings.Self.NodeInfo.InternalSecureTcp, - clusterSettings.Self.NodeInfo.ExternalTcp, - clusterSettings.Self.NodeInfo.ExternalSecureTcp, - clusterSettings.Self.NodeInfo.HttpEndPoint, null, 0, 0, + clusterSettings.Self.NodeInfo.HttpEndPoint, null, 0, clusterSettings.Self.NodePriority, clusterSettings.Self.ReadOnlyReplica); ElectionsService = new Core.Services.ElectionsService(Publisher, @@ -83,11 +79,7 @@ private ClusterInfo BuildClusterInfo(ClusterSettings clusterSettings) InitialDate, VNodeState.Unknown, true, - clusterSettings.Self.NodeInfo.InternalTcp, - clusterSettings.Self.NodeInfo.InternalSecureTcp, - clusterSettings.Self.NodeInfo.ExternalTcp, - clusterSettings.Self.NodeInfo.ExternalSecureTcp, - clusterSettings.Self.NodeInfo.HttpEndPoint, null, 0, 0, + clusterSettings.Self.NodeInfo.HttpEndPoint, null, 0, LastCommitPosition, WriterCheckpoint, ChaserCheckpoint, -1, -1, @@ -98,11 +90,7 @@ private ClusterInfo BuildClusterInfo(ClusterSettings clusterSettings) InitialDate, VNodeState.Unknown, true, - x.NodeInfo.InternalTcp, - x.NodeInfo.InternalSecureTcp, - x.NodeInfo.ExternalTcp, - x.NodeInfo.ExternalSecureTcp, - x.NodeInfo.HttpEndPoint, null, 0, 0, + x.NodeInfo.HttpEndPoint, null, 0, LastCommitPosition, WriterCheckpoint, ChaserCheckpoint, -1, -1, @@ -196,9 +184,7 @@ public IEnumerable ListMembers(Func predicate = nu ? MemberInfo.ForManager(x.InstanceId, x.TimeStamp, x.IsAlive, x.HttpEndPoint) : MemberInfo.ForVNode(x.InstanceId, x.TimeStamp, x.State, x.IsAlive, - x.InternalTcpEndPoint, x.InternalSecureTcpEndPoint, - x.ExternalTcpEndPoint, x.ExternalSecureTcpEndPoint, - x.HttpEndPoint, null, 0, 0, + x.HttpEndPoint, null, 0, x.LastCommitPosition, x.WriterCheckpoint, x.ChaserCheckpoint, x.EpochPosition, x.EpochNumber, x.EpochId, x.NodePriority, x.IsReadOnlyReplica)); } diff --git a/src/EventStore.Core.Tests/Services/ElectionsService/ElectionsServiceTests.cs b/src/EventStore.Core.Tests/Services/ElectionsService/ElectionsServiceTests.cs index bcf9e0938f..378cc8075c 100644 --- a/src/EventStore.Core.Tests/Services/ElectionsService/ElectionsServiceTests.cs +++ b/src/EventStore.Core.Tests/Services/ElectionsService/ElectionsServiceTests.cs @@ -29,18 +29,12 @@ public abstract class ElectionsFixture protected static Func NodeFactory = (id) => new VNodeInfo( Guid.Parse($"00000000-0000-0000-0000-00000000000{id}"), id, - new IPEndPoint(IPAddress.Loopback, id), - new IPEndPoint(IPAddress.Loopback, id), - new IPEndPoint(IPAddress.Loopback, id), - new IPEndPoint(IPAddress.Loopback, id), new IPEndPoint(IPAddress.Loopback, id), false); protected static readonly Func MemberInfoFromVNode = (nodeInfo, timestamp, state, isAlive, epochNumber, epochId, priority) => MemberInfo.ForVNode( nodeInfo.InstanceId, timestamp, state, isAlive, - nodeInfo.InternalTcp, - nodeInfo.InternalSecureTcp, nodeInfo.ExternalTcp, nodeInfo.ExternalSecureTcp, - nodeInfo.HttpEndPoint, null, 0, 0, + nodeInfo.HttpEndPoint, null, 0, 0, 0, 0, 0, epochNumber, epochId, priority, nodeInfo.IsReadOnlyReplica); @@ -924,9 +918,7 @@ public void should_send_an_acceptance_to_other_members() new ElectionMessage.ElectionsDone(0,0, MemberInfo.ForVNode( _nodeThree.InstanceId, _timeProvider.UtcNow, VNodeState.Unknown, true, - _nodeThree.InternalTcp, - _nodeThree.InternalSecureTcp, _nodeThree.ExternalTcp, _nodeThree.ExternalSecureTcp, - _nodeThree.HttpEndPoint, null, 0, 0, 0, 0, 0, 0, 0, _epochId, 0, + _nodeThree.HttpEndPoint, null, 0, 0, 0, 0, 0, 0, _epochId, 0, _nodeThree.IsReadOnlyReplica)), new GrpcMessage.SendOverGrpc(_nodeThree.HttpEndPoint, new ElectionMessage.Accept(_node.InstanceId, _node.HttpEndPoint, @@ -1077,9 +1069,7 @@ public void should_complete_elections() new ElectionMessage.ElectionsDone(0,0, MemberInfo.ForVNode( _nodeTwo.InstanceId, _timeProvider.UtcNow, VNodeState.Unknown, true, - _nodeTwo.InternalTcp, - _nodeTwo.InternalSecureTcp, _nodeTwo.ExternalTcp, _nodeTwo.ExternalSecureTcp, - _nodeTwo.HttpEndPoint, null, 0, 0, 0, 0, 0, 0, 0, _epochId, 0, + _nodeTwo.HttpEndPoint, null, 0, 0, 0, 0, 0, 0, _epochId, 0, _nodeTwo.IsReadOnlyReplica)), }; _publisher.Messages.Should().BeEquivalentTo(expected); @@ -1314,9 +1304,7 @@ public void should_attempt_not_to_elect_previously_elected_leader() new ElectionMessage.ElectionsDone(3,1, MemberInfo.ForVNode( _nodeTwo.InstanceId, _timeProvider.UtcNow, VNodeState.Unknown, true, - _nodeTwo.InternalTcp, - _nodeTwo.InternalSecureTcp, _nodeTwo.ExternalTcp, _nodeTwo.ExternalSecureTcp, - _nodeTwo.HttpEndPoint, null, 0, 0, 0, 0, 0, 0, 0, _epochId, 0, + _nodeTwo.HttpEndPoint, null, 0, 0, 0, 0, 0, 0, _epochId, 0, _nodeTwo.IsReadOnlyReplica)), }; _publisher.Messages.Should().BeEquivalentTo(expected); @@ -1357,11 +1345,7 @@ public void should_throw_argument_exception() var endpoint = new IPEndPoint(IPAddress.Loopback, 1234); var nodeInfo = MemberInfo.Initial(Guid.NewGuid(), DateTime.UtcNow, VNodeState.ReadOnlyLeaderless, true, - endpoint, - endpoint, - endpoint, - endpoint, - endpoint, null, 0, 0, + endpoint, null, 0, 0, true); @@ -1405,9 +1389,7 @@ public void previous_leader_should_be_elected() new ElectionMessage.ElectionsDone(0,0, MemberInfo.ForVNode( _nodeThree.InstanceId, _timeProvider.UtcNow, VNodeState.Unknown, true, - _nodeThree.InternalTcp, - _nodeThree.InternalSecureTcp, _nodeThree.ExternalTcp, _nodeThree.ExternalSecureTcp, - _nodeThree.HttpEndPoint, null, 0, 0, + _nodeThree.HttpEndPoint, null, 0, 0, 0, 0, 0, 0, _epochId, 0, _nodeThree.IsReadOnlyReplica)), }; @@ -1505,9 +1487,7 @@ public void previous_leader_should_not_be_elected() new ElectionMessage.ElectionsDone(0,0, MemberInfo.ForVNode( _nodeTwo.InstanceId, _timeProvider.UtcNow, VNodeState.Unknown, true, - _nodeTwo.InternalTcp, - _nodeTwo.InternalSecureTcp, _nodeTwo.ExternalTcp, _nodeTwo.ExternalSecureTcp, - _nodeTwo.HttpEndPoint, null, 0, 0, + _nodeTwo.HttpEndPoint, null, 0, 0, 0, 0, 0, 0, _epochId, 0, _nodeTwo.IsReadOnlyReplica)), }; @@ -1541,9 +1521,7 @@ public void previous_leader_should_not_be_elected() new ElectionMessage.ElectionsDone(0,0, MemberInfo.ForVNode( _nodeTwo.InstanceId, _timeProvider.UtcNow, VNodeState.Unknown, true, - _nodeTwo.InternalTcp, - _nodeTwo.InternalSecureTcp, _nodeTwo.ExternalTcp, _nodeTwo.ExternalSecureTcp, - _nodeTwo.HttpEndPoint, null, 0, 0, + _nodeTwo.HttpEndPoint, null, 0, 0, 0, 0, 0, 0, _epochId, 0, _nodeTwo.IsReadOnlyReplica)), }; @@ -1598,9 +1576,7 @@ public void previous_leader_should_not_be_elected() new ElectionMessage.ElectionsDone(0,0, MemberInfo.ForVNode( _nodeTwo.InstanceId, _timeProvider.UtcNow, VNodeState.Unknown, true, - _nodeTwo.InternalTcp, - _nodeTwo.InternalSecureTcp, _nodeTwo.ExternalTcp, _nodeTwo.ExternalSecureTcp, - _nodeTwo.HttpEndPoint, null, 0, 0, + _nodeTwo.HttpEndPoint, null, 0, 0, 0, 0, 0, 0, _epochId, 0, _nodeTwo.IsReadOnlyReplica)), }; @@ -1634,9 +1610,7 @@ public void previous_leader_should_not_be_elected() new ElectionMessage.ElectionsDone(0,0, MemberInfo.ForVNode( _nodeTwo.InstanceId, _timeProvider.UtcNow, VNodeState.Unknown, true, - _nodeTwo.InternalTcp, - _nodeTwo.InternalSecureTcp, _nodeTwo.ExternalTcp, _nodeTwo.ExternalSecureTcp, - _nodeTwo.HttpEndPoint, null, 0, 0, + _nodeTwo.HttpEndPoint, null, 0, 0, 0, 0, 0, 0, _epochId, 0, _nodeTwo.IsReadOnlyReplica)), }; diff --git a/src/EventStore.Core.Tests/Services/ElectionsService/LeaderNode/ElectionsServiceUnitTests.cs b/src/EventStore.Core.Tests/Services/ElectionsService/LeaderNode/ElectionsServiceUnitTests.cs index 11c8462e3d..6e3a013cf5 100644 --- a/src/EventStore.Core.Tests/Services/ElectionsService/LeaderNode/ElectionsServiceUnitTests.cs +++ b/src/EventStore.Core.Tests/Services/ElectionsService/LeaderNode/ElectionsServiceUnitTests.cs @@ -40,7 +40,7 @@ public void Setup() seeds.Add(endPoint); var instanceId = Guid.Parse($"101EFD13-F9CD-49BE-9C6D-E6AF9AF5540{i}"); var memberInfo = MemberInfo.ForVNode(instanceId, DateTime.UtcNow, VNodeState.Unknown, true, - endPoint, null, endPoint, null, endPoint, null, 0, 0, -1, 0, 0, -1, -1, Guid.Empty, 0, false); + endPoint, null, 0, -1, 0, 0, -1, -1, Guid.Empty, 0, false); members.Add(memberInfo); _fakeTimeProvider = new FakeTimeProvider(); _scheduler = new FakeScheduler(new FakeTimer(), _fakeTimeProvider); @@ -379,7 +379,7 @@ Func epochNumber { var id = IdForNode(i); var ep = EndpointForNode(i); - return MemberInfo.ForVNode(id, DateTime.Now, VNodeState.Follower, true, ep, ep, ep, ep, ep, null, 0, 0, + return MemberInfo.ForVNode(id, DateTime.Now, VNodeState.Follower, true, ep, null, 0, -1, writerCheckpoint(i), chaserCheckpoint(i), 1, epochNumber(i), epochId, nodePriority(i), false); } diff --git a/src/EventStore.Core.Tests/Services/ElectionsService/Randomized/RandomizedElectionsTestCase.cs b/src/EventStore.Core.Tests/Services/ElectionsService/Randomized/RandomizedElectionsTestCase.cs index 68b6ce5fd9..168f65a466 100644 --- a/src/EventStore.Core.Tests/Services/ElectionsService/Randomized/RandomizedElectionsTestCase.cs +++ b/src/EventStore.Core.Tests/Services/ElectionsService/Randomized/RandomizedElectionsTestCase.cs @@ -74,7 +74,7 @@ public virtual void Init() var outputBus = new SynchronousScheduler($"ELECTIONS-OUTPUT-BUS-{i}"); var endPoint = new IPEndPoint(BaseEndPoint.Address, BaseEndPoint.Port + i); var memberInfo = MemberInfo.Initial(Guid.NewGuid(), DateTime.UtcNow, VNodeState.Unknown, true, - endPoint, endPoint, endPoint, endPoint, endPoint, null, 0, 0, 0, false); + endPoint, null, 0, 0, false); _instances.Add(new ElectionsInstance(memberInfo.InstanceId, endPoint, inputBus, outputBus)); sendOverHttpHandler.RegisterEndPoint(endPoint, inputBus); @@ -125,8 +125,7 @@ protected virtual GossipMessage.GossipUpdated GetInitialGossipFor(ElectionsInsta { var members = allInstances.Select( x => MemberInfo.ForVNode(x.InstanceId, DateTime.UtcNow, VNodeState.Unknown, true, - x.EndPoint, null, x.EndPoint, null, - x.EndPoint, null, 0, 0, -1, 0, 0, -1, -1, Guid.Empty, 0, false)); + x.EndPoint, null, 0, -1, 0, 0, -1, -1, Guid.Empty, 0, false)); var gossip = new GossipMessage.GossipUpdated(new ClusterInfo(members.ToArray())); return gossip; } diff --git a/src/EventStore.Core.Tests/Services/ElectionsService/Randomized/UpdateGossipProcessor.cs b/src/EventStore.Core.Tests/Services/ElectionsService/Randomized/UpdateGossipProcessor.cs index 7ed5cd5fc9..b9381ca8dc 100644 --- a/src/EventStore.Core.Tests/Services/ElectionsService/Randomized/UpdateGossipProcessor.cs +++ b/src/EventStore.Core.Tests/Services/ElectionsService/Randomized/UpdateGossipProcessor.cs @@ -63,7 +63,7 @@ public void Process(int iteration, RandTestQueueItem item) previousMembers[leaderIndex] = MemberInfo.ForVNode(previousLeaderInfo.InstanceId, DateTime.UtcNow, VNodeState.Leader, - previousLeaderInfo.IsAlive, leaderEndPoint, null, leaderEndPoint, null, leaderEndPoint, null, 0, 0, + previousLeaderInfo.IsAlive, leaderEndPoint, null, 0, -1, 0, 0, -1, -1, Guid.Empty, 0, false); } } @@ -82,7 +82,7 @@ public void Process(int iteration, RandTestQueueItem item) foreach (var memberInfo in updatedGossip) { - _sendOverGrpcProcessor.RegisterEndpointToSkip(memberInfo.ExternalTcpEndPoint, !memberInfo.IsAlive); + _sendOverGrpcProcessor.RegisterEndpointToSkip(memberInfo.HttpEndPoint, !memberInfo.IsAlive); } var updateGossipMessage = new GossipMessage.GossipUpdated(new ClusterInfo(updatedGossip)); diff --git a/src/EventStore.Core.Tests/Services/ElectionsService/Randomized/elections_service_5_nodes_with_1_known_when_started_and_set_full_imediately.cs b/src/EventStore.Core.Tests/Services/ElectionsService/Randomized/elections_service_5_nodes_with_1_known_when_started_and_set_full_imediately.cs index 339ca873bd..6722010f0f 100644 --- a/src/EventStore.Core.Tests/Services/ElectionsService/Randomized/elections_service_5_nodes_with_1_known_when_started_and_set_full_imediately.cs +++ b/src/EventStore.Core.Tests/Services/ElectionsService/Randomized/elections_service_5_nodes_with_1_known_when_started_and_set_full_imediately.cs @@ -37,7 +37,7 @@ private MemberInfo[] CreateInitialGossip(ElectionsInstance instance, ElectionsIn { return new[] { MemberInfo.ForVNode(instance.InstanceId, DateTime.UtcNow, VNodeState.Unknown, true, - instance.EndPoint, null, instance.EndPoint, null, instance.EndPoint, null, 0, 0, + instance.EndPoint, null, 0, -1, 0, 0, -1, -1, Guid.Empty, 0, false) }; } @@ -53,7 +53,7 @@ private MemberInfo[] CreateUpdatedGossip(int iteration, Console.WriteLine("Update item: {0} : {1}", iteration, item.EndPoint.GetPort()); return instances.Select((x, i) => MemberInfo.ForVNode(x.InstanceId, DateTime.UtcNow, VNodeState.Unknown, true, - x.EndPoint, null, x.EndPoint, null, x.EndPoint, null, 0, 0, + x.EndPoint, null, 0, -1, 0, 0, -1, -1, Guid.Empty, 0, false)).ToArray(); } diff --git a/src/EventStore.Core.Tests/Services/ElectionsService/Randomized/elections_service_5_nodes_with_1_known_when_started_and_set_to_full_later.cs b/src/EventStore.Core.Tests/Services/ElectionsService/Randomized/elections_service_5_nodes_with_1_known_when_started_and_set_to_full_later.cs index 05f165cfab..cdf621e6ab 100644 --- a/src/EventStore.Core.Tests/Services/ElectionsService/Randomized/elections_service_5_nodes_with_1_known_when_started_and_set_to_full_later.cs +++ b/src/EventStore.Core.Tests/Services/ElectionsService/Randomized/elections_service_5_nodes_with_1_known_when_started_and_set_to_full_later.cs @@ -37,7 +37,7 @@ private MemberInfo[] CreateInitialGossip(ElectionsInstance instance, ElectionsIn { return new[] { MemberInfo.ForVNode(instance.InstanceId, DateTime.UtcNow, VNodeState.Unknown, true, - instance.EndPoint, null, instance.EndPoint, null, instance.EndPoint, null, 0, 0, + instance.EndPoint, null, 0, -1, 0, 0, -1, -1, Guid.Empty, 0, false) }; } @@ -57,7 +57,7 @@ private MemberInfo[] CreateUpdatedGossip(int iteration, { return instances.Select((x, i) => MemberInfo.ForVNode(x.InstanceId, DateTime.UtcNow, VNodeState.Unknown, true, - x.EndPoint, null, x.EndPoint, null, x.EndPoint, null, 0, 0, + x.EndPoint, null, 0, -1, 0, 0, -1, -1, Guid.Empty, 0, false)) .ToArray(); } diff --git a/src/EventStore.Core.Tests/Services/GossipService/NodeGossipServiceTests.cs b/src/EventStore.Core.Tests/Services/GossipService/NodeGossipServiceTests.cs index 11ebbfbf88..2c877c6645 100644 --- a/src/EventStore.Core.Tests/Services/GossipService/NodeGossipServiceTests.cs +++ b/src/EventStore.Core.Tests/Services/GossipService/NodeGossipServiceTests.cs @@ -43,31 +43,15 @@ public NodeGossipServiceTestFixture() _currentNode = new VNodeInfo( Guid.Parse("00000000-0000-0000-0000-000000000001"), 1, - new IPEndPoint(IPAddress.Loopback, 1111), - new IPEndPoint(IPAddress.Loopback, 1111), - new IPEndPoint(IPAddress.Loopback, 1111), - new IPEndPoint(IPAddress.Loopback, 1111), new IPEndPoint(IPAddress.Loopback, 1111), false); _nodeTwo = new VNodeInfo( Guid.Parse("00000000-0000-0000-0000-000000000002"), 2, - new IPEndPoint(IPAddress.Loopback, 2222), - new IPEndPoint(IPAddress.Loopback, 2222), - new IPEndPoint(IPAddress.Loopback, 2222), - new IPEndPoint(IPAddress.Loopback, 2222), new IPEndPoint(IPAddress.Loopback, 2222), false); _nodeThree = new VNodeInfo( Guid.Parse("00000000-0000-0000-0000-000000000003"), 3, - new IPEndPoint(IPAddress.Loopback, 3333), - new IPEndPoint(IPAddress.Loopback, 3333), - new IPEndPoint(IPAddress.Loopback, 3333), - new IPEndPoint(IPAddress.Loopback, 3333), new IPEndPoint(IPAddress.Loopback, 3333), false); _nodeFour = new VNodeInfo( Guid.Parse("00000000-0000-0000-0000-000000000004"), 4, - new IPEndPoint(IPAddress.Loopback, 4444), - new IPEndPoint(IPAddress.Loopback, 4444), - new IPEndPoint(IPAddress.Loopback, 4444), - new IPEndPoint(IPAddress.Loopback, 4444), new IPEndPoint(IPAddress.Loopback, 4444), false); _getNodeToGossipTo = infos => infos.First(x => Equals(x.HttpEndPoint, _nodeTwo.HttpEndPoint)); @@ -120,8 +104,7 @@ protected static MemberInfo MemberInfoForVNode(VNodeInfo nodeInfo, DateTime utcN VNodeState nodeState = VNodeState.Initializing, string esVersion = VersionInfo.DefaultVersion, bool isAlive = true) { return MemberInfo.ForVNode(nodeInfo.InstanceId, utcNow, nodeState, isAlive, - nodeInfo.InternalTcp, nodeInfo.InternalSecureTcp, nodeInfo.ExternalTcp, - nodeInfo.ExternalSecureTcp, nodeInfo.HttpEndPoint, null, 0, 0, + nodeInfo.HttpEndPoint, null, 0, 0, writerCheckpoint ?? 0, 0, -1, epochNumber ?? -1, Guid.Empty, nodePriority ?? 0, false, esVersion); } @@ -743,7 +726,7 @@ protected override Message[] Given() => _nodeTwo.HttpEndPoint); [Test] - public void should_ignore_message_and_wait_for_tcp_to_decide() + public void should_ignore_message_and_wait_for_connection_state_to_change() { ExpectNoMessages(); } @@ -1031,7 +1014,7 @@ private static MemberInfo TestNodeFor(int identifier, bool isAlive, DateTime tim { var ipEndpoint = new IPEndPoint(IPAddress.Loopback, identifier); return MemberInfo.ForVNode(Guid.NewGuid(), timeStamp, VNodeState.Initializing, isAlive, - ipEndpoint, ipEndpoint, ipEndpoint, ipEndpoint, ipEndpoint, null, 0, 0, + ipEndpoint, null, 0, 0, 0, 0, -1, -1, Guid.Empty, 0, false); } @@ -1108,7 +1091,7 @@ private static MemberInfo TestNodeFor(int identifier, bool isAlive, DateTime tim { var ipEndpoint = new IPEndPoint(IPAddress.Loopback, identifier); return MemberInfo.ForVNode(Guid.NewGuid(), timeStamp, nodeState, isAlive, - ipEndpoint, ipEndpoint, ipEndpoint, ipEndpoint, ipEndpoint, null, 0, 0, + ipEndpoint, null, 0, 0, 0, 0, -1, -1, Guid.Empty, 0, false); } diff --git a/src/EventStore.Core.Tests/Services/Replication/LogReplication/LogReplicationFixture.cs b/src/EventStore.Core.Tests/Services/Replication/LogReplication/LogReplicationFixture.cs index a7514a3a51..1dfac124f9 100644 --- a/src/EventStore.Core.Tests/Services/Replication/LogReplication/LogReplicationFixture.cs +++ b/src/EventStore.Core.Tests/Services/Replication/LogReplication/LogReplicationFixture.cs @@ -157,14 +157,9 @@ private async ValueTask> CreateLeader(TFChunkDb db, Cancel timeStamp: DateTime.Now, state: VNodeState.Leader, isAlive: true, - internalTcpEndPoint: FakeEndPoint, - internalSecureTcpEndPoint: null, - externalTcpEndPoint: null, - externalSecureTcpEndPoint: null, httpEndPoint: FakeEndPoint, advertiseHostToClientAs: null, advertiseHttpPortToClientAs: 0, - advertiseTcpPortToClientAs: 0, lastCommitPosition: 0, writerCheckpoint: 0, chaserCheckpoint: 0, diff --git a/src/EventStore.Core.Tests/Services/Replication/ReadOnlyReplica/connecting_to_read_only_replica.cs b/src/EventStore.Core.Tests/Services/Replication/ReadOnlyReplica/connecting_to_read_only_replica.cs index f8e76a11ab..9faa415dc9 100644 --- a/src/EventStore.Core.Tests/Services/Replication/ReadOnlyReplica/connecting_to_read_only_replica.cs +++ b/src/EventStore.Core.Tests/Services/Replication/ReadOnlyReplica/connecting_to_read_only_replica.cs @@ -1,11 +1,18 @@ +using System; using System.Net; +using System.Net.Http; +using System.Text; using System.Threading.Tasks; -using EventStore.ClientAPI; -using EventStore.ClientAPI.Exceptions; -using EventStore.Core.Tests.ClientAPI.Helpers; +using EventStore.Client.Streams; +using EventStore.Core.Services.Transport.Grpc; using EventStore.Core.Tests.Helpers; using EventStore.Core.Tests.Integration; +using Google.Protobuf; +using Grpc.Core; +using Grpc.Net.Client; using NUnit.Framework; +using Empty = EventStore.Client.Empty; +using GrpcMetadata = EventStore.Core.Services.Transport.Grpc.Constants.Metadata; namespace EventStore.Core.Tests.Replication.ReadOnlyReplica; @@ -13,13 +20,15 @@ namespace EventStore.Core.Tests.Replication.ReadOnlyReplica; [TestFixture(typeof(LogFormat.V2), typeof(string))] public class connecting_to_read_only_replica : specification_with_cluster { + protected override async Task Given() => + await _nodes[2].AdminUserCreated.WithTimeout(TimeSpan.FromSeconds(30)); + protected override MiniClusterNode CreateNode(int index, Endpoints endpoints, EndPoint[] gossipSeeds, bool wait = true) { var isReadOnly = index == 2; var node = new MiniClusterNode( - PathName, index, endpoints.InternalTcp, - endpoints.ExternalTcp, endpoints.HttpEndPoint, gossipSeeds, + PathName, index, endpoints.NodeEndPoint, endpoints.ReplicationEndPoint, gossipSeeds, readOnlyReplica: isReadOnly); if (wait && !isReadOnly) { @@ -29,35 +38,82 @@ protected override MiniClusterNode CreateNode(int index, return node; } - protected override IEventStoreConnection CreateConnection() + private static CallOptions GetCallOptions() { - var settings = ConnectionSettings.Create() - .DisableServerCertificateValidation() - .PerformOnAnyNode(); - return EventStoreConnection.Create(settings, _nodes[2].ExternalTcpEndPoint); + var credentials = CallCredentials.FromInterceptor((_, metadata) => + { + metadata.Add("authorization", + $"Basic {Convert.ToBase64String(Encoding.ASCII.GetBytes("admin:changeit"))}"); + return Task.CompletedTask; + }); + return new CallOptions(credentials: credentials, deadline: DateTime.UtcNow.AddSeconds(30)); } - [Test] - public async Task append_to_stream_should_fail_with_not_supported_exception() + private Streams.StreamsClient CreateClient(out GrpcChannel channel, out HttpClient httpClient) { - const string stream = "append_to_stream_should_fail_with_not_supported_exception"; - await AssertEx.ThrowsAsync( - () => _conn.AppendToStreamAsync(stream, ExpectedVersion.Any, TestEvent.NewTestEvent())); + httpClient = new HttpClient(new SocketsHttpHandler + { + SslOptions = { RemoteCertificateValidationCallback = delegate { return true; } } + }); + channel = GrpcChannel.ForAddress(new Uri($"https://{_nodes[2].HttpEndPoint}"), + new GrpcChannelOptions { HttpClient = httpClient }); + return new Streams.StreamsClient(channel); } [Test] - public async Task delete_stream_should_fail_with_not_supported_exception() + public async Task append_to_stream_is_rejected() { - const string stream = "delete_stream_should_fail_with_not_supported_exception"; - await AssertEx.ThrowsAsync(() => - _conn.DeleteStreamAsync(stream, ExpectedVersion.Any)); + var client = CreateClient(out var channel, out var httpClient); + using (channel) + using (httpClient) + using (var call = client.Append(GetCallOptions())) + { + await call.RequestStream.WriteAsync(new AppendReq + { + Options = new() + { + Any = new Empty(), + StreamIdentifier = new() { StreamName = ByteString.CopyFromUtf8(nameof(append_to_stream_is_rejected)) } + } + }); + await call.RequestStream.WriteAsync(new AppendReq + { + ProposedMessage = new() + { + Id = Uuid.NewUuid().ToDto(), + Data = ByteString.Empty, + CustomMetadata = ByteString.Empty, + Metadata = + { + [GrpcMetadata.Type] = "test", + [GrpcMetadata.ContentType] = GrpcMetadata.ContentTypes.ApplicationJson + } + } + }); + await call.RequestStream.CompleteAsync(); + + var exception = Assert.ThrowsAsync(async () => await call.ResponseAsync); + Assert.That(exception.StatusCode, Is.EqualTo(StatusCode.NotFound)); + } } [Test] - public async Task start_transaction_should_fail_with_not_supported_exception() + public async Task delete_stream_is_rejected() { - const string stream = "start_transaction_should_fail_with_not_supported_exception"; - await AssertEx.ThrowsAsync(() => - _conn.StartTransactionAsync(stream, ExpectedVersion.Any)); + var client = CreateClient(out var channel, out var httpClient); + using (channel) + using (httpClient) + using (var call = client.DeleteAsync(new DeleteReq + { + Options = new() + { + Any = new Empty(), + StreamIdentifier = new() { StreamName = ByteString.CopyFromUtf8(nameof(delete_stream_is_rejected)) } + } + }, GetCallOptions())) + { + var exception = Assert.ThrowsAsync(async () => await call.ResponseAsync); + Assert.That(exception.StatusCode, Is.EqualTo(StatusCode.NotFound)); + } } } diff --git a/src/EventStore.Core.Tests/Services/RequestForwarding/GrpcRequestForwardingServiceTests.cs b/src/EventStore.Core.Tests/Services/RequestForwarding/GrpcRequestForwardingServiceTests.cs index b333bd58be..70d7ef4a93 100644 --- a/src/EventStore.Core.Tests/Services/RequestForwarding/GrpcRequestForwardingServiceTests.cs +++ b/src/EventStore.Core.Tests/Services/RequestForwarding/GrpcRequestForwardingServiceTests.cs @@ -571,7 +571,7 @@ public void pre_replica_connects_to_the_leader_http_endpoint() fixture.Supervisor.Handle(new SystemMessage.BecomePreReplica( Guid.NewGuid(), Guid.NewGuid(), fixture.Leader)); - Assert.That(fixture.Factory.EndPoints.Single(), Is.EqualTo(fixture.Leader.ReplicationEndPoint)); + Assert.That(fixture.Factory.EndPoints.Single(), Is.EqualTo(fixture.Leader.HttpEndPoint)); } [Test] @@ -791,10 +791,6 @@ private static MemberInfo CreateLeader(Guid? instanceId = null, int httpPort = 2 DateTime.UtcNow, VNodeState.Leader, true, - new DnsEndPoint("leader-replication.internal", 1112), - new DnsEndPoint("leader-replication.internal", 1113), - null, - null, new DnsEndPoint("leader.internal", httpPort), null, 0, @@ -803,11 +799,9 @@ private static MemberInfo CreateLeader(Guid? instanceId = null, int httpPort = 2 0, 0, 0, - 0, Guid.NewGuid(), 0, - false, - replicationEndPoint: new DnsEndPoint("leader-forwarding.internal", 3112)); + false); private static async Task WaitUntil(Func condition) { diff --git a/src/EventStore.Core.Tests/Services/RequestForwarding/GrpcRequestForwardingTransportSecurityTests.cs b/src/EventStore.Core.Tests/Services/RequestForwarding/GrpcRequestForwardingTransportSecurityTests.cs index 479f678e6f..50186afc83 100644 --- a/src/EventStore.Core.Tests/Services/RequestForwarding/GrpcRequestForwardingTransportSecurityTests.cs +++ b/src/EventStore.Core.Tests/Services/RequestForwarding/GrpcRequestForwardingTransportSecurityTests.cs @@ -53,7 +53,7 @@ public async Task cleartext_credential_rejection_completes_the_proxy_correlation supervisor.Handle(new ClientMessage.ForwardMessage(request)); - var response = publisher.Messages.OfType().Single(); + var response = publisher.Messages.OfType().Single(); Assert.That(response.CorrelationId, Is.EqualTo(request.InternalCorrId)); } @@ -119,10 +119,6 @@ private static MemberInfo CreateLeader() => MemberInfo.ForVNode( DateTime.UtcNow, VNodeState.Leader, true, - new DnsEndPoint("leader-replication.internal", 1112), - new DnsEndPoint("leader-replication.internal", 1113), - null, - null, new DnsEndPoint("leader.internal", 2113), null, 0, @@ -131,11 +127,9 @@ private static MemberInfo CreateLeader() => MemberInfo.ForVNode( 0, 0, 0, - 0, Guid.NewGuid(), 0, - false, - replicationEndPoint: new DnsEndPoint("leader-forwarding.internal", 3112)); + false); private sealed class RejectingServiceFactory : IGrpcRequestForwardingServiceFactory { diff --git a/src/EventStore.Core.Tests/Services/RequestForwarding/RequestForwardingServiceTests.cs b/src/EventStore.Core.Tests/Services/RequestForwarding/RequestForwardingServiceTests.cs index d0ed4eb5e5..8eb4dccb3c 100644 --- a/src/EventStore.Core.Tests/Services/RequestForwarding/RequestForwardingServiceTests.cs +++ b/src/EventStore.Core.Tests/Services/RequestForwarding/RequestForwardingServiceTests.cs @@ -55,13 +55,13 @@ public void not_authenticated_survives_the_client_correlation_rewrite() clientCorrelationId, new CallbackEnvelope(message => response = message), TimeSpan.FromMinutes(1), - new TcpMessage.NotAuthenticated(clientCorrelationId, "timeout")); + new ClientMessage.NotAuthenticated(clientCorrelationId, "timeout")); var service = new RequestForwardingService( new NoopPublisher(), forwardingProxy, TimeSpan.FromSeconds(1)); - service.Handle(new TcpMessage.NotAuthenticated(internalCorrelationId, "not authenticated")); + service.Handle(new ClientMessage.NotAuthenticated(internalCorrelationId, "not authenticated")); - var completion = (TcpMessage.NotAuthenticated)response; + var completion = (ClientMessage.NotAuthenticated)response; Assert.Multiple(() => { Assert.That(completion.CorrelationId, Is.EqualTo(clientCorrelationId)); diff --git a/src/EventStore.Core.Tests/Services/RequestManagement/Service/when_writing_and_deposed_as_leader.cs b/src/EventStore.Core.Tests/Services/RequestManagement/Service/when_writing_and_deposed_as_leader.cs index 8ef3d8d03b..6f3bf4ff30 100644 --- a/src/EventStore.Core.Tests/Services/RequestManagement/Service/when_writing_and_deposed_as_leader.cs +++ b/src/EventStore.Core.Tests/Services/RequestManagement/Service/when_writing_and_deposed_as_leader.cs @@ -42,10 +42,6 @@ private static MemberInfo FakeMemberInfo() return EventStore.Core.Cluster.MemberInfo.Initial(Guid.Empty, DateTime.UtcNow, VNodeState.Unknown, true, new IPEndPoint(IPAddress.Parse(ipAddress), port), - new IPEndPoint(IPAddress.Parse(ipAddress), port), - new IPEndPoint(IPAddress.Parse(ipAddress), port), - new IPEndPoint(IPAddress.Parse(ipAddress), port), - new IPEndPoint(IPAddress.Parse(ipAddress), port), - null, 0, 0, 0, false); + null, 0, 0, false); } } diff --git a/src/EventStore.Core.Tests/Services/RequestManagement/Service/when_writing_and_deposed_as_leader_and_replica_moves_forward.cs b/src/EventStore.Core.Tests/Services/RequestManagement/Service/when_writing_and_deposed_as_leader_and_replica_moves_forward.cs index 994a62f28d..4b7ba26312 100644 --- a/src/EventStore.Core.Tests/Services/RequestManagement/Service/when_writing_and_deposed_as_leader_and_replica_moves_forward.cs +++ b/src/EventStore.Core.Tests/Services/RequestManagement/Service/when_writing_and_deposed_as_leader_and_replica_moves_forward.cs @@ -37,10 +37,6 @@ private static MemberInfo FakeMemberInfo() return EventStore.Core.Cluster.MemberInfo.Initial(Guid.Empty, DateTime.UtcNow, VNodeState.Unknown, true, new IPEndPoint(IPAddress.Parse(ipAddress), port), - new IPEndPoint(IPAddress.Parse(ipAddress), port), - new IPEndPoint(IPAddress.Parse(ipAddress), port), - new IPEndPoint(IPAddress.Parse(ipAddress), port), - new IPEndPoint(IPAddress.Parse(ipAddress), port), - null, 0, 0, 0, false); + null, 0, 0, false); } } diff --git a/src/EventStore.Core.Tests/Services/Transport/Grpc/Forwarding/ForwardingGrpcCodecTests.cs b/src/EventStore.Core.Tests/Services/Transport/Grpc/Forwarding/ForwardingGrpcCodecTests.cs index 828810fab5..e4396f5693 100644 --- a/src/EventStore.Core.Tests/Services/Transport/Grpc/Forwarding/ForwardingGrpcCodecTests.cs +++ b/src/EventStore.Core.Tests/Services/Transport/Grpc/Forwarding/ForwardingGrpcCodecTests.cs @@ -415,8 +415,6 @@ public void not_handled_leader_info_round_trips() Guid.NewGuid(), ClientMessage.NotHandled.Types.NotHandledReason.NotLeader, new ClientMessage.NotHandled.Types.LeaderInfo( - new DnsEndPoint("leader-tcp.internal", 1113), - true, new DnsEndPoint("leader-http.internal", 2113))); var decoded = RoundTripResponse(message); @@ -425,28 +423,6 @@ public void not_handled_leader_info_round_trips() { Assert.That(decoded.CorrelationId, Is.EqualTo(message.CorrelationId)); Assert.That(decoded.Reason, Is.EqualTo(message.Reason)); - Assert.That(decoded.LeaderInfo.IsSecure, Is.True); - Assert.That(decoded.LeaderInfo.ExternalTcp, Is.EqualTo(message.LeaderInfo.ExternalTcp)); - Assert.That(decoded.LeaderInfo.Http, Is.EqualTo(message.LeaderInfo.Http)); - }); - } - - [Test] - public void not_handled_leader_info_without_external_tcp_round_trips_as_null() - { - var message = new ClientMessage.NotHandled( - Guid.NewGuid(), - ClientMessage.NotHandled.Types.NotHandledReason.NotLeader, - new ClientMessage.NotHandled.Types.LeaderInfo( - null, - false, - new DnsEndPoint("leader-http.internal", 2113))); - - var decoded = RoundTripResponse(message); - - Assert.Multiple(() => - { - Assert.That(decoded.LeaderInfo.ExternalTcp, Is.Null); Assert.That(decoded.LeaderInfo.Http, Is.EqualTo(message.LeaderInfo.Http)); }); } @@ -454,9 +430,9 @@ public void not_handled_leader_info_without_external_tcp_round_trips_as_null() [Test] public void not_authenticated_round_trips() { - var message = new TcpMessage.NotAuthenticated(Guid.NewGuid(), "not authenticated"); + var message = new ClientMessage.NotAuthenticated(Guid.NewGuid(), "not authenticated"); - var decoded = RoundTripResponse(message); + var decoded = RoundTripResponse(message); Assert.Multiple(() => { diff --git a/src/EventStore.Core.Tests/Services/Transport/Grpc/Replication/GrpcReplicaServiceSupervisorTests.cs b/src/EventStore.Core.Tests/Services/Transport/Grpc/Replication/GrpcReplicaServiceSupervisorTests.cs index 024d596827..dcd662b4f8 100644 --- a/src/EventStore.Core.Tests/Services/Transport/Grpc/Replication/GrpcReplicaServiceSupervisorTests.cs +++ b/src/EventStore.Core.Tests/Services/Transport/Grpc/Replication/GrpcReplicaServiceSupervisorTests.cs @@ -432,10 +432,6 @@ private static MemberInfo CreateLeader() => MemberInfo.ForVNode( DateTime.UtcNow, VNodeState.Leader, true, - new DnsEndPoint("leader-replication.internal", 1112), - null, - null, - null, new DnsEndPoint("leader.internal", 2113), null, 0, @@ -444,7 +440,6 @@ private static MemberInfo CreateLeader() => MemberInfo.ForVNode( 0, 0, 0, - 0, Guid.NewGuid(), 0, false, diff --git a/src/EventStore.Core.Tests/Services/Transport/Tcp/core_tcp_package.cs b/src/EventStore.Core.Tests/Services/Transport/Tcp/core_tcp_package.cs deleted file mode 100644 index ffcb22e037..0000000000 --- a/src/EventStore.Core.Tests/Services/Transport/Tcp/core_tcp_package.cs +++ /dev/null @@ -1,199 +0,0 @@ -using System; -using EventStore.Core.Services.Transport.Tcp; -using NUnit.Framework; - -namespace EventStore.Core.Tests.Services.Transport.Tcp; - -[TestFixture] -public class core_tcp_package -{ - [Test] - public void should_throw_argument_null_exception_when_created_as_authorized_but_login_not_provided() - { - Assert.Throws(() => - new TcpPackage(TcpCommand.BadRequest, TcpFlags.Authenticated, Guid.NewGuid(), null, "pa$$", - new byte[] { 1, 2, 3 })); - } - - [Test] - public void should_throw_argument_null_exception_when_created_as_authorized_but_password_not_provided() - { - Assert.Throws(() => - new TcpPackage(TcpCommand.BadRequest, TcpFlags.Authenticated, Guid.NewGuid(), "login", null, - new byte[] { 1, 2, 3 })); - } - - [Test] - public void should_throw_argument_null_exception_when_created_as_authorized_but_token_not_provided() - { - Assert.Throws(() => - new TcpPackage(TcpCommand.BadRequest, TcpFlags.Authenticated, Guid.NewGuid(), null, - new byte[] { 1, 2, 3 })); - } - - [Test] - public void should_throw_argument_exception_when_created_as_not_authorized_but_login_is_provided() - { - Assert.Throws(() => - new TcpPackage(TcpCommand.BadRequest, TcpFlags.None, Guid.NewGuid(), "login", null, - new byte[] { 1, 2, 3 })); - } - - [Test] - public void should_throw_argument_exception_when_created_as_not_authorized_but_password_is_provided() - { - Assert.Throws(() => - new TcpPackage(TcpCommand.BadRequest, TcpFlags.None, Guid.NewGuid(), null, "pa$$", - new byte[] { 1, 2, 3 })); - } - - [Test] - public void should_throw_argument_exception_when_created_as_not_authorized_but_token_is_provided() - { - Assert.Throws(() => - new TcpPackage(TcpCommand.BadRequest, TcpFlags.None, Guid.NewGuid(), "token", - new byte[] { 1, 2, 3 })); - } - - [Test] - public void not_authorized_with_data_should_serialize_and_deserialize_correctly() - { - var corrId = Guid.NewGuid(); - var refPkg = new TcpPackage(TcpCommand.BadRequest, TcpFlags.None, corrId, null, null, new byte[] { 1, 2, 3 }); - var bytes = refPkg.AsArraySegment(); - - var pkg = TcpPackage.FromArraySegment(bytes); - Assert.AreEqual(TcpCommand.BadRequest, pkg.Command); - Assert.AreEqual(TcpFlags.None, pkg.Flags); - Assert.AreEqual(corrId, pkg.CorrelationId); - Assert.False(pkg.Tokens.TryGetValue("uid", out _)); - Assert.False(pkg.Tokens.TryGetValue("pwd", out _)); - Assert.False(pkg.Tokens.TryGetValue("jwt", out _)); - - Assert.AreEqual(3, pkg.Data.Count); - Assert.AreEqual(1, pkg.Data.Array[pkg.Data.Offset + 0]); - Assert.AreEqual(2, pkg.Data.Array[pkg.Data.Offset + 1]); - Assert.AreEqual(3, pkg.Data.Array[pkg.Data.Offset + 2]); - } - - [Test] - public void not_authorized_with_empty_data_should_serialize_and_deserialize_correctly() - { - var corrId = Guid.NewGuid(); - var refPkg = new TcpPackage(TcpCommand.BadRequest, TcpFlags.None, corrId, null, null, new byte[0]); - var bytes = refPkg.AsArraySegment(); - - var pkg = TcpPackage.FromArraySegment(bytes); - Assert.AreEqual(TcpCommand.BadRequest, pkg.Command); - Assert.AreEqual(TcpFlags.None, pkg.Flags); - Assert.AreEqual(corrId, pkg.CorrelationId); - Assert.False(pkg.Tokens.TryGetValue("uid", out _)); - Assert.False(pkg.Tokens.TryGetValue("pwd", out _)); - Assert.False(pkg.Tokens.TryGetValue("jwt", out _)); - - Assert.AreEqual(0, pkg.Data.Count); - } - - [Test] - public void authorized_with_data_should_serialize_and_deserialize_correctly() - { - var corrId = Guid.NewGuid(); - var refPkg = new TcpPackage(TcpCommand.BadRequest, TcpFlags.Authenticated, corrId, "login", "pa$$", - new byte[] { 1, 2, 3 }); - var bytes = refPkg.AsArraySegment(); - - var pkg = TcpPackage.FromArraySegment(bytes); - Assert.AreEqual(TcpCommand.BadRequest, pkg.Command); - Assert.AreEqual(TcpFlags.Authenticated, pkg.Flags); - Assert.AreEqual(corrId, pkg.CorrelationId); - Assert.AreEqual("login", pkg.Tokens["uid"]); - Assert.AreEqual("pa$$", pkg.Tokens["pwd"]); - Assert.False(pkg.Tokens.TryGetValue("jwt", out _)); - - Assert.AreEqual(3, pkg.Data.Count); - Assert.AreEqual(1, pkg.Data.Array[pkg.Data.Offset + 0]); - Assert.AreEqual(2, pkg.Data.Array[pkg.Data.Offset + 1]); - Assert.AreEqual(3, pkg.Data.Array[pkg.Data.Offset + 2]); - } - - [Test] - public void authorized_with_empty_data_should_serialize_and_deserialize_correctly() - { - var corrId = Guid.NewGuid(); - var refPkg = new TcpPackage(TcpCommand.BadRequest, TcpFlags.Authenticated, corrId, "login", "pa$$", - new byte[0]); - var bytes = refPkg.AsArraySegment(); - - var pkg = TcpPackage.FromArraySegment(bytes); - Assert.AreEqual(TcpCommand.BadRequest, pkg.Command); - Assert.AreEqual(TcpFlags.Authenticated, pkg.Flags); - Assert.AreEqual(corrId, pkg.CorrelationId); - Assert.AreEqual("login", pkg.Tokens["uid"]); - Assert.AreEqual("pa$$", pkg.Tokens["pwd"]); - Assert.False(pkg.Tokens.TryGetValue("jwt", out _)); - - Assert.AreEqual(0, pkg.Data.Count); - } - - [Test] - public void token_authorized_with_data_should_serialize_and_deserialize_correctly() - { - var corrId = Guid.NewGuid(); - var refPkg = new TcpPackage(TcpCommand.BadRequest, TcpFlags.Authenticated, corrId, "token", - new byte[] { 1, 2, 3 }); - var bytes = refPkg.AsArraySegment(); - - var pkg = TcpPackage.FromArraySegment(bytes); - Assert.AreEqual(TcpCommand.BadRequest, pkg.Command); - Assert.AreEqual(TcpFlags.Authenticated, pkg.Flags); - Assert.AreEqual(corrId, pkg.CorrelationId); - Assert.AreEqual("token", pkg.Tokens["jwt"]); - Assert.False(pkg.Tokens.TryGetValue("uid", out _)); - Assert.False(pkg.Tokens.TryGetValue("pwd", out _)); - - Assert.AreEqual(3, pkg.Data.Count); - Assert.AreEqual(1, pkg.Data.Array[pkg.Data.Offset + 0]); - Assert.AreEqual(2, pkg.Data.Array[pkg.Data.Offset + 1]); - Assert.AreEqual(3, pkg.Data.Array[pkg.Data.Offset + 2]); - } - - [Test] - public void token_authorized_with_empty_data_should_serialize_and_deserialize_correctly() - { - var corrId = Guid.NewGuid(); - var refPkg = new TcpPackage(TcpCommand.BadRequest, TcpFlags.Authenticated, corrId, "token", - new byte[0]); - var bytes = refPkg.AsArraySegment(); - - var pkg = TcpPackage.FromArraySegment(bytes); - Assert.AreEqual(TcpCommand.BadRequest, pkg.Command); - Assert.AreEqual(TcpFlags.Authenticated, pkg.Flags); - Assert.AreEqual(corrId, pkg.CorrelationId); - Assert.AreEqual("token", pkg.Tokens["jwt"]); - Assert.False(pkg.Tokens.TryGetValue("uid", out _)); - Assert.False(pkg.Tokens.TryGetValue("pwd", out _)); - - Assert.AreEqual(0, pkg.Data.Count); - } - - [Test] - public void should_throw_argument_exception_when_login_too_long() - { - Assert.Throws(() => new TcpPackage(TcpCommand.BadRequest, TcpFlags.Authenticated, - Guid.NewGuid(), new string('*', TcpPackage.MaxLoginLength + 1), "pa$$", new byte[] { 1, 2, 3 })); - } - - [Test] - public void should_throw_argument_exception_when_password_too_long() - { - Assert.Throws(() => new TcpPackage(TcpCommand.BadRequest, TcpFlags.Authenticated, - Guid.NewGuid(), "login", new string('*', TcpPackage.MaxPasswordLength + 1), new byte[] { 1, 2, 3 })); - } - - [Test] - public void should_throw_argument_exception_when_token_too_long() - { - Assert.Throws(() => new TcpPackage(TcpCommand.BadRequest, TcpFlags.Authenticated, - Guid.NewGuid(), new string('*', TcpPackage.MaxTokenLength + 1), new byte[] { 1, 2, 3 })); - } -} diff --git a/src/EventStore.Core.Tests/Services/Transport/Tcp/ssl_connection.cs b/src/EventStore.Core.Tests/Services/Transport/Tcp/ssl_connection.cs index f6165d0f82..40a61a7f5d 100644 --- a/src/EventStore.Core.Tests/Services/Transport/Tcp/ssl_connection.cs +++ b/src/EventStore.Core.Tests/Services/Transport/Tcp/ssl_connection.cs @@ -8,7 +8,6 @@ using System.Text; using System.Threading; using EventStore.Common.Utils; -using EventStore.Core.Services.Transport.Tcp; using EventStore.Core.Tests.Helpers; using EventStore.Transport.Tcp; using NUnit.Framework; @@ -89,7 +88,7 @@ public void should_connect_to_each_other_and_send_data() { return (true, null); }, null, new TcpClientConnector(), - TcpConnectionManager.ConnectionTimeout, + TimeSpan.FromSeconds(1), conn => { Log.Information("Sending bytes..."); diff --git a/src/EventStore.Core.Tests/Services/Transport/Tcp/ssl_connections_mutual_auth.cs b/src/EventStore.Core.Tests/Services/Transport/Tcp/ssl_connections_mutual_auth.cs index 1a80ae435e..3a0c30b772 100644 --- a/src/EventStore.Core.Tests/Services/Transport/Tcp/ssl_connections_mutual_auth.cs +++ b/src/EventStore.Core.Tests/Services/Transport/Tcp/ssl_connections_mutual_auth.cs @@ -8,7 +8,6 @@ using System.Security.Cryptography.X509Certificates; using System.Threading; using EventStore.Common.Utils; -using EventStore.Core.Services.Transport.Tcp; using EventStore.Core.Tests.Helpers; using EventStore.Transport.Tcp; using NUnit.Framework; @@ -116,7 +115,7 @@ bool shouldConnectSuccessfully (cert, chain, err, _) => validateServerCertificate ? ClusterVNode.ValidateServerCertificate(cert, chain, err, () => null, () => rootCertificates, null) : (true, null), () => new X509CertificateCollection { clientCertificate }, new TcpClientConnector(), - TcpConnectionManager.ConnectionTimeout, + TimeSpan.FromSeconds(1), conn => { Log.Information("Sending bytes..."); diff --git a/src/EventStore.Core.Tests/Services/VNode/InaugurationManager/InaugurationManagerTests.cs b/src/EventStore.Core.Tests/Services/VNode/InaugurationManager/InaugurationManagerTests.cs index 8ae7429f0c..fc9ce572a8 100644 --- a/src/EventStore.Core.Tests/Services/VNode/InaugurationManager/InaugurationManagerTests.cs +++ b/src/EventStore.Core.Tests/Services/VNode/InaugurationManager/InaugurationManagerTests.cs @@ -20,8 +20,7 @@ public abstract class InaugurationManagerTests protected readonly MemberInfo _leader = MemberInfo.ForVNode( default, default, default, default, - new DnsEndPoint("localhost", default), default, default, default, - new DnsEndPoint("localhost", default), default, default, default, + new DnsEndPoint("localhost", default), default, default, default, default, default, default, default, default, default, default); protected readonly long _replicationTarget = 400; protected readonly long _indexTarget = 400; diff --git a/src/EventStore.Core.Tests/Services/VNode/ShutdownServiceTests.cs b/src/EventStore.Core.Tests/Services/VNode/ShutdownServiceTests.cs index f2adb06c98..d1011fa934 100644 --- a/src/EventStore.Core.Tests/Services/VNode/ShutdownServiceTests.cs +++ b/src/EventStore.Core.Tests/Services/VNode/ShutdownServiceTests.cs @@ -1,5 +1,4 @@ using System; -using System.Net; using DotNext.Net.Http; using EventStore.Core.Data; using EventStore.Core.Messages; @@ -16,10 +15,6 @@ public class ShutdownServiceTests = new( Guid.NewGuid(), 0, - new IPEndPoint(0, 0), - new IPEndPoint(IPAddress.Loopback, 1), - new IPEndPoint(IPAddress.Loopback, 2), - new IPEndPoint(IPAddress.Loopback, 3), new HttpEndPoint(new Uri("http://www.trogondb.com")), true); [Test] diff --git a/src/EventStore.Core.Tests/Services/VNode/leader_info_provider.cs b/src/EventStore.Core.Tests/Services/VNode/leader_info_provider.cs index 68de1f0112..9a34d143df 100644 --- a/src/EventStore.Core.Tests/Services/VNode/leader_info_provider.cs +++ b/src/EventStore.Core.Tests/Services/VNode/leader_info_provider.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Net; -using EventStore.Common.Utils; using EventStore.Core.Cluster; using EventStore.Core.Data; using EventStore.Core.Services.VNode; @@ -13,252 +11,45 @@ namespace EventStore.Core.Tests.Services.VNode; [TestFixture] public class leader_info_provider { - private const string DefaultHttpEndPoint = "9.9.9.9:9"; - - public static IEnumerable TestCases() - { - var leaderInfoCases = new TestCase[] { - new ("Leader: NoEndPoints", - Given: new (Leader: new()), - Expected: new (TcpEndPoint: null)), - new ("Leader: TcpEndPoint", - Given: new (Leader: new(TcpEndPoint: "1.1.1.1:1")), - Expected: new (TcpEndPoint: "1.1.1.1:1", IsTcpSecure: false)), - new ("Leader: SecureTcpEndPoint", - Given: new (Leader: new(SecureTcpEndPoint: "2.2.2.2:2")), - Expected: new (TcpEndPoint: "2.2.2.2:2", IsTcpSecure: true)), - new ("Leader: TcpEndPoint + SecureTcpEndPoint", - Given: new (Leader: new(TcpEndPoint: "1.1.1.1:1", SecureTcpEndPoint: "2.2.2.2:2")), - Expected: new (TcpEndPoint: "1.1.1.1:1", IsTcpSecure: true)), - // AdvertiseTcpPort - new ("Leader: AdvertiseTcpPort & NoEndPoints", - Given: new (Leader: new(AdvertiseTcpPort: 9)), - Expected: new (TcpEndPoint: null, IsTcpSecure: false)), - new ("Leader: AdvertiseTcpPort & TcpEndPoint", - Given: new (Leader: new(AdvertiseTcpPort: 9, TcpEndPoint: "1.1.1.1:1")), - Expected: new (TcpEndPoint: "1.1.1.1:9")), - new ("Leader: AdvertiseTcpPort & SecureTcpEndPoint", - Given: new (Leader: new(AdvertiseTcpPort: 9, SecureTcpEndPoint: "2.2.2.2:2")), - Expected: new (TcpEndPoint: "2.2.2.2:9", IsTcpSecure: true)), - new ("Leader: AdvertiseTcpPort & TcpEndPoint + SecureTcpEndPoint", - Given: new (Leader: new(TcpEndPoint: "1.1.1.1:1", SecureTcpEndPoint: "2.2.2.2:2", AdvertiseTcpPort: 9)), - Expected: new (TcpEndPoint: "1.1.1.1:9", IsTcpSecure: true)), - // AdvertiseHost - new ("Leader: AdvertiseHost & NoEndPoints", - Given: new (Leader: new(AdvertiseHost: "host", HttpEndPoint: "3.3.3.3:3")), - Expected: new (TcpEndPoint: null, IsTcpSecure: false, HttpEndPoint: "host:3")), - new ("Leader: AdvertiseHost & TcpEndPoint", - Given: new (Leader: new(AdvertiseHost: "host", TcpEndPoint: "1.1.1.1:1", HttpEndPoint: "3.3.3.3:3")), - Expected: new (TcpEndPoint: "host:1", HttpEndPoint: "host:3")), - new ("Leader: AdvertiseHost & SecureTcpEndPoint", - Given: new (Leader: new(AdvertiseHost: "host", SecureTcpEndPoint: "2.2.2.2:2", HttpEndPoint: "3.3.3.3:3")), - Expected: new (TcpEndPoint: "host:2", IsTcpSecure: true, HttpEndPoint: "host:3")), - new ("Leader: AdvertiseHost & TcpEndPoint + SecureTcpEndPoint", - Given: new (Leader: new(AdvertiseHost: "host", TcpEndPoint: "1.1.1.1:1", SecureTcpEndPoint: "2.2.2.2:2", HttpEndPoint: "3.3.3.3:3")), - Expected: new (TcpEndPoint: "host:1", IsTcpSecure: true, HttpEndPoint: "host:3")), //?? is this intended, secure with port of insecure - // AdvertiseHost + AdvertisePort - new ("Leader: AdvertiseHost + Port & NoEndPoints", - Given: new (Leader: new(AdvertiseHost: "host", AdvertiseTcpPort: 9, HttpEndPoint: "3.3.3.3:3")), - Expected: new (TcpEndPoint: null, IsTcpSecure: false, HttpEndPoint: "host:3")), - new ("Leader: AdvertiseHost + Port & TcpEndPoint", - Given: new (Leader: new(AdvertiseHost: "host", AdvertiseTcpPort: 9, TcpEndPoint: "1.1.1.1:1", HttpEndPoint: "3.3.3.3:3")), - Expected: new (TcpEndPoint: "host:9", IsTcpSecure: false, HttpEndPoint: "host:3")), - new ("Leader: AdvertiseHost + Port & SecureTcpEndPoint", - Given: new (Leader: new(AdvertiseHost: "host", AdvertiseTcpPort: 9, SecureTcpEndPoint: "2.2.2.2:2", HttpEndPoint: "3.3.3.3:3")), - Expected: new (TcpEndPoint: "host:9", IsTcpSecure: true, HttpEndPoint: "host:3")), - // AdvertiseHttpPort - new ("Leader: AdvertiseHttpPort & NoEndPoints", - Given: new (Leader: new(AdvertiseHttpPort: 8, HttpEndPoint: "3.3.3.3:3")), - Expected: new (HttpEndPoint: "3.3.3.3:8")), - new ("Leader: AdvertiseHttpPort & AdvertiseHost", - Given: new (Leader: new(AdvertiseHttpPort: 8, AdvertiseHost: "host", HttpEndPoint: "3.3.3.3:3")), - Expected: new (HttpEndPoint: "host:8")), - }; - - var nodeInfoCases = new TestCase[] { - new ("Gossip: NoEndPoints", - Given: new (Gossip: new()), - Expected: new (TcpEndPoint: null)), - new ("Gossip: TcpEndPoint", - Given: new (Gossip: new(TcpEndPoint: "4.4.4.4:4")), - Expected: new (TcpEndPoint: "4.4.4.4:4", IsTcpSecure: false)), - new ("Gossip: SecureTcpEndPoint", - Given: new (Gossip: new(SecureTcpEndPoint: "5.5.5.5:5")), - Expected: new (TcpEndPoint: "5.5.5.5:5", IsTcpSecure: true)), - // AnyIP SecureTcpEndPoint - new ("Gossip: TcpEndPoint + SecureTcpEndPoint", - Given: new (Gossip: new(TcpEndPoint: "4.4.4.4:4", SecureTcpEndPoint: "5.5.5.5:5")), - Expected: new (TcpEndPoint: "4.4.4.4:4", IsTcpSecure: true)), - // AnyIP & HttpEndPoint - new ("Gossip: HttpEndPoint", - Given: new (Gossip: new(HttpEndPoint: "8.8.8.8:8")), - Expected: new (HttpEndPoint: "8.8.8.8:8")), - new ("Gossip: HttpEndPoint & AdvertiseHost", - Given: new (Gossip: new(HttpEndPoint: "8.8.8.8:8", AdvertiseHost: "host")), - Expected: new (HttpEndPoint: "host:8")), - new ("Gossip: HttpEndPoint & AdvertisePort", - Given: new (Gossip: new(HttpEndPoint: "8.8.8.8:8", AdvertiseHttpPort: 9)), - Expected: new (HttpEndPoint: "8.8.8.8:9")), - new ("Gossip: HttpEndPoint & AdvertiseHost + Port", - Given: new (Gossip: new(HttpEndPoint: "8.8.8.8:8", AdvertiseHost: "host", AdvertiseHttpPort: 9)), - Expected: new (HttpEndPoint: "host:9")), - }; - - foreach (var testCase in leaderInfoCases.Concat(nodeInfoCases)) - { - yield return new object[] { testCase }; - } - } - - [TestCaseSource(nameof(TestCases))] - public void should_provide_as_expected(TestCase t) - { - - var given = t.BuildGiven(); - var expected = t.BuildExpected(); - - LeaderInfoProvider leaderInfoProvider = new LeaderInfoProvider( - given.GossipInfo, - given.LeaderInfo); - - var result = leaderInfoProvider.GetLeaderInfoEndPoints(); - - AssertAreEqual(expected.TcpEndPoint, result.AdvertisedTcpEndPoint, "TcpEndPoint"); - AssertAreEqual(expected.HttpEndPoint, result.AdvertisedHttpEndPoint, "HttpEndPoint"); - Assert.AreEqual(expected.IsSecure, result.IsTcpEndPointSecure); - } - - private void AssertAreEqual(EndPoint expected, EndPoint actual, string msg) - { - - if (expected == null) - { - Assert.IsNull(actual); - return; - } - - Assert.AreEqual(expected.GetHost(), actual.GetHost(), $"{msg} host"); - Assert.AreEqual(expected.GetPort(), actual.GetPort(), $"{msg} port"); - } - - public record TestCase(string Test, GivenInput Given, ExpectedInput Expected) - { - public override string ToString() => Test; - - public Given BuildGiven() => Given.Build(); - public Expected BuildExpected() => Expected.Build(); - } - - public record Given(MemberInfo LeaderInfo, GossipAdvertiseInfo GossipInfo); - - public record Expected(EndPoint TcpEndPoint, bool IsSecure, EndPoint HttpEndPoint); - - public record LeaderInput( - string TcpEndPoint = null, - string SecureTcpEndPoint = null, - string HttpEndPoint = null, - int AdvertiseTcpPort = 0, - int AdvertiseHttpPort = 0, - string AdvertiseHost = null); - - public record GossipInput( - string TcpEndPoint = null, - string SecureTcpEndPoint = null, - string HttpEndPoint = null, - int AdvertiseTcpPort = 0, - int AdvertiseHttpPort = 0, - string AdvertiseHost = null); - - public record GivenInput( - LeaderInput Leader = null, - GossipInput Gossip = null) - { - - public MemberInfo BuildLeaderInfo() - { - if (Leader == null) - { - return null; - } - - return MemberInfo.Initial( - instanceId: Guid.NewGuid(), - timeStamp: DateTime.Now, - state: VNodeState.Initializing, - isAlive: false, - internalTcpEndPoint: IPEndPoint.Parse("1.1.2.2:3"), - internalSecureTcpEndPoint: IPEndPoint.Parse("4.4.5.5:6"), - externalTcpEndPoint: Leader.TcpEndPoint == null ? null : IPEndPoint.Parse(Leader.TcpEndPoint), - externalSecureTcpEndPoint: Leader.SecureTcpEndPoint == null ? null : IPEndPoint.Parse(Leader.SecureTcpEndPoint), - httpEndPoint: IPEndPoint.Parse(Leader.HttpEndPoint ?? DefaultHttpEndPoint), - advertiseHostToClientAs: Leader.AdvertiseHost, - advertiseHttpPortToClientAs: Leader.AdvertiseHttpPort, - advertiseTcpPortToClientAs: Leader.AdvertiseTcpPort, - nodePriority: 1, - isReadOnlyReplica: false); - } - - public GossipAdvertiseInfo BuildGossipInfo() - { - return new GossipAdvertiseInfo( - externalTcp: ParseDnsEndPoint(Gossip?.TcpEndPoint), - externalSecureTcp: ParseDnsEndPoint(Gossip?.SecureTcpEndPoint), - httpEndPoint: ParseDnsEndPoint(Gossip?.HttpEndPoint ?? DefaultHttpEndPoint), - advertiseHostToClientAs: Gossip?.AdvertiseHost, - advertiseHttpPortToClientAs: Gossip?.AdvertiseHttpPort ?? 8, - advertiseTcpPortToClientAs: Gossip?.AdvertiseTcpPort ?? 9, - internalTcp: new DnsEndPoint("internal.tcp", 1), - internalSecureTcp: new DnsEndPoint("secure.tcp", 2), - advertiseInternalHostAs: null, - advertiseExternalHostAs: null, - advertiseHttpPortAs: 0); - } - - public Given Build() - { - return new Given( - BuildLeaderInfo(), - BuildGossipInfo()); - } - } - - public record ExpectedInput(string TcpEndPoint = null, string HttpEndPoint = null, bool IsTcpSecure = false) - { - public Expected Build() - { - return new Expected( - ParseEndPoint(TcpEndPoint), - IsTcpSecure, - ParseEndPoint(HttpEndPoint ?? DefaultHttpEndPoint)); - } - } - - private static EndPoint ParseEndPoint(string s) - { - if (s == null) - { - return null; - } - - if (IPEndPoint.TryParse(s, out var ipEndPoint)) - { - return ipEndPoint; - } - - return ParseDnsEndPoint(s); - } - - private static DnsEndPoint ParseDnsEndPoint(string s) - { - if (s == null) - { - return null; - } - - var parts = s.Split(":"); - var host = parts[0]; - var port = int.Parse(parts[1]); - - return new DnsEndPoint(host, port); - } + public static IEnumerable Cases() + { + yield return Case("leader endpoint", Leader("1.1.1.1", 2113), Gossip("2.2.2.2", 2113), "1.1.1.1", 2113); + yield return Case("leader advertised host", Leader("1.1.1.1", 2113, "leader.example"), + Gossip("2.2.2.2", 2113), "leader.example", 2113); + yield return Case("leader advertised port", Leader("1.1.1.1", 2113, advertisePort: 3113), + Gossip("2.2.2.2", 2113), "1.1.1.1", 3113); + yield return Case("leader advertised endpoint", Leader("1.1.1.1", 2113, "leader.example", 3113), + Gossip("2.2.2.2", 2113), "leader.example", 3113); + yield return Case("local gossip endpoint", null, Gossip("2.2.2.2", 2113), "2.2.2.2", 2113); + yield return Case("local advertised endpoint", null, Gossip("2.2.2.2", 2113, "node.example", 3113), + "node.example", 3113); + } + + [TestCaseSource(nameof(Cases))] + public void returns_the_advertised_http_endpoint(MemberInfo leader, GossipAdvertiseInfo gossip, EndPoint expected) + { + var result = new LeaderInfoProvider(gossip, leader).GetLeaderInfoEndPoint(); + + Assert.AreEqual(expected, result); + } + + private static TestCaseData Case(string name, MemberInfo leader, GossipAdvertiseInfo gossip, + string expectedHost, int expectedPort) => + new TestCaseData(leader, gossip, new DnsEndPoint(expectedHost, expectedPort)).SetName(name); + + private static MemberInfo Leader(string host, int port, string advertiseHost = null, int advertisePort = 0) => + MemberInfo.Initial( + Guid.NewGuid(), + DateTime.UtcNow, + VNodeState.Leader, + true, + new DnsEndPoint(host, port), + advertiseHost, + advertisePort, + 0, + false); + + private static GossipAdvertiseInfo Gossip(string host, int port, string advertiseHost = null, + int advertisePort = 0) => + new(new DnsEndPoint(host, port), advertiseHost, advertisePort); } diff --git a/src/EventStore.Core.Tests/TcpApiTestPlugin/PublicTcpApiTestService.cs b/src/EventStore.Core.Tests/TcpApiTestPlugin/PublicTcpApiTestService.cs deleted file mode 100644 index 2cc5eb14dd..0000000000 --- a/src/EventStore.Core.Tests/TcpApiTestPlugin/PublicTcpApiTestService.cs +++ /dev/null @@ -1,123 +0,0 @@ -#nullable enable - -using System; -using System.Net; -using System.Security.Cryptography.X509Certificates; -using System.Threading; -using System.Threading.Tasks; -using EventStore.Core; -using EventStore.Core.Bus; -using EventStore.Core.Certificates; -using EventStore.Core.Messages; -using EventStore.Core.Services; -using EventStore.Core.Services.Transport.Tcp; -using EventStore.Plugins.Authentication; -using Microsoft.Extensions.Hosting; -using Serilog; - -namespace EventStore.TcpUnitTestPlugin; - -public class PublicTcpApiTestService : IHostedService -{ - static readonly ILogger Logger = Log.ForContext(); - private readonly TcpService _tcpService; - private int _systemInitialized; - - PublicTcpApiTestService(TcpService tcpService, ISubscriber bus) - { - _tcpService = tcpService; - - bus.Subscribe(new AdHocHandler(_ => StartTcpService())); - bus.Subscribe(new AdHocHandler(_ => StartTcpService())); - bus.Subscribe(tcpService); - - _ = Task.Run(async () => - { - await Task.Delay(TimeSpan.FromHours(1)); - Logger.Warning("Shutting down TCP unit tests"); - tcpService.Handle(new SystemMessage.BecomeShuttingDown(Guid.NewGuid(), true, true)); - }); - } - - public static PublicTcpApiTestService Insecure( - TcpApiTestOptions options, - IAuthenticationProvider authProvider, - AuthorizationGateway authGateway, - StandardComponents components - ) - { - var endpoint = new IPEndPoint(IPAddress.Loopback, options.NodeTcpPort); - - var tcpService = new TcpService( - publisher: components.MainQueue, - serverEndPoint: endpoint, - networkSendQueue: components.NetworkSendService, - serviceType: TcpServiceType.External, securityType: TcpSecurityType.Normal, - dispatcher: new ClientTcpDispatcher(options.WriteTimeoutMs), - heartbeatInterval: TimeSpan.FromMilliseconds(options.NodeHeartbeatInterval), - heartbeatTimeout: TimeSpan.FromMilliseconds(options.NodeHeartbeatTimeout), - authProvider: authProvider, - authorizationGateway: authGateway, - certificateSelector: null, - intermediatesSelector: null, - sslClientCertValidator: null, - connectionPendingSendBytesThreshold: options.ConnectionPendingSendBytesThreshold, - connectionQueueSizeThreshold: options.ConnectionQueueSizeThreshold - ); - - return new(tcpService, components.MainBus); - } - - public static PublicTcpApiTestService Secure( - TcpApiTestOptions options, - IAuthenticationProvider authProvider, - AuthorizationGateway authGateway, - StandardComponents components, - CertificateProvider? certificateProvider - ) - { - var endpoint = new IPEndPoint(IPAddress.Loopback, options.NodeTcpPort); - - var tcpService = new TcpService( - publisher: components.MainQueue, - serverEndPoint: endpoint, - networkSendQueue: components.NetworkSendService, - serviceType: TcpServiceType.External, securityType: TcpSecurityType.Secure, - dispatcher: new ClientTcpDispatcher(options.WriteTimeoutMs), - heartbeatInterval: TimeSpan.FromMilliseconds(options.NodeHeartbeatInterval), - heartbeatTimeout: TimeSpan.FromMilliseconds(options.NodeHeartbeatTimeout), - authProvider: authProvider, - authorizationGateway: authGateway, - certificateSelector: () => certificateProvider?.Certificate, - intermediatesSelector: () => - { - var intermediates = certificateProvider?.IntermediateCerts; - return intermediates == null ? null : new X509Certificate2Collection(intermediates); - }, - sslClientCertValidator: delegate - { return (true, null); }, - connectionPendingSendBytesThreshold: options.ConnectionPendingSendBytesThreshold, - connectionQueueSizeThreshold: options.ConnectionQueueSizeThreshold - ); - - return new(tcpService, components.MainBus); - } - - public Task StartAsync(CancellationToken cancellationToken) - { - StartTcpService(); - return Task.CompletedTask; - } - - public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - private void StartTcpService() - { - if (Interlocked.Exchange(ref _systemInitialized, 1) == 1) - { - return; - } - - _tcpService.Handle(new SystemMessage.SystemInit()); - } -} diff --git a/src/EventStore.Core.Tests/TcpApiTestPlugin/TcpApiTestOptions.cs b/src/EventStore.Core.Tests/TcpApiTestPlugin/TcpApiTestOptions.cs deleted file mode 100644 index a2be50f225..0000000000 --- a/src/EventStore.Core.Tests/TcpApiTestPlugin/TcpApiTestOptions.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace EventStore.TcpUnitTestPlugin; - -public class TcpApiTestOptions -{ - public int NodeTcpPort { get; init; } = 1113; - public int NodeHeartbeatInterval { get; init; } = 2_000; - public int NodeHeartbeatTimeout { get; init; } = 1_000; - public int ConnectionPendingSendBytesThreshold { get; set; } = 10 * 1_024 * 1_024; - public int ConnectionQueueSizeThreshold { get; set; } = 50_000; - public int WriteTimeoutMs { get; set; } = 2_000; - public bool Insecure { get; init; } = false; -} diff --git a/src/EventStore.Core.Tests/TcpApiTestPlugin/TcpApiTestPlugin.cs b/src/EventStore.Core.Tests/TcpApiTestPlugin/TcpApiTestPlugin.cs deleted file mode 100644 index 543fe02ca7..0000000000 --- a/src/EventStore.Core.Tests/TcpApiTestPlugin/TcpApiTestPlugin.cs +++ /dev/null @@ -1,38 +0,0 @@ -using System.Threading.Tasks; -using EventStore.Core; -using EventStore.Core.Certificates; -using EventStore.Core.Services; -using EventStore.Plugins; -using EventStore.Plugins.Authentication; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Serilog; - -namespace EventStore.TcpUnitTestPlugin; - -public class TcpApiTestPlugin() : SubsystemsPlugin(name: "TcpTestApi") -{ - static readonly ILogger Logger = Log.ForContext(); - - public override void ConfigureServices(IServiceCollection services, IConfiguration configuration) - { - var options = configuration.GetSection("EventStore:TcpUnitTestPlugin").Get() ?? new(); - - services.AddHostedService(serviceProvider => - { - var components = serviceProvider.GetRequiredService(); - var authGateway = serviceProvider.GetRequiredService(); - var authProvider = serviceProvider.GetRequiredService(); - - return options.Insecure - ? PublicTcpApiTestService.Insecure(options, authProvider, authGateway, components) - : PublicTcpApiTestService.Secure(options, authProvider, authGateway, components, serviceProvider.GetService()); - }); - } - - public override Task Start() - { - Logger.Debug("{Name}-{Version} test plugin is loaded", Name, Version); - return Task.CompletedTask; - } -} diff --git a/src/EventStore.Core.Tests/TransactionLog/Truncation/when_truncating_database.cs b/src/EventStore.Core.Tests/TransactionLog/Truncation/when_truncating_database.cs index a21d8f3546..7cca661583 100644 --- a/src/EventStore.Core.Tests/TransactionLog/Truncation/when_truncating_database.cs +++ b/src/EventStore.Core.Tests/TransactionLog/Truncation/when_truncating_database.cs @@ -20,8 +20,7 @@ public async Task everything_should_go_fine() var miniNode = new MiniNode(PathName); await miniNode.Start(); - var tcpPort = miniNode.TcpEndPoint.Port; - var httpPort = miniNode.HttpEndPoint.Port; + var nodePort = miniNode.HttpEndPoint.Port; const int cnt = 50; var countdown = new CountdownEvent(cnt); @@ -43,12 +42,12 @@ public async Task everything_should_go_fine() await miniNode.Shutdown(keepDb: true); // --- first restart and truncation - miniNode = new MiniNode(PathName, tcpPort, httpPort); + miniNode = new MiniNode(PathName, httpPort: nodePort); await miniNode.Start(); await miniNode.Shutdown(keepDb: true); // --- second restart after truncation - miniNode = new MiniNode(PathName, tcpPort, httpPort); + miniNode = new MiniNode(PathName, httpPort: nodePort); await miniNode.Start(); Assert.AreEqual(-1, miniNode.Db.Config.TruncateCheckpoint.Read()); Assert.That(miniNode.Db.Config.WriterCheckpoint.Read(), Is.GreaterThanOrEqualTo(truncatePosition)); @@ -61,7 +60,7 @@ public async Task everything_should_go_fine() await miniNode.Shutdown(keepDb: true); // -- third restart - miniNode = new MiniNode(PathName, tcpPort, httpPort); + miniNode = new MiniNode(PathName, httpPort: nodePort); Assert.AreEqual(-1, miniNode.Db.Config.TruncateCheckpoint.Read()); await miniNode.Start(); @@ -78,7 +77,7 @@ public async Task with_truncate_position_in_completed_chunk_everything_should_go var miniNode = new MiniNode(PathName, chunkSize: chunkSize, cachedChunkSize: cachedSize); await miniNode.Start(); - var httpPort = miniNode.HttpEndPoint.Port; + var nodePort = miniNode.HttpEndPoint.Port; const int cnt = 1; var countdown = new CountdownEvent(cnt); @@ -99,16 +98,14 @@ public async Task with_truncate_position_in_completed_chunk_everything_should_go await miniNode.Shutdown(keepDb: true); - var tcpPort = miniNode.TcpEndPoint.Port; - // --- first restart and truncation - miniNode = new MiniNode(PathName, tcpPort, httpPort, chunkSize: chunkSize, + miniNode = new MiniNode(PathName, httpPort: nodePort, chunkSize: chunkSize, cachedChunkSize: cachedSize); await miniNode.Start(); await miniNode.Shutdown(keepDb: true); // --- second restart after truncation - miniNode = new MiniNode(PathName, tcpPort, httpPort, chunkSize: chunkSize, + miniNode = new MiniNode(PathName, httpPort: nodePort, chunkSize: chunkSize, cachedChunkSize: cachedSize); await miniNode.Start(); Assert.AreEqual(-1, miniNode.Db.Config.TruncateCheckpoint.Read()); diff --git a/src/EventStore.Core.XUnit.Tests/Configuration/ClusterNodeOptionsTests/when_building/with_default_settings.cs b/src/EventStore.Core.XUnit.Tests/Configuration/ClusterNodeOptionsTests/when_building/with_default_settings.cs index 3ec6433bfd..179fd7e1d8 100644 --- a/src/EventStore.Core.XUnit.Tests/Configuration/ClusterNodeOptionsTests/when_building/with_default_settings.cs +++ b/src/EventStore.Core.XUnit.Tests/Configuration/ClusterNodeOptionsTests/when_building/with_default_settings.cs @@ -35,7 +35,6 @@ public void should_create_single_cluster_node() [Test] public void should_have_default_endpoints() { - Assert.AreEqual(new IPEndPoint(IPAddress.Loopback, 1112), _node.NodeInfo.InternalSecureTcp); Assert.AreEqual(new IPEndPoint(IPAddress.Loopback, 2113), _node.NodeInfo.HttpEndPoint); } @@ -69,11 +68,6 @@ public void should_set_command_line_args_to_default_values() Assert.AreEqual(Opts.MaxProjectionStateSizeDefault, _options.Projection.MaxProjectionStateSize, "MaxProjectionStateSize"); - Assert.AreEqual(700, _options.Interface.ReplicationHeartbeatInterval, "ReplicationHeartbeatInterval"); - - Assert.AreEqual(700, _options.Interface.ReplicationHeartbeatTimeout, - "ReplicationHeartbeatTimeout"); - Assert.AreEqual(TFConsts.ChunkSize, _node.Db.Config.ChunkSize, "ChunkSize"); Assert.AreEqual(TFConsts.ChunksCacheSize, _node.Db.Config.MaxChunksCacheSize, "MaxChunksCacheSize"); } @@ -92,15 +86,14 @@ public void should_create_single_cluster_node() } [Test] - public void should_have_default_secure_endpoints() + public void should_have_default_endpoint() { - var internalTcp = new IPEndPoint(IPAddress.Loopback, 1112); + var replicationEndPoint = new IPEndPoint(IPAddress.Loopback, 1112); var httpEndPoint = new IPEndPoint(IPAddress.Loopback, 2113); - Assert.AreEqual(internalTcp, _node.NodeInfo.InternalSecureTcp); + Assert.AreEqual(replicationEndPoint, _node.NodeInfo.ReplicationEndPoint); Assert.AreEqual(httpEndPoint, _node.NodeInfo.HttpEndPoint); - - Assert.AreEqual(internalTcp.ToDnsEndPoint(), _node.GossipAdvertiseInfo.InternalSecureTcp); + Assert.AreEqual(replicationEndPoint.ToDnsEndPoint(), _node.GossipAdvertiseInfo.ReplicationEndPoint); Assert.AreEqual(httpEndPoint.ToDnsEndPoint(), _node.GossipAdvertiseInfo.HttpEndPoint); } @@ -125,13 +118,12 @@ public void should_create_single_cluster_node() [Test] public void should_have_default_endpoints() { - var internalTcp = new IPEndPoint(IPAddress.Loopback, 1112); + var replicationEndPoint = new IPEndPoint(IPAddress.Loopback, 1112); var httpEndPoint = new IPEndPoint(IPAddress.Loopback, 2113); - Assert.AreEqual(internalTcp, _node.NodeInfo.InternalTcp); + Assert.AreEqual(replicationEndPoint, _node.NodeInfo.ReplicationEndPoint); Assert.AreEqual(httpEndPoint, _node.NodeInfo.HttpEndPoint); - - Assert.AreEqual(internalTcp.ToDnsEndPoint(), _node.GossipAdvertiseInfo.InternalTcp); + Assert.AreEqual(replicationEndPoint.ToDnsEndPoint(), _node.GossipAdvertiseInfo.ReplicationEndPoint); Assert.AreEqual(httpEndPoint.ToDnsEndPoint(), _node.GossipAdvertiseInfo.HttpEndPoint); } diff --git a/src/EventStore.Core.XUnit.Tests/Configuration/ClusterVNodeOptionsValidatorTests.cs b/src/EventStore.Core.XUnit.Tests/Configuration/ClusterVNodeOptionsValidatorTests.cs index 37a3703c9c..0278976316 100644 --- a/src/EventStore.Core.XUnit.Tests/Configuration/ClusterVNodeOptionsValidatorTests.cs +++ b/src/EventStore.Core.XUnit.Tests/Configuration/ClusterVNodeOptionsValidatorTests.cs @@ -47,32 +47,6 @@ public void node_and_replication_listeners_can_use_the_same_port_on_distinct_add ClusterVNodeOptionsValidator.Validate(options); } - [Theory] - [InlineData(0)] - [InlineData(-1)] - public void replication_heartbeat_interval_must_be_positive(int interval) - { - var options = new ClusterVNodeOptions - { - Interface = new() { ReplicationHeartbeatInterval = interval } - }; - - Assert.Throws(() => ClusterVNodeOptionsValidator.Validate(options)); - } - - [Theory] - [InlineData(0)] - [InlineData(-1)] - public void replication_heartbeat_timeout_must_be_positive(int timeout) - { - var options = new ClusterVNodeOptions - { - Interface = new() { ReplicationHeartbeatTimeout = timeout } - }; - - Assert.Throws(() => ClusterVNodeOptionsValidator.Validate(options)); - } - [Theory] [InlineData(false, false, true)] [InlineData(false, true, true)] diff --git a/src/EventStore.Core.XUnit.Tests/Metrics/ElectionsCounterTrackerTests.cs b/src/EventStore.Core.XUnit.Tests/Metrics/ElectionsCounterTrackerTests.cs index 4b2e6ba989..8e47744ed8 100644 --- a/src/EventStore.Core.XUnit.Tests/Metrics/ElectionsCounterTrackerTests.cs +++ b/src/EventStore.Core.XUnit.Tests/Metrics/ElectionsCounterTrackerTests.cs @@ -25,8 +25,7 @@ public ElectionsCounterTrackerTests() var endPoint = new DnsEndPoint("127.0.0.1", 1113); var memberInfo = Cluster.MemberInfo.Initial(Guid.Empty, DateTime.UtcNow, VNodeState.Unknown, true, - endPoint, endPoint, endPoint, endPoint, endPoint, - null, 0, 0, 0, false); + endPoint, null, 0, 0, false); _electionsDoneMessage = new ElectionMessage.ElectionsDone(1, 1, memberInfo); } diff --git a/src/EventStore.Core.XUnit.Tests/Services/Storage/InMemory/GossipListenerServiceTests.cs b/src/EventStore.Core.XUnit.Tests/Services/Storage/InMemory/GossipListenerServiceTests.cs index 84e972bfba..888b7ddda2 100644 --- a/src/EventStore.Core.XUnit.Tests/Services/Storage/InMemory/GossipListenerServiceTests.cs +++ b/src/EventStore.Core.XUnit.Tests/Services/Storage/InMemory/GossipListenerServiceTests.cs @@ -47,14 +47,9 @@ public async Task notify_state_change() timeStamp: DateTime.Now, state: Data.VNodeState.DiscoverLeader, isAlive: true, - internalTcpEndPoint: default, - internalSecureTcpEndPoint: new DnsEndPoint("myhost", random()), - externalTcpEndPoint: default, - externalSecureTcpEndPoint: new DnsEndPoint("myhost", random()), httpEndPoint: new DnsEndPoint("myhost", random()), advertiseHostToClientAs: "advertiseHostToClientAs", advertiseHttpPortToClientAs: random(), - advertiseTcpPortToClientAs: random(), lastCommitPosition: random(), writerCheckpoint: random(), chaserCheckpoint: random(), diff --git a/src/EventStore.Core.XUnit.Tests/Telemetry/TelemetryServiceTests.cs b/src/EventStore.Core.XUnit.Tests/Telemetry/TelemetryServiceTests.cs index 1193feeb54..ccaa036e64 100644 --- a/src/EventStore.Core.XUnit.Tests/Telemetry/TelemetryServiceTests.cs +++ b/src/EventStore.Core.XUnit.Tests/Telemetry/TelemetryServiceTests.cs @@ -77,14 +77,9 @@ private static MemberInfo CreateMemberInfo(Guid instanceId, VNodeState state, bo timeStamp: DateTime.Now, state: state, isAlive: true, - internalTcpEndPoint: default, - internalSecureTcpEndPoint: new DnsEndPoint("myhost", random()), - externalTcpEndPoint: default, - externalSecureTcpEndPoint: new DnsEndPoint("myhost", random()), httpEndPoint: new DnsEndPoint("myhost", random()), advertiseHostToClientAs: "advertiseHostToClientAs", advertiseHttpPortToClientAs: random(), - advertiseTcpPortToClientAs: random(), lastCommitPosition: random(), writerCheckpoint: random(), chaserCheckpoint: random(), diff --git a/src/EventStore.Core/Authorization/AuthorizationPolicies/LegacyPolicySelectorFactory.cs b/src/EventStore.Core/Authorization/AuthorizationPolicies/LegacyPolicySelectorFactory.cs index 951bed56fa..70c44939ae 100644 --- a/src/EventStore.Core/Authorization/AuthorizationPolicies/LegacyPolicySelectorFactory.cs +++ b/src/EventStore.Core/Authorization/AuthorizationPolicies/LegacyPolicySelectorFactory.cs @@ -62,7 +62,6 @@ public IPolicySelector Create(IPublisher publisher) addToPolicy(Operations.Node.Options); addToPolicy(Operations.Node.Statistics.Read); addToPolicy(Operations.Node.Statistics.Replication); - addToPolicy(Operations.Node.Statistics.Tcp); addToPolicy(Operations.Node.Statistics.Custom); policy.AddMatchAnyAssertion(Operations.Node.Information.Subsystems, Grant.Allow, OperationsOrAdmins); diff --git a/src/EventStore.Core/Cluster/ClientClusterInfo.cs b/src/EventStore.Core/Cluster/ClientClusterInfo.cs index b1e516d8f8..d490a77a8c 100644 --- a/src/EventStore.Core/Cluster/ClientClusterInfo.cs +++ b/src/EventStore.Core/Cluster/ClientClusterInfo.cs @@ -37,14 +37,6 @@ public class ClientMemberInfo public VNodeState State { get; set; } public bool IsAlive { get; set; } - public string InternalTcpIp { get; set; } - public int InternalTcpPort { get; set; } - public int InternalSecureTcpPort { get; set; } - - public string ExternalTcpIp { get; set; } - public int ExternalTcpPort { get; set; } - public int ExternalSecureTcpPort { get; set; } - public string InternalHttpEndPointIp { get; set; } public int InternalHttpEndPointPort { get; set; } public string ReplicationEndPointIp { get; set; } @@ -78,12 +70,6 @@ public ClientMemberInfo(MemberInfo member) State = member.State; IsAlive = member.IsAlive; - InternalTcpIp = member.InternalTcpEndPoint is null - ? member.InternalSecureTcpEndPoint.GetHost() - : member.InternalTcpEndPoint.GetHost(); - InternalSecureTcpPort = member.InternalSecureTcpEndPoint?.GetPort() ?? 0; - InternalTcpPort = member.InternalTcpEndPoint?.GetPort() ?? 0; - InternalHttpEndPointIp = member.HttpEndPoint.GetHost(); InternalHttpEndPointPort = member.HttpEndPoint.GetPort(); ReplicationEndPointIp = member.ReplicationEndPoint.GetHost(); @@ -96,18 +82,6 @@ public ClientMemberInfo(MemberInfo member) ? member.HttpEndPoint.GetPort() : member.AdvertiseHttpPortToClientAs; - ExternalTcpIp = string.IsNullOrEmpty(member.AdvertiseHostToClientAs) - ? member.ExternalSecureTcpEndPoint?.GetHost() ?? - member.ExternalTcpEndPoint?.GetHost() ?? member.HttpEndPoint.GetHost() - : member.AdvertiseHostToClientAs; - - ExternalTcpPort = member.AdvertiseTcpPortToClientAs == 0 - ? member.ExternalTcpEndPoint?.GetPort() ?? 0 - : member.AdvertiseTcpPortToClientAs; - ExternalSecureTcpPort = member.AdvertiseTcpPortToClientAs == 0 - ? member.ExternalSecureTcpEndPoint?.GetPort() ?? 0 - : member.AdvertiseTcpPortToClientAs; - LastCommitPosition = member.LastCommitPosition; WriterCheckpoint = member.WriterCheckpoint; ChaserCheckpoint = member.ChaserCheckpoint; @@ -126,8 +100,6 @@ public override string ToString() { return $"InstanceId: {InstanceId:B}, TimeStamp: {TimeStamp:yyyy-MM-dd HH:mm:ss.fff}, State: {State}, IsAlive: {IsAlive}, " + - $"InternalTcpIp: {InternalTcpIp}, InternalTcpPort: {InternalTcpPort}, InternalSecureTcpPort: {InternalSecureTcpPort}, " + - $"ExternalTcpIp: {ExternalTcpIp}, ExternalTcpPort: {ExternalTcpPort}, ExternalSecureTcpPort: {ExternalSecureTcpPort}, " + $"InternalHttpEndPointIp: {InternalHttpEndPointIp}, InternalHttpEndPointPort: {InternalHttpEndPointPort}, " + $"ReplicationEndPointIp: {ReplicationEndPointIp}, ReplicationEndPointPort: {ReplicationEndPointPort}, " + $"HttpEndPointIp: {HttpEndPointIp}, HttpEndPointPort: {HttpEndPointPort}, " + diff --git a/src/EventStore.Core/Cluster/ClusterInfo.cs b/src/EventStore.Core/Cluster/ClusterInfo.cs index 6276119ed3..a9b9cd9119 100644 --- a/src/EventStore.Core/Cluster/ClusterInfo.cs +++ b/src/EventStore.Core/Cluster/ClusterInfo.cs @@ -5,7 +5,6 @@ using EventStore.Client; using EventStore.Common.Utils; using EventStore.Core.Data; -using EventStore.Core.Messages; using EventStore.Core.Services.Transport.Grpc; namespace EventStore.Core.Cluster @@ -26,12 +25,6 @@ public ClusterInfo(IEnumerable members) .ToArray(); } - public ClusterInfo(ClusterInfoDto dto) - { - Members = dto.Members.Safe().Select(x => new MemberInfo(x)) - .OrderByDescending(x => x.HttpEndPoint, Comparer).ToArray(); - } - public override string ToString() { return string.Join("\n", Members.Select(s => s.ToString())); @@ -71,16 +64,8 @@ internal static ClusterInfo FromGrpcClusterInfo(EventStore.Cluster.ClusterInfo g new MemberInfo( Uuid.FromDto(x.InstanceId).ToGuid(), x.TimeStamp.FromTicksSinceEpoch(), (VNodeState)x.State, x.IsAlive, - !x.InternalTcpUsesTls ? new DnsEndPoint(x.InternalTcp.Address, (int)x.InternalTcp.Port).WithClusterDns(clusterDns) : null, - x.InternalTcpUsesTls ? new DnsEndPoint(x.InternalTcp.Address, (int)x.InternalTcp.Port).WithClusterDns(clusterDns) : null, - !x.ExternalTcpUsesTls && x.ExternalTcp != null - ? new DnsEndPoint(x.ExternalTcp.Address, (int)x.ExternalTcp.Port).WithClusterDns(clusterDns) - : null, - x.ExternalTcpUsesTls && x.ExternalTcp != null - ? new DnsEndPoint(x.ExternalTcp.Address, (int)x.ExternalTcp.Port).WithClusterDns(clusterDns) - : null, new DnsEndPoint(x.HttpEndPoint.Address, (int)x.HttpEndPoint.Port).WithClusterDns(clusterDns), - x.AdvertiseHostToClientAs, (int)x.AdvertiseHttpPortToClientAs, (int)x.AdvertiseTcpPortToClientAs, + x.AdvertiseHostToClientAs, (int)x.AdvertiseHttpPortToClientAs, x.LastCommitPosition, x.WriterCheckpoint, x.ChaserCheckpoint, x.EpochPosition, x.EpochNumber, Uuid.FromDto(x.EpochId).ToGuid(), x.NodePriority, x.IsReadOnlyReplica, x.EsVersion == String.Empty ? null : x.EsVersion, @@ -106,23 +91,6 @@ internal static EventStore.Cluster.ClusterInfo ToGrpcClusterInfo(ClusterInfo clu ReplicationEndPoint = new EventStore.Cluster.EndPoint( x.ReplicationEndPoint.GetHost(), (uint)x.ReplicationEndPoint.GetPort()), - InternalTcp = x.InternalSecureTcpEndPoint != null ? - new EventStore.Cluster.EndPoint( - x.InternalSecureTcpEndPoint.GetHost(), - (uint)x.InternalSecureTcpEndPoint.GetPort()) : - new EventStore.Cluster.EndPoint( - x.InternalTcpEndPoint.GetHost(), - (uint)x.InternalTcpEndPoint.GetPort()), - InternalTcpUsesTls = x.InternalSecureTcpEndPoint != null, - ExternalTcp = x.ExternalSecureTcpEndPoint != null ? - new EventStore.Cluster.EndPoint( - x.ExternalSecureTcpEndPoint.GetHost(), - (uint)x.ExternalSecureTcpEndPoint.GetPort()) : - x.ExternalTcpEndPoint != null ? - new EventStore.Cluster.EndPoint( - x.ExternalTcpEndPoint.GetHost(), - (uint)x.ExternalTcpEndPoint.GetPort()) : null, - ExternalTcpUsesTls = x.ExternalSecureTcpEndPoint != null, LastCommitPosition = x.LastCommitPosition, WriterCheckpoint = x.WriterCheckpoint, ChaserCheckpoint = x.ChaserCheckpoint, @@ -133,7 +101,6 @@ internal static EventStore.Cluster.ClusterInfo ToGrpcClusterInfo(ClusterInfo clu IsReadOnlyReplica = x.IsReadOnlyReplica, AdvertiseHostToClientAs = x.AdvertiseHostToClientAs ?? "", AdvertiseHttpPortToClientAs = (uint)x.AdvertiseHttpPortToClientAs, - AdvertiseTcpPortToClientAs = (uint)x.AdvertiseTcpPortToClientAs, EsVersion = x.ESVersion ?? String.Empty }).ToArray(); var info = new EventStore.Cluster.ClusterInfo(); diff --git a/src/EventStore.Core/Cluster/MemberInfo.cs b/src/EventStore.Core/Cluster/MemberInfo.cs index fbcacf901c..4fe81259c2 100644 --- a/src/EventStore.Core/Cluster/MemberInfo.cs +++ b/src/EventStore.Core/Cluster/MemberInfo.cs @@ -2,7 +2,6 @@ using System.Net; using EventStore.Common.Utils; using EventStore.Core.Data; -using EventStore.Core.Messages; using EventStore.Core.TransactionLog.LogRecords; namespace EventStore.Core.Cluster @@ -15,15 +14,10 @@ public class MemberInfo : IEquatable public readonly VNodeState State; public readonly bool IsAlive; - public readonly EndPoint InternalTcpEndPoint; - public readonly EndPoint InternalSecureTcpEndPoint; - public readonly EndPoint ExternalTcpEndPoint; - public readonly EndPoint ExternalSecureTcpEndPoint; - public readonly EndPoint HttpEndPoint; public readonly EndPoint ReplicationEndPoint; + public readonly EndPoint HttpEndPoint; public readonly string AdvertiseHostToClientAs; public readonly int AdvertiseHttpPortToClientAs; - public readonly int AdvertiseTcpPortToClientAs; public readonly long LastCommitPosition; public readonly long WriterCheckpoint; @@ -42,8 +36,7 @@ public static MemberInfo ForManager(Guid instanceId, DateTime timeStamp, bool is EndPoint replicationEndPoint = null) { return new MemberInfo(instanceId, timeStamp, VNodeState.Manager, isAlive, - httpEndPoint, null, httpEndPoint, null, - httpEndPoint, null, 0, 0, + httpEndPoint, null, 0, -1, -1, -1, -1, -1, Guid.Empty, 0, false, esVersion, replicationEndPoint); } @@ -51,14 +44,9 @@ public static MemberInfo ForVNode(Guid instanceId, DateTime timeStamp, VNodeState state, bool isAlive, - EndPoint internalTcpEndPoint, - EndPoint internalSecureTcpEndPoint, - EndPoint externalTcpEndPoint, - EndPoint externalSecureTcpEndPoint, EndPoint httpEndPoint, string advertiseHostToClientAs, int advertiseHttpPortToClientAs, - int advertiseTcpPortToClientAs, long lastCommitPosition, long writerCheckpoint, long chaserCheckpoint, @@ -75,9 +63,7 @@ public static MemberInfo ForVNode(Guid instanceId, } return new MemberInfo(instanceId, timeStamp, state, isAlive, - internalTcpEndPoint, internalSecureTcpEndPoint, - externalTcpEndPoint, externalSecureTcpEndPoint, - httpEndPoint, advertiseHostToClientAs, advertiseHttpPortToClientAs, advertiseTcpPortToClientAs, + httpEndPoint, advertiseHostToClientAs, advertiseHttpPortToClientAs, lastCommitPosition, writerCheckpoint, chaserCheckpoint, epochPosition, epochNumber, epochId, nodePriority, isReadOnlyReplica, esVersion, replicationEndPoint); @@ -87,14 +73,9 @@ public static MemberInfo Initial(Guid instanceId, DateTime timeStamp, VNodeState state, bool isAlive, - EndPoint internalTcpEndPoint, - EndPoint internalSecureTcpEndPoint, - EndPoint externalTcpEndPoint, - EndPoint externalSecureTcpEndPoint, EndPoint httpEndPoint, string advertiseHostToClientAs, int advertiseHttpPortToClientAs, - int advertiseTcpPortToClientAs, int nodePriority, bool isReadOnlyReplica, string esVersion = VersionInfo.UnknownVersion, EndPoint replicationEndPoint = null) @@ -105,22 +86,17 @@ public static MemberInfo Initial(Guid instanceId, } return new MemberInfo(instanceId, timeStamp, state, isAlive, - internalTcpEndPoint, internalSecureTcpEndPoint, - externalTcpEndPoint, externalSecureTcpEndPoint, - httpEndPoint, advertiseHostToClientAs, advertiseHttpPortToClientAs, advertiseTcpPortToClientAs, + httpEndPoint, advertiseHostToClientAs, advertiseHttpPortToClientAs, -1, -1, -1, -1, -1, Guid.Empty, nodePriority, isReadOnlyReplica, esVersion, replicationEndPoint); } internal MemberInfo(Guid instanceId, DateTime timeStamp, VNodeState state, bool isAlive, - EndPoint internalTcpEndPoint, EndPoint internalSecureTcpEndPoint, - EndPoint externalTcpEndPoint, EndPoint externalSecureTcpEndPoint, - EndPoint httpEndPoint, string advertiseHostToClientAs, int advertiseHttpPortToClientAs, int advertiseTcpPortToClientAs, + EndPoint httpEndPoint, string advertiseHostToClientAs, int advertiseHttpPortToClientAs, long lastCommitPosition, long writerCheckpoint, long chaserCheckpoint, long epochPosition, int epochNumber, Guid epochId, int nodePriority, bool isReadOnlyReplica, string esVersion = null, EndPoint replicationEndPoint = null) { - Ensure.Equal(false, internalTcpEndPoint == null && internalSecureTcpEndPoint == null, "Both internal TCP endpoints are null"); Ensure.NotNull(httpEndPoint, nameof(httpEndPoint)); InstanceId = instanceId; @@ -129,15 +105,10 @@ internal MemberInfo(Guid instanceId, DateTime timeStamp, VNodeState state, bool State = state; IsAlive = isAlive; - InternalTcpEndPoint = internalTcpEndPoint; - InternalSecureTcpEndPoint = internalSecureTcpEndPoint; - ExternalTcpEndPoint = externalTcpEndPoint; - ExternalSecureTcpEndPoint = externalSecureTcpEndPoint; - HttpEndPoint = httpEndPoint; ReplicationEndPoint = replicationEndPoint ?? httpEndPoint; + HttpEndPoint = httpEndPoint; AdvertiseHostToClientAs = advertiseHostToClientAs; AdvertiseHttpPortToClientAs = advertiseHttpPortToClientAs; - AdvertiseTcpPortToClientAs = advertiseTcpPortToClientAs; LastCommitPosition = lastCommitPosition; WriterCheckpoint = writerCheckpoint; @@ -153,44 +124,10 @@ internal MemberInfo(Guid instanceId, DateTime timeStamp, VNodeState state, bool ESVersion = esVersion; } - internal MemberInfo(MemberInfoDto dto) - { - InstanceId = dto.InstanceId; - TimeStamp = dto.TimeStamp; - State = dto.State; - IsAlive = dto.IsAlive; - InternalTcpEndPoint = new DnsEndPoint(dto.InternalTcpIp, dto.InternalTcpPort); - InternalSecureTcpEndPoint = dto.InternalSecureTcpPort > 0 - ? new DnsEndPoint(dto.InternalTcpIp, dto.InternalSecureTcpPort) - : null; - ExternalTcpEndPoint = dto.ExternalTcpIp != null ? new DnsEndPoint(dto.ExternalTcpIp, dto.ExternalTcpPort) : null; - ExternalSecureTcpEndPoint = dto.ExternalTcpIp != null && dto.ExternalSecureTcpPort > 0 - ? new DnsEndPoint(dto.ExternalTcpIp, dto.ExternalSecureTcpPort) - : null; - HttpEndPoint = new DnsEndPoint(dto.HttpEndPointIp, dto.HttpEndPointPort); - ReplicationEndPoint = HttpEndPoint; - AdvertiseHostToClientAs = dto.AdvertiseHostToClientAs; - AdvertiseHttpPortToClientAs = dto.AdvertiseHttpPortToClientAs; - AdvertiseTcpPortToClientAs = dto.AdvertiseTcpPortToClientAs; - LastCommitPosition = dto.LastCommitPosition; - WriterCheckpoint = dto.WriterCheckpoint; - ChaserCheckpoint = dto.ChaserCheckpoint; - EpochPosition = dto.EpochPosition; - EpochNumber = dto.EpochNumber; - EpochId = dto.EpochId; - NodePriority = dto.NodePriority; - IsReadOnlyReplica = dto.IsReadOnlyReplica; - } - public bool Is(EndPoint endPoint) { - return endPoint != null - && (HttpEndPoint.EndPointEquals(endPoint) - || ReplicationEndPoint.EndPointEquals(endPoint) - || (InternalTcpEndPoint != null && InternalTcpEndPoint.EndPointEquals(endPoint)) - || (InternalSecureTcpEndPoint != null && InternalSecureTcpEndPoint.EndPointEquals(endPoint)) - || (ExternalTcpEndPoint != null && ExternalTcpEndPoint.EndPointEquals(endPoint)) - || (ExternalSecureTcpEndPoint != null && ExternalSecureTcpEndPoint.EndPointEquals(endPoint))); + return endPoint != null && + (HttpEndPoint.EndPointEquals(endPoint) || ReplicationEndPoint.EndPointEquals(endPoint)); } public MemberInfo Updated(DateTime utcNow, @@ -206,14 +143,9 @@ public MemberInfo Updated(DateTime utcNow, utcNow, state ?? State, isAlive ?? IsAlive, - InternalTcpEndPoint, - InternalSecureTcpEndPoint, - ExternalTcpEndPoint, - ExternalSecureTcpEndPoint, HttpEndPoint, AdvertiseHostToClientAs, AdvertiseHttpPortToClientAs, - AdvertiseTcpPortToClientAs, lastCommitPosition ?? LastCommitPosition, writerCheckpoint ?? WriterCheckpoint, chaserCheckpoint ?? ChaserCheckpoint, @@ -229,17 +161,12 @@ public override string ToString() if (State == VNodeState.Manager) { return - $"MAN {InstanceId:B} <{(IsAlive ? "LIVE" : "DEAD")}> [{State}, {HttpEndPoint}] | {TimeStamp:yyyy-MM-dd HH:mm:ss.fff}"; + $"MAN {InstanceId:B} <{(IsAlive ? "LIVE" : "DEAD")}> [{State}, {ReplicationEndPoint}, {HttpEndPoint}] | {TimeStamp:yyyy-MM-dd HH:mm:ss.fff}"; } return $"Priority: {NodePriority} VND {InstanceId:B} <{(IsAlive ? "LIVE" : "DEAD")}> [{State}, " + - $"{(InternalTcpEndPoint == null ? "n/a" : InternalTcpEndPoint.ToString())}, " + - $"{(InternalSecureTcpEndPoint == null ? "n/a" : InternalSecureTcpEndPoint.ToString())}, " + - $"{(ExternalTcpEndPoint == null ? "n/a" : ExternalTcpEndPoint.ToString())}, " + - $"{(ExternalSecureTcpEndPoint == null ? "n/a" : ExternalSecureTcpEndPoint.ToString())}, " + - $"Replication:{ReplicationEndPoint}, " + - $"{HttpEndPoint}, (ADVERTISED: HTTP:{AdvertiseHostToClientAs}:{AdvertiseHttpPortToClientAs}, TCP:{AdvertiseHostToClientAs}:{AdvertiseTcpPortToClientAs}), " + + $"Replication:{ReplicationEndPoint}, {HttpEndPoint}, (ADVERTISED: HTTP:{AdvertiseHostToClientAs}:{AdvertiseHttpPortToClientAs}), " + $"Version: {ESVersion}] " + $"{LastCommitPosition}/{WriterCheckpoint}/{ChaserCheckpoint}/E{EpochNumber}@{EpochPosition}:{EpochId:B} | {TimeStamp:yyyy-MM-dd HH:mm:ss.fff}"; } @@ -260,15 +187,10 @@ public bool Equals(MemberInfo other) return other.InstanceId == InstanceId && other.State == State && other.IsAlive == IsAlive - && Equals(other.InternalTcpEndPoint, InternalTcpEndPoint) - && Equals(other.InternalSecureTcpEndPoint, InternalSecureTcpEndPoint) - && Equals(other.ExternalTcpEndPoint, ExternalTcpEndPoint) - && Equals(other.ExternalSecureTcpEndPoint, ExternalSecureTcpEndPoint) - && Equals(other.HttpEndPoint, HttpEndPoint) && Equals(other.ReplicationEndPoint, ReplicationEndPoint) + && Equals(other.HttpEndPoint, HttpEndPoint) && other.AdvertiseHostToClientAs == AdvertiseHostToClientAs && other.AdvertiseHttpPortToClientAs == AdvertiseHttpPortToClientAs - && other.AdvertiseTcpPortToClientAs == AdvertiseTcpPortToClientAs && other.EpochPosition == EpochPosition && other.EpochNumber == EpochNumber && other.EpochId == EpochId @@ -304,17 +226,10 @@ public override int GetHashCode() int result = InstanceId.GetHashCode(); result = (result * 397) ^ State.GetHashCode(); result = (result * 397) ^ IsAlive.GetHashCode(); - result = (result * 397) ^ (InternalTcpEndPoint != null ? InternalTcpEndPoint.GetHashCode() : 0); - result = (result * 397) ^ - (InternalSecureTcpEndPoint != null ? InternalSecureTcpEndPoint.GetHashCode() : 0); - result = (result * 397) ^ (ExternalTcpEndPoint != null ? ExternalTcpEndPoint.GetHashCode() : 0); - result = (result * 397) ^ - (ExternalSecureTcpEndPoint != null ? ExternalSecureTcpEndPoint.GetHashCode() : 0); - result = (result * 397) ^ HttpEndPoint.GetHashCode(); result = (result * 397) ^ ReplicationEndPoint.GetHashCode(); + result = (result * 397) ^ HttpEndPoint.GetHashCode(); result = (result * 397) ^ (AdvertiseHostToClientAs != null ? AdvertiseHostToClientAs.GetHashCode() : 0); result = (result * 397) ^ AdvertiseHttpPortToClientAs.GetHashCode(); - result = (result * 397) ^ AdvertiseTcpPortToClientAs.GetHashCode(); result = (result * 397) ^ EpochPosition.GetHashCode(); result = (result * 397) ^ EpochNumber.GetHashCode(); result = (result * 397) ^ EpochId.GetHashCode(); diff --git a/src/EventStore.Core/ClusterVNode.cs b/src/EventStore.Core/ClusterVNode.cs index ecb61b7f31..4ed416f955 100644 --- a/src/EventStore.Core/ClusterVNode.cs +++ b/src/EventStore.Core/ClusterVNode.cs @@ -55,7 +55,6 @@ using EventStore.Core.Services.Transport.Http; using EventStore.Core.Services.Transport.Http.Authentication; using EventStore.Core.Services.Transport.Http.NodeHttpClientFactory; -using EventStore.Core.Services.Transport.Tcp; using EventStore.Core.Services.VNode; using EventStore.Core.Settings; using EventStore.Core.Synchronization; @@ -291,51 +290,15 @@ public ClusterVNode(ClusterVNodeOptions options, archiveOptions.Validate(); OptionsFormatter.LogConfig("Archive", archiveOptions); - var disableInternalTcpTls = options.Application.TlsDisabled(); - var disableExternalTcpTls = options.Application.TlsDisabled(); - var nodeTcpOptions = configuration.GetSection("EventStore:TcpPlugin").Get() ?? new(); - var enableExternalTcp = nodeTcpOptions.EnableExternalTcp; - var httpEndPoint = new IPEndPoint(options.Interface.NodeIp, options.Interface.NodePort); var replicationEndPoint = new IPEndPoint( options.Interface.ReplicationIp, options.Interface.ReplicationPort); - var intTcp = disableInternalTcpTls - ? new IPEndPoint(options.Interface.ReplicationIp, - options.Interface.ReplicationPort) - : null; - var intSecIp = !disableInternalTcpTls - ? new IPEndPoint(options.Interface.ReplicationIp, - options.Interface.ReplicationPort) - : null; - - var extTcp = disableExternalTcpTls && enableExternalTcp - ? new IPEndPoint(options.Interface.NodeIp, - nodeTcpOptions.NodeTcpPort) - : null; - var extSecIp = !disableExternalTcpTls && enableExternalTcp - ? new IPEndPoint(options.Interface.NodeIp, - nodeTcpOptions.NodeTcpPort) - : null; - - var replicationPortAdvertiseAs = options.Interface.GetReplicationPortAdvertiseAs(); - var intTcpPortAdvertiseAs = disableInternalTcpTls ? replicationPortAdvertiseAs : 0; - var intSecTcpPortAdvertiseAs = !disableInternalTcpTls ? replicationPortAdvertiseAs : 0; - - var extTcpPortAdvertiseAs = - enableExternalTcp && disableExternalTcpTls && nodeTcpOptions.NodeTcpPortAdvertiseAs.HasValue - ? nodeTcpOptions.NodeTcpPortAdvertiseAs.Value! - : 0; - var extSecTcpPortAdvertiseAs = - enableExternalTcp && !disableExternalTcpTls && nodeTcpOptions.NodeTcpPortAdvertiseAs.HasValue - ? nodeTcpOptions.NodeTcpPortAdvertiseAs.Value! - : 0; - Log.Information("Quorum size set to {quorum}.", options.Cluster.QuorumSize); - NodeInfo = new VNodeInfo(instanceId.Value, debugIndex, intTcp, intSecIp, extTcp, extSecIp, - httpEndPoint, options.Cluster.ReadOnlyReplica, replicationEndPoint); + NodeInfo = new VNodeInfo(instanceId.Value, debugIndex, httpEndPoint, options.Cluster.ReadOnlyReplica, + replicationEndPoint); var metricsConfiguration = MetricsConfiguration.Get(configuration); var trackers = new Trackers(); @@ -635,8 +598,6 @@ void StartSubsystems() TimeSpan.FromSeconds(options.Application.StatsPeriodSec), NodeInfo.HttpEndPoint, options.Database.StatsStorage, - NodeInfo.ExternalTcp, - NodeInfo.ExternalSecureTcp, statsHelper); _mainBus.Subscribe(monitoringQueue); @@ -650,7 +611,6 @@ void StartSubsystems() monitoringInnerBus.Subscribe(monitoring); monitoringInnerBus.Subscribe(monitoring); monitoringInnerBus.Subscribe(monitoring); - monitoringInnerBus.Subscribe(monitoring); _threadPoolBacklogMonitor = new ThreadPoolBacklogMonitor(_queueStatsManager, trackers.QueueTrackers); _threadPoolBacklogMonitor.Start(); @@ -918,68 +878,42 @@ void StartSubsystems() GossipAdvertiseInfo GetGossipAdvertiseInfo() { - IPAddress intIpAddress = options.Interface.ReplicationIp; - - IPAddress extIpAddress = options.Interface.NodeIp; - - var intHostToAdvertise = options.Interface.ReplicationHostAdvertiseAs ?? intIpAddress.ToString(); - var extHostToAdvertise = options.Interface.NodeHostAdvertiseAs ?? extIpAddress.ToString(); - - if (intIpAddress.Equals(IPAddress.Any) || extIpAddress.Equals(IPAddress.Any)) + var nodeIpAddress = options.Interface.NodeIp; + var hostToAdvertise = options.Interface.NodeHostAdvertiseAs ?? nodeIpAddress.ToString(); + var replicationIpAddress = options.Interface.ReplicationIp; + var replicationHostToAdvertise = + options.Interface.ReplicationHostAdvertiseAs ?? replicationIpAddress.ToString(); + + if ((nodeIpAddress.Equals(IPAddress.Any) && options.Interface.NodeHostAdvertiseAs == null) || + (replicationIpAddress.Equals(IPAddress.Any) && options.Interface.ReplicationHostAdvertiseAs == null)) { IPAddress nonLoopbackAddress = IPFinder.GetNonLoopbackAddress(); - IPAddress addressToAdvertise = - options.Cluster.ClusterSize > 1 ? nonLoopbackAddress : IPAddress.Loopback; - - if (intIpAddress.Equals(IPAddress.Any) && options.Interface.ReplicationHostAdvertiseAs == null) + var addressToAdvertise = + (options.Cluster.ClusterSize > 1 ? nonLoopbackAddress : IPAddress.Loopback).ToString(); + if (nodeIpAddress.Equals(IPAddress.Any) && options.Interface.NodeHostAdvertiseAs == null) { - intHostToAdvertise = addressToAdvertise.ToString(); + hostToAdvertise = addressToAdvertise; } - if (extIpAddress.Equals(IPAddress.Any) && options.Interface.NodeHostAdvertiseAs == null) + if (replicationIpAddress.Equals(IPAddress.Any) && + options.Interface.ReplicationHostAdvertiseAs == null) { - extHostToAdvertise = addressToAdvertise.ToString(); + replicationHostToAdvertise = addressToAdvertise; } } - var intTcpEndPoint = NodeInfo.InternalTcp == null - ? null - : new DnsEndPoint(intHostToAdvertise, intTcpPortAdvertiseAs > 0 - ? intTcpPortAdvertiseAs - : NodeInfo.InternalTcp.Port); - - var intSecureTcpEndPoint = NodeInfo.InternalSecureTcp == null - ? null - : new DnsEndPoint(intHostToAdvertise, intSecTcpPortAdvertiseAs > 0 - ? intSecTcpPortAdvertiseAs - : NodeInfo.InternalSecureTcp.Port); - - var extTcpEndPoint = NodeInfo.ExternalTcp == null - ? null - : new DnsEndPoint(extHostToAdvertise, extTcpPortAdvertiseAs > 0 - ? extTcpPortAdvertiseAs - : NodeInfo.ExternalTcp.Port); - - var extSecureTcpEndPoint = NodeInfo.ExternalSecureTcp == null - ? null - : new DnsEndPoint(extHostToAdvertise, extSecTcpPortAdvertiseAs > 0 - ? extSecTcpPortAdvertiseAs - : NodeInfo.ExternalSecureTcp.Port); - - var httpEndPoint = new DnsEndPoint(extHostToAdvertise, + var httpEndPoint = new DnsEndPoint(hostToAdvertise, options.Interface.NodePortAdvertiseAs > 0 ? options.Interface.NodePortAdvertiseAs : NodeInfo.HttpEndPoint.GetPort()); - var advertisedReplicationEndPoint = new DnsEndPoint(intHostToAdvertise, + var replicationPortAdvertiseAs = options.Interface.GetReplicationPortAdvertiseAs(); + var advertisedReplicationEndPoint = new DnsEndPoint(replicationHostToAdvertise, replicationPortAdvertiseAs > 0 ? replicationPortAdvertiseAs : NodeInfo.ReplicationEndPoint.GetPort()); - return new GossipAdvertiseInfo(intTcpEndPoint, intSecureTcpEndPoint, extTcpEndPoint, - extSecureTcpEndPoint, httpEndPoint, options.Interface.ReplicationHostAdvertiseAs, - options.Interface.NodeHostAdvertiseAs, options.Interface.NodePortAdvertiseAs, - options.Interface.AdvertiseHostToClientAs, options.Interface.AdvertiseNodePortToClientAs, - nodeTcpOptions?.NodeTcpPortAdvertiseAs ?? 0, advertisedReplicationEndPoint); + return new GossipAdvertiseInfo(httpEndPoint, options.Interface.AdvertiseHostToClientAs, + options.Interface.AdvertiseNodePortToClientAs, advertisedReplicationEndPoint); } _httpService = new KestrelHttpService(_mainQueue, NodeInfo.HttpEndPoint); @@ -1023,15 +957,6 @@ GossipAdvertiseInfo GetGossipAdvertiseInfo() var authorizationGateway = new AuthorizationGateway(_authorizationProvider); - SubscribeWorkers(bus => - { - var tcpSendService = new TcpSendService(); - // ReSharper disable RedundantTypeArgumentsOfMethod - bus.Subscribe(tcpSendService); - // ReSharper restore RedundantTypeArgumentsOfMethod - }); - - var httpAuthenticationProviders = new List(); foreach (var authenticationScheme in _authenticationProvider.GetSupportedAuthenticationSchemes() ?? @@ -1106,7 +1031,7 @@ GossipAdvertiseInfo GetGossipAdvertiseInfo() _mainBus.Subscribe(forwardingService); _mainBus.Subscribe(forwardingService); _mainBus.Subscribe(forwardingService); - _mainBus.Subscribe(forwardingService); + _mainBus.Subscribe(forwardingService); _mainBus.Subscribe(forwardingService); _mainBus.Subscribe(forwardingService); _mainBus.Subscribe(forwardingService); @@ -1153,7 +1078,6 @@ GossipAdvertiseInfo GetGossipAdvertiseInfo() subscriptionQueueSlowMessageThreshold); _mainBus.Subscribe(subscrQueue); _mainBus.Subscribe(subscrQueue); - _mainBus.Subscribe(subscrQueue); _mainBus.Subscribe(subscrQueue); _mainBus.Subscribe(subscrQueue); _mainBus.Subscribe(subscrQueue); @@ -1168,7 +1092,6 @@ GossipAdvertiseInfo GetGossipAdvertiseInfo() virtualStreamReader); subscrBus.Subscribe(subscription); subscrBus.Subscribe(subscription); - subscrBus.Subscribe(subscription); subscrBus.Subscribe(subscription); subscrBus.Subscribe(subscription); subscrBus.Subscribe(subscription); @@ -1203,7 +1126,6 @@ GossipAdvertiseInfo GetGossipAdvertiseInfo() perSubscrBus.Subscribe(psubDispatcher); perSubscrBus.Subscribe(psubDispatcher); _mainBus.Subscribe(perSubscrQueue); - _mainBus.Subscribe(perSubscrQueue); _mainBus.Subscribe(perSubscrQueue); _mainBus.Subscribe(perSubscrQueue); _mainBus.Subscribe(perSubscrQueue); @@ -1237,7 +1159,6 @@ GossipAdvertiseInfo GetGossipAdvertiseInfo() perSubscrBus.Subscribe(persistentSubscription); perSubscrBus.Subscribe(persistentSubscription); perSubscrBus.Subscribe(persistentSubscription); - perSubscrBus.Subscribe(persistentSubscription); perSubscrBus.Subscribe(persistentSubscription); perSubscrBus.Subscribe(persistentSubscription); perSubscrBus.Subscribe(persistentSubscription); @@ -1484,14 +1405,9 @@ GossipAdvertiseInfo GetGossipAdvertiseInfo() _mainBus.Subscribe(_timerService); var memberInfo = MemberInfo.Initial(NodeInfo.InstanceId, _timeProvider.UtcNow, VNodeState.Unknown, true, - GossipAdvertiseInfo.InternalTcp, - GossipAdvertiseInfo.InternalSecureTcp, - GossipAdvertiseInfo.ExternalTcp, - GossipAdvertiseInfo.ExternalSecureTcp, GossipAdvertiseInfo.HttpEndPoint, GossipAdvertiseInfo.AdvertiseHostToClientAs, GossipAdvertiseInfo.AdvertiseHttpPortToClientAs, - GossipAdvertiseInfo.AdvertiseTcpPortToClientAs, options.Cluster.NodePriority, options.Cluster.ReadOnlyReplica, VersionInfo.Version, GossipAdvertiseInfo.ReplicationEndPoint); @@ -2276,6 +2192,5 @@ private void ReloadCertificates(ClusterVNodeOptions options) } public override string ToString() => - $"[{NodeInfo.InstanceId:B}, {NodeInfo.InternalTcp}, {NodeInfo.ExternalTcp}, " + - $"{NodeInfo.ReplicationEndPoint}, {NodeInfo.HttpEndPoint}]"; + $"[{NodeInfo.InstanceId:B}, {NodeInfo.ReplicationEndPoint}, {NodeInfo.HttpEndPoint}]"; } diff --git a/src/EventStore.Core/Configuration/ClusterVNodeOptions.cs b/src/EventStore.Core/Configuration/ClusterVNodeOptions.cs index 968310fdd9..a4818f1874 100644 --- a/src/EventStore.Core/Configuration/ClusterVNodeOptions.cs +++ b/src/EventStore.Core/Configuration/ClusterVNodeOptions.cs @@ -178,10 +178,10 @@ public record ApplicationOptions [Description("The maximum size of appends, in bytes. May not exceed 16MB.")] public int MaxAppendSize { get; init; } = 1_024 * 1_024; - [Description("Disable Authentication, Authorization and TLS on all TCP/HTTP interfaces.")] + [Description("Disable authentication, authorization, and TLS on the HTTP/gRPC listeners.")] public bool Insecure { get; init; } = false; - [Description("Disable TLS on all TCP/HTTP interfaces while keeping authentication and authorization enabled.")] + [Description("Disable TLS on the HTTP/gRPC listeners while keeping authentication and authorization enabled.")] public bool DisableTls { get; init; } = false; public bool AuthDisabled() => Insecure; @@ -635,7 +635,7 @@ public record InterfaceOptions [Description("Advertise the Node's host name to other nodes and external clients as.")] public string? NodeHostAdvertiseAs { get; init; } = null; - [Description("Advertise the Replication host name to other nodes in the cluster as.")] + [Description("Advertise the gRPC replication host name to other nodes in the cluster as.")] public string? ReplicationHostAdvertiseAs { get; init; } = null; [Description("Advertise Host in Gossip to Client As.")] @@ -672,12 +672,6 @@ public int GetReplicationPortAdvertiseAs() => [Description("Whether to allow local connections via a UNIX domain socket.")] public bool EnableUnixSocket { get; init; } = false; - [Description("The maximum number of pending send bytes allowed before a connection is closed.")] - public int ConnectionPendingSendBytesThreshold { get; init; } = 10 * 1_024 * 1_024; - - [Description("The maximum number of pending connection operations allowed before a connection is closed.")] - public int ConnectionQueueSizeThreshold { get; init; } = 50_000; - [Description("Disables the admin ui on the HTTP endpoint.")] public bool DisableAdminUi { get; init; } = false; diff --git a/src/EventStore.Core/Configuration/ClusterVNodeOptionsExtensions.cs b/src/EventStore.Core/Configuration/ClusterVNodeOptionsExtensions.cs index 052f6bb17a..9a4ef773da 100644 --- a/src/EventStore.Core/Configuration/ClusterVNodeOptionsExtensions.cs +++ b/src/EventStore.Core/Configuration/ClusterVNodeOptionsExtensions.cs @@ -89,21 +89,18 @@ options with }; /// - /// Sets the external tcp endpoint to the specified value + /// Sets the http endpoint to the specified value /// /// The - /// The external endpoint to use + /// The http endpoint to use /// A with the options set - public static ClusterVNodeOptions WithExternalTcpOn( + public static ClusterVNodeOptions WithNodeEndpointOn( this ClusterVNodeOptions options, IPEndPoint endPoint) => - options with { Interface = options.Interface with { NodeIp = endPoint.Address, } }; + options with { Interface = options.Interface with { NodeIp = endPoint.Address, NodePort = endPoint.Port } }; /// - /// Sets the gRPC replication endpoint to the specified value + /// Sets the endpoint used by node-to-node gRPC replication. /// - /// The - /// The internal endpoint to use - /// A with the options set public static ClusterVNodeOptions WithReplicationEndpointOn( this ClusterVNodeOptions options, IPEndPoint endPoint) => options with @@ -111,16 +108,6 @@ options with Interface = options.Interface with { ReplicationIp = endPoint.Address, ReplicationPort = endPoint.Port } }; - /// - /// Sets the http endpoint to the specified value - /// - /// The - /// The http endpoint to use - /// A with the options set - public static ClusterVNodeOptions WithNodeEndpointOn( - this ClusterVNodeOptions options, IPEndPoint endPoint) => - options with { Interface = options.Interface with { NodeIp = endPoint.Address, NodePort = endPoint.Port } }; - /// /// Sets up the External Host that would be advertised /// @@ -132,25 +119,10 @@ public static ClusterVNodeOptions options with { Interface = options.Interface with { NodeHostAdvertiseAs = endPoint.GetHost(), } }; /// - /// Sets up the Internal Host that would be advertised + /// Sets the endpoint advertised to other nodes for gRPC replication. /// - /// The - /// The advertised host - /// A with the options set - public static ClusterVNodeOptions - AdvertiseInternalHostAs(this ClusterVNodeOptions options, EndPoint endPoint) => - options with - { - Interface = options.Interface with - { - ReplicationHostAdvertiseAs = endPoint.GetHost(), - ReplicationTcpPortAdvertiseAs = endPoint.GetPort() - } - }; - public static ClusterVNodeOptions AdvertiseReplicationHostAs( - this ClusterVNodeOptions options, - EndPoint endPoint) => + this ClusterVNodeOptions options, EndPoint endPoint) => options with { Interface = options.Interface with diff --git a/src/EventStore.Core/Data/GossipAdvertiseInfo.cs b/src/EventStore.Core/Data/GossipAdvertiseInfo.cs index 4fcfcca51f..dc8de85350 100644 --- a/src/EventStore.Core/Data/GossipAdvertiseInfo.cs +++ b/src/EventStore.Core/Data/GossipAdvertiseInfo.cs @@ -5,50 +5,25 @@ namespace EventStore.Core.Data { public class GossipAdvertiseInfo { - public DnsEndPoint InternalTcp { get; } - public DnsEndPoint InternalSecureTcp { get; } - public DnsEndPoint ExternalTcp { get; } - public DnsEndPoint ExternalSecureTcp { get; } - public DnsEndPoint HttpEndPoint { get; } public DnsEndPoint ReplicationEndPoint { get; } - public string AdvertiseInternalHostAs { get; } - public string AdvertiseExternalHostAs { get; } - public int AdvertiseHttpPortAs { get; } + public DnsEndPoint HttpEndPoint { get; } public string AdvertiseHostToClientAs { get; } public int AdvertiseHttpPortToClientAs { get; } - public int AdvertiseTcpPortToClientAs { get; } - public GossipAdvertiseInfo(DnsEndPoint internalTcp, DnsEndPoint internalSecureTcp, - DnsEndPoint externalTcp, DnsEndPoint externalSecureTcp, - DnsEndPoint httpEndPoint, - string advertiseInternalHostAs, string advertiseExternalHostAs, int advertiseHttpPortAs, - string advertiseHostToClientAs, int advertiseHttpPortToClientAs, int advertiseTcpPortToClientAs, + public GossipAdvertiseInfo(DnsEndPoint httpEndPoint, + string advertiseHostToClientAs, int advertiseHttpPortToClientAs, DnsEndPoint replicationEndPoint = null) { - Ensure.Equal(false, internalTcp == null && internalSecureTcp == null, "Both internal TCP endpoints are null"); - - InternalTcp = internalTcp; - InternalSecureTcp = internalSecureTcp; - ExternalTcp = externalTcp; - ExternalSecureTcp = externalSecureTcp; - HttpEndPoint = httpEndPoint; + Ensure.NotNull(httpEndPoint, nameof(httpEndPoint)); ReplicationEndPoint = replicationEndPoint ?? httpEndPoint; - AdvertiseInternalHostAs = advertiseInternalHostAs; - AdvertiseExternalHostAs = advertiseExternalHostAs; - AdvertiseHttpPortAs = advertiseHttpPortAs; + HttpEndPoint = httpEndPoint; AdvertiseHostToClientAs = advertiseHostToClientAs; AdvertiseHttpPortToClientAs = advertiseHttpPortToClientAs; - AdvertiseTcpPortToClientAs = advertiseTcpPortToClientAs; } public override string ToString() { - return string.Format( - $"IntTcp: {InternalTcp}, IntSecureTcp: {InternalSecureTcp}\n" + - $"ExtTcp: {ExternalTcp}, ExtSecureTcp: {ExternalSecureTcp}\n" + - $"Http: {HttpEndPoint}, Replication: {ReplicationEndPoint}, HttpAdvertiseAs: {AdvertiseExternalHostAs}:{AdvertiseHttpPortAs},\n" + - $"HttpAdvertiseToClientAs: {AdvertiseHostToClientAs}:{AdvertiseHttpPortToClientAs},\n" + - $"TcpAdvertiseToClientAs: {AdvertiseHostToClientAs}:{AdvertiseTcpPortToClientAs}"); + return $"Replication: {ReplicationEndPoint}, Http: {HttpEndPoint}, HttpAdvertiseToClientAs: {AdvertiseHostToClientAs}:{AdvertiseHttpPortToClientAs}"; } } } diff --git a/src/EventStore.Core/Data/VNodeInfo.cs b/src/EventStore.Core/Data/VNodeInfo.cs index d8aa86d500..9361344b21 100644 --- a/src/EventStore.Core/Data/VNodeInfo.cs +++ b/src/EventStore.Core/Data/VNodeInfo.cs @@ -8,60 +8,32 @@ public class VNodeInfo { public readonly Guid InstanceId; public readonly int DebugIndex; - public readonly IPEndPoint InternalTcp; - public readonly IPEndPoint InternalSecureTcp; - public readonly IPEndPoint ExternalTcp; - public readonly IPEndPoint ExternalSecureTcp; - public readonly EndPoint HttpEndPoint; public readonly EndPoint ReplicationEndPoint; + public readonly EndPoint HttpEndPoint; public readonly bool IsReadOnlyReplica; - public VNodeInfo(Guid instanceId, int debugIndex, - IPEndPoint internalTcp, IPEndPoint internalSecureTcp, - IPEndPoint externalTcp, IPEndPoint externalSecureTcp, - EndPoint httpEndPoint, - bool isReadOnlyReplica, - EndPoint replicationEndPoint = null) + public VNodeInfo(Guid instanceId, int debugIndex, EndPoint httpEndPoint, + bool isReadOnlyReplica, EndPoint replicationEndPoint = null) { Ensure.NotEmptyGuid(instanceId, "instanceId"); - Ensure.Equal(false, internalTcp == null && internalSecureTcp == null, "Both internal TCP endpoints are null"); Ensure.NotNull(httpEndPoint, nameof(httpEndPoint)); DebugIndex = debugIndex; InstanceId = instanceId; - InternalTcp = internalTcp; - InternalSecureTcp = internalSecureTcp; - ExternalTcp = externalTcp; - ExternalSecureTcp = externalSecureTcp; - HttpEndPoint = httpEndPoint; ReplicationEndPoint = replicationEndPoint ?? httpEndPoint; + HttpEndPoint = httpEndPoint; IsReadOnlyReplica = isReadOnlyReplica; } public bool Is(EndPoint endPoint) { - return endPoint != null - && (HttpEndPoint.Equals(endPoint) - || ReplicationEndPoint.Equals(endPoint) - || (InternalTcp != null && InternalTcp.Equals(endPoint)) - || (InternalSecureTcp != null && InternalSecureTcp.Equals(endPoint)) - || (ExternalTcp != null && ExternalTcp.Equals(endPoint)) - || (ExternalSecureTcp != null && ExternalSecureTcp.Equals(endPoint))); + return endPoint != null && + (HttpEndPoint.Equals(endPoint) || ReplicationEndPoint.Equals(endPoint)); } public override string ToString() { - return string.Format("InstanceId: {0:B}, InternalTcp: {1}, InternalSecureTcp: {2}, " + - "ExternalTcp: {3}, ExternalSecureTcp: {4}, HttpEndPoint: {5}, " + - "ReplicationEndPoint: {6}, IsReadOnlyReplica: {7}", - InstanceId, - InternalTcp, - InternalSecureTcp, - ExternalTcp, - ExternalSecureTcp, - HttpEndPoint, - ReplicationEndPoint, - IsReadOnlyReplica); + return $"InstanceId: {InstanceId:B}, ReplicationEndPoint: {ReplicationEndPoint}, HttpEndPoint: {HttpEndPoint}, IsReadOnlyReplica: {IsReadOnlyReplica}"; } } } diff --git a/src/EventStore.Core/EventStore.Core.csproj b/src/EventStore.Core/EventStore.Core.csproj index fbe5f47564..5c3b635cda 100644 --- a/src/EventStore.Core/EventStore.Core.csproj +++ b/src/EventStore.Core/EventStore.Core.csproj @@ -43,15 +43,11 @@ - - - Services\Transport\Tcp\protos\ClientMessageDtos.proto - Services\Transport\Grpc\Protos\code.proto @@ -98,7 +94,4 @@ Services\Transport\Grpc\Protos\forwarding.proto - - - diff --git a/src/EventStore.Core/Helpers/IAsyncMessageFramer.cs b/src/EventStore.Core/Helpers/IAsyncMessageFramer.cs new file mode 100644 index 0000000000..23820cb0f5 --- /dev/null +++ b/src/EventStore.Core/Helpers/IAsyncMessageFramer.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace EventStore.Core.Helpers; + +public interface IAsyncMessageFramer +{ + bool HasData { get; } + IEnumerable> FrameData(ArraySegment data); + ValueTask UnFrameData(IEnumerable> data, CancellationToken token); + ValueTask UnFrameData(ArraySegment data, CancellationToken token); + void RegisterMessageArrivedCallback(Func handler); + void Reset(); +} diff --git a/src/EventStore.Core/Helpers/LengthPrefixSuffixFramer.cs b/src/EventStore.Core/Helpers/LengthPrefixSuffixFramer.cs index 64f7ea7dcd..afd89966de 100644 --- a/src/EventStore.Core/Helpers/LengthPrefixSuffixFramer.cs +++ b/src/EventStore.Core/Helpers/LengthPrefixSuffixFramer.cs @@ -7,7 +7,6 @@ using System.Threading.Tasks; using EventStore.Common.Utils; using EventStore.Core.TransactionLog.Chunks; -using EventStore.Transport.Tcp.Framing; using ILogger = Serilog.ILogger; namespace EventStore.Core.Helpers; @@ -85,7 +84,7 @@ private async ValueTask Parse(ArraySegment bytes, CancellationToken token) if (_packageLength <= 0 || _packageLength > _maxPackageSize) { Log.Error("FRAMING ERROR! Data:\n{data}", Helper.FormatBinaryDump(bytes)); - throw new PackageFramingException(string.Format( + throw new MessageFramingException(string.Format( "Package size is out of bounds: {0} (max: {1}).", _packageLength, _maxPackageSize)); } diff --git a/src/EventStore.Core/Helpers/MessageFramingException.cs b/src/EventStore.Core/Helpers/MessageFramingException.cs new file mode 100644 index 0000000000..f8f10f9f03 --- /dev/null +++ b/src/EventStore.Core/Helpers/MessageFramingException.cs @@ -0,0 +1,10 @@ +using System; + +namespace EventStore.Core.Helpers; + +public sealed class MessageFramingException : Exception +{ + public MessageFramingException(string message) : base(message) + { + } +} diff --git a/src/EventStore.Core/Messages/ClientMessage.cs b/src/EventStore.Core/Messages/ClientMessage.cs index 3b0f349bfa..5903247d7d 100644 --- a/src/EventStore.Core/Messages/ClientMessage.cs +++ b/src/EventStore.Core/Messages/ClientMessage.cs @@ -203,20 +203,29 @@ public enum NotHandledReason public class LeaderInfo { - public LeaderInfo(EndPoint externalTcp, bool isSecure, EndPoint http) + public LeaderInfo(EndPoint http) { - ExternalTcp = externalTcp; - IsSecure = isSecure; Http = http; } - public bool IsSecure { get; } - public EndPoint ExternalTcp { get; } public EndPoint Http { get; } } } } + [DerivedMessage(CoreMessage.Client)] + public partial class NotAuthenticated : Message + { + public readonly Guid CorrelationId; + public readonly string Reason; + + public NotAuthenticated(Guid correlationId, string reason) + { + CorrelationId = correlationId; + Reason = reason; + } + } + [DerivedMessage(CoreMessage.Client)] public partial class WriteEvents : WriteRequestMessage { diff --git a/src/EventStore.Core/Messages/ClusterInfoDto.cs b/src/EventStore.Core/Messages/ClusterInfoDto.cs deleted file mode 100644 index 68ab95c1e1..0000000000 --- a/src/EventStore.Core/Messages/ClusterInfoDto.cs +++ /dev/null @@ -1,32 +0,0 @@ -using System.Linq; -using System.Net; -using EventStore.Common.Utils; -using EventStore.Core.Cluster; - -namespace EventStore.Core.Messages -{ - public class ClusterInfoDto - { - public MemberInfoDto[] Members { get; set; } - public string ServerIp { get; set; } - public int ServerPort { get; set; } - - public ClusterInfoDto() - { - } - - public ClusterInfoDto(ClusterInfo clusterInfo, EndPoint serverEndPoint) - { - Members = clusterInfo.Members.Select(x => new MemberInfoDto(x)).ToArray(); - ServerIp = serverEndPoint.GetHost(); - ServerPort = serverEndPoint.GetPort(); - } - - public override string ToString() - { - return string.Format("Server: {0}:{1}, Members: [{2}]", - ServerIp, ServerPort, - Members != null ? string.Join(",", Members.Select(m => m.ToString())) : "null"); - } - } -} diff --git a/src/EventStore.Core/Messages/MemberInfoDto.cs b/src/EventStore.Core/Messages/MemberInfoDto.cs deleted file mode 100644 index eb491f64d6..0000000000 --- a/src/EventStore.Core/Messages/MemberInfoDto.cs +++ /dev/null @@ -1,95 +0,0 @@ -using System; -using EventStore.Common.Utils; -using EventStore.Core.Cluster; -using EventStore.Core.Data; - -namespace EventStore.Core.Messages -{ - public class MemberInfoDto - { - public Guid InstanceId { get; set; } - - public DateTime TimeStamp { get; set; } - public VNodeState State { get; set; } - public bool IsAlive { get; set; } - - public string InternalTcpIp { get; set; } - public int InternalTcpPort { get; set; } - public int InternalSecureTcpPort { get; set; } - - public string ExternalTcpIp { get; set; } - public int ExternalTcpPort { get; set; } - public int ExternalSecureTcpPort { get; set; } - - public string HttpEndPointIp { get; set; } - public int HttpEndPointPort { get; set; } - public string AdvertiseHostToClientAs { get; set; } - public int AdvertiseHttpPortToClientAs { get; set; } - public int AdvertiseTcpPortToClientAs { get; set; } - - public long LastCommitPosition { get; set; } - public long WriterCheckpoint { get; set; } - public long ChaserCheckpoint { get; set; } - - public long EpochPosition { get; set; } - public int EpochNumber { get; set; } - public Guid EpochId { get; set; } - - public int NodePriority { get; set; } - public bool IsReadOnlyReplica { get; set; } - - public MemberInfoDto() - { - } - - public MemberInfoDto(MemberInfo member) - { - InstanceId = member.InstanceId; - - TimeStamp = member.TimeStamp; - State = member.State; - IsAlive = member.IsAlive; - - InternalTcpIp = member.InternalTcpEndPoint?.GetHost() ?? member.InternalSecureTcpEndPoint?.GetHost(); - InternalTcpPort = member.InternalTcpEndPoint == null ? 0 : member.InternalTcpEndPoint.GetPort(); - InternalSecureTcpPort = - member.InternalSecureTcpEndPoint == null ? 0 : member.InternalSecureTcpEndPoint.GetPort(); - - ExternalTcpIp = member.ExternalTcpEndPoint?.GetHost() ?? member.ExternalSecureTcpEndPoint?.GetHost(); - ExternalTcpPort = member.ExternalTcpEndPoint == null ? 0 : member.ExternalTcpEndPoint.GetPort(); - ExternalSecureTcpPort = - member.ExternalSecureTcpEndPoint == null ? 0 : member.ExternalSecureTcpEndPoint.GetPort(); - - HttpEndPointIp = member.HttpEndPoint.GetHost(); - HttpEndPointPort = member.HttpEndPoint.GetPort(); - AdvertiseHostToClientAs = member.AdvertiseHostToClientAs; - AdvertiseHttpPortToClientAs = member.AdvertiseHttpPortToClientAs; - AdvertiseTcpPortToClientAs = member.AdvertiseTcpPortToClientAs; - - LastCommitPosition = member.LastCommitPosition; - WriterCheckpoint = member.WriterCheckpoint; - ChaserCheckpoint = member.ChaserCheckpoint; - - EpochPosition = member.EpochPosition; - EpochNumber = member.EpochNumber; - EpochId = member.EpochId; - - NodePriority = member.NodePriority; - IsReadOnlyReplica = member.IsReadOnlyReplica; - } - - public override string ToString() - { - return - $"InstanceId: {InstanceId:B}, TimeStamp: {TimeStamp:yyyy-MM-dd HH:mm:ss.fff}, State: {State}, IsAlive: {IsAlive}, " + - $"InternalTcpIp: {InternalTcpIp}, InternalTcpPort: {InternalTcpPort}, InternalSecureTcpPort: {InternalSecureTcpPort}, " + - $"ExternalTcpIp: {ExternalTcpIp}, ExternalTcpPort: {ExternalTcpPort}, ExternalSecureTcpPort: {ExternalSecureTcpPort}, " + - $"HttpEndPointIp: {HttpEndPointIp}, HttpEndPointPort: {HttpEndPointPort}, " + - $"{nameof(AdvertiseHostToClientAs)}: {AdvertiseHostToClientAs}, {nameof(AdvertiseHttpPortToClientAs)}: {AdvertiseHttpPortToClientAs}, " + - $"{nameof(AdvertiseTcpPortToClientAs)}: {AdvertiseTcpPortToClientAs}, " + - $"LastCommitPosition: {LastCommitPosition}, WriterCheckpoint: {WriterCheckpoint}, ChaserCheckpoint: {ChaserCheckpoint}, " + - $"EpochPosition: {EpochPosition}, EpochNumber: {EpochNumber}, EpochId: {EpochId:B}, NodePriority: {NodePriority}, " + - $"IsReadOnlyReplica: {IsReadOnlyReplica}"; - } - } -} diff --git a/src/EventStore.Core/Messages/TcpClientMessageDto.cs b/src/EventStore.Core/Messages/TcpClientMessageDto.cs deleted file mode 100755 index b62f9cf1bd..0000000000 --- a/src/EventStore.Core/Messages/TcpClientMessageDto.cs +++ /dev/null @@ -1,670 +0,0 @@ -using System; -using System.Net; -using EventStore.Common.Utils; -using EventStore.Core.Messages; -using Google.Protobuf; - -// ReSharper disable once CheckNamespace -namespace EventStore.Client.Messages -{ - partial class NewEvent - { - public NewEvent(byte[] eventId, string eventType, int dataContentType, int metadataContentType, byte[] data, byte[] metadata) - { - - EventId = ByteString.CopyFrom(eventId); - EventType = eventType; - DataContentType = dataContentType; - MetadataContentType = metadataContentType; - Data = ByteString.CopyFrom(data); - Metadata = ByteString.CopyFrom(metadata); - } - } - - partial class EventRecord - { - public EventRecord(string eventStreamId, long eventNumber, byte[] eventId, string eventType, int dataContentType, int metadataContentType, byte[] data, byte[] metadata, long created, long createdEpoch) - { - AssignValues(eventStreamId, eventNumber, eventId, eventType, dataContentType, metadataContentType, data, metadata, created, createdEpoch); - } - - private void AssignValues(string eventStreamId, long eventNumber, byte[] eventId, string eventType, int dataContentType, - int metadataContentType, byte[] data, byte[] metadata, long created, long createdEpoch) - { - EventStreamId = eventStreamId; - EventNumber = eventNumber; - EventId = ByteString.CopyFrom(eventId); - EventType = eventType; - DataContentType = dataContentType; - MetadataContentType = metadataContentType; - Data = ByteString.CopyFrom(data); - Metadata = ByteString.CopyFrom(metadata); - Created = created; - CreatedEpoch = createdEpoch; - } - - public EventRecord(Core.Data.EventRecord eventRecord) - { - if (eventRecord == null) - { - return; - } - - AssignValues( - eventRecord.EventStreamId, - eventRecord.EventNumber, - eventRecord.EventId.ToByteArray(), - eventRecord.EventType, - eventRecord.IsJson ? 1 : 0, - eventRecord.IsJson ? 1 : 0, - eventRecord.Data.ToArray(), - eventRecord.Metadata.ToArray(), - eventRecord.TimeStamp.ToBinary(), - (long)(eventRecord.TimeStamp - new DateTime(1970, 1, 1)).TotalMilliseconds - ); - } - - public EventRecord(Core.Data.EventRecord eventRecord, long eventNumber) - { - if (eventRecord == null) - { - return; - } - - AssignValues( - eventRecord.EventStreamId, - eventNumber, - eventRecord.EventId.ToByteArray(), - eventRecord.EventType, - eventRecord.IsJson ? 1 : 0, - eventRecord.IsJson ? 1 : 0, - eventRecord.Data.ToArray(), - eventRecord.Metadata.ToArray(), - eventRecord.TimeStamp.ToBinary(), - (long)(eventRecord.TimeStamp - new DateTime(1970, 1, 1)).TotalMilliseconds - ); - } - } - - partial class ResolvedIndexedEvent - { - public ResolvedIndexedEvent(EventRecord @event, EventRecord link) - { - Event = @event; - Link = link; - } - - public ResolvedIndexedEvent(Core.Data.EventRecord @event, Core.Data.EventRecord link) - { - if (@event != null) - { - Event = new EventRecord(@event); - } - - if (link != null) - { - Link = new EventRecord(link); - } - } - } - - partial class ResolvedEvent - { - public ResolvedEvent(EventRecord @event, EventRecord link, long commitPosition, long preparePosition) - { - if (@event != null) - { - Event = @event; - } - - if (link != null) - { - Link = link; - } - - CommitPosition = commitPosition; - PreparePosition = preparePosition; - } - - public ResolvedEvent(Core.Data.ResolvedEvent pair) - : this(pair.Event != null ? new EventRecord(pair.Event) : null, - pair.Link != null ? new EventRecord(pair.Link) : null, - pair.OriginalPosition?.CommitPosition ?? -1, - pair.OriginalPosition?.PreparePosition ?? -1) - { - } - } - - partial class WriteEvents - { - public WriteEvents(string eventStreamId, long expectedVersion, NewEvent[] events, bool requireLeader) - { - EventStreamId = eventStreamId; - ExpectedVersion = expectedVersion; - Events.AddRange(events); - RequireLeader = requireLeader; - } - } - - partial class WriteEventsCompleted - { - public WriteEventsCompleted(OperationResult result, string message, long firstEventNumber, long lastEventNumber, long preparePosition, long commitPosition, long currentVersion) - { - Result = result; - if (message != null) - { - Message = message; - } - - FirstEventNumber = firstEventNumber; - LastEventNumber = lastEventNumber; - PreparePosition = preparePosition; - CommitPosition = commitPosition; - CurrentVersion = currentVersion; - } - } - - partial class DeleteStream - { - public DeleteStream(string eventStreamId, long expectedVersion, bool requireLeader, bool hardDelete) - { - EventStreamId = eventStreamId; - ExpectedVersion = expectedVersion; - RequireLeader = requireLeader; - HardDelete = hardDelete; - } - } - - partial class DeleteStreamCompleted - { - public DeleteStreamCompleted(OperationResult result, string message, long currentVersion, long preparePosition, - long commitPosition) - { - Result = result; - if (!string.IsNullOrEmpty(message)) - { - Message = message; - } - - CurrentVersion = currentVersion; - PreparePosition = preparePosition; - CommitPosition = commitPosition; - } - } - - partial class TransactionStart - { - public TransactionStart(string eventStreamId, long expectedVersion, bool requireLeader) - { - EventStreamId = eventStreamId; - ExpectedVersion = expectedVersion; - RequireLeader = requireLeader; - } - } - - partial class TransactionStartCompleted - { - public TransactionStartCompleted(long transactionId, OperationResult result, string message) - { - TransactionId = transactionId; - Result = result; - if (message != null) - { - Message = message; - } - } - } - - partial class TransactionWrite - { - public TransactionWrite(long transactionId, NewEvent[] events, bool requireLeader) - { - TransactionId = transactionId; - Events.AddRange(events); - RequireLeader = requireLeader; - } - } - - partial class TransactionWriteCompleted - { - public TransactionWriteCompleted(long transactionId, OperationResult result, string message) - { - TransactionId = transactionId; - Result = result; - if (message != null) - { - Message = message; - } - } - } - - partial class TransactionCommit - { - public TransactionCommit(long transactionId, bool requireLeader) - { - TransactionId = transactionId; - RequireLeader = requireLeader; - } - } - - partial class TransactionCommitCompleted - { - public TransactionCommitCompleted(long transactionId, OperationResult result, string message, long firstEventNumber, long lastEventNumber, long preparePosition, long commitPosition) - { - TransactionId = transactionId; - Result = result; - if (message != null) - { - Message = message; - } - - FirstEventNumber = firstEventNumber; - LastEventNumber = lastEventNumber; - PreparePosition = preparePosition; - CommitPosition = commitPosition; - } - } - - partial class ReadEvent - { - public ReadEvent(string eventStreamId, long eventNumber, bool resolveLinkTos, bool requireLeader) - { - EventStreamId = eventStreamId; - EventNumber = eventNumber; - ResolveLinkTos = resolveLinkTos; - RequireLeader = requireLeader; - } - } - - partial class ReadEventCompleted - { - public ReadEventCompleted(ReadEventCompleted.Types.ReadEventResult result, ResolvedIndexedEvent @event, string error) - { - Result = result; - if (@event != null) - { - Event = @event; - } - - if (error != null) - { - Error = error; - } - } - } - - partial class ReadStreamEvents - { - public ReadStreamEvents(string eventStreamId, long fromEventNumber, int maxCount, bool resolveLinkTos, bool requireLeader) - { - EventStreamId = eventStreamId; - FromEventNumber = fromEventNumber; - MaxCount = maxCount; - ResolveLinkTos = resolveLinkTos; - RequireLeader = requireLeader; - } - } - - partial class ReadStreamEventsCompleted - { - public ReadStreamEventsCompleted(ResolvedIndexedEvent[] events, ReadStreamEventsCompleted.Types.ReadStreamResult result, long nextEventNumber, long lastEventNumber, bool isEndOfStream, long lastCommitPosition, string error) - { - Events.AddRange(events); - Result = result; - NextEventNumber = nextEventNumber; - LastEventNumber = lastEventNumber; - IsEndOfStream = isEndOfStream; - LastCommitPosition = lastCommitPosition; - if (error != null) - { - Error = error; - } - } - } - - partial class ReadAllEvents - { - public ReadAllEvents(long commitPosition, long preparePosition, int maxCount, bool resolveLinkTos, bool requireLeader) - { - CommitPosition = commitPosition; - PreparePosition = preparePosition; - MaxCount = maxCount; - ResolveLinkTos = resolveLinkTos; - RequireLeader = requireLeader; - } - } - - partial class ReadAllEventsCompleted - { - public ReadAllEventsCompleted(long commitPosition, long preparePosition, ResolvedEvent[] events, long nextCommitPosition, long nextPreparePosition, ReadAllEventsCompleted.Types.ReadAllResult result, string error) - { - CommitPosition = commitPosition; - PreparePosition = preparePosition; - Events.AddRange(events); - NextCommitPosition = nextCommitPosition; - NextPreparePosition = nextPreparePosition; - Result = result; - if (error != null) - { - Error = error; - } - } - } - - partial class Filter - { - public Filter(Filter.Types.FilterContext context, Filter.Types.FilterType type, string[] data) - { - Context = context; - Type = type; - Data.AddRange(data); - } - } - - partial class FilteredReadAllEvents - { - public FilteredReadAllEvents(long commitPosition, long preparePosition, int maxCount, int maxSearchWindow, bool resolveLinkTos, bool requireLeader, Filter filter) - { - CommitPosition = commitPosition; - PreparePosition = preparePosition; - MaxCount = maxCount; - MaxSearchWindow = maxSearchWindow; - ResolveLinkTos = resolveLinkTos; - RequireLeader = requireLeader; - Filter = filter; - } - } - - partial class FilteredReadAllEventsCompleted - { - public FilteredReadAllEventsCompleted(long commitPosition, long preparePosition, ResolvedEvent[] events, long nextCommitPosition, long nextPreparePosition, bool isEndOfStream, FilteredReadAllEventsCompleted.Types.FilteredReadAllResult result, string error) - { - CommitPosition = commitPosition; - PreparePosition = preparePosition; - Events.AddRange(events); - NextCommitPosition = nextCommitPosition; - NextPreparePosition = nextPreparePosition; - IsEndOfStream = isEndOfStream; - Result = result; - if (error != null) - { - Error = error; - } - } - } - - partial class CreatePersistentSubscription - { - public CreatePersistentSubscription(string subscriptionGroupName, string eventStreamId, bool resolveLinkTos, long startFrom, int messageTimeoutMilliseconds, bool recordStatistics, int liveBufferSize, int readBatchSize, int bufferSize, int maxRetryCount, bool preferRoundRobin, int checkpointAfterTime, int checkpointMaxCount, int checkpointMinCount, int subscriberMaxCount, string namedConsumerStrategy) - { - SubscriptionGroupName = subscriptionGroupName; - EventStreamId = eventStreamId; - ResolveLinkTos = resolveLinkTos; - StartFrom = startFrom; - MessageTimeoutMilliseconds = messageTimeoutMilliseconds; - RecordStatistics = recordStatistics; - LiveBufferSize = liveBufferSize; - ReadBatchSize = readBatchSize; - BufferSize = bufferSize; - MaxRetryCount = maxRetryCount; - PreferRoundRobin = preferRoundRobin; - CheckpointAfterTime = checkpointAfterTime; - CheckpointMaxCount = checkpointMaxCount; - CheckpointMinCount = checkpointMinCount; - SubscriberMaxCount = subscriberMaxCount; - NamedConsumerStrategy = namedConsumerStrategy; - } - } - - partial class CreatePersistentSubscriptionCompleted - { - public CreatePersistentSubscriptionCompleted(CreatePersistentSubscriptionCompleted.Types.CreatePersistentSubscriptionResult result, string reason) - { - Result = result; - if (reason != null) - { - Reason = reason; - } - } - } - - partial class DeletePersistentSubscription - { - public DeletePersistentSubscription(string subscriptionGroupName, string eventStreamId) - { - SubscriptionGroupName = subscriptionGroupName; - EventStreamId = eventStreamId; - } - } - - partial class DeletePersistentSubscriptionCompleted - { - public DeletePersistentSubscriptionCompleted(Types.DeletePersistentSubscriptionResult result, string reason) - { - Result = result; - Reason = reason; - } - } - - partial class UpdatePersistentSubscription - { - public UpdatePersistentSubscription(string subscriptionGroupName, string eventStreamId, bool resolveLinkTos, long startFrom, int messageTimeoutMilliseconds, bool recordStatistics, int liveBufferSize, int readBatchSize, int bufferSize, int maxRetryCount, bool preferRoundRobin, int checkpointAfterTime, int checkpointMaxCount, int checkpointMinCount, int subscriberMaxCount, string namedConsumerStrategy) - { - SubscriptionGroupName = subscriptionGroupName; - EventStreamId = eventStreamId; - ResolveLinkTos = resolveLinkTos; - StartFrom = startFrom; - MessageTimeoutMilliseconds = messageTimeoutMilliseconds; - RecordStatistics = recordStatistics; - LiveBufferSize = liveBufferSize; - ReadBatchSize = readBatchSize; - BufferSize = bufferSize; - MaxRetryCount = maxRetryCount; - PreferRoundRobin = preferRoundRobin; - CheckpointAfterTime = checkpointAfterTime; - CheckpointMaxCount = checkpointMaxCount; - CheckpointMinCount = checkpointMinCount; - SubscriberMaxCount = subscriberMaxCount; - NamedConsumerStrategy = namedConsumerStrategy; - } - } - - partial class UpdatePersistentSubscriptionCompleted - { - public UpdatePersistentSubscriptionCompleted(UpdatePersistentSubscriptionCompleted.Types.UpdatePersistentSubscriptionResult result, string reason) - { - Result = result; - if (reason != null) - { - Reason = reason; - } - } - } - - partial class ConnectToPersistentSubscription - { - public ConnectToPersistentSubscription(string subscriptionId, string eventStreamId, int allowedInFlightMessages) - { - SubscriptionId = subscriptionId; - EventStreamId = eventStreamId; - AllowedInFlightMessages = allowedInFlightMessages; - } - } - - partial class PersistentSubscriptionConfirmation - { - public PersistentSubscriptionConfirmation(long lastCommitPosition, string subscriptionId, long lastEventNumber) - { - LastCommitPosition = lastCommitPosition; - SubscriptionId = subscriptionId; - LastEventNumber = lastEventNumber; - } - } - - partial class PersistentSubscriptionAckEvents - { - public PersistentSubscriptionAckEvents(string subscriptionId, byte[][] processedEventIds) - { - SubscriptionId = subscriptionId; - for (int i = 0; i < processedEventIds.Length; i++) - { - ProcessedEventIds.Add(ByteString.CopyFrom(processedEventIds[i])); - } - } - } - - partial class PersistentSubscriptionNakEvents - { - public PersistentSubscriptionNakEvents(string subscriptionId, byte[][] processedEventIds, string message, PersistentSubscriptionNakEvents.Types.NakAction action) - { - SubscriptionId = subscriptionId; - for (int i = 0; i < processedEventIds.Length; i++) - { - ProcessedEventIds.Add(ByteString.CopyFrom(processedEventIds[i])); - } - - if (message != null) - { - Message = message; - } - - Action = action; - } - } - - partial class PersistentSubscriptionStreamEventAppeared - { - public PersistentSubscriptionStreamEventAppeared(ResolvedIndexedEvent @event, int retryCount) - { - Event = @event; - RetryCount = retryCount; - } - } - - partial class SubscribeToStream - { - public SubscribeToStream(string eventStreamId, bool resolveLinkTos) - { - EventStreamId = eventStreamId; - ResolveLinkTos = resolveLinkTos; - } - } - - partial class FilteredSubscribeToStream - { - public FilteredSubscribeToStream(string eventStreamId, bool resolveLinkTos, Filter filter, int checkpointInterval) - { - EventStreamId = eventStreamId; - ResolveLinkTos = resolveLinkTos; - Filter = filter; - CheckpointInterval = checkpointInterval; - } - } - - partial class CheckpointReached - { - public CheckpointReached(long commitPosition, long preparePosition) - { - CommitPosition = commitPosition; - PreparePosition = preparePosition; - } - } - - partial class SubscriptionConfirmation - { - public SubscriptionConfirmation(long lastCommitPosition, long lastEventNumber) - { - LastCommitPosition = lastCommitPosition; - LastEventNumber = lastEventNumber; - } - } - - partial class StreamEventAppeared - { - public StreamEventAppeared(ResolvedEvent @event) - { - Event = @event; - } - } - - partial class SubscriptionDropped - { - public SubscriptionDropped(SubscriptionDropped.Types.SubscriptionDropReason reason) - { - Reason = reason; - } - } - - partial class NotHandled - { - public NotHandled(ClientMessage.NotHandled source) - { - Reason = source.Reason switch - { - ClientMessage.NotHandled.Types.NotHandledReason.NotReady => Types.NotHandledReason.NotReady, - ClientMessage.NotHandled.Types.NotHandledReason.TooBusy => Types.NotHandledReason.TooBusy, - ClientMessage.NotHandled.Types.NotHandledReason.NotLeader => Types.NotHandledReason.NotLeader, - ClientMessage.NotHandled.Types.NotHandledReason.IsReadOnly => Types.NotHandledReason.IsReadOnly, - _ => throw new ArgumentOutOfRangeException() - }; - //this is horrible and only for transport compatibility purposes - if (source.LeaderInfo != null) - { - AdditionalInfo = new Types.LeaderInfo(source.LeaderInfo.ExternalTcp, source.LeaderInfo.IsSecure, source.LeaderInfo.Http).ToByteString(); - } - } - partial class Types - { - partial class LeaderInfo - { - public LeaderInfo(string externalTcpAddress, int externalTcpPort, string httpAddress, int httpPort, string externalSecureTcpAddress, int externalSecureTcpPort) - { - ExternalTcpAddress = externalTcpAddress; - ExternalTcpPort = externalTcpPort; - HttpAddress = httpAddress; - HttpPort = httpPort; - ExternalSecureTcpAddress = externalSecureTcpAddress; - ExternalSecureTcpPort = externalSecureTcpPort; - } - - public LeaderInfo(EndPoint tcpEndPoint, bool isTcpEndPointSecure, EndPoint httpEndPoint) - { - if (isTcpEndPointSecure) - { - ExternalSecureTcpAddress = tcpEndPoint?.GetHost(); - ExternalSecureTcpPort = tcpEndPoint?.GetPort() ?? 0; - } - else - { - ExternalTcpAddress = tcpEndPoint?.GetHost(); - ExternalTcpPort = tcpEndPoint?.GetPort() ?? 0; - } - - HttpAddress = httpEndPoint.GetHost(); - HttpPort = httpEndPoint.GetPort(); - } - } - } - } - - partial class ScavengeDatabaseResponse - { - public ScavengeDatabaseResponse(ScavengeDatabaseResponse.Types.ScavengeResult result, string scavengeId) - { - Result = result; - ScavengeId = scavengeId; - } - } - - partial class IdentifyClient - { - public IdentifyClient(int version, string connectionName) - { - Version = version; - if (connectionName != null) - { - ConnectionName = connectionName; - } - } - } -} diff --git a/src/EventStore.Core/Messages/TcpMessage.cs b/src/EventStore.Core/Messages/TcpMessage.cs deleted file mode 100644 index 40f1425898..0000000000 --- a/src/EventStore.Core/Messages/TcpMessage.cs +++ /dev/null @@ -1,127 +0,0 @@ -using System; -using System.Net.Sockets; -using EventStore.Core.Messaging; -using EventStore.Core.Services.Transport.Tcp; - -namespace EventStore.Core.Messages -{ - public static partial class TcpMessage - { - [DerivedMessage(CoreMessage.Tcp)] - public partial class TcpSend : Message - { - public override object SynchronizationGroup - { - get { return ConnectionManager; } - } - - public readonly TcpConnectionManager ConnectionManager; - public readonly Message Message; - - public TcpSend(TcpConnectionManager connectionManager, Message message) - { - ConnectionManager = connectionManager; - Message = message; - } - } - - [DerivedMessage(CoreMessage.Tcp)] - public partial class Heartbeat : Message - { - public readonly long ReceiveProgressIndicator; - public readonly long SendProgressIndicator; - - public Heartbeat(long receiveProgressIndicator, long sendProgressIndicator) - { - ReceiveProgressIndicator = receiveProgressIndicator; - SendProgressIndicator = sendProgressIndicator; - } - } - - [DerivedMessage(CoreMessage.Tcp)] - public partial class HeartbeatTimeout : Message - { - public readonly long ReceiveProgressIndicator; - - public HeartbeatTimeout(long receiveProgressIndicator) - { - ReceiveProgressIndicator = receiveProgressIndicator; - } - } - - [DerivedMessage(CoreMessage.Tcp)] - public partial class PongMessage : Message - { - public readonly Guid CorrelationId; - public readonly byte[] Payload; - - public PongMessage(Guid correlationId, byte[] payload) - { - CorrelationId = correlationId; - Payload = payload; - } - } - - [DerivedMessage(CoreMessage.Tcp)] - public partial class ConnectionEstablished : Message - { - public readonly TcpConnectionManager Connection; - - public ConnectionEstablished(TcpConnectionManager connection) - { - Connection = connection; - } - } - - [DerivedMessage(CoreMessage.Tcp)] - public partial class ConnectionClosed : Message - { - public readonly TcpConnectionManager Connection; - public readonly SocketError SocketError; - - public ConnectionClosed(TcpConnectionManager connection, SocketError socketError) - { - Connection = connection; - SocketError = socketError; - } - } - - [DerivedMessage(CoreMessage.Tcp)] - public partial class NotReady : Message - { - public readonly Guid CorrelationId; - public readonly string Reason; - - public NotReady(Guid correlationId, string reason) - { - CorrelationId = correlationId; - Reason = reason; - } - } - - - [DerivedMessage(CoreMessage.Tcp)] - public partial class NotAuthenticated : Message - { - public readonly Guid CorrelationId; - public readonly string Reason; - - public NotAuthenticated(Guid correlationId, string reason) - { - CorrelationId = correlationId; - Reason = reason; - } - } - - [DerivedMessage(CoreMessage.Tcp)] - public partial class Authenticated : Message - { - public readonly Guid CorrelationId; - - public Authenticated(Guid correlationId) - { - CorrelationId = correlationId; - } - } - } -} diff --git a/src/EventStore.Core/Messaging/CoreMessage.cs b/src/EventStore.Core/Messaging/CoreMessage.cs index 89d29acc6f..4e6a16e0de 100644 --- a/src/EventStore.Core/Messaging/CoreMessage.cs +++ b/src/EventStore.Core/Messaging/CoreMessage.cs @@ -22,7 +22,6 @@ public enum CoreMessage Storage, Subscription, System, - Tcp, Telemetry, Timer, UserManagement diff --git a/src/EventStore.Core/NodeTcpOptions.cs b/src/EventStore.Core/NodeTcpOptions.cs deleted file mode 100644 index 880c10b9d5..0000000000 --- a/src/EventStore.Core/NodeTcpOptions.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace EventStore.Core; - -// Still needed for gossip and stats. -public class NodeTcpOptions -{ - public int NodeTcpPort { get; init; } = 1113; - public bool EnableExternalTcp { get; init; } - public int? NodeTcpPortAdvertiseAs { get; init; } -} diff --git a/src/EventStore.Core/Services/AuthorizationGateway.cs b/src/EventStore.Core/Services/AuthorizationGateway.cs index 927e83bb27..e95f692661 100644 --- a/src/EventStore.Core/Services/AuthorizationGateway.cs +++ b/src/EventStore.Core/Services/AuthorizationGateway.cs @@ -211,7 +211,7 @@ public void Authorize(Message toValidate, IPublisher destination) case ClientMessage.PersistentSubscriptionNackEvents _: case ClientMessage.UnsubscribeFromStream _: case ClientMessage.NotHandled _: - case TcpMessage.NotAuthenticated _: + case ClientMessage.NotAuthenticated _: case ClientMessage.WriteEventsCompleted _: case ClientMessage.TransactionStartCompleted _: case ClientMessage.TransactionWriteCompleted _: @@ -244,8 +244,6 @@ public void Authorize(Message toValidate, IPublisher destination) break; default: #if DEBUG - //This sucks, because if new tcp messages are added there is no way to be sure they have to be authorized... - //They should be caught by debug builds though throw new ArgumentOutOfRangeException(nameof(toValidate), toValidate.GetType().FullName, "Unhandled client message"); #else diff --git a/src/EventStore.Core/Services/ElectionsService.cs b/src/EventStore.Core/Services/ElectionsService.cs index d0a6f9bed8..a220a232ce 100644 --- a/src/EventStore.Core/Services/ElectionsService.cs +++ b/src/EventStore.Core/Services/ElectionsService.cs @@ -137,10 +137,8 @@ public ElectionsService(IPublisher publisher, _timeProvider.UtcNow, VNodeState.Initializing, true, - memberInfo.InternalTcpEndPoint, memberInfo.InternalSecureTcpEndPoint, - memberInfo.ExternalTcpEndPoint, memberInfo.ExternalSecureTcpEndPoint, memberInfo.HttpEndPoint, - memberInfo.AdvertiseHostToClientAs, memberInfo.AdvertiseHttpPortToClientAs, memberInfo.AdvertiseTcpPortToClientAs, + memberInfo.AdvertiseHostToClientAs, memberInfo.AdvertiseHttpPortToClientAs, ownInfo.LastCommitPosition, ownInfo.WriterCheckpoint, ownInfo.ChaserCheckpoint, ownInfo.EpochPosition, ownInfo.EpochNumber, ownInfo.EpochId, ownInfo.NodePriority, memberInfo.IsReadOnlyReplica, VersionInfo.Version, memberInfo.ReplicationEndPoint) diff --git a/src/EventStore.Core/Services/Gossip/GossipServiceBase.cs b/src/EventStore.Core/Services/Gossip/GossipServiceBase.cs index 643147e6ed..9b1720c796 100644 --- a/src/EventStore.Core/Services/Gossip/GossipServiceBase.cs +++ b/src/EventStore.Core/Services/Gossip/GossipServiceBase.cs @@ -274,7 +274,7 @@ public void Handle(GossipMessage.GossipSendFailed message) if (node.InstanceId == CurrentLeader?.InstanceId) { Log.Information( - "Leader [{leaderEndPoint}, {instanceId:B}] appears to be DEAD (Gossip send failed); wait for TCP to decide.", + "Leader [{leaderEndPoint}, {instanceId:B}] appears to be DEAD (Gossip send failed); wait for replication transport to decide.", message.Recipient, node.InstanceId); return; } @@ -302,7 +302,7 @@ public void Handle(SystemMessage.VNodeConnectionLost message) return; } - Log.Information("Looks like node [{nodeEndPoint}] is DEAD (TCP connection lost). Issuing a gossip to confirm.", + Log.Information("Looks like node [{nodeEndPoint}] is DEAD (replication connection lost). Issuing a gossip to confirm.", message.VNodeEndPoint); _bus.Publish(new GrpcMessage.SendOverGrpc(node.HttpEndPoint, new GossipMessage.GetGossip(), @@ -354,7 +354,7 @@ public void Handle(GossipMessage.GetGossipFailed message) if (_cluster.HasChangedSince(oldCluster)) { LogClusterChange(oldCluster, _cluster, - string.Format("TCP connection lost to [{0}]", message.Recipient)); + string.Format("Replication connection lost to [{0}]", message.Recipient)); } _bus.Publish(new GossipMessage.GossipUpdated(_cluster)); @@ -371,7 +371,7 @@ public void Handle(SystemMessage.VNodeConnectionEstablished message) if (_cluster.HasChangedSince(oldCluster)) { LogClusterChange(oldCluster, _cluster, - string.Format("TCP connection established to [{0}]", message.VNodeEndPoint)); + string.Format("Replication connection established to [{0}]", message.VNodeEndPoint)); } _bus.Publish(new GossipMessage.GossipUpdated(_cluster)); diff --git a/src/EventStore.Core/Services/Gossip/NodeGossipService.cs b/src/EventStore.Core/Services/Gossip/NodeGossipService.cs index 3014089eaf..cca9ede01e 100644 --- a/src/EventStore.Core/Services/Gossip/NodeGossipService.cs +++ b/src/EventStore.Core/Services/Gossip/NodeGossipService.cs @@ -57,14 +57,9 @@ protected override MemberInfo GetInitialMe() _timeProvider.UtcNow, initialState, true, - _memberInfo.InternalTcpEndPoint, - _memberInfo.InternalSecureTcpEndPoint, - _memberInfo.ExternalTcpEndPoint, - _memberInfo.ExternalSecureTcpEndPoint, _memberInfo.HttpEndPoint, _memberInfo.AdvertiseHostToClientAs, _memberInfo.AdvertiseHttpPortToClientAs, - _memberInfo.AdvertiseTcpPortToClientAs, _getLastCommitPosition(), _writerCheckpoint.Read(), _chaserCheckpoint.Read(), @@ -72,7 +67,8 @@ protected override MemberInfo GetInitialMe() lastEpoch == null ? -1 : lastEpoch.EpochNumber, lastEpoch == null ? Guid.Empty : lastEpoch.EpochId, _nodePriority, - _memberInfo.IsReadOnlyReplica, _memberInfo.ESVersion, _memberInfo.ReplicationEndPoint); + _memberInfo.IsReadOnlyReplica, _memberInfo.ESVersion, + _memberInfo.ReplicationEndPoint); } protected override MemberInfo GetUpdatedMe(MemberInfo me) diff --git a/src/EventStore.Core/Services/Monitoring/MonitoringService.cs b/src/EventStore.Core/Services/Monitoring/MonitoringService.cs index f342fdfcdc..c4db88ef49 100644 --- a/src/EventStore.Core/Services/Monitoring/MonitoringService.cs +++ b/src/EventStore.Core/Services/Monitoring/MonitoringService.cs @@ -13,7 +13,6 @@ using EventStore.Core.Messaging; using EventStore.Core.Services.Monitoring.Stats; using EventStore.Core.Services.UserManagement; -using EventStore.Transport.Tcp; using ILogger = Serilog.ILogger; using Timeout = System.Threading.Timeout; @@ -33,8 +32,7 @@ public class MonitoringService : IHandle, IAsyncHandle, IHandle, IHandle, - IAsyncHandle, - IHandle + IAsyncHandle { private static readonly ILogger RegularLog = Serilog.Log.ForContext(Serilog.Core.Constants.SourceContextPropertyName, "REGULAR-STATS-LOGGER"); @@ -62,19 +60,12 @@ public class MonitoringService : IHandle, private readonly string _nodeStatsStream; private bool _statsStreamCreated; private Guid _streamMetadataWriteCorrId; - private IMonitoredTcpConnection[] _memoizedTcpConnections; - private DateTime _lastTcpConnectionsRequestTime; - private IPEndPoint _tcpEndpoint; - private IPEndPoint _tcpSecureEndpoint; - public MonitoringService(IQueuedHandler monitoringQueue, IAsyncHandle statsCollectionDispatcher, IPublisher mainQueue, TimeSpan statsCollectionPeriod, EndPoint nodeEndpoint, StatsStorage statsStorage, - IPEndPoint tcpEndpoint, - IPEndPoint tcpSecureEndpoint, SystemStatsHelper systemStatsHelper) { Ensure.NotNull(monitoringQueue, "monitoringQueue"); @@ -100,8 +91,6 @@ public MonitoringService(IQueuedHandler monitoringQueue, } _nodeStatsStream = $"{SystemStreams.StatsStreamPrefix}-{nodeEndpoint}"; - _tcpEndpoint = tcpEndpoint; - _tcpSecureEndpoint = tcpSecureEndpoint; _timer = Task.CompletedTask; _systemStats = systemStatsHelper; @@ -343,74 +332,6 @@ public async ValueTask HandleAsync(MonitoringMessage.GetFreshStats message, Canc } } - public void Handle(MonitoringMessage.GetFreshTcpConnectionStats message) - { - try - { - IMonitoredTcpConnection[] connections = null; - if (!TryGetMemoizedTcpConnections(out connections)) - { - connections = TcpConnectionMonitor.Default.GetTcpConnectionStats(); - if (connections != null) - { - _memoizedTcpConnections = connections; - _lastTcpConnectionsRequestTime = DateTime.UtcNow; - } - } - - List connStats = new List(); - foreach (var conn in connections) - { - var tcpConn = conn as TcpConnection; - if (tcpConn != null) - { - var isExternalConnection = _tcpEndpoint != null && _tcpEndpoint.Port == tcpConn.LocalEndPoint.GetPort(); - connStats.Add(new MonitoringMessage.TcpConnectionStats - { - IsExternalConnection = isExternalConnection, - RemoteEndPoint = tcpConn.RemoteEndPoint.ToString(), - LocalEndPoint = tcpConn.LocalEndPoint.ToString(), - ConnectionId = tcpConn.ConnectionId, - ClientConnectionName = tcpConn.ClientConnectionName, - TotalBytesSent = tcpConn.TotalBytesSent, - TotalBytesReceived = tcpConn.TotalBytesReceived, - PendingSendBytes = tcpConn.PendingSendBytes, - PendingReceivedBytes = tcpConn.PendingReceivedBytes, - IsSslConnection = false - }); - } - - var tcpConnSsl = conn as TcpConnectionSsl; - if (tcpConnSsl != null) - { - var isExternalConnection = _tcpSecureEndpoint != null && - _tcpSecureEndpoint.Port == tcpConnSsl.LocalEndPoint.GetPort(); - connStats.Add(new MonitoringMessage.TcpConnectionStats - { - IsExternalConnection = isExternalConnection, - RemoteEndPoint = tcpConnSsl.RemoteEndPoint.ToString(), - LocalEndPoint = tcpConnSsl.LocalEndPoint.ToString(), - ConnectionId = tcpConnSsl.ConnectionId, - ClientConnectionName = tcpConnSsl.ClientConnectionName, - TotalBytesSent = tcpConnSsl.TotalBytesSent, - TotalBytesReceived = tcpConnSsl.TotalBytesReceived, - PendingSendBytes = tcpConnSsl.PendingSendBytes, - PendingReceivedBytes = tcpConnSsl.PendingReceivedBytes, - IsSslConnection = true - }); - } - } - - message.Envelope.ReplyWith( - new MonitoringMessage.GetFreshTcpConnectionStatsCompleted(connStats) - ); - } - catch (Exception ex) - { - Log.Error(ex, "Error on getting fresh tcp connection stats"); - } - } - private bool TryGetMemoizedStats(out StatsContainer stats) { if (_memoizedStats == null || DateTime.UtcNow - _lastStatsRequestTime > MemoizePeriod) @@ -423,16 +344,5 @@ private bool TryGetMemoizedStats(out StatsContainer stats) return true; } - private bool TryGetMemoizedTcpConnections(out IMonitoredTcpConnection[] connections) - { - if (_memoizedTcpConnections == null || DateTime.UtcNow - _lastTcpConnectionsRequestTime > MemoizePeriod) - { - connections = null; - return false; - } - - connections = _memoizedTcpConnections; - return true; - } } } diff --git a/src/EventStore.Core/Services/Monitoring/SystemStatsHelper.cs b/src/EventStore.Core/Services/Monitoring/SystemStatsHelper.cs index e81c4a859c..7ffa6d8323 100644 --- a/src/EventStore.Core/Services/Monitoring/SystemStatsHelper.cs +++ b/src/EventStore.Core/Services/Monitoring/SystemStatsHelper.cs @@ -8,7 +8,6 @@ using EventStore.Core.Services.Monitoring.Stats; using EventStore.Core.Services.Monitoring.Utils; using EventStore.Core.TransactionLog.Checkpoint; -using EventStore.Transport.Tcp; using ILogger = Serilog.ILogger; namespace EventStore.Core.Services.Monitoring @@ -48,19 +47,6 @@ public IDictionary GetSystemStats() stats["proc-diskIo-readOps"] = diskIo.ReadOps; stats["proc-diskIo-writeOps"] = diskIo.WriteOps; - var tcp = TcpConnectionMonitor.Default.GetTcpStats(); - stats["proc-tcp-connections"] = tcp.Connections; - stats["proc-tcp-receivingSpeed"] = tcp.ReceivingSpeed; - stats["proc-tcp-sendingSpeed"] = tcp.SendingSpeed; - stats["proc-tcp-inSend"] = tcp.InSend; - stats["proc-tcp-measureTime"] = tcp.MeasureTime; - stats["proc-tcp-pendingReceived"] = tcp.PendingReceived; - stats["proc-tcp-pendingSend"] = tcp.PendingSend; - stats["proc-tcp-receivedBytesSinceLastRun"] = tcp.ReceivedBytesSinceLastRun; - stats["proc-tcp-receivedBytesTotal"] = tcp.ReceivedBytesTotal; - stats["proc-tcp-sentBytesSinceLastRun"] = tcp.SentBytesSinceLastRun; - stats["proc-tcp-sentBytesTotal"] = tcp.SentBytesTotal; - stats["es-checksum"] = _writerCheckpoint.Read(); stats["es-checksumNonFlushed"] = _writerCheckpoint.ReadNonFlushed(); diff --git a/src/EventStore.Core/Services/PersistentSubscription/PersistentSubscriptionService.cs b/src/EventStore.Core/Services/PersistentSubscription/PersistentSubscriptionService.cs index c1d3213f9d..9669f2ed46 100644 --- a/src/EventStore.Core/Services/PersistentSubscription/PersistentSubscriptionService.cs +++ b/src/EventStore.Core/Services/PersistentSubscription/PersistentSubscriptionService.cs @@ -29,7 +29,6 @@ public abstract class PersistentSubscriptionService public class PersistentSubscriptionService : PersistentSubscriptionService, IHandle, - IHandle, IHandle, IHandle, IHandle, @@ -1109,24 +1108,6 @@ private void UnsubscribeFromStream(Guid correlationId, bool sendDropNotification } } - public void Handle(TcpMessage.ConnectionClosed message) - { - if (_subscriptionsById == null) - { - return; //haven't built yet. - } - - foreach (var subscription in _subscriptionsById.Values) - { - if (subscription.RemoveClientByConnectionId(message.Connection.ConnectionId)) - { - Log.Debug("Persistent subscription {subscription} lost connection from {remoteEndPoint}", - subscription.SubscriptionId, - message.Connection.RemoteEndPoint); - } - } - } - private void DisconnectFromStream(Guid connectionId) { foreach (var subscription in _subscriptionsById.Values) @@ -1138,7 +1119,6 @@ private void DisconnectFromStream(Guid connectionId) } } } - public async ValueTask ConnectToPersistentSubscription( IPersistentSubscriptionEventSource eventSource, string groupName, diff --git a/src/EventStore.Core/Services/Replication/LogRecordFramer.cs b/src/EventStore.Core/Services/Replication/LogRecordFramer.cs index 117e626d6a..b4c870ae24 100644 --- a/src/EventStore.Core/Services/Replication/LogRecordFramer.cs +++ b/src/EventStore.Core/Services/Replication/LogRecordFramer.cs @@ -6,8 +6,8 @@ using System.Threading.Tasks; using DotNext.IO; using EventStore.Common.Utils; +using EventStore.Core.Helpers; using EventStore.Core.TransactionLog.LogRecords; -using EventStore.Transport.Tcp.Framing; namespace EventStore.Core.Services.Replication; diff --git a/src/EventStore.Core/Services/Replication/TransactionFramer.cs b/src/EventStore.Core/Services/Replication/TransactionFramer.cs index 8b6788e635..e1166610df 100644 --- a/src/EventStore.Core/Services/Replication/TransactionFramer.cs +++ b/src/EventStore.Core/Services/Replication/TransactionFramer.cs @@ -3,8 +3,8 @@ using System.Threading; using System.Threading.Tasks; using EventStore.Common.Utils; +using EventStore.Core.Helpers; using EventStore.Core.TransactionLog.LogRecords; -using EventStore.Transport.Tcp.Framing; namespace EventStore.Core.Services.Replication; diff --git a/src/EventStore.Core/Services/RequestForwarding/GrpcRequestForwardingSupervisor.cs b/src/EventStore.Core/Services/RequestForwarding/GrpcRequestForwardingSupervisor.cs index 91c6792a28..cfd7d7a51b 100644 --- a/src/EventStore.Core/Services/RequestForwarding/GrpcRequestForwardingSupervisor.cs +++ b/src/EventStore.Core/Services/RequestForwarding/GrpcRequestForwardingSupervisor.cs @@ -136,7 +136,7 @@ public void Handle(ClientMessage.ForwardMessage message) "Request forwarding stream is closed.")); break; case RequestForwardingAdmission.CredentialsRequireTls: - PublishIfActive(active, new TcpMessage.NotAuthenticated( + PublishIfActive(active, new ClientMessage.NotAuthenticated( request.InternalCorrId, "Credentials cannot be forwarded unless transport security is enabled.")); break; @@ -214,11 +214,11 @@ private void ConnectToLeader(MemberInfo leader) IGrpcRequestForwardingService service = null; try { - var active = new ActiveStream(leader.InstanceId, leader.ReplicationEndPoint, connectionGeneration); + var active = new ActiveStream(leader.InstanceId, leader.HttpEndPoint, connectionGeneration); service = _factory.Create( message => TryPublishIfActive(active, message), _publisher.Publish, - leader.ReplicationEndPoint, + leader.HttpEndPoint, new ForwardingSessionGeneration(connectionGeneration)); active.Service = service; _active = active; @@ -237,7 +237,7 @@ private void ConnectToLeader(MemberInfo leader) service?.Stop(); Log.Warning(exception, "Failed to start request forwarding stream to [{leaderEndPoint}].", - leader.ReplicationEndPoint); + leader.HttpEndPoint); if (connectionGeneration == _connectionGeneration) { ScheduleReconnect(leader.InstanceId, connectionGeneration); @@ -280,7 +280,7 @@ private bool HasHealthyStreamTo(MemberInfo leader) => _active is not null && !_active.Service.Task.IsCompleted && _active.LeaderId == leader.InstanceId && - Equals(_active.LeaderEndPoint, leader.ReplicationEndPoint); + Equals(_active.LeaderEndPoint, leader.HttpEndPoint); private void PublishIfActive(ActiveStream active, Message message) { diff --git a/src/EventStore.Core/Services/RequestForwardingService.cs b/src/EventStore.Core/Services/RequestForwardingService.cs index 00ec4690c2..94670ddd75 100644 --- a/src/EventStore.Core/Services/RequestForwardingService.cs +++ b/src/EventStore.Core/Services/RequestForwardingService.cs @@ -10,7 +10,7 @@ namespace EventStore.Core.Services public class RequestForwardingService : IHandle, IHandle, IHandle, - IHandle, + IHandle, IHandle, IHandle, IHandle, @@ -54,11 +54,11 @@ public void Handle(ClientMessage.NotHandled message) (clientCorrId, m) => new ClientMessage.NotHandled(clientCorrId, m.Reason, m.LeaderInfo)); } - public void Handle(TcpMessage.NotAuthenticated message) + public void Handle(ClientMessage.NotAuthenticated message) { _forwardingProxy.TryForwardReply( message.CorrelationId, message, - (clientCorrId, m) => new TcpMessage.NotAuthenticated(clientCorrId, m.Reason)); + (clientCorrId, m) => new ClientMessage.NotAuthenticated(clientCorrId, m.Reason)); } public void Handle(ClientMessage.WriteEventsCompleted message) diff --git a/src/EventStore.Core/Services/Storage/ReaderIndex/EventFilter.cs b/src/EventStore.Core/Services/Storage/ReaderIndex/EventFilter.cs index fbb3ff716f..52326bdf74 100644 --- a/src/EventStore.Core/Services/Storage/ReaderIndex/EventFilter.cs +++ b/src/EventStore.Core/Services/Storage/ReaderIndex/EventFilter.cs @@ -36,31 +36,6 @@ public static IEventFilter Regex(bool isAllStream, string regex) => new EventTypeRegexStrategy(isAllStream, regex); } - public static IEventFilter Get(bool isAllStream, Client.Messages.Filter filter) - { - if (filter == null || filter.Data.Count == 0) - { - return isAllStream ? (IEventFilter)new DefaultAllFilterStrategy() : new DefaultStreamFilterStrategy(); - } - - return filter.Context switch - { - Client.Messages.Filter.Types.FilterContext.EventType when filter.Type == - Client.Messages.Filter.Types.FilterType.Prefix => - EventType.Prefixes(isAllStream, filter.Data.ToArray()), - Client.Messages.Filter.Types.FilterContext.EventType when filter.Type == - Client.Messages.Filter.Types.FilterType.Regex => - EventType.Regex(isAllStream, filter.Data[0]), - Client.Messages.Filter.Types.FilterContext.StreamId when filter.Type == - Client.Messages.Filter.Types.FilterType.Prefix => - StreamName.Prefixes(isAllStream, filter.Data.ToArray()), - Client.Messages.Filter.Types.FilterContext.StreamId when filter.Type == - Client.Messages.Filter.Types.FilterType.Regex => - StreamName.Regex(isAllStream, filter.Data[0]), - _ => throw new Exception() // Invalid filter - }; - } - private sealed class DefaultStreamFilterStrategy : IEventFilter { [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] @@ -281,34 +256,16 @@ public static (bool success, string reason) TryParse(EventFilterDto dto, out IEv public static (bool Success, string Reason) TryParse(string context, bool isAllStream, string type, string data, out IEventFilter filter) { - Client.Messages.Filter.Types.FilterContext parsedContext; - switch (context) + if (context is not EventTypeContext and not StreamIdContext) { - case EventTypeContext: - parsedContext = Client.Messages.Filter.Types.FilterContext.EventType; - break; - case StreamIdContext: - parsedContext = Client.Messages.Filter.Types.FilterContext.StreamId; - break; - default: - filter = null; - var names = string.Join(", ", Enum.GetNames(typeof(Client.Messages.Filter.Types.FilterContext))); - return (false, $"Invalid context please provide one of the following: {names}."); + filter = null; + return (false, "Invalid context please provide one of the following: EventType, StreamId."); } - Client.Messages.Filter.Types.FilterType parsedType; - switch (type) + if (type is not RegexType and not PrefixType) { - case RegexType: - parsedType = Client.Messages.Filter.Types.FilterType.Regex; - break; - case PrefixType: - parsedType = Client.Messages.Filter.Types.FilterType.Prefix; - break; - default: - filter = null; - var names = string.Join(", ", Enum.GetNames(typeof(Client.Messages.Filter.Types.FilterType))); - return (false, $"Invalid type please provide one of the following: {names}."); + filter = null; + return (false, "Invalid type please provide one of the following: Regex, Prefix."); } if (string.IsNullOrEmpty(data)) @@ -317,14 +274,18 @@ public static (bool Success, string Reason) TryParse(string context, bool isAllS return (false, "Please provide a comma delimited list of data with at least one item."); } - if (parsedType == Client.Messages.Filter.Types.FilterType.Regex) + if (type == RegexType) { - filter = Get(isAllStream, new Client.Messages.Filter(parsedContext, parsedType, new[] { data })); + filter = context == EventTypeContext + ? EventType.Regex(isAllStream, data) + : StreamName.Regex(isAllStream, data); return (true, null); } - filter = Get(isAllStream, new Client.Messages.Filter(parsedContext, parsedType, - data.Split(new[] { "," }, StringSplitOptions.RemoveEmptyEntries))); + var prefixes = data.Split(new[] { "," }, StringSplitOptions.RemoveEmptyEntries); + filter = context == EventTypeContext + ? EventType.Prefixes(isAllStream, prefixes) + : StreamName.Prefixes(isAllStream, prefixes); return (true, null); } } diff --git a/src/EventStore.Core/Services/SubscriptionsService.cs b/src/EventStore.Core/Services/SubscriptionsService.cs index dba64708b5..6709c71ae0 100644 --- a/src/EventStore.Core/Services/SubscriptionsService.cs +++ b/src/EventStore.Core/Services/SubscriptionsService.cs @@ -37,7 +37,6 @@ public class SubscriptionsService : SubscriptionsService, IHandle, IHandle, - IHandle, IAsyncHandle, IAsyncHandle, IHandle, @@ -111,41 +110,6 @@ public void Handle(SystemMessage.BecomeShuttingDown message) _queuedHandler.RequestStop(); } - public void Handle(TcpMessage.ConnectionClosed message) - { - List subscriptionGroupsToRemove = null; - foreach (var subscriptionGroup in _subscriptionTopics) - { - var subscriptions = subscriptionGroup.Value; - for (int i = 0, n = subscriptions.Count; i < n; ++i) - { - if (subscriptions[i].ConnectionId == message.Connection.ConnectionId) - { - _subscriptionsById.Remove(subscriptions[i].CorrelationId); - } - } - - subscriptions.RemoveAll(x => x.ConnectionId == message.Connection.ConnectionId); - if (subscriptions.Count == 0) // schedule removal of list instance - { - if (subscriptionGroupsToRemove == null) - { - subscriptionGroupsToRemove = new List(); - } - - subscriptionGroupsToRemove.Add(subscriptionGroup.Key); - } - } - - if (subscriptionGroupsToRemove != null) - { - for (int i = 0, n = subscriptionGroupsToRemove.Count; i < n; ++i) - { - _subscriptionTopics.Remove(subscriptionGroupsToRemove[i]); - } - } - } - async ValueTask IAsyncHandle.HandleAsync(ClientMessage.SubscribeToStream msg, CancellationToken token) { diff --git a/src/EventStore.Core/Services/TcpSendService.cs b/src/EventStore.Core/Services/TcpSendService.cs deleted file mode 100644 index e3e2d31bcd..0000000000 --- a/src/EventStore.Core/Services/TcpSendService.cs +++ /dev/null @@ -1,14 +0,0 @@ -using EventStore.Core.Bus; -using EventStore.Core.Messages; - -namespace EventStore.Core.Services -{ - public class TcpSendService : IHandle - { - public void Handle(TcpMessage.TcpSend message) - { - // todo: histogram metric? - message.ConnectionManager.SendMessage(message.Message); - } - } -} diff --git a/src/EventStore.Core/Services/Transport/Grpc/Forwarding/ForwardingGrpcCodec.cs b/src/EventStore.Core/Services/Transport/Grpc/Forwarding/ForwardingGrpcCodec.cs index be24786c2e..c9e857c9de 100644 --- a/src/EventStore.Core/Services/Transport/Grpc/Forwarding/ForwardingGrpcCodec.cs +++ b/src/EventStore.Core/Services/Transport/Grpc/Forwarding/ForwardingGrpcCodec.cs @@ -172,7 +172,7 @@ public static ClientMessage.WriteRequestMessage FromGrpc( ClientMessage.TransactionCommitCompleted completed => ToGrpc(completed), ClientMessage.DeleteStreamCompleted completed => ToGrpc(completed), ClientMessage.NotHandled notHandled => ToGrpc(notHandled), - TcpMessage.NotAuthenticated notAuthenticated => ToGrpc(notAuthenticated), + ClientMessage.NotAuthenticated notAuthenticated => ToGrpc(notAuthenticated), _ => throw new ArgumentOutOfRangeException(nameof(message), message.GetType().FullName, "Unsupported forwarding response") } @@ -195,7 +195,7 @@ public static Message FromGrpc(Proto.LeaderFrame frame) Proto.ForwardResponse.PayloadOneofCase.DeleteStream => FromGrpc(correlationId, response.DeleteStream), Proto.ForwardResponse.PayloadOneofCase.NotHandled => FromGrpc(correlationId, response.NotHandled), Proto.ForwardResponse.PayloadOneofCase.NotAuthenticated => - new TcpMessage.NotAuthenticated(correlationId, response.NotAuthenticated.Reason), + new ClientMessage.NotAuthenticated(correlationId, response.NotAuthenticated.Reason), _ => throw new ArgumentOutOfRangeException(nameof(frame), response.PayloadCase, "Unknown forwarding response") }; @@ -457,8 +457,6 @@ private static Proto.ForwardResponse ToGrpc(ClientMessage.NotHandled message) { notHandled.LeaderInfo = new Proto.LeaderInfo { - ExternalTcp = ToGrpc(message.LeaderInfo.ExternalTcp), - IsSecure = message.LeaderInfo.IsSecure, Http = ToGrpc(message.LeaderInfo.Http) }; } @@ -471,7 +469,7 @@ private static Proto.ForwardResponse ToGrpc(ClientMessage.NotHandled message) return response; } - private static Proto.ForwardResponse ToGrpc(TcpMessage.NotAuthenticated message) + private static Proto.ForwardResponse ToGrpc(ClientMessage.NotAuthenticated message) { var response = NewResponse(message.CorrelationId); response.NotAuthenticated = new Proto.NotAuthenticated { Reason = message.Reason ?? string.Empty }; @@ -552,8 +550,6 @@ private static ClientMessage.NotHandled FromGrpc(Guid correlationId, Proto.NotHa correlationId, reason, new ClientMessage.NotHandled.Types.LeaderInfo( - FromGrpc(message.LeaderInfo.ExternalTcp), - message.LeaderInfo.IsSecure, FromGrpc(message.LeaderInfo.Http))), Proto.NotHandled.DetailOneofCase.Description => new ClientMessage.NotHandled(correlationId, reason, message.Description), diff --git a/src/EventStore.Core/Services/Transport/Grpc/Forwarding/ForwardingService.cs b/src/EventStore.Core/Services/Transport/Grpc/Forwarding/ForwardingService.cs index 39b5ef358d..c08f7d546f 100644 --- a/src/EventStore.Core/Services/Transport/Grpc/Forwarding/ForwardingService.cs +++ b/src/EventStore.Core/Services/Transport/Grpc/Forwarding/ForwardingService.cs @@ -355,7 +355,7 @@ public static ForwardedAuthentication Authenticated( public static ForwardedAuthentication NotAuthenticated(string reason) => new( null, null, - requestId => new TcpMessage.NotAuthenticated(requestId, reason)); + requestId => new ClientMessage.NotAuthenticated(requestId, reason)); public static ForwardedAuthentication NotReady(string reason) => new( null, diff --git a/src/EventStore.Core/Services/Transport/Tcp/ClientTcpDispatcher.cs b/src/EventStore.Core/Services/Transport/Tcp/ClientTcpDispatcher.cs deleted file mode 100644 index fd95d28cff..0000000000 --- a/src/EventStore.Core/Services/Transport/Tcp/ClientTcpDispatcher.cs +++ /dev/null @@ -1,600 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Security.Claims; -using EventStore.Client.Messages; -using EventStore.Common.Utils; -using EventStore.Core.Messages; -using EventStore.Core.Messaging; -using EventStore.Core.Services.Storage.ReaderIndex; -using EventStore.Core.Util; -using ResolvedEvent = EventStore.Core.Data.ResolvedEvent; - -namespace EventStore.Core.Services.Transport.Tcp; - -public class ClientTcpDispatcher : ClientWriteTcpDispatcher -{ - public ClientTcpDispatcher(int writeTimeoutMs) - : this(TimeSpan.FromMilliseconds(writeTimeoutMs)) - { - } - - public ClientTcpDispatcher(TimeSpan writeTimeout) : base(writeTimeout) - { - AddUnwrapper(TcpCommand.Ping, UnwrapPing, ClientVersion.V2); - AddWrapper(WrapPong, ClientVersion.V2); - - AddUnwrapper(TcpCommand.IdentifyClient, UnwrapIdentifyClient, ClientVersion.V2); - - AddUnwrapper(TcpCommand.ReadEvent, UnwrapReadEvent, ClientVersion.V2); - AddWrapper(WrapReadEventCompleted, ClientVersion.V2); - - AddUnwrapper(TcpCommand.ReadStreamEventsForward, UnwrapReadStreamEventsForward, ClientVersion.V2); - AddWrapper(WrapReadStreamEventsForwardCompleted, - ClientVersion.V2); - AddUnwrapper(TcpCommand.ReadStreamEventsBackward, UnwrapReadStreamEventsBackward, ClientVersion.V2); - AddWrapper(WrapReadStreamEventsBackwardCompleted, - ClientVersion.V2); - - AddUnwrapper(TcpCommand.ReadAllEventsForward, UnwrapReadAllEventsForward, ClientVersion.V2); - AddWrapper(WrapReadAllEventsForwardCompleted, - ClientVersion.V2); - AddUnwrapper(TcpCommand.ReadAllEventsBackward, UnwrapReadAllEventsBackward, ClientVersion.V2); - AddWrapper(WrapReadAllEventsBackwardCompleted, - ClientVersion.V2); - - AddUnwrapper(TcpCommand.FilteredReadAllEventsForward, UnwrapFilteredReadAllEventsForward, ClientVersion.V2); - AddWrapper(WrapFilteredReadAllEventsForwardCompleted, - ClientVersion.V2); - - AddUnwrapper(TcpCommand.FilteredReadAllEventsBackward, UnwrapFilteredReadAllEventsBackward, - ClientVersion.V2); - AddWrapper(WrapFilteredReadAllEventsBackwardCompleted, - ClientVersion.V2); - - AddUnwrapper(TcpCommand.SubscribeToStream, UnwrapSubscribeToStream, ClientVersion.V2); - AddUnwrapper(TcpCommand.FilteredSubscribeToStream, UnwrapFilteredSubscribeToStream, ClientVersion.V2); - AddUnwrapper(TcpCommand.UnsubscribeFromStream, UnwrapUnsubscribeFromStream, ClientVersion.V2); - - AddWrapper(WrapCheckpointReached, ClientVersion.V2); - - AddWrapper(WrapSubscribedToStream, ClientVersion.V2); - AddWrapper(WrapStreamEventAppeared, ClientVersion.V2); - AddWrapper(WrapSubscriptionDropped, ClientVersion.V2); - AddUnwrapper(TcpCommand.CreatePersistentSubscription, UnwrapCreatePersistentSubscription, ClientVersion.V2); - AddUnwrapper(TcpCommand.DeletePersistentSubscription, UnwrapDeletePersistentSubscription, ClientVersion.V2); - AddWrapper( - WrapCreatePersistentSubscriptionCompleted, - ClientVersion.V2); - AddWrapper( - WrapDeletePersistentSubscriptionCompleted, - ClientVersion.V2); - AddUnwrapper(TcpCommand.UpdatePersistentSubscription, UnwrapUpdatePersistentSubscription, ClientVersion.V2); - AddWrapper( - WrapUpdatePersistentSubscriptionCompleted, - ClientVersion.V2); - - - AddUnwrapper(TcpCommand.ConnectToPersistentSubscription, UnwrapConnectToPersistentSubscription, - ClientVersion.V2); - AddUnwrapper(TcpCommand.PersistentSubscriptionAckEvents, UnwrapPersistentSubscriptionAckEvents, - ClientVersion.V2); - AddUnwrapper(TcpCommand.PersistentSubscriptionNakEvents, UnwrapPersistentSubscriptionNackEvents, - ClientVersion.V2); - AddWrapper(WrapPersistentSubscriptionConfirmation, - ClientVersion.V2); - AddWrapper( - WrapPersistentSubscriptionStreamEventAppeared, ClientVersion.V2); - - AddUnwrapper(TcpCommand.ScavengeDatabase, UnwrapScavengeDatabase, ClientVersion.V2); - AddWrapper(WrapScavengeDatabaseResponse, ClientVersion.V2); - AddWrapper(WrapScavengeDatabaseResponse, ClientVersion.V2); - AddWrapper(WrapScavengeDatabaseResponse, ClientVersion.V2); - - AddWrapper(WrapAuthenticated, ClientVersion.V2); - } - - - private TcpPackage WrapCheckpointReached(ClientMessage.CheckpointReached msg) - { - var dto = new CheckpointReached(msg.Position.Value.CommitPosition, - msg.Position.Value.PreparePosition); - return new TcpPackage(TcpCommand.CheckpointReached, msg.CorrelationId, dto.Serialize()); - } - - private static Message UnwrapPing(TcpPackage package, IEnvelope envelope) - { - var data = new byte[package.Data.Count]; - Buffer.BlockCopy(package.Data.Array, package.Data.Offset, data, 0, package.Data.Count); - var pongMessage = new TcpMessage.PongMessage(package.CorrelationId, data); - envelope.ReplyWith(pongMessage); - return pongMessage; - } - - private static Message UnwrapIdentifyClient(TcpPackage package, IEnvelope envelope) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - return new ClientMessage.IdentifyClient(package.CorrelationId, dto.Version, dto.ConnectionName); - } - - private static TcpPackage WrapPong(TcpMessage.PongMessage message) - { - return new TcpPackage(TcpCommand.Pong, message.CorrelationId, message.Payload); - } - - private static ClientMessage.ReadEvent - UnwrapReadEvent(TcpPackage package, IEnvelope envelope, ClaimsPrincipal user) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - return new ClientMessage.ReadEvent(Guid.NewGuid(), package.CorrelationId, envelope, dto.EventStreamId, - dto.EventNumber, dto.ResolveLinkTos, dto.RequireLeader, user); - } - - private static TcpPackage WrapReadEventCompleted(ClientMessage.ReadEventCompleted msg) - { - var dto = new ReadEventCompleted( - (ReadEventCompleted.Types.ReadEventResult)msg.Result, - new ResolvedIndexedEvent(msg.Record.Event, msg.Record.Link), msg.Error); - return new TcpPackage(TcpCommand.ReadEventCompleted, msg.CorrelationId, dto.Serialize()); - } - - private static ClientMessage.ReadStreamEventsForward UnwrapReadStreamEventsForward(TcpPackage package, - IEnvelope envelope, ClaimsPrincipal user) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - return new ClientMessage.ReadStreamEventsForward(Guid.NewGuid(), package.CorrelationId, envelope, - dto.EventStreamId, dto.FromEventNumber, dto.MaxCount, - dto.ResolveLinkTos, dto.RequireLeader, null, user, - replyOnExpired: false); - } - - private static TcpPackage WrapReadStreamEventsForwardCompleted( - ClientMessage.ReadStreamEventsForwardCompleted msg) - { - var dto = new ReadStreamEventsCompleted( - ConvertToResolvedIndexedEvents(msg.Events), - (ReadStreamEventsCompleted.Types.ReadStreamResult)msg.Result, - msg.NextEventNumber, msg.LastEventNumber, msg.IsEndOfStream, msg.TfLastCommitPosition, msg.Error); - return new TcpPackage(TcpCommand.ReadStreamEventsForwardCompleted, msg.CorrelationId, dto.Serialize()); - } - - private static ClientMessage.ReadStreamEventsBackward UnwrapReadStreamEventsBackward(TcpPackage package, - IEnvelope envelope, ClaimsPrincipal user) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - return new ClientMessage.ReadStreamEventsBackward(Guid.NewGuid(), package.CorrelationId, envelope, - dto.EventStreamId, dto.FromEventNumber, dto.MaxCount, - dto.ResolveLinkTos, dto.RequireLeader, null, user); - } - - private static TcpPackage WrapReadStreamEventsBackwardCompleted( - ClientMessage.ReadStreamEventsBackwardCompleted msg) - { - var dto = new ReadStreamEventsCompleted( - ConvertToResolvedIndexedEvents(msg.Events), - (ReadStreamEventsCompleted.Types.ReadStreamResult)msg.Result, - msg.NextEventNumber, msg.LastEventNumber, msg.IsEndOfStream, msg.TfLastCommitPosition, msg.Error); - return new TcpPackage(TcpCommand.ReadStreamEventsBackwardCompleted, msg.CorrelationId, dto.Serialize()); - } - - private static ResolvedIndexedEvent[] - ConvertToResolvedIndexedEvents(IReadOnlyList events) - { - var result = new ResolvedIndexedEvent[events.Count]; - for (int i = 0; i < events.Count; ++i) - { - result[i] = new ResolvedIndexedEvent(events[i].Event, events[i].Link); - } - - return result; - } - - private static ClientMessage.ReadAllEventsForward UnwrapReadAllEventsForward(TcpPackage package, - IEnvelope envelope, ClaimsPrincipal user) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - return new ClientMessage.ReadAllEventsForward(Guid.NewGuid(), package.CorrelationId, envelope, - dto.CommitPosition, dto.PreparePosition, dto.MaxCount, - dto.ResolveLinkTos, dto.RequireLeader, null, user, - replyOnExpired: false, - longPollTimeout: null); - } - - - private static TcpPackage WrapReadAllEventsForwardCompleted(ClientMessage.ReadAllEventsForwardCompleted msg) - { - var dto = new ReadAllEventsCompleted( - msg.CurrentPos.CommitPosition, msg.CurrentPos.PreparePosition, ConvertToResolvedEvents(msg.Events), - msg.NextPos.CommitPosition, msg.NextPos.PreparePosition, - (ReadAllEventsCompleted.Types.ReadAllResult)msg.Result, msg.Error); - return new TcpPackage(TcpCommand.ReadAllEventsForwardCompleted, msg.CorrelationId, dto.Serialize()); - } - - private static ClientMessage.ReadAllEventsBackward UnwrapReadAllEventsBackward(TcpPackage package, - IEnvelope envelope, ClaimsPrincipal user) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - return new ClientMessage.ReadAllEventsBackward(Guid.NewGuid(), package.CorrelationId, envelope, - dto.CommitPosition, dto.PreparePosition, dto.MaxCount, - dto.ResolveLinkTos, dto.RequireLeader, null, user); - } - - private static TcpPackage WrapReadAllEventsBackwardCompleted(ClientMessage.ReadAllEventsBackwardCompleted msg) - { - var dto = new ReadAllEventsCompleted( - msg.CurrentPos.CommitPosition, msg.CurrentPos.PreparePosition, ConvertToResolvedEvents(msg.Events), - msg.NextPos.CommitPosition, msg.NextPos.PreparePosition, - (ReadAllEventsCompleted.Types.ReadAllResult)msg.Result, msg.Error); - return new TcpPackage(TcpCommand.ReadAllEventsBackwardCompleted, msg.CorrelationId, dto.Serialize()); - } - - private static ClientMessage.FilteredReadAllEventsForward UnwrapFilteredReadAllEventsForward(TcpPackage package, - IEnvelope envelope, ClaimsPrincipal user) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - IEventFilter eventFilter = EventFilter.Get(true, dto.Filter); - - int maxSearchWindow = dto.MaxCount; - if (dto.MaxSearchWindow > 0) - { - maxSearchWindow = dto.MaxSearchWindow; - } - - return new ClientMessage.FilteredReadAllEventsForward(Guid.NewGuid(), package.CorrelationId, envelope, - dto.CommitPosition, dto.PreparePosition, dto.MaxCount, - dto.ResolveLinkTos, dto.RequireLeader, maxSearchWindow, null, eventFilter, user, - replyOnExpired: false, - longPollTimeout: null); - } - - private static TcpPackage WrapFilteredReadAllEventsForwardCompleted( - ClientMessage.FilteredReadAllEventsForwardCompleted msg) - { - var dto = new FilteredReadAllEventsCompleted( - msg.CurrentPos.CommitPosition, msg.CurrentPos.PreparePosition, ConvertToResolvedEvents(msg.Events), - msg.NextPos.CommitPosition, msg.NextPos.PreparePosition, msg.IsEndOfStream, - (FilteredReadAllEventsCompleted.Types.FilteredReadAllResult)msg.Result, msg.Error); - return new TcpPackage(TcpCommand.FilteredReadAllEventsForwardCompleted, msg.CorrelationId, dto.Serialize()); - } - - private static Client.Messages.ResolvedEvent[] ConvertToResolvedEvents(IReadOnlyList events) - { - var result = new Client.Messages.ResolvedEvent[events.Count]; - for (int i = 0; i < events.Count; ++i) - { - result[i] = new Client.Messages.ResolvedEvent(events[i]); - } - - return result; - } - - private static ClientMessage.FilteredReadAllEventsBackward UnwrapFilteredReadAllEventsBackward( - TcpPackage package, - IEnvelope envelope, ClaimsPrincipal user) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - IEventFilter eventFilter = EventFilter.Get(true, dto.Filter); - - int maxSearchWindow = dto.MaxCount; - if (dto.MaxSearchWindow > 0) - { - maxSearchWindow = dto.MaxSearchWindow; - } - - return new ClientMessage.FilteredReadAllEventsBackward(Guid.NewGuid(), package.CorrelationId, envelope, - dto.CommitPosition, dto.PreparePosition, dto.MaxCount, - dto.ResolveLinkTos, dto.RequireLeader, maxSearchWindow, null, eventFilter, user, null); - } - - private static TcpPackage WrapFilteredReadAllEventsBackwardCompleted( - ClientMessage.FilteredReadAllEventsBackwardCompleted msg) - { - var dto = new FilteredReadAllEventsCompleted( - msg.CurrentPos.CommitPosition, msg.CurrentPos.PreparePosition, ConvertToResolvedEvents(msg.Events), - msg.NextPos.CommitPosition, msg.NextPos.PreparePosition, msg.IsEndOfStream, - (FilteredReadAllEventsCompleted.Types.FilteredReadAllResult)msg.Result, msg.Error); - return new TcpPackage(TcpCommand.FilteredReadAllEventsBackwardCompleted, msg.CorrelationId, - dto.Serialize()); - } - - private ClientMessage.SubscribeToStream UnwrapSubscribeToStream(TcpPackage package, - IEnvelope envelope, - ClaimsPrincipal user, - TcpConnectionManager connection) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - return new ClientMessage.SubscribeToStream(Guid.NewGuid(), package.CorrelationId, envelope, - connection.ConnectionId, dto.EventStreamId, dto.ResolveLinkTos, user); - } - - private ClientMessage.FilteredSubscribeToStream UnwrapFilteredSubscribeToStream(TcpPackage package, - IEnvelope envelope, - ClaimsPrincipal user, - TcpConnectionManager connection) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - IEventFilter eventFilter = EventFilter.Get(dto.EventStreamId.IsEmptyString(), dto.Filter); - - return new ClientMessage.FilteredSubscribeToStream(Guid.NewGuid(), package.CorrelationId, envelope, - connection.ConnectionId, dto.EventStreamId, dto.ResolveLinkTos, user, eventFilter, - dto.CheckpointInterval, checkpointIntervalCurrent: 0); - } - - private ClientMessage.UnsubscribeFromStream UnwrapUnsubscribeFromStream(TcpPackage package, IEnvelope envelope, - ClaimsPrincipal user) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - return new ClientMessage.UnsubscribeFromStream(Guid.NewGuid(), package.CorrelationId, envelope, user); - } - - private TcpPackage WrapSubscribedToStream(ClientMessage.SubscriptionConfirmation msg) - { - var dto = new SubscriptionConfirmation(msg.LastIndexedPosition, msg.LastEventNumber ?? 0); - return new TcpPackage(TcpCommand.SubscriptionConfirmation, msg.CorrelationId, dto.Serialize()); - } - - private ClientMessage.CreatePersistentSubscriptionToStream UnwrapCreatePersistentSubscription( - TcpPackage package, IEnvelope envelope, ClaimsPrincipal user, TcpConnectionManager connection) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - var namedConsumerStrategy = dto.NamedConsumerStrategy; - if (string.IsNullOrEmpty(namedConsumerStrategy)) - { - namedConsumerStrategy = dto.PreferRoundRobin - ? SystemConsumerStrategies.RoundRobin - : SystemConsumerStrategies.DispatchToSingle; - } - - return new ClientMessage.CreatePersistentSubscriptionToStream(Guid.NewGuid(), package.CorrelationId, envelope, - dto.EventStreamId, dto.SubscriptionGroupName, dto.ResolveLinkTos, dto.StartFrom, - dto.MessageTimeoutMilliseconds, - dto.RecordStatistics, dto.MaxRetryCount, dto.BufferSize, dto.LiveBufferSize, - dto.ReadBatchSize, dto.CheckpointAfterTime, dto.CheckpointMinCount, - dto.CheckpointMaxCount, dto.SubscriberMaxCount, namedConsumerStrategy, - user); - } - - private ClientMessage.UpdatePersistentSubscriptionToStream UnwrapUpdatePersistentSubscription( - TcpPackage package, IEnvelope envelope, ClaimsPrincipal user, TcpConnectionManager connection) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - var namedConsumerStrategy = dto.NamedConsumerStrategy; - if (string.IsNullOrEmpty(namedConsumerStrategy)) - { - namedConsumerStrategy = dto.PreferRoundRobin - ? SystemConsumerStrategies.RoundRobin - : SystemConsumerStrategies.DispatchToSingle; - } - - return new ClientMessage.UpdatePersistentSubscriptionToStream(Guid.NewGuid(), package.CorrelationId, envelope, - dto.EventStreamId, dto.SubscriptionGroupName, dto.ResolveLinkTos, dto.StartFrom, - dto.MessageTimeoutMilliseconds, - dto.RecordStatistics, dto.MaxRetryCount, dto.BufferSize, dto.LiveBufferSize, - dto.ReadBatchSize, dto.CheckpointAfterTime, dto.CheckpointMinCount, - dto.CheckpointMaxCount, dto.SubscriberMaxCount, namedConsumerStrategy, - user); - } - - private ClientMessage.DeletePersistentSubscriptionToStream UnwrapDeletePersistentSubscription( - TcpPackage package, IEnvelope envelope, ClaimsPrincipal user, TcpConnectionManager connection) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - return new ClientMessage.DeletePersistentSubscriptionToStream(Guid.NewGuid(), package.CorrelationId, envelope, - dto.EventStreamId, dto.SubscriptionGroupName, user); - } - - private TcpPackage WrapDeletePersistentSubscriptionCompleted( - ClientMessage.DeletePersistentSubscriptionToStreamCompleted msg) - { - var dto = new DeletePersistentSubscriptionCompleted( - (DeletePersistentSubscriptionCompleted.Types.DeletePersistentSubscriptionResult)msg - .Result, msg.Reason); - return new TcpPackage(TcpCommand.DeletePersistentSubscriptionCompleted, msg.CorrelationId, dto.Serialize()); - } - - private TcpPackage WrapCreatePersistentSubscriptionCompleted( - ClientMessage.CreatePersistentSubscriptionToStreamCompleted msg) - { - var dto = new CreatePersistentSubscriptionCompleted( - (CreatePersistentSubscriptionCompleted.Types.CreatePersistentSubscriptionResult)msg - .Result, msg.Reason); - return new TcpPackage(TcpCommand.CreatePersistentSubscriptionCompleted, msg.CorrelationId, dto.Serialize()); - } - - private TcpPackage WrapUpdatePersistentSubscriptionCompleted( - ClientMessage.UpdatePersistentSubscriptionToStreamCompleted msg) - { - var dto = new UpdatePersistentSubscriptionCompleted( - (UpdatePersistentSubscriptionCompleted.Types.UpdatePersistentSubscriptionResult)msg - .Result, msg.Reason); - return new TcpPackage(TcpCommand.UpdatePersistentSubscriptionCompleted, msg.CorrelationId, dto.Serialize()); - } - - - private ClientMessage.ConnectToPersistentSubscriptionToStream UnwrapConnectToPersistentSubscription( - TcpPackage package, IEnvelope envelope, ClaimsPrincipal user, TcpConnectionManager connection) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - return new ClientMessage.ConnectToPersistentSubscriptionToStream(Guid.NewGuid(), package.CorrelationId, - envelope, - connection.ConnectionId, connection.ClientConnectionName, dto.SubscriptionId, dto.EventStreamId, - dto.AllowedInFlightMessages, - connection.RemoteEndPoint.ToString(), user); - } - - private ClientMessage.PersistentSubscriptionAckEvents UnwrapPersistentSubscriptionAckEvents( - TcpPackage package, IEnvelope envelope, ClaimsPrincipal user, TcpConnectionManager connection) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - return new ClientMessage.PersistentSubscriptionAckEvents( - Guid.NewGuid(), package.CorrelationId, envelope, dto.SubscriptionId, - dto.ProcessedEventIds.Select(x => new Guid(x.ToByteArray())).ToArray(), user); - } - - private ClientMessage.PersistentSubscriptionNackEvents UnwrapPersistentSubscriptionNackEvents( - TcpPackage package, IEnvelope envelope, ClaimsPrincipal user, TcpConnectionManager connection) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - return new ClientMessage.PersistentSubscriptionNackEvents( - Guid.NewGuid(), package.CorrelationId, envelope, dto.SubscriptionId, - dto.Message, (ClientMessage.PersistentSubscriptionNackEvents.NakAction)dto.Action, - dto.ProcessedEventIds.Select(x => new Guid(x.ToByteArray())).ToArray(), user); - } - - private TcpPackage - WrapPersistentSubscriptionConfirmation(ClientMessage.PersistentSubscriptionConfirmation msg) - { - var dto = new PersistentSubscriptionConfirmation(msg.LastIndexedPosition, - msg.SubscriptionId, msg.LastEventNumber ?? 0); - return new TcpPackage(TcpCommand.PersistentSubscriptionConfirmation, msg.CorrelationId, dto.Serialize()); - } - - private TcpPackage WrapPersistentSubscriptionStreamEventAppeared( - ClientMessage.PersistentSubscriptionStreamEventAppeared msg) - { - var dto = new PersistentSubscriptionStreamEventAppeared( - new ResolvedIndexedEvent(msg.Event.Event, msg.Event.Link), msg.RetryCount); - return new TcpPackage(TcpCommand.PersistentSubscriptionStreamEventAppeared, msg.CorrelationId, - dto.Serialize()); - } - - private TcpPackage WrapStreamEventAppeared(ClientMessage.StreamEventAppeared msg) - { - var dto = new StreamEventAppeared(new Client.Messages.ResolvedEvent(msg.Event)); - return new TcpPackage(TcpCommand.StreamEventAppeared, msg.CorrelationId, dto.Serialize()); - } - - private TcpPackage WrapSubscriptionDropped(ClientMessage.SubscriptionDropped msg) - { - var dto = new SubscriptionDropped( - (SubscriptionDropped.Types.SubscriptionDropReason)msg.Reason); - return new TcpPackage(TcpCommand.SubscriptionDropped, msg.CorrelationId, dto.Serialize()); - } - - private ClientMessage.ScavengeDatabase UnwrapScavengeDatabase(TcpPackage package, IEnvelope envelope, - ClaimsPrincipal user) - { - return new ClientMessage.ScavengeDatabase(envelope, package.CorrelationId, user, 0, 1, null, null, false); - } - - private TcpPackage WrapScavengeDatabaseResponse(Message msg) - { - ScavengeDatabaseResponse.Types.ScavengeResult result; - string scavengeId; - Guid correlationId; - - switch (msg) - { - case ClientMessage.ScavengeDatabaseStartedResponse startedResponse: - result = ScavengeDatabaseResponse.Types.ScavengeResult.Started; - scavengeId = startedResponse.ScavengeId; - correlationId = startedResponse.CorrelationId; - break; - case ClientMessage.ScavengeDatabaseInProgressResponse inProgressResponse: - result = ScavengeDatabaseResponse.Types.ScavengeResult.InProgress; - scavengeId = inProgressResponse.ScavengeId; - correlationId = inProgressResponse.CorrelationId; - break; - case ClientMessage.ScavengeDatabaseUnauthorizedResponse unauthorizedResponse: - result = ScavengeDatabaseResponse.Types.ScavengeResult.Unauthorized; - scavengeId = unauthorizedResponse.ScavengeId; - correlationId = unauthorizedResponse.CorrelationId; - break; - default: - throw new ArgumentOutOfRangeException(); - } - - var dto = new ScavengeDatabaseResponse(result, scavengeId); - return new TcpPackage(TcpCommand.ScavengeDatabaseResponse, correlationId, dto.Serialize()); - } - - private TcpPackage WrapAuthenticated(TcpMessage.Authenticated msg) - { - return new TcpPackage(TcpCommand.Authenticated, msg.CorrelationId, Empty.ByteArray); - } -} diff --git a/src/EventStore.Core/Services/Transport/Tcp/ClientWriteTcpDispatcher.cs b/src/EventStore.Core/Services/Transport/Tcp/ClientWriteTcpDispatcher.cs deleted file mode 100644 index 3b8bf388c8..0000000000 --- a/src/EventStore.Core/Services/Transport/Tcp/ClientWriteTcpDispatcher.cs +++ /dev/null @@ -1,439 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net; -using System.Security.Claims; -using System.Security.Principal; -using System.Threading; -using EventStore.Client.Messages; -using EventStore.Common.Utils; -using EventStore.Core.Authentication; -using EventStore.Core.Authentication.DelegatedAuthentication; -using EventStore.Core.Data; -using EventStore.Core.Helpers; -using EventStore.Core.Messages; -using EventStore.Core.Messaging; -using EventStore.Core.Util; -using Google.Protobuf; -using Microsoft.Diagnostics.Tracing.Parsers.IIS_Trace; -using OperationResult = EventStore.Core.Messages.OperationResult; - -namespace EventStore.Core.Services.Transport.Tcp -{ - public enum ClientVersion : byte - { - V1 = 0, - V2 = 1 - } - - public class ClientWriteTcpDispatcher : TcpDispatcher - { - private readonly TimeSpan _writeTimeout; - - protected ClientWriteTcpDispatcher(TimeSpan writeTimeout) - { - _writeTimeout = writeTimeout; - AddUnwrapper(TcpCommand.WriteEvents, UnwrapWriteEvents, ClientVersion.V2); - AddWrapper(WrapWriteEvents, ClientVersion.V2); - AddUnwrapper(TcpCommand.WriteEventsCompleted, UnwrapWriteEventsCompleted, ClientVersion.V2); - AddWrapper(WrapWriteEventsCompleted, ClientVersion.V2); - - AddUnwrapper(TcpCommand.TransactionStart, UnwrapTransactionStart, ClientVersion.V2); - AddWrapper(WrapTransactionStart, ClientVersion.V2); - AddUnwrapper(TcpCommand.TransactionStartCompleted, UnwrapTransactionStartCompleted, ClientVersion.V2); - AddWrapper(WrapTransactionStartCompleted, ClientVersion.V2); - - AddUnwrapper(TcpCommand.TransactionWrite, UnwrapTransactionWrite, ClientVersion.V2); - AddWrapper(WrapTransactionWrite, ClientVersion.V2); - AddUnwrapper(TcpCommand.TransactionWriteCompleted, UnwrapTransactionWriteCompleted, ClientVersion.V2); - AddWrapper(WrapTransactionWriteCompleted, ClientVersion.V2); - - AddUnwrapper(TcpCommand.TransactionCommit, UnwrapTransactionCommit, ClientVersion.V2); - AddWrapper(WrapTransactionCommit, ClientVersion.V2); - AddUnwrapper(TcpCommand.TransactionCommitCompleted, UnwrapTransactionCommitCompleted, ClientVersion.V2); - AddWrapper(WrapTransactionCommitCompleted, ClientVersion.V2); - - AddUnwrapper(TcpCommand.DeleteStream, UnwrapDeleteStream, ClientVersion.V2); - AddWrapper(WrapDeleteStream, ClientVersion.V2); - AddUnwrapper(TcpCommand.DeleteStreamCompleted, UnwrapDeleteStreamCompleted, ClientVersion.V2); - AddWrapper(WrapDeleteStreamCompleted, ClientVersion.V2); - - AddUnwrapper(TcpCommand.NotHandled, UnwrapNotHandled, ClientVersion.V2); - AddWrapper(WrapNotHandled, ClientVersion.V2); - - AddUnwrapper(TcpCommand.NotAuthenticated, UnwrapNotAuthenticated, ClientVersion.V2); - AddWrapper(WrapNotAuthenticated, ClientVersion.V2); - } - private ClientMessage.WriteEvents UnwrapWriteEvents(TcpPackage package, IEnvelope envelope, - ClaimsPrincipal user) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - var events = new Event[dto.Events.Count]; - for (int i = 0; i < events.Length; ++i) - { - // ReSharper disable PossibleNullReferenceException - var e = dto.Events[i]; - // ReSharper restore PossibleNullReferenceException - events[i] = new Event(new Guid(e.EventId.ToByteArray()), e.EventType, e.DataContentType == 1, e.Data.ToByteArray(), e.Metadata.ToByteArray()); - } - - var cts = new CancellationTokenSource(); - var envelopeWrapper = new CallbackEnvelope(OnMessage); - cts.CancelAfter(_writeTimeout); - - return new ClientMessage.WriteEvents(Guid.NewGuid(), package.CorrelationId, envelopeWrapper, dto.RequireLeader, - dto.EventStreamId, dto.ExpectedVersion, events, user, package.Tokens, cts.Token); - - void OnMessage(Message m) - { - cts.Dispose(); - envelope.ReplyWith(m); - } - } - - private static TcpPackage WrapWriteEvents(ClientMessage.WriteEvents msg) - { - var events = new NewEvent[msg.Events.Length]; - for (int i = 0; i < events.Length; ++i) - { - var e = msg.Events[i]; - events[i] = new NewEvent(e.EventId.ToByteArray(), - e.EventType, - e.IsJson ? 1 : 0, - 0, e.Data, - e.Metadata); - } - - var dto = new WriteEvents(msg.EventStreamId, msg.ExpectedVersion, events, - msg.RequireLeader); - return CreateWriteRequestPackage(TcpCommand.WriteEvents, msg, dto); - } - - private static TcpPackage CreateWriteRequestPackage(TcpCommand command, ClientMessage.WriteRequestMessage msg, - T dto) where T : IMessage - { - // we forwarding with InternalCorrId, not client's CorrelationId!!! - if (msg.User == UserManagement.SystemAccounts.System) - { - return new TcpPackage(command, TcpFlags.TrustedWrite, msg.InternalCorrId, null, null, dto.Serialize()); - } - - foreach (var identity in msg.User.Identities) - { - if (!(identity is DelegatedClaimsIdentity dci)) - { - continue; - } - - var jwtClaim = dci.FindFirst(AuthenticationTokenKeys.Jwt); - if (jwtClaim != null) - { - return new TcpPackage(command, TcpFlags.Authenticated, msg.InternalCorrId, jwtClaim.Value, - dto.Serialize()); - } - - var uidClaim = dci.FindFirst(AuthenticationTokenKeys.Username); - var pwdClaim = dci.FindFirst(AuthenticationTokenKeys.Password); - - if (uidClaim != null && pwdClaim != null) - { - return new TcpPackage(command, TcpFlags.Authenticated, msg.InternalCorrId, uidClaim.Value, - pwdClaim.Value, dto.Serialize()); - } - } - - return msg.Login != null && msg.Password != null - ? new TcpPackage(command, TcpFlags.Authenticated, msg.InternalCorrId, msg.Login, msg.Password, - dto.Serialize()) - : new TcpPackage(command, TcpFlags.None, msg.InternalCorrId, null, null, dto.Serialize()); - } - - private static ClientMessage.WriteEventsCompleted UnwrapWriteEventsCompleted(TcpPackage package, - IEnvelope envelope) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - if (dto.Result == Client.Messages.OperationResult.Success) - { - return new ClientMessage.WriteEventsCompleted(package.CorrelationId, - dto.FirstEventNumber, - dto.LastEventNumber, - dto.PreparePosition, - dto.CommitPosition); - } - - return new ClientMessage.WriteEventsCompleted(package.CorrelationId, - (OperationResult)dto.Result, - dto.Message, - dto.CurrentVersion); - } - - private static TcpPackage WrapWriteEventsCompleted(ClientMessage.WriteEventsCompleted msg) - { - var dto = new WriteEventsCompleted((Client.Messages.OperationResult)msg.Result, - msg.Message, - msg.FirstEventNumber, - msg.LastEventNumber, - msg.PreparePosition, - msg.CommitPosition, - msg.CurrentVersion); - return new TcpPackage(TcpCommand.WriteEventsCompleted, msg.CorrelationId, dto.Serialize()); - } - - private static ClientMessage.TransactionStart UnwrapTransactionStart(TcpPackage package, IEnvelope envelope, - ClaimsPrincipal user) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - return new ClientMessage.TransactionStart(Guid.NewGuid(), package.CorrelationId, envelope, - dto.RequireLeader, - dto.EventStreamId, dto.ExpectedVersion, user, package.Tokens); - } - - private static TcpPackage WrapTransactionStart(ClientMessage.TransactionStart msg) - { - var dto = new TransactionStart(msg.EventStreamId, msg.ExpectedVersion, - msg.RequireLeader); - return CreateWriteRequestPackage(TcpCommand.TransactionStart, msg, dto); - } - - private static ClientMessage.TransactionStartCompleted UnwrapTransactionStartCompleted(TcpPackage package, - IEnvelope envelope) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - return new ClientMessage.TransactionStartCompleted(package.CorrelationId, dto.TransactionId, - (OperationResult)dto.Result, dto.Message); - } - - private static TcpPackage WrapTransactionStartCompleted(ClientMessage.TransactionStartCompleted msg) - { - var dto = new TransactionStartCompleted(msg.TransactionId, - (Client.Messages.OperationResult)msg.Result, msg.Message); - return new TcpPackage(TcpCommand.TransactionStartCompleted, msg.CorrelationId, dto.Serialize()); - } - - private static ClientMessage.TransactionWrite UnwrapTransactionWrite(TcpPackage package, IEnvelope envelope, - ClaimsPrincipal user) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - var events = new Event[dto.Events.Count]; - for (int i = 0; i < events.Length; ++i) - { - // ReSharper disable PossibleNullReferenceException - var e = dto.Events[i]; - // ReSharper restore PossibleNullReferenceException - events[i] = new Event(new Guid(e.EventId.ToByteArray()), e.EventType, e.DataContentType == 1, e.Data.ToByteArray(), e.Metadata.ToByteArray()); - } - - return new ClientMessage.TransactionWrite(Guid.NewGuid(), package.CorrelationId, envelope, - dto.RequireLeader, - dto.TransactionId, events, user, package.Tokens); - } - - private static TcpPackage WrapTransactionWrite(ClientMessage.TransactionWrite msg) - { - var events = new NewEvent[msg.Events.Length]; - for (int i = 0; i < events.Length; ++i) - { - var e = msg.Events[i]; - events[i] = new NewEvent(e.EventId.ToByteArray(), e.EventType, e.IsJson ? 1 : 0, 0, - e.Data, e.Metadata); - } - - var dto = new TransactionWrite(msg.TransactionId, events, msg.RequireLeader); - return CreateWriteRequestPackage(TcpCommand.TransactionWrite, msg, dto); - } - - private static ClientMessage.TransactionWriteCompleted UnwrapTransactionWriteCompleted(TcpPackage package, - IEnvelope envelope) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - return new ClientMessage.TransactionWriteCompleted(package.CorrelationId, dto.TransactionId, - (OperationResult)dto.Result, dto.Message); - } - - private static TcpPackage WrapTransactionWriteCompleted(ClientMessage.TransactionWriteCompleted msg) - { - var dto = new TransactionWriteCompleted(msg.TransactionId, - (Client.Messages.OperationResult)msg.Result, msg.Message); - return new TcpPackage(TcpCommand.TransactionWriteCompleted, msg.CorrelationId, dto.Serialize()); - } - - private static ClientMessage.TransactionCommit UnwrapTransactionCommit(TcpPackage package, IEnvelope envelope, - ClaimsPrincipal user) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - return new ClientMessage.TransactionCommit(Guid.NewGuid(), package.CorrelationId, envelope, - dto.RequireLeader, dto.TransactionId, user, package.Tokens); - } - - private static TcpPackage WrapTransactionCommit(ClientMessage.TransactionCommit msg) - { - var dto = new TransactionCommit(msg.TransactionId, msg.RequireLeader); - return CreateWriteRequestPackage(TcpCommand.TransactionCommit, msg, dto); - } - - private static ClientMessage.TransactionCommitCompleted UnwrapTransactionCommitCompleted(TcpPackage package, - IEnvelope envelope) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - if (dto.Result == Client.Messages.OperationResult.Success) - { - return new ClientMessage.TransactionCommitCompleted(package.CorrelationId, dto.TransactionId, - dto.FirstEventNumber, dto.LastEventNumber, dto.PreparePosition, dto.CommitPosition); - } - - return new ClientMessage.TransactionCommitCompleted(package.CorrelationId, dto.TransactionId, - (OperationResult)dto.Result, dto.Message); - } - - private static TcpPackage WrapTransactionCommitCompleted(ClientMessage.TransactionCommitCompleted msg) - { - var dto = new TransactionCommitCompleted(msg.TransactionId, - (Client.Messages.OperationResult)msg.Result, - msg.Message, msg.FirstEventNumber, msg.LastEventNumber, msg.PreparePosition, msg.CommitPosition); - return new TcpPackage(TcpCommand.TransactionCommitCompleted, msg.CorrelationId, dto.Serialize()); - } - - private ClientMessage.DeleteStream UnwrapDeleteStream(TcpPackage package, IEnvelope envelope, - ClaimsPrincipal user) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - var cts = new CancellationTokenSource(); - var envelopeWrapper = new CallbackEnvelope(OnMessage); - cts.CancelAfter(_writeTimeout); - - return new ClientMessage.DeleteStream(Guid.NewGuid(), package.CorrelationId, envelopeWrapper, dto.RequireLeader, - dto.EventStreamId, dto.ExpectedVersion, dto.HardDelete, user, package.Tokens, cts.Token); - - void OnMessage(Message m) - { - cts.Dispose(); - envelope.ReplyWith(m); - } - } - - private static TcpPackage WrapDeleteStream(ClientMessage.DeleteStream msg) - { - var dto = new DeleteStream(msg.EventStreamId, msg.ExpectedVersion, msg.RequireLeader, - msg.HardDelete); - return CreateWriteRequestPackage(TcpCommand.DeleteStream, msg, dto); - } - - private static ClientMessage.DeleteStreamCompleted UnwrapDeleteStreamCompleted(TcpPackage package, - IEnvelope envelope) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - return new ClientMessage.DeleteStreamCompleted(package.CorrelationId, (OperationResult)dto.Result, - dto.Message, - dto.CurrentVersion, - dto.PreparePosition, - dto.CommitPosition); - } - - private static TcpPackage WrapDeleteStreamCompleted(ClientMessage.DeleteStreamCompleted msg) - { - var dto = new DeleteStreamCompleted((Client.Messages.OperationResult)msg.Result, - msg.Message, - msg.CurrentVersion, - msg.PreparePosition, - msg.CommitPosition); - return new TcpPackage(TcpCommand.DeleteStreamCompleted, msg.CorrelationId, dto.Serialize()); - } - - private static ClientMessage.NotHandled UnwrapNotHandled(TcpPackage package, IEnvelope envelope) - { - var dto = package.Data.Deserialize(); - if (dto == null) - { - return null; - } - - var reason = dto.Reason switch - { - NotHandled.Types.NotHandledReason.NotReady => ClientMessage.NotHandled.Types.NotHandledReason.NotReady, - NotHandled.Types.NotHandledReason.TooBusy => ClientMessage.NotHandled.Types.NotHandledReason.TooBusy, - NotHandled.Types.NotHandledReason.NotLeader => ClientMessage.NotHandled.Types.NotHandledReason.NotLeader, - NotHandled.Types.NotHandledReason.IsReadOnly => ClientMessage.NotHandled.Types.NotHandledReason.IsReadOnly, - _ => ClientMessage.NotHandled.Types.NotHandledReason.NotReady - }; - var leaderInfoDto = dto.AdditionalInfo.IsEmpty - ? null - : dto.AdditionalInfo.ToByteArray().Deserialize(); - - var leaderInfo = leaderInfoDto switch - { - { ExternalTcpAddress.Length: > 0 } => new ClientMessage.NotHandled.Types.LeaderInfo( - new DnsEndPoint(leaderInfoDto.ExternalTcpAddress, leaderInfoDto.ExternalTcpPort), false, - new DnsEndPoint(leaderInfoDto.HttpAddress, leaderInfoDto.HttpPort)), - { ExternalSecureTcpAddress.Length: > 0 } => new ClientMessage.NotHandled.Types.LeaderInfo( - new DnsEndPoint(leaderInfoDto.ExternalSecureTcpAddress, leaderInfoDto.ExternalSecureTcpPort), true, - new DnsEndPoint(leaderInfoDto.HttpAddress, leaderInfoDto.HttpPort)), - _ => null - }; - return new ClientMessage.NotHandled(package.CorrelationId, reason, leaderInfo); - } - - private static TcpPackage WrapNotHandled(ClientMessage.NotHandled msg) - { - var dto = new Client.Messages.NotHandled(msg); - return new TcpPackage(TcpCommand.NotHandled, msg.CorrelationId, dto.Serialize()); - } - - private static TcpMessage.NotAuthenticated UnwrapNotAuthenticated(TcpPackage package, IEnvelope envelope) => - new(package.CorrelationId, - Helper.UTF8NoBom.GetString(package.Data.Array, package.Data.Offset, package.Data.Count)); - - private static TcpPackage WrapNotAuthenticated(TcpMessage.NotAuthenticated msg) => - new(TcpCommand.NotAuthenticated, msg.CorrelationId, - Helper.UTF8NoBom.GetBytes(msg.Reason ?? string.Empty)); - } -} diff --git a/src/EventStore.Core/Services/Transport/Tcp/ITcpDispatcher.cs b/src/EventStore.Core/Services/Transport/Tcp/ITcpDispatcher.cs deleted file mode 100644 index d629eb6ab7..0000000000 --- a/src/EventStore.Core/Services/Transport/Tcp/ITcpDispatcher.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System.Collections.Generic; -using System.Security.Claims; -using EventStore.Core.Messaging; - -namespace EventStore.Core.Services.Transport.Tcp -{ - public interface ITcpDispatcher - { - TcpPackage? WrapMessage(Message message, byte version); - - Message UnwrapPackage(TcpPackage package, IEnvelope envelope, ClaimsPrincipal user, - IReadOnlyDictionary tokens, TcpConnectionManager connection, byte version); - } -} diff --git a/src/EventStore.Core/Services/Transport/Tcp/ProtobufExtensions.cs b/src/EventStore.Core/Services/Transport/Tcp/ProtobufExtensions.cs deleted file mode 100644 index 4a2e7bc89e..0000000000 --- a/src/EventStore.Core/Services/Transport/Tcp/ProtobufExtensions.cs +++ /dev/null @@ -1,112 +0,0 @@ -using System; -using System.Collections.Concurrent; -using System.IO; -using System.Threading; -using Google.Protobuf; -using ILogger = Serilog.ILogger; - -namespace EventStore.Core.Services.Transport.Tcp -{ - public static class ProtobufExtensions - { - private static readonly ConcurrentStack _streams; - - static ProtobufExtensions() - { - _streams = new ConcurrentStack(); - for (var i = 0; i < 300; i++) - { - _streams.Push(new MemoryStream(2048)); - } - } - - private static readonly ILogger Log = - Serilog.Log.ForContext(Serilog.Core.Constants.SourceContextPropertyName, "ProtobufExtensions"); - - static MemoryStream AcquireStream() - { - for (var i = 0; i < 1000; i++) - { - MemoryStream ret; - if (_streams.TryPop(out ret)) - { - ret.SetLength(0); - return ret; - } - - if ((i + 1) % 5 == 0) - { - Thread.Sleep(1); //need to do better than this - } - } - - throw new UnableToAcquireStreamException(); - } - - static void ReleaseStream(MemoryStream stream) - { - _streams.Push(stream); - } - - public static T Deserialize(this byte[] data) where T : IMessage, new() - { - return Deserialize(new ArraySegment(data)); - } - - public static T Deserialize(this ArraySegment data) where T : IMessage, new() - { - try - { - using (var memory = new MemoryStream(data.Array, data.Offset, data.Count) - ) //uses original buffer as memory - { - var res = new T(); - res.MergeFrom(memory); - return res; - } - } - catch (Exception e) - { - Log.Information(e, "Deserialization to {type} failed", typeof(T).FullName); - return default(T); - } - } - - public static ArraySegment Serialize(this T protoContract) where T : IMessage - { - MemoryStream stream = null; - try - { - stream = AcquireStream(); - protoContract.WriteTo(stream); - var res = new ArraySegment(stream.ToArray(), 0, (int)stream.Length); - return res; - } - finally - { - if (stream != null) - { - ReleaseStream(stream); - } - } - } - - public static byte[] SerializeToArray(this T protoContract) where T : IMessage - { - MemoryStream stream = null; - try - { - stream = AcquireStream(); - protoContract.WriteTo(stream); - return stream.ToArray(); - } - finally - { - if (stream != null) - { - ReleaseStream(stream); - } - } - } - } -} diff --git a/src/EventStore.Core/Services/Transport/Tcp/SendOverTcpEnvelope.cs b/src/EventStore.Core/Services/Transport/Tcp/SendOverTcpEnvelope.cs deleted file mode 100644 index 2eeb6d6c87..0000000000 --- a/src/EventStore.Core/Services/Transport/Tcp/SendOverTcpEnvelope.cs +++ /dev/null @@ -1,29 +0,0 @@ -using EventStore.Common.Utils; -using EventStore.Core.Bus; -using EventStore.Core.Messages; -using EventStore.Core.Messaging; - -namespace EventStore.Core.Services.Transport.Tcp -{ - public class SendOverTcpEnvelope : IEnvelope - { - private readonly IPublisher _networkSendQueue; - private readonly TcpConnectionManager _manager; - - public SendOverTcpEnvelope(TcpConnectionManager manager, IPublisher networkSendQueue) - { - Ensure.NotNull(manager, "manager"); - Ensure.NotNull(networkSendQueue, "networkSendQueue"); - _networkSendQueue = networkSendQueue; - _manager = manager; - } - - public void ReplyWith(T message) where T : Message - { - if (_manager != null && !_manager.IsClosed) - { - _networkSendQueue.Publish(new TcpMessage.TcpSend(_manager, message)); - } - } - } -} diff --git a/src/EventStore.Core/Services/Transport/Tcp/TcpCommand.cs b/src/EventStore.Core/Services/Transport/Tcp/TcpCommand.cs deleted file mode 100644 index d36a90d10e..0000000000 --- a/src/EventStore.Core/Services/Transport/Tcp/TcpCommand.cs +++ /dev/null @@ -1,73 +0,0 @@ -namespace EventStore.Core.Services.Transport.Tcp -{ - public enum TcpCommand : byte - { - HeartbeatRequestCommand = 0x01, - HeartbeatResponseCommand = 0x02, - - Ping = 0x03, - Pong = 0x04, - - // CLIENT COMMANDS - // CreateStream = 0x80, - // CreateStreamCompleted = 0x81, - - WriteEvents = 0x82, - WriteEventsCompleted = 0x83, - - TransactionStart = 0x84, - TransactionStartCompleted = 0x85, - TransactionWrite = 0x86, - TransactionWriteCompleted = 0x87, - TransactionCommit = 0x88, - TransactionCommitCompleted = 0x89, - - DeleteStream = 0x8A, - DeleteStreamCompleted = 0x8B, - - ReadEvent = 0xB0, - ReadEventCompleted = 0xB1, - ReadStreamEventsForward = 0xB2, - ReadStreamEventsForwardCompleted = 0xB3, - ReadStreamEventsBackward = 0xB4, - ReadStreamEventsBackwardCompleted = 0xB5, - ReadAllEventsForward = 0xB6, - ReadAllEventsForwardCompleted = 0xB7, - ReadAllEventsBackward = 0xB8, - ReadAllEventsBackwardCompleted = 0xB9, - FilteredReadAllEventsForward = 0xBA, - FilteredReadAllEventsForwardCompleted = 0xBB, - FilteredReadAllEventsBackward = 0xBC, - FilteredReadAllEventsBackwardCompleted = 0xBE, - - SubscribeToStream = 0xC0, - SubscriptionConfirmation = 0xC1, - StreamEventAppeared = 0xC2, - UnsubscribeFromStream = 0xC3, - SubscriptionDropped = 0xC4, - ConnectToPersistentSubscription = 0xC5, - PersistentSubscriptionConfirmation = 0xC6, - PersistentSubscriptionStreamEventAppeared = 0xC7, - CreatePersistentSubscription = 0xC8, - CreatePersistentSubscriptionCompleted = 0xC9, - DeletePersistentSubscription = 0xCA, - DeletePersistentSubscriptionCompleted = 0xCB, - PersistentSubscriptionAckEvents = 0xCC, - PersistentSubscriptionNakEvents = 0xCD, - UpdatePersistentSubscription = 0xCE, - UpdatePersistentSubscriptionCompleted = 0xCF, - FilteredSubscribeToStream = 0xD2, - CheckpointReached = 0xD3, - - ScavengeDatabase = 0xD0, - ScavengeDatabaseResponse = 0xD1, - - BadRequest = 0xF0, - NotHandled = 0xF1, - Authenticate = 0xF2, - Authenticated = 0xF3, - NotAuthenticated = 0xF4, - IdentifyClient = 0xF5, - ClientIdentified = 0xF6, - } -} diff --git a/src/EventStore.Core/Services/Transport/Tcp/TcpConnectionManager.cs b/src/EventStore.Core/Services/Transport/Tcp/TcpConnectionManager.cs deleted file mode 100644 index 035825907b..0000000000 --- a/src/EventStore.Core/Services/Transport/Tcp/TcpConnectionManager.cs +++ /dev/null @@ -1,682 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Net; -using System.Net.Security; -using System.Net.Sockets; -using System.Security.Claims; -using System.Security.Cryptography.X509Certificates; -using System.Threading; -using EventStore.Client.Messages; -using EventStore.Common.Utils; -using EventStore.Core.Bus; -using EventStore.Core.Messages; -using EventStore.Core.Messaging; -using EventStore.Core.Services.TimerService; -using EventStore.Core.Settings; -using EventStore.Plugins.Authentication; -using EventStore.Transport.Tcp; -using EventStore.Transport.Tcp.Framing; -using ILogger = Serilog.ILogger; - -namespace EventStore.Core.Services.Transport.Tcp -{ - /// - /// Manager for individual TCP connection. It handles connection lifecycle, - /// heartbeats, message framing and dispatch to the memory bus. - /// - public class TcpConnectionManager : IHandle, IHandle - { - public static readonly TimeSpan ConnectionTimeout = TimeSpan.FromMilliseconds(1000); - - private static readonly ILogger Log = Serilog.Log.ForContext(); - private static readonly ClaimsPrincipal Anonymous = new ClaimsPrincipal(new ClaimsIdentity(new[] { new Claim(ClaimTypes.Anonymous, ""), })); - public readonly Guid ConnectionId; - public readonly string ConnectionName; - public readonly EndPoint RemoteEndPoint; - - public EndPoint LocalEndPoint - { - get { return _connection.LocalEndPoint; } - } - - public bool IsClosed - { - get { return _isClosed != 0; } - } - - public int SendQueueSize - { - get { return _connection.SendQueueSize; } - } - - public string ClientConnectionName - { - get { return _clientConnectionName; } - } - - private readonly ITcpConnection _connection; - private readonly IEnvelope _tcpEnvelope; - private readonly IPublisher _publisher; - private readonly ITcpDispatcher _dispatcher; - private readonly IMessageFramer> _framer; - private long _receiveProgressIndicator => _connection?.TotalBytesReceived ?? 0L; - private long _sendProgressIndicator => _connection?.TotalBytesSent ?? 0L; - private int _isClosed; - private string _clientConnectionName; - - private byte _version; - - private readonly Action _connectionClosed; - private readonly Action _connectionEstablished; - - private readonly SendToWeakThisEnvelope _weakThisEnvelope; - private readonly TimeSpan _heartbeatInterval; - private readonly TimeSpan _heartbeatTimeout; - private bool _awaitingHeartbeatTimeoutCheck; - private readonly int _connectionPendingSendBytesThreshold; - private readonly int _connectionQueueSizeThreshold; - - private readonly IAuthenticationProvider _authProvider; - private readonly AuthorizationGateway _authorization; - private UserCredentials _defaultUser; - private TcpServiceType _serviceType; - - public TcpConnectionManager(string connectionName, - TcpServiceType serviceType, - ITcpDispatcher dispatcher, - IPublisher publisher, - ITcpConnection openedConnection, - IPublisher networkSendQueue, - IAuthenticationProvider authProvider, - AuthorizationGateway authorization, - TimeSpan heartbeatInterval, - TimeSpan heartbeatTimeout, - Action onConnectionClosed, - int connectionPendingSendBytesThreshold, - int connectionQueueSizeThreshold) - { - Ensure.NotNull(dispatcher, "dispatcher"); - Ensure.NotNull(publisher, "publisher"); - Ensure.NotNull(openedConnection, "openedConnnection"); - Ensure.NotNull(networkSendQueue, "networkSendQueue"); - Ensure.NotNull(authProvider, "authProvider"); - Ensure.NotNull(authorization, "authorization"); - ConnectionId = openedConnection.ConnectionId; - ConnectionName = connectionName; - - _serviceType = serviceType; - _tcpEnvelope = new SendOverTcpEnvelope(this, networkSendQueue); - _publisher = publisher; - _dispatcher = dispatcher; - _authProvider = authProvider; - _authorization = authorization; - - _framer = new LengthPrefixMessageFramer(); - _framer.RegisterMessageArrivedCallback(OnMessageArrived); - - _weakThisEnvelope = new SendToWeakThisEnvelope(this); - _heartbeatInterval = heartbeatInterval; - _heartbeatTimeout = heartbeatTimeout; - _connectionPendingSendBytesThreshold = connectionPendingSendBytesThreshold; - _connectionQueueSizeThreshold = connectionQueueSizeThreshold; - - _connectionClosed = onConnectionClosed; - - RemoteEndPoint = openedConnection.RemoteEndPoint; - _connection = openedConnection; - _connection.ConnectionClosed += OnConnectionClosed; - if (_connection.IsClosed) - { - OnConnectionClosed(_connection, SocketError.Success); - return; - } - - ScheduleHeartbeat(0L, 0L); - } - - public TcpConnectionManager(string connectionName, - Guid connectionId, - ITcpDispatcher dispatcher, - IPublisher publisher, - string targetHost, - string[] otherNames, - EndPoint remoteEndPoint, - TcpClientConnector connector, - bool useSsl, - CertificateDelegates.ServerCertificateValidator sslServerCertValidator, - Func sslClientCertificatesSelector, - IPublisher networkSendQueue, - IAuthenticationProvider authProvider, - AuthorizationGateway authorization, - TimeSpan heartbeatInterval, - TimeSpan heartbeatTimeout, - Action onConnectionEstablished, - Action onConnectionClosed) - { - Ensure.NotEmptyGuid(connectionId, "connectionId"); - Ensure.NotNull(dispatcher, "dispatcher"); - Ensure.NotNull(publisher, "publisher"); - Ensure.NotNull(authProvider, "authProvider"); - Ensure.NotNull(authorization, "authorization"); - Ensure.NotNull(remoteEndPoint, "remoteEndPoint"); - Ensure.NotNull(connector, "connector"); - - ConnectionId = connectionId; - ConnectionName = connectionName; - - _tcpEnvelope = new SendOverTcpEnvelope(this, networkSendQueue); - _publisher = publisher; - _dispatcher = dispatcher; - _authProvider = authProvider; - _authorization = authorization; - - _framer = new LengthPrefixMessageFramer(); - _framer.RegisterMessageArrivedCallback(OnMessageArrived); - - _weakThisEnvelope = new SendToWeakThisEnvelope(this); - _heartbeatInterval = heartbeatInterval; - _heartbeatTimeout = heartbeatTimeout; - _connectionPendingSendBytesThreshold = ESConsts.UnrestrictedPendingSendBytes; - _connectionQueueSizeThreshold = ESConsts.MaxConnectionQueueSize; - - _connectionEstablished = onConnectionEstablished; - _connectionClosed = onConnectionClosed; - - RemoteEndPoint = remoteEndPoint; - _connection = useSsl - ? connector.ConnectSslTo(ConnectionId, targetHost, otherNames, remoteEndPoint.ResolveDnsToIPAddress(), ConnectionTimeout, - sslServerCertValidator, sslClientCertificatesSelector, OnConnectionEstablished, OnConnectionFailed) - : connector.ConnectTo(ConnectionId, remoteEndPoint.ResolveDnsToIPAddress(), ConnectionTimeout, OnConnectionEstablished, - OnConnectionFailed); - _connection.ConnectionClosed += OnConnectionClosed; - if (_connection.IsClosed) - { - OnConnectionClosed(_connection, SocketError.Success); - } - } - - private void OnConnectionEstablished(ITcpConnection connection) - { - Log.Information("Connection '{connectionName}' ({connectionId:B}) to [{remoteEndPoint}] established.", - ConnectionName, ConnectionId, connection.RemoteEndPoint); - - ScheduleHeartbeat(0L, 0L); - - var handler = _connectionEstablished; - if (handler != null) - { - handler(this); - } - } - - private void OnConnectionFailed(ITcpConnection connection, SocketError socketError) - { - if (Interlocked.CompareExchange(ref _isClosed, 1, 0) != 0) - { - return; - } - - Log.Information("Connection '{connectionName}' ({connectionId:B}) to [{remoteEndPoint}] failed: {e}.", - ConnectionName, ConnectionId, connection.RemoteEndPoint, socketError); - if (_connectionClosed != null) - { - _connectionClosed(this, socketError); - } - } - - private void OnConnectionClosed(ITcpConnection connection, SocketError socketError) - { - if (Interlocked.CompareExchange(ref _isClosed, 1, 0) != 0) - { - return; - } - - Log.Information( - "Connection '{connectionName}{clientConnectionName}' [{remoteEndPoint}, {connectionId:B}] closed: {e}.", - ConnectionName, ClientConnectionName.IsEmptyString() ? string.Empty : ":" + ClientConnectionName, - connection.RemoteEndPoint, ConnectionId, socketError); - - if (_serviceType == TcpServiceType.Internal && _connection.TotalBytesSent == 0) - { - Log.Warning( - "Unable to connect to Remote EndPoint : {remoteEndPoint}. Connection is potentially blocked. Total bytes sent: {TotalBytesSent}, Total bytes received: {TotalBytesReceived}", - RemoteEndPoint, _connection.TotalBytesSent, _connection.TotalBytesReceived); - } - - if (_connectionClosed != null) - { - _connectionClosed(this, socketError); - } - } - - public void StartReceiving() - { - _connection.ReceiveAsync(OnRawDataReceived); - } - - private void OnRawDataReceived(ITcpConnection connection, IEnumerable> data) - { - try - { - _framer.UnFrameData(data); - } - catch (PackageFramingException exc) - { - SendBadRequestAndClose(Guid.Empty, - string.Format("Invalid TCP frame received. Error: {0}.", exc.Message)); - return; - } - - _connection.ReceiveAsync(OnRawDataReceived); - } - - private void OnMessageArrived(ArraySegment data) - { - TcpPackage package; - try - { - package = TcpPackage.FromArraySegment(data); - } - catch (Exception e) - { - SendBadRequestAndClose(Guid.Empty, string.Format("Received bad network package. Error: {0}", e)); - return; - } - - OnPackageReceived(package); - } - - private void OnPackageReceived(TcpPackage package) - { - try - { - ProcessPackage(package); - } - catch (Exception e) - { - SendBadRequestAndClose(package.CorrelationId, - string.Format("Error while processing package. Error: {0}", e)); - } - } - - public void ProcessPackage(TcpPackage package) - { - if (_serviceType == TcpServiceType.External && (package.Flags & TcpFlags.TrustedWrite) != 0) - { - SendBadRequestAndClose(package.CorrelationId, - "Trusted writes aren't accepted over the external TCP interface"); - return; - } - - switch (package.Command) - { - case TcpCommand.HeartbeatResponseCommand: - break; - case TcpCommand.HeartbeatRequestCommand: - SendPackage(new TcpPackage(TcpCommand.HeartbeatResponseCommand, package.CorrelationId, null)); - break; - case TcpCommand.IdentifyClient: - { - try - { - var message = - (ClientMessage.IdentifyClient)_dispatcher.UnwrapPackage(package, _tcpEnvelope, null, null, - this, _version); - Log.Information( - "Connection '{connectionName}' ({connectionId:B}) identified by client. Client connection name: '{clientConnectionName}', Client version: {clientVersion}.", - ConnectionName, ConnectionId, message.ConnectionName, (ClientVersion)message.Version); - _version = (byte)message.Version; - _clientConnectionName = message.ConnectionName; - _connection.SetClientConnectionName(_clientConnectionName); - SendPackage(new TcpPackage(TcpCommand.ClientIdentified, package.CorrelationId, null)); - } - catch (Exception ex) - { - Log.Error("Error identifying client: {e}", ex); - } - - break; - } - case TcpCommand.BadRequest: - { - var reason = string.Empty; - Helper.EatException(() => - reason = Helper.UTF8NoBom.GetString(package.Data.Array, package.Data.Offset, - package.Data.Count)); - Log.Error( - "Bad request received from '{connectionName}{clientConnectionName}' [{remoteEndPoint}, L{localEndPoint}, {connectionId:B}]. CorrelationId: {correlationId:B}, Error: {e}.", - ConnectionName, - ClientConnectionName.IsEmptyString() ? string.Empty : ":" + ClientConnectionName, - RemoteEndPoint, - LocalEndPoint, ConnectionId, package.CorrelationId, - reason.IsEmptyString() ? "" : reason); - break; - } - case TcpCommand.Authenticate: - { - if ((package.Flags & TcpFlags.Authenticated) == 0) - { - ReplyNotAuthenticated(package.CorrelationId, "No user credentials provided."); - } - else - { - var defaultUser = new UserCredentials(package.Tokens, null); - Interlocked.Exchange(ref _defaultUser, defaultUser); - _authProvider.Authenticate(new TcpDefaultAuthRequest(this, package.CorrelationId, defaultUser)); - } - - break; - } - default: - { - var defaultUser = _defaultUser; - if ((package.Flags & TcpFlags.TrustedWrite) != 0) - { - UnwrapAndPublishPackage(package, UserManagement.SystemAccounts.System, null); - } - else if ((package.Flags & TcpFlags.Authenticated) != 0) - { - _authProvider.Authenticate(new TcpAuthRequest(this, package)); - } - else if (defaultUser != null) - { - if (defaultUser.User != null) - { - UnwrapAndPublishPackage(package, defaultUser.User, defaultUser.Tokens); - } - else - { - // The default credentials aren't authenticated - ReplyNotAuthenticated(package.CorrelationId, "Not Authenticated"); - } - } - else - { - UnwrapAndPublishPackage(package, Anonymous, null); - } - - break; - } - } - } - - private void UnwrapAndPublishPackage(TcpPackage package, ClaimsPrincipal user, IReadOnlyDictionary tokens) - { - Message message = null; - string error = ""; - try - { - message = _dispatcher.UnwrapPackage(package, _tcpEnvelope, user, tokens, this, _version); - } - catch (Exception ex) - { - error = ex.Message; - } - - if (message != null) - { - _authorization.Authorize(message, _publisher); - } - else - { - SendBadRequest(package.CorrelationId, - string.Format("Could not unwrap network package for command {0}.\n{1}", package.Command, error)); - } - } - - private void ReplyNotAuthenticated(Guid correlationId, string description) - { - _tcpEnvelope.ReplyWith(new TcpMessage.NotAuthenticated(correlationId, description)); - } - - private void ReplyNotReady(Guid correlationId, string description) - { - _tcpEnvelope.ReplyWith(new ClientMessage.NotHandled(correlationId, - ClientMessage.NotHandled.Types.NotHandledReason.NotReady, description)); - } - - private void ReplyAuthenticated(Guid correlationId, UserCredentials userCredentials, ClaimsPrincipal user) - { - var authCredentials = new UserCredentials(userCredentials.Tokens, user); - Interlocked.CompareExchange(ref _defaultUser, authCredentials, userCredentials); - _tcpEnvelope.ReplyWith(new TcpMessage.Authenticated(correlationId)); - } - - public void SendBadRequestAndClose(Guid correlationId, string message) - { - Ensure.NotNull(message, "message"); - - SendPackage(new TcpPackage(TcpCommand.BadRequest, correlationId, Helper.UTF8NoBom.GetBytes(message)), - checkQueueSize: false); - Log.Error( - "Closing connection '{connectionName}{clientConnectionName}' [{remoteEndPoint}, L{localEndPoint}, {connectionId:B}] due to error. Reason: {e}", - ConnectionName, ClientConnectionName.IsEmptyString() ? string.Empty : ":" + ClientConnectionName, - RemoteEndPoint, LocalEndPoint, ConnectionId, message); - _connection.Close(message); - } - - public void SendBadRequest(Guid correlationId, string message) - { - Ensure.NotNull(message, "message"); - - SendPackage(new TcpPackage(TcpCommand.BadRequest, correlationId, Helper.UTF8NoBom.GetBytes(message)), - checkQueueSize: false); - } - - public void Stop(string reason = null) - { - Log.Debug( - "Closing connection '{connectionName}{clientConnectionName}' [{remoteEndPoint}, L{localEndPoint}, {connectionId:B}] cleanly.{reason}", - ConnectionName, ClientConnectionName.IsEmptyString() ? string.Empty : ":" + ClientConnectionName, - RemoteEndPoint, LocalEndPoint, ConnectionId, - reason.IsEmpty() ? string.Empty : " Reason: " + reason); - _connection.Close(reason); - } - - public void SendMessage(Message message) - { - var package = _dispatcher.WrapMessage(message, _version); - if (package != null) - { - SendPackage(package.Value); - } - } - - private void SendPackage(TcpPackage package, bool checkQueueSize = true) - { - if (IsClosed) - { - return; - } - - int queueSize; - int queueSendBytes; - if (checkQueueSize) - { - if ((queueSize = _connection.SendQueueSize) > _connectionQueueSizeThreshold) - { - SendBadRequestAndClose(Guid.Empty, - string.Format("Connection queue size is too large: {0}.", queueSize)); - return; - } - - if (_connectionPendingSendBytesThreshold > ESConsts.UnrestrictedPendingSendBytes && - (queueSendBytes = _connection.PendingSendBytes) > _connectionPendingSendBytesThreshold) - { - SendBadRequestAndClose(Guid.Empty, - string.Format("Connection pending send bytes is too large: {0}.", queueSendBytes)); - return; - } - } - - var data = package.AsArraySegment(); - var framed = _framer.FrameData(data); - _connection.EnqueueSend(framed); - } - - public void Handle(TcpMessage.Heartbeat message) - { - if (IsClosed) - { - return; - } - - var receiveProgressIndicator = _receiveProgressIndicator; - var sendProgressIndicator = _sendProgressIndicator; - - /* - * If we have not received any data within the heartbeat interval and we are not waiting for a heartbeat timeout check, then we send a heartbeat request. - */ - if (message.ReceiveProgressIndicator == receiveProgressIndicator && !_awaitingHeartbeatTimeoutCheck) - { - SendPackage(new TcpPackage(TcpCommand.HeartbeatRequestCommand, Guid.NewGuid(), null)); - _awaitingHeartbeatTimeoutCheck = true; - _publisher.Publish(TimerMessage.Schedule.Create(_heartbeatTimeout, _weakThisEnvelope, - new TcpMessage.HeartbeatTimeout(receiveProgressIndicator))); - } - /* - * As a proactive measure, if we have not sent any data to the remote party within the heartbeat interval, - * we also send a heartbeat request just to generate some data for the remote party so that it can clear its heartbeat timeouts. - * This is particularly useful when the remote party's heartbeat request is stuck behind a large (multi-megabyte) TCP message. - */ - else if (message.SendProgressIndicator == sendProgressIndicator) - { - SendPackage(new TcpPackage(TcpCommand.HeartbeatRequestCommand, Guid.NewGuid(), null)); - Log.Verbose( - "Connection '{connectionName}{clientConnectionName}' [{remoteEndPoint}, {connectionId:B}] Proactive heartbeat request sent to the remote party since no data was sent during the last heartbeat interval.", - ConnectionName, ClientConnectionName.IsEmptyString() ? string.Empty : ":" + ClientConnectionName, - _connection.RemoteEndPoint, ConnectionId); - } - - //schedule the next heartbeat regardless of whether or not there's an active heartbeat request - ScheduleHeartbeat(receiveProgressIndicator, sendProgressIndicator); - } - - public void Handle(TcpMessage.HeartbeatTimeout message) - { - _awaitingHeartbeatTimeoutCheck = false; - if (IsClosed) - { - return; - } - - var receiveProgressIndicator = _receiveProgressIndicator; - var sendProgressIndicator = _sendProgressIndicator; - if (message.ReceiveProgressIndicator == receiveProgressIndicator) - { - Stop(string.Format($"HEARTBEAT TIMEOUT at receiveProgressIndicator={receiveProgressIndicator}, sendProgressIndicator={sendProgressIndicator}")); - } - } - - private void ScheduleHeartbeat(long receiveProgressIndicator, long sendProgressIndicator) - { - _publisher.Publish(TimerMessage.Schedule.Create(_heartbeatInterval, _weakThisEnvelope, - new TcpMessage.Heartbeat(receiveProgressIndicator, sendProgressIndicator))); - } - - // Same health warnings as SendToThisEnvelope - private class SendToWeakThisEnvelope : IEnvelope - { - private readonly WeakReference _receiver; - - public SendToWeakThisEnvelope(object receiver) - { - _receiver = new WeakReference(receiver); - } - - public void ReplyWith(T message) where T : Message - { - if (_receiver.Target is IHandle handle) - { - handle.Handle(message); - } - else if (_receiver is IAsyncHandle) - { - throw new Exception($"SendToWeakThisEnvelope does not support asynchronous receivers. Receiver: {_receiver}"); - } - } - } - - private class TcpAuthRequest : AuthenticationRequest - { - private readonly TcpConnectionManager _manager; - private readonly TcpPackage _package; - - public TcpAuthRequest(TcpConnectionManager manager, TcpPackage package) - : base($"(TCP) {manager.RemoteEndPoint}", package.Tokens) - { - _manager = manager; - _package = package; - } - - public override void Unauthorized() - { - _manager.ReplyNotAuthenticated(_package.CorrelationId, "Not Authenticated"); - } - - public override void Authenticated(ClaimsPrincipal principal) - { - _manager.UnwrapAndPublishPackage(_package, principal, _package.Tokens); - } - - public override void Error() - { - _manager.ReplyNotAuthenticated(_package.CorrelationId, "Internal Server Error"); - } - - public override void NotReady() - { - _manager.ReplyNotReady(_package.CorrelationId, "Server not ready"); - } - } - - - private class TcpDefaultAuthRequest : AuthenticationRequest - { - private readonly TcpConnectionManager _manager; - private readonly Guid _correlationId; - private readonly UserCredentials _userCredentials; - - public TcpDefaultAuthRequest(TcpConnectionManager manager, Guid correlationId, - UserCredentials userCredentials) - : base($"(TCP) {manager.RemoteEndPoint}", userCredentials.Tokens) - { - _manager = manager; - _correlationId = correlationId; - _userCredentials = userCredentials; - } - - public override void Unauthorized() - { - _manager.ReplyNotAuthenticated(_correlationId, "Unauthorized"); - } - - public override void Authenticated(ClaimsPrincipal principal) - { - _manager.ReplyAuthenticated(_correlationId, _userCredentials, principal); - } - - public override void Error() - { - _manager.ReplyNotAuthenticated(_correlationId, "Internal Server Error"); - } - - public override void NotReady() - { - _manager.ReplyNotReady(_correlationId, "Server not yet ready"); - } - } - - private class UserCredentials - { - public readonly ClaimsPrincipal User; - public readonly IReadOnlyDictionary Tokens; - - public UserCredentials(IReadOnlyDictionary tokens, ClaimsPrincipal user) - { - Tokens = tokens; - User = user; - } - } - } -} diff --git a/src/EventStore.Core/Services/Transport/Tcp/TcpDispatcher.cs b/src/EventStore.Core/Services/Transport/Tcp/TcpDispatcher.cs deleted file mode 100644 index b62ae907d0..0000000000 --- a/src/EventStore.Core/Services/Transport/Tcp/TcpDispatcher.cs +++ /dev/null @@ -1,134 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Security.Claims; -using EventStore.Core.Messaging; -using ILogger = Serilog.ILogger; - -namespace EventStore.Core.Services.Transport.Tcp -{ - public abstract class TcpDispatcher : ITcpDispatcher - { - private static readonly ILogger Log = Serilog.Log.ForContext(); - - private readonly Func, TcpConnectionManager, Message>[][] - _unwrappers; - - private readonly IDictionary>[] _wrappers; - - protected TcpDispatcher() - { - _unwrappers = - new Func, TcpConnectionManager, Message>[2][]; - _unwrappers[0] = - new Func, TcpConnectionManager, Message>[255]; - _unwrappers[1] = - new Func, TcpConnectionManager, Message>[255]; - - _wrappers = new IDictionary>[2]; - _wrappers[0] = new Dictionary>(); - _wrappers[1] = new Dictionary>(); - } - - protected void AddWrapper(Func wrapper, ClientVersion version) where T : Message - { - _wrappers[(byte)version][typeof(T)] = x => wrapper((T)x); - } - - protected void AddUnwrapper(TcpCommand command, Func unwrapper, - ClientVersion version) where T : Message - { - _unwrappers[(byte)version][(byte)command] = (pkg, env, user, tokens, conn) => unwrapper(pkg, env); - } - - protected void AddUnwrapper(TcpCommand command, - Func unwrapper, ClientVersion version) where T : Message - { - _unwrappers[(byte)version][(byte)command] = - (pkg, env, user, tokens, conn) => unwrapper(pkg, env, conn); - } - - protected void AddUnwrapper(TcpCommand command, Func unwrapper, - ClientVersion version) where T : Message - { - _unwrappers[(byte)version][(byte)command] = - (pkg, env, user, tokens, conn) => unwrapper(pkg, env, user, conn); - } - - protected void AddUnwrapper(TcpCommand command, Func unwrapper, - ClientVersion version) where T : Message - { - _unwrappers[(byte)version][(byte)command] = - (pkg, env, user, tokens, conn) => unwrapper(pkg, env, user); - } - - protected void AddUnwrapper(TcpCommand command, - Func, T> unwrapper, ClientVersion version) - where T : Message - { - _unwrappers[(byte)version][(byte)command] = (pkg, env, user, tokens, conn) => - unwrapper(pkg, env, user, tokens); - } - - protected void AddUnwrapper(TcpCommand command, - Func, TcpConnectionManager, T> unwrapper, - ClientVersion version) - where T : Message - { - // ReSharper disable RedundantCast - _unwrappers[(byte)version][(byte)command] = - (Func, TcpConnectionManager, Message>)unwrapper; - // ReSharper restore RedundantCast - } - - public TcpPackage? WrapMessage(Message message, byte version) - { - if (message == null) - { - throw new ArgumentNullException(nameof(message)); - } - - try - { - if (_wrappers[version].TryGetValue(message.GetType(), out var wrapper)) - { - return wrapper(message); - } - - if (_wrappers[^1].TryGetValue(message.GetType(), out wrapper)) - { - return wrapper(message); - } - } - catch (Exception exc) - { - Log.Error(exc, "Error while wrapping message {message}.", message); - } - - return null; - } - - public Message UnwrapPackage(TcpPackage package, IEnvelope envelope, ClaimsPrincipal user, - IReadOnlyDictionary tokens, TcpConnectionManager connection, byte version) - { - if (envelope == null) - { - throw new ArgumentNullException(nameof(envelope)); - } - - var unwrapper = _unwrappers[version][(byte)package.Command] ?? - _unwrappers[^1][(byte)package.Command]; - - try - { - return unwrapper?.Invoke(package, envelope, user, tokens, connection); - } - catch (Exception exc) - { - Log.Error(exc, "Error while unwrapping TcpPackage with command {command}.", - package.Command); - } - - return null; - } - } -} diff --git a/src/EventStore.Core/Services/Transport/Tcp/TcpPackage.cs b/src/EventStore.Core/Services/Transport/Tcp/TcpPackage.cs deleted file mode 100644 index 685f97f957..0000000000 --- a/src/EventStore.Core/Services/Transport/Tcp/TcpPackage.cs +++ /dev/null @@ -1,236 +0,0 @@ -using System; -using System.Collections.Generic; -using EventStore.Common.Utils; -using EventStore.Core.Authentication; - -namespace EventStore.Core.Services.Transport.Tcp -{ - [Flags] - public enum TcpFlags : byte - { - None = 0x00, - Authenticated = 0x01, - TrustedWrite = 0x02 - } - - public struct TcpPackage - { - private static readonly IReadOnlyDictionary NotAuthenticated = new Dictionary(); - public const int CommandOffset = 0; - public const int FlagsOffset = CommandOffset + 1; - public const int CorrelationOffset = FlagsOffset + 1; - public const int AuthOffset = CorrelationOffset + 16; - - public const int MandatorySize = AuthOffset; - - public const int MaxLoginLength = 127; - public const int MaxPasswordLength = 127; - public const int MaxTokenLength = short.MaxValue - 2; - - public readonly TcpCommand Command; - public readonly TcpFlags Flags; - public readonly Guid CorrelationId; - public readonly ArraySegment Data; - - private readonly string _login; - private readonly string _password; - private readonly string _authToken; - - public IReadOnlyDictionary Tokens => (_login, _authToken) switch - { - (null, null) => NotAuthenticated, - (null, _) => new Dictionary { [AuthenticationTokenKeys.Jwt] = _authToken }, - _ => new Dictionary - { - [AuthenticationTokenKeys.Username] = _login, - [AuthenticationTokenKeys.Password] = _password - } - }; - - public static TcpPackage FromArraySegment(ArraySegment data) - { - if (data.Count < MandatorySize) - { - throw new ArgumentException($"ArraySegment too short, length: {data.Count}", nameof(data)); - } - - var command = (TcpCommand)data.Array[data.Offset + CommandOffset]; - var flags = (TcpFlags)data.Array[data.Offset + FlagsOffset]; - - var guidBytes = new byte[16]; - Buffer.BlockCopy(data.Array, data.Offset + CorrelationOffset, guidBytes, 0, 16); - var correlationId = new Guid(guidBytes); - - var headerSize = MandatorySize; - string login = null; - string pass = null; - if ((flags & TcpFlags.Authenticated) != 0) - { - var firstByte = data.Array[data.Offset + AuthOffset]; - var tokenLength = BitConverter.ToInt16(data.Array, AuthOffset); - if (Math.Sign(tokenLength) == -1) - { - var token = Helper.UTF8NoBom.GetString(data.Array, AuthOffset + 2, -tokenLength); - - headerSize += token.Length + 2; - - return new TcpPackage(command, flags, correlationId, token, - new ArraySegment(data.Array, data.Offset + headerSize, data.Count - headerSize)); - } - - var loginLen = firstByte; - login = Helper.UTF8NoBom.GetString(data.Array, data.Offset + AuthOffset + 1, loginLen); - - var passLen = data.Array[data.Offset + AuthOffset + 1 + loginLen]; - pass = Helper.UTF8NoBom.GetString(data.Array, data.Offset + AuthOffset + 1 + loginLen + 1, passLen); - - headerSize += 1 + loginLen + 1 + passLen; - } - - return new TcpPackage(command, - flags, - correlationId, - login, - pass, - new ArraySegment(data.Array, data.Offset + headerSize, data.Count - headerSize)); - } - - public TcpPackage(TcpCommand command, Guid correlationId, byte[] data) - : this(command, TcpFlags.None, correlationId, null, new ArraySegment(data ?? Empty.ByteArray)) - { - } - - public TcpPackage(TcpCommand command, Guid correlationId, ArraySegment data) - : this(command, TcpFlags.None, correlationId, null, data) - { - } - - public TcpPackage(TcpCommand command, TcpFlags flags, Guid correlationId, string login, string password, - byte[] data) - : this(command, flags, correlationId, login, password, new ArraySegment(data ?? Empty.ByteArray)) - { - } - - public TcpPackage(TcpCommand command, TcpFlags flags, Guid correlationId, string login, string password, - ArraySegment data) : this(command, flags, correlationId, data) - { - if ((flags & TcpFlags.Authenticated) != 0) - { - Ensure.NotNull(login, nameof(login)); - Ensure.NotNull(password, nameof(password)); - if (Helper.UTF8NoBom.GetByteCount(login) > MaxLoginLength) - { - throw new ArgumentException($"Login length must be less than {MaxLoginLength} bytes.", - nameof(login)); - } - - if (Helper.UTF8NoBom.GetByteCount(password) > MaxPasswordLength) - { - throw new ArgumentException($"Password length must be less than {MaxPasswordLength} bytes.", - nameof(password)); - } - } - else - { - if (login != null) - { - throw new ArgumentException("Login provided for non-authorized TcpPackage", nameof(login)); - } - - if (password != null) - { - throw new ArgumentException("Password provided for non-authorized TcpPackage", nameof(password)); - } - } - - _login = login; - _password = password; - } - - public TcpPackage(TcpCommand command, TcpFlags flags, Guid correlationId, string authToken, - ArraySegment data) : this(command, flags, correlationId, data) - { - if ((flags & TcpFlags.Authenticated) != 0) - { - Ensure.NotNull(authToken, nameof(authToken)); - - if (Helper.UTF8NoBom.GetByteCount(authToken) > MaxTokenLength) - { - throw new ArgumentException($"Token length must be smaller than {MaxTokenLength} bytes", - nameof(authToken)); - } - } - - if ((flags & TcpFlags.Authenticated) == 0 && authToken != null) - { - throw new ArgumentException("Token provided for non-authorized TcpPackage", nameof(authToken)); - } - - _authToken = authToken; - } - - - private TcpPackage(TcpCommand command, TcpFlags flags, Guid correlationId, ArraySegment data) - { - Command = command; - Flags = flags; - CorrelationId = correlationId; - Data = data; - _login = null; - _password = null; - _authToken = null; - } - - public byte[] AsByteArray() - { - if ((Flags & TcpFlags.Authenticated) != 0) - { - if (_authToken == null) - { - var loginLen = Helper.UTF8NoBom.GetByteCount(_login); - var passLen = Helper.UTF8NoBom.GetByteCount(_password); - var res = new byte[MandatorySize + 2 + loginLen + passLen + Data.Count]; - res[CommandOffset] = (byte)Command; - res[FlagsOffset] = (byte)Flags; - Buffer.BlockCopy(CorrelationId.ToByteArray(), 0, res, CorrelationOffset, 16); - - res[AuthOffset] = (byte)loginLen; - Helper.UTF8NoBom.GetBytes(_login, 0, _login.Length, res, AuthOffset + 1); - res[AuthOffset + 1 + loginLen] = (byte)passLen; - Helper.UTF8NoBom.GetBytes(_password, 0, _password.Length, res, AuthOffset + 1 + loginLen + 1); - - Buffer.BlockCopy(Data.Array, Data.Offset, res, res.Length - Data.Count, Data.Count); - return res; - } - else - { - var authTokenLength = Helper.UTF8NoBom.GetByteCount(_authToken); - var res = new byte[MandatorySize + 2 + authTokenLength + Data.Count]; - res[CommandOffset] = (byte)Command; - res[FlagsOffset] = (byte)Flags; - Buffer.BlockCopy(CorrelationId.ToByteArray(), 0, res, CorrelationOffset, 16); - - Buffer.BlockCopy(BitConverter.GetBytes(-Convert.ToInt16(authTokenLength)), 0, res, AuthOffset, 2); - Helper.UTF8NoBom.GetBytes(_authToken, 0, _authToken.Length, res, AuthOffset + 2); - - Buffer.BlockCopy(Data.Array, Data.Offset, res, res.Length - Data.Count, Data.Count); - return res; - } - } - else - { - var res = new byte[MandatorySize + Data.Count]; - res[CommandOffset] = (byte)Command; - res[FlagsOffset] = (byte)Flags; - Buffer.BlockCopy(CorrelationId.ToByteArray(), 0, res, CorrelationOffset, 16); - Buffer.BlockCopy(Data.Array, Data.Offset, res, res.Length - Data.Count, Data.Count); - return res; - } - } - - public ArraySegment AsArraySegment() - { - return new ArraySegment(AsByteArray()); - } - } -} diff --git a/src/EventStore.Core/Services/Transport/Tcp/TcpService.cs b/src/EventStore.Core/Services/Transport/Tcp/TcpService.cs deleted file mode 100644 index b1749e8949..0000000000 --- a/src/EventStore.Core/Services/Transport/Tcp/TcpService.cs +++ /dev/null @@ -1,160 +0,0 @@ -using System; -using System.Net; -using System.Net.Security; -using System.Net.Sockets; -using System.Security.Cryptography.X509Certificates; -using EventStore.Common.Utils; -using EventStore.Core.Bus; -using EventStore.Core.Messages; -using EventStore.Plugins.Authentication; -using EventStore.Transport.Tcp; -using ILogger = Serilog.ILogger; - -namespace EventStore.Core.Services.Transport.Tcp -{ - public enum TcpServiceType - { - Internal, - External - } - - public enum TcpSecurityType - { - Normal, - Secure - } - - public class TcpService : IHandle, - IHandle, - IHandle - { - private static readonly ILogger Log = Serilog.Log.ForContext(); - - private readonly IPublisher _publisher; - private readonly IPEndPoint _serverEndPoint; - private readonly TcpServerListener _serverListener; - private readonly IPublisher _networkSendQueue; - private readonly TcpServiceType _serviceType; - private readonly TcpSecurityType _securityType; - private readonly Func _dispatcherFactory; - private readonly TimeSpan _heartbeatInterval; - private readonly TimeSpan _heartbeatTimeout; - private readonly IAuthenticationProvider _authProvider; - private readonly Func _certificateSelector; - private readonly Func _intermediatesSelector; - private readonly CertificateDelegates.ClientCertificateValidator _sslClientCertValidator; - private readonly int _connectionPendingSendBytesThreshold; - private readonly int _connectionQueueSizeThreshold; - private readonly AuthorizationGateway _authorizationGateway; - - public TcpService(IPublisher publisher, - IPEndPoint serverEndPoint, - IPublisher networkSendQueue, - TcpServiceType serviceType, - TcpSecurityType securityType, - ITcpDispatcher dispatcher, - TimeSpan heartbeatInterval, - TimeSpan heartbeatTimeout, - IAuthenticationProvider authProvider, - AuthorizationGateway authorizationGateway, - Func certificateSelector, - Func intermediatesSelector, - CertificateDelegates.ClientCertificateValidator sslClientCertValidator, - int connectionPendingSendBytesThreshold, - int connectionQueueSizeThreshold) - : this(publisher, serverEndPoint, networkSendQueue, serviceType, securityType, (_, __) => dispatcher, - heartbeatInterval, heartbeatTimeout, authProvider, authorizationGateway, certificateSelector, intermediatesSelector, sslClientCertValidator, connectionPendingSendBytesThreshold, connectionQueueSizeThreshold) - { - } - - public TcpService(IPublisher publisher, - IPEndPoint serverEndPoint, - IPublisher networkSendQueue, - TcpServiceType serviceType, - TcpSecurityType securityType, - Func dispatcherFactory, - TimeSpan heartbeatInterval, - TimeSpan heartbeatTimeout, - IAuthenticationProvider authProvider, - AuthorizationGateway authorizationGateway, - Func certificateSelector, - Func intermediatesSelector, - CertificateDelegates.ClientCertificateValidator sslClientCertValidator, - int connectionPendingSendBytesThreshold, - int connectionQueueSizeThreshold) - { - Ensure.NotNull(publisher, "publisher"); - Ensure.NotNull(serverEndPoint, "serverEndPoint"); - Ensure.NotNull(networkSendQueue, "networkSendQueue"); - Ensure.NotNull(dispatcherFactory, "dispatcherFactory"); - Ensure.NotNull(authProvider, "authProvider"); - Ensure.NotNull(authorizationGateway, "authorizationGateway"); - - _publisher = publisher; - _serverEndPoint = serverEndPoint; - _serverListener = new TcpServerListener(_serverEndPoint); - _networkSendQueue = networkSendQueue; - _serviceType = serviceType; - _securityType = securityType; - _dispatcherFactory = dispatcherFactory; - _heartbeatInterval = heartbeatInterval; - _heartbeatTimeout = heartbeatTimeout; - _connectionPendingSendBytesThreshold = connectionPendingSendBytesThreshold; - _connectionQueueSizeThreshold = connectionQueueSizeThreshold; - _authProvider = authProvider; - _authorizationGateway = authorizationGateway; - _certificateSelector = certificateSelector; - _intermediatesSelector = intermediatesSelector; - _sslClientCertValidator = sslClientCertValidator; - } - - public void Handle(SystemMessage.SystemInit message) - { - try - { - _serverListener.StartListening(OnConnectionAccepted, _securityType.ToString()); - } - catch (Exception e) - { - Application.Exit(ExitCode.Error, e.Message); - } - } - - public void Handle(SystemMessage.SystemStart message) - { - } - - public void Handle(SystemMessage.BecomeShuttingDown message) - { - _serverListener.Stop(); - } - - private void OnConnectionAccepted(IPEndPoint endPoint, Socket socket) - { - var conn = _securityType == TcpSecurityType.Secure - ? TcpConnectionSsl.CreateServerFromSocket(Guid.NewGuid(), endPoint, socket, _certificateSelector, _intermediatesSelector, _sslClientCertValidator, verbose: true) - : TcpConnection.CreateAcceptedTcpConnection(Guid.NewGuid(), endPoint, socket, verbose: true); - Log.Information( - "{serviceType} TCP connection accepted: [{securityType}, {remoteEndPoint}, L{localEndPoint}, {connectionId:B}].", - _serviceType, _securityType, conn.RemoteEndPoint, conn.LocalEndPoint, conn.ConnectionId); - - var dispatcher = _dispatcherFactory(conn.ConnectionId, _serverEndPoint); - var manager = new TcpConnectionManager( - $"{_serviceType.ToString().ToLower()}-{_securityType.ToString().ToLower()}", - _serviceType, - dispatcher, - _publisher, - conn, - _networkSendQueue, - _authProvider, - _authorizationGateway, - _heartbeatInterval, - _heartbeatTimeout, - (m, e) => _publisher.Publish(new TcpMessage.ConnectionClosed(m, e)), - _connectionPendingSendBytesThreshold, - _connectionQueueSizeThreshold); // TODO AN: race condition - _publisher.Publish(new TcpMessage.ConnectionEstablished(manager)); - manager.StartReceiving(); - } - } -} diff --git a/src/EventStore.Core/Services/Transport/Tcp/UnableToAcquireStreamException.cs b/src/EventStore.Core/Services/Transport/Tcp/UnableToAcquireStreamException.cs deleted file mode 100644 index a8a0457b55..0000000000 --- a/src/EventStore.Core/Services/Transport/Tcp/UnableToAcquireStreamException.cs +++ /dev/null @@ -1,8 +0,0 @@ -using System; - -namespace EventStore.Core.Services.Transport.Tcp -{ - class UnableToAcquireStreamException : Exception - { - } -} diff --git a/src/EventStore.Core/Services/VNode/ClusterVNodeController.cs b/src/EventStore.Core/Services/VNode/ClusterVNodeController.cs index f75bcb77ff..eb126bf830 100644 --- a/src/EventStore.Core/Services/VNode/ClusterVNodeController.cs +++ b/src/EventStore.Core/Services/VNode/ClusterVNodeController.cs @@ -1101,14 +1101,10 @@ private ValueTask ForwardRequest(ClientMessage.WriteRequestMessage msg, Message private void DenyRequestBecauseNotLeader(Guid correlationId, IEnvelope envelope) { LeaderInfoProvider leaderInfoProvider = new LeaderInfoProvider(_node.GossipAdvertiseInfo, _leader); - var endpoints = leaderInfoProvider.GetLeaderInfoEndPoints(); envelope.ReplyWith( new ClientMessage.NotHandled(correlationId, ClientMessage.NotHandled.Types.NotHandledReason.NotLeader, - new ClientMessage.NotHandled.Types.LeaderInfo(endpoints.AdvertisedTcpEndPoint, - endpoints.IsTcpEndPointSecure, - endpoints.AdvertisedHttpEndPoint - ))); + new ClientMessage.NotHandled.Types.LeaderInfo(leaderInfoProvider.GetLeaderInfoEndPoint()))); } private ValueTask HandleAsReadOnlyReplica(ClientMessage.WriteEvents message, CancellationToken token) @@ -1217,14 +1213,10 @@ private ValueTask HandleAsReadOnlyReplica(ClientMessage.DeleteStream message, Ca private void DenyRequestBecauseReadOnly(Guid correlationId, IEnvelope envelope) { LeaderInfoProvider leaderInfoProvider = new LeaderInfoProvider(_node.GossipAdvertiseInfo, _leader); - var endpoints = leaderInfoProvider.GetLeaderInfoEndPoints(); envelope.ReplyWith( new ClientMessage.NotHandled(correlationId, ClientMessage.NotHandled.Types.NotHandledReason.IsReadOnly, - new ClientMessage.NotHandled.Types.LeaderInfo(endpoints.AdvertisedTcpEndPoint, - endpoints.IsTcpEndPointSecure, - endpoints.AdvertisedHttpEndPoint - ))); + new ClientMessage.NotHandled.Types.LeaderInfo(leaderInfoProvider.GetLeaderInfoEndPoint()))); } private void DenyRequestBecauseNotReady(IEnvelope envelope, Guid correlationId) @@ -1536,11 +1528,8 @@ private async ValueTask Handle(ReplicationMessage.FollowerAssignment message, Ca if (IsLegitimateReplicationMessage(message)) { Log.Information( - "========== [{httpEndPoint}] FOLLOWER ASSIGNMENT RECEIVED FROM [{internalTcp},{internalSecureTcp},{leaderId:B}].", - _nodeInfo.HttpEndPoint, - _leader.InternalTcpEndPoint == null ? "n/a" : _leader.InternalTcpEndPoint.ToString(), - _leader.InternalSecureTcpEndPoint == null ? "n/a" : _leader.InternalSecureTcpEndPoint.ToString(), - message.LeaderId); + "========== [{httpEndPoint}] FOLLOWER ASSIGNMENT RECEIVED FROM [{leaderEndPoint},{leaderId:B}].", + _nodeInfo.HttpEndPoint, _leader.HttpEndPoint, message.LeaderId); await _outputBus.DispatchAsync(message, token); await _fsm.HandleAsync(new SystemMessage.BecomeFollower(_stateCorrelationId, _leader), token); } @@ -1551,11 +1540,8 @@ private async ValueTask Handle(ReplicationMessage.CloneAssignment message, Cance if (IsLegitimateReplicationMessage(message)) { Log.Information( - "========== [{httpEndPoint}] CLONE ASSIGNMENT RECEIVED FROM [{internalTcp},{internalSecureTcp},{leaderId:B}].", - _nodeInfo.HttpEndPoint, - _leader.InternalTcpEndPoint == null ? "n/a" : _leader.InternalTcpEndPoint.ToString(), - _leader.InternalSecureTcpEndPoint == null ? "n/a" : _leader.InternalSecureTcpEndPoint.ToString(), - message.LeaderId); + "========== [{httpEndPoint}] CLONE ASSIGNMENT RECEIVED FROM [{leaderEndPoint},{leaderId:B}].", + _nodeInfo.HttpEndPoint, _leader.HttpEndPoint, message.LeaderId); await _outputBus.DispatchAsync(message, token); await _fsm.HandleAsync(new SystemMessage.BecomeClone(_stateCorrelationId, _leader), token); } @@ -1569,11 +1555,8 @@ private ValueTask Handle(ReplicationMessage.DropSubscription message, Cancellati if (IsLegitimateReplicationMessage(message)) { Log.Information( - "========== [{httpEndPoint}] DROP SUBSCRIPTION REQUEST RECEIVED FROM [{internalTcp},{internalSecureTcp},{leaderId:B}]. THIS MEANS THAT THERE IS A SURPLUS OF NODES IN THE CLUSTER, SHUTTING DOWN.", - _nodeInfo.HttpEndPoint, - _leader.InternalTcpEndPoint == null ? "n/a" : _leader.InternalTcpEndPoint.ToString(), - _leader.InternalSecureTcpEndPoint == null ? "n/a" : _leader.InternalSecureTcpEndPoint.ToString(), - message.LeaderId); + "========== [{httpEndPoint}] DROP SUBSCRIPTION REQUEST RECEIVED FROM [{leaderEndPoint},{leaderId:B}]. THIS MEANS THAT THERE IS A SURPLUS OF NODES IN THE CLUSTER, SHUTTING DOWN.", + _nodeInfo.HttpEndPoint, _leader.HttpEndPoint, message.LeaderId); task = _outputBus.DispatchAsync( new ClientMessage.RequestShutdown(exitProcess: true, shutdownHttp: true), token); } diff --git a/src/EventStore.Core/Services/VNode/LeaderInfoProvider.cs b/src/EventStore.Core/Services/VNode/LeaderInfoProvider.cs index 8189b16503..f4a6e75e4b 100644 --- a/src/EventStore.Core/Services/VNode/LeaderInfoProvider.cs +++ b/src/EventStore.Core/Services/VNode/LeaderInfoProvider.cs @@ -20,38 +20,21 @@ public LeaderInfoProvider(GossipAdvertiseInfo gossipInfo, Cluster.MemberInfo lea _leaderInfo = leaderInfo; } - public (EndPoint AdvertisedTcpEndPoint, bool IsTcpEndPointSecure, EndPoint AdvertisedHttpEndPoint) - GetLeaderInfoEndPoints() + public EndPoint GetLeaderInfoEndPoint() { - var endpoints = _leaderInfo != null - ? (TcpEndPoint: _leaderInfo.ExternalTcpEndPoint ?? _leaderInfo.ExternalSecureTcpEndPoint, - IsTcpEndPointSecure: _leaderInfo.ExternalSecureTcpEndPoint != null, - HttpEndPoint: _leaderInfo.HttpEndPoint, + ? (HttpEndPoint: _leaderInfo.HttpEndPoint, AdvertiseHost: _leaderInfo.AdvertiseHostToClientAs, - AdvertiseHttpPort: _leaderInfo.AdvertiseHttpPortToClientAs, - AdvertiseTcpPort: _leaderInfo.AdvertiseTcpPortToClientAs) - : (TcpEndPoint: _gossipInfo.ExternalTcp ?? _gossipInfo.ExternalSecureTcp, - IsTcpEndPointSecure: _gossipInfo.ExternalSecureTcp != null, - HttpEndPoint: _gossipInfo.HttpEndPoint, + AdvertiseHttpPort: _leaderInfo.AdvertiseHttpPortToClientAs) + : (HttpEndPoint: (EndPoint)_gossipInfo.HttpEndPoint, AdvertiseHost: _gossipInfo.AdvertiseHostToClientAs, - AdvertiseHttpPort: _gossipInfo.AdvertiseHttpPortToClientAs, - AdvertiseTcpPort: _gossipInfo.AdvertiseTcpPortToClientAs); + AdvertiseHttpPort: _gossipInfo.AdvertiseHttpPortToClientAs); - var advertisedTcpEndPoint = endpoints.TcpEndPoint == null - ? null - : new DnsEndPoint( - string.IsNullOrEmpty(endpoints.AdvertiseHost) - ? endpoints.TcpEndPoint.GetHost() - : endpoints.AdvertiseHost, - endpoints.AdvertiseTcpPort == 0 ? endpoints.TcpEndPoint.GetPort() : endpoints.AdvertiseTcpPort); - var advertisedHttpEndPoint = new DnsEndPoint( + return new DnsEndPoint( string.IsNullOrEmpty(endpoints.AdvertiseHost) ? endpoints.HttpEndPoint.GetHost() : endpoints.AdvertiseHost, endpoints.AdvertiseHttpPort == 0 ? endpoints.HttpEndPoint.GetPort() : endpoints.AdvertiseHttpPort); - - return (advertisedTcpEndPoint, endpoints.IsTcpEndPointSecure, advertisedHttpEndPoint); } } } diff --git a/src/EventStore.Core/Settings/VNodeSettings.cs b/src/EventStore.Core/Settings/VNodeSettings.cs deleted file mode 100644 index 1e1d7b149c..0000000000 --- a/src/EventStore.Core/Settings/VNodeSettings.cs +++ /dev/null @@ -1,105 +0,0 @@ -using System; -using System.Net; -using System.Security.Cryptography.X509Certificates; -using EventStore.Common.Utils; -using EventStore.Core.Services.Monitoring; - -namespace EventStore.Core.Settings -{ - public class SingleVNodeSettings - { - public readonly IPEndPoint ExternalTcpEndPoint; - public readonly IPEndPoint ExternalSecureTcpEndPoint; - public readonly IPEndPoint HttpEndPoint; - public readonly string[] HttpPrefixes; - public readonly bool EnableTrustedAuth; - public readonly X509Certificate2 Certificate; - public readonly int WorkerThreads; - - public readonly TimeSpan MinFlushDelay; - public readonly TimeSpan PrepareTimeout; - public readonly TimeSpan CommitTimeout; - - public readonly bool DisableScavengeMerging; - - public readonly TimeSpan StatsPeriod; - public readonly StatsStorage StatsStorage; - - public readonly bool SkipInitializeStandardUsersCheck; - - public readonly TimeSpan TcpTimeout; - - public SingleVNodeSettings(IPEndPoint externalTcpEndPoint, - IPEndPoint externalSecureTcpEndPoint, - IPEndPoint httpEndPoint, - string[] httpPrefixes, - bool enableTrustedAuth, - X509Certificate2 certificate, - int workerThreads, - TimeSpan minFlushDelay, - TimeSpan prepareTimeout, - TimeSpan commitTimeout, - TimeSpan statsPeriod, - TimeSpan tcpTimeout, - StatsStorage statsStorage = StatsStorage.StreamAndFile, - bool skipInitializeStandardUsersCheck = false, - bool disableScavengeMerging = false) - { - Ensure.NotNull(httpEndPoint, nameof(httpEndPoint)); - Ensure.NotNull(httpPrefixes, "httpPrefixes"); - if (externalSecureTcpEndPoint != null) - { - Ensure.NotNull(certificate, "certificate"); - } - - Ensure.Positive(workerThreads, "workerThreads"); - - ExternalTcpEndPoint = externalTcpEndPoint; - ExternalSecureTcpEndPoint = externalSecureTcpEndPoint; - HttpEndPoint = httpEndPoint; - HttpPrefixes = httpPrefixes; - EnableTrustedAuth = enableTrustedAuth; - Certificate = certificate; - WorkerThreads = workerThreads; - - MinFlushDelay = minFlushDelay; - PrepareTimeout = prepareTimeout; - CommitTimeout = commitTimeout; - - StatsPeriod = statsPeriod; - StatsStorage = statsStorage; - - SkipInitializeStandardUsersCheck = skipInitializeStandardUsersCheck; - DisableScavengeMerging = disableScavengeMerging; - TcpTimeout = tcpTimeout; - } - - public override string ToString() - { - return string.Format("ExternalTcpEndPoint: {0},\n" - + "ExternalSecureTcpEndPoint: {1},\n" - + "HttpEndPoint: {2},\n" - + "HttpPrefixes: {3},\n" - + "EnableTrustedAuth: {4},\n" - + "Certificate: {5},\n" - + "WorkerThreads: {6}\n" - + "MinFlushDelay: {7}\n" - + "PrepareTimeout: {8}\n" - + "CommitTimeout: {9}\n" - + "StatsPeriod: {10}\n" - + "StatsStorage: {11}", - ExternalTcpEndPoint == null ? "n/a" : ExternalTcpEndPoint.ToString(), - ExternalSecureTcpEndPoint == null ? "n/a" : ExternalSecureTcpEndPoint.ToString(), - HttpEndPoint, - string.Join(", ", HttpPrefixes), - EnableTrustedAuth, - Certificate == null ? "n/a" : Certificate.ToString(verbose: true), - WorkerThreads, - MinFlushDelay, - PrepareTimeout, - CommitTimeout, - StatsPeriod, - StatsStorage); - } - } -} diff --git a/src/EventStore.Core/Util/Opts.cs b/src/EventStore.Core/Util/Opts.cs index 358400fa67..dac39b9cf6 100644 --- a/src/EventStore.Core/Util/Opts.cs +++ b/src/EventStore.Core/Util/Opts.cs @@ -1,16 +1,7 @@ -using System; -using System.Net; -using EventStore.Common.Options; -using EventStore.Core.TransactionLog.Chunks; - namespace EventStore.Core.Util { public static class Opts { - public const int ConnectionPendingSendBytesThresholdDefault = 10 * 1024 * 1024; - - public const int ConnectionQueueSizeThresholdDefault = 50000; - public const int HashCollisionReadLimitDefault = 100; public const bool FaultOutOfOrderProjectionsDefault = false; diff --git a/src/EventStore.Projections.Core.Tests/ClientAPI/Cluster/specification_with_standard_projections_runnning.cs b/src/EventStore.Projections.Core.Tests/ClientAPI/Cluster/specification_with_standard_projections_runnning.cs index 7e379c45be..606fee115d 100644 --- a/src/EventStore.Projections.Core.Tests/ClientAPI/Cluster/specification_with_standard_projections_runnning.cs +++ b/src/EventStore.Projections.Core.Tests/ClientAPI/Cluster/specification_with_standard_projections_runnning.cs @@ -44,19 +44,17 @@ public abstract class specification_with_standard_projections_runnning Ports => _ports; @@ -70,7 +68,6 @@ public override async Task TestFixtureSetUp() for (var index = 0; index < _nodeEndpoints.Length; index++) { _nodeEndpoints[index] = new Endpoints( - PortsHelper.GetAvailablePort(IPAddress.Loopback), PortsHelper.GetAvailablePort(IPAddress.Loopback), PortsHelper.GetAvailablePort(IPAddress.Loopback)); } @@ -79,7 +76,7 @@ public override async Task TestFixtureSetUp() { var gossipSeeds = _nodeEndpoints .Where((_, otherIndex) => otherIndex != index) - .Select(x => (EndPoint)x.HttpEndPoint) + .Select(x => (EndPoint)x.NodeEndPoint) .ToArray(); _nodes[index] = CreateNode(index, _nodeEndpoints[index], gossipSeeds); } @@ -130,9 +127,8 @@ private MiniClusterNode CreateNode(int index, Endpoints e return new MiniClusterNode( PathName, index, - endpoints.InternalTcp, - endpoints.ExternalTcp, - endpoints.HttpEndPoint, + endpoints.NodeEndPoint, + endpoints.ReplicationEndPoint, subsystems: [_projections[index]], gossipSeeds: gossipSeeds); } diff --git a/src/EventStore.Projections.Core.Tests/Services/projections_system/when_starting_up.cs b/src/EventStore.Projections.Core.Tests/Services/projections_system/when_starting_up.cs index 83cfb94daf..f970cd0c05 100644 --- a/src/EventStore.Projections.Core.Tests/Services/projections_system/when_starting_up.cs +++ b/src/EventStore.Projections.Core.Tests/Services/projections_system/when_starting_up.cs @@ -59,11 +59,7 @@ protected override IEnumerable PreWhen() { yield return (new SystemMessage.BecomeFollower(Guid.NewGuid(), MemberInfo.Initial(Guid.NewGuid(), DateTime.UtcNow, VNodeState.Unknown, true, - new IPEndPoint(IPAddress.Loopback, 1111), - new IPEndPoint(IPAddress.Loopback, 1112), - new IPEndPoint(IPAddress.Loopback, 1113), - new IPEndPoint(IPAddress.Loopback, 1114), - new IPEndPoint(IPAddress.Loopback, 1115), null, 0, 0, + new IPEndPoint(IPAddress.Loopback, 1115), null, 0, 1, false ))); diff --git a/src/Protos/ClientAPI/ClientMessageDtos.proto b/src/Protos/ClientAPI/ClientMessageDtos.proto deleted file mode 100755 index a94dc2fdc7..0000000000 --- a/src/Protos/ClientAPI/ClientMessageDtos.proto +++ /dev/null @@ -1,450 +0,0 @@ -syntax = "proto3"; - -package EventStore.Client.Messages; - -enum OperationResult -{ - Success = 0; - PrepareTimeout = 1; - CommitTimeout = 2; - ForwardTimeout = 3; - WrongExpectedVersion = 4; - StreamDeleted = 5; - InvalidTransaction = 6; - AccessDenied = 7; -} - -message NewEvent { - bytes event_id = 1; - string event_type = 2; - int32 data_content_type = 3; - int32 metadata_content_type = 4; - bytes data = 5; - bytes metadata = 6; -} - -message EventRecord { - string event_stream_id = 1; - int64 event_number = 2; - bytes event_id = 3; - string event_type = 4; - int32 data_content_type = 5; - int32 metadata_content_type = 6; - bytes data = 7; - bytes metadata = 8; - int64 created = 9; - int64 created_epoch = 10; -} - -message ResolvedIndexedEvent { - EventRecord event = 1; - EventRecord link = 2; -} - -message ResolvedEvent { - EventRecord event = 1; - EventRecord link = 2; - int64 commit_position = 3; - int64 prepare_position = 4; -} - -message WriteEvents { - string event_stream_id = 1; - int64 expected_version = 2; - repeated NewEvent events = 3; - bool require_leader = 4; -} - -message WriteEventsCompleted { - OperationResult result = 1; - string message = 2; - int64 first_event_number = 3; - int64 last_event_number = 4; - int64 prepare_position = 5; - int64 commit_position = 6; - int64 current_version = 7; -} - -message DeleteStream { - string event_stream_id = 1; - int64 expected_version = 2; - bool require_leader = 3; - bool hard_delete = 4; -} - -message DeleteStreamCompleted { - OperationResult result = 1; - string message = 2; - int64 prepare_position = 3; - int64 commit_position = 4; - int64 current_version = 5; -} - -message TransactionStart { - string event_stream_id = 1; - int64 expected_version = 2; - bool require_leader = 3; -} - -message TransactionStartCompleted { - int64 transaction_id = 1; - OperationResult result = 2; - string message = 3; -} - -message TransactionWrite { - int64 transaction_id = 1; - repeated NewEvent events = 2; - bool require_leader = 3; -} - -message TransactionWriteCompleted { - int64 transaction_id = 1; - OperationResult result = 2; - string message = 3; -} - -message TransactionCommit { - int64 transaction_id = 1; - bool require_leader = 2; -} - -message TransactionCommitCompleted { - int64 transaction_id = 1; - OperationResult result = 2; - string message = 3; - int64 first_event_number = 4; - int64 last_event_number = 5; - int64 prepare_position = 6; - int64 commit_position = 7; -} - -message ReadEvent { - string event_stream_id = 1; - int64 event_number = 2; - bool resolve_link_tos = 3; - bool require_leader = 4; -} - -message ReadEventCompleted { - - enum ReadEventResult { - Success = 0; - NotFound = 1; - NoStream = 2; - StreamDeleted = 3; - Error = 4; - AccessDenied = 5; - } - - ReadEventResult result = 1; - ResolvedIndexedEvent event = 2; - - string error = 3; -} - -message ReadStreamEvents { - string event_stream_id = 1; - int64 from_event_number = 2; - int32 max_count = 3; - bool resolve_link_tos = 4; - bool require_leader = 5; -} - -message ReadStreamEventsCompleted { - - enum ReadStreamResult { - Success = 0; - NoStream = 1; - StreamDeleted = 2; - NotModified = 3; - Error = 4; - AccessDenied = 5; - } - - repeated ResolvedIndexedEvent events = 1; - ReadStreamResult result = 2; - int64 next_event_number = 3; - int64 last_event_number = 4; - bool is_end_of_stream = 5; - int64 last_commit_position = 6; - - string error = 7; -} - -message ReadAllEvents { - int64 commit_position = 1; - int64 prepare_position = 2; - int32 max_count = 3; - bool resolve_link_tos = 4; - bool require_leader = 5; -} - -message ReadAllEventsCompleted { - - enum ReadAllResult { - Success = 0; - NotModified = 1; - Error = 2; - AccessDenied = 3; - } - - int64 commit_position = 1; - int64 prepare_position = 2; - repeated ResolvedEvent events = 3; - int64 next_commit_position = 4; - int64 next_prepare_position = 5; - - ReadAllResult result = 6; - string error = 7; -} - -message Filter{ - - enum FilterContext { - StreamId = 0; - EventType = 1; - - } - - enum FilterType { - Regex = 0; - Prefix = 1; - } - - FilterContext context = 1; - FilterType type = 2; - repeated string data = 3; -} - -message FilteredReadAllEvents { - int64 commit_position = 1; - int64 prepare_position = 2; - int32 max_count = 3; - int32 max_search_window = 4; - bool resolve_link_tos = 5; - bool require_leader = 6; - Filter filter = 7; -} - -message FilteredReadAllEventsCompleted { - - enum FilteredReadAllResult { - Success = 0; - NotModified = 1; - Error = 2; - AccessDenied = 3; - } - - int64 commit_position = 1; - int64 prepare_position = 2; - repeated ResolvedEvent events = 3; - int64 next_commit_position = 4; - int64 next_prepare_position = 5; - bool is_end_of_stream = 6; - - FilteredReadAllResult result = 7; - string error = 8; -} - -message CreatePersistentSubscription { - string subscription_group_name = 1; - string event_stream_id = 2; - bool resolve_link_tos = 3; - int64 start_from = 4; - int32 message_timeout_milliseconds = 5; - bool record_statistics = 6; - int32 live_buffer_size = 7; - int32 read_batch_size = 8; - int32 buffer_size = 9; - int32 max_retry_count = 10; - bool prefer_round_robin = 11; - int32 checkpoint_after_time = 12; - int32 checkpoint_max_count = 13; - int32 checkpoint_min_count = 14; - int32 subscriber_max_count = 15; - string named_consumer_strategy = 16; -} - -message DeletePersistentSubscription { - string subscription_group_name = 1; - string event_stream_id = 2; -} - -message UpdatePersistentSubscription { - string subscription_group_name = 1; - string event_stream_id = 2; - bool resolve_link_tos = 3; - int64 start_from = 4; - int32 message_timeout_milliseconds = 5; - bool record_statistics = 6; - int32 live_buffer_size = 7; - int32 read_batch_size = 8; - int32 buffer_size = 9; - int32 max_retry_count = 10; - bool prefer_round_robin = 11; - int32 checkpoint_after_time = 12; - int32 checkpoint_max_count = 13; - int32 checkpoint_min_count = 14; - int32 subscriber_max_count = 15; - string named_consumer_strategy = 16; -} - -message UpdatePersistentSubscriptionCompleted { - enum UpdatePersistentSubscriptionResult { - Success = 0; - DoesNotExist = 1; - Fail = 2; - AccessDenied=3; - } - UpdatePersistentSubscriptionResult result = 1; - string reason = 2; -} - -message CreatePersistentSubscriptionCompleted { - enum CreatePersistentSubscriptionResult { - Success = 0; - AlreadyExists = 1; - Fail = 2; - AccessDenied=3; - } - CreatePersistentSubscriptionResult result = 1; - string reason = 2; -} - -message DeletePersistentSubscriptionCompleted { - enum DeletePersistentSubscriptionResult { - Success = 0; - DoesNotExist = 1; - Fail = 2; - AccessDenied = 3; - } - DeletePersistentSubscriptionResult result = 1; - string reason = 2; -} - -message ConnectToPersistentSubscription { - string subscription_id = 1; - string event_stream_id = 2; - int32 allowed_in_flight_messages = 3; - -} - -message PersistentSubscriptionAckEvents { - string subscription_id = 1; - repeated bytes processed_event_ids = 2; -} - -message PersistentSubscriptionNakEvents { - enum NakAction { - Unknown = 0; - Park = 1; - Retry = 2; - Skip = 3; - Stop = 4; - } - - string subscription_id = 1; - repeated bytes processed_event_ids = 2; - string message = 3; - NakAction action = 4; -} - -message PersistentSubscriptionConfirmation { - int64 last_commit_position = 1; - string subscription_id = 2; - int64 last_event_number = 3; -} - -message PersistentSubscriptionStreamEventAppeared { - ResolvedIndexedEvent event = 1; - int32 retryCount = 2; -} - -message SubscribeToStream { - string event_stream_id = 1; - bool resolve_link_tos = 2; -} - -message FilteredSubscribeToStream { - string event_stream_id = 1; - bool resolve_link_tos = 2; - Filter filter = 3; - int32 checkpoint_interval = 4; -} - -message CheckpointReached { - int64 commit_position = 1; - int64 prepare_position = 2; -} - -message SubscriptionConfirmation { - int64 last_commit_position = 1; - int64 last_event_number = 2; -} - -message StreamEventAppeared { - ResolvedEvent event = 1; -} - -message UnsubscribeFromStream { -} - -message SubscriptionDropped { - - enum SubscriptionDropReason { - Unsubscribed = 0; - AccessDenied = 1; - NotFound=2; - PersistentSubscriptionDeleted=3; - SubscriberMaxCountReached=4; - } - - SubscriptionDropReason reason = 1; -} - -message NotHandled { - - enum NotHandledReason { - NotReady = 0; - TooBusy = 1; - NotLeader = 2; - IsReadOnly = 3; - } - - NotHandledReason reason = 1; - bytes additional_info = 2; - - message LeaderInfo { - string external_tcp_address = 1; - int32 external_tcp_port = 2; - string http_address = 3; - int32 http_port = 4; - string external_secure_tcp_address = 5; - int32 external_secure_tcp_port = 6; - } -} - -message ScavengeDatabase { -} - -message ScavengeDatabaseResponse { - - enum ScavengeResult { - Started = 0; - InProgress = 1; - Unauthorized = 2; - } - - ScavengeResult result = 1; - string scavengeId = 2; - -} - -message IdentifyClient { - int32 version = 1; - string connection_name = 2; -} - -message ClientIdentified { -} diff --git a/src/Protos/Grpc/cluster.proto b/src/Protos/Grpc/cluster.proto index 5d1b4e9d1d..e53b81756b 100644 --- a/src/Protos/Grpc/cluster.proto +++ b/src/Protos/Grpc/cluster.proto @@ -128,10 +128,8 @@ message MemberInfo { VNodeState state = 3; bool is_alive = 4; EndPoint http_end_point = 5; - EndPoint internal_tcp = 6; - EndPoint external_tcp = 7; - bool internal_tcp_uses_tls = 8; - bool external_tcp_uses_tls = 9; + reserved 6, 7, 8, 9; + reserved "internal_tcp", "external_tcp", "internal_tcp_uses_tls", "external_tcp_uses_tls"; int64 last_commit_position = 10; int64 writer_checkpoint = 11; @@ -142,9 +140,10 @@ message MemberInfo { int32 node_priority = 16; bool is_read_only_replica = 17; - string advertise_host_to_client_as = 18; - uint32 advertise_http_port_to_client_as = 19; - uint32 advertise_tcp_port_to_client_as = 20; + string advertise_host_to_client_as = 18; + uint32 advertise_http_port_to_client_as = 19; + reserved 20; + reserved "advertise_tcp_port_to_client_as"; string es_version = 21; EndPoint replication_end_point = 22; } diff --git a/src/Protos/Grpc/forwarding.proto b/src/Protos/Grpc/forwarding.proto index 621ceb77d4..47387edf76 100644 --- a/src/Protos/Grpc/forwarding.proto +++ b/src/Protos/Grpc/forwarding.proto @@ -188,8 +188,8 @@ enum NotHandledReason { } message LeaderInfo { - EndPoint external_tcp = 1; - bool is_secure = 2; + reserved 1, 2; + reserved "external_tcp", "is_secure"; EndPoint http = 3; } From 2e91033290a322a47f608800afef8783b1149332 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Mon, 21 Sep 2026 13:32:07 -0400 Subject: [PATCH 07/13] fix(tests): prove read-only delete rejection Signed-off-by: Yordis Prieto --- .../connecting_to_read_only_replica.cs | 61 +++++++++++++++++-- 1 file changed, 56 insertions(+), 5 deletions(-) diff --git a/src/EventStore.Core.Tests/Services/Replication/ReadOnlyReplica/connecting_to_read_only_replica.cs b/src/EventStore.Core.Tests/Services/Replication/ReadOnlyReplica/connecting_to_read_only_replica.cs index 9faa415dc9..52b3aad26a 100644 --- a/src/EventStore.Core.Tests/Services/Replication/ReadOnlyReplica/connecting_to_read_only_replica.cs +++ b/src/EventStore.Core.Tests/Services/Replication/ReadOnlyReplica/connecting_to_read_only_replica.cs @@ -49,13 +49,16 @@ private static CallOptions GetCallOptions() return new CallOptions(credentials: credentials, deadline: DateTime.UtcNow.AddSeconds(30)); } - private Streams.StreamsClient CreateClient(out GrpcChannel channel, out HttpClient httpClient) + private static Streams.StreamsClient CreateClient( + MiniClusterNode node, + out GrpcChannel channel, + out HttpClient httpClient) { httpClient = new HttpClient(new SocketsHttpHandler { SslOptions = { RemoteCertificateValidationCallback = delegate { return true; } } }); - channel = GrpcChannel.ForAddress(new Uri($"https://{_nodes[2].HttpEndPoint}"), + channel = GrpcChannel.ForAddress(new Uri($"https://{node.HttpEndPoint}"), new GrpcChannelOptions { HttpClient = httpClient }); return new Streams.StreamsClient(channel); } @@ -63,7 +66,7 @@ private Streams.StreamsClient CreateClient(out GrpcChannel channel, out HttpClie [Test] public async Task append_to_stream_is_rejected() { - var client = CreateClient(out var channel, out var httpClient); + var client = CreateClient(_nodes[2], out var channel, out var httpClient); using (channel) using (httpClient) using (var call = client.Append(GetCallOptions())) @@ -100,7 +103,17 @@ await call.RequestStream.WriteAsync(new AppendReq [Test] public async Task delete_stream_is_rejected() { - var client = CreateClient(out var channel, out var httpClient); + const string stream = nameof(delete_stream_is_rejected); + var leader = GetLeader(); + await AppendToStream(leader, stream); + var leaderWriterPosition = leader.Db.Config.WriterCheckpoint.Read(); + AssertEx.IsOrBecomesTrue( + () => _nodes[2].Db.Config.WriterCheckpoint.Read() >= leaderWriterPosition, + timeout: TimeSpan.FromSeconds(30), + onFail: MiniNodeLogging.WriteLogs, + msg: "The stream was not replicated to the read-only replica."); + + var client = CreateClient(_nodes[2], out var channel, out var httpClient); using (channel) using (httpClient) using (var call = client.DeleteAsync(new DeleteReq @@ -108,7 +121,7 @@ public async Task delete_stream_is_rejected() Options = new() { Any = new Empty(), - StreamIdentifier = new() { StreamName = ByteString.CopyFromUtf8(nameof(delete_stream_is_rejected)) } + StreamIdentifier = new() { StreamName = ByteString.CopyFromUtf8(stream) } } }, GetCallOptions())) { @@ -116,4 +129,42 @@ public async Task delete_stream_is_rejected() Assert.That(exception.StatusCode, Is.EqualTo(StatusCode.NotFound)); } } + + private static async Task AppendToStream( + MiniClusterNode node, + string stream) + { + var client = CreateClient(node, out var channel, out var httpClient); + using (channel) + using (httpClient) + using (var call = client.Append(GetCallOptions())) + { + await call.RequestStream.WriteAsync(new AppendReq + { + Options = new() + { + NoStream = new Empty(), + StreamIdentifier = new() { StreamName = ByteString.CopyFromUtf8(stream) } + } + }); + await call.RequestStream.WriteAsync(new AppendReq + { + ProposedMessage = new() + { + Id = Uuid.NewUuid().ToDto(), + Data = ByteString.Empty, + CustomMetadata = ByteString.Empty, + Metadata = + { + [GrpcMetadata.Type] = "test", + [GrpcMetadata.ContentType] = GrpcMetadata.ContentTypes.ApplicationJson + } + } + }); + await call.RequestStream.CompleteAsync(); + + var response = await call.ResponseAsync; + Assert.That(response.ResultCase, Is.EqualTo(AppendResp.ResultOneofCase.Success)); + } + } } From 2e33e61a305af73e92aa7e82550cf7c688c5678d Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Mon, 21 Sep 2026 13:38:01 -0400 Subject: [PATCH 08/13] fix(tests): follow the gRPC replication endpoint Signed-off-by: Yordis Prieto --- .../when_building/with_cluster_node_and_custom_settings.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/EventStore.Core.XUnit.Tests/Configuration/ClusterNodeOptionsTests/when_building/with_cluster_node_and_custom_settings.cs b/src/EventStore.Core.XUnit.Tests/Configuration/ClusterNodeOptionsTests/when_building/with_cluster_node_and_custom_settings.cs index e9eb8027df..7f4f5637fc 100644 --- a/src/EventStore.Core.XUnit.Tests/Configuration/ClusterNodeOptionsTests/when_building/with_cluster_node_and_custom_settings.cs +++ b/src/EventStore.Core.XUnit.Tests/Configuration/ClusterNodeOptionsTests/when_building/with_cluster_node_and_custom_settings.cs @@ -155,7 +155,6 @@ public void should_advertise_the_configured_replication_port() { Assert.AreEqual(_options.Interface.ReplicationPort, _node.NodeInfo.ReplicationEndPoint.GetPort()); Assert.AreEqual(3112, _node.GossipAdvertiseInfo.ReplicationEndPoint.Port); - Assert.AreEqual(3112, _node.GossipAdvertiseInfo.InternalSecureTcp.Port); }); } } From 43d6ce1eebb00e2155cd3852530126a110b50f24 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Mon, 21 Sep 2026 13:41:12 -0400 Subject: [PATCH 09/13] fix(tests): follow the gRPC membership matcher Signed-off-by: Yordis Prieto --- .../Regression/ClusterStatusServiceTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/EventStore.Core.Tests/Regression/ClusterStatusServiceTests.cs b/src/EventStore.Core.Tests/Regression/ClusterStatusServiceTests.cs index 32429f1ab1..bbba18105c 100644 --- a/src/EventStore.Core.Tests/Regression/ClusterStatusServiceTests.cs +++ b/src/EventStore.Core.Tests/Regression/ClusterStatusServiceTests.cs @@ -19,7 +19,7 @@ public void replication_statistics_match_the_member_replication_endpoint() }; var result = (ClientClusterInfo.ClientMemberInfo)typeof(ClusterStatusService) - .GetMethod("FindMemberByInternalEndpoint", BindingFlags.NonPublic | BindingFlags.Static)! + .GetMethod("FindMemberByEndpoint", BindingFlags.NonPublic | BindingFlags.Static)! .Invoke(null, [new List { member }, "replica.internal:1112"])!; Assert.That(result, Is.SameAs(member)); From d79424b3e348898c63125bf96107dfa778778905 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Mon, 21 Sep 2026 13:55:30 -0400 Subject: [PATCH 10/13] fix(protocol): keep retired API out of the lockfile Signed-off-by: Yordis Prieto --- proto.lock | 1501 ---------------------------------------------------- 1 file changed, 1501 deletions(-) diff --git a/proto.lock b/proto.lock index facf535cd7..c4343920b3 100644 --- a/proto.lock +++ b/proto.lock @@ -1,1506 +1,5 @@ { "definitions": [ - { - "protopath": "ClientAPI:/:ClientMessageDtos.proto", - "def": { - "enums": [ - { - "name": "OperationResult", - "enum_fields": [ - { - "name": "Success" - }, - { - "name": "PrepareTimeout", - "integer": 1 - }, - { - "name": "CommitTimeout", - "integer": 2 - }, - { - "name": "ForwardTimeout", - "integer": 3 - }, - { - "name": "WrongExpectedVersion", - "integer": 4 - }, - { - "name": "StreamDeleted", - "integer": 5 - }, - { - "name": "InvalidTransaction", - "integer": 6 - }, - { - "name": "AccessDenied", - "integer": 7 - } - ] - }, - { - "name": "ReadEventCompleted.ReadEventResult", - "enum_fields": [ - { - "name": "Success" - }, - { - "name": "NotFound", - "integer": 1 - }, - { - "name": "NoStream", - "integer": 2 - }, - { - "name": "StreamDeleted", - "integer": 3 - }, - { - "name": "Error", - "integer": 4 - }, - { - "name": "AccessDenied", - "integer": 5 - } - ] - }, - { - "name": "ReadStreamEventsCompleted.ReadStreamResult", - "enum_fields": [ - { - "name": "Success" - }, - { - "name": "NoStream", - "integer": 1 - }, - { - "name": "StreamDeleted", - "integer": 2 - }, - { - "name": "NotModified", - "integer": 3 - }, - { - "name": "Error", - "integer": 4 - }, - { - "name": "AccessDenied", - "integer": 5 - } - ] - }, - { - "name": "ReadAllEventsCompleted.ReadAllResult", - "enum_fields": [ - { - "name": "Success" - }, - { - "name": "NotModified", - "integer": 1 - }, - { - "name": "Error", - "integer": 2 - }, - { - "name": "AccessDenied", - "integer": 3 - } - ] - }, - { - "name": "Filter.FilterContext", - "enum_fields": [ - { - "name": "StreamId" - }, - { - "name": "EventType", - "integer": 1 - } - ] - }, - { - "name": "Filter.FilterType", - "enum_fields": [ - { - "name": "Regex" - }, - { - "name": "Prefix", - "integer": 1 - } - ] - }, - { - "name": "FilteredReadAllEventsCompleted.FilteredReadAllResult", - "enum_fields": [ - { - "name": "Success" - }, - { - "name": "NotModified", - "integer": 1 - }, - { - "name": "Error", - "integer": 2 - }, - { - "name": "AccessDenied", - "integer": 3 - } - ] - }, - { - "name": "UpdatePersistentSubscriptionCompleted.UpdatePersistentSubscriptionResult", - "enum_fields": [ - { - "name": "Success" - }, - { - "name": "DoesNotExist", - "integer": 1 - }, - { - "name": "Fail", - "integer": 2 - }, - { - "name": "AccessDenied", - "integer": 3 - } - ] - }, - { - "name": "CreatePersistentSubscriptionCompleted.CreatePersistentSubscriptionResult", - "enum_fields": [ - { - "name": "Success" - }, - { - "name": "AlreadyExists", - "integer": 1 - }, - { - "name": "Fail", - "integer": 2 - }, - { - "name": "AccessDenied", - "integer": 3 - } - ] - }, - { - "name": "DeletePersistentSubscriptionCompleted.DeletePersistentSubscriptionResult", - "enum_fields": [ - { - "name": "Success" - }, - { - "name": "DoesNotExist", - "integer": 1 - }, - { - "name": "Fail", - "integer": 2 - }, - { - "name": "AccessDenied", - "integer": 3 - } - ] - }, - { - "name": "PersistentSubscriptionNakEvents.NakAction", - "enum_fields": [ - { - "name": "Unknown" - }, - { - "name": "Park", - "integer": 1 - }, - { - "name": "Retry", - "integer": 2 - }, - { - "name": "Skip", - "integer": 3 - }, - { - "name": "Stop", - "integer": 4 - } - ] - }, - { - "name": "SubscriptionDropped.SubscriptionDropReason", - "enum_fields": [ - { - "name": "Unsubscribed" - }, - { - "name": "AccessDenied", - "integer": 1 - }, - { - "name": "NotFound", - "integer": 2 - }, - { - "name": "PersistentSubscriptionDeleted", - "integer": 3 - }, - { - "name": "SubscriberMaxCountReached", - "integer": 4 - } - ] - }, - { - "name": "NotHandled.NotHandledReason", - "enum_fields": [ - { - "name": "NotReady" - }, - { - "name": "TooBusy", - "integer": 1 - }, - { - "name": "NotLeader", - "integer": 2 - }, - { - "name": "IsReadOnly", - "integer": 3 - } - ] - }, - { - "name": "ScavengeDatabaseResponse.ScavengeResult", - "enum_fields": [ - { - "name": "Started" - }, - { - "name": "InProgress", - "integer": 1 - }, - { - "name": "Unauthorized", - "integer": 2 - } - ] - } - ], - "messages": [ - { - "name": "NewEvent", - "fields": [ - { - "id": 1, - "name": "event_id", - "type": "bytes" - }, - { - "id": 2, - "name": "event_type", - "type": "string" - }, - { - "id": 3, - "name": "data_content_type", - "type": "int32" - }, - { - "id": 4, - "name": "metadata_content_type", - "type": "int32" - }, - { - "id": 5, - "name": "data", - "type": "bytes" - }, - { - "id": 6, - "name": "metadata", - "type": "bytes" - } - ] - }, - { - "name": "EventRecord", - "fields": [ - { - "id": 1, - "name": "event_stream_id", - "type": "string" - }, - { - "id": 2, - "name": "event_number", - "type": "int64" - }, - { - "id": 3, - "name": "event_id", - "type": "bytes" - }, - { - "id": 4, - "name": "event_type", - "type": "string" - }, - { - "id": 5, - "name": "data_content_type", - "type": "int32" - }, - { - "id": 6, - "name": "metadata_content_type", - "type": "int32" - }, - { - "id": 7, - "name": "data", - "type": "bytes" - }, - { - "id": 8, - "name": "metadata", - "type": "bytes" - }, - { - "id": 9, - "name": "created", - "type": "int64" - }, - { - "id": 10, - "name": "created_epoch", - "type": "int64" - } - ] - }, - { - "name": "ResolvedIndexedEvent", - "fields": [ - { - "id": 1, - "name": "event", - "type": "EventRecord" - }, - { - "id": 2, - "name": "link", - "type": "EventRecord" - } - ] - }, - { - "name": "ResolvedEvent", - "fields": [ - { - "id": 1, - "name": "event", - "type": "EventRecord" - }, - { - "id": 2, - "name": "link", - "type": "EventRecord" - }, - { - "id": 3, - "name": "commit_position", - "type": "int64" - }, - { - "id": 4, - "name": "prepare_position", - "type": "int64" - } - ] - }, - { - "name": "WriteEvents", - "fields": [ - { - "id": 1, - "name": "event_stream_id", - "type": "string" - }, - { - "id": 2, - "name": "expected_version", - "type": "int64" - }, - { - "id": 3, - "name": "events", - "type": "NewEvent", - "is_repeated": true - }, - { - "id": 4, - "name": "require_leader", - "type": "bool" - } - ] - }, - { - "name": "WriteEventsCompleted", - "fields": [ - { - "id": 1, - "name": "result", - "type": "OperationResult" - }, - { - "id": 2, - "name": "message", - "type": "string" - }, - { - "id": 3, - "name": "first_event_number", - "type": "int64" - }, - { - "id": 4, - "name": "last_event_number", - "type": "int64" - }, - { - "id": 5, - "name": "prepare_position", - "type": "int64" - }, - { - "id": 6, - "name": "commit_position", - "type": "int64" - }, - { - "id": 7, - "name": "current_version", - "type": "int64" - } - ] - }, - { - "name": "DeleteStream", - "fields": [ - { - "id": 1, - "name": "event_stream_id", - "type": "string" - }, - { - "id": 2, - "name": "expected_version", - "type": "int64" - }, - { - "id": 3, - "name": "require_leader", - "type": "bool" - }, - { - "id": 4, - "name": "hard_delete", - "type": "bool" - } - ] - }, - { - "name": "DeleteStreamCompleted", - "fields": [ - { - "id": 1, - "name": "result", - "type": "OperationResult" - }, - { - "id": 2, - "name": "message", - "type": "string" - }, - { - "id": 3, - "name": "prepare_position", - "type": "int64" - }, - { - "id": 4, - "name": "commit_position", - "type": "int64" - }, - { - "id": 5, - "name": "current_version", - "type": "int64" - } - ] - }, - { - "name": "TransactionStart", - "fields": [ - { - "id": 1, - "name": "event_stream_id", - "type": "string" - }, - { - "id": 2, - "name": "expected_version", - "type": "int64" - }, - { - "id": 3, - "name": "require_leader", - "type": "bool" - } - ] - }, - { - "name": "TransactionStartCompleted", - "fields": [ - { - "id": 1, - "name": "transaction_id", - "type": "int64" - }, - { - "id": 2, - "name": "result", - "type": "OperationResult" - }, - { - "id": 3, - "name": "message", - "type": "string" - } - ] - }, - { - "name": "TransactionWrite", - "fields": [ - { - "id": 1, - "name": "transaction_id", - "type": "int64" - }, - { - "id": 2, - "name": "events", - "type": "NewEvent", - "is_repeated": true - }, - { - "id": 3, - "name": "require_leader", - "type": "bool" - } - ] - }, - { - "name": "TransactionWriteCompleted", - "fields": [ - { - "id": 1, - "name": "transaction_id", - "type": "int64" - }, - { - "id": 2, - "name": "result", - "type": "OperationResult" - }, - { - "id": 3, - "name": "message", - "type": "string" - } - ] - }, - { - "name": "TransactionCommit", - "fields": [ - { - "id": 1, - "name": "transaction_id", - "type": "int64" - }, - { - "id": 2, - "name": "require_leader", - "type": "bool" - } - ] - }, - { - "name": "TransactionCommitCompleted", - "fields": [ - { - "id": 1, - "name": "transaction_id", - "type": "int64" - }, - { - "id": 2, - "name": "result", - "type": "OperationResult" - }, - { - "id": 3, - "name": "message", - "type": "string" - }, - { - "id": 4, - "name": "first_event_number", - "type": "int64" - }, - { - "id": 5, - "name": "last_event_number", - "type": "int64" - }, - { - "id": 6, - "name": "prepare_position", - "type": "int64" - }, - { - "id": 7, - "name": "commit_position", - "type": "int64" - } - ] - }, - { - "name": "ReadEvent", - "fields": [ - { - "id": 1, - "name": "event_stream_id", - "type": "string" - }, - { - "id": 2, - "name": "event_number", - "type": "int64" - }, - { - "id": 3, - "name": "resolve_link_tos", - "type": "bool" - }, - { - "id": 4, - "name": "require_leader", - "type": "bool" - } - ] - }, - { - "name": "ReadEventCompleted", - "fields": [ - { - "id": 1, - "name": "result", - "type": "ReadEventResult" - }, - { - "id": 2, - "name": "event", - "type": "ResolvedIndexedEvent" - }, - { - "id": 3, - "name": "error", - "type": "string" - } - ] - }, - { - "name": "ReadStreamEvents", - "fields": [ - { - "id": 1, - "name": "event_stream_id", - "type": "string" - }, - { - "id": 2, - "name": "from_event_number", - "type": "int64" - }, - { - "id": 3, - "name": "max_count", - "type": "int32" - }, - { - "id": 4, - "name": "resolve_link_tos", - "type": "bool" - }, - { - "id": 5, - "name": "require_leader", - "type": "bool" - } - ] - }, - { - "name": "ReadStreamEventsCompleted", - "fields": [ - { - "id": 1, - "name": "events", - "type": "ResolvedIndexedEvent", - "is_repeated": true - }, - { - "id": 2, - "name": "result", - "type": "ReadStreamResult" - }, - { - "id": 3, - "name": "next_event_number", - "type": "int64" - }, - { - "id": 4, - "name": "last_event_number", - "type": "int64" - }, - { - "id": 5, - "name": "is_end_of_stream", - "type": "bool" - }, - { - "id": 6, - "name": "last_commit_position", - "type": "int64" - }, - { - "id": 7, - "name": "error", - "type": "string" - } - ] - }, - { - "name": "ReadAllEvents", - "fields": [ - { - "id": 1, - "name": "commit_position", - "type": "int64" - }, - { - "id": 2, - "name": "prepare_position", - "type": "int64" - }, - { - "id": 3, - "name": "max_count", - "type": "int32" - }, - { - "id": 4, - "name": "resolve_link_tos", - "type": "bool" - }, - { - "id": 5, - "name": "require_leader", - "type": "bool" - } - ] - }, - { - "name": "ReadAllEventsCompleted", - "fields": [ - { - "id": 1, - "name": "commit_position", - "type": "int64" - }, - { - "id": 2, - "name": "prepare_position", - "type": "int64" - }, - { - "id": 3, - "name": "events", - "type": "ResolvedEvent", - "is_repeated": true - }, - { - "id": 4, - "name": "next_commit_position", - "type": "int64" - }, - { - "id": 5, - "name": "next_prepare_position", - "type": "int64" - }, - { - "id": 6, - "name": "result", - "type": "ReadAllResult" - }, - { - "id": 7, - "name": "error", - "type": "string" - } - ] - }, - { - "name": "Filter", - "fields": [ - { - "id": 1, - "name": "context", - "type": "FilterContext" - }, - { - "id": 2, - "name": "type", - "type": "FilterType" - }, - { - "id": 3, - "name": "data", - "type": "string", - "is_repeated": true - } - ] - }, - { - "name": "FilteredReadAllEvents", - "fields": [ - { - "id": 1, - "name": "commit_position", - "type": "int64" - }, - { - "id": 2, - "name": "prepare_position", - "type": "int64" - }, - { - "id": 3, - "name": "max_count", - "type": "int32" - }, - { - "id": 4, - "name": "max_search_window", - "type": "int32" - }, - { - "id": 5, - "name": "resolve_link_tos", - "type": "bool" - }, - { - "id": 6, - "name": "require_leader", - "type": "bool" - }, - { - "id": 7, - "name": "filter", - "type": "Filter" - } - ] - }, - { - "name": "FilteredReadAllEventsCompleted", - "fields": [ - { - "id": 1, - "name": "commit_position", - "type": "int64" - }, - { - "id": 2, - "name": "prepare_position", - "type": "int64" - }, - { - "id": 3, - "name": "events", - "type": "ResolvedEvent", - "is_repeated": true - }, - { - "id": 4, - "name": "next_commit_position", - "type": "int64" - }, - { - "id": 5, - "name": "next_prepare_position", - "type": "int64" - }, - { - "id": 6, - "name": "is_end_of_stream", - "type": "bool" - }, - { - "id": 7, - "name": "result", - "type": "FilteredReadAllResult" - }, - { - "id": 8, - "name": "error", - "type": "string" - } - ] - }, - { - "name": "CreatePersistentSubscription", - "fields": [ - { - "id": 1, - "name": "subscription_group_name", - "type": "string" - }, - { - "id": 2, - "name": "event_stream_id", - "type": "string" - }, - { - "id": 3, - "name": "resolve_link_tos", - "type": "bool" - }, - { - "id": 4, - "name": "start_from", - "type": "int64" - }, - { - "id": 5, - "name": "message_timeout_milliseconds", - "type": "int32" - }, - { - "id": 6, - "name": "record_statistics", - "type": "bool" - }, - { - "id": 7, - "name": "live_buffer_size", - "type": "int32" - }, - { - "id": 8, - "name": "read_batch_size", - "type": "int32" - }, - { - "id": 9, - "name": "buffer_size", - "type": "int32" - }, - { - "id": 10, - "name": "max_retry_count", - "type": "int32" - }, - { - "id": 11, - "name": "prefer_round_robin", - "type": "bool" - }, - { - "id": 12, - "name": "checkpoint_after_time", - "type": "int32" - }, - { - "id": 13, - "name": "checkpoint_max_count", - "type": "int32" - }, - { - "id": 14, - "name": "checkpoint_min_count", - "type": "int32" - }, - { - "id": 15, - "name": "subscriber_max_count", - "type": "int32" - }, - { - "id": 16, - "name": "named_consumer_strategy", - "type": "string" - } - ] - }, - { - "name": "DeletePersistentSubscription", - "fields": [ - { - "id": 1, - "name": "subscription_group_name", - "type": "string" - }, - { - "id": 2, - "name": "event_stream_id", - "type": "string" - } - ] - }, - { - "name": "UpdatePersistentSubscription", - "fields": [ - { - "id": 1, - "name": "subscription_group_name", - "type": "string" - }, - { - "id": 2, - "name": "event_stream_id", - "type": "string" - }, - { - "id": 3, - "name": "resolve_link_tos", - "type": "bool" - }, - { - "id": 4, - "name": "start_from", - "type": "int64" - }, - { - "id": 5, - "name": "message_timeout_milliseconds", - "type": "int32" - }, - { - "id": 6, - "name": "record_statistics", - "type": "bool" - }, - { - "id": 7, - "name": "live_buffer_size", - "type": "int32" - }, - { - "id": 8, - "name": "read_batch_size", - "type": "int32" - }, - { - "id": 9, - "name": "buffer_size", - "type": "int32" - }, - { - "id": 10, - "name": "max_retry_count", - "type": "int32" - }, - { - "id": 11, - "name": "prefer_round_robin", - "type": "bool" - }, - { - "id": 12, - "name": "checkpoint_after_time", - "type": "int32" - }, - { - "id": 13, - "name": "checkpoint_max_count", - "type": "int32" - }, - { - "id": 14, - "name": "checkpoint_min_count", - "type": "int32" - }, - { - "id": 15, - "name": "subscriber_max_count", - "type": "int32" - }, - { - "id": 16, - "name": "named_consumer_strategy", - "type": "string" - } - ] - }, - { - "name": "UpdatePersistentSubscriptionCompleted", - "fields": [ - { - "id": 1, - "name": "result", - "type": "UpdatePersistentSubscriptionResult" - }, - { - "id": 2, - "name": "reason", - "type": "string" - } - ] - }, - { - "name": "CreatePersistentSubscriptionCompleted", - "fields": [ - { - "id": 1, - "name": "result", - "type": "CreatePersistentSubscriptionResult" - }, - { - "id": 2, - "name": "reason", - "type": "string" - } - ] - }, - { - "name": "DeletePersistentSubscriptionCompleted", - "fields": [ - { - "id": 1, - "name": "result", - "type": "DeletePersistentSubscriptionResult" - }, - { - "id": 2, - "name": "reason", - "type": "string" - } - ] - }, - { - "name": "ConnectToPersistentSubscription", - "fields": [ - { - "id": 1, - "name": "subscription_id", - "type": "string" - }, - { - "id": 2, - "name": "event_stream_id", - "type": "string" - }, - { - "id": 3, - "name": "allowed_in_flight_messages", - "type": "int32" - } - ] - }, - { - "name": "PersistentSubscriptionAckEvents", - "fields": [ - { - "id": 1, - "name": "subscription_id", - "type": "string" - }, - { - "id": 2, - "name": "processed_event_ids", - "type": "bytes", - "is_repeated": true - } - ] - }, - { - "name": "PersistentSubscriptionNakEvents", - "fields": [ - { - "id": 1, - "name": "subscription_id", - "type": "string" - }, - { - "id": 2, - "name": "processed_event_ids", - "type": "bytes", - "is_repeated": true - }, - { - "id": 3, - "name": "message", - "type": "string" - }, - { - "id": 4, - "name": "action", - "type": "NakAction" - } - ] - }, - { - "name": "PersistentSubscriptionConfirmation", - "fields": [ - { - "id": 1, - "name": "last_commit_position", - "type": "int64" - }, - { - "id": 2, - "name": "subscription_id", - "type": "string" - }, - { - "id": 3, - "name": "last_event_number", - "type": "int64" - } - ] - }, - { - "name": "PersistentSubscriptionStreamEventAppeared", - "fields": [ - { - "id": 1, - "name": "event", - "type": "ResolvedIndexedEvent" - }, - { - "id": 2, - "name": "retryCount", - "type": "int32" - } - ] - }, - { - "name": "SubscribeToStream", - "fields": [ - { - "id": 1, - "name": "event_stream_id", - "type": "string" - }, - { - "id": 2, - "name": "resolve_link_tos", - "type": "bool" - } - ] - }, - { - "name": "FilteredSubscribeToStream", - "fields": [ - { - "id": 1, - "name": "event_stream_id", - "type": "string" - }, - { - "id": 2, - "name": "resolve_link_tos", - "type": "bool" - }, - { - "id": 3, - "name": "filter", - "type": "Filter" - }, - { - "id": 4, - "name": "checkpoint_interval", - "type": "int32" - } - ] - }, - { - "name": "CheckpointReached", - "fields": [ - { - "id": 1, - "name": "commit_position", - "type": "int64" - }, - { - "id": 2, - "name": "prepare_position", - "type": "int64" - } - ] - }, - { - "name": "SubscriptionConfirmation", - "fields": [ - { - "id": 1, - "name": "last_commit_position", - "type": "int64" - }, - { - "id": 2, - "name": "last_event_number", - "type": "int64" - } - ] - }, - { - "name": "StreamEventAppeared", - "fields": [ - { - "id": 1, - "name": "event", - "type": "ResolvedEvent" - } - ] - }, - { - "name": "UnsubscribeFromStream" - }, - { - "name": "SubscriptionDropped", - "fields": [ - { - "id": 1, - "name": "reason", - "type": "SubscriptionDropReason" - } - ] - }, - { - "name": "NotHandled", - "fields": [ - { - "id": 1, - "name": "reason", - "type": "NotHandledReason" - }, - { - "id": 2, - "name": "additional_info", - "type": "bytes" - } - ], - "messages": [ - { - "name": "LeaderInfo", - "fields": [ - { - "id": 1, - "name": "external_tcp_address", - "type": "string" - }, - { - "id": 2, - "name": "external_tcp_port", - "type": "int32" - }, - { - "id": 3, - "name": "http_address", - "type": "string" - }, - { - "id": 4, - "name": "http_port", - "type": "int32" - }, - { - "id": 5, - "name": "external_secure_tcp_address", - "type": "string" - }, - { - "id": 6, - "name": "external_secure_tcp_port", - "type": "int32" - } - ] - } - ] - }, - { - "name": "ScavengeDatabase" - }, - { - "name": "ScavengeDatabaseResponse", - "fields": [ - { - "id": 1, - "name": "result", - "type": "ScavengeResult" - }, - { - "id": 2, - "name": "scavengeId", - "type": "string" - } - ] - }, - { - "name": "IdentifyClient", - "fields": [ - { - "id": 1, - "name": "version", - "type": "int32" - }, - { - "id": 2, - "name": "connection_name", - "type": "string" - } - ] - }, - { - "name": "ClientIdentified" - } - ], - "package": { - "name": "EventStore.Client.Messages" - } - } - }, { "protopath": "Grpc:/:cluster.proto", "def": { From 1c35ce72f42bc310f01b7d7f7466f196c2efb7d1 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Tue, 22 Sep 2026 14:45:45 -0400 Subject: [PATCH 11/13] fix(cluster): avoid drift from gRPC service contracts Signed-off-by: Yordis Prieto --- .../Components/Services/ReplicationEndpointPolicy.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/EventStore.ClusterNode/Components/Services/ReplicationEndpointPolicy.cs b/src/EventStore.ClusterNode/Components/Services/ReplicationEndpointPolicy.cs index a83b6f0494..514ef9e8b1 100644 --- a/src/EventStore.ClusterNode/Components/Services/ReplicationEndpointPolicy.cs +++ b/src/EventStore.ClusterNode/Components/Services/ReplicationEndpointPolicy.cs @@ -6,8 +6,10 @@ namespace EventStore.ClusterNode.Components.Services; public sealed class ReplicationEndpointPolicy(IPEndPoint listenEndPoint) { - private static readonly PathString ReplicationServicePath = "/event_store.replication.Replication"; - private static readonly PathString RequestForwardingServicePath = "/event_store.forwarding.RequestForwarding"; + private static readonly PathString ReplicationServicePath = + new($"/{EventStore.Replication.Replication.Descriptor.FullName}"); + private static readonly PathString RequestForwardingServicePath = + new($"/{EventStore.Forwarding.RequestForwarding.Descriptor.FullName}"); public bool Allows(HttpContext context) { From 5b387fb9da9d2dbd0a44e4092bd904cb10d7567d Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Tue, 22 Sep 2026 15:41:42 -0400 Subject: [PATCH 12/13] fix(tests): preserve read-only replica write coverage Signed-off-by: Yordis Prieto --- .../connecting_to_read_only_replica.cs | 62 ++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/src/EventStore.Core.Tests/Services/Replication/ReadOnlyReplica/connecting_to_read_only_replica.cs b/src/EventStore.Core.Tests/Services/Replication/ReadOnlyReplica/connecting_to_read_only_replica.cs index 52b3aad26a..881b0b07a9 100644 --- a/src/EventStore.Core.Tests/Services/Replication/ReadOnlyReplica/connecting_to_read_only_replica.cs +++ b/src/EventStore.Core.Tests/Services/Replication/ReadOnlyReplica/connecting_to_read_only_replica.cs @@ -1,9 +1,11 @@ using System; +using System.Linq; using System.Net; using System.Net.Http; using System.Text; using System.Threading.Tasks; using EventStore.Client.Streams; +using EventStore.Core.Data; using EventStore.Core.Services.Transport.Grpc; using EventStore.Core.Tests.Helpers; using EventStore.Core.Tests.Integration; @@ -12,6 +14,7 @@ using Grpc.Net.Client; using NUnit.Framework; using Empty = EventStore.Client.Empty; +using GrpcExceptions = EventStore.Core.Services.Transport.Grpc.Constants.Exceptions; using GrpcMetadata = EventStore.Core.Services.Transport.Grpc.Constants.Metadata; namespace EventStore.Core.Tests.Replication.ReadOnlyReplica; @@ -20,8 +23,13 @@ namespace EventStore.Core.Tests.Replication.ReadOnlyReplica; [TestFixture(typeof(LogFormat.V2), typeof(string))] public class connecting_to_read_only_replica : specification_with_cluster { - protected override async Task Given() => + protected override async Task Given() + { await _nodes[2].AdminUserCreated.WithTimeout(TimeSpan.FromSeconds(30)); + AssertEx.IsOrBecomesTrue(() => _nodes[2].NodeState == VNodeState.ReadOnlyReplica, + timeout: TimeSpan.FromSeconds(30), + onFail: MiniNodeLogging.WriteLogs); + } protected override MiniClusterNode CreateNode(int index, Endpoints endpoints, EndPoint[] gossipSeeds, bool wait = true) @@ -97,6 +105,56 @@ await call.RequestStream.WriteAsync(new AppendReq var exception = Assert.ThrowsAsync(async () => await call.ResponseAsync); Assert.That(exception.StatusCode, Is.EqualTo(StatusCode.NotFound)); + Assert.That(exception.Trailers.Select(x => (x.Key, x.Value)), + Does.Contain((GrpcExceptions.ExceptionKey, GrpcExceptions.NotLeader))); + } + } + + [Test] + public async Task batch_append_is_rejected() + { + var client = CreateClient(_nodes[2], out var channel, out var httpClient); + using (channel) + using (httpClient) + using (var call = client.BatchAppend(GetCallOptions())) + { + await call.RequestStream.WriteAsync(new BatchAppendReq + { + CorrelationId = Uuid.NewUuid().ToDto(), + Options = new() + { + Any = new Google.Protobuf.WellKnownTypes.Empty(), + StreamIdentifier = new() { StreamName = ByteString.CopyFromUtf8(nameof(batch_append_is_rejected)) } + }, + IsFinal = true, + ProposedMessages = + { + new BatchAppendReq.Types.ProposedMessage + { + Id = Uuid.NewUuid().ToDto(), + Metadata = + { + [GrpcMetadata.Type] = "test", + [GrpcMetadata.ContentType] = GrpcMetadata.ContentTypes.ApplicationJson + } + }, + new BatchAppendReq.Types.ProposedMessage + { + Id = Uuid.NewUuid().ToDto(), + Metadata = + { + [GrpcMetadata.Type] = "test", + [GrpcMetadata.ContentType] = GrpcMetadata.ContentTypes.ApplicationJson + } + } + } + }); + await call.RequestStream.CompleteAsync(); + + var exception = Assert.ThrowsAsync(async () => await call.ResponseStream.MoveNext()); + Assert.That(exception.StatusCode, Is.EqualTo(StatusCode.NotFound)); + Assert.That(exception.Trailers.Select(x => (x.Key, x.Value)), + Does.Contain((GrpcExceptions.ExceptionKey, GrpcExceptions.NotLeader))); } } @@ -127,6 +185,8 @@ public async Task delete_stream_is_rejected() { var exception = Assert.ThrowsAsync(async () => await call.ResponseAsync); Assert.That(exception.StatusCode, Is.EqualTo(StatusCode.NotFound)); + Assert.That(exception.Trailers.Select(x => (x.Key, x.Value)), + Does.Contain((GrpcExceptions.ExceptionKey, GrpcExceptions.NotLeader))); } } From fc98ac5be7e07c73076928f7ad8d2c755b97ee10 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Wed, 23 Sep 2026 11:31:23 -0400 Subject: [PATCH 13/13] fix(grpc): prevent orphaned subscriptions on disconnect Signed-off-by: Yordis Prieto --- .../SubscriptionDisposalOrderingTests.cs | 94 +++++++++++++++++++ .../SubscriptionDisconnectTests.cs | 83 ++++++++++++++++ .../Enumerators/Enumerator.AllSubscription.cs | 22 +++-- .../Enumerator.AllSubscriptionFiltered.cs | 22 +++-- .../Enumerator.StreamSubscription.cs | 22 +++-- 5 files changed, 219 insertions(+), 24 deletions(-) create mode 100644 src/EventStore.Core.Tests/Services/Transport/Enumerators/SubscriptionDisposalOrderingTests.cs create mode 100644 src/EventStore.Core.Tests/Services/Transport/Grpc/StreamsTests/SubscriptionDisconnectTests.cs diff --git a/src/EventStore.Core.Tests/Services/Transport/Enumerators/SubscriptionDisposalOrderingTests.cs b/src/EventStore.Core.Tests/Services/Transport/Enumerators/SubscriptionDisposalOrderingTests.cs new file mode 100644 index 0000000000..45df456276 --- /dev/null +++ b/src/EventStore.Core.Tests/Services/Transport/Enumerators/SubscriptionDisposalOrderingTests.cs @@ -0,0 +1,94 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using EventStore.Core.Bus; +using EventStore.Core.Messages; +using EventStore.Core.Messaging; +using EventStore.Core.Services.Storage.ReaderIndex; +using EventStore.Core.Services.Transport.Common; +using EventStore.Core.Services.Transport.Enumerators; +using EventStore.Core.Services.UserManagement; +using NUnit.Framework; + +namespace EventStore.Core.Tests.Services.Transport.Enumerators; + +[TestFixture] +public class SubscriptionDisposalOrderingTests +{ + [TestCase("stream")] + [TestCase("all")] + [TestCase("filtered-all")] + public async Task disposing_while_live_subscription_is_being_registered_does_not_leave_it_active(string kind) + { + var publisher = new DelayedSubscriptionPublisher(); + var enumerator = CreateEnumerator(kind, publisher); + try + { + await publisher.SubscribeEntered.Task.WaitAsync(TimeSpan.FromSeconds(10)); + var disposal = enumerator.DisposeAsync().AsTask(); + publisher.ReleaseSubscribe(); + await disposal.WaitAsync(TimeSpan.FromSeconds(10)); + await publisher.SubscribeCompleted.Task.WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.That(publisher.ActiveSubscriptions, Is.Empty); + } + finally + { + publisher.ReleaseSubscribe(); + await enumerator.DisposeAsync(); + } + } + + private static IAsyncEnumerator CreateEnumerator(string kind, IPublisher publisher) => kind switch + { + "stream" => new Enumerator.StreamSubscription( + publisher, new DefaultExpiryStrategy(), "subscription-disposal-ordering", StreamRevision.End, + false, SystemAccounts.System, false, CancellationToken.None), + "all" => new Enumerator.AllSubscription( + publisher, new DefaultExpiryStrategy(), Position.End, + false, SystemAccounts.System, false, CancellationToken.None), + "filtered-all" => new Enumerator.AllSubscriptionFiltered( + publisher, new DefaultExpiryStrategy(), Position.End, false, + EventFilter.EventType.Prefixes(false, "matching"), SystemAccounts.System, false, + null, 1, CancellationToken.None), + _ => throw new ArgumentOutOfRangeException(nameof(kind)) + }; + + private sealed class DelayedSubscriptionPublisher : IPublisher + { + private readonly ManualResetEventSlim _continueSubscribe = new(false); + private readonly ConcurrentDictionary _activeSubscriptions = new(); + public TaskCompletionSource SubscribeEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource SubscribeCompleted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public Guid[] ActiveSubscriptions => [.. _activeSubscriptions.Keys]; + + public void ReleaseSubscribe() => _continueSubscribe.Set(); + + public void Publish(Message message) + { + switch (message) + { + case ClientMessage.SubscribeToStream subscription: + Register(subscription.CorrelationId, subscription.Envelope); + break; + case ClientMessage.FilteredSubscribeToStream subscription: + Register(subscription.CorrelationId, subscription.Envelope); + break; + case ClientMessage.UnsubscribeFromStream unsubscribe: + _activeSubscriptions.TryRemove(unsubscribe.CorrelationId, out _); + break; + } + } + + private void Register(Guid id, IEnvelope envelope) + { + SubscribeEntered.TrySetResult(true); + _continueSubscribe.Wait(TimeSpan.FromSeconds(10)); + _activeSubscriptions.TryAdd(id, 0); + envelope.ReplyWith(new ClientMessage.SubscriptionConfirmation(id, 0, 0)); + SubscribeCompleted.TrySetResult(true); + } + } +} diff --git a/src/EventStore.Core.Tests/Services/Transport/Grpc/StreamsTests/SubscriptionDisconnectTests.cs b/src/EventStore.Core.Tests/Services/Transport/Grpc/StreamsTests/SubscriptionDisconnectTests.cs new file mode 100644 index 0000000000..ee17e023ea --- /dev/null +++ b/src/EventStore.Core.Tests/Services/Transport/Grpc/StreamsTests/SubscriptionDisconnectTests.cs @@ -0,0 +1,83 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using EventStore.Client.Streams; +using EventStore.Core.Bus; +using EventStore.Core.Messages; +using Google.Protobuf; +using NUnit.Framework; + +namespace EventStore.Core.Tests.Services.Transport.Grpc.StreamsTests; + +[TestFixture(typeof(LogFormat.V2), typeof(string))] +public class SubscriptionDisconnectTests : GrpcSpecification +{ + protected override Task Given() => Task.CompletedTask; + protected override Task When() => Task.CompletedTask; + + [TestCase("stream")] + [TestCase("all")] + [TestCase("filtered-all")] + public async Task disposing_the_grpc_read_call_unsubscribes_its_live_subscription(string kind) + { + var unsubscribed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var handler = new AdHocHandler(message => + unsubscribed.TrySetResult(message.CorrelationId)); + Node.Node.MainBus.Subscribe(handler); + try + { + using var call = StreamsClient.Read(CreateReadRequest(kind), GetCallOptions(AdminCredentials)); + Assert.That(await call.ResponseStream.MoveNext(CancellationToken.None), Is.True); + var confirmation = call.ResponseStream.Current; + Assert.That(confirmation.ContentCase, Is.EqualTo(ReadResp.ContentOneofCase.Confirmation)); + var subscriptionId = Guid.Parse(confirmation.Confirmation.SubscriptionId); + + call.Dispose(); + + Assert.That(await unsubscribed.Task.WaitAsync(TimeSpan.FromSeconds(10)), Is.EqualTo(subscriptionId)); + } + finally + { + Node.Node.MainBus.Unsubscribe(handler); + } + } + + private static ReadReq CreateReadRequest(string kind) + { + var options = new ReadReq.Types.Options + { + Subscription = new(), + UuidOption = new() { Structured = new() }, + ReadDirection = ReadReq.Types.Options.Types.ReadDirection.Forwards, + }; + + switch (kind) + { + case "stream": + options.Stream = new() + { + End = new(), + StreamIdentifier = new() { StreamName = ByteString.CopyFromUtf8("subscription-disconnect") } + }; + options.NoFilter = new(); + break; + case "all": + options.All = new() { End = new() }; + options.NoFilter = new(); + break; + case "filtered-all": + options.All = new() { End = new() }; + options.Filter = new() + { + Max = 32, + CheckpointIntervalMultiplier = 1, + StreamIdentifier = new() { Prefix = { "subscription-disconnect" } } + }; + break; + default: + throw new ArgumentOutOfRangeException(nameof(kind)); + } + + return new ReadReq { Options = options }; + } +} diff --git a/src/EventStore.Core/Services/Transport/Enumerators/Enumerator.AllSubscription.cs b/src/EventStore.Core/Services/Transport/Enumerators/Enumerator.AllSubscription.cs index dc64da5bf6..d98465c619 100644 --- a/src/EventStore.Core/Services/Transport/Enumerators/Enumerator.AllSubscription.cs +++ b/src/EventStore.Core/Services/Transport/Enumerators/Enumerator.AllSubscription.cs @@ -30,6 +30,7 @@ public class AllSubscription : IAsyncEnumerator private readonly CancellationToken _cancellationToken; private readonly Channel _channel; private readonly Channel<(ulong SequenceNumber, ResolvedEvent ResolvedEvent)> _liveEvents; + private readonly Task _subscriptionStartupTask; private ReadResponse _current; private bool _disposed; @@ -62,25 +63,30 @@ public AllSubscription( SubscriptionId = _subscriptionId.ToString(); - Subscribe(checkpoint, _cts.Token); + _subscriptionStartupTask = Subscribe(checkpoint, _cts.Token); } - public ValueTask DisposeAsync() + public async ValueTask DisposeAsync() { if (_disposed) { - return ValueTask.CompletedTask; + return; } Log.Verbose("Subscription {subscriptionId} to $all disposed.", _subscriptionId); _disposed = true; - Unsubscribe(); _cts.Cancel(); + try + { + await _subscriptionStartupTask; + } + catch (OperationCanceledException) + { + } + Unsubscribe(); _cts.Dispose(); - - return ValueTask.CompletedTask; } public async ValueTask MoveNextAsync() @@ -131,9 +137,9 @@ private async ValueTask MoveNextCoreAsync() return true; } - private void Subscribe(Position? checkpoint, CancellationToken ct) + private Task Subscribe(Position? checkpoint, CancellationToken ct) { - Task.Factory.StartNew(() => MainLoop(checkpoint, ct), ct); + return Task.Factory.StartNew(() => MainLoop(checkpoint, ct), ct); } private static TFPos ConvertCheckpoint(Position? checkpoint, TFPos lastLivePos) diff --git a/src/EventStore.Core/Services/Transport/Enumerators/Enumerator.AllSubscriptionFiltered.cs b/src/EventStore.Core/Services/Transport/Enumerators/Enumerator.AllSubscriptionFiltered.cs index 720062cbdc..154f8072ac 100644 --- a/src/EventStore.Core/Services/Transport/Enumerators/Enumerator.AllSubscriptionFiltered.cs +++ b/src/EventStore.Core/Services/Transport/Enumerators/Enumerator.AllSubscriptionFiltered.cs @@ -33,6 +33,7 @@ public class AllSubscriptionFiltered : IAsyncEnumerator private readonly CancellationToken _cancellationToken; private readonly Channel _channel; private readonly Channel<(ulong SequenceNumber, ResolvedEvent? ResolvedEvent, TFPos? Checkpoint)> _liveEvents; + private readonly Task _subscriptionStartupTask; private ReadResponse _current; private bool _disposed; @@ -73,25 +74,30 @@ public AllSubscriptionFiltered(IPublisher bus, SubscriptionId = _subscriptionId.ToString(); - Subscribe(checkpoint, _cts.Token); + _subscriptionStartupTask = Subscribe(checkpoint, _cts.Token); } - public ValueTask DisposeAsync() + public async ValueTask DisposeAsync() { if (_disposed) { - return ValueTask.CompletedTask; + return; } Log.Verbose("Subscription {subscriptionId} to $all:{eventFilter} disposed.", _subscriptionId, _eventFilter); _disposed = true; - Unsubscribe(); _cts.Cancel(); + try + { + await _subscriptionStartupTask; + } + catch (OperationCanceledException) + { + } + Unsubscribe(); _cts.Dispose(); - - return ValueTask.CompletedTask; } public async ValueTask MoveNextAsync() @@ -162,9 +168,9 @@ private async ValueTask MoveNextCoreAsync() return true; } - private void Subscribe(Position? checkpoint, CancellationToken ct) + private Task Subscribe(Position? checkpoint, CancellationToken ct) { - Task.Factory.StartNew(() => MainLoop(checkpoint, ct), ct); + return Task.Factory.StartNew(() => MainLoop(checkpoint, ct), ct); } private static TFPos ConvertCheckpoint(Position? checkpoint, TFPos lastLivePos) diff --git a/src/EventStore.Core/Services/Transport/Enumerators/Enumerator.StreamSubscription.cs b/src/EventStore.Core/Services/Transport/Enumerators/Enumerator.StreamSubscription.cs index b58380a9d0..24b16ed350 100644 --- a/src/EventStore.Core/Services/Transport/Enumerators/Enumerator.StreamSubscription.cs +++ b/src/EventStore.Core/Services/Transport/Enumerators/Enumerator.StreamSubscription.cs @@ -39,6 +39,7 @@ public class StreamSubscription : StreamSubscription private readonly CancellationToken _cancellationToken; private readonly Channel _channel; private readonly Channel<(ulong SequenceNumber, ResolvedEvent ResolvedEvent)> _liveEvents; + private readonly Task _subscriptionStartupTask; private ReadResponse _current; private bool _disposed; @@ -75,26 +76,31 @@ public StreamSubscription( SubscriptionId = _subscriptionId.ToString(); - Subscribe(checkpoint, _cts.Token); + _subscriptionStartupTask = Subscribe(checkpoint, _cts.Token); } - public override ValueTask DisposeAsync() + public override async ValueTask DisposeAsync() { if (_disposed) { - return ValueTask.CompletedTask; + return; } Log.Verbose("Subscription {subscriptionId} to {streamName} disposed.", _subscriptionId, _streamName); _disposed = true; + _cts.Cancel(); + try + { + await _subscriptionStartupTask; + } + catch (OperationCanceledException) + { + } Unsubscribe(); - _cts.Cancel(); _cts.Dispose(); - - return ValueTask.CompletedTask; } public override async ValueTask MoveNextAsync() @@ -148,9 +154,9 @@ private async ValueTask MoveNextCoreAsync() return true; } - private void Subscribe(StreamRevision? checkpoint, CancellationToken ct) + private Task Subscribe(StreamRevision? checkpoint, CancellationToken ct) { - Task.Factory.StartNew(() => MainLoop(checkpoint, ct), ct); + return Task.Factory.StartNew(() => MainLoop(checkpoint, ct), ct); } private static long ConvertCheckpoint(StreamRevision? checkpoint, long lastLiveEventNumber)