Skip to content

Commit ea44f29

Browse files
yordisyordis
committed
feat(monitoring): preserve connection visibility over gRPC
Signed-off-by: Yordis Prieto <yordis.prieto@gmail.com>
1 parent 3f51781 commit ea44f29

15 files changed

Lines changed: 721 additions & 2149 deletions

File tree

‎proto.lock‎

Lines changed: 52 additions & 1548 deletions
Large diffs are not rendered by default.

‎src/EventStore.ClusterNode/Components/Pages/Observability.razor‎

Lines changed: 121 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
<div>
1111
<p class="text-sm font-black uppercase tracking-[0.28em] text-es-green">Observability</p>
1212
<h1 class="mt-4 text-5xl font-black tracking-tight text-es-ink sm:text-6xl">A map of the signals the node already emits.</h1>
13-
<p class="mt-5 text-lg leading-8 text-es-muted">Inspect live queue pressure, processing throughput, health, metrics, replication, TCP, and grouped runtime statistics.</p>
13+
<p class="mt-5 text-lg leading-8 text-es-muted">Inspect live queue pressure, processing throughput, health, metrics, replication, and grouped runtime statistics.</p>
1414
</div>
1515
<div class="grid gap-3 rounded-[2rem] border border-white/80 bg-white/85 p-6 shadow-[0_20px_70px_rgba(23,32,51,0.09)] sm:grid-cols-2">
1616
<MetricCard Label="Queues" Value="@(Page?.QueueCountLabel ?? "-")" QueueMetric="queue-count" />
@@ -20,6 +20,112 @@
2020
</div>
2121
</section>
2222

23+
<section class="mt-8">
24+
<div class="mb-4 flex flex-col gap-2 sm:flex-row sm:items-end sm:justify-between">
25+
<div>
26+
<p class="text-xs font-black uppercase tracking-[0.22em] text-es-green">Network boundary</p>
27+
<h2 class="mt-2 text-3xl font-black tracking-tight text-es-ink">Shared HTTP and gRPC connections</h2>
28+
<p class="mt-2 text-sm text-es-muted">Active connections accepted by the node endpoint, including per-second traffic rates and pending bytes.</p>
29+
</div>
30+
<div class="flex flex-wrap gap-2">
31+
<span class="rounded-full border border-es-green/20 bg-es-green/10 px-4 py-2 text-sm font-bold text-es-forest" data-network-status>@NetworkStatusLabel</span>
32+
<span class="rounded-full border border-es-ink/10 bg-white px-4 py-2 text-sm font-bold text-es-muted" data-network-page-status>@NetworkPageStatusLabel</span>
33+
<button type="button" class="rounded-full border border-es-ink/10 bg-white px-4 py-2 text-sm font-bold text-es-ink transition hover:border-es-green/30 hover:text-es-green disabled:cursor-not-allowed disabled:opacity-40" data-network-page="previous" disabled>Previous</button>
34+
<button type="button" class="rounded-full border border-es-ink/10 bg-white px-4 py-2 text-sm font-bold text-es-ink transition hover:border-es-green/30 hover:text-es-green disabled:cursor-not-allowed disabled:opacity-40" data-network-page="next" disabled>Next</button>
35+
</div>
36+
</div>
37+
38+
<div class="overflow-hidden rounded-[2rem] border border-white/80 bg-white/85 shadow-[0_20px_70px_rgba(23,32,51,0.09)]">
39+
<div class="overflow-x-auto">
40+
<table class="min-w-full divide-y divide-es-ink/10 text-left text-sm">
41+
<thead class="bg-es-ink/5 text-xs font-black uppercase tracking-[0.18em] text-es-muted">
42+
<tr>
43+
<th class="px-5 py-4">Connection</th>
44+
<th class="px-5 py-4">Client</th>
45+
<th class="px-5 py-4">Type</th>
46+
<th class="px-5 py-4">Remote endpoint</th>
47+
<th class="px-5 py-4 text-right">Sent rate</th>
48+
<th class="px-5 py-4 text-right">Sent current</th>
49+
<th class="px-5 py-4 text-right">Sent pending</th>
50+
<th class="px-5 py-4 text-right">Received rate</th>
51+
<th class="px-5 py-4 text-right">Received current</th>
52+
<th class="px-5 py-4 text-right">Received pending</th>
53+
</tr>
54+
</thead>
55+
<tbody class="divide-y divide-es-ink/10" data-network-table-body>
56+
@if (NetworkRows.Count == 0)
57+
{
58+
<tr><td class="px-5 py-4 text-es-muted" colspan="10">No active shared-endpoint connections.</td></tr>
59+
}
60+
else
61+
{
62+
foreach (var connection in NetworkRows.Take(5))
63+
{
64+
<tr>
65+
<td class="px-5 py-4 font-mono text-xs text-es-muted">@connection.ConnectionId</td>
66+
<td class="px-5 py-4 font-bold text-es-ink">@Display(connection.ClientName)</td>
67+
<td class="px-5 py-4 text-es-muted">@connection.Application · @Display(connection.Protocol) · @(connection.IsTls ? "TLS" : "Cleartext")</td>
68+
<td class="px-5 py-4 font-mono text-xs text-es-muted" title="@ConnectionDetails(connection)">@connection.RemoteEndPoint</td>
69+
<td class="px-5 py-4 text-right font-mono text-es-muted">Waiting</td>
70+
<td class="px-5 py-4 text-right font-mono text-es-ink">@FormatBytes(connection.TotalBytesSent)</td>
71+
<td class="px-5 py-4 text-right font-mono text-es-ink">@FormatBytes(connection.PendingSendBytes)</td>
72+
<td class="px-5 py-4 text-right font-mono text-es-muted">Waiting</td>
73+
<td class="px-5 py-4 text-right font-mono text-es-ink">@FormatBytes(connection.TotalBytesReceived)</td>
74+
<td class="px-5 py-4 text-right font-mono text-es-ink">@FormatBytes(connection.PendingReceivedBytes)</td>
75+
</tr>
76+
}
77+
}
78+
</tbody>
79+
</table>
80+
</div>
81+
</div>
82+
</section>
83+
84+
<section class="mt-8">
85+
<div class="mb-4">
86+
<p class="text-xs font-black uppercase tracking-[0.22em] text-es-green">Cluster transport</p>
87+
<h2 class="mt-2 text-3xl font-black tracking-tight text-es-ink">gRPC replication connections</h2>
88+
<p class="mt-2 text-sm text-es-muted">Live database replication sessions on the shared HTTP/2 endpoint.</p>
89+
</div>
90+
91+
<div class="overflow-hidden rounded-[2rem] border border-white/80 bg-white/85 shadow-[0_20px_70px_rgba(23,32,51,0.09)]">
92+
<div class="overflow-x-auto">
93+
<table class="min-w-full divide-y divide-es-ink/10 text-left text-sm">
94+
<thead class="bg-es-ink/5 text-xs font-black uppercase tracking-[0.18em] text-es-muted">
95+
<tr>
96+
<th class="px-5 py-4">Endpoint</th>
97+
<th class="px-5 py-4">Connection</th>
98+
<th class="px-5 py-4 text-right">Sent</th>
99+
<th class="px-5 py-4 text-right">Received</th>
100+
<th class="px-5 py-4 text-right">Pending</th>
101+
<th class="px-5 py-4 text-right">Send queue</th>
102+
</tr>
103+
</thead>
104+
<tbody class="divide-y divide-es-ink/10" data-replication-table-body>
105+
@if (Page is not null && Page.ReplicationConnections.Count > 0)
106+
{
107+
@foreach (var connection in Page.ReplicationConnections)
108+
{
109+
<tr>
110+
<td class="px-5 py-4 font-bold text-es-ink">@connection.Endpoint</td>
111+
<td class="px-5 py-4 font-mono text-xs text-es-muted" title="Subscription @connection.SubscriptionId">@connection.ConnectionId</td>
112+
<td class="px-5 py-4 text-right font-mono text-es-ink">@FormatBytes(connection.TotalBytesSent)</td>
113+
<td class="px-5 py-4 text-right font-mono text-es-ink">@FormatBytes(connection.TotalBytesReceived)</td>
114+
<td class="px-5 py-4 text-right font-mono text-es-ink">@FormatBytes(connection.PendingSendBytes + connection.PendingReceivedBytes)</td>
115+
<td class="px-5 py-4 text-right font-mono text-es-ink">@connection.SendQueueSize.ToString("N0", CultureInfo.InvariantCulture)</td>
116+
</tr>
117+
}
118+
}
119+
else
120+
{
121+
<tr><td class="px-5 py-4 text-es-muted" colspan="6">No active gRPC replication connections.</td></tr>
122+
}
123+
</tbody>
124+
</table>
125+
</div>
126+
</div>
127+
</section>
128+
23129
<section class="mt-8">
24130
<details id="dashboard-snapshot" class="rounded-[2rem] border border-white/80 bg-white/85 p-6 shadow-[0_20px_70px_rgba(23,32,51,0.09)]">
25131
<summary class="cursor-pointer text-sm font-black uppercase tracking-[0.22em] text-es-green">Dashboard snapshot</summary>
@@ -87,58 +193,6 @@
87193
}
88194
</section>
89195

90-
<section class="mt-8">
91-
<div class="mb-4 flex flex-col gap-2 sm:flex-row sm:items-end sm:justify-between">
92-
<div>
93-
<p class="text-xs font-black uppercase tracking-[0.22em] text-es-green">TCP dashboard</p>
94-
<h2 class="mt-2 text-3xl font-black tracking-tight text-es-ink">Realtime connections</h2>
95-
<p class="mt-2 text-sm text-es-muted">Shows active connections with per-second sent and received byte rates.</p>
96-
</div>
97-
<div class="flex flex-wrap gap-2">
98-
<span class="rounded-full border border-es-green/20 bg-es-green/10 px-4 py-2 text-sm font-bold text-es-forest" data-tcp-status>@TcpStatusLabel</span>
99-
<span class="rounded-full border border-es-ink/10 bg-white px-4 py-2 text-sm font-bold text-es-muted" data-tcp-page-status>@TcpPageStatusLabel</span>
100-
<button type="button" class="rounded-full border border-es-ink/10 bg-white px-4 py-2 text-sm font-bold text-es-ink transition hover:border-es-green/30 hover:text-es-green disabled:cursor-not-allowed disabled:opacity-40" data-tcp-page="previous" disabled>Previous</button>
101-
<button type="button" class="rounded-full border border-es-ink/10 bg-white px-4 py-2 text-sm font-bold text-es-ink transition hover:border-es-green/30 hover:text-es-green disabled:cursor-not-allowed disabled:opacity-40" data-tcp-page="next" disabled>Next</button>
102-
</div>
103-
</div>
104-
105-
<div class="overflow-hidden rounded-[2rem] border border-white/80 bg-white/85 shadow-[0_20px_70px_rgba(23,32,51,0.09)]">
106-
<div class="overflow-x-auto">
107-
<table class="min-w-full divide-y divide-es-ink/10 text-left text-sm">
108-
<thead class="bg-es-ink/5 text-xs font-black uppercase tracking-[0.18em] text-es-muted">
109-
<tr>
110-
<th class="px-5 py-4">Connection</th>
111-
<th class="px-5 py-4">Client</th>
112-
<th class="px-5 py-4">Type</th>
113-
<th class="px-5 py-4">IP Address</th>
114-
<th class="px-5 py-4 text-right">Sent rate</th>
115-
<th class="px-5 py-4 text-right">Sent current</th>
116-
<th class="px-5 py-4 text-right">Sent pending</th>
117-
<th class="px-5 py-4 text-right">Received rate</th>
118-
<th class="px-5 py-4 text-right">Received current</th>
119-
<th class="px-5 py-4 text-right">Received pending</th>
120-
</tr>
121-
</thead>
122-
<tbody class="divide-y divide-es-ink/10" data-tcp-table-body>
123-
@if (TcpRows.Count == 0)
124-
{
125-
<tr>
126-
<td class="px-5 py-4 text-es-muted" colspan="10">@TcpEmptyMessage</td>
127-
</tr>
128-
}
129-
else
130-
{
131-
foreach (var connection in TcpRows.Take(5))
132-
{
133-
@RenderTcpRow(connection)
134-
}
135-
}
136-
</tbody>
137-
</table>
138-
</div>
139-
</div>
140-
</section>
141-
142196
</div>
143197

144198
@code {
@@ -148,29 +202,21 @@
148202
private string Expanded { get; set; } = "";
149203

150204
private string DashboardPayloadJson => Page?.ClientPayloadJson ?? "{}";
151-
private IReadOnlyList<TcpConnectionRow> TcpRows => Page?.TcpConnections ?? Array.Empty<TcpConnectionRow>();
152-
private string TcpErrorMessage => !string.IsNullOrWhiteSpace(Page?.TcpMessage)
153-
? Page.TcpMessage
154-
: Page?.Message ?? "";
155-
private string TcpStatusLabel {
156-
get {
157-
if (Page is null)
158-
return "Connecting TCP stats...";
159-
160-
if (!string.IsNullOrWhiteSpace(TcpErrorMessage))
161-
return "TCP unavailable";
162-
163-
return TcpRows.Count == 0
164-
? "TCP live"
165-
: string.Create(CultureInfo.InvariantCulture, $"TCP live · {TcpRows.Count} connection{(TcpRows.Count == 1 ? "" : "s")}");
166-
}
167-
}
168-
private string TcpPageStatusLabel => TcpRows.Count == 0
205+
private IReadOnlyList<NodeConnectionSnapshot> NetworkRows =>
206+
Page?.NodeConnections ?? Array.Empty<NodeConnectionSnapshot>();
207+
private string NetworkStatusLabel => NetworkRows.Count == 0
208+
? "Network live"
209+
: string.Create(CultureInfo.InvariantCulture, $"Network live · {NetworkRows.Count} connection{(NetworkRows.Count == 1 ? "" : "s")}");
210+
private string NetworkPageStatusLabel => NetworkRows.Count == 0
169211
? "No pages"
170-
: string.Create(CultureInfo.InvariantCulture, $"Page 1 of {Math.Max(1, (int)Math.Ceiling(TcpRows.Count / 5.0))}");
171-
private string TcpEmptyMessage => string.IsNullOrWhiteSpace(TcpErrorMessage)
172-
? "No TCP connections are currently reported."
173-
: TcpErrorMessage;
212+
: string.Create(CultureInfo.InvariantCulture, $"Page 1 of {Math.Max(1, (int)Math.Ceiling(NetworkRows.Count / 5.0))}");
213+
214+
private static string FormatBytes(long value) =>
215+
value.ToString("N0", CultureInfo.InvariantCulture) + " B";
216+
private static string Display(string value) => string.IsNullOrWhiteSpace(value) ? "<none>" : value;
217+
private static string ConnectionDetails(NodeConnectionSnapshot connection) =>
218+
string.Create(CultureInfo.InvariantCulture,
219+
$"Local {connection.LocalEndPoint}, connected {connection.ConnectedAt:u}");
174220

175221
protected override async Task OnParametersSetAsync() {
176222
Page = null;
@@ -216,19 +262,6 @@
216262
<td class="px-5 py-4 font-mono text-xs text-es-muted">@row.CurrentLastMessageLabel</td>
217263
</tr>;
218264

219-
private RenderFragment RenderTcpRow(TcpConnectionRow connection) => @<tr class="bg-white/70 text-es-ink">
220-
<td class="max-w-[14rem] truncate px-5 py-4 font-mono text-xs text-es-muted">@connection.IdLabel</td>
221-
<td class="px-5 py-4 font-bold text-es-ink">@connection.ClientLabel</td>
222-
<td class="px-5 py-4 text-es-muted">@connection.TypeLabel</td>
223-
<td class="px-5 py-4 font-mono text-xs text-es-muted">@connection.RemoteEndPointLabel</td>
224-
<td class="px-5 py-4 text-right font-mono text-es-ink">@connection.SentRateLabel</td>
225-
<td class="px-5 py-4 text-right font-mono text-es-ink">@connection.TotalBytesSentLabel</td>
226-
<td class="px-5 py-4 text-right font-mono text-es-ink">@connection.PendingSendBytesLabel</td>
227-
<td class="px-5 py-4 text-right font-mono text-es-ink">@connection.ReceivedRateLabel</td>
228-
<td class="px-5 py-4 text-right font-mono text-es-ink">@connection.TotalBytesReceivedLabel</td>
229-
<td class="px-5 py-4 text-right font-mono text-es-ink">@connection.PendingReceivedBytesLabel</td>
230-
</tr>;
231-
232265
private bool IsExpanded(string groupName) =>
233266
ParseExpandedGroups().Contains(groupName);
234267

‎src/EventStore.ClusterNode/Components/Services/NodeConnectionTracker.cs‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,12 @@
66
using System.Linq;
77
using System.Threading;
88
using System.Threading.Tasks;
9+
using EventStore.Core.Services.Transport.Grpc;
910
using Microsoft.AspNetCore.Connections;
1011

1112
namespace EventStore.ClusterNode.Components.Services;
1213

13-
public sealed class NodeConnectionTracker
14+
public sealed class NodeConnectionTracker : IConnectionStatsProvider
1415
{
1516
private readonly ConcurrentDictionary<string, NodeConnectionState> _connections = new();
1617

@@ -20,6 +21,8 @@ public IReadOnlyList<NodeConnectionSnapshot> Snapshot() =>
2021
.ThenBy(x => x.ConnectionId, StringComparer.Ordinal)
2122
.ToArray();
2223

24+
IReadOnlyList<ConnectionStatsSnapshot> IConnectionStatsProvider.Snapshot() => Snapshot();
25+
2326
public async Task Track(ConnectionContext context, ConnectionDelegate next, bool isTls)
2427
{
2528
var state = new NodeConnectionState(
@@ -49,7 +52,9 @@ public void ObserveRequest(
4952
string userAgent)
5053
{
5154
if (_connections.TryGetValue(connectionId, out var connection))
55+
{
5256
connection.ObserveRequest(protocol, isGrpc, connectionName, userAgent);
57+
}
5358
}
5459
}
5560

@@ -65,7 +70,19 @@ public sealed record NodeConnectionSnapshot(
6570
long TotalBytesSent,
6671
long TotalBytesReceived,
6772
long PendingSendBytes,
68-
long PendingReceivedBytes);
73+
long PendingReceivedBytes) : ConnectionStatsSnapshot(
74+
ConnectionId,
75+
RemoteEndPoint,
76+
LocalEndPoint,
77+
ClientName,
78+
Application,
79+
Protocol,
80+
IsTls,
81+
ConnectedAt,
82+
TotalBytesSent,
83+
TotalBytesReceived,
84+
PendingSendBytes,
85+
PendingReceivedBytes);
6986

7087
internal sealed class NodeConnectionState
7188
{
@@ -172,7 +189,10 @@ public NodeConnectionSnapshot Snapshot()
172189
private static string Merge(string current, string observed)
173190
{
174191
if (string.IsNullOrWhiteSpace(observed) || current == observed)
192+
{
175193
return current;
194+
}
195+
176196
return string.IsNullOrWhiteSpace(current) ? observed : "Mixed";
177197
}
178198
}
@@ -228,7 +248,9 @@ public override async ValueTask<ReadResult> ReadAsync(CancellationToken cancella
228248
public override bool TryRead(out ReadResult result)
229249
{
230250
if (!_inner.TryRead(out result))
251+
{
231252
return false;
253+
}
232254

233255
Observe(result);
234256
return true;
@@ -268,7 +290,10 @@ public override async ValueTask<FlushResult> FlushAsync(CancellationToken cancel
268290
{
269291
var result = await _inner.FlushAsync(cancellationToken);
270292
if (!result.IsCanceled)
293+
{
271294
_state.Sent();
295+
}
296+
272297
return result;
273298
}
274299

0 commit comments

Comments
 (0)