-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathobserver.go
More file actions
403 lines (355 loc) · 13.1 KB
/
Copy pathobserver.go
File metadata and controls
403 lines (355 loc) · 13.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
package clientkit
import (
"context"
"time"
"github.com/jaredjakacky/opskit"
)
// Observer receives backend-neutral client lifecycle events. Implementations
// must be safe for concurrent use. Callbacks are synchronous and should return
// quickly. Event attributes must remain bounded and safe for production
// telemetry.
type Observer interface {
// StartOperation observes an operation start and may return a derived context
// used by the operation and its later observer callbacks.
StartOperation(context.Context, OperationStartEvent) (context.Context, OperationObservation)
// ObserveAttempt observes one completed execution attempt.
ObserveAttempt(context.Context, AttemptEvent)
// ObserveRetry observes one retry that has been scheduled.
ObserveRetry(context.Context, RetryEvent)
// ObserveHealth observes one completed health check.
ObserveHealth(context.Context, HealthEvent)
}
// OperationObservation completes an observation started by Observer.
type OperationObservation interface {
// End observes the final operation outcome exactly once.
End(context.Context, OperationEndEvent)
}
// OperationKind describes whether an observed operation coordinates logical
// client policy or directly represents one remote interaction.
type OperationKind uint8
const (
// OperationKindLogical identifies an operation that may coordinate retries,
// backoff, classification, or multiple remote interactions.
OperationKindLogical OperationKind = iota
// OperationKindRemote identifies an operation that directly represents one
// remote interaction.
OperationKindRemote
)
// OperationStartEvent describes the beginning of a client operation. The zero
// Kind is OperationKindLogical for compatibility with ordinary event literals.
type OperationStartEvent struct {
// Kind identifies the operation boundary for tracing adapters.
Kind OperationKind
// Client is the stable configured client name.
Client string
// Protocol identifies the client protocol.
Protocol string
// Operation identifies the bounded operation type.
Operation string
// StartedAt is the UTC operation start time.
StartedAt time.Time
// Attributes contains bounded, production-safe operation details.
Attributes []opskit.Attribute
}
// OperationEndEvent describes the completion of a client operation. Err may be
// recorded by error-aware telemetry but must never be used directly as a metric
// label.
type OperationEndEvent struct {
// Client is the stable configured client name.
Client string
// Protocol identifies the client protocol.
Protocol string
// Operation identifies the bounded operation type.
Operation string
// StartedAt is the UTC operation start time.
StartedAt time.Time
// EndedAt is the UTC operation completion time.
EndedAt time.Time
// Duration is the complete operation duration.
Duration time.Duration
// Attempts is the number of actual execution attempts.
Attempts int
// Outcome is the protocol implementation's bounded final outcome.
Outcome string
// Succeeded is the protocol implementation's authoritative decision that the
// operation met its configured acceptance criteria. A true value requires a
// nil Err and FailureNone; adapters treat contradictory events as failures.
Succeeded bool
// FailureClass is the protocol implementation's stable classified failure.
// It supplements Outcome and Err and is empty on success.
FailureClass FailureClass
// Err is the original terminal error and must not be used as a metric label.
Err error
// Attributes contains bounded, production-safe operation details.
Attributes []opskit.Attribute
}
// AttemptEvent describes one actual protocol execution attempt. Err may be
// recorded by error-aware telemetry but must never be used directly as a metric
// label.
type AttemptEvent struct {
// Client is the stable configured client name.
Client string
// Protocol identifies the client protocol.
Protocol string
// Operation identifies the bounded operation type.
Operation string
// Number is the one-based attempt number.
Number int
// StartedAt is the UTC attempt start time.
StartedAt time.Time
// EndedAt is the UTC attempt completion time.
EndedAt time.Time
// Duration is the attempt duration.
Duration time.Duration
// Outcome is the protocol implementation's bounded attempt outcome.
Outcome string
// Succeeded is the protocol implementation's authoritative decision that the
// attempt met its configured acceptance criteria. A true value requires a nil
// Err and FailureNone; adapters treat contradictory events as failures.
Succeeded bool
// FailureClass is the protocol implementation's stable classified failure.
// It supplements Outcome and Err and is empty on success.
FailureClass FailureClass
// Err is the original attempt error and must not be used as a metric label.
Err error
// Attributes contains bounded, production-safe attempt details.
Attributes []opskit.Attribute
}
// RetryEvent describes one retry that has been scheduled.
type RetryEvent struct {
// Client is the stable configured client name.
Client string
// Protocol identifies the client protocol.
Protocol string
// Operation identifies the bounded operation type.
Operation string
// AfterAttempt is the completed attempt that caused the retry.
AfterAttempt int
// At is the UTC time at which the retry was scheduled.
At time.Time
// Delay is the exact selected retry delay.
Delay time.Duration
// Cause is the bounded outcome that caused the retry.
Cause string
// FailureClass is the stable classified failure that caused the retry.
FailureClass FailureClass
// Attributes contains bounded, production-safe retry details.
Attributes []opskit.Attribute
}
// HealthEvent describes a completed client health check.
type HealthEvent struct {
// Client is the stable configured client name.
Client string
// Protocol identifies the client protocol.
Protocol string
// State is the final health state.
State HealthState
// FailureClass is the stable classified health-check failure and is empty
// when no execution failure caused the health state.
FailureClass FailureClass
// CheckedAt is the UTC health-check completion time.
CheckedAt time.Time
// Duration is the complete health-check duration.
Duration time.Duration
// Message is the bounded health result message.
Message string
// Attributes contains bounded, production-safe health details.
Attributes []opskit.Attribute
}
// NopObserver is an Observer that performs no work. Supplying it in a protocol
// client configuration explicitly disables the protocol's default observer.
type NopObserver struct{}
// StartOperation returns the incoming context and a no-op observation.
func (NopObserver) StartOperation(ctx context.Context, _ OperationStartEvent) (context.Context, OperationObservation) {
return ctx, NopOperationObservation{}
}
// ObserveAttempt performs no work.
func (NopObserver) ObserveAttempt(context.Context, AttemptEvent) {}
// ObserveRetry performs no work.
func (NopObserver) ObserveRetry(context.Context, RetryEvent) {}
// ObserveHealth performs no work.
func (NopObserver) ObserveHealth(context.Context, HealthEvent) {}
// NopOperationObservation is an OperationObservation that performs no work.
type NopOperationObservation struct{}
// End performs no work.
func (NopOperationObservation) End(context.Context, OperationEndEvent) {}
// OperationObservationFunc adapts a function into an OperationObservation.
type OperationObservationFunc func(context.Context, OperationEndEvent)
// End invokes fn when it is non-nil.
func (fn OperationObservationFunc) End(ctx context.Context, event OperationEndEvent) {
if fn != nil {
fn(ctx, event)
}
}
// SafeObserver wraps an observer so telemetry panics cannot affect client
// execution. Event attribute slices are cloned before callbacks, a panic from
// StartOperation preserves the incoming context, and a nil observer becomes
// NopObserver.
func SafeObserver(observer Observer) Observer {
if observer == nil {
return NopObserver{}
}
switch observer.(type) {
case NopObserver, safeObserver, multiObserver:
return observer
}
return safeObserver{observer: observer}
}
type safeObserver struct {
observer Observer
}
func (o safeObserver) StartOperation(ctx context.Context, event OperationStartEvent) (context.Context, OperationObservation) {
next, observation, ok := startOperationSafely(o.observer, ctx, event)
if !ok {
return ctx, NopOperationObservation{}
}
if observation == nil {
observation = NopOperationObservation{}
}
return next, safeOperationObservation{observation: observation}
}
func (o safeObserver) ObserveAttempt(ctx context.Context, event AttemptEvent) {
observeAttemptSafely(o.observer, ctx, event)
}
func (o safeObserver) ObserveRetry(ctx context.Context, event RetryEvent) {
observeRetrySafely(o.observer, ctx, event)
}
func (o safeObserver) ObserveHealth(ctx context.Context, event HealthEvent) {
observeHealthSafely(o.observer, ctx, event)
}
type safeOperationObservation struct {
observation OperationObservation
}
func (o safeOperationObservation) End(ctx context.Context, event OperationEndEvent) {
endOperationSafely(o.observation, ctx, event)
}
// MultiObserver explicitly composes non-nil observers in registration order.
// Derived operation contexts are chained in the same order, callback panics are
// contained independently, and operation observations end in reverse order to
// support stacked spans and cleanup.
func MultiObserver(observers ...Observer) Observer {
usable := make([]Observer, 0, len(observers))
for _, observer := range observers {
if observer != nil {
usable = append(usable, observer)
}
}
if len(usable) == 0 {
return NopObserver{}
}
if len(usable) == 1 {
return SafeObserver(usable[0])
}
return multiObserver{observers: usable}
}
type multiObserver struct {
observers []Observer
}
func (o multiObserver) StartOperation(ctx context.Context, event OperationStartEvent) (context.Context, OperationObservation) {
current := ctx
observations := make([]OperationObservation, 0, len(o.observers))
for _, observer := range o.observers {
next, observation, ok := startOperationSafely(observer, current, event)
if !ok {
continue
}
current = next
if observation != nil {
observations = append(observations, observation)
}
}
if len(observations) == 0 {
return current, NopOperationObservation{}
}
return current, multiOperationObservation{observations: observations}
}
func (o multiObserver) ObserveAttempt(ctx context.Context, event AttemptEvent) {
for _, observer := range o.observers {
observeAttemptSafely(observer, ctx, event)
}
}
func (o multiObserver) ObserveRetry(ctx context.Context, event RetryEvent) {
for _, observer := range o.observers {
observeRetrySafely(observer, ctx, event)
}
}
func (o multiObserver) ObserveHealth(ctx context.Context, event HealthEvent) {
for _, observer := range o.observers {
observeHealthSafely(observer, ctx, event)
}
}
type multiOperationObservation struct {
observations []OperationObservation
}
func (o multiOperationObservation) End(ctx context.Context, event OperationEndEvent) {
for index := len(o.observations) - 1; index >= 0; index-- {
endOperationSafely(o.observations[index], ctx, event)
}
}
func startOperationSafely(observer Observer, ctx context.Context, event OperationStartEvent) (next context.Context, observation OperationObservation, ok bool) {
next = ctx
defer func() {
if recover() != nil {
next = ctx
observation = nil
ok = false
}
}()
returnedContext, observation := observer.StartOperation(ctx, cloneOperationStartEvent(event))
if returnedContext != nil {
next = returnedContext
}
return next, observation, true
}
func endOperationSafely(observation OperationObservation, ctx context.Context, event OperationEndEvent) {
defer func() {
_ = recover()
}()
observation.End(ctx, cloneOperationEndEvent(event))
}
func observeAttemptSafely(observer Observer, ctx context.Context, event AttemptEvent) {
defer func() {
_ = recover()
}()
observer.ObserveAttempt(ctx, cloneAttemptEvent(event))
}
func observeRetrySafely(observer Observer, ctx context.Context, event RetryEvent) {
defer func() {
_ = recover()
}()
observer.ObserveRetry(ctx, cloneRetryEvent(event))
}
func observeHealthSafely(observer Observer, ctx context.Context, event HealthEvent) {
defer func() {
_ = recover()
}()
observer.ObserveHealth(ctx, cloneHealthEvent(event))
}
func cloneOperationStartEvent(event OperationStartEvent) OperationStartEvent {
event.Attributes = cloneObserverAttributes(event.Attributes)
return event
}
func cloneOperationEndEvent(event OperationEndEvent) OperationEndEvent {
event.Attributes = cloneObserverAttributes(event.Attributes)
return event
}
func cloneAttemptEvent(event AttemptEvent) AttemptEvent {
event.Attributes = cloneObserverAttributes(event.Attributes)
return event
}
func cloneRetryEvent(event RetryEvent) RetryEvent {
event.Attributes = cloneObserverAttributes(event.Attributes)
return event
}
func cloneHealthEvent(event HealthEvent) HealthEvent {
event.Attributes = cloneObserverAttributes(event.Attributes)
return event
}
func cloneObserverAttributes(attributes []opskit.Attribute) []opskit.Attribute {
if attributes == nil {
return nil
}
cloned := make([]opskit.Attribute, len(attributes))
copy(cloned, attributes)
return cloned
}