-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathloop.go
More file actions
431 lines (380 loc) · 11.4 KB
/
Copy pathloop.go
File metadata and controls
431 lines (380 loc) · 11.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
package workerkit
import (
"context"
"errors"
"fmt"
"sync"
"time"
)
const loopFailureStopHookTimeout = 5 * time.Second
var (
// ErrLoopExitedUnexpectedly reports that a LoopWorker loop returned nil before
// Stop canceled it.
ErrLoopExitedUnexpectedly = errors.New("loop worker exited unexpectedly")
// ErrLoopWorkerActive reports that Start found an existing loop lifecycle in
// progress instead of launching a new loop.
ErrLoopWorkerActive = errors.New("loop worker already active")
errLoopCleanupPanicked = errors.New("loop worker cleanup panicked")
)
// LoopFunc is the long-running background function managed by LoopWorker.
type LoopFunc func(context.Context, WorkerRuntime) error
// LoopWorkerOption configures a LoopWorker.
type LoopWorkerOption func(*loopWorkerConfig)
type loopWorkerConfig struct {
onStart func(context.Context, WorkerRuntime) error
onStop func(context.Context, WorkerRuntime) error
autoReady bool
}
type loopWorkerState int
const (
loopIdle loopWorkerState = iota
loopStarting
loopRunning
loopStopping
loopStopped
)
func (s loopWorkerState) String() string {
switch s {
case loopIdle:
return "idle"
case loopStarting:
return "starting"
case loopRunning:
return "running"
case loopStopping:
return "stopping"
case loopStopped:
return "stopped"
default:
return fmt.Sprintf("unknown(%d)", int(s))
}
}
// WithLoopStart sets an optional hook run before the loop goroutine starts.
func WithLoopStart(fn func(context.Context, WorkerRuntime) error) LoopWorkerOption {
return func(cfg *loopWorkerConfig) {
cfg.onStart = fn
}
}
// WithLoopStop sets an optional cleanup hook run after the loop goroutine
// stops. Only one cleanup attempt runs at a time, including after an unexpected
// loop failure. Stop waits for an in-progress attempt before returning. A hook
// error leaves cleanup pending so a later Stop can retry it; Start remains
// blocked until the loop has exited and one cleanup attempt succeeds. Hook
// panics follow the worker's configured PanicPolicy.
func WithLoopStop(fn func(context.Context, WorkerRuntime) error) LoopWorkerOption {
return func(cfg *loopWorkerConfig) {
cfg.onStop = fn
}
}
// WithLoopAutoReady controls whether Start marks the worker ready after
// launching the loop goroutine.
//
// Auto-ready is a convenience for loops whose successful launch is enough to
// consider the worker operational. It does not prove that the loop completed
// domain warmup, acquired leases, connected to brokers, completed a first poll,
// or validated external dependencies. Disable auto-ready when readiness depends
// on work performed inside the loop, and call WorkerRuntime.SetReady(true) from
// the loop after that condition is met.
func WithLoopAutoReady(enabled bool) LoopWorkerOption {
return func(cfg *loopWorkerConfig) {
cfg.autoReady = enabled
}
}
// LoopWorker is a Worker implementation for long-running background loops.
//
// Use NewLoopWorker to construct one with production-oriented lifecycle
// behavior: Start launches the loop in a goroutine, Stop cancels the loop and
// waits for it to exit, and unexpected loop exits are reported through
// WorkerRuntime.ReportFailure before stop completion is published. Cancellation
// errors caused by Stop are treated as normal exits, while independent errors
// racing with Stop remain failures. By default, Start marks the worker ready
// after the loop goroutine starts. This is only a launch-readiness signal. Use
// WithLoopAutoReady(false) when readiness depends on domain warmup inside the
// loop, such as acquiring a lease, connecting to a broker, loading initial
// state, completing a first poll, or validating external dependencies. In that
// mode, the loop should call WorkerRuntime.SetReady(true) when it is actually
// ready.
//
// The loop context preserves values from the Start call for telemetry and
// correlation, but is detached from Start cancellation because Start returns
// after launching the background loop. Stop owns loop cancellation through the
// LoopWorker's internal cancel function. Loop functions should still observe
// ctx.Done() so Stop can shut them down cleanly.
type LoopWorker struct {
loop LoopFunc
onStart func(context.Context, WorkerRuntime) error
onStop func(context.Context, WorkerRuntime) error
autoReady bool
mu sync.Mutex
state loopWorkerState
cancel context.CancelFunc
done chan struct{}
runtime WorkerRuntime
stopHookRunning bool
stopHookComplete bool
stopHookAttempt *loopStopHookAttempt
}
type loopStopHookAttempt struct {
done chan struct{}
err error
}
type loopCleanupFailureReporter interface {
reportLoopCleanupFailure(error, bool)
}
type loopCleanupPanicPolicy interface {
crashOnLoopCleanupPanic() bool
}
// NewLoopWorker constructs a LoopWorker for a long-running background loop.
// It enables auto-ready by default. Use WithLoopAutoReady(false) for
// domain-gated readiness.
func NewLoopWorker(loop LoopFunc, opts ...LoopWorkerOption) *LoopWorker {
cfg := loopWorkerConfig{
autoReady: true,
}
for _, opt := range opts {
if opt != nil {
opt(&cfg)
}
}
return &LoopWorker{
loop: loop,
onStart: cfg.onStart,
onStop: cfg.onStop,
autoReady: cfg.autoReady,
}
}
// Start implements Worker.
func (w *LoopWorker) Start(ctx context.Context) error {
if w.loop == nil {
return errors.New("loop worker loop must not be nil")
}
runtime, ok := WorkerRuntimeFromContext(ctx)
if !ok {
return errors.New("worker runtime handle unavailable")
}
if err := w.beginStart(); err != nil {
return err
}
started := false
defer func() {
if !started {
w.finishFailedStart()
}
}()
if w.onStart != nil {
if err := w.onStart(ctx, runtime); err != nil {
return err
}
}
if !w.autoReady {
if err := runtime.SetReady(false); err != nil {
return err
}
}
// Preserve Start context values for telemetry/correlation, but detach from
// Start cancellation because the loop outlives the Start call. Stop uses this
// cancel function to shut the loop down.
loopCtx, cancel := context.WithCancel(context.WithoutCancel(ctx))
done := make(chan struct{})
w.finishStart(runtime, cancel, done)
started = true
go w.runLoop(loopCtx, runtime, done)
if !w.autoReady {
return nil
}
if err := runtime.SetReady(true); err != nil {
w.markStopping(done)
cancel()
select {
case <-done:
case <-ctx.Done():
return fmt.Errorf("mark loop worker ready: %w", err)
}
if stopErr := w.runStopHook(ctx, runtime); stopErr != nil {
return errors.Join(err, stopErr)
}
return err
}
return nil
}
// Stop implements Worker.
func (w *LoopWorker) Stop(ctx context.Context) error {
cancel, done, runtime, ok := w.beginStop()
if ok {
cancel()
select {
case <-done:
case <-ctx.Done():
return ctx.Err()
}
}
if runtime == nil {
var ok bool
runtime, ok = WorkerRuntimeFromContext(ctx)
if !ok {
return errors.New("worker runtime handle unavailable")
}
}
return w.runStopHook(ctx, runtime)
}
func (w *LoopWorker) beginStart() error {
w.mu.Lock()
defer w.mu.Unlock()
if !w.stopHookComplete && (w.stopHookAttempt != nil || w.state == loopStopped) {
return newLoopCleanupError(fmt.Errorf("%w: state=%s cleanup=pending", ErrLoopWorkerActive, w.state))
}
if w.state == loopStarting || w.state == loopRunning || w.state == loopStopping {
return fmt.Errorf("%w: state=%s", ErrLoopWorkerActive, w.state)
}
w.state = loopStarting
w.stopHookRunning = false
w.stopHookComplete = false
w.stopHookAttempt = nil
return nil
}
func (w *LoopWorker) finishStart(runtime WorkerRuntime, cancel context.CancelFunc, done chan struct{}) {
w.mu.Lock()
defer w.mu.Unlock()
w.runtime = runtime
w.cancel = cancel
w.done = done
w.state = loopRunning
}
func (w *LoopWorker) finishFailedStart() {
w.mu.Lock()
defer w.mu.Unlock()
w.state = loopIdle
}
func (w *LoopWorker) beginStop() (context.CancelFunc, chan struct{}, WorkerRuntime, bool) {
w.mu.Lock()
defer w.mu.Unlock()
if w.cancel == nil || w.done == nil {
return nil, nil, w.runtime, false
}
switch w.state {
case loopRunning:
w.state = loopStopping
case loopStopping:
default:
return nil, nil, w.runtime, false
}
return w.cancel, w.done, w.runtime, true
}
func (w *LoopWorker) markStopping(done chan struct{}) {
w.mu.Lock()
defer w.mu.Unlock()
if w.done == done {
w.state = loopStopping
}
}
func (w *LoopWorker) runLoop(ctx context.Context, runtime WorkerRuntime, done chan struct{}) {
err := w.loop(ctx, runtime)
w.mu.Lock()
stopRequested := w.state == loopStopping && ctx.Err() != nil
if w.done == done && !stopRequested {
w.state = loopStopping
}
w.mu.Unlock()
if stopRequested && intentionalLoopStop(err, ctx.Err()) {
w.finishLoop(done)
return
}
if err == nil {
err = ErrLoopExitedUnexpectedly
}
if err != nil {
_ = runtime.ReportFailure(err)
// Run an initial failure-cleanup attempt here because the loop has already
// exited and Stop can no longer initiate cleanup by canceling it. Preserve
// loop context values for telemetry and bound this best-effort attempt. An
// unsuccessful attempt leaves cleanup pending for a later Stop retry.
stopCtx := context.WithoutCancel(ctx)
stopCtx, cancel := context.WithTimeout(stopCtx, loopFailureStopHookTimeout)
defer cancel()
w.runFailureStopHook(stopCtx, runtime)
}
w.finishLoop(done)
}
func intentionalLoopStop(loopErr, contextErr error) bool {
return contextErr != nil && (loopErr == nil || errors.Is(loopErr, contextErr))
}
func (w *LoopWorker) finishLoop(done chan struct{}) {
w.mu.Lock()
if w.done == done {
w.cancel = nil
w.done = nil
w.runtime = nil
w.state = loopStopped
}
w.mu.Unlock()
close(done)
}
func (w *LoopWorker) runStopHook(ctx context.Context, runtime WorkerRuntime) (err error) {
attempt, run := w.beginStopHook()
if !run {
select {
case <-attempt.done:
return attempt.err
case <-ctx.Done():
return ctx.Err()
}
}
defer func() {
if recovered := recover(); recovered != nil {
w.finishStopHook(attempt, newLoopCleanupError(errLoopCleanupPanicked))
panic(recovered)
}
w.finishStopHook(attempt, err)
}()
if w.onStop != nil {
err = w.onStop(ctx, runtime)
}
if err != nil {
err = newLoopCleanupError(err)
}
return err
}
func (w *LoopWorker) runFailureStopHook(ctx context.Context, runtime WorkerRuntime) {
defer func() {
if recovered := recover(); recovered != nil {
w.reportLoopCleanupFailure(runtime, newLoopCleanupError(errLoopCleanupPanicked), true)
if policy, ok := runtime.(loopCleanupPanicPolicy); ok && policy.crashOnLoopCleanupPanic() {
panic(recovered)
}
}
}()
if err := w.runStopHook(ctx, runtime); err != nil {
w.reportLoopCleanupFailure(runtime, err, false)
}
}
func (w *LoopWorker) beginStopHook() (*loopStopHookAttempt, bool) {
w.mu.Lock()
defer w.mu.Unlock()
if w.stopHookComplete {
return w.stopHookAttempt, false
}
if w.stopHookRunning {
return w.stopHookAttempt, false
}
attempt := &loopStopHookAttempt{done: make(chan struct{})}
w.stopHookAttempt = attempt
w.stopHookRunning = true
return attempt, true
}
func (w *LoopWorker) finishStopHook(attempt *loopStopHookAttempt, err error) {
w.mu.Lock()
if w.stopHookAttempt == attempt && w.stopHookRunning {
attempt.err = err
w.stopHookRunning = false
w.stopHookComplete = err == nil
close(attempt.done)
}
w.mu.Unlock()
}
func (w *LoopWorker) reportLoopCleanupFailure(runtime WorkerRuntime, err error, panicked bool) {
reporter, ok := runtime.(loopCleanupFailureReporter)
if !ok {
return
}
reporter.reportLoopCleanupFailure(err, panicked)
}