Skip to content

fix(reminders): enforce lifecycle provider consistency #10908

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ReubenBond wants to merge 3 commits into
base: main
Choose a base branch
from
+80 −27
Open

fix(reminders): enforce lifecycle provider consistency #10908

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ee071ee
fix(reminders): normalize provider lifecycle schedules
ReubenBond Aug 28, 2026
30c6762
test(reminders): improve lifecycle failure diagnostics
ReubenBond Aug 28, 2026
258320e
fix(reminders): enforce lifecycle read consistency
ReubenBond Aug 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 80 additions & 27 deletions src/Orleans.Reminders.TestKit/ReminderServiceLifecycleTestRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,10 +198,18 @@ await ExecuteWithCleanupAsync(
cancellationToken,
async () =>
{
await grain.RegisterOrUpdateAsync(Name, TimeSpan.FromSeconds(3), Period).WaitAsync(cancellationToken);
var initialDue = TimeSpan.FromSeconds(3);
var originalExpectedStart = _harness.UtcNow.UtcDateTime + initialDue;
await grain.RegisterOrUpdateAsync(Name, initialDue, Period).WaitAsync(cancellationToken);
await WaitForOwnersAfterRefreshAsync([(grain, Name)], cancellationToken);
await _harness.WaitForScheduleAsync(grain.GetGrainId(), Name, cancellationToken);
var original = await ReadRequiredAsync(Guarantee, grain.GetGrainId(), Name, cancellationToken);
var original = await AssertPersistedAsync(
Guarantee,
grain.GetGrainId(),
Name,
originalExpectedStart,
Period,
cancellationToken);
var owners = _harness.GetOwners(grain.GetGrainId(), Name).ToArray();
var starts = _harness.GetLocalStartCount(grain.GetGrainId(), Name);
var stops = _harness.GetLocalStopCount(grain.GetGrainId(), Name);
Expand All @@ -218,18 +226,27 @@ await ExecuteWithCleanupAsync(
await _harness.RefreshAsync(cancellationToken);
await changed;
await _harness.WaitForScheduleAsync(grain.GetGrainId(), Name, cancellationToken);
var updated = await ReadRequiredAsync(Guarantee, grain.GetGrainId(), Name, cancellationToken);
var updated = await AssertPersistedAsync(
Guarantee,
grain.GetGrainId(),
Name,
expectedStart,
Period + TimeSpan.FromMinutes(1),
cancellationToken,
original.ETag);

if (!owners.SequenceEqual(_harness.GetOwners(grain.GetGrainId(), Name))
|| starts != _harness.GetLocalStartCount(grain.GetGrainId(), Name)
|| stops != _harness.GetLocalStopCount(grain.GetGrainId(), Name)
|| string.Equals(original.ETag, updated.ETag, StringComparison.Ordinal)
|| updated.StartAt != expectedStart
|| NormalizeStartAt(updated.StartAt) != NormalizeStartAt(expectedStart)
|| updated.Period != Period + TimeSpan.FromMinutes(1))
{
Fail(Guarantee, "RegisterOrUpdateReminder")
Comment thread
ReubenBond marked this conversation as resolved.
.WithIdentity(grain.GetGrainId(), Name)
.WithExpected($"same single owner, starts={starts}, stops={stops}, rotated ETag, StartAt={expectedStart:O}")
.WithExpected(
$"same single owner, starts={starts}, stops={stops}, rotated ETag, "
+ $"StartAt={NormalizeStartAt(expectedStart):O} at whole-second precision")
.WithObserved(
$"owners=[{string.Join(", ", _harness.GetOwners(grain.GetGrainId(), Name))}], "
+ $"starts={_harness.GetLocalStartCount(grain.GetGrainId(), Name)}, "
Expand Down Expand Up @@ -289,9 +306,7 @@ public async Task RunReminderService_ExactDueRecovery(CancellationToken cancella
const string Guarantee = nameof(ReminderService_ExactDueRecovery);
const string Name = "exact-due-recovery";
var grain = CreateGrain(Guarantee);
var due = _harness.ReminderLoadingWindow
+ _harness.ReminderRefreshPeriod
+ _harness.ReminderRefreshPeriod;
var due = _harness.ReminderLoadingWindow + TimeSpan.FromSeconds(3);
await ExecuteWithCleanupAsync(
Guarantee,
cancellationToken,
Expand Down Expand Up @@ -500,15 +515,11 @@ await Task.WhenAll(
sentinel.RegisterOrUpdateAsync(SentinelName, TimeSpan.FromMinutes(1), Period).WaitAsync(cancellationToken),
subject.RegisterOrUpdateAsync(SubjectName, TimeSpan.FromMinutes(1), Period).WaitAsync(cancellationToken));
await CleanupAsync(Guarantee, subject, SubjectName, cancellationToken);
var sentinelRow = await _harness.ReminderTable.ReadRow(sentinel.GetGrainId(), SentinelName).WaitAsync(cancellationToken);
if (sentinelRow is null)
{
Fail(Guarantee, "cleanup")
.WithIdentity(sentinel.GetGrainId(), SentinelName)
.WithExpected("unrelated sentinel remains registered")
.WithObserved("sentinel row was removed")
.Throw();
}
_ = await ReadRequiredAsync(
Guarantee,
sentinel.GetGrainId(),
SentinelName,
cancellationToken);
},
async cleanupToken =>
{
Expand Down Expand Up @@ -654,6 +665,7 @@ private async Task RemovePersistedRowAsync(
return;
}

RequireETag(guarantee, grainId, name, row!);
if (!await _harness.ReminderTable.RemoveRow(grainId, name, row.ETag!).WaitAsync(cancellationToken))
{
Fail(guarantee, "scenario cleanup remove")
Expand All @@ -665,25 +677,44 @@ private async Task RemovePersistedRowAsync(
}
}

private async Task AssertPersistedAsync(
private async Task<ReminderEntry> AssertPersistedAsync(
string guarantee,
GrainId grainId,
string name,
DateTime expectedStart,
TimeSpan expectedPeriod,
CancellationToken cancellationToken)
CancellationToken cancellationToken,
string? previousETag = null)
{
var row = await ReadRequiredAsync(guarantee, grainId, name, cancellationToken);
if (row.StartAt != expectedStart || row.Period != expectedPeriod || string.IsNullOrEmpty(row.ETag))
var row = await _harness.ReminderTable.ReadRow(grainId, name).WaitAsync(cancellationToken);
if (row is null)
{
Fail(guarantee, "ReadRow")
.WithIdentity(grainId, name)
.WithExpected($"StartAt={expectedStart:O}, Period={expectedPeriod}, non-empty ETag")
.WithObserved(Describe(row))
.WithSchedule(row.StartAt, row.Period)
.WithETags(row.ETag)
.WithExpected("one immediately visible persisted row")
.WithObserved("<null>")
.Throw();
}

var persisted = row!;
RequireETag(guarantee, grainId, name, persisted);
if (NormalizeStartAt(persisted.StartAt) != NormalizeStartAt(expectedStart)
|| persisted.Period != expectedPeriod
|| previousETag is not null && StringComparer.Ordinal.Equals(previousETag, persisted.ETag))
{
Fail(guarantee, "ReadRow")
.WithIdentity(grainId, name)
.WithExpected(
$"immediately visible StartAt={NormalizeStartAt(expectedStart):O} at whole-second precision, "
+ $"Period={expectedPeriod}, "
+ (previousETag is null ? "non-empty ETag" : $"ETag different from '{previousETag}'"))
.WithObserved(Describe(persisted))
.WithSchedule(persisted.StartAt, persisted.Period)
.WithETags(persisted.ETag, previousETag)
.Throw();
}

return persisted;
}

private async Task<ReminderEntry> ReadRequiredAsync(
Expand All @@ -697,11 +728,12 @@ private async Task<ReminderEntry> ReadRequiredAsync(
{
Fail(guarantee, "ReadRow")
.WithIdentity(grainId, name)
.WithExpected("one persisted row")
.WithExpected("one immediately visible persisted row")
.WithObserved("<null>")
.Throw();
}

RequireETag(guarantee, grainId, name, row!);
return row!;
}

Expand All @@ -716,13 +748,34 @@ private async Task AssertAbsentAsync(
{
Fail(guarantee, "cleanup")
.WithIdentity(grainId, name)
.WithExpected("row absent")
.WithExpected("row immediately absent")
.WithObserved(Describe(row))
.WithETags(row.ETag)
.Throw();
}
}

private void RequireETag(string guarantee, GrainId grainId, string name, ReminderEntry row)
{
if (string.IsNullOrEmpty(row.ETag))
{
Fail(guarantee, "ReadRow")
.WithIdentity(grainId, name)
.WithExpected("every persisted row has a non-empty ETag")
.WithObserved(Describe(row))
.WithETags(row.ETag)
.Throw();
}
}

private static DateTime NormalizeStartAt(DateTime value)
{
var utc = value.Kind == DateTimeKind.Utc ? value : DateTime.SpecifyKind(value, DateTimeKind.Utc);
return new DateTime(
utc.Ticks - (utc.Ticks % TimeSpan.TicksPerSecond),
DateTimeKind.Utc);
}

private void AssertCounts(
string guarantee,
IReminderServiceTestGrain grain,
Expand Down