diff --git a/src/Orleans.Reminders.TestKit/README.md b/src/Orleans.Reminders.TestKit/README.md index 7df327d4bb6..2f7465ef9e2 100644 --- a/src/Orleans.Reminders.TestKit/README.md +++ b/src/Orleans.Reminders.TestKit/README.md @@ -12,6 +12,8 @@ cardinality setup and cleanup, bounded retries, diagnostics, and failure message | --- | --- | | `ReminderTableTestRunner` | Direct conformance facts for every documented table guarantee. | | `ReminderServiceTestRunner` | Cluster-level registration, replacement, lookup, enumeration, and removal conformance. | +| `ReminderServiceLifecycleTestRunner` | Deterministic startup, ownership, exact-due, reconciliation, churn, and cleanup-isolation conformance. | +| `IReminderServiceLifecycleHarness` | Adapter contract for one cluster, one reminder clock, diagnostics, and explicit topology barriers. | | `ReminderTableModelBasedTestRunner` | Generated sequential conformance against the same full contract. | | `IdealizedReminderTable` | Deterministic, strongly consistent reference implementation and fault-injection oracle. | | `ReminderTableTestFixture` | In-process cluster fixture which deploys and resolves a provider. | @@ -173,6 +175,54 @@ public sealed class MyReminderServiceTests } ``` +### Run lifecycle and churn conformance + +`ReminderServiceLifecycleTestRunner` adds the shared service-level contract. The same eight scenarios run for every +service provider: startup readiness, single registration ownership, in-place schedule update, removal quiescence, +exact-due recovery, stale-owner registration reconciliation, one-silo join/leave transfer, and cleanup isolation. + +Use `ReminderTestClock` as the sole time driver and `ReminderDiagnosticObserver` as the lifecycle/tick source. The +`ReminderServiceLifecycleHarness` adapter for `InProcessTestCluster` supplies explicit membership and reminder-range +reconciliation barriers: + +```csharp +var clock = builder.AddReminderTestClock(); +var cluster = builder.Build(); +await cluster.DeployAsync(); + +var options = cluster.Silos[0].ServiceProvider + .GetRequiredService>().Value; +var harness = new ReminderServiceLifecycleHarness( + cluster, + clock, + clock.DiagnosticObserver, + options.ReminderLoadingWindow); + +public sealed class MyLifecycleTests : ReminderServiceLifecycleTestRunner +{ + public MyLifecycleTests(IReminderServiceLifecycleHarness harness) + : base(harness, "MyProvider", seed: 42) + { + } + + [Fact] + public override Task ReminderService_OneSiloJoinLeaveTransfersOwnership() + => base.ReminderService_OneSiloJoinLeaveTransfersOwnership(); +} +``` + +Do not replace harness barriers with delays, retry loops, longer timeouts, or provider-specific skips. Scenario cleanup +uses its own bounded token, removes only deterministic scenario rows using their current ETags, advances one explicit +refresh for owner quiescence, and verifies their absence; it never clears unrelated provider rows or replaces the +original scenario failure. Ownership assertions count local reminder instance identities, including duplicate +instances on one silo, rather than counting distinct silo addresses. + +The built-in in-memory, Azure Table, Cosmos DB, ADO.NET SQL Server, PostgreSQL, MySQL, Redis, DynamoDB, and Firestore +providers all expose these same inherited facts. Their adapters contain only backend precondition/setup and provider +registration. External-service availability can skip fixture construction, but no provider disables individual +lifecycle guarantees. Add a documented capability boundary here before omitting a future provider which cannot host +the Orleans reminder service or participate in in-process silo churn. + ## Deterministic oracle and cluster testing `IdealizedReminderTable` supplies a strongly consistent reference implementation for TestKit self-tests and @@ -200,7 +250,9 @@ The oracle exposes: - `FreezeReads` for stale-read convergence scenarios; and - lifecycle cancellation and invariant checks. -The TestKit cluster integration suite uses these controls to cover exact-due delivery, exact-due storage recovery, +`ReminderTestClock` creates its lifecycle observer before the cluster is built, allowing startup conformance to await +one `ReminderServiceStarted` event for every silo before liveness and range-reconciliation barriers. The TestKit +cluster integration suite uses these controls to cover exact-due delivery, exact-due storage recovery, due times beyond the platform timer limit, stale-refresh suppression after unregister, and single-owner delivery in a multi-silo cluster. diff --git a/src/Orleans.Reminders.TestKit/ReminderServiceLifecycleTestRunner.cs b/src/Orleans.Reminders.TestKit/ReminderServiceLifecycleTestRunner.cs new file mode 100644 index 00000000000..06e7737c550 --- /dev/null +++ b/src/Orleans.Reminders.TestKit/ReminderServiceLifecycleTestRunner.cs @@ -0,0 +1,763 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Runtime.ExceptionServices; +using System.Threading; +using System.Threading.Tasks; +using Orleans.Runtime; + +namespace Orleans.Reminders.TestKit; + +/// +/// Supplies deterministic clock, diagnostics, and topology controls to +/// . +/// +public interface IReminderServiceLifecycleHarness +{ + /// Gets the deployed cluster's grain factory. + IGrainFactory GrainFactory { get; } + + /// Gets the provider table used by the reminder service. + IReminderTable ReminderTable { get; } + + /// Gets the deterministic reminder clock time. + DateTimeOffset UtcNow { get; } + + /// Gets the configured reminder loading window. + TimeSpan ReminderLoadingWindow { get; } + + /// Gets the configured reminder-table refresh period. + TimeSpan ReminderRefreshPeriod { get; } + + /// Gets the currently active silos. + IReadOnlyList ActiveSilos { get; } + + /// Waits until every active reminder service is ready. + Task WaitForStartupReadinessAsync(CancellationToken cancellationToken); + + /// Advances the one reminder clock driver. + Task AdvanceAsync(TimeSpan amount, CancellationToken cancellationToken); + + /// Refreshes every active reminder service without advancing time. + Task RefreshAsync(CancellationToken cancellationToken); + + /// Waits for exactly local owners. + Task WaitForOwnerCountAsync( + GrainId grainId, + string reminderName, + int count, + CancellationToken cancellationToken); + + /// Gets the current local owners. + IReadOnlyList GetOwners(GrainId grainId, string reminderName); + + /// Returns whether a silo's current ring range owns a grain identity. + bool IsOwner(SiloAddress siloAddress, GrainId grainId); + + /// Waits until the current owner has armed its persisted schedule. + Task WaitForScheduleAsync(GrainId grainId, string reminderName, CancellationToken cancellationToken); + + /// Gets the number of local reminder instances started for a reminder. + int GetLocalStartCount(GrainId grainId, string reminderName); + + /// Gets the number of local reminder instances stopped for a reminder. + int GetLocalStopCount(GrainId grainId, string reminderName); + + /// Gets the number of local schedule changes for a reminder. + int GetScheduleChangeCount(GrainId grainId, string reminderName); + + /// Waits for the local schedule-change count. + Task WaitForScheduleChangeCountAsync( + GrainId grainId, + string reminderName, + int count, + CancellationToken cancellationToken); + + /// Waits for exactly the requested completed tick count. + Task WaitForTickCountAsync( + GrainId grainId, + string reminderName, + int count, + CancellationToken cancellationToken); + + /// Gets the current completed tick count. + int GetTickCount(GrainId grainId, string reminderName); + + /// Starts one silo and waits for its reminder service to become ready. + Task JoinOneSiloAsync(CancellationToken cancellationToken); + + /// Stops the specified silo. + Task LeaveSiloAsync(SiloAddress siloAddress, CancellationToken cancellationToken); + + /// Waits for membership and reminder-range reconciliation on all active silos. + Task WaitForTopologyReconciliationAsync(CancellationToken cancellationToken); +} + +/// +/// Shared deterministic conformance scenarios for reminder service lifecycle, ownership, and churn. +/// +/// +/// Provider suites supply only an . The runner owns identities, +/// schedules, topology transitions, exact assertions, and cleanup. Each scenario cleans only the reminders which +/// it created and verifies their absence, so unrelated rows and concurrently running provider suites are isolated. +/// +public abstract class ReminderServiceLifecycleTestRunner +{ + private static readonly TimeSpan Period = TimeSpan.FromMinutes(2); + private readonly IReminderServiceLifecycleHarness _harness; + private readonly int _seed; + private int _grainCounter; + + /// Initializes the runner. + protected ReminderServiceLifecycleTestRunner( + IReminderServiceLifecycleHarness harness, + string providerName, + int seed = 0) + { + _harness = harness ?? throw new ArgumentNullException(nameof(harness)); + ArgumentException.ThrowIfNullOrWhiteSpace(providerName); + ProviderName = providerName; + _seed = seed; + } + + /// Gets the provider name used in diagnostics. + protected string ProviderName { get; } + + /// Guarantee: every active silo reports reminder-service readiness before operations begin. + public virtual Task ReminderService_StartupReadiness() + => RunReminderService_StartupReadiness(CancellationToken.None); + + /// Runs the startup-readiness scenario. + public async Task RunReminderService_StartupReadiness(CancellationToken cancellationToken) + { + const string Guarantee = nameof(ReminderService_StartupReadiness); + await ExecuteWithCleanupAsync( + Guarantee, + cancellationToken, + async () => + { + await _harness.WaitForStartupReadinessAsync(cancellationToken); + if (_harness.ActiveSilos.Count == 0) + { + Fail(Guarantee, "startup") + .WithExpected("at least one ready initial silo") + .WithObserved($"activeSilos=[{string.Join(", ", _harness.ActiveSilos)}]") + .Throw(); + } + }, + _ => Task.CompletedTask); + } + + /// Guarantee: a registration has one owner and one exact delivery. + public virtual Task ReminderService_RegistrationHasSingleOwner() + => RunReminderService_RegistrationHasSingleOwner(CancellationToken.None); + + /// Runs the registration-ownership scenario. + public async Task RunReminderService_RegistrationHasSingleOwner(CancellationToken cancellationToken) + { + const string Guarantee = nameof(ReminderService_RegistrationHasSingleOwner); + const string Name = "registration-owner"; + var grain = CreateGrain(Guarantee); + var due = TimeSpan.FromSeconds(3); + await ExecuteWithCleanupAsync( + Guarantee, + cancellationToken, + async () => + { + var expectedStart = _harness.UtcNow.UtcDateTime + due; + await grain.RegisterOrUpdateAsync(Name, due, Period).WaitAsync(cancellationToken); + await WaitForOwnersAfterRefreshAsync([(grain, Name)], cancellationToken); + if (_harness.GetOwners(grain.GetGrainId(), Name).Count != 1) + { + OwnershipFailure(Guarantee, grain.GetGrainId(), Name, 1).Throw(); + } + + await _harness.WaitForScheduleAsync(grain.GetGrainId(), Name, cancellationToken); + await AssertPersistedAsync(Guarantee, grain.GetGrainId(), Name, expectedStart, Period, cancellationToken); + var tick = _harness.WaitForTickCountAsync(grain.GetGrainId(), Name, 1, cancellationToken); + await _harness.AdvanceAsync(expectedStart - _harness.UtcNow.UtcDateTime, cancellationToken); + await tick; + AssertCounts(Guarantee, grain, Name, owners: null, ticks: 1); + }, + cleanupToken => CleanupAsync(Guarantee, grain, Name, cleanupToken)); + } + + /// Guarantee: updating a reminder changes its schedule without restarting its local owner. + public virtual Task ReminderService_UpdateDoesNotRestartLocalOwner() + => RunReminderService_UpdateDoesNotRestartLocalOwner(CancellationToken.None); + + /// Runs the in-place update scenario. + public async Task RunReminderService_UpdateDoesNotRestartLocalOwner(CancellationToken cancellationToken) + { + const string Guarantee = nameof(ReminderService_UpdateDoesNotRestartLocalOwner); + const string Name = "in-place-update"; + var grain = CreateGrain(Guarantee); + await ExecuteWithCleanupAsync( + Guarantee, + cancellationToken, + async () => + { + await grain.RegisterOrUpdateAsync(Name, TimeSpan.FromSeconds(3), Period).WaitAsync(cancellationToken); + await WaitForOwnersAfterRefreshAsync([(grain, Name)], cancellationToken); + await _harness.WaitForScheduleAsync(grain.GetGrainId(), Name, cancellationToken); + var original = await ReadRequiredAsync(Guarantee, grain.GetGrainId(), Name, cancellationToken); + var owners = _harness.GetOwners(grain.GetGrainId(), Name).ToArray(); + var starts = _harness.GetLocalStartCount(grain.GetGrainId(), Name); + var stops = _harness.GetLocalStopCount(grain.GetGrainId(), Name); + var scheduleChanges = _harness.GetScheduleChangeCount(grain.GetGrainId(), Name); + var due = TimeSpan.FromSeconds(4); + var expectedStart = _harness.UtcNow.UtcDateTime + due; + + var changed = _harness.WaitForScheduleChangeCountAsync( + grain.GetGrainId(), + Name, + scheduleChanges + 1, + cancellationToken); + await grain.RegisterOrUpdateAsync(Name, due, Period + TimeSpan.FromMinutes(1)).WaitAsync(cancellationToken); + await _harness.RefreshAsync(cancellationToken); + await changed; + await _harness.WaitForScheduleAsync(grain.GetGrainId(), Name, cancellationToken); + var updated = await ReadRequiredAsync(Guarantee, grain.GetGrainId(), Name, cancellationToken); + + if (!owners.SequenceEqual(_harness.GetOwners(grain.GetGrainId(), Name)) + || starts != _harness.GetLocalStartCount(grain.GetGrainId(), Name) + || stops != _harness.GetLocalStopCount(grain.GetGrainId(), Name) + || string.Equals(original.ETag, updated.ETag, StringComparison.Ordinal) + || updated.StartAt != expectedStart + || updated.Period != Period + TimeSpan.FromMinutes(1)) + { + Fail(Guarantee, "RegisterOrUpdateReminder") + .WithIdentity(grain.GetGrainId(), Name) + .WithExpected($"same single owner, starts={starts}, stops={stops}, rotated ETag, StartAt={expectedStart:O}") + .WithObserved( + $"owners=[{string.Join(", ", _harness.GetOwners(grain.GetGrainId(), Name))}], " + + $"starts={_harness.GetLocalStartCount(grain.GetGrainId(), Name)}, " + + $"stops={_harness.GetLocalStopCount(grain.GetGrainId(), Name)}, row={Describe(updated)}") + .WithETags(updated.ETag, original.ETag) + .Throw(); + } + }, + cleanupToken => CleanupAsync(Guarantee, grain, Name, cleanupToken)); + } + + /// Guarantee: removal reaches quiescence and cannot deliver a later occurrence. + public virtual Task ReminderService_RemovalReachesQuiescence() + => RunReminderService_RemovalReachesQuiescence(CancellationToken.None); + + /// Runs the removal/quiescence scenario. + public async Task RunReminderService_RemovalReachesQuiescence(CancellationToken cancellationToken) + { + const string Guarantee = nameof(ReminderService_RemovalReachesQuiescence); + const string Name = "removal-quiescence"; + var grain = CreateGrain(Guarantee); + await ExecuteWithCleanupAsync( + Guarantee, + cancellationToken, + async () => + { + await grain.RegisterOrUpdateAsync(Name, TimeSpan.FromSeconds(3), Period).WaitAsync(cancellationToken); + await WaitForOwnersAfterRefreshAsync([(grain, Name)], cancellationToken); + await _harness.WaitForScheduleAsync(grain.GetGrainId(), Name, cancellationToken); + + if (!await grain.UnregisterAsync(Name).WaitAsync(cancellationToken)) + { + Fail(Guarantee, "UnregisterReminder") + .WithIdentity(grain.GetGrainId(), Name) + .WithExpected("successful removal") + .WithObserved("removal returned false") + .Throw(); + } + + var quiescence = _harness.WaitForOwnerCountAsync(grain.GetGrainId(), Name, 0, cancellationToken); + await _harness.RefreshAsync(cancellationToken); + await quiescence; + await AssertAbsentAsync(Guarantee, grain.GetGrainId(), Name, cancellationToken); + await _harness.AdvanceAsync(Period, cancellationToken); + AssertCounts(Guarantee, grain, Name, owners: 0, ticks: 0); + }, + cleanupToken => CleanupAsync(Guarantee, grain, Name, cleanupToken)); + } + + /// Guarantee: a persisted reminder entering the loading window fires at its exact due time. + public virtual Task ReminderService_ExactDueRecovery() + => RunReminderService_ExactDueRecovery(CancellationToken.None); + + /// Runs the exact-due recovery scenario. + public async Task RunReminderService_ExactDueRecovery(CancellationToken cancellationToken) + { + const string Guarantee = nameof(ReminderService_ExactDueRecovery); + const string Name = "exact-due-recovery"; + var grain = CreateGrain(Guarantee); + var due = _harness.ReminderLoadingWindow + + _harness.ReminderRefreshPeriod + + _harness.ReminderRefreshPeriod; + await ExecuteWithCleanupAsync( + Guarantee, + cancellationToken, + async () => + { + var expectedStart = _harness.UtcNow.UtcDateTime + due; + await grain.RegisterOrUpdateAsync(Name, due, Period).WaitAsync(cancellationToken); + var boundaryStep = TimeSpan.FromTicks(1); + var preWindowAdvance = due - _harness.ReminderLoadingWindow - boundaryStep; + await _harness.AdvanceAsync(preWindowAdvance, cancellationToken); + if (_harness.GetOwners(grain.GetGrainId(), Name).Count != 0) + { + Fail(Guarantee, "before loading window") + .WithIdentity(grain.GetGrainId(), Name) + .WithExpected("no local owner immediately before entering the loading window") + .WithObserved($"owners=[{string.Join(", ", _harness.GetOwners(grain.GetGrainId(), Name))}]") + .Throw(); + } + + var owner = _harness.WaitForOwnerCountAsync(grain.GetGrainId(), Name, 1, cancellationToken); + await _harness.AdvanceAsync(boundaryStep, cancellationToken); + await _harness.RefreshAsync(cancellationToken); + await owner; + await _harness.WaitForTopologyReconciliationAsync(cancellationToken); + await _harness.WaitForOwnerCountAsync(grain.GetGrainId(), Name, 1, cancellationToken); + await _harness.WaitForScheduleAsync(grain.GetGrainId(), Name, cancellationToken); + var tick = _harness.WaitForTickCountAsync(grain.GetGrainId(), Name, 1, cancellationToken); + await _harness.AdvanceAsync(expectedStart - _harness.UtcNow.UtcDateTime, cancellationToken); + await tick; + await AssertPersistedAsync(Guarantee, grain.GetGrainId(), Name, expectedStart, Period, cancellationToken); + AssertCounts(Guarantee, grain, Name, owners: null, ticks: 1); + }, + cleanupToken => CleanupAsync(Guarantee, grain, Name, cleanupToken)); + } + + /// Guarantee: registration and ownership reconcile to one owner after a silo joins. + public virtual Task ReminderService_StaleOwnerRegistrationReconciles() + => RunReminderService_StaleOwnerRegistrationReconciles(CancellationToken.None); + + /// Runs stale-owner registration reconciliation. + public async Task RunReminderService_StaleOwnerRegistrationReconciles(CancellationToken cancellationToken) + { + const string Guarantee = nameof(ReminderService_StaleOwnerRegistrationReconciles); + var reminders = CreateGrains(Guarantee, 16); + var phase = "join"; + SiloAddress? joined = null; + await ExecuteWithCleanupAsync( + Guarantee, + cancellationToken, + async () => + { + try + { + joined = await _harness.JoinOneSiloAsync(cancellationToken); + phase = "registration"; + await Task.WhenAll(reminders.Select(item => + item.Grain.RegisterOrUpdateAsync(item.Name, TimeSpan.FromSeconds(3), Period) + .WaitAsync(cancellationToken))); + + phase = "topology reconciliation"; + await _harness.WaitForTopologyReconciliationAsync(cancellationToken); + phase = "owner reconciliation"; + var ownerWaits = reminders + .Select(item => _harness.WaitForOwnerCountAsync( + item.Grain.GetGrainId(), + item.Name, + 1, + cancellationToken)) + .ToArray(); + await _harness.RefreshAsync(cancellationToken); + await Task.WhenAll(ownerWaits); + await _harness.WaitForTopologyReconciliationAsync(cancellationToken); + await Task.WhenAll(reminders.Select(item => + _harness.WaitForOwnerCountAsync( + item.Grain.GetGrainId(), + item.Name, + 1, + cancellationToken))); + foreach (var (grain, name) in reminders) + { + phase = $"schedule reconciliation for {grain.GetGrainId()}/{name}"; + await _harness.WaitForScheduleAsync(grain.GetGrainId(), name, cancellationToken); + if (_harness.GetOwners(grain.GetGrainId(), name).Count != 1) + { + OwnershipFailure(Guarantee, grain.GetGrainId(), name, 1).Throw(); + } + } + } + catch (OperationCanceledException exception) when (cancellationToken.IsCancellationRequested) + { + Fail(Guarantee, phase) + .WithExpected("all registrations reconcile to one current owner with an armed schedule") + .WithObserved(string.Join( + "; ", + reminders.Select(item => + $"{item.Grain.GetGrainId()}/{item.Name}=" + + $"[{string.Join(", ", _harness.GetOwners(item.Grain.GetGrainId(), item.Name))}]"))) + .Throw(exception); + } + }, + async cleanupToken => + { + if (joined is not null && _harness.ActiveSilos.Contains(joined)) + { + await _harness.LeaveSiloAsync(joined, cleanupToken); + await _harness.WaitForTopologyReconciliationAsync(cleanupToken); + } + + await CleanupAsync(Guarantee, reminders, cleanupToken); + }); + } + + /// Guarantee: one-silo join/leave transfers ownership without duplicates or missed delivery. + public virtual Task ReminderService_OneSiloJoinLeaveTransfersOwnership() + => RunReminderService_OneSiloJoinLeaveTransfersOwnership(CancellationToken.None); + + /// Runs the one-silo join/leave ownership-transfer scenario. + public async Task RunReminderService_OneSiloJoinLeaveTransfersOwnership(CancellationToken cancellationToken) + { + const string Guarantee = nameof(ReminderService_OneSiloJoinLeaveTransfersOwnership); + var reminders = new List<(IReminderServiceTestGrain Grain, string Name)>(); + SiloAddress? joined = null; + await ExecuteWithCleanupAsync( + Guarantee, + cancellationToken, + async () => + { + joined = await _harness.JoinOneSiloAsync(cancellationToken); + await _harness.WaitForTopologyReconciliationAsync(cancellationToken); + var grain = CreateGrainOwnedBy(joined, Guarantee); + const string Name = "join-leave-owner"; + reminders.Add((grain, Name)); + var due = TimeSpan.FromSeconds(3); + var firstTickTime = _harness.UtcNow.UtcDateTime + due; + + await grain.RegisterOrUpdateAsync(Name, due, Period).WaitAsync(cancellationToken); + await WaitForOwnersAfterRefreshAsync(reminders, cancellationToken); + var joinedOwner = _harness.GetOwners(grain.GetGrainId(), Name); + if (joinedOwner.Count != 1 || !joinedOwner[0].Equals(joined)) + { + OwnershipFailure(Guarantee, grain.GetGrainId(), Name, 1) + .WithExpected($"the joined silo {joined} owns the selected reminder") + .WithObserved($"owners=[{string.Join(", ", joinedOwner)}]") + .Throw(); + } + + await _harness.LeaveSiloAsync(joined, cancellationToken); + joined = null; + await _harness.WaitForTopologyReconciliationAsync(cancellationToken); + await _harness.WaitForOwnerCountAsync( + grain.GetGrainId(), + Name, + 1, + cancellationToken); + await _harness.WaitForTopologyReconciliationAsync(cancellationToken); + await _harness.WaitForOwnerCountAsync( + grain.GetGrainId(), + Name, + 1, + cancellationToken); + await _harness.WaitForScheduleAsync(grain.GetGrainId(), Name, cancellationToken); + var tick = _harness.WaitForTickCountAsync( + grain.GetGrainId(), + Name, + 1, + cancellationToken); + await _harness.AdvanceAsync(firstTickTime - _harness.UtcNow.UtcDateTime, cancellationToken); + await tick; + AssertCounts( + Guarantee, + grain, + Name, + owners: null, + ticks: 1); + }, + async cleanupToken => + { + if (joined is not null && _harness.ActiveSilos.Contains(joined)) + { + await _harness.LeaveSiloAsync(joined, cleanupToken); + await _harness.WaitForTopologyReconciliationAsync(cleanupToken); + } + + await CleanupAsync(Guarantee, reminders, cleanupToken); + }); + } + + /// Guarantee: scenario cleanup removes only rows owned by that scenario. + public virtual Task ReminderService_CleanupIsIsolated() + => RunReminderService_CleanupIsIsolated(CancellationToken.None); + + /// Runs cleanup isolation. + public async Task RunReminderService_CleanupIsIsolated(CancellationToken cancellationToken) + { + const string Guarantee = nameof(ReminderService_CleanupIsIsolated); + var sentinel = CreateGrain($"{Guarantee}/sentinel"); + var subject = CreateGrain($"{Guarantee}/subject"); + const string SentinelName = "unrelated-sentinel"; + const string SubjectName = "scenario-owned"; + await ExecuteWithCleanupAsync( + Guarantee, + cancellationToken, + async () => + { + await Task.WhenAll( + sentinel.RegisterOrUpdateAsync(SentinelName, TimeSpan.FromMinutes(1), Period).WaitAsync(cancellationToken), + subject.RegisterOrUpdateAsync(SubjectName, TimeSpan.FromMinutes(1), Period).WaitAsync(cancellationToken)); + await CleanupAsync(Guarantee, subject, SubjectName, cancellationToken); + var sentinelRow = await _harness.ReminderTable.ReadRow(sentinel.GetGrainId(), SentinelName).WaitAsync(cancellationToken); + if (sentinelRow is null) + { + Fail(Guarantee, "cleanup") + .WithIdentity(sentinel.GetGrainId(), SentinelName) + .WithExpected("unrelated sentinel remains registered") + .WithObserved("sentinel row was removed") + .Throw(); + } + }, + async cleanupToken => + { + await CleanupAsync(Guarantee, sentinel, SentinelName, cleanupToken); + await CleanupAsync(Guarantee, subject, SubjectName, cleanupToken); + }); + } + + private List<(IReminderServiceTestGrain Grain, string Name)> CreateGrains(string guarantee, int count) + => Enumerable.Range(0, count) + .Select(index => (CreateGrain($"{guarantee}/{index}"), $"churn-{index.ToString("D2", CultureInfo.InvariantCulture)}")) + .ToList(); + + private IReminderServiceTestGrain CreateGrain(string label) + { + var ordinal = Interlocked.Increment(ref _grainCounter); + var key = ReminderTestData.CreateGuid(_seed, $"{ProviderName}/{label}/{ordinal}"); + return _harness.GrainFactory.GetGrain(key); + } + + private IReminderServiceTestGrain CreateGrainOwnedBy(SiloAddress owner, string label) + { + var ordinal = Interlocked.Increment(ref _grainCounter); + for (var candidate = 0; candidate < ushort.MaxValue; candidate++) + { + var key = ReminderTestData.CreateGuid( + _seed, + $"{ProviderName}/{label}/{ordinal}/{candidate.ToString(CultureInfo.InvariantCulture)}"); + var grain = _harness.GrainFactory.GetGrain(key); + if (_harness.IsOwner(owner, grain.GetGrainId())) + { + return grain; + } + } + + Fail(label, "identity selection") + .WithExpected($"a deterministic grain identity owned by {owner}") + .WithObserved("no owned identity in 65,535 deterministic candidates") + .Throw(); + return null!; + } + + private async Task ExecuteWithCleanupAsync( + string guarantee, + CancellationToken scenarioCancellationToken, + Func scenario, + Func cleanup) + { + ExceptionDispatchInfo? scenarioFailure = null; + try + { + await _harness.WaitForStartupReadinessAsync(scenarioCancellationToken); + await scenario(); + } + catch (Exception exception) + { + scenarioFailure = ExceptionDispatchInfo.Capture(exception); + } + + Exception? cleanupFailure = null; + using (var cleanupCancellation = new CancellationTokenSource(TimeSpan.FromSeconds(30))) + { + try + { + await cleanup(cleanupCancellation.Token); + } + catch (Exception exception) + { + cleanupFailure = exception; + } + } + + if (scenarioFailure is not null) + { + if (cleanupFailure is not null) + { + scenarioFailure.SourceException.Data["ReminderServiceLifecycleCleanupFailure"] = cleanupFailure.ToString(); + } + + scenarioFailure.Throw(); + } + + if (cleanupFailure is not null) + { + Fail(guarantee, "scenario cleanup") + .WithExpected("all scenario-owned reminders absent and all temporary topology restored") + .WithObserved($"{cleanupFailure.GetType().FullName}: {cleanupFailure.Message}") + .Throw(cleanupFailure); + } + } + + private async Task WaitForOwnersAfterRefreshAsync( + IReadOnlyList<(IReminderServiceTestGrain Grain, string Name)> reminders, + CancellationToken cancellationToken) + { + var ownerWaits = reminders + .Select(item => _harness.WaitForOwnerCountAsync( + item.Grain.GetGrainId(), + item.Name, + 1, + cancellationToken)) + .ToArray(); + await _harness.RefreshAsync(cancellationToken); + await Task.WhenAll(ownerWaits); + } + + private async Task CleanupAsync( + string guarantee, + IReadOnlyList<(IReminderServiceTestGrain Grain, string Name)> reminders, + CancellationToken cancellationToken) + { + await Task.WhenAll(reminders.Select(item => + RemovePersistedRowAsync(guarantee, item.Grain.GetGrainId(), item.Name, cancellationToken))); + var quiescence = reminders + .Select(item => _harness.WaitForOwnerCountAsync( + item.Grain.GetGrainId(), + item.Name, + 0, + cancellationToken)) + .ToArray(); + await _harness.RefreshAsync(cancellationToken); + await Task.WhenAll(quiescence); + await Task.WhenAll(reminders.Select(item => + AssertAbsentAsync(guarantee, item.Grain.GetGrainId(), item.Name, cancellationToken))); + } + + private Task CleanupAsync( + string guarantee, + IReminderServiceTestGrain grain, + string name, + CancellationToken cancellationToken) + => CleanupAsync(guarantee, [(grain, name)], cancellationToken); + + private async Task RemovePersistedRowAsync( + string guarantee, + GrainId grainId, + string name, + CancellationToken cancellationToken) + { + var row = await _harness.ReminderTable.ReadRow(grainId, name).WaitAsync(cancellationToken); + if (row is null) + { + return; + } + + if (!await _harness.ReminderTable.RemoveRow(grainId, name, row.ETag!).WaitAsync(cancellationToken)) + { + Fail(guarantee, "scenario cleanup remove") + .WithIdentity(grainId, name) + .WithExpected("scenario-owned row removed using its current ETag") + .WithObserved(Describe(row)) + .WithETags(row.ETag, supplied: row.ETag) + .Throw(); + } + } + + private async Task AssertPersistedAsync( + string guarantee, + GrainId grainId, + string name, + DateTime expectedStart, + TimeSpan expectedPeriod, + CancellationToken cancellationToken) + { + var row = await ReadRequiredAsync(guarantee, grainId, name, cancellationToken); + if (row.StartAt != expectedStart || row.Period != expectedPeriod || string.IsNullOrEmpty(row.ETag)) + { + Fail(guarantee, "ReadRow") + .WithIdentity(grainId, name) + .WithExpected($"StartAt={expectedStart:O}, Period={expectedPeriod}, non-empty ETag") + .WithObserved(Describe(row)) + .WithSchedule(row.StartAt, row.Period) + .WithETags(row.ETag) + .Throw(); + } + } + + private async Task ReadRequiredAsync( + string guarantee, + GrainId grainId, + string name, + CancellationToken cancellationToken) + { + var row = await _harness.ReminderTable.ReadRow(grainId, name).WaitAsync(cancellationToken); + if (row is null) + { + Fail(guarantee, "ReadRow") + .WithIdentity(grainId, name) + .WithExpected("one persisted row") + .WithObserved("") + .Throw(); + } + + return row!; + } + + private async Task AssertAbsentAsync( + string guarantee, + GrainId grainId, + string name, + CancellationToken cancellationToken) + { + var row = await _harness.ReminderTable.ReadRow(grainId, name).WaitAsync(cancellationToken); + if (row is not null) + { + Fail(guarantee, "cleanup") + .WithIdentity(grainId, name) + .WithExpected("row absent") + .WithObserved(Describe(row)) + .WithETags(row.ETag) + .Throw(); + } + } + + private void AssertCounts( + string guarantee, + IReminderServiceTestGrain grain, + string name, + int? owners, + int ticks) + { + var diagnosticTicks = _harness.GetTickCount(grain.GetGrainId(), name); + var actualOwners = _harness.GetOwners(grain.GetGrainId(), name); + if ((owners is { } expectedOwners && actualOwners.Count != expectedOwners) + || diagnosticTicks != ticks) + { + Fail(guarantee, "exact counters") + .WithIdentity(grain.GetGrainId(), name) + .WithExpected($"owners={(owners?.ToString(CultureInfo.InvariantCulture) ?? "")}, completedTicks={ticks}") + .WithObserved( + $"owners={actualOwners.Count} [{string.Join(", ", actualOwners)}], " + + $"completedTicks={diagnosticTicks}") + .Throw(); + } + } + + private ReminderFailureReport OwnershipFailure(string guarantee, GrainId grainId, string name, int expected) + => Fail(guarantee, "ownership reconciliation") + .WithIdentity(grainId, name) + .WithExpected($"exactly {expected} local owner(s)") + .WithObserved($"owners=[{string.Join(", ", _harness.GetOwners(grainId, name))}]"); + + private ReminderFailureReport Fail(string guarantee, string operation) + => ReminderFailureReport.Create(ProviderName, guarantee, operation) + .WithDetail("seed", _seed.ToString(CultureInfo.InvariantCulture)) + .WithDetail("clock", _harness.UtcNow.ToString("O", CultureInfo.InvariantCulture)) + .WithDetail("activeSilos", string.Join(", ", _harness.ActiveSilos)); + + private static string Describe(ReminderEntry row) + => $"(GrainId={row.GrainId}, ReminderName='{row.ReminderName}', StartAt={row.StartAt:O}, " + + $"Period={row.Period}, ETag='{row.ETag}')"; +} diff --git a/src/Orleans.Reminders/Orleans.Reminders.csproj b/src/Orleans.Reminders/Orleans.Reminders.csproj index 734c52a6098..f7d5c661867 100644 --- a/src/Orleans.Reminders/Orleans.Reminders.csproj +++ b/src/Orleans.Reminders/Orleans.Reminders.csproj @@ -21,9 +21,9 @@ + - diff --git a/src/Orleans.Runtime/Orleans.Runtime.csproj b/src/Orleans.Runtime/Orleans.Runtime.csproj index a541e66f68f..66035b1b7ab 100644 --- a/src/Orleans.Runtime/Orleans.Runtime.csproj +++ b/src/Orleans.Runtime/Orleans.Runtime.csproj @@ -40,6 +40,7 @@ + diff --git a/test/Extensions/Orleans.AWS.Tests/Orleans.AWS.Tests.csproj b/test/Extensions/Orleans.AWS.Tests/Orleans.AWS.Tests.csproj index 6e467d28a39..714ddf05d5a 100644 --- a/test/Extensions/Orleans.AWS.Tests/Orleans.AWS.Tests.csproj +++ b/test/Extensions/Orleans.AWS.Tests/Orleans.AWS.Tests.csproj @@ -19,6 +19,7 @@ + \ No newline at end of file diff --git a/test/Extensions/Orleans.AWS.Tests/Reminder/DynamoDBRemindersTableTests.cs b/test/Extensions/Orleans.AWS.Tests/Reminder/DynamoDBRemindersTableTests.cs index 38fda15b490..01e9e2b63e3 100644 --- a/test/Extensions/Orleans.AWS.Tests/Reminder/DynamoDBRemindersTableTests.cs +++ b/test/Extensions/Orleans.AWS.Tests/Reminder/DynamoDBRemindersTableTests.cs @@ -2,14 +2,75 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Orleans.Configuration; +using Orleans.Hosting; using Orleans.Reminders.DynamoDB; +using Orleans.Testing.Reminders; +using Orleans.TestingHost; using TestExtensions; using UnitTests; using UnitTests.RemindersTest; +using UnitTests.TimerTests; using Xunit; namespace AWSUtils.Tests.RemindersTest { + public sealed class DynamoDBReminderServiceLifecycleFixture : BaseInProcessTestClusterFixture + { + private ReminderTestClock? _clock; + + public ReminderTestClock Clock + { + get + { + EnsurePreconditionsMet(); + return _clock ?? throw new InvalidOperationException("The reminder clock has not been configured."); + } + } + + protected override void CheckPreconditionsOrThrow() + { + if (!AWSTestConstants.IsDynamoDbAvailable) + { + throw Xunit.Sdk.SkipException.ForSkip("Unable to connect to AWS DynamoDB simulator"); + } + } + + protected override void ConfigureTestCluster(InProcessTestClusterBuilder builder) + { + _clock = builder.AddReminderTestClock(); + builder.ConfigureSilo((_, siloBuilder) => + siloBuilder.UseDynamoDBReminderService(options => + options.ParseConnectionString($"Service={AWSTestConstants.DynamoDbService}"))); + } + + public override async ValueTask DisposeAsync() + { + try + { + await base.DisposeAsync(); + } + finally + { + _clock?.Dispose(); + } + } + } + + [TestCategory("Reminders"), TestCategory("AWS"), TestCategory("DynamoDb")] + [Collection(TestEnvironmentFixture.DefaultCollection)] + [TestSuite("Functional")] + [TestProvider("DynamoDB")] + [TestArea("Reminders")] + public sealed class DynamoDBReminderServiceLifecycleTests + : ReminderServiceLifecycleTestsBase, IClassFixture + { + public DynamoDBReminderServiceLifecycleTests(DynamoDBReminderServiceLifecycleFixture fixture) + : base(fixture.Clock, fixture.HostedCluster, "DynamoDB") + { + fixture.EnsurePreconditionsMet(); + } + } + /// /// Tests DynamoDB implementation of the Orleans reminders table for storing and retrieving grain reminders. /// diff --git a/test/Extensions/Orleans.AdoNet.Tests/Reminders/ReminderServiceLifecycleTests_AdoNet.cs b/test/Extensions/Orleans.AdoNet.Tests/Reminders/ReminderServiceLifecycleTests_AdoNet.cs new file mode 100644 index 00000000000..c6f8e9682af --- /dev/null +++ b/test/Extensions/Orleans.AdoNet.Tests/Reminders/ReminderServiceLifecycleTests_AdoNet.cs @@ -0,0 +1,110 @@ +using Orleans.Hosting; +using Orleans.Tests.SqlUtils; +using Orleans.Testing.Reminders; +using Orleans.TestingHost; +using TestExtensions; +using UnitTests.TimerTests; + +namespace Tester.AdoNet.Reminders; + +public abstract class AdoNetReminderServiceLifecycleFixture : BaseInProcessTestClusterFixture +{ + private string _connectionString = null!; + private ReminderTestClock? _clock; + + protected abstract string Invariant { get; } + + protected abstract string DatabaseName { get; } + + public ReminderTestClock Clock + { + get + { + EnsurePreconditionsMet(); + return _clock ?? throw new InvalidOperationException("The reminder clock has not been configured."); + } + } + + protected override void CheckPreconditionsOrThrow() + => UnitTests.General.RelationalStorageForTesting.CheckPreconditionsOrThrow(Invariant); + + public override async ValueTask InitializeAsync() + { + if (!PreconditionsMet) + { + return; + } + + var relationalStorage = await UnitTests.General.RelationalStorageForTesting.SetupInstance( + Invariant, + DatabaseName, + cancellationToken: TestContext.Current.CancellationToken); + _connectionString = relationalStorage.CurrentConnectionString; + await base.InitializeAsync(); + } + + protected override void ConfigureTestCluster(InProcessTestClusterBuilder builder) + { + _clock = builder.AddReminderTestClock(); + builder.ConfigureSilo((_, siloBuilder) => + siloBuilder.UseAdoNetReminderService(options => + { + options.ConnectionString = _connectionString; + options.Invariant = Invariant; + })); + } + + public override async ValueTask DisposeAsync() + { + try + { + await base.DisposeAsync(); + } + finally + { + _clock?.Dispose(); + } + } +} + +public sealed class PostgreSqlReminderServiceLifecycleFixture : AdoNetReminderServiceLifecycleFixture +{ + protected override string Invariant => AdoNetInvariants.InvariantNamePostgreSql; + + protected override string DatabaseName => "OrleansTest_PostgreSql_ReminderLifecycle"; +} + +public sealed class MySqlReminderServiceLifecycleFixture : AdoNetReminderServiceLifecycleFixture +{ + protected override string Invariant => AdoNetInvariants.InvariantNameMySql; + + protected override string DatabaseName => "OrleansTest_MySql_ReminderLifecycle"; +} + +[TestSuite("Functional")] +[TestProvider("PostgreSql")] +[TestArea("Reminders")] +[TestCategory("Functional"), TestCategory("Reminders"), TestCategory("AdoNet"), TestCategory("PostgreSql")] +public sealed class PostgreSqlReminderServiceLifecycleTests + : ReminderServiceLifecycleTestsBase, IClassFixture +{ + public PostgreSqlReminderServiceLifecycleTests(PostgreSqlReminderServiceLifecycleFixture fixture) + : base(fixture.Clock, fixture.HostedCluster, "AdoNet.PostgreSql") + { + fixture.EnsurePreconditionsMet(); + } +} + +[TestSuite("Functional")] +[TestProvider("MySql")] +[TestArea("Reminders")] +[TestCategory("Functional"), TestCategory("Reminders"), TestCategory("AdoNet"), TestCategory("MySql")] +public sealed class MySqlReminderServiceLifecycleTests + : ReminderServiceLifecycleTestsBase, IClassFixture +{ + public MySqlReminderServiceLifecycleTests(MySqlReminderServiceLifecycleFixture fixture) + : base(fixture.Clock, fixture.HostedCluster, "AdoNet.MySql") + { + fixture.EnsurePreconditionsMet(); + } +} diff --git a/test/Extensions/Orleans.AdoNet.Tests/Reminders/ReminderTests_AdoNet_SqlServer.cs b/test/Extensions/Orleans.AdoNet.Tests/Reminders/ReminderTests_AdoNet_SqlServer.cs index c48915abe07..db4114f4700 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Reminders/ReminderTests_AdoNet_SqlServer.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Reminders/ReminderTests_AdoNet_SqlServer.cs @@ -17,6 +17,20 @@ namespace Tester.AdoNet.Reminders { + [TestSuite("Functional")] + [TestProvider("SqlServer")] + [TestArea("Reminders")] + [TestCategory("Reminders"), TestCategory("AdoNet"), TestCategory("SqlServer")] + public sealed class ReminderServiceLifecycleTests_AdoNet_SqlServer + : ReminderServiceLifecycleTestsBase, IClassFixture + { + public ReminderServiceLifecycleTests_AdoNet_SqlServer(ReminderTests_AdoNet_SqlServer.Fixture fixture) + : base(fixture.ReminderClock, fixture.HostedCluster, "AdoNet.SqlServer") + { + fixture.EnsurePreconditionsMet(); + } + } + /// /// Integration tests for Orleans reminders functionality using SQL Server as the reminder service backend. /// @@ -101,7 +115,7 @@ public async ValueTask InitializeAsync() await ClearReminderTableAsync(TestContext.Current.CancellationToken) .WaitAsync(TestConstants.InitTimeout, TestContext.Current.CancellationToken); } - + // Basic tests [Fact] diff --git a/test/Extensions/Orleans.Azure.Tests/Reminder/ReminderTests_AzureTable.cs b/test/Extensions/Orleans.Azure.Tests/Reminder/ReminderTests_AzureTable.cs index 74569db5563..29e17e0d4f6 100644 --- a/test/Extensions/Orleans.Azure.Tests/Reminder/ReminderTests_AzureTable.cs +++ b/test/Extensions/Orleans.Azure.Tests/Reminder/ReminderTests_AzureTable.cs @@ -14,6 +14,20 @@ namespace Tester.AzureUtils.TimerTests { + [TestSuite("Functional")] + [TestProvider("AzureStorage")] + [TestArea("Reminders")] + [TestCategory("Reminders"), TestCategory("AzureStorage")] + public sealed class ReminderServiceLifecycleTests_AzureTable + : ReminderServiceLifecycleTestsBase, IClassFixture + { + public ReminderServiceLifecycleTests_AzureTable(ReminderTests_AzureTable.Fixture fixture) + : base(fixture.ReminderClock, fixture.HostedCluster, "AzureStorage") + { + fixture.EnsurePreconditionsMet(); + } + } + /// /// Tests for Azure Table Storage-based reminder service, including basic operations, failover, and multi-grain scenarios. /// diff --git a/test/Extensions/Orleans.Cosmos.Tests/ReminderTests_Cosmos.cs b/test/Extensions/Orleans.Cosmos.Tests/ReminderTests_Cosmos.cs index 59320f9c42c..630958f363b 100644 --- a/test/Extensions/Orleans.Cosmos.Tests/ReminderTests_Cosmos.cs +++ b/test/Extensions/Orleans.Cosmos.Tests/ReminderTests_Cosmos.cs @@ -11,6 +11,20 @@ namespace Tester.Cosmos.Reminders; +[TestSuite("Functional")] +[TestProvider("Cosmos")] +[TestArea("Reminders")] +[TestCategory("Reminders"), TestCategory("Cosmos")] +public sealed class ReminderServiceLifecycleTests_Cosmos + : ReminderServiceLifecycleTestsBase, IClassFixture +{ + public ReminderServiceLifecycleTests_Cosmos(ReminderTests_Cosmos.Fixture fixture) + : base(fixture.ReminderClock, fixture.HostedCluster, "Cosmos") + { + fixture.EnsurePreconditionsMet(); + } +} + /// /// Tests for Orleans reminders functionality using Azure Cosmos DB as the reminder service backing store. /// diff --git a/test/Extensions/Orleans.Redis.Tests/Orleans.Redis.Tests.csproj b/test/Extensions/Orleans.Redis.Tests/Orleans.Redis.Tests.csproj index 2234e120da4..bd826f71e0f 100644 --- a/test/Extensions/Orleans.Redis.Tests/Orleans.Redis.Tests.csproj +++ b/test/Extensions/Orleans.Redis.Tests/Orleans.Redis.Tests.csproj @@ -22,6 +22,7 @@ + diff --git a/test/Extensions/Orleans.Redis.Tests/Reminders/RedisReminderTableTests.cs b/test/Extensions/Orleans.Redis.Tests/Reminders/RedisReminderTableTests.cs index b45a18f9206..aa67670f45c 100644 --- a/test/Extensions/Orleans.Redis.Tests/Reminders/RedisReminderTableTests.cs +++ b/test/Extensions/Orleans.Redis.Tests/Reminders/RedisReminderTableTests.cs @@ -3,16 +3,75 @@ using Microsoft.Extensions.Options; using Newtonsoft.Json; using Orleans.Configuration; +using Orleans.Hosting; using Orleans.Reminders.Redis; using Orleans.Runtime; +using Orleans.Testing.Reminders; +using Orleans.TestingHost; using StackExchange.Redis; using TestExtensions; using UnitTests; using UnitTests.RemindersTest; +using UnitTests.TimerTests; using Xunit; namespace Tester.Redis.Reminders { + public sealed class RedisReminderServiceLifecycleFixture : BaseInProcessTestClusterFixture + { + private ReminderTestClock? _clock; + + public ReminderTestClock Clock + { + get + { + EnsurePreconditionsMet(); + return _clock ?? throw new InvalidOperationException("The reminder clock has not been configured."); + } + } + + protected override void CheckPreconditionsOrThrow() => TestUtils.CheckForRedis(); + + protected override void ConfigureTestCluster(InProcessTestClusterBuilder builder) + { + _clock = builder.AddReminderTestClock(); + builder.ConfigureSilo((_, siloBuilder) => + siloBuilder.UseRedisReminderService(options => + { + options.ConfigurationOptions = ConfigurationOptions.Parse( + TestDefaultConfiguration.RedisConnectionString!); + options.EntryExpiry = TimeSpan.FromHours(1); + })); + } + + public override async ValueTask DisposeAsync() + { + try + { + await base.DisposeAsync(); + } + finally + { + _clock?.Dispose(); + } + } + } + + [TestCategory("Redis"), TestCategory("Reminders"), TestCategory("Functional")] + [Collection(TestEnvironmentFixture.DefaultCollection)] + [TestSuite("Functional")] + [TestProvider("Redis")] + [TestArea("Reminders")] + public sealed class RedisReminderServiceLifecycleTests + : ReminderServiceLifecycleTestsBase, IClassFixture + { + public RedisReminderServiceLifecycleTests(RedisReminderServiceLifecycleFixture fixture) + : base(fixture.Clock, fixture.HostedCluster, "Redis") + { + fixture.EnsurePreconditionsMet(); + } + } + /// /// Tests for Redis reminder table implementation. /// @@ -23,7 +82,7 @@ namespace Tester.Redis.Reminders [TestArea("Reminders")] public class RedisRemindersTableTests : ReminderTableTestsBase { - public RedisRemindersTableTests(ConnectionStringFixture fixture, CommonFixture clusterFixture) : base (fixture, clusterFixture, CreateFilters()) + public RedisRemindersTableTests(ConnectionStringFixture fixture, CommonFixture clusterFixture) : base(fixture, clusterFixture, CreateFilters()) { TestUtils.CheckForRedis(); } @@ -46,7 +105,7 @@ protected override IReminderTable CreateRemindersTable() { ConfigurationOptions = ConfigurationOptions.Parse(GetConnectionString().Result), EntryExpiry = TimeSpan.FromHours(1) - })); + })); if (reminderTable == null) { diff --git a/test/Extensions/Orleans.Reminders.Firestore.Tests/FirestoreRemindersTests.cs b/test/Extensions/Orleans.Reminders.Firestore.Tests/FirestoreRemindersTests.cs index 40697a25c18..1a297ffd988 100644 --- a/test/Extensions/Orleans.Reminders.Firestore.Tests/FirestoreRemindersTests.cs +++ b/test/Extensions/Orleans.Reminders.Firestore.Tests/FirestoreRemindersTests.cs @@ -1,14 +1,71 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using UnitTests; -using TestExtensions; -using UnitTests.RemindersTest; +using Orleans.Hosting; using Orleans.Reminders.Firestore; using Orleans.Runtime; - +using Orleans.Testing.Reminders; +using Orleans.TestingHost; +using TestExtensions; +using UnitTests; +using UnitTests.RemindersTest; +using UnitTests.TimerTests; namespace Orleans.Reminders.Firestore.Tests; +public sealed class FirestoreReminderServiceLifecycleFixture : BaseInProcessTestClusterFixture +{ + private ReminderTestClock? _clock; + + public ReminderTestClock Clock + { + get + { + EnsurePreconditionsMet(); + return _clock ?? throw new InvalidOperationException("The reminder clock has not been configured."); + } + } + + protected override void CheckPreconditionsOrThrow() => _ = GoogleEmulatorHost.FirestoreEndpoint; + + protected override void ConfigureTestCluster(InProcessTestClusterBuilder builder) + { + _clock = builder.AddReminderTestClock(); + builder.ConfigureSilo((_, siloBuilder) => + siloBuilder.UseFirestoreReminderService(options => + { + options.ProjectId = GoogleEmulatorHost.ProjectId; + options.EmulatorHost = GoogleEmulatorHost.FirestoreEndpoint; + })); + } + + public override async ValueTask DisposeAsync() + { + try + { + await base.DisposeAsync(); + } + finally + { + _clock?.Dispose(); + } + } +} + +[TestSuite("Functional")] +[TestProvider("GoogleCloud")] +[TestArea("Reminders")] +[TestCategory("Reminders"), TestCategory("Firestore"), TestCategory("GoogleCloud"), TestCategory("Functional")] +[Collection(TestEnvironmentFixture.DefaultCollection)] +public sealed class FirestoreReminderServiceLifecycleTests + : ReminderServiceLifecycleTestsBase, IClassFixture +{ + public FirestoreReminderServiceLifecycleTests(FirestoreReminderServiceLifecycleFixture fixture) + : base(fixture.Clock, fixture.HostedCluster, "Firestore") + { + fixture.EnsurePreconditionsMet(); + } +} + [TestSuite("Functional")] [TestProvider("GoogleCloud")] [TestArea("Reminders")] diff --git a/test/Extensions/Orleans.Reminders.Firestore.Tests/Orleans.Reminders.Firestore.Tests.csproj b/test/Extensions/Orleans.Reminders.Firestore.Tests/Orleans.Reminders.Firestore.Tests.csproj index 494bc07168d..37313411031 100644 --- a/test/Extensions/Orleans.Reminders.Firestore.Tests/Orleans.Reminders.Firestore.Tests.csproj +++ b/test/Extensions/Orleans.Reminders.Firestore.Tests/Orleans.Reminders.Firestore.Tests.csproj @@ -9,6 +9,7 @@ + diff --git a/test/Orleans.Reminders.TestKit.Tests/ReminderServiceLifecycleConformanceTests.cs b/test/Orleans.Reminders.TestKit.Tests/ReminderServiceLifecycleConformanceTests.cs new file mode 100644 index 00000000000..6b557e074d4 --- /dev/null +++ b/test/Orleans.Reminders.TestKit.Tests/ReminderServiceLifecycleConformanceTests.cs @@ -0,0 +1,407 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Configuration; +using Orleans.Reminders.TestKit; +using Orleans.Runtime; +using Orleans.Testing.Reminders; +using Orleans.TestingHost; +using Xunit; + +namespace Orleans.Reminders.TestKit.Tests; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class ReminderServiceLifecycleCollection +{ + public const string Name = "Reminder service lifecycle conformance"; +} + +public sealed class ReminderServiceLifecycleFixture : IAsyncLifetime +{ + private static readonly TimeSpan LoadingWindow = TimeSpan.FromSeconds(5); + private InProcessTestCluster? _cluster; + private ReminderTestClock? _clock; + + public ReminderServiceLifecycleFixture() + { + } + + internal ReminderServiceLifecycleFixture(ReminderTestClock clock) + { + _clock = clock; + } + + public ReminderServiceLifecycleHarness Harness { get; private set; } = null!; + + public async ValueTask InitializeAsync() + { + try + { + var builder = new InProcessTestClusterBuilder(1); + builder.ConfigureSilo((_, siloBuilder) => + siloBuilder.Configure(options => options.UseVirtualBucketsConsistentRing = false)); + builder.UseIdealizedReminderTable( + configureReminderOptions: options => options.ReminderLoadingWindow = LoadingWindow); + _clock = ReminderTestClock.Attach( + builder, + minimumReminderPeriod: TimeSpan.FromSeconds(1), + refreshReminderListPeriod: TimeSpan.FromSeconds(1)); + _cluster = builder.Build(); + await _cluster.DeployAsync(TestContext.Current.CancellationToken); + Harness = new ReminderServiceLifecycleHarness( + _cluster, + _clock, + _clock.DiagnosticObserver, + LoadingWindow); + } + catch (Exception initializationException) + { + try + { + await DisposeAsync(); + } + catch (Exception cleanupException) + { + initializationException.Data["ReminderServiceLifecycleFixture.CleanupException"] = cleanupException; + } + + throw; + } + } + + public async ValueTask DisposeAsync() + { + var cluster = _cluster; + var clock = _clock; + _cluster = null; + _clock = null; + + try + { + if (cluster is not null) + { + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(1)); + await cluster.StopAllSilosAsync(cancellation.Token); + } + } + finally + { + try + { + if (cluster is not null) + { + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(1)); + await cluster.DisposeAsync().AsTask().WaitAsync(cancellation.Token); + } + } + finally + { + clock?.Dispose(); + } + } + } +} + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Reminders")] +[TestCategory("BVT"), TestCategory("Reminders"), TestCategory("ReminderTestKit")] +[Collection(ReminderServiceLifecycleCollection.Name)] +public sealed class ReminderServiceLifecycleConformanceTests +{ + [Fact] + public Task ReminderService_StartupReadiness() + => RunAsync((runner, token) => runner.RunReminderService_StartupReadiness(token)); + + [Fact] + public Task ReminderService_RegistrationHasSingleOwner() + => RunAsync((runner, token) => runner.RunReminderService_RegistrationHasSingleOwner(token)); + + [Fact] + public Task ReminderService_UpdateDoesNotRestartLocalOwner() + => RunAsync((runner, token) => runner.RunReminderService_UpdateDoesNotRestartLocalOwner(token)); + + [Fact] + public Task ReminderService_RemovalReachesQuiescence() + => RunAsync((runner, token) => runner.RunReminderService_RemovalReachesQuiescence(token)); + + [Fact] + public Task ReminderService_ExactDueRecovery() + => RunAsync((runner, token) => runner.RunReminderService_ExactDueRecovery(token)); + + [Fact] + public Task ReminderService_StaleOwnerRegistrationReconciles() + => RunAsync((runner, token) => runner.RunReminderService_StaleOwnerRegistrationReconciles(token)); + + [Fact] + public Task ReminderService_OneSiloJoinLeaveTransfersOwnership() + => RunAsync((runner, token) => runner.RunReminderService_OneSiloJoinLeaveTransfersOwnership(token)); + + [Fact] + public Task ReminderService_CleanupIsIsolated() + => RunAsync((runner, token) => runner.RunReminderService_CleanupIsIsolated(token)); + + private static async Task RunAsync( + Func scenario) + { + var fixture = new ReminderServiceLifecycleFixture(); + await fixture.InitializeAsync(); + try + { + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource( + TestContext.Current.CancellationToken); + cancellation.CancelAfter(TimeSpan.FromMinutes(2)); + await scenario( + new LifecycleRunner(fixture.Harness, "IdealizedReminderTable"), + cancellation.Token); + } + finally + { + await fixture.DisposeAsync(); + } + } + + private sealed class LifecycleRunner(IReminderServiceLifecycleHarness harness, string providerName) + : ReminderServiceLifecycleTestRunner(harness, providerName, seed: 42); +} + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Reminders")] +[TestCategory("BVT"), TestCategory("Reminders"), TestCategory("ReminderTestKit")] +[Collection(ReminderServiceLifecycleCollection.Name)] +public sealed class FaultyReminderServiceLifecycleTests +{ + [Fact] + public async Task PartiallyInitializedFixtureDisposesClock() + { + var clock = ReminderTestClock.Attach(new InProcessTestClusterBuilder(1)); + var fixture = new ReminderServiceLifecycleFixture(clock); + + await fixture.DisposeAsync(); + + await Assert.ThrowsAsync( + () => clock.AdvanceAsync(TimeSpan.Zero, TestContext.Current.CancellationToken)); + } + + [Fact] + public Task DuplicateOwnerImplementationIsRejected() + => RunFaultAsync( + harness => new LifecycleRunner(new DuplicateOwnerHarness(harness), "DuplicateOwner"), + async (runner, token) => + { + var exception = await Assert.ThrowsAsync( + () => runner.RunReminderService_RegistrationHasSingleOwner(token)); + Assert.Contains("exactly 1 local owner", exception.Message, StringComparison.Ordinal); + Assert.Contains("owners=[", exception.Message, StringComparison.Ordinal); + }); + + [Fact] + public Task RestartingUpdateImplementationIsRejected() + => RunFaultAsync( + harness => new LifecycleRunner(new RestartingUpdateHarness(harness), "RestartingUpdate"), + async (runner, token) => + { + var exception = await Assert.ThrowsAsync( + () => runner.RunReminderService_UpdateDoesNotRestartLocalOwner(token)); + Assert.Contains("same single owner", exception.Message, StringComparison.Ordinal); + Assert.Contains("starts=2", exception.Message, StringComparison.Ordinal); + }); + + [Fact] + public async Task CanceledScenarioPreservesCancellationAndStillCleansItsRows() + { + var fixture = new ReminderServiceLifecycleFixture(); + await fixture.InitializeAsync(); + try + { + var runner = new LifecycleRunner(new BlockingScheduleHarness(fixture.Harness), "CanceledScenario"); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource( + TestContext.Current.CancellationToken); + cancellation.CancelAfter(TimeSpan.FromMilliseconds(100)); + + await Assert.ThrowsAnyAsync( + () => runner.RunReminderService_RegistrationHasSingleOwner(cancellation.Token)); + + var table = Assert.IsType(fixture.Harness.ReminderTable); + Assert.Empty(table.Snapshot()); + } + finally + { + await fixture.DisposeAsync(); + } + } + + [Fact] + public Task UpdateWaitsForRefreshBeforeExpectingScheduleReconciliation() + { + RefreshGatedScheduleHarness? gatedHarness = null; + return RunFaultAsync( + harness => new LifecycleRunner( + gatedHarness = new RefreshGatedScheduleHarness(harness), + "RefreshGatedUpdate"), + async (runner, token) => + { + await runner.RunReminderService_UpdateDoesNotRestartLocalOwner(token); + Assert.NotNull(gatedHarness); + Assert.True(gatedHarness.ReleasedByRefresh); + }); + } + + [Fact] + public Task JoinLeaveAdvancesToTheExactRemainingDueTime() + { + RecordingAdvanceHarness? recordingHarness = null; + return RunFaultAsync( + harness => new LifecycleRunner( + recordingHarness = new RecordingAdvanceHarness(harness), + "ExactJoinLeaveDue"), + async (runner, token) => + { + await runner.RunReminderService_OneSiloJoinLeaveTransfersOwnership(token); + Assert.NotNull(recordingHarness); + Assert.Equal([TimeSpan.FromSeconds(3)], recordingHarness.Advances); + }); + } + + private static async Task RunFaultAsync( + Func createRunner, + Func scenario) + { + var fixture = new ReminderServiceLifecycleFixture(); + await fixture.InitializeAsync(); + try + { + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource( + TestContext.Current.CancellationToken); + cancellation.CancelAfter(TimeSpan.FromMinutes(2)); + await scenario(createRunner(fixture.Harness), cancellation.Token); + } + finally + { + await fixture.DisposeAsync(); + } + } + + private sealed class LifecycleRunner(IReminderServiceLifecycleHarness harness, string providerName) + : ReminderServiceLifecycleTestRunner(harness, providerName); + + private class DelegatingHarness(IReminderServiceLifecycleHarness inner) : IReminderServiceLifecycleHarness + { + protected IReminderServiceLifecycleHarness Inner { get; } = inner; + + public IGrainFactory GrainFactory => Inner.GrainFactory; + public IReminderTable ReminderTable => Inner.ReminderTable; + public DateTimeOffset UtcNow => Inner.UtcNow; + public TimeSpan ReminderLoadingWindow => Inner.ReminderLoadingWindow; + public TimeSpan ReminderRefreshPeriod => Inner.ReminderRefreshPeriod; + public IReadOnlyList ActiveSilos => Inner.ActiveSilos; + public Task WaitForStartupReadinessAsync(CancellationToken cancellationToken) => Inner.WaitForStartupReadinessAsync(cancellationToken); + public virtual Task AdvanceAsync(TimeSpan amount, CancellationToken cancellationToken) => Inner.AdvanceAsync(amount, cancellationToken); + public virtual Task RefreshAsync(CancellationToken cancellationToken) => Inner.RefreshAsync(cancellationToken); + public virtual Task WaitForOwnerCountAsync(GrainId grainId, string reminderName, int count, CancellationToken cancellationToken) => Inner.WaitForOwnerCountAsync(grainId, reminderName, count, cancellationToken); + public virtual IReadOnlyList GetOwners(GrainId grainId, string reminderName) => Inner.GetOwners(grainId, reminderName); + public bool IsOwner(SiloAddress siloAddress, GrainId grainId) => Inner.IsOwner(siloAddress, grainId); + public virtual Task WaitForScheduleAsync(GrainId grainId, string reminderName, CancellationToken cancellationToken) => Inner.WaitForScheduleAsync(grainId, reminderName, cancellationToken); + public virtual int GetLocalStartCount(GrainId grainId, string reminderName) => Inner.GetLocalStartCount(grainId, reminderName); + public int GetLocalStopCount(GrainId grainId, string reminderName) => Inner.GetLocalStopCount(grainId, reminderName); + public int GetScheduleChangeCount(GrainId grainId, string reminderName) => Inner.GetScheduleChangeCount(grainId, reminderName); + public virtual Task WaitForScheduleChangeCountAsync(GrainId grainId, string reminderName, int count, CancellationToken cancellationToken) => Inner.WaitForScheduleChangeCountAsync(grainId, reminderName, count, cancellationToken); + public Task WaitForTickCountAsync(GrainId grainId, string reminderName, int count, CancellationToken cancellationToken) => Inner.WaitForTickCountAsync(grainId, reminderName, count, cancellationToken); + public int GetTickCount(GrainId grainId, string reminderName) => Inner.GetTickCount(grainId, reminderName); + public Task JoinOneSiloAsync(CancellationToken cancellationToken) => Inner.JoinOneSiloAsync(cancellationToken); + public Task LeaveSiloAsync(SiloAddress siloAddress, CancellationToken cancellationToken) => Inner.LeaveSiloAsync(siloAddress, cancellationToken); + public Task WaitForTopologyReconciliationAsync(CancellationToken cancellationToken) => Inner.WaitForTopologyReconciliationAsync(cancellationToken); + } + + private sealed class DuplicateOwnerHarness(IReminderServiceLifecycleHarness inner) : DelegatingHarness(inner) + { + private object? _duplicateIdentity; + + public override async Task WaitForOwnerCountAsync( + GrainId grainId, + string reminderName, + int count, + CancellationToken cancellationToken) + { + var harness = Assert.IsType(Inner); + if (count == 0 && _duplicateIdentity is { } identity) + { + harness.RemoveDuplicateOwnerForTesting(grainId, reminderName, identity); + _duplicateIdentity = null; + } + + await base.WaitForOwnerCountAsync(grainId, reminderName, count, cancellationToken); + if (count == 1 && _duplicateIdentity is null) + { + _duplicateIdentity = harness.AddDuplicateOwnerForTesting(grainId, reminderName); + } + } + } + + private sealed class RestartingUpdateHarness(IReminderServiceLifecycleHarness inner) : DelegatingHarness(inner) + { + private bool _updated; + + public override int GetLocalStartCount(GrainId grainId, string reminderName) + => base.GetLocalStartCount(grainId, reminderName) + (_updated ? 1 : 0); + + public override async Task WaitForScheduleChangeCountAsync( + GrainId grainId, + string reminderName, + int count, + CancellationToken cancellationToken) + { + await base.WaitForScheduleChangeCountAsync(grainId, reminderName, count, cancellationToken); + _updated = true; + } + } + + private sealed class BlockingScheduleHarness(IReminderServiceLifecycleHarness inner) : DelegatingHarness(inner) + { + public override Task WaitForScheduleAsync( + GrainId grainId, + string reminderName, + CancellationToken cancellationToken) + => Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + } + + private sealed class RefreshGatedScheduleHarness(IReminderServiceLifecycleHarness inner) : DelegatingHarness(inner) + { + private TaskCompletionSource? _refreshGate; + + public bool ReleasedByRefresh { get; private set; } + + public override async Task WaitForScheduleChangeCountAsync( + GrainId grainId, + string reminderName, + int count, + CancellationToken cancellationToken) + { + var innerWait = base.WaitForScheduleChangeCountAsync( + grainId, + reminderName, + count, + cancellationToken); + var refreshGate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _refreshGate = refreshGate; + await Task.WhenAll(innerWait, refreshGate.Task.WaitAsync(cancellationToken)); + ReleasedByRefresh = true; + } + + public override async Task RefreshAsync(CancellationToken cancellationToken) + { + await base.RefreshAsync(cancellationToken); + Interlocked.Exchange(ref _refreshGate, null)?.TrySetResult(); + } + } + + private sealed class RecordingAdvanceHarness(IReminderServiceLifecycleHarness inner) : DelegatingHarness(inner) + { + public List Advances { get; } = []; + + public override async Task AdvanceAsync(TimeSpan amount, CancellationToken cancellationToken) + { + Advances.Add(amount); + await base.AdvanceAsync(amount, cancellationToken); + } + } +} diff --git a/test/Orleans.Reminders.Tests/Diagnostics/ReminderEventsTests.cs b/test/Orleans.Reminders.Tests/Diagnostics/ReminderEventsTests.cs index a967bab00d3..5e9121b29a1 100644 --- a/test/Orleans.Reminders.Tests/Diagnostics/ReminderEventsTests.cs +++ b/test/Orleans.Reminders.Tests/Diagnostics/ReminderEventsTests.cs @@ -234,6 +234,47 @@ public async Task ReminderDiagnosticObserver_GlobalQuiescenceIgnoresUnrelatedSil unrelatedSilo); } + [TestSuite("BVT")] + [TestProvider("None")] + [Fact, TestCategory("BVT")] + public async Task ReminderDiagnosticObserver_CountsDuplicateInstancesOnOneSilo() + { + using var observer = ReminderDiagnosticObserver.Create(); + var grainId = GrainId.Create("test", "duplicate-owner"); + const string reminderName = "reminder"; + var siloAddress = SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 14010), 11); + var firstIdentity = new object(); + var secondIdentity = new object(); + + var twoOwners = observer.WaitForActiveReminderCountAsync( + grainId, + 2, + TestContext.Current.CancellationToken, + reminderName, + [siloAddress]); + ReminderEvents.EmitLocalReminderStarted(grainId, reminderName, firstIdentity, siloAddress); + ReminderEvents.EmitLocalReminderStarted(grainId, reminderName, secondIdentity, siloAddress); + + await twoOwners; + Assert.Equal( + [siloAddress, siloAddress], + observer.GetActiveReminderOwnerSilos(grainId, reminderName)); + Assert.Single(observer.GetActiveReminderSilos(grainId, reminderName)); + + ReminderEvents.EmitLocalReminderStopped( + grainId, + reminderName, + firstIdentity, + ReminderEvents.LocalReminderStopReason.Unregistered, + siloAddress); + ReminderEvents.EmitLocalReminderStopped( + grainId, + reminderName, + secondIdentity, + ReminderEvents.LocalReminderStopReason.Unregistered, + siloAddress); + } + [TestSuite("BVT")] [TestProvider("None")] [Fact, TestCategory("BVT")] diff --git a/test/Orleans.Reminders.Tests/TimerTests/ReminderServiceLifecycleTestsBase.cs b/test/Orleans.Reminders.Tests/TimerTests/ReminderServiceLifecycleTestsBase.cs new file mode 100644 index 00000000000..e2ab1f2e205 --- /dev/null +++ b/test/Orleans.Reminders.Tests/TimerTests/ReminderServiceLifecycleTestsBase.cs @@ -0,0 +1,80 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.Reminders.TestKit; +using Orleans.Testing.Reminders; +using Orleans.TestingHost; +using TestExtensions; +using Xunit; + +namespace UnitTests.TimerTests; + +/// +/// Exposes the shared reminder-service lifecycle contract without inheriting provider whole-table cleanup. +/// +[Collection(TestEnvironmentFixture.DefaultCollection)] +public abstract class ReminderServiceLifecycleTestsBase +{ + private static readonly TimeSpan ScenarioTimeout = TimeSpan.FromMinutes(2); + private readonly ReminderServiceLifecycleTestRunner _runner; + + protected ReminderServiceLifecycleTestsBase( + ReminderTestClock reminderClock, + InProcessTestCluster cluster, + string providerName) + { + ArgumentNullException.ThrowIfNull(reminderClock); + ArgumentNullException.ThrowIfNull(cluster); + var services = cluster.GetActiveSilos().First().ServiceProvider; + var options = services.GetRequiredService>().Value; + var harness = new ReminderServiceLifecycleHarness( + cluster, + reminderClock, + reminderClock.DiagnosticObserver, + options.ReminderLoadingWindow); + _runner = new ProviderRunner(harness, providerName); + } + + [Fact] + public Task ReminderService_StartupReadiness() + => RunAsync(_runner.RunReminderService_StartupReadiness); + + [Fact] + public Task ReminderService_RegistrationHasSingleOwner() + => RunAsync(_runner.RunReminderService_RegistrationHasSingleOwner); + + [Fact] + public Task ReminderService_UpdateDoesNotRestartLocalOwner() + => RunAsync(_runner.RunReminderService_UpdateDoesNotRestartLocalOwner); + + [Fact] + public Task ReminderService_RemovalReachesQuiescence() + => RunAsync(_runner.RunReminderService_RemovalReachesQuiescence); + + [Fact] + public Task ReminderService_ExactDueRecovery() + => RunAsync(_runner.RunReminderService_ExactDueRecovery); + + [Fact] + public Task ReminderService_StaleOwnerRegistrationReconciles() + => RunAsync(_runner.RunReminderService_StaleOwnerRegistrationReconciles); + + [Fact] + public Task ReminderService_OneSiloJoinLeaveTransfersOwnership() + => RunAsync(_runner.RunReminderService_OneSiloJoinLeaveTransfersOwnership); + + [Fact] + public Task ReminderService_CleanupIsIsolated() + => RunAsync(_runner.RunReminderService_CleanupIsIsolated); + + private static async Task RunAsync(Func scenario) + { + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource( + TestContext.Current.CancellationToken); + cancellation.CancelAfter(ScenarioTimeout); + await scenario(cancellation.Token); + } + + private sealed class ProviderRunner(IReminderServiceLifecycleHarness harness, string providerName) + : ReminderServiceLifecycleTestRunner(harness, providerName); +} diff --git a/test/Orleans.Reminders.Tests/TimerTests/ReminderTests_TableGrain.cs b/test/Orleans.Reminders.Tests/TimerTests/ReminderTests_TableGrain.cs index 99c3384b8d2..da3ac658eef 100644 --- a/test/Orleans.Reminders.Tests/TimerTests/ReminderTests_TableGrain.cs +++ b/test/Orleans.Reminders.Tests/TimerTests/ReminderTests_TableGrain.cs @@ -20,6 +20,19 @@ namespace UnitTests.TimerTests { + [TestSuite("Functional")] + [TestProvider("None")] + [TestArea("Reminders")] + [TestCategory("Functional"), TestCategory("Reminders")] + public sealed class ReminderServiceLifecycleTests_TableGrain + : ReminderServiceLifecycleTestsBase, IClassFixture + { + public ReminderServiceLifecycleTests_TableGrain(ReminderTests_TableGrain.Fixture fixture) + : base(fixture.ReminderClock, fixture.HostedCluster, "InMemory") + { + } + } + /// /// Tests for grain-based reminder functionality using in-memory reminder service as table storage. /// diff --git a/test/Orleans.Testing.Reminders/Orleans.Testing.Reminders.csproj b/test/Orleans.Testing.Reminders/Orleans.Testing.Reminders.csproj index b9e72c34b03..e97e2678109 100644 --- a/test/Orleans.Testing.Reminders/Orleans.Testing.Reminders.csproj +++ b/test/Orleans.Testing.Reminders/Orleans.Testing.Reminders.csproj @@ -15,7 +15,12 @@ + + + + + diff --git a/test/Orleans.Testing.Reminders/ReminderDiagnosticObserver.cs b/test/Orleans.Testing.Reminders/ReminderDiagnosticObserver.cs index 4a40a4a229a..36de95f2809 100644 --- a/test/Orleans.Testing.Reminders/ReminderDiagnosticObserver.cs +++ b/test/Orleans.Testing.Reminders/ReminderDiagnosticObserver.cs @@ -29,11 +29,15 @@ public sealed class ReminderDiagnosticObserver : IDisposable private readonly IDisposable _storageSubscription; private readonly Dictionary _tickCountsByGrain = []; private readonly Dictionary _tickCountsByReminder = []; + private readonly Dictionary _localStartCounts = []; + private readonly Dictionary _localStopCounts = []; + private readonly Dictionary _scheduleChangeCounts = []; private readonly Dictionary> _activeLocalReminders = []; private readonly List _tickCountWaiters = []; private readonly List _activeReminderCountWaiters = []; private readonly List _localReminderScheduleWaiters = []; private readonly List _globalQuiescenceWaiters = []; + private readonly List _scheduleChangeCountWaiters = []; /// /// Creates a new instance of the observer and starts listening for reminder diagnostic events. @@ -84,6 +88,7 @@ private void StoreEvent(ReminderEvents.ReminderEvent value) break; case ReminderEvents.LocalReminderStarted localReminderStarted: var startedKey = new ReminderTickKey(localReminderStarted.GrainId, localReminderStarted.ReminderName); + _localStartCounts[startedKey] = _localStartCounts.GetValueOrDefault(startedKey) + 1; if (!_activeLocalReminders.TryGetValue(startedKey, out var startedInstances)) { startedInstances = new Dictionary(); @@ -97,6 +102,7 @@ private void StoreEvent(ReminderEvents.ReminderEvent value) break; case ReminderEvents.LocalReminderStopped localReminderStopped: var stoppedKey = new ReminderTickKey(localReminderStopped.GrainId, localReminderStopped.ReminderName); + _localStopCounts[stoppedKey] = _localStopCounts.GetValueOrDefault(stoppedKey) + 1; if (_activeLocalReminders.TryGetValue(stoppedKey, out var stoppedInstances)) { stoppedInstances.Remove(new LocalReminderInstanceKey( @@ -114,6 +120,7 @@ private void StoreEvent(ReminderEvents.ReminderEvent value) break; case ReminderEvents.LocalReminderScheduleChanged localReminderScheduleChanged: var changedKey = new ReminderTickKey(localReminderScheduleChanged.GrainId, localReminderScheduleChanged.ReminderName); + _scheduleChangeCounts[changedKey] = _scheduleChangeCounts.GetValueOrDefault(changedKey) + 1; if (TryGetLocalReminderInstance(changedKey, localReminderScheduleChanged.Identity, out var changedInstance)) { changedInstance.ScheduleVersion = Math.Max( @@ -121,6 +128,7 @@ private void StoreEvent(ReminderEvents.ReminderEvent value) localReminderScheduleChanged.ScheduleVersion); } + ReleaseReadyScheduleChangeWaiters(ready); break; case ReminderEvents.LocalReminderTickWaitArmed localReminderTickWaitArmed: var tickWaitArmedKey = new ReminderTickKey(localReminderTickWaitArmed.GrainId, localReminderTickWaitArmed.ReminderName); @@ -245,6 +253,63 @@ public int GetTickCount(GrainId grainId, string? reminderName = null) } } + /// Gets the number of local reminder instances started for a reminder. + public int GetLocalStartCount(GrainId grainId, string reminderName) + { + lock (_lock) + { + return _localStartCounts.GetValueOrDefault(new ReminderTickKey(grainId, reminderName)); + } + } + + /// Gets the number of local reminder instances stopped for a reminder. + public int GetLocalStopCount(GrainId grainId, string reminderName) + { + lock (_lock) + { + return _localStopCounts.GetValueOrDefault(new ReminderTickKey(grainId, reminderName)); + } + } + + /// Gets the number of local schedule changes for a reminder. + public int GetScheduleChangeCount(GrainId grainId, string reminderName) + { + lock (_lock) + { + return _scheduleChangeCounts.GetValueOrDefault(new ReminderTickKey(grainId, reminderName)); + } + } + + /// Waits for the requested number of local schedule changes for a reminder. + public Task WaitForScheduleChangeCountAsync( + GrainId grainId, + string reminderName, + int expectedCount, + CancellationToken cancellationToken) + { + ArgumentOutOfRangeException.ThrowIfNegative(expectedCount); + ArgumentException.ThrowIfNullOrEmpty(reminderName); + if (cancellationToken.IsCancellationRequested) + { + return Task.FromCanceled(cancellationToken); + } + + ScheduleChangeCountWaiter waiter; + lock (_lock) + { + if (_scheduleChangeCounts.GetValueOrDefault(new ReminderTickKey(grainId, reminderName)) >= expectedCount) + { + return Task.CompletedTask; + } + + waiter = new ScheduleChangeCountWaiter(grainId, reminderName, expectedCount); + _scheduleChangeCountWaiters.Add(waiter); + RegisterCancellation(waiter, _scheduleChangeCountWaiters, cancellationToken); + } + + return waiter.TaskSource.Task; + } + /// /// Gets the count of active local reminder owners for a specific reminder. /// @@ -273,6 +338,29 @@ public SiloAddress[] GetActiveReminderSilos(GrainId grainId, string reminderName } } + /// + /// Gets one silo address per active local reminder instance, preserving duplicate instances on the same silo. + /// + public SiloAddress[] GetActiveReminderOwnerSilos( + GrainId grainId, + string reminderName, + IEnumerable? activeSilos = null) + { + ArgumentException.ThrowIfNullOrEmpty(reminderName); + var activeSiloSet = activeSilos?.ToHashSet(); + + lock (_lock) + { + return _activeLocalReminders.TryGetValue(new ReminderTickKey(grainId, reminderName), out var instances) + ? instances.Values + .Select(instance => instance.SiloAddress) + .OfType() + .Where(siloAddress => activeSiloSet is null || activeSiloSet.Contains(siloAddress)) + .ToArray() + : []; + } + } + /// /// Waits for a specific number of active local reminder owners for a reminder. /// @@ -280,7 +368,28 @@ public Task WaitForActiveReminderCountAsync(GrainId grainId, int expectedCount, { ArgumentOutOfRangeException.ThrowIfNegative(expectedCount); ArgumentException.ThrowIfNullOrEmpty(reminderName); - return WaitForActiveReminderCountCoreAsync(grainId, expectedCount, reminderName, cancellationToken); + return WaitForActiveReminderCountCoreAsync(grainId, expectedCount, reminderName, activeSilos: null, cancellationToken); + } + + /// + /// Waits for a specific number of active local reminder instances on . + /// + public Task WaitForActiveReminderCountAsync( + GrainId grainId, + int expectedCount, + CancellationToken cancellationToken, + string reminderName, + IEnumerable activeSilos) + { + ArgumentOutOfRangeException.ThrowIfNegative(expectedCount); + ArgumentException.ThrowIfNullOrEmpty(reminderName); + ArgumentNullException.ThrowIfNull(activeSilos); + return WaitForActiveReminderCountCoreAsync( + grainId, + expectedCount, + reminderName, + activeSilos.ToHashSet(), + cancellationToken); } /// @@ -298,7 +407,7 @@ public Task WaitForLocalReminderScheduleAsync(GrainId grainId, string reminderNa public Task WaitForReminderQuiescenceAsync(GrainId grainId, string reminderName, CancellationToken cancellationToken) { ArgumentException.ThrowIfNullOrEmpty(reminderName); - return WaitForActiveReminderCountCoreAsync(grainId, 0, reminderName, cancellationToken); + return WaitForActiveReminderCountCoreAsync(grainId, 0, reminderName, activeSilos: null, cancellationToken); } /// @@ -359,7 +468,12 @@ private Task WaitForTickCountCoreAsync(GrainId grainId, int targetCount, string? return waiter.TaskSource.Task; } - private Task WaitForActiveReminderCountCoreAsync(GrainId grainId, int targetCount, string reminderName, CancellationToken cancellationToken) + private Task WaitForActiveReminderCountCoreAsync( + GrainId grainId, + int targetCount, + string reminderName, + HashSet? activeSilos, + CancellationToken cancellationToken) { if (cancellationToken.IsCancellationRequested) { @@ -369,12 +483,12 @@ private Task WaitForActiveReminderCountCoreAsync(GrainId grainId, int targetCoun ActiveReminderCountWaiter? waiter; lock (_lock) { - if (GetActiveReminderCountCore(grainId, reminderName) == targetCount) + if (GetActiveReminderCountCore(grainId, reminderName, activeSilos) == targetCount) { return Task.CompletedTask; } - waiter = new ActiveReminderCountWaiter(grainId, reminderName, targetCount); + waiter = new ActiveReminderCountWaiter(grainId, reminderName, targetCount, activeSilos); _activeReminderCountWaiters.Add(waiter); RegisterCancellation(waiter, _activeReminderCountWaiters, cancellationToken); } @@ -437,10 +551,15 @@ private int GetTickCountCore(GrainId grainId, string? reminderName) return _tickCountsByReminder.GetValueOrDefault(new ReminderTickKey(grainId, reminderName)); } - private int GetActiveReminderCountCore(GrainId grainId, string reminderName) + private int GetActiveReminderCountCore( + GrainId grainId, + string reminderName, + HashSet? activeSilos = null) { return _activeLocalReminders.TryGetValue(new ReminderTickKey(grainId, reminderName), out var instances) - ? instances.Count + ? instances.Values.Count(instance => + instance.SiloAddress is { } siloAddress + && (activeSilos is null || activeSilos.Contains(siloAddress))) : 0; } @@ -498,7 +617,7 @@ private void ReleaseReadyActiveReminderWaiters(List ready) for (var i = _activeReminderCountWaiters.Count - 1; i >= 0; i--) { var waiter = _activeReminderCountWaiters[i]; - if (GetActiveReminderCountCore(waiter.GrainId, waiter.ReminderName) != waiter.TargetCount) + if (GetActiveReminderCountCore(waiter.GrainId, waiter.ReminderName, waiter.ActiveSilos) != waiter.TargetCount) { continue; } @@ -545,6 +664,23 @@ private bool IsGloballyQuiescentCore(IReadOnlySet siloAddresses) .Any(instance => instance.SiloAddress is { } address && siloAddresses.Contains(address)); } + private void ReleaseReadyScheduleChangeWaiters(List ready) + { + for (var i = _scheduleChangeCountWaiters.Count - 1; i >= 0; i--) + { + var waiter = _scheduleChangeCountWaiters[i]; + var count = _scheduleChangeCounts.GetValueOrDefault( + new ReminderTickKey(waiter.GrainId, waiter.ReminderName)); + if (count < waiter.TargetCount) + { + continue; + } + + _scheduleChangeCountWaiters.RemoveAt(i); + ready.Add(waiter); + } + } + private readonly record struct ReminderTickKey(GrainId GrainId, string ReminderName); private readonly record struct LocalReminderInstanceKey(object Identity) { @@ -581,11 +717,16 @@ private sealed class TickCountWaiter(GrainId grainId, string? reminderName, int public int TargetCount { get; } = targetCount; } - private sealed class ActiveReminderCountWaiter(GrainId grainId, string reminderName, int targetCount) : Waiter + private sealed class ActiveReminderCountWaiter( + GrainId grainId, + string reminderName, + int targetCount, + HashSet? activeSilos) : Waiter { public GrainId GrainId { get; } = grainId; public string ReminderName { get; } = reminderName; public int TargetCount { get; } = targetCount; + public HashSet? ActiveSilos { get; } = activeSilos; } private sealed class LocalReminderScheduleWaiter(GrainId grainId, string reminderName) : Waiter @@ -599,6 +740,16 @@ private sealed class GlobalQuiescenceWaiter(IReadOnlySet siloAddres public IReadOnlySet SiloAddresses { get; } = siloAddresses.ToHashSet(); } + private sealed class ScheduleChangeCountWaiter( + GrainId grainId, + string reminderName, + int targetCount) : Waiter + { + public GrainId GrainId { get; } = grainId; + public string ReminderName { get; } = reminderName; + public int TargetCount { get; } = targetCount; + } + /// public void Dispose() { diff --git a/test/Orleans.Testing.Reminders/ReminderServiceLifecycleHarness.cs b/test/Orleans.Testing.Reminders/ReminderServiceLifecycleHarness.cs new file mode 100644 index 00000000000..6b05fcf3fa9 --- /dev/null +++ b/test/Orleans.Testing.Reminders/ReminderServiceLifecycleHarness.cs @@ -0,0 +1,208 @@ +#nullable enable + +using Microsoft.Extensions.DependencyInjection; +using Orleans.Reminders.Diagnostics; +using Orleans.Reminders.TestKit; +using Orleans.Runtime; +using Orleans.Runtime.ConsistentRing; +using Orleans.Runtime.ReminderService; +using Orleans.TestingHost; + +namespace Orleans.Testing.Reminders; + +/// +/// Adapts an in-process cluster, , and +/// to the shared service lifecycle conformance runner. +/// +public sealed class ReminderServiceLifecycleHarness : IReminderServiceLifecycleHarness +{ + private readonly InProcessTestCluster _cluster; + private readonly ReminderTestClock _clock; + private readonly ReminderDiagnosticObserver _observer; + + /// Initializes the harness. + public ReminderServiceLifecycleHarness( + InProcessTestCluster cluster, + ReminderTestClock clock, + ReminderDiagnosticObserver observer, + TimeSpan reminderLoadingWindow) + { + _cluster = cluster ?? throw new ArgumentNullException(nameof(cluster)); + _clock = clock ?? throw new ArgumentNullException(nameof(clock)); + _observer = observer ?? throw new ArgumentNullException(nameof(observer)); + ReminderLoadingWindow = reminderLoadingWindow; + } + + /// + public IGrainFactory GrainFactory => _cluster.Client; + + /// + public IReminderTable ReminderTable + => _cluster.GetActiveSilos().First().ServiceProvider.GetRequiredService(); + + /// + public DateTimeOffset UtcNow => _clock.UtcNow; + + /// + public TimeSpan ReminderLoadingWindow { get; } + + /// + public TimeSpan ReminderRefreshPeriod => _clock.RefreshReminderListPeriod; + + /// + public IReadOnlyList ActiveSilos + => _cluster.GetActiveSilos().Select(silo => silo.SiloAddress).Order().ToArray(); + + /// + public async Task WaitForStartupReadinessAsync(CancellationToken cancellationToken) + { + var startupEvents = ActiveSilos + .Select(silo => _observer.WaitForReminderServiceStartedAsync(cancellationToken, silo)) + .ToArray(); + await Task.WhenAll(startupEvents); + await WaitForTopologyReconciliationAsync(cancellationToken); + } + + /// + public Task AdvanceAsync(TimeSpan amount, CancellationToken cancellationToken) + => _clock.AdvanceAsync(amount, cancellationToken); + + /// + public async Task RefreshAsync(CancellationToken cancellationToken) + { + foreach (var silo in _cluster.GetActiveSilos()) + { + await silo.ServiceProvider.GetRequiredService() + .TestOnlyRefresh() + .WaitAsync(cancellationToken); + } + } + + /// + public Task WaitForOwnerCountAsync( + GrainId grainId, + string reminderName, + int count, + CancellationToken cancellationToken) + => _observer.WaitForActiveReminderCountAsync( + grainId, + count, + cancellationToken, + reminderName, + ActiveSilos); + + /// + public IReadOnlyList GetOwners(GrainId grainId, string reminderName) + { + return _observer.GetActiveReminderOwnerSilos(grainId, reminderName, ActiveSilos); + } + + /// + public bool IsOwner(SiloAddress siloAddress, GrainId grainId) + { + var silo = _cluster.GetSiloForAddress(siloAddress) + ?? throw new InvalidOperationException($"Silo {siloAddress} is not active."); + return silo.ServiceProvider + .GetRequiredService() + .GetMyRange() + .InRange(grainId); + } + + /// + public Task WaitForScheduleAsync(GrainId grainId, string reminderName, CancellationToken cancellationToken) + => _observer.WaitForLocalReminderScheduleAsync(grainId, reminderName, cancellationToken); + + /// + public int GetLocalStartCount(GrainId grainId, string reminderName) + => _observer.GetLocalStartCount(grainId, reminderName); + + /// + public int GetLocalStopCount(GrainId grainId, string reminderName) + => _observer.GetLocalStopCount(grainId, reminderName); + + /// + public int GetScheduleChangeCount(GrainId grainId, string reminderName) + => _observer.GetScheduleChangeCount(grainId, reminderName); + + /// + public Task WaitForScheduleChangeCountAsync( + GrainId grainId, + string reminderName, + int count, + CancellationToken cancellationToken) + => _observer.WaitForScheduleChangeCountAsync(grainId, reminderName, count, cancellationToken); + + /// + public Task WaitForTickCountAsync( + GrainId grainId, + string reminderName, + int count, + CancellationToken cancellationToken) + => _observer.WaitForTickCountAsync(grainId, count, cancellationToken, reminderName); + + /// + public int GetTickCount(GrainId grainId, string reminderName) + => _observer.GetTickCount(grainId, reminderName); + + /// + public async Task JoinOneSiloAsync(CancellationToken cancellationToken) + { + var silo = AssertSingle(await _cluster.StartSilosAsync(1).WaitAsync(cancellationToken)); + await Task.WhenAll( + _observer.WaitForReminderServiceStartedAsync(cancellationToken, silo.SiloAddress), + _cluster.WaitForLivenessToStabilizeAsync().WaitAsync(cancellationToken), + _cluster.WaitForClusterManifestToStabilizeAsync().WaitAsync(cancellationToken)); + return silo.SiloAddress; + } + + /// + public async Task LeaveSiloAsync(SiloAddress siloAddress, CancellationToken cancellationToken) + { + var silo = _cluster.GetSiloForAddress(siloAddress) + ?? throw new InvalidOperationException($"Silo {siloAddress} is not active."); + await _cluster.StopSiloAsync(silo, cancellationToken); + await Task.WhenAll( + _cluster.WaitForLivenessToStabilizeAsync().WaitAsync(cancellationToken), + _cluster.WaitForClusterManifestToStabilizeAsync().WaitAsync(cancellationToken)); + } + + /// + public async Task WaitForTopologyReconciliationAsync(CancellationToken cancellationToken) + { + await Task.WhenAll( + _cluster.WaitForLivenessToStabilizeAsync().WaitAsync(cancellationToken), + _cluster.WaitForClusterManifestToStabilizeAsync().WaitAsync(cancellationToken)); + var barriers = _cluster.GetActiveSilos().Select(silo => + silo.ServiceProvider.GetRequiredService() + .TestOnlyWaitForRangeChangeReconciliation(cancellationToken)); + await Task.WhenAll(barriers); + await RefreshAsync(cancellationToken); + } + + private static InProcessSiloHandle AssertSingle(IReadOnlyList silos) + => silos.Count == 1 + ? silos[0] + : throw new InvalidOperationException($"Expected one new silo, observed {silos.Count}."); + + internal object AddDuplicateOwnerForTesting(GrainId grainId, string reminderName) + { + var identity = new object(); + ReminderEvents.EmitLocalReminderStarted( + grainId, + reminderName, + identity, + ActiveSilos.First()); + return identity; + } + + internal void RemoveDuplicateOwnerForTesting( + GrainId grainId, + string reminderName, + object identity) + => ReminderEvents.EmitLocalReminderStopped( + grainId, + reminderName, + identity, + ReminderEvents.LocalReminderStopReason.Unregistered, + ActiveSilos.First()); +} diff --git a/test/Orleans.Testing.Reminders/ReminderTestClock.cs b/test/Orleans.Testing.Reminders/ReminderTestClock.cs index 5e8734f3508..72c7c8e4a1d 100644 --- a/test/Orleans.Testing.Reminders/ReminderTestClock.cs +++ b/test/Orleans.Testing.Reminders/ReminderTestClock.cs @@ -28,6 +28,7 @@ private ReminderTestClock( TimeSpan initializationTimeout) { TimeProvider = new FakeTimeProvider(initialTime); + DiagnosticObserver = ReminderDiagnosticObserver.Create(); MinimumReminderPeriod = minimumReminderPeriod; RefreshReminderListPeriod = refreshReminderListPeriod; InitializationTimeout = initializationTimeout; @@ -35,6 +36,12 @@ private ReminderTestClock( internal FakeTimeProvider TimeProvider { get; } + /// + /// Gets the diagnostic observer attached before the cluster is built, so reminder-service startup events are + /// available to deterministic lifecycle tests. + /// + public ReminderDiagnosticObserver DiagnosticObserver { get; } + /// /// Gets the current reminder clock time. /// @@ -146,6 +153,7 @@ public void Dispose() } _disposed = true; + DiagnosticObserver.Dispose(); _advanceLock.Dispose(); }