From 636775e063bf328a7ff7025dc773f5d52340541a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 12 Aug 2026 11:58:33 -0700 Subject: [PATCH 01/28] refactor(runtime): start grain activation synchronously Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 408b1948-f836-4328-9b59-c1a6d9504551 --- .../ActivationDataActivatorProvider.cs | 27 +++++++--- .../Utils/DefaultExecutionContext.cs | 50 +++++++++++++++++++ .../Runtime/DefaultExecutionContextTests.cs | 44 ++++++++++++++++ 3 files changed, 114 insertions(+), 7 deletions(-) create mode 100644 src/Orleans.Runtime/Utils/DefaultExecutionContext.cs create mode 100644 test/Orleans.Core.Tests/Runtime/DefaultExecutionContextTests.cs diff --git a/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs b/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs index 4d14d65ce1c..21611eb3bff 100644 --- a/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs +++ b/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs @@ -57,6 +57,7 @@ private partial class ActivationDataActivator : IGrainContextActivator private readonly GrainTypeSharedContext _sharedComponents; private readonly Func _createWorkItemGroup; private readonly Action _startActivation; + private readonly ContextCallback _startActivationSynchronously; public ActivationDataActivator( IGrainActivator grainActivator, @@ -74,6 +75,24 @@ public ActivationDataActivator( _schedulingOptions, schedulerInstruments); _startActivation = state => ((ActivationData)state!).Start(_grainActivator); + _startActivationSynchronously = state => + { + var context = (ActivationData)state!; + RuntimeContext.SetExecutionContext(context, out var originalContext); + try + { + var task = new Task( + _startActivation, + context, + CancellationToken.None, + TaskCreationOptions.DenyChildAttach); + task.RunSynchronously(context.ActivationTaskScheduler); + } + finally + { + RuntimeContext.ResetExecutionContext(originalContext); + } + }; } public IGrainContext CreateContext(GrainAddress activationAddress, IConfigureGrainContext[] configureActions) @@ -89,13 +108,7 @@ public IGrainContext CreateContext(GrainAddress activationAddress, IConfigureGra configure.Configure(context); } - using var ecSuppressor = ExecutionContext.SuppressFlow(); - _ = Task.Factory.StartNew( - _startActivation, - context, - CancellationToken.None, - TaskCreationOptions.DenyChildAttach, - context.ActivationTaskScheduler); + ExecutionContext.Run(DefaultExecutionContext.Instance, _startActivationSynchronously, context); return context; } } diff --git a/src/Orleans.Runtime/Utils/DefaultExecutionContext.cs b/src/Orleans.Runtime/Utils/DefaultExecutionContext.cs new file mode 100644 index 00000000000..8b1cb3f4cb5 --- /dev/null +++ b/src/Orleans.Runtime/Utils/DefaultExecutionContext.cs @@ -0,0 +1,50 @@ +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; + +namespace Orleans.Runtime; + +/// +/// Provides an which contains no ambient state. +/// +internal static class DefaultExecutionContext +{ + public static ExecutionContext Instance { get; } = GetInstance(); + + private static ExecutionContext GetInstance() + { + try + { + return GetRuntimeDefault(null!); + } + catch (MissingFieldException) + { + return CaptureDefault(); + } + } + + [UnsafeAccessor(UnsafeAccessorKind.StaticField, Name = "Default")] + private static extern ref ExecutionContext GetRuntimeDefault(ExecutionContext _); + + internal static ExecutionContext CaptureDefault() + { + Task captureTask; + if (ExecutionContext.IsFlowSuppressed()) + { + captureTask = CaptureAsync(); + } + else + { + using (ExecutionContext.SuppressFlow()) + { + captureTask = CaptureAsync(); + } + } + + return captureTask.GetAwaiter().GetResult(); + + static Task CaptureAsync() => Task.Run( + static () => ExecutionContext.Capture() + ?? throw new InvalidOperationException("Could not capture the default execution context.")); + } +} diff --git a/test/Orleans.Core.Tests/Runtime/DefaultExecutionContextTests.cs b/test/Orleans.Core.Tests/Runtime/DefaultExecutionContextTests.cs new file mode 100644 index 00000000000..5d8bde32936 --- /dev/null +++ b/test/Orleans.Core.Tests/Runtime/DefaultExecutionContextTests.cs @@ -0,0 +1,44 @@ +using System.Threading; +using Orleans.Runtime; +using TestExtensions; +using Xunit; + +namespace UnitTests.Runtime; + +public class DefaultExecutionContextTests +{ + [Fact, TestCategory("BVT")] + public void InstanceDoesNotContainAmbientState() => AssertDoesNotContainAmbientState(DefaultExecutionContext.Instance); + + [Fact, TestCategory("BVT")] + public void FallbackDoesNotContainAmbientState() => AssertDoesNotContainAmbientState(DefaultExecutionContext.CaptureDefault()); + + private static void AssertDoesNotContainAmbientState(ExecutionContext executionContext) + { + var ambientState = new AsyncLocal(); + var expected = new object(); + ambientState.Value = expected; + object? observed = expected; + var flowSuppressed = true; + + try + { + ExecutionContext.Run( + executionContext, + _ => + { + observed = ambientState.Value; + flowSuppressed = ExecutionContext.IsFlowSuppressed(); + }, + null); + + Assert.Null(observed); + Assert.False(flowSuppressed); + Assert.Same(expected, ambientState.Value); + } + finally + { + ambientState.Value = null; + } + } +} From 23c181e3515bc14a8a9ea4906ea3725ba6d88041 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 19 Aug 2026 15:47:56 -0700 Subject: [PATCH 02/28] test(runtime): cover synchronous activation start Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../GrainActivatorTests.cs | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/test/Orleans.Runtime.Tests/GrainActivatorTests.cs b/test/Orleans.Runtime.Tests/GrainActivatorTests.cs index 89edd22ec37..ca3f7438012 100644 --- a/test/Orleans.Runtime.Tests/GrainActivatorTests.cs +++ b/test/Orleans.Runtime.Tests/GrainActivatorTests.cs @@ -112,6 +112,45 @@ public async Task GrainContextIsConfiguredBeforeGrainConstruction() Assert.True(state.WasConfiguredAtConstruction); } + [Fact, TestCategory("BVT")] + public async Task ContextCreationStartsActivationSynchronouslyOnActivationSchedulerWithCleanExecutionContext() + { + var state = ActivationOrderingState.Instance; + var ambientState = new object(); + state.Arm(); + state.SetAmbientState(ambientState); + + var primary = Assert.IsType(fixture.HostedCluster.Primary); + var services = primary.ServiceProvider; + var grainType = services.GetRequiredService().GetGrainType(typeof(ExplicitlyRegisteredSimpleDIGrain)); + var grainId = GrainId.Create(grainType, Guid.NewGuid().ToString()); + var address = GrainAddress.NewActivationAddress(primary.SiloAddress, grainId); + ActivationData? context = null; + + try + { + context = Assert.IsType(services.GetRequiredService().CreateInstance(address)); + + Assert.NotNull(context.GrainInstance); + Assert.True(state.WasConfiguredAtConstruction); + Assert.True(state.HadSchedulerAffinityAtConstruction); + Assert.True(state.HadRuntimeContextAtConstruction); + Assert.Null(state.RequestContextAtConstruction); + Assert.Null(state.TransactionStateAtConstruction); + Assert.Same(ambientState, RequestContext.Get(ActivationOrderingState.RequestContextKey)); + Assert.Same(ambientState, state.TransactionState); + } + finally + { + state.ClearAmbientState(); + if (context is not null) + { + context.Deactivate(new(DeactivationReasonCode.ApplicationRequested, "Test completed.")); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(10)); + } + } + } + /// /// Custom grain activator that bypasses dependency injection entirely. /// Implements both IGrainActivator (for creation/disposal) and IConfigureGrainTypeComponents @@ -181,19 +220,47 @@ public bool TryGetConfigurator( private sealed class ActivationOrderingState : IConfigureGrainContext { + private readonly AsyncLocal _transactionState = new(); private int _armed; + private int _hadRuntimeContextAtConstruction; + private int _hadSchedulerAffinityAtConstruction; private int _wasConfiguredAtConstruction; + private object? _requestContextAtConstruction; + private object? _transactionStateAtConstruction; public static ActivationOrderingState Instance { get; } = new(); + public const string RequestContextKey = "activation-ordering"; + + public bool HadRuntimeContextAtConstruction => Volatile.Read(ref _hadRuntimeContextAtConstruction) != 0; + public bool HadSchedulerAffinityAtConstruction => Volatile.Read(ref _hadSchedulerAffinityAtConstruction) != 0; + public object? RequestContextAtConstruction => Volatile.Read(ref _requestContextAtConstruction); + public object? TransactionState => _transactionState.Value; + public object? TransactionStateAtConstruction => Volatile.Read(ref _transactionStateAtConstruction); public bool WasConfiguredAtConstruction => Volatile.Read(ref _wasConfiguredAtConstruction) != 0; public void Arm() { + Volatile.Write(ref _hadRuntimeContextAtConstruction, 0); + Volatile.Write(ref _hadSchedulerAffinityAtConstruction, 0); + Volatile.Write(ref _requestContextAtConstruction, null); + Volatile.Write(ref _transactionStateAtConstruction, null); Volatile.Write(ref _wasConfiguredAtConstruction, 0); Volatile.Write(ref _armed, 1); } + public void SetAmbientState(object value) + { + RequestContext.Set(RequestContextKey, value); + _transactionState.Value = value; + } + + public void ClearAmbientState() + { + RequestContext.Remove(RequestContextKey); + _transactionState.Value = null; + } + public void Configure(IGrainContext context) { if (Volatile.Read(ref _armed) == 0) @@ -214,6 +281,14 @@ public void ObserveConstruction(IGrainContext context) Volatile.Write( ref _wasConfiguredAtConstruction, context.GetComponent() is not null ? 1 : 0); + Volatile.Write( + ref _hadSchedulerAffinityAtConstruction, + ReferenceEquals(((ActivationData)context).ActivationTaskScheduler, TaskScheduler.Current) ? 1 : 0); + Volatile.Write( + ref _hadRuntimeContextAtConstruction, + ReferenceEquals(context, RuntimeContext.Current) ? 1 : 0); + Volatile.Write(ref _requestContextAtConstruction, RequestContext.Get(RequestContextKey)); + Volatile.Write(ref _transactionStateAtConstruction, _transactionState.Value); } } From 6d706490be6d397c3f2351b4ee527de024f31287 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 19 Aug 2026 15:52:16 -0700 Subject: [PATCH 03/28] test(runtime): verify concurrent default context use Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Runtime/DefaultExecutionContextTests.cs | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/test/Orleans.Core.Tests/Runtime/DefaultExecutionContextTests.cs b/test/Orleans.Core.Tests/Runtime/DefaultExecutionContextTests.cs index 5d8bde32936..3d9a6944ab4 100644 --- a/test/Orleans.Core.Tests/Runtime/DefaultExecutionContextTests.cs +++ b/test/Orleans.Core.Tests/Runtime/DefaultExecutionContextTests.cs @@ -1,4 +1,5 @@ using System.Threading; +using System.Threading.Tasks; using Orleans.Runtime; using TestExtensions; using Xunit; @@ -13,6 +14,31 @@ public class DefaultExecutionContextTests [Fact, TestCategory("BVT")] public void FallbackDoesNotContainAmbientState() => AssertDoesNotContainAmbientState(DefaultExecutionContext.CaptureDefault()); + [Fact, TestCategory("BVT")] + public async Task InstanceSupportsConcurrentExecution() + { + var tasks = new Task[Math.Max(4, Environment.ProcessorCount)]; + for (var i = 0; i < tasks.Length; i++) + { + var expected = new object(); + tasks[i] = Task.Run(() => + { + var ambientState = new AsyncLocal { Value = expected }; + object? observed = expected; + + ExecutionContext.Run( + DefaultExecutionContext.Instance, + _ => observed = ambientState.Value, + null); + + Assert.Null(observed); + Assert.Same(expected, ambientState.Value); + }); + } + + await Task.WhenAll(tasks); + } + private static void AssertDoesNotContainAmbientState(ExecutionContext executionContext) { var ambientState = new AsyncLocal(); From ee07701efb874c5d3e8d059f8e525faae84fcf98 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 19 Aug 2026 17:07:45 -0700 Subject: [PATCH 04/28] refactor(runtime): capture clean context via thread pool Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: ae2e4673-3220-4e33-9073-c2dd36d0fed9 --- .../Utils/DefaultExecutionContext.cs | 51 +++++++------------ 1 file changed, 18 insertions(+), 33 deletions(-) diff --git a/src/Orleans.Runtime/Utils/DefaultExecutionContext.cs b/src/Orleans.Runtime/Utils/DefaultExecutionContext.cs index 8b1cb3f4cb5..9f65209c1bf 100644 --- a/src/Orleans.Runtime/Utils/DefaultExecutionContext.cs +++ b/src/Orleans.Runtime/Utils/DefaultExecutionContext.cs @@ -1,4 +1,3 @@ -using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -9,42 +8,28 @@ namespace Orleans.Runtime; /// internal static class DefaultExecutionContext { - public static ExecutionContext Instance { get; } = GetInstance(); - - private static ExecutionContext GetInstance() - { - try - { - return GetRuntimeDefault(null!); - } - catch (MissingFieldException) - { - return CaptureDefault(); - } - } - - [UnsafeAccessor(UnsafeAccessorKind.StaticField, Name = "Default")] - private static extern ref ExecutionContext GetRuntimeDefault(ExecutionContext _); + public static ExecutionContext Instance { get; } = CaptureDefault(); internal static ExecutionContext CaptureDefault() { - Task captureTask; - if (ExecutionContext.IsFlowSuppressed()) - { - captureTask = CaptureAsync(); - } - else - { - using (ExecutionContext.SuppressFlow()) + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + ThreadPool.UnsafeQueueUserWorkItem( + static completion => { - captureTask = CaptureAsync(); - } - } - - return captureTask.GetAwaiter().GetResult(); + try + { + completion.SetResult( + ExecutionContext.Capture() + ?? throw new InvalidOperationException("Could not capture the default execution context.")); + } + catch (Exception exception) + { + completion.SetException(exception); + } + }, + completion, + preferLocal: false); - static Task CaptureAsync() => Task.Run( - static () => ExecutionContext.Capture() - ?? throw new InvalidOperationException("Could not capture the default execution context.")); + return completion.Task.GetAwaiter().GetResult(); } } From 6b55ad4bd163b2f865d12fd1b7eb6d70017f1caf Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 19 Aug 2026 20:31:49 -0700 Subject: [PATCH 05/28] fix(runtime): preserve activation scheduler ownership Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: ae2e4673-3220-4e33-9073-c2dd36d0fed9 --- .../ActivationDataActivatorProvider.cs | 21 +-- .../Scheduler/ActivationTaskScheduler.cs | 3 +- .../Scheduler/WorkItemGroup.cs | 120 ++++++++++++++---- .../OrleansTaskSchedulerBasicTests.cs | 102 +++++++++++++++ 4 files changed, 207 insertions(+), 39 deletions(-) diff --git a/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs b/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs index 21611eb3bff..7c738daa47e 100644 --- a/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs +++ b/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs @@ -78,20 +78,13 @@ public ActivationDataActivator( _startActivationSynchronously = state => { var context = (ActivationData)state!; - RuntimeContext.SetExecutionContext(context, out var originalContext); - try - { - var task = new Task( - _startActivation, - context, - CancellationToken.None, - TaskCreationOptions.DenyChildAttach); - task.RunSynchronously(context.ActivationTaskScheduler); - } - finally - { - RuntimeContext.ResetExecutionContext(originalContext); - } + var task = new Task( + _startActivation, + context, + CancellationToken.None, + TaskCreationOptions.DenyChildAttach); + context.ActivationTaskScheduler.RunOrQueueTask(task); + task.GetAwaiter().GetResult(); }; } diff --git a/src/Orleans.Runtime/Scheduler/ActivationTaskScheduler.cs b/src/Orleans.Runtime/Scheduler/ActivationTaskScheduler.cs index a854e04b108..21cf145b88c 100644 --- a/src/Orleans.Runtime/Scheduler/ActivationTaskScheduler.cs +++ b/src/Orleans.Runtime/Scheduler/ActivationTaskScheduler.cs @@ -56,6 +56,8 @@ internal void RunTaskFromWorkItemGroup(Task task) } } + internal void RunOrQueueTask(Task task) => workerGroup.RunOrQueueTask(task); + /// Queues a task to the scheduler. /// The task to be queued. protected override void QueueTask(Task task) @@ -101,7 +103,6 @@ protected override bool TryExecuteTaskInline(Task task, bool taskWasPreviouslyQu #if DEBUG LogTraceTryExecuteTaskInlineYes(myId, task.Id, System.Environment.CurrentManagedThreadId); #endif - // Try to run the task. bool done = TryExecuteTask(task); #if DEBUG if (!done) diff --git a/src/Orleans.Runtime/Scheduler/WorkItemGroup.cs b/src/Orleans.Runtime/Scheduler/WorkItemGroup.cs index 8320c332a8d..f218d922320 100644 --- a/src/Orleans.Runtime/Scheduler/WorkItemGroup.cs +++ b/src/Orleans.Runtime/Scheduler/WorkItemGroup.cs @@ -119,6 +119,80 @@ public void EnqueueTask(Task task) } } + internal void RunOrQueueTask(Task task) + { + if (Equals(RuntimeContext.Current, GrainContext)) + { + task.RunSynchronously(TaskScheduler); + return; + } + + if (!TryAcquireExecution(task, out var taskStart)) + { + task.Start(TaskScheduler); + return; + } + + RuntimeContext.SetExecutionContext(GrainContext, out var originalContext); + try + { +#if DEBUG + LogTaskStart(task); +#endif + task.RunSynchronously(TaskScheduler); + } + finally + { + RuntimeContext.ResetExecutionContext(originalContext); + try + { + CompleteTask( + task, + taskStart, + (long)Math.Ceiling(_schedulingOptions.TurnWarningLengthThreshold.TotalMilliseconds)); + } + finally + { + ReleaseExecution(); + } + } + } + + private bool TryAcquireExecution(Task task, out long taskStart) + { + lock (_lockObj) + { + if (_state != WorkGroupStatus.Waiting) + { + taskStart = 0; + return false; + } + + _state = WorkGroupStatus.Running; + _currentTask = task; + _currentTaskStarted = taskStart = Environment.TickCount64; + _totalItemsEnqueued++; + return true; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void ReleaseExecution() + { + lock (_lockObj) + { + if (_workItems.Count > 0) + { + _state = WorkGroupStatus.Runnable; + ScheduleExecution(this); + } + else + { + _state = WorkGroupStatus.Waiting; + } + } + } + [MethodImpl(MethodImplOptions.NoInlining)] private void LogTooManyTasksInQueue(int count, int maxPendingItemsLimit) { @@ -179,17 +253,8 @@ public void Execute() } finally { - _totalItemsProcessed++; - taskEnd = Environment.TickCount64; - var taskDurationMs = taskEnd - taskStart; + taskEnd = CompleteTask(task, taskStart, turnWarningDurationMs); taskStart = taskEnd; - if (taskDurationMs > turnWarningDurationMs) - { - _schedulerInstruments.OnLongRunningTurn(); - LogLongRunningTurn(task, taskDurationMs); - } - - _currentTask = null; } } while (activationSchedulingQuantumMs <= 0 || taskEnd - loopStart < activationSchedulingQuantumMs); @@ -200,24 +265,31 @@ public void Execute() } finally { - // Now we're not Running anymore. - // If we left work items on our run list, we're Runnable, and need to go back on the silo run queue; - // If our run list is empty, then we're waiting. - lock (_lockObj) + try { - if (_workItems.Count > 0) - { - _state = WorkGroupStatus.Runnable; - ScheduleExecution(this); - } - else - { - _state = WorkGroupStatus.Waiting; - } + ReleaseExecution(); + } + finally + { + RuntimeContext.ResetExecutionContext(originalContext); } + } + } - RuntimeContext.ResetExecutionContext(originalContext); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private long CompleteTask(Task task, long taskStart, long turnWarningDurationMs) + { + _totalItemsProcessed++; + var taskEnd = Environment.TickCount64; + var taskDurationMs = taskEnd - taskStart; + if (taskDurationMs > turnWarningDurationMs) + { + _schedulerInstruments.OnLongRunningTurn(); + LogLongRunningTurn(task, taskDurationMs); } + + _currentTask = null; + return taskEnd; } #if DEBUG diff --git a/test/Orleans.Core.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs b/test/Orleans.Core.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs index cce2d6e5aa2..9dfd7889ced 100644 --- a/test/Orleans.Core.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs +++ b/test/Orleans.Core.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs @@ -116,6 +116,108 @@ public async Task Async_Task_Start_ActivationTaskScheduler() Assert.Equal(expected, received); } + [Fact] + public async Task Sched_RunOrQueueTask_OwnsWorkItemGroup() + { + Task? queuedTask = null; + var inlineCompleted = 0; + var queuedTaskObservedInlineCompletion = 0; + var inlineTask = new Task(() => + { + queuedTask = new Task( + () => + { + Volatile.Write( + ref queuedTaskObservedInlineCompletion, + Volatile.Read(ref inlineCompleted)); + + var nestedTaskRan = false; + var nestedTask = new Task(() => nestedTaskRan = true); + nestedTask.RunSynchronously(_rootContext.WorkItemGroup.TaskScheduler); + Assert.True(nestedTaskRan); + }); + queuedTask.Start(_rootContext.WorkItemGroup.TaskScheduler); + + Assert.Contains("WorkGroupStatus=Running", _rootContext.WorkItemGroup.DumpStatus()); + Assert.False(queuedTask.IsCompleted); + Volatile.Write(ref inlineCompleted, 1); + }); + + _rootContext.WorkItemGroup.RunOrQueueTask(inlineTask); + Assert.True(inlineTask.IsCompletedSuccessfully, inlineTask.Exception?.ToString()); + + await queuedTask!.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(1, queuedTaskObservedInlineCompletion); + } + + [Fact] + public async Task Sched_RunOrQueueTask_QueuesWhenWorkItemGroupIsBusy() + { + using var releaseBlockingTask = new ManualResetEventSlim(); + var blockingTaskStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var blockingTask = new Task( + () => + { + blockingTaskStarted.SetResult(); + releaseBlockingTask.Wait(); + }); + blockingTask.Start(_rootContext.WorkItemGroup.TaskScheduler); + await blockingTaskStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + var queuedTaskRan = false; + var queuedTask = new Task(() => queuedTaskRan = true); + _rootContext.WorkItemGroup.RunOrQueueTask(queuedTask); + + Assert.False(queuedTask.IsCompleted); + releaseBlockingTask.Set(); + await Task.WhenAll(blockingTask, queuedTask).WaitAsync(TimeSpan.FromSeconds(5)); + Assert.True(queuedTaskRan); + } + + [Fact] + public async Task Sched_RunSynchronously_InlinesWhenCurrentSchedulerIsHidden() + { + var nestedTaskRan = false; + var outerTask = new Task( + () => + { + Assert.Same(TaskScheduler.Default, TaskScheduler.Current); + var nestedTask = new Task(() => nestedTaskRan = true); + nestedTask.RunSynchronously(_rootContext.WorkItemGroup.TaskScheduler); + }, + CancellationToken.None, + TaskCreationOptions.HideScheduler); + + outerTask.Start(_rootContext.WorkItemGroup.TaskScheduler); + await outerTask.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.True(nestedTaskRan); + } + + [Fact] + public async Task Sched_RunOrQueueTask_RestoresParentWorkItemGroup() + { + using var childContext = UnitTestSchedulingContext.Create(_loggerFactory); + var parentTaskRestored = false; + var parentTask = new Task( + () => + { + Assert.Same(_rootContext, RuntimeContext.Current); + var childTask = new Task(() => Assert.Same(childContext, RuntimeContext.Current)); + childContext.WorkItemGroup.RunOrQueueTask(childTask); + Assert.True(childTask.IsCompletedSuccessfully, childTask.Exception?.ToString()); + + Assert.Same(_rootContext, RuntimeContext.Current); + var nestedParentTask = new Task(() => parentTaskRestored = true); + nestedParentTask.RunSynchronously(_rootContext.WorkItemGroup.TaskScheduler); + }, + CancellationToken.None, + TaskCreationOptions.HideScheduler); + + parentTask.Start(_rootContext.WorkItemGroup.TaskScheduler); + await parentTask.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.True(parentTaskRestored); + } + [Fact] public async Task Sched_SimpleFifoTest() { From 30d0efd2ea8dfd680a9b498dcb156a62e44e2872 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 19 Aug 2026 22:04:36 -0700 Subject: [PATCH 06/28] fix(runtime): preserve activation continuation context Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: ae2e4673-3220-4e33-9073-c2dd36d0fed9 --- .../ActivationDataActivatorProvider.cs | 3 +- .../Scheduler/ActivationTaskScheduler.cs | 3 +- .../Scheduler/WorkItemGroup.cs | 120 +++------------ .../OrleansTaskSchedulerBasicTests.cs | 145 +++++++++--------- 4 files changed, 99 insertions(+), 172 deletions(-) diff --git a/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs b/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs index 7c738daa47e..446712adca5 100644 --- a/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs +++ b/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs @@ -83,8 +83,7 @@ public ActivationDataActivator( context, CancellationToken.None, TaskCreationOptions.DenyChildAttach); - context.ActivationTaskScheduler.RunOrQueueTask(task); - task.GetAwaiter().GetResult(); + task.RunSynchronously(context.ActivationTaskScheduler); }; } diff --git a/src/Orleans.Runtime/Scheduler/ActivationTaskScheduler.cs b/src/Orleans.Runtime/Scheduler/ActivationTaskScheduler.cs index 21cf145b88c..a854e04b108 100644 --- a/src/Orleans.Runtime/Scheduler/ActivationTaskScheduler.cs +++ b/src/Orleans.Runtime/Scheduler/ActivationTaskScheduler.cs @@ -56,8 +56,6 @@ internal void RunTaskFromWorkItemGroup(Task task) } } - internal void RunOrQueueTask(Task task) => workerGroup.RunOrQueueTask(task); - /// Queues a task to the scheduler. /// The task to be queued. protected override void QueueTask(Task task) @@ -103,6 +101,7 @@ protected override bool TryExecuteTaskInline(Task task, bool taskWasPreviouslyQu #if DEBUG LogTraceTryExecuteTaskInlineYes(myId, task.Id, System.Environment.CurrentManagedThreadId); #endif + // Try to run the task. bool done = TryExecuteTask(task); #if DEBUG if (!done) diff --git a/src/Orleans.Runtime/Scheduler/WorkItemGroup.cs b/src/Orleans.Runtime/Scheduler/WorkItemGroup.cs index f218d922320..8320c332a8d 100644 --- a/src/Orleans.Runtime/Scheduler/WorkItemGroup.cs +++ b/src/Orleans.Runtime/Scheduler/WorkItemGroup.cs @@ -119,80 +119,6 @@ public void EnqueueTask(Task task) } } - internal void RunOrQueueTask(Task task) - { - if (Equals(RuntimeContext.Current, GrainContext)) - { - task.RunSynchronously(TaskScheduler); - return; - } - - if (!TryAcquireExecution(task, out var taskStart)) - { - task.Start(TaskScheduler); - return; - } - - RuntimeContext.SetExecutionContext(GrainContext, out var originalContext); - try - { -#if DEBUG - LogTaskStart(task); -#endif - task.RunSynchronously(TaskScheduler); - } - finally - { - RuntimeContext.ResetExecutionContext(originalContext); - try - { - CompleteTask( - task, - taskStart, - (long)Math.Ceiling(_schedulingOptions.TurnWarningLengthThreshold.TotalMilliseconds)); - } - finally - { - ReleaseExecution(); - } - } - } - - private bool TryAcquireExecution(Task task, out long taskStart) - { - lock (_lockObj) - { - if (_state != WorkGroupStatus.Waiting) - { - taskStart = 0; - return false; - } - - _state = WorkGroupStatus.Running; - _currentTask = task; - _currentTaskStarted = taskStart = Environment.TickCount64; - _totalItemsEnqueued++; - return true; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void ReleaseExecution() - { - lock (_lockObj) - { - if (_workItems.Count > 0) - { - _state = WorkGroupStatus.Runnable; - ScheduleExecution(this); - } - else - { - _state = WorkGroupStatus.Waiting; - } - } - } - [MethodImpl(MethodImplOptions.NoInlining)] private void LogTooManyTasksInQueue(int count, int maxPendingItemsLimit) { @@ -253,8 +179,17 @@ public void Execute() } finally { - taskEnd = CompleteTask(task, taskStart, turnWarningDurationMs); + _totalItemsProcessed++; + taskEnd = Environment.TickCount64; + var taskDurationMs = taskEnd - taskStart; taskStart = taskEnd; + if (taskDurationMs > turnWarningDurationMs) + { + _schedulerInstruments.OnLongRunningTurn(); + LogLongRunningTurn(task, taskDurationMs); + } + + _currentTask = null; } } while (activationSchedulingQuantumMs <= 0 || taskEnd - loopStart < activationSchedulingQuantumMs); @@ -265,31 +200,24 @@ public void Execute() } finally { - try - { - ReleaseExecution(); - } - finally + // Now we're not Running anymore. + // If we left work items on our run list, we're Runnable, and need to go back on the silo run queue; + // If our run list is empty, then we're waiting. + lock (_lockObj) { - RuntimeContext.ResetExecutionContext(originalContext); + if (_workItems.Count > 0) + { + _state = WorkGroupStatus.Runnable; + ScheduleExecution(this); + } + else + { + _state = WorkGroupStatus.Waiting; + } } - } - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private long CompleteTask(Task task, long taskStart, long turnWarningDurationMs) - { - _totalItemsProcessed++; - var taskEnd = Environment.TickCount64; - var taskDurationMs = taskEnd - taskStart; - if (taskDurationMs > turnWarningDurationMs) - { - _schedulerInstruments.OnLongRunningTurn(); - LogLongRunningTurn(task, taskDurationMs); + RuntimeContext.ResetExecutionContext(originalContext); } - - _currentTask = null; - return taskEnd; } #if DEBUG diff --git a/test/Orleans.Core.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs b/test/Orleans.Core.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs index 9dfd7889ced..b4ae4455719 100644 --- a/test/Orleans.Core.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs +++ b/test/Orleans.Core.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs @@ -117,105 +117,106 @@ public async Task Async_Task_Start_ActivationTaskScheduler() } [Fact] - public async Task Sched_RunOrQueueTask_OwnsWorkItemGroup() + public async Task Sched_RunSynchronously_QueuesWhenContextIsNotCurrent() { Task? queuedTask = null; - var inlineCompleted = 0; - var queuedTaskObservedInlineCompletion = 0; - var inlineTask = new Task(() => + var startCompleted = 0; + var queuedTaskObservedStartCompletion = 0; + var startTask = new Task(() => { + Assert.Same(_rootContext, RuntimeContext.Current); + Assert.Same(_rootContext.WorkItemGroup.TaskScheduler, TaskScheduler.Current); queuedTask = new Task( () => { Volatile.Write( - ref queuedTaskObservedInlineCompletion, - Volatile.Read(ref inlineCompleted)); - - var nestedTaskRan = false; - var nestedTask = new Task(() => nestedTaskRan = true); - nestedTask.RunSynchronously(_rootContext.WorkItemGroup.TaskScheduler); - Assert.True(nestedTaskRan); + ref queuedTaskObservedStartCompletion, + Volatile.Read(ref startCompleted)); }); queuedTask.Start(_rootContext.WorkItemGroup.TaskScheduler); Assert.Contains("WorkGroupStatus=Running", _rootContext.WorkItemGroup.DumpStatus()); Assert.False(queuedTask.IsCompleted); - Volatile.Write(ref inlineCompleted, 1); + Volatile.Write(ref startCompleted, 1); }); - _rootContext.WorkItemGroup.RunOrQueueTask(inlineTask); - Assert.True(inlineTask.IsCompletedSuccessfully, inlineTask.Exception?.ToString()); + startTask.RunSynchronously(_rootContext.WorkItemGroup.TaskScheduler); + Assert.True(startTask.IsCompletedSuccessfully, startTask.Exception?.ToString()); await queuedTask!.WaitAsync(TimeSpan.FromSeconds(5)); - Assert.Equal(1, queuedTaskObservedInlineCompletion); + Assert.Equal(1, queuedTaskObservedStartCompletion); } [Fact] - public async Task Sched_RunOrQueueTask_QueuesWhenWorkItemGroupIsBusy() + public async Task Sched_RunSynchronously_PreservesContextAcrossAsynchronousSignals() { - using var releaseBlockingTask = new ManualResetEventSlim(); - var blockingTaskStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var blockingTask = new Task( - () => + const int IterationCount = 1_000; + var signal = new SingleWaiterAutoResetEvent { RunContinuationsAsynchronously = true }; + var observations = new TaskCompletionSource[IterationCount]; + for (var i = 0; i < observations.Length; i++) + { + observations[i] = new(TaskCreationOptions.RunContinuationsAsynchronously); + } + + Task? observationLoop = null; + var initialObservation = 0; + var asyncInitialObservation = 0; + var startTask = new Task(() => + { + if (ReferenceEquals(RuntimeContext.Current, _rootContext)) { - blockingTaskStarted.SetResult(); - releaseBlockingTask.Wait(); - }); - blockingTask.Start(_rootContext.WorkItemGroup.TaskScheduler); - await blockingTaskStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + initialObservation |= 1; + } - var queuedTaskRan = false; - var queuedTask = new Task(() => queuedTaskRan = true); - _rootContext.WorkItemGroup.RunOrQueueTask(queuedTask); + if (ReferenceEquals(TaskScheduler.Current, _rootContext.WorkItemGroup.TaskScheduler)) + { + initialObservation |= 2; + } - Assert.False(queuedTask.IsCompleted); - releaseBlockingTask.Set(); - await Task.WhenAll(blockingTask, queuedTask).WaitAsync(TimeSpan.FromSeconds(5)); - Assert.True(queuedTaskRan); - } + observationLoop = ObserveSignals(); + }); + startTask.RunSynchronously(_rootContext.WorkItemGroup.TaskScheduler); + Assert.True(startTask.IsCompletedSuccessfully, startTask.Exception?.ToString()); + Assert.Equal(3, initialObservation); + Assert.Equal(3, asyncInitialObservation); - [Fact] - public async Task Sched_RunSynchronously_InlinesWhenCurrentSchedulerIsHidden() - { - var nestedTaskRan = false; - var outerTask = new Task( - () => + for (var i = 0; i < observations.Length; i++) + { + signal.Signal(); + Assert.Equal(3, await observations[i].Task.WaitAsync(TimeSpan.FromSeconds(5))); + } + + await observationLoop!.WaitAsync(TimeSpan.FromSeconds(5)); + + async Task ObserveSignals() + { + if (ReferenceEquals(RuntimeContext.Current, _rootContext)) { - Assert.Same(TaskScheduler.Default, TaskScheduler.Current); - var nestedTask = new Task(() => nestedTaskRan = true); - nestedTask.RunSynchronously(_rootContext.WorkItemGroup.TaskScheduler); - }, - CancellationToken.None, - TaskCreationOptions.HideScheduler); - - outerTask.Start(_rootContext.WorkItemGroup.TaskScheduler); - await outerTask.WaitAsync(TimeSpan.FromSeconds(5)); - Assert.True(nestedTaskRan); - } + asyncInitialObservation |= 1; + } - [Fact] - public async Task Sched_RunOrQueueTask_RestoresParentWorkItemGroup() - { - using var childContext = UnitTestSchedulingContext.Create(_loggerFactory); - var parentTaskRestored = false; - var parentTask = new Task( - () => + if (ReferenceEquals(TaskScheduler.Current, _rootContext.WorkItemGroup.TaskScheduler)) + { + asyncInitialObservation |= 2; + } + + for (var i = 0; i < observations.Length; i++) { - Assert.Same(_rootContext, RuntimeContext.Current); - var childTask = new Task(() => Assert.Same(childContext, RuntimeContext.Current)); - childContext.WorkItemGroup.RunOrQueueTask(childTask); - Assert.True(childTask.IsCompletedSuccessfully, childTask.Exception?.ToString()); - - Assert.Same(_rootContext, RuntimeContext.Current); - var nestedParentTask = new Task(() => parentTaskRestored = true); - nestedParentTask.RunSynchronously(_rootContext.WorkItemGroup.TaskScheduler); - }, - CancellationToken.None, - TaskCreationOptions.HideScheduler); - - parentTask.Start(_rootContext.WorkItemGroup.TaskScheduler); - await parentTask.WaitAsync(TimeSpan.FromSeconds(5)); - Assert.True(parentTaskRestored); + await signal.WaitAsync(); + var observation = 0; + if (ReferenceEquals(RuntimeContext.Current, _rootContext)) + { + observation |= 1; + } + + if (ReferenceEquals(TaskScheduler.Current, _rootContext.WorkItemGroup.TaskScheduler)) + { + observation |= 2; + } + + observations[i].SetResult(observation); + } + } } [Fact] From 8de959caf1af04287e3666322a1c654a730d3308 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 20 Aug 2026 03:01:53 -0700 Subject: [PATCH 07/28] fix(runtime): preserve activation startup ordering Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: ae2e4673-3220-4e33-9073-c2dd36d0fed9 --- .../ActivationDataActivatorProvider.cs | 34 ++++----- .../Activation/IGrainContextActivator.cs | 13 ++++ src/Orleans.Runtime/Catalog/ActivationData.cs | 53 ++++++++++++-- src/Orleans.Runtime/Catalog/Catalog.cs | 8 ++- .../Catalog/StatelessWorkerGrainContext.cs | 6 +- .../Scheduler/ActivationTaskScheduler.cs | 24 +++++++ .../Scheduler/WorkItemGroup.cs | 69 +++++++++++++++++++ .../Utils/DefaultExecutionContext.cs | 33 +++++---- .../OrleansTaskSchedulerBasicTests.cs | 26 +++++-- 9 files changed, 221 insertions(+), 45 deletions(-) diff --git a/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs b/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs index 446712adca5..723aebde164 100644 --- a/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs +++ b/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs @@ -49,15 +49,13 @@ public bool TryGet(GrainType grainType, [NotNullWhen(true)] out IGrainContextAct return true; } - private partial class ActivationDataActivator : IGrainContextActivator + private partial class ActivationDataActivator : IDeferredGrainContextActivator { private readonly IOptions _schedulingOptions; private readonly IGrainActivator _grainActivator; private readonly IServiceProvider _serviceProvider; private readonly GrainTypeSharedContext _sharedComponents; private readonly Func _createWorkItemGroup; - private readonly Action _startActivation; - private readonly ContextCallback _startActivationSynchronously; public ActivationDataActivator( IGrainActivator grainActivator, @@ -74,33 +72,37 @@ public ActivationDataActivator( context, _schedulingOptions, schedulerInstruments); - _startActivation = state => ((ActivationData)state!).Start(_grainActivator); - _startActivationSynchronously = state => - { - var context = (ActivationData)state!; - var task = new Task( - _startActivation, - context, - CancellationToken.None, - TaskCreationOptions.DenyChildAttach); - task.RunSynchronously(context.ActivationTaskScheduler); - }; } public IGrainContext CreateContext(GrainAddress activationAddress, IConfigureGrainContext[] configureActions) + { + var context = CreateDeferredContext(activationAddress, configureActions); + context.Start(); + context.StartMessageLoop(); + return context; + } + + IGrainContext IDeferredGrainContextActivator.CreateDeferredContext( + GrainAddress activationAddress, + IConfigureGrainContext[] configureActions) + => CreateDeferredContext(activationAddress, configureActions); + + private ActivationData CreateDeferredContext( + GrainAddress activationAddress, + IConfigureGrainContext[] configureActions) { var context = new ActivationData( activationAddress, _createWorkItemGroup, _serviceProvider, - _sharedComponents); + _sharedComponents, + _grainActivator); foreach (var configure in configureActions) { configure.Configure(context); } - ExecutionContext.Run(DefaultExecutionContext.Instance, _startActivationSynchronously, context); return context; } } diff --git a/src/Orleans.Runtime/Activation/IGrainContextActivator.cs b/src/Orleans.Runtime/Activation/IGrainContextActivator.cs index ab496fbccb9..c5ac1491401 100644 --- a/src/Orleans.Runtime/Activation/IGrainContextActivator.cs +++ b/src/Orleans.Runtime/Activation/IGrainContextActivator.cs @@ -57,6 +57,9 @@ public GrainContextActivator( /// The grain address. /// The grain context. public IGrainContext CreateInstance(GrainAddress address) + => CreateInstance(address, deferActivation: false); + + internal IGrainContext CreateInstance(GrainAddress address, bool deferActivation) { var grainId = address.GrainId; if (!_activators.TryGetValue(grainId.Type, out var activator)) @@ -64,6 +67,11 @@ public IGrainContext CreateInstance(GrainAddress address) activator = this.CreateActivator(grainId.Type); } + if (deferActivation && activator.Activator is IDeferredGrainContextActivator deferredActivator) + { + return deferredActivator.CreateDeferredContext(address, activator.ConfigureActions); + } + return activator.Activator.CreateContext(address, activator.ConfigureActions); } @@ -134,6 +142,11 @@ public interface IGrainContextActivator public IGrainContext CreateContext(GrainAddress address, IConfigureGrainContext[] configureActions); } + internal interface IDeferredGrainContextActivator : IGrainContextActivator + { + IGrainContext CreateDeferredContext(GrainAddress address, IConfigureGrainContext[] configureActions); + } + /// /// Provides a instance for the provided grain type. /// diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 0ce9b85710e..0f470066ac3 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -42,6 +42,7 @@ internal sealed partial class ActivationData : { private const string GrainAddressMigrationContextKey = "sys.addr"; private readonly GrainTypeSharedContext _shared; + private readonly IGrainActivator _grainActivator; private readonly IServiceScope _serviceScope; private readonly WorkItemGroup _workItemGroup; private readonly List<(Message Message, CoarseStopwatch QueuedTime)> _waitingRequests = new(); @@ -65,6 +66,7 @@ internal sealed partial class ActivationData : #pragma warning disable IDE0052 // Remove unread private members private Task? _messageLoopTask; #pragma warning restore IDE0052 // Remove unread private members + private int _messageLoopStarted; private Activity? _activationActivity; @@ -88,18 +90,29 @@ public ActivationData( GrainAddress grainAddress, Func createWorkItemGroup, IServiceProvider applicationServices, - GrainTypeSharedContext shared) + GrainTypeSharedContext shared, + IGrainActivator grainActivator) { ArgumentNullException.ThrowIfNull(grainAddress); ArgumentNullException.ThrowIfNull(createWorkItemGroup); ArgumentNullException.ThrowIfNull(applicationServices); ArgumentNullException.ThrowIfNull(shared); + ArgumentNullException.ThrowIfNull(grainActivator); _shared = shared; + _grainActivator = grainActivator; Address = grainAddress; _serviceScope = applicationServices.CreateScope(); Debug.Assert(_serviceScope != null, "_serviceScope must not be null."); _workItemGroup = createWorkItemGroup(this); Debug.Assert(_workItemGroup != null, "_workItemGroup must not be null."); + _workItemGroup.ReserveExecution(); + _workItemGroup.QueueAction( + static state => + { + var context = (ActivationData)state; + context._messageLoopTask = context.RunMessageLoop(); + }, + this); } internal void SetActivationActivity(Activity activity) @@ -116,15 +129,38 @@ internal void SetActivationActivity(Activity activity) return _activationActivity?.Context; } - public void Start(IGrainActivator grainActivator) + internal void Start() + { + ExecutionContext.Run( + DefaultExecutionContext.Instance, + static state => + { + var context = (ActivationData)state!; + var task = new Task( + static state => ((ActivationData)state!).StartCore(), + context, + CancellationToken.None, + TaskCreationOptions.DenyChildAttach); + context._workItemGroup.RunTaskSynchronously(task); + task.GetAwaiter().GetResult(); + }, + this); + } + + private void StartCore() { Debug.Assert(Equals(ActivationTaskScheduler, TaskScheduler.Current)); // locking on `this` is intentional as there are other places in the codebase taking locks on ActivationData instances lock (this) { + if (State is not ActivationState.Creating) + { + return; + } + try { - var instance = grainActivator.CreateInstance(this); + var instance = _grainActivator.CreateInstance(this); SetGrainInstance(instance); _activationActivity?.AddEvent(new ActivityEvent("instance-created")); @@ -137,10 +173,19 @@ public void Start(IGrainActivator grainActivator) Deactivate(new(DeactivationReasonCode.ActivationFailed, exception, "Error constructing grain instance."), _activationActivity?.Context, CancellationToken.None); } - _messageLoopTask = RunMessageLoop(); } } + internal void StartMessageLoop() + { + if (Interlocked.Exchange(ref _messageLoopStarted, 1) != 0) + { + throw new InvalidOperationException("The activation message loop has already been started."); + } + + _workItemGroup.ReleaseExecution(); + } + public ActivationTaskScheduler ActivationTaskScheduler => _workItemGroup.TaskScheduler; public IGrainRuntime GrainRuntime => _shared.Runtime; public object? GrainInstance { get; private set; } diff --git a/src/Orleans.Runtime/Catalog/Catalog.cs b/src/Orleans.Runtime/Catalog/Catalog.cs index b7902bf98a7..74b06fefe50 100644 --- a/src/Orleans.Runtime/Catalog/Catalog.cs +++ b/src/Orleans.Runtime/Catalog/Catalog.cs @@ -164,7 +164,7 @@ internal int UnregisterGrainForTesting(GrainId grain) MembershipVersion = MembershipVersion.MinValue, }; - result = this.grainActivator.CreateInstance(address); + result = this.grainActivator.CreateInstance(address, deferActivation: true); activations.RecordNewTarget(result); } } // End lock @@ -194,6 +194,11 @@ internal int UnregisterGrainForTesting(GrainId grain) } } + if (result is ActivationData activationData) + { + activationData.Start(); + } + _catalogInstruments.OnActivationCreated(); // Rehydration occurs before activation. @@ -204,6 +209,7 @@ internal int UnregisterGrainForTesting(GrainId grain) // Initialize the new activation asynchronously. result.Activate(requestContextData); + (result as ActivationData)?.StartMessageLoop(); return result; [MethodImpl(MethodImplOptions.NoInlining)] diff --git a/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs b/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs index b244328caf0..a6775c3e9d8 100644 --- a/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs +++ b/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs @@ -326,17 +326,19 @@ private ActivationData CreateWorker(object? message) { Debug.Assert(!_terminated, "CreateWorker must not be called on a terminated stateless worker context."); var address = GrainAddress.GetAddress(Address.SiloAddress, Address.GrainId, ActivationId.NewId()); - var newWorker = (ActivationData)_innerActivator.CreateContext(address, []); + var newWorker = (ActivationData)((IDeferredGrainContextActivator)_innerActivator).CreateDeferredContext(address, []); // Observe the create/destroy lifecycle of the activation newWorker.SetComponent(this); + _workers.Add(newWorker); + newWorker.Start(); // If this is a new worker and there is a message in scope, try to get the request context and activate the worker var requestContext = (message as Message)?.RequestContextData ?? []; var cancellation = new CancellationTokenSource(_shared.Shared.InternalRuntime.CollectionOptions.Value.ActivationTimeout); newWorker.Activate(requestContext, cancellation.Token); - _workers.Add(newWorker); + newWorker.StartMessageLoop(); StatelessWorkerEvents.EmitWorkerCreated(this, newWorker, _workers.Count); return newWorker; diff --git a/src/Orleans.Runtime/Scheduler/ActivationTaskScheduler.cs b/src/Orleans.Runtime/Scheduler/ActivationTaskScheduler.cs index a854e04b108..9e1042cf33b 100644 --- a/src/Orleans.Runtime/Scheduler/ActivationTaskScheduler.cs +++ b/src/Orleans.Runtime/Scheduler/ActivationTaskScheduler.cs @@ -21,6 +21,7 @@ internal sealed partial class ActivationTaskScheduler : TaskScheduler private readonly long myId; #endif private readonly WorkItemGroup workerGroup; + private Task? _synchronouslyQueuedTask; #if EXTRA_STATS private readonly CounterStatistic turnsExecutedStatistic; #endif @@ -56,10 +57,33 @@ internal void RunTaskFromWorkItemGroup(Task task) } } + internal void RunTaskSynchronously(Task task) + { + Debug.Assert(_synchronouslyQueuedTask is null); + _synchronouslyQueuedTask = task; + try + { + task.Start(this); + if (!TryExecuteTask(task)) + { + throw new InvalidOperationException($"Unable to execute synchronous task {task.Id}."); + } + } + finally + { + _synchronouslyQueuedTask = null; + } + } + /// Queues a task to the scheduler. /// The task to be queued. protected override void QueueTask(Task task) { + if (ReferenceEquals(task, _synchronouslyQueuedTask)) + { + return; + } + #if DEBUG LogTraceQueueTask(myId, task.Id); #endif diff --git a/src/Orleans.Runtime/Scheduler/WorkItemGroup.cs b/src/Orleans.Runtime/Scheduler/WorkItemGroup.cs index 8320c332a8d..481015dce22 100644 --- a/src/Orleans.Runtime/Scheduler/WorkItemGroup.cs +++ b/src/Orleans.Runtime/Scheduler/WorkItemGroup.cs @@ -119,6 +119,75 @@ public void EnqueueTask(Task task) } } + internal void ReserveExecution() + { + lock (_lockObj) + { + if (_state != WorkGroupStatus.Waiting) + { + throw new InvalidOperationException($"Cannot reserve execution while {this} is {_state}."); + } + + _state = WorkGroupStatus.Running; + } + } + + internal void RunTaskSynchronously(Task task) + { + long taskStart; + lock (_lockObj) + { + if (_state != WorkGroupStatus.Running || _currentTask is not null) + { + throw new InvalidOperationException($"Synchronous execution requires a reserved {this}."); + } + + _currentTask = task; + _currentTaskStarted = taskStart = Environment.TickCount64; + _totalItemsEnqueued++; + } + + RuntimeContext.SetExecutionContext(GrainContext, out var originalContext); + try + { +#if DEBUG + LogTaskStart(task); +#endif + TaskScheduler.RunTaskSynchronously(task); + } + finally + { + RuntimeContext.ResetExecutionContext(originalContext); + _totalItemsProcessed++; + var taskDurationMs = Environment.TickCount64 - taskStart; + if (taskDurationMs > (long)Math.Ceiling(_schedulingOptions.TurnWarningLengthThreshold.TotalMilliseconds)) + { + _schedulerInstruments.OnLongRunningTurn(); + LogLongRunningTurn(task, taskDurationMs); + } + + _currentTask = null; + } + } + + internal void ReleaseExecution() + { + lock (_lockObj) + { + Debug.Assert(_state == WorkGroupStatus.Running); + Debug.Assert(_currentTask is null); + if (_workItems.Count > 0) + { + _state = WorkGroupStatus.Runnable; + ScheduleExecution(this); + } + else + { + _state = WorkGroupStatus.Waiting; + } + } + } + [MethodImpl(MethodImplOptions.NoInlining)] private void LogTooManyTasksInQueue(int count, int maxPendingItemsLimit) { diff --git a/src/Orleans.Runtime/Utils/DefaultExecutionContext.cs b/src/Orleans.Runtime/Utils/DefaultExecutionContext.cs index 9f65209c1bf..97708c831e1 100644 --- a/src/Orleans.Runtime/Utils/DefaultExecutionContext.cs +++ b/src/Orleans.Runtime/Utils/DefaultExecutionContext.cs @@ -13,22 +13,25 @@ internal static class DefaultExecutionContext internal static ExecutionContext CaptureDefault() { var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - ThreadPool.UnsafeQueueUserWorkItem( - static completion => - { - try + if (!ThreadPool.UnsafeQueueUserWorkItem( + static completion => { - completion.SetResult( - ExecutionContext.Capture() - ?? throw new InvalidOperationException("Could not capture the default execution context.")); - } - catch (Exception exception) - { - completion.SetException(exception); - } - }, - completion, - preferLocal: false); + try + { + completion.SetResult( + ExecutionContext.Capture() + ?? throw new InvalidOperationException("Could not capture the default execution context.")); + } + catch (Exception exception) + { + completion.SetException(exception); + } + }, + completion, + preferLocal: false)) + { + throw new InvalidOperationException("Could not queue work to capture the default execution context."); + } return completion.Task.GetAwaiter().GetResult(); } diff --git a/test/Orleans.Core.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs b/test/Orleans.Core.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs index b4ae4455719..d8033b50244 100644 --- a/test/Orleans.Core.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs +++ b/test/Orleans.Core.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs @@ -117,7 +117,7 @@ public async Task Async_Task_Start_ActivationTaskScheduler() } [Fact] - public async Task Sched_RunSynchronously_QueuesWhenContextIsNotCurrent() + public async Task Sched_RunTaskSynchronously_DefersQueuedWorkUntilExecutionIsReleased() { Task? queuedTask = null; var startCompleted = 0; @@ -135,20 +135,23 @@ public async Task Sched_RunSynchronously_QueuesWhenContextIsNotCurrent() }); queuedTask.Start(_rootContext.WorkItemGroup.TaskScheduler); - Assert.Contains("WorkGroupStatus=Running", _rootContext.WorkItemGroup.DumpStatus()); + Assert.Equal(1, _rootContext.WorkItemGroup.ExternalWorkItemCount); Assert.False(queuedTask.IsCompleted); Volatile.Write(ref startCompleted, 1); }); - startTask.RunSynchronously(_rootContext.WorkItemGroup.TaskScheduler); + _rootContext.WorkItemGroup.ReserveExecution(); + _rootContext.WorkItemGroup.RunTaskSynchronously(startTask); Assert.True(startTask.IsCompletedSuccessfully, startTask.Exception?.ToString()); + Assert.False(queuedTask!.IsCompleted); + _rootContext.WorkItemGroup.ReleaseExecution(); - await queuedTask!.WaitAsync(TimeSpan.FromSeconds(5)); + await queuedTask.WaitAsync(TimeSpan.FromSeconds(5)); Assert.Equal(1, queuedTaskObservedStartCompletion); } [Fact] - public async Task Sched_RunSynchronously_PreservesContextAcrossAsynchronousSignals() + public async Task Sched_RunTaskSynchronously_PreservesContextForReleasedAsynchronousWork() { const int IterationCount = 1_000; var signal = new SingleWaiterAutoResetEvent { RunContinuationsAsynchronously = true }; @@ -161,6 +164,12 @@ public async Task Sched_RunSynchronously_PreservesContextAcrossAsynchronousSigna Task? observationLoop = null; var initialObservation = 0; var asyncInitialObservation = 0; + var loopStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var loopStarter = new Task(() => + { + observationLoop = ObserveSignals(); + loopStarted.SetResult(); + }); var startTask = new Task(() => { if (ReferenceEquals(RuntimeContext.Current, _rootContext)) @@ -173,11 +182,14 @@ public async Task Sched_RunSynchronously_PreservesContextAcrossAsynchronousSigna initialObservation |= 2; } - observationLoop = ObserveSignals(); + loopStarter.Start(_rootContext.WorkItemGroup.TaskScheduler); }); - startTask.RunSynchronously(_rootContext.WorkItemGroup.TaskScheduler); + _rootContext.WorkItemGroup.ReserveExecution(); + _rootContext.WorkItemGroup.RunTaskSynchronously(startTask); Assert.True(startTask.IsCompletedSuccessfully, startTask.Exception?.ToString()); Assert.Equal(3, initialObservation); + _rootContext.WorkItemGroup.ReleaseExecution(); + await loopStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); Assert.Equal(3, asyncInitialObservation); for (var i = 0; i < observations.Length; i++) From 00028fd658817ce16c8e3a036a2c2199e8dbd3e9 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 20 Aug 2026 21:32:17 -0700 Subject: [PATCH 08/28] refactor(runtime): use one-shot activation startup Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: ae2e4673-3220-4e33-9073-c2dd36d0fed9 --- .../ActivationDataActivatorProvider.cs | 32 ++--- .../Activation/IGrainContextActivator.cs | 58 +++++++-- src/Orleans.Runtime/Catalog/ActivationData.cs | 85 ++++++++---- src/Orleans.Runtime/Catalog/Catalog.cs | 123 ++++++++++++++---- .../Catalog/StatelessWorkerGrainContext.cs | 57 ++++++-- .../Scheduler/ActivationTaskScheduler.cs | 18 +-- .../Scheduler/WorkItemGroup.cs | 57 +++++++- .../OrleansTaskSchedulerBasicTests.cs | 61 +++++++-- .../GrainActivatorTests.cs | 31 +++++ 9 files changed, 401 insertions(+), 121 deletions(-) diff --git a/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs b/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs index 723aebde164..26a39c15939 100644 --- a/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs +++ b/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs @@ -49,7 +49,7 @@ public bool TryGet(GrainType grainType, [NotNullWhen(true)] out IGrainContextAct return true; } - private partial class ActivationDataActivator : IDeferredGrainContextActivator + private partial class ActivationDataActivator : IPreparedGrainContextActivator { private readonly IOptions _schedulingOptions; private readonly IGrainActivator _grainActivator; @@ -76,18 +76,12 @@ public ActivationDataActivator( public IGrainContext CreateContext(GrainAddress activationAddress, IConfigureGrainContext[] configureActions) { - var context = CreateDeferredContext(activationAddress, configureActions); - context.Start(); - context.StartMessageLoop(); - return context; + var preparedContext = CreatePreparedContext(activationAddress, configureActions); + using var startup = preparedContext.Start(); + return preparedContext.Context; } - IGrainContext IDeferredGrainContextActivator.CreateDeferredContext( - GrainAddress activationAddress, - IConfigureGrainContext[] configureActions) - => CreateDeferredContext(activationAddress, configureActions); - - private ActivationData CreateDeferredContext( + public PreparedGrainContext CreatePreparedContext( GrainAddress activationAddress, IConfigureGrainContext[] configureActions) { @@ -98,12 +92,20 @@ private ActivationData CreateDeferredContext( _sharedComponents, _grainActivator); - foreach (var configure in configureActions) + try { - configure.Configure(context); - } + foreach (var configure in configureActions) + { + configure.Configure(context); + } - return context; + return new(context, (IGrainContextStartup)context); + } + catch + { + ((IGrainContextStartup)context).Abort(); + throw; + } } } } diff --git a/src/Orleans.Runtime/Activation/IGrainContextActivator.cs b/src/Orleans.Runtime/Activation/IGrainContextActivator.cs index c5ac1491401..165e96cc09f 100644 --- a/src/Orleans.Runtime/Activation/IGrainContextActivator.cs +++ b/src/Orleans.Runtime/Activation/IGrainContextActivator.cs @@ -14,6 +14,7 @@ using Orleans.GrainReferences; using Orleans.Metadata; using Orleans.Runtime.Placement; +using Orleans.Runtime.Scheduler; using Orleans.Serialization.Invocation; using Orleans.Serialization.Session; @@ -57,9 +58,13 @@ public GrainContextActivator( /// The grain address. /// The grain context. public IGrainContext CreateInstance(GrainAddress address) - => CreateInstance(address, deferActivation: false); + { + var preparedContext = CreatePreparedContext(address); + using var lease = preparedContext.Start(); + return preparedContext.Context; + } - internal IGrainContext CreateInstance(GrainAddress address, bool deferActivation) + internal PreparedGrainContext CreatePreparedContext(GrainAddress address) { var grainId = address.GrainId; if (!_activators.TryGetValue(grainId.Type, out var activator)) @@ -67,12 +72,9 @@ internal IGrainContext CreateInstance(GrainAddress address, bool deferActivation activator = this.CreateActivator(grainId.Type); } - if (deferActivation && activator.Activator is IDeferredGrainContextActivator deferredActivator) - { - return deferredActivator.CreateDeferredContext(address, activator.ConfigureActions); - } - - return activator.Activator.CreateContext(address, activator.ConfigureActions); + return activator.Activator is IPreparedGrainContextActivator preparedActivator + ? preparedActivator.CreatePreparedContext(address, activator.ConfigureActions) + : new(activator.Activator.CreateContext(address, activator.ConfigureActions), startup: null); } private (IGrainContextActivator, IConfigureGrainContext[]) CreateActivator(GrainType grainType) @@ -139,12 +141,48 @@ public interface IGrainContextActivator /// The grain address. /// The actions which must be used to configure the context before grain construction begins. /// The newly created grain context. + /// + /// Use to create and start a context. + /// This method only performs activator-specific context creation. Custom activators remain responsible + /// for any custom startup behavior which their context requires. + /// public IGrainContext CreateContext(GrainAddress address, IConfigureGrainContext[] configureActions); } - internal interface IDeferredGrainContextActivator : IGrainContextActivator + internal interface IPreparedGrainContextActivator : IGrainContextActivator { - IGrainContext CreateDeferredContext(GrainAddress address, IConfigureGrainContext[] configureActions); + PreparedGrainContext CreatePreparedContext( + GrainAddress address, + IConfigureGrainContext[] configureActions); + } + + internal readonly struct PreparedGrainContext(IGrainContext context, IGrainContextStartup? startup) + { + private readonly IGrainContext? _context = context; + private readonly IGrainContextStartup? _startup = startup; + + public IGrainContext Context + => _context ?? throw new InvalidOperationException("The grain context activation is not initialized."); + + public IDisposable Start() => _startup?.Start() ?? NoopDisposable.Instance; + + public void Abort() => _startup?.Abort(); + + private sealed class NoopDisposable : IDisposable + { + public static NoopDisposable Instance { get; } = new(); + + public void Dispose() + { + } + } + } + + internal interface IGrainContextStartup + { + IDisposable Start(); + + void Abort(); } /// diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 0f470066ac3..4ed9dd31687 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -13,6 +13,7 @@ using Orleans.Internal; using Orleans.Runtime.Diagnostics; using Orleans.Runtime.GrainDirectory; +using Orleans.Runtime.Internal; using Orleans.Runtime.Placement; using Orleans.Runtime.Scheduler; using Orleans.Serialization.Invocation; @@ -37,6 +38,7 @@ internal sealed partial class ActivationData : IGrainManagementExtension, IGrainCallCancellationExtension, ICallChainReentrantGrainContext, + IGrainContextStartup, IAsyncDisposable, IDisposable { @@ -45,6 +47,7 @@ internal sealed partial class ActivationData : private readonly IGrainActivator _grainActivator; private readonly IServiceScope _serviceScope; private readonly WorkItemGroup _workItemGroup; + private WorkItemGroup.ActivationStartup? _startup; private readonly List<(Message Message, CoarseStopwatch QueuedTime)> _waitingRequests = new(); private readonly Dictionary _runningRequests = new(); private readonly SingleWaiterAutoResetEvent _workSignal = new() { RunContinuationsAsynchronously = true }; @@ -66,7 +69,7 @@ internal sealed partial class ActivationData : #pragma warning disable IDE0052 // Remove unread private members private Task? _messageLoopTask; #pragma warning restore IDE0052 // Remove unread private members - private int _messageLoopStarted; + private int _started; private Activity? _activationActivity; @@ -105,7 +108,7 @@ public ActivationData( Debug.Assert(_serviceScope != null, "_serviceScope must not be null."); _workItemGroup = createWorkItemGroup(this); Debug.Assert(_workItemGroup != null, "_workItemGroup must not be null."); - _workItemGroup.ReserveExecution(); + _startup = _workItemGroup.BeginActivationStartup(); _workItemGroup.QueueAction( static state => { @@ -129,22 +132,62 @@ internal void SetActivationActivity(Activity activity) return _activationActivity?.Context; } - internal void Start() + IDisposable IGrainContextStartup.Start() { - ExecutionContext.Run( - DefaultExecutionContext.Instance, - static state => - { - var context = (ActivationData)state!; - var task = new Task( - static state => ((ActivationData)state!).StartCore(), - context, + if (Interlocked.Exchange(ref _started, 1) != 0 || _startup is not { } startup) + { + throw new InvalidOperationException("The activation has already been started."); + } + + try + { + ExecutionContext.Run( + DefaultExecutionContext.Instance, + static state => + { + var context = (ActivationData)state!; + var task = new Task( + static state => ((ActivationData)state!).StartCore(), + context, + CancellationToken.None, + TaskCreationOptions.DenyChildAttach); + context._startup!.RunConstructor(task); + task.GetAwaiter().GetResult(); + }, + this); + _startup = null; + return startup; + } + catch (Exception exception) + { + _startup = null; + Deactivate( + new DeactivationReason( + DeactivationReasonCode.ActivationFailed, + exception, + "Error starting grain construction."), + CancellationToken.None); + startup.Dispose(); + throw; + } + } + + void IGrainContextStartup.Abort() + { + if (Interlocked.Exchange(ref _startup, null) is { } startup) + { + startup.Abort(); + using var suppressExecutionContext = new ExecutionContextSuppressor(); + Task.Factory.StartNew( + static state => DisposeAsync(state!).AsTask(), + _serviceScope, CancellationToken.None, - TaskCreationOptions.DenyChildAttach); - context._workItemGroup.RunTaskSynchronously(task); - task.GetAwaiter().GetResult(); - }, - this); + TaskCreationOptions.DenyChildAttach, + TaskScheduler.Default) + .Unwrap() + .GetAwaiter() + .GetResult(); + } } private void StartCore() @@ -176,16 +219,6 @@ private void StartCore() } } - internal void StartMessageLoop() - { - if (Interlocked.Exchange(ref _messageLoopStarted, 1) != 0) - { - throw new InvalidOperationException("The activation message loop has already been started."); - } - - _workItemGroup.ReleaseExecution(); - } - public ActivationTaskScheduler ActivationTaskScheduler => _workItemGroup.TaskScheduler; public IGrainRuntime GrainRuntime => _shared.Runtime; public object? GrainInstance { get; private set; } diff --git a/src/Orleans.Runtime/Catalog/Catalog.cs b/src/Orleans.Runtime/Catalog/Catalog.cs index 74b06fefe50..916e4bfb77e 100644 --- a/src/Orleans.Runtime/Catalog/Catalog.cs +++ b/src/Orleans.Runtime/Catalog/Catalog.cs @@ -3,6 +3,7 @@ using Microsoft.Extensions.Logging; using Orleans.GrainDirectory; using Orleans.Runtime.GrainDirectory; +using Orleans.Runtime.Scheduler; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using Orleans.Diagnostics; @@ -135,6 +136,7 @@ internal int UnregisterGrainForTesting(GrainId grain) Dictionary? requestContextData, MigrationContext? rehydrationContext) { + PreparedGrainContext preparedContext = default; if (TryGetGrainContext(grainId, out var result)) { rehydrationContext?.Dispose(); @@ -164,7 +166,8 @@ internal int UnregisterGrainForTesting(GrainId grain) MembershipVersion = MembershipVersion.MinValue, }; - result = this.grainActivator.CreateInstance(address, deferActivation: true); + preparedContext = this.grainActivator.CreatePreparedContext(address); + result = preparedContext.Context; activations.RecordNewTarget(result); } } // End lock @@ -175,42 +178,106 @@ internal int UnregisterGrainForTesting(GrainId grain) return UnableToCreateActivation(this, grainId); } - // Start activation span with parent context from request if available - var parentContext = requestContextData.TryGetActivityContext(); - var activationActivity = parentContext.HasValue - ? ActivitySources.LifecycleGrainSource.StartActivity(ActivityNames.ActivateGrain, ActivityKind.Internal, parentContext.Value) - : ActivitySources.LifecycleGrainSource.StartActivity(ActivityNames.ActivateGrain, ActivityKind.Internal); - if (activationActivity is not null) + IDisposable activationStartup; + var startAttempted = false; + try { - activationActivity.SetTag(ActivityTagKeys.GrainId, grainId.ToString()); - activationActivity.SetTag(ActivityTagKeys.GrainType, grainId.Type.ToString()); - activationActivity.SetTag(ActivityTagKeys.SiloId, Silo.ToString()); - activationActivity.SetTag(ActivityTagKeys.ActivationCause, rehydrationContext is null ? "new" : "rehydrate"); - if (result is ActivationData act) + // Start activation span with parent context from request if available + var parentContext = requestContextData.TryGetActivityContext(); + var activationActivity = parentContext.HasValue + ? ActivitySources.LifecycleGrainSource.StartActivity(ActivityNames.ActivateGrain, ActivityKind.Internal, parentContext.Value) + : ActivitySources.LifecycleGrainSource.StartActivity(ActivityNames.ActivateGrain, ActivityKind.Internal); + if (activationActivity is not null) { - activationActivity.SetTag(ActivityTagKeys.ActivationId, act.ActivationId.ToString()); - act.SetActivationActivity(activationActivity); - activationActivity.AddEvent(new ActivityEvent("creating")); + activationActivity.SetTag(ActivityTagKeys.GrainId, grainId.ToString()); + activationActivity.SetTag(ActivityTagKeys.GrainType, grainId.Type.ToString()); + activationActivity.SetTag(ActivityTagKeys.SiloId, Silo.ToString()); + activationActivity.SetTag(ActivityTagKeys.ActivationCause, rehydrationContext is null ? "new" : "rehydrate"); + if (result is ActivationData act) + { + activationActivity.SetTag(ActivityTagKeys.ActivationId, act.ActivationId.ToString()); + act.SetActivationActivity(activationActivity); + activationActivity.AddEvent(new ActivityEvent("creating")); + } } - } - if (result is ActivationData activationData) - { - activationData.Start(); + startAttempted = true; + activationStartup = preparedContext.Start(); } + catch (Exception exception) + { + List? cleanupExceptions = null; + if (!startAttempted) + { + try + { + result.Deactivate( + new DeactivationReason( + DeactivationReasonCode.ActivationFailed, + exception, + "Error preparing grain activation."), + CancellationToken.None); + } + catch (Exception cleanupException) + { + (cleanupExceptions ??= []).Add(cleanupException); + } - _catalogInstruments.OnActivationCreated(); + try + { + using var startup = preparedContext.Start(); + } + catch (Exception cleanupException) + { + (cleanupExceptions ??= []).Add(cleanupException); + } + } - // Rehydration occurs before activation. - if (rehydrationContext is not null) - { - result.Rehydrate(rehydrationContext); + try + { + rehydrationContext?.Dispose(); + } + catch (Exception cleanupException) + { + (cleanupExceptions ??= []).Add(cleanupException); + } + + if (cleanupExceptions is not null) + { + cleanupExceptions.Insert(0, exception); + throw new AggregateException("Error preparing grain activation and cleaning up the failed context.", cleanupExceptions); + } + + throw; } - // Initialize the new activation asynchronously. - result.Activate(requestContextData); - (result as ActivationData)?.StartMessageLoop(); - return result; + using (activationStartup) + { + _catalogInstruments.OnActivationCreated(); + + try + { + // Rehydration occurs before activation. + if (rehydrationContext is not null) + { + result.Rehydrate(rehydrationContext); + } + + // Initialize the new activation asynchronously. + result.Activate(requestContextData); + return result; + } + catch (Exception exception) + { + result.Deactivate( + new DeactivationReason( + DeactivationReasonCode.ActivationFailed, + exception, + "Error starting grain activation."), + CancellationToken.None); + throw; + } + } [MethodImpl(MethodImplOptions.NoInlining)] static IGrainContext? UnableToCreateActivation(Catalog self, GrainId grainId) diff --git a/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs b/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs index a6775c3e9d8..c4155fc0c24 100644 --- a/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs +++ b/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs @@ -9,6 +9,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Orleans.Runtime.Diagnostics; +using Orleans.Runtime.Scheduler; namespace Orleans.Runtime; @@ -326,22 +327,52 @@ private ActivationData CreateWorker(object? message) { Debug.Assert(!_terminated, "CreateWorker must not be called on a terminated stateless worker context."); var address = GrainAddress.GetAddress(Address.SiloAddress, Address.GrainId, ActivationId.NewId()); - var newWorker = (ActivationData)((IDeferredGrainContextActivator)_innerActivator).CreateDeferredContext(address, []); - - // Observe the create/destroy lifecycle of the activation - newWorker.SetComponent(this); - _workers.Add(newWorker); - newWorker.Start(); + var preparedContext = ((IPreparedGrainContextActivator)_innerActivator).CreatePreparedContext(address, []); + var newWorker = (ActivationData)preparedContext.Context; + IDisposable activationStartup; + var startAttempted = false; + try + { + // Observe the create/destroy lifecycle of the activation + newWorker.SetComponent(this); + _workers.Add(newWorker); + startAttempted = true; + activationStartup = preparedContext.Start(); + } + catch + { + if (!startAttempted) + { + preparedContext.Abort(); + _workers.Remove(newWorker); + } - // If this is a new worker and there is a message in scope, try to get the request context and activate the worker - var requestContext = (message as Message)?.RequestContextData ?? []; - var cancellation = new CancellationTokenSource(_shared.Shared.InternalRuntime.CollectionOptions.Value.ActivationTimeout); + throw; + } - newWorker.Activate(requestContext, cancellation.Token); - newWorker.StartMessageLoop(); - StatelessWorkerEvents.EmitWorkerCreated(this, newWorker, _workers.Count); + using (activationStartup) + { + try + { + // If this is a new worker and there is a message in scope, try to get the request context and activate the worker + var requestContext = (message as Message)?.RequestContextData ?? []; + var cancellation = new CancellationTokenSource(_shared.Shared.InternalRuntime.CollectionOptions.Value.ActivationTimeout); + newWorker.Activate(requestContext, cancellation.Token); + StatelessWorkerEvents.EmitWorkerCreated(this, newWorker, _workers.Count); - return newWorker; + return newWorker; + } + catch (Exception exception) + { + newWorker.Deactivate( + new DeactivationReason( + DeactivationReasonCode.ActivationFailed, + exception, + "Error starting stateless worker activation."), + CancellationToken.None); + throw; + } + } } private void DeactivateInternal(DeactivationReason reason, CancellationToken cancellationToken) diff --git a/src/Orleans.Runtime/Scheduler/ActivationTaskScheduler.cs b/src/Orleans.Runtime/Scheduler/ActivationTaskScheduler.cs index 9e1042cf33b..5eaad97fbc0 100644 --- a/src/Orleans.Runtime/Scheduler/ActivationTaskScheduler.cs +++ b/src/Orleans.Runtime/Scheduler/ActivationTaskScheduler.cs @@ -21,7 +21,6 @@ internal sealed partial class ActivationTaskScheduler : TaskScheduler private readonly long myId; #endif private readonly WorkItemGroup workerGroup; - private Task? _synchronouslyQueuedTask; #if EXTRA_STATS private readonly CounterStatistic turnsExecutedStatistic; #endif @@ -59,19 +58,10 @@ internal void RunTaskFromWorkItemGroup(Task task) internal void RunTaskSynchronously(Task task) { - Debug.Assert(_synchronouslyQueuedTask is null); - _synchronouslyQueuedTask = task; - try + task.Start(this); + if (!TryExecuteTask(task)) { - task.Start(this); - if (!TryExecuteTask(task)) - { - throw new InvalidOperationException($"Unable to execute synchronous task {task.Id}."); - } - } - finally - { - _synchronouslyQueuedTask = null; + throw new InvalidOperationException($"Unable to execute synchronous task {task.Id}."); } } @@ -79,7 +69,7 @@ internal void RunTaskSynchronously(Task task) /// The task to be queued. protected override void QueueTask(Task task) { - if (ReferenceEquals(task, _synchronouslyQueuedTask)) + if (workerGroup.IsCurrentTask(task)) { return; } diff --git a/src/Orleans.Runtime/Scheduler/WorkItemGroup.cs b/src/Orleans.Runtime/Scheduler/WorkItemGroup.cs index 481015dce22..1ba689cd795 100644 --- a/src/Orleans.Runtime/Scheduler/WorkItemGroup.cs +++ b/src/Orleans.Runtime/Scheduler/WorkItemGroup.cs @@ -56,6 +56,8 @@ internal int ExternalWorkItemCount get { lock (_lockObj) { return _workItems.Count; } } } + internal bool IsCurrentTask(Task task) => ReferenceEquals(_currentTask, task); + public WorkItemGroup( IGrainContext grainContext, IOptions schedulingOptions, @@ -119,7 +121,7 @@ public void EnqueueTask(Task task) } } - internal void ReserveExecution() + internal ActivationStartup BeginActivationStartup() { lock (_lockObj) { @@ -129,10 +131,11 @@ internal void ReserveExecution() } _state = WorkGroupStatus.Running; + return new(this); } } - internal void RunTaskSynchronously(Task task) + private void RunTaskSynchronously(Task task) { long taskStart; lock (_lockObj) @@ -170,7 +173,7 @@ internal void RunTaskSynchronously(Task task) } } - internal void ReleaseExecution() + private void ReleaseExecution() { lock (_lockObj) { @@ -188,6 +191,54 @@ internal void ReleaseExecution() } } + private void AbortExecution() + { + lock (_lockObj) + { + if (_state != WorkGroupStatus.Running || _currentTask is not null) + { + throw new InvalidOperationException($"Cannot abort execution while {this} is {_state}."); + } + + _workItems.Clear(); + _state = WorkGroupStatus.Waiting; + } + } + // One-shot scheduler reservation used while an activation is published and constructed. + internal sealed class ActivationStartup : IDisposable + { + private WorkItemGroup? _owner; + private int _constructorStarted; + + internal ActivationStartup(WorkItemGroup owner) + { + _owner = owner; + } + + public void RunConstructor(Task task) + { + if (Interlocked.Exchange(ref _constructorStarted, 1) != 0) + { + throw new InvalidOperationException("The activation constructor has already been run."); + } + + (_owner ?? throw new ObjectDisposedException(nameof(ActivationStartup))) + .RunTaskSynchronously(task); + } + + public void Dispose() => Interlocked.Exchange(ref _owner, null)?.ReleaseExecution(); + + public void Abort() + { + if (Volatile.Read(ref _constructorStarted) != 0) + { + throw new InvalidOperationException("An activation startup cannot be aborted after construction has begun."); + } + + Interlocked.Exchange(ref _owner, null)?.AbortExecution(); + } + } + [MethodImpl(MethodImplOptions.NoInlining)] private void LogTooManyTasksInQueue(int count, int maxPendingItemsLimit) { diff --git a/test/Orleans.Core.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs b/test/Orleans.Core.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs index d8033b50244..53afa717937 100644 --- a/test/Orleans.Core.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs +++ b/test/Orleans.Core.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs @@ -117,7 +117,7 @@ public async Task Async_Task_Start_ActivationTaskScheduler() } [Fact] - public async Task Sched_RunTaskSynchronously_DefersQueuedWorkUntilExecutionIsReleased() + public async Task Sched_ActivationStartup_DefersQueuedWorkUntilDisposed() { Task? queuedTask = null; var startCompleted = 0; @@ -140,18 +140,22 @@ public async Task Sched_RunTaskSynchronously_DefersQueuedWorkUntilExecutionIsRel Volatile.Write(ref startCompleted, 1); }); - _rootContext.WorkItemGroup.ReserveExecution(); - _rootContext.WorkItemGroup.RunTaskSynchronously(startTask); - Assert.True(startTask.IsCompletedSuccessfully, startTask.Exception?.ToString()); - Assert.False(queuedTask!.IsCompleted); - _rootContext.WorkItemGroup.ReleaseExecution(); + using (var startup = _rootContext.WorkItemGroup.BeginActivationStartup()) + { + startup.RunConstructor(startTask); + Assert.True(startTask.IsCompletedSuccessfully, startTask.Exception?.ToString()); + Assert.False(queuedTask!.IsCompleted); + Assert.Throws( + () => startup.RunConstructor(new Task(static () => { }))); + Assert.Throws(startup.Abort); + } await queuedTask.WaitAsync(TimeSpan.FromSeconds(5)); Assert.Equal(1, queuedTaskObservedStartCompletion); } [Fact] - public async Task Sched_RunTaskSynchronously_PreservesContextForReleasedAsynchronousWork() + public async Task Sched_ActivationStartup_PreservesContextForReleasedAsynchronousWork() { const int IterationCount = 1_000; var signal = new SingleWaiterAutoResetEvent { RunContinuationsAsynchronously = true }; @@ -184,11 +188,12 @@ public async Task Sched_RunTaskSynchronously_PreservesContextForReleasedAsynchro loopStarter.Start(_rootContext.WorkItemGroup.TaskScheduler); }); - _rootContext.WorkItemGroup.ReserveExecution(); - _rootContext.WorkItemGroup.RunTaskSynchronously(startTask); - Assert.True(startTask.IsCompletedSuccessfully, startTask.Exception?.ToString()); - Assert.Equal(3, initialObservation); - _rootContext.WorkItemGroup.ReleaseExecution(); + using (var startup = _rootContext.WorkItemGroup.BeginActivationStartup()) + { + startup.RunConstructor(startTask); + Assert.True(startTask.IsCompletedSuccessfully, startTask.Exception?.ToString()); + Assert.Equal(3, initialObservation); + } await loopStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); Assert.Equal(3, asyncInitialObservation); @@ -231,6 +236,38 @@ async Task ObserveSignals() } } + [Fact] + public async Task Sched_ActivationStartup_DisposeIsIdempotent() + { + var queuedTaskRan = false; + var queuedTask = new Task(() => queuedTaskRan = true); + var startup = _rootContext.WorkItemGroup.BeginActivationStartup(); + queuedTask.Start(_rootContext.WorkItemGroup.TaskScheduler); + + startup.Dispose(); + startup.Dispose(); + + await queuedTask.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.True(queuedTaskRan); + } + + [Fact] + public async Task Sched_ActivationStartup_AbortDiscardsQueuedWorkAndAllowsReuse() + { + var discardedTask = new Task(static () => throw new InvalidOperationException("This task must not execute.")); + var startup = _rootContext.WorkItemGroup.BeginActivationStartup(); + discardedTask.Start(_rootContext.WorkItemGroup.TaskScheduler); + startup.Abort(); + + Assert.False(discardedTask.IsCompleted); + + var subsequentTaskRan = false; + var subsequentTask = new Task(() => subsequentTaskRan = true); + subsequentTask.Start(_rootContext.WorkItemGroup.TaskScheduler); + await subsequentTask.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.True(subsequentTaskRan); + } + [Fact] public async Task Sched_SimpleFifoTest() { diff --git a/test/Orleans.Runtime.Tests/GrainActivatorTests.cs b/test/Orleans.Runtime.Tests/GrainActivatorTests.cs index ca3f7438012..f25730f3159 100644 --- a/test/Orleans.Runtime.Tests/GrainActivatorTests.cs +++ b/test/Orleans.Runtime.Tests/GrainActivatorTests.cs @@ -151,6 +151,37 @@ public async Task ContextCreationStartsActivationSynchronouslyOnActivationSchedu } } + [Fact, TestCategory("BVT")] + public async Task BuiltInActivatorCreateContextStartsContext() + { + var primary = Assert.IsType(fixture.HostedCluster.Primary); + var services = primary.ServiceProvider; + var grainType = services.GetRequiredService().GetGrainType(typeof(ExplicitlyRegisteredSimpleDIGrain)); + var address = GrainAddress.NewActivationAddress( + primary.SiloAddress, + GrainId.Create(grainType, Guid.NewGuid().ToString())); + IGrainContextActivator? activator = null; + foreach (var provider in services.GetServices()) + { + if (provider.TryGet(grainType, out activator)) + { + break; + } + } + + Assert.NotNull(activator); + var context = Assert.IsType(activator.CreateContext(address, [])); + try + { + Assert.NotNull(context.GrainInstance); + } + finally + { + context.Deactivate(new(DeactivationReasonCode.ApplicationRequested, "Test completed.")); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(10)); + } + } + /// /// Custom grain activator that bypasses dependency injection entirely. /// Implements both IGrainActivator (for creation/disposal) and IConfigureGrainTypeComponents From 01a2f6625dc2b3312f834f8f14eb9b5edda8c6f1 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 20 Aug 2026 22:53:27 -0700 Subject: [PATCH 09/28] refactor(runtime): expose grain context startup --- .../Core/IGrainContext.cs | 19 +++++++ .../ActivationDataActivatorProvider.cs | 15 +---- .../Activation/IGrainContextActivator.cs | 55 +++---------------- src/Orleans.Runtime/Catalog/ActivationData.cs | 26 +++++---- src/Orleans.Runtime/Catalog/Catalog.cs | 10 ++-- .../Catalog/StatelessWorkerGrainContext.cs | 9 ++- .../Orleans.Core.Abstractions.cs | 2 + .../Runtime/DefaultExecutionContextTests.cs | 9 ++- .../GrainActivatorTests.cs | 12 ++-- 9 files changed, 70 insertions(+), 87 deletions(-) diff --git a/src/Orleans.Core.Abstractions/Core/IGrainContext.cs b/src/Orleans.Core.Abstractions/Core/IGrainContext.cs index 3e78f7435f2..888b6cb6c67 100644 --- a/src/Orleans.Core.Abstractions/Core/IGrainContext.cs +++ b/src/Orleans.Core.Abstractions/Core/IGrainContext.cs @@ -69,6 +69,25 @@ public interface IGrainContext : ITargetHolder, IEquatable /// The message. void ReceiveMessage(object message); + /// + /// Starts this context. + /// + /// + /// A token which releases queued work when disposed, or if the context starts eagerly. + /// + /// + /// This method is called at most once. Implementations which throw from this method are responsible for + /// releasing any startup resources which they acquired. + /// + IDisposable? Start() => null; + + /// + /// Aborts this context before startup begins. + /// + void Abort() + { + } + /// /// Start activating this instance. /// diff --git a/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs b/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs index 26a39c15939..6f3e0dde5eb 100644 --- a/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs +++ b/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs @@ -49,7 +49,7 @@ public bool TryGet(GrainType grainType, [NotNullWhen(true)] out IGrainContextAct return true; } - private partial class ActivationDataActivator : IPreparedGrainContextActivator + private partial class ActivationDataActivator : IGrainContextActivator { private readonly IOptions _schedulingOptions; private readonly IGrainActivator _grainActivator; @@ -75,15 +75,6 @@ public ActivationDataActivator( } public IGrainContext CreateContext(GrainAddress activationAddress, IConfigureGrainContext[] configureActions) - { - var preparedContext = CreatePreparedContext(activationAddress, configureActions); - using var startup = preparedContext.Start(); - return preparedContext.Context; - } - - public PreparedGrainContext CreatePreparedContext( - GrainAddress activationAddress, - IConfigureGrainContext[] configureActions) { var context = new ActivationData( activationAddress, @@ -99,11 +90,11 @@ public PreparedGrainContext CreatePreparedContext( configure.Configure(context); } - return new(context, (IGrainContextStartup)context); + return context; } catch { - ((IGrainContextStartup)context).Abort(); + context.Abort(); throw; } } diff --git a/src/Orleans.Runtime/Activation/IGrainContextActivator.cs b/src/Orleans.Runtime/Activation/IGrainContextActivator.cs index 165e96cc09f..5db24219052 100644 --- a/src/Orleans.Runtime/Activation/IGrainContextActivator.cs +++ b/src/Orleans.Runtime/Activation/IGrainContextActivator.cs @@ -14,7 +14,6 @@ using Orleans.GrainReferences; using Orleans.Metadata; using Orleans.Runtime.Placement; -using Orleans.Runtime.Scheduler; using Orleans.Serialization.Invocation; using Orleans.Serialization.Session; @@ -59,12 +58,12 @@ public GrainContextActivator( /// The grain context. public IGrainContext CreateInstance(GrainAddress address) { - var preparedContext = CreatePreparedContext(address); - using var lease = preparedContext.Start(); - return preparedContext.Context; + var context = CreateContext(address); + using var startup = context.Start(); + return context; } - internal PreparedGrainContext CreatePreparedContext(GrainAddress address) + internal IGrainContext CreateContext(GrainAddress address) { var grainId = address.GrainId; if (!_activators.TryGetValue(grainId.Type, out var activator)) @@ -72,9 +71,7 @@ internal PreparedGrainContext CreatePreparedContext(GrainAddress address) activator = this.CreateActivator(grainId.Type); } - return activator.Activator is IPreparedGrainContextActivator preparedActivator - ? preparedActivator.CreatePreparedContext(address, activator.ConfigureActions) - : new(activator.Activator.CreateContext(address, activator.ConfigureActions), startup: null); + return activator.Activator.CreateContext(address, activator.ConfigureActions); } private (IGrainContextActivator, IConfigureGrainContext[]) CreateActivator(GrainType grainType) @@ -142,49 +139,13 @@ public interface IGrainContextActivator /// The actions which must be used to configure the context before grain construction begins. /// The newly created grain context. /// - /// Use to create and start a context. - /// This method only performs activator-specific context creation. Custom activators remain responsible - /// for any custom startup behavior which their context requires. + /// Call on the returned context, or use + /// to create and start it. + /// Custom contexts which start eagerly can use the default implementation. /// public IGrainContext CreateContext(GrainAddress address, IConfigureGrainContext[] configureActions); } - internal interface IPreparedGrainContextActivator : IGrainContextActivator - { - PreparedGrainContext CreatePreparedContext( - GrainAddress address, - IConfigureGrainContext[] configureActions); - } - - internal readonly struct PreparedGrainContext(IGrainContext context, IGrainContextStartup? startup) - { - private readonly IGrainContext? _context = context; - private readonly IGrainContextStartup? _startup = startup; - - public IGrainContext Context - => _context ?? throw new InvalidOperationException("The grain context activation is not initialized."); - - public IDisposable Start() => _startup?.Start() ?? NoopDisposable.Instance; - - public void Abort() => _startup?.Abort(); - - private sealed class NoopDisposable : IDisposable - { - public static NoopDisposable Instance { get; } = new(); - - public void Dispose() - { - } - } - } - - internal interface IGrainContextStartup - { - IDisposable Start(); - - void Abort(); - } - /// /// Provides a instance for the provided grain type. /// diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 4ed9dd31687..60aa38b2221 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -38,7 +38,6 @@ internal sealed partial class ActivationData : IGrainManagementExtension, IGrainCallCancellationExtension, ICallChainReentrantGrainContext, - IGrainContextStartup, IAsyncDisposable, IDisposable { @@ -132,7 +131,7 @@ internal void SetActivationActivity(Activity activity) return _activationActivity?.Context; } - IDisposable IGrainContextStartup.Start() + public IDisposable? Start() { if (Interlocked.Exchange(ref _started, 1) != 0 || _startup is not { } startup) { @@ -161,18 +160,25 @@ IDisposable IGrainContextStartup.Start() catch (Exception exception) { _startup = null; - Deactivate( - new DeactivationReason( - DeactivationReasonCode.ActivationFailed, - exception, - "Error starting grain construction."), - CancellationToken.None); - startup.Dispose(); + try + { + Deactivate( + new DeactivationReason( + DeactivationReasonCode.ActivationFailed, + exception, + "Error starting grain construction."), + CancellationToken.None); + } + finally + { + startup.Dispose(); + } + throw; } } - void IGrainContextStartup.Abort() + public void Abort() { if (Interlocked.Exchange(ref _startup, null) is { } startup) { diff --git a/src/Orleans.Runtime/Catalog/Catalog.cs b/src/Orleans.Runtime/Catalog/Catalog.cs index 916e4bfb77e..f987847a50a 100644 --- a/src/Orleans.Runtime/Catalog/Catalog.cs +++ b/src/Orleans.Runtime/Catalog/Catalog.cs @@ -136,7 +136,6 @@ internal int UnregisterGrainForTesting(GrainId grain) Dictionary? requestContextData, MigrationContext? rehydrationContext) { - PreparedGrainContext preparedContext = default; if (TryGetGrainContext(grainId, out var result)) { rehydrationContext?.Dispose(); @@ -166,8 +165,7 @@ internal int UnregisterGrainForTesting(GrainId grain) MembershipVersion = MembershipVersion.MinValue, }; - preparedContext = this.grainActivator.CreatePreparedContext(address); - result = preparedContext.Context; + result = this.grainActivator.CreateContext(address); activations.RecordNewTarget(result); } } // End lock @@ -178,7 +176,7 @@ internal int UnregisterGrainForTesting(GrainId grain) return UnableToCreateActivation(this, grainId); } - IDisposable activationStartup; + IDisposable? activationStartup; var startAttempted = false; try { @@ -202,7 +200,7 @@ internal int UnregisterGrainForTesting(GrainId grain) } startAttempted = true; - activationStartup = preparedContext.Start(); + activationStartup = result.Start(); } catch (Exception exception) { @@ -225,7 +223,7 @@ internal int UnregisterGrainForTesting(GrainId grain) try { - using var startup = preparedContext.Start(); + using var startup = result.Start(); } catch (Exception cleanupException) { diff --git a/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs b/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs index c4155fc0c24..837940b160e 100644 --- a/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs +++ b/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs @@ -327,9 +327,8 @@ private ActivationData CreateWorker(object? message) { Debug.Assert(!_terminated, "CreateWorker must not be called on a terminated stateless worker context."); var address = GrainAddress.GetAddress(Address.SiloAddress, Address.GrainId, ActivationId.NewId()); - var preparedContext = ((IPreparedGrainContextActivator)_innerActivator).CreatePreparedContext(address, []); - var newWorker = (ActivationData)preparedContext.Context; - IDisposable activationStartup; + var newWorker = (ActivationData)_innerActivator.CreateContext(address, []); + IDisposable? activationStartup; var startAttempted = false; try { @@ -337,13 +336,13 @@ private ActivationData CreateWorker(object? message) newWorker.SetComponent(this); _workers.Add(newWorker); startAttempted = true; - activationStartup = preparedContext.Start(); + activationStartup = newWorker.Start(); } catch { if (!startAttempted) { - preparedContext.Abort(); + newWorker.Abort(); _workers.Remove(newWorker); } diff --git a/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs b/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs index bd4ddd6a280..a0c99caa4ba 100644 --- a/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs +++ b/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs @@ -2560,12 +2560,14 @@ public partial interface IGrainContext : Orleans.Serialization.Invocation.ITarge IWorkItemScheduler Scheduler { get; } void Activate(System.Collections.Generic.Dictionary? requestContext, System.Threading.CancellationToken cancellationToken = default); + void Abort(); void Deactivate(DeactivationReason deactivationReason, System.Threading.CancellationToken cancellationToken = default); void Migrate(System.Collections.Generic.Dictionary? requestContext, System.Threading.CancellationToken cancellationToken = default); void ReceiveMessage(object message); void Rehydrate(IRehydrationContext context); void SetComponent(TComponent? value) where TComponent : class; + System.IDisposable? Start(); } public partial interface IGrainContextAccessor diff --git a/test/Orleans.Core.Tests/Runtime/DefaultExecutionContextTests.cs b/test/Orleans.Core.Tests/Runtime/DefaultExecutionContextTests.cs index 3d9a6944ab4..2d8120bb1e9 100644 --- a/test/Orleans.Core.Tests/Runtime/DefaultExecutionContextTests.cs +++ b/test/Orleans.Core.Tests/Runtime/DefaultExecutionContextTests.cs @@ -6,15 +6,18 @@ namespace UnitTests.Runtime; +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Runtime")] public class DefaultExecutionContextTests { - [Fact, TestCategory("BVT")] + [Fact] public void InstanceDoesNotContainAmbientState() => AssertDoesNotContainAmbientState(DefaultExecutionContext.Instance); - [Fact, TestCategory("BVT")] + [Fact] public void FallbackDoesNotContainAmbientState() => AssertDoesNotContainAmbientState(DefaultExecutionContext.CaptureDefault()); - [Fact, TestCategory("BVT")] + [Fact] public async Task InstanceSupportsConcurrentExecution() { var tasks = new Task[Math.Max(4, Environment.ProcessorCount)]; diff --git a/test/Orleans.Runtime.Tests/GrainActivatorTests.cs b/test/Orleans.Runtime.Tests/GrainActivatorTests.cs index f25730f3159..bf65919b971 100644 --- a/test/Orleans.Runtime.Tests/GrainActivatorTests.cs +++ b/test/Orleans.Runtime.Tests/GrainActivatorTests.cs @@ -112,7 +112,7 @@ public async Task GrainContextIsConfiguredBeforeGrainConstruction() Assert.True(state.WasConfiguredAtConstruction); } - [Fact, TestCategory("BVT")] + [Fact] public async Task ContextCreationStartsActivationSynchronouslyOnActivationSchedulerWithCleanExecutionContext() { var state = ActivationOrderingState.Instance; @@ -151,8 +151,8 @@ public async Task ContextCreationStartsActivationSynchronouslyOnActivationSchedu } } - [Fact, TestCategory("BVT")] - public async Task BuiltInActivatorCreateContextStartsContext() + [Fact] + public async Task BuiltInActivatorCreateContextReturnsUnstartedContext() { var primary = Assert.IsType(fixture.HostedCluster.Primary); var services = primary.ServiceProvider; @@ -173,7 +173,11 @@ public async Task BuiltInActivatorCreateContextStartsContext() var context = Assert.IsType(activator.CreateContext(address, [])); try { - Assert.NotNull(context.GrainInstance); + Assert.Null(context.GrainInstance); + using (context.Start()) + { + Assert.NotNull(context.GrainInstance); + } } finally { From 15f74595183447d2d33e2b311fcf8240131db50a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 20 Aug 2026 23:59:55 -0700 Subject: [PATCH 10/28] style(runtime): remove unused scheduler imports Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: ae2e4673-3220-4e33-9073-c2dd36d0fed9 --- src/Orleans.Runtime/Catalog/Catalog.cs | 1 - src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs | 1 - 2 files changed, 2 deletions(-) diff --git a/src/Orleans.Runtime/Catalog/Catalog.cs b/src/Orleans.Runtime/Catalog/Catalog.cs index f987847a50a..8c6a298e8dc 100644 --- a/src/Orleans.Runtime/Catalog/Catalog.cs +++ b/src/Orleans.Runtime/Catalog/Catalog.cs @@ -3,7 +3,6 @@ using Microsoft.Extensions.Logging; using Orleans.GrainDirectory; using Orleans.Runtime.GrainDirectory; -using Orleans.Runtime.Scheduler; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using Orleans.Diagnostics; diff --git a/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs b/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs index 837940b160e..b6f839a4bdf 100644 --- a/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs +++ b/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs @@ -9,7 +9,6 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Orleans.Runtime.Diagnostics; -using Orleans.Runtime.Scheduler; namespace Orleans.Runtime; From 5e6ae67d06ecade01a8299bfc264c24deaff0c9d Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 23 Aug 2026 23:35:33 -0700 Subject: [PATCH 11/28] refactor(runtime): simplify activation startup protocol --- .../Core/IGrainContext.cs | 19 ------- .../ActivationDataActivatorProvider.cs | 15 ++++-- .../Activation/IGrainContextActivator.cs | 53 +++++++++++++++---- src/Orleans.Runtime/Catalog/ActivationData.cs | 7 +-- src/Orleans.Runtime/Catalog/Catalog.cs | 10 ++-- .../Catalog/StatelessWorkerGrainContext.cs | 9 ++-- .../Orleans.Core.Abstractions.cs | 2 - 7 files changed, 70 insertions(+), 45 deletions(-) diff --git a/src/Orleans.Core.Abstractions/Core/IGrainContext.cs b/src/Orleans.Core.Abstractions/Core/IGrainContext.cs index 888b6cb6c67..3e78f7435f2 100644 --- a/src/Orleans.Core.Abstractions/Core/IGrainContext.cs +++ b/src/Orleans.Core.Abstractions/Core/IGrainContext.cs @@ -69,25 +69,6 @@ public interface IGrainContext : ITargetHolder, IEquatable /// The message. void ReceiveMessage(object message); - /// - /// Starts this context. - /// - /// - /// A token which releases queued work when disposed, or if the context starts eagerly. - /// - /// - /// This method is called at most once. Implementations which throw from this method are responsible for - /// releasing any startup resources which they acquired. - /// - IDisposable? Start() => null; - - /// - /// Aborts this context before startup begins. - /// - void Abort() - { - } - /// /// Start activating this instance. /// diff --git a/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs b/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs index 6f3e0dde5eb..26a39c15939 100644 --- a/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs +++ b/src/Orleans.Runtime/Activation/ActivationDataActivatorProvider.cs @@ -49,7 +49,7 @@ public bool TryGet(GrainType grainType, [NotNullWhen(true)] out IGrainContextAct return true; } - private partial class ActivationDataActivator : IGrainContextActivator + private partial class ActivationDataActivator : IPreparedGrainContextActivator { private readonly IOptions _schedulingOptions; private readonly IGrainActivator _grainActivator; @@ -75,6 +75,15 @@ public ActivationDataActivator( } public IGrainContext CreateContext(GrainAddress activationAddress, IConfigureGrainContext[] configureActions) + { + var preparedContext = CreatePreparedContext(activationAddress, configureActions); + using var startup = preparedContext.Start(); + return preparedContext.Context; + } + + public PreparedGrainContext CreatePreparedContext( + GrainAddress activationAddress, + IConfigureGrainContext[] configureActions) { var context = new ActivationData( activationAddress, @@ -90,11 +99,11 @@ public IGrainContext CreateContext(GrainAddress activationAddress, IConfigureGra configure.Configure(context); } - return context; + return new(context, (IGrainContextStartup)context); } catch { - context.Abort(); + ((IGrainContextStartup)context).Abort(); throw; } } diff --git a/src/Orleans.Runtime/Activation/IGrainContextActivator.cs b/src/Orleans.Runtime/Activation/IGrainContextActivator.cs index 5db24219052..fbdd4ee62d9 100644 --- a/src/Orleans.Runtime/Activation/IGrainContextActivator.cs +++ b/src/Orleans.Runtime/Activation/IGrainContextActivator.cs @@ -58,12 +58,12 @@ public GrainContextActivator( /// The grain context. public IGrainContext CreateInstance(GrainAddress address) { - var context = CreateContext(address); - using var startup = context.Start(); - return context; + var preparedContext = CreatePreparedContext(address); + using var startup = preparedContext.Start(); + return preparedContext.Context; } - internal IGrainContext CreateContext(GrainAddress address) + internal PreparedGrainContext CreatePreparedContext(GrainAddress address) { var grainId = address.GrainId; if (!_activators.TryGetValue(grainId.Type, out var activator)) @@ -71,7 +71,9 @@ internal IGrainContext CreateContext(GrainAddress address) activator = this.CreateActivator(grainId.Type); } - return activator.Activator.CreateContext(address, activator.ConfigureActions); + return activator.Activator is IPreparedGrainContextActivator preparedActivator + ? preparedActivator.CreatePreparedContext(address, activator.ConfigureActions) + : new(activator.Activator.CreateContext(address, activator.ConfigureActions), startup: null); } private (IGrainContextActivator, IConfigureGrainContext[]) CreateActivator(GrainType grainType) @@ -138,14 +140,45 @@ public interface IGrainContextActivator /// The grain address. /// The actions which must be used to configure the context before grain construction begins. /// The newly created grain context. - /// - /// Call on the returned context, or use - /// to create and start it. - /// Custom contexts which start eagerly can use the default implementation. - /// public IGrainContext CreateContext(GrainAddress address, IConfigureGrainContext[] configureActions); } + internal interface IPreparedGrainContextActivator : IGrainContextActivator + { + PreparedGrainContext CreatePreparedContext( + GrainAddress address, + IConfigureGrainContext[] configureActions); + } + + internal readonly struct PreparedGrainContext(IGrainContext context, IGrainContextStartup? startup) + { + private readonly IGrainContext? _context = context; + private readonly IGrainContextStartup? _startup = startup; + + public IGrainContext Context + => _context ?? throw new InvalidOperationException("The grain context activation is not initialized."); + + public IDisposable Start() => _startup?.Start() ?? NoopDisposable.Instance; + + public void Abort() => _startup?.Abort(); + + private sealed class NoopDisposable : IDisposable + { + public static NoopDisposable Instance { get; } = new(); + + public void Dispose() + { + } + } + } + + internal interface IGrainContextStartup + { + IDisposable Start(); + + void Abort(); + } + /// /// Provides a instance for the provided grain type. /// diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 60aa38b2221..e5bb72da3d8 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -38,6 +38,7 @@ internal sealed partial class ActivationData : IGrainManagementExtension, IGrainCallCancellationExtension, ICallChainReentrantGrainContext, + IGrainContextStartup, IAsyncDisposable, IDisposable { @@ -131,7 +132,7 @@ internal void SetActivationActivity(Activity activity) return _activationActivity?.Context; } - public IDisposable? Start() + IDisposable IGrainContextStartup.Start() { if (Interlocked.Exchange(ref _started, 1) != 0 || _startup is not { } startup) { @@ -178,7 +179,7 @@ internal void SetActivationActivity(Activity activity) } } - public void Abort() + void IGrainContextStartup.Abort() { if (Interlocked.Exchange(ref _startup, null) is { } startup) { @@ -659,7 +660,7 @@ internal bool TryStartMigration(Dictionary? requestContext, Canc { lock (this) { - if (State is not (ActivationState.Activating or ActivationState.Valid or ActivationState.Deactivating)) + if (State is not (ActivationState.Valid or ActivationState.Deactivating)) { return false; } diff --git a/src/Orleans.Runtime/Catalog/Catalog.cs b/src/Orleans.Runtime/Catalog/Catalog.cs index 8c6a298e8dc..3f8237dece0 100644 --- a/src/Orleans.Runtime/Catalog/Catalog.cs +++ b/src/Orleans.Runtime/Catalog/Catalog.cs @@ -135,6 +135,7 @@ internal int UnregisterGrainForTesting(GrainId grain) Dictionary? requestContextData, MigrationContext? rehydrationContext) { + PreparedGrainContext preparedContext = default; if (TryGetGrainContext(grainId, out var result)) { rehydrationContext?.Dispose(); @@ -164,7 +165,8 @@ internal int UnregisterGrainForTesting(GrainId grain) MembershipVersion = MembershipVersion.MinValue, }; - result = this.grainActivator.CreateContext(address); + preparedContext = this.grainActivator.CreatePreparedContext(address); + result = preparedContext.Context; activations.RecordNewTarget(result); } } // End lock @@ -175,7 +177,7 @@ internal int UnregisterGrainForTesting(GrainId grain) return UnableToCreateActivation(this, grainId); } - IDisposable? activationStartup; + IDisposable activationStartup; var startAttempted = false; try { @@ -199,7 +201,7 @@ internal int UnregisterGrainForTesting(GrainId grain) } startAttempted = true; - activationStartup = result.Start(); + activationStartup = preparedContext.Start(); } catch (Exception exception) { @@ -222,7 +224,7 @@ internal int UnregisterGrainForTesting(GrainId grain) try { - using var startup = result.Start(); + using var startup = preparedContext.Start(); } catch (Exception cleanupException) { diff --git a/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs b/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs index b6f839a4bdf..97e7598cd0e 100644 --- a/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs +++ b/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs @@ -326,8 +326,9 @@ private ActivationData CreateWorker(object? message) { Debug.Assert(!_terminated, "CreateWorker must not be called on a terminated stateless worker context."); var address = GrainAddress.GetAddress(Address.SiloAddress, Address.GrainId, ActivationId.NewId()); - var newWorker = (ActivationData)_innerActivator.CreateContext(address, []); - IDisposable? activationStartup; + var preparedContext = ((IPreparedGrainContextActivator)_innerActivator).CreatePreparedContext(address, []); + var newWorker = (ActivationData)preparedContext.Context; + IDisposable activationStartup; var startAttempted = false; try { @@ -335,13 +336,13 @@ private ActivationData CreateWorker(object? message) newWorker.SetComponent(this); _workers.Add(newWorker); startAttempted = true; - activationStartup = newWorker.Start(); + activationStartup = preparedContext.Start(); } catch { if (!startAttempted) { - newWorker.Abort(); + preparedContext.Abort(); _workers.Remove(newWorker); } diff --git a/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs b/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs index a0c99caa4ba..bd4ddd6a280 100644 --- a/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs +++ b/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs @@ -2560,14 +2560,12 @@ public partial interface IGrainContext : Orleans.Serialization.Invocation.ITarge IWorkItemScheduler Scheduler { get; } void Activate(System.Collections.Generic.Dictionary? requestContext, System.Threading.CancellationToken cancellationToken = default); - void Abort(); void Deactivate(DeactivationReason deactivationReason, System.Threading.CancellationToken cancellationToken = default); void Migrate(System.Collections.Generic.Dictionary? requestContext, System.Threading.CancellationToken cancellationToken = default); void ReceiveMessage(object message); void Rehydrate(IRehydrationContext context); void SetComponent(TComponent? value) where TComponent : class; - System.IDisposable? Start(); } public partial interface IGrainContextAccessor From 0d06b14a2370d8c3d70073a5fd5b11076e41f1f0 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 23 Aug 2026 23:35:41 -0700 Subject: [PATCH 12/28] test(runtime): cover activation startup lifecycle --- .../ActivationStartupTestGrain.cs | 121 +++ .../ActivationStartupTestHooks.cs | 381 +++++++++ ...atelessWorkerActivationStartupTestGrain.cs | 127 +++ .../Runtime/GrainContextActivatorTests.cs | 88 +++ .../OrleansTaskSchedulerBasicTests.cs | 85 ++- .../ActivationDataMigrationTests.cs | 485 ++++++++++++ .../ActivationStartupTestFixture.cs | 207 +++++ .../ActivationStartupTests.cs | 570 ++++++++++++++ .../StatelessWorkerActivationStartupTests.cs | 721 ++++++++++++++++++ .../GrainActivatorTests.cs | 8 +- 10 files changed, 2778 insertions(+), 15 deletions(-) create mode 100644 test/Grains/TestInternalGrains/ActivationStartupTestGrain.cs create mode 100644 test/Grains/TestInternalGrains/ActivationStartupTestHooks.cs create mode 100644 test/Grains/TestInternalGrains/StatelessWorkerActivationStartupTestGrain.cs create mode 100644 test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTestFixture.cs create mode 100644 test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs create mode 100644 test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/StatelessWorkerActivationStartupTests.cs diff --git a/test/Grains/TestInternalGrains/ActivationStartupTestGrain.cs b/test/Grains/TestInternalGrains/ActivationStartupTestGrain.cs new file mode 100644 index 00000000000..308e016b412 --- /dev/null +++ b/test/Grains/TestInternalGrains/ActivationStartupTestGrain.cs @@ -0,0 +1,121 @@ +using Orleans.Runtime; + +namespace UnitTests.Grains; + +public interface IActivationStartupTestGrain : IGrainWithStringKey +{ + Task Invoke(string payload); +} + +public sealed class ActivationStartupTestGrain : + IActivationStartupTestGrain, + IGrainBase, + ILifecycleParticipant +{ + private readonly ActivationStartupScenario _scenario; + + public ActivationStartupTestGrain(IGrainContext grainContext, ActivationStartupTestHooks hooks) + { + GrainContext = grainContext; + _scenario = hooks.GetRequiredScenario(grainContext.GrainId); + _scenario.ObserveConstructor(grainContext); + } + + public IGrainContext GrainContext { get; } + + public void Participate(IGrainLifecycle lifecycle) + { + lifecycle.Subscribe( + "ActivationStartup-Low", + GrainLifecycleStage.SetupState, + _ => + { + _scenario.Record("LifecycleStartLow", GrainContext); + return Task.CompletedTask; + }, + _ => + { + _scenario.Record("LifecycleStopLow", GrainContext); + return Task.CompletedTask; + }); + lifecycle.Subscribe( + "ActivationStartup-High", + GrainLifecycleStage.Activate, + _ => + { + _scenario.Record("LifecycleStartHigh", GrainContext); + return Task.CompletedTask; + }, + _ => + { + _scenario.Record("LifecycleStopHigh", GrainContext); + return Task.CompletedTask; + }); + } + + public Task OnActivateAsync(CancellationToken cancellationToken) + { + _scenario.ObserveOnActivate(GrainContext); + return _scenario.Completion switch + { + ActivationStartupCompletion.ImmediateSuccess => CompleteActivation(), + ActivationStartupCompletion.ImmediateFailure => FailActivation(), + ActivationStartupCompletion.AsynchronousSuccess => CompleteActivationAfterRelease(cancellationToken), + ActivationStartupCompletion.AsynchronousFailure => FailActivationAfterRelease(cancellationToken), + ActivationStartupCompletion.Cancellation => CompleteActivationAfterRelease(cancellationToken), + _ => throw new ArgumentOutOfRangeException(), + }; + } + + public Task OnDeactivateAsync(DeactivationReason reason, CancellationToken cancellationToken) + { + _scenario.ObserveOnDeactivate(GrainContext); + _scenario.Record("OnDeactivateCompleted", GrainContext); + return Task.CompletedTask; + } + + public Task Invoke(string payload) + { + _scenario.ObserveRequest(GrainContext); + return Task.FromResult($"{payload}:{_scenario.RequestContextValue}"); + } + + private Task CompleteActivation() + { + _scenario.Record("OnActivateCompleted", GrainContext); + return Task.CompletedTask; + } + + private Task FailActivation() + { + _scenario.Record("OnActivateFailed", GrainContext); + throw _scenario.ActivationException; + } + + private async Task CompleteActivationAfterRelease(CancellationToken cancellationToken) + { + using var registration = cancellationToken.Register( + static state => + { + var (scenario, context) = ((ActivationStartupScenario, IGrainContext))state!; + scenario.Record("CancellationObserved", context); + }, + (_scenario, GrainContext)); + await _scenario.ActivationRelease.WaitAsync(cancellationToken); + _scenario.Record("OnActivateCompleted", GrainContext); + } + + private async Task FailActivationAfterRelease(CancellationToken cancellationToken) + { + using var registration = cancellationToken.Register( + static state => + { + var (scenario, context) = ((ActivationStartupScenario, IGrainContext))state!; + scenario.Record("CancellationObserved", context); + }, + (_scenario, GrainContext)); + await _scenario.ActivationRelease.WaitAsync(cancellationToken); + _scenario.Record("OnActivateFailed", GrainContext); + throw _scenario.ActivationException; + } +} diff --git a/test/Grains/TestInternalGrains/ActivationStartupTestHooks.cs b/test/Grains/TestInternalGrains/ActivationStartupTestHooks.cs new file mode 100644 index 00000000000..28a10371f8d --- /dev/null +++ b/test/Grains/TestInternalGrains/ActivationStartupTestHooks.cs @@ -0,0 +1,381 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Runtime; + +namespace UnitTests.Grains; + +public enum ActivationStartupCompletion +{ + ImmediateSuccess, + AsynchronousSuccess, + ImmediateFailure, + AsynchronousFailure, + Cancellation, +} + +public enum ActivationStartupDisposal +{ + Synchronous, + Asynchronous, +} + +public readonly record struct ActivationStartupEvent(string Name, GrainId GrainId, ActivationId ActivationId); + +public sealed class ActivationStartupTestHooks +{ + public const string RequestContextKey = "activation-startup"; + + private readonly AsyncLocal _ambientValue = new(); + private readonly ConcurrentDictionary _scenarios = new(); + + public string? AmbientValue + { + get => _ambientValue.Value; + set => _ambientValue.Value = value; + } + + public ActivationStartupScenario CreateScenario( + GrainId grainId, + ActivationStartupCompletion completion, + ActivationStartupDisposal disposal) + { + var scenario = new ActivationStartupScenario(this, grainId, completion, disposal); + if (!_scenarios.TryAdd(grainId, scenario)) + { + throw new InvalidOperationException($"A startup scenario already exists for grain '{grainId}'."); + } + + return scenario; + } + + public ActivationStartupScenario GetRequiredScenario(GrainId grainId) => + _scenarios.TryGetValue(grainId, out var scenario) + ? scenario + : throw new InvalidOperationException($"No startup scenario exists for grain '{grainId}'."); + + public bool TryGetScenario(GrainId grainId, out ActivationStartupScenario? scenario) => + _scenarios.TryGetValue(grainId, out scenario); + + public void RemoveScenario(GrainId grainId) + { + _scenarios.TryRemove(grainId, out _); + } +} + +public sealed class ActivationStartupScenario +{ + private readonly ActivationStartupTestHooks _hooks; + private readonly TaskCompletionSource _activationRelease = CreateCompletionSource(); + private readonly TaskCompletionSource _disposalRelease = CreateCompletionSource(); + private readonly ConcurrentQueue _events = new(); + private readonly ConcurrentDictionary> _eventWaiters = new(); + private IGrainContext? _context; + private string? _constructorAmbientValue; + private string? _constructorRequestContextValue; + private string? _onActivateAmbientValue; + private string? _onActivateRequestContextValue; + private string? _requestAmbientValue; + private string? _requestContextValue; + private int _constructorCount; + private int _createCount; + private int _disposeCompletedCount; + private int _disposeStartedCount; + private int _onActivateCount; + private int _onDeactivateCount; + private int _requestInvocationCount; + private int _scopeDisposeCount; + + internal ActivationStartupScenario( + ActivationStartupTestHooks hooks, + GrainId grainId, + ActivationStartupCompletion completion, + ActivationStartupDisposal disposal) + { + _hooks = hooks; + GrainId = grainId; + Completion = completion; + Disposal = disposal; + ActivationException = new InvalidOperationException("activate-fault"); + } + + public GrainId GrainId { get; } + + public ActivationStartupCompletion Completion { get; } + + public ActivationStartupDisposal Disposal { get; } + + public InvalidOperationException ActivationException { get; } + + public IReadOnlyList Events => _events.ToArray(); + + public int ConstructorCount => Volatile.Read(ref _constructorCount); + + public int CreateCount => Volatile.Read(ref _createCount); + + public int DisposeCompletedCount => Volatile.Read(ref _disposeCompletedCount); + + public int DisposeStartedCount => Volatile.Read(ref _disposeStartedCount); + + public int OnActivateCount => Volatile.Read(ref _onActivateCount); + + public int OnDeactivateCount => Volatile.Read(ref _onDeactivateCount); + + public int RequestInvocationCount => Volatile.Read(ref _requestInvocationCount); + + public int ScopeDisposeCount => Volatile.Read(ref _scopeDisposeCount); + + public string? ConstructorAmbientValue => Volatile.Read(ref _constructorAmbientValue); + + public string? ConstructorRequestContextValue => Volatile.Read(ref _constructorRequestContextValue); + + public string? OnActivateAmbientValue => Volatile.Read(ref _onActivateAmbientValue); + + public string? OnActivateRequestContextValue => Volatile.Read(ref _onActivateRequestContextValue); + + public string? RequestAmbientValue => Volatile.Read(ref _requestAmbientValue); + + public string? RequestContextValue => Volatile.Read(ref _requestContextValue); + + internal IGrainContext Context => + Volatile.Read(ref _context) + ?? throw new InvalidOperationException($"The startup scenario for grain '{GrainId}' has no activation context."); + + internal Task ActivationRelease => _activationRelease.Task; + + internal Task DisposalRelease => _disposalRelease.Task; + + public Task WaitForEvent(string name) => + _eventWaiters.GetOrAdd(name, static _ => CreateEventCompletionSource()).Task; + + public void ReleaseActivation() => _activationRelease.TrySetResult(); + + public void ReleaseDisposal() => _disposalRelease.TrySetResult(); + + public int GetEventCount(string name) => _events.Count(entry => entry.Name == name); + + public void Record(string name, IGrainContext context) + { + Attach(context); + var entry = new ActivationStartupEvent(name, context.GrainId, context.ActivationId); + _events.Enqueue(entry); + _eventWaiters.GetOrAdd(name, static _ => CreateEventCompletionSource()).TrySetResult(entry); + } + + internal void ObserveCreate(IGrainContext context) + { + Attach(context); + Interlocked.Increment(ref _createCount); + } + + internal void ObserveConstructor(IGrainContext context) + { + Attach(context); + Interlocked.Increment(ref _constructorCount); + Volatile.Write(ref _constructorAmbientValue, _hooks.AmbientValue); + Volatile.Write( + ref _constructorRequestContextValue, + RequestContext.Get(ActivationStartupTestHooks.RequestContextKey) as string); + } + + internal void ObserveOnActivate(IGrainContext context) + { + Interlocked.Increment(ref _onActivateCount); + Volatile.Write(ref _onActivateAmbientValue, _hooks.AmbientValue); + Volatile.Write( + ref _onActivateRequestContextValue, + RequestContext.Get(ActivationStartupTestHooks.RequestContextKey) as string); + Record("OnActivateEntered", context); + } + + internal void ObserveOnDeactivate(IGrainContext context) + { + Interlocked.Increment(ref _onDeactivateCount); + Record("OnDeactivateEntered", context); + } + + internal void ObserveRequest(IGrainContext context) + { + Interlocked.Increment(ref _requestInvocationCount); + Volatile.Write(ref _requestAmbientValue, _hooks.AmbientValue); + Volatile.Write( + ref _requestContextValue, + RequestContext.Get(ActivationStartupTestHooks.RequestContextKey) as string); + Record("RequestInvoked", context); + } + + internal void ObserveDisposeStarted(IGrainContext context) + { + Interlocked.Increment(ref _disposeStartedCount); + Record("ActivatorDisposeStarted", context); + } + + internal void ObserveDisposeCompleted(IGrainContext context) + { + Interlocked.Increment(ref _disposeCompletedCount); + Record("ActivatorDisposeCompleted", context); + } + + internal void ObserveScopeDisposed(IGrainContext context) + { + Interlocked.Increment(ref _scopeDisposeCount); + Record("ScopeDisposed", context); + } + + private void Attach(IGrainContext context) + { + if (!context.GrainId.Equals(GrainId)) + { + throw new InvalidOperationException( + $"Scenario grain '{GrainId}' cannot observe activation '{context.GrainId}'."); + } + + var existing = Interlocked.CompareExchange(ref _context, context, null); + if (existing is not null && !ReferenceEquals(existing, context)) + { + throw new InvalidOperationException($"Scenario grain '{GrainId}' observed multiple activation contexts."); + } + } + + private static TaskCompletionSource CreateCompletionSource() => + new(TaskCreationOptions.RunContinuationsAsynchronously); + + private static TaskCompletionSource CreateEventCompletionSource() => + new(TaskCreationOptions.RunContinuationsAsynchronously); +} + +public sealed class ActivationStartupScopedResource(ActivationStartupTestHooks hooks) : IDisposable +{ + private IGrainContext? _context; + private int _disposed; + + public void Attach(IGrainContext context) => _context = context; + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0 + && _context is { } context + && hooks.TryGetScenario(context.GrainId, out var scenario)) + { + scenario!.ObserveScopeDisposed(context); + } + } +} + +public enum CatalogActivationStartupOutcome +{ + Success, + ConstructorFailure, +} + +public sealed class CatalogActivationStartupTestHooks +{ + private readonly ConcurrentDictionary _gates = new(); + + public CatalogActivationStartupGate CreateGate( + GrainId grainId, + CatalogActivationStartupOutcome outcome) + { + var gate = new CatalogActivationStartupGate(grainId, outcome); + if (!_gates.TryAdd(grainId, gate)) + { + throw new InvalidOperationException($"A catalog startup gate already exists for grain '{grainId}'."); + } + + return gate; + } + + public bool TryGetGate(GrainId grainId, out CatalogActivationStartupGate? gate) => + _gates.TryGetValue(grainId, out gate); + + public void RemoveGate(GrainId grainId) => _gates.TryRemove(grainId, out _); +} + +public sealed class CatalogActivationStartupGate( + GrainId grainId, + CatalogActivationStartupOutcome outcome) +{ + private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(30); + private readonly TaskCompletionSource _entered = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _release = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _entryCount; + + public GrainId GrainId { get; } = grainId; + + public CatalogActivationStartupOutcome Outcome { get; } = outcome; + + public InvalidOperationException ConstructorException { get; } = + new("constructor-fault"); + + public Task Entered => _entered.Task; + + public int EntryCount => Volatile.Read(ref _entryCount); + + public void Release() => _release.TrySetResult(); + + internal void EnterAndWait(IGrainContext context, ActivationStartupScenario scenario) + { + if (!context.GrainId.Equals(GrainId)) + { + throw new InvalidOperationException( + $"Catalog startup gate for grain '{GrainId}' cannot observe activation '{context.GrainId}'."); + } + + if (Interlocked.Increment(ref _entryCount) != 1) + { + throw new InvalidOperationException( + $"Catalog startup gate for grain '{GrainId}' was entered more than once."); + } + + scenario.Record("ConstructorBlocked", context); + _entered.TrySetResult(context); + try + { + _release.Task.WaitAsync(Timeout).GetAwaiter().GetResult(); + } + catch (TimeoutException exception) + { + throw new TimeoutException( + $"Timed out releasing catalog startup for grain '{context.GrainId}', activation '{context.ActivationId}', outcome '{Outcome}'.", + exception); + } + } +} + +public sealed class CatalogActivationStartupTestActivator( + ActivationStartupTestHooks hooks, + CatalogActivationStartupTestHooks catalogHooks) : IGrainActivator +{ + public object CreateInstance(IGrainContext context) + { + var scenario = hooks.GetRequiredScenario(context.GrainId); + scenario.ObserveCreate(context); + context.ActivationServices.GetRequiredService().Attach(context); + var instance = new ActivationStartupTestGrain(context, hooks); + + if (catalogHooks.TryGetGate(context.GrainId, out var gate)) + { + gate!.EnterAndWait(context, scenario); + if (gate.Outcome is CatalogActivationStartupOutcome.ConstructorFailure) + { + scenario.Record("ConstructorFailed", context); + throw gate.ConstructorException; + } + } + + return instance; + } + + public async ValueTask DisposeInstance(IGrainContext context, object instance) + { + var scenario = hooks.GetRequiredScenario(context.GrainId); + scenario.ObserveDisposeStarted(context); + if (scenario.Disposal is ActivationStartupDisposal.Asynchronous) + { + await scenario.DisposalRelease; + } + + scenario.ObserveDisposeCompleted(context); + } +} diff --git a/test/Grains/TestInternalGrains/StatelessWorkerActivationStartupTestGrain.cs b/test/Grains/TestInternalGrains/StatelessWorkerActivationStartupTestGrain.cs new file mode 100644 index 00000000000..22383300bc2 --- /dev/null +++ b/test/Grains/TestInternalGrains/StatelessWorkerActivationStartupTestGrain.cs @@ -0,0 +1,127 @@ +using Orleans.Concurrency; +using Orleans.Runtime; + +namespace UnitTests.Grains; + +public interface IStatelessWorkerActivationStartupTestGrain : IGrainWithStringKey +{ + Task Invoke(string payload); +} + +[StatelessWorker(1)] +public sealed class StatelessWorkerActivationStartupTestGrain : + IStatelessWorkerActivationStartupTestGrain, + IGrainBase, + ILifecycleParticipant +{ + private readonly ActivationStartupScenario _scenario; + + public StatelessWorkerActivationStartupTestGrain( + IGrainContext grainContext, + ActivationStartupTestHooks hooks) + { + GrainContext = grainContext; + _scenario = hooks.GetRequiredScenario(grainContext.GrainId); + _scenario.ObserveConstructor(grainContext); + _scenario.Record("ConstructorEntered", grainContext); + } + + public IGrainContext GrainContext { get; } + + public void Participate(IGrainLifecycle lifecycle) + { + lifecycle.Subscribe( + "StatelessWorkerActivationStartup-Low", + GrainLifecycleStage.SetupState, + _ => + { + _scenario.Record("LifecycleStartLow", GrainContext); + return Task.CompletedTask; + }, + _ => + { + _scenario.Record("LifecycleStopLow", GrainContext); + return Task.CompletedTask; + }); + lifecycle.Subscribe( + "StatelessWorkerActivationStartup-High", + GrainLifecycleStage.Activate, + _ => + { + _scenario.Record("LifecycleStartHigh", GrainContext); + return Task.CompletedTask; + }, + _ => + { + _scenario.Record("LifecycleStopHigh", GrainContext); + return Task.CompletedTask; + }); + } + + public Task OnActivateAsync(CancellationToken cancellationToken) + { + _scenario.ObserveOnActivate(GrainContext); + return _scenario.Completion switch + { + ActivationStartupCompletion.ImmediateSuccess => CompleteActivation(), + ActivationStartupCompletion.ImmediateFailure => FailActivation(), + ActivationStartupCompletion.AsynchronousSuccess => CompleteActivationAfterRelease(cancellationToken), + ActivationStartupCompletion.AsynchronousFailure => FailActivationAfterRelease(cancellationToken), + ActivationStartupCompletion.Cancellation => CompleteActivationAfterRelease(cancellationToken), + _ => throw new ArgumentOutOfRangeException(), + }; + } + + public Task OnDeactivateAsync(DeactivationReason reason, CancellationToken cancellationToken) + { + _scenario.ObserveOnDeactivate(GrainContext); + _scenario.Record("OnDeactivateCompleted", GrainContext); + return Task.CompletedTask; + } + + public Task Invoke(string payload) + { + _scenario.ObserveRequest(GrainContext); + return Task.FromResult( + $"{payload}:{GrainContext.ActivationId}:{_scenario.RequestContextValue}"); + } + + private Task CompleteActivation() + { + _scenario.Record("OnActivateCompleted", GrainContext); + return Task.CompletedTask; + } + + private Task FailActivation() + { + _scenario.Record("OnActivateFailed", GrainContext); + throw _scenario.ActivationException; + } + + private async Task CompleteActivationAfterRelease(CancellationToken cancellationToken) + { + using var registration = cancellationToken.Register( + static state => + { + var (scenario, context) = ((ActivationStartupScenario, IGrainContext))state!; + scenario.Record("CancellationObserved", context); + }, + (_scenario, GrainContext)); + await _scenario.ActivationRelease.WaitAsync(cancellationToken); + _scenario.Record("OnActivateCompleted", GrainContext); + } + + private async Task FailActivationAfterRelease(CancellationToken cancellationToken) + { + using var registration = cancellationToken.Register( + static state => + { + var (scenario, context) = ((ActivationStartupScenario, IGrainContext))state!; + scenario.Record("CancellationObserved", context); + }, + (_scenario, GrainContext)); + await _scenario.ActivationRelease.WaitAsync(cancellationToken); + _scenario.Record("OnActivateFailed", GrainContext); + throw _scenario.ActivationException; + } +} diff --git a/test/Orleans.Core.Tests/Runtime/GrainContextActivatorTests.cs b/test/Orleans.Core.Tests/Runtime/GrainContextActivatorTests.cs index 5ace698222b..f77a7bbda63 100644 --- a/test/Orleans.Core.Tests/Runtime/GrainContextActivatorTests.cs +++ b/test/Orleans.Core.Tests/Runtime/GrainContextActivatorTests.cs @@ -8,6 +8,9 @@ namespace UnitTests.Runtime; +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Runtime")] public class GrainContextActivatorTests { [Fact, TestCategory("BVT")] @@ -26,6 +29,42 @@ [new TestConfigureGrainContextProvider(events)], Assert.Equal(["configure", "activate"], events); } + [Fact] + public void CreatePreparedContext_CustomActivatorRemainsEager() + { + var events = new List(); + var context = Substitute.For(); + var activator = CreateActivator(new TestGrainContextActivator(context, events), events); + var address = new GrainAddress { GrainId = GrainId.Create("test", "grain") }; + + var preparedContext = activator.CreatePreparedContext(address); + using var startup = preparedContext.Start(); + preparedContext.Abort(); + + Assert.Same(context, preparedContext.Context); + Assert.Equal(["configure", "activate"], events); + } + + [Fact] + public void CreateInstance_PreparedContextStartsAndReleasesExactlyOnce() + { + var events = new List(); + var context = Substitute.For(); + var activator = CreateActivator(new TestPreparedGrainContextActivator(context, events), events); + var address = new GrainAddress { GrainId = GrainId.Create("test", "grain") }; + + Assert.Same(context, activator.CreateInstance(address)); + Assert.Equal(["configure", "create", "start", "release"], events); + } + + private static GrainContextActivator CreateActivator( + IGrainContextActivator contextActivator, + List events) => + new( + [new TestGrainContextActivatorProvider(contextActivator)], + [new TestConfigureGrainContextProvider(events)], + new GrainPropertiesResolver(Substitute.For())); + private sealed class TestGrainContextActivatorProvider(IGrainContextActivator activator) : IGrainContextActivatorProvider { public bool TryGet(GrainType grainType, [NotNullWhen(true)] out IGrainContextActivator? result) @@ -65,4 +104,53 @@ public IGrainContext CreateContext(GrainAddress address, IConfigureGrainContext[ return context; } } + + private sealed class TestPreparedGrainContextActivator( + IGrainContext context, + List events) : IPreparedGrainContextActivator + { + public IGrainContext CreateContext(GrainAddress address, IConfigureGrainContext[] configureActions) + { + var preparedContext = CreatePreparedContext(address, configureActions); + using var startup = preparedContext.Start(); + return preparedContext.Context; + } + + public PreparedGrainContext CreatePreparedContext( + GrainAddress address, + IConfigureGrainContext[] configureActions) + { + foreach (var configure in configureActions) + { + configure.Configure(context); + } + + events.Add("create"); + return new(context, new TestGrainContextStartup(events)); + } + } + + private sealed class TestGrainContextStartup(List events) : IGrainContextStartup + { + public IDisposable Start() + { + events.Add("start"); + return new TestStartupLease(events); + } + + public void Abort() => events.Add("abort"); + } + + private sealed class TestStartupLease(List events) : IDisposable + { + private int _disposed; + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + { + events.Add("release"); + } + } + } } diff --git a/test/Orleans.Core.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs b/test/Orleans.Core.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs index 53afa717937..bfe05d0a8b5 100644 --- a/test/Orleans.Core.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs +++ b/test/Orleans.Core.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs @@ -239,33 +239,52 @@ async Task ObserveSignals() [Fact] public async Task Sched_ActivationStartup_DisposeIsIdempotent() { - var queuedTaskRan = false; - var queuedTask = new Task(() => queuedTaskRan = true); + var queuedWorkCount = 0; + var subsequentWorkCount = 0; + var queuedTask = new Task(() => Interlocked.Increment(ref queuedWorkCount)); var startup = _rootContext.WorkItemGroup.BeginActivationStartup(); queuedTask.Start(_rootContext.WorkItemGroup.TaskScheduler); startup.Dispose(); + await queuedTask.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(1, Volatile.Read(ref queuedWorkCount)); + Assert.Equal(0, _rootContext.WorkItemGroup.ExternalWorkItemCount); + startup.Dispose(); + Assert.Equal(1, Volatile.Read(ref queuedWorkCount)); + Assert.Equal(0, _rootContext.WorkItemGroup.ExternalWorkItemCount); - await queuedTask.WaitAsync(TimeSpan.FromSeconds(5)); - Assert.True(queuedTaskRan); + var subsequentTask = new Task(() => Interlocked.Increment(ref subsequentWorkCount)); + subsequentTask.Start(_rootContext.WorkItemGroup.TaskScheduler); + await subsequentTask.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(1, Volatile.Read(ref queuedWorkCount)); + Assert.Equal(1, Volatile.Read(ref subsequentWorkCount)); + Assert.Equal(0, _rootContext.WorkItemGroup.ExternalWorkItemCount); } [Fact] public async Task Sched_ActivationStartup_AbortDiscardsQueuedWorkAndAllowsReuse() { - var discardedTask = new Task(static () => throw new InvalidOperationException("This task must not execute.")); + var discardedWorkCount = 0; + var subsequentWorkCount = 0; + var discardedTask = new Task(() => Interlocked.Increment(ref discardedWorkCount)); var startup = _rootContext.WorkItemGroup.BeginActivationStartup(); discardedTask.Start(_rootContext.WorkItemGroup.TaskScheduler); + Assert.Equal(1, _rootContext.WorkItemGroup.ExternalWorkItemCount); + startup.Abort(); - Assert.False(discardedTask.IsCompleted); + Assert.Equal(0, Volatile.Read(ref discardedWorkCount)); + Assert.Equal(0, _rootContext.WorkItemGroup.ExternalWorkItemCount); - var subsequentTaskRan = false; - var subsequentTask = new Task(() => subsequentTaskRan = true); + var subsequentTask = new Task(() => Interlocked.Increment(ref subsequentWorkCount)); subsequentTask.Start(_rootContext.WorkItemGroup.TaskScheduler); await subsequentTask.WaitAsync(TimeSpan.FromSeconds(5)); - Assert.True(subsequentTaskRan); + + Assert.Equal(0, Volatile.Read(ref discardedWorkCount)); + Assert.Equal(1, Volatile.Read(ref subsequentWorkCount)); + Assert.Equal(0, _rootContext.WorkItemGroup.ExternalWorkItemCount); } [Fact] @@ -595,6 +614,54 @@ internal static ILoggerFactory InitSchedulerLogging() var loggerFactory = TestingUtils.CreateDefaultLoggerFactory(TestingUtils.CreateTraceFileName("Silo", DateTime.UtcNow.ToString("yyyyMMdd_hhmmss")), filters); return loggerFactory; } + + [Fact] + public async Task Sched_ActivationStartup_ConcurrentReleaseExecutesQueuedWorkExactlyOnce() + { + const int ParticipantCount = 8; + var queuedWorkCount = 0; + var subsequentWorkCount = 0; + var queuedWork = new Task(() => Interlocked.Increment(ref queuedWorkCount)); + var startup = _rootContext.WorkItemGroup.BeginActivationStartup(); + queuedWork.Start(_rootContext.WorkItemGroup.TaskScheduler); + + var releaseParticipants = new Task[ParticipantCount]; + var participantReady = new TaskCompletionSource[ParticipantCount]; + var releaseParticipantsBarrier = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + for (var i = 0; i < ParticipantCount; i++) + { + var participant = participantReady[i] = new(TaskCreationOptions.RunContinuationsAsynchronously); + releaseParticipants[i] = Task.Run(async () => + { + participant.SetResult(); + await releaseParticipantsBarrier.Task; + startup.Dispose(); + }); + } + + try + { + await Task.WhenAll(Array.ConvertAll(participantReady, static participant => participant.Task)) + .WaitAsync(TimeSpan.FromSeconds(5)); + Assert.False(queuedWork.IsCompleted); + + releaseParticipantsBarrier.SetResult(); + await Task.WhenAll(releaseParticipants).WaitAsync(TimeSpan.FromSeconds(5)); + await queuedWork.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(1, Volatile.Read(ref queuedWorkCount)); + + var subsequentWork = new Task(() => Interlocked.Increment(ref subsequentWorkCount)); + subsequentWork.Start(_rootContext.WorkItemGroup.TaskScheduler); + await subsequentWork.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(1, Volatile.Read(ref subsequentWorkCount)); + } + finally + { + releaseParticipantsBarrier.TrySetResult(); + startup.Dispose(); + await Task.WhenAll(releaseParticipants).WaitAsync(TimeSpan.FromSeconds(5)); + } + } } } diff --git a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationDataMigrationTests.cs b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationDataMigrationTests.cs index bdc810b4df0..2df8c49139c 100644 --- a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationDataMigrationTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationDataMigrationTests.cs @@ -1,15 +1,21 @@ #nullable enable using Microsoft.Extensions.DependencyInjection; +using Orleans.CodeGeneration; +using Orleans.Metadata; using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.Serialization.Invocation; using Orleans.TestingHost; using TestExtensions; using UnitTests.GrainInterfaces; +using UnitTests.Grains; using Xunit; namespace UnitTests.ActivationsLifeCycleTests; [TestSuite("BVT")] [TestProvider("None")] +[TestArea("Runtime")] [TestCategory("BVT"), TestCategory("Migration")] public class ActivationDataMigrationTests(ActivationDataMigrationTests.Fixture fixture) : IClassFixture { @@ -53,6 +59,485 @@ private async Task GetActivation(CancellationToken cancellationT return Assert.IsType(directory.FindTarget(grainId)); } + [Fact] + public async Task TryStartMigrationReturnsFalseDuringSynchronousActivationStartup() + { + var startupFixture = new SynchronousMigrationFixture(); + await startupFixture.InitializeAsync(); + var (grainId, scenario) = startupFixture.CreateScenario(); + using var lifecycleSubscription = startupFixture.ObserveLifecycle(); + Task? startTask = null; + ActivationData? context = null; + + try + { + startTask = Task.Run(() => startupFixture.StartActivation(grainId)); + var observation = await startupFixture.Probe.Observation.WaitAsync( + TimeSpan.FromSeconds(30), + TestContext.Current.CancellationToken); + context = observation.Context; + + Assert.False(observation.Result); + Assert.Equal(ActivationState.Creating, context.State); + Assert.Same(context, startupFixture.ActivationDirectory.FindTarget(grainId)); + Assert.Equal(1, scenario.CreateCount); + Assert.Equal(1, scenario.ConstructorCount); + Assert.Equal(0, scenario.OnActivateCount); + Assert.Equal(0, scenario.GetEventCount("Deactivating")); + Assert.Equal(0, scenario.GetEventCount("Deactivated")); + Assert.Equal(0, scenario.DisposeStartedCount); + + startupFixture.Probe.Release(); + Assert.Same( + context, + await startTask.WaitAsync( + TimeSpan.FromSeconds(30), + TestContext.Current.CancellationToken)); + await scenario.WaitForEvent("Activated").WaitAsync( + TimeSpan.FromSeconds(30), + TestContext.Current.CancellationToken); + var (message, request) = startupFixture.CreateRequest( + context, + scenario, + payload: "synchronous-migration", + requestContextValue: "synchronous-migration-context"); + context.ReceiveMessage(message); + await scenario.WaitForEvent("Response").WaitAsync( + TimeSpan.FromSeconds(30), + TestContext.Current.CancellationToken); + + Assert.Equal( + "synchronous-migration:synchronous-migration-context", + request.Result); + Assert.Equal(ActivationState.Valid, context.State); + Assert.Equal(1, scenario.GetEventCount("Created")); + Assert.Equal(1, scenario.GetEventCount("Activated")); + Assert.Equal(1, scenario.OnActivateCount); + Assert.Equal(1, scenario.RequestInvocationCount); + Assert.Same(context, startupFixture.ActivationDirectory.FindTarget(grainId)); + Assert.Single(scenario.Events.Select(entry => entry.ActivationId).Distinct()); + Assert.All(scenario.Events, entry => Assert.Equal(grainId, entry.GrainId)); + AssertEventOrder( + scenario, + "SynchronousMigrationAttempted", + "Created", + "LifecycleStartLow", + "LifecycleStartHigh", + "OnActivateEntered", + "OnActivateCompleted", + "Activated", + "RequestInvoked", + "Response"); + } + finally + { + startupFixture.Probe.Release(); + scenario.ReleaseActivation(); + scenario.ReleaseDisposal(); + await AwaitIgnoringFailure(startTask); + await CleanupAsync(startupFixture.ActivationDirectory, context, scenario); + startupFixture.Hooks.RemoveScenario(grainId); + await startupFixture.DisposeAsync(); + } + } + + [Fact] + public async Task TryStartMigrationReturnsFalseDuringAsyncActivationStartupAndSucceedsAfterActivation() + { + var startupFixture = new ActivationStartupTestFixture(); + await startupFixture.InitializeAsync(); + var (grainId, scenario) = startupFixture.CreateScenario( + ActivationStartupCompletion.AsynchronousSuccess, + ActivationStartupDisposal.Synchronous); + using var lifecycleProbe = new MigrationLifecycleProbe(scenario); + ActivationData? context = null; + + try + { + context = startupFixture.StartActivation(grainId); + await scenario.WaitForEvent("OnActivateEntered").WaitAsync( + TimeSpan.FromSeconds(30), + TestContext.Current.CancellationToken); + + Assert.Equal(ActivationState.Activating, context.State); + Assert.False(context.TryStartMigration(requestContext: null)); + Assert.Equal(ActivationState.Activating, context.State); + Assert.Same(context, startupFixture.ActivationDirectory.FindTarget(grainId)); + Assert.DoesNotContain( + lifecycleProbe.Events, + static evt => evt is GrainLifecycleEvents.Deactivating); + Assert.DoesNotContain( + lifecycleProbe.Events, + static evt => evt is GrainLifecycleEvents.Deactivated); + Assert.Equal(0, scenario.DisposeStartedCount); + Assert.Equal(0, scenario.ScopeDisposeCount); + + scenario.ReleaseActivation(); + await scenario.WaitForEvent("Activated").WaitAsync( + TimeSpan.FromSeconds(30), + TestContext.Current.CancellationToken); + Assert.Equal(ActivationState.Valid, context.State); + + var deactivating = scenario.WaitForEvent("Deactivating"); + var deactivated = scenario.WaitForEvent("Deactivated"); + Assert.True(context.TryStartMigration(requestContext: null)); + await deactivating.WaitAsync( + TimeSpan.FromSeconds(30), + TestContext.Current.CancellationToken); + + Assert.Equal(ActivationState.Deactivating, context.State); + var migration = Assert.Single( + lifecycleProbe.Events.OfType()); + Assert.Same(context, migration.GrainContext); + Assert.Equal(DeactivationReasonCode.Migrating, migration.Reason.ReasonCode); + + await deactivated.WaitAsync( + TimeSpan.FromSeconds(30), + TestContext.Current.CancellationToken); + Assert.Equal(ActivationState.Invalid, context.State); + Assert.Null(startupFixture.ActivationDirectory.FindTarget(grainId)); + Assert.Equal(1, scenario.CreateCount); + Assert.Equal(1, scenario.ConstructorCount); + Assert.Equal(1, scenario.OnActivateCount); + Assert.Equal(1, scenario.OnDeactivateCount); + Assert.Equal(1, scenario.GetEventCount("Created")); + Assert.Equal(1, scenario.GetEventCount("Activated")); + Assert.Equal(1, scenario.GetEventCount("Deactivating")); + Assert.Equal(1, scenario.GetEventCount("Deactivated")); + Assert.Equal(1, scenario.DisposeStartedCount); + Assert.Equal(1, scenario.DisposeCompletedCount); + Assert.Equal(1, scenario.ScopeDisposeCount); + Assert.Single(scenario.Events.Select(entry => entry.ActivationId).Distinct()); + Assert.All(scenario.Events, entry => Assert.Equal(grainId, entry.GrainId)); + AssertEventOrder( + scenario, + "Created", + "LifecycleStartLow", + "LifecycleStartHigh", + "OnActivateEntered", + "OnActivateCompleted", + "Activated", + "Deactivating", + "OnDeactivateEntered", + "OnDeactivateCompleted", + "LifecycleStopHigh", + "LifecycleStopLow", + "ActivatorDisposeStarted", + "ActivatorDisposeCompleted", + "ScopeDisposed", + "Deactivated"); + } + finally + { + scenario.ReleaseActivation(); + scenario.ReleaseDisposal(); + await CleanupAsync(startupFixture.ActivationDirectory, context, scenario); + startupFixture.RemoveScenario(grainId); + await startupFixture.DisposeAsync(); + } + } + + private static void AssertEventOrder( + ActivationStartupScenario scenario, + params string[] expected) + { + var expectedNames = expected.ToHashSet(); + Assert.Equal( + expected, + scenario.Events + .Where(entry => expectedNames.Contains(entry.Name)) + .Select(entry => entry.Name)); + Assert.All(expected, name => Assert.Equal(1, scenario.GetEventCount(name))); + } + + private static async Task CleanupAsync( + ActivationDirectory directory, + ActivationData? context, + ActivationStartupScenario scenario) + { + if (context is null || context.State is ActivationState.Invalid) + { + return; + } + + var deactivated = scenario.WaitForEvent("Deactivated"); + context.Deactivate(new( + DeactivationReasonCode.ApplicationRequested, + "Migration startup test cleanup.")); + await deactivated.WaitAsync( + TimeSpan.FromSeconds(30), + TestContext.Current.CancellationToken); + Assert.Null(directory.FindTarget(context.GrainId)); + } + + private static async Task AwaitIgnoringFailure(Task? task) + { + if (task is null) + { + return; + } + + try + { + await task.WaitAsync( + TimeSpan.FromSeconds(30), + TestContext.Current.CancellationToken); + } + catch + { + } + } + + private sealed class MigrationLifecycleProbe : + IObserver, + IDisposable + { + private readonly ActivationStartupScenario _scenario; + private readonly IDisposable _subscription; + private readonly List _events = []; + + public MigrationLifecycleProbe(ActivationStartupScenario scenario) + { + _scenario = scenario; + _subscription = GrainLifecycleEvents.AllEvents.Subscribe(this); + } + + public IReadOnlyList Events + { + get + { + lock (_events) + { + return _events.ToArray(); + } + } + } + + public void OnCompleted() + { + } + + public void OnError(Exception error) + { + } + + public void OnNext(GrainLifecycleEvents.LifecycleEvent value) + { + if (!value.GrainContext.GrainId.Equals(_scenario.GrainId)) + { + return; + } + + lock (_events) + { + _events.Add(value); + } + + var name = value switch + { + GrainLifecycleEvents.Created => "Created", + GrainLifecycleEvents.Activated => "Activated", + GrainLifecycleEvents.Deactivating => "Deactivating", + GrainLifecycleEvents.Deactivated => "Deactivated", + _ => null, + }; + if (name is not null) + { + _scenario.Record(name, value.GrainContext); + } + } + + public void Dispose() => _subscription.Dispose(); + } + + private sealed class SynchronousMigrationFixture : BaseTestClusterFixture + { + protected override void ConfigureTestCluster(TestClusterBuilder builder) + { + builder.Options.InitialSilosCount = 1; + builder.AddSiloBuilderConfigurator(); + } + + private InProcessSiloHandle PrimarySilo => (InProcessSiloHandle)HostedCluster.Primary!; + + private IServiceProvider Services => PrimarySilo.SiloHost.Services; + + public ActivationStartupTestHooks Hooks => + Services.GetRequiredService(); + + public SynchronousMigrationProbe Probe => + Services.GetRequiredService(); + + public ActivationDirectory ActivationDirectory => + Services.GetRequiredService(); + + public (GrainId GrainId, ActivationStartupScenario Scenario) CreateScenario() + { + var grainType = Services.GetRequiredService() + .GetGrainType(typeof(ActivationStartupTestGrain)); + var grainId = GrainId.Create(grainType, Guid.NewGuid().ToString("N")); + return ( + grainId, + Hooks.CreateScenario( + grainId, + ActivationStartupCompletion.ImmediateSuccess, + ActivationStartupDisposal.Synchronous)); + } + + public ActivationData StartActivation(GrainId grainId) => + Assert.IsType( + Services.GetRequiredService() + .GetOrCreateActivation( + grainId, + requestContextData: null, + rehydrationContext: null)); + + public (Message Message, ActivationStartupRequest Request) CreateRequest( + ActivationData context, + ActivationStartupScenario scenario, + string payload, + string requestContextValue) + { + var request = new ActivationStartupRequest(scenario, payload, recordResponse: true); + var message = Services.GetRequiredService() + .CreateMessage(request, InvokeMethodOptions.OneWay); + message.SetInfiniteTimeToLive(); + message.RequestContextData = new() + { + [ActivationStartupTestHooks.RequestContextKey] = requestContextValue, + }; + message.SendingGrain = GrainId.Create( + "migration-startup-sender", + Guid.NewGuid().ToString("N")); + message.SendingSilo = PrimarySilo.SiloAddress; + message.TargetGrain = context.GrainId; + message.TargetSilo = PrimarySilo.SiloAddress; + return (message, request); + } + + public IDisposable ObserveLifecycle() => + GrainLifecycleEvents.AllEvents.Subscribe(new LifecycleObserver(Hooks)); + + private sealed class SiloConfigurator : ISiloConfigurator + { + public void Configure(ISiloBuilder hostBuilder) + { + hostBuilder.Services.AddSingleton(); + hostBuilder.Services.AddSingleton(); + hostBuilder.Services.AddScoped(); + hostBuilder.Services.AddSingleton(); + hostBuilder.Services.AddSingleton< + IConfigureGrainTypeComponents, + ActivationStartupConfigurator>(); + } + } + + private sealed class ActivationStartupConfigurator( + GrainClassMap grainClassMap, + SynchronousMigrationActivator activator) + : IConfigureGrainTypeComponents + { + public void Configure( + GrainType grainType, + GrainProperties properties, + GrainTypeSharedContext shared) + { + if (grainClassMap.TryGetGrainClass(grainType, out var grainClass) + && grainClass == typeof(ActivationStartupTestGrain)) + { + shared.SetComponent(activator); + } + } + } + + private sealed class LifecycleObserver(ActivationStartupTestHooks hooks) + : IObserver + { + public void OnCompleted() + { + } + + public void OnError(Exception error) + { + } + + public void OnNext(GrainLifecycleEvents.LifecycleEvent value) + { + if (!hooks.TryGetScenario(value.GrainContext.GrainId, out var scenario)) + { + return; + } + + var name = value switch + { + GrainLifecycleEvents.Created => "Created", + GrainLifecycleEvents.Activated => "Activated", + GrainLifecycleEvents.Deactivating => "Deactivating", + GrainLifecycleEvents.Deactivated => "Deactivated", + _ => null, + }; + if (name is not null) + { + scenario!.Record(name, value.GrainContext); + } + } + } + } + + private sealed class SynchronousMigrationProbe + { + private readonly TaskCompletionSource<(ActivationData Context, bool Result)> _observation = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _release = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task<(ActivationData Context, bool Result)> Observation => _observation.Task; + + public void ObserveAndWait(ActivationData context, bool result) + { + _observation.TrySetResult((context, result)); + try + { + _release.Task.WaitAsync(TimeSpan.FromSeconds(30)).GetAwaiter().GetResult(); + } + catch (TimeoutException exception) + { + throw new TimeoutException( + $"Timed out releasing synchronous migration probe for grain '{context.GrainId}', activation '{context.ActivationId}', result '{result}'.", + exception); + } + } + + public void Release() => _release.TrySetResult(); + } + + private sealed class SynchronousMigrationActivator( + ActivationStartupTestHooks hooks, + SynchronousMigrationProbe probe) : IGrainActivator + { + public object CreateInstance(IGrainContext context) + { + var scenario = hooks.GetRequiredScenario(context.GrainId); + scenario.ObserveCreate(context); + context.ActivationServices.GetRequiredService() + .Attach(context); + var instance = new ActivationStartupTestGrain(context, hooks); + var activation = Assert.IsType(context); + var result = activation.TryStartMigration(requestContext: null); + scenario.Record("SynchronousMigrationAttempted", context); + probe.ObserveAndWait(activation, result); + return instance; + } + + public ValueTask DisposeInstance(IGrainContext context, object instance) + { + var scenario = hooks.GetRequiredScenario(context.GrainId); + scenario.ObserveDisposeStarted(context); + scenario.ObserveDisposeCompleted(context); + return ValueTask.CompletedTask; + } + } + public class Fixture : BaseTestClusterFixture { protected override void ConfigureTestCluster(TestClusterBuilder builder) diff --git a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTestFixture.cs b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTestFixture.cs new file mode 100644 index 00000000000..9cda1796338 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTestFixture.cs @@ -0,0 +1,207 @@ +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Orleans.CodeGeneration; +using Orleans.Metadata; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.Serialization.Invocation; +using Orleans.TestingHost; +using TestExtensions; +using UnitTests.Grains; +using Xunit; + +namespace UnitTests.ActivationsLifeCycleTests; + +public sealed class ActivationStartupTestFixture : BaseTestClusterFixture +{ + protected override void ConfigureTestCluster(TestClusterBuilder builder) + { + builder.Options.InitialSilosCount = 1; + builder.AddSiloBuilderConfigurator(); + } + + public InProcessSiloHandle PrimarySilo => (InProcessSiloHandle)HostedCluster.Primary!; + + public IServiceProvider Services => PrimarySilo.SiloHost.Services; + + public ActivationStartupTestHooks Hooks => Services.GetRequiredService(); + + internal ActivationDirectory ActivationDirectory => Services.GetRequiredService(); + + public (GrainId GrainId, ActivationStartupScenario Scenario) CreateScenario( + ActivationStartupCompletion completion, + ActivationStartupDisposal disposal) + { + var grainType = Services.GetRequiredService().GetGrainType(typeof(ActivationStartupTestGrain)); + var grainId = GrainId.Create(grainType, Guid.NewGuid().ToString("N")); + return (grainId, Hooks.CreateScenario(grainId, completion, disposal)); + } + + internal ActivationData StartActivation(GrainId grainId, string? requestContextValue = null) + { + Dictionary? requestContext = requestContextValue is null + ? null + : new() { [ActivationStartupTestHooks.RequestContextKey] = requestContextValue }; + return Assert.IsType( + Services.GetRequiredService().GetOrCreateActivation(grainId, requestContext, rehydrationContext: null)); + } + + internal (Message Message, ActivationStartupRequest Request) CreateRequest( + ActivationData context, + ActivationStartupScenario scenario, + string payload, + string requestContextValue, + bool recordResponse, + InvokeMethodOptions invokeMethodOptions = InvokeMethodOptions.OneWay) + { + var request = new ActivationStartupRequest(scenario, payload, recordResponse); + var message = Services.GetRequiredService().CreateMessage(request, invokeMethodOptions); + message.SetInfiniteTimeToLive(); + message.RequestContextData = new() + { + [ActivationStartupTestHooks.RequestContextKey] = requestContextValue, + }; + message.SendingGrain = GrainId.Create("activation-startup-sender", Guid.NewGuid().ToString("N")); + message.SendingSilo = PrimarySilo.SiloAddress; + message.TargetGrain = context.GrainId; + message.TargetSilo = PrimarySilo.SiloAddress; + return (message, request); + } + + public IDisposable ObserveLifecycle() => + GrainLifecycleEvents.AllEvents.Subscribe(new LifecycleObserver(Hooks)); + + public void RemoveScenario(GrainId grainId) => Hooks.RemoveScenario(grainId); + + private sealed class SiloConfigurator : ISiloConfigurator + { + public void Configure(ISiloBuilder hostBuilder) + { + hostBuilder.Services.AddSingleton(); + hostBuilder.Services.AddScoped(); + hostBuilder.Services.AddSingleton(); + } + } + + private sealed class ActivationStartupTestActivator( + GrainClassMap grainClassMap, + ActivationStartupTestHooks hooks) : IGrainActivator, IConfigureGrainTypeComponents + { + public void Configure(GrainType grainType, GrainProperties properties, GrainTypeSharedContext shared) + { + if (grainClassMap.TryGetGrainClass(grainType, out var grainClass) + && grainClass == typeof(ActivationStartupTestGrain)) + { + shared.SetComponent(this); + } + } + + public object CreateInstance(IGrainContext context) + { + var scenario = hooks.GetRequiredScenario(context.GrainId); + scenario.ObserveCreate(context); + context.ActivationServices.GetRequiredService().Attach(context); + return new ActivationStartupTestGrain(context, hooks); + } + + public async ValueTask DisposeInstance(IGrainContext context, object instance) + { + var scenario = hooks.GetRequiredScenario(context.GrainId); + scenario.ObserveDisposeStarted(context); + if (scenario.Disposal is ActivationStartupDisposal.Asynchronous) + { + await scenario.DisposalRelease; + } + + scenario.ObserveDisposeCompleted(context); + } + } + + private sealed class LifecycleObserver(ActivationStartupTestHooks hooks) + : IObserver + { + public void OnCompleted() + { + } + + public void OnError(Exception error) + { + } + + public void OnNext(GrainLifecycleEvents.LifecycleEvent value) + { + if (!hooks.TryGetScenario(value.GrainContext.GrainId, out var scenario)) + { + return; + } + + var name = value switch + { + GrainLifecycleEvents.Created => "Created", + GrainLifecycleEvents.Activated => "Activated", + GrainLifecycleEvents.Deactivating => "Deactivating", + GrainLifecycleEvents.Deactivated => "Deactivated", + _ => null, + }; + if (name is not null) + { + scenario!.Record(name, value.GrainContext); + } + } + } +} + +internal sealed class ActivationStartupRequest( + ActivationStartupScenario scenario, + string payload, + bool recordResponse) : IInvokable +{ + private static readonly MethodInfo Method = + typeof(IActivationStartupTestGrain).GetMethod(nameof(IActivationStartupTestGrain.Invoke))!; + + private IActivationStartupTestGrain? _target; + private string? _result; + + public string? Result => Volatile.Read(ref _result); + + public object? GetTarget() => _target; + + public void SetTarget(ITargetHolder holder) + { + _target = (IActivationStartupTestGrain)holder.GetTarget()!; + } + + public async ValueTask Invoke() + { + var result = await _target!.Invoke(payload); + Volatile.Write(ref _result, result); + if (recordResponse) + { + scenario.Record("Response", ((IGrainBase)_target).GrainContext); + } + + return Response.FromResult(result); + } + + public int GetArgumentCount() => 1; + + public object? GetArgument(int index) => + index == 0 ? payload : throw new ArgumentOutOfRangeException(nameof(index)); + + public void SetArgument(int index, object value) => + throw new NotSupportedException("The activation startup request is immutable."); + + public string GetMethodName() => nameof(IActivationStartupTestGrain.Invoke); + + public string GetInterfaceName() => typeof(IActivationStartupTestGrain).FullName!; + + public string GetActivityName() => $"{GetInterfaceName()}/{GetMethodName()}"; + + public MethodInfo GetMethod() => Method; + + public Type GetInterfaceType() => typeof(IActivationStartupTestGrain); + + public void Dispose() + { + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs new file mode 100644 index 00000000000..5ba5c56be3e --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs @@ -0,0 +1,570 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.CodeGeneration; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.TestingHost.Diagnostics; +using TestExtensions; +using UnitTests.Grains; +using Xunit; + +namespace UnitTests.ActivationsLifeCycleTests; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Runtime")] +public sealed class ActivationStartupTests(ActivationStartupTestFixture fixture) + : IClassFixture +{ + private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(30); + + [Fact] + public async Task AsyncActivation_OrdersLifecycleCallbacksAndCleanup() + { + var (grainId, scenario) = fixture.CreateScenario( + ActivationStartupCompletion.AsynchronousSuccess, + ActivationStartupDisposal.Asynchronous); + using var lifecycleSubscription = fixture.ObserveLifecycle(); + var expected = new[] + { + "Created", + "LifecycleStartLow", + "LifecycleStartHigh", + "OnActivateEntered", + "OnActivateCompleted", + "Activated", + "RequestInvoked", + "Deactivating", + "OnDeactivateEntered", + "OnDeactivateCompleted", + "LifecycleStopHigh", + "LifecycleStopLow", + "ActivatorDisposeStarted", + "ActivatorDisposeCompleted", + "ScopeDisposed", + "Deactivated", + }; + var eventTasks = expected.Select(scenario.WaitForEvent).ToArray(); + ActivationData? context = null; + + try + { + context = fixture.StartActivation(grainId); + await scenario.WaitForEvent("OnActivateEntered").WaitAsync(Timeout, TestContext.Current.CancellationToken); + + scenario.ReleaseActivation(); + await scenario.WaitForEvent("Activated").WaitAsync(Timeout, TestContext.Current.CancellationToken); + + var (message, _) = fixture.CreateRequest( + context, + scenario, + payload: "lifecycle", + requestContextValue: "lifecycle-request", + recordResponse: false); + context.ReceiveMessage(message); + await scenario.WaitForEvent("RequestInvoked").WaitAsync(Timeout, TestContext.Current.CancellationToken); + + context.Deactivate(new(DeactivationReasonCode.ApplicationRequested, "Lifecycle test complete.")); + await scenario.WaitForEvent("ActivatorDisposeStarted").WaitAsync(Timeout, TestContext.Current.CancellationToken); + Assert.Equal(0, scenario.DisposeCompletedCount); + Assert.Equal(0, scenario.ScopeDisposeCount); + + scenario.ReleaseDisposal(); + await scenario.WaitForEvent("Deactivated").WaitAsync(Timeout, TestContext.Current.CancellationToken); + await Task.WhenAll(eventTasks).WaitAsync(Timeout, TestContext.Current.CancellationToken); + + Assert.Equal(expected, scenario.Events.Select(static entry => entry.Name)); + Assert.All(expected, name => Assert.Equal(1, scenario.GetEventCount(name))); + Assert.Equal(1, scenario.ConstructorCount); + Assert.Equal(1, scenario.OnActivateCount); + Assert.Equal(1, scenario.OnDeactivateCount); + Assert.Equal(1, scenario.RequestInvocationCount); + AssertIdentity(scenario, grainId); + Assert.Null(fixture.ActivationDirectory.FindTarget(grainId)); + } + finally + { + await CleanupAsync(context, scenario); + fixture.RemoveScenario(grainId); + } + } + + [Fact] + public async Task RequestAdmittedDuringAsyncActivation_CompletesAfterStartup() + { + var (grainId, scenario) = fixture.CreateScenario( + ActivationStartupCompletion.AsynchronousSuccess, + ActivationStartupDisposal.Synchronous); + using var lifecycleSubscription = fixture.ObserveLifecycle(); + var entered = scenario.WaitForEvent("OnActivateEntered"); + var response = scenario.WaitForEvent("Response"); + ActivationData? context = null; + + try + { + context = fixture.StartActivation(grainId); + await entered.WaitAsync(Timeout, TestContext.Current.CancellationToken); + var (message, request) = fixture.CreateRequest( + context, + scenario, + payload: "success", + requestContextValue: "request-success", + recordResponse: true); + + context.ReceiveMessage(message); + scenario.Record("RequestAdmitted", context); + + Assert.Equal(0, scenario.RequestInvocationCount); + Assert.Null(request.Result); + + scenario.ReleaseActivation(); + await response.WaitAsync(Timeout, TestContext.Current.CancellationToken); + + Assert.Equal("success:request-success", request.Result); + Assert.Equal(1, scenario.RequestInvocationCount); + Assert.Equal("request-success", scenario.RequestContextValue); + AssertEventSubset( + scenario, + "RequestAdmitted", + "OnActivateCompleted", + "Activated", + "RequestInvoked", + "Response"); + AssertIdentity(scenario, grainId); + } + finally + { + await CleanupAsync(context, scenario); + fixture.RemoveScenario(grainId); + } + } + + [Fact] + public async Task RequestAdmittedDuringAsyncActivationFailure_IsRejectedWithoutInvocation() + { + var (grainId, scenario) = fixture.CreateScenario( + ActivationStartupCompletion.AsynchronousFailure, + ActivationStartupDisposal.Synchronous); + using var lifecycleSubscription = fixture.ObserveLifecycle(); + using var collector = new DiagnosticEventCollector(DispatcherEvents.ListenerName); + var entered = scenario.WaitForEvent("OnActivateEntered"); + var failed = scenario.WaitForEvent("OnActivateFailed"); + var deactivated = scenario.WaitForEvent("Deactivated"); + ActivationData? context = null; + + try + { + context = fixture.StartActivation(grainId); + await entered.WaitAsync(Timeout, TestContext.Current.CancellationToken); + var (message, request) = fixture.CreateRequest( + context, + scenario, + payload: "must-not-run", + requestContextValue: "request-failure", + recordResponse: true); + var rejected = collector.WaitForEventAsync( + nameof(DispatcherEvents.Rejected), + diagnosticEvent => diagnosticEvent.Payload is DispatcherEvents.Rejected rejection + && ReferenceEquals(rejection.Message, message), + Timeout, + TestContext.Current.CancellationToken); + + context.ReceiveMessage(message); + scenario.Record("RequestAdmitted", context); + Assert.Equal(0, scenario.RequestInvocationCount); + + scenario.ReleaseActivation(); + await failed.WaitAsync(Timeout, TestContext.Current.CancellationToken); + var rejectionEvent = await rejected; + await deactivated.WaitAsync(Timeout, TestContext.Current.CancellationToken); + + var rejection = Assert.IsType(rejectionEvent.Payload); + var exception = Assert.IsType(rejection.Exception); + Assert.Equal("activate-fault", exception.Message); + Assert.Equal(Message.RejectionTypes.Transient, rejection.RejectionType); + Assert.Contains("Failed to activate grain", rejection.Reason); + Assert.Same(message, rejection.Message); + Assert.Null(request.Result); + Assert.Equal(0, scenario.RequestInvocationCount); + Assert.Equal(0, scenario.GetEventCount("Activated")); + Assert.Equal(1, scenario.GetEventCount("Deactivating")); + Assert.Equal(1, scenario.GetEventCount("Deactivated")); + Assert.Equal(1, scenario.DisposeStartedCount); + Assert.Equal(1, scenario.DisposeCompletedCount); + Assert.Equal(1, scenario.ScopeDisposeCount); + Assert.Null(fixture.ActivationDirectory.FindTarget(grainId)); + AssertIdentity(scenario, grainId); + } + finally + { + await CleanupAsync(context, scenario); + fixture.RemoveScenario(grainId); + } + } + + [Fact] + public async Task CancellationDuringAsyncActivation_AbortsAndCleansUpExactlyOnce() + { + var (grainId, scenario) = fixture.CreateScenario( + ActivationStartupCompletion.Cancellation, + ActivationStartupDisposal.Asynchronous); + using var lifecycleSubscription = fixture.ObserveLifecycle(); + var entered = scenario.WaitForEvent("OnActivateEntered"); + var cancellationObserved = scenario.WaitForEvent("CancellationObserved"); + var disposeStarted = scenario.WaitForEvent("ActivatorDisposeStarted"); + var deactivated = scenario.WaitForEvent("Deactivated"); + ActivationData? context = null; + + try + { + context = fixture.StartActivation(grainId); + await entered.WaitAsync(Timeout, TestContext.Current.CancellationToken); + + context.Deactivate(new(DeactivationReasonCode.RuntimeRequested, "Cancel startup.")); + await cancellationObserved.WaitAsync(Timeout, TestContext.Current.CancellationToken); + await disposeStarted.WaitAsync(Timeout, TestContext.Current.CancellationToken); + Assert.Equal(0, scenario.DisposeCompletedCount); + Assert.Equal(0, scenario.ScopeDisposeCount); + + scenario.ReleaseDisposal(); + await deactivated.WaitAsync(Timeout, TestContext.Current.CancellationToken); + + Assert.Equal(0, scenario.GetEventCount("Activated")); + Assert.Equal(0, scenario.RequestInvocationCount); + Assert.Equal(0, scenario.OnDeactivateCount); + Assert.Equal(1, scenario.GetEventCount("Deactivating")); + Assert.Equal(1, scenario.GetEventCount("Deactivated")); + Assert.Equal(1, scenario.DisposeStartedCount); + Assert.Equal(1, scenario.DisposeCompletedCount); + Assert.Equal(1, scenario.ScopeDisposeCount); + Assert.Null(fixture.ActivationDirectory.FindTarget(grainId)); + AssertIdentity(scenario, grainId); + } + finally + { + await CleanupAsync(context, scenario); + fixture.RemoveScenario(grainId); + } + } + + [Fact] + public async Task SiloShutdownDuringAsyncActivation_CancelsAndCleansUpExactlyOnce() + { + var shutdownFixture = new ActivationStartupTestFixture(); + await shutdownFixture.InitializeAsync(); + var (grainId, scenario) = shutdownFixture.CreateScenario( + ActivationStartupCompletion.Cancellation, + ActivationStartupDisposal.Asynchronous); + var shutdownHooks = shutdownFixture.Hooks; + var activationDirectory = shutdownFixture.ActivationDirectory; + var primarySilo = shutdownFixture.PrimarySilo; + using var lifecycleSubscription = shutdownFixture.ObserveLifecycle(); + var entered = scenario.WaitForEvent("OnActivateEntered"); + var cancellationObserved = scenario.WaitForEvent("CancellationObserved"); + var disposeStarted = scenario.WaitForEvent("ActivatorDisposeStarted"); + var deactivated = scenario.WaitForEvent("Deactivated"); + using var collector = new DiagnosticEventCollector(DispatcherEvents.ListenerName); + Task? stopTask = null; + ActivationData? context = null; + + try + { + context = shutdownFixture.StartActivation(grainId); + await entered.WaitAsync(Timeout, TestContext.Current.CancellationToken); + var (message, request) = shutdownFixture.CreateRequest( + context, + scenario, + payload: "shutdown-pending", + requestContextValue: "shutdown-request", + recordResponse: true, + invokeMethodOptions: InvokeMethodOptions.None); + var rejected = collector.WaitForEventAsync( + nameof(DispatcherEvents.Rejected), + diagnosticEvent => diagnosticEvent.Payload is DispatcherEvents.Rejected rejection + && ReferenceEquals(rejection.Message, message), + Timeout, + TestContext.Current.CancellationToken); + + context.ReceiveMessage(message); + Assert.Equal(0, scenario.RequestInvocationCount); + Assert.Null(request.Result); + + stopTask = shutdownFixture.HostedCluster.StopSiloAsync(primarySilo); + await cancellationObserved.WaitAsync(Timeout, TestContext.Current.CancellationToken); + await disposeStarted.WaitAsync(Timeout, TestContext.Current.CancellationToken); + + scenario.ReleaseDisposal(); + await deactivated.WaitAsync(Timeout, TestContext.Current.CancellationToken); + await stopTask.WaitAsync(Timeout, TestContext.Current.CancellationToken); + var rejectionEvent = await rejected; + var rejection = Assert.IsType(rejectionEvent.Payload); + + Assert.Same(message, rejection.Message); + Assert.Equal(Message.RejectionTypes.Unrecoverable, rejection.RejectionType); + var unavailable = Assert.IsType(rejection.Exception); + Assert.Equal( + $"Silo '{primarySilo.SiloAddress}' is shutting down.", + unavailable.Message); + Assert.Null(rejection.Reason); + Assert.Null(request.Result); + Assert.Equal(0, scenario.GetEventCount("Activated")); + Assert.Equal(0, scenario.RequestInvocationCount); + Assert.Equal(1, scenario.GetEventCount("Deactivating")); + Assert.Equal(1, scenario.GetEventCount("Deactivated")); + Assert.Equal(1, scenario.DisposeStartedCount); + Assert.Equal(1, scenario.DisposeCompletedCount); + Assert.Equal(1, scenario.ScopeDisposeCount); + Assert.Equal(ActivationState.Invalid, context.State); + Assert.Null(activationDirectory.FindTarget(grainId)); + AssertIdentity(scenario, grainId); + } + finally + { + scenario.ReleaseActivation(); + scenario.ReleaseDisposal(); + if (stopTask is not null) + { + await stopTask.WaitAsync(Timeout, TestContext.Current.CancellationToken); + } + + shutdownHooks.RemoveScenario(grainId); + await shutdownFixture.DisposeAsync(); + } + } + + [Fact] + public async Task ActivationStartup_DoesNotFlowExecutionContextOrRequestContext() + { + var (grainIdA, scenarioA) = fixture.CreateScenario( + ActivationStartupCompletion.AsynchronousSuccess, + ActivationStartupDisposal.Synchronous); + var (grainIdB, scenarioB) = fixture.CreateScenario( + ActivationStartupCompletion.AsynchronousSuccess, + ActivationStartupDisposal.Synchronous); + using var lifecycleSubscription = fixture.ObserveLifecycle(); + var enteredA = scenarioA.WaitForEvent("OnActivateEntered"); + var enteredB = scenarioB.WaitForEvent("OnActivateEntered"); + var responseA = scenarioA.WaitForEvent("Response"); + var responseB = scenarioB.WaitForEvent("Response"); + var originalAmbient = fixture.Hooks.AmbientValue; + var originalRequestContext = RequestContext.Get(ActivationStartupTestHooks.RequestContextKey); + ActivationData? contextA = null; + ActivationData? contextB = null; + + try + { + fixture.Hooks.AmbientValue = "caller-ambient"; + RequestContext.Set(ActivationStartupTestHooks.RequestContextKey, "caller-request"); + + contextA = fixture.StartActivation(grainIdA, "activate-a"); + contextB = fixture.StartActivation(grainIdB, "activate-b"); + await Task.WhenAll(enteredA, enteredB).WaitAsync(Timeout, TestContext.Current.CancellationToken); + + var (messageA, requestA) = fixture.CreateRequest( + contextA, + scenarioA, + payload: "a", + requestContextValue: "request-a", + recordResponse: true); + var (messageB, requestB) = fixture.CreateRequest( + contextB, + scenarioB, + payload: "b", + requestContextValue: "request-b", + recordResponse: true); + contextA.ReceiveMessage(messageA); + scenarioA.Record("RequestAdmitted", contextA); + contextB.ReceiveMessage(messageB); + scenarioB.Record("RequestAdmitted", contextB); + Assert.Equal(0, scenarioA.RequestInvocationCount); + Assert.Equal(0, scenarioB.RequestInvocationCount); + + scenarioA.ReleaseActivation(); + scenarioB.ReleaseActivation(); + await Task.WhenAll(responseA, responseB).WaitAsync(Timeout, TestContext.Current.CancellationToken); + + Assert.Null(scenarioA.ConstructorAmbientValue); + Assert.Null(scenarioB.ConstructorAmbientValue); + Assert.Null(scenarioA.ConstructorRequestContextValue); + Assert.Null(scenarioB.ConstructorRequestContextValue); + Assert.Null(scenarioA.OnActivateAmbientValue); + Assert.Null(scenarioB.OnActivateAmbientValue); + Assert.Equal("activate-a", scenarioA.OnActivateRequestContextValue); + Assert.Equal("activate-b", scenarioB.OnActivateRequestContextValue); + Assert.Null(scenarioA.RequestAmbientValue); + Assert.Null(scenarioB.RequestAmbientValue); + Assert.Equal("request-a", scenarioA.RequestContextValue); + Assert.Equal("request-b", scenarioB.RequestContextValue); + Assert.Equal("a:request-a", requestA.Result); + Assert.Equal("b:request-b", requestB.Result); + Assert.Equal(1, scenarioA.ConstructorCount); + Assert.Equal(1, scenarioB.ConstructorCount); + Assert.Equal(1, scenarioA.OnActivateCount); + Assert.Equal(1, scenarioB.OnActivateCount); + Assert.Equal(1, scenarioA.RequestInvocationCount); + Assert.Equal(1, scenarioB.RequestInvocationCount); + Assert.Equal("caller-ambient", fixture.Hooks.AmbientValue); + Assert.Equal("caller-request", RequestContext.Get(ActivationStartupTestHooks.RequestContextKey)); + } + finally + { + fixture.Hooks.AmbientValue = originalAmbient; + if (originalRequestContext is null) + { + RequestContext.Remove(ActivationStartupTestHooks.RequestContextKey); + } + else + { + RequestContext.Set(ActivationStartupTestHooks.RequestContextKey, originalRequestContext); + } + await CleanupAsync(contextA, scenarioA); + await CleanupAsync(contextB, scenarioB); + fixture.RemoveScenario(grainIdA); + fixture.RemoveScenario(grainIdB); + } + } + + [Theory] + [InlineData(ActivationStartupCompletion.ImmediateSuccess, ActivationStartupDisposal.Synchronous)] + [InlineData(ActivationStartupCompletion.AsynchronousSuccess, ActivationStartupDisposal.Asynchronous)] + [InlineData(ActivationStartupCompletion.ImmediateFailure, ActivationStartupDisposal.Synchronous)] + [InlineData(ActivationStartupCompletion.AsynchronousFailure, ActivationStartupDisposal.Asynchronous)] + [InlineData(ActivationStartupCompletion.Cancellation, ActivationStartupDisposal.Asynchronous)] + public async Task ActivationStartup_CleanupOccursExactlyOnce( + ActivationStartupCompletion completion, + ActivationStartupDisposal disposal) + { + var (grainId, scenario) = fixture.CreateScenario(completion, disposal); + using var lifecycleSubscription = fixture.ObserveLifecycle(); + var activated = scenario.WaitForEvent("Activated"); + var entered = scenario.WaitForEvent("OnActivateEntered"); + var failed = scenario.WaitForEvent("OnActivateFailed"); + var cancellationObserved = scenario.WaitForEvent("CancellationObserved"); + var disposeStarted = scenario.WaitForEvent("ActivatorDisposeStarted"); + var disposeCompleted = scenario.WaitForEvent("ActivatorDisposeCompleted"); + var scopeDisposed = scenario.WaitForEvent("ScopeDisposed"); + var deactivated = scenario.WaitForEvent("Deactivated"); + ActivationData? context = null; + + try + { + context = fixture.StartActivation(grainId); + switch (completion) + { + case ActivationStartupCompletion.ImmediateSuccess: + await activated.WaitAsync(Timeout, TestContext.Current.CancellationToken); + context.Deactivate(new(DeactivationReasonCode.ApplicationRequested, "Matrix success complete.")); + break; + case ActivationStartupCompletion.AsynchronousSuccess: + await entered.WaitAsync(Timeout, TestContext.Current.CancellationToken); + scenario.ReleaseActivation(); + await activated.WaitAsync(Timeout, TestContext.Current.CancellationToken); + context.Deactivate(new(DeactivationReasonCode.ApplicationRequested, "Matrix success complete.")); + break; + case ActivationStartupCompletion.ImmediateFailure: + await failed.WaitAsync(Timeout, TestContext.Current.CancellationToken); + break; + case ActivationStartupCompletion.AsynchronousFailure: + await entered.WaitAsync(Timeout, TestContext.Current.CancellationToken); + scenario.ReleaseActivation(); + await failed.WaitAsync(Timeout, TestContext.Current.CancellationToken); + break; + case ActivationStartupCompletion.Cancellation: + await entered.WaitAsync(Timeout, TestContext.Current.CancellationToken); + context.Deactivate(new(DeactivationReasonCode.RuntimeRequested, "Matrix cancellation.")); + await cancellationObserved.WaitAsync(Timeout, TestContext.Current.CancellationToken); + break; + default: + throw new ArgumentOutOfRangeException(nameof(completion)); + } + + await disposeStarted.WaitAsync(Timeout, TestContext.Current.CancellationToken); + if (disposal is ActivationStartupDisposal.Asynchronous) + { + Assert.Equal(0, scenario.DisposeCompletedCount); + Assert.Equal(0, scenario.ScopeDisposeCount); + scenario.ReleaseDisposal(); + } + + await deactivated.WaitAsync(Timeout, TestContext.Current.CancellationToken); + await Task.WhenAll(disposeCompleted, scopeDisposed).WaitAsync(Timeout, TestContext.Current.CancellationToken); + + Assert.Equal(1, scenario.CreateCount); + Assert.Equal(1, scenario.ConstructorCount); + Assert.Equal(1, scenario.OnActivateCount); + Assert.Equal(1, scenario.DisposeStartedCount); + Assert.Equal(1, scenario.DisposeCompletedCount); + Assert.Equal(1, scenario.ScopeDisposeCount); + Assert.Equal( + completion is ActivationStartupCompletion.ImmediateSuccess + or ActivationStartupCompletion.AsynchronousSuccess ? 1 : 0, + scenario.OnDeactivateCount); + Assert.Equal( + completion is ActivationStartupCompletion.ImmediateSuccess + or ActivationStartupCompletion.AsynchronousSuccess ? 1 : 0, + scenario.GetEventCount("Activated")); + Assert.Equal(1, scenario.GetEventCount("Deactivated")); + Assert.Null(fixture.ActivationDirectory.FindTarget(grainId)); + + var events = scenario.Events; + Assert.True(IndexOf(events, "ActivatorDisposeStarted") < IndexOf(events, "ActivatorDisposeCompleted")); + Assert.True(IndexOf(events, "ActivatorDisposeCompleted") < IndexOf(events, "ScopeDisposed")); + Assert.True(IndexOf(events, "ScopeDisposed") < IndexOf(events, "Deactivated")); + Assert.Equal("Deactivated", events[^1].Name); + AssertIdentity(scenario, grainId); + + var eventSnapshot = scenario.Events; + await context.DisposeAsync(); + await context.DisposeAsync(); + Assert.Equal(eventSnapshot, scenario.Events); + Assert.Equal(1, scenario.DisposeCompletedCount); + Assert.Equal(1, scenario.ScopeDisposeCount); + } + finally + { + await CleanupAsync(context, scenario); + fixture.RemoveScenario(grainId); + } + } + + private static async Task CleanupAsync(ActivationData? context, ActivationStartupScenario scenario) + { + scenario.ReleaseActivation(); + scenario.ReleaseDisposal(); + if (context is null) + { + return; + } + + var deactivated = scenario.WaitForEvent("Deactivated"); + context.Deactivate(new(DeactivationReasonCode.ApplicationRequested, "Test cleanup.")); + await deactivated.WaitAsync(Timeout, TestContext.Current.CancellationToken); + } + + private static void AssertEventSubset(ActivationStartupScenario scenario, params string[] expected) + { + var expectedSet = expected.ToHashSet(); + Assert.Equal(expected, scenario.Events.Where(entry => expectedSet.Contains(entry.Name)).Select(static entry => entry.Name)); + Assert.All(expected, name => Assert.Equal(1, scenario.GetEventCount(name))); + } + + private static void AssertIdentity(ActivationStartupScenario scenario, GrainId grainId) + { + var events = scenario.Events; + Assert.NotEmpty(events); + Assert.All(events, entry => Assert.Equal(grainId, entry.GrainId)); + Assert.Single(events.Select(static entry => entry.ActivationId).Distinct()); + } + + private static int IndexOf(IReadOnlyList events, string name) + { + for (var i = 0; i < events.Count; i++) + { + if (events[i].Name == name) + { + return i; + } + } + + return -1; + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/StatelessWorkerActivationStartupTests.cs b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/StatelessWorkerActivationStartupTests.cs new file mode 100644 index 00000000000..f0ce3016406 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/StatelessWorkerActivationStartupTests.cs @@ -0,0 +1,721 @@ +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Orleans.CodeGeneration; +using Orleans.Metadata; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.Serialization.Invocation; +using Orleans.TestingHost; +using Orleans.TestingHost.Diagnostics; +using TestExtensions; +using UnitTests.Grains; +using Xunit; + +namespace UnitTests.ActivationsLifeCycleTests; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Runtime")] +public sealed class StatelessWorkerActivationStartupTests( + StatelessWorkerActivationStartupTests.Fixture fixture) + : IClassFixture +{ + private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(30); + private static readonly FieldInfo WorkersField = + typeof(StatelessWorkerGrainContext).GetField( + "_workers", + BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException("The stateless-worker child list was not found."); + + [Fact] + public async Task StatelessWorkerChildIsPublishedBeforeSynchronousStartAndDoesNotInvokeEarly() + { + var (grainId, scenario) = fixture.CreateScenario( + ActivationStartupCompletion.ImmediateSuccess, + ActivationStartupDisposal.Synchronous); + var gate = fixture.CreateConstructorGate(grainId, CatalogActivationStartupOutcome.Success); + using var eventSubscription = fixture.ObserveEvents(); + using var collector = new DiagnosticEventCollector(StatelessWorkerEvents.ListenerName); + var wrapper = fixture.GetOrCreateContext(grainId); + var workerCreatedTask = WaitForWorkerCreatedAsync(collector, wrapper); + var responseTask = scenario.WaitForEvent("Response"); + ActivationData? worker = null; + Task? admissionTask = null; + + try + { + var (message, request) = fixture.CreateRequest( + wrapper, + scenario, + payload: "synchronous", + requestContextValue: "sync-request"); + + admissionTask = Task.Run(() => wrapper.ReceiveMessage(message)); + worker = Assert.IsType( + await gate.Entered.WaitAsync(Timeout, TestContext.Current.CancellationToken)); + + Assert.Same(worker, Assert.Single(GetWorkers(wrapper))); + Assert.Equal(1, gate.EntryCount); + Assert.Equal(1, scenario.CreateCount); + Assert.Equal(1, scenario.ConstructorCount); + Assert.Equal(0, scenario.OnActivateCount); + Assert.Equal(0, scenario.RequestInvocationCount); + Assert.Null(request.Result); + Assert.Empty(GetWorkerCreatedEvents(collector, wrapper)); + + gate.Release(); + await admissionTask.WaitAsync(Timeout, TestContext.Current.CancellationToken); + var workerCreated = await workerCreatedTask; + await responseTask.WaitAsync(Timeout, TestContext.Current.CancellationToken); + + Assert.Same(wrapper, workerCreated.Context); + Assert.Same(worker, workerCreated.WorkerContext); + Assert.Equal(1, workerCreated.WorkerCount); + Assert.Equal( + $"synchronous:{worker.ActivationId}:sync-request", + request.Result); + Assert.Equal(1, scenario.CreateCount); + Assert.Equal(1, scenario.ConstructorCount); + Assert.Equal(1, scenario.OnActivateCount); + Assert.Equal(1, scenario.RequestInvocationCount); + Assert.Same(worker, Assert.Single(GetWorkers(wrapper))); + AssertScenarioEventOrder( + scenario, + "ConstructorEntered", + "ConstructorBlocked", + "WorkerCreated", + "LifecycleStartLow", + "LifecycleStartHigh", + "OnActivateEntered", + "OnActivateCompleted", + "RequestInvoked", + "Response"); + AssertIdentity(scenario, grainId, worker.ActivationId); + Assert.Empty(GetContextTerminatedEvents(collector, wrapper)); + Assert.Empty(GetMessageForwardedEvents(collector, grainId)); + } + finally + { + gate.Release(); + if (admissionTask is not null) + { + await admissionTask.WaitAsync(Timeout, TestContext.Current.CancellationToken); + } + + await CleanupAsync(wrapper, worker, scenario, collector); + fixture.RemoveScenario(grainId); + } + } + + [Fact] + public async Task StatelessWorkerRequestWaitsForAsyncStartupBeforeInvocation() + { + var (grainId, scenario) = fixture.CreateScenario( + ActivationStartupCompletion.AsynchronousSuccess, + ActivationStartupDisposal.Synchronous); + using var eventSubscription = fixture.ObserveEvents(); + using var collector = new DiagnosticEventCollector(StatelessWorkerEvents.ListenerName); + var wrapper = fixture.GetOrCreateContext(grainId); + var workerCreatedTask = WaitForWorkerCreatedAsync(collector, wrapper); + var activationEnteredTask = scenario.WaitForEvent("OnActivateEntered"); + var responseTask = scenario.WaitForEvent("Response"); + ActivationData? worker = null; + + try + { + var (message, request) = fixture.CreateRequest( + wrapper, + scenario, + payload: "asynchronous", + requestContextValue: "async-request"); + + wrapper.ReceiveMessage(message); + var workerCreated = await workerCreatedTask; + worker = Assert.IsType(workerCreated.WorkerContext); + await activationEnteredTask.WaitAsync(Timeout, TestContext.Current.CancellationToken); + + Assert.Same(worker, Assert.Single(GetWorkers(wrapper))); + Assert.Equal(1, workerCreated.WorkerCount); + Assert.Equal(1, scenario.ConstructorCount); + Assert.Equal(1, scenario.OnActivateCount); + Assert.Equal(0, scenario.RequestInvocationCount); + Assert.Null(request.Result); + + scenario.ReleaseActivation(); + await responseTask.WaitAsync(Timeout, TestContext.Current.CancellationToken); + + Assert.Equal( + $"asynchronous:{worker.ActivationId}:async-request", + request.Result); + Assert.Equal(1, scenario.RequestInvocationCount); + Assert.Equal("async-request", scenario.RequestContextValue); + AssertScenarioEventOrder( + scenario, + "WorkerCreated", + "OnActivateEntered", + "OnActivateCompleted", + "RequestInvoked", + "Response"); + AssertIdentity(scenario, grainId, worker.ActivationId); + Assert.Same(worker, Assert.Single(GetWorkers(wrapper))); + Assert.Single(GetWorkerCreatedEvents(collector, wrapper)); + Assert.Empty(GetContextTerminatedEvents(collector, wrapper)); + Assert.Empty(GetMessageForwardedEvents(collector, grainId)); + } + finally + { + await CleanupAsync(wrapper, worker, scenario, collector); + fixture.RemoveScenario(grainId); + } + } + + [Theory] + [InlineData(StatelessWorkerStartupFailure.Constructor)] + [InlineData(StatelessWorkerStartupFailure.AsynchronousActivation)] + public async Task StatelessWorkerStartupFailureRemovesChildAndDisposesResourcesExactlyOnce( + StatelessWorkerStartupFailure failure) + { + var completion = failure is StatelessWorkerStartupFailure.Constructor + ? ActivationStartupCompletion.ImmediateSuccess + : ActivationStartupCompletion.AsynchronousFailure; + var disposal = failure is StatelessWorkerStartupFailure.Constructor + ? ActivationStartupDisposal.Synchronous + : ActivationStartupDisposal.Asynchronous; + var (grainId, scenario) = fixture.CreateScenario(completion, disposal); + var gate = failure is StatelessWorkerStartupFailure.Constructor + ? fixture.CreateConstructorGate(grainId, CatalogActivationStartupOutcome.ConstructorFailure) + : null; + using var eventSubscription = fixture.ObserveEvents(); + using var statelessEvents = new DiagnosticEventCollector(StatelessWorkerEvents.ListenerName); + using var dispatcherEvents = new DiagnosticEventCollector(DispatcherEvents.ListenerName); + var wrapper = fixture.GetOrCreateContext(grainId); + var workerCreatedTask = WaitForWorkerCreatedAsync(statelessEvents, wrapper); + var contextTerminatedTask = WaitForContextTerminatedAsync(statelessEvents, wrapper); + var deactivatedTask = scenario.WaitForEvent("Deactivated"); + var scopeDisposedTask = scenario.WaitForEvent("ScopeDisposed"); + ActivationData? worker = null; + Task? admissionTask = null; + + try + { + var (message, request) = fixture.CreateRequest( + wrapper, + scenario, + payload: "must-not-run", + requestContextValue: $"failure-{failure}"); + var rejectedTask = WaitForRejectionAsync(dispatcherEvents, message); + + if (gate is not null) + { + admissionTask = Task.Run(() => wrapper.ReceiveMessage(message)); + worker = Assert.IsType( + await gate.Entered.WaitAsync(Timeout, TestContext.Current.CancellationToken)); + Assert.Same(worker, Assert.Single(GetWorkers(wrapper))); + Assert.Equal(0, scenario.RequestInvocationCount); + gate.Release(); + await admissionTask.WaitAsync(Timeout, TestContext.Current.CancellationToken); + } + else + { + wrapper.ReceiveMessage(message); + await scenario.WaitForEvent("OnActivateEntered") + .WaitAsync(Timeout, TestContext.Current.CancellationToken); + worker = Assert.IsType((await workerCreatedTask).WorkerContext); + Assert.Same(worker, Assert.Single(GetWorkers(wrapper))); + Assert.Equal(0, scenario.RequestInvocationCount); + scenario.ReleaseActivation(); + await scenario.WaitForEvent("OnActivateFailed") + .WaitAsync(Timeout, TestContext.Current.CancellationToken); + await scenario.WaitForEvent("ActivatorDisposeStarted") + .WaitAsync(Timeout, TestContext.Current.CancellationToken); + Assert.Equal(0, scenario.DisposeCompletedCount); + Assert.Equal(0, scenario.ScopeDisposeCount); + scenario.ReleaseDisposal(); + } + + var workerCreated = await workerCreatedTask; + worker ??= Assert.IsType(workerCreated.WorkerContext); + var rejectionEvent = await rejectedTask; + var terminated = await contextTerminatedTask; + await Task.WhenAll(deactivatedTask, scopeDisposedTask) + .WaitAsync(Timeout, TestContext.Current.CancellationToken); + + var rejection = Assert.IsType(rejectionEvent.Payload); + var expectedException = gate?.ConstructorException ?? scenario.ActivationException; + var expectedExceptionMessage = failure is StatelessWorkerStartupFailure.Constructor + ? "constructor-fault" + : "activate-fault"; + Assert.Same(expectedException, rejection.Exception); + Assert.Equal(expectedExceptionMessage, rejection.Exception!.Message); + Assert.Equal(Message.RejectionTypes.Transient, rejection.RejectionType); + Assert.Equal( + failure is StatelessWorkerStartupFailure.Constructor + ? "Error constructing grain instance." + : "Failed to activate grain.", + rejection.Reason); + Assert.Same(message, rejection.Message); + Assert.Null(request.Result); + Assert.Equal(0, scenario.RequestInvocationCount); + Assert.Equal(0, scenario.GetEventCount("Activated")); + Assert.Equal(1, scenario.GetEventCount("Deactivated")); + Assert.Equal( + failure is StatelessWorkerStartupFailure.Constructor ? 0 : 1, + scenario.DisposeStartedCount); + Assert.Equal( + failure is StatelessWorkerStartupFailure.Constructor ? 0 : 1, + scenario.DisposeCompletedCount); + Assert.Equal(1, scenario.ScopeDisposeCount); + Assert.Equal(0, terminated.WorkerCount); + Assert.Empty(GetWorkers(wrapper)); + Assert.Single(GetWorkerCreatedEvents(statelessEvents, wrapper)); + Assert.Single(GetContextTerminatedEvents(statelessEvents, wrapper)); + Assert.Empty(GetMessageForwardedEvents(statelessEvents, grainId)); + if (failure is StatelessWorkerStartupFailure.Constructor) + { + AssertScenarioEventOrder( + scenario, + "ConstructorEntered", + "ConstructorBlocked", + "ConstructorFailed", + "Deactivating", + "WorkerCreated", + "ScopeDisposed", + "Deactivated"); + } + else + { + AssertScenarioEventOrder( + scenario, + "ConstructorEntered", + "WorkerCreated", + "OnActivateEntered", + "OnActivateFailed", + "Deactivating", + "ActivatorDisposeStarted", + "ActivatorDisposeCompleted", + "ScopeDisposed", + "Deactivated"); + } + + AssertIdentity(scenario, grainId, worker.ActivationId); + + var eventSnapshot = scenario.Events.ToArray(); + await worker.DisposeAsync(); + await worker.DisposeAsync(); + Assert.Equal(eventSnapshot, scenario.Events); + Assert.Equal(1, scenario.ScopeDisposeCount); + Assert.Equal( + failure is StatelessWorkerStartupFailure.Constructor ? 0 : 1, + scenario.DisposeCompletedCount); + Assert.Single(GetContextTerminatedEvents(statelessEvents, wrapper)); + } + finally + { + gate?.Release(); + if (admissionTask is not null) + { + await admissionTask.WaitAsync(Timeout, TestContext.Current.CancellationToken); + } + + await CleanupAsync(wrapper, worker, scenario, statelessEvents); + fixture.RemoveScenario(grainId); + } + } + + private static IReadOnlyList GetWorkers(StatelessWorkerGrainContext context) => + [.. Assert.IsType>(WorkersField.GetValue(context))]; + + private static IReadOnlyList GetWorkerCreatedEvents( + DiagnosticEventCollector collector, + IGrainContext context) => + [.. collector.Events + .Select(static evt => evt.Payload) + .OfType() + .Where(evt => ReferenceEquals(evt.Context, context))]; + + private static IReadOnlyList GetContextTerminatedEvents( + DiagnosticEventCollector collector, + IGrainContext context) => + [.. collector.Events + .Select(static evt => evt.Payload) + .OfType() + .Where(evt => ReferenceEquals(evt.Context, context))]; + + private static IReadOnlyList GetMessageForwardedEvents( + DiagnosticEventCollector collector, + GrainId grainId) => + [.. collector.Events + .Select(static evt => evt.Payload) + .OfType() + .Where(evt => evt.GrainId.Equals(grainId))]; + + private static async Task WaitForWorkerCreatedAsync( + DiagnosticEventCollector collector, + IGrainContext context) + { + var evt = await collector.WaitForEventAsync( + nameof(StatelessWorkerEvents.WorkerCreated), + diagnosticEvent => diagnosticEvent.Payload is StatelessWorkerEvents.WorkerCreated created + && ReferenceEquals(created.Context, context), + Timeout, + TestContext.Current.CancellationToken); + return Assert.IsType(evt.Payload); + } + + private static async Task WaitForContextTerminatedAsync( + DiagnosticEventCollector collector, + IGrainContext context) + { + var evt = await collector.WaitForEventAsync( + nameof(StatelessWorkerEvents.ContextTerminated), + diagnosticEvent => diagnosticEvent.Payload is StatelessWorkerEvents.ContextTerminated terminated + && ReferenceEquals(terminated.Context, context), + Timeout, + TestContext.Current.CancellationToken); + return Assert.IsType(evt.Payload); + } + + private static Task WaitForRejectionAsync( + DiagnosticEventCollector collector, + Message message) => + collector.WaitForEventAsync( + nameof(DispatcherEvents.Rejected), + diagnosticEvent => diagnosticEvent.Payload is DispatcherEvents.Rejected rejected + && ReferenceEquals(rejected.Message, message), + Timeout, + TestContext.Current.CancellationToken); + + private static async Task CleanupAsync( + StatelessWorkerGrainContext wrapper, + ActivationData? worker, + ActivationStartupScenario scenario, + DiagnosticEventCollector collector) + { + scenario.ReleaseActivation(); + scenario.ReleaseDisposal(); + if (worker is not null && scenario.GetEventCount("Deactivated") == 0) + { + var deactivated = scenario.WaitForEvent("Deactivated"); + worker.Deactivate(new(DeactivationReasonCode.ApplicationRequested, "Test cleanup.")); + await deactivated.WaitAsync(Timeout, TestContext.Current.CancellationToken); + } + + if (worker is not null) + { + await WaitForContextTerminatedAsync(collector, wrapper); + } + + await wrapper.DisposeAsync(); + } + + private static void AssertScenarioEventOrder( + ActivationStartupScenario scenario, + params string[] expected) + { + var expectedSet = expected.ToHashSet(); + Assert.Equal( + expected, + scenario.Events + .Where(entry => expectedSet.Contains(entry.Name)) + .Select(static entry => entry.Name)); + Assert.All(expected, name => Assert.Equal(1, scenario.GetEventCount(name))); + } + + private static void AssertIdentity( + ActivationStartupScenario scenario, + GrainId grainId, + ActivationId activationId) + { + Assert.NotEmpty(scenario.Events); + Assert.All( + scenario.Events, + entry => + { + Assert.Equal(grainId, entry.GrainId); + Assert.Equal(activationId, entry.ActivationId); + }); + } + + public enum StatelessWorkerStartupFailure + { + Constructor, + AsynchronousActivation, + } + + public sealed class Fixture : BaseTestClusterFixture + { + protected override void ConfigureTestCluster(TestClusterBuilder builder) + { + builder.Options.InitialSilosCount = 1; + builder.AddSiloBuilderConfigurator(); + } + + internal IServiceProvider Services => + ((InProcessSiloHandle)HostedCluster.Primary!).SiloHost.Services; + + internal ActivationDirectory ActivationDirectory => + Services.GetRequiredService(); + + private ActivationStartupTestHooks Hooks => + Services.GetRequiredService(); + + private CatalogActivationStartupTestHooks CatalogHooks => + Services.GetRequiredService(); + + internal (GrainId GrainId, ActivationStartupScenario Scenario) CreateScenario( + ActivationStartupCompletion completion, + ActivationStartupDisposal disposal) + { + var grainType = Services.GetRequiredService() + .GetGrainType(typeof(StatelessWorkerActivationStartupTestGrain)); + var grainId = GrainId.Create(grainType, Guid.NewGuid().ToString("N")); + return (grainId, Hooks.CreateScenario(grainId, completion, disposal)); + } + + internal ActivationStartupScenario CreateScenario( + GrainId grainId, + ActivationStartupCompletion completion, + ActivationStartupDisposal disposal) => + Hooks.CreateScenario(grainId, completion, disposal); + + internal CatalogActivationStartupGate CreateConstructorGate( + GrainId grainId, + CatalogActivationStartupOutcome outcome) => + CatalogHooks.CreateGate(grainId, outcome); + + internal StatelessWorkerGrainContext GetOrCreateContext(GrainId grainId) => + Assert.IsType( + Services.GetRequiredService() + .GetOrCreateActivation( + grainId, + requestContextData: null, + rehydrationContext: null)); + + internal (Message Message, StatelessWorkerActivationStartupRequest Request) CreateRequest( + StatelessWorkerGrainContext context, + ActivationStartupScenario scenario, + string payload, + string requestContextValue) + { + var request = new StatelessWorkerActivationStartupRequest(scenario, payload); + var message = Services.GetRequiredService() + .CreateMessage(request, InvokeMethodOptions.OneWay); + message.SetInfiniteTimeToLive(); + message.RequestContextData = new() + { + [ActivationStartupTestHooks.RequestContextKey] = requestContextValue, + }; + message.SendingGrain = GrainId.Create( + "stateless-worker-activation-startup-sender", + Guid.NewGuid().ToString("N")); + message.SendingSilo = ((InProcessSiloHandle)HostedCluster.Primary!).SiloAddress; + message.TargetGrain = context.GrainId; + message.TargetSilo = ((InProcessSiloHandle)HostedCluster.Primary!).SiloAddress; + return (message, request); + } + + internal IDisposable ObserveEvents() => + new CompositeDisposable( + GrainLifecycleEvents.AllEvents.Subscribe(new LifecycleObserver(Hooks)), + StatelessWorkerEvents.AllEvents.Subscribe(new StatelessWorkerObserver(Hooks))); + + internal void RemoveScenario(GrainId grainId) + { + CatalogHooks.RemoveGate(grainId); + Hooks.RemoveScenario(grainId); + } + + private sealed class SiloConfigurator : ISiloConfigurator + { + public void Configure(ISiloBuilder hostBuilder) + { + hostBuilder.Services.AddSingleton(); + hostBuilder.Services.AddSingleton(); + hostBuilder.Services.AddScoped(); + hostBuilder.Services.AddSingleton(); + hostBuilder.Services.AddSingleton< + IConfigureGrainTypeComponents, + StatelessWorkerActivationStartupConfigurator>(); + } + } + + private sealed class StatelessWorkerActivationStartupConfigurator( + GrainClassMap grainClassMap, + StatelessWorkerActivationStartupTestActivator activator) + : IConfigureGrainTypeComponents + { + public void Configure( + GrainType grainType, + GrainProperties properties, + GrainTypeSharedContext shared) + { + if (grainClassMap.TryGetGrainClass(grainType, out var grainClass) + && grainClass == typeof(StatelessWorkerActivationStartupTestGrain)) + { + shared.SetComponent(activator); + } + } + } + + private sealed class StatelessWorkerActivationStartupTestActivator( + ActivationStartupTestHooks hooks, + CatalogActivationStartupTestHooks catalogHooks) : IGrainActivator + { + public object CreateInstance(IGrainContext context) + { + var scenario = hooks.GetRequiredScenario(context.GrainId); + scenario.ObserveCreate(context); + context.ActivationServices + .GetRequiredService() + .Attach(context); + var instance = new StatelessWorkerActivationStartupTestGrain(context, hooks); + + if (catalogHooks.TryGetGate(context.GrainId, out var gate)) + { + gate!.EnterAndWait(context, scenario); + if (gate.Outcome is CatalogActivationStartupOutcome.ConstructorFailure) + { + scenario.Record("ConstructorFailed", context); + throw gate.ConstructorException; + } + } + + return instance; + } + + public async ValueTask DisposeInstance(IGrainContext context, object instance) + { + var scenario = hooks.GetRequiredScenario(context.GrainId); + scenario.ObserveDisposeStarted(context); + if (scenario.Disposal is ActivationStartupDisposal.Asynchronous) + { + await scenario.DisposalRelease; + } + + scenario.ObserveDisposeCompleted(context); + } + } + + private sealed class LifecycleObserver(ActivationStartupTestHooks hooks) + : IObserver + { + public void OnCompleted() + { + } + + public void OnError(Exception error) + { + } + + public void OnNext(GrainLifecycleEvents.LifecycleEvent value) + { + if (!hooks.TryGetScenario(value.GrainContext.GrainId, out var scenario)) + { + return; + } + + var name = value switch + { + GrainLifecycleEvents.Created => "Created", + GrainLifecycleEvents.Activated => "Activated", + GrainLifecycleEvents.Deactivating => "Deactivating", + GrainLifecycleEvents.Deactivated => "Deactivated", + _ => null, + }; + if (name is not null) + { + scenario!.Record(name, value.GrainContext); + } + } + } + + private sealed class StatelessWorkerObserver(ActivationStartupTestHooks hooks) + : IObserver + { + public void OnCompleted() + { + } + + public void OnError(Exception error) + { + } + + public void OnNext(StatelessWorkerEvents.StatelessWorkerEvent value) + { + if (value is StatelessWorkerEvents.WorkerCreated created + && hooks.TryGetScenario(created.GrainId, out var scenario)) + { + scenario!.Record("WorkerCreated", created.WorkerContext); + } + } + } + + private sealed class CompositeDisposable(params IDisposable[] disposables) : IDisposable + { + public void Dispose() + { + foreach (var disposable in disposables) + { + disposable.Dispose(); + } + } + } + } + + internal sealed class StatelessWorkerActivationStartupRequest( + ActivationStartupScenario scenario, + string payload) : IInvokable + { + private static readonly MethodInfo Method = + typeof(IStatelessWorkerActivationStartupTestGrain) + .GetMethod(nameof(IStatelessWorkerActivationStartupTestGrain.Invoke))!; + + private IStatelessWorkerActivationStartupTestGrain? _target; + private readonly TaskCompletionSource _completion = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private string? _result; + + public Task Completion => _completion.Task; + + public string? Result => Volatile.Read(ref _result); + + public object? GetTarget() => _target; + + public void SetTarget(ITargetHolder holder) + { + _target = (IStatelessWorkerActivationStartupTestGrain)holder.GetTarget()!; + } + + public async ValueTask Invoke() + { + var result = await _target!.Invoke(payload); + Volatile.Write(ref _result, result); + scenario.Record("Response", ((IGrainBase)_target).GrainContext); + _completion.TrySetResult(); + return Response.FromResult(result); + } + + public int GetArgumentCount() => 1; + + public object? GetArgument(int index) => + index == 0 ? payload : throw new ArgumentOutOfRangeException(nameof(index)); + + public void SetArgument(int index, object value) => + throw new NotSupportedException("The stateless-worker startup request is immutable."); + + public string GetMethodName() => nameof(IStatelessWorkerActivationStartupTestGrain.Invoke); + + public string GetInterfaceName() => + typeof(IStatelessWorkerActivationStartupTestGrain).FullName!; + + public string GetActivityName() => $"{GetInterfaceName()}/{GetMethodName()}"; + + public MethodInfo GetMethod() => Method; + + public Type GetInterfaceType() => typeof(IStatelessWorkerActivationStartupTestGrain); + + public void Dispose() + { + } + } +} diff --git a/test/Orleans.Runtime.Tests/GrainActivatorTests.cs b/test/Orleans.Runtime.Tests/GrainActivatorTests.cs index bf65919b971..76a5289877d 100644 --- a/test/Orleans.Runtime.Tests/GrainActivatorTests.cs +++ b/test/Orleans.Runtime.Tests/GrainActivatorTests.cs @@ -152,7 +152,7 @@ public async Task ContextCreationStartsActivationSynchronouslyOnActivationSchedu } [Fact] - public async Task BuiltInActivatorCreateContextReturnsUnstartedContext() + public async Task BuiltInActivatorCreateContextStartsContext() { var primary = Assert.IsType(fixture.HostedCluster.Primary); var services = primary.ServiceProvider; @@ -173,11 +173,7 @@ public async Task BuiltInActivatorCreateContextReturnsUnstartedContext() var context = Assert.IsType(activator.CreateContext(address, [])); try { - Assert.Null(context.GrainInstance); - using (context.Start()) - { - Assert.NotNull(context.GrainInstance); - } + Assert.NotNull(context.GrainInstance); } finally { From c0bcd39cddf261ab1b21fcfbc4f903eab47e6e9a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 23 Aug 2026 23:53:24 -0700 Subject: [PATCH 13/28] fix(runtime): balance activation metrics on abort --- src/Orleans.Runtime/Catalog/ActivationData.cs | 6 +- .../GrainActivatorTests.cs | 102 ++++++++++++++++++ 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index e5bb72da3d8..46560795f01 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -973,7 +973,11 @@ public async ValueTask DisposeAsync() try { - _shared.OnDestroyActivation(this); + if (GrainInstance is not null) + { + _shared.OnDestroyActivation(this); + } + GetComponent()?.OnDestroyActivation(this); } catch (ObjectDisposedException) diff --git a/test/Orleans.Runtime.Tests/GrainActivatorTests.cs b/test/Orleans.Runtime.Tests/GrainActivatorTests.cs index 76a5289877d..c46f182c3b3 100644 --- a/test/Orleans.Runtime.Tests/GrainActivatorTests.cs +++ b/test/Orleans.Runtime.Tests/GrainActivatorTests.cs @@ -1,5 +1,7 @@ using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Metrics; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.Metrics.Testing; using Microsoft.Extensions.Logging.Abstractions; using Orleans.Metadata; using Orleans.Runtime; @@ -50,6 +52,7 @@ public void Configure(ISiloBuilder hostBuilder) // This allows it to selectively apply to specific grain types services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); }); } } @@ -182,6 +185,40 @@ public async Task BuiltInActivatorCreateContextStartsContext() } } + [Fact] + public void ConfigurationFailureBeforeConstructionDoesNotDecrementActiveGrainCount() + { + var primary = Assert.IsType(fixture.HostedCluster.Primary); + var services = primary.ServiceProvider; + var grainType = services.GetRequiredService() + .GetGrainType(typeof(ExplicitlyRegisteredSimpleDIGrain)); + var grainId = GrainId.Create(grainType, Guid.NewGuid().ToString()); + var address = GrainAddress.NewActivationAddress(primary.SiloAddress, grainId); + var grainTypeName = services.GetRequiredService() + .GetComponents(grainType) + .GrainTypeName; + using var collector = new MetricCollector( + services.GetRequiredService(), + "Microsoft.Orleans", + InstrumentNames.GRAIN_COUNTS); + + ConfigurationFailureState.Arm(grainId); + try + { + var exception = Assert.Throws( + () => services.GetRequiredService().CreateInstance(address)); + Assert.Equal("configuration-fault", exception.Message); + } + finally + { + ConfigurationFailureState.Clear(); + } + + Assert.DoesNotContain( + collector.GetMeasurementSnapshot(), + measurement => Equals(measurement.Tags["type"], grainTypeName)); + } + /// /// Custom grain activator that bypasses dependency injection entirely. /// Implements both IGrainActivator (for creation/disposal) and IConfigureGrainTypeComponents @@ -249,6 +286,71 @@ public bool TryGetConfigurator( } } + private sealed class ConfigurationFailureConfiguratorProvider(GrainClassMap grainClassMap) : IConfigureGrainContextProvider + { + public bool TryGetConfigurator( + GrainType grainType, + GrainProperties properties, + [NotNullWhen(true)] out IConfigureGrainContext? configurator) + { + if (grainClassMap.TryGetGrainClass(grainType, out var grainClass) + && grainClass == typeof(ExplicitlyRegisteredSimpleDIGrain)) + { + configurator = ConfigurationFailureConfigurator.Instance; + return true; + } + + configurator = null; + return false; + } + } + + private sealed class ConfigurationFailureConfigurator : IConfigureGrainContext + { + public static ConfigurationFailureConfigurator Instance { get; } = new(); + + public void Configure(IGrainContext context) + { + if (ConfigurationFailureState.ShouldFail(context.GrainId)) + { + throw new InvalidOperationException("configuration-fault"); + } + } + } + + private static class ConfigurationFailureState + { + private static readonly object Lock = new(); + private static GrainId _grainId; + private static bool _armed; + + public static void Arm(GrainId grainId) + { + lock (Lock) + { + _grainId = grainId; + _armed = true; + } + } + + public static bool ShouldFail(GrainId grainId) + { + lock (Lock) + { + return _armed && _grainId.Equals(grainId); + } + } + + public static void Clear() + { + lock (Lock) + { + _armed = false; + _grainId = default; + } + } + } + private sealed class ActivationOrderingState : IConfigureGrainContext { private readonly AsyncLocal _transactionState = new(); From 9a38d0fffb428a01113c0712763806c834cb387b Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 24 Aug 2026 08:25:50 -0700 Subject: [PATCH 14/28] fix(runtime): harden activation startup cleanup --- .../Activation/IGrainContextActivator.cs | 30 ++++++++++-- src/Orleans.Runtime/Catalog/Catalog.cs | 2 +- .../Catalog/StatelessWorkerGrainContext.cs | 19 +++++--- .../Runtime/GrainContextActivatorTests.cs | 48 +++++++++++++++++-- 4 files changed, 84 insertions(+), 15 deletions(-) diff --git a/src/Orleans.Runtime/Activation/IGrainContextActivator.cs b/src/Orleans.Runtime/Activation/IGrainContextActivator.cs index fbdd4ee62d9..891c2638e99 100644 --- a/src/Orleans.Runtime/Activation/IGrainContextActivator.cs +++ b/src/Orleans.Runtime/Activation/IGrainContextActivator.cs @@ -71,9 +71,7 @@ internal PreparedGrainContext CreatePreparedContext(GrainAddress address) activator = this.CreateActivator(grainId.Type); } - return activator.Activator is IPreparedGrainContextActivator preparedActivator - ? preparedActivator.CreatePreparedContext(address, activator.ConfigureActions) - : new(activator.Activator.CreateContext(address, activator.ConfigureActions), startup: null); + return PreparedGrainContext.Create(activator.Activator, address, activator.ConfigureActions); } private (IGrainContextActivator, IConfigureGrainContext[]) CreateActivator(GrainType grainType) @@ -158,7 +156,31 @@ internal readonly struct PreparedGrainContext(IGrainContext context, IGrainConte public IGrainContext Context => _context ?? throw new InvalidOperationException("The grain context activation is not initialized."); - public IDisposable Start() => _startup?.Start() ?? NoopDisposable.Instance; + public static PreparedGrainContext Create( + IGrainContextActivator activator, + GrainAddress address, + IConfigureGrainContext[] configureActions) => + activator is IPreparedGrainContextActivator preparedActivator + ? preparedActivator.CreatePreparedContext(address, configureActions) + : new(activator.CreateContext(address, configureActions), startup: null); + + public IDisposable Start() + { + if (_startup is not { } startup) + { + return NoopDisposable.Instance; + } + + try + { + return startup.Start(); + } + catch + { + startup.Abort(); + throw; + } + } public void Abort() => _startup?.Abort(); diff --git a/src/Orleans.Runtime/Catalog/Catalog.cs b/src/Orleans.Runtime/Catalog/Catalog.cs index 3f8237dece0..660bff81f32 100644 --- a/src/Orleans.Runtime/Catalog/Catalog.cs +++ b/src/Orleans.Runtime/Catalog/Catalog.cs @@ -224,7 +224,7 @@ internal int UnregisterGrainForTesting(GrainId grain) try { - using var startup = preparedContext.Start(); + preparedContext.Abort(); } catch (Exception cleanupException) { diff --git a/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs b/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs index 97e7598cd0e..6272d058c2f 100644 --- a/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs +++ b/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs @@ -326,23 +326,29 @@ private ActivationData CreateWorker(object? message) { Debug.Assert(!_terminated, "CreateWorker must not be called on a terminated stateless worker context."); var address = GrainAddress.GetAddress(Address.SiloAddress, Address.GrainId, ActivationId.NewId()); - var preparedContext = ((IPreparedGrainContextActivator)_innerActivator).CreatePreparedContext(address, []); + var preparedContext = PreparedGrainContext.Create(_innerActivator, address, []); var newWorker = (ActivationData)preparedContext.Context; IDisposable activationStartup; - var startAttempted = false; + var startInvoked = false; try { // Observe the create/destroy lifecycle of the activation newWorker.SetComponent(this); _workers.Add(newWorker); - startAttempted = true; + startInvoked = true; activationStartup = preparedContext.Start(); } catch { - if (!startAttempted) + try + { + if (!startInvoked) + { + preparedContext.Abort(); + } + } + finally { - preparedContext.Abort(); _workers.Remove(newWorker); } @@ -355,8 +361,7 @@ private ActivationData CreateWorker(object? message) { // If this is a new worker and there is a message in scope, try to get the request context and activate the worker var requestContext = (message as Message)?.RequestContextData ?? []; - var cancellation = new CancellationTokenSource(_shared.Shared.InternalRuntime.CollectionOptions.Value.ActivationTimeout); - newWorker.Activate(requestContext, cancellation.Token); + newWorker.Activate(requestContext, CancellationToken.None); StatelessWorkerEvents.EmitWorkerCreated(this, newWorker, _workers.Count); return newWorker; diff --git a/test/Orleans.Core.Tests/Runtime/GrainContextActivatorTests.cs b/test/Orleans.Core.Tests/Runtime/GrainContextActivatorTests.cs index f77a7bbda63..51757d70a74 100644 --- a/test/Orleans.Core.Tests/Runtime/GrainContextActivatorTests.cs +++ b/test/Orleans.Core.Tests/Runtime/GrainContextActivatorTests.cs @@ -30,14 +30,15 @@ [new TestConfigureGrainContextProvider(events)], } [Fact] - public void CreatePreparedContext_CustomActivatorRemainsEager() + public void PreparedContextCreate_CustomActivatorRemainsEager() { var events = new List(); var context = Substitute.For(); - var activator = CreateActivator(new TestGrainContextActivator(context, events), events); + var activator = new TestGrainContextActivator(context, events); var address = new GrainAddress { GrainId = GrainId.Create("test", "grain") }; + IConfigureGrainContext[] configureActions = [new TestConfigureGrainContext(events)]; - var preparedContext = activator.CreatePreparedContext(address); + var preparedContext = PreparedGrainContext.Create(activator, address, configureActions); using var startup = preparedContext.Start(); preparedContext.Abort(); @@ -57,6 +58,34 @@ public void CreateInstance_PreparedContextStartsAndReleasesExactlyOnce() Assert.Equal(["configure", "create", "start", "release"], events); } + [Fact] + public void PreparedContext_AbortDoesNotStart() + { + var events = new List(); + var context = Substitute.For(); + var preparedContext = new PreparedGrainContext(context, new TestGrainContextStartup(events)); + + preparedContext.Abort(); + + Assert.Equal(["abort"], events); + } + + [Fact] + public void PreparedContext_StartFailureAbortsExactlyOnce() + { + var events = new List(); + var context = Substitute.For(); + var expected = new InvalidOperationException("start-fault"); + var preparedContext = new PreparedGrainContext( + context, + new ThrowingGrainContextStartup(events, expected)); + + var actual = Assert.Throws(preparedContext.Start); + + Assert.Same(expected, actual); + Assert.Equal(["start", "abort"], events); + } + private static GrainContextActivator CreateActivator( IGrainContextActivator contextActivator, List events) => @@ -153,4 +182,17 @@ public void Dispose() } } } + + private sealed class ThrowingGrainContextStartup( + List events, + Exception exception) : IGrainContextStartup + { + public IDisposable Start() + { + events.Add("start"); + throw exception; + } + + public void Abort() => events.Add("abort"); + } } From 1f68aabccd5bd508343a5411c481e7a43555b040 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 24 Aug 2026 08:56:57 -0700 Subject: [PATCH 15/28] fix(runtime): unpublish aborted activation startup --- src/Orleans.Runtime/Catalog/Catalog.cs | 13 +------ .../ActivationStartupTests.cs | 39 +++++++++++++++++++ 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/src/Orleans.Runtime/Catalog/Catalog.cs b/src/Orleans.Runtime/Catalog/Catalog.cs index 660bff81f32..8a08728957f 100644 --- a/src/Orleans.Runtime/Catalog/Catalog.cs +++ b/src/Orleans.Runtime/Catalog/Catalog.cs @@ -208,18 +208,9 @@ internal int UnregisterGrainForTesting(GrainId grain) List? cleanupExceptions = null; if (!startAttempted) { - try + if (activations.RemoveTarget(result)) { - result.Deactivate( - new DeactivationReason( - DeactivationReasonCode.ActivationFailed, - exception, - "Error preparing grain activation."), - CancellationToken.None); - } - catch (Exception cleanupException) - { - (cleanupExceptions ??= []).Add(cleanupException); + LogTraceUnregisteredActivation(result); } try diff --git a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs index 5ba5c56be3e..01f5f6f9aaf 100644 --- a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs @@ -1,5 +1,7 @@ +using System.Diagnostics; using Microsoft.Extensions.DependencyInjection; using Orleans.CodeGeneration; +using Orleans.Diagnostics; using Orleans.Runtime; using Orleans.Runtime.Diagnostics; using Orleans.TestingHost.Diagnostics; @@ -17,6 +19,43 @@ public sealed class ActivationStartupTests(ActivationStartupTestFixture fixture) { private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(30); + [Fact] + public void FailureBeforeStartAbortsAndUnregistersPreparedContext() + { + var (grainId, scenario) = fixture.CreateScenario( + ActivationStartupCompletion.ImmediateSuccess, + ActivationStartupDisposal.Synchronous); + var failActivityCreation = new AsyncLocal(); + var expected = new InvalidOperationException("activity-start-fault"); + using var listener = new ActivityListener + { + ShouldListenTo = static source => + source.Name == ActivitySources.LifecycleActivitySourceName, + Sample = (ref ActivityCreationOptions _) => + failActivityCreation.Value ? throw expected : ActivitySamplingResult.None, + SampleUsingParentId = (ref ActivityCreationOptions _) => + failActivityCreation.Value ? throw expected : ActivitySamplingResult.None, + }; + ActivitySource.AddActivityListener(listener); + + try + { + failActivityCreation.Value = true; + var actual = Assert.Throws(() => fixture.StartActivation(grainId)); + + Assert.Same(expected, actual); + Assert.Null(fixture.ActivationDirectory.FindTarget(grainId)); + Assert.Equal(0, scenario.CreateCount); + Assert.Equal(0, scenario.ConstructorCount); + Assert.Equal(0, scenario.OnActivateCount); + } + finally + { + failActivityCreation.Value = false; + fixture.RemoveScenario(grainId); + } + } + [Fact] public async Task AsyncActivation_OrdersLifecycleCallbacksAndCleanup() { From 7459ad893c0c956cf619ee26354d173abf4005d9 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 24 Aug 2026 09:40:18 -0700 Subject: [PATCH 16/28] fix(runtime): unpublish failed activation start --- src/Orleans.Runtime/Catalog/Catalog.cs | 18 ++++++++++- .../Runtime/GrainContextActivatorTests.cs | 31 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Runtime/Catalog/Catalog.cs b/src/Orleans.Runtime/Catalog/Catalog.cs index 8a08728957f..c0ee3cd8caa 100644 --- a/src/Orleans.Runtime/Catalog/Catalog.cs +++ b/src/Orleans.Runtime/Catalog/Catalog.cs @@ -201,7 +201,7 @@ internal int UnregisterGrainForTesting(GrainId grain) } startAttempted = true; - activationStartup = preparedContext.Start(); + activationStartup = StartPreparedContext(preparedContext, result, activations); } catch (Exception exception) { @@ -303,6 +303,22 @@ internal int UnregisterGrainForTesting(GrainId grain) } } + internal static IDisposable StartPreparedContext( + PreparedGrainContext preparedContext, + IGrainContext context, + ActivationDirectory activations) + { + try + { + return preparedContext.Start(); + } + catch + { + activations.RemoveTarget(context); + throw; + } + } + private async Task UnregisterNonExistentActivation(GrainAddress address) { try diff --git a/test/Orleans.Core.Tests/Runtime/GrainContextActivatorTests.cs b/test/Orleans.Core.Tests/Runtime/GrainContextActivatorTests.cs index 51757d70a74..569bcb4f211 100644 --- a/test/Orleans.Core.Tests/Runtime/GrainContextActivatorTests.cs +++ b/test/Orleans.Core.Tests/Runtime/GrainContextActivatorTests.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.DependencyInjection; using NSubstitute; using Orleans.Metadata; using Orleans.Runtime; @@ -86,6 +87,36 @@ public void PreparedContext_StartFailureAbortsExactlyOnce() Assert.Equal(["start", "abort"], events); } + [Fact] + public void CatalogStartPreparedContext_StartFailureRemovesRecordedTarget() + { + var events = new List(); + var grainId = GrainId.Create("test", "grain"); + var context = Substitute.For(); + context.GrainId.Returns(grainId); + context.Equals(Arg.Any()) + .Returns(call => ReferenceEquals(context, call.Arg())); + var expected = new InvalidOperationException("start-fault"); + var preparedContext = new PreparedGrainContext( + context, + new ThrowingGrainContextStartup(events, expected)); + using var services = new ServiceCollection() + .AddMetrics() + .AddSingleton() + .AddSingleton() + .BuildServiceProvider(); + var activations = new ActivationDirectory(services.GetRequiredService()); + activations.RecordNewTarget(context); + + var actual = Assert.Throws( + () => Catalog.StartPreparedContext(preparedContext, context, activations)); + + Assert.Same(expected, actual); + Assert.Null(activations.FindTarget(grainId)); + Assert.Equal(0, activations.Count); + Assert.Equal(["start", "abort"], events); + } + private static GrainContextActivator CreateActivator( IGrainContextActivator contextActivator, List events) => From c22f60aeb29133a051609e88db9fbc51fc14e4e3 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 27 Aug 2026 11:58:37 -0700 Subject: [PATCH 17/28] fix(runtime): complete aborted activation startup --- src/Orleans.Runtime/Catalog/ActivationData.cs | 20 ++++---- .../ActivationStartupTests.cs | 49 +++++++++++++++++-- 2 files changed, 53 insertions(+), 16 deletions(-) diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 46560795f01..c2aeb43a748 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -183,17 +183,15 @@ void IGrainContextStartup.Abort() { if (Interlocked.Exchange(ref _startup, null) is { } startup) { - startup.Abort(); - using var suppressExecutionContext = new ExecutionContextSuppressor(); - Task.Factory.StartNew( - static state => DisposeAsync(state!).AsTask(), - _serviceScope, - CancellationToken.None, - TaskCreationOptions.DenyChildAttach, - TaskScheduler.Default) - .Unwrap() - .GetAwaiter() - .GetResult(); + var exception = new InvalidOperationException("Activation startup was aborted."); + Deactivate( + new DeactivationReason( + DeactivationReasonCode.ActivationFailed, + exception, + "Activation startup was aborted."), + CancellationToken.None); + startup.Dispose(); + Deactivated.GetAwaiter().GetResult(); } } diff --git a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs index 01f5f6f9aaf..daa3002d64b 100644 --- a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs @@ -20,21 +20,23 @@ public sealed class ActivationStartupTests(ActivationStartupTestFixture fixture) private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(30); [Fact] - public void FailureBeforeStartAbortsAndUnregistersPreparedContext() + public async Task FailureBeforeStartAbortsRejectsRequestAndUnregistersPreparedContext() { var (grainId, scenario) = fixture.CreateScenario( ActivationStartupCompletion.ImmediateSuccess, ActivationStartupDisposal.Synchronous); var failActivityCreation = new AsyncLocal(); var expected = new InvalidOperationException("activity-start-fault"); + using var collector = new DiagnosticEventCollector(DispatcherEvents.ListenerName); + ActivationData? context = null; + ActivationStartupRequest? request = null; + Task? rejected = null; using var listener = new ActivityListener { ShouldListenTo = static source => source.Name == ActivitySources.LifecycleActivitySourceName, - Sample = (ref ActivityCreationOptions _) => - failActivityCreation.Value ? throw expected : ActivitySamplingResult.None, - SampleUsingParentId = (ref ActivityCreationOptions _) => - failActivityCreation.Value ? throw expected : ActivitySamplingResult.None, + Sample = (ref ActivityCreationOptions _) => OnSample(), + SampleUsingParentId = (ref ActivityCreationOptions _) => OnSample(), }; ActivitySource.AddActivityListener(listener); @@ -44,16 +46,53 @@ public void FailureBeforeStartAbortsAndUnregistersPreparedContext() var actual = Assert.Throws(() => fixture.StartActivation(grainId)); Assert.Same(expected, actual); + Assert.NotNull(context); + await context.Deactivated.WaitAsync(Timeout, TestContext.Current.CancellationToken); + Assert.NotNull(rejected); + var rejectionEvent = await rejected; + var rejection = Assert.IsType(rejectionEvent.Payload); + Assert.Same(request, rejection.Message.BodyObject); + Assert.Equal(Message.RejectionTypes.Transient, rejection.RejectionType); + Assert.Equal( + "Activation startup was aborted.", + Assert.IsType(rejection.Exception).Message); Assert.Null(fixture.ActivationDirectory.FindTarget(grainId)); Assert.Equal(0, scenario.CreateCount); Assert.Equal(0, scenario.ConstructorCount); Assert.Equal(0, scenario.OnActivateCount); + Assert.Equal(ActivationState.Invalid, context.State); } finally { failActivityCreation.Value = false; fixture.RemoveScenario(grainId); } + + ActivitySamplingResult OnSample() + { + if (!failActivityCreation.Value) + { + return ActivitySamplingResult.None; + } + + context = Assert.IsType(fixture.ActivationDirectory.FindTarget(grainId)); + var requestData = fixture.CreateRequest( + context, + scenario, + payload: "must-not-run", + requestContextValue: "pre-start", + recordResponse: true); + request = requestData.Request; + rejected = collector.WaitForEventAsync( + nameof(DispatcherEvents.Rejected), + diagnosticEvent => diagnosticEvent.Payload is DispatcherEvents.Rejected rejection + && ReferenceEquals(rejection.Message, requestData.Message), + Timeout, + TestContext.Current.CancellationToken); + context.ReceiveMessage(requestData.Message); + failActivityCreation.Value = false; + throw expected; + } } [Fact] From dd75566257bc73fa2380608defe1f69bdb95dab6 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 27 Aug 2026 12:17:21 -0700 Subject: [PATCH 18/28] fix(runtime): terminate aborted activation startup --- src/Orleans.Runtime/Catalog/ActivationData.cs | 53 +++++++++++++++---- .../ActivationStartupTests.cs | 4 +- 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index c2aeb43a748..db73d6915a1 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -55,6 +55,7 @@ internal sealed partial class ActivationData : private Queue? _pendingOperations; private Message? _blockingRequest; private bool _isInWorkingSet = true; + private bool _wasActivated; private CoarseStopwatch _busyDuration; private CoarseStopwatch _idleDuration; private GrainReference? _selfReference; @@ -183,15 +184,42 @@ void IGrainContextStartup.Abort() { if (Interlocked.Exchange(ref _startup, null) is { } startup) { - var exception = new InvalidOperationException("Activation startup was aborted."); - Deactivate( - new DeactivationReason( - DeactivationReasonCode.ActivationFailed, - exception, - "Activation startup was aborted."), - CancellationToken.None); - startup.Dispose(); - Deactivated.GetAwaiter().GetResult(); + startup.Abort(); + using var suppressExecutionContext = new ExecutionContextSuppressor(); + Task.Factory.StartNew( + static state => ((ActivationData)state!).AbortStartupAsync().AsTask(), + this, + CancellationToken.None, + TaskCreationOptions.DenyChildAttach, + TaskScheduler.Default) + .Unwrap() + .GetAwaiter() + .GetResult(); + } + } + + private async ValueTask AbortStartupAsync() + { + var exception = new InvalidOperationException("Activation startup was aborted."); + DeactivationReason = new( + DeactivationReasonCode.ActivationFailed, + exception, + exception.Message); + + lock (this) + { + SetState(ActivationState.Invalid); + } + + try + { + RejectAllQueuedMessages(); + } + finally + { + await DisposeAsync(); + GetDeactivationCompletionSource().TrySetResult(true); + _workSignal.Signal(); } } @@ -951,7 +979,11 @@ public async ValueTask DisposeAsync() lock (this) { - _shared.InternalRuntime.ActivationWorkingSet.OnDeactivated(this); + if (_wasActivated) + { + _shared.InternalRuntime.ActivationWorkingSet.OnDeactivated(this); + } + SetState(ActivationState.Invalid); } @@ -1937,6 +1969,7 @@ private async Task ActivateAsync(Dictionary? requestContextData, { SetState(ActivationState.Valid); _shared.InternalRuntime.ActivationWorkingSet.OnActivated(this); + _wasActivated = true; } } _activationActivity?.AddEvent(new ActivityEvent("state-valid")); diff --git a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs index daa3002d64b..faba8fd5283 100644 --- a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs @@ -28,6 +28,8 @@ public async Task FailureBeforeStartAbortsRejectsRequestAndUnregistersPreparedCo var failActivityCreation = new AsyncLocal(); var expected = new InvalidOperationException("activity-start-fault"); using var collector = new DiagnosticEventCollector(DispatcherEvents.ListenerName); + var activationCollector = fixture.Services.GetRequiredService(); + var baselineActivationCount = activationCollector._activationCount; ActivationData? context = null; ActivationStartupRequest? request = null; Task? rejected = null; @@ -61,6 +63,7 @@ public async Task FailureBeforeStartAbortsRejectsRequestAndUnregistersPreparedCo Assert.Equal(0, scenario.ConstructorCount); Assert.Equal(0, scenario.OnActivateCount); Assert.Equal(ActivationState.Invalid, context.State); + Assert.Equal(baselineActivationCount, activationCollector._activationCount); } finally { @@ -90,7 +93,6 @@ ActivitySamplingResult OnSample() Timeout, TestContext.Current.CancellationToken); context.ReceiveMessage(requestData.Message); - failActivityCreation.Value = false; throw expected; } } From 202dba9ba824ace4db893bc354edd56521ee1358 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 27 Aug 2026 12:29:08 -0700 Subject: [PATCH 19/28] fix(runtime): reject requests after startup abort --- src/Orleans.Runtime/Catalog/ActivationData.cs | 24 +++++++++++++++++-- .../ActivationStartupTests.cs | 20 ++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index db73d6915a1..3ff39398e5c 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -1673,9 +1673,29 @@ private void ReceiveRequest(Message message) return; } + var invalid = false; lock (this) { - _waitingRequests.Add((message, CoarseStopwatch.StartNew())); + if (State is ActivationState.Invalid) + { + invalid = true; + } + else + { + _waitingRequests.Add((message, CoarseStopwatch.StartNew())); + } + } + + if (invalid) + { + _shared.InternalRuntime.MessageCenter.ProcessRequestsToInvalidActivation( + [message], + Address, + forwardingAddress: ForwardingAddress, + failedOperation: DeactivationReason.Description, + exc: DeactivationException, + rejectMessages: true); + return; } _workSignal.Signal(); @@ -1968,8 +1988,8 @@ private async Task ActivateAsync(Dictionary? requestContextData, if (State is ActivationState.Activating) { SetState(ActivationState.Valid); - _shared.InternalRuntime.ActivationWorkingSet.OnActivated(this); _wasActivated = true; + _shared.InternalRuntime.ActivationWorkingSet.OnActivated(this); } } _activationActivity?.AddEvent(new ActivityEvent("state-valid")); diff --git a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs index faba8fd5283..2dd9862f8fc 100644 --- a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs @@ -64,6 +64,26 @@ public async Task FailureBeforeStartAbortsRejectsRequestAndUnregistersPreparedCo Assert.Equal(0, scenario.OnActivateCount); Assert.Equal(ActivationState.Invalid, context.State); Assert.Equal(baselineActivationCount, activationCollector._activationCount); + + var lateRequestData = fixture.CreateRequest( + context, + scenario, + payload: "late-must-not-run", + requestContextValue: "late-pre-start", + recordResponse: true); + var lateRejected = collector.WaitForEventAsync( + nameof(DispatcherEvents.Rejected), + diagnosticEvent => diagnosticEvent.Payload is DispatcherEvents.Rejected rejection + && ReferenceEquals(rejection.Message, lateRequestData.Message), + Timeout, + TestContext.Current.CancellationToken); + context.ReceiveMessage(lateRequestData.Message); + var lateRejection = Assert.IsType((await lateRejected).Payload); + Assert.Equal(Message.RejectionTypes.Transient, lateRejection.RejectionType); + Assert.Equal( + "Activation startup was aborted.", + Assert.IsType(lateRejection.Exception).Message); + Assert.Equal(0, scenario.RequestInvocationCount); } finally { From cd82eb50d36c2eed85336e0357c0d961bdfca21d Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 27 Aug 2026 12:49:05 -0700 Subject: [PATCH 20/28] fix(runtime): preserve abort compatibility --- .../Activation/IGrainContextActivator.cs | 2 ++ src/Orleans.Runtime/Catalog/ActivationData.cs | 2 +- src/Orleans.Runtime/Catalog/Catalog.cs | 23 +++++++++++++- .../Runtime/GrainContextActivatorTests.cs | 30 +++++++++++++++++++ 4 files changed, 55 insertions(+), 2 deletions(-) diff --git a/src/Orleans.Runtime/Activation/IGrainContextActivator.cs b/src/Orleans.Runtime/Activation/IGrainContextActivator.cs index 891c2638e99..7dba7e39eac 100644 --- a/src/Orleans.Runtime/Activation/IGrainContextActivator.cs +++ b/src/Orleans.Runtime/Activation/IGrainContextActivator.cs @@ -156,6 +156,8 @@ internal readonly struct PreparedGrainContext(IGrainContext context, IGrainConte public IGrainContext Context => _context ?? throw new InvalidOperationException("The grain context activation is not initialized."); + public bool HasStartup => _startup is not null; + public static PreparedGrainContext Create( IGrainContextActivator activator, GrainAddress address, diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 3ff39398e5c..69106964d7b 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -1694,7 +1694,7 @@ private void ReceiveRequest(Message message) forwardingAddress: ForwardingAddress, failedOperation: DeactivationReason.Description, exc: DeactivationException, - rejectMessages: true); + rejectMessages: DeactivationException is not null && ForwardingAddress is null); return; } diff --git a/src/Orleans.Runtime/Catalog/Catalog.cs b/src/Orleans.Runtime/Catalog/Catalog.cs index c0ee3cd8caa..c82bc079524 100644 --- a/src/Orleans.Runtime/Catalog/Catalog.cs +++ b/src/Orleans.Runtime/Catalog/Catalog.cs @@ -215,7 +215,13 @@ internal int UnregisterGrainForTesting(GrainId grain) try { - preparedContext.Abort(); + AbortPreparedContext( + preparedContext, + result, + new DeactivationReason( + DeactivationReasonCode.ActivationFailed, + exception, + "Error preparing grain activation.")); } catch (Exception cleanupException) { @@ -319,6 +325,21 @@ internal static IDisposable StartPreparedContext( } } + internal static void AbortPreparedContext( + PreparedGrainContext preparedContext, + IGrainContext context, + DeactivationReason reason) + { + if (preparedContext.HasStartup) + { + preparedContext.Abort(); + } + else + { + context.Deactivate(reason, CancellationToken.None); + } + } + private async Task UnregisterNonExistentActivation(GrainAddress address) { try diff --git a/test/Orleans.Core.Tests/Runtime/GrainContextActivatorTests.cs b/test/Orleans.Core.Tests/Runtime/GrainContextActivatorTests.cs index 569bcb4f211..005a47d81b2 100644 --- a/test/Orleans.Core.Tests/Runtime/GrainContextActivatorTests.cs +++ b/test/Orleans.Core.Tests/Runtime/GrainContextActivatorTests.cs @@ -117,6 +117,36 @@ public void CatalogStartPreparedContext_StartFailureRemovesRecordedTarget() Assert.Equal(["start", "abort"], events); } + [Fact] + public void CatalogAbortPreparedContext_EagerCustomContextIsDeactivated() + { + var context = Substitute.For(); + var reason = new DeactivationReason( + DeactivationReasonCode.ActivationFailed, + "pre-start-failure"); + var preparedContext = new PreparedGrainContext(context, startup: null); + + Catalog.AbortPreparedContext(preparedContext, context, reason); + + context.Received(1).Deactivate(reason, CancellationToken.None); + } + + [Fact] + public void CatalogAbortPreparedContext_PreparedContextIsAborted() + { + var events = new List(); + var context = Substitute.For(); + var reason = new DeactivationReason( + DeactivationReasonCode.ActivationFailed, + "pre-start-failure"); + var preparedContext = new PreparedGrainContext(context, new TestGrainContextStartup(events)); + + Catalog.AbortPreparedContext(preparedContext, context, reason); + + Assert.Equal(["abort"], events); + context.DidNotReceive().Deactivate(Arg.Any(), Arg.Any()); + } + private static GrainContextActivator CreateActivator( IGrainContextActivator contextActivator, List events) => From a919a1ccc1470a4df715100cca429fc910921a5a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 27 Aug 2026 13:06:40 -0700 Subject: [PATCH 21/28] fix(runtime): defer aborted startup cleanup --- src/Orleans.Runtime/Catalog/ActivationData.cs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 69106964d7b..03e95744d00 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -193,8 +193,7 @@ void IGrainContextStartup.Abort() TaskCreationOptions.DenyChildAttach, TaskScheduler.Default) .Unwrap() - .GetAwaiter() - .GetResult(); + .Ignore(); } } @@ -217,9 +216,15 @@ private async ValueTask AbortStartupAsync() } finally { - await DisposeAsync(); - GetDeactivationCompletionSource().TrySetResult(true); - _workSignal.Signal(); + try + { + await DisposeAsync(); + } + finally + { + GetDeactivationCompletionSource().TrySetResult(true); + _workSignal.Signal(); + } } } From 9164679bad4301ee715fa63986837ffdf6060868 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 27 Aug 2026 13:34:23 -0700 Subject: [PATCH 22/28] fix(runtime): dispose failed rehydration context --- src/Orleans.Runtime/Catalog/Catalog.cs | 53 +++++++++++++------ .../GrainActivatorTests.cs | 30 +++++++++++ 2 files changed, 66 insertions(+), 17 deletions(-) diff --git a/src/Orleans.Runtime/Catalog/Catalog.cs b/src/Orleans.Runtime/Catalog/Catalog.cs index c82bc079524..5c2dad4541b 100644 --- a/src/Orleans.Runtime/Catalog/Catalog.cs +++ b/src/Orleans.Runtime/Catalog/Catalog.cs @@ -147,29 +147,48 @@ internal int UnregisterGrainForTesting(GrainId grain) return null; } - lock (GetStripedLock(grainId)) + try + { + lock (GetStripedLock(grainId)) + { + if (TryGetGrainContext(grainId, out result)) + { + rehydrationContext?.Dispose(); + return result; + } + + if (_siloStatusOracle.CurrentStatus == SiloStatus.Active) + { + var address = new GrainAddress + { + SiloAddress = Silo, + GrainId = grainId, + ActivationId = ActivationId.NewId(), + MembershipVersion = MembershipVersion.MinValue, + }; + + preparedContext = this.grainActivator.CreatePreparedContext(address); + result = preparedContext.Context; + activations.RecordNewTarget(result); + } + } + } + catch (Exception exception) when (result is null) { - if (TryGetGrainContext(grainId, out result)) + try { rehydrationContext?.Dispose(); - return result; } - - if (_siloStatusOracle.CurrentStatus == SiloStatus.Active) + catch (Exception cleanupException) { - var address = new GrainAddress - { - SiloAddress = Silo, - GrainId = grainId, - ActivationId = ActivationId.NewId(), - MembershipVersion = MembershipVersion.MinValue, - }; - - preparedContext = this.grainActivator.CreatePreparedContext(address); - result = preparedContext.Context; - activations.RecordNewTarget(result); + throw new AggregateException( + "Error creating grain activation and disposing its rehydration context.", + exception, + cleanupException); } - } // End lock + + throw; + } if (result is null) { diff --git a/test/Orleans.Runtime.Tests/GrainActivatorTests.cs b/test/Orleans.Runtime.Tests/GrainActivatorTests.cs index c46f182c3b3..8c7a6cbd8c8 100644 --- a/test/Orleans.Runtime.Tests/GrainActivatorTests.cs +++ b/test/Orleans.Runtime.Tests/GrainActivatorTests.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Logging.Abstractions; using Orleans.Metadata; using Orleans.Runtime; +using Orleans.Serialization.Session; using Orleans.TestingHost; using TestExtensions; using UnitTests.GrainInterfaces; @@ -219,6 +220,35 @@ public void ConfigurationFailureBeforeConstructionDoesNotDecrementActiveGrainCou measurement => Equals(measurement.Tags["type"], grainTypeName)); } + [Fact] + public void ConfigurationFailureDisposesRehydrationContext() + { + var primary = Assert.IsType(fixture.HostedCluster.Primary); + var services = primary.ServiceProvider; + var grainType = services.GetRequiredService() + .GetGrainType(typeof(ExplicitlyRegisteredSimpleDIGrain)); + var grainId = GrainId.Create(grainType, Guid.NewGuid().ToString()); + using var migrationContext = new MigrationContext( + services.GetRequiredService()); + migrationContext.AddBytes("test", [1, 2, 3]); + + ConfigurationFailureState.Arm(grainId); + try + { + var exception = Assert.Throws( + () => services.GetRequiredService() + .GetOrCreateActivation(grainId, requestContextData: null, migrationContext)); + Assert.Equal("configuration-fault", exception.Message); + } + finally + { + ConfigurationFailureState.Clear(); + } + + Assert.False(migrationContext.TryGetBytes("test", out _)); + Assert.Null(services.GetRequiredService().FindTarget(grainId)); + } + /// /// Custom grain activator that bypasses dependency injection entirely. /// Implements both IGrainActivator (for creation/disposal) and IConfigureGrainTypeComponents From 5d5901100c77a248e0ff5e1d79064d753c13a9b8 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 27 Aug 2026 21:11:29 -0700 Subject: [PATCH 23/28] test(runtime): propagate test cancellation --- .../Runtime/DefaultExecutionContextTests.cs | 24 +++++----- .../OrleansTaskSchedulerBasicTests.cs | 44 +++++++++++-------- .../ActivationDataMigrationTests.cs | 10 ++++- .../ActivationStartupTests.cs | 24 +++++++--- .../StatelessWorkerActivationStartupTests.cs | 8 +++- .../GrainActivatorTests.cs | 16 +++++-- 6 files changed, 83 insertions(+), 43 deletions(-) diff --git a/test/Orleans.Core.Tests/Runtime/DefaultExecutionContextTests.cs b/test/Orleans.Core.Tests/Runtime/DefaultExecutionContextTests.cs index 2d8120bb1e9..078855372f1 100644 --- a/test/Orleans.Core.Tests/Runtime/DefaultExecutionContextTests.cs +++ b/test/Orleans.Core.Tests/Runtime/DefaultExecutionContextTests.cs @@ -24,19 +24,21 @@ public async Task InstanceSupportsConcurrentExecution() for (var i = 0; i < tasks.Length; i++) { var expected = new object(); - tasks[i] = Task.Run(() => - { - var ambientState = new AsyncLocal { Value = expected }; - object? observed = expected; + tasks[i] = Task.Run( + () => + { + var ambientState = new AsyncLocal { Value = expected }; + object? observed = expected; - ExecutionContext.Run( - DefaultExecutionContext.Instance, - _ => observed = ambientState.Value, - null); + ExecutionContext.Run( + DefaultExecutionContext.Instance, + _ => observed = ambientState.Value, + null); - Assert.Null(observed); - Assert.Same(expected, ambientState.Value); - }); + Assert.Null(observed); + Assert.Same(expected, ambientState.Value); + }, + TestContext.Current.CancellationToken); } await Task.WhenAll(tasks); diff --git a/test/Orleans.Core.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs b/test/Orleans.Core.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs index bfe05d0a8b5..9800c25768a 100644 --- a/test/Orleans.Core.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs +++ b/test/Orleans.Core.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs @@ -150,7 +150,7 @@ public async Task Sched_ActivationStartup_DefersQueuedWorkUntilDisposed() Assert.Throws(startup.Abort); } - await queuedTask.WaitAsync(TimeSpan.FromSeconds(5)); + await queuedTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.Equal(1, queuedTaskObservedStartCompletion); } @@ -194,16 +194,20 @@ public async Task Sched_ActivationStartup_PreservesContextForReleasedAsynchronou Assert.True(startTask.IsCompletedSuccessfully, startTask.Exception?.ToString()); Assert.Equal(3, initialObservation); } - await loopStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await loopStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.Equal(3, asyncInitialObservation); for (var i = 0; i < observations.Length; i++) { signal.Signal(); - Assert.Equal(3, await observations[i].Task.WaitAsync(TimeSpan.FromSeconds(5))); + Assert.Equal( + 3, + await observations[i].Task.WaitAsync( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken)); } - await observationLoop!.WaitAsync(TimeSpan.FromSeconds(5)); + await observationLoop!.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); async Task ObserveSignals() { @@ -246,7 +250,7 @@ public async Task Sched_ActivationStartup_DisposeIsIdempotent() queuedTask.Start(_rootContext.WorkItemGroup.TaskScheduler); startup.Dispose(); - await queuedTask.WaitAsync(TimeSpan.FromSeconds(5)); + await queuedTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.Equal(1, Volatile.Read(ref queuedWorkCount)); Assert.Equal(0, _rootContext.WorkItemGroup.ExternalWorkItemCount); @@ -256,7 +260,7 @@ public async Task Sched_ActivationStartup_DisposeIsIdempotent() var subsequentTask = new Task(() => Interlocked.Increment(ref subsequentWorkCount)); subsequentTask.Start(_rootContext.WorkItemGroup.TaskScheduler); - await subsequentTask.WaitAsync(TimeSpan.FromSeconds(5)); + await subsequentTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.Equal(1, Volatile.Read(ref queuedWorkCount)); Assert.Equal(1, Volatile.Read(ref subsequentWorkCount)); @@ -280,7 +284,7 @@ public async Task Sched_ActivationStartup_AbortDiscardsQueuedWorkAndAllowsReuse( var subsequentTask = new Task(() => Interlocked.Increment(ref subsequentWorkCount)); subsequentTask.Start(_rootContext.WorkItemGroup.TaskScheduler); - await subsequentTask.WaitAsync(TimeSpan.FromSeconds(5)); + await subsequentTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.Equal(0, Volatile.Read(ref discardedWorkCount)); Assert.Equal(1, Volatile.Read(ref subsequentWorkCount)); @@ -631,35 +635,39 @@ public async Task Sched_ActivationStartup_ConcurrentReleaseExecutesQueuedWorkExa for (var i = 0; i < ParticipantCount; i++) { var participant = participantReady[i] = new(TaskCreationOptions.RunContinuationsAsynchronously); - releaseParticipants[i] = Task.Run(async () => - { - participant.SetResult(); - await releaseParticipantsBarrier.Task; - startup.Dispose(); - }); + releaseParticipants[i] = Task.Run( + async () => + { + participant.SetResult(); + await releaseParticipantsBarrier.Task; + startup.Dispose(); + }, + TestContext.Current.CancellationToken); } try { await Task.WhenAll(Array.ConvertAll(participantReady, static participant => participant.Task)) - .WaitAsync(TimeSpan.FromSeconds(5)); + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.False(queuedWork.IsCompleted); releaseParticipantsBarrier.SetResult(); - await Task.WhenAll(releaseParticipants).WaitAsync(TimeSpan.FromSeconds(5)); - await queuedWork.WaitAsync(TimeSpan.FromSeconds(5)); + await Task.WhenAll(releaseParticipants) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await queuedWork.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.Equal(1, Volatile.Read(ref queuedWorkCount)); var subsequentWork = new Task(() => Interlocked.Increment(ref subsequentWorkCount)); subsequentWork.Start(_rootContext.WorkItemGroup.TaskScheduler); - await subsequentWork.WaitAsync(TimeSpan.FromSeconds(5)); + await subsequentWork.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.Equal(1, Volatile.Read(ref subsequentWorkCount)); } finally { releaseParticipantsBarrier.TrySetResult(); startup.Dispose(); - await Task.WhenAll(releaseParticipants).WaitAsync(TimeSpan.FromSeconds(5)); + await Task.WhenAll(releaseParticipants) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); } } } diff --git a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationDataMigrationTests.cs b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationDataMigrationTests.cs index 2df8c49139c..41d7256a7b9 100644 --- a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationDataMigrationTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationDataMigrationTests.cs @@ -160,7 +160,10 @@ await scenario.WaitForEvent("OnActivateEntered").WaitAsync( TestContext.Current.CancellationToken); Assert.Equal(ActivationState.Activating, context.State); - Assert.False(context.TryStartMigration(requestContext: null)); + Assert.False( + context.TryStartMigration( + requestContext: null, + TestContext.Current.CancellationToken)); Assert.Equal(ActivationState.Activating, context.State); Assert.Same(context, startupFixture.ActivationDirectory.FindTarget(grainId)); Assert.DoesNotContain( @@ -180,7 +183,10 @@ await scenario.WaitForEvent("Activated").WaitAsync( var deactivating = scenario.WaitForEvent("Deactivating"); var deactivated = scenario.WaitForEvent("Deactivated"); - Assert.True(context.TryStartMigration(requestContext: null)); + Assert.True( + context.TryStartMigration( + requestContext: null, + TestContext.Current.CancellationToken)); await deactivating.WaitAsync( TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); diff --git a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs index 2dd9862f8fc..fb928163331 100644 --- a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs @@ -163,7 +163,9 @@ public async Task AsyncActivation_OrdersLifecycleCallbacksAndCleanup() context.ReceiveMessage(message); await scenario.WaitForEvent("RequestInvoked").WaitAsync(Timeout, TestContext.Current.CancellationToken); - context.Deactivate(new(DeactivationReasonCode.ApplicationRequested, "Lifecycle test complete.")); + context.Deactivate( + new(DeactivationReasonCode.ApplicationRequested, "Lifecycle test complete."), + TestContext.Current.CancellationToken); await scenario.WaitForEvent("ActivatorDisposeStarted").WaitAsync(Timeout, TestContext.Current.CancellationToken); Assert.Equal(0, scenario.DisposeCompletedCount); Assert.Equal(0, scenario.ScopeDisposeCount); @@ -319,7 +321,9 @@ public async Task CancellationDuringAsyncActivation_AbortsAndCleansUpExactlyOnce context = fixture.StartActivation(grainId); await entered.WaitAsync(Timeout, TestContext.Current.CancellationToken); - context.Deactivate(new(DeactivationReasonCode.RuntimeRequested, "Cancel startup.")); + context.Deactivate( + new(DeactivationReasonCode.RuntimeRequested, "Cancel startup."), + TestContext.Current.CancellationToken); await cancellationObserved.WaitAsync(Timeout, TestContext.Current.CancellationToken); await disposeStarted.WaitAsync(Timeout, TestContext.Current.CancellationToken); Assert.Equal(0, scenario.DisposeCompletedCount); @@ -388,7 +392,9 @@ public async Task SiloShutdownDuringAsyncActivation_CancelsAndCleansUpExactlyOnc Assert.Equal(0, scenario.RequestInvocationCount); Assert.Null(request.Result); - stopTask = shutdownFixture.HostedCluster.StopSiloAsync(primarySilo); + stopTask = shutdownFixture.HostedCluster.StopSiloAsync( + primarySilo, + cancellationToken: TestContext.Current.CancellationToken); await cancellationObserved.WaitAsync(Timeout, TestContext.Current.CancellationToken); await disposeStarted.WaitAsync(Timeout, TestContext.Current.CancellationToken); @@ -552,13 +558,17 @@ public async Task ActivationStartup_CleanupOccursExactlyOnce( { case ActivationStartupCompletion.ImmediateSuccess: await activated.WaitAsync(Timeout, TestContext.Current.CancellationToken); - context.Deactivate(new(DeactivationReasonCode.ApplicationRequested, "Matrix success complete.")); + context.Deactivate( + new(DeactivationReasonCode.ApplicationRequested, "Matrix success complete."), + TestContext.Current.CancellationToken); break; case ActivationStartupCompletion.AsynchronousSuccess: await entered.WaitAsync(Timeout, TestContext.Current.CancellationToken); scenario.ReleaseActivation(); await activated.WaitAsync(Timeout, TestContext.Current.CancellationToken); - context.Deactivate(new(DeactivationReasonCode.ApplicationRequested, "Matrix success complete.")); + context.Deactivate( + new(DeactivationReasonCode.ApplicationRequested, "Matrix success complete."), + TestContext.Current.CancellationToken); break; case ActivationStartupCompletion.ImmediateFailure: await failed.WaitAsync(Timeout, TestContext.Current.CancellationToken); @@ -570,7 +580,9 @@ public async Task ActivationStartup_CleanupOccursExactlyOnce( break; case ActivationStartupCompletion.Cancellation: await entered.WaitAsync(Timeout, TestContext.Current.CancellationToken); - context.Deactivate(new(DeactivationReasonCode.RuntimeRequested, "Matrix cancellation.")); + context.Deactivate( + new(DeactivationReasonCode.RuntimeRequested, "Matrix cancellation."), + TestContext.Current.CancellationToken); await cancellationObserved.WaitAsync(Timeout, TestContext.Current.CancellationToken); break; default: diff --git a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/StatelessWorkerActivationStartupTests.cs b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/StatelessWorkerActivationStartupTests.cs index f0ce3016406..9c2b5d93bb4 100644 --- a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/StatelessWorkerActivationStartupTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/StatelessWorkerActivationStartupTests.cs @@ -50,7 +50,9 @@ public async Task StatelessWorkerChildIsPublishedBeforeSynchronousStartAndDoesNo payload: "synchronous", requestContextValue: "sync-request"); - admissionTask = Task.Run(() => wrapper.ReceiveMessage(message)); + admissionTask = Task.Run( + () => wrapper.ReceiveMessage(message), + TestContext.Current.CancellationToken); worker = Assert.IsType( await gate.Entered.WaitAsync(Timeout, TestContext.Current.CancellationToken)); @@ -207,7 +209,9 @@ public async Task StatelessWorkerStartupFailureRemovesChildAndDisposesResourcesE if (gate is not null) { - admissionTask = Task.Run(() => wrapper.ReceiveMessage(message)); + admissionTask = Task.Run( + () => wrapper.ReceiveMessage(message), + TestContext.Current.CancellationToken); worker = Assert.IsType( await gate.Entered.WaitAsync(Timeout, TestContext.Current.CancellationToken)); Assert.Same(worker, Assert.Single(GetWorkers(wrapper))); diff --git a/test/Orleans.Runtime.Tests/GrainActivatorTests.cs b/test/Orleans.Runtime.Tests/GrainActivatorTests.cs index 8c7a6cbd8c8..3f11f9dda36 100644 --- a/test/Orleans.Runtime.Tests/GrainActivatorTests.cs +++ b/test/Orleans.Runtime.Tests/GrainActivatorTests.cs @@ -149,8 +149,12 @@ public async Task ContextCreationStartsActivationSynchronouslyOnActivationSchedu state.ClearAmbientState(); if (context is not null) { - context.Deactivate(new(DeactivationReasonCode.ApplicationRequested, "Test completed.")); - await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(10)); + context.Deactivate( + new(DeactivationReasonCode.ApplicationRequested, "Test completed."), + TestContext.Current.CancellationToken); + await context.Deactivated.WaitAsync( + TimeSpan.FromSeconds(10), + TestContext.Current.CancellationToken); } } } @@ -181,8 +185,12 @@ public async Task BuiltInActivatorCreateContextStartsContext() } finally { - context.Deactivate(new(DeactivationReasonCode.ApplicationRequested, "Test completed.")); - await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(10)); + context.Deactivate( + new(DeactivationReasonCode.ApplicationRequested, "Test completed."), + TestContext.Current.CancellationToken); + await context.Deactivated.WaitAsync( + TimeSpan.FromSeconds(10), + TestContext.Current.CancellationToken); } } From 1cdaf2d18d653cd4b190f8c646bca5bf0330b379 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 28 Aug 2026 02:15:15 -0700 Subject: [PATCH 24/28] style(test): remove unused activation imports --- .../ActivationsLifeCycleTests/ActivationDataMigrationTests.cs | 1 - .../ActivationsLifeCycleTests/ActivationStartupTestFixture.cs | 1 - .../ActivationsLifeCycleTests/ActivationStartupTests.cs | 1 - .../StatelessWorkerActivationStartupTests.cs | 1 - test/Orleans.Runtime.Tests/GrainActivatorTests.cs | 1 - 5 files changed, 5 deletions(-) diff --git a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationDataMigrationTests.cs b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationDataMigrationTests.cs index 41d7256a7b9..069afadeb46 100644 --- a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationDataMigrationTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationDataMigrationTests.cs @@ -1,6 +1,5 @@ #nullable enable using Microsoft.Extensions.DependencyInjection; -using Orleans.CodeGeneration; using Orleans.Metadata; using Orleans.Runtime; using Orleans.Runtime.Diagnostics; diff --git a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTestFixture.cs b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTestFixture.cs index 9cda1796338..a389acb9af6 100644 --- a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTestFixture.cs +++ b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTestFixture.cs @@ -1,6 +1,5 @@ using System.Reflection; using Microsoft.Extensions.DependencyInjection; -using Orleans.CodeGeneration; using Orleans.Metadata; using Orleans.Runtime; using Orleans.Runtime.Diagnostics; diff --git a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs index fb928163331..bcf02559e3a 100644 --- a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs @@ -1,6 +1,5 @@ using System.Diagnostics; using Microsoft.Extensions.DependencyInjection; -using Orleans.CodeGeneration; using Orleans.Diagnostics; using Orleans.Runtime; using Orleans.Runtime.Diagnostics; diff --git a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/StatelessWorkerActivationStartupTests.cs b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/StatelessWorkerActivationStartupTests.cs index 9c2b5d93bb4..e64978dbaf8 100644 --- a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/StatelessWorkerActivationStartupTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/StatelessWorkerActivationStartupTests.cs @@ -1,6 +1,5 @@ using System.Reflection; using Microsoft.Extensions.DependencyInjection; -using Orleans.CodeGeneration; using Orleans.Metadata; using Orleans.Runtime; using Orleans.Runtime.Diagnostics; diff --git a/test/Orleans.Runtime.Tests/GrainActivatorTests.cs b/test/Orleans.Runtime.Tests/GrainActivatorTests.cs index 3f11f9dda36..9ce96ee8242 100644 --- a/test/Orleans.Runtime.Tests/GrainActivatorTests.cs +++ b/test/Orleans.Runtime.Tests/GrainActivatorTests.cs @@ -1,5 +1,4 @@ using System.Diagnostics.CodeAnalysis; -using System.Diagnostics.Metrics; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.Metrics.Testing; using Microsoft.Extensions.Logging.Abstractions; From c867151f0fb4e598d51f08742ebe9f0fe675e600 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 28 Aug 2026 02:38:26 -0700 Subject: [PATCH 25/28] fix(test): restore metrics namespace --- test/Orleans.Runtime.Tests/GrainActivatorTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/Orleans.Runtime.Tests/GrainActivatorTests.cs b/test/Orleans.Runtime.Tests/GrainActivatorTests.cs index 9ce96ee8242..3f11f9dda36 100644 --- a/test/Orleans.Runtime.Tests/GrainActivatorTests.cs +++ b/test/Orleans.Runtime.Tests/GrainActivatorTests.cs @@ -1,4 +1,5 @@ using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Metrics; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.Metrics.Testing; using Microsoft.Extensions.Logging.Abstractions; From dd0952bd4a1c5383b5bd69f16bb3bc047eae24aa Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 28 Aug 2026 02:52:18 -0700 Subject: [PATCH 26/28] fix(test): restore invocation namespace --- .../ActivationsLifeCycleTests/ActivationStartupTestFixture.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTestFixture.cs b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTestFixture.cs index a389acb9af6..9cda1796338 100644 --- a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTestFixture.cs +++ b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTestFixture.cs @@ -1,5 +1,6 @@ using System.Reflection; using Microsoft.Extensions.DependencyInjection; +using Orleans.CodeGeneration; using Orleans.Metadata; using Orleans.Runtime; using Orleans.Runtime.Diagnostics; From cb08b32d98ba4af4dcbb2e2cbdb78740e1c8ae51 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 28 Aug 2026 03:03:35 -0700 Subject: [PATCH 27/28] fix(test): restore code generation namespace --- .../ActivationsLifeCycleTests/ActivationDataMigrationTests.cs | 1 + .../ActivationsLifeCycleTests/ActivationStartupTests.cs | 1 + .../StatelessWorkerActivationStartupTests.cs | 1 + 3 files changed, 3 insertions(+) diff --git a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationDataMigrationTests.cs b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationDataMigrationTests.cs index 069afadeb46..41d7256a7b9 100644 --- a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationDataMigrationTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationDataMigrationTests.cs @@ -1,5 +1,6 @@ #nullable enable using Microsoft.Extensions.DependencyInjection; +using Orleans.CodeGeneration; using Orleans.Metadata; using Orleans.Runtime; using Orleans.Runtime.Diagnostics; diff --git a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs index bcf02559e3a..fb928163331 100644 --- a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationStartupTests.cs @@ -1,5 +1,6 @@ using System.Diagnostics; using Microsoft.Extensions.DependencyInjection; +using Orleans.CodeGeneration; using Orleans.Diagnostics; using Orleans.Runtime; using Orleans.Runtime.Diagnostics; diff --git a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/StatelessWorkerActivationStartupTests.cs b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/StatelessWorkerActivationStartupTests.cs index e64978dbaf8..9c2b5d93bb4 100644 --- a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/StatelessWorkerActivationStartupTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/StatelessWorkerActivationStartupTests.cs @@ -1,5 +1,6 @@ using System.Reflection; using Microsoft.Extensions.DependencyInjection; +using Orleans.CodeGeneration; using Orleans.Metadata; using Orleans.Runtime; using Orleans.Runtime.Diagnostics; From 19d2e79199f34fb611360bdf334ee208e927bd2f Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 28 Aug 2026 05:10:22 -0700 Subject: [PATCH 28/28] fix(runtime): preserve startup failure details --- .../Activation/IGrainContextActivator.cs | 15 ++++++- .../Runtime/GrainContextActivatorTests.cs | 41 +++++++++++++++++++ 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/src/Orleans.Runtime/Activation/IGrainContextActivator.cs b/src/Orleans.Runtime/Activation/IGrainContextActivator.cs index 7dba7e39eac..1a520072617 100644 --- a/src/Orleans.Runtime/Activation/IGrainContextActivator.cs +++ b/src/Orleans.Runtime/Activation/IGrainContextActivator.cs @@ -177,9 +177,20 @@ public IDisposable Start() { return startup.Start(); } - catch + catch (Exception startupException) { - startup.Abort(); + try + { + startup.Abort(); + } + catch (Exception abortException) + { + throw new AggregateException( + "Grain context startup failed and aborting the startup also failed.", + startupException, + abortException); + } + throw; } } diff --git a/test/Orleans.Core.Tests/Runtime/GrainContextActivatorTests.cs b/test/Orleans.Core.Tests/Runtime/GrainContextActivatorTests.cs index 005a47d81b2..5cb03352cf8 100644 --- a/test/Orleans.Core.Tests/Runtime/GrainContextActivatorTests.cs +++ b/test/Orleans.Core.Tests/Runtime/GrainContextActivatorTests.cs @@ -87,6 +87,29 @@ public void PreparedContext_StartFailureAbortsExactlyOnce() Assert.Equal(["start", "abort"], events); } + [Fact] + public void PreparedContext_StartAndAbortFailureReportsBothExceptions() + { + var events = new List(); + var context = Substitute.For(); + var startupException = new InvalidOperationException("start-fault"); + var abortException = new InvalidOperationException("abort-fault"); + var preparedContext = new PreparedGrainContext( + context, + new ThrowingStartAndAbortGrainContextStartup( + events, + startupException, + abortException)); + + var actual = Assert.Throws(preparedContext.Start); + + Assert.Equal( + "Grain context startup failed and aborting the startup also failed.", + actual.Message.Split(" (")[0]); + Assert.Equal([startupException, abortException], actual.InnerExceptions); + Assert.Equal(["start", "abort"], events); + } + [Fact] public void CatalogStartPreparedContext_StartFailureRemovesRecordedTarget() { @@ -256,4 +279,22 @@ public IDisposable Start() public void Abort() => events.Add("abort"); } + + private sealed class ThrowingStartAndAbortGrainContextStartup( + List events, + Exception startupException, + Exception abortException) : IGrainContextStartup + { + public IDisposable Start() + { + events.Add("start"); + throw startupException; + } + + public void Abort() + { + events.Add("abort"); + throw abortException; + } + } }