diff --git a/docs/site/src/content/docs/grains/event-sourcing/event-sourcing-configuration.md b/docs/site/src/content/docs/grains/event-sourcing/event-sourcing-configuration.md index 25a45aa03fc..640b46ffa5c 100644 --- a/docs/site/src/content/docs/grains/event-sourcing/event-sourcing-configuration.md +++ b/docs/site/src/content/docs/grains/event-sourcing/event-sourcing-configuration.md @@ -1,7 +1,7 @@ --- title: Event sourcing configuration description: Configure JournaledGrain log consistency and storage in Orleans. -ms.date: 08/02/2026 +ms.date: 08/23/2026 ms.topic: how-to --- @@ -20,6 +20,7 @@ Available registration methods are: - - - +- Each also has an `AsDefault` form. If a default log-consistency provider and default grain storage provider are registered, provider attributes can be omitted. @@ -35,6 +36,16 @@ Custom storage doesn't use . The grain imple :::code language="csharp" source="../../snippets/compiled/EventSourcing/EventSourcingSnippets.cs" id="custom_storage_grain"::: +## Journaled-state provider + +The journaled-state provider stores the event log in the same Orleans journal as the activation's other durable states. One write atomically publishes the captured event log, write marker, and auxiliary durable state. Snapshot replacement failures leave the previous journal generation published and retain the captured changes for retry. + +This provider runs on a single turn-serialized grain activation. Grain types configured for reentrancy, selective interleaving, always-interleaved methods, or stateless-worker placement are rejected during activation. + +The provider's persisted write marker supports a versioned length-prefixed encoding, so every valid `ClusterId`, including identifiers containing commas and punctuation, has an exact identity. Existing comma-token markers remain readable and continue using that representation while all cluster identifiers are delimiter-safe, preserving rolling upgrades with previous Orleans versions. + +Complete the Orleans rolling upgrade before configuring a comma-containing `ClusterId`. The first write from that cluster upgrades the marker to the versioned representation, which previous Orleans versions don't understand. After that upgrade, rollback requires restoring a journal generation written with the legacy marker format. + ## Multi-cluster responsibility Custom storage owns the write-topology rules needed by a multi-cluster deployment. The `primaryCluster` registration argument is retained by the provider but doesn't restrict submissions, configure Orleans multi-cluster networking, replicate storage, or provide failover. Enforce any single-writer or regional-write rule in the application and storage implementation. diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index 063b8f85c61..838cf9d4531 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -1,7 +1,7 @@ --- title: Journaling runtime behavior and consistency description: Understand Orleans Journaling activation, write, recovery, compaction, concurrency, and failure semantics. -ms.date: 08/21/2026 +ms.date: 08/23/2026 ms.topic: conceptual --- @@ -49,7 +49,11 @@ Design commands to tolerate retries at the application boundary. Use operation i A normal append failure leaves encoded pending entries available for a later write attempt. -A compaction write has two storage stages: it first appends committed pending entries, then publishes a snapshot replacement. The append can succeed before the replacement fails. In that outcome, faults even though the state mutation is durable in the append history. Treat every failed write as an uncertain application outcome and retry commands using an operation identifier or another idempotency mechanism. +A snapshot failure leaves the previously published journal unchanged and keeps the captured in-memory changes available for a later write attempt. Commands added while the replacement is awaiting storage remain outside the captured snapshot and are persisted by a later operation. + +An optimistic-concurrency conflict identifies a competing journal generation. The manager recovers that winning generation and discards the losing activation's uncommitted in-memory changes before reporting the conflict. + +Storage acknowledgement can still have an uncertain network outcome. Treat a failed write as an uncertain application outcome and retry commands using an operation identifier or another idempotency mechanism. Recovery exceptions fault activation or queued work rather than replacing or truncating stored data. Restore the required format/codec registration or repair the backing data before retrying activation. @@ -57,9 +61,10 @@ Recovery exceptions fault activation or queued work rather than replacing or tru Each provider reports when its journal crosses a configured storage threshold. The next : -1. Persists any already-buffered append data. -1. Builds a snapshot containing the state directory and every active durable state. -1. Atomically replaces the published journal with the snapshot. +1. Captures the pending journal prefix and builds a snapshot containing the state directory and every active durable state. +1. Atomically replaces the published journal with the complete snapshot. +1. Consumes the captured pending prefix after storage acknowledges the replacement. +1. Leaves commands added during the replacement pending for the next write. 1. Clears the compaction request after storage acknowledges the replacement. Compaction bounds replay work and storage growth according to provider thresholds. Snapshot size still scales with the complete durable state owned by the grain, so capacity tests must include hot and large grain identities. diff --git a/src/Orleans.EventSourcing/Common/StringEncodedWriteVector.cs b/src/Orleans.EventSourcing/Common/StringEncodedWriteVector.cs index 1df9265c90a..6de0a77a2a9 100644 --- a/src/Orleans.EventSourcing/Common/StringEncodedWriteVector.cs +++ b/src/Orleans.EventSourcing/Common/StringEncodedWriteVector.cs @@ -1,52 +1,157 @@ -namespace Orleans.EventSourcing.Common +using System.Globalization; +using System.Text; + +namespace Orleans.EventSourcing.Common; + +/// +/// Encodes a set of replica write bits in a string. +/// +/// +/// New values use the versioned format v1: followed by UTF-16 length-prefixed replica identifiers. +/// This representation supports every valid cluster identifier without delimiter restrictions. Legacy values +/// containing comma-prefixed tokens remain readable. Updates remain in the legacy format while every identifier +/// is representable by it, preserving rolling-upgrade compatibility. A comma-containing identifier upgrades the +/// value to the current format. +/// In legacy values, commas are interpreted as token delimiters because the previous format did not escape them. +/// Malformed or unsupported versioned values throw . +/// +public static class StringEncodedWriteVector { - public static class StringEncodedWriteVector + private const string CurrentFormatPrefix = "v1:"; + + /// + /// Gets one of the bits in . + /// + /// The write vector. + /// The replica whose bit is returned. + /// when the replica's bit is set. + public static bool GetBit(string writeVector, string Replica) { + ArgumentNullException.ThrowIfNull(writeVector); + ArgumentException.ThrowIfNullOrEmpty(Replica); + return Decode(writeVector, out _).Contains(Replica, StringComparer.Ordinal); + } - // BitVector of replicas is implemented as a set of replica strings encoded within a string - // The bitvector is represented as the set of replica ids whose bit is 1 - // This set is written as a string that contains the replica ids preceded by a comma each - // - // Assuming our replicas are named A, B, and BB, then - // "" represents {} represents 000 - // ",A" represents {A} represents 100 - // ",A,B" represents {A,B} represents 110 - // ",BB,A,B" represents {A,B,BB} represents 111 - - /// - /// Gets one of the bits in writeVector - /// - /// The write vector which we want get the bit from - /// The replica for which we want to look up the bit - /// - public static bool GetBit(string writeVector, string Replica) - { - var pos = writeVector.IndexOf(Replica); - return pos != -1 && writeVector[pos - 1] == ','; - } - - /// - /// toggle one of the bits in writeVector and return the new value. - /// - /// The write vector in which we want to flip the bit - /// The replica for which we want to flip the bit - /// the state of the bit after flipping it - public static bool FlipBit(ref string writeVector, string Replica) - { - var pos = writeVector.IndexOf(Replica); - if (pos != -1 && writeVector[pos - 1] == ',') + /// + /// Toggles one of the bits in . + /// + /// The write vector. + /// The replica whose bit is toggled. + /// The bit value after it is toggled. + public static bool FlipBit(ref string writeVector, string Replica) + { + ArgumentNullException.ThrowIfNull(writeVector); + ArgumentException.ThrowIfNullOrEmpty(Replica); + + var replicas = Decode(writeVector, out var isLegacy); + var removed = false; + for (var index = replicas.Count - 1; index >= 0; index--) + { + if (string.Equals(replicas[index], Replica, StringComparison.Ordinal)) { - var pos2 = writeVector.IndexOf(',', pos + 1); - if (pos2 == -1) - pos2 = writeVector.Length; - writeVector = writeVector.Remove(pos - 1, pos2 - pos + 1); - return false; + replicas.RemoveAt(index); + removed = true; } - else + } + + if (!removed) + { + replicas.Add(Replica); + } + + writeVector = isLegacy && replicas.All(static replica => !replica.Contains(',')) + ? EncodeLegacy(replicas) + : EncodeCurrent(replicas); + return !removed; + } + + private static List Decode(string writeVector, out bool isLegacy) + { + if (writeVector.Length == 0) + { + isLegacy = true; + return []; + } + + if (!writeVector.StartsWith(CurrentFormatPrefix, StringComparison.Ordinal)) + { + isLegacy = true; + return DecodeLegacy(writeVector); + } + + isLegacy = false; + var result = new List(); + var position = CurrentFormatPrefix.Length; + if (position == writeVector.Length) + { + throw new FormatException("The versioned write vector does not contain any replica identifiers."); + } + + while (position < writeVector.Length) + { + var separator = writeVector.IndexOf(':', position); + if (separator < 0 + || separator == position + || !int.TryParse( + writeVector.AsSpan(position, separator - position), + NumberStyles.None, + CultureInfo.InvariantCulture, + out var length) + || length <= 0 + || length > writeVector.Length - separator - 1) { - writeVector = string.Format(",{0}{1}", Replica, writeVector); - return true; + throw new FormatException("The write vector contains an invalid length-prefixed replica identifier."); } + + position = separator + 1; + result.Add(writeVector.Substring(position, length)); + position += length; } + + return result; + } + + private static List DecodeLegacy(string writeVector) + { + if (writeVector[0] != ',') + { + throw new FormatException("The write vector has an unsupported format."); + } + + var tokens = writeVector[1..].Split(','); + if (tokens.Any(static token => token.Length == 0)) + { + throw new FormatException("The legacy write vector contains an empty replica identifier."); + } + + return [.. tokens]; + } + + private static string EncodeCurrent(List replicas) + { + if (replicas.Count == 0) + { + return string.Empty; + } + + var builder = new StringBuilder(CurrentFormatPrefix); + foreach (var replica in replicas) + { + builder.Append(replica.Length.ToString(CultureInfo.InvariantCulture)); + builder.Append(':'); + builder.Append(replica); + } + + return builder.ToString(); + } + + private static string EncodeLegacy(List replicas) + { + if (replicas.Count == 0) + { + return string.Empty; + } + + return string.Concat(",", string.Join(',', replicas)); } } diff --git a/src/Orleans.EventSourcing/Hosting/JournaledStateSiloBuilderExtensions.cs b/src/Orleans.EventSourcing/Hosting/JournaledStateSiloBuilderExtensions.cs new file mode 100644 index 00000000000..aca6554306d --- /dev/null +++ b/src/Orleans.EventSourcing/Hosting/JournaledStateSiloBuilderExtensions.cs @@ -0,0 +1,57 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Orleans.EventSourcing; +using Orleans.EventSourcing.JournaledState; +using Orleans.Journaling; +using Orleans.Providers; +using Orleans.Runtime; + +#nullable disable +#pragma warning disable ORLEANSEXP005 +namespace Orleans.Hosting; + +/// +/// Extensions for configuring journaled-state log consistency. +/// +public static class JournaledStateSiloBuilderExtensions +{ + /// + /// Adds a journaled-state log consistency provider as the default consistency provider. + /// + /// The silo builder. + /// The silo builder. + public static ISiloBuilder AddJournaledStateBasedLogConsistencyProviderAsDefault(this ISiloBuilder builder) + { + ArgumentNullException.ThrowIfNull(builder); + + return builder.AddJournaledStateBasedLogConsistencyProvider(ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME); + } + + /// + /// Adds a journaled-state log consistency provider. + /// + /// The silo builder. + /// The provider name. + /// The silo builder. + public static ISiloBuilder AddJournaledStateBasedLogConsistencyProvider(this ISiloBuilder builder, string name = "JournaledState") + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrWhiteSpace(name); + + builder.AddJournalStorage(); + return builder.ConfigureServices(services => services.AddJournaledStateBasedLogConsistencyProvider(name)); + } + + internal static IServiceCollection AddJournaledStateBasedLogConsistencyProvider(this IServiceCollection services, string name) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentException.ThrowIfNullOrWhiteSpace(name); + + services.AddLogConsistencyProtocolServicesFactory(); + services.TryAddSingleton( + serviceProvider => serviceProvider.GetKeyedService(ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME)!); + return services.AddKeyedSingleton(name); + } +} + +#pragma warning restore ORLEANSEXP005 diff --git a/src/Orleans.EventSourcing/JournaledState/LogConsistencyProvider.cs b/src/Orleans.EventSourcing/JournaledState/LogConsistencyProvider.cs new file mode 100644 index 00000000000..c5bef8be9e4 --- /dev/null +++ b/src/Orleans.EventSourcing/JournaledState/LogConsistencyProvider.cs @@ -0,0 +1,29 @@ +using Orleans.Storage; +#nullable disable +namespace Orleans.EventSourcing.JournaledState; + +/// +/// A log-consistency provider that stores event-sourcing events in the host grain's journaled state. +/// +public sealed class LogConsistencyProvider : ILogViewAdaptorFactory +{ + /// + public bool UsesStorageProvider => false; + + /// + public ILogViewAdaptor MakeLogViewAdaptor( + ILogViewAdaptorHost hostGrain, + TView initialState, + string grainTypeName, + IGrainStorage grainStorage, + ILogConsistencyProtocolServices services) + where TView : class, new() + where TEntry : class + { + ArgumentNullException.ThrowIfNull(hostGrain); + ArgumentNullException.ThrowIfNull(initialState); + ArgumentNullException.ThrowIfNull(services); + + return new LogViewAdaptor(hostGrain, initialState, services); + } +} diff --git a/src/Orleans.EventSourcing/JournaledState/LogViewAdaptor.cs b/src/Orleans.EventSourcing/JournaledState/LogViewAdaptor.cs new file mode 100644 index 00000000000..bf610687bda --- /dev/null +++ b/src/Orleans.EventSourcing/JournaledState/LogViewAdaptor.cs @@ -0,0 +1,389 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.ExceptionServices; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Orleans.Concurrency; +using Orleans.EventSourcing.Common; +using Orleans.Journaling; +using Orleans.Storage; + +#nullable disable +#pragma warning disable ORLEANSEXP005 +namespace Orleans.EventSourcing.JournaledState; + +/// +/// A log view adaptor that persists the event log as journaled state owned by the host grain. +/// +/// Type of log view. +/// Type of log entry. +internal sealed class LogViewAdaptor : PrimaryBasedLogViewAdaptor> + where TLogView : class, new() + where TLogEntry : class +{ + private const string EventLogStateName = "Orleans.EventSourcing.JournaledState.EventLog"; + private const string WriteVectorStateName = "Orleans.EventSourcing.JournaledState.WriteVector"; + + private readonly IGrainBase _grain; + private readonly IJournaledStateManager _stateManager; + private readonly IDurableList _eventLog; + private readonly IDurableValue _writeVector; + + private Task _initializationTask; + private TLogView _confirmedView; + private int _confirmedVersion; + private Exception _terminalFailure; + + public LogViewAdaptor( + ILogViewAdaptorHost host, + TLogView initialState, + ILogConsistencyProtocolServices services) + : base(host, initialState, services) + { + if (host is not IGrainBase grain) + { + throw new BadProviderConfigException("The JournaledState log-consistency provider can only be used by grain classes."); + } + + var grainType = grain.GetType(); + var allowsInterleaving = grainType.IsDefined(typeof(ReentrantAttribute), inherit: true) + || grainType.IsDefined(typeof(MayInterleaveAttribute), inherit: true) + || grainType.IsDefined(typeof(StatelessWorkerAttribute), inherit: true) + || grainType.GetInterfaces() + .SelectMany(static interfaceType => interfaceType.GetMethods()) + .Any(static method => method.IsDefined(typeof(AlwaysInterleaveAttribute), inherit: true)); + if (allowsInterleaving) + { + throw new BadProviderConfigException("The JournaledState log-consistency provider requires a single, turn-serialized grain activation."); + } + + var serviceProvider = grain.GrainContext.ActivationServices; + _grain = grain; + _stateManager = serviceProvider.GetRequiredService(); + _eventLog = serviceProvider.GetRequiredKeyedService>(EventLogStateName); + _writeVector = serviceProvider.GetRequiredKeyedService>(WriteVectorStateName); + } + + /// + public override async Task PreOnActivate() + { + await EnsureInitializedAsync(); + await base.PreOnActivate(); + } + + /// + public override Task> RetrieveLogSegment(int fromVersion, int toVersion) + { + if (fromVersion < 0 || toVersion < fromVersion || toVersion > _confirmedVersion) + { + throw new ArgumentException("Invalid log segment range."); + } + + var length = toVersion - fromVersion; + if (length == 0) + { + return Task.FromResult>(Array.Empty()); + } + + var result = new TLogEntry[length]; + for (var index = 0; index < length; index++) + { + result[index] = _eventLog[fromVersion + index]; + } + + return Task.FromResult>(result); + } + + /// + protected override TLogView LastConfirmedView() => _confirmedView; + + /// + protected override int GetConfirmedVersion() => _confirmedVersion; + + /// + protected override void InitializeConfirmedView(TLogView initialState) + { + _confirmedView = initialState; + _confirmedVersion = 0; + } + + /// + protected override async Task ClearPrimaryLogAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + ThrowIfTerminated(); + + await EnsureInitializedAsync(); + _eventLog.Clear(); + var writeBit = FlipWriteVectorBit(); + await PersistStateAsync(writeBit, cancellationToken, "clear"); + } + + /// + protected override SubmissionEntry MakeSubmissionEntry(TLogEntry entry) + { + return new SubmissionEntry { Entry = entry }; + } + + /// + protected override async Task ReadAsync() + { + ThrowIfTerminated(); + EnterOperation("ReadAsync"); + + while (true) + { + try + { + if (_stateManager.HasPendingWrites) + { + var writeBit = FlipWriteVectorBit(); + await PersistStateAsync(writeBit, CancellationToken.None, "refresh flush"); + } + + await _stateManager.RevertPendingChangesAsync(CancellationToken.None); + UpdateConfirmedViewFromJournal(rebuild: true); + + Services.Log(LogLevel.Debug, "read success v{0}", _confirmedVersion); + + LastPrimaryIssue.Resolve(Host, Services); + break; + } + catch (Exception exception) + { + ThrowIfTerminated(); + LastPrimaryIssue.Record(new ReadFromJournaledStateFailed { Exception = exception }, Host, Services); + } + + Services.Log(LogLevel.Debug, "read failed {0}", LastPrimaryIssue); + + await LastPrimaryIssue.DelayBeforeRetry(); + } + + ExitOperation("ReadAsync"); + } + + /// + protected override async Task WriteAsync() + { + ThrowIfTerminated(); + EnterOperation("WriteAsync"); + + var updates = GetCurrentBatchOfUpdates(); + if (updates.Length == 0) + { + ExitOperation("WriteAsync"); + return 0; + } + + bool writeBit; + try + { + writeBit = FlipWriteVectorBit(); + foreach (var update in updates) + { + _eventLog.Add(update.Entry); + } + } + catch (Exception exception) + { + await TerminallyFailAfterStagingErrorAsync(exception); + throw; + } + + await PersistStateAsync(writeBit, CancellationToken.None, "write"); + + Services.Log(LogLevel.Debug, "write ({0} updates) success v{1}", updates.Length, _eventLog.Count); + + UpdateConfirmedViewFromJournal(); + LastPrimaryIssue.Resolve(Host, Services); + + ExitOperation("WriteAsync"); + return updates.Length; + } + + private async Task EnsureInitializedAsync() + { + if (_initializationTask is null || _initializationTask.IsCanceled || _initializationTask.IsFaulted) + { + _initializationTask = _stateManager.InitializeAsync(CancellationToken.None).AsTask(); + } + + await _initializationTask; + } + + private async Task PersistStateAsync(bool writeBit, CancellationToken cancellationToken, string operation) + { + while (true) + { + try + { + await _stateManager.WriteStateAsync(cancellationToken); + LastPrimaryIssue.Resolve(Host, Services); + return; + } + catch (OperationCanceledException exception) when (cancellationToken.IsCancellationRequested) + { + _terminalFailure = exception; + _grain.DeactivateOnIdle(); + throw; + } + catch (Exception exception) + { + LastPrimaryIssue.Record(new UpdateJournaledStateFailed { Exception = exception }, Host, Services); + Services.Log(LogLevel.Debug, "{0} failed {1}", operation, LastPrimaryIssue); + + if (cancellationToken.IsCancellationRequested) + { + _terminalFailure = new OperationCanceledException(cancellationToken); + _grain.DeactivateOnIdle(); + cancellationToken.ThrowIfCancellationRequested(); + } + + await LastPrimaryIssue.DelayBeforeRetry(); + if (cancellationToken.IsCancellationRequested) + { + _terminalFailure = new OperationCanceledException(cancellationToken); + _grain.DeactivateOnIdle(); + cancellationToken.ThrowIfCancellationRequested(); + } + + await RecoverAfterWriteFailureAsync(); + if (writeBit == GetWriteVectorBit()) + { + Services.Log(LogLevel.Debug, "last {0} was actually a success v{1}", operation, _eventLog.Count); + LastPrimaryIssue.Resolve(Host, Services); + return; + } + + _terminalFailure = exception; + _grain.DeactivateOnIdle(); + throw; + } + } + } + + private async Task RecoverAfterWriteFailureAsync() + { + while (true) + { + try + { + await _stateManager.RevertPendingChangesAsync(CancellationToken.None); + UpdateConfirmedViewFromJournal(rebuild: true); + return; + } + catch (Exception exception) + { + LastPrimaryIssue.Record(new ReadFromJournaledStateFailed { Exception = exception }, Host, Services); + } + + Services.Log(LogLevel.Debug, "read failed {0}", LastPrimaryIssue); + + await LastPrimaryIssue.DelayBeforeRetry(); + } + } + + private async Task TerminallyFailAfterStagingErrorAsync(Exception exception) + { + try + { + await RecoverAfterWriteFailureAsync(); + } + finally + { + _terminalFailure = exception; + _grain.DeactivateOnIdle(); + } + } + + private void ThrowIfTerminated() + { + if (_terminalFailure is { } exception) + { + ExceptionDispatchInfo.Capture(exception).Throw(); + } + } + + private void UpdateConfirmedViewFromJournal(bool rebuild = false) + { + if (rebuild || _eventLog.Count < _confirmedVersion) + { + InitializeConfirmedView(InitialState); + } + + for (var index = _confirmedVersion; index < _eventLog.Count; index++) + { + try + { + Host.UpdateView(_confirmedView, _eventLog[index]); + } + catch (Exception exception) + { + Services.CaughtUserCodeException("UpdateView", nameof(UpdateConfirmedViewFromJournal), exception); + } + } + + _confirmedVersion = _eventLog.Count; + } + + private bool FlipWriteVectorBit() + { + var value = _writeVector.Value ?? string.Empty; + var result = StringEncodedWriteVector.FlipBit(ref value, Services.MyClusterId); + _writeVector.Value = value; + return result; + } + + private bool GetWriteVectorBit() => StringEncodedWriteVector.GetBit(_writeVector.Value ?? string.Empty, Services.MyClusterId); + + [Serializable] + [GenerateSerializer] + public sealed class UpdateJournaledStateFailed : PrimaryOperationFailed + { + /// + public override string ToString() + { + return $"write event log to journaled state failed: caught {Exception.GetType().Name}: {Exception.Message}"; + } + } + + [Serializable] + [GenerateSerializer] + public sealed class ReadFromJournaledStateFailed : PrimaryOperationFailed + { + /// + public override string ToString() + { + return $"read event log from journaled state failed: caught {Exception.GetType().Name}: {Exception.Message}"; + } + } + +#if DEBUG + private bool _operationInProgress; +#endif + + [System.Diagnostics.Conditional("DEBUG")] + private void EnterOperation(string name) + { +#if DEBUG + Services.Log(LogLevel.Trace, "/-- enter {0}", name); + System.Diagnostics.Debug.Assert(!_operationInProgress); + _operationInProgress = true; +#endif + } + + [System.Diagnostics.Conditional("DEBUG")] + private void ExitOperation(string name) + { +#if DEBUG + Services.Log(LogLevel.Trace, "\\-- exit {0}", name); + System.Diagnostics.Debug.Assert(_operationInProgress); + _operationInProgress = false; +#endif + } +} + +#pragma warning restore ORLEANSEXP005 diff --git a/src/Orleans.EventSourcing/Orleans.EventSourcing.csproj b/src/Orleans.EventSourcing/Orleans.EventSourcing.csproj index ecb5461dbe6..0b68d046089 100644 --- a/src/Orleans.EventSourcing/Orleans.EventSourcing.csproj +++ b/src/Orleans.EventSourcing/Orleans.EventSourcing.csproj @@ -12,6 +12,7 @@ + diff --git a/src/Orleans.Journaling/DurableDictionary.cs b/src/Orleans.Journaling/DurableDictionary.cs index 11a1a8325ec..a756730e96e 100644 --- a/src/Orleans.Journaling/DurableDictionary.cs +++ b/src/Orleans.Journaling/DurableDictionary.cs @@ -59,6 +59,8 @@ public V this[K key] public bool IsReadOnly => ((ICollection>)_items).IsReadOnly; + bool IJournaledState.HasPendingChanges => false; + void IJournaledState.ReplayEntry(JournalEntry entry, JournalReplayContext context) => context.GetRequiredCommandCodec(entry.FormatKey, _codec).Apply(entry.Reader, this); diff --git a/src/Orleans.Journaling/DurableList.cs b/src/Orleans.Journaling/DurableList.cs index ffff4cfe8ed..6d7c580a97b 100644 --- a/src/Orleans.Journaling/DurableList.cs +++ b/src/Orleans.Journaling/DurableList.cs @@ -58,6 +58,8 @@ public T this[int index] bool ICollection.IsReadOnly => false; + bool IJournaledState.HasPendingChanges => false; + void IJournaledState.ReplayEntry(JournalEntry entry, JournalReplayContext context) => context.GetRequiredCommandCodec(entry.FormatKey, _codec).Apply(entry.Reader, this); diff --git a/src/Orleans.Journaling/DurableNothing.cs b/src/Orleans.Journaling/DurableNothing.cs index 76333a3271e..9d0b53318f5 100644 --- a/src/Orleans.Journaling/DurableNothing.cs +++ b/src/Orleans.Journaling/DurableNothing.cs @@ -14,6 +14,8 @@ public interface IDurableNothing /// internal sealed class DurableNothing : IDurableNothing, IJournaledState { + bool IJournaledState.HasPendingChanges => false; + public DurableNothing([ServiceKey] string key, IJournaledStateManager manager) { ArgumentNullException.ThrowIfNullOrEmpty(key); diff --git a/src/Orleans.Journaling/DurableQueue.cs b/src/Orleans.Journaling/DurableQueue.cs index 50468f0f0c8..4f1a676c7e8 100644 --- a/src/Orleans.Journaling/DurableQueue.cs +++ b/src/Orleans.Journaling/DurableQueue.cs @@ -45,6 +45,8 @@ internal DurableQueue(string key, IJournaledStateManager manager, IDurableQueueC public int Count => _items.Count; + bool IJournaledState.HasPendingChanges => false; + void IJournaledState.ReplayEntry(JournalEntry entry, JournalReplayContext context) => context.GetRequiredCommandCodec(entry.FormatKey, _codec).Apply(entry.Reader, this); diff --git a/src/Orleans.Journaling/DurableSet.cs b/src/Orleans.Journaling/DurableSet.cs index 243db397356..0861a1d75db 100644 --- a/src/Orleans.Journaling/DurableSet.cs +++ b/src/Orleans.Journaling/DurableSet.cs @@ -46,6 +46,8 @@ internal DurableSet(string key, IJournaledStateManager manager, IDurableSetComma public int Count => _items.Count; public bool IsReadOnly => false; + bool IJournaledState.HasPendingChanges => false; + void IJournaledState.ReplayEntry(JournalEntry entry, JournalReplayContext context) => context.GetRequiredCommandCodec(entry.FormatKey, _codec).Apply(entry.Reader, this); diff --git a/src/Orleans.Journaling/DurableState.cs b/src/Orleans.Journaling/DurableState.cs index 97f5abbeba7..09fcab326cd 100644 --- a/src/Orleans.Journaling/DurableState.cs +++ b/src/Orleans.Journaling/DurableState.cs @@ -15,6 +15,9 @@ internal sealed class DurableState : IPersistentState, IJournaledState, IP private PendingWriteKind _pendingWrite; private bool _hasState; private bool _clearRequested; + private bool _isDirty; + private ulong _changeVersion; + private ulong _stagedChangeVersion; public DurableState( [ServiceKey] string key, @@ -42,6 +45,8 @@ T IStorage.State get { _hasState = true; + _isDirty = true; + _changeVersion++; return _value ??= Activator.CreateInstance(); } set @@ -49,12 +54,16 @@ T IStorage.State _value = value; _hasState = true; _clearRequested = false; + _isDirty = true; + _changeVersion++; } } string IStorage.Etag => $"{_version}"; bool IStorage.RecordExists => _version > 0; + bool IJournaledState.HasPendingChanges => _clearRequested || _isDirty; + void IJournaledState.ReplayEntry(JournalEntry entry, JournalReplayContext context) => context.GetRequiredCommandCodec(entry.FormatKey, _codec).Apply(entry.Reader, this); @@ -64,18 +73,29 @@ void IJournaledState.OnWriteCompleted() { case PendingWriteKind.Set: _version = _pendingVersion; - _clearRequested = false; - _hasState = true; + if (_stagedChangeVersion == _changeVersion) + { + _clearRequested = false; + _hasState = true; + } break; case PendingWriteKind.Clear: _version = 0; - _clearRequested = false; - _hasState = false; + if (_stagedChangeVersion == _changeVersion) + { + _clearRequested = false; + _hasState = false; + } break; } _pendingWrite = PendingWriteKind.None; _pendingVersion = 0; + if (_stagedChangeVersion == _changeVersion) + { + _isDirty = false; + } + OnPersisted?.Invoke(); } @@ -87,6 +107,9 @@ void IJournaledState.Reset(JournalStreamWriter writer) _pendingWrite = PendingWriteKind.None; _hasState = false; _clearRequested = false; + _isDirty = false; + _changeVersion = 0; + _stagedChangeVersion = 0; } void IJournaledState.AppendEntries(JournalStreamWriter writer) @@ -94,10 +117,15 @@ void IJournaledState.AppendEntries(JournalStreamWriter writer) if (_clearRequested) { WriteClear(writer); + _stagedChangeVersion = _changeVersion; + _clearRequested = false; + _isDirty = false; } - else if (_hasState) + else if (_hasState && _isDirty) { WriteState(writer); + _stagedChangeVersion = _changeVersion; + _isDirty = false; } } @@ -107,10 +135,12 @@ void IJournaledState.AppendSnapshot(JournalStreamWriter snapshotWriter) { _pendingWrite = PendingWriteKind.Clear; _pendingVersion = 0; + _stagedChangeVersion = _changeVersion; } else if (_hasState) { WriteState(snapshotWriter); + _stagedChangeVersion = _changeVersion; } } @@ -137,6 +167,9 @@ void IPersistentStateCommandHandler.ApplySet(T state, ulong version) _version = version; _hasState = true; _clearRequested = false; + _isDirty = false; + _changeVersion = 0; + _stagedChangeVersion = 0; } void IPersistentStateCommandHandler.ApplyClear() @@ -145,6 +178,9 @@ void IPersistentStateCommandHandler.ApplyClear() _version = 0; _hasState = false; _clearRequested = false; + _isDirty = false; + _changeVersion = 0; + _stagedChangeVersion = 0; } Task IStorage.ClearStateAsync() => ((IStorage)this).ClearStateAsync(CancellationToken.None); @@ -153,6 +189,8 @@ async Task IStorage.ClearStateAsync(CancellationToken cancellationToken) _value = default; _hasState = false; _clearRequested = true; + _isDirty = true; + _changeVersion++; await _manager.WriteStateAsync(cancellationToken); } diff --git a/src/Orleans.Journaling/DurableTaskCompletionSource.cs b/src/Orleans.Journaling/DurableTaskCompletionSource.cs index bf712889e4e..7096e38f33c 100644 --- a/src/Orleans.Journaling/DurableTaskCompletionSource.cs +++ b/src/Orleans.Journaling/DurableTaskCompletionSource.cs @@ -25,6 +25,9 @@ internal sealed class DurableTaskCompletionSource : IDurableTaskCompletionSou private DurableTaskCompletionSourceStatus _status; private T? _value; private Exception? _exception; + private bool _isDirty; + private ulong _changeVersion; + private ulong _stagedChangeVersion; public DurableTaskCompletionSource( [ServiceKey] string key, @@ -64,6 +67,8 @@ public bool TrySetResult(T value) _status = DurableTaskCompletionSourceStatus.Completed; _value = _copier.Copy(value); + _isDirty = true; + _changeVersion++; return true; } @@ -76,6 +81,8 @@ public bool TrySetException(Exception exception) _status = DurableTaskCompletionSourceStatus.Faulted; _exception = _exceptionCopier.Copy(exception); + _isDirty = true; + _changeVersion++; return true; } @@ -87,6 +94,8 @@ public bool TrySetCanceled() } _status = DurableTaskCompletionSourceStatus.Canceled; + _isDirty = true; + _changeVersion++; return true; } @@ -101,6 +110,8 @@ public bool TrySetCanceled() _ => throw new InvalidOperationException($"Unexpected status, \"{_status}\""), }; + bool IJournaledState.HasPendingChanges => _isDirty; + void IJournaledState.ReplayEntry(JournalEntry entry, JournalReplayContext context) => context.GetRequiredCommandCodec(entry.FormatKey, _codec).Apply(entry.Reader, this); @@ -122,14 +133,31 @@ private void OnValuePersisted() } } - void IJournaledState.OnRecoveryCompleted() => OnValuePersisted(); - void IJournaledState.OnWriteCompleted() => OnValuePersisted(); + void IJournaledState.OnRecoveryCompleted() + { + _isDirty = false; + _changeVersion = 0; + _stagedChangeVersion = 0; + OnValuePersisted(); + } + + void IJournaledState.OnWriteCompleted() + { + if (_stagedChangeVersion == _changeVersion) + { + _isDirty = false; + OnValuePersisted(); + } + } void IJournaledState.Reset(JournalStreamWriter writer) { _status = DurableTaskCompletionSourceStatus.Pending; _value = default; _exception = null; + _isDirty = false; + _changeVersion = 0; + _stagedChangeVersion = 0; // Reset the task completion source if necessary. if (_completion.Task.IsCompleted) @@ -140,13 +168,19 @@ void IJournaledState.Reset(JournalStreamWriter writer) void IJournaledState.AppendEntries(JournalStreamWriter writer) { - if (_status is not DurableTaskCompletionSourceStatus.Pending) + if (_isDirty) { WriteState(writer); + _stagedChangeVersion = _changeVersion; + _isDirty = false; } } - void IJournaledState.AppendSnapshot(JournalStreamWriter snapshotWriter) => WriteState(snapshotWriter); + void IJournaledState.AppendSnapshot(JournalStreamWriter snapshotWriter) + { + WriteState(snapshotWriter); + _stagedChangeVersion = _changeVersion; + } private void WriteState(JournalStreamWriter writer) { @@ -167,20 +201,38 @@ private void WriteState(JournalStreamWriter writer) } } - void IDurableTaskCompletionSourceCommandHandler.ApplyPending() => _status = DurableTaskCompletionSourceStatus.Pending; + void IDurableTaskCompletionSourceCommandHandler.ApplyPending() + { + _status = DurableTaskCompletionSourceStatus.Pending; + _isDirty = false; + _changeVersion = 0; + _stagedChangeVersion = 0; + } void IDurableTaskCompletionSourceCommandHandler.ApplyCompleted(T value) { _status = DurableTaskCompletionSourceStatus.Completed; _value = value; + _isDirty = false; + _changeVersion = 0; + _stagedChangeVersion = 0; } void IDurableTaskCompletionSourceCommandHandler.ApplyFaulted(Exception exception) { _status = DurableTaskCompletionSourceStatus.Faulted; _exception = exception; + _isDirty = false; + _changeVersion = 0; + _stagedChangeVersion = 0; } - void IDurableTaskCompletionSourceCommandHandler.ApplyCanceled() => _status = DurableTaskCompletionSourceStatus.Canceled; + void IDurableTaskCompletionSourceCommandHandler.ApplyCanceled() + { + _status = DurableTaskCompletionSourceStatus.Canceled; + _isDirty = false; + _changeVersion = 0; + _stagedChangeVersion = 0; + } public IJournaledState DeepCopy() => throw new NotImplementedException(); } @@ -206,4 +258,3 @@ public readonly struct DurableTaskCompletionSourceState [Id(2)] public Exception? Exception { get; init; } } - diff --git a/src/Orleans.Journaling/DurableValue.cs b/src/Orleans.Journaling/DurableValue.cs index 5495c407a94..32b5973f10c 100644 --- a/src/Orleans.Journaling/DurableValue.cs +++ b/src/Orleans.Journaling/DurableValue.cs @@ -14,6 +14,8 @@ internal sealed class DurableValue : IDurableValue, IJournaledState, IDura private readonly IDurableValueCommandCodec _codec; private T? _value; private bool _isDirty; + private ulong _changeVersion; + private ulong _stagedChangeVersion; public DurableValue( [ServiceKey] string key, @@ -45,20 +47,43 @@ public T? Value public Action? OnPersisted { get; set; } + bool IJournaledState.HasPendingChanges => _isDirty; + private void OnValuePersisted() => OnPersisted?.Invoke(); - public void OnModified() => _isDirty = true; + public void OnModified() + { + _isDirty = true; + _changeVersion++; + } void IJournaledState.ReplayEntry(JournalEntry entry, JournalReplayContext context) => context.GetRequiredCommandCodec(entry.FormatKey, _codec).Apply(entry.Reader, this); - void IJournaledState.OnRecoveryCompleted() => OnValuePersisted(); - void IJournaledState.OnWriteCompleted() => OnValuePersisted(); + void IJournaledState.OnRecoveryCompleted() + { + _isDirty = false; + _changeVersion = 0; + _stagedChangeVersion = 0; + OnValuePersisted(); + } + + void IJournaledState.OnWriteCompleted() + { + if (_stagedChangeVersion == _changeVersion) + { + _isDirty = false; + } + + OnValuePersisted(); + } void IJournaledState.Reset(JournalStreamWriter writer) { _value = default; _isDirty = false; + _changeVersion = 0; + _stagedChangeVersion = 0; } void IJournaledState.AppendEntries(JournalStreamWriter writer) @@ -66,11 +91,16 @@ void IJournaledState.AppendEntries(JournalStreamWriter writer) if (_isDirty) { WriteState(writer); + _stagedChangeVersion = _changeVersion; _isDirty = false; } } - void IJournaledState.AppendSnapshot(JournalStreamWriter snapshotWriter) => WriteState(snapshotWriter); + void IJournaledState.AppendSnapshot(JournalStreamWriter snapshotWriter) + { + WriteState(snapshotWriter); + _stagedChangeVersion = _changeVersion; + } public IJournaledState DeepCopy() => throw new NotImplementedException(); @@ -79,5 +109,11 @@ private void WriteState(JournalStreamWriter writer) _codec.WriteSet(_value!, writer); } - void IDurableValueCommandHandler.ApplySet(T value) => _value = value; + void IDurableValueCommandHandler.ApplySet(T value) + { + _value = value; + _isDirty = false; + _changeVersion = 0; + _stagedChangeVersion = 0; + } } diff --git a/src/Orleans.Journaling/IJournaledState.cs b/src/Orleans.Journaling/IJournaledState.cs index 79d9230d0a4..6145f3e16d6 100644 --- a/src/Orleans.Journaling/IJournaledState.cs +++ b/src/Orleans.Journaling/IJournaledState.cs @@ -39,6 +39,15 @@ namespace Orleans.Journaling; /// public interface IJournaledState { + /// + /// Gets a value indicating whether this state has changes which have not been materialized in the journal buffer. + /// + /// + /// Implementations which stage mutations outside the shared journal buffer must override this property. + /// States which write directly to their assigned are tracked by the buffer itself. + /// + bool HasPendingChanges => false; + /// /// Replays one entry during journal recovery. /// diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index 2861d6e3deb..f87427ec643 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -13,6 +13,10 @@ public interface IJournaledStateManager : IAsyncDisposable /// /// Initializes the state manager. /// + /// + /// Cancellation is observed before initialization is queued. Once recovery begins, the operation completes + /// before returning so callers never observe state while recovery is still mutating it. + /// /// The cancellation token. /// A which represents the operation. ValueTask InitializeAsync(CancellationToken cancellationToken); @@ -34,6 +38,12 @@ public interface IJournaledStateManager : IAsyncDisposable /// /// Prepares and persists an update to the journal. /// + /// + /// When the operation writes a snapshot, the complete captured state is replaced atomically. If storage + /// fails without reporting an optimistic-concurrency conflict, the captured changes remain pending and can be retried. + /// Changes made while storage is awaiting are not consumed by the completed operation. + /// An optimistic-concurrency conflict recovers the winning journal generation and discards the losing in-memory changes. + /// /// The cancellation token. /// A which represents the operation. ValueTask WriteStateAsync(CancellationToken cancellationToken); @@ -48,6 +58,9 @@ public interface IJournaledStateManager : IAsyncDisposable /// /// Resets this instance, removing any persistent state. /// + /// + /// Cancellation is observed before deletion is queued. Once deletion begins, the operation completes before returning. + /// /// The cancellation token. /// A which represents the operation. ValueTask DeleteStateAsync(CancellationToken cancellationToken); @@ -62,4 +75,9 @@ public interface IJournaledStateManager : IAsyncDisposable /// concurrent writers and should not be used for correctness decisions. /// long PendingWriteByteCount => -1; + + /// + /// Gets a value indicating whether any registered state has changes which have not been written to storage. + /// + bool HasPendingWrites => PendingWriteByteCount != 0; } diff --git a/src/Orleans.Journaling/JournalBufferWriter.cs b/src/Orleans.Journaling/JournalBufferWriter.cs index 60a745e0d98..fbd96eaf84e 100644 --- a/src/Orleans.Journaling/JournalBufferWriter.cs +++ b/src/Orleans.Journaling/JournalBufferWriter.cs @@ -208,6 +208,18 @@ internal int CommittedLength } } + internal int BufferedLength + { + get + { + lock (_lock) + { + ThrowIfDisposed(); + return _buffer.Length; + } + } + } + internal void Consume(ArcBuffer buffer) { lock (_lock) diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 658e3594695..a9311269f1f 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -401,6 +401,9 @@ private async Task WorkLoop() { if (isSnapshot) { + // The snapshot includes all state represented by the captured journal prefix. + // Replace it atomically, then consume only that prefix so commands added while + // storage is awaiting remain pending for the next write. await ReplaceStorageAsync(writeSequence, _shutdownCancellation.Token).ConfigureAwait(true); } else @@ -711,7 +714,7 @@ public async ValueTask DeleteStateAsync(CancellationToken cancellationToken) var startTimestamp = _shared.TimeProvider.GetTimestamp(); try { - await task.WaitAsync(cancellationToken); + await task; _shared.Instruments.OnStateDeleteRequest(_shared.TimeProvider.GetElapsedTime(startTimestamp), succeeded: true); } catch @@ -744,6 +747,11 @@ private async Task RecoverAsync(CancellationToken cancellationToken) { foreach ((var name, var state) in _states) { + if (state is not RetiredState && !_journalStreamDirectory.ContainsKey(name)) + { + _journalStreamDirectory.Set(name, _journalStreamDirectory.GetNextJournalStreamId()); + } + state.OnRecoveryCompleted(); if (state is RetiredState) @@ -773,6 +781,13 @@ private void ResetForRecovery() { (retiredNames ??= []).Add(name); } + else + { + var streamId = _journalStreamDirectory.TryGetValue(name, out var id) + ? new JournalStreamId(id) + : new JournalStreamId(MinApplicationJournalStreamId); + state.Reset(CreateJournalStreamWriter(streamId)); + } } if (retiredNames is not null) @@ -1051,7 +1066,31 @@ private void BindState(string name, uint id) public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState? state) => _states.TryGetValue(name, out state); - public long PendingWriteByteCount => _journalWriter.CommittedLength; + public long PendingWriteByteCount => _journalWriter.BufferedLength; + + public bool HasPendingWrites + { + get + { + lock (_lock) + { + if (_journalWriter.BufferedLength > 0) + { + return true; + } + + foreach (var state in _states.Values) + { + if (state.HasPendingChanges) + { + return true; + } + } + + return false; + } + } + } void ILifecycleParticipant.Participate(IGrainLifecycle observer) => observer.Subscribe(GrainLifecycleStage.SetupState, this); Task ILifecycleObserver.OnStart(CancellationToken cancellationToken) => InitializeAsync(cancellationToken).AsTask(); @@ -1229,6 +1268,8 @@ private sealed class StateDirectory( public uint this[string name] => _ids[name]; + bool IJournaledState.HasPendingChanges => false; + void IJournaledState.ReplayEntry(JournalEntry entry, JournalReplayContext context) => context.GetRequiredCommandCodec(entry.FormatKey, _codec).Apply(entry.Reader, this); @@ -1327,6 +1368,8 @@ private sealed class RetiredState(JournalStreamId streamId) : IJournaledState public IReadOnlyList PreservedEntries => _preservedEntries; + bool IJournaledState.HasPendingChanges => false; + void IJournaledState.ReplayEntry(JournalEntry entry, JournalReplayContext context) => _preservedEntries.Add(new PreservedJournalEntry(entry.FormatKey, entry.Reader)); diff --git a/src/api/Orleans.EventSourcing/Orleans.EventSourcing.cs b/src/api/Orleans.EventSourcing/Orleans.EventSourcing.cs index d3ca6a70da2..0088f18f916 100644 --- a/src/api/Orleans.EventSourcing/Orleans.EventSourcing.cs +++ b/src/api/Orleans.EventSourcing/Orleans.EventSourcing.cs @@ -444,6 +444,20 @@ public sealed partial class LogStateWithMetaData } } +namespace Orleans.EventSourcing.JournaledState +{ + public sealed partial class LogConsistencyProvider : ILogViewAdaptorFactory + { + public LogConsistencyProvider() { } + + public bool UsesStorageProvider { get { throw null; } } + + public ILogViewAdaptor MakeLogViewAdaptor(ILogViewAdaptorHost hostGrain, TView initialState, string grainTypeName, Storage.IGrainStorage grainStorage, ILogConsistencyProtocolServices services) + where TView : class, new() + where TEntry : class { throw null; } + } +} + namespace Orleans.EventSourcing.StateStorage { [GenerateSerializer] @@ -517,6 +531,13 @@ public static partial class LogStorageSiloBuilderExtensions public static ISiloBuilder AddLogStorageBasedLogConsistencyProviderAsDefault(this ISiloBuilder builder) { throw null; } } + public static partial class JournaledStateSiloBuilderExtensions + { + public static ISiloBuilder AddJournaledStateBasedLogConsistencyProvider(this ISiloBuilder builder, string name = "JournaledState") { throw null; } + + public static ISiloBuilder AddJournaledStateBasedLogConsistencyProviderAsDefault(this ISiloBuilder builder) { throw null; } + } + public static partial class StateStorageSiloBuilderExtensions { public static ISiloBuilder AddStateStorageBasedLogConsistencyProvider(this ISiloBuilder builder, string name = "StateStorage") { throw null; } @@ -916,4 +937,4 @@ public Copier_GrainStateWithMetaData(global::Orleans.Serialization.Serializers.I public global::Orleans.EventSourcing.StateStorage.GrainStateWithMetaData DeepCopy(global::Orleans.EventSourcing.StateStorage.GrainStateWithMetaData original, global::Orleans.Serialization.Cloning.CopyContext context) { throw null; } } -} \ No newline at end of file +} diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index e7fcd7aa32d..43bddd98baa 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -210,6 +210,8 @@ public partial interface IDurableValue public partial interface IJournaledState { + bool HasPendingChanges { get; } + void AppendEntries(JournalStreamWriter writer); void AppendSnapshot(JournalStreamWriter writer); IJournaledState DeepCopy(); @@ -221,6 +223,8 @@ public partial interface IJournaledState public partial interface IJournaledStateManager : System.IAsyncDisposable { + bool HasPendingWrites { get; } + long PendingWriteByteCount { get; } System.Threading.Tasks.ValueTask DeleteStateAsync(System.Threading.CancellationToken cancellationToken); diff --git a/test/Grains/TestGrainInterfaces/ILogTestGrain.cs b/test/Grains/TestGrainInterfaces/ILogTestGrain.cs index 85942fbc013..9f403474b15 100644 --- a/test/Grains/TestGrainInterfaces/ILogTestGrain.cs +++ b/test/Grains/TestGrainInterfaces/ILogTestGrain.cs @@ -1,83 +1,96 @@ -namespace UnitTests.GrainInterfaces +namespace UnitTests.GrainInterfaces; + +/// +/// A grain used for testing log-consistency providers. +/// The content of this class is pretty arbitrary and messy; +/// (don't use this as an introduction on how to use JournaledGrain) +/// it started from SimpleGrain, but a lot of stuff got added over time +/// +public interface ILogTestGrain: IGrainWithIntegerKey { - /// - /// A grain used for testing log-consistency providers. - /// The content of this class is pretty arbitrary and messy; - /// (don't use this as an introduction on how to use JournaledGrain) - /// it started from SimpleGrain, but a lot of stuff got added over time - /// - public interface ILogTestGrain: IGrainWithIntegerKey - { - // read A + // read A - Task GetAGlobal(); + Task GetAGlobal(); - Task GetALocal(); + Task GetALocal(); - // read both + // read both - Task GetBothGlobal(); + Task GetBothGlobal(); - Task GetBothLocal(); + Task GetBothLocal(); - // reservations + // reservations - Task GetReservationsGlobal(); + Task GetReservationsGlobal(); - // version + // version - Task GetConfirmedVersion(); + Task GetConfirmedVersion(); - // set or increment A + // set or increment A - Task SetAGlobal(int a); + Task SetAGlobal(int a); - Task> SetAConditional(int a); + Task> SetAConditional(int a); - Task SetALocal(int a); + Task SetALocal(int a); - Task IncrementALocal(); + Task IncrementALocal(); - Task IncrementAGlobal(); + Task IncrementAGlobal(); - // set B + // set B - Task SetBGlobal(int b); + Task SetBGlobal(int b); - Task SetBLocal(int b); + Task SetBLocal(int b); - // reservations + // reservations - Task AddReservationLocal(int x); + Task AddReservationLocal(int x); - Task RemoveReservationLocal(int x); + Task RemoveReservationLocal(int x); - Task> Read(); - Task Update(IReadOnlyList updates, int expectedversion); + Task> Read(); + Task Update(IReadOnlyList updates, int expectedversion); - Task> GetEventLog(); + Task> GetEventLog(); + Task> GetEventLogSegment(int fromVersion, int toVersion); - // other operations - Task SynchronizeGlobalState(); - Task Clear(); - Task Deactivate(); - } + // other operations - /// - /// Used by unit tests. - /// The fields don't really have any meaning. - /// The point of the struct is just that a grain method can return both A and B at the same time. - /// - [GenerateSerializer] - public struct AB - { - [Id(0)] - public int A; + Task SynchronizeGlobalState(); + Task RaiseEventsWithUnsupportedSecond(); + Task Clear(); + Task Deactivate(); +} + +public interface ILogTestGrainWithAuxiliaryState : ILogTestGrain +{ + Task GetAuxiliaryValue(); + + Task SetAuxiliaryValue(int value); + + Task SetAuxiliaryValueAndAGlobal(int auxiliaryValue, int value); + + Task SetAuxiliaryValueAndSynchronize(int value); +} + +/// +/// Used by unit tests. +/// The fields don't really have any meaning. +/// The point of the struct is just that a grain method can return both A and B at the same time. +/// +[GenerateSerializer] +public struct AB +{ + [Id(0)] + public int A; - [Id(1)] - public int B; - } + [Id(1)] + public int B; } diff --git a/test/Grains/TestGrains/LogTestGrain.cs b/test/Grains/TestGrains/LogTestGrain.cs index 6f380e9735c..9393539712b 100644 --- a/test/Grains/TestGrains/LogTestGrain.cs +++ b/test/Grains/TestGrains/LogTestGrain.cs @@ -160,6 +160,13 @@ public Task SynchronizeGlobalState() return RefreshNow(); } + public async Task RaiseEventsWithUnsupportedSecond() + { + RaiseEvent(new UpdateA { Val = 41 }); + RaiseEvent(new UnsupportedLogEvent()); + await ConfirmEvents(); + } + public Task GetConfirmedVersion() { return Task.FromResult(this.Version); @@ -194,5 +201,12 @@ public Task> GetEventLog() { return this.RetrieveConfirmedEvents(0, Version); } + public Task> GetEventLogSegment(int fromVersion, int toVersion) + { + return RetrieveConfirmedEvents(fromVersion, toVersion); + } + } + + internal sealed class UnsupportedLogEvent; } diff --git a/test/Grains/TestGrains/LogTestGrainVariations.cs b/test/Grains/TestGrains/LogTestGrainVariations.cs index fd47761763d..13e405b3d52 100644 --- a/test/Grains/TestGrains/LogTestGrainVariations.cs +++ b/test/Grains/TestGrains/LogTestGrainVariations.cs @@ -1,136 +1,180 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Concurrency; +using Orleans.Journaling; using Orleans.Providers; using Orleans.Serialization; using UnitTests.GrainInterfaces; -namespace TestGrains +#pragma warning disable ORLEANSEXP005 +namespace TestGrains; + +// variations of the log consistent grain are used to test a variety of provider and configurations + +// use azure storage and a explicitly configured consistency provider +[StorageProvider(ProviderName = "AzureStore")] +[LogConsistencyProvider(ProviderName = "StateStorage")] +public class LogTestGrainSharedStateStorage : LogTestGrain +{ +} + +// use azure storage and a explicitly configured consistency provider +[StorageProvider(ProviderName = "AzureStore")] +[LogConsistencyProvider(ProviderName = "LogStorage")] +public class LogTestGrainSharedLogStorage : LogTestGrain +{ +} + +[LogConsistencyProvider(ProviderName = "JournaledState")] +public class LogTestGrainJournaledStateStorage : LogTestGrain { - // variations of the log consistent grain are used to test a variety of provider and configurations +} - // use azure storage and a explicitly configured consistency provider - [StorageProvider(ProviderName = "AzureStore")] - [LogConsistencyProvider(ProviderName = "StateStorage")] - public class LogTestGrainSharedStateStorage : LogTestGrain +[Reentrant] +[LogConsistencyProvider(ProviderName = "JournaledState")] +public class LogTestGrainJournaledStateReentrantStorage : LogTestGrain +{ +} + +[LogConsistencyProvider(ProviderName = "JournaledState")] +public class LogTestGrainJournaledStateStorageWithAuxiliaryState( + [FromKeyedServices("auxiliary-state")] IDurableValue auxiliaryState) + : LogTestGrain, + ILogTestGrainWithAuxiliaryState +{ + public Task GetAuxiliaryValue() { + return Task.FromResult(auxiliaryState.Value); } - // use azure storage and a explicitly configured consistency provider - [StorageProvider(ProviderName = "AzureStore")] - [LogConsistencyProvider(ProviderName = "LogStorage")] - public class LogTestGrainSharedLogStorage : LogTestGrain + public Task SetAuxiliaryValue(int value) { + auxiliaryState.Value = value; + return Task.CompletedTask; } - // use the default storage provider as the shared storage - public class LogTestGrainDefaultStorage : LogTestGrain + public async Task SetAuxiliaryValueAndAGlobal(int auxiliaryValue, int value) { + auxiliaryState.Value = auxiliaryValue; + await SetAGlobal(value); } - // use MemoryStore (which uses GSI grain) - [StorageProvider(ProviderName = "MemoryStore")] - public class LogTestGrainMemoryStorage : LogTestGrain + public async Task SetAuxiliaryValueAndSynchronize(int value) { + auxiliaryState.Value = value; + await SynchronizeGlobalState(); + return auxiliaryState.Value; } +} - // use the explictly specified "CustomStorage" log-consistency provider with symmetric access from all clusters - [LogConsistencyProvider(ProviderName = "CustomStorage")] - public class LogTestGrainCustomStorage : LogTestGrain, - Orleans.EventSourcing.CustomStorage.ICustomStorageInterface - { +// use the default storage provider as the shared storage +public class LogTestGrainDefaultStorage : LogTestGrain +{ +} - // we use another impl of this grain as the primary. - private ILogTestGrain? storagegrain; +// use MemoryStore (which uses GSI grain) +[StorageProvider(ProviderName = "MemoryStore")] +public class LogTestGrainMemoryStorage : LogTestGrain +{ +} - private ILogTestGrain GetStorageGrain() - { - if (storagegrain == null) - { - storagegrain = GrainFactory.GetGrain(this.GetPrimaryKeyLong(), "TestGrains.LogTestGrainSharedStateStorage"); - } - return storagegrain; - } - +// use the explictly specified "CustomStorage" log-consistency provider with symmetric access from all clusters +[LogConsistencyProvider(ProviderName = "CustomStorage")] +public class LogTestGrainCustomStorage : LogTestGrain, + Orleans.EventSourcing.CustomStorage.ICustomStorageInterface +{ - public Task ApplyUpdatesToStorage(IReadOnlyList updates, int expectedversion) - { - return GetStorageGrain().Update(updates, expectedversion); - } + // we use another impl of this grain as the primary. + private ILogTestGrain? storagegrain; - public async Task> ReadStateFromStorage() + private ILogTestGrain GetStorageGrain() + { + if (storagegrain == null) { - var kvp = await GetStorageGrain().Read(); - return new KeyValuePair(kvp.Key, (MyGrainState)kvp.Value); + storagegrain = GrainFactory.GetGrain(this.GetPrimaryKeyLong(), "TestGrains.LogTestGrainSharedStateStorage"); } + return storagegrain; + } - public Task ClearStoredState() - { - return GetStorageGrain().Clear(); - } + + public Task ApplyUpdatesToStorage(IReadOnlyList updates, int expectedversion) + { + return GetStorageGrain().Update(updates, expectedversion); } - // use the explictly specified "CustomStorage" log-consistency provider with access from primary cluster only - [LogConsistencyProvider(ProviderName = "CustomStoragePrimaryCluster")] - public class LogTestGrainCustomStoragePrimaryCluster : LogTestGrain, - Orleans.EventSourcing.CustomStorage.ICustomStorageInterface + public async Task> ReadStateFromStorage() { - private readonly DeepCopier copier; + var kvp = await GetStorageGrain().Read(); + return new KeyValuePair(kvp.Key, (MyGrainState)kvp.Value); + } - // we use fake in-memory state as the storage - private MyGrainState? state; - private int version; + public Task ClearStoredState() + { + return GetStorageGrain().Clear(); + } +} - public LogTestGrainCustomStoragePrimaryCluster(DeepCopier copier) - { - this.copier = copier; - } +// use the explictly specified "CustomStorage" log-consistency provider with access from primary cluster only +[LogConsistencyProvider(ProviderName = "CustomStoragePrimaryCluster")] +public class LogTestGrainCustomStoragePrimaryCluster : LogTestGrain, + Orleans.EventSourcing.CustomStorage.ICustomStorageInterface +{ + private readonly DeepCopier copier; + + // we use fake in-memory state as the storage + private MyGrainState? state; + private int version; + + public LogTestGrainCustomStoragePrimaryCluster(DeepCopier copier) + { + this.copier = copier; + } - // simulate an async call during activation. This caused deadlock in earlier version, - // so I add it here to catch regressions. - public override async Task OnActivateAsync(CancellationToken cancellationToken) + // simulate an async call during activation. This caused deadlock in earlier version, + // so I add it here to catch regressions. + public override async Task OnActivateAsync(CancellationToken cancellationToken) + { + await Task.Run(async () => { - await Task.Run(async () => - { - await Task.Delay(10); - }); - } + await Task.Delay(10); + }); + } - public Task ApplyUpdatesToStorage(IReadOnlyList updates, int expectedversion) + public Task ApplyUpdatesToStorage(IReadOnlyList updates, int expectedversion) + { + if (state == null) { - if (state == null) - { - state = new MyGrainState(); - version = 0; - } - - if (expectedversion != version) - return Task.FromResult(false); - - foreach (var u in updates) - { - this.TransitionState(state, u); - version++; - } - - return Task.FromResult(true); + state = new MyGrainState(); + version = 0; } - public Task> ReadStateFromStorage() + if (expectedversion != version) + return Task.FromResult(false); + + foreach (var u in updates) { - if (state == null) - { - state = new MyGrainState(); - version = 0; - } - return Task.FromResult(new KeyValuePair(version, this.copier.Copy(state!)!)); // State is initialized in the branch above. + this.TransitionState(state, u); + version++; } - public Task ClearStoredState() + return Task.FromResult(true); + } + + public Task> ReadStateFromStorage() + { + if (state == null) { - state = null; + state = new MyGrainState(); version = 0; - return Task.CompletedTask; } + return Task.FromResult(new KeyValuePair(version, this.copier.Copy(state!)!)); // State is initialized in the branch above. } - + public Task ClearStoredState() + { + state = null; + version = 0; + return Task.CompletedTask; + } } +#pragma warning restore ORLEANSEXP005 diff --git a/test/Orleans.EventSourcing.Tests/EventSourcingTests/EventSourcingClusterFixture.cs b/test/Orleans.EventSourcing.Tests/EventSourcingTests/EventSourcingClusterFixture.cs index 1c0bff8aa3f..23cc79eb6d8 100644 --- a/test/Orleans.EventSourcing.Tests/EventSourcingTests/EventSourcingClusterFixture.cs +++ b/test/Orleans.EventSourcing.Tests/EventSourcingTests/EventSourcingClusterFixture.cs @@ -1,42 +1,265 @@ +using System.Buffers; +using System.Collections.Concurrent; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Orleans.EventSourcing.CustomStorage; +using Orleans.Journaling; +using Orleans.Journaling.Json; using Orleans.Storage; using Orleans.TestingHost; using TestExtensions; +using TestGrains; -namespace Tester.EventSourcingTests +#pragma warning disable ORLEANSEXP005 +namespace Tester.EventSourcingTests; + +/// +/// We use a special fixture for event sourcing tests +/// so we can add the required log consistency providers, and +/// do more tracing +/// +public class EventSourcingClusterFixture : BaseTestClusterFixture { - /// - /// We use a special fixture for event sourcing tests - /// so we can add the required log consistency providers, and - /// do more tracing - /// - public class EventSourcingClusterFixture : BaseTestClusterFixture + protected override void ConfigureTestCluster(TestClusterBuilder builder) + { + builder.AddSiloBuilderConfigurator(); + } + + public void FailNextJournalAppend(IAddressable grain, Exception exception, bool afterWrite = false) + { + JournalStorageProvider.FailNextAppend(JournalId.FromGrainId(grain.GetGrainId()), exception, afterWrite); + } + + public Task BlockNextJournalAppend(IAddressable grain) + { + return JournalStorageProvider.BlockNextAppend(JournalId.FromGrainId(grain.GetGrainId())); + } + + public void ReleaseBlockedJournalAppend(IAddressable grain) { - protected override void ConfigureTestCluster(TestClusterBuilder builder) + JournalStorageProvider.ReleaseBlockedAppend(JournalId.FromGrainId(grain.GetGrainId())); + } + + private static readonly FaultInjectingJournalStorageProvider JournalStorageProvider = new(); + + private class TestSiloConfigurator : ISiloConfigurator + { + public void Configure(ISiloBuilder hostBuilder) { - builder.AddSiloBuilderConfigurator(); + // we use a slowed-down memory storage provider + hostBuilder + .AddLogStorageBasedLogConsistencyProvider("LogStorage") + .AddStateStorageBasedLogConsistencyProvider("StateStorage") + .AddJournaledStateBasedLogConsistencyProvider("JournaledState") + .UseJsonJournalFormat(options => + { + options.SerializerOptions.IncludeFields = true; + options.SerializerOptions.Converters.Add(new LogTestEventJsonConverter()); + options.AddTypeInfoResolver(EventSourcingTestsJsonContext.Default); + }) + .AddCustomStorageBasedLogConsistencyProvider("CustomStoragePrimaryCluster") + .ConfigureLogging(builder => + { + builder.AddFilter(typeof(MemoryGrainStorage).FullName, LogLevel.Debug); + builder.AddFilter(typeof(LogConsistencyProvider).Namespace, LogLevel.Debug); + }) + .AddMemoryGrainStorageAsDefault() + .AddMemoryGrainStorage("AzureStore") + .AddMemoryGrainStorage("MemoryStore") + .AddFaultInjectionMemoryStorage("SlowMemoryStore", options => options.NumStorageGrains = 10, faultyOptions => faultyOptions.Latency = TimeSpan.FromMilliseconds(15)); + + hostBuilder.Services.AddSingleton(JournalStorageProvider); } - private class TestSiloConfigurator : ISiloConfigurator + private sealed class LogTestEventJsonConverter : JsonConverter { - public void Configure(ISiloBuilder hostBuilder) + public override bool CanConvert(Type typeToConvert) => typeToConvert == typeof(object); + + public override object? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { - // we use a slowed-down memory storage provider - hostBuilder - .AddLogStorageBasedLogConsistencyProvider("LogStorage") - .AddStateStorageBasedLogConsistencyProvider("StateStorage") - .AddCustomStorageBasedLogConsistencyProvider("CustomStoragePrimaryCluster") - .ConfigureLogging(builder => - { - builder.AddFilter(typeof(MemoryGrainStorage).FullName, LogLevel.Debug); - builder.AddFilter(typeof(LogConsistencyProvider).Namespace, LogLevel.Debug); - }) - .AddMemoryGrainStorageAsDefault() - .AddMemoryGrainStorage("AzureStore") - .AddMemoryGrainStorage("MemoryStore") - .AddFaultInjectionMemoryStorage("SlowMemoryStore", options=>options.NumStorageGrains = 10, faultyOptions => faultyOptions.Latency = TimeSpan.FromMilliseconds(15)); + if (reader.TokenType is JsonTokenType.Null) + { + return null; + } + + using var document = JsonDocument.ParseValue(ref reader); + var root = document.RootElement; + var type = root.GetProperty("$type").GetString(); + var value = root.TryGetProperty(nameof(UpdateA.Val), out var valueProperty) ? valueProperty.GetInt32() : 0; + + return type switch + { + nameof(UpdateA) => new UpdateA { Val = value }, + nameof(UpdateB) => new UpdateB { Val = value }, + nameof(IncrementA) => new IncrementA { Val = value }, + nameof(AddReservation) => new AddReservation { Val = value }, + nameof(RemoveReservation) => new RemoveReservation { Val = value }, + _ => throw new JsonException($"Unknown log test event type '{type}'.") + }; + } + + public override void Write(Utf8JsonWriter writer, object value, JsonSerializerOptions options) + { + writer.WriteStartObject(); + switch (value) + { + case UpdateA update: + WriteEvent(writer, nameof(UpdateA), update.Val); + break; + case UpdateB update: + WriteEvent(writer, nameof(UpdateB), update.Val); + break; + case IncrementA update: + WriteEvent(writer, nameof(IncrementA), update.Val); + break; + case AddReservation update: + WriteEvent(writer, nameof(AddReservation), update.Val); + break; + case RemoveReservation update: + WriteEvent(writer, nameof(RemoveReservation), update.Val); + break; + default: + throw new JsonException($"Unsupported log test event type '{value.GetType()}'."); + } + + writer.WriteEndObject(); + } + + private static void WriteEvent(Utf8JsonWriter writer, string type, int value) + { + writer.WriteString("$type", type); + writer.WriteNumber(nameof(UpdateA.Val), value); + } + } + } + + private sealed class FaultInjectingJournalStorageProvider : IJournalStorageProvider + { + private readonly ConcurrentDictionary _appendFailures = new(); + private readonly ConcurrentDictionary _appendBlocks = new(); + private readonly VolatileJournalStorageProvider _inner = new(); + + public IJournalStorage CreateStorage(JournalId journalId) => new FaultInjectingJournalStorage(this, journalId, _inner.CreateStorage(journalId)); + + public void FailNextAppend(JournalId journalId, Exception exception, bool afterWrite) + { + ArgumentNullException.ThrowIfNull(exception); + if (!_appendFailures.TryAdd(journalId, new(exception, afterWrite))) + { + throw new InvalidOperationException($"An append failure is already configured for journal '{journalId}'."); } } + + public Task BlockNextAppend(JournalId journalId) + { + var block = new AppendBlock(); + if (!_appendBlocks.TryAdd(journalId, block)) + { + throw new InvalidOperationException($"An append block is already configured for journal '{journalId}'."); + } + + return block.Started.Task; + } + + public void ReleaseBlockedAppend(JournalId journalId) + { + if (!_appendBlocks.TryGetValue(journalId, out var block)) + { + throw new InvalidOperationException($"No append block is configured for journal '{journalId}'."); + } + + block.Allow.TrySetResult(); + } + + private bool TryTakeAppendFailure(JournalId journalId, out AppendFailure failure) => _appendFailures.TryRemove(journalId, out failure); + + private readonly record struct AppendFailure(Exception Exception, bool AfterWrite); + + private sealed class AppendBlock + { + public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource Allow { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + } + + private sealed class FaultInjectingJournalStorage( + FaultInjectingJournalStorageProvider provider, + JournalId journalId, + IJournalStorage inner) : IJournalStorage + { + public bool IsCompactionRequested => inner.IsCompactionRequested; + + public ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken) => + inner.ReadAsync(consumer, cancellationToken); + + public ValueTask CreateIfNotExistsAsync( + IReadOnlyDictionary? metadata = null, + CancellationToken cancellationToken = default) => + inner.CreateIfNotExistsAsync(metadata, cancellationToken); + + public ValueTask GetMetadataAsync(CancellationToken cancellationToken = default) => + inner.GetMetadataAsync(cancellationToken); + + public ValueTask UpdateMetadataAsync( + IReadOnlyDictionary? set = null, + IEnumerable? remove = null, + string? expectedETag = null, + CancellationToken cancellationToken = default) => + inner.UpdateMetadataAsync(set, remove, expectedETag, cancellationToken); + + public ValueTask ReplaceAsync(ReadOnlySequence value, CancellationToken cancellationToken) => + inner.ReplaceAsync(value, cancellationToken); + + public async ValueTask AppendAsync(ReadOnlySequence value, CancellationToken cancellationToken) + { + if (provider._appendBlocks.TryGetValue(journalId, out var block)) + { + block.Started.TrySetResult(); + await block.Allow.Task.WaitAsync(cancellationToken); + provider._appendBlocks.TryRemove(new KeyValuePair(journalId, block)); + } + + var hasFailure = provider.TryTakeAppendFailure(journalId, out var failure); + if (hasFailure && !failure.AfterWrite) + { + throw failure.Exception; + } + + await inner.AppendAsync(value, cancellationToken); + if (hasFailure) + { + throw failure.Exception; + } + } + + public ValueTask DeleteAsync(CancellationToken cancellationToken) => inner.DeleteAsync(cancellationToken); + } + } +} + +[JsonSourceGenerationOptions(IncludeFields = true)] +[JsonSerializable(typeof(DateTime))] +[JsonSerializable(typeof(int))] +[JsonSerializable(typeof(object))] +[JsonSerializable(typeof(string))] +[JsonSerializable(typeof(uint))] +[JsonSerializable(typeof(UpdateA))] +[JsonSerializable(typeof(UpdateB))] +[JsonSerializable(typeof(IncrementA))] +[JsonSerializable(typeof(AddReservation))] +[JsonSerializable(typeof(RemoveReservation))] +internal sealed partial class EventSourcingTestsJsonContext : JsonSerializerContext; +#pragma warning restore ORLEANSEXP005 + +public sealed class CommaClusterIdEventSourcingClusterFixture : EventSourcingClusterFixture +{ + public const string ClusterId = "west,prod-v2.canary"; + + protected override void ConfigureTestCluster(TestClusterBuilder builder) + { + base.ConfigureTestCluster(builder); + builder.Options.ClusterId = ClusterId; } } diff --git a/test/Orleans.EventSourcing.Tests/EventSourcingTests/LogTestGrainClearTests.cs b/test/Orleans.EventSourcing.Tests/EventSourcingTests/LogTestGrainClearTests.cs index a69b044735f..7024027df15 100644 --- a/test/Orleans.EventSourcing.Tests/EventSourcingTests/LogTestGrainClearTests.cs +++ b/test/Orleans.EventSourcing.Tests/EventSourcingTests/LogTestGrainClearTests.cs @@ -6,107 +6,426 @@ using Xunit; using Assert = Xunit.Assert; -namespace Tester.EventSourcingTests +namespace Tester.EventSourcingTests; + +/// +/// Integration tests for clear-log behavior on non-Azure log test grain configurations. +/// +[TestSuite("Functional")] +[TestProvider("None")] +[TestArea("EventSourcing")] +public class LogTestGrainClearTests : IClassFixture { - /// - /// Integration tests for clear-log behavior on non-Azure log test grain configurations. - /// - [TestSuite("Functional")] - [TestProvider("None")] - [TestArea("EventSourcing")] - public class LogTestGrainClearTests : IClassFixture + private readonly EventSourcingClusterFixture fixture; + + public LogTestGrainClearTests(EventSourcingClusterFixture fixture) + { + this.fixture = fixture; + } + + [Theory, TestCategory("EventSourcing"), TestCategory("Functional")] + [InlineData("TestGrains.LogTestGrainDefaultStorage", 721001L)] + [InlineData("TestGrains.LogTestGrainSharedLogStorage", 721002L)] + [InlineData("TestGrains.LogTestGrainCustomStoragePrimaryCluster", 721003L)] + [InlineData("TestGrains.LogTestGrainJournaledStateStorage", 721004L)] + public async Task ClearLog_ResetDropsTentativeAndAllowsFurtherWrites(string grainClass, long grainId) + { + var grain = this.fixture.GrainFactory.GetGrain(grainId, grainClass); + + await grain.Clear(); + await grain.SetAGlobal(10); + Assert.Equal(10, await grain.GetAGlobal()); + Assert.Equal(1, await grain.GetConfirmedVersion()); + + await grain.SetALocal(99); + await grain.SetBLocal(77); + var tentativeBeforeClear = await grain.GetBothLocal(); + Assert.Equal(99, tentativeBeforeClear.A); + Assert.Equal(77, tentativeBeforeClear.B); + + await grain.Clear(); + Assert.Equal(0, await grain.GetConfirmedVersion()); + + var confirmedAfterClear = await grain.GetBothGlobal(); + Assert.Equal(0, confirmedAfterClear.A); + Assert.Equal(0, confirmedAfterClear.B); + + var tentativeAfterClear = await grain.GetBothLocal(); + Assert.Equal(0, tentativeAfterClear.A); + Assert.Equal(0, tentativeAfterClear.B); + + await grain.SetAGlobal(41); + await grain.IncrementAGlobal(); + Assert.Equal(42, await grain.GetAGlobal()); + Assert.Equal(2, await grain.GetConfirmedVersion()); + + await grain.Clear(); + var exceptions = await RunConcurrentOperationsAroundClear(grain); + Assert.DoesNotContain(exceptions, static ex => ex is InconsistentStateException); + Assert.Empty(exceptions); + + await grain.Clear(); + await grain.SetAGlobal(7); + Assert.Equal(7, await grain.GetAGlobal()); + Assert.Equal(1, await grain.GetConfirmedVersion()); + } + + [Fact, TestCategory("EventSourcing"), TestCategory("Functional")] + public async Task JournaledStateLogStorage_PersistsEventsAcrossActivation() { - private readonly EventSourcingClusterFixture fixture; + var grain = this.fixture.GrainFactory.GetGrain(721005L, "TestGrains.LogTestGrainJournaledStateStorage"); - public LogTestGrainClearTests(EventSourcingClusterFixture fixture) + await grain.Clear(); + await grain.SetAGlobal(10); + await grain.IncrementAGlobal(); + + var eventLog = await grain.GetEventLog(); + Assert.Equal(2, eventLog.Count); + Assert.Equal(11, await grain.GetAGlobal()); + + await this.fixture.HostedCluster.DeactivateAsync(grain); + + grain = this.fixture.GrainFactory.GetGrain(721005L, "TestGrains.LogTestGrainJournaledStateStorage"); + Assert.Equal(11, await grain.GetAGlobal()); + Assert.Equal(2, await grain.GetConfirmedVersion()); + + eventLog = await grain.GetEventLog(); + Assert.Equal(2, eventLog.Count); + } + + [Fact, TestCategory("EventSourcing"), TestCategory("Functional")] + public async Task JournaledStateLogStorage_RetrievesIndexedLogSegments() + { + var grain = this.fixture.GrainFactory.GetGrain(721017L, "TestGrains.LogTestGrainJournaledStateStorage"); + + await grain.Clear(); + await grain.SetAGlobal(10); + await grain.SetBGlobal(20); + await grain.IncrementAGlobal(); + + var segment = await grain.GetEventLogSegment(1, 3); + Assert.Collection( + segment, + entry => Assert.Equal(20, Assert.IsType(entry).Val), + entry => Assert.IsType(entry)); + Assert.Empty(await grain.GetEventLogSegment(2, 2)); + await Assert.ThrowsAsync(() => grain.GetEventLogSegment(-1, 1)); + await Assert.ThrowsAsync(() => grain.GetEventLogSegment(2, 4)); + } + + [Fact, TestCategory("EventSourcing"), TestCategory("Functional")] + public async Task JournaledStateLogStorage_DoesNotRetrieveTentativeLogEntries() + { + var grain = this.fixture.GrainFactory.GetGrain(721018L, "TestGrains.LogTestGrainJournaledStateStorage"); + + await grain.Clear(); + var appendStarted = this.fixture.BlockNextJournalAppend(grain); + try { - this.fixture = fixture; - } + await grain.SetALocal(41); + await appendStarted.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - [Theory, TestCategory("EventSourcing"), TestCategory("Functional")] - [InlineData("TestGrains.LogTestGrainDefaultStorage", 721001L)] - [InlineData("TestGrains.LogTestGrainSharedLogStorage", 721002L)] - [InlineData("TestGrains.LogTestGrainCustomStoragePrimaryCluster", 721003L)] - public async Task ClearLog_ResetDropsTentativeAndAllowsFurtherWrites(string grainClass, long grainId) + Assert.Equal(0, await grain.GetConfirmedVersion()); + await Assert.ThrowsAsync(() => grain.GetEventLogSegment(0, 1)); + } + finally { - var grain = this.fixture.GrainFactory.GetGrain(grainId, grainClass); + this.fixture.ReleaseBlockedJournalAppend(grain); + } + + await grain.SynchronizeGlobalState(); + var segment = await grain.GetEventLogSegment(0, 1); + Assert.Equal(41, Assert.IsType(Assert.Single(segment)).Val); + } - await grain.Clear(); - await grain.SetAGlobal(10); - Assert.Equal(10, await grain.GetAGlobal()); - Assert.Equal(1, await grain.GetConfirmedVersion()); + [Fact, TestCategory("EventSourcing"), TestCategory("Functional")] + public async Task JournaledStateLogStorage_ClearPreservesOtherJournaledState() + { + var grain = this.fixture.GrainFactory.GetGrain(721006L, "TestGrains.LogTestGrainJournaledStateStorageWithAuxiliaryState"); - await grain.SetALocal(99); - await grain.SetBLocal(77); - var tentativeBeforeClear = await grain.GetBothLocal(); - Assert.Equal(99, tentativeBeforeClear.A); - Assert.Equal(77, tentativeBeforeClear.B); + await grain.Clear(); + await grain.SetAuxiliaryValue(17); + await grain.SetAGlobal(10); - await grain.Clear(); - Assert.Equal(0, await grain.GetConfirmedVersion()); + Assert.Equal(17, await grain.GetAuxiliaryValue()); + Assert.Equal(10, await grain.GetAGlobal()); + Assert.Equal(1, await grain.GetConfirmedVersion()); - var confirmedAfterClear = await grain.GetBothGlobal(); - Assert.Equal(0, confirmedAfterClear.A); - Assert.Equal(0, confirmedAfterClear.B); + await grain.Clear(); - var tentativeAfterClear = await grain.GetBothLocal(); - Assert.Equal(0, tentativeAfterClear.A); - Assert.Equal(0, tentativeAfterClear.B); + Assert.Equal(17, await grain.GetAuxiliaryValue()); + Assert.Equal(0, await grain.GetConfirmedVersion()); + Assert.Equal(0, await grain.GetAGlobal()); - await grain.SetAGlobal(41); - await grain.IncrementAGlobal(); - Assert.Equal(42, await grain.GetAGlobal()); - Assert.Equal(2, await grain.GetConfirmedVersion()); + await this.fixture.HostedCluster.DeactivateAsync(grain); - await grain.Clear(); - var exceptions = await RunConcurrentOperationsAroundClear(grain); - Assert.DoesNotContain(exceptions, static ex => ex is InconsistentStateException); - Assert.Empty(exceptions); + grain = this.fixture.GrainFactory.GetGrain(721006L, "TestGrains.LogTestGrainJournaledStateStorageWithAuxiliaryState"); - await grain.Clear(); - await grain.SetAGlobal(7); - Assert.Equal(7, await grain.GetAGlobal()); - Assert.Equal(1, await grain.GetConfirmedVersion()); - } + Assert.Equal(17, await grain.GetAuxiliaryValue()); + Assert.Equal(0, await grain.GetConfirmedVersion()); + Assert.Equal(0, await grain.GetAGlobal()); - private static async Task> RunConcurrentOperationsAroundClear(ILogTestGrain grain) - { - var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var exceptions = new List(); - var syncLock = new object(); + await grain.SetAGlobal(42); + Assert.Equal(17, await grain.GetAuxiliaryValue()); + Assert.Equal(1, await grain.GetConfirmedVersion()); + } + + [Fact, TestCategory("EventSourcing"), TestCategory("Functional")] + public async Task JournaledStateLogStorage_UncommittedFailureFailsCompleteJournalBatch() + { + var grain = this.fixture.GrainFactory.GetGrain(721007L, "TestGrains.LogTestGrainJournaledStateStorageWithAuxiliaryState"); + + await grain.Clear(); + await grain.SetAuxiliaryValueAndAGlobal(17, 10); + var deactivated = this.fixture.HostedCluster.WaitForDeactivationAsync(grain); + this.fixture.FailNextJournalAppend(grain, new IOException("Expected transient append failure.")); + + await Assert.ThrowsAsync(() => grain.SetAuxiliaryValueAndAGlobal(23, 41)); + await deactivated.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + grain = this.fixture.GrainFactory.GetGrain(721007L, "TestGrains.LogTestGrainJournaledStateStorageWithAuxiliaryState"); + Assert.Equal(17, await grain.GetAuxiliaryValue()); + Assert.Equal(10, await grain.GetAGlobal()); + Assert.Equal(1, await grain.GetConfirmedVersion()); + } + + [Fact, TestCategory("EventSourcing"), TestCategory("Functional")] + public async Task JournaledStateLogStorage_ConflictFailsCompleteJournalBatch() + { + var grain = this.fixture.GrainFactory.GetGrain(721008L, "TestGrains.LogTestGrainJournaledStateStorageWithAuxiliaryState"); + await grain.Clear(); + await grain.SetAuxiliaryValueAndAGlobal(17, 10); + + var deactivated = this.fixture.HostedCluster.WaitForDeactivationAsync(grain); + this.fixture.FailNextJournalAppend(grain, new InconsistentStateException("Expected append conflict.")); + + await Assert.ThrowsAsync(() => grain.SetAuxiliaryValueAndAGlobal(23, 41)); + await deactivated.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + grain = this.fixture.GrainFactory.GetGrain(721008L, "TestGrains.LogTestGrainJournaledStateStorageWithAuxiliaryState"); + Assert.Equal(17, await grain.GetAuxiliaryValue()); + Assert.Equal(10, await grain.GetAGlobal()); + Assert.Equal(1, await grain.GetConfirmedVersion()); + } + + [Fact, TestCategory("EventSourcing"), TestCategory("Functional")] + public async Task JournaledStateLogStorage_AmbiguousConflictRecognizesCommittedJournalBatch() + { + var grain = this.fixture.GrainFactory.GetGrain(721009L, "TestGrains.LogTestGrainJournaledStateStorageWithAuxiliaryState"); + await grain.Clear(); + this.fixture.FailNextJournalAppend(grain, new InconsistentStateException("Expected post-commit conflict."), afterWrite: true); + + await grain.SetAuxiliaryValueAndAGlobal(23, 41); + + Assert.Equal(23, await grain.GetAuxiliaryValue()); + Assert.Equal(41, await grain.GetAGlobal()); + Assert.Equal(1, await grain.GetConfirmedVersion()); + + await this.fixture.HostedCluster.DeactivateAsync(grain); + grain = this.fixture.GrainFactory.GetGrain(721009L, "TestGrains.LogTestGrainJournaledStateStorageWithAuxiliaryState"); + Assert.Equal(23, await grain.GetAuxiliaryValue()); + Assert.Equal(41, await grain.GetAGlobal()); + Assert.Equal(1, await grain.GetConfirmedVersion()); + } + + [Fact, TestCategory("EventSourcing"), TestCategory("Functional")] + public async Task JournaledStateLogStorage_AmbiguousTransientFailureRecognizesCommittedJournalBatch() + { + var grain = this.fixture.GrainFactory.GetGrain(721012L, "TestGrains.LogTestGrainJournaledStateStorageWithAuxiliaryState"); + await grain.Clear(); + this.fixture.FailNextJournalAppend(grain, new IOException("Expected post-commit transient failure."), afterWrite: true); + + await grain.SetAuxiliaryValueAndAGlobal(23, 41); - Task Run(Func operation) + Assert.Equal(23, await grain.GetAuxiliaryValue()); + Assert.Equal(41, await grain.GetAGlobal()); + Assert.Equal(1, await grain.GetConfirmedVersion()); + + await this.fixture.HostedCluster.DeactivateAsync(grain); + grain = this.fixture.GrainFactory.GetGrain(721012L, "TestGrains.LogTestGrainJournaledStateStorageWithAuxiliaryState"); + Assert.Equal(23, await grain.GetAuxiliaryValue()); + Assert.Equal(41, await grain.GetAGlobal()); + Assert.Equal(1, await grain.GetConfirmedVersion()); + } + + [Fact, TestCategory("EventSourcing"), TestCategory("Functional")] + public async Task JournaledStateLogStorage_ClearFailureFailsAndPreservesDurableState() + { + var grain = this.fixture.GrainFactory.GetGrain(721010L, "TestGrains.LogTestGrainJournaledStateStorageWithAuxiliaryState"); + await grain.Clear(); + await grain.SetAuxiliaryValueAndAGlobal(17, 10); + var deactivated = this.fixture.HostedCluster.WaitForDeactivationAsync(grain); + this.fixture.FailNextJournalAppend(grain, new IOException("Expected transient clear failure.")); + + await Assert.ThrowsAsync(() => grain.Clear()); + await deactivated.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + grain = this.fixture.GrainFactory.GetGrain(721010L, "TestGrains.LogTestGrainJournaledStateStorageWithAuxiliaryState"); + Assert.Equal(17, await grain.GetAuxiliaryValue()); + Assert.Equal(10, await grain.GetAGlobal()); + Assert.Equal(1, await grain.GetConfirmedVersion()); + } + + [Fact, TestCategory("EventSourcing"), TestCategory("Functional")] + public async Task JournaledStateLogStorage_ClearConflictFailsAndPreservesDurableState() + { + var grain = this.fixture.GrainFactory.GetGrain(721011L, "TestGrains.LogTestGrainJournaledStateStorageWithAuxiliaryState"); + await grain.Clear(); + await grain.SetAuxiliaryValueAndAGlobal(17, 10); + + var deactivated = this.fixture.HostedCluster.WaitForDeactivationAsync(grain); + this.fixture.FailNextJournalAppend(grain, new InconsistentStateException("Expected clear conflict.")); + + await Assert.ThrowsAsync(() => grain.Clear()); + await deactivated.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + grain = this.fixture.GrainFactory.GetGrain(721011L, "TestGrains.LogTestGrainJournaledStateStorageWithAuxiliaryState"); + Assert.Equal(17, await grain.GetAuxiliaryValue()); + Assert.Equal(10, await grain.GetAGlobal()); + Assert.Equal(1, await grain.GetConfirmedVersion()); + } + + [Fact, TestCategory("EventSourcing"), TestCategory("Functional")] + public async Task JournaledStateLogStorage_RefreshPreservesUnflushedAuxiliaryState() + { + var grain = this.fixture.GrainFactory.GetGrain(721013L, "TestGrains.LogTestGrainJournaledStateStorageWithAuxiliaryState"); + await grain.Clear(); + + Assert.Equal(23, await grain.SetAuxiliaryValueAndSynchronize(23)); + await grain.SetAGlobal(41); + + await this.fixture.HostedCluster.DeactivateAsync(grain); + grain = this.fixture.GrainFactory.GetGrain(721013L, "TestGrains.LogTestGrainJournaledStateStorageWithAuxiliaryState"); + Assert.Equal(23, await grain.GetAuxiliaryValue()); + Assert.Equal(41, await grain.GetAGlobal()); + Assert.Equal(1, await grain.GetConfirmedVersion()); + } + + [Fact, TestCategory("EventSourcing"), TestCategory("Functional")] + public async Task JournaledStateLogStorage_StagingFailureDoesNotPersistPartialBatch() + { + var grain = this.fixture.GrainFactory.GetGrain(721014L, "TestGrains.LogTestGrainJournaledStateStorage"); + await grain.Clear(); + var deactivated = this.fixture.HostedCluster.WaitForDeactivationAsync(grain); + + await Assert.ThrowsAsync(() => grain.RaiseEventsWithUnsupportedSecond()); + await deactivated.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + grain = this.fixture.GrainFactory.GetGrain(721014L, "TestGrains.LogTestGrainJournaledStateStorage"); + Assert.Equal(0, await grain.GetAGlobal()); + Assert.Equal(0, await grain.GetConfirmedVersion()); + Assert.Empty(await grain.GetEventLog()); + } + + [Fact, TestCategory("EventSourcing"), TestCategory("Functional")] + public async Task JournaledStateLogStorage_RejectsReentrantGrain() + { + var grain = this.fixture.GrainFactory.GetGrain(721015L, "TestGrains.LogTestGrainJournaledStateReentrantStorage"); + + var exception = await Assert.ThrowsAnyAsync(() => grain.GetAGlobal()); + + Assert.Contains("requires a single, turn-serialized grain activation", exception.ToString(), StringComparison.Ordinal); + } + + private static async Task> RunConcurrentOperationsAroundClear(ILogTestGrain grain) + { + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var exceptions = new List(); + var syncLock = new object(); + + Task Run(Func operation) + { + return Task.Run(async () => { - return Task.Run(async () => + await gate.Task; + try { - await gate.Task; - try - { - await operation(); - } - catch (Exception exception) + await operation(); + } + catch (Exception exception) + { + lock (syncLock) { - lock (syncLock) - { - exceptions.Add(exception); - } + exceptions.Add(exception); } - }); - } + } + }); + } - var operations = new[] - { - Run(() => grain.SetALocal(1)), - Run(() => grain.SetAGlobal(2)), - Run(() => grain.IncrementAGlobal()), - Run(() => grain.Clear()), - Run(() => grain.SetAGlobal(3)), - Run(async () => _ = await grain.GetAGlobal()), - }; - - gate.SetResult(true); - await Task.WhenAll(operations); - return exceptions; + var operations = new[] + { + Run(() => grain.SetALocal(1)), + Run(() => grain.SetAGlobal(2)), + Run(() => grain.IncrementAGlobal()), + Run(() => grain.Clear()), + Run(() => grain.SetAGlobal(3)), + Run(async () => _ = await grain.GetAGlobal()), + }; + + gate.SetResult(true); + await Task.WhenAll(operations); + return exceptions; + } +} + +[TestSuite("Functional")] +[TestProvider("None")] +[TestArea("EventSourcing")] +public class LogTestGrainClearCommaClusterIdTests : IClassFixture +{ + private const long GrainId = 721016L; + private const string GrainClass = "TestGrains.LogTestGrainJournaledStateStorageWithAuxiliaryState"; + private readonly CommaClusterIdEventSourcingClusterFixture fixture; + + public LogTestGrainClearCommaClusterIdTests(CommaClusterIdEventSourcingClusterFixture fixture) + { + this.fixture = fixture; + } + + [Fact, TestCategory("EventSourcing"), TestCategory("Functional")] + public async Task JournaledStateLogStorage_AmbiguousConflictWithCommaClusterId_RecognizesCommittedBatchBeforeAndAfterReactivation() + { + Assert.Equal("west,prod-v2.canary", this.fixture.HostedCluster.Options.ClusterId); + + var grain = this.fixture.GrainFactory.GetGrain(GrainId, GrainClass); + + await grain.Clear(); + await grain.SetAuxiliaryValueAndAGlobal(17, 10); + + Assert.Equal(17, await grain.GetAuxiliaryValue()); + Assert.Equal(10, await grain.GetAGlobal()); + Assert.Equal(1, await grain.GetConfirmedVersion()); + AssertEventLog(await grain.GetEventLog(), 10); + + this.fixture.FailNextJournalAppend( + grain, + new InconsistentStateException( + $"Expected post-commit conflict for ClusterId '{CommaClusterIdEventSourcingClusterFixture.ClusterId}'."), + afterWrite: true); + + await grain.SetAuxiliaryValueAndAGlobal(23, 41); + + Assert.Equal(23, await grain.GetAuxiliaryValue()); + Assert.Equal(41, await grain.GetAGlobal()); + Assert.Equal(2, await grain.GetConfirmedVersion()); + AssertEventLog(await grain.GetEventLog(), 10, 41); + + await this.fixture.HostedCluster.DeactivateAsync(grain); + grain = this.fixture.GrainFactory.GetGrain(GrainId, GrainClass); + + Assert.Equal(23, await grain.GetAuxiliaryValue()); + Assert.Equal(41, await grain.GetAGlobal()); + Assert.Equal(2, await grain.GetConfirmedVersion()); + AssertEventLog(await grain.GetEventLog(), 10, 41); + } + + private static void AssertEventLog(IReadOnlyList eventLog, params int[] expectedValues) + { + Assert.Equal(expectedValues.Length, eventLog.Count); + for (var index = 0; index < expectedValues.Length; index++) + { + var update = Assert.IsType(eventLog[index]); + Assert.Equal(expectedValues[index], update.Val); } } } diff --git a/test/Orleans.EventSourcing.Tests/EventSourcingTests/StringEncodedWriteVectorTests.cs b/test/Orleans.EventSourcing.Tests/EventSourcingTests/StringEncodedWriteVectorTests.cs new file mode 100644 index 00000000000..45f2acb2d07 --- /dev/null +++ b/test/Orleans.EventSourcing.Tests/EventSourcingTests/StringEncodedWriteVectorTests.cs @@ -0,0 +1,191 @@ +using Orleans.EventSourcing.Common; +using Xunit; + +namespace Tester.EventSourcingTests; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("EventSourcing")] +public sealed class StringEncodedWriteVectorTests +{ + [Fact, TestCategory("EventSourcing"), TestCategory("BVT")] + public void GetBit_RequiresExactReplicaToken() + { + const string writeVector = ",cluster10"; + + Assert.False(StringEncodedWriteVector.GetBit(writeVector, "cluster1")); + Assert.True(StringEncodedWriteVector.GetBit(writeVector, "cluster10")); + } + + [Fact, TestCategory("EventSourcing"), TestCategory("BVT")] + public void FlipBit_OnlyRemovesExactReplicaToken() + { + var writeVector = ",cluster10,cluster1"; + + Assert.False(StringEncodedWriteVector.FlipBit(ref writeVector, "cluster1")); + Assert.Equal(",cluster10", writeVector); + Assert.True(StringEncodedWriteVector.GetBit(writeVector, "cluster10")); + } + + [Theory, TestCategory("EventSourcing"), TestCategory("BVT")] + [InlineData("west,prod", "west", "prod", "west,prod-canary", "West,prod")] + [InlineData("cluster:blue/1", "cluster:blue", "blue/1", "cluster:blue/10", "Cluster:blue/1")] + [InlineData("cluster.1+canary@west", "cluster.1", "canary@west", "cluster.1+canary@west-prod", "Cluster.1+canary@west")] + public void GetBit_CommaAndPunctuationReplicaIds_MatchOnlyExactIds( + string replica, + string prefix, + string suffix, + string extended, + string caseVariant) + { + var writeVector = string.Empty; + + Assert.True(StringEncodedWriteVector.FlipBit(ref writeVector, replica)); + Assert.True(StringEncodedWriteVector.GetBit(writeVector, replica)); + Assert.False(StringEncodedWriteVector.GetBit(writeVector, prefix)); + Assert.False(StringEncodedWriteVector.GetBit(writeVector, suffix)); + Assert.False(StringEncodedWriteVector.GetBit(writeVector, extended)); + Assert.False(StringEncodedWriteVector.GetBit(writeVector, caseVariant)); + } + + [Fact, TestCategory("EventSourcing"), TestCategory("BVT")] + public void FlipBit_CommaContainingReplica_TogglesWithoutChangingNeighborReplicas() + { + var writeVector = string.Empty; + + Assert.True(StringEncodedWriteVector.FlipBit(ref writeVector, "prod")); + Assert.True(StringEncodedWriteVector.FlipBit(ref writeVector, "west")); + Assert.True(StringEncodedWriteVector.FlipBit(ref writeVector, "west,prod")); + Assert.True(StringEncodedWriteVector.GetBit(writeVector, "west")); + Assert.True(StringEncodedWriteVector.GetBit(writeVector, "prod")); + Assert.True(StringEncodedWriteVector.GetBit(writeVector, "west,prod")); + + Assert.False(StringEncodedWriteVector.FlipBit(ref writeVector, "west,prod")); + Assert.True(StringEncodedWriteVector.GetBit(writeVector, "west")); + Assert.True(StringEncodedWriteVector.GetBit(writeVector, "prod")); + Assert.False(StringEncodedWriteVector.GetBit(writeVector, "west,prod")); + + Assert.True(StringEncodedWriteVector.FlipBit(ref writeVector, "west,prod")); + Assert.True(StringEncodedWriteVector.GetBit(writeVector, "west,prod")); + Assert.False(StringEncodedWriteVector.FlipBit(ref writeVector, "west,prod")); + Assert.True(StringEncodedWriteVector.GetBit(writeVector, "west")); + Assert.True(StringEncodedWriteVector.GetBit(writeVector, "prod")); + Assert.False(StringEncodedWriteVector.GetBit(writeVector, "west,prod")); + } + + [Fact, TestCategory("EventSourcing"), TestCategory("BVT")] + public void FlipBit_PrefixAndSuffixLikeReplicaIds_RoundTripIndependently() + { + var replicas = new[] { "cluster1", "cluster10", "1", "prod", "west-prod", "prod-west" }; + var expected = new HashSet(StringComparer.Ordinal); + var writeVector = string.Empty; + + foreach (var replica in replicas) + { + expected.Add(replica); + Assert.True(StringEncodedWriteVector.FlipBit(ref writeVector, replica)); + + foreach (var candidate in replicas) + { + Assert.Equal(expected.Contains(candidate), StringEncodedWriteVector.GetBit(writeVector, candidate)); + } + } + + foreach (var replica in replicas) + { + expected.Remove(replica); + Assert.False(StringEncodedWriteVector.FlipBit(ref writeVector, replica)); + + foreach (var candidate in replicas) + { + Assert.Equal(expected.Contains(candidate), StringEncodedWriteVector.GetBit(writeVector, candidate)); + } + } + + Assert.Empty(expected); + Assert.All(replicas, replica => Assert.False(StringEncodedWriteVector.GetBit(writeVector, replica))); + } + + [Fact, TestCategory("EventSourcing"), TestCategory("BVT")] + public void GetBit_LegacyDelimitedVector_DecodesExactLegacyTokens() + { + const string writeVector = ",clusterA,clusterB"; + + Assert.True(StringEncodedWriteVector.GetBit(writeVector, "clusterA")); + Assert.True(StringEncodedWriteVector.GetBit(writeVector, "clusterB")); + Assert.False(StringEncodedWriteVector.GetBit(writeVector, "cluster")); + Assert.False(StringEncodedWriteVector.GetBit(writeVector, "clusterAB")); + Assert.False(StringEncodedWriteVector.GetBit(writeVector, "ClusterA")); + Assert.False(StringEncodedWriteVector.GetBit(writeVector, "clusterA,clusterB")); + } + + [Fact, TestCategory("EventSourcing"), TestCategory("BVT")] + public void FlipBit_LegacyDelimitedVector_PreservesUntoggledLegacyTokens() + { + var writeVector = ",clusterA,clusterB"; + + Assert.False(StringEncodedWriteVector.FlipBit(ref writeVector, "clusterA")); + Assert.False(StringEncodedWriteVector.GetBit(writeVector, "clusterA")); + Assert.True(StringEncodedWriteVector.GetBit(writeVector, "clusterB")); + + Assert.True(StringEncodedWriteVector.FlipBit(ref writeVector, "clusterA")); + Assert.True(StringEncodedWriteVector.GetBit(writeVector, "clusterA")); + Assert.True(StringEncodedWriteVector.GetBit(writeVector, "clusterB")); + + Assert.True(StringEncodedWriteVector.FlipBit(ref writeVector, "cluster:blue/1")); + Assert.StartsWith(",", writeVector, StringComparison.Ordinal); + Assert.True(StringEncodedWriteVector.GetBit(writeVector, "cluster:blue/1")); + Assert.True(StringEncodedWriteVector.GetBit(writeVector, "clusterA")); + Assert.True(StringEncodedWriteVector.GetBit(writeVector, "clusterB")); + + Assert.False(StringEncodedWriteVector.FlipBit(ref writeVector, "cluster:blue/1")); + Assert.False(StringEncodedWriteVector.GetBit(writeVector, "cluster:blue/1")); + Assert.True(StringEncodedWriteVector.GetBit(writeVector, "clusterA")); + Assert.True(StringEncodedWriteVector.GetBit(writeVector, "clusterB")); + } + + [Fact, TestCategory("EventSourcing"), TestCategory("BVT")] + public void FlipBit_LegacySafeIds_RemainsReadableByPreviousFormat() + { + var writeVector = ",clusterA"; + + Assert.True(StringEncodedWriteVector.FlipBit(ref writeVector, "clusterB")); + Assert.Equal(",clusterA,clusterB", writeVector); + Assert.True(GetBitUsingLegacyReader(writeVector, "clusterA")); + Assert.True(GetBitUsingLegacyReader(writeVector, "clusterB")); + + Assert.False(StringEncodedWriteVector.FlipBit(ref writeVector, "clusterA")); + Assert.Equal(",clusterB", writeVector); + Assert.False(GetBitUsingLegacyReader(writeVector, "clusterA")); + Assert.True(GetBitUsingLegacyReader(writeVector, "clusterB")); + } + + [Fact, TestCategory("EventSourcing"), TestCategory("BVT")] + public void FlipBit_CurrentFormat_UsesVersionedUtf16LengthPrefixes() + { + var writeVector = string.Empty; + + Assert.True(StringEncodedWriteVector.FlipBit(ref writeVector, "west,prod")); + Assert.Equal("v1:9:west,prod", writeVector); + + Assert.True(StringEncodedWriteVector.FlipBit(ref writeVector, "cluster:blue/1")); + Assert.Equal("v1:9:west,prod14:cluster:blue/1", writeVector); + } + + [Theory, TestCategory("EventSourcing"), TestCategory("BVT")] + [InlineData("v2:1:A")] + [InlineData("v1:")] + [InlineData("v1:x:A")] + [InlineData("v1:5:abc")] + [InlineData("missing-prefix")] + public void VersionedFormat_MalformedOrUnsupportedValueThrows(string writeVector) + { + Assert.Throws(() => StringEncodedWriteVector.GetBit(writeVector, "A")); + } + + private static bool GetBitUsingLegacyReader(string writeVector, string replica) + { + var position = writeVector.IndexOf(replica, StringComparison.Ordinal); + return position > 0 && writeVector[position - 1] == ','; + } +} diff --git a/test/Orleans.Journaling.Json.Tests/CodecRecoveryTests.cs b/test/Orleans.Journaling.Json.Tests/CodecRecoveryTests.cs index cf910fc581c..f95875845dc 100644 --- a/test/Orleans.Journaling.Json.Tests/CodecRecoveryTests.cs +++ b/test/Orleans.Journaling.Json.Tests/CodecRecoveryTests.cs @@ -40,7 +40,7 @@ public async Task OrleansBinaryCodec_WriteAndRecover() dict.Add("alpha", 1); dict.Add("beta", 2); dict.Add("gamma", 3); - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); // Recovery phase — new manager, same storage var sut2 = CreateTestSystem(storage); @@ -72,7 +72,7 @@ public async Task JsonCodec_WriteAndRecover() dict.Add("alpha", 1); dict.Add("beta", 2); - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); var journal = Encoding.UTF8.GetString(storage.Segments.Single()); Assert.Equal( """[0,["set","dict",8]]""" + "\n" + @@ -107,7 +107,7 @@ public async Task JsonCodec_DurableList_WriteAndRecover() list.Add("one"); list.Add("two"); list.Add("three"); - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); // Recovery phase var sut2 = CreateTestSystemWithJsonCodec(storage, jsonOptions); @@ -135,7 +135,7 @@ public async Task JsonCodec_DurableValue_WriteAndRecover() await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); value.Value = 42; - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); // Recovery phase var sut2 = CreateTestSystemWithJsonCodec(storage, jsonOptions); @@ -169,7 +169,7 @@ public async Task JsonCodec_DurableQueueSetStateAndTcs_WriteAndRecover() set.Add("b"); ((IStorage)state).State = "state-value"; Assert.True(tcs.TrySetResult(17)); - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); var sut2 = CreateTestSystemWithJsonCodec(storage, jsonOptions); var queue2 = new DurableQueue("queue", sut2.Manager, new JsonDurableQueueCommandCodec(jsonOptions)); @@ -201,7 +201,7 @@ public async Task Recovery_BinaryJournalWithJsonFormat_MigratesOnFirstWrite() var dict = CreateFormatAwareDictionary(first, OrleansBinaryJournalFormat.JournalFormatKey); await first.Lifecycle.OnStart(TestContext.Current.CancellationToken); dict.Add("alpha", 1); - await first.Manager.WriteStateAsync(CancellationToken.None); + await first.Manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Equal(OrleansBinaryJournalFormat.JournalFormatKey, storage.StoredJournalFormatKey); storage.SetConfiguredJournalFormatKey(JsonJournalExtensions.JournalFormatKey); @@ -212,7 +212,7 @@ public async Task Recovery_BinaryJournalWithJsonFormat_MigratesOnFirstWrite() Assert.Equal(1, recoveredDict["alpha"]); recoveredDict.Add("beta", 2); - await recovered.Manager.WriteStateAsync(CancellationToken.None); + await recovered.Manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Equal(JsonJournalExtensions.JournalFormatKey, storage.StoredJournalFormatKey); Assert.Single(storage.Segments); @@ -229,7 +229,7 @@ public async Task Recovery_JsonJournalWithBinaryFormat_MigratesOnFirstWrite() var dict = CreateFormatAwareDictionary(first, JsonJournalExtensions.JournalFormatKey); await first.Lifecycle.OnStart(TestContext.Current.CancellationToken); dict.Add("alpha", 1); - await first.Manager.WriteStateAsync(CancellationToken.None); + await first.Manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Equal(JsonJournalExtensions.JournalFormatKey, storage.StoredJournalFormatKey); storage.SetConfiguredJournalFormatKey(OrleansBinaryJournalFormat.JournalFormatKey); @@ -240,7 +240,7 @@ public async Task Recovery_JsonJournalWithBinaryFormat_MigratesOnFirstWrite() Assert.Equal(1, recoveredDict["alpha"]); recoveredDict.Add("beta", 2); - await recovered.Manager.WriteStateAsync(CancellationToken.None); + await recovered.Manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Equal(OrleansBinaryJournalFormat.JournalFormatKey, storage.StoredJournalFormatKey); Assert.Single(storage.Segments); @@ -252,6 +252,36 @@ public async Task Recovery_JsonJournalWithBinaryFormat_MigratesOnFirstWrite() Assert.Equal(2, finalDict["beta"]); } + [Fact] + public async Task Recovery_JsonJournalWithBinaryFormat_ReplacesWithoutAppendingDuringMigration() + { + var innerStorage = new VolatileJournalStorage(JsonJournalExtensions.JournalFormatKey); + using var first = CreateFormatAwareTestSystem(innerStorage, JsonJournalExtensions.JournalFormatKey); + var dict = CreateFormatAwareDictionary(first, JsonJournalExtensions.JournalFormatKey); + await first.Lifecycle.OnStart(TestContext.Current.CancellationToken); + dict.Add("alpha", 1); + await first.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + + innerStorage.SetConfiguredJournalFormatKey(OrleansBinaryJournalFormat.JournalFormatKey); + var storage = new CountingStorage(innerStorage); + using var recovered = CreateFormatAwareTestSystem(storage, OrleansBinaryJournalFormat.JournalFormatKey); + var recoveredDict = CreateFormatAwareDictionary(recovered, OrleansBinaryJournalFormat.JournalFormatKey); + await recovered.Lifecycle.OnStart(TestContext.Current.CancellationToken); + + recoveredDict.Add("beta", 2); + await recovered.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + + Assert.Equal(0, storage.AppendCount); + Assert.Equal(1, storage.ReplaceCount); + Assert.Equal(OrleansBinaryJournalFormat.JournalFormatKey, innerStorage.StoredJournalFormatKey); + + using var final = CreateFormatAwareTestSystem(innerStorage, OrleansBinaryJournalFormat.JournalFormatKey); + var finalDict = CreateFormatAwareDictionary(final, OrleansBinaryJournalFormat.JournalFormatKey); + await final.Lifecycle.OnStart(TestContext.Current.CancellationToken); + Assert.Equal(1, finalDict["alpha"]); + Assert.Equal(2, finalDict["beta"]); + } + [Fact] public async Task Recovery_MetadataLessJournal_UsesConfiguredFormat() { @@ -260,7 +290,7 @@ public async Task Recovery_MetadataLessJournal_UsesConfiguredFormat() var dict = CreateFormatAwareDictionary(first, JsonJournalExtensions.JournalFormatKey); await first.Lifecycle.OnStart(TestContext.Current.CancellationToken); dict.Add("alpha", 1); - await first.Manager.WriteStateAsync(CancellationToken.None); + await first.Manager.WriteStateAsync(TestContext.Current.CancellationToken); var metadataLessStorage = new MetadataOverridingStorage(storage, storedJournalFormatKey: null); using var recovered = CreateFormatAwareTestSystem(metadataLessStorage, JsonJournalExtensions.JournalFormatKey); @@ -270,7 +300,7 @@ public async Task Recovery_MetadataLessJournal_UsesConfiguredFormat() Assert.Equal(1, recoveredDict["alpha"]); recoveredDict.Add("beta", 2); - await recovered.Manager.WriteStateAsync(CancellationToken.None); + await recovered.Manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Equal(JsonJournalExtensions.JournalFormatKey, storage.StoredJournalFormatKey); Assert.Equal(2, storage.Segments.Count); @@ -287,7 +317,7 @@ public async Task Recovery_EmptyJournalWithStaleMetadata_WritesConfiguredFormat( await system.Lifecycle.OnStart(TestContext.Current.CancellationToken); dict.Add("alpha", 1); - await system.Manager.WriteStateAsync(CancellationToken.None); + await system.Manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Equal(JsonJournalExtensions.JournalFormatKey, storage.StoredJournalFormatKey); Assert.Contains("""[8,["set","alpha",1]]""", Encoding.UTF8.GetString(storage.Segments.Single()), StringComparison.Ordinal); @@ -301,7 +331,7 @@ public async Task Migration_WithUnregisteredRetiredState_ThrowsClearError() var dict = CreateFormatAwareDictionary(first, OrleansBinaryJournalFormat.JournalFormatKey, "dict"); await first.Lifecycle.OnStart(TestContext.Current.CancellationToken); dict.Add("alpha", 1); - await first.Manager.WriteStateAsync(CancellationToken.None); + await first.Manager.WriteStateAsync(TestContext.Current.CancellationToken); storage.SetConfiguredJournalFormatKey(JsonJournalExtensions.JournalFormatKey); using var recovered = CreateFormatAwareTestSystem(storage, JsonJournalExtensions.JournalFormatKey); @@ -310,7 +340,7 @@ public async Task Migration_WithUnregisteredRetiredState_ThrowsClearError() other.Add("beta", 2); var exception = await Assert.ThrowsAsync( - () => recovered.Manager.WriteStateAsync(CancellationToken.None).AsTask()); + () => recovered.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()); Assert.Contains("Cannot migrate journal", exception.Message, StringComparison.Ordinal); Assert.Contains("not currently registered", exception.Message, StringComparison.Ordinal); @@ -332,6 +362,83 @@ public async Task JsonRecovery_MalformedJournal_ThrowsFormatKeyError(string json Assert.Contains(expectedInnerMessage, exception.InnerException!.Message, StringComparison.Ordinal); } + [Fact] + public async Task Recovery_MigrationReplaceFails_PreservesOldMixedJournalAndRetriesWithoutAppend() + { + var innerStorage = new VolatileJournalStorage(JsonJournalExtensions.JournalFormatKey); + using (var first = CreateFormatAwareTestSystem(innerStorage, JsonJournalExtensions.JournalFormatKey)) + { + var list = CreateFormatAwareList(first, JsonJournalExtensions.JournalFormatKey); + var value = CreateFormatAwareValue(first, JsonJournalExtensions.JournalFormatKey); + await first.Lifecycle.OnStart(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + list.Add("baseline"); + value.Value = 10; + await first.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + list.Add("old-batch"); + value.Value = 20; + await first.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + } + + var oldCheckpoint = innerStorage.Segments.Select(static segment => segment.ToArray()).ToArray(); + innerStorage.SetConfiguredJournalFormatKey(OrleansBinaryJournalFormat.JournalFormatKey); + var expected = new IOException("Expected migration replacement failure."); + var storage = new FaultingCountingStorage(innerStorage) { NextReplaceException = expected }; + using var migrating = CreateFormatAwareTestSystem(storage, OrleansBinaryJournalFormat.JournalFormatKey); + var migratingList = CreateFormatAwareList(migrating, OrleansBinaryJournalFormat.JournalFormatKey); + var migratingValue = CreateFormatAwareValue(migrating, OrleansBinaryJournalFormat.JournalFormatKey); + await migrating.Lifecycle.OnStart(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + migratingList.Add("migration-pending"); + migratingValue.Value = 30; + var exception = await Assert.ThrowsAsync( + () => migrating.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + var failedCheckpoint = innerStorage.Segments.Select(static segment => segment.ToArray()).ToArray(); + var replaceAttemptsAfterFailure = storage.ReplaceAttempts; + var committedReplacesAfterFailure = storage.CommittedReplaces; + var appendAttemptsAfterFailure = storage.AppendAttempts; + var committedAppendsAfterFailure = storage.CommittedAppends; + var pendingAfterFailure = migrating.Manager.HasPendingWrites; + + var oldRecoveryStorage = new VolatileJournalStorage(JsonJournalExtensions.JournalFormatKey); + foreach (var segment in failedCheckpoint) + { + await oldRecoveryStorage.AppendAsync(new ReadOnlySequence(segment), TestContext.Current.CancellationToken); + } + + using var failedRecovery = CreateFormatAwareTestSystem(oldRecoveryStorage, JsonJournalExtensions.JournalFormatKey); + var failedList = CreateFormatAwareList(failedRecovery, JsonJournalExtensions.JournalFormatKey); + var failedValue = CreateFormatAwareValue(failedRecovery, JsonJournalExtensions.JournalFormatKey); + await failedRecovery.Lifecycle.OnStart(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + await migrating.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + using var final = CreateFormatAwareTestSystem(innerStorage, OrleansBinaryJournalFormat.JournalFormatKey); + var finalList = CreateFormatAwareList(final, OrleansBinaryJournalFormat.JournalFormatKey); + var finalValue = CreateFormatAwareValue(final, OrleansBinaryJournalFormat.JournalFormatKey); + await final.Lifecycle.OnStart(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + Assert.Same(expected, exception); + Assert.Equal(1, replaceAttemptsAfterFailure); + Assert.Equal(0, committedReplacesAfterFailure); + Assert.Equal(0, appendAttemptsAfterFailure); + Assert.Equal(0, committedAppendsAfterFailure); + Assert.Equal(oldCheckpoint.SelectMany(static segment => segment), failedCheckpoint.SelectMany(static segment => segment)); + Assert.Equal(["baseline", "old-batch"], failedList.ToArray()); + Assert.Equal(20, failedValue.Value); + Assert.True(pendingAfterFailure); + Assert.Equal(["baseline", "old-batch", "migration-pending"], migratingList.ToArray()); + Assert.Equal(30, migratingValue.Value); + Assert.Equal(2, storage.ReplaceAttempts); + Assert.Equal(1, storage.CommittedReplaces); + Assert.Equal(0, storage.AppendAttempts); + Assert.Equal(0, storage.CommittedAppends); + Assert.Equal(OrleansBinaryJournalFormat.JournalFormatKey, innerStorage.StoredJournalFormatKey); + Assert.Single(innerStorage.Segments); + Assert.Equal(["baseline", "old-batch", "migration-pending"], finalList.ToArray()); + Assert.Equal(30, finalValue.Value); + } + internal (IJournaledStateManager Manager, IJournalStorage Storage, ILifecycleSubject Lifecycle) CreateTestSystemWithJsonCodec(IJournalStorage? storage = null, System.Text.Json.JsonSerializerOptions? jsonOptions = null) { storage ??= CreateJsonStorage(); @@ -392,6 +499,14 @@ private FormatAwareTestSystem CreateFormatAwareTestSystem( typeof(IDurableDictionaryCommandCodec<,>), OrleansBinaryJournalFormat.JournalFormatKey, typeof(OrleansBinaryDurableDictionaryCommandCodec<,>)); + services.AddKeyedSingleton( + typeof(IDurableListCommandCodec<>), + OrleansBinaryJournalFormat.JournalFormatKey, + typeof(OrleansBinaryDurableListCommandCodec<>)); + services.AddKeyedSingleton( + typeof(IDurableValueCommandCodec<>), + OrleansBinaryJournalFormat.JournalFormatKey, + typeof(OrleansBinaryDurableValueCommandCodec<>)); var jsonOptions = CreateJsonOptions(); services.Configure(options => options.SerializerOptions = jsonOptions); @@ -400,6 +515,14 @@ private FormatAwareTestSystem CreateFormatAwareTestSystem( typeof(IDurableDictionaryCommandCodec<,>), JsonJournalExtensions.JournalFormatKey, typeof(JsonDurableDictionaryCommandCodecService<,>)); + services.AddKeyedSingleton( + typeof(IDurableListCommandCodec<>), + JsonJournalExtensions.JournalFormatKey, + typeof(JsonDurableListCommandCodecService<>)); + services.AddKeyedSingleton( + typeof(IDurableValueCommandCodec<>), + JsonJournalExtensions.JournalFormatKey, + typeof(JsonDurableValueCommandCodecService<>)); var serviceProvider = services.BuildServiceProvider(); var managerOptions = new JournaledStateManagerOptions @@ -429,6 +552,26 @@ private static DurableDictionary CreateFormatAwareDictionary( system.ServiceProvider, writeJournalFormatKey)); + private static DurableList CreateFormatAwareList( + FormatAwareTestSystem system, + string writeJournalFormatKey) + => new( + "list", + system.Manager, + JournalFormatServices.GetRequiredCommandCodec>( + system.ServiceProvider, + writeJournalFormatKey)); + + private static DurableValue CreateFormatAwareValue( + FormatAwareTestSystem system, + string writeJournalFormatKey) + => new( + "value", + system.Manager, + JournalFormatServices.GetRequiredCommandCodec>( + system.ServiceProvider, + writeJournalFormatKey)); + private OrleansBinaryDurableDictionaryCommandCodec CreateBinaryDictionaryCodec() where TKey : notnull => new(ValueCodec(), ValueCodec(), SessionPool); @@ -509,6 +652,74 @@ private IEnumerable> ReadSegments(CancellationToken cancell } } + private sealed class CountingStorage(VolatileJournalStorage inner) : IJournalStorage + { + public int AppendCount { get; private set; } + + public int ReplaceCount { get; private set; } + + public bool IsCompactionRequested => inner.IsCompactionRequested; + + public ValueTask AppendAsync(ReadOnlySequence value, CancellationToken cancellationToken) + { + AppendCount++; + return inner.AppendAsync(value, cancellationToken); + } + + public ValueTask DeleteAsync(CancellationToken cancellationToken) + => inner.DeleteAsync(cancellationToken); + + public ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken) + => inner.ReadAsync(consumer, cancellationToken); + + public ValueTask ReplaceAsync(ReadOnlySequence value, CancellationToken cancellationToken) + { + ReplaceCount++; + return inner.ReplaceAsync(value, cancellationToken); + } + } + + private sealed class FaultingCountingStorage(VolatileJournalStorage inner) : IJournalStorage + { + public int AppendAttempts { get; private set; } + + public int ReplaceAttempts { get; private set; } + + public int CommittedAppends { get; private set; } + + public int CommittedReplaces { get; private set; } + + public Exception? NextReplaceException { get; set; } + + public bool IsCompactionRequested => inner.IsCompactionRequested; + + public async ValueTask AppendAsync(ReadOnlySequence value, CancellationToken cancellationToken) + { + AppendAttempts++; + await inner.AppendAsync(value, cancellationToken); + CommittedAppends++; + } + + public ValueTask DeleteAsync(CancellationToken cancellationToken) + => inner.DeleteAsync(cancellationToken); + + public ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken) + => inner.ReadAsync(consumer, cancellationToken); + + public async ValueTask ReplaceAsync(ReadOnlySequence value, CancellationToken cancellationToken) + { + ReplaceAttempts++; + if (NextReplaceException is { } exception) + { + NextReplaceException = null; + throw exception; + } + + await inner.ReplaceAsync(value, cancellationToken); + CommittedReplaces++; + } + } + private sealed class FormatAwareTestSystem(ServiceProvider serviceProvider, JournaledStateManager manager, TestGrainLifecycle lifecycle) : IDisposable { public ServiceProvider ServiceProvider { get; } = serviceProvider; diff --git a/test/Orleans.Journaling.Tests/DurableStateAndTcsRecoveryTests.cs b/test/Orleans.Journaling.Tests/DurableStateAndTcsRecoveryTests.cs index 91004b3d90b..0109b495757 100644 --- a/test/Orleans.Journaling.Tests/DurableStateAndTcsRecoveryTests.cs +++ b/test/Orleans.Journaling.Tests/DurableStateAndTcsRecoveryTests.cs @@ -28,7 +28,7 @@ public async Task OrleansBinaryCodec_StateAndTcs_WriteAndRecover() ((IStorage)state).State = "state-value"; Assert.True(tcs.TrySetResult(17)); - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); var sut2 = CreateTestSystem(storage: sut.Storage); var state2 = new DurableState("state", sut2.Manager, new OrleansBinaryPersistentStateCommandCodec(ValueCodec(), SessionPool)); @@ -60,8 +60,8 @@ public async Task OrleansBinaryCodec_StateClear_WritesClearAndRecoversNoRecord() await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); grainState.State = "state-value"; - await grainState.WriteStateAsync(CancellationToken.None); - await grainState.ClearStateAsync(CancellationToken.None); + await grainState.WriteStateAsync(TestContext.Current.CancellationToken); + await grainState.ClearStateAsync(TestContext.Current.CancellationToken); Assert.Equal(1, codec.WriteClearCount); Assert.False(grainState.RecordExists); @@ -88,16 +88,121 @@ public async Task DurableTaskCompletionSource_DeleteState_ResetsToPending() Copier()); await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); Assert.True(tcs.TrySetResult(17)); - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Equal(17, await tcs.Task); - await sut.Manager.DeleteStateAsync(CancellationToken.None); + await sut.Manager.DeleteStateAsync(TestContext.Current.CancellationToken); Assert.Equal(DurableTaskCompletionSourceStatus.Pending, tcs.State.Status); Assert.False(tcs.Task.IsCompleted); Assert.True(tcs.TrySetResult(18)); } + [Fact] + public async Task DurableState_SetRetry_ReusesStagedCommand() + { + var storage = new RetryCapturingStorage(); + var codec = new TrackingPersistentStateCommandCodec(ValueCodec(), SessionPool); + var sut = CreateTestSystem(storage: storage); + var state = new DurableState("state", sut.Manager, codec); + var grainState = (IStorage)state; + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); + + grainState.State = "state-value"; + storage.FailNextAppend(); + await Assert.ThrowsAsync(() => grainState.WriteStateAsync(TestContext.Current.CancellationToken)); + var firstAttempt = Assert.Single(storage.AppendAttempts); + + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + + Assert.Equal(1, codec.WriteSetCount); + Assert.Equal(2, storage.AppendAttempts.Count); + Assert.Equal(firstAttempt, storage.AppendAttempts[1]); + + var recovered = CreateTestSystem(storage: storage); + var recoveredState = new DurableState( + "state", + recovered.Manager, + new OrleansBinaryPersistentStateCommandCodec(ValueCodec(), SessionPool)); + await recovered.Lifecycle.OnStart(TestContext.Current.CancellationToken); + Assert.Equal("state-value", ((IStorage)recoveredState).State); + } + + [Fact] + public async Task DurableState_ClearRetry_ReusesStagedCommand() + { + var storage = new RetryCapturingStorage(); + var codec = new TrackingPersistentStateCommandCodec(ValueCodec(), SessionPool); + var sut = CreateTestSystem(storage: storage); + var state = new DurableState("state", sut.Manager, codec); + var grainState = (IStorage)state; + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); + grainState.State = "state-value"; + await grainState.WriteStateAsync(TestContext.Current.CancellationToken); + storage.ClearAttempts(); + + storage.FailNextAppend(); + await Assert.ThrowsAsync(() => grainState.ClearStateAsync(TestContext.Current.CancellationToken)); + var firstAttempt = Assert.Single(storage.AppendAttempts); + + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + + Assert.Equal(1, codec.WriteClearCount); + Assert.Equal(2, storage.AppendAttempts.Count); + Assert.Equal(firstAttempt, storage.AppendAttempts[1]); + + var recovered = CreateTestSystem(storage: storage); + var recoveredState = new DurableState( + "state", + recovered.Manager, + new OrleansBinaryPersistentStateCommandCodec(ValueCodec(), SessionPool)); + await recovered.Lifecycle.OnStart(TestContext.Current.CancellationToken); + Assert.False(((IStorage)recoveredState).RecordExists); + } + + [Fact] + public async Task DurableTaskCompletionSource_Retry_ReusesStagedCommand() + { + var storage = new RetryCapturingStorage(); + var codec = new TrackingTaskCompletionSourceCommandCodec( + ValueCodec(), + ValueCodec(), + SessionPool); + var sut = CreateTestSystem(storage: storage); + var tcs = new DurableTaskCompletionSource( + "tcs", + sut.Manager, + codec, + Copier(), + Copier()); + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); + + Assert.True(tcs.TrySetResult(17)); + storage.FailNextAppend(); + await Assert.ThrowsAsync(() => sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()); + var firstAttempt = Assert.Single(storage.AppendAttempts); + + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + + Assert.Equal(1, codec.WriteCompletedCount); + Assert.Equal(2, storage.AppendAttempts.Count); + Assert.Equal(firstAttempt, storage.AppendAttempts[1]); + Assert.Equal(17, await tcs.Task); + + var recovered = CreateTestSystem(storage: storage); + var recoveredTcs = new DurableTaskCompletionSource( + "tcs", + recovered.Manager, + new OrleansBinaryDurableTaskCompletionSourceCommandCodec( + ValueCodec(), + ValueCodec(), + SessionPool), + Copier(), + Copier()); + await recovered.Lifecycle.OnStart(TestContext.Current.CancellationToken); + Assert.Equal(17, await recoveredTcs.Task); + } + private IFieldCodec ValueCodec() => CodecProvider.GetCodec(); private DeepCopier Copier() => ServiceProvider.GetRequiredService>(); @@ -108,7 +213,13 @@ private sealed class TrackingPersistentStateCommandCodec(IFieldCodec value public int WriteClearCount { get; private set; } - public void WriteSet(T state, ulong version, JournalStreamWriter writer) => _inner.WriteSet(state, version, writer); + public int WriteSetCount { get; private set; } + + public void WriteSet(T state, ulong version, JournalStreamWriter writer) + { + WriteSetCount++; + _inner.WriteSet(state, version, writer); + } public void WriteClear(JournalStreamWriter writer) { @@ -118,4 +229,63 @@ public void WriteClear(JournalStreamWriter writer) public void Apply(JournalBufferReader input, IPersistentStateCommandHandler consumer) => _inner.Apply(input, consumer); } + + private sealed class TrackingTaskCompletionSourceCommandCodec( + IFieldCodec valueCodec, + IFieldCodec exceptionCodec, + SerializerSessionPool sessionPool) : IDurableTaskCompletionSourceCommandCodec + { + private readonly OrleansBinaryDurableTaskCompletionSourceCommandCodec _inner = new(valueCodec, exceptionCodec, sessionPool); + + public int WriteCompletedCount { get; private set; } + + public void Apply(JournalBufferReader input, IDurableTaskCompletionSourceCommandHandler consumer) => _inner.Apply(input, consumer); + + public void WritePending(JournalStreamWriter writer) => _inner.WritePending(writer); + + public void WriteCompleted(T value, JournalStreamWriter writer) + { + WriteCompletedCount++; + _inner.WriteCompleted(value, writer); + } + + public void WriteFaulted(Exception exception, JournalStreamWriter writer) => _inner.WriteFaulted(exception, writer); + + public void WriteCanceled(JournalStreamWriter writer) => _inner.WriteCanceled(writer); + } + + private sealed class RetryCapturingStorage : IJournalStorage + { + private readonly VolatileJournalStorage _inner = new(); + private bool _failNextAppend; + + public List AppendAttempts { get; } = []; + + public bool IsCompactionRequested => false; + + public void FailNextAppend() => _failNextAppend = true; + + public void ClearAttempts() => AppendAttempts.Clear(); + + public ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken) => + _inner.ReadAsync(consumer, cancellationToken); + + public ValueTask ReplaceAsync(ReadOnlySequence value, CancellationToken cancellationToken) => + _inner.ReplaceAsync(value, cancellationToken); + + public async ValueTask AppendAsync(ReadOnlySequence value, CancellationToken cancellationToken) + { + var bytes = value.ToArray(); + AppendAttempts.Add(bytes); + if (_failNextAppend) + { + _failNextAppend = false; + throw new IOException("Expected append failure."); + } + + await _inner.AppendAsync(new ReadOnlySequence(bytes), cancellationToken); + } + + public ValueTask DeleteAsync(CancellationToken cancellationToken) => _inner.DeleteAsync(cancellationToken); + } } diff --git a/test/Orleans.Journaling.Tests/JournalBatchTests.cs b/test/Orleans.Journaling.Tests/JournalBatchTests.cs index e5637b195ec..89e0df5ab51 100644 --- a/test/Orleans.Journaling.Tests/JournalBatchTests.cs +++ b/test/Orleans.Journaling.Tests/JournalBatchTests.cs @@ -188,6 +188,26 @@ public async Task DurableList_Persistence_Test() Assert.Equal("three", list2[2]); } + [Fact] + public async Task RevertPendingChangesAsync_ReplacesUnflushedChanges() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + var grainId = GrainId.Create("test-grain", $"readState-{Guid.NewGuid()}"); + var (manager, list, _) = CreateTestComponents("readStateList", grainId); + await manager.InitializeAsync(cts.Token); + + list.Add("persisted"); + await manager.WriteStateAsync(cts.Token); + + list.Add("unflushed"); + Assert.Equal(2, list.Count); + + await manager.RevertPendingChangesAsync(cts.Token); + + Assert.Single(list); + Assert.Equal("persisted", list[0]); + } + /// /// Tests storing and retrieving complex objects, including updates to mutable properties. /// diff --git a/test/Orleans.Journaling.Tests/StateManagerTests.cs b/test/Orleans.Journaling.Tests/StateManagerTests.cs index 4345ee51e88..c4450c0e33b 100644 --- a/test/Orleans.Journaling.Tests/StateManagerTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerTests.cs @@ -2,8 +2,11 @@ using System.Collections.Concurrent; using System.Diagnostics.Metrics; using System.Runtime.ExceptionServices; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Time.Testing; +using Orleans.Core; +using Orleans.Serialization; using Orleans.Serialization.Buffers; using Orleans.Serialization.Buffers.Adaptors; using Orleans.Serialization.Session; @@ -50,7 +53,7 @@ public async Task StateManager_RegisterState_Test() queue.Enqueue(42); // Write state - await manager.WriteStateAsync(CancellationToken.None); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); // Assert - Data is correctly stored Assert.Equal(1, dictionary["key1"]); @@ -75,7 +78,7 @@ public async Task StateManager_Initialize_ThrowsWhenCompletedReadLeavesData() var storage = new VolatileJournalStorage(); using (var data = CreateBuffer([1, 2, 3])) { - await storage.AppendAsync(data.AsReadOnlySequence(), CancellationToken.None); + await storage.AppendAsync(data.AsReadOnlySequence(), TestContext.Current.CancellationToken); } var sut = CreateTestSystem(storage: storage, journalFormat: new NonConsumingJournalFormat()); @@ -94,9 +97,9 @@ public async Task StateManager_WriteOperations_RequireInitialization() var sut = CreateTestSystem(); var writeException = await Assert.ThrowsAsync( - () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask()); + () => sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()); var deleteException = await Assert.ThrowsAsync( - () => sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask()); + () => sut.Manager.DeleteStateAsync(TestContext.Current.CancellationToken).AsTask()); Assert.Contains("not been initialized", writeException.Message, StringComparison.Ordinal); Assert.Contains("not been initialized", deleteException.Message, StringComparison.Ordinal); @@ -150,12 +153,12 @@ public async Task StateManager_Recovery_PreservesUnknownStateEntry() using var data = segment.GetBuffer(); var originalBytes = data.ToArray(); - await storage.AppendAsync(data.AsReadOnlySequence(), CancellationToken.None); + await storage.AppendAsync(data.AsReadOnlySequence(), TestContext.Current.CancellationToken); var sut = CreateTestSystem(storage: storage); await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken) .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); var segmentBytes = Assert.Single(storage.Segments); Assert.Equal(originalBytes, segmentBytes); @@ -169,7 +172,7 @@ public async Task StateManager_UnknownStateCompaction_PreservesDecodedPayloadThr var storage = new CapturingStorage { IsCompactionRequested = true }; using (var data = CreateBuffer(physicalBytes)) { - await storage.AppendAsync(data.AsReadOnlySequence(), CancellationToken.None); + await storage.AppendAsync(data.AsReadOnlySequence(), TestContext.Current.CancellationToken); } var format = new DecodedPayloadOnlyJournalFormat(new JournalStreamId(99), decodedPayload, SessionPool); @@ -177,7 +180,7 @@ public async Task StateManager_UnknownStateCompaction_PreservesDecodedPayloadThr await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken) .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); var replacement = Assert.Single(storage.Replaces); Assert.NotEqual(physicalBytes, replacement); @@ -198,14 +201,14 @@ public async Task StateManager_RetiredStateCompaction_WritesPreservedPayloadThro await initial.Lifecycle.OnStart(TestContext.Current.CancellationToken); dictionary.Add("key", 7); - await initial.Manager.WriteStateAsync(CancellationToken.None); + await initial.Manager.WriteStateAsync(TestContext.Current.CancellationToken); storage.IsCompactionRequested = true; var format = new TrackingJournalFormat(SessionPool); var compacting = CreateTestSystem(storage: storage, journalFormat: format); await compacting.Lifecycle.OnStart(TestContext.Current.CancellationToken); - await compacting.Manager.WriteStateAsync(CancellationToken.None); + await compacting.Manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Contains(format.Writers, writer => writer.BeganEntryIds.Any(id => id >= 8)); Assert.Single(storage.Replaces); @@ -227,7 +230,7 @@ public async Task StateManager_DirectWrites_UseFormatOwnedCurrentSegmentWriter() await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); dictionary.Add("key", 1); - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); var writer = Assert.Single(format.Writers); Assert.Single(storage.Appends); @@ -246,7 +249,7 @@ public async Task StateManager_AppendJournalFlush_UsesFormatOwnedWriter() await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); value.Value = 42; - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); var writer = Assert.Single(format.Writers); Assert.Single(storage.Appends); @@ -263,7 +266,7 @@ public async Task StateManager_BinaryAppend_StoresBinaryVarUIntEntries() await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); dictionary.Add("key", 1); - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); var append = Assert.Single(storage.Appends); Assert.Empty(storage.Replaces); @@ -279,7 +282,7 @@ public async Task StateManager_AppendBufferIsBorrowedUntilStorageCompletes() await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); value.Value = 42; - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.NotNull(storage.AppendBytesAfterYield); Assert.NotEmpty(storage.AppendBytesAfterYield); @@ -294,12 +297,118 @@ public async Task StateManager_ReplaceBufferIsBorrowedUntilStorageCompletes() await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); value.Value = 42; - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.NotNull(storage.ReplaceBytesAfterYield); Assert.NotEmpty(storage.ReplaceBytesAfterYield); } + [Fact] + public async Task StateManager_SnapshotCompletionPreservesReentrantDurableValueMutation() + { + var storage = new CapturingStorage { IsCompactionRequested = true, DelayReplace = true }; + var sut = CreateTestSystem(storage: storage); + var value = new DurableValue("value", sut.Manager, CreateValueCodec()); + + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); + value.Value = 1; + var firstWrite = sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await storage.ReplaceStarted.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + value.Value = 2; + storage.AllowReplace.SetResult(); + await firstWrite.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + Assert.True(sut.Manager.HasPendingWrites); + storage.IsCompactionRequested = false; + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + + var recovered = CreateTestSystem(storage: storage); + var recoveredValue = new DurableValue("value", recovered.Manager, CreateValueCodec()); + await recovered.Lifecycle.OnStart(TestContext.Current.CancellationToken); + Assert.Equal(2, recoveredValue.Value); + } + + [Fact] + public async Task StateManager_SnapshotCompletionPreservesReentrantDurableStateClear() + { + var storage = new CapturingStorage { IsCompactionRequested = true, DelayReplace = true }; + var sut = CreateTestSystem(storage: storage); + var state = new DurableState( + "state", + sut.Manager, + new OrleansBinaryPersistentStateCommandCodec(CodecProvider.GetCodec(), SessionPool)); + + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); + ((IStorage)state).State = "value"; + var firstWrite = sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await storage.ReplaceStarted.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + storage.IsCompactionRequested = false; + var clear = ((IStorage)state).ClearStateAsync(TestContext.Current.CancellationToken); + storage.AllowReplace.SetResult(); + await Task.WhenAll(firstWrite, clear).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + var recovered = CreateTestSystem(storage: storage); + var recoveredState = new DurableState( + "state", + recovered.Manager, + new OrleansBinaryPersistentStateCommandCodec(CodecProvider.GetCodec(), SessionPool)); + await recovered.Lifecycle.OnStart(TestContext.Current.CancellationToken); + Assert.False(((IStorage)recoveredState).RecordExists); + } + + [Fact] + public async Task StateManager_SnapshotDoesNotCompleteDurableTaskBeforeReplace() + { + var storage = new CapturingStorage { IsCompactionRequested = true, DelayReplace = true }; + var sut = CreateTestSystem(storage: storage); + var completion = new DurableTaskCompletionSource( + "completion", + sut.Manager, + new OrleansBinaryDurableTaskCompletionSourceCommandCodec( + CodecProvider.GetCodec(), + CodecProvider.GetCodec(), + SessionPool), + ServiceProvider.GetRequiredService>(), + ServiceProvider.GetRequiredService>()); + + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); + Assert.True(completion.TrySetResult(42)); + var write = sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await storage.ReplaceStarted.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + Assert.False(completion.Task.IsCompleted); + + storage.AllowReplace.SetResult(); + await write.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(42, await completion.Task); + } + + [Fact] + public async Task StateManager_FailedSnapshotPreservesDurableValuePendingState() + { + var expected = new IOException("Expected snapshot failure."); + var storage = new CapturingStorage { IsCompactionRequested = true, NextReplaceException = expected }; + var sut = CreateTestSystem(storage: storage); + var value = new DurableValue("value", sut.Manager, CreateValueCodec()); + + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); + value.Value = 42; + + var exception = await Assert.ThrowsAsync(() => sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()); + Assert.Same(expected, exception); + Assert.True(sut.Manager.HasPendingWrites); + + storage.IsCompactionRequested = false; + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + + var recovered = CreateTestSystem(storage: storage); + var recoveredValue = new DurableValue("value", recovered.Manager, CreateValueCodec()); + await recovered.Lifecycle.OnStart(TestContext.Current.CancellationToken); + Assert.Equal(42, recoveredValue.Value); + } + [Fact] public async Task StateManager_BinarySnapshot_StoresBinaryVarUIntEntries() { @@ -309,7 +418,7 @@ public async Task StateManager_BinarySnapshot_StoresBinaryVarUIntEntries() await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); dictionary.Add("key", 1); - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); var replacement = Assert.Single(storage.Replaces); Assert.Empty(storage.Appends); @@ -327,10 +436,11 @@ public async Task StateManager_StorageOperationBytesMetric_RecordsWritePayloadSi await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); value.Value = 42; - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); var replacement = Assert.Single(storage.Replaces); Assert.Empty(storage.Appends); + Assert.DoesNotContain(observedBytes, measurement => measurement.Operation == "append"); Assert.Contains(observedBytes, measurement => measurement.Operation == "replace" && measurement.Status == "ok" && measurement.Value == replacement.Length); } @@ -360,10 +470,10 @@ public async Task StateManager_StorageOperationQueueDurationMetric_RecordsWaitBe sut.Manager.RegisterState("state", state); await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); - var firstWrite = sut.Manager.WriteStateAsync(CancellationToken.None).AsTask(); + var firstWrite = sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); await storage.FirstAppendStarted.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - var secondWrite = sut.Manager.WriteStateAsync(CancellationToken.None).AsTask(); + var secondWrite = sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); timeProvider.Advance(TimeSpan.FromMilliseconds(250)); storage.AllowFirstAppend.SetResult(); @@ -382,11 +492,11 @@ public async Task StateManager_DeleteState_ReallocatesApplicationStreamsAboveInt await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); dictionary.Add("before", 1); - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); - await sut.Manager.DeleteStateAsync(CancellationToken.None); + await sut.Manager.DeleteStateAsync(TestContext.Current.CancellationToken); dictionary.Add("after", 2); - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Equal(2, storage.Appends.Count); var entries = ReadBinaryEntries(storage.Appends[^1]); @@ -410,11 +520,11 @@ public async Task StateManager_DeleteState_ClearsDurableValueDirtyFlag() await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); value.Value = 1; - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); value.Value = 2; - await sut.Manager.DeleteStateAsync(CancellationToken.None); - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.DeleteStateAsync(TestContext.Current.CancellationToken); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Equal(2, storage.Appends.Count); var postDeleteEntries = ReadBinaryEntries(storage.Appends[^1]); @@ -432,7 +542,7 @@ public async Task StateManager_SnapshotFlush_ResetsPendingAppendData() await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); dictionary.Add("key", 1); - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Empty(storage.Appends); Assert.Single(storage.Replaces); @@ -514,7 +624,7 @@ public async Task StateManager_DirectWriteFailure_AbortsEntryBeforeMutation() Assert.Empty(dictionary); Assert.Equal(lengthBefore, GetCommittedLength(writer)); - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); } [Fact] @@ -532,7 +642,7 @@ public async Task StateManager_DirectWriteFailure_DoesNotPersistPartialEntry() codec.ThrowOnSet = false; dictionary.Add(1, 1); - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); var recovered = CreateTestSystem(storage: storage, journalFormat: new TrackingJournalFormat(SessionPool)); var recoveredDictionary = new DurableDictionary("dict", recovered.Manager, CreateDictionaryCodec()); @@ -551,14 +661,14 @@ public async Task StateManager_WriteStateAsync_RecoversAfterInconsistentStateExc await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); dictionary.Add("first", 1); - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); var expected = new InconsistentStateException("Expected storage write conflict."); storage.NextAppendException = expected; storage.BlockNextRead = true; dictionary.Add("second", 2); - var failedWrite = sut.Manager.WriteStateAsync(CancellationToken.None).AsTask(); + var failedWrite = sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); await storage.BlockedReadStarted.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); Assert.False(failedWrite.IsCompleted); @@ -590,7 +700,7 @@ public async Task StateManager_WriteStateAsync_PreservesPendingStateAfterTransie await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); dictionary.Add("first", 1); - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); storage.ResetReadConsumeCount(); var expected = new IOException("Expected storage write failure."); @@ -598,7 +708,7 @@ public async Task StateManager_WriteStateAsync_PreservesPendingStateAfterTransie dictionary.Add("second", 2); var exception = await Assert.ThrowsAsync( - () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask()); + () => sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()); Assert.Same(expected, exception); await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask() @@ -624,7 +734,7 @@ public async Task StateManager_WriteStateAsync_RetriesRecoveryAfterRepeatedFailu await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); dictionary.Add("first", 1); - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); var conflict = new InconsistentStateException("Expected storage write conflict."); var firstRecoveryFailure = new IOException("Expected first recovery failure."); @@ -738,6 +848,54 @@ await storage.RecoveryReadStarted.Task.WaitAsync( await recovery.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); } + [Fact] + public async Task StateManager_InitializeAsync_CallerCancellationAfterStart_WaitsForRecovery() + { + var storage = new CapturingStorage { BlockNextRead = true }; + var sut = CreateTestSystem(storage: storage); + using var cancellation = new CancellationTokenSource(); + var cancellationSignaled = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var registration = cancellation.Token.Register(static state => ((TaskCompletionSource)state!).TrySetResult(), cancellationSignaled); + + var initialize = sut.Manager.InitializeAsync(cancellation.Token).AsTask(); + await storage.BlockedReadStarted.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + cancellation.Cancel(); + await cancellationSignaled.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.False(initialize.IsCompleted); + + storage.AllowBlockedRead.SetResult(); + await initialize.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + } + + [Fact] + public async Task StateManager_DeleteStateAsync_CallerCancellationAfterStart_WaitsForDeletion() + { + var storage = new CapturingStorage(); + var sut = CreateTestSystem(storage: storage); + var dictionary = new DurableDictionary("dict", sut.Manager, CreateDictionaryCodec()); + + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); + dictionary.Add("persisted", 1); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + storage.BlockNextDelete = true; + + using var cancellation = new CancellationTokenSource(); + var cancellationSignaled = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var registration = cancellation.Token.Register(static state => ((TaskCompletionSource)state!).TrySetResult(), cancellationSignaled); + var delete = sut.Manager.DeleteStateAsync(cancellation.Token).AsTask(); + await storage.DeleteStarted.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + cancellation.Cancel(); + await cancellationSignaled.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.False(delete.IsCompleted); + + storage.AllowDelete.SetResult(); + await delete.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(1, storage.DeleteCount); + Assert.Empty(dictionary); + } + [Fact] public async Task StateManager_WriteStateAsync_CoalescesQueuedWrites() { @@ -857,7 +1015,7 @@ Task StartSnapshotAndCommitActiveEntry() try { entry.Writer.Write(new byte[] { 1, 2, 3 }); - writeTask = sut.Manager.WriteStateAsync(CancellationToken.None).AsTask(); + writeTask = sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); Assert.True(SpinWait.SpinUntil(() => storage.ReplaceStarted.Task.IsCompleted, TimeSpan.FromSeconds(10)), writeTask.Exception?.ToString()); Assert.False(writeTask.IsCompleted); @@ -874,6 +1032,33 @@ Task StartSnapshotAndCommitActiveEntry() } } + [Fact] + public async Task StateManager_DirectWritingState_DefaultPendingFlagUsesJournalBuffer() + { + var storage = new CapturingStorage(); + var sut = CreateTestSystem(storage: storage); + var state = new ManualDirectWriteState(); + sut.Manager.RegisterState("manual", state); + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + + Assert.False(sut.Manager.HasPendingWrites); + + using (var entry = state.BeginEntry()) + { + entry.Writer.Write(new byte[] { 1, 2, 3 }); + entry.Commit(); + } + state.MarkEntryClosing(); + + Assert.True(sut.Manager.HasPendingWrites); + + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + + Assert.False(sut.Manager.HasPendingWrites); + Assert.Equal(2, storage.Appends.Count); + } + /// /// Tests that all registered states are correctly recovered together. /// Verifies that the manager maintains consistency across multiple collections @@ -895,7 +1080,7 @@ public async Task StateManager_StateRecovery_Test() list.Add("item1"); list.Add("item2"); - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); // Act - Create new manager with same storage var sut2 = CreateTestSystem(storage: sut.Storage); @@ -922,7 +1107,7 @@ public async Task StateManager_Recovery_UsesSelectedJournalFormatRead() await initial.Lifecycle.OnStart(TestContext.Current.CancellationToken); value.Value = 42; - await initial.Manager.WriteStateAsync(CancellationToken.None); + await initial.Manager.WriteStateAsync(TestContext.Current.CancellationToken); var format = new TrackingJournalFormat(SessionPool); var recovered = CreateTestSystem(storage: storage, journalFormat: format); @@ -944,7 +1129,7 @@ public async Task StateManager_RegisterState_AfterRecovery_AllocatesAboveRecover AppendDirectorySet(segment, "existing", recoveredStreamId); CreateValueCodec().WriteSet(42, segment.CreateJournalStreamWriter(recoveredStreamId)); using var committed = segment.GetBuffer(); - await storage.AppendAsync(committed.AsReadOnlySequence(), CancellationToken.None); + await storage.AppendAsync(committed.AsReadOnlySequence(), TestContext.Current.CancellationToken); } var sut = CreateTestSystem(storage: storage); @@ -953,7 +1138,7 @@ public async Task StateManager_RegisterState_AfterRecovery_AllocatesAboveRecover await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); next.Value = 99; - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Equal(42, existing.Value); var entries = ReadBinaryEntries(storage.Appends[^1]); @@ -969,9 +1154,9 @@ public async Task StateManager_Recovery_ReadsConcatenatedJournalData() await initial.Lifecycle.OnStart(TestContext.Current.CancellationToken); value.Value = 1; - await initial.Manager.WriteStateAsync(CancellationToken.None); + await initial.Manager.WriteStateAsync(TestContext.Current.CancellationToken); value.Value = 2; - await initial.Manager.WriteStateAsync(CancellationToken.None); + await initial.Manager.WriteStateAsync(TestContext.Current.CancellationToken); storage.ConcatenateReads = true; var recovered = CreateTestSystem(storage: storage); @@ -992,9 +1177,9 @@ public async Task StateManager_Recovery_BuffersEntriesSplitAcrossStorageChunks() await initial.Lifecycle.OnStart(TestContext.Current.CancellationToken); value.Value = 1; - await initial.Manager.WriteStateAsync(CancellationToken.None); + await initial.Manager.WriteStateAsync(TestContext.Current.CancellationToken); value.Value = 2; - await initial.Manager.WriteStateAsync(CancellationToken.None); + await initial.Manager.WriteStateAsync(TestContext.Current.CancellationToken); var persistedBytes = storage.Appends.SelectMany(static segment => segment).ToArray(); var splitStorage = new ChunkedReadStorage(persistedBytes, chunkSize: 1); @@ -1035,7 +1220,7 @@ public async Task StateManager_Recovery_RejectsMalformedTrailingData() } finally { - await sut.Lifecycle.OnStop(CancellationToken.None); + await sut.Lifecycle.OnStop(TestContext.Current.CancellationToken); } Assert.Contains("journal format key 'orleans-binary'", exception.Message, StringComparison.Ordinal); @@ -1123,7 +1308,7 @@ public async Task StateManager_Recovery_DoesNotWrapStorageReadException() } finally { - await sut.Lifecycle.OnStop(CancellationToken.None); + await sut.Lifecycle.OnStop(TestContext.Current.CancellationToken); } Assert.Same(storage.Exception, exception); @@ -1146,16 +1331,16 @@ public async Task StateManager_MultipleWriteStates_Test() // Act - Multiple operations with WriteState in between dictionary.Add("key1", 1); - await manager.WriteStateAsync(CancellationToken.None); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); dictionary.Add("key2", 2); - await manager.WriteStateAsync(CancellationToken.None); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); dictionary["key1"] = 10; - await manager.WriteStateAsync(CancellationToken.None); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); dictionary.Remove("key2"); - await manager.WriteStateAsync(CancellationToken.None); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); // Assert - Final state is correct Assert.Single(dictionary); @@ -1200,7 +1385,7 @@ public async Task StateManager_MultipleStates_Test() personValue.Value = new TestPerson { Id = 100, Name = "Test Person", Age = 30 }; - await manager.WriteStateAsync(CancellationToken.None); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); // Assert - All should have correct values Assert.Equal(2, intDict.Count); @@ -1253,7 +1438,7 @@ public async Task StateManager_Concurrency_Test() dict1.Add("key2", 2); dict2.Add("key2", 200); - await manager.WriteStateAsync(CancellationToken.None); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); // Assert - Both states should have their correct values Assert.Equal(2, dict1.Count); @@ -1286,7 +1471,7 @@ public async Task StateManager_LargeStateRecovery_Test() largeDict.Add(i, $"Value {i}"); } - await sut.Manager.WriteStateAsync(CancellationToken.None); + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken); // Create new manager for recovery var sut2 = CreateTestSystem(storage: sut.Storage); @@ -1328,7 +1513,7 @@ public async Task StateManager_AutoRetiringStates() dictToKeep1.Add("a", 1); dictToRetire2.Add("b", 1); - await sut1.Manager.WriteStateAsync(CancellationToken.None); + await sut1.Manager.WriteStateAsync(TestContext.Current.CancellationToken); // -------------- STEP 2 -------------- @@ -1423,11 +1608,478 @@ static async Task TriggerCompaction(IJournaledStateManager manager, DurableDicti for (var i = 0; i < 11; i++) { dict["a"] = i; - await manager.WriteStateAsync(CancellationToken.None); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); } } } + [Fact] + public async Task WriteStateAsync_OrdinarySnapshotReplaceFails_DoesNotAppendAndRetryRecoversMixedStateExactlyOnce() + { + var storage = new CheckpointingJournalStorage(); + var sut = CreateMixedStateTestSystem(storage); + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + sut.List.Add("baseline"); + sut.Value.Value = 10; + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var baselineCheckpoint = storage.GetDurableCheckpoint(); + storage.ClearOperationHistory(); + + sut.List.Add("new"); + sut.Value.Value = 20; + storage.IsCompactionRequested = true; + var expected = new IOException("Expected ordinary snapshot replacement failure."); + storage.NextReplaceException = expected; + + var exception = await Assert.ThrowsAsync( + () => sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + var pendingAfterFailure = sut.Manager.HasPendingWrites; + var pendingBytesAfterFailure = sut.Manager.PendingWriteByteCount; + var failedCheckpoint = storage.GetDurableCheckpoint(); + var appendAttemptsAfterFailure = storage.AppendAttempts.Count; + var committedAppendsAfterFailure = storage.CommittedAppends.Count; + var replaceAttemptsAfterFailure = storage.ReplaceAttempts.Count; + var committedReplacesAfterFailure = storage.CommittedReplaces.Count; + var failedRecovery = await RecoverMixedStateAsync(new CheckpointingJournalStorage(failedCheckpoint)); + + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var finalRecovery = await RecoverMixedStateAsync(storage.CreateRecoveryStorage()); + + Assert.Same(expected, exception); + Assert.Equal(1, replaceAttemptsAfterFailure); + Assert.Equal(0, committedReplacesAfterFailure); + Assert.Equal(0, appendAttemptsAfterFailure); + Assert.Equal(0, committedAppendsAfterFailure); + Assert.True(pendingAfterFailure); + Assert.True(pendingBytesAfterFailure > 0); + Assert.Equal(Flatten(baselineCheckpoint), Flatten(failedCheckpoint)); + Assert.Equal(["baseline"], failedRecovery.List.ToArray()); + Assert.Equal(10, failedRecovery.Value.Value); + Assert.Equal(2, storage.ReplaceAttempts.Count); + Assert.Single(storage.CommittedReplaces); + Assert.Empty(storage.AppendAttempts); + Assert.Empty(storage.CommittedAppends); + Assert.Equal(["baseline", "new"], finalRecovery.List.ToArray()); + Assert.Equal(20, finalRecovery.Value.Value); + } + + [Fact] + public async Task WriteStateAsync_SnapshotConflictRecoversWinningStateWithoutPartialAppend() + { + var storage = new CheckpointingJournalStorage(); + var sut = CreateMixedStateTestSystem(storage); + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + sut.List.Add("baseline"); + sut.Value.Value = 10; + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var baselineCheckpoint = storage.GetDurableCheckpoint(); + storage.ClearOperationHistory(); + + sut.List.Add("losing"); + sut.Value.Value = 20; + storage.IsCompactionRequested = true; + var expected = new InconsistentStateException("Expected snapshot conflict."); + storage.NextReplaceException = expected; + + var exception = await Assert.ThrowsAsync( + () => sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + + Assert.Same(expected, exception); + Assert.Single(storage.ReplaceAttempts); + Assert.Empty(storage.CommittedReplaces); + Assert.Empty(storage.AppendAttempts); + Assert.Empty(storage.CommittedAppends); + Assert.Equal(Flatten(baselineCheckpoint), Flatten(storage.GetDurableCheckpoint())); + Assert.Equal(["baseline"], sut.List.ToArray()); + Assert.Equal(10, sut.Value.Value); + Assert.False(sut.Manager.HasPendingWrites); + } + + [Fact] + public async Task WriteStateAsync_SnapshotConflictWithEmptyWinnerResetsAllRegisteredState() + { + var storage = new CheckpointingJournalStorage(); + var sut = CreateMixedStateTestSystem(storage); + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + sut.List.Add("baseline"); + sut.Value.Value = 10; + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + storage.ClearOperationHistory(); + + sut.List.Add("losing"); + sut.Value.Value = 20; + storage.IsCompactionRequested = true; + storage.ClearDurableCheckpoint(); + var expected = new InconsistentStateException("Expected conflict with an empty winning generation."); + storage.NextReplaceException = expected; + + var exception = await Assert.ThrowsAsync( + () => sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + + Assert.Same(expected, exception); + Assert.Empty(storage.GetDurableCheckpoint()); + Assert.Empty(storage.AppendAttempts); + Assert.Empty(storage.CommittedAppends); + Assert.Empty(storage.CommittedReplaces); + Assert.Empty(sut.List); + Assert.Equal(0, sut.Value.Value); + Assert.True(sut.Manager.HasPendingWrites); + } + + [Fact] + public async Task WriteStateAsync_MutationsDuringBlockedReplace_RemainPendingForNextWrite() + { + var storage = new CheckpointingJournalStorage(); + var sut = CreateMixedStateTestSystem(storage); + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + sut.List.Add("baseline"); + sut.Value.Value = 10; + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + storage.ClearOperationHistory(); + + sut.List.Add("before-capture"); + sut.Value.Value = 20; + storage.IsCompactionRequested = true; + storage.DelayReplace = true; + var firstWrite = sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await storage.ReplaceStarted.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + var capturedRecovery = await RecoverMixedStateAsync(storage.CreateCapturedReplacementStorage()); + sut.List.Add("during-replace"); + sut.Value.Value = 30; + + storage.AllowReplace.TrySetResult(); + await firstWrite.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + await storage.ReplaceCompleted.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var pendingAfterFirstWrite = sut.Manager.HasPendingWrites; + var pendingBytesAfterFirstWrite = sut.Manager.PendingWriteByteCount; + var replacementsAfterFirstWrite = storage.CommittedReplaces.Count; + + storage.DelayReplace = false; + storage.IsCompactionRequested = false; + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var finalRecovery = await RecoverMixedStateAsync(storage.CreateRecoveryStorage()); + + Assert.Equal(["baseline", "before-capture"], capturedRecovery.List.ToArray()); + Assert.Equal(20, capturedRecovery.Value.Value); + Assert.DoesNotContain("during-replace", capturedRecovery.List); + Assert.True(pendingAfterFirstWrite); + Assert.True(pendingBytesAfterFirstWrite > 0); + Assert.Equal(1, replacementsAfterFirstWrite); + Assert.Equal(["baseline", "before-capture", "during-replace"], finalRecovery.List.ToArray()); + Assert.Equal(30, finalRecovery.Value.Value); + } + + [Fact] + public async Task WriteStateAsync_ReplaceOperationCanceled_DoesNotAppendAndRemainsRetryable() + { + var storage = new CheckpointingJournalStorage(); + var sut = CreateMixedStateTestSystem(storage); + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + sut.List.Add("baseline"); + sut.Value.Value = 10; + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var baselineCheckpoint = storage.GetDurableCheckpoint(); + storage.ClearOperationHistory(); + + sut.List.Add("new"); + sut.Value.Value = 20; + storage.IsCompactionRequested = true; + var expected = new OperationCanceledException("Expected storage-side cancellation."); + storage.NextReplaceException = expected; + + var exception = await Assert.ThrowsAsync( + () => sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + var pendingAfterFailure = sut.Manager.HasPendingWrites; + var pendingBytesAfterFailure = sut.Manager.PendingWriteByteCount; + var failedCheckpoint = storage.GetDurableCheckpoint(); + var appendAttemptsAfterFailure = storage.AppendAttempts.Count; + var committedAppendsAfterFailure = storage.CommittedAppends.Count; + var replaceAttemptsAfterFailure = storage.ReplaceAttempts.Count; + var committedReplacesAfterFailure = storage.CommittedReplaces.Count; + var failedRecovery = await RecoverMixedStateAsync(new CheckpointingJournalStorage(failedCheckpoint)); + + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var finalRecovery = await RecoverMixedStateAsync(storage.CreateRecoveryStorage()); + + Assert.Same(expected, exception); + Assert.Equal(1, replaceAttemptsAfterFailure); + Assert.Equal(0, committedReplacesAfterFailure); + Assert.Equal(0, appendAttemptsAfterFailure); + Assert.Equal(0, committedAppendsAfterFailure); + Assert.Equal(Flatten(baselineCheckpoint), Flatten(failedCheckpoint)); + Assert.True(pendingAfterFailure); + Assert.True(pendingBytesAfterFailure > 0); + Assert.Equal(["baseline"], failedRecovery.List.ToArray()); + Assert.Equal(10, failedRecovery.Value.Value); + Assert.Equal(2, storage.ReplaceAttempts.Count); + Assert.Single(storage.CommittedReplaces); + Assert.Empty(storage.AppendAttempts); + Assert.Empty(storage.CommittedAppends); + Assert.Equal(["baseline", "new"], finalRecovery.List.ToArray()); + Assert.Equal(20, finalRecovery.Value.Value); + } + + [Fact] + public async Task WriteStateAsync_CallerCancellationWhileReplaceBlocked_DoesNotCancelStorageReplace() + { + var storage = new CheckpointingJournalStorage + { + IsCompactionRequested = true, + DelayReplace = true + }; + var sut = CreateMixedStateTestSystem(storage); + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + sut.List.Add("direct"); + sut.Value.Value = 42; + using var callerCancellation = new CancellationTokenSource(); + var callerWrite = sut.Manager.WriteStateAsync(callerCancellation.Token).AsTask(); + await storage.ReplaceStarted.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + callerCancellation.Cancel(); + await Assert.ThrowsAnyAsync( + () => callerWrite.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + var storageTokenCanceledByCaller = storage.LastReplaceCancellationToken.IsCancellationRequested; + var replacementCompletedBeforeRelease = storage.ReplaceCompleted.Task.IsCompleted; + var committedReplacesBeforeRelease = storage.CommittedReplaces.Count; + + storage.AllowReplace.TrySetResult(); + await storage.ReplaceCompleted.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + storage.IsCompactionRequested = false; + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var recovered = await RecoverMixedStateAsync(storage.CreateRecoveryStorage()); + + Assert.False(storageTokenCanceledByCaller); + Assert.False(replacementCompletedBeforeRelease); + Assert.Equal(0, committedReplacesBeforeRelease); + Assert.Single(storage.CommittedReplaces); + Assert.Empty(storage.CommittedAppends); + Assert.False(sut.Manager.HasPendingWrites); + Assert.Equal(["direct"], recovered.List.ToArray()); + Assert.Equal(42, recovered.Value.Value); + } + + [Fact] + public async Task WriteStateAsync_AppendFails_RecoveryNeverObservesMixedState() + { + var storage = new CheckpointingJournalStorage(); + var sut = CreateMixedStateTestSystem(storage); + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + sut.List.Add("baseline"); + sut.Value.Value = 10; + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var baselineCheckpoint = storage.GetDurableCheckpoint(); + storage.ClearOperationHistory(); + + sut.List.Add("new"); + sut.Value.Value = 20; + var expected = new IOException("Expected append failure."); + storage.NextAppendException = expected; + + var exception = await Assert.ThrowsAsync( + () => sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + var failedCheckpoint = storage.GetDurableCheckpoint(); + var failedRecovery = await RecoverMixedStateAsync(new CheckpointingJournalStorage(failedCheckpoint)); + var appendAttemptsAfterFailure = storage.AppendAttempts.Count; + var committedAppendsAfterFailure = storage.CommittedAppends.Count; + var replaceAttemptsAfterFailure = storage.ReplaceAttempts.Count; + var committedReplacesAfterFailure = storage.CommittedReplaces.Count; + + await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var finalRecovery = await RecoverMixedStateAsync(storage.CreateRecoveryStorage()); + + Assert.Same(expected, exception); + Assert.Equal(1, appendAttemptsAfterFailure); + Assert.Equal(0, committedAppendsAfterFailure); + Assert.Equal(0, replaceAttemptsAfterFailure); + Assert.Equal(0, committedReplacesAfterFailure); + Assert.Equal(Flatten(baselineCheckpoint), Flatten(failedCheckpoint)); + Assert.Equal(["baseline"], failedRecovery.List.ToArray()); + Assert.DoesNotContain("new", failedRecovery.List); + Assert.Equal(10, failedRecovery.Value.Value); + Assert.Equal(2, storage.AppendAttempts.Count); + Assert.Single(storage.CommittedAppends); + Assert.Empty(storage.ReplaceAttempts); + Assert.Empty(storage.CommittedReplaces); + Assert.Equal(["baseline", "new"], finalRecovery.List.ToArray()); + Assert.Equal(20, finalRecovery.Value.Value); + } + + private ( + IJournaledStateManager Manager, + ILifecycleSubject Lifecycle, + DurableList List, + DurableValue Value) CreateMixedStateTestSystem(IJournalStorage storage) + { + var sut = CreateTestSystem(storage: storage); + var list = new DurableList("list", sut.Manager, new OrleansBinaryDurableListCommandCodec(CodecProvider.GetCodec(), SessionPool)); + var value = new DurableValue("value", sut.Manager, CreateValueCodec()); + return (sut.Manager, sut.Lifecycle, list, value); + } + + private async Task<( + IJournaledStateManager Manager, + ILifecycleSubject Lifecycle, + DurableList List, + DurableValue Value)> RecoverMixedStateAsync(IJournalStorage storage) + { + var recovered = CreateMixedStateTestSystem(storage); + await recovered.Lifecycle.OnStart(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + return recovered; + } + + private static byte[] Flatten(IEnumerable segments) => segments.SelectMany(static segment => segment).ToArray(); + + private sealed class CheckpointingJournalStorage : IJournalStorage + { + private readonly object _lock = new(); + private readonly List _segments = []; + + public CheckpointingJournalStorage() + { + } + + public CheckpointingJournalStorage(IEnumerable segments) + { + foreach (var segment in segments) + { + _segments.Add(segment.ToArray()); + } + } + + public List AppendAttempts { get; } = []; + + public List ReplaceAttempts { get; } = []; + + public List CommittedAppends { get; } = []; + + public List CommittedReplaces { get; } = []; + + public Exception? NextAppendException { get; set; } + + public Exception? NextReplaceException { get; set; } + + public bool IsCompactionRequested { get; set; } + + public bool DelayReplace { get; set; } + + public TaskCompletionSource ReplaceStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource AllowReplace { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource ReplaceCompleted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public CancellationToken LastReplaceCancellationToken { get; private set; } + + public byte[][] GetDurableCheckpoint() + { + lock (_lock) + { + return _segments.Select(static segment => segment.ToArray()).ToArray(); + } + } + + public CheckpointingJournalStorage CreateRecoveryStorage() => new(GetDurableCheckpoint()); + + public CheckpointingJournalStorage CreateCapturedReplacementStorage() + { + var replacement = Assert.Single(ReplaceAttempts); + return new CheckpointingJournalStorage([replacement]); + } + + public void ClearOperationHistory() + { + AppendAttempts.Clear(); + ReplaceAttempts.Clear(); + CommittedAppends.Clear(); + CommittedReplaces.Clear(); + } + + public void ClearDurableCheckpoint() + { + lock (_lock) + { + _segments.Clear(); + } + } + + public ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(consumer); + cancellationToken.ThrowIfCancellationRequested(); + consumer.Read(GetDurableCheckpoint().Select(static segment => (ReadOnlyMemory)segment.AsMemory()), metadata: null, complete: true); + return default; + } + + public async ValueTask ReplaceAsync(ReadOnlySequence value, CancellationToken cancellationToken) + { + LastReplaceCancellationToken = cancellationToken; + var bytes = value.ToArray(); + ReplaceAttempts.Add(bytes); + + if (NextReplaceException is { } exception) + { + NextReplaceException = null; + throw exception; + } + + if (DelayReplace) + { + ReplaceStarted.TrySetResult(); + await AllowReplace.Task.WaitAsync(TimeSpan.FromSeconds(10), cancellationToken); + } + + cancellationToken.ThrowIfCancellationRequested(); + lock (_lock) + { + _segments.Clear(); + _segments.Add(bytes.ToArray()); + } + + CommittedReplaces.Add(bytes); + ReplaceCompleted.TrySetResult(); + } + + public ValueTask AppendAsync(ReadOnlySequence value, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var bytes = value.ToArray(); + AppendAttempts.Add(bytes); + + if (NextAppendException is { } exception) + { + NextAppendException = null; + return ValueTask.FromException(exception); + } + + lock (_lock) + { + _segments.Add(bytes.ToArray()); + } + + CommittedAppends.Add(bytes); + return default; + } + + public ValueTask DeleteAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + lock (_lock) + { + _segments.Clear(); + } + + return default; + } + } + private sealed class StreamingOnlyStorage : IJournalStorage { public bool StreamingReadCalled { get; private set; } @@ -1837,16 +2489,24 @@ private sealed class CapturingStorage : IJournalStorage public Exception? NextReadException { get; set; } + public Queue ReadExceptions { get; } = new(); + + public Exception? NextReplaceException { get; set; } + public bool BlockNextRead { get; set; } + public bool BlockNextDelete { get; set; } + public TaskCompletionSource BlockedReadStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public TaskCompletionSource AllowBlockedRead { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); - public Exception? NextReplaceException { get; set; } - public int ReplaceAttemptCount { get; private set; } + public TaskCompletionSource DeleteStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource AllowDelete { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public int ReadConsumeCount { get; private set; } public void ResetReadConsumeCount() => ReadConsumeCount = 0; @@ -1955,12 +2615,18 @@ public ValueTask AppendAsync(ReadOnlySequence value, CancellationToken can return default; } - public ValueTask DeleteAsync(CancellationToken cancellationToken) + public async ValueTask DeleteAsync(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); + if (BlockNextDelete) + { + BlockNextDelete = false; + DeleteStarted.SetResult(); + await AllowDelete.Task.WaitAsync(cancellationToken); + } + DeleteCount++; _segments.Clear(); - return default; } }