diff --git a/docs/site/src/content/docs/host/monitoring/signals.md b/docs/site/src/content/docs/host/monitoring/signals.md index 2b7d8bc0a22..eb474519606 100644 --- a/docs/site/src/content/docs/host/monitoring/signals.md +++ b/docs/site/src/content/docs/host/monitoring/signals.md @@ -73,4 +73,4 @@ Page on user impact or imminent data/availability risk. Route isolated warnings ## Dashboard and telemetry backends -The Orleans Dashboard provides a current operational view and method profiling. It isn't a replacement for retained logs, metrics, traces, or alerts. Secure it as an administrative endpoint and use OTLP for durable telemetry. See [Orleans Dashboard](../../dashboard/index.md). +The Orleans Dashboard provides a current operational view and method profiling. Use logs, metrics, traces, and alerts for durable telemetry. Secure the dashboard as an administrative endpoint and export telemetry through OTLP. See [Orleans Dashboard](../../dashboard/index.md). diff --git a/docs/site/src/content/docs/implementation/streams-implementation/index.md b/docs/site/src/content/docs/implementation/streams-implementation/index.md index 6f34c3036b8..34de34b232a 100644 --- a/docs/site/src/content/docs/implementation/streams-implementation/index.md +++ b/docs/site/src/content/docs/implementation/streams-implementation/index.md @@ -81,7 +81,7 @@ The default maximum adapter batch-container batch size is 1 and the empty-poll p An decouples queue reads from consumer delivery. Each subscription has an , so a slow consumer does not directly block a fast consumer at a later cursor. -The cache tracks the earliest delivery progress across active subscriptions. Purging must not remove an item still needed by any cursor. uses pressure buckets to stop or slow reads as lag grows instead of discarding undelivered events. Its default capacity is 4,096 batch containers. +The cache tracks the earliest contiguous partition position which is safe across active subscriptions. A matching record becomes safe after delivery or intentional filtering. A cursor also advances safely across records for other streams when no earlier matching delivery is pending, so a quiet stream does not pin an otherwise busy partition. Purging must not remove an item still needed by any cursor. uses pressure buckets to stop or slow reads as lag grows instead of discarding undelivered events. Its default capacity is 4,096 batch containers. ```mermaid flowchart TB @@ -97,11 +97,13 @@ flowchart TB Cache capacity is not durability. The queue remains the durable boundary, subject to the adapter's acknowledgement contract. +Recoverable partitioned stream providers can compose a stream partition pipeline from , a partition source, and a data adapter. The pipeline admits immutable stream records into pooled storage, reconstructs batches lazily, reconciles the earliest safe subscription scan/delivery watermark, and persists a checkpoint which resumes strictly after that position. + ## Pub-sub handshake The agent registers as a producer for each stream and obtains subscription records from stream pub-sub. It holds a pin cursor while subscription handshakes complete so cache cleanup cannot pass the requested start token. New subscription notifications update the agent's local pub-sub cache. -Sequence tokens allow a rewindable adapter to start from a supported historical position. An adapter whose property is `false` must reject unsupported tokens rather than pretending to honor them. +Sequence tokens allow a rewindable adapter to start from a supported historical position. A start token is inclusive and remains unsafe until its record is delivered or intentionally filtered. A delivery handshake token confirms that its position was already processed. Exact `EventSequenceToken` and `EventSequenceTokenV2` values interoperate for legacy compatibility. Derived tokens compare only with the same concrete type unless the provider overrides equality, ordering, and hashing together. An adapter whose property is `false` must reject unsupported tokens rather than pretending to honor them. ## Delivery and failure semantics diff --git a/docs/site/src/content/docs/streaming/custom-queue-adapter.md b/docs/site/src/content/docs/streaming/custom-queue-adapter.md index 60ca4636c4e..13aca58e3fa 100644 --- a/docs/site/src/content/docs/streaming/custom-queue-adapter.md +++ b/docs/site/src/content/docs/streaming/custom-queue-adapter.md @@ -51,7 +51,7 @@ The factory composes the adapter with queue mapping, caching, and failure handli `SimpleQueueAdapterCache` is suitable for a non-rewindable adapter whose queue remains the durability boundary. A rewindable adapter usually needs a cache and sequence-token implementation which can position cursors at retained historical messages. -`AddPersistentStreams` leaves checkpointing to the adapter. The non-rewindable example acknowledges completed messages through its receiver and therefore has no independent checkpoint. For a retained-log transport, implement an , have the receiver or cache load and update the per-partition position, and register it as a named component with `ConfigureComponent`. Persist a checkpoint only after all consumers have advanced beyond the corresponding cached messages. A no-op checkpointer is suitable only when replay position is deliberately disposable. +`AddPersistentStreams` leaves checkpointing to the adapter. The non-rewindable example acknowledges completed messages through its receiver and therefore has no independent checkpoint. For a partitioned stream transport, implement an , have the receiver or cache load and update the stream partition position, and register it as a named component with `ConfigureComponent`. Treat a requested cursor start as inclusive: selecting that position does not confirm its record. Persist only the earliest contiguous partition position which every subscription has delivered, intentionally filtered, or safely scanned as belonging to another stream. A no-op checkpointer is suitable only when replay position is deliberately disposable. ## Register the provider @@ -75,6 +75,9 @@ Test the adapter against the real queue service, including: 1. queue ownership moving between silos during membership changes; 1. duplicate delivery and consumer idempotency; 1. stable stream-to-partition mapping across restarts and upgrades; and -1. sustained load beyond cache capacity to verify backpressure and queue retention. +1. sustained load beyond cache capacity to verify backpressure and queue retention; +1. quiet and busy streams sharing a partition, including restart after the quiet cursor scans unrelated records; +1. cancellation while partition ownership acquisition is blocked, followed by reassignment and late command completion; and +1. sequence-token equality, ordering, and hashing in both comparison directions. Monitor queue depth and oldest-message age by partition, receive and acknowledgement latency, redelivery count, throttling, pulling-agent errors, and consumer delivery failures. Alert before retention or visibility limits can cause data loss or a redelivery storm. diff --git a/docs/site/src/content/docs/streaming/data-adapters.md b/docs/site/src/content/docs/streaming/data-adapters.md index 8554da96bde..6b3dfe4048a 100644 --- a/docs/site/src/content/docs/streaming/data-adapters.md +++ b/docs/site/src/content/docs/streaming/data-adapters.md @@ -62,7 +62,7 @@ For Event Hubs, derive from to encode events published through Orleans; and - to select the physical Event Hubs partition key. -The adapter also participates in cache conversion and sequence positioning through . Preserve the Event Hubs offset and sequence number when constructing batch tokens so checkpoint and rewind behavior remains aligned with the partition log. +The adapter also participates in cache conversion and sequence positioning through . Preserve the Event Hubs offset and sequence number when constructing batch tokens so checkpoint and rewind behavior remains aligned with the stream partition. Register the adapter and Event Hubs connection under the same provider name on silos and publishing clients. The silo registration also configures durable Azure Table checkpoints: diff --git a/docs/site/src/content/docs/streaming/stream-providers.md b/docs/site/src/content/docs/streaming/stream-providers.md index 8eccee648a8..aa3b303a498 100644 --- a/docs/site/src/content/docs/streaming/stream-providers.md +++ b/docs/site/src/content/docs/streaming/stream-providers.md @@ -18,7 +18,7 @@ A stream provider connects the Orleans streaming API to a transport and defines | Azure Event Hubs | [`Microsoft.Orleans.Streaming.EventHubs`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.EventHubs) | Stable | Yes, within Event Hubs retention | Yes | Event Hubs namespace, hub, consumer group, and checkpoint storage | | Amazon Kinesis | [`Microsoft.Orleans.Streaming.Kinesis`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.Kinesis) | Stable | Yes, within Kinesis retention | Yes | Kinesis data stream, AWS credentials, region, and durable checkpoint storage | | Amazon SQS | [`Microsoft.Orleans.Streaming.SQS`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.SQS) | Stable | Yes, within SQS retention | No | AWS account, queue permissions, region/endpoint configuration | -| ADO.NET | [`Microsoft.Orleans.Streaming.AdoNet`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.AdoNet) | **Alpha** | Yes, in relational tables until expiry/dead-letter eviction | No | Supported database, ADO.NET driver, and Orleans streaming SQL schema | +| ADO.NET partitioned stream | [`Microsoft.Orleans.Streaming.AdoNet`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.AdoNet) | **Alpha** | Yes, in relational stream partitions | No; resumes after its durable safe checkpoint | Supported database, ADO.NET driver, and matching Orleans streaming SQL schema | | NATS JetStream | [`Microsoft.Orleans.Streaming.NATS`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.NATS) | **Alpha** | Configurable; file storage is the default | No | NATS server with JetStream and sufficient storage; subject/stream administration | | Redis Streams | [`Microsoft.Orleans.Streaming.Redis`](https://www.nuget.org/packages/Microsoft.Orleans.Streaming.Redis) | **Alpha** | Configurable through Redis persistence and stream retention | Yes, while entries remain | Redis deployment, persistence/HA policy, and retention sizing | @@ -58,15 +58,17 @@ The Event Hubs provider supports a custom data adapter for provider-specific wir ## Amazon Kinesis -Register [Amazon Kinesis Data Streams](https://docs.aws.amazon.com/streams/latest/dev/introduction.html) with . Kinesis retains events independently of Orleans, and the provider persists each shard's last delivered sequence number so that delivery can resume after shutdown or queue reassignment. See [Stream with Amazon Kinesis](kinesis-streaming.md) for configuration, checkpoint choices, and operational constraints. +Register [Amazon Kinesis Data Streams](https://docs.aws.amazon.com/streams/latest/dev/introduction.html) with . Kinesis retains events independently of Orleans, and the provider persists each shard's earliest safe scan/delivery position so that delivery can resume after shutdown or queue reassignment. Kinesis sequence tokens compare only with other Kinesis tokens using the numeric shard sequence and event index. See [Stream with Amazon Kinesis](kinesis-streaming.md) for configuration, checkpoint choices, and operational constraints. ## Amazon SQS Register [Amazon SQS](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/welcome.html) with . Standard queues provide at-least-once delivery, while FIFO queues preserve ordering within each Orleans stream. SQS redelivers after the [visibility timeout](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html) when processing isn't acknowledged. See [Stream with Amazon SQS](sqs-streaming.md) for standard and FIFO configuration, custom data adapters, permissions, and operational guidance. -## ADO.NET streaming (alpha) +## ADO.NET partitioned stream provider (alpha) -Register [ADO.NET](https://learn.microsoft.com/dotnet/framework/data/adonet/ado-net-overview) streaming with `AddAdoNetStreams`. Install the matching database driver and apply the SQL Server, PostgreSQL, or MySQL streaming schema shipped in the package source. Messages are durable in relational tables but expire and can move to dead letters according to `AdoNetStreamOptions`. The provider isn't rewindable. +Register the [ADO.NET](https://learn.microsoft.com/dotnet/framework/data/adonet/ado-net-overview) partitioned stream provider with . Install the matching database driver and apply the SQL Server, PostgreSQL, or MySQL streaming schema shipped in the package source. Each queue maps to an immutable, ordered stream partition with a durable, ownership-fenced checkpoint. The stream partition pipeline resumes strictly after the earliest safe scan/delivery checkpoint, redelivering uncheckpointed records after recovery. Ownership acquisition propagates cancellation, and a receiver retains its queue reservation until an in-flight acquisition command settles. retains checkpointed records for one day by default before cleanup. can impose a hard storage ceiling; deleting unprocessed records produces retention gap diagnostics. Retention and cleanup intervals round fractional seconds upward. Cleanup is bounded by and deletes a contiguous eligible prefix. can fault one failing subscription while preserving the multicast partition record. + +The current alpha schema is versioned and isn't compatible with the former queue, visibility-timeout, confirmation, or dead-letter schema. There is no in-place migration for alpha data. Before upgrading, stop producers and consumers, drop the former `OrleansStreamMessage`, `OrleansStreamDeadLetter`, `OrleansStreamControl`, and `OrleansStreamMessageSequence` objects, remove their streaming routines and `OrleansQuery` rows, and then apply the current streaming script. Drop `OrleansStreamPartition` too if a partial installation exists. Existing alpha rows aren't read or silently converted; preserve them externally first if the payloads are required. ## NATS JetStream streaming (alpha) diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamCheckpointStore.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamCheckpointStore.cs index 0f023d86359..a1fbe7e1551 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamCheckpointStore.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamCheckpointStore.cs @@ -13,15 +13,7 @@ namespace Orleans.Streaming.Kinesis { - internal interface IDynamoDBStreamCheckpointStore - { - ValueTask Load(CancellationToken cancellationToken); - - ValueTask Update( - string checkpoint, - string expectedCheckpoint, - CancellationToken cancellationToken); - } + internal interface IDynamoDBStreamCheckpointStore : IStreamCheckpointStore; internal sealed partial class DynamoDBStreamCheckpointStore : IDynamoDBStreamCheckpointStore { @@ -57,13 +49,13 @@ public DynamoDBStreamCheckpointStore( }; } - public async ValueTask Load(CancellationToken cancellationToken) + public async ValueTask Load(CancellationToken cancellationToken) { await _mutex.WaitAsync(cancellationToken); try { await LoadCore(cancellationToken); - return _checkpoint; + return GetState(); } finally { @@ -71,13 +63,13 @@ public async ValueTask Load(CancellationToken cancellationToken) } } - public async ValueTask Update( + public async ValueTask Update( string checkpoint, - string expectedCheckpoint, + string expectedVersion, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(checkpoint); - ArgumentNullException.ThrowIfNull(expectedCheckpoint); + ArgumentNullException.ThrowIfNull(expectedVersion); await _mutex.WaitAsync(cancellationToken); try @@ -87,9 +79,12 @@ public async ValueTask Update( await LoadCore(cancellationToken); } - if (!string.Equals(_checkpoint, expectedCheckpoint, StringComparison.Ordinal)) + var currentVersion = _version == 0 + ? string.Empty + : _version.ToString(CultureInfo.InvariantCulture); + if (!string.Equals(currentVersion, expectedVersion, StringComparison.Ordinal)) { - return _checkpoint; + return GetState(); } try @@ -137,14 +132,20 @@ public async ValueTask Update( await LoadCore(cancellationToken); } - return _checkpoint; + return GetState(); } + finally { _mutex.Release(); } } + private StreamCheckpointStoreState GetState() + => new( + _checkpoint, + _version == 0 ? string.Empty : _version.ToString(CultureInfo.InvariantCulture)); + internal static async Task InitializeTable( IAmazonDynamoDB client, DynamoDBStreamQueueCheckpointerOptions options, diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs index 56ba751fb24..6703cbc8605 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/DynamoDBStreamQueueCheckpointer.cs @@ -11,15 +11,15 @@ namespace Orleans.Streaming.Kinesis /// internal sealed class DynamoDBStreamQueueCheckpointer : IStreamQueueCheckpointer { - private readonly GrainStreamQueueCheckpointer _inner; + private readonly StreamQueueCheckpointer _inner; internal DynamoDBStreamQueueCheckpointer( IDynamoDBStreamCheckpointStore store, DynamoDBStreamQueueCheckpointerOptions options) { - _inner = new GrainStreamQueueCheckpointer( - new StreamCheckpointStoreAdapter(store), - new GrainStreamQueueCheckpointerOptions + _inner = new StreamQueueCheckpointer( + store, + new StreamQueueCheckpointerOptions { CheckpointComparer = StreamCheckpointComparers.Numeric, PersistInterval = options.PersistInterval, @@ -63,15 +63,5 @@ public void Update(string offset, DateTime utcNow, CancellationToken cancellatio /// public Task FlushAsync(CancellationToken cancellationToken) => _inner.FlushAsync(cancellationToken); - private sealed class StreamCheckpointStoreAdapter(IDynamoDBStreamCheckpointStore store) : IStreamCheckpointerGrain - { - public ValueTask Load(CancellationToken cancellationToken) => store.Load(cancellationToken); - - public ValueTask Update( - string checkpoint, - string expectedCheckpoint, - CancellationToken cancellationToken) - => store.Update(checkpoint, expectedCheckpoint, cancellationToken); - } } } diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs index 8fbd667b379..9eb953cc0c4 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterFactory.cs @@ -21,13 +21,13 @@ namespace Orleans.Streaming.Kinesis /// /// Queue adapter factory which allows the PersistentStreamProvider to use AWS Kinesis Data Streams as its backend persistent event queue. /// - internal class KinesisAdapterFactory : IQueueAdapterFactory, IQueueAdapter, IDisposable + internal class KinesisAdapterFactory : IQueueAdapterFactory, IQueueAdapter, IQueueAdapterCache, IDisposable { private readonly KinesisStreamOptions _options; private readonly Serializer _serializer; private readonly IStreamQueueCheckpointerFactory? _checkpointerFactory; private readonly ILoggerFactory _loggerFactory; - private readonly IQueueAdapterCache _adapterCache; + private readonly SimpleQueueCacheOptions _cacheOptions; private readonly ILogger _logger; private readonly Func _queueMapperFactory; private readonly IAmazonKinesis _client; @@ -35,6 +35,7 @@ internal class KinesisAdapterFactory : IQueueAdapterFactory, IQueueAdapter, IDis private HashRingBasedPartitionedStreamQueueMapper _streamQueueMapper = null!; private KinesisShardTopologyMonitor _topologyMonitor = null!; + private QueueAdapterReceiverRegistry _receivers = null!; private int _disposed; public KinesisAdapterFactory( @@ -53,22 +54,17 @@ public KinesisAdapterFactory( _serializer = serializer; _checkpointerFactory = checkpointerFactory; _loggerFactory = loggerFactory; + _cacheOptions = cacheOptions; _logger = loggerFactory.CreateLogger(); _timeProvider = timeProvider ?? TimeProvider.System; - _adapterCache = new SimpleQueueAdapterCache( - cacheOptions, - name, - loggerFactory - ); - _queueMapperFactory = partitions => new HashRingBasedPartitionedStreamQueueMapper(partitions, Name); _client = CreateClient(); } public string Name { get; } - public bool IsRewindable => false; + public bool IsRewindable => true; public StreamProviderDirection Direction => StreamProviderDirection.ReadWrite; @@ -106,13 +102,14 @@ public async Task CreateAdapter(CancellationToken cancellationTok _options.TopologyCheckInterval, _timeProvider, _loggerFactory.CreateLogger()); + _receivers = new QueueAdapterReceiverRegistry(MakeReceiver); } return this; } public IQueueAdapterCache GetQueueAdapterCache() - => _adapterCache; + => this; public IStreamQueueMapper GetStreamQueueMapper() => _streamQueueMapper; @@ -135,6 +132,12 @@ public async Task QueueMessageBatchAsync(StreamId streamId, IEnumerable ev } public IQueueAdapterReceiver CreateReceiver(QueueId queueId) + => GetOrCreateReceiver(queueId); + + public IQueueCache CreateQueueCache(QueueId queueId) + => GetOrCreateReceiver(queueId); + + private KinesisPooledAdapterReceiver GetOrCreateReceiver(QueueId queueId) { if (_checkpointerFactory is null) { @@ -142,18 +145,24 @@ public IQueueAdapterReceiver CreateReceiver(QueueId queueId) $"No {nameof(IStreamQueueCheckpointerFactory)} is configured for the Kinesis stream provider '{Name}'."); } - var partition = _streamQueueMapper.QueueToPartition(queueId); + return _receivers.GetOrCreate(queueId); + } - return new KinesisAdapterReceiver( + private KinesisPooledAdapterReceiver MakeReceiver(QueueId queueId) + { + var partition = _streamQueueMapper.QueueToPartition(queueId); + return new KinesisPooledAdapterReceiver( CreateClient(), _options.StreamName, partition, - _checkpointerFactory, + _checkpointerFactory!, + _cacheOptions, _serializer, _loggerFactory, _topologyMonitor, _options.GetRecordsInterval, - _timeProvider + _timeProvider, + receiver => _receivers.Remove(queueId, receiver) ); } diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterReceiver.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterReceiver.cs index dcfb81b18c0..d4e0159a17c 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterReceiver.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisAdapterReceiver.cs @@ -163,7 +163,8 @@ public async Task> GetQueueMessagesAsync( foreach (var record in getRecordsResponse.Records) { - // Kinesis only has a long string sequence ID, so we fake one based on the order we read from the partition. + // Retain the receiver-local ordinal for compatibility with existing serialized tokens. + // KinesisSequenceToken orders records using the durable shard sequence. batch.Add(KinesisBatchContainer.FromKinesisRecord(_serializer, record, _lastReadMessage++)); } diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisBatchContainer.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisBatchContainer.cs index 97ea841a767..99be23cfb43 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisBatchContainer.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisBatchContainer.cs @@ -21,6 +21,12 @@ internal class KinesisBatchContainer : IBatchContainer, IComparable Serializer { get; set; } = null!; @@ -37,6 +43,20 @@ private KinesisBatchContainer(Record record, Serializer serializer, + StreamId streamId, + string shardSequence, + long sequenceId) + { + Serializer = serializer; + _rawRecord = rawRecord; + _streamId = streamId; + _hasStreamId = true; + Token = new KinesisSequenceToken(shardSequence, sequenceId, 0); + } + [GeneratedActivatorConstructor] internal KinesisBatchContainer(Serializer serializer) { @@ -46,7 +66,7 @@ internal KinesisBatchContainer(Serializer serializer /// /// Stream identifier for the stream this batch is part of. /// - public StreamId StreamId => GetPayload().StreamId; + public StreamId StreamId => _hasStreamId ? _streamId : GetPayload().StreamId; /// /// Stream Sequence Token for the start of this batch. @@ -81,7 +101,7 @@ public bool ImportRequestContext() } public int CompareTo(KinesisBatchContainer? other) - => other is null ? 1 : Token.SequenceNumber.CompareTo(other.SequenceToken.SequenceNumber); + => other is null ? 1 : Token.CompareTo(other.Token); [Serializable] [GenerateSerializer] @@ -113,5 +133,13 @@ internal static KinesisBatchContainer FromKinesisRecord(Serializer serializer, + StreamId streamId, + byte[] rawRecord, + string shardSequence, + long sequenceId) + => new(rawRecord, serializer, streamId, shardSequence, sequenceId); } } diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs new file mode 100644 index 00000000000..b826b376d67 --- /dev/null +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisPooledAdapterReceiver.cs @@ -0,0 +1,269 @@ +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.ExceptionServices; +using Amazon.Kinesis; +using Microsoft.Extensions.Logging; +using Orleans.Configuration; +using Orleans.Providers.Streams.Common; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Streams; + +namespace Orleans.Streaming.Kinesis; + +internal sealed class KinesisPooledAdapterReceiver : IQueueAdapterReceiver, IQueueCache +{ + private const int BufferSize = 1024 * 1024; + private readonly IStreamQueueCheckpointerFactory _checkpointerFactory; + private readonly string _partition; + private readonly KinesisRecoverableStreamSource _source; + private readonly KinesisRecoverableStreamDataAdapter _dataAdapter; + private readonly RecoverableStreamQueueCache _cache; + private readonly Action? _onShutdown; + private readonly object _lifecycleLock = new(); + private readonly CancellationTokenSource _lifecycleCancellation = new(); + private RecoverableStreamReceiver? _inner; + private Task? _initializationTask; + private CancellationToken _initializationOwnerToken; + private int _initialized; + private int _shutdown; + + internal CancellationToken LifecycleCancellationToken => _lifecycleCancellation.Token; + + public KinesisPooledAdapterReceiver( + IAmazonKinesis client, + string streamName, + string partition, + IStreamQueueCheckpointerFactory checkpointerFactory, + SimpleQueueCacheOptions cacheOptions, + Serializer serializer, + ILoggerFactory loggerFactory, + KinesisShardTopologyMonitor topologyMonitor, + TimeSpan getRecordsInterval, + TimeProvider timeProvider, + Action? onShutdown = null) + { + _checkpointerFactory = checkpointerFactory; + _partition = partition; + _onShutdown = onShutdown; + var logger = loggerFactory.CreateLogger(); + _source = new( + client, + streamName, + partition, + topologyMonitor, + getRecordsInterval, + timeProvider); + _dataAdapter = new(serializer); + var evictionStrategy = new ChronologicalEvictionStrategy( + logger, + new TimePurgePredicate(TimeSpan.MaxValue, TimeSpan.MaxValue), + cacheMonitor: null, + monitorWriteInterval: null); + _cache = new( + Math.Min(1000, cacheOptions.CacheSize), + new ObjectPool(() => new FixedSizeBuffer(BufferSize)), + _dataAdapter, + evictionStrategy, + logger, + maxCacheSize: cacheOptions.CacheSize); + } + + public async Task Initialize(TimeSpan timeout) + { + using var cancellation = new CancellationTokenSource(timeout); + await EnsureInitialized(cancellation.Token); + } + + public async Task> GetQueueMessagesAsync(int maxCount) + => await GetQueueMessagesAsync(maxCount, CancellationToken.None); + + public async Task> GetQueueMessagesAsync(int maxCount, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (Volatile.Read(ref _shutdown) != 0 || maxCount <= 0) + { + return []; + } + + await EnsureInitialized(cancellationToken); + if (Volatile.Read(ref _shutdown) != 0) + { + return []; + } + + return await GetInner().GetQueueMessagesAsync(maxCount, cancellationToken); + } + + public Task MessagesDeliveredAsync(IList messages) + => Task.CompletedTask; + + public Task MessagesDeliveredAsync(IList messages, CancellationToken cancellationToken) + => cancellationToken.IsCancellationRequested ? Task.FromCanceled(cancellationToken) : Task.CompletedTask; + + public async Task Shutdown(TimeSpan timeout) + { + if (Interlocked.Exchange(ref _shutdown, 1) != 0) + { + return; + } + + try + { + _lifecycleCancellation.Cancel(); + var shutdownWatch = Stopwatch.StartNew(); + using var cancellation = timeout == Timeout.InfiniteTimeSpan + ? null + : new CancellationTokenSource(timeout); + var cancellationToken = cancellation?.Token ?? CancellationToken.None; + List? exceptions = null; + Task? initializationTask; + lock (_lifecycleLock) + { + initializationTask = _initializationTask; + } + + if (initializationTask is not null) + { + try + { + await initializationTask.WaitAsync(cancellationToken); + } + catch (OperationCanceledException) + when (_lifecycleCancellation.IsCancellationRequested + && !cancellationToken.IsCancellationRequested) + { + } + catch (Exception exception) + { + (exceptions ??= []).Add(exception); + } + } + + try + { + if (_inner is null) + { + await _source.Shutdown(cancellationToken); + } + else + { + var remaining = timeout == Timeout.InfiniteTimeSpan + ? Timeout.InfiniteTimeSpan + : timeout > shutdownWatch.Elapsed + ? timeout - shutdownWatch.Elapsed + : TimeSpan.Zero; + await _inner.Shutdown(remaining); + } + } + catch (Exception exception) + { + (exceptions ??= []).Add(exception); + } + + if (exceptions is [var singleException]) + { + ExceptionDispatchInfo.Capture(singleException).Throw(); + } + + if (exceptions is { Count: > 1 }) + { + throw new AggregateException(exceptions); + } + } + finally + { + _lifecycleCancellation.Dispose(); + _onShutdown?.Invoke(this); + } + } + + public int GetMaxAddCount() => _cache.GetMaxAddCount(); + + public void AddToCache(IList messages) + { + } + + public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList purgedItems) + => _cache.TryPurgeFromCache(out purgedItems); + + public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) + => _cache.GetCacheCursor(streamId, token); + + public bool IsUnderPressure() => _cache.IsUnderPressure(); + + public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToken, DateTime utcNow) + => _inner?.UpdateDeliveryProgress(earliestSubscriptionToken, utcNow); + + private async Task EnsureInitialized(CancellationToken cancellationToken) + { + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + Task initializationTask; + CancellationToken initializationOwnerToken; + lock (_lifecycleLock) + { + if (Volatile.Read(ref _shutdown) != 0 || Volatile.Read(ref _initialized) != 0) + { + return; + } + + if (_initializationTask is null || _initializationTask.IsCompleted) + { + _initializationOwnerToken = cancellationToken; + _initializationTask = InitializeCore(cancellationToken); + } + + initializationTask = _initializationTask; + initializationOwnerToken = _initializationOwnerToken; + } + + try + { + await initializationTask.WaitAsync(cancellationToken); + return; + } + catch (OperationCanceledException) when ( + !cancellationToken.IsCancellationRequested + && initializationOwnerToken.IsCancellationRequested + && initializationTask.IsCanceled) + { + // The caller which owned the shared initialization attempt canceled. + // An unaffected caller retries after that attempt has settled. + } + } + } + + private async Task InitializeCore(CancellationToken initializationToken) + { + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource( + _lifecycleCancellation.Token, + initializationToken); + var lifecycleToken = cancellation.Token; + if (_inner is null) + { + var checkpointer = await _checkpointerFactory.Create(_partition, lifecycleToken); + if (Volatile.Read(ref _shutdown) != 0) + { + return; + } + + _inner = new( + _source, + _dataAdapter, + _cache, + checkpointer, + startFromNow: false); + } + + await _inner.Initialize(lifecycleToken); + if (Volatile.Read(ref _shutdown) == 0) + { + Volatile.Write(ref _initialized, 1); + } + } + + private RecoverableStreamReceiver GetInner() + => _inner ?? throw new InvalidOperationException("The Kinesis receiver has not been initialized."); +} diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisRecoverableStream.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisRecoverableStream.cs new file mode 100644 index 00000000000..c11ff68d18c --- /dev/null +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisRecoverableStream.cs @@ -0,0 +1,276 @@ +using Amazon.Kinesis; +using Amazon.Kinesis.Model; +using Orleans.Providers.Streams.Common; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Streams; + +namespace Orleans.Streaming.Kinesis; + +internal sealed class KinesisCacheRecord( + Record record, + long sequenceNumber) +{ + private byte[]? _rawPayload; + + public Record Record { get; } = record; + + public long SequenceNumber { get; } = sequenceNumber; + + public KinesisBatchContainer.Body? Body { get; set; } + + public byte[] RawPayload => _rawPayload ??= Record.Data.ToArray(); +} + +internal sealed class KinesisRecoverableStreamSource( + IAmazonKinesis client, + string streamName, + string partition, + KinesisShardTopologyMonitor topologyMonitor, + TimeSpan getRecordsInterval, + TimeProvider timeProvider) : IRecoverableStreamSource +{ + private string? _shardIterator; + private string? _readOffset; + private long _nextSequenceNumber; + private DateTimeOffset _nextGetRecordsUtc; + private bool _shardExhausted; + private bool _topologyCheckRequired; + private bool _resetRequired; + + public async Task Initialize( + RecoverableStreamStartPosition position, + CancellationToken cancellationToken) + { + _readOffset = position.Checkpoint; + await ResetShardIterator(cancellationToken); + } + + public async Task> Read( + int maxCount, + CancellationToken cancellationToken) + { + if (_resetRequired) + { + await ResetShardIterator(cancellationToken); + _resetRequired = false; + } + + if (!await topologyMonitor.CheckTopology(_topologyCheckRequired, cancellationToken)) + { + return []; + } + + _topologyCheckRequired = false; + if (_shardExhausted) + { + return []; + } + + await WaitForGetRecordsInterval(cancellationToken); + var request = new GetRecordsRequest + { + Limit = maxCount, + ShardIterator = _shardIterator, + }; + + GetRecordsResponse response; + try + { + response = await client.GetRecordsAsync(request, cancellationToken); + } + catch (ExpiredIteratorException) + { + await ResetShardIterator(cancellationToken); + if (_shardExhausted) + { + return []; + } + + await WaitForGetRecordsInterval(cancellationToken); + request.ShardIterator = _shardIterator; + response = await client.GetRecordsAsync(request, cancellationToken); + } + + _shardIterator = response.NextShardIterator; + if (string.IsNullOrEmpty(_shardIterator)) + { + _shardExhausted = true; + _topologyCheckRequired = true; + } + + if (response.Records is not { Count: > 0 } records) + { + return []; + } + + var result = new List(records.Count); + for (var i = 0; i < records.Count; i++) + { + result.Add(new KinesisCacheRecord(records[i], _nextSequenceNumber + i)); + } + + return result; + } + + public void MessagesAdded(IReadOnlyList messages) + { + if (messages.Count == 0) + { + return; + } + + _nextSequenceNumber += messages.Count; + _readOffset = messages[^1].Record.SequenceNumber; + } + + public void MessagesAddFailed(IReadOnlyList messages) + { + _resetRequired = true; + } + + public Task Shutdown(CancellationToken cancellationToken) + { + client.Dispose(); + return cancellationToken.IsCancellationRequested + ? Task.FromCanceled(cancellationToken) + : Task.CompletedTask; + } + + private async Task ResetShardIterator(CancellationToken cancellationToken) + { + var request = new GetShardIteratorRequest + { + StreamName = streamName, + ShardId = partition, + ShardIteratorType = string.IsNullOrEmpty(_readOffset) + ? ShardIteratorType.TRIM_HORIZON + : ShardIteratorType.AFTER_SEQUENCE_NUMBER, + StartingSequenceNumber = _readOffset, + }; + var response = await client.GetShardIteratorAsync(request, cancellationToken); + _shardIterator = response.ShardIterator; + _shardExhausted = string.IsNullOrEmpty(_shardIterator); + } + + private async Task WaitForGetRecordsInterval(CancellationToken cancellationToken) + { + var delay = _nextGetRecordsUtc - timeProvider.GetUtcNow(); + if (delay > TimeSpan.Zero) + { + await Task.Delay(delay, timeProvider, cancellationToken); + } + + _nextGetRecordsUtc = timeProvider.GetUtcNow() + getRecordsInterval; + } +} + +internal sealed class KinesisRecoverableStreamDataAdapter( + Serializer serializer) : IRecoverableStreamDataAdapter +{ + public StreamPosition GetStreamPosition(KinesisCacheRecord queueMessage) + { + queueMessage.Body ??= serializer.Deserialize(queueMessage.RawPayload)!; + return new( + queueMessage.Body.StreamId, + new KinesisSequenceToken( + queueMessage.Record.SequenceNumber, + queueMessage.SequenceNumber, + 0)); + } + + public CachedMessage FromQueueMessage( + StreamPosition streamPosition, + KinesisCacheRecord queueMessage, + DateTime dequeueTimeUtc, + Func> getSegment) + { + var payload = queueMessage.RawPayload; + var size = SegmentBuilder.CalculateAppendSize(queueMessage.Record.SequenceNumber) + + SegmentBuilder.CalculateAppendSize(payload); + var segment = getSegment(size); + var offset = 0; + SegmentBuilder.Append(segment, ref offset, queueMessage.Record.SequenceNumber); + SegmentBuilder.Append(segment, ref offset, payload); + return new CachedMessage + { + StreamId = streamPosition.StreamId, + SequenceNumber = queueMessage.SequenceNumber, + EventIndex = streamPosition.SequenceToken.EventIndex, + EnqueueTimeUtc = queueMessage.Record.ApproximateArrivalTimestamp?.ToUniversalTime() ?? dequeueTimeUtc, + DequeueTimeUtc = dequeueTimeUtc, + Segment = segment, + }; + } + + public IBatchContainer GetBatchContainer(ref CachedMessage cachedMessage) + { + var offset = 0; + var shardSequence = SegmentBuilder.ReadNextString(cachedMessage.Segment, ref offset)!; + var payload = SegmentBuilder.ReadNextBytes(cachedMessage.Segment, ref offset).ToArray(); + return KinesisBatchContainer.FromCachedRecord( + serializer, + cachedMessage.StreamId, + payload, + shardSequence, + cachedMessage.SequenceNumber); + } + + public StreamSequenceToken GetSequenceToken(ref CachedMessage cachedMessage) + { + var offset = 0; + var shardSequence = SegmentBuilder.ReadNextString(cachedMessage.Segment, ref offset)!; + return new KinesisSequenceToken(shardSequence, cachedMessage.SequenceNumber, cachedMessage.EventIndex); + } + + public int Compare(ref CachedMessage cachedMessage, StreamSequenceToken token) + { + if (token is not KinesisSequenceToken kinesisToken) + { + throw new ArgumentOutOfRangeException(nameof(token)); + } + + var offset = 0; + var shardSequence = SegmentBuilder.ReadNextString(cachedMessage.Segment, ref offset)!; + var difference = CompareShardSequences(shardSequence, kinesisToken.ShardSequence); + return difference != 0 ? difference : cachedMessage.EventIndex.CompareTo(kinesisToken.EventIndex); + } + + private static int CompareShardSequences(string left, string right) + { + var leftStart = 0; + while (leftStart < left.Length && left[leftStart] == '0') + { + leftStart++; + } + + var rightStart = 0; + while (rightStart < right.Length && right[rightStart] == '0') + { + rightStart++; + } + + var lengthComparison = (left.Length - leftStart).CompareTo(right.Length - rightStart); + return lengthComparison != 0 + ? lengthComparison + : left.AsSpan(leftStart).SequenceCompareTo(right.AsSpan(rightStart)); + } + + public string GetOffset(ref CachedMessage cachedMessage) + { + var offset = 0; + return SegmentBuilder.ReadNextString(cachedMessage.Segment, ref offset)!; + } + + public bool TryGetOffset(StreamSequenceToken token, out string offset) + { + if (token is KinesisSequenceToken kinesisToken) + { + offset = kinesisToken.ShardSequence; + return true; + } + + offset = string.Empty; + return false; + } +} diff --git a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisSequenceToken.cs b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisSequenceToken.cs index ad81c1ad62d..142f736d0c5 100644 --- a/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisSequenceToken.cs +++ b/src/AWS/Orleans.Streaming.Kinesis/Streams/KinesisSequenceToken.cs @@ -1,24 +1,30 @@ using Newtonsoft.Json; using Orleans.Providers.Streams.Common; +using Orleans.Streams; using System; using System.Globalization; +using System.Numerics; namespace Orleans.Streaming.Kinesis { [Serializable] [GenerateSerializer] - internal class KinesisSequenceToken : EventSequenceTokenV2 + internal sealed class KinesisSequenceToken : EventSequenceTokenV2 { + [NonSerialized] + private BigInteger? _numericShardSequence; + /// /// Initializes a new instance of the class. /// /// Kinesis offset within the shard (partition) from which this message came. /// Receiver-generated sequenceNumber for this message. /// Index into a batch of events, if multiple events were delivered within a single Kinesis record. + [JsonConstructor] public KinesisSequenceToken(string shardSequence, long sequenceNumber, int eventIndex) : base(sequenceNumber, eventIndex) { - ShardSequence = shardSequence; + ShardSequence = shardSequence ?? throw new ArgumentNullException(nameof(shardSequence)); } /// @@ -38,6 +44,37 @@ public KinesisSequenceToken() : base() [JsonProperty] public string ShardSequence { get; } = null!; + /// + public override bool Equals(object? obj) => obj is StreamSequenceToken token && Equals(token); + + /// + public override bool Equals(StreamSequenceToken? other) + { + return other is KinesisSequenceToken token + && NumericShardSequence.Equals(token.NumericShardSequence) + && EventIndex == token.EventIndex; + } + + /// + public override int CompareTo(StreamSequenceToken? other) + { + if (other is null) + { + return 1; + } + + if (other is not KinesisSequenceToken token) + { + throw new ArgumentOutOfRangeException(nameof(other)); + } + + var difference = NumericShardSequence.CompareTo(token.NumericShardSequence); + return difference != 0 ? difference : EventIndex.CompareTo(token.EventIndex); + } + + /// + public override int GetHashCode() => HashCode.Combine(NumericShardSequence, EventIndex); + /// Returns a string that represents the current object. /// A string that represents the current object. /// 2 @@ -45,5 +82,11 @@ public override string ToString() { return string.Format(CultureInfo.InvariantCulture, "KinesisSequenceToken(ShardSequence: {0}, SequenceNumber: {1}, EventIndex: {2})", ShardSequence, SequenceNumber, EventIndex); } + + private BigInteger NumericShardSequence + => _numericShardSequence ??= BigInteger.Parse( + ShardSequence, + NumberStyles.None, + CultureInfo.InvariantCulture); } } diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetBatchContainer.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetBatchContainer.cs index 2b2cd1a87ab..77e4325addb 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetBatchContainer.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetBatchContainer.cs @@ -4,8 +4,8 @@ namespace Orleans.Streaming.AdoNet; /// The implementation for the ADONET provider. /// /// -/// 1. This class only supports binary serialization as performance and data size is the priority for database storage. -/// 2. Though the is supported here, it is not yet used, as the ADO.NET provider is not rewindable. +/// This class uses binary serialization to prioritize database storage performance and payload size. +/// Its identifies a record position in the partition history. /// [GenerateSerializer] [Alias("Orleans.Streaming.AdoNet.AdoNetBatchContainer")] @@ -81,7 +81,7 @@ public static AdoNetBatchContainer FromMessage(Serializer // A stored stream message always contains a serialized batch container. var container = serializer.Deserialize(message.Payload)!; container.SequenceToken = new(message.MessageId); - container.Dequeued = message.Dequeued; + container.Dequeued = 0; return container; } diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapter.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapter.cs index 479de28c0b2..3bdedc728fa 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapter.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapter.cs @@ -3,17 +3,48 @@ namespace Orleans.Streaming.AdoNet; /// /// Stream queue storage adapter for ADO.NET providers. /// -internal partial class AdoNetQueueAdapter(string name, AdoNetStreamOptions streamOptions, ClusterOptions clusterOptions, SimpleQueueCacheOptions cacheOptions, AdoNetStreamQueueMapper mapper, RelationalOrleansQueries queries, Serializer serializer, ILogger logger, IServiceProvider serviceProvider) : IQueueAdapter +internal partial class AdoNetQueueAdapter : IQueueAdapter, IQueueAdapterCache { - private readonly ILogger _logger = logger; + private readonly AdoNetStreamOptions _streamOptions; + private readonly ClusterOptions _clusterOptions; + private readonly SimpleQueueCacheOptions _cacheOptions; + private readonly AdoNetStreamQueueMapper _mapper; + private readonly RelationalOrleansQueries _queries; + private readonly Serializer _serializer; + private readonly ILogger _logger; + private readonly IServiceProvider _serviceProvider; + private readonly QueueAdapterReceiverRegistry _receivers; + + public AdoNetQueueAdapter( + string name, + AdoNetStreamOptions streamOptions, + ClusterOptions clusterOptions, + SimpleQueueCacheOptions cacheOptions, + AdoNetStreamQueueMapper mapper, + RelationalOrleansQueries queries, + Serializer serializer, + ILogger logger, + IServiceProvider serviceProvider) + { + Name = name; + _streamOptions = streamOptions; + _clusterOptions = clusterOptions; + _cacheOptions = cacheOptions; + _mapper = mapper; + _queries = queries; + _serializer = serializer; + _logger = logger; + _serviceProvider = serviceProvider; + _receivers = new QueueAdapterReceiverRegistry(CreateReceiverCore); + } /// /// Maps to the ProviderId in the database. /// - public string Name { get; } = name; + public string Name { get; } /// - /// The ADO.NET provider is not yet rewindable. + /// The ADO.NET partitioned stream provider resumes from its durable partition checkpoint. /// public bool IsRewindable => false; @@ -23,49 +54,65 @@ internal partial class AdoNetQueueAdapter(string name, AdoNetStreamOptions strea public StreamProviderDirection Direction => StreamProviderDirection.ReadWrite; public IQueueAdapterReceiver CreateReceiver(QueueId queueId) - { - // map the queue id - var adoNetQueueId = mapper.GetAdoNetQueueId(queueId); + => _receivers.GetOrCreate(queueId); - // create the receiver - return ReceiverFactory(serviceProvider, [Name, adoNetQueueId, streamOptions, clusterOptions, cacheOptions, queries]); - } + public IQueueCache CreateQueueCache(QueueId queueId) + => _receivers.GetOrCreate(queueId); - public async Task QueueMessageBatchAsync(StreamId streamId, IEnumerable events, StreamSequenceToken? token, Dictionary? requestContext) + public async Task QueueMessageBatchAsync( + StreamId streamId, + IEnumerable events, + StreamSequenceToken? token, + Dictionary? requestContext) { - // the ADO.NET provider is not rewindable so we do not support user supplied tokens + // Producer-supplied tokens are not supported. Replay positions are consumer-side tokens. if (token is not null) { throw new ArgumentException($"{nameof(AdoNetQueueAdapter)} does not support a user supplied {nameof(StreamSequenceToken)}."); } - // map the Orleans stream id to the corresponding queue id - var queueId = mapper.GetAdoNetQueueId(streamId); - - // create the payload from the events - var payload = AdoNetBatchContainer.ToMessagePayload(serializer, streamId, events.Cast().ToList(), requestContext); + var queueId = _mapper.GetAdoNetQueueId(streamId); + var payload = AdoNetBatchContainer.ToMessagePayload( + _serializer, + streamId, + events.Cast().ToList(), + requestContext); - // we can enqueue the message now try { - await queries.QueueStreamMessageAsync(clusterOptions.ServiceId, Name, queueId, payload, streamOptions.ExpiryTimeout.TotalSecondsCeiling()); + await _queries.AppendStreamMessageAsync( + _clusterOptions.ServiceId, + Name, + queueId, + streamId.FullKey.ToArray(), + streamId.Namespace.Length, + payload); } - catch (Exception ex) + catch (Exception exception) { - LogFailedToQueueStreamMessage(ex, clusterOptions.ServiceId, Name, queueId); + LogFailedToAppendStreamMessage( + exception, + _clusterOptions.ServiceId, + Name, + queueId); throw; } } - /// - /// The receiver factory. - /// - private static readonly ObjectFactory ReceiverFactory = ActivatorUtilities.CreateFactory([typeof(string), typeof(string), typeof(AdoNetStreamOptions), typeof(ClusterOptions), typeof(SimpleQueueCacheOptions), typeof(RelationalOrleansQueries)]); - - #region Logging - - [LoggerMessage(1, LogLevel.Error, "Failed to queue stream message with ({ServiceId}, {ProviderId}, {QueueId})")] - private partial void LogFailedToQueueStreamMessage(Exception ex, string serviceId, string providerId, string queueId); + private AdoNetQueueAdapterReceiver CreateReceiverCore(QueueId queueId) + { + var receiver = ActivatorUtilities.CreateInstance( + _serviceProvider, + Name, + _mapper.GetAdoNetQueueId(queueId), + _streamOptions, + _clusterOptions, + _cacheOptions, + _queries); + receiver.OnShutdown = current => _receivers.Remove(queueId, current); + return receiver; + } - #endregion Logging -} \ No newline at end of file + [LoggerMessage(1, LogLevel.Error, "Failed to append stream message with ({ServiceId}, {ProviderId}, {QueueId})")] + private partial void LogFailedToAppendStreamMessage(Exception ex, string serviceId, string providerId, string queueId); +} diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterFactory.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterFactory.cs index f681cc5b7bd..c3d5cb3d76c 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterFactory.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterFactory.cs @@ -3,7 +3,7 @@ namespace Orleans.Streaming.AdoNet; -internal class AdoNetQueueAdapterFactory : IQueueAdapterFactory +internal class AdoNetQueueAdapterFactory : IQueueAdapterFactory, IQueueAdapterCache { public AdoNetQueueAdapterFactory(string name, AdoNetStreamOptions streamOptions, ClusterOptions clusterOptions, SimpleQueueCacheOptions cacheOptions, HashRingStreamQueueMapperOptions hashOptions, ILoggerFactory loggerFactory, IHostApplicationLifetime lifetime, IServiceProvider serviceProvider) { @@ -15,7 +15,6 @@ public AdoNetQueueAdapterFactory(string name, AdoNetStreamOptions streamOptions, _serviceProvider = serviceProvider; _streamQueueMapper = new HashRingBasedStreamQueueMapper(hashOptions, name); - _cache = new SimpleQueueAdapterCache(cacheOptions, name, loggerFactory); _adoNetQueueMapper = new AdoNetStreamQueueMapper(_streamQueueMapper); } @@ -27,10 +26,10 @@ public AdoNetQueueAdapterFactory(string name, AdoNetStreamOptions streamOptions, private readonly IServiceProvider _serviceProvider; private readonly HashRingBasedStreamQueueMapper _streamQueueMapper; - private readonly SimpleQueueAdapterCache _cache; private readonly AdoNetStreamQueueMapper _adoNetQueueMapper; private RelationalOrleansQueries? _queries; + private AdoNetQueueAdapter? _adapter; /// /// Unfortunate implementation detail to account for lack of async lifetime. @@ -74,17 +73,23 @@ public async Task CreateAdapter() { var queries = await GetQueriesAsync(); - return AdapterFactory(_serviceProvider, [_name, _streamOptions, _clusterOptions, _cacheOptions, _adoNetQueueMapper, queries]); + return _adapter ??= (AdoNetQueueAdapter)AdapterFactory( + _serviceProvider, + [_name, _streamOptions, _clusterOptions, _cacheOptions, _adoNetQueueMapper, queries]); } public async Task GetDeliveryFailureHandler(QueueId queueId) { var queries = await GetQueriesAsync(); - return HandlerFactory(_serviceProvider, [false, _streamOptions, _clusterOptions, _adoNetQueueMapper, queries]); + return HandlerFactory(_serviceProvider, [_streamOptions.FaultOnDeliveryFailure, _streamOptions, _clusterOptions, _adoNetQueueMapper, queries]); } - public IQueueAdapterCache GetQueueAdapterCache() => _cache; + public IQueueAdapterCache GetQueueAdapterCache() => this; + + public IQueueCache CreateQueueCache(QueueId queueId) + => (_adapter ?? throw new InvalidOperationException("The ADO.NET stream adapter must be created before its queue cache.")) + .CreateQueueCache(queueId); public IStreamQueueMapper GetStreamQueueMapper() => _streamQueueMapper; diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs index 3a6bcdca5af..33e7c149b96 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetQueueAdapterReceiver.cs @@ -1,47 +1,18 @@ -namespace Orleans.Streaming.AdoNet; - -internal interface IStreamMessageQueries -{ - Task> GetStreamMessagesAsync( - string serviceId, - string providerId, - string queueId, - int maxCount, - int maxAttempts, - int visibilityTimeout, - int removalTimeout, - int evictionInterval, - int evictionBatchSize); +using System.Diagnostics.CodeAnalysis; - Task> ConfirmStreamMessagesAsync( - string serviceId, - string providerId, - string queueId, - IList messages); - - Task> ReleaseStreamMessagesAsync( - string serviceId, - string providerId, - string queueId, - IList messages); -} +namespace Orleans.Streaming.AdoNet; /// -/// Receives message batches from an individual queue of an ADO.NET provider. +/// Receives records from one stream partition through the recoverable stream partition pipeline. /// -internal partial class AdoNetQueueAdapterReceiver(string providerId, string queueId, AdoNetStreamOptions streamOptions, ClusterOptions clusterOptions, SimpleQueueCacheOptions cacheOptions, IStreamMessageQueries queries, Serializer serializer, ILogger logger) : IQueueAdapterReceiver +internal sealed class AdoNetQueueAdapterReceiver : IQueueAdapterReceiver, IQueueCache { - private readonly ILogger _logger = logger; - private readonly object _lock = new(); - private readonly Dictionary _pendingMessages = []; - - /// - /// Flags that no further work should be attempted. - /// - private bool _shutdown; + private const int BufferSize = 1024 * 1024; + private readonly RecoverableStreamReceiver _inner; + private readonly AdoNetRecoverableStream _source; + private int _shutdownNotified; - private int _activeOperations; - private TaskCompletionSource? _operationsCompleted; + internal Action? OnShutdown { get; set; } public AdoNetQueueAdapterReceiver( string providerId, @@ -52,256 +23,111 @@ public AdoNetQueueAdapterReceiver( RelationalOrleansQueries queries, Serializer serializer, ILogger logger) - : this(providerId, queueId, streamOptions, clusterOptions, cacheOptions, new RelationalStreamMessageQueries(queries), serializer, logger) { - } - - /// - /// This receiver does not require initialization. - /// - public Task Initialize(TimeSpan timeout) => Task.CompletedTask; - - /// - /// Waits for any outstanding work before shutting down. - /// - public async Task Shutdown(TimeSpan timeout) - { - Task? operationsCompleted; - lock (_lock) - { - _shutdown = true; - operationsCompleted = _operationsCompleted?.Task; - } - - if (operationsCompleted is not null) - { - try - { - await operationsCompleted.WaitAsync(timeout); - } - catch (Exception ex) + _source = new AdoNetRecoverableStream( + clusterOptions.ServiceId, + providerId, + queueId, + streamOptions, + queries, + logger); + var checkpointer = new StreamQueueCheckpointer( + _source, + new StreamQueueCheckpointerOptions { - LogShutdownFault(ex, clusterOptions.ServiceId, providerId, queueId); - return; - } - } - - List pending; - lock (_lock) - { - RemoveExpiredPendingMessages(); - if (_pendingMessages.Count == 0) - { - return; - } - - pending = _pendingMessages - .Select(static item => new AdoNetStreamConfirmation(item.Key, item.Value.Dequeued)) - .ToList(); - } - - try - { - var released = await queries.ReleaseStreamMessagesAsync(clusterOptions.ServiceId, providerId, queueId, pending).WaitAsync(timeout); - lock (_lock) - { - foreach (var message in released) - { - _pendingMessages.Remove(message.MessageId); - } - } - } - catch (Exception ex) - { - LogReleaseFailed(ex, clusterOptions.ServiceId, providerId, queueId, pending); - } + CheckpointComparer = StreamCheckpointComparers.Numeric, + PersistInterval = streamOptions.CheckpointPersistInterval, + }); + var dataAdapter = new AdoNetRecoverableStreamDataAdapter(serializer); + var bufferPool = new ObjectPool(() => new FixedSizeBuffer(BufferSize)); + var evictionStrategy = new ChronologicalEvictionStrategy( + logger, + new TimePurgePredicate(TimeSpan.MaxValue, TimeSpan.MaxValue), + cacheMonitor: null, + monitorWriteInterval: null); + var cache = new RecoverableStreamQueueCache( + Math.Min(streamOptions.MaxMessagesPerRead, cacheOptions.CacheSize), + bufferPool, + dataAdapter, + evictionStrategy, + logger, + maxCacheSize: cacheOptions.CacheSize); + _inner = new RecoverableStreamReceiver( + _source, + dataAdapter, + cache, + checkpointer, + streamOptions.StartFromNow); } - /// - public async Task> GetQueueMessagesAsync(int maxCount) - { - if (!TryBeginOperation()) - { - return []; - } - - // cap max count as appropriate - maxCount = Math.Min(maxCount, cacheOptions.CacheSize); - - try - { - var messages = await queries.GetStreamMessagesAsync( - clusterOptions.ServiceId, - providerId, - queueId, - maxCount, - streamOptions.MaxAttempts, - streamOptions.VisibilityTimeout.TotalSecondsCeiling(), - streamOptions.DeadLetterEvictionTimeout.TotalSecondsCeiling(), - streamOptions.EvictionInterval.TotalSecondsCeiling(), - streamOptions.EvictionBatchSize); - - lock (_lock) - { - RemoveExpiredPendingMessages(); - foreach (var message in messages) - { - _pendingMessages[message.MessageId] = new(message.Dequeued, message.ExpiresOn); - } - } - - // convert the messages into standard batch containers - return messages.Select(x => AdoNetBatchContainer.FromMessage(serializer, x)).Cast().ToList(); - } - catch (Exception ex) - { - LogDequeueFailed(ex, clusterOptions.ServiceId, providerId, queueId); - throw; - } - finally - { - EndOperation(); - } - } + public Task Initialize(TimeSpan timeout) => _inner.Initialize(timeout); - /// - public async Task MessagesDeliveredAsync(IList messages) + public async Task Shutdown(TimeSpan timeout) { - // skip work if there are no messages to deliver - if (messages.Count == 0) - { - return; - } - - if (!TryBeginOperation()) - { - return; - } - - // get the identifiers for the messages to confirm - var items = messages.Cast().Select(x => new AdoNetStreamConfirmation(x.SequenceToken.SequenceNumber, x.Dequeued)).ToList(); - try { - try - { - var confirmed = await queries.ConfirmStreamMessagesAsync(clusterOptions.ServiceId, providerId, queueId, items); - var receipts = items.ToDictionary(static item => item.MessageId, static item => item.Dequeued); - lock (_lock) - { - foreach (var message in confirmed) - { - if (receipts.TryGetValue(message.MessageId, out var receipt) - && _pendingMessages.TryGetValue(message.MessageId, out var pending) - && receipt == pending.Dequeued) - { - _pendingMessages.Remove(message.MessageId); - } - } - } - } - catch (Exception ex) - { - LogConfirmationFailed(ex, clusterOptions.ServiceId, providerId, queueId, items); - throw; - } + await _inner.Shutdown(timeout); } finally { - EndOperation(); - } - } - - private bool TryBeginOperation() - { - lock (_lock) - { - if (_shutdown) + var acquisitionCompletion = _source.AcquisitionCompletion; + if (acquisitionCompletion.IsCompleted) { - return false; + NotifyShutdown(); } - - if (_activeOperations++ == 0) + else { - _operationsCompleted = new(TaskCreationOptions.RunContinuationsAsynchronously); + _ = NotifyShutdownAfterAcquisition(acquisitionCompletion, NotifyShutdown); } - - return true; } } - private void EndOperation() + internal static async Task NotifyShutdownAfterAcquisition( + Task acquisitionCompletion, + Action notifyShutdown) { - TaskCompletionSource? operationsCompleted = null; - lock (_lock) - { - if (--_activeOperations == 0) - { - operationsCompleted = _operationsCompleted; - _operationsCompleted = null; - } - } - - operationsCompleted?.TrySetResult(); + ArgumentNullException.ThrowIfNull(acquisitionCompletion); + ArgumentNullException.ThrowIfNull(notifyShutdown); + await acquisitionCompletion.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); + notifyShutdown(); } - private void RemoveExpiredPendingMessages() + private void NotifyShutdown() { - var now = DateTime.UtcNow; - var expired = _pendingMessages - .Where(message => message.Value.ExpiresOn <= now) - .Select(static message => message.Key) - .ToList(); - foreach (var messageId in expired) + if (Interlocked.Exchange(ref _shutdownNotified, 1) == 0) { - _pendingMessages.Remove(messageId); + OnShutdown?.Invoke(this); } } - private readonly record struct PendingMessage(int Dequeued, DateTime ExpiresOn); + public Task> GetQueueMessagesAsync(int maxCount) + => _inner.GetQueueMessagesAsync(maxCount, CancellationToken.None); - private sealed class RelationalStreamMessageQueries(RelationalOrleansQueries queries) : IStreamMessageQueries - { - public Task> GetStreamMessagesAsync( - string serviceId, - string providerId, - string queueId, - int maxCount, - int maxAttempts, - int visibilityTimeout, - int removalTimeout, - int evictionInterval, - int evictionBatchSize) => - queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize); + public Task> GetQueueMessagesAsync( + int maxCount, + CancellationToken cancellationToken) + => _inner.GetQueueMessagesAsync(maxCount, cancellationToken); - public Task> ConfirmStreamMessagesAsync( - string serviceId, - string providerId, - string queueId, - IList messages) => - queries.ConfirmStreamMessagesAsync(serviceId, providerId, queueId, messages); + public Task MessagesDeliveredAsync(IList messages) + => _inner.MessagesDeliveredAsync(messages, CancellationToken.None); - public Task> ReleaseStreamMessagesAsync( - string serviceId, - string providerId, - string queueId, - IList messages) => - queries.ReleaseStreamMessagesAsync(serviceId, providerId, queueId, messages); - } + public Task MessagesDeliveredAsync( + IList messages, + CancellationToken cancellationToken) + => _inner.MessagesDeliveredAsync(messages, cancellationToken); - #region Logging + public int GetMaxAddCount() => _inner.GetMaxAddCount(); - [LoggerMessage(1, LogLevel.Error, "Failed to get messages from ({ServiceId}, {ProviderId}, {QueueId})")] - private partial void LogDequeueFailed(Exception exception, string serviceId, string providerId, string queueId); + public void AddToCache(IList messages) => _inner.AddToCache(messages); - [LoggerMessage(2, LogLevel.Error, "Failed to confirm messages for ({ServiceId}, {ProviderId}, {QueueId}, {@Items})")] - private partial void LogConfirmationFailed(Exception exception, string serviceId, string providerId, string queueId, List items); + public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList purgedItems) + => _inner.TryPurgeFromCache(out purgedItems); - [LoggerMessage(3, LogLevel.Warning, "Handled fault while shutting down receiver for ({ServiceId}, {ProviderId}, {QueueId})")] - private partial void LogShutdownFault(Exception exception, string serviceId, string providerId, string queueId); + public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) + => _inner.GetCacheCursor(streamId, token); - [LoggerMessage(4, LogLevel.Warning, "Failed to release messages while shutting down receiver for ({ServiceId}, {ProviderId}, {QueueId}, {@Items})")] - private partial void LogReleaseFailed(Exception exception, string serviceId, string providerId, string queueId, List items); + public bool IsUnderPressure() => _inner.IsUnderPressure(); - #endregion Logging + public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToken, DateTime utcNow) + => _inner.UpdateDeliveryProgress(earliestSubscriptionToken, utcNow); } diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs new file mode 100644 index 00000000000..82f71385a1d --- /dev/null +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetRecoverableStream.cs @@ -0,0 +1,237 @@ +using System.Globalization; +using Orleans.Providers.Streams.Common; + +namespace Orleans.Streaming.AdoNet; + +internal sealed partial class AdoNetRecoverableStream( + string serviceId, + string providerId, + string queueId, + AdoNetStreamOptions options, + RelationalOrleansQueries queries, + ILogger logger) : IRecoverableStreamSource, IStreamCheckpointStore +{ + private AdoNetStreamPartitionState? _partition; + private long _readOffset; + private Task? _acquisitionTask; + + internal Task AcquisitionCompletion => Volatile.Read(ref _acquisitionTask) ?? Task.CompletedTask; + + public async ValueTask Load(CancellationToken cancellationToken) + { + var acquisitionTask = queries.AcquireStreamPartitionAsync( + serviceId, + providerId, + queueId, + options.StartFromNow, + cancellationToken); + Volatile.Write(ref _acquisitionTask, acquisitionTask); + var partition = await acquisitionTask; + cancellationToken.ThrowIfCancellationRequested(); + _partition = partition; + _readOffset = partition.Checkpoint ?? 0; + ThrowIfRetentionGap(partition); + var checkpoint = partition.Checkpoint?.ToString(CultureInfo.InvariantCulture) ?? string.Empty; + return new(checkpoint, partition.OwnerEpoch.ToString(CultureInfo.InvariantCulture)); + } + + public async ValueTask Update( + string checkpoint, + string expectedVersion, + CancellationToken cancellationToken) + { + if (_partition is not { } partition) + { + throw new InvalidOperationException("The ADO.NET stream partition checkpoint must be loaded before it can be updated."); + } + + var checkpointValue = long.Parse(checkpoint, NumberStyles.None, CultureInfo.InvariantCulture); + var ownerEpoch = long.Parse(expectedVersion, NumberStyles.None, CultureInfo.InvariantCulture); + var update = await queries.AdvanceStreamCheckpointAsync( + serviceId, + providerId, + queueId, + ownerEpoch, + checkpointValue, + cancellationToken); + return ResolveCheckpointUpdate( + $"{serviceId}/{providerId}/{queueId}", + partition.OwnerEpoch, + update); + } + + internal static StreamCheckpointStoreState ResolveCheckpointUpdate( + string partitionId, + long acquiredOwnerEpoch, + AdoNetStreamCheckpointUpdate? update) + { + if (update is not null && update.OwnerEpoch == acquiredOwnerEpoch) + { + return new( + (update.Checkpoint ?? 0).ToString(CultureInfo.InvariantCulture), + update.OwnerEpoch.ToString(CultureInfo.InvariantCulture)); + } + + throw new InvalidOperationException( + $"ADO.NET stream partition ownership was lost for '{partitionId}' at epoch {acquiredOwnerEpoch}. The stale receiver cannot advance its checkpoint."); + } + + public Task Initialize( + RecoverableStreamStartPosition position, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (_partition is null) + { + throw new InvalidOperationException("The ADO.NET stream partition checkpoint must be loaded before initializing its source."); + } + + _readOffset = position.Checkpoint is null + ? 0 + : long.Parse(position.Checkpoint, NumberStyles.None, CultureInfo.InvariantCulture); + return Task.CompletedTask; + } + + public async Task> Read( + int maxCount, + CancellationToken cancellationToken) + { + var messages = await queries.ReadStreamMessagesAsync( + serviceId, + providerId, + queueId, + _readOffset, + Math.Min(maxCount, options.MaxMessagesPerRead), + cancellationToken); + + var cleanup = await queries.CleanupStreamMessagesAsync( + serviceId, + providerId, + queueId, + AdoNetStreamTime.ToSqlSeconds(options.RetentionPeriod), + options.MaximumRetentionPeriod is { } maximum + ? AdoNetStreamTime.ToSqlSeconds(maximum) + : null, + AdoNetStreamTime.ToSqlSeconds(options.CleanupInterval), + options.CleanupBatchSize, + cancellationToken); + if (cleanup.HardDeletedCount > 0) + { + LogHardRetentionCrossed( + logger, + serviceId, + providerId, + queueId, + cleanup.HardDeletedCount, + cleanup.HardDeletedFromMessageId, + cleanup.HardDeletedThroughMessageId, + cleanup.Checkpoint); + } + + return messages as IReadOnlyList ?? messages.ToList(); + } + + public void MessagesAdded(IReadOnlyList messages) + { + if (messages.Count > 0) + { + _readOffset = messages[^1].MessageId; + } + } + + public Task Shutdown(CancellationToken cancellationToken) + => cancellationToken.IsCancellationRequested + ? Task.FromCanceled(cancellationToken) + : Task.CompletedTask; + + private void ThrowIfRetentionGap(AdoNetStreamPartitionState state) + { + if (HasRetentionGap(state)) + { + throw new DataNotAvailableException( + $"ADO.NET stream partition '{serviceId}/{providerId}/{queueId}' has a retention gap: " + + $"checkpoint {state.Checkpoint}, earliest retained record {state.EarliestMessageId?.ToString(CultureInfo.InvariantCulture) ?? ""}, " + + $"next message id {state.NextMessageId}, tail {state.TailMessageId?.ToString(CultureInfo.InvariantCulture) ?? ""}."); + } + } + + internal static bool HasRetentionGap(AdoNetStreamPartitionState state) + { + if (state.Checkpoint is not { } checkpoint) + { + return false; + } + + var earliestAvailablePosition = state.EarliestMessageId ?? state.NextMessageId; + return checkpoint < earliestAvailablePosition - 1; + } + + [LoggerMessage( + Level = LogLevel.Warning, + Message = "Hard stream retention deleted {DeletedCount} records for {ServiceId}/{ProviderId}/{QueueId} from {DeletedFrom} through {DeletedThrough}, crossing checkpoint {Checkpoint}.")] + private static partial void LogHardRetentionCrossed( + ILogger logger, + string serviceId, + string providerId, + string queueId, + int deletedCount, + long? deletedFrom, + long? deletedThrough, + long? checkpoint); +} + +internal sealed class AdoNetRecoverableStreamDataAdapter( + Serializer serializer) : IRecoverableStreamDataAdapter +{ + public StreamPosition GetStreamPosition(AdoNetStreamMessage queueMessage) + => new(queueMessage.StreamId, new EventSequenceTokenV2(queueMessage.MessageId)); + + public CachedMessage FromQueueMessage( + StreamPosition streamPosition, + AdoNetStreamMessage queueMessage, + DateTime dequeueTimeUtc, + Func> getSegment) + { + var size = SegmentBuilder.CalculateAppendSize(queueMessage.Payload); + var segment = getSegment(size); + var offset = 0; + SegmentBuilder.Append(segment, ref offset, queueMessage.Payload); + return new CachedMessage + { + StreamId = streamPosition.StreamId, + SequenceNumber = queueMessage.MessageId, + EventIndex = streamPosition.SequenceToken.EventIndex, + EnqueueTimeUtc = queueMessage.CreatedOn, + DequeueTimeUtc = dequeueTimeUtc, + Segment = segment, + }; + } + + public IBatchContainer GetBatchContainer(ref CachedMessage cachedMessage) + { + var offset = 0; + var payload = SegmentBuilder.ReadNextBytes(cachedMessage.Segment, ref offset).ToArray(); + var message = new AdoNetStreamMessage( + string.Empty, + string.Empty, + string.Empty, + cachedMessage.SequenceNumber, + cachedMessage.StreamId.FullKey.ToArray(), + cachedMessage.StreamId.Namespace.Length, + cachedMessage.EnqueueTimeUtc, + payload); + return AdoNetBatchContainer.FromMessage(serializer, message); + } + + public StreamSequenceToken GetSequenceToken(ref CachedMessage cachedMessage) + => new EventSequenceTokenV2(cachedMessage.SequenceNumber, cachedMessage.EventIndex); + + public string GetOffset(ref CachedMessage cachedMessage) + => cachedMessage.SequenceNumber.ToString(CultureInfo.InvariantCulture); + + public bool TryGetOffset(StreamSequenceToken token, out string offset) + { + offset = token.SequenceNumber.ToString(CultureInfo.InvariantCulture); + return token is EventSequenceTokenV2; + } +} diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamCheckpointUpdate.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamCheckpointUpdate.cs new file mode 100644 index 00000000000..2c7ab53dba1 --- /dev/null +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamCheckpointUpdate.cs @@ -0,0 +1,12 @@ +namespace Orleans.Streaming.AdoNet; + +/// +/// Describes the result of an epoch-fenced checkpoint update. +/// +internal record AdoNetStreamCheckpointUpdate( + string ServiceId, + string ProviderId, + string QueueId, + long OwnerEpoch, + long? Checkpoint, + bool Updated); diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamCleanupResult.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamCleanupResult.cs new file mode 100644 index 00000000000..00fb1027756 --- /dev/null +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamCleanupResult.cs @@ -0,0 +1,15 @@ +namespace Orleans.Streaming.AdoNet; + +/// +/// Describes one bounded stream partition retention cleanup operation. +/// +internal record AdoNetStreamCleanupResult( + bool Ran, + int DeletedCount, + long? DeletedThroughMessageId, + int HardDeletedCount, + long? HardDeletedFromMessageId, + long? HardDeletedThroughMessageId, + long? Checkpoint, + long? EarliestMessageId, + long? TailMessageId); diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamConfirmation.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamConfirmation.cs deleted file mode 100644 index 6f9ec48c3a3..00000000000 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamConfirmation.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace Orleans.Streaming.AdoNet; - -/// -/// The model that represents a message that can be confirmed. -/// -internal record AdoNetStreamConfirmation( - long MessageId, - int Dequeued) -{ - public AdoNetStreamConfirmation() : this(0, 0) - { - } -} \ No newline at end of file diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamConfirmationAck.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamConfirmationAck.cs deleted file mode 100644 index 1ec232487f5..00000000000 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamConfirmationAck.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace Orleans.Streaming.AdoNet; - -/// -/// The model that represents a message that was successfully confirmed. -/// -internal record AdoNetStreamConfirmationAck( - string ServiceId, - string ProviderId, - string QueueId, - long MessageId) -{ - public AdoNetStreamConfirmationAck() : this("", "", "", 0) - { - } -} \ No newline at end of file diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamDeadLetter.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamDeadLetter.cs deleted file mode 100644 index 54816031bf4..00000000000 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamDeadLetter.cs +++ /dev/null @@ -1,23 +0,0 @@ -namespace Orleans.Streaming.AdoNet; - -/// -/// The model that represents a dead letter in an ADONET streaming provider. -/// -internal record AdoNetStreamDeadLetter( - string ServiceId, - string ProviderId, - string QueueId, - long MessageId, - int Dequeued, - DateTime VisibleOn, - DateTime ExpiresOn, - DateTime CreatedOn, - DateTime ModifiedOn, - DateTime DeadOn, - DateTime RemoveOn, - byte[] Payload) -{ - public AdoNetStreamDeadLetter() : this("", "", "", 0, 0, DateTime.MinValue, DateTime.MinValue, DateTime.MinValue, DateTime.MinValue, DateTime.MinValue, DateTime.MinValue, []) - { - } -} \ No newline at end of file diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamFailureHandler.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamFailureHandler.cs index e9e0ba906ef..b94c945002f 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamFailureHandler.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamFailureHandler.cs @@ -1,57 +1,70 @@ namespace Orleans.Streaming.AdoNet; /// -/// An that attempts to move the message to dead letters. +/// Logs subscriber failures while preserving shared partition records. /// -internal partial class AdoNetStreamFailureHandler(bool faultOnFailure, AdoNetStreamOptions streamOptions, ClusterOptions clusterOptions, AdoNetStreamQueueMapper mapper, RelationalOrleansQueries queries, ILogger logger) : IStreamFailureHandler +internal partial class AdoNetStreamFailureHandler : IStreamFailureHandler { - private readonly ILogger _logger = logger; + private readonly ILogger _logger; - /// - /// Gets a value indicating whether the subscription should fault when there is an error. - /// - public bool ShouldFaultSubsriptionOnError { get; } = faultOnFailure; - - /// - /// Attempts to move the message to dead letters on delivery failure. - /// - public Task OnDeliveryFailure(GuidId subscriptionId, string streamProviderName, StreamId streamIdentity, StreamSequenceToken? sequenceToken) => OnFailureAsync(streamProviderName, streamIdentity, sequenceToken); - - /// - /// Attempts to move the message to dead letters on delivery failure. - /// - public Task OnSubscriptionFailure(GuidId subscriptionId, string streamProviderName, StreamId streamIdentity, StreamSequenceToken? sequenceToken) => OnFailureAsync(streamProviderName, streamIdentity, sequenceToken); - - /// - /// Attempts to move the message to dead letters on delivery failure. - /// - private async Task OnFailureAsync(string streamProviderName, StreamId streamIdentity, StreamSequenceToken? sequenceToken) + public AdoNetStreamFailureHandler( + bool faultOnFailure, + AdoNetStreamOptions streamOptions, + ClusterOptions clusterOptions, + AdoNetStreamQueueMapper mapper, + RelationalOrleansQueries queries, + ILogger logger) { - ArgumentNullException.ThrowIfNull(streamProviderName); - ArgumentNullException.ThrowIfNull(sequenceToken); - - var queueId = mapper.GetAdoNetQueueId(streamIdentity); + ShouldFaultSubsriptionOnError = faultOnFailure; + _logger = logger; + _ = streamOptions; + _ = clusterOptions; + _ = mapper; + _ = queries; + } - try - { - await queries.FailStreamMessageAsync(clusterOptions.ServiceId, streamProviderName, queueId, sequenceToken.SequenceNumber, streamOptions.MaxAttempts, streamOptions.DeadLetterEvictionTimeout.TotalSecondsCeiling()); + /// + public bool ShouldFaultSubsriptionOnError { get; } - LogMovedMessage(clusterOptions.ServiceId, streamProviderName, queueId, sequenceToken.SequenceNumber); - } - catch (Exception ex) - { - LogFailedToMoveMessage(ex, clusterOptions.ServiceId, streamProviderName, queueId, sequenceToken.SequenceNumber); - throw; - } + /// + public Task OnDeliveryFailure( + GuidId subscriptionId, + string streamProviderName, + StreamId streamIdentity, + StreamSequenceToken? sequenceToken) + { + LogDeliveryFailure(_logger, subscriptionId, streamProviderName, streamIdentity, sequenceToken); + return Task.CompletedTask; } - #region Logging - - [LoggerMessage(1, LogLevel.Warning, "Moved failed delivery to dead letters: ({ServiceId}, {ProviderId}, {QueueId}, {MessageId})")] - private partial void LogMovedMessage(string serviceId, string providerId, string queueId, long messageId); + /// + public Task OnSubscriptionFailure( + GuidId subscriptionId, + string streamProviderName, + StreamId streamIdentity, + StreamSequenceToken? sequenceToken) + { + LogSubscriptionFailure(_logger, subscriptionId, streamProviderName, streamIdentity, sequenceToken); + return Task.CompletedTask; + } - [LoggerMessage(2, LogLevel.Error, "Failed to move failed delivery to dead letters: ({ServiceId}, {ProviderId}, {QueueId}, {MessageId}")] - private partial void LogFailedToMoveMessage(Exception ex, string serviceId, string providerId, string queueId, long messageId); + [LoggerMessage( + Level = LogLevel.Warning, + Message = "ADO.NET stream delivery failed for subscription {SubscriptionId} on provider {ProviderName}, stream {StreamId}, at {SequenceToken}. The partition record remains available.")] + private static partial void LogDeliveryFailure( + ILogger logger, + GuidId subscriptionId, + string providerName, + StreamId streamId, + StreamSequenceToken? sequenceToken); - #endregion Logging -} \ No newline at end of file + [LoggerMessage( + Level = LogLevel.Warning, + Message = "ADO.NET stream subscription {SubscriptionId} failed on provider {ProviderName}, stream {StreamId}, at {SequenceToken}. The partition record remains available.")] + private static partial void LogSubscriptionFailure( + ILogger logger, + GuidId subscriptionId, + string providerName, + StreamId streamId, + StreamSequenceToken? sequenceToken); +} diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamMessage.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamMessage.cs index 9bdea11805f..c5b2231ecf9 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamMessage.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamMessage.cs @@ -8,14 +8,19 @@ internal record AdoNetStreamMessage( string ProviderId, string QueueId, long MessageId, - int Dequeued, - DateTime VisibleOn, - DateTime ExpiresOn, + byte[] StreamIdBytes, + int StreamNamespaceLength, DateTime CreatedOn, - DateTime ModifiedOn, byte[] Payload) { - public AdoNetStreamMessage() : this("", "", "", 0, 0, DateTime.MinValue, DateTime.MinValue, DateTime.MinValue, DateTime.MinValue, []) + public AdoNetStreamMessage() : this("", "", "", 0, [], 0, DateTime.MinValue, []) { } + + /// + /// Gets the canonical stream identifier stored with this message. + /// + public StreamId StreamId => StreamId.Create( + StreamIdBytes.AsSpan(0, StreamNamespaceLength), + StreamIdBytes.AsSpan(StreamNamespaceLength)); } \ No newline at end of file diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs index 7823292b4ea..aa9074ff0a7 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptions.cs @@ -28,38 +28,57 @@ public class AdoNetStreamOptions public DbDataSource? DataSource { get; set; } /// - /// The maximum number of attempts to deliver a message. - /// The message is eventually moved to dead letters if these many attempts are made without success. + /// Gets or sets a value indicating whether a new partition checkpoint starts at the current partition history tail. /// - public int MaxAttempts { get; set; } = 5; + /// + /// When , a new checkpoint starts immediately before the earliest retained record. + /// This setting is only used while initializing a partition which does not have a checkpoint. + /// + public bool StartFromNow { get; set; } + + /// + /// Gets or sets a value indicating whether a subscription is faulted after delivery failure handling. + /// + public bool FaultOnDeliveryFailure { get; set; } + + /// + /// Gets or sets the maximum number of stream records returned by a partition read. + /// + public int MaxMessagesPerRead { get; set; } = 1_000; /// - /// The timeout until a message is allowed to be dequeued again if not yet confirmed. + /// Gets or sets the interval between checkpoint persistence attempts. /// - public TimeSpan VisibilityTimeout { get; set; } = TimeSpan.FromMinutes(1); + public TimeSpan CheckpointPersistInterval { get; set; } = TimeSpan.FromSeconds(5); /// - /// The expiry timeout until a message is considered expired and moved to dead letters regardless of attempts. - /// The message is only moved if the current attempt is also past its visibility timeout. + /// Gets or sets the minimum amount of time that a stream record is retained after it is checkpointed. /// - public TimeSpan ExpiryTimeout { get; set; } = TimeSpan.FromMinutes(10); + /// + /// Storage routines use whole seconds. Fractional values are rounded upward so configured retention is never shortened. + /// + public TimeSpan RetentionPeriod { get; set; } = TimeSpan.FromDays(1); /// - /// The removal timeout until a failed message is deleted from the dead letters table. + /// Gets or sets an optional hard retention ceiling. /// - public TimeSpan DeadLetterEvictionTimeout { get; set; } = TimeSpan.FromDays(7); + /// + /// Stream records older than this value can be deleted even when they are newer than the checkpoint. + /// Such deletions are reported by storage so that the receiver can emit gap diagnostics. + /// Fractional values are rounded upward to whole seconds. + /// + public TimeSpan? MaximumRetentionPeriod { get; set; } /// - /// The period of time between eviction activities. - /// These include moving expired messages to dead letters and removing dead letters after their own lifetime. - /// This period is cluster wide and will not change with the number of silos. + /// Gets or sets the interval between cleanup attempts for a partition. /// - public TimeSpan EvictionInterval { get; set; } = TimeSpan.FromSeconds(10); + /// Fractional values are rounded upward to whole seconds. + public TimeSpan CleanupInterval { get; set; } = TimeSpan.FromMinutes(1); /// - /// The maximum number of messages affected by an eviction batch. + /// Gets or sets the maximum number of stream records deleted by one cleanup operation. /// - public int EvictionBatchSize { get; set; } = 1000; + public int CleanupBatchSize { get; set; } = 1_000; /// /// A safety timeout for underlying database initialization. diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptionsValidator.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptionsValidator.cs index 906505be414..f974ca95834 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptionsValidator.cs +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamOptionsValidator.cs @@ -1,4 +1,5 @@ using static System.String; +using Orleans.Streaming.AdoNet; namespace Orleans.Configuration; @@ -20,34 +21,43 @@ public void ValidateConfiguration() throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': configure exactly one of {nameof(options.ConnectionString)} or {nameof(options.DataSource)}."); } - if (options.MaxAttempts < 0) + if (options.MaxMessagesPerRead <= 0) { - throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.MaxAttempts)} must be greater than zero."); + throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.MaxMessagesPerRead)} must be greater than zero."); } - if (options.VisibilityTimeout < TimeSpan.Zero) + if (options.CheckpointPersistInterval <= TimeSpan.Zero) { - throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.VisibilityTimeout)} must be greater than zero."); + throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.CheckpointPersistInterval)} must be greater than zero."); } - if (options.EvictionInterval < TimeSpan.Zero) + if (IsInvalidSqlInterval(options.RetentionPeriod)) { - throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.EvictionInterval)} must be greater than zero."); + throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.RetentionPeriod)} must be between one second and {int.MaxValue} seconds."); } - if (options.ExpiryTimeout < TimeSpan.Zero) + if (options.MaximumRetentionPeriod is { } maximumRetentionPeriod + && (IsInvalidSqlInterval(maximumRetentionPeriod) || maximumRetentionPeriod < options.RetentionPeriod)) { - throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.ExpiryTimeout)} must be greater than zero."); + throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.MaximumRetentionPeriod)} must fit in SQL integer seconds and be greater than or equal to {nameof(options.RetentionPeriod)}."); } - if (options.DeadLetterEvictionTimeout < TimeSpan.Zero) + if (IsInvalidSqlInterval(options.CleanupInterval)) { - throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.DeadLetterEvictionTimeout)} must be greater than zero."); + throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.CleanupInterval)} must be between one second and {int.MaxValue} seconds."); } - if (options.EvictionBatchSize < 0) + if (options.CleanupBatchSize <= 0) { - throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.EvictionBatchSize)} must be greater than zero."); + throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.CleanupBatchSize)} must be greater than zero."); + } + + if (options.InitializationTimeout <= TimeSpan.Zero) + { + throw new OrleansConfigurationException($"Invalid {nameof(AdoNetStreamOptions)} values for ADO.NET Streaming Provider '{name}': {nameof(options.InitializationTimeout)} must be greater than zero."); } } + + private static bool IsInvalidSqlInterval(TimeSpan value) + => !AdoNetStreamTime.IsValidSqlInterval(value); } \ No newline at end of file diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamPartitionState.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamPartitionState.cs new file mode 100644 index 00000000000..a84356007e9 --- /dev/null +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamPartitionState.cs @@ -0,0 +1,14 @@ +namespace Orleans.Streaming.AdoNet; + +/// +/// Describes the durable position and retained bounds of a stream partition. +/// +internal record AdoNetStreamPartitionState( + string ServiceId, + string ProviderId, + string QueueId, + long OwnerEpoch, + long NextMessageId, + long? Checkpoint, + long? EarliestMessageId, + long? TailMessageId); diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamTime.cs b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamTime.cs new file mode 100644 index 00000000000..74da9aa349f --- /dev/null +++ b/src/AdoNet/Orleans.Streaming.AdoNet/AdoNetStreamTime.cs @@ -0,0 +1,25 @@ +namespace Orleans.Streaming.AdoNet; + +internal static class AdoNetStreamTime +{ + private static readonly TimeSpan MaxSqlInterval = TimeSpan.FromSeconds(int.MaxValue); + + internal static bool IsValidSqlInterval(TimeSpan value) + => value >= TimeSpan.FromSeconds(1) && value <= MaxSqlInterval; + + internal static int ToSqlSeconds(TimeSpan value) + { + if (value < TimeSpan.Zero || value > MaxSqlInterval) + { + throw new OverflowException("The interval does not fit in SQL integer seconds."); + } + + var seconds = value.Ticks / TimeSpan.TicksPerSecond; + if (value.Ticks % TimeSpan.TicksPerSecond != 0) + { + seconds++; + } + + return checked((int)seconds); + } +} diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/Extensions.cs b/src/AdoNet/Orleans.Streaming.AdoNet/Extensions.cs deleted file mode 100644 index 2e2f2b8172e..00000000000 --- a/src/AdoNet/Orleans.Streaming.AdoNet/Extensions.cs +++ /dev/null @@ -1,20 +0,0 @@ -namespace Orleans.Streaming.AdoNet; - -/// -/// Internal syntax sugar. -/// -internal static class Extensions -{ - /// - public static int Int32Ceiling(this double value) => (int)Math.Ceiling(value); - - /// - /// Rounds up the specified time span to the nearest upper second and returns the total number of seconds as an integer. - /// - public static int TotalSecondsCeiling(this TimeSpan value) => value.TotalSeconds.Int32Ceiling(); - - /// - /// Rounds up the specified time span to the nearest upper second and returns the total number of seconds as an integer. - /// - public static TimeSpan SecondsCeiling(this TimeSpan value) => TimeSpan.FromSeconds(value.TotalSecondsCeiling()); -} \ No newline at end of file diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql b/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql index 7caa8b2e4e0..3adeea780a4 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql +++ b/src/AdoNet/Orleans.Streaming.AdoNet/MySQL-Streaming.sql @@ -1,389 +1,169 @@ -CREATE TABLE OrleansStreamMessageSequence -( - MessageId BIGINT NOT NULL -); -INSERT INTO OrleansStreamMessageSequence -SELECT 0 -WHERE NOT EXISTS (SELECT * FROM OrleansStreamMessageSequence); - -DELIMITER $$ - -CREATE TABLE OrleansStreamMessage -( - /* Identifies the application */ - ServiceId NVARCHAR(150) NOT NULL, - - /* Identifies the provider within the application */ - ProviderId NVARCHAR(150) NOT NULL, - - /* Identifies the individual queue shard as configured in the provider*/ - QueueId NVARCHAR(150) NOT NULL, - - /* The unique ascending number of the queued message */ - MessageId BIGINT NOT NULL, - - /* The number of times the event was dequeued */ - Dequeued INT NOT NULL, - - /* The UTC time at which the event will become visible */ - VisibleOn DATETIME(6) NOT NULL, - - /* The UTC time at which the event will expire */ - ExpiresOn DATETIME(6) NOT NULL, +/* +ADO.NET streaming schema version 2. - /* The UTC time at which the event was created - troubleshooting only */ - CreatedOn DATETIME(6) NOT NULL, +This alpha schema is intentionally incompatible with the former destructive queue schema. +Drop the former streaming tables, sequence, routines, and OrleansQuery rows before applying +this script. Existing queue rows are not migrated. +*/ - /* The UTC time at which the event was updated - troubleshooting only */ - ModifiedOn DATETIME(6) NOT NULL, - - /* The arbitrarily large payload of the event */ - Payload LONGBLOB NOT NULL, - - /* This PK supports the various ordered scanning queries. */ - PRIMARY KEY (ServiceId, ProviderId, QueueId, MessageId) -); +DROP PROCEDURE IF EXISTS ValidateOrleansStreamingSchemaUpgrade; DELIMITER $$ -CREATE TABLE OrleansStreamDeadLetter -( - /* Identifies the application */ - ServiceId NVARCHAR(150) NOT NULL, - - /* Identifies the provider within the application */ - ProviderId NVARCHAR(150) NOT NULL, - - /* Identifies the individual queue shard as configured in the provider*/ - QueueId NVARCHAR(150) NOT NULL, - - /* The unique ascending number of the queued message */ - MessageId BIGINT NOT NULL, - - /* The number of times the event was dequeued */ - Dequeued INT NOT NULL, - - /* The UTC time at which the event will become visible */ - VisibleOn DATETIME(6) NOT NULL, - - /* The UTC time at which the event will expire */ - ExpiresOn DATETIME(6) NOT NULL, - - /* The UTC time at which the event was created - troubleshooting only */ - CreatedOn DATETIME(6) NOT NULL, - - /* The UTC time at which the event was updated - troubleshooting only */ - ModifiedOn DATETIME(6) NOT NULL, - - /* The UTC time at which the event was given up on - troubleshooting only */ - DeadOn DATETIME(6) NOT NULL, - - /* The UTC time at which the event is scheduled to be removed from dead letters */ - RemoveOn DATETIME(6) NOT NULL, - - /* The arbitrarily large payload of the event */ - Payload LONGBLOB NULL, - - /* This PK supports the various ordered scanning queries. */ - PRIMARY KEY (ServiceId, ProviderId, QueueId, MessageId) -); - -DELIMITER $$ - -CREATE TABLE OrleansStreamControl -( - /* Identifies the application */ - ServiceId NVARCHAR(150) NOT NULL, - - /* Identifies the provider within the application */ - ProviderId NVARCHAR(150) NOT NULL, - - /* Identifies the individual queue shard as configured in the provider */ - QueueId NVARCHAR(150) NOT NULL, - - /* The next due schedule for messages to be evicted */ - EvictOn DATETIME(6) NOT NULL, - - /* Each row represents a flat configuration object for an individual queue */ - PRIMARY KEY (ServiceId, ProviderId, QueueId) -); - -DELIMITER $$ - -CREATE PROCEDURE QueueStreamMessage -( - IN _ServiceId NVARCHAR(150), - IN _ProviderId NVARCHAR(150), - IN _QueueId NVARCHAR(150), - IN _Payload LONGBLOB, - IN _ExpiryTimeout INT -) +CREATE PROCEDURE ValidateOrleansStreamingSchemaUpgrade() BEGIN + IF EXISTS + ( + SELECT 1 + FROM information_schema.tables + WHERE table_schema = DATABASE() + AND table_name IN + ( + 'OrleansStreamPartition', + 'OrleansStreamMessage', + 'OrleansStreamDeadLetter', + 'OrleansStreamControl', + 'OrleansStreamMessageSequence' + ) + ) + OR EXISTS + ( + SELECT 1 + FROM OrleansQuery + WHERE QueryKey IN + ( + 'QueueStreamMessageKey', + 'GetStreamMessagesKey', + 'ConfirmStreamMessagesKey', + 'FailStreamMessageKey', + 'EvictStreamMessagesKey', + 'EvictStreamDeadLettersKey', + 'StreamSchemaVersionKey' + ) + ) + THEN + SIGNAL SQLSTATE '45000' + SET MESSAGE_TEXT = 'Incompatible alpha ADO.NET streaming schema. Drop old streaming objects and query rows; no in-place migration.'; + END IF; +END$$ -DECLARE _Now DATETIME(6) DEFAULT UTC_TIMESTAMP(6); -DECLARE _ExpiresOn DATETIME(6) DEFAULT DATE_ADD(_Now, INTERVAL _ExpiryTimeout SECOND); -DECLARE _MessageId BIGINT; - -UPDATE OrleansStreamMessageSequence -SET MessageId = LAST_INSERT_ID(MessageId + 1); +DELIMITER ; -SET _MessageId = LAST_INSERT_ID(); +CALL ValidateOrleansStreamingSchemaUpgrade(); +DROP PROCEDURE ValidateOrleansStreamingSchemaUpgrade; -INSERT INTO OrleansStreamMessage +CREATE TABLE OrleansStreamPartition ( - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - Payload -) -VALUES -( - _ServiceId, - _ProviderId, - _QueueId, - _MessageId, - 0, - _Now, - _ExpiresOn, - _Now, - _Now, - _Payload -); - -SELECT - _ServiceId AS ServiceId, - _ProviderId AS ProviderId, - _QueueId AS QueueId, - _MessageId AS MessageId; - -END; + ServiceId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + ProviderId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + QueueId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + NextMessageId BIGINT NOT NULL, + Checkpoint BIGINT NULL, + OwnerEpoch BIGINT NOT NULL, + CleanupOn DATETIME(6) NOT NULL, + CreatedOn DATETIME(6) NOT NULL, + ModifiedOn DATETIME(6) NOT NULL, + + PRIMARY KEY (ServiceId, ProviderId, QueueId) +) ENGINE = InnoDB; -DELIMITER $$ - -INSERT INTO OrleansQuery +CREATE TABLE OrleansStreamMessage ( - QueryKey, - QueryText -) -SELECT - 'QueueStreamMessageKey', - 'CALL QueueStreamMessage(@ServiceId, @ProviderId, @QueueId, @Payload, @ExpiryTimeout)' + ServiceId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + ProviderId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + QueueId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + MessageId BIGINT NOT NULL, + StreamIdBytes LONGBLOB NOT NULL, + StreamNamespaceLength INT NOT NULL, + CreatedOn DATETIME(6) NOT NULL, + CheckpointedOn DATETIME(6) NULL, + Payload LONGBLOB NOT NULL, + + PRIMARY KEY (ServiceId, ProviderId, QueueId, MessageId) +) ENGINE = InnoDB; DELIMITER $$ -CREATE PROCEDURE GetStreamMessages +CREATE PROCEDURE AppendStreamMessage ( - IN _ServiceId NVARCHAR(150), - IN _ProviderId NVARCHAR(150), - IN _QueueId NVARCHAR(150), - IN _MaxCount INT, - IN _MaxAttempts INT, - IN _VisibilityTimeout INT, - IN _RemovalTimeout INT, - IN _EvictionInterval INT, - IN _EvictionBatchSize INT + IN _ServiceId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + IN _ProviderId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + IN _QueueId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + IN _StreamIdBytes LONGBLOB, + IN _StreamNamespaceLength INT, + IN _Payload LONGBLOB, + IN _ManageTransaction BOOLEAN ) BEGIN + DECLARE _Now DATETIME(6); + DECLARE _MessageId BIGINT; + + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + IF _ManageTransaction THEN + ROLLBACK; + END IF; + RESIGNAL; + END; + + IF _ManageTransaction THEN + START TRANSACTION; + END IF; -DECLARE _Now DATETIME(6) DEFAULT UTC_TIMESTAMP(6); -DECLARE _VisibleOn DATETIME(6) DEFAULT DATE_ADD(_Now, INTERVAL _VisibilityTimeout SECOND); -DECLARE _NextEvictOn TIMESTAMP(6) DEFAULT DATE_ADD(_Now, INTERVAL _EvictionInterval SECOND); -DECLARE _EvictOn DATETIME(6); -DECLARE _Count INT; - --- get the next eviction schedule -SET _EvictOn = -( - SELECT EvictOn - FROM OrleansStreamControl - WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId -); - --- initialize the control row as necessary -IF _EvictOn IS NULL THEN - - -- race to initialize the control row - INSERT OrleansStreamControl + INSERT INTO OrleansStreamPartition ( ServiceId, ProviderId, QueueId, - EvictOn + NextMessageId, + Checkpoint, + OwnerEpoch, + CleanupOn, + CreatedOn, + ModifiedOn ) VALUES ( _ServiceId, _ProviderId, _QueueId, - _NextEvictOn + 1, + NULL, + 0, + UTC_TIMESTAMP(6), + UTC_TIMESTAMP(6), + UTC_TIMESTAMP(6) ) - ON DUPLICATE KEY - UPDATE - EvictOn = EvictOn; - - -- read the winning update - SET _EvictOn = - ( - SELECT EvictOn - FROM OrleansStreamControl - WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - ); - -END IF; - -IF _EvictOn <= _Now THEN + ON DUPLICATE KEY UPDATE NextMessageId = NextMessageId; - -- race to update the control row - UPDATE OrleansStreamControl - SET EvictOn = _NextEvictOn - WHERE - ServiceId = _ServiceId + SELECT NextMessageId + INTO _MessageId + FROM OrleansStreamPartition + WHERE ServiceId = _ServiceId AND ProviderId = _ProviderId AND QueueId = _QueueId - AND EvictOn <= _Now; - - -- if we won the race then we also run eviction - IF ROW_COUNT() > 0 THEN - CALL EvictStreamMessages(_ServiceId, _ProviderId, _QueueId, _MaxAttempts, _RemovalTimeout, _EvictionBatchSize); - CALL EvictStreamDeadLetters(_ServiceId, _ProviderId, _QueueId, _EvictionBatchSize); - END IF; - -END IF; - -START TRANSACTION; - -/* elect the batch of messages to dequeue and lock them in order */ -CREATE TEMPORARY TABLE _Batch AS -SELECT - ServiceId, - ProviderId, - QueueId, - MessageId -FROM - OrleansStreamMessage -WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - AND Dequeued < _MaxAttempts - AND VisibleOn <= _Now - AND ExpiresOn > _Now -ORDER BY - ServiceId, - ProviderId, - QueueId, - MessageId -LIMIT _MaxCount -FOR UPDATE SKIP LOCKED; - -/* update the message batch */ -UPDATE OrleansStreamMessage AS M -INNER JOIN _Batch AS B - ON M.ServiceId = B.ServiceId - AND M.ProviderId = B.ProviderId - AND M.QueueId = B.QueueId - AND M.MessageId = B.MessageId -SET - M.Dequeued = M.Dequeued + 1, - M.VisibleOn = _VisibleOn, - M.ModifiedOn = _Now; - -/* return the updated batch */ -SELECT - M.ServiceId, - M.ProviderId, - M.QueueId, - M.MessageId, - M.Dequeued, - M.VisibleOn, - M.ExpiresOn, - M.CreatedOn, - M.ModifiedOn, - M.Payload -FROM - OrleansStreamMessage AS M - INNER JOIN _Batch AS B - ON M.ServiceId = B.ServiceId - AND M.ProviderId = B.ProviderId - AND M.QueueId = B.QueueId - AND M.MessageId = B.MessageId -ORDER BY - M.MessageId; - -DROP TEMPORARY TABLE _Batch; - -COMMIT; - -END; - -DELIMITER $$ - -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'GetStreamMessagesKey', - 'CALL GetStreamMessages(@ServiceId, @ProviderId, @QueueId, @MaxCount, @MaxAttempts, @VisibilityTimeout, @RemovalTimeout, @EvictionInterval, @EvictionBatchSize)'; - -DELIMITER $$ - -CREATE PROCEDURE ConfirmStreamMessages -( - IN _ServiceId NVARCHAR(150), - IN _ProviderId NVARCHAR(150), - IN _QueueId NVARCHAR(150), - IN _Items LONGTEXT -) -BEGIN - -DECLARE _Delimiter1 NVARCHAR(1) DEFAULT '|'; -DECLARE _Delimiter2 NVARCHAR(1) DEFAULT ':'; -DECLARE _Value LONGTEXT; -DECLARE _MessageId BIGINT; -DECLARE _Dequeued INT; - -SET _Items = CONCAT(_Items, _Delimiter1); - -/* parse the message identifiers to be deleted */ -DROP TEMPORARY TABLE IF EXISTS _ItemsTable; -CREATE TEMPORARY TABLE _ItemsTable -( - ServiceId NVARCHAR(150) NOT NULL, - ProviderId NVARCHAR(150) NOT NULL, - QueueId NVARCHAR(150) NOT NULL, - MessageId BIGINT NOT NULL, - Dequeued INT NOT NULL, - - PRIMARY KEY (ServiceId, ProviderId, QueueId, MessageId) -); + FOR UPDATE; -WHILE LOCATE(_Delimiter1, _Items) > 0 DO + SET _Now = UTC_TIMESTAMP(6); - SET _Value = SUBSTRING_INDEX(_Items, _Delimiter1, 1); - SET _MessageId = CAST(SUBSTRING_INDEX(_Value, _Delimiter2, 1) AS UNSIGNED); - SET _Dequeued = CAST(SUBSTRING_INDEX(_Value, _Delimiter2, -1) AS SIGNED); + UPDATE OrleansStreamPartition + SET + NextMessageId = _MessageId + 1, + ModifiedOn = _Now + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId; - INSERT INTO _ItemsTable + INSERT INTO OrleansStreamMessage ( ServiceId, ProviderId, QueueId, MessageId, - Dequeued + StreamIdBytes, + StreamNamespaceLength, + CreatedOn, + Payload ) VALUES ( @@ -391,353 +171,385 @@ WHILE LOCATE(_Delimiter1, _Items) > 0 DO _ProviderId, _QueueId, _MessageId, - _Dequeued + _StreamIdBytes, + _StreamNamespaceLength, + _Now, + _Payload ); - SET _Items = SUBSTRING(_Items, LOCATE(_Delimiter1, _Items) + 1); - -END WHILE; - -START TRANSACTION; - -/* elect the batch of messages to confirm and lock them in order */ -CREATE TEMPORARY TABLE _Batch AS -SELECT - M.ServiceId, - M.ProviderId, - M.QueueId, - M.MessageId -FROM - OrleansStreamMessage AS M - INNER JOIN _ItemsTable AS I - ON M.ServiceId = I.ServiceId - AND M.ProviderId = I.ProviderId - AND M.QueueId = I.QueueId - AND M.MessageId = I.MessageId - AND M.Dequeued = ABS(I.Dequeued) -ORDER BY - M.ServiceId, - M.ProviderId, - M.QueueId, - M.MessageId -FOR UPDATE; - -IF EXISTS (SELECT 1 FROM _ItemsTable WHERE Dequeued < 0) THEN - /* negative dequeue receipts release messages for immediate redelivery */ - UPDATE OrleansStreamMessage AS M - INNER JOIN _Batch AS B - ON M.ServiceId = B.ServiceId - AND M.ProviderId = B.ProviderId - AND M.QueueId = B.QueueId - AND M.MessageId = B.MessageId - SET - M.VisibleOn = UTC_TIMESTAMP(6), - M.ModifiedOn = UTC_TIMESTAMP(6); -ELSE - /* delete the elected batch */ - DELETE M - FROM OrleansStreamMessage AS M - INNER JOIN _Batch AS B - ON M.ServiceId = B.ServiceId - AND M.ProviderId = B.ProviderId - AND M.QueueId = B.QueueId - AND M.MessageId = B.MessageId; -END IF; - -/* return the ack */ -SELECT - ServiceId, - ProviderId, - QueueId, - MessageId -FROM - _Batch; - -DROP TEMPORARY TABLE _Batch; -DROP TEMPORARY TABLE _ItemsTable; - -COMMIT; -END; - -DELIMITER $$ - -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'ConfirmStreamMessagesKey', - 'CALL ConfirmStreamMessages(@ServiceId, @ProviderId, @QueueId, @Items)'; + IF _ManageTransaction THEN + COMMIT; + END IF; -DELIMITER $$ + SELECT + _ServiceId AS ServiceId, + _ProviderId AS ProviderId, + _QueueId AS QueueId, + _MessageId AS MessageId; +END$$ -CREATE PROCEDURE FailStreamMessage +CREATE PROCEDURE AcquireStreamPartition ( - IN _ServiceId NVARCHAR(150), - IN _ProviderId NVARCHAR(150), - IN _QueueId NVARCHAR(150), - IN _MessageId BIGINT, - IN _MaxAttempts INT, - IN _RemovalTimeout INT + IN _ServiceId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + IN _ProviderId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + IN _QueueId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + IN _StartFromNow BOOLEAN, + IN _ManageTransaction BOOLEAN ) BEGIN + DECLARE _Now DATETIME(6); + DECLARE _NextMessageId BIGINT; + DECLARE _Checkpoint BIGINT; + DECLARE _OwnerEpoch BIGINT; + DECLARE _EarliestMessageId BIGINT; + DECLARE _TailMessageId BIGINT; + + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + IF _ManageTransaction THEN + ROLLBACK; + END IF; + RESIGNAL; + END; + + IF _ManageTransaction THEN + START TRANSACTION; + END IF; -DECLARE _Now DATETIME(6) DEFAULT UTC_TIMESTAMP(6); -DECLARE _RemoveOn DATETIME(6) DEFAULT DATE_ADD(_Now, INTERVAL _RemovalTimeout SECOND); - -/* if the message can still be dequeued then attempt to mark it visible again */ -UPDATE OrleansStreamMessage -SET - VisibleOn = _Now, - ModifiedOn = _Now -WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - AND MessageId = _MessageId - AND Dequeued < _MaxAttempts; - -IF ROW_COUNT() = 0 THEN - - START TRANSACTION; - - /* otherwise attempt to move the message to dead letters */ - CREATE TEMPORARY TABLE Deleted AS - SELECT - * - FROM - OrleansStreamMessage - WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - AND MessageId = _MessageId; - - DELETE FROM OrleansStreamMessage - WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - AND MessageId = _MessageId; - - INSERT INTO OrleansStreamDeadLetter + INSERT INTO OrleansStreamPartition ( ServiceId, ProviderId, QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, + NextMessageId, + Checkpoint, + OwnerEpoch, + CleanupOn, CreatedOn, - ModifiedOn, - DeadOn, - RemoveOn, - Payload + ModifiedOn ) - SELECT - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - _Now AS DeadOn, - _RemoveOn AS RemoveOn, - Payload - FROM - Deleted; + VALUES + ( + _ServiceId, + _ProviderId, + _QueueId, + 1, + NULL, + 0, + UTC_TIMESTAMP(6), + UTC_TIMESTAMP(6), + UTC_TIMESTAMP(6) + ) + ON DUPLICATE KEY UPDATE NextMessageId = NextMessageId; - COMMIT; + SELECT NextMessageId, Checkpoint + INTO _NextMessageId, _Checkpoint + FROM OrleansStreamPartition + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId + FOR UPDATE; -END IF; + SET _Now = UTC_TIMESTAMP(6); -END; + SELECT MIN(MessageId), MAX(MessageId) + INTO _EarliestMessageId, _TailMessageId + FROM OrleansStreamMessage + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId; -DELIMITER $$ + IF _Checkpoint IS NULL THEN + SET _Checkpoint = CASE + WHEN _StartFromNow THEN _NextMessageId - 1 + ELSE COALESCE(_EarliestMessageId - 1, _NextMessageId - 1) + END; + END IF; -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'FailStreamMessageKey', - 'CALL FailStreamMessage(@ServiceId, @ProviderId, @QueueId, @MessageId, @MaxAttempts, @RemovalTimeout)' + UPDATE OrleansStreamPartition + SET + Checkpoint = _Checkpoint, + OwnerEpoch = OwnerEpoch + 1, + ModifiedOn = _Now + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId; -DELIMITER $$ + UPDATE OrleansStreamMessage + SET CheckpointedOn = COALESCE(CheckpointedOn, _Now) + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId + AND MessageId <= _Checkpoint + AND CheckpointedOn IS NULL; -CREATE PROCEDURE EvictStreamMessages + SELECT OwnerEpoch + INTO _OwnerEpoch + FROM OrleansStreamPartition + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId; + + IF _ManageTransaction THEN + COMMIT; + END IF; + + SELECT + _ServiceId AS ServiceId, + _ProviderId AS ProviderId, + _QueueId AS QueueId, + _OwnerEpoch AS OwnerEpoch, + _NextMessageId AS NextMessageId, + _Checkpoint AS Checkpoint, + _EarliestMessageId AS EarliestMessageId, + _TailMessageId AS TailMessageId; +END$$ + +CREATE PROCEDURE AdvanceStreamCheckpoint ( - IN _ServiceId NVARCHAR(150), - IN _ProviderId NVARCHAR(150), - IN _QueueId NVARCHAR(150), - IN _BatchSize INT, - IN _MaxAttempts INT, - IN _RemovalTimeout INT + IN _ServiceId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + IN _ProviderId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + IN _QueueId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + IN _OwnerEpoch BIGINT, + IN _Checkpoint BIGINT, + IN _ManageTransaction BOOLEAN ) BEGIN + DECLARE _Now DATETIME(6); + DECLARE _CurrentOwnerEpoch BIGINT; + DECLARE _CurrentCheckpoint BIGINT; + DECLARE _Updated BOOLEAN DEFAULT FALSE; + + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + IF _ManageTransaction THEN + ROLLBACK; + END IF; + RESIGNAL; + END; + + IF _ManageTransaction THEN + START TRANSACTION; + END IF; -DECLARE _Now DATETIME(6) DEFAULT UTC_TIMESTAMP(); -DECLARE _RemoveOn DATETIME(6) DEFAULT DATE_ADD(_Now, INTERVAL _RemovalTimeout SECOND); - -START TRANSACTION; - -/* elect the batch of messages to move and lock them in order */ -CREATE TEMPORARY TABLE _Batch AS -SELECT - ServiceId, - ProviderId, - QueueId, - MessageId -FROM - OrleansStreamMessage -WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - AND - ( - -- a message is no longer dequeueable if the last attempt timed out - (Dequeued >= _MaxAttempts AND VisibleOn <= _Now) - OR - -- a message is no longer dequeueable if it has expired regardless - (ExpiresOn <= _Now) - ) -ORDER BY - ServiceId, - ProviderId, - QueueId, - MessageId -LIMIT _BatchSize -FOR UPDATE SKIP LOCKED; - -/* copy the messages to dead letters */ -INSERT INTO OrleansStreamDeadLetter -( - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - DeadOn, - RemoveOn, - Payload -) -SELECT - M.ServiceId, - M.ProviderId, - M.QueueId, - M.MessageId, - M.Dequeued, - M.VisibleOn, - M.ExpiresOn, - M.CreatedOn, - M.ModifiedOn, - _Now, - _RemoveOn, - M.Payload -FROM - OrleansStreamMessage AS M - INNER JOIN _Batch AS B - ON M.ServiceId = B.ServiceId - AND M.ProviderId = B.ProviderId - AND M.QueueId = B.QueueId - AND M.MessageId = B.MessageId; - -/* delete elected messages from the source now */ -DELETE M -FROM OrleansStreamMessage AS M -INNER JOIN _Batch AS B - ON M.ServiceId = B.ServiceId - AND M.ProviderId = B.ProviderId - AND M.QueueId = B.QueueId - AND M.MessageId = B.MessageId; - -DROP TEMPORARY TABLE _Batch; - -COMMIT; - -END; + SELECT OwnerEpoch, Checkpoint + INTO _CurrentOwnerEpoch, _CurrentCheckpoint + FROM OrleansStreamPartition + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId + FOR UPDATE; -DELIMITER $$ + SET _Now = UTC_TIMESTAMP(6); -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'EvictStreamMessagesKey', - 'CALL EvictStreamMessages(@ServiceId, @ProviderId, @QueueId, @BatchSize, @MaxAttempts, @RemovalTimeout)' -; + UPDATE OrleansStreamPartition + SET + Checkpoint = _Checkpoint, + ModifiedOn = _Now + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId + AND OwnerEpoch = _OwnerEpoch + AND (Checkpoint IS NULL OR Checkpoint < _Checkpoint) + AND _Checkpoint < NextMessageId; -DELIMITER $$ + SET _Updated = ROW_COUNT() = 1; -CREATE PROCEDURE EvictStreamDeadLetters + IF _Updated THEN + UPDATE OrleansStreamMessage + SET CheckpointedOn = COALESCE(CheckpointedOn, _Now) + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId + AND (_CurrentCheckpoint IS NULL OR MessageId > _CurrentCheckpoint) + AND MessageId <= _Checkpoint + AND CheckpointedOn IS NULL; + END IF; + + SELECT OwnerEpoch, Checkpoint + INTO _CurrentOwnerEpoch, _CurrentCheckpoint + FROM OrleansStreamPartition + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId + FOR UPDATE; + + IF _ManageTransaction THEN + COMMIT; + END IF; + + SELECT + _ServiceId AS ServiceId, + _ProviderId AS ProviderId, + _QueueId AS QueueId, + _CurrentOwnerEpoch AS OwnerEpoch, + _CurrentCheckpoint AS Checkpoint, + _Updated AS Updated + FROM DUAL + WHERE _CurrentOwnerEpoch IS NOT NULL; +END$$ + +CREATE PROCEDURE CleanupStreamMessages ( - _ServiceId NVARCHAR(150), - _ProviderId NVARCHAR(150), - _QueueId NVARCHAR(150), - _BatchSize INT + IN _ServiceId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + IN _ProviderId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + IN _QueueId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + IN _RetentionPeriodSeconds INT, + IN _MaximumRetentionPeriodSeconds INT, + IN _CleanupIntervalSeconds INT, + IN _CleanupBatchSize INT, + IN _ManageTransaction BOOLEAN ) BEGIN + DECLARE _Now DATETIME(6) DEFAULT UTC_TIMESTAMP(6); + DECLARE _Checkpoint BIGINT; + DECLARE _CleanupOn DATETIME(6); + DECLARE _DeletedCount INT DEFAULT 0; + DECLARE _DeletedThroughMessageId BIGINT; + DECLARE _HardDeletedCount INT DEFAULT 0; + DECLARE _HardDeletedFromMessageId BIGINT; + DECLARE _HardDeletedThroughMessageId BIGINT; + DECLARE _EarliestMessageId BIGINT; + DECLARE _TailMessageId BIGINT; + DECLARE _PartitionExists BOOLEAN DEFAULT FALSE; + + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + IF _ManageTransaction THEN + ROLLBACK; + END IF; + RESIGNAL; + END; + + IF _ManageTransaction THEN + START TRANSACTION; + END IF; -DECLARE _Now DATETIME(6) DEFAULT UTC_TIMESTAMP(); - -/* elect the batch of messages to remove */ -CREATE TEMPORARY TABLE _Batch AS -SELECT - ServiceId, - ProviderId, - QueueId, - MessageId -FROM - OrleansStreamDeadLetter -WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - AND RemoveOn <= _Now -ORDER BY - ServiceId, - ProviderId, - QueueId, - MessageId -LIMIT _BatchSize -FOR UPDATE SKIP LOCKED; - -/* now delete the locked messages */ -DELETE M -FROM OrleansStreamDeadLetter AS M -INNER JOIN _Batch AS B - ON M.ServiceId = B.ServiceId - AND M.ProviderId = B.ProviderId - AND M.QueueId = B.QueueId - AND M.MessageId = B.MessageId; - -DROP TEMPORARY TABLE _Batch; - -END; + SELECT TRUE, Checkpoint, CleanupOn + INTO _PartitionExists, _Checkpoint, _CleanupOn + FROM OrleansStreamPartition + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId + FOR UPDATE; -DELIMITER $$ + IF NOT _PartitionExists OR _CleanupOn > _Now THEN + SELECT MIN(MessageId), MAX(MessageId) + INTO _EarliestMessageId, _TailMessageId + FROM OrleansStreamMessage + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId; + + IF _ManageTransaction THEN + COMMIT; + END IF; + + SELECT + FALSE AS Ran, + 0 AS DeletedCount, + NULL AS DeletedThroughMessageId, + 0 AS HardDeletedCount, + NULL AS HardDeletedFromMessageId, + NULL AS HardDeletedThroughMessageId, + _Checkpoint AS Checkpoint, + _EarliestMessageId AS EarliestMessageId, + _TailMessageId AS TailMessageId; + ELSE + UPDATE OrleansStreamPartition + SET + CleanupOn = DATE_ADD(_Now, INTERVAL _CleanupIntervalSeconds SECOND), + ModifiedOn = _Now + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId; + + DROP TEMPORARY TABLE IF EXISTS OrleansStreamCleanupBatch; + CREATE TEMPORARY TABLE OrleansStreamCleanupBatch + ( + MessageId BIGINT NOT NULL, + PRIMARY KEY (MessageId) + ); + + INSERT INTO OrleansStreamCleanupBatch (MessageId) + SELECT MessageId + FROM OrleansStreamMessage + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId + AND + ( + ( + _Checkpoint IS NOT NULL + AND MessageId <= _Checkpoint + AND CheckpointedOn < DATE_SUB(_Now, INTERVAL _RetentionPeriodSeconds SECOND) + ) + OR + ( + _MaximumRetentionPeriodSeconds IS NOT NULL + AND CreatedOn < DATE_SUB(_Now, INTERVAL _MaximumRetentionPeriodSeconds SECOND) + ) + ) + ORDER BY MessageId + LIMIT _CleanupBatchSize + FOR UPDATE; + + SELECT + COUNT(*), + MAX(MessageId), + COALESCE(SUM(_Checkpoint IS NULL OR MessageId > _Checkpoint), 0), + MIN(CASE WHEN _Checkpoint IS NULL OR MessageId > _Checkpoint THEN MessageId END), + MAX(CASE WHEN _Checkpoint IS NULL OR MessageId > _Checkpoint THEN MessageId END) + INTO + _DeletedCount, + _DeletedThroughMessageId, + _HardDeletedCount, + _HardDeletedFromMessageId, + _HardDeletedThroughMessageId + FROM OrleansStreamCleanupBatch; + + DELETE M + FROM OrleansStreamMessage AS M + INNER JOIN OrleansStreamCleanupBatch AS B + ON B.MessageId = M.MessageId + WHERE M.ServiceId = _ServiceId + AND M.ProviderId = _ProviderId + AND M.QueueId = _QueueId; + + SELECT MIN(MessageId), MAX(MessageId) + INTO _EarliestMessageId, _TailMessageId + FROM OrleansStreamMessage + WHERE ServiceId = _ServiceId + AND ProviderId = _ProviderId + AND QueueId = _QueueId; + + DROP TEMPORARY TABLE OrleansStreamCleanupBatch; + + IF _ManageTransaction THEN + COMMIT; + END IF; + + SELECT + TRUE AS Ran, + _DeletedCount AS DeletedCount, + _DeletedThroughMessageId AS DeletedThroughMessageId, + _HardDeletedCount AS HardDeletedCount, + _HardDeletedFromMessageId AS HardDeletedFromMessageId, + _HardDeletedThroughMessageId AS HardDeletedThroughMessageId, + _Checkpoint AS Checkpoint, + _EarliestMessageId AS EarliestMessageId, + _TailMessageId AS TailMessageId; + END IF; +END$$ -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'EvictStreamDeadLettersKey', - 'CALL EvictStreamDeadLetters(@ServiceId, @ProviderId, @QueueId, @BatchSize)' -; +DELIMITER ; -DELIMITER $$ \ No newline at end of file +INSERT INTO OrleansQuery (QueryKey, QueryText) +VALUES + ('StreamSchemaVersionKey', '2'), + ('AppendStreamMessageKey', 'CALL AppendStreamMessage(@ServiceId, @ProviderId, @QueueId, @StreamIdBytes, @StreamNamespaceLength, @Payload, TRUE)'), + ('AcquireStreamPartitionKey', 'CALL AcquireStreamPartition(@ServiceId, @ProviderId, @QueueId, @StartFromNow, TRUE)'), + ('ReadStreamMessagesKey', 'SELECT ServiceId, ProviderId, QueueId, MessageId, StreamIdBytes, StreamNamespaceLength, CreatedOn, Payload FROM OrleansStreamMessage WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId AND QueueId = @QueueId AND MessageId > @AfterMessageId ORDER BY MessageId LIMIT @MaxCount'), + ('AdvanceStreamCheckpointKey', 'CALL AdvanceStreamCheckpoint(@ServiceId, @ProviderId, @QueueId, @OwnerEpoch, @Checkpoint, TRUE)'), + ('GetStreamPartitionBoundsKey', 'SELECT P.ServiceId, P.ProviderId, P.QueueId, P.OwnerEpoch, P.NextMessageId, P.Checkpoint, MIN(M.MessageId) AS EarliestMessageId, MAX(M.MessageId) AS TailMessageId FROM OrleansStreamPartition AS P LEFT JOIN OrleansStreamMessage AS M ON M.ServiceId = P.ServiceId AND M.ProviderId = P.ProviderId AND M.QueueId = P.QueueId WHERE P.ServiceId = @ServiceId AND P.ProviderId = @ProviderId AND P.QueueId = @QueueId GROUP BY P.ServiceId, P.ProviderId, P.QueueId, P.OwnerEpoch, P.NextMessageId, P.Checkpoint'), + ('CleanupStreamMessagesKey', 'CALL CleanupStreamMessages(@ServiceId, @ProviderId, @QueueId, @RetentionPeriodSeconds, @MaximumRetentionPeriodSeconds, @CleanupIntervalSeconds, @CleanupBatchSize, TRUE)'); diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql b/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql index 817742dab8f..8b1e66e8d50 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql +++ b/src/AdoNet/Orleans.Streaming.AdoNet/PostgreSQL-Streaming.sql @@ -1,707 +1,511 @@ -CREATE SEQUENCE OrleansStreamMessageSequence -AS BIGINT -START WITH 1 -INCREMENT BY 1 -NO MAXVALUE -NO CYCLE; +/* +ADO.NET streaming schema version 2. -CREATE TABLE OrleansStreamMessage -( - ServiceId VARCHAR(150) NOT NULL, - ProviderId VARCHAR(150) NOT NULL, - QueueId VARCHAR(150) NOT NULL, - MessageId BIGINT NOT NULL, - Dequeued INT NOT NULL, - VisibleOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, - ExpiresOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, - CreatedOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, - ModifiedOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, - Payload BYTEA NOT NULL, - - CONSTRAINT PK_OrleansStreamMessage PRIMARY KEY - ( - ServiceId, - ProviderId, - QueueId, - MessageId - ) -); +This alpha schema is intentionally incompatible with the former destructive queue schema. +Drop the former streaming tables, sequence, routines, and OrleansQuery rows before applying +this script. Existing queue rows are not migrated. +*/ + +DO $$ +BEGIN + IF to_regclass('orleansstreampartition') IS NOT NULL + OR to_regclass('orleansstreammessage') IS NOT NULL + OR to_regclass('orleansstreamdeadletter') IS NOT NULL + OR to_regclass('orleansstreamcontrol') IS NOT NULL + OR to_regclass('orleansstreammessagesequence') IS NOT NULL + OR EXISTS + ( + SELECT 1 + FROM OrleansQuery + WHERE QueryKey IN + ( + 'QueueStreamMessageKey', + 'GetStreamMessagesKey', + 'ConfirmStreamMessagesKey', + 'FailStreamMessageKey', + 'EvictStreamMessagesKey', + 'EvictStreamDeadLettersKey', + 'StreamSchemaVersionKey' + ) + ) + THEN + RAISE EXCEPTION 'Incompatible alpha ADO.NET streaming schema. Drop old streaming tables, sequence, routines, and OrleansQuery rows before applying version 2; no in-place migration is supported.'; + END IF; +END; +$$; -CREATE TABLE OrleansStreamDeadLetter +CREATE TABLE OrleansStreamPartition ( - ServiceId VARCHAR(150) NOT NULL, + ServiceId VARCHAR(150) NOT NULL, ProviderId VARCHAR(150) NOT NULL, - QueueId VARCHAR(150) NOT NULL, - MessageId BIGINT NOT NULL, - Dequeued INT NOT NULL, - VisibleOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, - ExpiresOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, - CreatedOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, - ModifiedOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, - DeadOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, - RemoveOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, - Payload BYTEA, - - CONSTRAINT PK_OrleansStreamDeadLetter PRIMARY KEY + QueueId VARCHAR(150) NOT NULL, + NextMessageId BIGINT NOT NULL, + Checkpoint BIGINT NULL, + OwnerEpoch BIGINT NOT NULL, + CleanupOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, + CreatedOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, + ModifiedOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, + + CONSTRAINT PK_OrleansStreamPartition PRIMARY KEY ( ServiceId, ProviderId, - QueueId, - MessageId + QueueId ) ); -CREATE TABLE OrleansStreamControl +CREATE TABLE OrleansStreamMessage ( - ServiceId VARCHAR(150) NOT NULL, + ServiceId VARCHAR(150) NOT NULL, ProviderId VARCHAR(150) NOT NULL, - QueueId VARCHAR(150) NOT NULL, - EvictOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, - - CONSTRAINT PK_OrleansStreamControl PRIMARY KEY + QueueId VARCHAR(150) NOT NULL, + MessageId BIGINT NOT NULL, + StreamIdBytes BYTEA NOT NULL, + StreamNamespaceLength INT NOT NULL, + CreatedOn TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, + CheckpointedOn TIMESTAMP(6) WITHOUT TIME ZONE NULL, + Payload BYTEA NOT NULL, + + CONSTRAINT PK_OrleansStreamMessage PRIMARY KEY ( ServiceId, ProviderId, - QueueId + QueueId, + MessageId ) ); -CREATE OR REPLACE FUNCTION QueueStreamMessage -( - _ServiceId VARCHAR(150), - _ProviderId VARCHAR(150), - _QueueId VARCHAR(150), - _Payload BYTEA, - _ExpiryTimeout INT -) -RETURNS TABLE -( - ServiceId VARCHAR(150), - ProviderId VARCHAR(150), - QueueId VARCHAR(150), - MessageId BIGINT -) -LANGUAGE plpgsql -AS $$ -#VARIABLE_CONFLICT USE_COLUMN -DECLARE - _MessageId BIGINT := nextval('OrleansStreamMessageSequence'); - _Now TIMESTAMP(6) WITHOUT TIME ZONE := CURRENT_TIMESTAMP AT TIME ZONE 'UTC'; - _ExpiresOn TIMESTAMP(6) WITHOUT TIME ZONE := _Now + INTERVAL '1 SECOND' * _ExpiryTimeout; -BEGIN - -RETURN QUERY -INSERT INTO OrleansStreamMessage -( - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - Payload -) -VALUES +CREATE OR REPLACE FUNCTION AppendStreamMessage ( - _ServiceId, - _ProviderId, - _QueueId, - _MessageId, - 0, - _Now, - _ExpiresOn, - _Now, - _Now, - _Payload -) -RETURNING - ServiceId, - ProviderId, - QueueId, - MessageId; - -END; -$$; - -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'QueueStreamMessageKey', - 'SELECT * FROM QueueStreamMessage(@ServiceId, @ProviderId, @QueueId, @Payload, @ExpiryTimeout)' -; - -CREATE OR REPLACE FUNCTION GetStreamMessages -( - _ServiceId VARCHAR(150), + _ServiceId VARCHAR(150), _ProviderId VARCHAR(150), - _QueueId VARCHAR(150), - _MaxCount INT, - _MaxAttempts INT, - _VisibilityTimeout INT, - _RemovalTimeout INT, - _EvictionInterval INT, - _EvictionBatchSize INT + _QueueId VARCHAR(150), + _StreamIdBytes BYTEA, + _StreamNamespaceLength INT, + _Payload BYTEA ) RETURNS TABLE ( - ServiceId VARCHAR(150), + ServiceId VARCHAR(150), ProviderId VARCHAR(150), - QueueId VARCHAR(150), - MessageId BIGINT, - Dequeued INT, - VisibleOn TIMESTAMP(6) WITHOUT TIME ZONE, - ExpiresOn TIMESTAMP(6) WITHOUT TIME ZONE, - CreatedOn TIMESTAMP(6) WITHOUT TIME ZONE, - ModifiedOn TIMESTAMP(6) WITHOUT TIME ZONE, - Payload BYTEA + QueueId VARCHAR(150), + MessageId BIGINT ) LANGUAGE plpgsql AS $$ #VARIABLE_CONFLICT USE_COLUMN DECLARE - _Now TIMESTAMP(6) WITHOUT TIME ZONE := CURRENT_TIMESTAMP AT TIME ZONE 'UTC'; - _VisibleOn TIMESTAMP(6) WITHOUT TIME ZONE := _Now + INTERVAL '1 SECOND' * _VisibilityTimeout; - _EvictOn TIMESTAMP(6) WITHOUT TIME ZONE; - _NextEvictOn TIMESTAMP(6) WITHOUT TIME ZONE := _Now + INTERVAL '1 SECOND' * _EvictionInterval; + _Now TIMESTAMP(6) WITHOUT TIME ZONE; + _MessageId BIGINT; BEGIN + INSERT INTO OrleansStreamPartition + ( + ServiceId, + ProviderId, + QueueId, + NextMessageId, + Checkpoint, + OwnerEpoch, + CleanupOn, + CreatedOn, + ModifiedOn + ) + VALUES + ( + _ServiceId, + _ProviderId, + _QueueId, + 1, + NULL, + 0, + clock_timestamp() AT TIME ZONE 'UTC', + clock_timestamp() AT TIME ZONE 'UTC', + clock_timestamp() AT TIME ZONE 'UTC' + ) + ON CONFLICT (ServiceId, ProviderId, QueueId) DO NOTHING; -/* get the next eviction schedule */ -SELECT EvictOn -INTO _EvictOn -FROM OrleansStreamControl -WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId; + UPDATE OrleansStreamPartition AS P + SET + NextMessageId = P.NextMessageId + 1, + ModifiedOn = clock_timestamp() AT TIME ZONE 'UTC' + WHERE P.ServiceId = _ServiceId + AND P.ProviderId = _ProviderId + AND P.QueueId = _QueueId + RETURNING P.NextMessageId - 1 INTO _MessageId; -/* initialize the control row if necessary */ -IF _EvictOn IS NULL THEN + _Now := clock_timestamp() AT TIME ZONE 'UTC'; - /* initialize with a past date so eviction runs immediately */ - INSERT INTO OrleansStreamControl + INSERT INTO OrleansStreamMessage ( ServiceId, ProviderId, QueueId, - EvictOn + MessageId, + StreamIdBytes, + StreamNamespaceLength, + CreatedOn, + Payload ) VALUES ( _ServiceId, _ProviderId, _QueueId, - _Now - INTERVAL '1 SECOND' - ) - ON CONFLICT (ServiceId, ProviderId, QueueId) - DO NOTHING; - - /* get the next eviction schedule again */ - SELECT EvictOn - INTO _EvictOn - FROM OrleansStreamControl - WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId; - -END IF; - -/* evict messages if necessary */ -IF _EvictOn <= _Now THEN - - /* race to set the next schedule */ - UPDATE OrleansStreamControl - SET EvictOn = _NextEvictOn - WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - AND EvictOn <= _Now; - - /* if we won the race then we also run the due eviction */ - IF (FOUND) THEN - CALL EvictStreamMessages(_ServiceId, _ProviderId, _QueueId, _EvictionBatchSize, _MaxAttempts, _RemovalTimeout); - CALL EvictStreamDeadLetters(_ServiceId, _ProviderId, _QueueId, _EvictionBatchSize); - END IF; - -END IF; - -RETURN QUERY -WITH Batch AS -( - /* elect the next batch of visible messages */ - SELECT - ServiceId, - ProviderId, - QueueId, - MessageId - FROM - OrleansStreamMessage - WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - AND Dequeued < _MaxAttempts - AND VisibleOn <= _Now - AND ExpiresOn > _Now - - /* the criteria below helps prevent deadlocks while improving queue-like throughput */ - ORDER BY - ServiceId, - ProviderId, - QueueId, - MessageId - FOR UPDATE - LIMIT _MaxCount -) -UPDATE OrleansStreamMessage AS M -SET - Dequeued = Dequeued + 1, - VisibleOn = _VisibleOn, - ModifiedOn = _Now -FROM - Batch AS B -WHERE - M.ServiceId = B.ServiceId - AND M.ProviderId = B.ProviderId - AND M.QueueId = B.QueueId - AND M.MessageId = B.MessageId -RETURNING - M.ServiceId, - M.ProviderId, - M.QueueId, - M.MessageId, - M.Dequeued, - M.VisibleOn, - M.ExpiresOn, - M.CreatedOn, - M.ModifiedOn, - M.Payload; + _MessageId, + _StreamIdBytes, + _StreamNamespaceLength, + _Now, + _Payload + ); + RETURN QUERY + SELECT _ServiceId, _ProviderId, _QueueId, _MessageId; END; $$; -INSERT INTO OrleansQuery +CREATE OR REPLACE FUNCTION AcquireStreamPartition ( - QueryKey, - QueryText -) -SELECT - 'GetStreamMessagesKey', - 'SELECT * FROM GetStreamMessages(@ServiceId, @ProviderId, @QueueId, @MaxCount, @MaxAttempts, @VisibilityTimeout, @RemovalTimeout, @EvictionInterval, @EvictionBatchSize)' -; - -CREATE OR REPLACE FUNCTION ConfirmStreamMessages -( - _ServiceId VARCHAR(150), + _ServiceId VARCHAR(150), _ProviderId VARCHAR(150), - _QueueId VARCHAR(150), - _Items TEXT + _QueueId VARCHAR(150), + _StartFromNow BOOLEAN ) RETURNS TABLE ( - ServiceId VARCHAR(150), + ServiceId VARCHAR(150), ProviderId VARCHAR(150), - QueueId VARCHAR(150), - MessageId BIGINT + QueueId VARCHAR(150), + OwnerEpoch BIGINT, + NextMessageId BIGINT, + Checkpoint BIGINT, + EarliestMessageId BIGINT, + TailMessageId BIGINT ) LANGUAGE plpgsql AS $$ #VARIABLE_CONFLICT USE_COLUMN DECLARE - _Count INT; - _Now TIMESTAMP(6) WITHOUT TIME ZONE := CURRENT_TIMESTAMP AT TIME ZONE 'UTC'; + _Now TIMESTAMP(6) WITHOUT TIME ZONE; + _NextMessageId BIGINT; + _Checkpoint BIGINT; + _OwnerEpoch BIGINT; + _EarliestMessageId BIGINT; + _TailMessageId BIGINT; BEGIN - -CREATE TEMP TABLE _ItemsTable -( - MessageId BIGINT PRIMARY KEY NOT NULL, - Dequeued INT NOT NULL -) ON COMMIT DROP; - -INSERT INTO _ItemsTable -( - MessageId, - Dequeued -) -SELECT - CAST(split_part(Value, ':', 1) AS BIGINT) AS MessageId, - CAST(split_part(Value, ':', 2) AS INT) AS Dequeued -FROM - UNNEST(string_to_array(_Items, '|')) AS Value; - -/* negative dequeue receipts release messages for immediate redelivery */ -IF EXISTS (SELECT 1 FROM _ItemsTable WHERE Dequeued < 0) THEN - RETURN QUERY - WITH Batch AS + INSERT INTO OrleansStreamPartition ( - SELECT - M.* - FROM - OrleansStreamMessage AS M - INNER JOIN _ItemsTable AS I - ON I.MessageId = M.MessageId - AND -I.Dequeued = M.Dequeued - WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - ORDER BY - ServiceId, - ProviderId, - QueueId, - MessageId - FOR UPDATE + ServiceId, + ProviderId, + QueueId, + NextMessageId, + Checkpoint, + OwnerEpoch, + CleanupOn, + CreatedOn, + ModifiedOn ) - UPDATE OrleansStreamMessage AS M + VALUES + ( + _ServiceId, + _ProviderId, + _QueueId, + 1, + NULL, + 0, + clock_timestamp() AT TIME ZONE 'UTC', + clock_timestamp() AT TIME ZONE 'UTC', + clock_timestamp() AT TIME ZONE 'UTC' + ) + ON CONFLICT (ServiceId, ProviderId, QueueId) DO NOTHING; + + SELECT P.NextMessageId, P.Checkpoint + INTO _NextMessageId, _Checkpoint + FROM OrleansStreamPartition AS P + WHERE P.ServiceId = _ServiceId + AND P.ProviderId = _ProviderId + AND P.QueueId = _QueueId + FOR UPDATE; + + _Now := clock_timestamp() AT TIME ZONE 'UTC'; + + SELECT MIN(M.MessageId), MAX(M.MessageId) + INTO _EarliestMessageId, _TailMessageId + FROM OrleansStreamMessage AS M + WHERE M.ServiceId = _ServiceId + AND M.ProviderId = _ProviderId + AND M.QueueId = _QueueId; + + IF _Checkpoint IS NULL THEN + _Checkpoint := CASE + WHEN _StartFromNow THEN _NextMessageId - 1 + ELSE COALESCE(_EarliestMessageId - 1, _NextMessageId - 1) + END; + END IF; + + UPDATE OrleansStreamPartition AS P SET - VisibleOn = _Now, + Checkpoint = _Checkpoint, + OwnerEpoch = P.OwnerEpoch + 1, ModifiedOn = _Now - FROM - Batch AS B - WHERE - M.ServiceId = B.ServiceId - AND M.ProviderId = B.ProviderId - AND M.QueueId = B.QueueId - AND M.MessageId = B.MessageId - RETURNING - M.ServiceId, - M.ProviderId, - M.QueueId, - M.MessageId; - RETURN; -END IF; - -RETURN QUERY -WITH Batch AS -( - SELECT - M.* - FROM - OrleansStreamMessage AS M - INNER JOIN _ItemsTable AS I - ON I.MessageId = M.MessageId - AND I.Dequeued = M.Dequeued - WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - - /* the criteria below helps prevent deadlocks */ - ORDER BY - ServiceId, - ProviderId, - QueueId, - MessageId - FOR UPDATE -) -DELETE FROM OrleansStreamMessage AS M -USING Batch AS B -WHERE - M.ServiceId = B.ServiceId - AND M.ProviderId = B.ProviderId - AND M.QueueId = B.QueueId - AND M.MessageId = B.MessageId -RETURNING - M.ServiceId, - M.ProviderId, - M.QueueId, - M.MessageId; + WHERE P.ServiceId = _ServiceId + AND P.ProviderId = _ProviderId + AND P.QueueId = _QueueId + RETURNING P.OwnerEpoch INTO _OwnerEpoch; + UPDATE OrleansStreamMessage AS M + SET CheckpointedOn = COALESCE(M.CheckpointedOn, _Now) + WHERE M.ServiceId = _ServiceId + AND M.ProviderId = _ProviderId + AND M.QueueId = _QueueId + AND M.MessageId <= _Checkpoint + AND M.CheckpointedOn IS NULL; + + RETURN QUERY + SELECT + _ServiceId, + _ProviderId, + _QueueId, + _OwnerEpoch, + _NextMessageId, + _Checkpoint, + _EarliestMessageId, + _TailMessageId; END; $$; -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'ConfirmStreamMessagesKey', - 'SELECT * FROM ConfirmStreamMessages(@ServiceId, @ProviderId, @QueueId, @Items)' -; - -CREATE OR REPLACE PROCEDURE FailStreamMessage +CREATE OR REPLACE FUNCTION AdvanceStreamCheckpoint ( _ServiceId VARCHAR(150), _ProviderId VARCHAR(150), _QueueId VARCHAR(150), - _MessageId BIGINT, - _MaxAttempts INT, - _RemovalTimeout INT + _OwnerEpoch BIGINT, + _Checkpoint BIGINT ) -LANGUAGE plpgsql -AS $$ -#VARIABLE_CONFLICT USE_COLUMN -DECLARE - _Now TIMESTAMP(6) WITHOUT TIME ZONE := CURRENT_TIMESTAMP AT TIME ZONE 'UTC'; - _RemoveOn TIMESTAMP(6) WITHOUT TIME ZONE := _Now + INTERVAL '1 SECOND' * _RemovalTimeout; -BEGIN - -/* if the message can still be dequeued then attempt to mark it visible again */ -UPDATE OrleansStreamMessage -SET - VisibleOn = _Now, - ModifiedOn = _Now -WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - AND MessageId = _MessageId - AND Dequeued < _MaxAttempts; - -IF FOUND THEN - RETURN; -END IF; - -/* otherwise attempt to move the message to dead letters */ -WITH Deleted AS -( - DELETE FROM OrleansStreamMessage - WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - AND MessageId = _MessageId - RETURNING - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - Payload -) -INSERT INTO OrleansStreamDeadLetter -( - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - DeadOn, - RemoveOn, - Payload -) -SELECT - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - _Now AS DeadOn, - _RemoveOn AS RemoveOn, - Payload -FROM - Deleted; - -END; -$$; - -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'FailStreamMessageKey', - 'CALL FailStreamMessage(@ServiceId, @ProviderId, @QueueId, @MessageId, @MaxAttempts, @RemovalTimeout)' -; - -CREATE OR REPLACE PROCEDURE EvictStreamMessages +RETURNS TABLE ( - _ServiceId VARCHAR(150), - _ProviderId VARCHAR(150), - _QueueId VARCHAR(150), - _BatchSize INT, - _MaxAttempts INT, - _RemovalTimeout INT + ServiceId VARCHAR(150), + ProviderId VARCHAR(150), + QueueId VARCHAR(150), + OwnerEpoch BIGINT, + Checkpoint BIGINT, + Updated BOOLEAN ) LANGUAGE plpgsql AS $$ #VARIABLE_CONFLICT USE_COLUMN DECLARE - _Now TIMESTAMP(6) WITHOUT TIME ZONE := CURRENT_TIMESTAMP AT TIME ZONE 'UTC'; - _RemoveOn TIMESTAMP(6) WITHOUT TIME ZONE := _Now + INTERVAL '1 second' * _RemovalTimeout; + _Now TIMESTAMP(6) WITHOUT TIME ZONE; + _CurrentOwnerEpoch BIGINT; + _CurrentCheckpoint BIGINT; + _PreviousCheckpoint BIGINT; + _Updated BOOLEAN := FALSE; BEGIN - -/* elect the next batch of messages to evict */ -WITH Batch AS -( - SELECT - ServiceId, - ProviderId, - QueueId, - MessageId - FROM - OrleansStreamMessage - WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - - -- the message was given the opportunity to complete - AND VisibleOn <= _Now - AND - ( - -- the message was dequeued too many times - Dequeued >= _MaxAttempts - OR - -- the message expired - ExpiresOn <= _Now - ) - - /* the criteria below helps prevent deadlocks while improving queue-like throughput */ - ORDER BY - ServiceId, - ProviderId, - QueueId, - MessageId - FOR UPDATE - LIMIT _BatchSize -), - -/* delete the messages locked in the batch */ -Deleted AS -( - DELETE FROM OrleansStreamMessage AS M - USING Batch AS B - WHERE - M.ServiceId = B.ServiceId - AND M.ProviderId = B.ProviderId - AND M.QueueId = B.QueueId - AND M.MessageId = B.MessageId - RETURNING - M.ServiceId, - M.ProviderId, - M.QueueId, - M.MessageId, - M.Dequeued, - M.VisibleOn, - M.ExpiresOn, - M.CreatedOn, - M.ModifiedOn, - M.Payload -) - -/* copy the deleted messages to the dead-letter table */ -INSERT INTO OrleansStreamDeadLetter -( - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - DeadOn, - RemoveOn, - Payload -) -SELECT - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - _Now, - _RemoveOn, - Payload -FROM - Deleted AS D; - + SELECT P.OwnerEpoch, P.Checkpoint + INTO _CurrentOwnerEpoch, _CurrentCheckpoint + FROM OrleansStreamPartition AS P + WHERE P.ServiceId = _ServiceId + AND P.ProviderId = _ProviderId + AND P.QueueId = _QueueId + FOR UPDATE; + _PreviousCheckpoint := _CurrentCheckpoint; + + _Now := clock_timestamp() AT TIME ZONE 'UTC'; + + UPDATE OrleansStreamPartition AS P + SET + Checkpoint = _Checkpoint, + ModifiedOn = _Now + WHERE P.ServiceId = _ServiceId + AND P.ProviderId = _ProviderId + AND P.QueueId = _QueueId + AND P.OwnerEpoch = _OwnerEpoch + AND (P.Checkpoint IS NULL OR P.Checkpoint < _Checkpoint) + AND _Checkpoint < P.NextMessageId + RETURNING P.OwnerEpoch, P.Checkpoint + INTO _CurrentOwnerEpoch, _CurrentCheckpoint; + + IF FOUND THEN + _Updated := TRUE; + UPDATE OrleansStreamMessage AS M + SET CheckpointedOn = COALESCE(M.CheckpointedOn, _Now) + WHERE M.ServiceId = _ServiceId + AND M.ProviderId = _ProviderId + AND M.QueueId = _QueueId + AND (_PreviousCheckpoint IS NULL OR M.MessageId > _PreviousCheckpoint) + AND M.MessageId <= _Checkpoint + AND M.CheckpointedOn IS NULL; + ELSE + SELECT P.OwnerEpoch, P.Checkpoint + INTO _CurrentOwnerEpoch, _CurrentCheckpoint + FROM OrleansStreamPartition AS P + WHERE P.ServiceId = _ServiceId + AND P.ProviderId = _ProviderId + AND P.QueueId = _QueueId; + END IF; + + IF _CurrentOwnerEpoch IS NOT NULL THEN + RETURN QUERY + SELECT + _ServiceId, + _ProviderId, + _QueueId, + _CurrentOwnerEpoch, + _CurrentCheckpoint, + _Updated; + END IF; END; $$; -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'EvictStreamMessagesKey', - 'CALL EvictStreamMessages(@ServiceId, @ProviderId, @QueueId, @BatchSize, @MaxAttempts, @RemovalTimeout)' -; - -CREATE OR REPLACE PROCEDURE EvictStreamDeadLetters +CREATE OR REPLACE FUNCTION CleanupStreamMessages ( _ServiceId VARCHAR(150), _ProviderId VARCHAR(150), _QueueId VARCHAR(150), - _BatchSize INT + _RetentionPeriodSeconds INT, + _MaximumRetentionPeriodSeconds INT, + _CleanupIntervalSeconds INT, + _CleanupBatchSize INT +) +RETURNS TABLE +( + Ran BOOLEAN, + DeletedCount INT, + DeletedThroughMessageId BIGINT, + HardDeletedCount INT, + HardDeletedFromMessageId BIGINT, + HardDeletedThroughMessageId BIGINT, + Checkpoint BIGINT, + EarliestMessageId BIGINT, + TailMessageId BIGINT ) LANGUAGE plpgsql AS $$ #VARIABLE_CONFLICT USE_COLUMN DECLARE _Now TIMESTAMP(6) WITHOUT TIME ZONE := CURRENT_TIMESTAMP AT TIME ZONE 'UTC'; + _Checkpoint BIGINT; + _DeletedCount INT := 0; + _DeletedThroughMessageId BIGINT; + _HardDeletedCount INT := 0; + _HardDeletedFromMessageId BIGINT; + _HardDeletedThroughMessageId BIGINT; + _EarliestMessageId BIGINT; + _TailMessageId BIGINT; BEGIN - -/* elect the next batch of dead letters to evict */ -WITH Batch AS -( + UPDATE OrleansStreamPartition AS P + SET + CleanupOn = _Now + make_interval(secs => _CleanupIntervalSeconds), + ModifiedOn = _Now + WHERE P.ServiceId = _ServiceId + AND P.ProviderId = _ProviderId + AND P.QueueId = _QueueId + AND P.CleanupOn <= _Now + RETURNING P.Checkpoint INTO _Checkpoint; + + IF NOT FOUND THEN + SELECT P.Checkpoint + INTO _Checkpoint + FROM OrleansStreamPartition AS P + WHERE P.ServiceId = _ServiceId + AND P.ProviderId = _ProviderId + AND P.QueueId = _QueueId; + + SELECT MIN(M.MessageId), MAX(M.MessageId) + INTO _EarliestMessageId, _TailMessageId + FROM OrleansStreamMessage AS M + WHERE M.ServiceId = _ServiceId + AND M.ProviderId = _ProviderId + AND M.QueueId = _QueueId; + + RETURN QUERY + SELECT + FALSE, + 0, + NULL::BIGINT, + 0, + NULL::BIGINT, + NULL::BIGINT, + _Checkpoint, + _EarliestMessageId, + _TailMessageId; + RETURN; + END IF; + + WITH Candidate AS + ( + SELECT M.MessageId + FROM OrleansStreamMessage AS M + WHERE M.ServiceId = _ServiceId + AND M.ProviderId = _ProviderId + AND M.QueueId = _QueueId + AND + ( + ( + _Checkpoint IS NOT NULL + AND M.MessageId <= _Checkpoint + AND M.CheckpointedOn < _Now - make_interval(secs => _RetentionPeriodSeconds) + ) + OR + ( + _MaximumRetentionPeriodSeconds IS NOT NULL + AND M.CreatedOn < _Now - make_interval(secs => _MaximumRetentionPeriodSeconds) + ) + ) + ORDER BY M.MessageId + FOR UPDATE + LIMIT _CleanupBatchSize + ), + Deleted AS + ( + DELETE FROM OrleansStreamMessage AS M + USING Candidate AS C + WHERE M.ServiceId = _ServiceId + AND M.ProviderId = _ProviderId + AND M.QueueId = _QueueId + AND M.MessageId = C.MessageId + RETURNING M.MessageId + ) SELECT - ServiceId, - ProviderId, - QueueId, - MessageId - FROM - OrleansStreamDeadLetter - WHERE - ServiceId = _ServiceId - AND ProviderId = _ProviderId - AND QueueId = _QueueId - AND RemoveOn <= _Now - - /* the criteria below helps prevent deadlocks while improving queue-like throughput */ - ORDER BY - ServiceId, - ProviderId, - QueueId, - MessageId - FOR UPDATE - LIMIT _BatchSize -) -DELETE FROM OrleansStreamDeadLetter AS M -USING Batch AS B -WHERE - M.ServiceId = B.ServiceId - AND M.ProviderId = B.ProviderId - AND M.QueueId = B.QueueId - AND M.MessageId = B.MessageId; + COUNT(*)::INT, + MAX(D.MessageId), + COUNT(*) FILTER (WHERE _Checkpoint IS NULL OR D.MessageId > _Checkpoint)::INT, + MIN(D.MessageId) FILTER (WHERE _Checkpoint IS NULL OR D.MessageId > _Checkpoint), + MAX(D.MessageId) FILTER (WHERE _Checkpoint IS NULL OR D.MessageId > _Checkpoint) + INTO + _DeletedCount, + _DeletedThroughMessageId, + _HardDeletedCount, + _HardDeletedFromMessageId, + _HardDeletedThroughMessageId + FROM Deleted AS D; + + SELECT MIN(M.MessageId), MAX(M.MessageId) + INTO _EarliestMessageId, _TailMessageId + FROM OrleansStreamMessage AS M + WHERE M.ServiceId = _ServiceId + AND M.ProviderId = _ProviderId + AND M.QueueId = _QueueId; + RETURN QUERY + SELECT + TRUE, + _DeletedCount, + _DeletedThroughMessageId, + _HardDeletedCount, + _HardDeletedFromMessageId, + _HardDeletedThroughMessageId, + _Checkpoint, + _EarliestMessageId, + _TailMessageId; END; $$; -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'EvictStreamDeadLettersKey', - 'CALL EvictStreamDeadLetters(@ServiceId, @ProviderId, @QueueId, @BatchSize)' -; \ No newline at end of file +INSERT INTO OrleansQuery (QueryKey, QueryText) +VALUES + ('StreamSchemaVersionKey', '2'), + ('AppendStreamMessageKey', 'SELECT * FROM AppendStreamMessage(@ServiceId, @ProviderId, @QueueId, @StreamIdBytes, @StreamNamespaceLength, @Payload)'), + ('AcquireStreamPartitionKey', 'SELECT * FROM AcquireStreamPartition(@ServiceId, @ProviderId, @QueueId, @StartFromNow)'), + ('ReadStreamMessagesKey', 'SELECT ServiceId, ProviderId, QueueId, MessageId, StreamIdBytes, StreamNamespaceLength, CreatedOn, Payload FROM OrleansStreamMessage WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId AND QueueId = @QueueId AND MessageId > @AfterMessageId ORDER BY MessageId LIMIT @MaxCount'), + ('AdvanceStreamCheckpointKey', 'SELECT * FROM AdvanceStreamCheckpoint(@ServiceId, @ProviderId, @QueueId, @OwnerEpoch, @Checkpoint)'), + ('GetStreamPartitionBoundsKey', 'SELECT P.ServiceId, P.ProviderId, P.QueueId, P.OwnerEpoch, P.NextMessageId, P.Checkpoint, MIN(M.MessageId) AS EarliestMessageId, MAX(M.MessageId) AS TailMessageId FROM OrleansStreamPartition AS P LEFT JOIN OrleansStreamMessage AS M ON M.ServiceId = P.ServiceId AND M.ProviderId = P.ProviderId AND M.QueueId = P.QueueId WHERE P.ServiceId = @ServiceId AND P.ProviderId = @ProviderId AND P.QueueId = @QueueId GROUP BY P.ServiceId, P.ProviderId, P.QueueId, P.OwnerEpoch, P.NextMessageId, P.Checkpoint'), + ('CleanupStreamMessagesKey', 'SELECT * FROM CleanupStreamMessages(@ServiceId, @ProviderId, @QueueId, @RetentionPeriodSeconds, @MaximumRetentionPeriodSeconds, @CleanupIntervalSeconds, @CleanupBatchSize)'); diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/README.md b/src/AdoNet/Orleans.Streaming.AdoNet/README.md index 15e96f12b54..b99fe578b9d 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/README.md +++ b/src/AdoNet/Orleans.Streaming.AdoNet/README.md @@ -1,7 +1,7 @@ # Microsoft Orleans Streaming for ADO.NET ## Introduction -Microsoft Orleans Streaming for ADO.NET provides a stream provider implementation for Orleans using ADO.NET-compatible databases (SQL Server, MySQL, PostgreSQL, etc.). This allows for publishing and subscribing to streams of events with relational databases as the underlying infrastructure. +Microsoft Orleans Streaming for ADO.NET provides a partitioned stream provider for Orleans using ADO.NET-compatible databases (SQL Server, MySQL, PostgreSQL, etc.). This allows for publishing and subscribing to streams of events with relational databases as the underlying infrastructure. ## Getting Started To use this package, install it via NuGet: @@ -48,6 +48,28 @@ var builder = Host.CreateApplicationBuilder(args) await builder.RunAsync(); ``` +The provider stores each queue as an immutable, ordered stream partition. Its stream partition pipeline appends records, reads partition history, and advances an ownership-fenced checkpoint. Configure it with: + +- `StartFromNow`: initialize a new checkpoint at the current partition history tail instead of before the earliest retained record. +- `FaultOnDeliveryFailure`: optionally fault a failing subscription while preserving the shared partition records. +- `MaxMessagesPerRead`: bound each ordered storage read. +- `CheckpointPersistInterval`: throttle durable checkpoint updates. +- `RetentionPeriod`: retain checkpointed records for at least this period (one day by default). Fractional seconds round upward. +- `MaximumRetentionPeriod`: optionally delete older records even when they are not checkpointed. This is a hard capacity ceiling and can create a diagnosed retention gap. Fractional seconds round upward. +- `CleanupInterval` and `CleanupBatchSize`: bound cleanup frequency and work. Fractional cleanup intervals round upward. + +The partitioned stream provider resumes strictly after the durable, ownership-fenced queue checkpoint and can redeliver records after a crash without skipping uncheckpointed data. The checkpoint advances through the earliest contiguous position which is safe for every subscription, including unrelated partition records which quiet-stream cursors have scanned. Subscription starts attach to the live partition position. Partition state retains the next message identifier, so recovery detects a hard-retention gap even when the purge leaves no records. + +Partition acquisition is cancellation-aware. A receiver whose acquisition command is still completing retains its queue reservation, so a late database result settles before a replacement receiver acquires a newer ownership epoch. + +Message creation and checkpoint-eligibility timestamps are sampled after the partition lock is acquired. Lock contention therefore does not consume the configured retention window. + +## Alpha schema upgrade + +The current streaming scripts use schema version 2 and are intentionally incompatible with the former queue, visibility-timeout, confirmation, and dead-letter schema. The provider fails during initialization when it detects old or mixed streaming query keys. + +There is no in-place migration for this alpha package. Stop producers and consumers, drop `OrleansStreamMessage`, `OrleansStreamDeadLetter`, `OrleansStreamControl`, `OrleansStreamMessageSequence`, the old streaming routines, and their `OrleansQuery` rows. Drop `OrleansStreamPartition` too after a partial version 2 installation. Then apply the current SQL Server, PostgreSQL, or MySQL streaming script. Existing alpha rows are not read or silently converted, so export payloads first if they must be retained. + ## Example - Using ADO.NET Streams in a Grain ```csharp // Producer grain diff --git a/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql b/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql index af4461624a6..73ed997bb5f 100644 --- a/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql +++ b/src/AdoNet/Orleans.Streaming.AdoNet/SQLServer-Streaming.sql @@ -1,792 +1,672 @@ /* -Orleans Stream Message Sequence. -This sequence reduces contention on generation of [MessageId] values vs an identity column. -The CACHE parameter can be increased to further reduce contention. -*/ -CREATE SEQUENCE OrleansStreamMessageSequence -AS BIGINT -START WITH 1 -INCREMENT BY 1 -NO MAXVALUE -NO CYCLE -CACHE 1000; -GO - -/* -Orleans Streaming Message Queue. - -This table stores queued messages awaiting processing by Orleans. - -The demands for this table are as follows: - -1. The table will see inserts only at the tail, as new rows are added. -2. The table will be polled with high frequency to reserve the first batch of rows that matches a well-known criteria ("visible" and "not expired" and "under max attempts"). -3. The table will see rows being removed at the head as messages are confirmed. -4. The table will see rows being removed at the head as expired messages are moved to dead letters. -5. Due to the above queries touching more than one row at a time, there is a possibility of deadlocks. -6. A few faulted or poisoned messages can linger for some time at the head before being moved to dead letters. -7. The table will occasionaly become empty or at least sparse as the cluster succeeds to catch up to all messages. - -While [1-6] all cause page fragmentation over time, [7] self resolves this degradation by allowing sql server to eventually remove all pages. -Therefore the design attempts to optimise for [2] while assuming the resulting degradation eventually resolves itself. - -The design also attempts to minimize the possibility of deadlocks at the expense of higher locking contention. -This happens by forcing all queries to touch data in the exact same order of the clustered index. -This induces ordered resource lock acquisition while avoiding the cost of ordering itself. +ADO.NET streaming schema version 2. +This alpha schema is intentionally incompatible with the former destructive queue schema. +Drop the former streaming tables, sequence, routines, and OrleansQuery rows before applying +this script. Existing queue rows are not migrated. */ -CREATE TABLE OrleansStreamMessage -( - /* Identifies the application */ - ServiceId NVARCHAR(150) NOT NULL, - - /* Identifies the provider within the application */ - ProviderId NVARCHAR(150) NOT NULL, - /* Identifies the individual queue shard as configured in the provider*/ - QueueId NVARCHAR(150) NOT NULL, - - /* The unique ascending number of the queued message */ - MessageId BIGINT NOT NULL, - - /* The number of times the event was dequeued */ - Dequeued INT NOT NULL, - - /* The UTC time at which the event will become visible */ - VisibleOn DATETIME2(7) NOT NULL, - - /* The UTC time at which the event will expire */ - ExpiresOn DATETIME2(7) NOT NULL, - - /* The UTC time at which the event was created - troubleshooting only */ - CreatedOn DATETIME2(7) NOT NULL, - - /* The UTC time at which the event was updated - troubleshooting only */ - ModifiedOn DATETIME2(7) NOT NULL, - - /* The arbitrarily large payload of the event */ - Payload VARBINARY(MAX) NOT NULL, - - /* This Clustered PK supports the various ordered scanning queries. */ - CONSTRAINT PK_OrleansStreamMessage PRIMARY KEY CLUSTERED - ( - ServiceId ASC, - ProviderId ASC, - QueueId ASC, - MessageId ASC - ) -); +IF OBJECT_ID(N'OrleansStreamPartition', N'U') IS NOT NULL + OR OBJECT_ID(N'OrleansStreamMessage', N'U') IS NOT NULL + OR OBJECT_ID(N'OrleansStreamDeadLetter', N'U') IS NOT NULL + OR OBJECT_ID(N'OrleansStreamControl', N'U') IS NOT NULL + OR OBJECT_ID(N'OrleansStreamMessageSequence', N'SO') IS NOT NULL + OR EXISTS + ( + SELECT 1 + FROM OrleansQuery + WHERE QueryKey IN + ( + 'QueueStreamMessageKey', + 'GetStreamMessagesKey', + 'ConfirmStreamMessagesKey', + 'FailStreamMessageKey', + 'EvictStreamMessagesKey', + 'EvictStreamDeadLettersKey', + 'StreamSchemaVersionKey' + ) + ) +BEGIN + THROW 51001, 'Incompatible alpha ADO.NET streaming schema. Drop old streaming tables, sequence, routines, and OrleansQuery rows before applying version 2; no in-place migration is supported.', 1; +END; GO -/* -Orleans Streaming Dead Letters. - -This table holds events that could not be processed within the allowed number of attempts or that have expired. -*/ -CREATE TABLE OrleansStreamDeadLetter +CREATE TABLE OrleansStreamPartition ( - /* Identifies the application */ - ServiceId NVARCHAR(150) NOT NULL, - - /* Identifies the provider within the application */ + ServiceId NVARCHAR(150) NOT NULL, ProviderId NVARCHAR(150) NOT NULL, - - /* Identifies the individual queue shard as configured in the provider*/ - QueueId NVARCHAR(150) NOT NULL, - - /* The unique ascending number of the queued message */ - MessageId BIGINT NOT NULL, - - /* The number of times the event was dequeued */ - Dequeued INT NOT NULL, - - /* The UTC time at which the event will become visible */ - VisibleOn DATETIME2(7) NOT NULL, - - /* The UTC time at which the event will expire */ - ExpiresOn DATETIME2(7) NOT NULL, - - /* The UTC time at which the event was created - troubleshooting only */ - CreatedOn DATETIME2(7) NOT NULL, - - /* The UTC time at which the event was updated - troubleshooting only */ - ModifiedOn DATETIME2(7) NOT NULL, - - /* The UTC time at which the event was given up on - troubleshooting only */ - DeadOn DATETIME2(7) NOT NULL, - - /* The UTC time at which the event is scheduled to be removed from dead letters */ - RemoveOn DATETIME2(7) NOT NULL, - - /* The arbitrarily large payload of the event */ - Payload VARBINARY(MAX) NULL, - - /* This Clustered PK supports the various ordered scanning queries. */ - /* Its main purpose is to help partition the update row locks as to minimize dequeing contention. */ - CONSTRAINT PK_OrleansStreamDeadLetter PRIMARY KEY CLUSTERED - ( - ServiceId ASC, - ProviderId ASC, - QueueId ASC, - MessageId ASC - ) + QueueId NVARCHAR(150) NOT NULL, + NextMessageId BIGINT NOT NULL, + [Checkpoint] BIGINT NULL, + OwnerEpoch BIGINT NOT NULL, + CleanupOn DATETIME2(7) NOT NULL, + CreatedOn DATETIME2(7) NOT NULL, + ModifiedOn DATETIME2(7) NOT NULL, + + CONSTRAINT PK_OrleansStreamPartition PRIMARY KEY CLUSTERED + ( + ServiceId, + ProviderId, + QueueId + ) ); GO -/* -Orleans Streaming Control Table. -This table holds schedule variables to help providers self manage their own work. -*/ -CREATE TABLE OrleansStreamControl +CREATE TABLE OrleansStreamMessage ( - /* Identifies the application */ - ServiceId NVARCHAR(150) NOT NULL, - - /* Identifies the provider within the application */ + ServiceId NVARCHAR(150) NOT NULL, ProviderId NVARCHAR(150) NOT NULL, - - /* Identifies the individual queue shard as configured in the provider */ - QueueId NVARCHAR(150) NOT NULL, - - /* The next due schedule for messages to be evicted */ - EvictOn DATETIME2(7) NOT NULL, - - /* Each row represents a flat configuration object for an individual queue */ - CONSTRAINT PK_OrleansStreamControl PRIMARY KEY CLUSTERED - ( - ServiceId ASC, - ProviderId ASC, - QueueId ASC - ) + QueueId NVARCHAR(150) NOT NULL, + MessageId BIGINT NOT NULL, + StreamIdBytes VARBINARY(MAX) NOT NULL, + StreamNamespaceLength INT NOT NULL, + CreatedOn DATETIME2(7) NOT NULL, + CheckpointedOn DATETIME2(7) NULL, + Payload VARBINARY(MAX) NOT NULL, + + CONSTRAINT PK_OrleansStreamMessage PRIMARY KEY CLUSTERED + ( + ServiceId, + ProviderId, + QueueId, + MessageId + ) ); GO -/* Queues a message to the Orleans Streaming Message Queue */ -CREATE PROCEDURE QueueStreamMessage - @ServiceId NVARCHAR(150), +CREATE PROCEDURE AppendStreamMessage + @ServiceId NVARCHAR(150), @ProviderId NVARCHAR(150), - @QueueId NVARCHAR(150), - @Payload VARBINARY(MAX), - @ExpiryTimeout INT + @QueueId NVARCHAR(150), + @StreamIdBytes VARBINARY(MAX), + @StreamNamespaceLength INT, + @Payload VARBINARY(MAX) AS BEGIN - -SET NOCOUNT ON; -SET XACT_ABORT ON; - -/* -MessageIds must become visible in allocation order within each queue. Otherwise, -concurrent transactions can expose a later MessageId while an earlier insert is -still uncommitted, violating the stream cache's monotonic sequence invariant. -*/ -DECLARE @LockResource NVARCHAR(255) = CONCAT -( - N'OrleansStreamMessage:', - CONVERT - ( - VARCHAR(64), - HASHBYTES + SET NOCOUNT ON; + SET XACT_ABORT ON; + + DECLARE @StartedTransaction BIT = 0; + DECLARE @Now DATETIME2(7); + DECLARE @LockedNextMessageId BIGINT; + DECLARE @Allocated TABLE (MessageId BIGINT NOT NULL); + + BEGIN TRY + IF @@TRANCOUNT = 0 + BEGIN + BEGIN TRANSACTION; + SET @StartedTransaction = 1; + END; + + IF NOT EXISTS ( - 'SHA2_256', - CONCAT + SELECT 1 + FROM OrleansStreamPartition + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId + ) + BEGIN + DECLARE @InitializationLockResource NVARCHAR(255) = CONCAT ( - DATALENGTH(@ServiceId), N':', @ServiceId, - DATALENGTH(@ProviderId), N':', @ProviderId, - DATALENGTH(@QueueId), N':', @QueueId + N'OrleansStreamPartition:', + CONVERT + ( + VARCHAR(64), + HASHBYTES + ( + 'SHA2_256', + CONCAT + ( + DATALENGTH(@ServiceId), N':', @ServiceId, + DATALENGTH(@ProviderId), N':', @ProviderId, + DATALENGTH(@QueueId), N':', @QueueId + ) + ), + 2 + ) + ); + DECLARE @InitializationLockResult INT; + + EXECUTE @InitializationLockResult = sys.sp_getapplock + @Resource = @InitializationLockResource, + @LockMode = 'Exclusive', + @LockOwner = 'Transaction'; + + IF @InitializationLockResult < 0 + BEGIN + THROW 51000, 'Failed to acquire the stream partition initialization lock.', 1; + END; + + IF NOT EXISTS + ( + SELECT 1 + FROM OrleansStreamPartition + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId ) - ), - 2 - ) -); -DECLARE @LockResult INT; -DECLARE @StartedTransaction BIT = 0; - -BEGIN TRY - IF @@TRANCOUNT = 0 - BEGIN - BEGIN TRANSACTION; - SET @StartedTransaction = 1; - END; - - EXECUTE @LockResult = sys.sp_getapplock - @Resource = @LockResource, - @LockMode = 'Exclusive', - @LockOwner = 'Transaction'; - - IF @LockResult < 0 - BEGIN - THROW 51000, 'Failed to acquire the stream message queue lock.', 1; - END; - - DECLARE @MessageId BIGINT = NEXT VALUE FOR OrleansStreamMessageSequence; - DECLARE @Now DATETIME2(7) = SYSUTCDATETIME(); - DECLARE @ExpiresOn DATETIME2(7) = DATEADD(SECOND, @ExpiryTimeout, @Now); + BEGIN + INSERT INTO OrleansStreamPartition + ( + ServiceId, + ProviderId, + QueueId, + NextMessageId, + [Checkpoint], + OwnerEpoch, + CleanupOn, + CreatedOn, + ModifiedOn + ) + VALUES + ( + @ServiceId, + @ProviderId, + @QueueId, + 1, + NULL, + 0, + SYSUTCDATETIME(), + SYSUTCDATETIME(), + SYSUTCDATETIME() + ); + END; + END; + + SELECT @LockedNextMessageId = NextMessageId + FROM OrleansStreamPartition WITH (UPDLOCK, ROWLOCK) + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId; - INSERT INTO OrleansStreamMessage - ( - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - Payload - ) - OUTPUT - Inserted.ServiceId, - Inserted.ProviderId, - Inserted.QueueId, - Inserted.MessageId - VALUES - ( - @ServiceId, - @ProviderId, - @QueueId, - @MessageId, - 0, - @Now, - @ExpiresOn, - @Now, - @Now, - @Payload - ); + SET @Now = SYSUTCDATETIME(); - IF @StartedTransaction = 1 - BEGIN - COMMIT TRANSACTION; - END; -END TRY -BEGIN CATCH - IF @StartedTransaction = 1 AND XACT_STATE() <> 0 - BEGIN - ROLLBACK TRANSACTION; - END; - - THROW; -END CATCH; - -END -GO + UPDATE OrleansStreamPartition WITH (UPDLOCK, ROWLOCK) + SET + NextMessageId = NextMessageId + 1, + ModifiedOn = @Now + OUTPUT Inserted.NextMessageId - 1 INTO @Allocated (MessageId) + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId; -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'QueueStreamMessageKey', - 'EXECUTE QueueStreamMessage @ServiceId = @ServiceId, @ProviderId = @ProviderId, @QueueId = @QueueId, @Payload = @Payload, @ExpiryTimeout = @ExpiryTimeout' + INSERT INTO OrleansStreamMessage + ( + ServiceId, + ProviderId, + QueueId, + MessageId, + StreamIdBytes, + StreamNamespaceLength, + CreatedOn, + Payload + ) + SELECT + @ServiceId, + @ProviderId, + @QueueId, + MessageId, + @StreamIdBytes, + @StreamNamespaceLength, + @Now, + @Payload + FROM @Allocated; + + IF @StartedTransaction = 1 + BEGIN + COMMIT TRANSACTION; + END; + + SELECT + @ServiceId AS ServiceId, + @ProviderId AS ProviderId, + @QueueId AS QueueId, + MessageId + FROM @Allocated; + END TRY + BEGIN CATCH + IF @StartedTransaction = 1 AND XACT_STATE() <> 0 + BEGIN + ROLLBACK TRANSACTION; + END; + + THROW; + END CATCH; +END; GO -/* Gets message batches from the Orleans Streaming Message Queue */ -/* Also opportunistically performs eviction activities when they are due */ -CREATE PROCEDURE GetStreamMessages - @ServiceId NVARCHAR(150), +CREATE PROCEDURE AcquireStreamPartition + @ServiceId NVARCHAR(150), @ProviderId NVARCHAR(150), - @QueueId NVARCHAR(150), - @MaxCount INT, - @MaxAttempts INT, - @VisibilityTimeout INT, - @RemovalTimeout INT, - @EvictionInterval INT, - @EvictionBatchSize INT + @QueueId NVARCHAR(150), + @StartFromNow BIT AS BEGIN + SET NOCOUNT ON; + SET XACT_ABORT ON; + + DECLARE @StartedTransaction BIT = 0; + DECLARE @Now DATETIME2(7); + DECLARE @NextMessageId BIGINT; + DECLARE @Checkpoint BIGINT; + DECLARE @OwnerEpoch BIGINT; + DECLARE @EarliestMessageId BIGINT; + DECLARE @TailMessageId BIGINT; + + BEGIN TRY + IF @@TRANCOUNT = 0 + BEGIN + BEGIN TRANSACTION; + SET @StartedTransaction = 1; + END; + + IF NOT EXISTS + ( + SELECT 1 + FROM OrleansStreamPartition + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId + ) + BEGIN + DECLARE @InitializationLockResource NVARCHAR(255) = CONCAT + ( + N'OrleansStreamPartition:', + CONVERT + ( + VARCHAR(64), + HASHBYTES + ( + 'SHA2_256', + CONCAT + ( + DATALENGTH(@ServiceId), N':', @ServiceId, + DATALENGTH(@ProviderId), N':', @ProviderId, + DATALENGTH(@QueueId), N':', @QueueId + ) + ), + 2 + ) + ); + DECLARE @InitializationLockResult INT; + + EXECUTE @InitializationLockResult = sys.sp_getapplock + @Resource = @InitializationLockResource, + @LockMode = 'Exclusive', + @LockOwner = 'Transaction'; + + IF @InitializationLockResult < 0 + BEGIN + THROW 51000, 'Failed to acquire the stream partition initialization lock.', 1; + END; + + IF NOT EXISTS + ( + SELECT 1 + FROM OrleansStreamPartition + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId + ) + BEGIN + INSERT INTO OrleansStreamPartition + ( + ServiceId, + ProviderId, + QueueId, + NextMessageId, + [Checkpoint], + OwnerEpoch, + CleanupOn, + CreatedOn, + ModifiedOn + ) + VALUES + ( + @ServiceId, + @ProviderId, + @QueueId, + 1, + NULL, + 0, + SYSUTCDATETIME(), + SYSUTCDATETIME(), + SYSUTCDATETIME() + ); + END; + END; -SET NOCOUNT ON; -SET XACT_ABORT ON; - -DECLARE @Now DATETIME2(7) = SYSUTCDATETIME(); -DECLARE @VisibleOn DATETIME2(7) = DATEADD(SECOND, @VisibilityTimeout, @Now); - -/* lightweight check to see if an eviction activity is due */ -DECLARE @EvictOn DATETIME2(7) = -( - SELECT EvictOn - FROM OrleansStreamControl - WHERE - ServiceId = @ServiceId - AND ProviderId = @ProviderId - AND QueueId = @QueueId -); + SELECT + @NextMessageId = NextMessageId, + @Checkpoint = [Checkpoint] + FROM OrleansStreamPartition WITH (UPDLOCK, ROWLOCK) + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId; -/* escalate to a eviction attempt only if an activity is due */ -IF @EvictOn IS NULL OR @EvictOn < @Now -BEGIN + SET @Now = SYSUTCDATETIME(); - /* attempt to win a race to update the schedule */ - /* this will also initialize the table if necessary */ - WITH Candidate AS - ( SELECT - ServiceId = @ServiceId, - ProviderId = @ProviderId, - QueueId = @QueueId, - Now = @Now, - EvictOn = DATEADD(SECOND, @EvictionInterval, @Now) - ) - MERGE OrleansStreamControl WITH (UPDLOCK, HOLDLOCK) AS T - USING Candidate AS S - ON T.ServiceId = S.ServiceId - AND T.ProviderId = S.ProviderId - AND T.QueueId = S.QueueId - WHEN MATCHED AND T.EvictOn < S.Now THEN - UPDATE SET T.EvictOn = S.EvictOn - WHEN NOT MATCHED BY TARGET THEN - INSERT - ( - ServiceId, - ProviderId, - QueueId, - EvictOn - ) - VALUES - ( - ServiceId, - ProviderId, - QueueId, - EvictOn - ); + @EarliestMessageId = MIN(MessageId), + @TailMessageId = MAX(MessageId) + FROM OrleansStreamMessage + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId; + + IF @Checkpoint IS NULL + BEGIN + SET @Checkpoint = CASE + WHEN @StartFromNow = 1 THEN @NextMessageId - 1 + ELSE COALESCE(@EarliestMessageId - 1, @NextMessageId - 1) + END; + END; + + UPDATE OrleansStreamPartition WITH (UPDLOCK, ROWLOCK) + SET + [Checkpoint] = @Checkpoint, + OwnerEpoch = OwnerEpoch + 1, + ModifiedOn = @Now + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId; - /* if the above statement won the race then we also get to run the eviction */ - /* other concurrent queries will continue running as normal until the next due time */ - IF (@@ROWCOUNT > 0) - BEGIN - - /* evict messages */ - EXECUTE EvictStreamMessages - @ServiceId = @ServiceId, - @ProviderId = @ProviderId, - @QueueId = @QueueId, - @MaxAttempts = @MaxAttempts, - @RemovalTimeout = @RemovalTimeout, - @BatchSize = @EvictionBatchSize - - /* evict dead letters */ - EXECUTE EvictStreamDeadLetters - @ServiceId = @ServiceId, - @ProviderId = @ProviderId, - @QueueId = @QueueId, - @BatchSize = @EvictionBatchSize; - - END; + UPDATE OrleansStreamMessage + SET CheckpointedOn = COALESCE(CheckpointedOn, @Now) + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId + AND MessageId <= @Checkpoint + AND CheckpointedOn IS NULL; -END; + SELECT @OwnerEpoch = OwnerEpoch + FROM OrleansStreamPartition + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId; -/* update messages in the exact same order as the clustered index to avoid deadlocks with other queries */ -WITH Batch AS -( - SELECT TOP (@MaxCount) - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - Payload - FROM - OrleansStreamMessage WITH (UPDLOCK, READPAST, READCOMMITTEDLOCK) - WHERE - ServiceId = @ServiceId - AND ProviderId = @ProviderId - AND QueueId = @QueueId - AND Dequeued < @MaxAttempts - AND VisibleOn <= @Now - AND ExpiresOn > @Now - ORDER BY - ServiceId, - ProviderId, - QueueId, - MessageId -) -UPDATE Batch -SET - Dequeued += 1, - VisibleOn = @VisibleOn, - ModifiedOn = @Now -OUTPUT - Inserted.ServiceId, - Inserted.ProviderId, - Inserted.QueueId, - Inserted.MessageId, - Inserted.Dequeued, - Inserted.VisibleOn, - Inserted.ExpiresOn, - Inserted.CreatedOn, - Inserted.ModifiedOn, - Inserted.Payload -FROM - Batch; - -END -GO + IF @StartedTransaction = 1 + BEGIN + COMMIT TRANSACTION; + END; -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'GetStreamMessagesKey', - 'EXECUTE GetStreamMessages @ServiceId = @ServiceId, @ProviderId = @ProviderId, @QueueId = @QueueId, @MaxCount = @MaxCount, @MaxAttempts = @MaxAttempts, @VisibilityTimeout = @VisibilityTimeout, @RemovalTimeout = @RemovalTimeout, @EvictionInterval = @EvictionInterval, @EvictionBatchSize = @EvictionBatchSize' + SELECT + @ServiceId AS ServiceId, + @ProviderId AS ProviderId, + @QueueId AS QueueId, + @OwnerEpoch AS OwnerEpoch, + @NextMessageId AS NextMessageId, + @Checkpoint AS [Checkpoint], + @EarliestMessageId AS EarliestMessageId, + @TailMessageId AS TailMessageId; + END TRY + BEGIN CATCH + IF @StartedTransaction = 1 AND XACT_STATE() <> 0 + BEGIN + ROLLBACK TRANSACTION; + END; + + THROW; + END CATCH; +END; GO -/* Confirms delivery of a stream message. */ -CREATE PROCEDURE ConfirmStreamMessages - @ServiceId NVARCHAR(150), +CREATE PROCEDURE AdvanceStreamCheckpoint + @ServiceId NVARCHAR(150), @ProviderId NVARCHAR(150), - @QueueId NVARCHAR(150), - @Items NVARCHAR(MAX) + @QueueId NVARCHAR(150), + @OwnerEpoch BIGINT, + @Checkpoint BIGINT AS BEGIN + SET NOCOUNT ON; + SET XACT_ABORT ON; -SET NOCOUNT ON; -SET XACT_ABORT ON; - -/* parse the message identifiers to be deleted */ -DECLARE @ItemsTable TABLE -( - MessageId BIGINT PRIMARY KEY NOT NULL, - Dequeued INT NOT NULL -); -WITH Items AS -( - SELECT Value FROM STRING_SPLIT(@Items, '|') -) -INSERT INTO @ItemsTable -( - MessageId, - Dequeued -) -SELECT - CAST(SUBSTRING(Value, 1, CHARINDEX(':', Value, 1) - 1) AS BIGINT) AS MessageId, - CAST(SUBSTRING(Value, CHARINDEX(':', Value, 1) + 1, LEN(Value)) AS INT) AS Dequeued -FROM - Items; - -/* count the number of messages to delete so we can use order by in the next query */ -DECLARE @Count INT = (SELECT COUNT(*) FROM @ItemsTable); - -/* negative dequeue receipts release messages for immediate redelivery */ -IF EXISTS (SELECT 1 FROM @ItemsTable WHERE Dequeued < 0) -BEGIN - DECLARE @Now DATETIME2(7) = SYSUTCDATETIME(); - - WITH Batch AS + DECLARE @Result TABLE ( - SELECT TOP (@Count) - M.* - FROM - OrleansStreamMessage AS M WITH (UPDLOCK, READPAST, READCOMMITTEDLOCK, ROWLOCK) - WHERE - ServiceId = @ServiceId + ServiceId NVARCHAR(150) NOT NULL, + ProviderId NVARCHAR(150) NOT NULL, + QueueId NVARCHAR(150) NOT NULL, + OwnerEpoch BIGINT NOT NULL, + [Checkpoint] BIGINT NULL, + Updated BIT NOT NULL + ); + DECLARE @Now DATETIME2(7); + DECLARE @LockedCheckpoint BIGINT; + DECLARE @StartedTransaction BIT = 0; + + BEGIN TRY + IF @@TRANCOUNT = 0 + BEGIN + BEGIN TRANSACTION; + SET @StartedTransaction = 1; + END; + + SELECT @LockedCheckpoint = [Checkpoint] + FROM OrleansStreamPartition WITH (UPDLOCK, ROWLOCK) + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId; + + SET @Now = SYSUTCDATETIME(); + + UPDATE OrleansStreamPartition WITH (UPDLOCK, ROWLOCK) + SET + [Checkpoint] = @Checkpoint, + ModifiedOn = @Now + OUTPUT + Inserted.ServiceId, + Inserted.ProviderId, + Inserted.QueueId, + Inserted.OwnerEpoch, + Inserted.[Checkpoint], + CAST(1 AS BIT) + INTO @Result + WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId AND QueueId = @QueueId - AND EXISTS - ( - SELECT * - FROM @ItemsTable AS I - WHERE I.MessageId = M.MessageId - AND -I.Dequeued = M.Dequeued - ) - ORDER BY + AND OwnerEpoch = @OwnerEpoch + AND ([Checkpoint] IS NULL OR [Checkpoint] < @Checkpoint) + AND @Checkpoint < NextMessageId; + + IF EXISTS (SELECT 1 FROM @Result) + BEGIN + UPDATE OrleansStreamMessage + SET CheckpointedOn = COALESCE(CheckpointedOn, @Now) + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId + AND (@LockedCheckpoint IS NULL OR MessageId > @LockedCheckpoint) + AND MessageId <= @Checkpoint + AND CheckpointedOn IS NULL; + END; + + IF @StartedTransaction = 1 + BEGIN + COMMIT TRANSACTION; + END; + + IF EXISTS (SELECT 1 FROM @Result) + BEGIN + SELECT ServiceId, ProviderId, QueueId, OwnerEpoch, [Checkpoint], Updated + FROM @Result; + RETURN; + END; + + SELECT ServiceId, ProviderId, QueueId, - MessageId - ) - UPDATE Batch - SET - VisibleOn = @Now, - ModifiedOn = @Now - OUTPUT - Inserted.ServiceId, - Inserted.ProviderId, - Inserted.QueueId, - Inserted.MessageId; - - RETURN; + OwnerEpoch, + [Checkpoint], + CAST(0 AS BIT) AS Updated + FROM OrleansStreamPartition + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId; + END TRY + BEGIN CATCH + IF @StartedTransaction = 1 AND XACT_STATE() <> 0 + BEGIN + ROLLBACK TRANSACTION; + END; + THROW; + END CATCH; END; - -/* delete messages in the exact same order as the clustered index to avoid deadlocks with other queries */ -/* skip messages being changed concurrently since their dequeue receipt may no longer match */ -WITH Batch AS -( - SELECT TOP (@Count) - * - FROM - OrleansStreamMessage AS M WITH (UPDLOCK, READPAST, READCOMMITTEDLOCK, ROWLOCK) - WHERE - ServiceId = @ServiceId - AND ProviderId = @ProviderId - AND QueueId = @QueueId - AND EXISTS - ( - SELECT * - FROM @ItemsTable AS I - WHERE I.MessageId = M.MessageId - AND I.Dequeued = M.Dequeued - ) - ORDER BY - ServiceId, - ProviderId, - QueueId, - MessageId -) -DELETE FROM Batch -OUTPUT - Deleted.ServiceId, - Deleted.ProviderId, - Deleted.QueueId, - Deleted.MessageId; - -END -GO - -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'ConfirmStreamMessagesKey', - 'EXECUTE ConfirmStreamMessages @ServiceId = @ServiceId, @ProviderId = @ProviderId, @QueueId = @QueueId, @Items = @Items' -GO - -/* Applies delivery failure rules to the specified message. */ -/* If the message has been dequeued too many times, we move it to the dead letter table. */ -/* If the message has expired, we move to the dead letter table. */ -/* If the message is still eligible for delivery, it is made visible again. */ -CREATE PROCEDURE FailStreamMessage - @ServiceId NVARCHAR(150), - @ProviderId NVARCHAR(150), - @QueueId NVARCHAR(150), - @MessageId BIGINT, - @MaxAttempts INT, - @RemovalTimeout INT -AS -BEGIN - -SET NOCOUNT ON; -SET XACT_ABORT ON; - -DECLARE @Now DATETIME2(7) = SYSUTCDATETIME(); -DECLARE @RemoveOn DATETIME2(7) = DATEADD(SECOND, @RemovalTimeout, @Now); - -/* if the message can still be dequeued then attempt to mark it visible again */ -UPDATE OrleansStreamMessage -SET - VisibleOn = @Now, - ModifiedOn = @Now -WHERE - ServiceId = @ServiceId - AND ProviderId = @ProviderId - AND QueueId = @QueueId - AND MessageId = @MessageId - AND Dequeued < @MaxAttempts; - -IF @@ROWCOUNT > 0 RETURN; - -/* otherwise attempt to move the message to dead letters */ -DELETE FROM OrleansStreamMessage -OUTPUT - Deleted.ServiceId, - Deleted.ProviderId, - Deleted.QueueId, - Deleted.MessageId, - Deleted.Dequeued, - Deleted.VisibleOn, - Deleted.ExpiresOn, - Deleted.CreatedOn, - Deleted.ModifiedOn, - @Now AS DeadOn, - @RemoveOn AS RemoveOn, - Deleted.Payload -INTO OrleansStreamDeadLetter -( - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - DeadOn, - RemoveOn, - Payload -) -WHERE - ServiceId = @ServiceId - AND ProviderId = @ProviderId - AND QueueId = @QueueId - AND MessageId = @MessageId; - -END -GO - -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'FailStreamMessageKey', - 'EXECUTE FailStreamMessage @ServiceId = @ServiceId, @ProviderId = @ProviderId, @QueueId = @QueueId, @MessageId = @MessageId, @MaxAttempts = @MaxAttempts, @RemovalTimeout = @RemovalTimeout' -GO - -/* Moves non-delivered messages from the message table to the dead letter table for human troubleshooting. */ -CREATE PROCEDURE EvictStreamMessages - @ServiceId NVARCHAR(150), - @ProviderId NVARCHAR(150), - @QueueId NVARCHAR(150), - @BatchSize INT, - @MaxAttempts INT, - @RemovalTimeout INT -AS -BEGIN - -SET NOCOUNT ON; -SET XACT_ABORT ON; - -DECLARE @Now DATETIME2(7) = SYSUTCDATETIME(); -DECLARE @RemoveOn DATETIME2(7) = DATEADD(SECOND, @RemovalTimeout, @Now); - -/* delete messages in the exact same order as the clustered index to avoid deadlocks with other queries */ -WITH Batch AS -( - SELECT TOP (@BatchSize) - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - DeadOn = @Now, - RemoveOn = @RemoveOn, - Payload - FROM - OrleansStreamMessage WITH (UPDLOCK, READPAST, READCOMMITTEDLOCK) - WHERE - ServiceId = @ServiceId - AND ProviderId = @ProviderId - AND QueueId = @QueueId - - -- the message was given the opportunity to complete - AND VisibleOn <= @Now - AND - ( - -- the message was dequeued too many times - Dequeued >= @MaxAttempts - OR - -- the message expired - ExpiresOn <= @Now - ) - ORDER BY - ServiceId, - ProviderId, - QueueId, - MessageId -) -DELETE FROM Batch -OUTPUT - Deleted.ServiceId, - Deleted.ProviderId, - Deleted.QueueId, - Deleted.MessageId, - Deleted.Dequeued, - Deleted.VisibleOn, - Deleted.ExpiresOn, - Deleted.CreatedOn, - Deleted.ModifiedOn, - Deleted.DeadOn, - Deleted.RemoveOn, - Deleted.Payload -INTO OrleansStreamDeadLetter -( - ServiceId, - ProviderId, - QueueId, - MessageId, - Dequeued, - VisibleOn, - ExpiresOn, - CreatedOn, - ModifiedOn, - DeadOn, - RemoveOn, - Payload -); - -END GO -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'EvictStreamMessagesKey', - 'EXECUTE EvictStreamMessages @ServiceId = @ServiceId, @ProviderId = @ProviderId, @QueueId = @QueueId, @BatchSize = @BatchSize, @MaxAttempts = @MaxAttempts, @RemovalTimeout = @RemovalTimeout' -GO - -/* Removes messages from the dead letters table. */ -CREATE PROCEDURE EvictStreamDeadLetters - @ServiceId NVARCHAR(150), +CREATE PROCEDURE CleanupStreamMessages + @ServiceId NVARCHAR(150), @ProviderId NVARCHAR(150), - @QueueId NVARCHAR(150), - @BatchSize INT + @QueueId NVARCHAR(150), + @RetentionPeriodSeconds INT, + @MaximumRetentionPeriodSeconds INT = NULL, + @CleanupIntervalSeconds INT, + @CleanupBatchSize INT AS BEGIN + SET NOCOUNT ON; + SET XACT_ABORT ON; -SET NOCOUNT ON; -SET XACT_ABORT ON; - -DECLARE @Now DATETIME2(7) = SYSUTCDATETIME(); - -/* delete messages in the exact same order as the clustered index to avoid deadlocks with other queries */ -WITH Batch AS -( - SELECT TOP (@BatchSize) - ServiceId, - ProviderId, - QueueId, - MessageId - FROM - OrleansStreamDeadLetter WITH (UPDLOCK) - WHERE - ServiceId = @ServiceId - AND ProviderId = @ProviderId - AND QueueId = @QueueId - AND RemoveOn <= @Now - ORDER BY - ServiceId, - ProviderId, - QueueId, - MessageId -) -DELETE FROM Batch; + DECLARE @StartedTransaction BIT = 0; + DECLARE @Now DATETIME2(7) = SYSUTCDATETIME(); + DECLARE @Checkpoint BIGINT; + DECLARE @Deleted TABLE (MessageId BIGINT NOT NULL, HardDeleted BIT NOT NULL); + + BEGIN TRY + IF @@TRANCOUNT = 0 + BEGIN + BEGIN TRANSACTION; + SET @StartedTransaction = 1; + END; + + UPDATE OrleansStreamPartition WITH (UPDLOCK, ROWLOCK) + SET + CleanupOn = DATEADD(SECOND, @CleanupIntervalSeconds, @Now), + ModifiedOn = @Now, + @Checkpoint = [Checkpoint] + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId + AND CleanupOn <= @Now; + + IF @@ROWCOUNT = 0 + BEGIN + SELECT @Checkpoint = [Checkpoint] + FROM OrleansStreamPartition + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId; + + SELECT + CAST(0 AS BIT) AS Ran, + 0 AS DeletedCount, + CAST(NULL AS BIGINT) AS DeletedThroughMessageId, + 0 AS HardDeletedCount, + CAST(NULL AS BIGINT) AS HardDeletedFromMessageId, + CAST(NULL AS BIGINT) AS HardDeletedThroughMessageId, + @Checkpoint AS [Checkpoint], + MIN(MessageId) AS EarliestMessageId, + MAX(MessageId) AS TailMessageId + FROM OrleansStreamMessage + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId; + + IF @StartedTransaction = 1 + BEGIN + COMMIT TRANSACTION; + END; + + RETURN; + END; + + ;WITH Candidate AS + ( + SELECT TOP (@CleanupBatchSize) + MessageId + FROM OrleansStreamMessage WITH (UPDLOCK, READCOMMITTEDLOCK, ROWLOCK) + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId + AND + ( + ( + @Checkpoint IS NOT NULL + AND MessageId <= @Checkpoint + AND CheckpointedOn < DATEADD(SECOND, -@RetentionPeriodSeconds, @Now) + ) + OR + ( + @MaximumRetentionPeriodSeconds IS NOT NULL + AND CreatedOn < DATEADD(SECOND, -@MaximumRetentionPeriodSeconds, @Now) + ) + ) + ORDER BY MessageId + ) + DELETE Message + OUTPUT + Deleted.MessageId, + CASE + WHEN @Checkpoint IS NULL OR Deleted.MessageId > @Checkpoint THEN CAST(1 AS BIT) + ELSE CAST(0 AS BIT) + END + INTO @Deleted (MessageId, HardDeleted) + FROM OrleansStreamMessage AS Message + INNER JOIN Candidate + ON Candidate.MessageId = Message.MessageId + WHERE Message.ServiceId = @ServiceId + AND Message.ProviderId = @ProviderId + AND Message.QueueId = @QueueId; -END + SELECT + CAST(1 AS BIT) AS Ran, + COUNT(*) AS DeletedCount, + MAX(MessageId) AS DeletedThroughMessageId, + COALESCE(SUM(CASE WHEN HardDeleted = 1 THEN 1 ELSE 0 END), 0) AS HardDeletedCount, + MIN(CASE WHEN HardDeleted = 1 THEN MessageId END) AS HardDeletedFromMessageId, + MAX(CASE WHEN HardDeleted = 1 THEN MessageId END) AS HardDeletedThroughMessageId, + @Checkpoint AS [Checkpoint], + ( + SELECT MIN(MessageId) + FROM OrleansStreamMessage + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId + ) AS EarliestMessageId, + ( + SELECT MAX(MessageId) + FROM OrleansStreamMessage + WHERE ServiceId = @ServiceId + AND ProviderId = @ProviderId + AND QueueId = @QueueId + ) AS TailMessageId + FROM @Deleted; + + IF @StartedTransaction = 1 + BEGIN + COMMIT TRANSACTION; + END; + END TRY + BEGIN CATCH + IF @StartedTransaction = 1 AND XACT_STATE() <> 0 + BEGIN + ROLLBACK TRANSACTION; + END; + + THROW; + END CATCH; +END; GO -INSERT INTO OrleansQuery -( - QueryKey, - QueryText -) -SELECT - 'EvictStreamDeadLettersKey', - 'EXECUTE EvictStreamDeadLetters @ServiceId = @ServiceId, @ProviderId = @ProviderId, @QueueId = @QueueId, @BatchSize = @BatchSize' +INSERT INTO OrleansQuery (QueryKey, QueryText) +VALUES + ('StreamSchemaVersionKey', '2'), + ('AppendStreamMessageKey', 'EXECUTE AppendStreamMessage @ServiceId = @ServiceId, @ProviderId = @ProviderId, @QueueId = @QueueId, @StreamIdBytes = @StreamIdBytes, @StreamNamespaceLength = @StreamNamespaceLength, @Payload = @Payload'), + ('AcquireStreamPartitionKey', 'EXECUTE AcquireStreamPartition @ServiceId = @ServiceId, @ProviderId = @ProviderId, @QueueId = @QueueId, @StartFromNow = @StartFromNow'), + ('ReadStreamMessagesKey', 'SELECT ServiceId, ProviderId, QueueId, MessageId, StreamIdBytes, StreamNamespaceLength, CreatedOn, Payload FROM OrleansStreamMessage WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId AND QueueId = @QueueId AND MessageId > @AfterMessageId ORDER BY MessageId OFFSET 0 ROWS FETCH NEXT @MaxCount ROWS ONLY'), + ('AdvanceStreamCheckpointKey', 'EXECUTE AdvanceStreamCheckpoint @ServiceId = @ServiceId, @ProviderId = @ProviderId, @QueueId = @QueueId, @OwnerEpoch = @OwnerEpoch, @Checkpoint = @Checkpoint'), + ('GetStreamPartitionBoundsKey', 'SELECT P.ServiceId, P.ProviderId, P.QueueId, P.OwnerEpoch, P.NextMessageId, P.[Checkpoint] AS [Checkpoint], MIN(M.MessageId) AS EarliestMessageId, MAX(M.MessageId) AS TailMessageId FROM OrleansStreamPartition AS P LEFT JOIN OrleansStreamMessage AS M ON M.ServiceId = P.ServiceId AND M.ProviderId = P.ProviderId AND M.QueueId = P.QueueId WHERE P.ServiceId = @ServiceId AND P.ProviderId = @ProviderId AND P.QueueId = @QueueId GROUP BY P.ServiceId, P.ProviderId, P.QueueId, P.OwnerEpoch, P.NextMessageId, P.[Checkpoint]'), + ('CleanupStreamMessagesKey', 'EXECUTE CleanupStreamMessages @ServiceId = @ServiceId, @ProviderId = @ProviderId, @QueueId = @QueueId, @RetentionPeriodSeconds = @RetentionPeriodSeconds, @MaximumRetentionPeriodSeconds = @MaximumRetentionPeriodSeconds, @CleanupIntervalSeconds = @CleanupIntervalSeconds, @CleanupBatchSize = @CleanupBatchSize'); GO diff --git a/src/AdoNet/Shared/Storage/DbStoredQueries.cs b/src/AdoNet/Shared/Storage/DbStoredQueries.cs index 0936ddc716e..e14319f31ec 100644 --- a/src/AdoNet/Shared/Storage/DbStoredQueries.cs +++ b/src/AdoNet/Shared/Storage/DbStoredQueries.cs @@ -31,10 +31,39 @@ namespace Orleans.Tests.SqlUtils /// internal class DbStoredQueries { +#if STREAMING_ADONET || TESTER_SQLUTILS + private const string CurrentStreamingSchemaVersion = "2"; + + private static readonly string[] RequiredStreamingQueryKeys = + [ + nameof(StreamSchemaVersionKey), + nameof(AppendStreamMessageKey), + nameof(AcquireStreamPartitionKey), + nameof(ReadStreamMessagesKey), + nameof(AdvanceStreamCheckpointKey), + nameof(GetStreamPartitionBoundsKey), + nameof(CleanupStreamMessagesKey) + ]; + + private static readonly string[] LegacyStreamingQueryKeys = + [ + "QueueStreamMessageKey", + "GetStreamMessagesKey", + "ConfirmStreamMessagesKey", + "FailStreamMessageKey", + "EvictStreamMessagesKey", + "EvictStreamDeadLettersKey" + ]; +#endif + private readonly Dictionary queries; internal DbStoredQueries(Dictionary queries) { +#if STREAMING_ADONET || TESTER_SQLUTILS + ValidateStreamingSchema(queries); +#endif + var fields = typeof(DbStoredQueries).GetProperties(BindingFlags.Instance | BindingFlags.NonPublic) .Select(p => p.Name); var missingQueryKeys = fields.Except(queries.Keys).ToArray(); @@ -46,6 +75,34 @@ internal DbStoredQueries(Dictionary queries) this.queries = queries; } +#if STREAMING_ADONET || TESTER_SQLUTILS + private static void ValidateStreamingSchema(Dictionary queries) + { + var hasLegacyKeys = LegacyStreamingQueryKeys.Any(queries.ContainsKey); + var missingKeys = RequiredStreamingQueryKeys.Where(key => !queries.ContainsKey(key)).ToArray(); + var hasExpectedVersion = queries.TryGetValue(nameof(StreamSchemaVersionKey), out var version) + && string.Equals(version, CurrentStreamingSchemaVersion, StringComparison.Ordinal); + + if (hasExpectedVersion && !hasLegacyKeys && missingKeys.Length == 0) + { + return; + } + + var detectedVersion = version ?? "legacy or missing"; + var missingDescription = missingKeys.Length == 0 ? "none" : string.Join(", ", missingKeys); + var legacyDescription = LegacyStreamingQueryKeys.Where(queries.ContainsKey).ToArray() is { Length: > 0 } legacyKeys + ? string.Join(", ", legacyKeys) + : "none"; + throw new InvalidOperationException( + $"The ADO.NET streaming schema is incompatible. Expected stream partition schema version {CurrentStreamingSchemaVersion}, " + + $"but detected '{detectedVersion}'. Missing query keys: {missingDescription}. Legacy query keys: {legacyDescription}. " + + "This alpha schema has no in-place migration. Drop the legacy OrleansStreamMessage, " + + "OrleansStreamDeadLetter, OrleansStreamControl, and OrleansStreamMessageSequence objects, drop OrleansStreamPartition if it " + + "exists, remove the streaming routines and OrleansQuery rows, and then apply the current SQL Server, PostgreSQL, or MySQL " + + "streaming script. Existing alpha queue rows are not compatible and will not be migrated."); + } +#endif + /// /// The query that's used to get all the stored queries. /// this will probably be the same for all relational dbs. @@ -143,34 +200,39 @@ internal DbStoredQueries(Dictionary queries) #if STREAMING_ADONET || TESTER_SQLUTILS /// - /// A query template to enqueue a message into the stream table. + /// The stream partition schema version marker. /// - internal string QueueStreamMessageKey => queries[nameof(QueueStreamMessageKey)]; + internal string StreamSchemaVersionKey => queries[nameof(StreamSchemaVersionKey)]; /// - /// A query template to dequeue messages from the stream table. + /// A query template to append a record to a stream partition. /// - internal string GetStreamMessagesKey => queries[nameof(GetStreamMessagesKey)]; + internal string AppendStreamMessageKey => queries[nameof(AppendStreamMessageKey)]; /// - /// A query template to confirm message delivery from the stream table. + /// A query template to acquire stream partition ownership. /// - internal string ConfirmStreamMessagesKey => queries[nameof(ConfirmStreamMessagesKey)]; + internal string AcquireStreamPartitionKey => queries[nameof(AcquireStreamPartitionKey)]; /// - /// A query template to evict a single message (by moving it to dead letters). + /// A query template to read ordered stream records after an exclusive position. /// - internal string FailStreamMessageKey => queries[nameof(FailStreamMessageKey)]; + internal string ReadStreamMessagesKey => queries[nameof(ReadStreamMessagesKey)]; /// - /// A query template to batch evict messages (by moving them to dead letters). + /// A query template to advance an epoch-fenced checkpoint. /// - internal string EvictStreamMessagesKey => queries[nameof(EvictStreamMessagesKey)]; + internal string AdvanceStreamCheckpointKey => queries[nameof(AdvanceStreamCheckpointKey)]; /// - /// A query template to evict expired dead letters (by deleting them). + /// A query template to read a partition checkpoint and partition history bounds. /// - internal string EvictStreamDeadLettersKey => queries[nameof(EvictStreamDeadLettersKey)]; + internal string GetStreamPartitionBoundsKey => queries[nameof(GetStreamPartitionBoundsKey)]; + + /// + /// A query template to perform bounded stream partition retention cleanup. + /// + internal string CleanupStreamMessagesKey => queries[nameof(CleanupStreamMessagesKey)]; #endif @@ -504,44 +566,64 @@ internal long MessageId set => Add(nameof(MessageId), value); } + internal long AfterMessageId + { + set => Add(nameof(AfterMessageId), value); + } + internal byte[] Payload { set => Add(nameof(Payload), value); } - internal int ExpiryTimeout + internal byte[] StreamIdBytes { - set => Add(nameof(ExpiryTimeout), value); + set => Add(nameof(StreamIdBytes), value); } - internal int MaxCount + internal int StreamNamespaceLength { - set => Add(nameof(MaxCount), value); + set => Add(nameof(StreamNamespaceLength), value); } - internal int MaxAttempts + internal bool StartFromNow { - set => Add(nameof(MaxAttempts), value); + set => Add(nameof(StartFromNow), value); + } + + internal long OwnerEpoch + { + set => Add(nameof(OwnerEpoch), value); + } + + internal long Checkpoint + { + set => Add(nameof(Checkpoint), value); + } + + internal int MaxCount + { + set => Add(nameof(MaxCount), value); } - internal int RemovalTimeout + internal int RetentionPeriodSeconds { - set => Add(nameof(RemovalTimeout), value); + set => Add(nameof(RetentionPeriodSeconds), value); } - internal int VisibilityTimeout + internal int? MaximumRetentionPeriodSeconds { - set => Add(nameof(VisibilityTimeout), value); + set => Add(nameof(MaximumRetentionPeriodSeconds), value, DbType.Int32); } - internal int EvictionInterval + internal int CleanupIntervalSeconds { - set => Add(nameof(EvictionInterval), value); + set => Add(nameof(CleanupIntervalSeconds), value); } - internal int EvictionBatchSize + internal int CleanupBatchSize { - set => Add(nameof(EvictionBatchSize), value); + set => Add(nameof(CleanupBatchSize), value); } internal string EventIds diff --git a/src/AdoNet/Shared/Storage/RelationalOrleansQueries.cs b/src/AdoNet/Shared/Storage/RelationalOrleansQueries.cs index 228133391b4..02597f6f187 100644 --- a/src/AdoNet/Shared/Storage/RelationalOrleansQueries.cs +++ b/src/AdoNet/Shared/Storage/RelationalOrleansQueries.cs @@ -3,7 +3,7 @@ using System.Data; using System.Data.Common; using System.Linq; -using System.Text; +using System.Threading; using System.Threading.Tasks; using Orleans.Runtime; @@ -55,7 +55,7 @@ internal class RelationalOrleansQueries /// /// the underlying relational storage /// Orleans functional queries - private RelationalOrleansQueries(IRelationalStorage storage, DbStoredQueries dbStoredQueries) + internal RelationalOrleansQueries(IRelationalStorage storage, DbStoredQueries dbStoredQueries) { this.storage = storage; this.dbStoredQueries = dbStoredQueries; @@ -91,9 +91,14 @@ private Task ExecuteAsync(string query, Func ReadAsync(string query, Func selector, Func parameterProvider, - Func, TAggregate> aggregator) - { - var ret = await storage.ReadAsync(query, selector, command => parameterProvider(command)); + Func, TAggregate> aggregator, + CancellationToken cancellationToken = default) + { + var ret = await storage.ReadAsync( + query, + selector, + command => parameterProvider(command), + cancellationToken); return aggregator(ret); } @@ -393,22 +398,35 @@ private static MembershipTableData ConvertToMembershipTableData(IEnumerable - /// Queues a stream message to the stream message table. + /// Appends an immutable stream record to a stream partition. /// /// The service identifier. /// The provider identifier. /// The queue identifier. + /// The canonical bytes. + /// The namespace boundary within . /// The serialized event payload. - /// The expiry timeout for this event batch. - /// An acknowledgement that the message was queued. - internal Task QueueStreamMessageAsync(string serviceId, string providerId, string queueId, byte[] payload, int expiryTimeout) + /// An acknowledgement containing the allocated partition-local message identifier. + internal Task AppendStreamMessageAsync( + string serviceId, + string providerId, + string queueId, + byte[] streamIdBytes, + int streamNamespaceLength, + byte[] payload) { ArgumentNullException.ThrowIfNull(serviceId); ArgumentNullException.ThrowIfNull(providerId); ArgumentNullException.ThrowIfNull(queueId); + ArgumentNullException.ThrowIfNull(streamIdBytes); + ArgumentNullException.ThrowIfNull(payload); + if (streamIdBytes.Length == 0 || streamNamespaceLength < 0 || streamNamespaceLength >= streamIdBytes.Length) + { + throw new ArgumentOutOfRangeException(nameof(streamNamespaceLength), "The stream namespace boundary must leave a non-empty stream key."); + } return ReadAsync( - dbStoredQueries.QueueStreamMessageKey, + dbStoredQueries.AppendStreamMessageKey, record => new AdoNetStreamMessageAck( (string)record[nameof(AdoNetStreamMessageAck.ServiceId)], (string)record[nameof(AdoNetStreamMessageAck.ProviderId)], @@ -419,232 +437,213 @@ internal Task QueueStreamMessageAsync(string serviceId, ServiceId = serviceId, ProviderId = providerId, QueueId = queueId, - Payload = payload, - ExpiryTimeout = expiryTimeout, + StreamIdBytes = streamIdBytes, + StreamNamespaceLength = streamNamespaceLength, + Payload = payload }, result => result.Single()); } /// - /// Gets stream messages from the stream message table. + /// Acquires partition ownership and initializes its checkpoint when necessary. /// - /// The service identifier. - /// The provider identifier. - /// The queue identifier. - /// The maximum count of event batches to get. - /// The maximum attempts to lock an unprocessed event batch. - /// The visibility timeout for the retrieved event batches. - /// The timeout before the message is to be deleted from dead letters. - /// The interval between opportunistic data eviction. - /// The number of messages to evict in each batch. - /// A list of dequeued payloads. - internal Task> GetStreamMessagesAsync(string serviceId, string providerId, string queueId, int maxCount, int maxAttempts, int visibilityTimeout, int removalTimeout, int evictionInterval, int evictionBatchSize) + internal Task AcquireStreamPartitionAsync( + string serviceId, + string providerId, + string queueId, + bool startFromNow, + CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(serviceId); ArgumentNullException.ThrowIfNull(providerId); ArgumentNullException.ThrowIfNull(queueId); - return ReadAsync>( - dbStoredQueries.GetStreamMessagesKey, - record => new AdoNetStreamMessage( - (string)record[nameof(AdoNetStreamMessage.ServiceId)], - (string)record[nameof(AdoNetStreamMessage.ProviderId)], - (string)record[nameof(AdoNetStreamMessage.QueueId)], - (long)record[nameof(AdoNetStreamMessage.MessageId)], - (int)record[nameof(AdoNetStreamMessage.Dequeued)], - (DateTime)record[nameof(AdoNetStreamMessage.VisibleOn)], - (DateTime)record[nameof(AdoNetStreamMessage.ExpiresOn)], - (DateTime)record[nameof(AdoNetStreamMessage.CreatedOn)], - (DateTime)record[nameof(AdoNetStreamMessage.ModifiedOn)], - (byte[])record[nameof(AdoNetStreamMessage.Payload)]), + return ReadAsync( + dbStoredQueries.AcquireStreamPartitionKey, + record => new AdoNetStreamPartitionState( + (string)record[nameof(AdoNetStreamPartitionState.ServiceId)], + (string)record[nameof(AdoNetStreamPartitionState.ProviderId)], + (string)record[nameof(AdoNetStreamPartitionState.QueueId)], + (long)record[nameof(AdoNetStreamPartitionState.OwnerEpoch)], + (long)record[nameof(AdoNetStreamPartitionState.NextMessageId)], + GetNullableInt64(record, nameof(AdoNetStreamPartitionState.Checkpoint)), + GetNullableInt64(record, nameof(AdoNetStreamPartitionState.EarliestMessageId)), + GetNullableInt64(record, nameof(AdoNetStreamPartitionState.TailMessageId))), command => new DbStoredQueries.Columns(command) { ServiceId = serviceId, ProviderId = providerId, QueueId = queueId, - MaxCount = maxCount, - MaxAttempts = maxAttempts, - VisibilityTimeout = visibilityTimeout, - RemovalTimeout = removalTimeout, - EvictionInterval = evictionInterval, - EvictionBatchSize = evictionBatchSize + StartFromNow = startFromNow }, - result => - { - var messages = result.ToList(); - messages.Sort(static (left, right) => left.MessageId.CompareTo(right.MessageId)); - return messages; - }); + result => result.Single(), + cancellationToken); } /// - /// Confirms delivery of messages from the stream message table. + /// Reads retained stream records with identifiers strictly greater than . /// - /// The service identifier. - /// The provider identifier. - /// The queue identifier. - /// The messages to confirm. - /// A list of confirmations. - /// - /// If is empty then an empty confirmation list is returned. - /// - internal Task> ConfirmStreamMessagesAsync(string serviceId, string providerId, string queueId, IList messages) + internal Task> ReadStreamMessagesAsync( + string serviceId, + string providerId, + string queueId, + long afterMessageId, + int maxCount, + CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(serviceId); ArgumentNullException.ThrowIfNull(providerId); ArgumentNullException.ThrowIfNull(queueId); - ArgumentNullException.ThrowIfNull(messages); + ArgumentOutOfRangeException.ThrowIfNegative(afterMessageId); + ArgumentOutOfRangeException.ThrowIfLessThan(maxCount, 1); - if (messages.Count == 0) - { - return Task.FromResult>([]); - } - - return ReadAsync>( - dbStoredQueries.ConfirmStreamMessagesKey, - record => new AdoNetStreamConfirmationAck( - (string)record[nameof(AdoNetStreamConfirmationAck.ServiceId)], - (string)record[nameof(AdoNetStreamConfirmationAck.ProviderId)], - (string)record[nameof(AdoNetStreamConfirmationAck.QueueId)], - (long)record[nameof(AdoNetStreamConfirmationAck.MessageId)]), + return ReadAsync>( + dbStoredQueries.ReadStreamMessagesKey, + record => new AdoNetStreamMessage( + (string)record[nameof(AdoNetStreamMessage.ServiceId)], + (string)record[nameof(AdoNetStreamMessage.ProviderId)], + (string)record[nameof(AdoNetStreamMessage.QueueId)], + (long)record[nameof(AdoNetStreamMessage.MessageId)], + (byte[])record[nameof(AdoNetStreamMessage.StreamIdBytes)], + (int)record[nameof(AdoNetStreamMessage.StreamNamespaceLength)], + record.GetDateTimeValue(nameof(AdoNetStreamMessage.CreatedOn)), + (byte[])record[nameof(AdoNetStreamMessage.Payload)]), command => new DbStoredQueries.Columns(command) { ServiceId = serviceId, ProviderId = providerId, QueueId = queueId, - Items = FormatStreamConfirmations(messages, release: false) + AfterMessageId = afterMessageId, + MaxCount = maxCount }, - result => result.ToList()); + result => result.ToList(), + cancellationToken); } /// - /// Makes unconfirmed stream messages immediately available for redelivery. + /// Advances a checkpoint only when the caller owns the current epoch and the value moves forward. /// - /// The service identifier. - /// The provider identifier. - /// The queue identifier. - /// The messages to release. - /// A list of released messages. - /// - /// The dequeue counter acts as a receipt so that a stale receiver cannot release a message - /// which has already been dequeued by a new receiver. - /// - internal Task> ReleaseStreamMessagesAsync(string serviceId, string providerId, string queueId, IList messages) + internal Task AdvanceStreamCheckpointAsync( + string serviceId, + string providerId, + string queueId, + long ownerEpoch, + long checkpoint, + CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(serviceId); ArgumentNullException.ThrowIfNull(providerId); ArgumentNullException.ThrowIfNull(queueId); - ArgumentNullException.ThrowIfNull(messages); - - if (messages.Count == 0) - { - return Task.FromResult>([]); - } - - return ReadAsync>( - dbStoredQueries.ConfirmStreamMessagesKey, - record => new AdoNetStreamConfirmationAck( - (string)record[nameof(AdoNetStreamConfirmationAck.ServiceId)], - (string)record[nameof(AdoNetStreamConfirmationAck.ProviderId)], - (string)record[nameof(AdoNetStreamConfirmationAck.QueueId)], - (long)record[nameof(AdoNetStreamConfirmationAck.MessageId)]), + ArgumentOutOfRangeException.ThrowIfLessThan(ownerEpoch, 1); + ArgumentOutOfRangeException.ThrowIfNegative(checkpoint); + + return ReadAsync( + dbStoredQueries.AdvanceStreamCheckpointKey, + record => new AdoNetStreamCheckpointUpdate( + (string)record[nameof(AdoNetStreamCheckpointUpdate.ServiceId)], + (string)record[nameof(AdoNetStreamCheckpointUpdate.ProviderId)], + (string)record[nameof(AdoNetStreamCheckpointUpdate.QueueId)], + (long)record[nameof(AdoNetStreamCheckpointUpdate.OwnerEpoch)], + GetNullableInt64(record, nameof(AdoNetStreamCheckpointUpdate.Checkpoint)), + Convert.ToBoolean(record[nameof(AdoNetStreamCheckpointUpdate.Updated)])), command => new DbStoredQueries.Columns(command) { ServiceId = serviceId, ProviderId = providerId, QueueId = queueId, - Items = FormatStreamConfirmations(messages, release: true) + OwnerEpoch = ownerEpoch, + Checkpoint = checkpoint }, - result => result.ToList()); + result => result.SingleOrDefault(), + cancellationToken); } - // Builds a provider-neutral receipt list in the form "1:2|3:4|5:6". - private static string FormatStreamConfirmations(IList messages, bool release) => - messages.Aggregate( - new StringBuilder(), - (builder, message) => builder - .Append(builder.Length > 0 ? "|" : "") - .Append(message.MessageId) - .Append(':') - .Append(release ? -message.Dequeued : message.Dequeued), - static builder => builder.ToString()); - /// - /// Applies delivery failure logic to a stream message, such as making the message visible again or moving it to dead letters. + /// Reads the current checkpoint, ownership epoch, and retained bounds of partition history. /// - /// The service identifier. - /// The provider identifier. - /// The queue identifier. - /// The message identifier. - internal Task FailStreamMessageAsync(string serviceId, string providerId, string queueId, long messageId, int maxAttempts, int removalTimeout) + internal Task GetStreamPartitionBoundsAsync( + string serviceId, + string providerId, + string queueId) { ArgumentNullException.ThrowIfNull(serviceId); ArgumentNullException.ThrowIfNull(providerId); ArgumentNullException.ThrowIfNull(queueId); - return ExecuteAsync( - dbStoredQueries.FailStreamMessageKey, + return ReadAsync( + dbStoredQueries.GetStreamPartitionBoundsKey, + record => new AdoNetStreamPartitionState( + (string)record[nameof(AdoNetStreamPartitionState.ServiceId)], + (string)record[nameof(AdoNetStreamPartitionState.ProviderId)], + (string)record[nameof(AdoNetStreamPartitionState.QueueId)], + (long)record[nameof(AdoNetStreamPartitionState.OwnerEpoch)], + (long)record[nameof(AdoNetStreamPartitionState.NextMessageId)], + GetNullableInt64(record, nameof(AdoNetStreamPartitionState.Checkpoint)), + GetNullableInt64(record, nameof(AdoNetStreamPartitionState.EarliestMessageId)), + GetNullableInt64(record, nameof(AdoNetStreamPartitionState.TailMessageId))), command => new DbStoredQueries.Columns(command) { ServiceId = serviceId, ProviderId = providerId, - QueueId = queueId, - MessageId = messageId, - MaxAttempts = maxAttempts, - RemovalTimeout = removalTimeout - }); + QueueId = queueId + }, + result => result.SingleOrDefault()); } /// - /// Moves eligible messages from the stream message table to the dead letter table. + /// Deletes an ordered, bounded batch of retained stream records. /// - /// The service identifier. - /// The provider identifier. - /// The queue identifier. - /// The max number of messages to move in this batch. - /// The max number of times a message can be dequeued. - /// The timeout before the message is to be deleted from dead letters. - internal Task EvictStreamMessagesAsync(string serviceId, string providerId, string queueId, int maxCount, int maxAttempts, int removalTimeout) + internal Task CleanupStreamMessagesAsync( + string serviceId, + string providerId, + string queueId, + int retentionPeriodSeconds, + int? maximumRetentionPeriodSeconds, + int cleanupIntervalSeconds, + int cleanupBatchSize, + CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(serviceId); ArgumentNullException.ThrowIfNull(providerId); ArgumentNullException.ThrowIfNull(queueId); + ArgumentOutOfRangeException.ThrowIfLessThan(retentionPeriodSeconds, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(cleanupIntervalSeconds, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(cleanupBatchSize, 1); + if (maximumRetentionPeriodSeconds is { } maximum && maximum < retentionPeriodSeconds) + { + throw new ArgumentOutOfRangeException(nameof(maximumRetentionPeriodSeconds), "The maximum retention period must be greater than or equal to the normal retention period."); + } - return ExecuteAsync( - dbStoredQueries.EvictStreamMessagesKey, + return ReadAsync( + dbStoredQueries.CleanupStreamMessagesKey, + record => new AdoNetStreamCleanupResult( + Convert.ToBoolean(record[nameof(AdoNetStreamCleanupResult.Ran)]), + Convert.ToInt32(record[nameof(AdoNetStreamCleanupResult.DeletedCount)]), + GetNullableInt64(record, nameof(AdoNetStreamCleanupResult.DeletedThroughMessageId)), + Convert.ToInt32(record[nameof(AdoNetStreamCleanupResult.HardDeletedCount)]), + GetNullableInt64(record, nameof(AdoNetStreamCleanupResult.HardDeletedFromMessageId)), + GetNullableInt64(record, nameof(AdoNetStreamCleanupResult.HardDeletedThroughMessageId)), + GetNullableInt64(record, nameof(AdoNetStreamCleanupResult.Checkpoint)), + GetNullableInt64(record, nameof(AdoNetStreamCleanupResult.EarliestMessageId)), + GetNullableInt64(record, nameof(AdoNetStreamCleanupResult.TailMessageId))), command => new DbStoredQueries.Columns(command) { ServiceId = serviceId, ProviderId = providerId, QueueId = queueId, - MaxCount = maxCount, - MaxAttempts = maxAttempts, - RemovalTimeout = removalTimeout - }); + RetentionPeriodSeconds = retentionPeriodSeconds, + MaximumRetentionPeriodSeconds = maximumRetentionPeriodSeconds, + CleanupIntervalSeconds = cleanupIntervalSeconds, + CleanupBatchSize = cleanupBatchSize + }, + result => result.Single(), + cancellationToken); } - /// - /// Removes messages from the dead letter after their removal timeout expires. - /// - /// The service identifier. - /// The provider identifier. - /// The queue identifier. - /// The max number of messages to move in this batch. - internal Task EvictStreamDeadLettersAsync(string serviceId, string providerId, string queueId, int maxCount) + private static long? GetNullableInt64(IDataRecord record, string fieldName) { - ArgumentNullException.ThrowIfNull(serviceId); - ArgumentNullException.ThrowIfNull(providerId); - ArgumentNullException.ThrowIfNull(queueId); - - return ExecuteAsync( - dbStoredQueries.EvictStreamDeadLettersKey, - command => new DbStoredQueries.Columns(command) - { - ServiceId = serviceId, - ProviderId = providerId, - QueueId = queueId, - MaxCount = maxCount - }); + var ordinal = record.GetOrdinal(fieldName); + return record.IsDBNull(ordinal) ? null : Convert.ToInt64(record.GetValue(ordinal)); } #endif diff --git a/src/AdoNet/Shared/Storage/RelationalStorageExtensions.cs b/src/AdoNet/Shared/Storage/RelationalStorageExtensions.cs index 8b2a243d6e3..54570dba94b 100644 --- a/src/AdoNet/Shared/Storage/RelationalStorageExtensions.cs +++ b/src/AdoNet/Shared/Storage/RelationalStorageExtensions.cs @@ -36,10 +36,19 @@ internal static class RelationalStorageExtensions /// /// /// - public static Task> ReadAsync(this IRelationalStorage storage, string query, Func selector, Action? parameterProvider) + public static Task> ReadAsync( + this IRelationalStorage storage, + string query, + Func selector, + Action? parameterProvider, + CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(selector); - return storage.ReadAsync(query, parameterProvider, (record, i, cancellationToken) => Task.FromResult(selector(record))); + return storage.ReadAsync( + query, + parameterProvider, + (record, i, _) => Task.FromResult(selector(record)), + cancellationToken: cancellationToken); } /// diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs index 7c69c12b365..3bbf49451d4 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointer.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using Azure; using Microsoft.Extensions.Logging; using Orleans.Configuration; using Orleans.Streaming.EventHubs; @@ -13,16 +14,11 @@ namespace Orleans.Streams /// public partial class AzureTableStreamQueueCheckpointer : IStreamQueueCheckpointer { - private readonly AzureTableDataManager _dataManager; - private readonly TimeSpan _persistInterval; - private readonly IComparer? _checkpointComparer; - private readonly object _lock = new(); + private readonly IStreamCheckpointStore _store; + private readonly Func _initialize; + private readonly StreamQueueCheckpointer _inner; - private StreamQueueCheckpointEntity _entity; - private Task _inProgressSave = Task.CompletedTask; - private DateTime? _throttleSavesUntilUtc; - private string _latestCheckpoint = string.Empty; - private string _persistedCheckpoint = string.Empty; + internal IStreamCheckpointStore Store => _store; private AzureTableStreamQueueCheckpointer( AzureTableStreamCheckpointerOptions options, @@ -45,16 +41,25 @@ private AzureTableStreamQueueCheckpointer( $"{nameof(AzureTableStreamCheckpointerOptions.PersistInterval)} must be greater than zero."); } - _persistInterval = options.PersistInterval; - _checkpointComparer = options.CheckpointComparer ?? defaultComparer; - _dataManager = new AzureTableDataManager( + var dataManager = new AzureTableDataManager( options, loggerFactory.CreateLogger()); - _entity = StreamQueueCheckpointEntity.Create( - partitionKeyPrefix ?? options.PartitionKeyPrefix, - streamProviderName, - serviceId, - partition); + var store = new AzureTableCheckpointStore( + dataManager, + StreamQueueCheckpointEntity.Create( + partitionKeyPrefix ?? options.PartitionKeyPrefix, + streamProviderName, + serviceId, + partition)); + _store = store; + _initialize = store.Initialize; + _inner = new StreamQueueCheckpointer( + _store, + new StreamQueueCheckpointerOptions + { + CheckpointComparer = options.CheckpointComparer ?? defaultComparer, + PersistInterval = options.PersistInterval, + }); LogCreatingCheckpointer( loggerFactory.CreateLogger(), partition, @@ -62,18 +67,26 @@ private AzureTableStreamQueueCheckpointer( serviceId); } - /// - public bool CheckpointExists + internal AzureTableStreamQueueCheckpointer( + IStreamCheckpointStore store, + TimeSpan persistInterval, + IComparer? checkpointComparer) { - get - { - lock (_lock) + ArgumentNullException.ThrowIfNull(store); + _store = store; + _initialize = static _ => Task.CompletedTask; + _inner = new StreamQueueCheckpointer( + store, + new StreamQueueCheckpointerOptions { - return !string.IsNullOrEmpty(_latestCheckpoint); - } - } + CheckpointComparer = checkpointComparer, + PersistInterval = persistInterval, + }); } + /// + public bool CheckpointExists => _inner.CheckpointExists; + /// /// Creates and initializes an Azure Table stream queue checkpointer. /// @@ -84,7 +97,35 @@ public static Task> Create( string serviceId, ILoggerFactory loggerFactory) { - return Create(options, streamProviderName, partition, serviceId, loggerFactory, defaultComparer: null); + return Create( + options, + streamProviderName, + partition, + serviceId, + loggerFactory, + defaultComparer: null, + cancellationToken: CancellationToken.None); + } + + /// + /// Creates and initializes an Azure Table stream queue checkpointer. + /// + public static Task> Create( + AzureTableStreamCheckpointerOptions options, + string streamProviderName, + string partition, + string serviceId, + ILoggerFactory loggerFactory, + CancellationToken cancellationToken) + { + return Create( + options, + streamProviderName, + partition, + serviceId, + loggerFactory, + defaultComparer: null, + cancellationToken: cancellationToken); } internal static async Task> Create( @@ -94,7 +135,8 @@ internal static async Task> Create( string serviceId, ILoggerFactory loggerFactory, IComparer? defaultComparer, - string? partitionKeyPrefix = null) + string? partitionKeyPrefix = null, + CancellationToken cancellationToken = default) { var checkpointer = new AzureTableStreamQueueCheckpointer( options, @@ -104,112 +146,92 @@ internal static async Task> Create( loggerFactory, defaultComparer, partitionKeyPrefix); - await checkpointer._dataManager.InitTableAsync(); + await checkpointer._initialize(cancellationToken); return checkpointer; } /// - public async Task Load() - { - var result = await _dataManager.ReadSingleTableEntryAsync(_entity.PartitionKey, _entity.RowKey); - var checkpoint = result.Entity?.Offset ?? string.Empty; - lock (_lock) - { - if (result.Entity is not null) - { - _entity = result.Entity; - } + public Task Load() => Load(CancellationToken.None); - _latestCheckpoint = checkpoint; - _persistedCheckpoint = checkpoint; - } - - return checkpoint; - } + /// + public Task Load(CancellationToken cancellationToken) => _inner.Load(cancellationToken); /// public void Update(string offset, DateTime utcNow) + => Update(offset, utcNow, CancellationToken.None); + + /// + public void Update(string offset, DateTime utcNow, CancellationToken cancellationToken) + => _inner.Update(offset, utcNow, cancellationToken); + + /// + public Task FlushAsync(CancellationToken cancellationToken) + => _inner.FlushAsync(cancellationToken); + + private sealed class AzureTableCheckpointStore( + AzureTableDataManager dataManager, + StreamQueueCheckpointEntity entity) : IStreamCheckpointStore { - ArgumentNullException.ThrowIfNull(offset); + public StreamQueueCheckpointEntity Entity { get; private set; } = entity; - lock (_lock) - { - if (string.Equals(_latestCheckpoint, offset, StringComparison.Ordinal) - || (_checkpointComparer is { } comparer - && !string.IsNullOrEmpty(_latestCheckpoint) - && comparer.Compare(offset, _latestCheckpoint) <= 0)) - { - return; - } + public Task Initialize(CancellationToken cancellationToken) + => dataManager.InitTableAsync(cancellationToken); - _latestCheckpoint = offset; - if (_throttleSavesUntilUtc.HasValue - && (_throttleSavesUntilUtc.Value > utcNow || !_inProgressSave.IsCompleted)) + public async ValueTask Load(CancellationToken cancellationToken) + { + var result = await dataManager.ReadSingleTableEntryAsync( + Entity.PartitionKey, + Entity.RowKey, + cancellationToken); + if (result.Entity is null) { - return; + return new(string.Empty, string.Empty); } - _throttleSavesUntilUtc = utcNow + _persistInterval; - _inProgressSave = Save(offset); - _inProgressSave.Ignore(); + Entity = result.Entity; + return new(Entity.Offset, result.ETag ?? Entity.ETag.ToString()); } - } - /// - public async Task FlushAsync(CancellationToken cancellationToken) - { - var retryingSave = false; - while (true) + public async ValueTask Update( + string checkpoint, + string expectedVersion, + CancellationToken cancellationToken) { - Task inProgressSave; - lock (_lock) + var updatedEntity = new StreamQueueCheckpointEntity { - inProgressSave = _inProgressSave; - } + PartitionKey = Entity.PartitionKey, + RowKey = Entity.RowKey, + Offset = checkpoint, + }; - if (retryingSave) - { - await inProgressSave.WaitAsync(cancellationToken); - } - else + string version; + if (string.IsNullOrEmpty(expectedVersion)) { - try - { - await inProgressSave.WaitAsync(cancellationToken); - } - catch (Exception) when (!cancellationToken.IsCancellationRequested) + var result = await dataManager.InsertTableEntryAsync(updatedEntity, cancellationToken); + if (!result.isSuccess) { + return await Load(cancellationToken); } - cancellationToken.ThrowIfCancellationRequested(); + version = result.eTag!; } - - lock (_lock) + else { - if (!ReferenceEquals(inProgressSave, _inProgressSave)) - { - retryingSave = false; - continue; - } - - if (string.Equals(_persistedCheckpoint, _latestCheckpoint, StringComparison.Ordinal)) + var result = await dataManager.TryUpdateTableEntryAsync( + updatedEntity, + expectedVersion, + cancellationToken); + if (!result.isSuccess) { - return; + return await Load(cancellationToken); } - _inProgressSave = Save(_latestCheckpoint); - retryingSave = true; + version = result.eTag!; } - } - } - private async Task Save(string checkpoint) - { - _entity.Offset = checkpoint; - await _dataManager.UpsertTableEntryAsync(_entity); - lock (_lock) - { - _persistedCheckpoint = checkpoint; + updatedEntity.ETag = new ETag(version); + Entity = updatedEntity; + return new(checkpoint, version); } } diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointerFactory.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointerFactory.cs index 6aae03ee714..81f470b7342 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointerFactory.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/AzureTableStreamQueueCheckpointerFactory.cs @@ -1,4 +1,5 @@ using System; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -48,14 +49,22 @@ public static IStreamQueueCheckpointerFactory CreateFactory(IServiceProvider ser } /// + [Obsolete("Use the overload which accepts a CancellationToken.")] public Task> Create(string partition) + => Create(partition, CancellationToken.None); + + /// + public Task> Create( + string partition, + CancellationToken cancellationToken) { return AzureTableStreamQueueCheckpointer.Create( _options, _providerName, partition, _clusterOptions.ServiceId.ToString(), - _loggerFactory); + _loggerFactory, + cancellationToken); } } } diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterFactory.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterFactory.cs index 41575832e86..f2db6d34e5d 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterFactory.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubAdapterFactory.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Threading.Tasks; @@ -50,7 +49,7 @@ public class EventHubAdapterFactory : IQueueAdapterFactory, IQueueAdapter, IQueu private readonly StreamCacheEvictionOptions cacheEvictionOptions; private HashRingBasedPartitionedStreamQueueMapper streamQueueMapper = null!; private string[] partitionIds = null!; - private ConcurrentDictionary receivers = null!; + private QueueAdapterReceiverRegistry receivers = null!; private EventHubProducerClient client = null!; /// @@ -101,7 +100,7 @@ public class EventHubAdapterFactory : IQueueAdapterFactory, IQueueAdapter, IQueu /// Factory to create a IEventHubReceiver /// protected Func EventHubReceiverFactory = null!; - internal ConcurrentDictionary EventHubReceivers => receivers; + internal IReadOnlyDictionary EventHubReceivers => receivers.Receivers; internal HashRingBasedPartitionedStreamQueueMapper EventHubQueueMapper => streamQueueMapper; public EventHubAdapterFactory( @@ -131,7 +130,7 @@ public EventHubAdapterFactory( public virtual void Init() { - this.receivers = new ConcurrentDictionary(); + this.receivers = new QueueAdapterReceiverRegistry(MakeReceiver); InitEventHubClient(); @@ -246,7 +245,7 @@ public IQueueCache CreateQueueCache(QueueId queueId) private EventHubAdapterReceiver GetOrCreateReceiver(QueueId queueId) { - return this.receivers.GetOrAdd(queueId, (q, instance) => instance.MakeReceiver(q), this); + return this.receivers.GetOrCreate(queueId); } protected virtual void InitEventHubClient() diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs index 61d330448b2..d258ace89e0 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubCheckpointer.cs @@ -26,8 +26,19 @@ public EventHubCheckpointerFactory(string providerName, AzureTableStreamCheckpoi } public Task> Create(string partition) + => Create(partition, CancellationToken.None); + + public Task> Create( + string partition, + CancellationToken cancellationToken) { - return EventHubCheckpointer.Create(options, providerName, partition, this.clusterOptions.ServiceId.ToString(), loggerFactory); + return EventHubCheckpointer.Create( + options, + providerName, + partition, + this.clusterOptions.ServiceId.ToString(), + loggerFactory, + cancellationToken); } public static IStreamQueueCheckpointerFactory CreateFactory(IServiceProvider services, string providerName) @@ -60,6 +71,24 @@ public class EventHubCheckpointer : IStreamQueueCheckpointer /// /// public static async Task> Create(AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, ILoggerFactory loggerFactory) + => await Create( + options, + streamProviderName, + partition, + serviceId, + loggerFactory, + CancellationToken.None); + + /// + /// Factory function that creates and initializes the checkpointer. + /// + public static async Task> Create( + AzureTableStreamCheckpointerOptions options, + string streamProviderName, + string partition, + string serviceId, + ILoggerFactory loggerFactory, + CancellationToken cancellationToken) { var inner = await AzureTableStreamQueueCheckpointer.Create( options, @@ -68,7 +97,8 @@ public static async Task> Create(AzureTableStre serviceId, loggerFactory, StreamCheckpointComparers.Numeric, - StreamQueueCheckpointEntity.EventHubPartitionKeyPrefix); + StreamQueueCheckpointEntity.EventHubPartitionKeyPrefix, + cancellationToken); return new EventHubCheckpointer(inner); } diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubQueueCache.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubQueueCache.cs index 3a1e70de18e..3ee5ff1689c 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubQueueCache.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubQueueCache.cs @@ -32,7 +32,7 @@ public partial class EventHubQueueCache : IEventHubQueueCache private readonly ILogger logger; private readonly AggregatedCachePressureMonitor cachePressureMonitor; private readonly ICacheMonitor cacheMonitor; - private FixedSizeBuffer currentBuffer = null!; + private FixedSizeBuffer? currentBuffer; /// /// EventHub queue cache. @@ -77,6 +77,10 @@ public EventHubQueueCache( public void SignalPurge() { this.evictionStrategy.PerformPurge(DateTime.UtcNow); + if (this.cache.IsEmpty) + { + this.currentBuffer = null; + } } /// diff --git a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubSequenceToken.cs b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubSequenceToken.cs index 394876082c0..20de0c5f25b 100644 --- a/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubSequenceToken.cs +++ b/src/Azure/Orleans.Streaming.EventHubs/Providers/Streams/EventHub/EventHubSequenceToken.cs @@ -3,6 +3,7 @@ using System.Globalization; using Newtonsoft.Json; using Orleans.Providers.Streams.Common; +using Orleans.Streams; namespace Orleans.Streaming.EventHubs { @@ -31,6 +32,10 @@ public interface IEventHubPartitionLocation /// indicates which application layer event this token is for, within an EventHub message. It is required for uniqueness /// and ordering of application layer events within an EventHub message. /// + /// + /// Event Hub token versions compare with each other using the Event Hubs sequence number + /// and event index. They do not compare with generic event sequence tokens. + /// [Serializable] [GenerateSerializer] public class EventHubSequenceToken : EventSequenceToken, IEventHubPartitionLocation @@ -64,6 +69,35 @@ public EventHubSequenceToken() : base() { } + /// + public override bool Equals(StreamSequenceToken? other) + { + return other is not null + && IsCompatibleEventHubToken(other) + && other.SequenceNumber == SequenceNumber + && other.EventIndex == EventIndex; + } + + /// + public override int CompareTo(StreamSequenceToken? other) + { + if (other is null) + { + return 1; + } + + if (!IsCompatibleEventHubToken(other)) + { + throw new ArgumentOutOfRangeException(nameof(other)); + } + + var difference = SequenceNumber.CompareTo(other.SequenceNumber); + return difference != 0 ? difference : EventIndex.CompareTo(other.EventIndex); + } + + /// + public override int GetHashCode() => HashCode.Combine(SequenceNumber, EventIndex); + /// Returns a string that represents the current object. /// A string that represents the current object. /// 2 @@ -71,5 +105,19 @@ public override string ToString() { return string.Format(CultureInfo.InvariantCulture, "EventHubSequenceToken(EventHubOffset: {0}, SequenceNumber: {1}, EventIndex: {2})", EventHubOffset, SequenceNumber, EventIndex); } + + private bool IsCompatibleEventHubToken(StreamSequenceToken? other) + { + if (other is null) + { + return false; + } + + var currentType = GetType(); + var otherType = other.GetType(); + return currentType == otherType + || (currentType == typeof(EventHubSequenceToken) || currentType == typeof(EventHubSequenceTokenV2)) + && (otherType == typeof(EventHubSequenceToken) || otherType == typeof(EventHubSequenceTokenV2)); + } } } diff --git a/src/Azure/Shared/Storage/AzureTableDataManager.cs b/src/Azure/Shared/Storage/AzureTableDataManager.cs index 99d42571482..8c02d0fb0ec 100644 --- a/src/Azure/Shared/Storage/AzureTableDataManager.cs +++ b/src/Azure/Shared/Storage/AzureTableDataManager.cs @@ -72,16 +72,19 @@ public AzureTableDataManager(AzureStorageOperationOptions options, ILogger logge /// Connects to, or creates and initializes a new Azure table if it does not already exist. /// /// Completion promise for this operation. - public async Task InitTableAsync() + public async Task InitTableAsync(CancellationToken cancellationToken = default) { const string operation = "InitTable"; var startTime = DateTime.UtcNow; try { - TableServiceClient tableCreationClient = await GetCloudTableCreationClientAsync(); + cancellationToken.ThrowIfCancellationRequested(); + TableServiceClient tableCreationClient = await GetCloudTableCreationClientAsync() + .AsTask() + .WaitAsync(cancellationToken); var table = tableCreationClient.GetTableClient(TableName); - var response = await table.CreateIfNotExistsAsync(); + var response = await table.CreateIfNotExistsAsync(cancellationToken); var alreadyExisted = response.GetRawResponse().Status == (int)HttpStatusCode.Conflict; LogInfoTableCreation(Logger, alreadyExisted ? "Attached to" : "Created", TableName); @@ -211,7 +214,9 @@ public async Task UpsertTableEntryAsync(T data) /// /// Data to be inserted or replaced in the table. /// Value promise with new Etag for this data entry after completing this storage operation. - public async Task<(bool isSuccess, string? eTag)> InsertTableEntryAsync(T data) + public async Task<(bool isSuccess, string? eTag)> InsertTableEntryAsync( + T data, + CancellationToken cancellationToken = default) { const string operation = "InsertTableEntry"; var startTime = DateTime.UtcNow; @@ -220,7 +225,7 @@ public async Task UpsertTableEntryAsync(T data) { try { - var opResult = await Table.AddEntityAsync(data); + var opResult = await Table.AddEntityAsync(data, cancellationToken); return (true, opResult.Headers.ETag.GetValueOrDefault().ToString()); } catch (RequestFailedException storageException) when (storageException.Status == (int)HttpStatusCode.Conflict) @@ -239,6 +244,40 @@ public async Task UpsertTableEntryAsync(T data) } } + /// + /// Conditionally replaces a data entry using an ETag. + /// + /// The replacement data. + /// The expected ETag. + /// The cancellation token. + /// The update result and the new ETag when successful. + public async Task<(bool isSuccess, string? eTag)> TryUpdateTableEntryAsync( + T data, + string dataEtag, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(data); + ArgumentNullException.ThrowIfNull(dataEtag); + + data.ETag = new ETag(dataEtag); + try + { + var response = await Table.UpdateEntityAsync( + data, + data.ETag, + TableUpdateMode.Replace, + cancellationToken); + return (true, response.Headers.ETag.GetValueOrDefault().ToString()); + } + catch (RequestFailedException exception) + when (exception.Status is (int)HttpStatusCode.NotFound + or (int)HttpStatusCode.Conflict + or (int)HttpStatusCode.PreconditionFailed) + { + return (false, null); + } + } + /// /// Merges a data entry in the Azure table. /// diff --git a/src/Orleans.Streaming/Checkpointers/GrainStreamQueueCheckpointer.cs b/src/Orleans.Streaming/Checkpointers/GrainStreamQueueCheckpointer.cs index f1ae8216b91..e1ae927ae3e 100644 --- a/src/Orleans.Streaming/Checkpointers/GrainStreamQueueCheckpointer.cs +++ b/src/Orleans.Streaming/Checkpointers/GrainStreamQueueCheckpointer.cs @@ -13,14 +13,7 @@ public class GrainStreamQueueCheckpointer : IStreamQueueCheckpointer { private const char KeySeparator = '-'; private const string StorageProviderKeyPrefix = "__orleans_storage_provider__-"; - private readonly IStreamCheckpointerGrain _grain; - private readonly GrainStreamQueueCheckpointerOptions _options; - private readonly object _lock = new(); - - private string _latestCheckpoint = string.Empty; - private string _persistedCheckpoint = string.Empty; - private Task _inProgressSave = Task.CompletedTask; - private DateTime? _throttleSavesUntilUtc; + private readonly StreamQueueCheckpointer _inner; /// /// Initializes a new instance with default options. @@ -55,21 +48,17 @@ public GrainStreamQueueCheckpointer(IStreamCheckpointerGrain grain, GrainStreamQ nameof(options)); } - _grain = grain; - _options = options; + _inner = new StreamQueueCheckpointer( + new StreamCheckpointStoreAdapter(grain), + new StreamQueueCheckpointerOptions + { + CheckpointComparer = options.CheckpointComparer, + PersistInterval = options.PersistInterval, + }); } /// - public bool CheckpointExists - { - get - { - lock (_lock) - { - return !string.IsNullOrEmpty(_latestCheckpoint); - } - } - } + public bool CheckpointExists => _inner.CheckpointExists; /// /// Creates and initializes a grain-based checkpointer with default options. @@ -182,17 +171,7 @@ internal static string GetConfiguredStorageProviderName(ReadOnlySpan grain public Task Load() => Load(CancellationToken.None); /// - public async Task Load(CancellationToken cancellationToken) - { - var checkpoint = await _grain.Load(cancellationToken); - lock (_lock) - { - _latestCheckpoint = checkpoint; - _persistedCheckpoint = checkpoint; - } - - return checkpoint; - } + public Task Load(CancellationToken cancellationToken) => _inner.Load(cancellationToken); /// [Obsolete("Use the overload which accepts a CancellationToken.")] @@ -201,121 +180,30 @@ public void Update(string offset, DateTime utcNow) /// public void Update(string offset, DateTime utcNow, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(offset); - cancellationToken.ThrowIfCancellationRequested(); - - lock (_lock) - { - if (string.Equals(_latestCheckpoint, offset, StringComparison.Ordinal) - || (_options.CheckpointComparer is { } comparer - && !string.IsNullOrEmpty(_latestCheckpoint) - && comparer.Compare(offset, _latestCheckpoint) <= 0)) - { - return; - } - - _latestCheckpoint = offset; - if (_throttleSavesUntilUtc.HasValue && (_throttleSavesUntilUtc.Value > utcNow || !_inProgressSave.IsCompleted)) - { - return; - } - - _throttleSavesUntilUtc = utcNow + _options.PersistInterval; - _inProgressSave = Save(offset, cancellationToken); - _inProgressSave.Ignore(); - } - } + => _inner.Update(offset, utcNow, cancellationToken); /// - public async Task FlushAsync(CancellationToken cancellationToken) - { - var retryingSave = false; - while (true) - { - Task inProgressSave; - lock (_lock) - { - inProgressSave = _inProgressSave; - } - - if (retryingSave) - { - await inProgressSave.WaitAsync(cancellationToken); - } - else - { - try - { - await inProgressSave.WaitAsync(cancellationToken); - } - catch (Exception) when (!cancellationToken.IsCancellationRequested) - { - } - - cancellationToken.ThrowIfCancellationRequested(); - } - - lock (_lock) - { - if (!ReferenceEquals(inProgressSave, _inProgressSave)) - { - retryingSave = false; - continue; - } + public Task FlushAsync(CancellationToken cancellationToken) + => _inner.FlushAsync(cancellationToken); - if (string.Equals(_persistedCheckpoint, _latestCheckpoint, StringComparison.Ordinal)) - { - return; - } - - _inProgressSave = Save(_latestCheckpoint, cancellationToken); - retryingSave = true; - } - } - } - - private async Task Save(string checkpoint, CancellationToken cancellationToken) + private sealed class StreamCheckpointStoreAdapter(IStreamCheckpointerGrain grain) : IStreamCheckpointStore { - string expectedCheckpoint; - lock (_lock) + public async ValueTask Load(CancellationToken cancellationToken) { - expectedCheckpoint = _persistedCheckpoint; + var checkpoint = await grain.Load(cancellationToken).ConfigureAwait(false); + return new(checkpoint, checkpoint); } - while (true) + public async ValueTask Update( + string checkpoint, + string expectedVersion, + CancellationToken cancellationToken) { - var persistedCheckpoint = await _grain.Update( + var persistedCheckpoint = await grain.Update( checkpoint, - expectedCheckpoint, - cancellationToken); - - lock (_lock) - { - _persistedCheckpoint = persistedCheckpoint; - if (string.Equals(persistedCheckpoint, checkpoint, StringComparison.Ordinal)) - { - return; - } - - if (_options.CheckpointComparer is not { } comparer) - { - _latestCheckpoint = persistedCheckpoint; - return; - } - - if (comparer.Compare(_latestCheckpoint, persistedCheckpoint) <= 0) - { - _latestCheckpoint = persistedCheckpoint; - } - - if (comparer.Compare(checkpoint, persistedCheckpoint) <= 0) - { - return; - } - - expectedCheckpoint = persistedCheckpoint; - } + expectedVersion, + cancellationToken).ConfigureAwait(false); + return new(persistedCheckpoint, persistedCheckpoint); } } } diff --git a/src/Orleans.Streaming/Checkpointers/IStreamCheckpointStore.cs b/src/Orleans.Streaming/Checkpointers/IStreamCheckpointStore.cs new file mode 100644 index 00000000000..044586812bd --- /dev/null +++ b/src/Orleans.Streaming/Checkpointers/IStreamCheckpointStore.cs @@ -0,0 +1,33 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace Orleans.Streams +{ + /// + /// Stores a persistent stream checkpoint using conditional updates. + /// + public interface IStreamCheckpointStore + { + /// + /// Loads the current checkpoint and its version. + /// + /// The cancellation token. + /// The current checkpoint state. + ValueTask Load(CancellationToken cancellationToken); + + /// + /// Updates the checkpoint if matches the persisted version. + /// + /// The checkpoint to persist. + /// The expected persisted version. + /// The cancellation token. + /// + /// The persisted state after the update attempt. If the expected version did not match, + /// the returned state contains the conflicting persisted checkpoint and version. + /// + ValueTask Update( + string checkpoint, + string expectedVersion, + CancellationToken cancellationToken); + } +} diff --git a/src/Orleans.Streaming/Checkpointers/StreamCheckpointStoreState.cs b/src/Orleans.Streaming/Checkpointers/StreamCheckpointStoreState.cs new file mode 100644 index 00000000000..6ba1824f0ad --- /dev/null +++ b/src/Orleans.Streaming/Checkpointers/StreamCheckpointStoreState.cs @@ -0,0 +1,33 @@ +using System; + +namespace Orleans.Streams +{ + /// + /// Represents a persisted stream checkpoint and its backend version. + /// + public readonly struct StreamCheckpointStoreState + { + /// + /// Initializes a new instance of the struct. + /// + /// The checkpoint value. + /// The backend version or entity tag. + public StreamCheckpointStoreState(string checkpoint, string version) + { + ArgumentNullException.ThrowIfNull(checkpoint); + ArgumentNullException.ThrowIfNull(version); + Checkpoint = checkpoint; + Version = version; + } + + /// + /// Gets the checkpoint value. + /// + public string Checkpoint { get; } + + /// + /// Gets the backend version or entity tag. + /// + public string Version { get; } + } +} diff --git a/src/Orleans.Streaming/Checkpointers/StreamQueueCheckpointer.cs b/src/Orleans.Streaming/Checkpointers/StreamQueueCheckpointer.cs new file mode 100644 index 00000000000..9af04ec9eb1 --- /dev/null +++ b/src/Orleans.Streaming/Checkpointers/StreamQueueCheckpointer.cs @@ -0,0 +1,218 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Orleans.Streams +{ + /// + /// Coalesces and persists stream queue checkpoints using an . + /// + public sealed class StreamQueueCheckpointer : IStreamQueueCheckpointer + { + private readonly IStreamCheckpointStore _store; + private readonly StreamQueueCheckpointerOptions _options; + private readonly object _lock = new(); + + private string _latestCheckpoint = string.Empty; + private StreamCheckpointStoreState _persistedState = new(string.Empty, string.Empty); + private Task _inProgressSave = Task.CompletedTask; + private DateTime? _throttleSavesUntilUtc; + + /// + /// Initializes a new instance of the class. + /// + /// The checkpoint store. + /// The checkpointer options. + public StreamQueueCheckpointer(IStreamCheckpointStore store, StreamQueueCheckpointerOptions options) + { + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(options); + if (options.PersistInterval <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException( + nameof(options), + options.PersistInterval, + $"{nameof(StreamQueueCheckpointerOptions.PersistInterval)} must be greater than zero."); + } + + _store = store; + _options = options; + } + + /// + public bool CheckpointExists + { + get + { + lock (_lock) + { + return !string.IsNullOrEmpty(_latestCheckpoint); + } + } + } + + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task Load() => Load(CancellationToken.None); + + /// + public async Task Load(CancellationToken cancellationToken) + { + var state = await _store.Load(cancellationToken); + lock (_lock) + { + _latestCheckpoint = state.Checkpoint; + _persistedState = state; + } + + return state.Checkpoint; + } + + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public void Update(string offset, DateTime utcNow) + => Update(offset, utcNow, CancellationToken.None); + + /// + public void Update(string offset, DateTime utcNow, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(offset); + cancellationToken.ThrowIfCancellationRequested(); + + lock (_lock) + { + if (string.Equals(_latestCheckpoint, offset, StringComparison.Ordinal)) + { + if (string.Equals(_persistedState.Checkpoint, offset, StringComparison.Ordinal) + || !_inProgressSave.IsCompleted + || (_throttleSavesUntilUtc.HasValue && _throttleSavesUntilUtc.Value > utcNow)) + { + return; + } + + _throttleSavesUntilUtc = utcNow + _options.PersistInterval; + _inProgressSave = Save(offset, cancellationToken); + _inProgressSave.Ignore(); + return; + } + + if (_options.CheckpointComparer is { } comparer + && !string.IsNullOrEmpty(_latestCheckpoint) + && comparer.Compare(offset, _latestCheckpoint) <= 0) + { + return; + } + + _latestCheckpoint = offset; + if (_throttleSavesUntilUtc.HasValue + && (_throttleSavesUntilUtc.Value > utcNow || !_inProgressSave.IsCompleted)) + { + return; + } + + _throttleSavesUntilUtc = utcNow + _options.PersistInterval; + _inProgressSave = Save(offset, cancellationToken); + _inProgressSave.Ignore(); + } + } + + /// + public async Task FlushAsync(CancellationToken cancellationToken) + { + var retryingSave = false; + while (true) + { + Task inProgressSave; + lock (_lock) + { + inProgressSave = _inProgressSave; + } + + if (retryingSave) + { + await inProgressSave.WaitAsync(cancellationToken); + } + else + { + try + { + await inProgressSave.WaitAsync(cancellationToken); + } + catch (Exception) when (!cancellationToken.IsCancellationRequested) + { + } + + cancellationToken.ThrowIfCancellationRequested(); + } + + lock (_lock) + { + if (!ReferenceEquals(inProgressSave, _inProgressSave)) + { + retryingSave = false; + continue; + } + + if (string.Equals(_persistedState.Checkpoint, _latestCheckpoint, StringComparison.Ordinal)) + { + return; + } + + _inProgressSave = Save(_latestCheckpoint, cancellationToken); + retryingSave = true; + } + } + } + + private async Task Save(string checkpoint, CancellationToken cancellationToken) + { + string expectedVersion; + lock (_lock) + { + expectedVersion = _persistedState.Version; + } + + while (true) + { + var persistedState = await _store.Update(checkpoint, expectedVersion, cancellationToken); + + lock (_lock) + { + _persistedState = persistedState; + if (string.Equals(persistedState.Checkpoint, checkpoint, StringComparison.Ordinal)) + { + return; + } + + if (_options.CheckpointComparer is not { } comparer) + { + _latestCheckpoint = persistedState.Checkpoint; + return; + } + + if (Compare(comparer, _latestCheckpoint, persistedState.Checkpoint) <= 0) + { + _latestCheckpoint = persistedState.Checkpoint; + } + + if (Compare(comparer, checkpoint, persistedState.Checkpoint) <= 0) + { + return; + } + + expectedVersion = persistedState.Version; + } + } + } + + private static int Compare(IComparer comparer, string left, string right) + { + if (string.IsNullOrEmpty(left)) + { + return string.IsNullOrEmpty(right) ? 0 : -1; + } + + return string.IsNullOrEmpty(right) ? 1 : comparer.Compare(left, right); + } + } +} diff --git a/src/Orleans.Streaming/Checkpointers/StreamQueueCheckpointerOptions.cs b/src/Orleans.Streaming/Checkpointers/StreamQueueCheckpointerOptions.cs new file mode 100644 index 00000000000..f63ee55ceb0 --- /dev/null +++ b/src/Orleans.Streaming/Checkpointers/StreamQueueCheckpointerOptions.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; + +namespace Orleans.Streams +{ + /// + /// Configures a . + /// + public sealed class StreamQueueCheckpointerOptions + { + /// + /// Gets or sets the minimum interval between checkpoint writes. + /// + public TimeSpan PersistInterval { get; set; } = TimeSpan.FromMinutes(1); + + /// + /// Gets or sets the comparer used to prevent a checkpoint from moving backwards. + /// + /// + /// When this property is , checkpoints are assumed to arrive in increasing order. + /// + public IComparer? CheckpointComparer { get; set; } + } +} diff --git a/src/Orleans.Streaming/Common/EventSequenceToken.cs b/src/Orleans.Streaming/Common/EventSequenceToken.cs index b2d029e604a..ac2e9442fec 100644 --- a/src/Orleans.Streaming/Common/EventSequenceToken.cs +++ b/src/Orleans.Streaming/Common/EventSequenceToken.cs @@ -8,6 +8,11 @@ namespace Orleans.Providers.Streams.Common /// /// Stream sequence token that tracks sequence number and event index /// + /// + /// The exact and + /// types compare across versions. Derived tokens compare only with the same concrete + /// runtime type unless they override equality, ordering, and hashing together. + /// [Serializable] [GenerateSerializer] public class EventSequenceToken : StreamSequenceToken @@ -61,24 +66,27 @@ public EventSequenceToken() /// Creates a sequence token for a specific event in the current batch. /// /// The event index, for events which are part of a batch. - /// The sequence token. - public EventSequenceToken CreateSequenceTokenForEvent(int eventInd) + /// A token with the same concrete runtime type and position metadata, targeting the specified event. + public virtual EventSequenceToken CreateSequenceTokenForEvent(int eventInd) { - return new EventSequenceToken(SequenceNumber, eventInd); + var result = (EventSequenceToken)MemberwiseClone(); + result.EventIndex = eventInd; + return result; } /// public override bool Equals(object? obj) { - return Equals(obj as EventSequenceToken); + return obj is StreamSequenceToken token && Equals(token); } /// public override bool Equals(StreamSequenceToken? other) { - var token = other as EventSequenceToken; - return token != null && (token.SequenceNumber == SequenceNumber && - token.EventIndex == EventIndex); + return other is not null + && IsCompatibleLegacyToken(other) + && other.SequenceNumber == SequenceNumber + && other.EventIndex == EventIndex; } /// @@ -87,12 +95,11 @@ public override int CompareTo(StreamSequenceToken? other) if (other == null) return 1; - var token = other as EventSequenceToken; - if (token == null) + if (!IsCompatibleLegacyToken(other)) throw new ArgumentOutOfRangeException(nameof(other)); - - int difference = SequenceNumber.CompareTo(token.SequenceNumber); - return difference != 0 ? difference : EventIndex.CompareTo(token.EventIndex); + + int difference = SequenceNumber.CompareTo(other.SequenceNumber); + return difference != 0 ? difference : EventIndex.CompareTo(other.EventIndex); } /// @@ -103,5 +110,19 @@ public override string ToString() { return string.Format(CultureInfo.InvariantCulture, "[EventSequenceToken: SeqNum={0}, EventIndex={1}]", SequenceNumber, EventIndex); } + + private bool IsCompatibleLegacyToken(StreamSequenceToken? other) + { + if (other is null) + { + return false; + } + + var currentType = GetType(); + var otherType = other.GetType(); + return currentType == otherType + || currentType == typeof(EventSequenceToken) + && otherType == typeof(EventSequenceTokenV2); + } } } diff --git a/src/Orleans.Streaming/Common/EventSequenceTokenV2.cs b/src/Orleans.Streaming/Common/EventSequenceTokenV2.cs index db025a17992..40e6a297953 100644 --- a/src/Orleans.Streaming/Common/EventSequenceTokenV2.cs +++ b/src/Orleans.Streaming/Common/EventSequenceTokenV2.cs @@ -8,6 +8,11 @@ namespace Orleans.Providers.Streams.Common /// /// Stream sequence token that tracks sequence number and event index /// + /// + /// The exact and + /// types compare across versions. Derived tokens compare only with the same concrete + /// runtime type unless they override equality, ordering, and hashing together. + /// [Serializable] [GenerateSerializer] public class EventSequenceTokenV2 : StreamSequenceToken @@ -61,24 +66,27 @@ public EventSequenceTokenV2() /// Creates a sequence token for a specific event in the current batch /// /// The event index. - /// A new sequence token. - public EventSequenceTokenV2 CreateSequenceTokenForEvent(int eventInd) + /// A token with the same concrete runtime type and position metadata, targeting the specified event. + public virtual EventSequenceTokenV2 CreateSequenceTokenForEvent(int eventInd) { - return new EventSequenceTokenV2(SequenceNumber, eventInd); + var result = (EventSequenceTokenV2)MemberwiseClone(); + result.EventIndex = eventInd; + return result; } /// public override bool Equals(object? obj) { - return Equals(obj as EventSequenceTokenV2); + return obj is StreamSequenceToken token && Equals(token); } /// public override bool Equals(StreamSequenceToken? other) { - var token = other as EventSequenceTokenV2; - return token != null && (token.SequenceNumber == SequenceNumber && - token.EventIndex == EventIndex); + return other is not null + && IsCompatibleLegacyToken(other) + && other.SequenceNumber == SequenceNumber + && other.EventIndex == EventIndex; } /// @@ -87,12 +95,11 @@ public override int CompareTo(StreamSequenceToken? other) if (other == null) return 1; - var token = other as EventSequenceTokenV2; - if (token == null) + if (!IsCompatibleLegacyToken(other)) throw new ArgumentOutOfRangeException(nameof(other)); - int difference = SequenceNumber.CompareTo(token.SequenceNumber); - return difference != 0 ? difference : EventIndex.CompareTo(token.EventIndex); + int difference = SequenceNumber.CompareTo(other.SequenceNumber); + return difference != 0 ? difference : EventIndex.CompareTo(other.EventIndex); } /// @@ -103,5 +110,19 @@ public override string ToString() { return string.Format(CultureInfo.InvariantCulture, "[EventSequenceTokenV2: SeqNum={0}, EventIndex={1}]", SequenceNumber, EventIndex); } + + private bool IsCompatibleLegacyToken(StreamSequenceToken? other) + { + if (other is null) + { + return false; + } + + var currentType = GetType(); + var otherType = other.GetType(); + return currentType == otherType + || currentType == typeof(EventSequenceTokenV2) + && otherType == typeof(EventSequenceToken); + } } } diff --git a/src/Orleans.Streaming/Common/PooledCache/CachedMessageBlock.cs b/src/Orleans.Streaming/Common/PooledCache/CachedMessageBlock.cs index b7522bcf8fc..e9f6d2fc2e7 100644 --- a/src/Orleans.Streaming/Common/PooledCache/CachedMessageBlock.cs +++ b/src/Orleans.Streaming/Common/PooledCache/CachedMessageBlock.cs @@ -164,10 +164,10 @@ public StreamSequenceToken GetOldestSequenceToken(ICacheDataAdapter dataAdapter) } /// - /// Gets the index of the first message in this block that has a sequence token at or before the provided token + /// Gets the index of the newest message in this block whose sequence token is less than or equal to the provided token. /// /// The sequence token. - /// The index of the first message in this block that has a sequence token equal to or before the provided token. + /// The index of the newest message whose sequence token is less than or equal to the provided token. public int GetIndexOfFirstMessageLessThanOrEqualTo(StreamSequenceToken token) { for (int i = writeIndex - 1; i >= readIndex; i--) @@ -180,6 +180,28 @@ public int GetIndexOfFirstMessageLessThanOrEqualTo(StreamSequenceToken token) throw new ArgumentOutOfRangeException(nameof(token)); } + /// + /// Gets the index of the newest message in this block whose sequence token is less than or equal to the provided token. + /// + /// The sequence token. + /// The data adapter used to compare provider-specific positions. + /// The index of the newest message whose sequence token is less than or equal to the provided token. + public int GetIndexOfFirstMessageLessThanOrEqualTo( + StreamSequenceToken token, + ICacheDataAdapter dataAdapter) + { + ArgumentNullException.ThrowIfNull(dataAdapter); + for (int i = writeIndex - 1; i >= readIndex; i--) + { + if (dataAdapter.Compare(ref cachedMessages[i], token) <= 0) + { + return i; + } + } + + throw new ArgumentOutOfRangeException(nameof(token)); + } + /// /// Tries to find the first message in the block that is part of the provided stream. /// diff --git a/src/Orleans.Streaming/Common/PooledCache/ChronologicalEvictionStrategy.cs b/src/Orleans.Streaming/Common/PooledCache/ChronologicalEvictionStrategy.cs index ca25720ab79..1181483d9a3 100644 --- a/src/Orleans.Streaming/Common/PooledCache/ChronologicalEvictionStrategy.cs +++ b/src/Orleans.Streaming/Common/PooledCache/ChronologicalEvictionStrategy.cs @@ -112,7 +112,17 @@ private void PerformPurgeInternal(DateTime nowUtc) if (itemsPurged == 0) return; - //items got purged, time to conduct follow up actions + OnPurgeCompleted(lastMessagePurged, itemsPurged); + } + + /// + public void OnPurgeCompleted(CachedMessage? lastMessagePurged, int itemsPurged) + { + if (itemsPurged <= 0) + { + return; + } + this.cacheMonitor?.TrackMessagesPurged(itemsPurged); OnPurged?.Invoke(lastMessagePurged, this.PurgeObservable.Newest); FreePurgedBuffers(lastMessagePurged, this.PurgeObservable.Oldest); @@ -127,20 +137,33 @@ private void FreePurgedBuffers(CachedMessage? lastMessagePurged, CachedMessage? object? IdOfLastPurgedBufferId = lastMessagePurged?.Segment.Array; // IdOfLastBufferInCache will be null if cache is empty after purge object? IdOfLastBufferInCacheId = oldestMessageInCache?.Segment.Array; - //all buffers older than LastPurgedBuffer should be purged - while (this.inUseBuffers.Peek().Id != IdOfLastPurgedBufferId) + if (IdOfLastBufferInCacheId is null) { - var purgedBuffer = this.inUseBuffers.Dequeue(); - memoryReleasedInByte += purgedBuffer.SizeInByte; - purgedBuffer.Dispose(); + while (this.inUseBuffers.Count > 0) + { + var purgedBuffer = this.inUseBuffers.Dequeue(); + memoryReleasedInByte += purgedBuffer.SizeInByte; + purgedBuffer.Dispose(); + } } - // if last purged message does not share buffer with remaining messages in cache and cache is not empty - //then last purged buffer should be purged too - if (IdOfLastBufferInCacheId != null && IdOfLastPurgedBufferId != IdOfLastBufferInCacheId) + else { - var purgedBuffer = this.inUseBuffers.Dequeue(); - memoryReleasedInByte += purgedBuffer.SizeInByte; - purgedBuffer.Dispose(); + // All buffers older than the last purged buffer can be returned. + while (this.inUseBuffers.Peek().Id != IdOfLastPurgedBufferId) + { + var purgedBuffer = this.inUseBuffers.Dequeue(); + memoryReleasedInByte += purgedBuffer.SizeInByte; + purgedBuffer.Dispose(); + } + + // If the last purged message does not share a buffer with the oldest remaining message, + // the last purged buffer can also be returned. + if (IdOfLastPurgedBufferId != IdOfLastBufferInCacheId) + { + var purgedBuffer = this.inUseBuffers.Dequeue(); + memoryReleasedInByte += purgedBuffer.SizeInByte; + purgedBuffer.Dispose(); + } } //report metrics if (memoryReleasedInByte > 0) diff --git a/src/Orleans.Streaming/Common/PooledCache/FixedSizeBuffer.cs b/src/Orleans.Streaming/Common/PooledCache/FixedSizeBuffer.cs index 0bed302edd7..4cc4db692fb 100644 --- a/src/Orleans.Streaming/Common/PooledCache/FixedSizeBuffer.cs +++ b/src/Orleans.Streaming/Common/PooledCache/FixedSizeBuffer.cs @@ -22,6 +22,8 @@ public class FixedSizeBuffer : PooledResource /// public object Id => buffer; + internal int Position => count; + /// /// Manages access to a fixed size byte buffer. /// @@ -56,6 +58,13 @@ public bool TryGetSegment(int size, out ArraySegment value) return true; } + internal void ResetTo(int position) + { + ArgumentOutOfRangeException.ThrowIfNegative(position); + ArgumentOutOfRangeException.ThrowIfGreaterThan(position, count); + count = position; + } + /// public override void OnResetState() { diff --git a/src/Orleans.Streaming/Common/PooledCache/ICacheDataAdapter.cs b/src/Orleans.Streaming/Common/PooledCache/ICacheDataAdapter.cs index d5c499dafc5..a14e50c57b9 100644 --- a/src/Orleans.Streaming/Common/PooledCache/ICacheDataAdapter.cs +++ b/src/Orleans.Streaming/Common/PooledCache/ICacheDataAdapter.cs @@ -23,5 +23,18 @@ public interface ICacheDataAdapter /// The cached message. /// The sequence token. StreamSequenceToken GetSequenceToken(ref CachedMessage cachedMessage); + + /// + /// Compares a cached message with a stream sequence token. + /// + /// The cached message. + /// The sequence token. + /// A value indicating the relative order of the cached message and token. + /// + /// The default implementation uses the allocation-free sequence number and event index fields. + /// Providers with external offsets can override this method and compare encoded offset data directly. + /// + int Compare(ref CachedMessage cachedMessage, StreamSequenceToken token) + => cachedMessage.Compare(token); } } diff --git a/src/Orleans.Streaming/Common/PooledCache/IEvictionStrategy.cs b/src/Orleans.Streaming/Common/PooledCache/IEvictionStrategy.cs index 711bdab01e3..f2cad39198c 100644 --- a/src/Orleans.Streaming/Common/PooledCache/IEvictionStrategy.cs +++ b/src/Orleans.Streaming/Common/PooledCache/IEvictionStrategy.cs @@ -28,6 +28,14 @@ public interface IEvictionStrategy /// /// The new block. void OnBlockAllocated(FixedSizeBuffer newBlock); + + /// + /// Performs follow-up accounting after the owner removes messages directly. + /// + /// The last message removed. + /// The number of messages removed. + void OnPurgeCompleted(CachedMessage? lastMessagePurged, int itemsPurged) + => OnPurged?.Invoke(lastMessagePurged, null); } /// diff --git a/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs b/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs index 284393ad880..52aecbc82fd 100644 --- a/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs +++ b/src/Orleans.Streaming/Common/PooledCache/PooledQueueCache.cs @@ -202,28 +202,41 @@ private void TrackAndPurgeMetadata(CachedMessage messageToRemove) private void SetCursor(Cursor cursor, StreamSequenceToken? sequenceToken) { // If nothing in cache, unset token, and wait for more data. - if (messageBlocks.Count == 0) + if (IsEmpty) { - cursor.State = CursorStates.Unset; + cursor.State = sequenceToken is null ? CursorStates.Idle : CursorStates.Unset; cursor.SequenceToken = sequenceToken; return; } LinkedListNode newestBlock = messageBlocks.First!; // messageBlocks.Count != 0 (checked above). - // if sequenceToken is null, iterate from newest message in cache + // A cursor which waited on an empty cache starts at the oldest message; + // otherwise, a null token starts at the newest message. if (sequenceToken == null) { - cursor.State = CursorStates.Idle; - cursor.CurrentBlock = newestBlock; - cursor.Index = newestBlock.Value.NewestMessageIndex; - cursor.SequenceToken = newestBlock.Value.GetNewestSequenceToken(cacheDataAdapter); + if (cursor.State == CursorStates.Idle) + { + var waitingOldestBlock = messageBlocks.Last!; + cursor.State = CursorStates.Set; + cursor.CurrentBlock = waitingOldestBlock; + cursor.Index = waitingOldestBlock.Value.OldestMessageIndex; + cursor.SequenceToken = waitingOldestBlock.Value.GetOldestSequenceToken(cacheDataAdapter); + } + else + { + cursor.State = CursorStates.Idle; + cursor.CurrentBlock = newestBlock; + cursor.Index = newestBlock.Value.NewestMessageIndex; + cursor.SequenceToken = newestBlock.Value.GetNewestSequenceToken(cacheDataAdapter); + } + return; } // If sequenceToken is too new to be in cache, unset token, and wait for more data. CachedMessage newestMessage = newestBlock.Value.NewestMessage; - if (newestMessage.Compare(sequenceToken) < 0) + if (cacheDataAdapter.Compare(ref newestMessage, sequenceToken) < 0) { cursor.State = CursorStates.Unset; cursor.SequenceToken = sequenceToken; @@ -233,7 +246,7 @@ private void SetCursor(Cursor cursor, StreamSequenceToken? sequenceToken) // Check to see if sequenceToken is too old to be in cache var oldestBlock = messageBlocks.Last!; // messageBlocks.Count != 0 (checked above). var oldestMessage = oldestBlock.Value.OldestMessage; - if (oldestMessage.Compare(sequenceToken) > 0) + if (cacheDataAdapter.Compare(ref oldestMessage, sequenceToken) > 0) { // Check if we missed an event since we last purged the cache if (this.lastPurgedToken.TryGetValue(cursor.StreamId, out var entry) && sequenceToken.CompareTo(entry.Token) >= 0) @@ -258,7 +271,7 @@ private void SetCursor(Cursor cursor, StreamSequenceToken? sequenceToken) while (true) { CachedMessage oldestMessageInBlock = node!.Value.OldestMessage; // Loop invariant: node is non-null while the search has not exhausted the cache (guaranteed by the bounds checks above). - if (oldestMessageInBlock.Compare(sequenceToken) <= 0) + if (cacheDataAdapter.Compare(ref oldestMessageInBlock, sequenceToken) <= 0) { break; } @@ -267,7 +280,7 @@ private void SetCursor(Cursor cursor, StreamSequenceToken? sequenceToken) // return cursor from start. cursor.CurrentBlock = node; - cursor.Index = node!.Value.GetIndexOfFirstMessageLessThanOrEqualTo(sequenceToken); // See loop invariant above. + cursor.Index = node!.Value.GetIndexOfFirstMessageLessThanOrEqualTo(sequenceToken, cacheDataAdapter); // See loop invariant above. // if cursor has been idle, move to next message after message specified by sequenceToken if(cursor.State == CursorStates.Idle) { @@ -324,55 +337,93 @@ public bool TryGetNextMessage(object cursorObj, [NotNullWhen(true)] out IBatchCo // has this message been purged CachedMessage oldestMessage = messageBlocks.Last!.Value.OldestMessage; // Cursor is Set, so the cache is non-empty. - if (oldestMessage.Compare(cursor.SequenceToken!) > 0) // Cursor is Set, so SequenceToken is guaranteed non-null. + if (cacheDataAdapter.Compare(ref oldestMessage, cursor.SequenceToken!) > 0) // Cursor is Set, so SequenceToken is guaranteed non-null. { throw new QueueCacheMissException(cursor.SequenceToken!, // Cursor is Set, so SequenceToken is guaranteed non-null. messageBlocks.Last!.Value.GetOldestSequenceToken(cacheDataAdapter), // Cursor is Set, so the cache is non-empty. messageBlocks.First!.Value.GetNewestSequenceToken(cacheDataAdapter)); // Cursor is Set, so the cache is non-empty. } - // Iterate forward (in time) in the cache until we find a message on the stream or run out of cached messages. - // Note that we get the message from the current cursor location, then move it forward. This means that if we return true, the cursor - // will point to the next message after the one we're returning. + // Iterate forward in partition order. Records for other streams are safe as soon as they + // are scanned. A matching record and everything after it remain pending until its delivery + // is confirmed by the owner. while (cursor.State == CursorStates.Set) { CachedMessage currentMessage = cursor.Message; + var currentToken = cacheDataAdapter.GetSequenceToken(ref currentMessage); + MoveCursorForward(cursor, currentToken); - // Have we caught up to the newest event, if so set cursor to idle. - if (cursor.CurrentBlock == messageBlocks.First && cursor.IsNewestInBlock) - { - cursor.State = CursorStates.Idle; - cursor.SequenceToken = messageBlocks.First!.Value.GetNewestSequenceToken(cacheDataAdapter); // Just compared equal to cursor.CurrentBlock, which is non-null while cursor.State is Set. - } - else // move to next + // check if this message is in the cursor's stream + if (currentMessage.CompareStreamId(cursor.StreamId)) { - int index; - if (cursor.IsNewestInBlock) + if (cursor.DeliveredThroughToken is { } deliveredThrough + && cacheDataAdapter.Compare(ref currentMessage, deliveredThrough) <= 0) { - // cursor.CurrentBlock is non-null while cursor.State is Set. It is not messageBlocks.First here (checked above), - // so it is not the newest block in the cache, and therefore has a non-null Previous. - cursor.CurrentBlock = cursor.CurrentBlock!.Previous; - cursor.CurrentBlock!.Value.TryFindFirstMessage(cursor.StreamId, this.cacheDataAdapter, out index); + cursor.RecordScanned(currentToken); + continue; } - else - { - cursor.CurrentBlock!.Value.TryFindNextMessage(cursor.Index + 1, cursor.StreamId, this.cacheDataAdapter, out index); // Non-null while cursor.State is Set. - } - cursor.Index = index; - } - // check if this message is in the cursor's stream - if (currentMessage.CompareStreamId(cursor.StreamId)) - { + cursor.RecordPending(currentToken); message = cacheDataAdapter.GetBatchContainer(ref currentMessage); - cursor.SequenceToken = cursor.CurrentBlock!.Value.GetSequenceToken(cursor.Index, cacheDataAdapter); // Non-null while cursor.State is Set. return true; } + + cursor.RecordScanned(currentToken); } return false; } + internal StreamSequenceToken? GetSafeSequenceToken(object cursorObj) + => GetCursor(cursorObj).SafeSequenceToken; + + internal void SetCursorDeliveredThrough(object cursorObj, StreamSequenceToken token) + => GetCursor(cursorObj).DeliveredThroughToken = token; + + internal void RecordDeliverySuccess(object cursorObj) + => GetCursor(cursorObj).RecordDeliverySuccess(); + + internal void RecordDeliveryFailure(object cursorObj) + { + var cursor = GetCursor(cursorObj); + if (cursor.TakePendingStartToken() is not { } retryToken) + { + return; + } + + cursor.State = CursorStates.Unset; + cursor.CurrentBlock = null; + cursor.SequenceToken = retryToken; + SetCursor(cursor, retryToken); + } + + private Cursor GetCursor(object cursorObj) + => cursorObj as Cursor + ?? throw new ArgumentOutOfRangeException(nameof(cursorObj), "Cursor is bad"); + + private void MoveCursorForward(Cursor cursor, StreamSequenceToken currentToken) + { + if (cursor.CurrentBlock == messageBlocks.First && cursor.IsNewestInBlock) + { + cursor.State = CursorStates.Idle; + cursor.SequenceToken = currentToken; + return; + } + + if (cursor.IsNewestInBlock) + { + // The current block is not the newest block, so Previous is non-null. + cursor.CurrentBlock = cursor.CurrentBlock!.Previous; + cursor.Index = cursor.CurrentBlock!.Value.OldestMessageIndex; + } + else + { + cursor.Index++; + } + + cursor.SequenceToken = cursor.CurrentBlock!.Value.GetSequenceToken(cursor.Index, cacheDataAdapter); + } + /// /// Add a list of queue message to the cache /// @@ -437,8 +488,13 @@ public Cursor(StreamId streamId) public CursorStates State; - // current sequence token; null while Unset (no sequence token has been established yet) + // current sequence token; null while waiting for the first message to arrive public StreamSequenceToken? SequenceToken; + public StreamSequenceToken? SafeSequenceToken; + public StreamSequenceToken? DeliveredThroughToken; + private StreamSequenceToken? pendingSequenceToken; + private StreamSequenceToken? pendingStartToken; + private bool hasPendingDelivery; // reference into cache; non-null while State is Set public LinkedListNode? CurrentBlock; @@ -447,6 +503,57 @@ public Cursor(StreamId streamId) // utilities public bool IsNewestInBlock => Index == CurrentBlock!.Value.NewestMessageIndex; // Only accessed while State is Set, at which point CurrentBlock is non-null. public CachedMessage Message => CurrentBlock!.Value[Index]; // Only accessed while State is Set, at which point CurrentBlock is non-null. + + public void RecordScanned(StreamSequenceToken token) + { + if (hasPendingDelivery) + { + pendingSequenceToken = token; + } + else + { + SafeSequenceToken = token; + } + } + + public void RecordPending(StreamSequenceToken token) + { + if (!hasPendingDelivery) + { + pendingStartToken = token; + } + + hasPendingDelivery = true; + pendingSequenceToken = token; + } + + public void RecordDeliverySuccess() + { + if (!hasPendingDelivery) + { + return; + } + + SafeSequenceToken = pendingSequenceToken; + pendingSequenceToken = null; + pendingStartToken = null; + hasPendingDelivery = false; + } + + public StreamSequenceToken? TakePendingStartToken() + { + if (!hasPendingDelivery) + { + return null; + } + + var result = pendingStartToken; + pendingSequenceToken = null; + pendingStartToken = null; + hasPendingDelivery = false; + return result; + } + } } } diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamDataAdapter.cs b/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamDataAdapter.cs new file mode 100644 index 00000000000..db6c5a351d8 --- /dev/null +++ b/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamDataAdapter.cs @@ -0,0 +1,37 @@ +using System; +using Orleans.Runtime; +using Orleans.Streams; + +namespace Orleans.Providers.Streams.Common +{ + /// + /// Adapts immutable source records to and from pooled cache storage. + /// + /// The source record type. + public interface IRecoverableStreamDataAdapter : ICacheDataAdapter + { + /// + /// Gets the stream and provider position for a source record. + /// + StreamPosition GetStreamPosition(TQueueMessage queueMessage); + + /// + /// Packs a source record into pooled cache storage. + /// + CachedMessage FromQueueMessage( + StreamPosition streamPosition, + TQueueMessage queueMessage, + DateTime dequeueTimeUtc, + Func> getSegment); + + /// + /// Gets the provider offset encoded in a cached message. + /// + string GetOffset(ref CachedMessage cachedMessage); + + /// + /// Tries to extract a provider offset from a delivery token. + /// + bool TryGetOffset(StreamSequenceToken token, out string offset); + } +} diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamQueueCache.cs b/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamQueueCache.cs new file mode 100644 index 00000000000..15aa21eca78 --- /dev/null +++ b/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamQueueCache.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Orleans.Runtime; +using Orleans.Streams; + +namespace Orleans.Providers.Streams.Common +{ + /// + /// Cache contract used by . + /// + /// The immutable source record type. + public interface IRecoverableStreamQueueCache : IQueueCache, IDisposable + { + /// + /// Packs and adds ordered source records to the cache. + /// + IReadOnlyList Add( + IReadOnlyList messages, + DateTime dequeueTimeUtc); + + /// + /// Tries to get the newest cached provider position. + /// + bool TryGetNewestPosition( + [NotNullWhen(true)] out StreamSequenceToken? token, + [NotNullWhen(true)] out string? offset); + } +} diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamSource.cs b/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamSource.cs new file mode 100644 index 00000000000..97e4383fb8b --- /dev/null +++ b/src/Orleans.Streaming/Common/RecoverableStreams/IRecoverableStreamSource.cs @@ -0,0 +1,44 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Orleans.Providers.Streams.Common +{ + /// + /// Reads ordered records from a recoverable stream partition. + /// + /// The source record type. + public interface IRecoverableStreamSource + { + /// + /// Initializes the source at the requested position. + /// + /// The durable checkpoint or configured start policy. + /// The cancellation token. + /// A durable checkpoint always takes precedence and reads must begin strictly after it. + Task Initialize(RecoverableStreamStartPosition position, CancellationToken cancellationToken); + + /// + /// Reads an ordered batch of immutable source records. + /// + Task> Read(int maxCount, CancellationToken cancellationToken); + + /// + /// Notifies the source that records were successfully admitted to the cache. + /// + /// The admitted records. + /// Sources should advance volatile read offsets only in this callback. + void MessagesAdded(IReadOnlyList messages) { } + + /// + /// Notifies the source that records could not be admitted to the cache. + /// + /// The records which were not admitted. + void MessagesAddFailed(IReadOnlyList messages) { } + + /// + /// Shuts the partition source down. + /// + Task Shutdown(CancellationToken cancellationToken); + } +} diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs b/src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs new file mode 100644 index 00000000000..d3e098d450e --- /dev/null +++ b/src/Orleans.Streaming/Common/RecoverableStreams/QueueAdapterReceiverRegistry.cs @@ -0,0 +1,75 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using Orleans.Streams; + +namespace Orleans.Providers.Streams.Common +{ + /// + /// Ensures receiver and cache factories share one coordinator instance per queue. + /// + /// The combined receiver and cache type. + public sealed class QueueAdapterReceiverRegistry + where TReceiver : class, IQueueAdapterReceiver, IQueueCache + { + private readonly ConcurrentDictionary> _receivers = new(); + private readonly Func _factory; + + /// + /// Initializes a new instance of the class. + /// + public QueueAdapterReceiverRegistry(Func factory) + { + _factory = factory ?? throw new ArgumentNullException(nameof(factory)); + } + + /// + /// Gets the registered receiver instances. + /// + public IReadOnlyDictionary Receivers + => _receivers + .Where(pair => pair.Value.IsValueCreated) + .ToDictionary(pair => pair.Key, pair => pair.Value.Value); + + /// + /// Gets or creates the coordinator for a queue. + /// + public TReceiver GetOrCreate(QueueId queueId) + { + var receiver = _receivers.GetOrAdd( + queueId, + static (id, factory) => new( + () => factory(id), + LazyThreadSafetyMode.ExecutionAndPublication), + _factory); + try + { + return receiver.Value; + } + catch + { + ((ICollection>>)_receivers) + .Remove(new(queueId, receiver)); + throw; + } + } + + /// + /// Removes a receiver if it is still the registered instance for the queue. + /// + public bool Remove(QueueId queueId, TReceiver receiver) + { + if (!_receivers.TryGetValue(queueId, out var registered) + || !registered.IsValueCreated + || !ReferenceEquals(registered.Value, receiver)) + { + return false; + } + + return ((ICollection>>)_receivers) + .Remove(new(queueId, registered)); + } + } +} diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs new file mode 100644 index 00000000000..11aa7fe2492 --- /dev/null +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamQueueCache.cs @@ -0,0 +1,311 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.Logging; +using Orleans.Runtime; +using Orleans.Streams; + +namespace Orleans.Providers.Streams.Common +{ + /// + /// Provider-neutral pooled cache for immutable recoverable stream records. + /// + /// The source record type. + public sealed class RecoverableStreamQueueCache : IRecoverableStreamQueueCache + { + private readonly int _defaultMaxAddCount; + private readonly IObjectPool _bufferPool; + private readonly IRecoverableStreamDataAdapter _dataAdapter; + private readonly IEvictionStrategy _evictionStrategy; + private readonly IQueueFlowController? _flowController; + private readonly int? _maxCacheSize; + private readonly PooledQueueCache _cache; + private FixedSizeBuffer? _currentBuffer; + + /// + /// Initializes a new instance of the class. + /// + public RecoverableStreamQueueCache( + int defaultMaxAddCount, + IObjectPool bufferPool, + IRecoverableStreamDataAdapter dataAdapter, + IEvictionStrategy evictionStrategy, + ILogger logger, + IQueueFlowController? flowController = null, + ICacheMonitor? cacheMonitor = null, + TimeSpan? cacheMonitorWriteInterval = null, + TimeSpan? metadataMinTimeInCache = null, + int? maxCacheSize = null) + { + if (defaultMaxAddCount <= 0) + { + throw new ArgumentOutOfRangeException(nameof(defaultMaxAddCount)); + } + + _defaultMaxAddCount = defaultMaxAddCount; + _bufferPool = bufferPool ?? throw new ArgumentNullException(nameof(bufferPool)); + _dataAdapter = dataAdapter ?? throw new ArgumentNullException(nameof(dataAdapter)); + _evictionStrategy = evictionStrategy ?? throw new ArgumentNullException(nameof(evictionStrategy)); + _flowController = flowController; + if (maxCacheSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(maxCacheSize)); + } + + _maxCacheSize = maxCacheSize; + _cache = new PooledQueueCache( + dataAdapter, + logger ?? throw new ArgumentNullException(nameof(logger)), + cacheMonitor, + cacheMonitorWriteInterval, + metadataMinTimeInCache); + _evictionStrategy.PurgeObservable = _cache; + _evictionStrategy.OnPurged = OnPurged; + } + + /// + /// Gets the most recently purged provider offset. + /// + public string? LastPurgedOffset { get; private set; } + + /// + /// Gets the number of records currently held in the cache. + /// + public int ItemCount => _cache.ItemCount; + + /// + /// Tries to get the newest cached provider position. + /// + public bool TryGetNewestPosition( + [NotNullWhen(true)] out StreamSequenceToken? token, + [NotNullWhen(true)] out string? offset) + { + if (_cache.Newest is not { } newest) + { + token = null; + offset = null; + return false; + } + + token = _dataAdapter.GetSequenceToken(ref newest); + offset = _dataAdapter.GetOffset(ref newest); + return true; + } + + /// + /// Packs and adds ordered source records to the cache. + /// + public IReadOnlyList Add( + IReadOnlyList messages, + DateTime dequeueTimeUtc) + { + ArgumentNullException.ThrowIfNull(messages); + var positions = new List(messages.Count); + var cachedMessages = new List(messages.Count); + var allocatedBuffers = new List(); + var initialBuffer = _currentBuffer; + var initialBufferPosition = initialBuffer?.Position ?? 0; + var batchBuffer = initialBuffer; + try + { + foreach (var message in messages) + { + var position = _dataAdapter.GetStreamPosition(message); + cachedMessages.Add(_dataAdapter.FromQueueMessage(position, message, dequeueTimeUtc, GetBatchSegment)); + positions.Add(position); + } + } + catch + { + initialBuffer?.ResetTo(initialBufferPosition); + foreach (var buffer in allocatedBuffers) + { + buffer.Dispose(); + } + + throw; + } + + _cache.Add(cachedMessages, dequeueTimeUtc); + foreach (var buffer in allocatedBuffers) + { + _evictionStrategy.OnBlockAllocated(buffer); + } + + _currentBuffer = batchBuffer; + return positions; + + ArraySegment GetBatchSegment(int size) + { + if (batchBuffer is not null && batchBuffer.TryGetSegment(size, out var segment)) + { + return segment; + } + + var buffer = _bufferPool.Allocate(); + if (!buffer.TryGetSegment(size, out segment)) + { + buffer.Dispose(); + buffer = new FixedSizeBuffer(size); + _ = buffer.TryGetSegment(size, out segment); + } + + allocatedBuffers.Add(buffer); + batchBuffer = buffer; + return segment; + } + } + + /// + public int GetMaxAddCount() + { + var result = _defaultMaxAddCount; + if (_maxCacheSize is { } maxCacheSize) + { + result = Math.Min(result, Math.Max(0, maxCacheSize - _cache.ItemCount)); + } + + if (_flowController is not null) + { + var flowControlLimit = _flowController.GetMaxAddCount(); + if (flowControlLimit >= 0) + { + result = Math.Min(result, flowControlLimit); + } + } + + return result; + } + + /// + public void AddToCache(IList messages) + { + // Source records are admitted directly by the receiver coordinator. + } + + /// + public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList purgedItems) + { + purgedItems = null; + // Pressure indicates that a lagging cursor can still need the oldest records. Time-based + // eviction in that state would turn backpressure into data loss. UpdateDeliveryProgress + // removes records through the safe delivery watermark and releases pressure as consumers advance. + if (!IsUnderPressure()) + { + _evictionStrategy.PerformPurge(DateTime.UtcNow); + if (_cache.IsEmpty) + { + _currentBuffer = null; + } + } + + return false; + } + + /// + public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) + => new Cursor(_cache, streamId, token); + + /// + public bool IsUnderPressure() => GetMaxAddCount() <= 0; + + /// + public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToken, DateTime utcNow) + { + if (earliestSubscriptionToken is null) + { + return; + } + + CachedMessage? lastPurged = null; + var itemsPurged = 0; + while (_cache.Oldest is { } oldest + && _dataAdapter.Compare(ref oldest, earliestSubscriptionToken) <= 0) + { + LastPurgedOffset = _dataAdapter.GetOffset(ref oldest); + lastPurged = oldest; + itemsPurged++; + _cache.RemoveOldestMessage(); + } + + _evictionStrategy.OnPurgeCompleted(lastPurged, itemsPurged); + if (_cache.IsEmpty) + { + _currentBuffer = null; + } + } + + /// + public void Dispose() + { + CachedMessage? lastPurged = null; + var itemsPurged = 0; + while (_cache.Oldest is { } oldest) + { + lastPurged = oldest; + itemsPurged++; + _cache.RemoveOldestMessage(); + } + + _evictionStrategy.OnPurgeCompleted(lastPurged, itemsPurged); + _currentBuffer = null; + _evictionStrategy.OnPurged = null; + } + + private void OnPurged(CachedMessage? lastPurged, CachedMessage? newest) + { + if (lastPurged is { } message) + { + LastPurgedOffset = _dataAdapter.GetOffset(ref message); + } + } + + private sealed class Cursor : IQueueCacheCursor, IQueueCacheCursorProgress + { + private readonly PooledQueueCache _cache; + private readonly object _cursor; + private IBatchContainer? _current; + + public Cursor(PooledQueueCache cache, StreamId streamId, StreamSequenceToken? token) + { + _cache = cache; + _cursor = cache.GetCursor(streamId, token); + } + + public void Dispose() + { + } + + public StreamSequenceToken? SafeSequenceToken => _cache.GetSafeSequenceToken(_cursor); + + public void SetDeliveredThrough(StreamSequenceToken token) + => _cache.SetCursorDeliveredThrough(_cursor, token); + + public IBatchContainer? GetCurrent(out Exception? exception) + { + exception = null; + return _current; + } + + public bool MoveNext() + { + if (!_cache.TryGetNextMessage(_cursor, out var next)) + { + return false; + } + + _current = next; + return true; + } + + public void Refresh(StreamSequenceToken token) => _cache.Refresh(_cursor, token); + + public void RecordDeliveryFailure() + { + _cache.RecordDeliveryFailure(_cursor); + } + + public void RecordDeliverySuccess() => _cache.RecordDeliverySuccess(_cursor); + } + } +} diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs new file mode 100644 index 00000000000..c636a769ea4 --- /dev/null +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamReceiver.cs @@ -0,0 +1,326 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.ExceptionServices; +using System.Threading; +using System.Threading.Tasks; +using Orleans.Streams; + +namespace Orleans.Providers.Streams.Common +{ + /// + /// Coordinates a recoverable stream partition pipeline comprising a partition source, pooled cache, and durable checkpoint. + /// + /// The source record type. + public sealed class RecoverableStreamReceiver : IQueueAdapterReceiver, IQueueCache + { + private readonly IRecoverableStreamSource _source; + private readonly IRecoverableStreamDataAdapter _dataAdapter; + private readonly IRecoverableStreamQueueCache _cache; + private readonly IStreamQueueCheckpointer _checkpointer; + private readonly bool _startFromNow; + private readonly object _lifecycleLock = new(); + private readonly CancellationTokenSource _lifecycleCancellation = new(); + private Task? _initializeTask; + private CancellationToken _initializeTaskOwnerToken; + private int _running; + private int _shutdown; + + /// + /// Initializes a new instance of the class. + /// + public RecoverableStreamReceiver( + IRecoverableStreamSource source, + IRecoverableStreamDataAdapter dataAdapter, + RecoverableStreamQueueCache cache, + IStreamQueueCheckpointer checkpointer, + bool startFromNow) + : this(source, dataAdapter, (IRecoverableStreamQueueCache)cache, checkpointer, startFromNow) + { + } + + /// + /// Initializes a new instance of the class. + /// + public RecoverableStreamReceiver( + IRecoverableStreamSource source, + IRecoverableStreamDataAdapter dataAdapter, + IRecoverableStreamQueueCache cache, + IStreamQueueCheckpointer checkpointer, + bool startFromNow) + { + _source = source ?? throw new ArgumentNullException(nameof(source)); + _dataAdapter = dataAdapter ?? throw new ArgumentNullException(nameof(dataAdapter)); + _cache = cache ?? throw new ArgumentNullException(nameof(cache)); + _checkpointer = checkpointer ?? throw new ArgumentNullException(nameof(checkpointer)); + _startFromNow = startFromNow; + } + + /// + public async Task Initialize(TimeSpan timeout) + { + using var cancellation = timeout == Timeout.InfiniteTimeSpan + ? null + : new CancellationTokenSource(timeout); + var cancellationToken = cancellation?.Token ?? CancellationToken.None; + await EnsureInitialized(cancellationToken); + } + + /// + /// Initializes the receiver. + /// + /// The cancellation token. + public Task Initialize(CancellationToken cancellationToken) + => EnsureInitialized(cancellationToken); + + private async Task EnsureInitialized(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + while (true) + { + Task initializeTask; + CancellationToken initializeTaskOwnerToken; + lock (_lifecycleLock) + { + if (Volatile.Read(ref _running) != 0 || Volatile.Read(ref _shutdown) != 0) + { + return; + } + + if (_initializeTask is null || _initializeTask.IsCompleted) + { + _initializeTaskOwnerToken = cancellationToken; + _initializeTask = InitializeCore(cancellationToken); + } + + initializeTask = _initializeTask; + initializeTaskOwnerToken = _initializeTaskOwnerToken; + } + + try + { + await initializeTask.WaitAsync(cancellationToken); + return; + } + catch (OperationCanceledException) + when (!cancellationToken.IsCancellationRequested + && Volatile.Read(ref _shutdown) == 0 + && initializeTaskOwnerToken.IsCancellationRequested + && initializeTask.IsCanceled) + { + // The caller which started this shared initialization canceled it. Once that + // task has settled, loop and create a fresh attempt for this still-active caller. + } + } + } + + private async Task InitializeCore(CancellationToken initializationToken) + { + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource( + _lifecycleCancellation.Token, + initializationToken); + var lifecycleToken = cancellation.Token; + var checkpoint = await _checkpointer.Load(lifecycleToken); + await _source.Initialize( + new RecoverableStreamStartPosition( + _checkpointer.CheckpointExists ? checkpoint : null, + _startFromNow), + lifecycleToken); + if (Volatile.Read(ref _shutdown) != 0) + { + return; + } + + Volatile.Write(ref _running, 1); + } + + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task> GetQueueMessagesAsync(int maxCount) + => GetQueueMessagesAsync(maxCount, CancellationToken.None); + + /// + public async Task> GetQueueMessagesAsync( + int maxCount, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (Volatile.Read(ref _shutdown) != 0 || maxCount <= 0) + { + return []; + } + + await EnsureInitialized(cancellationToken); + if (Volatile.Read(ref _shutdown) != 0) + { + return []; + } + + var messages = await _source.Read(maxCount, cancellationToken); + if (messages.Count == 0) + { + return []; + } + + IReadOnlyList positions; + try + { + positions = _cache.Add(messages, DateTime.UtcNow); + _source.MessagesAdded(messages); + } + catch + { + _source.MessagesAddFailed(messages); + throw; + } + + var result = new List(positions.Count); + foreach (var position in positions) + { + result.Add(new StreamActivityNotificationBatch(position)); + } + + return result; + } + + /// + [Obsolete("Use the overload which accepts a CancellationToken.")] + public Task MessagesDeliveredAsync(IList messages) + => Task.CompletedTask; + + /// + public Task MessagesDeliveredAsync(IList messages, CancellationToken cancellationToken) + => cancellationToken.IsCancellationRequested + ? Task.FromCanceled(cancellationToken) + : Task.CompletedTask; + + /// + public async Task Shutdown(TimeSpan timeout) + { + if (Interlocked.Exchange(ref _shutdown, 1) != 0) + { + return; + } + + Volatile.Write(ref _running, 0); + _lifecycleCancellation.Cancel(); + using var cancellation = timeout == Timeout.InfiniteTimeSpan + ? null + : new CancellationTokenSource(timeout); + var cancellationToken = cancellation?.Token ?? CancellationToken.None; + List? exceptions = null; + Task? initializeTask; + lock (_lifecycleLock) + { + initializeTask = _initializeTask; + } + + if (initializeTask is not null) + { + try + { + await initializeTask.WaitAsync(cancellationToken); + } + catch (OperationCanceledException) + when (_lifecycleCancellation.IsCancellationRequested + && !cancellationToken.IsCancellationRequested) + { + } + catch (Exception exception) + { + (exceptions ??= []).Add(exception); + } + } + + try + { + await _checkpointer.FlushAsync(cancellationToken); + } + catch (Exception exception) + { + (exceptions ??= []).Add(exception); + } + + try + { + await _source.Shutdown(cancellationToken); + } + catch (Exception exception) + { + (exceptions ??= []).Add(exception); + } + + try + { + _cache.Dispose(); + } + catch (Exception exception) + { + (exceptions ??= []).Add(exception); + } + + if (exceptions is [var singleException]) + { + ExceptionDispatchInfo.Capture(singleException).Throw(); + } + + if (exceptions is { Count: > 1 }) + { + throw new AggregateException(exceptions); + } + } + + /// + public int GetMaxAddCount() => _cache.GetMaxAddCount(); + + /// + public void AddToCache(IList messages) + { + } + + /// + public bool TryPurgeFromCache([MaybeNullWhen(false)] out IList purgedItems) + => _cache.TryPurgeFromCache(out purgedItems); + + /// + public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) + => _cache.GetCacheCursor(streamId, token); + + /// + public bool IsUnderPressure() => _cache.IsUnderPressure(); + + /// + public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToken, DateTime utcNow) + { + if (Volatile.Read(ref _shutdown) != 0) + { + return; + } + + var progressToken = earliestSubscriptionToken; + string? offset = null; + if (progressToken is null) + { + _ = _cache.TryGetNewestPosition(out progressToken, out offset); + } + + _cache.UpdateDeliveryProgress(progressToken, utcNow); + if (progressToken is not null + && (offset is not null || _dataAdapter.TryGetOffset(progressToken, out offset))) + { + _checkpointer.Update(offset, utcNow, CancellationToken.None); + } + } + + private sealed class StreamActivityNotificationBatch(StreamPosition position) : IBatchContainer + { + public StreamId StreamId => position.StreamId; + + public StreamSequenceToken SequenceToken => position.SequenceToken; + + public IEnumerable> GetEvents() => throw new NotSupportedException(); + + public bool ImportRequestContext() => throw new NotSupportedException(); + } + } +} diff --git a/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamStartPosition.cs b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamStartPosition.cs new file mode 100644 index 00000000000..0b9af8d59a7 --- /dev/null +++ b/src/Orleans.Streaming/Common/RecoverableStreams/RecoverableStreamStartPosition.cs @@ -0,0 +1,29 @@ +namespace Orleans.Providers.Streams.Common +{ + /// + /// Describes where a recoverable stream source begins reading. + /// + public readonly struct RecoverableStreamStartPosition + { + /// + /// Initializes a new instance of the struct. + /// + /// The durable checkpoint, or if one does not exist. + /// Whether a source without a checkpoint starts at its current tail. + public RecoverableStreamStartPosition(string? checkpoint, bool startFromNow) + { + Checkpoint = checkpoint; + StartFromNow = startFromNow; + } + + /// + /// Gets the durable checkpoint. Sources must begin strictly after this value. + /// + public string? Checkpoint { get; } + + /// + /// Gets a value indicating whether a source without a checkpoint starts at its current tail. + /// + public bool StartFromNow { get; } + } +} diff --git a/src/Orleans.Streaming/Generator/GeneratorPooledCache.cs b/src/Orleans.Streaming/Generator/GeneratorPooledCache.cs index 5ecbc0cf662..1e5352e1aa8 100644 --- a/src/Orleans.Streaming/Generator/GeneratorPooledCache.cs +++ b/src/Orleans.Streaming/Generator/GeneratorPooledCache.cs @@ -30,11 +30,27 @@ public class GeneratorPooledCache : IQueueCache, ICacheDataAdapter /// The cache monitor. /// The monitor write interval. Only triggered for active caches public GeneratorPooledCache(IObjectPool bufferPool, ILogger logger, Serialization.Serializer serializer, ICacheMonitor? cacheMonitor, TimeSpan? monitorWriteInterval) + : this( + bufferPool, + logger, + serializer, + cacheMonitor, + monitorWriteInterval, + new TimePurgePredicate(TimeSpan.FromMinutes(5), TimeSpan.FromMinutes(10))) + { + } + + internal GeneratorPooledCache( + IObjectPool bufferPool, + ILogger logger, + Serialization.Serializer serializer, + ICacheMonitor? cacheMonitor, + TimeSpan? monitorWriteInterval, + TimePurgePredicate purgePredicate) { this.bufferPool = bufferPool; this.serializer = serializer; cache = new PooledQueueCache(this, logger, cacheMonitor, monitorWriteInterval); - TimePurgePredicate purgePredicate = new TimePurgePredicate(TimeSpan.FromMinutes(5), TimeSpan.FromMinutes(10)); this.evictionStrategy = new ChronologicalEvictionStrategy(logger, purgePredicate, cacheMonitor, monitorWriteInterval) {PurgeObservable = cache}; } @@ -170,6 +186,11 @@ public bool TryPurgeFromCache(out IList purgedItems) { purgedItems = null!; // Return value is always false, per [MaybeNullWhen(false)] on the interface. this.evictionStrategy.PerformPurge(DateTime.UtcNow); + if (cache.IsEmpty) + { + currentBuffer = null; + } + return false; } diff --git a/src/Orleans.Streaming/MemoryStreams/MemoryPooledCache.cs b/src/Orleans.Streaming/MemoryStreams/MemoryPooledCache.cs index 1d0f739e7af..8525191cb32 100644 --- a/src/Orleans.Streaming/MemoryStreams/MemoryPooledCache.cs +++ b/src/Orleans.Streaming/MemoryStreams/MemoryPooledCache.cs @@ -164,6 +164,11 @@ public bool TryPurgeFromCache(out IList purgedItems) { purgedItems = null!; // Return value is always false, per [MaybeNullWhen(false)] on the interface. this.evictionStrategy.PerformPurge(DateTime.UtcNow); + if (cache.IsEmpty) + { + currentBuffer = null; + } + return false; } diff --git a/src/Orleans.Streaming/PersistentStreams/Options/PersistentStreamProviderOptions.cs b/src/Orleans.Streaming/PersistentStreams/Options/PersistentStreamProviderOptions.cs index 0f98eef0ed7..ed73e0d3664 100644 --- a/src/Orleans.Streaming/PersistentStreams/Options/PersistentStreamProviderOptions.cs +++ b/src/Orleans.Streaming/PersistentStreams/Options/PersistentStreamProviderOptions.cs @@ -119,6 +119,16 @@ public class StreamPullingAgentOptions /// public static readonly TimeSpan DEFAULT_MAX_EVENT_DELIVERY_TIME = TimeSpan.FromMinutes(1); + /// + /// Gets or sets the period between delivery progress updates. + /// + public TimeSpan DeliveryProgressUpdateInterval { get; set; } = DEFAULT_DELIVERY_PROGRESS_UPDATE_INTERVAL; + + /// + /// The default period between delivery progress updates. + /// + public static readonly TimeSpan DEFAULT_DELIVERY_PROGRESS_UPDATE_INTERVAL = TimeSpan.FromSeconds(1); + /// /// Gets or sets the stream inactivity period. /// diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index 59e52145f17..bdd763637a2 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -42,9 +42,11 @@ internal sealed partial class PersistentStreamPullingAgent : SystemTarget, IPers private IQueueAdapterReceiver? receiver; private DateTime lastTimeCleanedPubSubCache; private IGrainTimer? timer; + private ITimer? deliveryProgressTimer; private Task? receiverInitTask; private Task _activePumpTask = Task.CompletedTask; + private int _hasUnprocessedRead; private bool IsShutdown => timer is null; private string StatisticUniquePostfix => $"{streamProviderName}.{QueueId}"; @@ -52,6 +54,8 @@ internal interface ITestAccessor { Task ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver? receiver, int maxCacheAddCount); Task RegisterStream(QualifiedStreamId streamId, StreamSequenceToken firstToken, DateTime now); + Task DoHandshakeWithConsumer(StreamConsumerData consumerData, StreamSequenceToken? cacheToken); + Task RunConsumerCursor(StreamConsumerData consumerData); Task> GetPubSubCache(); Task RunQueuePump(QueueId myQueueId, CancellationToken cancellationToken); Task Shutdown(); @@ -82,6 +86,14 @@ internal PersistentStreamPullingAgent( this.streamFilter = streamFilter; pubSubCache = new Dictionary(); this.options = options; + if (options.DeliveryProgressUpdateInterval <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException( + nameof(options.DeliveryProgressUpdateInterval), + options.DeliveryProgressUpdateInterval, + "The delivery progress update interval must be greater than zero."); + } + this.queueAdapter = queueAdapter ?? throw new ArgumentNullException(nameof(queueAdapter)); this.streamFailureHandler = streamFailureHandler ?? throw new ArgumentNullException(nameof(streamFailureHandler)); this.queueAdapterCache = queueAdapterCache; @@ -112,6 +124,12 @@ Task ITestAccessor.RegisterStream(QualifiedStreamId streamId, StreamSequenceToke return Task.CompletedTask; }).Unwrap(); + Task ITestAccessor.DoHandshakeWithConsumer(StreamConsumerData consumerData, StreamSequenceToken? cacheToken) + => this.RunOrQueueTaskResult(() => DoHandshakeWithConsumer(consumerData, cacheToken)).Unwrap(); + + Task ITestAccessor.RunConsumerCursor(StreamConsumerData consumerData) + => this.RunOrQueueTask(() => RunConsumerCursor(consumerData)); + Task> ITestAccessor.GetPubSubCache() => this.RunOrQueueTaskResult(() => (IReadOnlyDictionary)new Dictionary(pubSubCache)); @@ -136,6 +154,7 @@ public Task Initialize() LogInfoInit(GetType().Name, GrainId, Silo, new(QueueId)); _activePumpTask = Task.CompletedTask; + Volatile.Write(ref _hasUnprocessedRead, 0); lastTimeCleanedPubSubCache = _timeProvider.GetUtcNow().UtcDateTime; try @@ -182,6 +201,11 @@ public Task Initialize() // Even if the receiver failed to initialize, treat it as OK and start pumping it. It's receiver responsibility to retry initialization. var randomTimerOffset = RandomTimeSpan.Next(this.options.GetQueueMsgsTimerPeriod); timer = RegisterGrainTimer(RunQueuePump, QueueId, randomTimerOffset, this.options.GetQueueMsgsTimerPeriod); + deliveryProgressTimer = _timeProvider.CreateTimer( + static state => ((PersistentStreamPullingAgent)state!).ScheduleDeliveryProgressUpdate(), + this, + this.options.DeliveryProgressUpdateInterval, + this.options.DeliveryProgressUpdateInterval); StreamingEvents.EmitPullingAgentStarted(streamProviderName, Silo, QueueId, randomTimerOffset, this.options.GetQueueMsgsTimerPeriod); _streamInstruments?.RegisterPersistentStreamPubSubCacheSizeObserve(() => new Measurement(pubSubCache.Count, new KeyValuePair("name", StatisticUniquePostfix))); @@ -211,6 +235,9 @@ public async Task Shutdown() var asyncTimer = timer; timer = null; + var localDeliveryProgressTimer = deliveryProgressTimer; + deliveryProgressTimer = null; + localDeliveryProgressTimer?.Dispose(); if (asyncTimer is not null) { asyncTimer.Dispose(); @@ -228,7 +255,10 @@ public async Task Shutdown() // Final delivery progress scan so the receiver has the latest watermark // before FlushAsync persists the checkpoint. - NotifyDeliveryProgress(); + if (Volatile.Read(ref _hasUnprocessedRead) == 0) + { + NotifyDeliveryProgress(); + } this.queueCache = null; @@ -325,8 +355,6 @@ private async Task AddSubscriber_Impl( if (await DoHandshakeWithConsumer(data, cacheToken)) { - var startToken = data.LastToken?.Token ?? cacheToken ?? data.PendingStartToken; - data.LastProcessedToken = startToken; data.PendingStartToken = null; data.IsRegistered = true; StreamingEvents.EmitSubscriptionAttached(streamProviderName, streamId.StreamId, subscriptionId.Guid, streamConsumer, Silo); @@ -342,6 +370,8 @@ private async Task DoHandshakeWithConsumer( if (IsShutdown) return false; StreamHandshakeToken? requestedHandshakeToken = null; + var cursorStartToken = cacheToken ?? consumerData.PendingStartToken; + var cursorRepositioned = false; // if not cache, then we can't get cursor and there is no reason to ask consumer for token. if (queueCache != null) { @@ -359,23 +389,57 @@ private async Task DoHandshakeWithConsumer( var requestedToken = requestedHandshakeToken?.Token; if (requestedToken != null) { + var isDeliveryToken = requestedHandshakeToken is DeliveryToken; + cursorStartToken = isDeliveryToken + ? cacheToken ?? consumerData.PendingStartToken ?? requestedToken + : requestedToken; consumerData.SafeDisposeCursor(logger); try { - consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId, requestedToken); + var newCursor = queueCache.GetCacheCursor(consumerData.StreamId, cursorStartToken); + if (isDeliveryToken) + { + if (newCursor is IQueueCacheCursorProgress progressCursor) + { + progressCursor.SetDeliveredThrough(requestedToken); + } + else + { + if (!Equals(cursorStartToken, requestedToken)) + { + newCursor.Dispose(); + cursorStartToken = requestedToken; + newCursor = queueCache.GetCacheCursor(consumerData.StreamId, requestedToken); + } + } + } + + consumerData.Cursor = newCursor; + cursorRepositioned = true; } catch (QueueCacheMissException) when (cacheToken is not null) { // A cold stream's triggering batch is the receiver's first available // message, so resume there if the consumer's prior token was evicted. + cursorStartToken = cacheToken; consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId, cacheToken); + if (requestedHandshakeToken is DeliveryToken + && consumerData.Cursor is IQueueCacheCursorProgress progressCursor) + { + progressCursor.SetDeliveredThrough(requestedToken); + } + + cursorRepositioned = true; } } else { var registrationToken = cacheToken ?? consumerData.PendingStartToken; if (consumerData.Cursor == null) // if the consumer did not ask for a specific token and we already have a cursor, just keep using it. + { consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId, registrationToken); + cursorRepositioned = true; + } } } catch (Exception exception) @@ -398,13 +462,35 @@ private async Task DoHandshakeWithConsumer( try { var registrationToken = cacheToken ?? consumerData.PendingStartToken; + cursorStartToken = registrationToken; consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId, registrationToken); + cursorRepositioned = true; } catch (Exception) { consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId, null); // just in case last GetCacheCursor failed. + cursorStartToken = null; + cursorRepositioned = true; + } + } + + if (cursorRepositioned) + { + consumerData.CursorStartToken = cursorStartToken; + if (requestedHandshakeToken is DeliveryToken deliveryToken) + { + consumerData.LastProcessedToken = deliveryToken.Token; + consumerData.LastSafePartitionToken = null; + } + else + { + // Start/cache/pending tokens are inclusive positions. They become safe only + // after the matching record is delivered or intentionally filtered. + consumerData.LastProcessedToken = null; + consumerData.LastSafePartitionToken = null; } } + return true; } @@ -573,6 +659,11 @@ private async Task ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver? if (IsShutdown || cancellationToken.IsCancellationRequested) { + if (multiBatch is { Count: > 0 }) + { + Volatile.Write(ref _hasUnprocessedRead, 1); + } + return false; } @@ -589,13 +680,23 @@ private async Task ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver? LogTraceGotMessages(multiBatch.Count, new(myQueueId), numMessages); - foreach (var group in - multiBatch - .Where(m => m is not null) - .GroupBy(container => container.StreamId)) + var availableMessages = multiBatch.Where(m => m is not null).ToList(); + if (availableMessages.Count == 0) + { + return false; + } + + var partitionStartToken = availableMessages[0].SequenceToken; + foreach (var streamData in pubSubCache.Values) + { + StartInactiveCursors(streamData, partitionStartToken); + } + + foreach (var group in availableMessages.GroupBy(container => container.StreamId)) { if (IsShutdown || cancellationToken.IsCancellationRequested) { + Volatile.Write(ref _hasUnprocessedRead, 1); return false; } @@ -604,7 +705,6 @@ private async Task ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver? if (pubSubCache.TryGetValue(streamId, out var streamData)) { streamData.RefreshActivity(now); - StartInactiveCursors(streamData, startToken); } else { @@ -645,16 +745,23 @@ private void CleanupPubSubCache(DateTime now) } /// - /// Computes delivery progress before shutdown so the queue can persist the latest handoff checkpoint. + /// Computes delivery progress so the queue can persist the latest handoff checkpoint. /// private void NotifyDeliveryProgress() { if (queueCache is null) return; var utcNow = _timeProvider.GetUtcNow().UtcDateTime; - if (TryGetDeliveryProgress(out var earliest)) + try { - queueCache.UpdateDeliveryProgress(earliest, utcNow); + if (TryGetDeliveryProgress(out var earliest)) + { + queueCache.UpdateDeliveryProgress(earliest, utcNow); + } + } + catch (ArgumentException exception) + { + LogWarningDeliveryProgressComparison(new(QueueId), exception); } } @@ -676,7 +783,16 @@ private bool TryGetDeliveryProgress(out StreamSequenceToken? earliest) return false; } - var current = consumer.LastProcessedToken; + var current = consumer.Cursor is IQueueCacheCursorProgress + ? consumer.LastSafePartitionToken + : consumer.LastProcessedToken; + if (consumer.Cursor is not IQueueCacheCursorProgress + && consumer.LastSafePartitionToken is { } safePartition + && (current is null || IsBefore(current, safePartition))) + { + current = safePartition; + } + if (current is null) { return false; @@ -692,10 +808,40 @@ private bool TryGetDeliveryProgress(out StreamSequenceToken? earliest) return true; } - private static bool IsBefore(StreamSequenceToken current, StreamSequenceToken other) + private void ScheduleDeliveryProgressUpdate() { - var difference = current.SequenceNumber.CompareTo(other.SequenceNumber); - return difference < 0 || difference == 0 && current.EventIndex < other.EventIndex; + this.RunOrQueueTask(() => + { + if (!IsShutdown && deliveryProgressTimer is not null) + { + NotifyDeliveryProgress(); + } + + return Task.CompletedTask; + }) + .LogException( + logger, + ErrorCode.PersistentStreamPullingAgent_28, + $"Failed to update delivery progress for queue {QueueId}.") + .Ignore(); + } + + private static bool IsBefore(StreamSequenceToken current, StreamSequenceToken other) => current.CompareTo(other) < 0; + + private static void UpdateCursorProgress( + StreamConsumerData consumerData, + IQueueCacheCursorProgress? progressCursor) + { + if (progressCursor?.SafeSequenceToken is not { } safeToken) + { + return; + } + + if (consumerData.LastSafePartitionToken is null + || IsBefore(consumerData.LastSafePartitionToken, safeToken)) + { + consumerData.LastSafePartitionToken = safeToken; + } } private void RegisterStream(QualifiedStreamId streamId, StreamSequenceToken firstToken, DateTime now) @@ -841,6 +987,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData) var deliveredAny = false; while (!IsShutdown && consumerData.Cursor is not null) { + var progressCursor = consumerData.Cursor as IQueueCacheCursorProgress; var batchCursor = options.BatchContainerBatchSize > 1 ? consumerData.Cursor as IQueueCacheCursorBatchDelivery : null; @@ -850,6 +997,7 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData) try { nextBatch = GetBatchForConsumer(consumerData.Cursor, consumerData.StreamId, consumerData.FilterData); + UpdateCursorProgress(consumerData, progressCursor); if (!nextBatch.HasProgress) { // Only emit cursor-drained when we transitioned from delivering to empty, @@ -872,7 +1020,9 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData) if (nextBatch.Batch is null) { + progressCursor?.RecordDeliverySuccess(); consumerData.LastProcessedToken = nextBatch.ProgressToken; + UpdateCursorProgress(consumerData, progressCursor); continue; } } @@ -903,29 +1053,59 @@ private async Task RunConsumerCursor(StreamConsumerData consumerData) deliveryBackoffProvider); if (newToken is not null) { - consumerData.LastProcessedToken = newToken.Token; + var previousSafePartitionToken = consumerData.LastSafePartitionToken; consumerData.LastToken = newToken; IQueueCacheCursor newCursor; + var resumedFromFallback = false; try { - newCursor = queueCache!.GetCacheCursor(consumerData.StreamId, newToken.Token); // queueCache must be non-null here: consumerData.Cursor was only ever populated via queueCache.GetCacheCursor. - // The handshake token points to an already processed event, so advance past it. - newCursor.MoveNext(); + var restartToken = newToken is DeliveryToken + ? previousSafePartitionToken ?? newToken.Token + : newToken.Token; + newCursor = queueCache!.GetCacheCursor(consumerData.StreamId, restartToken); // queueCache must be non-null here: consumerData.Cursor was only ever populated via queueCache.GetCacheCursor. + if (newCursor is IQueueCacheCursorProgress repositioningProgress) + { + if (newToken is DeliveryToken confirmedToken) + { + repositioningProgress.SetDeliveredThrough(confirmedToken.Token); + } + } + else + { + // Legacy cursors use returned handshake tokens as an exclusive + // resume boundary, matching the historical stream handshake contract. + newCursor.MoveNext(); + } } catch (QueueCacheMissException) { // The current batch is the receiver's first available message. // Keep it pending when the consumer resumes from an evicted token. newCursor = queueCache!.GetCacheCursor(consumerData.StreamId, batch.SequenceToken); + resumedFromFallback = true; } consumerData.SafeDisposeCursor(logger); consumerData.Cursor = newCursor; + consumerData.CursorStartToken = resumedFromFallback ? batch.SequenceToken : newToken.Token; + if (newToken is DeliveryToken deliveryToken) + { + consumerData.LastProcessedToken = deliveryToken.Token; + consumerData.LastSafePartitionToken = previousSafePartitionToken; + UpdateCursorProgress(consumerData, newCursor as IQueueCacheCursorProgress); + } + else + { + consumerData.LastProcessedToken = null; + consumerData.LastSafePartitionToken = null; + } } else { // Track progress for the periodic delivery scan. + progressCursor?.RecordDeliverySuccess(); consumerData.LastProcessedToken = nextBatch.ProgressToken; + UpdateCursorProgress(consumerData, progressCursor); } } } @@ -1290,6 +1470,13 @@ private readonly struct QueueIdLogRecord(QueueId queueId) )] private partial void LogWarningMessagesDeliveredAsync(QueueIdLogRecord myQueueId, Exception exception); + [LoggerMessage( + Level = LogLevel.Warning, + EventId = (int)ErrorCode.PersistentStreamPullingAgent_28, + Message = "Unable to compare delivery progress tokens for queue {QueueId}. The checkpoint will not advance." + )] + private partial void LogWarningDeliveryProgressComparison(QueueIdLogRecord queueId, Exception exception); + [LoggerMessage( Level = LogLevel.Information, EventId = (int)ErrorCode.PersistentStreamPullingAgent_24, diff --git a/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs b/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs index afb821e8263..3950b385523 100644 --- a/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs +++ b/src/Orleans.Streaming/PersistentStreams/QueueStreamDataStructures.cs @@ -34,6 +34,8 @@ internal sealed class StreamConsumerData public bool IsRegistered = false; [NonSerialized] public StreamSequenceToken? PendingStartToken; + [NonSerialized] + public StreamSequenceToken? CursorStartToken; /// /// The sequence token of the last batch processed (delivered or filtered) by this subscription. @@ -42,6 +44,14 @@ internal sealed class StreamConsumerData [NonSerialized] public StreamSequenceToken? LastProcessedToken; + /// + /// The last contiguous partition record which is safe for this subscription. + /// This includes successfully processed matching records and scanned records + /// belonging to other streams. + /// + [NonSerialized] + public StreamSequenceToken? LastSafePartitionToken; + public StreamConsumerData(GuidId subscriptionId, QualifiedStreamId streamId, IStreamConsumerExtension streamConsumer, string? filterData) { SubscriptionId = subscriptionId; diff --git a/src/Orleans.Streaming/PubSub/PubSubRendezvousGrain.cs b/src/Orleans.Streaming/PubSub/PubSubRendezvousGrain.cs index 4b5989861ee..9b52155faaa 100644 --- a/src/Orleans.Streaming/PubSub/PubSubRendezvousGrain.cs +++ b/src/Orleans.Streaming/PubSub/PubSubRendezvousGrain.cs @@ -329,6 +329,11 @@ public async Task UnregisterConsumer(GuidId subscriptionId, QualifiedStreamId st if (await TryClearState()) { + if (numRemoved != 0) + { + StreamingEvents.EmitSubscriptionUnregistered(streamId.ProviderName, streamId.StreamId, subscriptionId.Guid, GrainContext.Address.SiloAddress); + } + // If state was cleared expedite Deactivation DeactivateOnIdle(); } diff --git a/src/Orleans.Streaming/QueueAdapters/IQueueCache.cs b/src/Orleans.Streaming/QueueAdapters/IQueueCache.cs index e297a555759..628bed86dab 100644 --- a/src/Orleans.Streaming/QueueAdapters/IQueueCache.cs +++ b/src/Orleans.Streaming/QueueAdapters/IQueueCache.cs @@ -39,7 +39,9 @@ public interface IQueueCache : IQueueFlowController /// Updates the cache with the current delivery progress of all active subscriptions. /// /// - /// The earliest last processed sequence token across registered subscriptions. + /// The earliest contiguous partition position which is safe across registered subscriptions. + /// A position becomes safe after a matching record is delivered or intentionally filtered, + /// or after a cursor scans an unrelated record without an earlier pending delivery. /// A value indicates that there are no active subscriptions. /// The token is only valid for the duration of the call and must not be stored. /// diff --git a/src/Orleans.Streaming/QueueAdapters/IQueueCacheCursorProgress.cs b/src/Orleans.Streaming/QueueAdapters/IQueueCacheCursorProgress.cs new file mode 100644 index 00000000000..8dd8ba7662b --- /dev/null +++ b/src/Orleans.Streaming/QueueAdapters/IQueueCacheCursorProgress.cs @@ -0,0 +1,10 @@ +namespace Orleans.Streams; + +internal interface IQueueCacheCursorProgress +{ + StreamSequenceToken? SafeSequenceToken { get; } + + void SetDeliveredThrough(StreamSequenceToken token); + + void RecordDeliverySuccess(); +} diff --git a/src/Redis/Orleans.Streaming.Redis/Streams/RedisStreamSequenceToken.cs b/src/Redis/Orleans.Streaming.Redis/Streams/RedisStreamSequenceToken.cs index 829a35f27fc..63a01089202 100644 --- a/src/Redis/Orleans.Streaming.Redis/Streams/RedisStreamSequenceToken.cs +++ b/src/Redis/Orleans.Streaming.Redis/Streams/RedisStreamSequenceToken.cs @@ -53,7 +53,8 @@ public RedisStreamSequenceToken() /// /// The event index within the Orleans batch. /// A token for the specified event. - public new RedisStreamSequenceToken CreateSequenceTokenForEvent(int eventIndex) => new(EntryId, SequenceNumber, RedisSequenceNumber, eventIndex); + public override RedisStreamSequenceToken CreateSequenceTokenForEvent(int eventIndex) + => new(EntryId, SequenceNumber, RedisSequenceNumber, eventIndex); /// public override bool Equals(StreamSequenceToken? other) @@ -73,22 +74,27 @@ public override int CompareTo(StreamSequenceToken? other) return 1; } + if (other is not RedisStreamSequenceToken token) + { + throw new ArgumentOutOfRangeException(nameof(other)); + } + var difference = SequenceNumber.CompareTo(other.SequenceNumber); if (difference != 0) { return difference; } - if (other is RedisStreamSequenceToken token) + difference = RedisSequenceNumber.CompareTo(token.RedisSequenceNumber); + if (difference != 0) { - difference = RedisSequenceNumber.CompareTo(token.RedisSequenceNumber); - if (difference != 0) - { - return difference; - } + return difference; } - return EventIndex.CompareTo(other.EventIndex); + difference = EventIndex.CompareTo(other.EventIndex); + return difference != 0 + ? difference + : string.CompareOrdinal(EntryId, token.EntryId); } /// diff --git a/src/api/AdoNet/Orleans.Streaming.AdoNet/Orleans.Streaming.AdoNet.cs b/src/api/AdoNet/Orleans.Streaming.AdoNet/Orleans.Streaming.AdoNet.cs index 632663fc594..df264c68161 100644 --- a/src/api/AdoNet/Orleans.Streaming.AdoNet/Orleans.Streaming.AdoNet.cs +++ b/src/api/AdoNet/Orleans.Streaming.AdoNet/Orleans.Streaming.AdoNet.cs @@ -10,27 +10,31 @@ namespace Orleans.Configuration { public partial class AdoNetStreamOptions { + public System.TimeSpan CheckpointPersistInterval { get { throw null; } set { } } + + public int CleanupBatchSize { get { throw null; } set { } } + + public System.TimeSpan CleanupInterval { get { throw null; } set { } } + [Redact] public string? ConnectionString { get { throw null; } set { } } [Redact] public System.Data.Common.DbDataSource? DataSource { get { throw null; } set { } } - public System.TimeSpan DeadLetterEvictionTimeout { get { throw null; } set { } } - - public int EvictionBatchSize { get { throw null; } set { } } - - public System.TimeSpan EvictionInterval { get { throw null; } set { } } - - public System.TimeSpan ExpiryTimeout { get { throw null; } set { } } + public bool FaultOnDeliveryFailure { get { throw null; } set { } } public System.TimeSpan InitializationTimeout { get { throw null; } set { } } public string Invariant { get { throw null; } set { } } - public int MaxAttempts { get { throw null; } set { } } + public System.TimeSpan? MaximumRetentionPeriod { get { throw null; } set { } } + + public int MaxMessagesPerRead { get { throw null; } set { } } + + public System.TimeSpan RetentionPeriod { get { throw null; } set { } } - public System.TimeSpan VisibilityTimeout { get { throw null; } set { } } + public bool StartFromNow { get { throw null; } set { } } } public partial class AdoNetStreamOptionsValidator : IConfigurationValidator diff --git a/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs b/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs index de3bc387527..21aec5500c0 100644 --- a/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs +++ b/src/api/Azure/Orleans.Streaming.EventHubs/Orleans.Streaming.EventHubs.cs @@ -303,6 +303,7 @@ public EventHubAdapterFactory(string name, Configuration.EventHubOptions ehOptio public static EventHubAdapterFactory Create(System.IServiceProvider services, string name) { throw null; } + [System.Diagnostics.DebuggerStepThrough] public System.Threading.Tasks.Task CreateAdapter() { throw null; } protected virtual IEventHubQueueCacheFactory CreateCacheFactory(Configuration.EventHubStreamCachePressureOptions eventHubCacheOptions) { throw null; } @@ -313,6 +314,7 @@ public EventHubAdapterFactory(string name, Configuration.EventHubOptions ehOptio public System.Threading.Tasks.Task GetDeliveryFailureHandler(Streams.QueueId queueId) { throw null; } + [System.Diagnostics.DebuggerStepThrough] protected virtual System.Threading.Tasks.Task GetPartitionIdsAsync() { throw null; } public Streams.IQueueAdapterCache GetQueueAdapterCache() { throw null; } @@ -368,6 +370,10 @@ internal EventHubCheckpointer() { } public bool CheckpointExists { get { throw null; } } + [System.Diagnostics.DebuggerStepThrough] + public static System.Threading.Tasks.Task> Create(Configuration.AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory, System.Threading.CancellationToken cancellationToken) { throw null; } + + [System.Diagnostics.DebuggerStepThrough] public static System.Threading.Tasks.Task> Create(Configuration.AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory) { throw null; } public System.Threading.Tasks.Task FlushAsync(System.Threading.CancellationToken cancellationToken) { throw null; } @@ -375,6 +381,7 @@ internal EventHubCheckpointer() { } [System.Obsolete("Use the overload which accepts a CancellationToken.")] public System.Threading.Tasks.Task Load() { throw null; } + [System.Diagnostics.DebuggerStepThrough] public System.Threading.Tasks.Task Load(System.Threading.CancellationToken cancellationToken) { throw null; } public void Update(string offset, System.DateTime utcNow, System.Threading.CancellationToken cancellationToken) { } @@ -387,6 +394,8 @@ public partial class EventHubCheckpointerFactory : Streams.IStreamQueueCheckpoin { public EventHubCheckpointerFactory(string providerName, Configuration.AzureTableStreamCheckpointerOptions options, Microsoft.Extensions.Options.IOptions clusterOptions, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory) { } + public System.Threading.Tasks.Task> Create(string partition, System.Threading.CancellationToken cancellationToken) { throw null; } + public System.Threading.Tasks.Task> Create(string partition) { throw null; } public static Streams.IStreamQueueCheckpointerFactory CreateFactory(System.IServiceProvider services, string providerName) { throw null; } @@ -530,6 +539,12 @@ public EventHubSequenceToken(string eventHubOffset, long sequenceNumber, int eve [Newtonsoft.Json.JsonProperty] public string EventHubOffset { get { throw null; } } + public override int CompareTo(Streams.StreamSequenceToken? other) { throw null; } + + public override bool Equals(Streams.StreamSequenceToken? other) { throw null; } + + public override int GetHashCode() { throw null; } + public override string ToString() { throw null; } } @@ -757,12 +772,18 @@ internal AzureTableStreamQueueCheckpointer() { } public bool CheckpointExists { get { throw null; } } + public static System.Threading.Tasks.Task> Create(Configuration.AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory, System.Threading.CancellationToken cancellationToken) { throw null; } + public static System.Threading.Tasks.Task> Create(Configuration.AzureTableStreamCheckpointerOptions options, string streamProviderName, string partition, string serviceId, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory) { throw null; } public System.Threading.Tasks.Task FlushAsync(System.Threading.CancellationToken cancellationToken) { throw null; } public System.Threading.Tasks.Task Load() { throw null; } + public System.Threading.Tasks.Task Load(System.Threading.CancellationToken cancellationToken) { throw null; } + + public void Update(string offset, System.DateTime utcNow, System.Threading.CancellationToken cancellationToken) { } + public void Update(string offset, System.DateTime utcNow) { } } @@ -770,6 +791,9 @@ public partial class AzureTableStreamQueueCheckpointerFactory : IStreamQueueChec { public AzureTableStreamQueueCheckpointerFactory(string providerName, Configuration.AzureTableStreamCheckpointerOptions options, Microsoft.Extensions.Options.IOptions clusterOptions, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory) { } + public System.Threading.Tasks.Task> Create(string partition, System.Threading.CancellationToken cancellationToken) { throw null; } + + [System.Obsolete("Use the overload which accepts a CancellationToken.")] public System.Threading.Tasks.Task> Create(string partition) { throw null; } public static IStreamQueueCheckpointerFactory CreateFactory(System.IServiceProvider services, string providerName) { throw null; } diff --git a/src/api/Orleans.Streaming/Orleans.Streaming.cs b/src/api/Orleans.Streaming/Orleans.Streaming.cs index a25c4b7c345..b2a3394f13a 100644 --- a/src/api/Orleans.Streaming/Orleans.Streaming.cs +++ b/src/api/Orleans.Streaming/Orleans.Streaming.cs @@ -146,12 +146,15 @@ public partial class StreamPubSubOptions public partial class StreamPullingAgentOptions { public static readonly int DEFAULT_BATCH_CONTAINER_BATCH_SIZE; + public static readonly System.TimeSpan DEFAULT_DELIVERY_PROGRESS_UPDATE_INTERVAL; public static readonly System.TimeSpan DEFAULT_GET_QUEUE_MESSAGES_TIMER_PERIOD; public static readonly System.TimeSpan DEFAULT_INIT_QUEUE_TIMEOUT; public static readonly System.TimeSpan DEFAULT_MAX_EVENT_DELIVERY_TIME; public static readonly System.TimeSpan DEFAULT_STREAM_INACTIVITY_PERIOD; public int BatchContainerBatchSize { get { throw null; } set { } } + public System.TimeSpan DeliveryProgressUpdateInterval { get { throw null; } set { } } + public System.TimeSpan GetQueueMsgsTimerPeriod { get { throw null; } set { } } public System.TimeSpan InitQueueTimeout { get { throw null; } set { } } @@ -409,6 +412,7 @@ public MemoryAdapterFactory(string providerName, Configuration.StreamCacheEvicti [System.Diagnostics.CodeAnalysis.MemberNotNull(new[] { "CacheMonitorFactory", "BlockPoolMonitorFactory", "ReceiverMonitorFactory" })] public void Init() { } + [System.Diagnostics.DebuggerStepThrough] public System.Threading.Tasks.Task QueueMessageBatchAsync(Runtime.StreamId streamId, System.Collections.Generic.IEnumerable events, Orleans.Streams.StreamSequenceToken? token, System.Collections.Generic.Dictionary? requestContext) { throw null; } } @@ -513,6 +517,8 @@ public CachedMessageBlock(int blockSize = 16384) { } public void Add(CachedMessage message) { } + public int GetIndexOfFirstMessageLessThanOrEqualTo(Orleans.Streams.StreamSequenceToken token, ICacheDataAdapter dataAdapter) { throw null; } + public int GetIndexOfFirstMessageLessThanOrEqualTo(Orleans.Streams.StreamSequenceToken token) { throw null; } public Orleans.Streams.StreamSequenceToken GetNewestSequenceToken(ICacheDataAdapter dataAdapter) { throw null; } @@ -555,6 +561,8 @@ public IPurgeObservable PurgeObservable { set { } } public void OnBlockAllocated(FixedSizeBuffer newBlock) { } + public void OnPurgeCompleted(CachedMessage? lastMessagePurged, int itemsPurged) { } + public void PerformPurge(System.DateTime nowUtc) { } protected virtual bool ShouldPurge(ref CachedMessage cachedMessage, ref CachedMessage newestCachedMessage, System.DateTime nowUtc) { throw null; } @@ -630,7 +638,7 @@ public EventSequenceToken(long sequenceNumber) { } public override int CompareTo(Orleans.Streams.StreamSequenceToken? other) { throw null; } - public EventSequenceToken CreateSequenceTokenForEvent(int eventInd) { throw null; } + public virtual EventSequenceToken CreateSequenceTokenForEvent(int eventInd) { throw null; } public override bool Equals(Orleans.Streams.StreamSequenceToken? other) { throw null; } @@ -660,7 +668,7 @@ public EventSequenceTokenV2(long seqNumber) { } public override int CompareTo(Orleans.Streams.StreamSequenceToken? other) { throw null; } - public EventSequenceTokenV2 CreateSequenceTokenForEvent(int eventInd) { throw null; } + public virtual EventSequenceTokenV2 CreateSequenceTokenForEvent(int eventInd) { throw null; } public override bool Equals(Orleans.Streams.StreamSequenceToken? other) { throw null; } @@ -692,6 +700,7 @@ public partial interface IBlockPoolMonitor public partial interface ICacheDataAdapter { + int Compare(ref CachedMessage cachedMessage, Orleans.Streams.StreamSequenceToken token); Orleans.Streams.IBatchContainer GetBatchContainer(ref CachedMessage cachedMessage); Orleans.Streams.StreamSequenceToken GetSequenceToken(ref CachedMessage cachedMessage); } @@ -714,6 +723,7 @@ public partial interface IEvictionStrategy IPurgeObservable PurgeObservable { set; } void OnBlockAllocated(FixedSizeBuffer newBlock); + void OnPurgeCompleted(CachedMessage? lastMessagePurged, int itemsPurged); void PerformPurge(System.DateTime utcNow); } @@ -752,6 +762,29 @@ public partial interface IQueueAdapterReceiverMonitor void TrackShutdown(bool success, System.TimeSpan callTime, System.Exception? exception); } + public partial interface IRecoverableStreamDataAdapter : ICacheDataAdapter + { + CachedMessage FromQueueMessage(Orleans.Streams.StreamPosition streamPosition, TQueueMessage queueMessage, System.DateTime dequeueTimeUtc, System.Func> getSegment); + string GetOffset(ref CachedMessage cachedMessage); + Orleans.Streams.StreamPosition GetStreamPosition(TQueueMessage queueMessage); + bool TryGetOffset(Orleans.Streams.StreamSequenceToken token, out string offset); + } + + public partial interface IRecoverableStreamQueueCache : Orleans.Streams.IQueueCache, Orleans.Streams.IQueueFlowController, System.IDisposable + { + System.Collections.Generic.IReadOnlyList Add(System.Collections.Generic.IReadOnlyList messages, System.DateTime dequeueTimeUtc); + bool TryGetNewestPosition(out Orleans.Streams.StreamSequenceToken? token, out string? offset); + } + + public partial interface IRecoverableStreamSource + { + System.Threading.Tasks.Task Initialize(RecoverableStreamStartPosition position, System.Threading.CancellationToken cancellationToken); + void MessagesAdded(System.Collections.Generic.IReadOnlyList messages); + void MessagesAddFailed(System.Collections.Generic.IReadOnlyList messages); + System.Threading.Tasks.Task> Read(int maxCount, System.Threading.CancellationToken cancellationToken); + System.Threading.Tasks.Task Shutdown(System.Threading.CancellationToken cancellationToken); + } + public partial class ObjectPoolMonitorBridge : IObjectPoolMonitor { public ObjectPoolMonitorBridge(IBlockPoolMonitor blockPoolMonitor, int blockSizeInBytes) { } @@ -841,6 +874,18 @@ public virtual void OnResetState() { } public virtual void SignalPurge() { } } + public sealed partial class QueueAdapterReceiverRegistry + where TReceiver : class, Orleans.Streams.IQueueAdapterReceiver, Orleans.Streams.IQueueCache + { + public QueueAdapterReceiverRegistry(System.Func factory) { } + + public System.Collections.Generic.IReadOnlyDictionary Receivers { get { throw null; } } + + public TReceiver GetOrCreate(Orleans.Streams.QueueId queueId) { throw null; } + + public bool Remove(Orleans.Streams.QueueId queueId, TReceiver receiver) { throw null; } + } + public partial class ReceiverMonitorDimensions { public ReceiverMonitorDimensions() { } @@ -850,6 +895,82 @@ public ReceiverMonitorDimensions(string queueId) { } public string QueueId { get { throw null; } set { } } } + public sealed partial class RecoverableStreamQueueCache : IRecoverableStreamQueueCache, Orleans.Streams.IQueueCache, Orleans.Streams.IQueueFlowController, System.IDisposable + { + public RecoverableStreamQueueCache(int defaultMaxAddCount, IObjectPool bufferPool, IRecoverableStreamDataAdapter dataAdapter, IEvictionStrategy evictionStrategy, Microsoft.Extensions.Logging.ILogger logger, Orleans.Streams.IQueueFlowController? flowController = null, ICacheMonitor? cacheMonitor = null, System.TimeSpan? cacheMonitorWriteInterval = null, System.TimeSpan? metadataMinTimeInCache = null, int? maxCacheSize = null) { } + + public int ItemCount { get { throw null; } } + + public string? LastPurgedOffset { get { throw null; } } + + public System.Collections.Generic.IReadOnlyList Add(System.Collections.Generic.IReadOnlyList messages, System.DateTime dequeueTimeUtc) { throw null; } + + public void AddToCache(System.Collections.Generic.IList messages) { } + + public void Dispose() { } + + public Orleans.Streams.IQueueCacheCursor GetCacheCursor(Runtime.StreamId streamId, Orleans.Streams.StreamSequenceToken? token) { throw null; } + + public int GetMaxAddCount() { throw null; } + + public bool IsUnderPressure() { throw null; } + + public bool TryGetNewestPosition(out Orleans.Streams.StreamSequenceToken? token, out string? offset) { throw null; } + + public bool TryPurgeFromCache(out System.Collections.Generic.IList purgedItems) { throw null; } + + public void UpdateDeliveryProgress(Orleans.Streams.StreamSequenceToken? earliestSubscriptionToken, System.DateTime utcNow) { } + } + + public sealed partial class RecoverableStreamReceiver : Orleans.Streams.IQueueAdapterReceiver, Orleans.Streams.IQueueCache, Orleans.Streams.IQueueFlowController + { + public RecoverableStreamReceiver(IRecoverableStreamSource source, IRecoverableStreamDataAdapter dataAdapter, IRecoverableStreamQueueCache cache, Orleans.Streams.IStreamQueueCheckpointer checkpointer, bool startFromNow) { } + + public RecoverableStreamReceiver(IRecoverableStreamSource source, IRecoverableStreamDataAdapter dataAdapter, RecoverableStreamQueueCache cache, Orleans.Streams.IStreamQueueCheckpointer checkpointer, bool startFromNow) { } + + public void AddToCache(System.Collections.Generic.IList messages) { } + + public Orleans.Streams.IQueueCacheCursor GetCacheCursor(Runtime.StreamId streamId, Orleans.Streams.StreamSequenceToken? token) { throw null; } + + public int GetMaxAddCount() { throw null; } + + [System.Diagnostics.DebuggerStepThrough] + public System.Threading.Tasks.Task> GetQueueMessagesAsync(int maxCount, System.Threading.CancellationToken cancellationToken) { throw null; } + + [System.Obsolete("Use the overload which accepts a CancellationToken.")] + public System.Threading.Tasks.Task> GetQueueMessagesAsync(int maxCount) { throw null; } + + public System.Threading.Tasks.Task Initialize(System.Threading.CancellationToken cancellationToken) { throw null; } + + [System.Diagnostics.DebuggerStepThrough] + public System.Threading.Tasks.Task Initialize(System.TimeSpan timeout) { throw null; } + + public bool IsUnderPressure() { throw null; } + + public System.Threading.Tasks.Task MessagesDeliveredAsync(System.Collections.Generic.IList messages, System.Threading.CancellationToken cancellationToken) { throw null; } + + [System.Obsolete("Use the overload which accepts a CancellationToken.")] + public System.Threading.Tasks.Task MessagesDeliveredAsync(System.Collections.Generic.IList messages) { throw null; } + + [System.Diagnostics.DebuggerStepThrough] + public System.Threading.Tasks.Task Shutdown(System.TimeSpan timeout) { throw null; } + + public bool TryPurgeFromCache(out System.Collections.Generic.IList purgedItems) { throw null; } + + public void UpdateDeliveryProgress(Orleans.Streams.StreamSequenceToken? earliestSubscriptionToken, System.DateTime utcNow) { } + } + + public readonly partial struct RecoverableStreamStartPosition + { + private readonly object _dummy; + private readonly int _dummyPrimitive; + public RecoverableStreamStartPosition(string? checkpoint, bool startFromNow) { } + + public string? Checkpoint { get { throw null; } } + + public bool StartFromNow { get { throw null; } } + } + public static partial class SegmentBuilder { public static void Append(System.ArraySegment segment, ref int writerOffset, System.ReadOnlySpan bytes) { } @@ -1468,13 +1589,17 @@ public GrainStreamQueueCheckpointer(IStreamCheckpointerGrain grain) { } public bool CheckpointExists { get { throw null; } } + [System.Diagnostics.DebuggerStepThrough] public static System.Threading.Tasks.Task> Create(string providerName, string partition, string serviceId, IClusterClient clusterClient, Configuration.GrainStreamQueueCheckpointerOptions options, System.Threading.CancellationToken cancellationToken) { throw null; } + [System.Diagnostics.DebuggerStepThrough] [System.Obsolete("Use the overload which accepts a CancellationToken.")] public static System.Threading.Tasks.Task> Create(string providerName, string partition, string serviceId, IClusterClient clusterClient, Configuration.GrainStreamQueueCheckpointerOptions options) { throw null; } + [System.Diagnostics.DebuggerStepThrough] public static System.Threading.Tasks.Task> Create(string providerName, string partition, string serviceId, IClusterClient clusterClient, System.Threading.CancellationToken cancellationToken) { throw null; } + [System.Diagnostics.DebuggerStepThrough] [System.Obsolete("Use the overload which accepts a CancellationToken.")] public static System.Threading.Tasks.Task> Create(string providerName, string partition, string serviceId, IClusterClient clusterClient) { throw null; } @@ -1678,6 +1803,12 @@ public partial interface IStreamCheckpointerGrain : IGrainWithStringKey, IGrain, System.Threading.Tasks.ValueTask Update(string offset, string expectedCheckpoint, System.Threading.CancellationToken cancellationToken); } + public partial interface IStreamCheckpointStore + { + System.Threading.Tasks.ValueTask Load(System.Threading.CancellationToken cancellationToken); + System.Threading.Tasks.ValueTask Update(string checkpoint, string expectedVersion, System.Threading.CancellationToken cancellationToken); + } + public partial interface IStreamFailureHandler { bool ShouldFaultSubsriptionOnError { get; } @@ -1780,10 +1911,12 @@ public LeaseBasedQueueBalancer(string name, Configuration.LeaseBasedQueueBalance public override System.Collections.Generic.IEnumerable GetMyQueues() { throw null; } + [System.Diagnostics.DebuggerStepThrough] public override System.Threading.Tasks.Task Initialize(IStreamQueueMapper queueMapper) { throw null; } protected override void OnClusterMembershipChange(System.Collections.Generic.HashSet activeSilos) { } + [System.Diagnostics.DebuggerStepThrough] public override System.Threading.Tasks.Task Shutdown() { throw null; } } @@ -1914,6 +2047,7 @@ protected QueueBalancerBase(System.IServiceProvider sp, Microsoft.Extensions.Log protected System.Threading.Tasks.Task NotifyListeners() { throw null; } protected abstract void OnClusterMembershipChange(System.Collections.Generic.HashSet activeSilos); + [System.Diagnostics.DebuggerStepThrough] public virtual System.Threading.Tasks.Task Shutdown() { throw null; } public bool SubscribeToQueueDistributionChangeEvents(IStreamQueueBalanceListener observer) { throw null; } @@ -2025,9 +2159,21 @@ public StreamCheckpointGrain(Runtime.IPersistentState Load(System.Threading.CancellationToken cancellationToken) { throw null; } + [System.Diagnostics.DebuggerStepThrough] public System.Threading.Tasks.ValueTask Update(string offset, string expectedCheckpoint, System.Threading.CancellationToken cancellationToken) { throw null; } } + public readonly partial struct StreamCheckpointStoreState + { + private readonly object _dummy; + private readonly int _dummyPrimitive; + public StreamCheckpointStoreState(string checkpoint, string version) { } + + public string Checkpoint { get { throw null; } } + + public string Version { get { throw null; } } + } + [GenerateSerializer] public sealed partial class StreamEventDeliveryFailureException : Runtime.OrleansException { @@ -2102,6 +2248,34 @@ public enum StreamPubSubType ImplicitOnly = 2 } + public sealed partial class StreamQueueCheckpointer : IStreamQueueCheckpointer + { + public StreamQueueCheckpointer(IStreamCheckpointStore store, StreamQueueCheckpointerOptions options) { } + + public bool CheckpointExists { get { throw null; } } + + [System.Diagnostics.DebuggerStepThrough] + public System.Threading.Tasks.Task FlushAsync(System.Threading.CancellationToken cancellationToken) { throw null; } + + [System.Obsolete("Use the overload which accepts a CancellationToken.")] + public System.Threading.Tasks.Task Load() { throw null; } + + [System.Diagnostics.DebuggerStepThrough] + public System.Threading.Tasks.Task Load(System.Threading.CancellationToken cancellationToken) { throw null; } + + public void Update(string offset, System.DateTime utcNow, System.Threading.CancellationToken cancellationToken) { } + + [System.Obsolete("Use the overload which accepts a CancellationToken.")] + public void Update(string offset, System.DateTime utcNow) { } + } + + public sealed partial class StreamQueueCheckpointerOptions + { + public System.Collections.Generic.IComparer? CheckpointComparer { get { throw null; } set { } } + + public System.TimeSpan PersistInterval { get { throw null; } set { } } + } + [GenerateSerializer] public abstract partial class StreamSequenceToken : System.IEquatable, System.IComparable { diff --git a/test/Benchmarks.AdoNet/Streaming/MessageDequeueingBenchmark.cs b/test/Benchmarks.AdoNet/Streaming/MessageDequeueingBenchmark.cs deleted file mode 100644 index 0f6de197c9c..00000000000 --- a/test/Benchmarks.AdoNet/Streaming/MessageDequeueingBenchmark.cs +++ /dev/null @@ -1,127 +0,0 @@ -using Microsoft.Data.SqlClient; -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Engines; -using Orleans.Streaming.AdoNet; -using Orleans.Tests.SqlUtils; -using UnitTests.General; -using static System.String; - -namespace Benchmarks.AdoNet.Streaming; - -public class SqlServerMessageDequeuingBenchmark() : MessageDequeuingBenchmark(AdoNetInvariants.InvariantNameSqlServer, "OrleansStreamTest") -{ - public override void GlobalSetup() - { - base.GlobalSetup(); - - SqlConnection.ClearAllPools(); - } -} - -/// -/// This benchmark measures the performance of message queueing. -/// -[WarmupCount(1), IterationCount(3), InvocationCount(1), MarkdownExporter] -public abstract class MessageDequeuingBenchmark(string invariant, string database) -{ - private const int OperationsPerInvoke = 1000; - - private readonly Consumer _consumer = new(); - // BenchmarkDotNet invokes GlobalSetup before any of these fields are read. - private IRelationalStorage _storage = default!; - private RelationalOrleansQueries _queries = default!; - private byte[] _payload = []; - private string[] _queueIds = default!; - private AdoNetStreamMessageAck[] _acks = []; - - /// - /// This highlights degradation from queue concurrency. - /// - [Params(1, 4, 8)] - public int QueueCount { get; set; } - - /// - /// This highlights degradation from payload size. - /// - [Params(10000)] - public int PayloadSize { get; set; } - - /// - /// This highlights variation according to batch size. - /// - [Params(1, 16, 32)] - public int BatchSize { get; set; } - - /// - /// This highlights variation from how full the table is. - /// - [Params(0, 0.5, 1)] - public double FullnessRatio { get; set; } - - [GlobalSetup] - public virtual void GlobalSetup() - { - Async().GetAwaiter().GetResult(); - - async Task Async() - { - // create an appropriate size payload - _payload = new byte[PayloadSize]; - Array.Fill(_payload, 0xFF); - - // define the set queues - _queueIds = Enumerable.Range(0, QueueCount).Select(i => $"QueueId-{i}").ToArray(); - - // setup the test database - var testing = await RelationalStorageForTesting.SetupInstance(invariant, database); - if (IsNullOrEmpty(testing.CurrentConnectionString)) - { - throw new InvalidOperationException($"Database '{database}' not initialized"); - } - _storage = RelationalStorage.CreateInstance(invariant, testing.CurrentConnectionString); - _queries = await RelationalOrleansQueries.CreateInstance(invariant, testing.CurrentConnectionString); - } - } - - [IterationSetup] - public void IterationSetup() - { - Async().GetAwaiter().GetResult(); - - async Task Async() - { - await _storage.ExecuteAsync("TRUNCATE TABLE OrleansStreamMessage"); - - // generate test data to dequeue - var count = (int)Math.Ceiling(OperationsPerInvoke * QueueCount * BatchSize * FullnessRatio); - _acks = new AdoNetStreamMessageAck[count]; - await Parallel.ForAsync(0, count, async (i, ct) => - { - // generate messages in round robin queue order to help simulate multiple agents - var queueId = _queueIds[i % _queueIds.Length]; - var ack = await _queries.QueueStreamMessageAsync("ServiceId-0", "ProviderId-0", queueId, _payload, 1000); - - _acks[i] = ack; - }); - } - } - - [Benchmark(OperationsPerInvoke = OperationsPerInvoke)] - public async Task GetStreamMessages() - { - var count = OperationsPerInvoke * QueueCount; - - await Parallel.ForAsync(0, count, new ParallelOptions { MaxDegreeOfParallelism = QueueCount }, async (i, ct) => - { - // get a queue id in round robin order to help simulate multiple agents - var queueId = _queueIds[i % _queueIds.Length]; - - // get messages for the queue of the ack - // the queue may or may not have data to dequeue depending on the fullness and batch size parameters - // we dequeue regardless in order to measure overhead - var messages = await _queries.GetStreamMessagesAsync("ServiceId-0", "ProviderId-0", queueId, BatchSize, 1, 1000, 1000, 1000, 1000); - - _consumer.Consume(messages); - }); - } -} diff --git a/test/Benchmarks.AdoNet/Streaming/MessageQueueingBenchmark.cs b/test/Benchmarks.AdoNet/Streaming/MessageQueueingBenchmark.cs deleted file mode 100644 index c160f612539..00000000000 --- a/test/Benchmarks.AdoNet/Streaming/MessageQueueingBenchmark.cs +++ /dev/null @@ -1,89 +0,0 @@ -using Microsoft.Data.SqlClient; -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Engines; -using Orleans.Tests.SqlUtils; -using UnitTests.General; -using static System.String; - -namespace Benchmarks.AdoNet.Streaming; - -public class SqlServerMessageQueueingBenchmark() : MessageQueueingBenchmark(AdoNetInvariants.InvariantNameSqlServer, "OrleansStreamTest") -{ - public override void GlobalSetup() - { - base.GlobalSetup(); - - SqlConnection.ClearAllPools(); - } -} - -/// -/// This benchmark measures the performance of message queueing. -/// -[WarmupCount(1), IterationCount(3), InvocationCount(1), MarkdownExporter] -public abstract class MessageQueueingBenchmark(string invariant, string database) -{ - private const int OperationsPerInvoke = 1000; - - private readonly Consumer _consumer = new(); - // BenchmarkDotNet invokes GlobalSetup before any of these fields are read. - private IRelationalStorage _storage = default!; - private RelationalOrleansQueries _queries = default!; - private byte[] _payload = []; - private string[] _queueIds = default!; - - /// - /// This highlights degradation from database locking. - /// - [Params(1, 4, 8)] - public int QueueCount { get; set; } - - /// - /// This highlights degradation from payload size. - /// - [Params(1000, 10000, 100000)] - public int PayloadSize { get; set; } - - /// - /// This highlights degradation from concurrency. - /// - [Params(1, 4, 8)] - public int Concurrency { get; set; } - - [GlobalSetup] - public virtual void GlobalSetup() - { - _payload = new byte[PayloadSize]; - Array.Fill(_payload, 0xFF); - - _queueIds = Enumerable.Range(0, QueueCount).Select(i => $"QueueId-{i}").ToArray(); - - var testing = RelationalStorageForTesting.SetupInstance(invariant, database).GetAwaiter().GetResult(); - - if (IsNullOrEmpty(testing.CurrentConnectionString)) - { - throw new InvalidOperationException($"Database '{database}' not initialized"); - } - - _storage = RelationalStorage.CreateInstance(invariant, testing.CurrentConnectionString); - - _queries = RelationalOrleansQueries.CreateInstance(invariant, testing.CurrentConnectionString).GetAwaiter().GetResult(); - } - - [IterationSetup] - public void IterationSetup() => _storage.ExecuteAsync("TRUNCATE TABLE OrleansStreamMessage").GetAwaiter().GetResult(); - - [Benchmark(OperationsPerInvoke = OperationsPerInvoke)] - public Task QueueStreamMessage() - { - var count = OperationsPerInvoke * Concurrency; - - return Parallel.ForAsync(0, count, new ParallelOptions { MaxDegreeOfParallelism = Concurrency }, async (i, ct) => - { - var queueId = _queueIds[Random.Shared.Next(_queueIds.Length)]; - var ack = await _queries.QueueStreamMessageAsync("ServiceId-0", "ProviderId-0", queueId, _payload, 1000); - - _consumer.Consume(ack); - }); - } -} diff --git a/test/Benchmarks.AdoNet/Streaming/StreamPartitionAppendBenchmark.cs b/test/Benchmarks.AdoNet/Streaming/StreamPartitionAppendBenchmark.cs new file mode 100644 index 00000000000..d7d7df3c95e --- /dev/null +++ b/test/Benchmarks.AdoNet/Streaming/StreamPartitionAppendBenchmark.cs @@ -0,0 +1,84 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Engines; +using Microsoft.Data.SqlClient; +using Orleans.Tests.SqlUtils; +using UnitTests.General; +using static System.String; + +namespace Benchmarks.AdoNet.Streaming; + +public class SqlServerStreamPartitionAppendBenchmark() : StreamPartitionAppendBenchmark(AdoNetInvariants.InvariantNameSqlServer, "OrleansStreamTest") +{ + public override void GlobalSetup() + { + base.GlobalSetup(); + SqlConnection.ClearAllPools(); + } +} + +/// +/// Measures stream partition append throughput while varying payload size, concurrency, and partition contention. +/// A partition count of one is the lock-wait baseline for database-side wait telemetry. +/// +[WarmupCount(1), IterationCount(3), InvocationCount(1), MarkdownExporter] +public abstract class StreamPartitionAppendBenchmark(string invariant, string database) +{ + private const int OperationsPerInvoke = 1_000; + + private readonly Consumer _consumer = new(); + private IRelationalStorage _storage = default!; + private RelationalOrleansQueries _queries = default!; + private byte[] _payload = []; + private byte[] _streamIdBytes = []; + private string[] _partitionIds = []; + + [Params(1, 8)] + public int PartitionCount { get; set; } + + [Params(1_000, 100_000)] + public int PayloadSize { get; set; } + + [Params(1, 8)] + public int Concurrency { get; set; } + + [GlobalSetup] + public virtual void GlobalSetup() + { + _payload = new byte[PayloadSize]; + new Random(42).NextBytes(_payload); + _streamIdBytes = "benchmarkstream-0"u8.ToArray(); + _partitionIds = Enumerable.Range(0, PartitionCount).Select(i => $"QueueId-{i}").ToArray(); + + var testing = RelationalStorageForTesting.SetupInstance(invariant, database).GetAwaiter().GetResult(); + if (IsNullOrEmpty(testing.CurrentConnectionString)) + { + throw new InvalidOperationException($"Database '{database}' not initialized"); + } + + _storage = RelationalStorage.CreateInstance(invariant, testing.CurrentConnectionString); + _queries = RelationalOrleansQueries.CreateInstance(invariant, testing.CurrentConnectionString).GetAwaiter().GetResult(); + } + + [IterationSetup] + public void IterationSetup() => _storage.ExecuteAsync( + "TRUNCATE TABLE OrleansStreamMessage; TRUNCATE TABLE OrleansStreamPartition").GetAwaiter().GetResult(); + + [Benchmark(OperationsPerInvoke = OperationsPerInvoke)] + [BenchmarkCategory("Append", "LockWait")] + public Task AppendWithPartitionContention() => + Parallel.ForAsync( + 0, + OperationsPerInvoke, + new ParallelOptions { MaxDegreeOfParallelism = Concurrency }, + async (i, cancellationToken) => + { + var result = await _queries.AppendStreamMessageAsync( + "ServiceId-0", + "ProviderId-0", + _partitionIds[i % _partitionIds.Length], + _streamIdBytes, + 9, + _payload); + _consumer.Consume(result); + }); +} diff --git a/test/Benchmarks.AdoNet/Streaming/StreamPartitionReadCheckpointBenchmark.cs b/test/Benchmarks.AdoNet/Streaming/StreamPartitionReadCheckpointBenchmark.cs new file mode 100644 index 00000000000..23af845f961 --- /dev/null +++ b/test/Benchmarks.AdoNet/Streaming/StreamPartitionReadCheckpointBenchmark.cs @@ -0,0 +1,262 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Engines; +using Microsoft.Data.SqlClient; +using Orleans.Streaming.AdoNet; +using Orleans.Tests.SqlUtils; +using UnitTests.General; +using static System.String; + +namespace Benchmarks.AdoNet.Streaming; + +public class SqlServerStreamPartitionReadCheckpointBenchmark() : StreamPartitionReadCheckpointBenchmark(AdoNetInvariants.InvariantNameSqlServer, "OrleansStreamTest") +{ + public override void GlobalSetup() + { + base.GlobalSetup(); + SqlConnection.ClearAllPools(); + } +} + +/// +/// Measures exclusive ordered reads and epoch-fenced checkpoint updates across partitions. +/// +[WarmupCount(1), IterationCount(3), InvocationCount(1), MarkdownExporter] +public abstract class StreamPartitionReadCheckpointBenchmark(string invariant, string database) +{ + private const int OperationsPerInvoke = 1_000; + + private readonly Consumer _consumer = new(); + private IRelationalStorage _storage = default!; + private RelationalOrleansQueries _queries = default!; + private byte[] _payload = []; + private byte[] _streamIdBytes = []; + private string[] _partitionIds = []; + private long[] _ownerEpochs = []; + + [Params(1, 8)] + public int PartitionCount { get; set; } + + [Params(1, 32, 256)] + public int BatchSize { get; set; } + + [Params(1_000)] + public int PayloadSize { get; set; } + + [GlobalSetup] + public virtual void GlobalSetup() + { + _payload = new byte[PayloadSize]; + new Random(42).NextBytes(_payload); + _streamIdBytes = "benchmarkstream-0"u8.ToArray(); + _partitionIds = Enumerable.Range(0, PartitionCount).Select(i => $"QueueId-{i}").ToArray(); + _ownerEpochs = new long[PartitionCount]; + + var testing = RelationalStorageForTesting.SetupInstance(invariant, database).GetAwaiter().GetResult(); + if (IsNullOrEmpty(testing.CurrentConnectionString)) + { + throw new InvalidOperationException($"Database '{database}' not initialized"); + } + + _storage = RelationalStorage.CreateInstance(invariant, testing.CurrentConnectionString); + _queries = RelationalOrleansQueries.CreateInstance(invariant, testing.CurrentConnectionString).GetAwaiter().GetResult(); + } + + [IterationSetup] + public void IterationSetup() + { + _storage.ExecuteAsync( + "TRUNCATE TABLE OrleansStreamMessage; TRUNCATE TABLE OrleansStreamPartition").GetAwaiter().GetResult(); + + for (var i = 0; i < OperationsPerInvoke * PartitionCount; i++) + { + _queries.AppendStreamMessageAsync( + "ServiceId-0", + "ProviderId-0", + _partitionIds[i % PartitionCount], + _streamIdBytes, + 9, + _payload).GetAwaiter().GetResult(); + } + + for (var i = 0; i < PartitionCount; i++) + { + _ownerEpochs[i] = _queries.AcquireStreamPartitionAsync( + "ServiceId-0", + "ProviderId-0", + _partitionIds[i], + startFromNow: false).GetAwaiter().GetResult().OwnerEpoch; + } + } + + [Benchmark(OperationsPerInvoke = OperationsPerInvoke)] + [BenchmarkCategory("OrderedRead")] + public async Task ReadExclusiveOrderedBatches() + { + await Parallel.ForAsync(0, OperationsPerInvoke, async (i, cancellationToken) => + { + var messages = await _queries.ReadStreamMessagesAsync( + "ServiceId-0", + "ProviderId-0", + _partitionIds[i % PartitionCount], + afterMessageId: 0, + BatchSize); + _consumer.Consume(messages); + }); + } + + [Benchmark(OperationsPerInvoke = OperationsPerInvoke)] + [BenchmarkCategory("Checkpoint")] + public async Task AdvanceEpochFencedCheckpoints() + { + await Parallel.ForAsync(0, Math.Min(PartitionCount, OperationsPerInvoke), async (partition, cancellationToken) => + { + for (var i = partition; i < OperationsPerInvoke; i += PartitionCount) + { + var checkpoint = (i / PartitionCount) + 1L; + var result = await _queries.AdvanceStreamCheckpointAsync( + "ServiceId-0", + "ProviderId-0", + _partitionIds[partition], + _ownerEpochs[partition], + checkpoint, + cancellationToken); + if (result is not { Updated: true }) + { + throw new InvalidOperationException($"Checkpoint {checkpoint} did not advance partition {partition}."); + } + + _consumer.Consume(result); + } + }); + } +} + +public class SqlServerStreamPartitionCleanupBenchmark() : StreamPartitionCleanupBenchmark(AdoNetInvariants.InvariantNameSqlServer, "OrleansStreamTest") +{ + public override void GlobalSetup() + { + base.GlobalSetup(); + SqlConnection.ClearAllPools(); + } +} + +/// +/// Measures bounded cleanup sweeps while varying partition count, batch size, and eligible-row density. +/// +[WarmupCount(1), IterationCount(3), InvocationCount(1), MarkdownExporter] +public abstract class StreamPartitionCleanupBenchmark(string invariant, string database) +{ + private const int MessagesPerPartition = 1_000; + + private readonly Consumer _consumer = new(); + private IRelationalStorage _storage = default!; + private RelationalOrleansQueries _queries = default!; + private byte[] _payload = []; + private byte[] _streamIdBytes = []; + private string[] _partitionIds = []; + + [Params(1, 8)] + public int PartitionCount { get; set; } + + [Params(16, 256)] + public int CleanupBatchSize { get; set; } + + [Params(0.0, 0.5, 1.0)] + public double CleanupImpactRatio { get; set; } + + [Params(1_000)] + public int PayloadSize { get; set; } + + [GlobalSetup] + public virtual void GlobalSetup() + { + _payload = new byte[PayloadSize]; + new Random(42).NextBytes(_payload); + _streamIdBytes = "benchmarkstream-0"u8.ToArray(); + _partitionIds = Enumerable.Range(0, PartitionCount).Select(i => $"QueueId-{i}").ToArray(); + + var testing = RelationalStorageForTesting.SetupInstance(invariant, database).GetAwaiter().GetResult(); + if (IsNullOrEmpty(testing.CurrentConnectionString)) + { + throw new InvalidOperationException($"Database '{database}' not initialized"); + } + + _storage = RelationalStorage.CreateInstance(invariant, testing.CurrentConnectionString); + _queries = RelationalOrleansQueries.CreateInstance(invariant, testing.CurrentConnectionString).GetAwaiter().GetResult(); + } + + [IterationSetup] + public void IterationSetup() + { + _storage.ExecuteAsync( + "TRUNCATE TABLE OrleansStreamMessage; TRUNCATE TABLE OrleansStreamPartition").GetAwaiter().GetResult(); + + for (var partition = 0; partition < PartitionCount; partition++) + { + for (var message = 0; message < MessagesPerPartition; message++) + { + _queries.AppendStreamMessageAsync( + "ServiceId-0", + "ProviderId-0", + _partitionIds[partition], + _streamIdBytes, + 9, + _payload).GetAwaiter().GetResult(); + } + + var state = _queries.AcquireStreamPartitionAsync( + "ServiceId-0", + "ProviderId-0", + _partitionIds[partition], + startFromNow: false).GetAwaiter().GetResult(); + _queries.AdvanceStreamCheckpointAsync( + "ServiceId-0", + "ProviderId-0", + _partitionIds[partition], + state.OwnerEpoch, + MessagesPerPartition).GetAwaiter().GetResult(); + } + + var eligiblePerPartition = (int)(MessagesPerPartition * CleanupImpactRatio); + _storage.ExecuteAsync( + $""" + WITH Ranked AS + ( + SELECT + ServiceId, + ProviderId, + QueueId, + MessageId, + ROW_NUMBER() OVER (PARTITION BY QueueId ORDER BY MessageId) AS RowNumber + FROM OrleansStreamMessage + ) + UPDATE Message + SET CheckpointedOn = DATEADD(DAY, -2, SYSUTCDATETIME()) + FROM OrleansStreamMessage AS Message + INNER JOIN Ranked + ON Ranked.ServiceId = Message.ServiceId + AND Ranked.ProviderId = Message.ProviderId + AND Ranked.QueueId = Message.QueueId + AND Ranked.MessageId = Message.MessageId + WHERE Ranked.RowNumber <= {eligiblePerPartition}; + + UPDATE OrleansStreamPartition SET CleanupOn = DATEADD(SECOND, -1, SYSUTCDATETIME()); + """).GetAwaiter().GetResult(); + } + + [Benchmark] + [BenchmarkCategory("Cleanup", "CleanupImpact")] + public async Task CleanupStreamPartitions() + { + var results = await Task.WhenAll(_partitionIds.Select(partitionId => + _queries.CleanupStreamMessagesAsync( + "ServiceId-0", + "ProviderId-0", + partitionId, + retentionPeriodSeconds: 86_400, + maximumRetentionPeriodSeconds: null, + cleanupIntervalSeconds: 60, + CleanupBatchSize))); + _consumer.Consume(results); + } +} diff --git a/test/Extensions/Orleans.AdoNet.Tests/AdoNetOptionsValidatorTests.cs b/test/Extensions/Orleans.AdoNet.Tests/AdoNetOptionsValidatorTests.cs index 80c2cdaba35..a6257f84bfd 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/AdoNetOptionsValidatorTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/AdoNetOptionsValidatorTests.cs @@ -5,6 +5,7 @@ using Orleans.GrainDirectory.AdoNet; using Orleans.Runtime; using Orleans.Storage; +using Orleans.Streaming.AdoNet; using Orleans.Tests.SqlUtils; using UnitTests.StorageTests.Relational; @@ -205,6 +206,184 @@ public void Streaming_RequiresInvariant() AssertInvariantRequired(new AdoNetStreamOptionsValidator(options, "stream").ValidateConfiguration); } + [Theory] + [InlineData(1, true)] + [InlineData(0, false)] + [InlineData(-1, false)] + public void Streaming_ValidatesMaxMessagesPerRead(int maxMessagesPerRead, bool valid) + { + var options = ValidStreamOptions(); + options.MaxMessagesPerRead = maxMessagesPerRead; + + AssertStreamingValidation(valid, options, nameof(AdoNetStreamOptions.MaxMessagesPerRead)); + } + + [Theory] + [InlineData(1, true)] + [InlineData(0, false)] + [InlineData(-1, false)] + public void Streaming_ValidatesCheckpointPersistInterval(int seconds, bool valid) + { + var options = ValidStreamOptions(); + options.CheckpointPersistInterval = TimeSpan.FromSeconds(seconds); + + AssertStreamingValidation(valid, options, nameof(AdoNetStreamOptions.CheckpointPersistInterval)); + } + + [Theory] + [InlineData(1, true)] + [InlineData(0, false)] + [InlineData(-1, false)] + public void Streaming_ValidatesRetentionPeriod(int seconds, bool valid) + { + var options = ValidStreamOptions(); + options.RetentionPeriod = TimeSpan.FromSeconds(seconds); + + AssertStreamingValidation(valid, options, nameof(AdoNetStreamOptions.RetentionPeriod)); + } + + [Fact] + public void Streaming_RejectsSubSecondRetentionPeriod() + { + var options = ValidStreamOptions(); + options.RetentionPeriod = TimeSpan.FromMilliseconds(500); + + AssertStreamingValidation(false, options, nameof(AdoNetStreamOptions.RetentionPeriod)); + } + + [Theory] + [InlineData(1.1)] + [InlineData(1.9)] + public void Streaming_AllowsFractionalRetentionWhichRoundsUp(double seconds) + { + var options = ValidStreamOptions(); + options.RetentionPeriod = TimeSpan.FromSeconds(seconds); + + new AdoNetStreamOptionsValidator(options, "stream").ValidateConfiguration(); + Assert.Equal(2, AdoNetStreamTime.ToSqlSeconds(options.RetentionPeriod)); + } + + [Fact] + public void Streaming_AllowsNullMaximumRetentionPeriod() + { + var options = ValidStreamOptions(); + options.RetentionPeriod = TimeSpan.FromSeconds(30); + options.MaximumRetentionPeriod = null; + + // Should not throw: a null hard ceiling means no hard-retention diagnostics are ever produced. + new AdoNetStreamOptionsValidator(options, "stream").ValidateConfiguration(); + } + + [Theory] + [InlineData(30, 30, true)] // equal to the normal retention period is allowed + [InlineData(30, 31, true)] // greater than the normal retention period is allowed + [InlineData(30, 29, false)] // a hard ceiling tighter than normal retention is invalid + public void Streaming_ValidatesMaximumRetentionPeriodAgainstRetentionPeriod(int retentionSeconds, int maximumRetentionSeconds, bool valid) + { + var options = ValidStreamOptions(); + options.RetentionPeriod = TimeSpan.FromSeconds(retentionSeconds); + options.MaximumRetentionPeriod = TimeSpan.FromSeconds(maximumRetentionSeconds); + + if (valid) + { + new AdoNetStreamOptionsValidator(options, "stream").ValidateConfiguration(); + } + else + { + var exception = Assert.Throws(() => new AdoNetStreamOptionsValidator(options, "stream").ValidateConfiguration()); + Assert.Contains(nameof(AdoNetStreamOptions.MaximumRetentionPeriod), exception.Message, StringComparison.Ordinal); + } + } + + [Theory] + [InlineData(1, true)] + [InlineData(0, false)] + [InlineData(-1, false)] + public void Streaming_ValidatesCleanupInterval(int seconds, bool valid) + { + var options = ValidStreamOptions(); + options.CleanupInterval = TimeSpan.FromSeconds(seconds); + + AssertStreamingValidation(valid, options, nameof(AdoNetStreamOptions.CleanupInterval)); + } + + [Fact] + public void Streaming_RejectsSubSecondCleanupInterval() + { + var options = ValidStreamOptions(); + options.CleanupInterval = TimeSpan.FromMilliseconds(500); + + AssertStreamingValidation(false, options, nameof(AdoNetStreamOptions.CleanupInterval)); + } + + [Fact] + public void Streaming_RejectsRetentionWhoseCeilingExceedsSqlIntegerRange() + { + var options = ValidStreamOptions(); + options.RetentionPeriod = TimeSpan.FromSeconds(int.MaxValue) + TimeSpan.FromTicks(1); + + AssertStreamingValidation(false, options, nameof(AdoNetStreamOptions.RetentionPeriod)); + } + + [Theory] + [InlineData(1, true)] + [InlineData(0, false)] + [InlineData(-1, false)] + public void Streaming_ValidatesCleanupBatchSize(int cleanupBatchSize, bool valid) + { + var options = ValidStreamOptions(); + options.CleanupBatchSize = cleanupBatchSize; + + AssertStreamingValidation(valid, options, nameof(AdoNetStreamOptions.CleanupBatchSize)); + } + + [Theory] + [InlineData(1, true)] + [InlineData(0, false)] + [InlineData(-1, false)] + public void Streaming_ValidatesInitializationTimeout(int seconds, bool valid) + { + var options = ValidStreamOptions(); + options.InitializationTimeout = TimeSpan.FromSeconds(seconds); + + AssertStreamingValidation(valid, options, nameof(AdoNetStreamOptions.InitializationTimeout)); + } + + [Fact] + public void Streaming_Options_HaveRetentionSafeDefaults() + { + var options = new AdoNetStreamOptions(); + + Assert.False(options.StartFromNow); + Assert.Equal(TimeSpan.FromDays(1), options.RetentionPeriod); + Assert.Null(options.MaximumRetentionPeriod); + } + + private static AdoNetStreamOptions ValidStreamOptions() => new() + { + Invariant = AdoNetInvariants.InvariantNameSqlLite, + ConnectionString = "Data Source=:memory:", + MaxMessagesPerRead = 100, + CheckpointPersistInterval = TimeSpan.FromSeconds(5), + RetentionPeriod = TimeSpan.FromMinutes(1), + MaximumRetentionPeriod = TimeSpan.FromMinutes(5), + CleanupInterval = TimeSpan.FromMinutes(1), + CleanupBatchSize = 1000, + }; + + private static void AssertStreamingValidation(bool valid, AdoNetStreamOptions options, string propertyName) + { + if (valid) + { + new AdoNetStreamOptionsValidator(options, "stream").ValidateConfiguration(); + } + else + { + var exception = Assert.Throws(() => new AdoNetStreamOptionsValidator(options, "stream").ValidateConfiguration()); + Assert.Contains(propertyName, exception.Message, StringComparison.Ordinal); + } + } + private static void AssertValidation(bool valid, Action validate) { if (valid) diff --git a/test/Extensions/Orleans.AdoNet.Tests/StorageTests/Relational/RelationalOrleansQueriesUnitTests.cs b/test/Extensions/Orleans.AdoNet.Tests/StorageTests/Relational/RelationalOrleansQueriesUnitTests.cs index 354c8938909..c70550b84e7 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/StorageTests/Relational/RelationalOrleansQueriesUnitTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/StorageTests/Relational/RelationalOrleansQueriesUnitTests.cs @@ -46,12 +46,13 @@ public sealed class RelationalOrleansQueriesUnitTests private static readonly string[] StreamingQueryKeys = [ - "QueueStreamMessageKey", - "GetStreamMessagesKey", - "ConfirmStreamMessagesKey", - "FailStreamMessageKey", - "EvictStreamMessagesKey", - "EvictStreamDeadLettersKey", + "StreamSchemaVersionKey", + "AppendStreamMessageKey", + "AcquireStreamPartitionKey", + "ReadStreamMessagesKey", + "AdvanceStreamCheckpointKey", + "GetStreamPartitionBoundsKey", + "CleanupStreamMessagesKey", ]; private static readonly string[] DirectoryQueryKeys = @@ -316,186 +317,6 @@ public async Task MembershipMutation_ReturnsResultAndCapturesAllParameters() storage.VerifyComplete(); } - [Fact] - public async Task GetStreamMessages_ReturnsRowsSortedByQueueAndSequence() - { - var now = new DateTime(2026, 8, 27, 16, 0, 0, DateTimeKind.Utc); - var storage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys) - .ExpectRead( - Sql("GetStreamMessagesKey"), - CreateTable( - StreamMessageColumns, - ["service-e", "provider-a", "queue-a", 20L, 2, now, now.AddHours(1), now.AddMinutes(-2), now.AddMinutes(-1), new byte[] { 2, 0 }], - ["service-e", "provider-a", "queue-a", 3L, 1, now, now.AddHours(2), now.AddMinutes(-4), now.AddMinutes(-3), new byte[] { 0, 3 }])); - var queries = await StreamingQueries.CreateInstance(storage); - - var result = await queries.GetStreamMessagesAsync("service-e", "provider-a", "queue-a", 25, 4, 30, 60, 90, 10); - - Assert.Equal([3L, 20L], result.Select(message => message.MessageId)); - Assert.Equal([0, 3], result[0].Payload); - Assert.Equal(2, result[1].Dequeued); - AssertParameters( - AssertOperationCall(storage, Sql("GetStreamMessagesKey")), - ("ServiceId", "service-e"), - ("ProviderId", "provider-a"), - ("QueueId", "queue-a"), - ("MaxCount", 25), - ("MaxAttempts", 4), - ("VisibilityTimeout", 30), - ("RemovalTimeout", 60), - ("EvictionInterval", 90), - ("EvictionBatchSize", 10)); - storage.VerifyComplete(); - } - - [Fact] - public async Task GetStreamMessages_ReturnsEmptyWhenNoRowsExist() - { - var storage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys) - .ExpectRead(Sql("GetStreamMessagesKey")); - var queries = await StreamingQueries.CreateInstance(storage); - - var result = await queries.GetStreamMessagesAsync("service-f", "provider-b", "queue-b", 5, 2, 10, 20, 30, 4); - - Assert.Empty(result); - AssertParameters( - AssertOperationCall(storage, Sql("GetStreamMessagesKey")), - ("ServiceId", "service-f"), - ("ProviderId", "provider-b"), - ("QueueId", "queue-b"), - ("MaxCount", 5), - ("MaxAttempts", 2), - ("VisibilityTimeout", 10), - ("RemovalTimeout", 20), - ("EvictionInterval", 30), - ("EvictionBatchSize", 4)); - storage.VerifyComplete(); - } - - [Fact] - public async Task ConfirmStreamMessages_EmptySet_DoesNotExecuteMutation() - { - var storage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys); - var queries = await StreamingQueries.CreateInstance(storage); - - var result = await queries.ConfirmStreamMessagesAsync("service-g", "provider-c", "queue-c", []); - - Assert.Empty(result); - Assert.Single(storage.Calls); - storage.VerifyComplete(); - } - - [Fact] - public async Task ConfirmStreamMessages_CapturesReceiptParameters() - { - var storage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys) - .ExpectRead( - Sql("ConfirmStreamMessagesKey"), - CreateTable( - StreamConfirmationColumns, - ["service-g", "provider-c", "queue-c", 11L], - ["service-g", "provider-c", "queue-c", 12L])); - var queries = await StreamingQueries.CreateInstance(storage); - - var result = await queries.ConfirmStreamMessagesAsync( - "service-g", - "provider-c", - "queue-c", - [new(11, 3), new(12, 4)]); - - Assert.Equal([11L, 12L], result.Select(ack => ack.MessageId)); - Assert.All(result, ack => Assert.Equal("queue-c", ack.QueueId)); - AssertParameters( - AssertOperationCall(storage, Sql("ConfirmStreamMessagesKey")), - ("ServiceId", "service-g"), - ("ProviderId", "provider-c"), - ("QueueId", "queue-c"), - ("Items", "11:3|12:4")); - storage.VerifyComplete(); - - var singletonStorage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys) - .ExpectRead( - Sql("ConfirmStreamMessagesKey"), - CreateTable(StreamConfirmationColumns, ["service-g", "provider-c", "queue-c", 13L])); - var singletonQueries = await StreamingQueries.CreateInstance(singletonStorage); - - var singletonResult = await singletonQueries.ConfirmStreamMessagesAsync( - "service-g", - "provider-c", - "queue-c", - [new(13, 5)]); - - Assert.Equal([13L], singletonResult.Select(ack => ack.MessageId)); - AssertParameters( - AssertOperationCall(singletonStorage, Sql("ConfirmStreamMessagesKey")), - ("ServiceId", "service-g"), - ("ProviderId", "provider-c"), - ("QueueId", "queue-c"), - ("Items", "13:5")); - singletonStorage.VerifyComplete(); - } - - [Fact] - public async Task ReleaseStreamMessages_EmptySet_DoesNotExecuteMutation() - { - var storage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys); - var queries = await StreamingQueries.CreateInstance(storage); - - var result = await queries.ReleaseStreamMessagesAsync("service-h", "provider-d", "queue-d", []); - - Assert.Empty(result); - Assert.Single(storage.Calls); - storage.VerifyComplete(); - } - - [Fact] - public async Task ReleaseStreamMessages_CapturesReceiptParameters() - { - var storage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys) - .ExpectRead( - Sql("ConfirmStreamMessagesKey"), - CreateTable( - StreamConfirmationColumns, - ["service-h", "provider-d", "queue-d", 21L], - ["service-h", "provider-d", "queue-d", 22L])); - var queries = await StreamingQueries.CreateInstance(storage); - - var result = await queries.ReleaseStreamMessagesAsync( - "service-h", - "provider-d", - "queue-d", - [new(21, 5), new(22, 6)]); - - Assert.Equal([21L, 22L], result.Select(ack => ack.MessageId)); - AssertParameters( - AssertOperationCall(storage, Sql("ConfirmStreamMessagesKey")), - ("ServiceId", "service-h"), - ("ProviderId", "provider-d"), - ("QueueId", "queue-d"), - ("Items", "21:-5|22:-6")); - storage.VerifyComplete(); - - var singletonStorage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys) - .ExpectRead( - Sql("ConfirmStreamMessagesKey"), - CreateTable(StreamConfirmationColumns, ["service-h", "provider-d", "queue-d", 23L])); - var singletonQueries = await StreamingQueries.CreateInstance(singletonStorage); - - var singletonResult = await singletonQueries.ReleaseStreamMessagesAsync( - "service-h", - "provider-d", - "queue-d", - [new(23, 7)]); - - Assert.Equal([23L], singletonResult.Select(ack => ack.MessageId)); - AssertParameters( - AssertOperationCall(singletonStorage, Sql("ConfirmStreamMessagesKey")), - ("ServiceId", "service-h"), - ("ProviderId", "provider-d"), - ("QueueId", "queue-d"), - ("Items", "23:-7")); - singletonStorage.VerifyComplete(); - } [Fact] public async Task GrainDirectoryLookup_ReturnsSingleEntry() @@ -585,12 +406,18 @@ public async Task Operation_PropagatesStorageErrorAcrossProductionCopies() membershipStorage.VerifyComplete(); var streamingStorage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys) - .ExpectReadException(Sql("GetStreamMessagesKey"), expected); + .ExpectReadException(Sql("ReadStreamMessagesKey"), expected); var streamingQueries = await StreamingQueries.CreateInstance(streamingStorage); var streamingError = await Assert.ThrowsAsync( - () => streamingQueries.GetStreamMessagesAsync("service-error", "provider-error", "queue-error", 5, 2, 10, 20, 30, 4)); + () => streamingQueries.ReadStreamMessagesAsync( + "service-error", + "provider-error", + "queue-error", + afterMessageId: 0, + maxCount: 5, + TestContext.Current.CancellationToken)); Assert.Same(expected, streamingError); - AssertOperationCall(streamingStorage, Sql("GetStreamMessagesKey")); + AssertOperationCall(streamingStorage, Sql("ReadStreamMessagesKey")); streamingStorage.VerifyComplete(); var directoryStorage = ExpectQueryLoad(new ScriptedRelationalStorage(), DirectoryQueryKeys) @@ -687,40 +514,6 @@ public async Task UnregisterGrainActivationsAsync_ReturnsCountAndCapturesParamet storage.VerifyComplete(); } - [Fact] - public async Task QueueStreamMessageAsync_ReturnsAcknowledgementAndCapturesParameters() - { - byte[] payload = [0, 1, 127, 255]; - var storage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys) - .ExpectRead( - Sql("QueueStreamMessageKey"), - CreateTable( - StreamConfirmationColumns, - ["service-queue", "provider-queue", "queue-queue", 9_876_543_210L])); - var queries = await StreamingQueries.CreateInstance(storage); - - var result = await queries.QueueStreamMessageAsync( - "service-queue", - "provider-queue", - "queue-queue", - payload, - 3_600); - - Assert.Equal( - new AdoNetStreamMessageAck("service-queue", "provider-queue", "queue-queue", 9_876_543_210L), - result); - var call = AssertOperationCall(storage, Sql("QueueStreamMessageKey")); - AssertParameters( - call, - ("ServiceId", "service-queue"), - ("ProviderId", "provider-queue"), - ("QueueId", "queue-queue"), - ("Payload", payload), - ("ExpiryTimeout", 3_600)); - Assert.Equal(payload, Assert.IsType(Parameter(call, "Payload").Value)); - storage.VerifyComplete(); - } - [Fact] public async Task UpsertReminderRowAsync_ReturnsVersionAndCapturesParameters() { @@ -992,57 +785,6 @@ public async Task InsertMembershipVersionRowAsync_ReturnsResultAndCapturesDeploy storage.VerifyComplete(); } - [Fact] - public async Task FailStreamMessageAsync_ExecutesSentinelQueryAndCapturesParameters() - { - var storage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys) - .ExpectExecute(Sql("FailStreamMessageKey"), affectedRows: 1); - var queries = await StreamingQueries.CreateInstance(storage); - - await queries.FailStreamMessageAsync( - "service-fail", - "provider-fail", - "queue-fail", - 4_294_967_297L, - 8, - 7_200); - - AssertParameters( - AssertOperationCall(storage, Sql("FailStreamMessageKey"), ExpectedCallKind.Execute), - ("ServiceId", "service-fail"), - ("ProviderId", "provider-fail"), - ("QueueId", "queue-fail"), - ("MessageId", 4_294_967_297L), - ("MaxAttempts", 8), - ("RemovalTimeout", 7_200)); - storage.VerifyComplete(); - } - - [Theory] - [InlineData(0, "serviceId")] - [InlineData(1, "providerId")] - [InlineData(2, "queueId")] - public async Task QueueStreamMessageAsync_WithNullRequiredArgument_ThrowsArgumentNullException( - int nullIndex, - string expectedParameterName) - { - var arguments = new string?[] { "service-null", "provider-null", "queue-null" }; - arguments[nullIndex] = null; - var storage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys); - var queries = await StreamingQueries.CreateInstance(storage); - - var exception = await Assert.ThrowsAsync( - () => queries.QueueStreamMessageAsync( - arguments[0]!, - arguments[1]!, - arguments[2]!, - [1, 2, 3], - 60)); - - Assert.Equal(expectedParameterName, exception.ParamName); - AssertOnlyQueryLoadCall(storage); - storage.VerifyComplete(); - } [Theory] [InlineData(0, "clusterId")] @@ -1140,53 +882,6 @@ public async Task UnregisterGrainActivationsAsync_WithNullRequiredArgument_Throw storage.VerifyComplete(); } - [Fact] - public async Task EvictStreamMessagesAsync_ExecutesSentinelQueryAndCapturesParameters() - { - var storage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys) - .ExpectExecute(Sql("EvictStreamMessagesKey"), affectedRows: 37); - var queries = await StreamingQueries.CreateInstance(storage); - - await queries.EvictStreamMessagesAsync( - "service-evict", - "provider-evict", - "queue-evict", - 37, - 8, - 7_200); - - AssertParameters( - AssertOperationCall(storage, Sql("EvictStreamMessagesKey"), ExpectedCallKind.Execute), - ("ServiceId", "service-evict"), - ("ProviderId", "provider-evict"), - ("QueueId", "queue-evict"), - ("MaxCount", 37), - ("MaxAttempts", 8), - ("RemovalTimeout", 7_200)); - storage.VerifyComplete(); - } - - [Fact] - public async Task EvictStreamDeadLettersAsync_ExecutesSentinelQueryAndCapturesParameters() - { - var storage = ExpectQueryLoad(new ScriptedRelationalStorage(), StreamingQueryKeys) - .ExpectExecute(Sql("EvictStreamDeadLettersKey"), affectedRows: 43); - var queries = await StreamingQueries.CreateInstance(storage); - - await queries.EvictStreamDeadLettersAsync( - "service-dead-letter", - "provider-dead-letter", - "queue-dead-letter", - 43); - - AssertParameters( - AssertOperationCall(storage, Sql("EvictStreamDeadLettersKey"), ExpectedCallKind.Execute), - ("ServiceId", "service-dead-letter"), - ("ProviderId", "provider-dead-letter"), - ("QueueId", "queue-dead-letter"), - ("MaxCount", 43)); - storage.VerifyComplete(); - } [Fact] public async Task UpdateIAmAliveTimeAsync_ExecutesSentinelQueryAndCapturesUtcValue() @@ -1289,27 +984,6 @@ public void GetQueryKeyAndValue_WithDuplicateColumns_ReturnsExpectedPair() Assert.False(reader.Read()); } - private static readonly (string Name, Type Type)[] StreamMessageColumns = - [ - ("ServiceId", typeof(string)), - ("ProviderId", typeof(string)), - ("QueueId", typeof(string)), - ("MessageId", typeof(long)), - ("Dequeued", typeof(int)), - ("VisibleOn", typeof(DateTime)), - ("ExpiresOn", typeof(DateTime)), - ("CreatedOn", typeof(DateTime)), - ("ModifiedOn", typeof(DateTime)), - ("Payload", typeof(byte[])), - ]; - - private static readonly (string Name, Type Type)[] StreamConfirmationColumns = - [ - ("ServiceId", typeof(string)), - ("ProviderId", typeof(string)), - ("QueueId", typeof(string)), - ("MessageId", typeof(long)), - ]; private static readonly (string Name, Type Type)[] DirectoryEntryColumns = [ @@ -1325,7 +999,9 @@ private static ScriptedRelationalStorage ExpectQueryLoad( IEnumerable keys) => storage.ExpectRead( GetQueriesSql, - CreateQueryTable(keys.Select(key => (key, Sql(key))).ToArray())); + CreateQueryTable(keys + .Select(key => (key, key == "StreamSchemaVersionKey" ? "2" : Sql(key))) + .ToArray())); private static DataTable CreateQueryTable(params (string Key, string Query)[] queries) => CreateTable( diff --git a/test/Extensions/Orleans.AdoNet.Tests/StorageTests/Relational/RelationalStorageExtensionsTests.cs b/test/Extensions/Orleans.AdoNet.Tests/StorageTests/Relational/RelationalStorageExtensionsTests.cs index 355ff1dba37..63f59b36835 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/StorageTests/Relational/RelationalStorageExtensionsTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/StorageTests/Relational/RelationalStorageExtensionsTests.cs @@ -25,7 +25,8 @@ record => record.GetInt32(0) * 2, "@tenant", "north", size: 32, - dbType: DbType.AnsiString)); + dbType: DbType.AnsiString), + TestContext.Current.CancellationToken); Assert.Equal([84], results); var call = Assert.Single(storage.Calls); @@ -48,7 +49,11 @@ public async Task ReadAsync_FlowsAcrossResultSetsInOrder() CreateTable(("Value", typeof(int), 10), ("Value", typeof(int), 20)), CreateTable(("Value", typeof(int), 30))); - var results = await storage.ReadAsync(Sql, record => record.GetInt32(0), parameterProvider: null); + var results = await storage.ReadAsync( + Sql, + record => record.GetInt32(0), + parameterProvider: null, + TestContext.Current.CancellationToken); Assert.Equal([10, 20, 30], results); Assert.Single(storage.Calls); @@ -111,7 +116,11 @@ public async Task ReadAsync_RejectsNullSelector() var storage = new ScriptedRelationalStorage(); var exception = await Assert.ThrowsAsync( - () => storage.ReadAsync(Sql, (Func)null!, parameterProvider: null)); + () => storage.ReadAsync( + Sql, + (Func)null!, + parameterProvider: null, + TestContext.Current.CancellationToken)); Assert.Equal("selector", exception.ParamName); Assert.Empty(storage.Calls); diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetBatchContainerTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetBatchContainerTests.cs index 30102f14e64..075079bf638 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetBatchContainerTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetBatchContainerTests.cs @@ -44,7 +44,15 @@ public void AdoNetBatchContainer_FromMessage_CreatesContainer() var temp = new AdoNetBatchContainer(streamId, events, requestContext); var serializer = fixture.Serializer.GetSerializer(); var payload = serializer.SerializeToArray(temp); - var message = new AdoNetStreamMessage("MyServiceId", "MyProviderId", "MyQueueId", 123, 234, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, payload); + var message = new AdoNetStreamMessage( + "MyServiceId", + "MyProviderId", + "MyQueueId", + 123, + streamId.FullKey.ToArray(), + streamId.Namespace.Length, + DateTime.UtcNow, + payload); // act var container = AdoNetBatchContainer.FromMessage(serializer, message); @@ -54,7 +62,7 @@ public void AdoNetBatchContainer_FromMessage_CreatesContainer() Assert.Equal(events, container.Events); Assert.Equal(requestContext, container.RequestContext); Assert.Equal(new EventSequenceTokenV2(123), container.SequenceToken); - Assert.Equal(234, container.Dequeued); + Assert.Equal(0, container.Dequeued); } [Fact] @@ -79,6 +87,42 @@ public void AdoNetBatchContainer_ToMessagePayload_CreatesPayload() Assert.Equal(0, container.Dequeued); } + [Fact] + public void RecoverableDataAdapter_UsesIdentityColumnsAndDecodesPayloadLazily() + { + var serializer = fixture.Serializer.GetSerializer(); + var streamId = StreamId.Create("MyNamespace", "MyKey"); + var payload = AdoNetBatchContainer.ToMessagePayload( + serializer, + streamId, + [new TestModel(1)], + requestContext: null); + var message = new AdoNetStreamMessage( + "service", + "provider", + "queue", + 42, + streamId.FullKey.ToArray(), + streamId.Namespace.Length, + DateTime.UtcNow, + payload); + var adapter = new AdoNetRecoverableStreamDataAdapter(serializer); + + var position = adapter.GetStreamPosition(message); + var cached = adapter.FromQueueMessage( + position, + message, + DateTime.UtcNow, + size => new byte[size]); + + Assert.Equal(streamId, cached.StreamId); + Assert.Equal("42", adapter.GetOffset(ref cached)); + var batch = Assert.IsType(adapter.GetBatchContainer(ref cached)); + Assert.Equal(streamId, batch.StreamId); + Assert.Equal([new TestModel(1)], batch.GetEvents().Select(item => item.Item1)); + Assert.Equal(new EventSequenceTokenV2(42), batch.SequenceToken); + } + [Fact] public void AdoNetBatchContainer_GetEvents_ThrowsOnHalfBaked() { @@ -104,7 +148,15 @@ public void AdoNetBatchContainer_GetEvents_FiltersEvents() var temp = new AdoNetBatchContainer(streamId, events, requestContext); var serializer = fixture.Serializer.GetSerializer(); var payload = serializer.SerializeToArray(temp); - var message = new AdoNetStreamMessage("MyServiceId", "MyProviderId", "MyQueueId", 123, 234, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, payload); + var message = new AdoNetStreamMessage( + "MyServiceId", + "MyProviderId", + "MyQueueId", + 123, + streamId.FullKey.ToArray(), + streamId.Namespace.Length, + DateTime.UtcNow, + payload); // act var container = AdoNetBatchContainer.FromMessage(serializer, message); diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetClientStreamTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetClientStreamTests.cs index ff154f54a17..35b7a31abb5 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetClientStreamTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetClientStreamTests.cs @@ -138,12 +138,8 @@ public virtual Task AdoNetStreamConsumerOnDroppedClientTest() AdoNetStreamProviderName, StreamNamespace, _output, - async () => (await _testing.Storage.ReadAsync( - "SELECT COUNT(*) FROM OrleansStreamDeadLetter", - _ => { }, - (record, i, ct) => Task.FromResult(record.GetInt32(0)), - cancellationToken: cancellationToken)) - .Single(), + getDeliveryFailureCount: null, + waitForRetryTimeouts: true, cancellationToken: cancellationToken); } } \ No newline at end of file diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterFactoryTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterFactoryTests.cs index af94c551972..b02f5122146 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterFactoryTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterFactoryTests.cs @@ -84,7 +84,7 @@ public async Task AdoNetQueueAdapterFactory_CreatesAdapter() var streamOptions = new AdoNetStreamOptions { Invariant = invariant, - ConnectionString = _storage.ConnectionString + ConnectionString = _storage.ConnectionString, }; var clusterOptions = new ClusterOptions { @@ -106,6 +106,17 @@ public async Task AdoNetQueueAdapterFactory_CreatesAdapter() Assert.Equal(name, adapter.Name); Assert.False(adapter.IsRewindable); Assert.Equal(StreamProviderDirection.ReadWrite, adapter.Direction); + var queueId = factory.GetStreamQueueMapper().GetAllQueues().First(); + Assert.Same( + adapter.CreateReceiver(queueId), + factory.GetQueueAdapterCache().CreateQueueCache(queueId)); + var firstReceiver = adapter.CreateReceiver(queueId); + await firstReceiver.Shutdown(TimeSpan.FromSeconds(5)); + var reassignedReceiver = adapter.CreateReceiver(queueId); + Assert.NotSame(firstReceiver, reassignedReceiver); + Assert.Same( + reassignedReceiver, + factory.GetQueueAdapterCache().CreateQueueCache(queueId)); } /// @@ -119,7 +130,8 @@ public async Task AdoNetQueueAdapterFactory_GetsDeliveryFailureHandler() var streamOptions = new AdoNetStreamOptions { Invariant = invariant, - ConnectionString = _storage.ConnectionString + ConnectionString = _storage.ConnectionString, + FaultOnDeliveryFailure = true, }; var clusterOptions = new ClusterOptions { @@ -139,11 +151,11 @@ public async Task AdoNetQueueAdapterFactory_GetsDeliveryFailureHandler() // assert Assert.NotNull(handler); Assert.IsType(handler); - Assert.False(handler.ShouldFaultSubsriptionOnError); + Assert.True(handler.ShouldFaultSubsriptionOnError); } /// - /// Tests that the gets a instance. + /// Tests that the exposes its shared receiver/cache registry. /// [Fact] public void AdoNetQueueAdapterFactory_GetsQueueAdapterCache() @@ -171,7 +183,7 @@ public void AdoNetQueueAdapterFactory_GetsQueueAdapterCache() // assert Assert.NotNull(cache); - Assert.IsType(cache); + Assert.Same(factory, cache); } /// diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterReceiverTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterReceiverTests.cs deleted file mode 100644 index a85a1748ee9..00000000000 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterReceiverTests.cs +++ /dev/null @@ -1,349 +0,0 @@ -using Microsoft.Extensions.Logging.Abstractions; -using MySql.Data.MySqlClient; -using Orleans.Configuration; -using Orleans.Streaming.AdoNet; -using Orleans.Tests.SqlUtils; -using System.Runtime.CompilerServices; -using TestExtensions; -using UnitTests.General; -using static System.String; -using RelationalOrleansQueries = Orleans.Streaming.AdoNet.Storage.RelationalOrleansQueries; - -namespace Tester.AdoNet.Streaming; - -/// -/// Provider-independent lifecycle tests for . -/// -[Collection(TestEnvironmentFixture.DefaultCollection)] -[TestCategory("BVT"), TestCategory("AdoNet"), TestCategory("Streaming")] -[TestProvider("None")] -[TestSuite("BVT")] -[TestArea("Streaming")] -public class AdoNetQueueAdapterReceiverLifecycleTests(TestEnvironmentFixture fixture) -{ - [Fact] - public void AdoNetQueueAdapterReceiver_CanBeCreatedByAdapterFactory() => - RuntimeHelpers.RunClassConstructor(typeof(AdoNetQueueAdapter).TypeHandle); - - [Fact] - public async Task AdoNetQueueAdapterReceiver_Shutdown_WaitsForDequeueBookkeepingBeforeRelease() - { - var cancellationToken = TestContext.Current.CancellationToken; - var serviceId = $"Service-{Guid.NewGuid()}"; - var providerId = $"Provider-{Guid.NewGuid()}"; - var queueId = $"Queue-{Guid.NewGuid()}"; - var clusterOptions = new ClusterOptions { ServiceId = serviceId }; - var streamOptions = new AdoNetStreamOptions - { - VisibilityTimeout = TimeSpan.FromMinutes(5), - EvictionBatchSize = 0 - }; - var cacheOptions = new SimpleQueueCacheOptions(); - var serializer = fixture.Serializer.GetSerializer(); - var logger = NullLogger.Instance; - var payload = serializer.SerializeToArray(new AdoNetBatchContainer(StreamId.Create("MyNamespace", "MyKey"), [new TestModel(1)], null!)); - var now = DateTime.UtcNow; - var message = new AdoNetStreamMessage(serviceId, providerId, queueId, 42, 1, now.AddMinutes(5), now.AddHours(1), now, now, payload); - var dequeueStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var continueDequeue = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var queries = new BlockingStreamMessageQueries(message, dequeueStarted, continueDequeue, cancellationToken); - - var receiver = new AdoNetQueueAdapterReceiver(providerId, queueId, streamOptions, clusterOptions, cacheOptions, queries, serializer, logger); - var getTask = receiver.GetQueueMessagesAsync(1); - await dequeueStarted.Task.WaitAsync(cancellationToken); - - var shutdownTask = receiver.Shutdown(TimeSpan.FromSeconds(10)); - Assert.False(shutdownTask.IsCompleted); - - continueDequeue.SetResult(); - var dequeued = Assert.IsType(Assert.Single(await getTask.WaitAsync(cancellationToken))); - await shutdownTask.WaitAsync(cancellationToken); - - Assert.Equal(message.MessageId, dequeued.SequenceToken.SequenceNumber); - var released = Assert.Single(queries.Released); - Assert.Equal(message.MessageId, released.MessageId); - Assert.Equal(message.Dequeued, released.Dequeued); - } - - [GenerateSerializer] - [Alias("Tester.AdoNet.Streaming.AdoNetQueueAdapterReceiverLifecycleTests.TestModel")] - public record TestModel( - [property: Id(0)] int Value); - - private sealed class BlockingStreamMessageQueries( - AdoNetStreamMessage message, - TaskCompletionSource dequeueStarted, - TaskCompletionSource continueDequeue, - CancellationToken cancellationToken) : IStreamMessageQueries - { - public IList Released { get; private set; } = []; - - public async Task> GetStreamMessagesAsync( - string serviceId, - string providerId, - string queueId, - int maxCount, - int maxAttempts, - int visibilityTimeout, - int removalTimeout, - int evictionInterval, - int evictionBatchSize) - { - dequeueStarted.SetResult(); - await continueDequeue.Task.WaitAsync(cancellationToken); - return [message]; - } - - public Task> ConfirmStreamMessagesAsync( - string serviceId, - string providerId, - string queueId, - IList messages) => - throw new NotSupportedException(); - - public Task> ReleaseStreamMessagesAsync( - string serviceId, - string providerId, - string queueId, - IList messages) - { - Released = messages.ToList(); - return Task.FromResult>( - [new(serviceId, providerId, queueId, message.MessageId)]); - } - } -} - -/// -/// Tests for against SQL Server. -/// -[TestCategory("SqlServer"), TestCategory("BVT"), TestCategory("AdoNet"), TestCategory("Streaming")] -[TestProvider("SqlServer")] -[TestSuite("Functional")] -public class SqlServerAdoNetQueueAdapterReceiverTests(TestEnvironmentFixture fixture) : AdoNetQueueAdapterReceiverTests(AdoNetInvariants.InvariantNameSqlServer, fixture) -{ -} - -/// -/// Tests for against MySQL. -/// -[TestCategory("MySql"), TestCategory("BVT"), TestCategory("AdoNet"), TestCategory("Streaming")] -[TestProvider("MySql")] -[TestSuite("Functional")] -public class MySqlAdoNetQueueAdapterReceiverTests : AdoNetQueueAdapterReceiverTests -{ - public MySqlAdoNetQueueAdapterReceiverTests(TestEnvironmentFixture fixture) : base(AdoNetInvariants.InvariantNameMySql, fixture) - { - MySqlConnection.ClearAllPools(); - } -} - -/// -/// Tests for against PostgreSQL. -/// -[TestCategory("PostgreSql"), TestCategory("BVT"), TestCategory("AdoNet"), TestCategory("Streaming")] -[TestProvider("PostgreSql")] -[TestSuite("Functional")] -public class PostgreSqlAdoNetQueueAdapterReceiverTests(TestEnvironmentFixture fixture) : AdoNetQueueAdapterReceiverTests(AdoNetInvariants.InvariantNamePostgreSql, fixture) -{ -} - -/// -/// Tests for . -/// -[Collection(TestEnvironmentFixture.DefaultCollection)] -[TestCategory("AdoNet"), TestCategory("Streaming")] -[TestSuite("Functional")] -[TestArea("Streaming")] -public abstract class AdoNetQueueAdapterReceiverTests(string invariant, TestEnvironmentFixture fixture) : IAsyncLifetime -{ - private readonly TestEnvironmentFixture _fixture = fixture; - private RelationalStorageForTesting _testing = null!; - private IRelationalStorage _storage = null!; - private RelationalOrleansQueries _queries = null!; - - private const string TestDatabaseName = "OrleansStreamTest"; - - public async ValueTask InitializeAsync() - { - _testing = await RelationalStorageForTesting.SetupInstance( - invariant, - TestDatabaseName, - cancellationToken: TestContext.Current.CancellationToken); - Assert.SkipWhen(IsNullOrEmpty(_testing.CurrentConnectionString), $"Database '{TestDatabaseName}' not initialized"); - - _storage = _testing.Storage; - _queries = await RelationalOrleansQueries.CreateInstance(invariant, _storage.ConnectionString); - } - - /// - /// Tests that the can get and confirm messages. - /// - [Fact] - public async Task AdoNetQueueAdapterReceiver_GetsMessages_ConfirmsMessages() - { - // arrange - receiver - var serviceId = "MyServiceId"; - var clusterOptions = new ClusterOptions - { - ServiceId = serviceId - }; - var providerId = "MyProviderId"; - var queueId = "MyQueueId"; - var maxCount = 10; - var streamOptions = new AdoNetStreamOptions - { - Invariant = invariant, - ConnectionString = _storage.ConnectionString, - - // disable eviction for this test - EvictionBatchSize = 0 - }; - var cacheOptions = new SimpleQueueCacheOptions(); - var serializer = _fixture.Serializer.GetSerializer(); - var logger = NullLogger.Instance; - var receiver = new AdoNetQueueAdapterReceiver(providerId, queueId, streamOptions, clusterOptions, cacheOptions, _queries, serializer, logger); - await receiver.Initialize(TimeSpan.FromSeconds(10)); - - // arrange - data - var streamId = StreamId.Create("MyNamespace", "MyKey"); - var events = new List { new TestModel(1), new TestModel(2), new TestModel(3) }; - var context = new Dictionary { { "MyKey", "MyValue" } }; - var container = new AdoNetBatchContainer(streamId, events, context); - var payload = serializer.SerializeToArray(container); - - // arrange - enqueue (via storage) some invalid messages followed by a valid message - var ackExpired = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, 0); - var ackOtherQueueId = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId + "X", payload, 100); - var ackOtherProviderId = await _queries.QueueStreamMessageAsync(serviceId, providerId + "X", queueId, payload, 100); - var ackOtherServiceId = await _queries.QueueStreamMessageAsync(serviceId + "X", providerId, queueId, payload, 100); - var ackValid = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, 100); - - // act - dequeue messages via receiver - var dequeued = await receiver.GetQueueMessagesAsync(maxCount); - Assert.NotNull(dequeued); - var storedDequeued = (await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken)).ToDictionary(x => x.MessageId); - - // act - confirm messages via receiver - await receiver.MessagesDeliveredAsync(dequeued); - var storedConfirmed = (await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken)).ToDictionary(x => x.MessageId); - - // assert - dequeued messages are as expected - var single = Assert.IsType(Assert.Single(dequeued)); - Assert.NotNull(single.RequestContext); - Assert.Equal(streamId, single.StreamId); - Assert.Equal(events, single.Events); - Assert.Equal(context.Select(x => (x.Key, x.Value)), single.RequestContext.Select(x => (x.Key, x.Value))); - Assert.Equal(ackValid.MessageId, single.SequenceToken.SequenceNumber); - Assert.Equal(1, single.Dequeued); - - // assert - storage is as expected after dequeuing - Assert.Equal(5, storedDequeued.Count); - Assert.Equal(0, storedDequeued[ackExpired.MessageId].Dequeued); - Assert.Equal(0, storedDequeued[ackOtherQueueId.MessageId].Dequeued); - Assert.Equal(0, storedDequeued[ackOtherProviderId.MessageId].Dequeued); - Assert.Equal(0, storedDequeued[ackOtherServiceId.MessageId].Dequeued); - Assert.Equal(1, storedDequeued[ackValid.MessageId].Dequeued); - - // assert - stored confirmed messages - Assert.Equal(4, storedConfirmed.Count); - Assert.True(storedConfirmed.ContainsKey(ackExpired.MessageId)); - Assert.True(storedConfirmed.ContainsKey(ackOtherQueueId.MessageId)); - Assert.True(storedConfirmed.ContainsKey(ackOtherProviderId.MessageId)); - Assert.True(storedConfirmed.ContainsKey(ackOtherServiceId.MessageId)); - Assert.False(storedConfirmed.ContainsKey(ackValid.MessageId)); - } - - /// - /// Tests that shutting down a receiver immediately releases its unconfirmed messages. - /// - [Fact] - public async Task AdoNetQueueAdapterReceiver_Shutdown_ReleasesUnconfirmedMessages() - { - var cancellationToken = TestContext.Current.CancellationToken; - var serviceId = $"Service-{Guid.NewGuid()}"; - var providerId = $"Provider-{Guid.NewGuid()}"; - var queueId = $"Queue-{Guid.NewGuid()}"; - var clusterOptions = new ClusterOptions { ServiceId = serviceId }; - var streamOptions = new AdoNetStreamOptions - { - Invariant = invariant, - ConnectionString = _storage.ConnectionString, - VisibilityTimeout = TimeSpan.FromMinutes(5), - EvictionBatchSize = 0 - }; - var cacheOptions = new SimpleQueueCacheOptions(); - var serializer = _fixture.Serializer.GetSerializer(); - var logger = NullLogger.Instance; - var streamId = StreamId.Create("MyNamespace", "MyKey"); - var payload = serializer.SerializeToArray(new AdoNetBatchContainer(streamId, [new TestModel(1)], null!)); - var ack = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, 100); - - var receiver = new AdoNetQueueAdapterReceiver(providerId, queueId, streamOptions, clusterOptions, cacheOptions, _queries, serializer, logger); - var first = Assert.IsType( - Assert.Single(await receiver.GetQueueMessagesAsync(1).WaitAsync(cancellationToken))); - Assert.Equal(1, first.Dequeued); - - await receiver.Shutdown(TimeSpan.FromSeconds(10)).WaitAsync(cancellationToken); - - var replacement = new AdoNetQueueAdapterReceiver(providerId, queueId, streamOptions, clusterOptions, cacheOptions, _queries, serializer, logger); - var redelivered = Assert.IsType( - Assert.Single(await replacement.GetQueueMessagesAsync(1).WaitAsync(cancellationToken))); - Assert.Equal(ack.MessageId, redelivered.SequenceToken.SequenceNumber); - Assert.Equal(2, redelivered.Dequeued); - await replacement.MessagesDeliveredAsync([redelivered]).WaitAsync(cancellationToken); - await replacement.Shutdown(TimeSpan.FromSeconds(10)).WaitAsync(cancellationToken); - } - - /// - /// Tests that waits for the outstanding task. - /// - [Fact] - public async Task AdoNetQueueAdapterReceiver_Shutdown_WaitsForOutstandingTask() - { - var cancellationToken = TestContext.Current.CancellationToken; - var serviceId = $"Service-{Guid.NewGuid()}"; - var providerId = $"Provider-{Guid.NewGuid()}"; - var queueId = $"Queue-{Guid.NewGuid()}"; - var clusterOptions = new ClusterOptions { ServiceId = serviceId }; - var streamOptions = new AdoNetStreamOptions - { - Invariant = invariant, - ConnectionString = _storage.ConnectionString, - VisibilityTimeout = TimeSpan.FromMinutes(5), - EvictionBatchSize = 0 - }; - var cacheOptions = new SimpleQueueCacheOptions(); - var serializer = _fixture.Serializer.GetSerializer(); - var logger = NullLogger.Instance; - var receiver = new AdoNetQueueAdapterReceiver(providerId, queueId, streamOptions, clusterOptions, cacheOptions, _queries, serializer, logger); - var payload = serializer.SerializeToArray(new AdoNetBatchContainer(StreamId.Create("MyNamespace", "MyKey"), [new TestModel(1)], null!)); - var ack = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, 100); - - var getTask = receiver.GetQueueMessagesAsync(1); - await receiver.Shutdown(TimeSpan.FromSeconds(10)).WaitAsync(cancellationToken); - - Assert.True(getTask.IsCompleted); - var first = Assert.IsType(Assert.Single(await getTask.WaitAsync(cancellationToken))); - Assert.Equal(1, first.Dequeued); - - var replacement = new AdoNetQueueAdapterReceiver(providerId, queueId, streamOptions, clusterOptions, cacheOptions, _queries, serializer, logger); - var redelivered = Assert.IsType( - Assert.Single(await replacement.GetQueueMessagesAsync(1).WaitAsync(cancellationToken))); - Assert.Equal(ack.MessageId, redelivered.SequenceToken.SequenceNumber); - Assert.Equal(2, redelivered.Dequeued); - await replacement.MessagesDeliveredAsync([redelivered]).WaitAsync(cancellationToken); - await replacement.Shutdown(TimeSpan.FromSeconds(10)).WaitAsync(cancellationToken); - } - - public ValueTask DisposeAsync() => ValueTask.CompletedTask; - - [GenerateSerializer] - [Alias("Tester.AdoNet.Streaming.AdoNetQueueAdapterReceiverTests.TestModel")] - public record TestModel( - [property: Id(0)] int Value); -} \ No newline at end of file diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs index 2e2a471f8e0..a05f46174b2 100644 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetQueueAdapterTests.cs @@ -123,8 +123,7 @@ public async Task AdoNetQueueAdapter_EnqueuesMessages() var streamOptions = new AdoNetStreamOptions { Invariant = invariant, - ConnectionString = _storage.ConnectionString, - ExpiryTimeout = TimeSpan.FromSeconds(100) + ConnectionString = _storage.ConnectionString }; var serializer = _fixture.Serializer.GetSerializer(); var logger = NullLogger.Instance; @@ -144,97 +143,16 @@ public async Task AdoNetQueueAdapter_EnqueuesMessages() var afterEnqueued = DateTime.UtcNow.AddSeconds(1); // assert - stored messages are as expected - var stored = (await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken)).ToList(); - for (var i = 0; i < stored.Count; i++) - { - var item = stored[i]; - - Assert.Equal(serviceId, item.ServiceId); - Assert.Equal(providerId, item.ProviderId); - Assert.Equal(adoNetQueueId, item.QueueId); - Assert.NotEqual(0, item.MessageId); - Assert.Equal(0, item.Dequeued); - Assert.True(item.VisibleOn >= beforeEnqueued); - Assert.True(item.VisibleOn <= afterEnqueued); - Assert.True(item.ExpiresOn >= beforeEnqueued.Add(streamOptions.ExpiryTimeout)); - Assert.True(item.ExpiresOn <= afterEnqueued.Add(streamOptions.ExpiryTimeout)); - Assert.Equal(item.VisibleOn, item.CreatedOn); - Assert.Equal(item.VisibleOn, item.ModifiedOn); - - var serializedContainer = serializer.Deserialize(item.Payload); - Assert.NotNull(serializedContainer); - Assert.NotNull(serializedContainer.RequestContext); - Assert.Equal(streamId, serializedContainer.StreamId); - Assert.Null(serializedContainer.SequenceToken); - Assert.Equal(new[] { new TestModel(i + 1) }, serializedContainer.Events); - Assert.Single(serializedContainer.RequestContext); - Assert.Equal("MyValue", serializedContainer.RequestContext["MyKey"]); - Assert.Equal(0, serializedContainer.Dequeued); - } - } - - /// - /// Tests that the can enqueue messages that are visible to its receivers. - /// - [Fact] - public async Task AdoNetQueueAdapter_WiresUpReceiver() - { - // arrange - var serviceId = "MyServiceId"; - var clusterOptions = new ClusterOptions - { - ServiceId = serviceId - }; - var cacheOptions = new SimpleQueueCacheOptions(); - var providerId = "MyProviderId"; - var streamOptions = new AdoNetStreamOptions - { - Invariant = invariant, - ConnectionString = _storage.ConnectionString - }; - var serializer = _fixture.Serializer.GetSerializer(); - var logger = NullLogger.Instance; - var streamId = StreamId.Create("MyNamespace", "MyKey"); - var hashOptions = new HashRingStreamQueueMapperOptions { TotalQueueCount = 8 }; - var hashMapper = new HashRingBasedStreamQueueMapper(hashOptions, "MyQueue"); - var queueId = hashMapper.GetQueueForStream(streamId); - var adoMapper = new AdoNetStreamQueueMapper(hashMapper); - var adoNetQueueId = adoMapper.GetAdoNetQueueId(streamId); - var adapter = new AdoNetQueueAdapter(providerId, streamOptions, clusterOptions, cacheOptions, adoMapper, _queries, serializer, logger, _fixture.Services); - - // act - enqueue (via adapter) some messages - var beforeEnqueued = DateTime.UtcNow.AddSeconds(-1); - await adapter.QueueMessageBatchAsync(streamId, new[] { new TestModel(1) }, null!, new Dictionary { { "MyKey", 1 } }); - await adapter.QueueMessageBatchAsync(streamId, new[] { new TestModel(2) }, null!, new Dictionary { { "MyKey", 2 } }); - await adapter.QueueMessageBatchAsync(streamId, new[] { new TestModel(3) }, null!, new Dictionary { { "MyKey", 3 } }); - var afterEnqueued = DateTime.UtcNow.AddSeconds(1); - - // act - grab receiver and dequeue messages - var receiver = adapter.CreateReceiver(queueId); - await receiver.Initialize(TimeSpan.FromSeconds(10)); - var beforeDequeued = DateTime.UtcNow.AddSeconds(-1); - var messages = await receiver.GetQueueMessagesAsync(10, TestContext.Current.CancellationToken); - var afterDequeued = DateTime.UtcNow.AddSeconds(1); - - // assert - dequeued messages are as expected - Assert.NotNull(messages); - Assert.Equal(3, messages.Count); - for (var i = 0; i < messages.Count; i++) - { - var message = messages[i]; - - Assert.Equal(streamId, message.StreamId); - Assert.Equal([new TestModel(i + 1)], message.GetEvents().Select(x => x.Item1)); - Assert.True(message.ImportRequestContext()); - Assert.Equal(i + 1, RequestContext.Get("MyKey")); - } - - // assert - stored messages are as expected - var stored = (await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken)).ToList(); + var stored = (await _queries.ReadStreamMessagesAsync( + serviceId, + providerId, + adoNetQueueId, + afterMessageId: 0, + maxCount: 100, + TestContext.Current.CancellationToken)) + .OrderBy(static message => message.MessageId) + .ToList(); + Assert.Equal(3, stored.Count); for (var i = 0; i < stored.Count; i++) { var item = stored[i]; @@ -243,15 +161,11 @@ public async Task AdoNetQueueAdapter_WiresUpReceiver() Assert.Equal(providerId, item.ProviderId); Assert.Equal(adoNetQueueId, item.QueueId); Assert.NotEqual(0, item.MessageId); - Assert.Equal(1, item.Dequeued); - Assert.True(item.VisibleOn >= beforeDequeued.Add(streamOptions.VisibilityTimeout)); - Assert.True(item.VisibleOn <= afterDequeued.Add(streamOptions.VisibilityTimeout)); - Assert.True(item.ExpiresOn >= beforeEnqueued.Add(streamOptions.ExpiryTimeout)); - Assert.True(item.ExpiresOn <= afterEnqueued.Add(streamOptions.ExpiryTimeout)); + Assert.Equal(streamId.FullKey.ToArray(), item.StreamIdBytes); + Assert.Equal(streamId.Namespace.Length, item.StreamNamespaceLength); + Assert.Equal(streamId, item.StreamId); Assert.True(item.CreatedOn >= beforeEnqueued); Assert.True(item.CreatedOn <= afterEnqueued); - Assert.True(item.ModifiedOn >= beforeDequeued); - Assert.True(item.ModifiedOn <= afterDequeued); var serializedContainer = serializer.Deserialize(item.Payload); Assert.NotNull(serializedContainer); @@ -260,7 +174,7 @@ public async Task AdoNetQueueAdapter_WiresUpReceiver() Assert.Null(serializedContainer.SequenceToken); Assert.Equal(new[] { new TestModel(i + 1) }, serializedContainer.Events); Assert.Single(serializedContainer.RequestContext); - Assert.Equal(i + 1, serializedContainer.RequestContext["MyKey"]); + Assert.Equal("MyValue", serializedContainer.RequestContext["MyKey"]); Assert.Equal(0, serializedContainer.Dequeued); } } diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs new file mode 100644 index 00000000000..3ca139422d6 --- /dev/null +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetRecoverableStreamTests.cs @@ -0,0 +1,442 @@ +using System.Data; +using System.Reflection; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Logging.Abstractions; +using Orleans.Configuration; +using Orleans.Providers.Streams.Common; +using Orleans.Streaming.AdoNet; +using Orleans.Streaming.AdoNet.Storage; +using Orleans.Streams; + +namespace Tester.AdoNet.Streaming; + +[TestCategory("BVT"), TestCategory("AdoNet"), TestCategory("Streaming")] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Streaming")] +public class AdoNetRecoverableStreamTests +{ + [Fact] + public void ResolveCheckpointUpdate_ReturnsAuthoritativeStateForExpectedVersionConflict() + { + var update = new AdoNetStreamCheckpointUpdate( + "service", + "provider", + "queue", + OwnerEpoch: 7, + Checkpoint: 42, + Updated: false); + + var result = AdoNetRecoverableStream.ResolveCheckpointUpdate("service/provider/queue", 7, update); + + Assert.Equal("42", result.Checkpoint); + Assert.Equal("7", result.Version); + } + + [Fact] + public void ResolveCheckpointUpdate_ThrowsWhenPartitionOwnershipIsLost() + { + var update = new AdoNetStreamCheckpointUpdate( + "service", + "provider", + "queue", + OwnerEpoch: 8, + Checkpoint: 42, + Updated: false); + + var exception = Assert.Throws( + () => AdoNetRecoverableStream.ResolveCheckpointUpdate("service/provider/queue", 7, update)); + + Assert.Contains("ownership was lost", exception.Message); + Assert.Contains("service/provider/queue", exception.Message); + Assert.Contains("epoch 7", exception.Message); + } + + [Theory] + [InlineData(1.1, 2)] + [InlineData(1.9, 2)] + [InlineData(2.0, 2)] + public void ToSqlSeconds_RoundsUpWithoutShorteningRetention(double seconds, int expected) + => Assert.Equal(expected, AdoNetStreamTime.ToSqlSeconds(TimeSpan.FromSeconds(seconds))); + + [Fact] + public void ToSqlSeconds_ThrowsWhenCeilingExceedsSqlIntegerRange() + { + var value = TimeSpan.FromSeconds(int.MaxValue) + TimeSpan.FromTicks(1); + + Assert.Throws(() => AdoNetStreamTime.ToSqlSeconds(value)); + } + + [Fact] + public void CleanupParameters_UseRoundedIntegerSecondsAndNullableMaximum() + { + using var command = new SqlCommand(); + _ = new DbStoredQueries.Columns(command) + { + RetentionPeriodSeconds = AdoNetStreamTime.ToSqlSeconds(TimeSpan.FromSeconds(1.1)), + MaximumRetentionPeriodSeconds = null, + CleanupIntervalSeconds = AdoNetStreamTime.ToSqlSeconds(TimeSpan.FromSeconds(1.9)), + }; + + Assert.Equal(2, command.Parameters[nameof(DbStoredQueries.Columns.RetentionPeriodSeconds)].Value); + var maximum = command.Parameters[nameof(DbStoredQueries.Columns.MaximumRetentionPeriodSeconds)]; + Assert.Equal(DBNull.Value, maximum.Value); + Assert.Equal(DbType.Int32, maximum.DbType); + Assert.Equal(2, command.Parameters[nameof(DbStoredQueries.Columns.CleanupIntervalSeconds)].Value); + } + + [Fact] + public async Task Load_PropagatesCancellationAndDiscardsLateAcquisition() + { + var storage = new BlockingRelationalStorage(); + var source = new AdoNetRecoverableStream( + "service", + "provider", + "queue", + new AdoNetStreamOptions { StartFromNow = false }, + CreateQueries(storage), + NullLogger.Instance); + using var cancellation = new CancellationTokenSource(); + + var loadTask = source.Load(cancellation.Token).AsTask(); + await storage.AcquisitionStarted.Task; + cancellation.Cancel(); + + Assert.True(storage.CapturedCancellationToken.IsCancellationRequested); + Assert.False(source.AcquisitionCompletion.IsCompleted); + + storage.CompleteAcquisition(ownerEpoch: 1); + await Assert.ThrowsAnyAsync(() => loadTask); + Assert.True(source.AcquisitionCompletion.IsCompletedSuccessfully); + await Assert.ThrowsAsync( + () => source.Update("1", "1", TestContext.Current.CancellationToken).AsTask()); + } + + [Theory] + [InlineData(AcquisitionCompletionKind.Success)] + [InlineData(AcquisitionCompletionKind.Fault)] + [InlineData(AcquisitionCompletionKind.Canceled)] + public async Task ShutdownNotification_WaitsForAnyAcquisitionCompletionBeforeAllowingReplacement( + AcquisitionCompletionKind completionKind) + { + var queueId = QueueId.GetQueueId("queue", 0, 0); + var registry = new QueueAdapterReceiverRegistry(_ => new ReservedReceiver()); + var first = registry.GetOrCreate(queueId); + var acquisition = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var released = AdoNetQueueAdapterReceiver.NotifyShutdownAfterAcquisition( + acquisition.Task, + () => registry.Remove(queueId, first)); + + Assert.Same(first, registry.GetOrCreate(queueId)); + switch (completionKind) + { + case AcquisitionCompletionKind.Success: + acquisition.SetResult(); + break; + case AcquisitionCompletionKind.Fault: + acquisition.SetException(new InvalidOperationException("late failure")); + break; + case AcquisitionCompletionKind.Canceled: + acquisition.SetCanceled(TestContext.Current.CancellationToken); + break; + } + + await released; + + var replacement = registry.GetOrCreate(queueId); + Assert.NotSame(first, replacement); + Assert.Same(replacement, Assert.Single(registry.Receivers).Value); + } + + [Theory] + [InlineData(StreamQueryKind.Read)] + [InlineData(StreamQueryKind.Advance)] + [InlineData(StreamQueryKind.Cleanup)] + public async Task StreamingQueries_PropagateCancellationToken(StreamQueryKind queryKind) + { + var storage = new CapturingRelationalStorage(); + var queries = CreateQueries(storage); + using var cancellation = new CancellationTokenSource(); + + switch (queryKind) + { + case StreamQueryKind.Read: + _ = await queries.ReadStreamMessagesAsync( + "service", "provider", "queue", 0, 1, cancellation.Token); + break; + case StreamQueryKind.Advance: + _ = await queries.AdvanceStreamCheckpointAsync( + "service", "provider", "queue", 1, 1, cancellation.Token); + break; + case StreamQueryKind.Cleanup: + _ = await queries.CleanupStreamMessagesAsync( + "service", "provider", "queue", 1, 2, 3, 4, cancellation.Token); + break; + default: + throw new ArgumentOutOfRangeException(nameof(queryKind)); + } + + Assert.Equal(cancellation.Token, storage.CapturedCancellationToken); + } + + [Fact] + public async Task Read_UsesDistinctRoundedRetentionParameters() + { + var storage = new CapturingRelationalStorage(); + var source = new AdoNetRecoverableStream( + "service", + "provider", + "queue", + new AdoNetStreamOptions + { + MaxMessagesPerRead = 10, + RetentionPeriod = TimeSpan.FromSeconds(2.1), + MaximumRetentionPeriod = TimeSpan.FromSeconds(5.1), + CleanupInterval = TimeSpan.FromSeconds(3.1), + CleanupBatchSize = 9, + }, + CreateQueries(storage), + NullLogger.Instance); + + Assert.Empty(await source.Read(10, TestContext.Current.CancellationToken)); + + var parameters = storage.Parameters[nameof(DbStoredQueries.CleanupStreamMessagesKey)]; + Assert.Equal(3, parameters[nameof(DbStoredQueries.Columns.RetentionPeriodSeconds)]); + Assert.Equal(6, parameters[nameof(DbStoredQueries.Columns.MaximumRetentionPeriodSeconds)]); + Assert.Equal(4, parameters[nameof(DbStoredQueries.Columns.CleanupIntervalSeconds)]); + Assert.Equal(9, parameters[nameof(DbStoredQueries.Columns.CleanupBatchSize)]); + } + + [Theory] + [InlineData(null, 1L, null, false)] + [InlineData(0L, 1L, null, false)] + [InlineData(3L, 4L, null, false)] + [InlineData(0L, 4L, null, true)] + [InlineData(0L, 4L, 2L, true)] + public void HasRetentionGap_UsesNextMessageIdWhenRetainedHistoryIsEmpty( + long? checkpoint, + long nextMessageId, + long? earliestMessageId, + bool expected) + { + var state = new AdoNetStreamPartitionState( + "service", + "provider", + "queue", + OwnerEpoch: 1, + NextMessageId: nextMessageId, + Checkpoint: checkpoint, + EarliestMessageId: earliestMessageId, + TailMessageId: null); + + Assert.Equal(expected, AdoNetRecoverableStream.HasRetentionGap(state)); + } + + private static RelationalOrleansQueries CreateQueries(IRelationalStorage storage) + { + var queryValues = typeof(DbStoredQueries) + .GetProperties(BindingFlags.Instance | BindingFlags.NonPublic) + .ToDictionary(property => property.Name, property => + property.Name == nameof(DbStoredQueries.StreamSchemaVersionKey) ? "2" : property.Name); + return new RelationalOrleansQueries(storage, new DbStoredQueries(queryValues)); + } + + private sealed class BlockingRelationalStorage : IRelationalStorage + { + private readonly TaskCompletionSource acquisition = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource AcquisitionStarted { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public CancellationToken CapturedCancellationToken { get; private set; } + + public string InvariantName => AdoNetInvariants.InvariantNameSqlServer; + + public string ConnectionString => string.Empty; + + public void CompleteAcquisition(long ownerEpoch) + => acquisition.SetResult(new( + "service", + "provider", + "queue", + ownerEpoch, + NextMessageId: 1, + Checkpoint: 0, + EarliestMessageId: null, + TailMessageId: null)); + + public async Task> ReadAsync( + string query, + Action? parameterProvider, + Func> selector, + CommandBehavior commandBehavior = CommandBehavior.Default, + CancellationToken cancellationToken = default) + { + CapturedCancellationToken = cancellationToken; + AcquisitionStarted.TrySetResult(); + var state = await acquisition.Task; + using var command = new SqlCommand(); + parameterProvider?.Invoke(command); + var record = new DictionaryDataRecord(new Dictionary + { + [nameof(AdoNetStreamPartitionState.ServiceId)] = state.ServiceId, + [nameof(AdoNetStreamPartitionState.ProviderId)] = state.ProviderId, + [nameof(AdoNetStreamPartitionState.QueueId)] = state.QueueId, + [nameof(AdoNetStreamPartitionState.OwnerEpoch)] = state.OwnerEpoch, + [nameof(AdoNetStreamPartitionState.NextMessageId)] = state.NextMessageId, + [nameof(AdoNetStreamPartitionState.Checkpoint)] = state.Checkpoint, + [nameof(AdoNetStreamPartitionState.EarliestMessageId)] = state.EarliestMessageId, + [nameof(AdoNetStreamPartitionState.TailMessageId)] = state.TailMessageId, + }); + return [await selector(record, 0, cancellationToken)]; + } + + public Task ExecuteAsync( + string query, + Action? parameterProvider, + CommandBehavior commandBehavior = CommandBehavior.Default, + CancellationToken cancellationToken = default) => throw new NotSupportedException(); + } + + private sealed class DictionaryDataRecord(IReadOnlyDictionary values) : IDataRecord + { + private readonly string[] names = values.Keys.ToArray(); + + public object this[int i] => GetValue(i); + public object this[string name] => values[name] ?? DBNull.Value; + public int FieldCount => names.Length; + public bool GetBoolean(int i) => (bool)GetValue(i); + public byte GetByte(int i) => (byte)GetValue(i); + public long GetBytes(int i, long fieldOffset, byte[]? buffer, int bufferoffset, int length) => throw new NotSupportedException(); + public char GetChar(int i) => (char)GetValue(i); + public long GetChars(int i, long fieldoffset, char[]? buffer, int bufferoffset, int length) => throw new NotSupportedException(); + public IDataReader GetData(int i) => throw new NotSupportedException(); + public string GetDataTypeName(int i) => GetFieldType(i).Name; + public DateTime GetDateTime(int i) => (DateTime)GetValue(i); + public decimal GetDecimal(int i) => (decimal)GetValue(i); + public double GetDouble(int i) => (double)GetValue(i); + public Type GetFieldType(int i) => GetValue(i).GetType(); + public float GetFloat(int i) => (float)GetValue(i); + public Guid GetGuid(int i) => (Guid)GetValue(i); + public short GetInt16(int i) => (short)GetValue(i); + public int GetInt32(int i) => (int)GetValue(i); + public long GetInt64(int i) => (long)GetValue(i); + public string GetName(int i) => names[i]; + public int GetOrdinal(string name) => Array.IndexOf(names, name); + public string GetString(int i) => (string)GetValue(i); + public object GetValue(int i) => values[names[i]] ?? DBNull.Value; + public int GetValues(object[] destination) + { + var count = Math.Min(destination.Length, FieldCount); + for (var i = 0; i < count; i++) + { + destination[i] = GetValue(i); + } + + return count; + } + public bool IsDBNull(int i) => values[names[i]] is null or DBNull; + } + + private sealed class CapturingRelationalStorage : IRelationalStorage + { + public CancellationToken CapturedCancellationToken { get; private set; } + + public Dictionary> Parameters { get; } = []; + + public string InvariantName => AdoNetInvariants.InvariantNameSqlServer; + + public string ConnectionString => string.Empty; + + public async Task> ReadAsync( + string query, + Action? parameterProvider, + Func> selector, + CommandBehavior commandBehavior = CommandBehavior.Default, + CancellationToken cancellationToken = default) + { + CapturedCancellationToken = cancellationToken; + using var command = new SqlCommand(); + parameterProvider?.Invoke(command); + Parameters[query] = command.Parameters.Cast() + .ToDictionary(parameter => parameter.ParameterName, parameter => + parameter.Value is DBNull ? null : parameter.Value); + var records = query switch + { + nameof(DbStoredQueries.ReadStreamMessagesKey) => Array.Empty(), + nameof(DbStoredQueries.AdvanceStreamCheckpointKey) => + [ + Record( + (nameof(AdoNetStreamCheckpointUpdate.ServiceId), "service"), + (nameof(AdoNetStreamCheckpointUpdate.ProviderId), "provider"), + (nameof(AdoNetStreamCheckpointUpdate.QueueId), "queue"), + (nameof(AdoNetStreamCheckpointUpdate.OwnerEpoch), 1L), + (nameof(AdoNetStreamCheckpointUpdate.Checkpoint), 1L), + (nameof(AdoNetStreamCheckpointUpdate.Updated), true)), + ], + nameof(DbStoredQueries.CleanupStreamMessagesKey) => + [ + Record( + (nameof(AdoNetStreamCleanupResult.Ran), true), + (nameof(AdoNetStreamCleanupResult.DeletedCount), 0), + (nameof(AdoNetStreamCleanupResult.DeletedThroughMessageId), null), + (nameof(AdoNetStreamCleanupResult.HardDeletedCount), 0), + (nameof(AdoNetStreamCleanupResult.HardDeletedFromMessageId), null), + (nameof(AdoNetStreamCleanupResult.HardDeletedThroughMessageId), null), + (nameof(AdoNetStreamCleanupResult.Checkpoint), 0L), + (nameof(AdoNetStreamCleanupResult.EarliestMessageId), null), + (nameof(AdoNetStreamCleanupResult.TailMessageId), null)), + ], + _ => throw new ArgumentOutOfRangeException(nameof(query), query, null), + }; + var results = new List(); + foreach (var record in records) + { + results.Add(await selector(record, 0, cancellationToken)); + } + + return results; + } + + public Task ExecuteAsync( + string query, + Action? parameterProvider, + CommandBehavior commandBehavior = CommandBehavior.Default, + CancellationToken cancellationToken = default) => throw new NotSupportedException(); + + private static IDataRecord Record(params (string Name, object? Value)[] values) + => new DictionaryDataRecord(values.ToDictionary(value => value.Name, value => value.Value)); + } + + private sealed class ReservedReceiver : IQueueAdapterReceiver, IQueueCache + { + public Task Initialize(TimeSpan timeout) => Task.CompletedTask; + public Task> GetQueueMessagesAsync(int maxCount) => Task.FromResult>([]); + public Task MessagesDeliveredAsync(IList messages) => Task.CompletedTask; + public Task Shutdown(TimeSpan timeout) => Task.CompletedTask; + public int GetMaxAddCount() => 1; + public void AddToCache(IList messages) { } + public bool TryPurgeFromCache(out IList purgedItems) + { + purgedItems = null!; + return false; + } + public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) => throw new NotSupportedException(); + public bool IsUnderPressure() => false; + } + + public enum AcquisitionCompletionKind + { + Success, + Fault, + Canceled, + } + + public enum StreamQueryKind + { + Read, + Advance, + Cleanup, + } +} diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamFailureHandlerTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamFailureHandlerTests.cs deleted file mode 100644 index 06e910b24af..00000000000 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamFailureHandlerTests.cs +++ /dev/null @@ -1,222 +0,0 @@ -using Microsoft.Extensions.Logging.Abstractions; -using MySql.Data.MySqlClient; -using Orleans.Configuration; -using Orleans.Providers.Streams.Common; -using Orleans.Streaming.AdoNet; -using Orleans.Streams; -using Orleans.Tests.SqlUtils; -using UnitTests.General; -using static System.String; -using RelationalOrleansQueries = Orleans.Streaming.AdoNet.Storage.RelationalOrleansQueries; - -namespace Tester.AdoNet.Streaming; - -/// -/// Tests for against SQL Server. -/// -[TestCategory("SqlServer"), TestCategory("BVT"), TestCategory("AdoNet"), TestCategory("Streaming")] -[TestProvider("SqlServer")] -[TestSuite("Functional")] -public class SqlServerAdoNetStreamFailureHandlerTests() : AdoNetStreamFailureHandlerTests(AdoNetInvariants.InvariantNameSqlServer) -{ -} - -/// -/// Tests for against MySQL. -/// -[TestCategory("MySql"), TestCategory("BVT"), TestCategory("AdoNet"), TestCategory("Streaming")] -[TestProvider("MySql")] -[TestSuite("Functional")] -public class MySqlAdoNetStreamFailureHandlerTests : AdoNetStreamFailureHandlerTests -{ - public MySqlAdoNetStreamFailureHandlerTests() : base(AdoNetInvariants.InvariantNameMySql) - { - MySqlConnection.ClearAllPools(); - } -} - -/// -/// Tests for against PostgreSQL. -/// -[TestCategory("PostgreSql"), TestCategory("BVT"), TestCategory("AdoNet"), TestCategory("Streaming")] -[TestProvider("PostgreSql")] -[TestSuite("Functional")] -public class PostgreSqlAdoNetStreamFailureHandlerTests() : AdoNetStreamFailureHandlerTests(AdoNetInvariants.InvariantNamePostgreSql) -{ -} - -/// -/// Tests for . -/// -[TestCategory("AdoNet"), TestCategory("Streaming")] -[TestSuite("Functional")] -[TestArea("Streaming")] -public abstract class AdoNetStreamFailureHandlerTests(string invariant) : IAsyncLifetime -{ - private RelationalStorageForTesting _testing = null!; - private IRelationalStorage _storage = null!; - private RelationalOrleansQueries _queries = null!; - - private const string TestDatabaseName = "OrleansStreamTest"; - - public async ValueTask InitializeAsync() - { - _testing = await RelationalStorageForTesting.SetupInstance( - invariant, - TestDatabaseName, - cancellationToken: TestContext.Current.CancellationToken); - - Assert.SkipWhen(IsNullOrEmpty(_testing.CurrentConnectionString), $"Database '{TestDatabaseName}' not initialized"); - - _storage = _testing.Storage; - _queries = await RelationalOrleansQueries.CreateInstance(invariant, _testing.CurrentConnectionString); - } - - /// - /// Tests that a can be constructed. - /// - [Fact] - public void AdoNetStreamFailureHandler_Constructs() - { - // arrange - var faultOnFailure = false; - var streamOptions = new AdoNetStreamOptions - { - Invariant = invariant, - ConnectionString = _storage.ConnectionString - }; - var clusterOptions = new ClusterOptions - { - ServiceId = "MyServiceId" - }; - var mapper = new AdoNetStreamQueueMapper(new HashRingBasedStreamQueueMapper(new HashRingStreamQueueMapperOptions(), "MyQueuePrefix")); - var logger = NullLogger.Instance; - - // act - var handler = new AdoNetStreamFailureHandler(faultOnFailure, streamOptions, clusterOptions, mapper, _queries, logger); - - // assert - Assert.Equal(faultOnFailure, handler.ShouldFaultSubsriptionOnError); - } - - /// - /// Tests that a can move a poisoned message to dead letters. - /// - [Fact] - public async Task AdoNetStreamFailureHandler_OnDeliveryFailure_MovesPoisonedMessageToDeadLetters() - { - // arrange - handler - var providerId = "MyProviderId"; - var faultOnFailure = false; - var streamOptions = new AdoNetStreamOptions - { - Invariant = invariant, - ConnectionString = _storage.ConnectionString, - MaxAttempts = 1 - }; - var cacheOptions = new SimpleQueueCacheOptions(); - var agentOptions = new StreamPullingAgentOptions - { - MaxEventDeliveryTime = TimeSpan.FromSeconds(0) - }; - var clusterOptions = new ClusterOptions - { - ServiceId = "MyServiceId" - }; - var mapper = new AdoNetStreamQueueMapper(new HashRingBasedStreamQueueMapper(new HashRingStreamQueueMapperOptions(), "MyQueuePrefix")); - var logger = NullLogger.Instance; - var handler = new AdoNetStreamFailureHandler(faultOnFailure, streamOptions, clusterOptions, mapper, _queries, logger); - - // arrange - queue an expired message - var streamId = StreamId.Create("MyNamespace", "MyKey"); - var queueId = mapper.GetAdoNetQueueId(streamId); - var payload = new byte[] { 0xFF }; - - var ack = await _queries.QueueStreamMessageAsync(clusterOptions.ServiceId, providerId, queueId, payload, streamOptions.ExpiryTimeout.TotalSecondsCeiling()); - - // arrange - dequeue the message and make immediately available - await _queries.GetStreamMessagesAsync( - ack.ServiceId, - ack.ProviderId, - ack.QueueId, - cacheOptions.CacheSize, - streamOptions.MaxAttempts, - agentOptions.MaxEventDeliveryTime.TotalSecondsCeiling(), - streamOptions.DeadLetterEvictionTimeout.TotalSecondsCeiling(), - streamOptions.EvictionInterval.TotalSecondsCeiling(), - streamOptions.EvictionBatchSize); - Assert.Empty(await _storage.ReadAsync( - "SELECT * FROM OrleansStreamDeadLetter", - TestContext.Current.CancellationToken)); - - // act - clean up with max attempts of one so the message above is flagged - await handler.OnDeliveryFailure(GuidId.GetNewGuidId(), providerId, streamId, new EventSequenceTokenV2(ack.MessageId)); - - // assert - var dead = Assert.Single(await _storage.ReadAsync( - "SELECT * FROM OrleansStreamDeadLetter", - TestContext.Current.CancellationToken)); - Assert.Equal(clusterOptions.ServiceId, dead.ServiceId); - Assert.Equal(providerId, dead.ProviderId); - Assert.Equal(queueId, dead.QueueId); - Assert.Equal(ack.MessageId, dead.MessageId); - Assert.Equal(1, dead.Dequeued); - Assert.Equal(dead.CreatedOn.Add(streamOptions.ExpiryTimeout.SecondsCeiling()), dead.ExpiresOn); - Assert.Equal(dead.ModifiedOn, dead.VisibleOn); - Assert.Equal(dead.DeadOn.Add(streamOptions.DeadLetterEvictionTimeout.SecondsCeiling()), dead.RemoveOn); - Assert.Equal(payload, dead.Payload); - } - - /// - /// Tests that a can move a poisoned message to dead letters. - /// - [Fact] - public async Task AdoNetStreamFailureHandler_OnDeliveryFailure_DoesNotMoveHealthyMessageToDeadLetters() - { - // arrange - handler - var providerId = "MyProviderId"; - var faultOnFailure = false; - var streamOptions = new AdoNetStreamOptions - { - Invariant = invariant, - ConnectionString = _storage.ConnectionString - }; - var cacheOptions = new SimpleQueueCacheOptions(); - var agentOptions = new StreamPullingAgentOptions(); - var clusterOptions = new ClusterOptions - { - ServiceId = "MyServiceId" - }; - var mapper = new AdoNetStreamQueueMapper(new HashRingBasedStreamQueueMapper(new HashRingStreamQueueMapperOptions(), "MyQueuePrefix")); - var logger = NullLogger.Instance; - var handler = new AdoNetStreamFailureHandler(faultOnFailure, streamOptions, clusterOptions, mapper, _queries, logger); - - // arrange - queue an expired message - var streamId = StreamId.Create("MyNamespace", "MyKey"); - var queueId = mapper.GetAdoNetQueueId(streamId); - var payload = new byte[] { 0xFF }; - var ack = await _queries.QueueStreamMessageAsync(clusterOptions.ServiceId, providerId, queueId, payload, streamOptions.ExpiryTimeout.TotalSecondsCeiling()); - - // arrange - dequeue the message and make immediately available - await _queries.GetStreamMessagesAsync( - ack.ServiceId, - ack.ProviderId, - ack.QueueId, - cacheOptions.CacheSize, - streamOptions.MaxAttempts, - agentOptions.MaxEventDeliveryTime.TotalSecondsCeiling(), - streamOptions.DeadLetterEvictionTimeout.TotalSecondsCeiling(), - streamOptions.EvictionInterval.TotalSecondsCeiling(), - streamOptions.EvictionBatchSize); - - // act - clean up with max attempts of one so the message above is flagged - await handler.OnDeliveryFailure(GuidId.GetNewGuidId(), providerId, streamId, new EventSequenceTokenV2(ack.MessageId)); - - // assert - Assert.Empty(await _storage.ReadAsync( - "SELECT * FROM OrleansStreamDeadLetter", - TestContext.Current.CancellationToken)); - } - - public ValueTask DisposeAsync() => ValueTask.CompletedTask; -} \ No newline at end of file diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamMessageStreamIdTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamMessageStreamIdTests.cs new file mode 100644 index 00000000000..7d1ff206b7b --- /dev/null +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamMessageStreamIdTests.cs @@ -0,0 +1,131 @@ +using System.Text; +using Orleans.Runtime; +using Orleans.Streaming.AdoNet; + +namespace Tester.AdoNet.Streaming; + +/// +/// Tests that reconstructs the canonical +/// bytes and namespace boundary exactly as they would be +/// persisted in the StreamIdBytes/StreamNamespaceLength columns of +/// OrleansStreamMessage. These tests require no database and always run. +/// +[TestCategory("AdoNet"), TestCategory("Streaming"), TestCategory("BVT")] +[TestProvider("None")] +[TestSuite("BVT")] +[TestArea("Streaming")] +public sealed class AdoNetStreamMessageStreamIdTests +{ + private static AdoNetStreamMessage CreateMessage(StreamId streamId, long messageId = 1) => + new( + ServiceId: "service", + ProviderId: "provider", + QueueId: "queue", + MessageId: messageId, + StreamIdBytes: streamId.FullKey.ToArray(), + StreamNamespaceLength: streamId.Namespace.Length, + CreatedOn: DateTime.UtcNow, + Payload: [1, 2, 3]); + + [Fact] + public void StreamId_RoundTrips_WithNamespace() + { + var original = StreamId.Create("orders-namespace", Guid.NewGuid()); + + var message = CreateMessage(original); + + Assert.Equal(original, message.StreamId); + Assert.True(original.FullKey.Span.SequenceEqual(message.StreamIdBytes)); + Assert.Equal(original.Namespace.Length, message.StreamNamespaceLength); + Assert.True(original.Namespace.Span.SequenceEqual(message.StreamId.Namespace.Span)); + Assert.True(original.Key.Span.SequenceEqual(message.StreamId.Key.Span)); + } + + [Fact] + public void StreamId_RoundTrips_WithoutNamespace() + { + var original = StreamId.Create(ns: null, key: Guid.NewGuid()); + + var message = CreateMessage(original); + + Assert.Equal(0, original.Namespace.Length); + Assert.Equal(0, message.StreamNamespaceLength); + Assert.Equal(original, message.StreamId); + Assert.True(original.Key.Span.SequenceEqual(message.StreamId.Key.Span)); + Assert.True(original.FullKey.Span.SequenceEqual(message.StreamId.FullKey.Span)); + } + + [Fact] + public void StreamId_RoundTrips_WithStringKeyAndNamespace() + { + var original = StreamId.Create("tenant/42", "order-key-123"); + + var message = CreateMessage(original); + + Assert.Equal(original, message.StreamId); + Assert.Equal(Encoding.UTF8.GetByteCount("tenant/42"), message.StreamNamespaceLength); + Assert.Equal("order-key-123", Encoding.UTF8.GetString(message.StreamId.Key.Span)); + } + + [Theory] + [InlineData(null, "just-a-key")] + [InlineData("", "key-with-empty-namespace")] + [InlineData("ns", "k")] + [InlineData("namespace-with-unicode-\u00e9\u00e8", "key-with-unicode-\u00fc")] + public void StreamId_NamespaceBoundary_SeparatesNamespaceAndKeyExactly(string? ns, string key) + { + var original = StreamId.Create(ns, key); + var message = CreateMessage(original); + + // Re-derive the namespace/key split purely from the stored bytes and boundary, + // exactly as a storage-layer reader would, rather than trusting StreamId.Equals + // (which compares only the full key bytes and ignores the namespace boundary). + var namespaceBytes = message.StreamIdBytes.AsSpan(0, message.StreamNamespaceLength).ToArray(); + var keyBytes = message.StreamIdBytes.AsSpan(message.StreamNamespaceLength).ToArray(); + + Assert.True(original.Namespace.Span.SequenceEqual(namespaceBytes)); + Assert.True(original.Key.Span.SequenceEqual(keyBytes)); + Assert.Equal(key, Encoding.UTF8.GetString(keyBytes)); + } + + [Fact] + public void StreamId_DifferentNamespaceBoundary_ProducesDifferentNamespaceAndKeySplit() + { + // StreamId.Equals compares only the full key bytes, so an off-by-one boundary mutation + // would NOT be caught by an equality assertion alone. Guard the boundary explicitly by + // asserting the derived Namespace/Key spans themselves differ when the stored + // StreamNamespaceLength is shifted by one byte. + var original = StreamId.Create("ns", "key"); + var shiftedMessage = new AdoNetStreamMessage( + "service", "provider", "queue", 1, + original.FullKey.ToArray(), + original.Namespace.Length + 1, + DateTime.UtcNow, + [1]); + + var shifted = shiftedMessage.StreamId; + + // Sanity: the underlying full key bytes are unchanged. + Assert.True(original.FullKey.Span.SequenceEqual(shifted.FullKey.Span)); + + // But the namespace/key split has moved, and must be observably different. + Assert.NotEqual(original.Namespace.Length, shifted.Namespace.Length); + Assert.False(original.Namespace.Span.SequenceEqual(shifted.Namespace.Span)); + Assert.False(original.Key.Span.SequenceEqual(shifted.Key.Span)); + } + + [Fact] + public void StreamId_FullKey_IsExactByteConcatenationOfNamespaceAndKey() + { + var original = StreamId.Create("orders", "order-42"); + var message = CreateMessage(original); + + var reconstructedFullKey = message.StreamIdBytes; + var expectedFullKey = new byte[original.Namespace.Length + original.Key.Length]; + original.Namespace.Span.CopyTo(expectedFullKey.AsSpan(0, original.Namespace.Length)); + original.Key.Span.CopyTo(expectedFullKey.AsSpan(original.Namespace.Length)); + + Assert.Equal(expectedFullKey, reconstructedFullKey); + Assert.Equal(original.Namespace.Length + original.Key.Length, reconstructedFullKey.Length); + } +} diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs new file mode 100644 index 00000000000..0589d25d496 --- /dev/null +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamPartitionTests.cs @@ -0,0 +1,1220 @@ +using System.Data; +using System.Data.Common; +using System.Globalization; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Logging.Abstractions; +using MySql.Data.MySqlClient; +using Npgsql; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Streaming.AdoNet; +using Orleans.Streaming.AdoNet.Storage; +using Orleans.Streams; +using UnitTests.General; +using static System.String; + +namespace Tester.AdoNet.Streaming; + +/// +/// Tests the stream partition storage layer via against Sql Server. +/// +[TestCategory("SqlServer"), TestCategory("Functional"), TestCategory("AdoNet"), TestCategory("Streaming")] +[TestProvider("SqlServer")] +[TestSuite("Functional")] +public class SqlServerAdoNetStreamPartitionTests() : AdoNetStreamPartitionTests(AdoNetInvariants.InvariantNameSqlServer) +{ + /// + /// Concurrent appends to the same partition must be serialized by the partition-row lock, and a + /// rolled-back append must not permanently burn its allocated message identifier. + /// + [Fact] + public Task AppendStreamMessage_ConcurrentAppends_AreSerializedAndRollbackDoesNotBurnIds() => + VerifySqlServerConcurrentAppendsAreSerializedAndRollbackDoesNotBurnIds(); + + /// + /// A reader must never observe a message allocated by a transaction that has not yet committed. + /// + [Fact] + public Task ReadStreamMessages_ExcludesUncommittedInFlightAppend() => + VerifySqlServerReadExcludesUncommittedInFlightAppend(); + + [Fact] + public Task AppendStreamMessage_DifferentPartitionsDoNotShareTheAllocationLock() => + VerifySqlServerPartitionsAreIndependent(); +} + +/// +/// Tests the stream partition storage layer via against MySQL. +/// +[TestCategory("MySql"), TestCategory("Functional"), TestCategory("AdoNet"), TestCategory("Streaming")] +[TestProvider("MySql")] +[TestSuite("Functional")] +public class MySqlAdoNetStreamPartitionTests : AdoNetStreamPartitionTests +{ + public MySqlAdoNetStreamPartitionTests() : base(AdoNetInvariants.InvariantNameMySql) + { + MySqlConnection.ClearAllPools(); + } + + [Fact] + public Task AppendStreamMessage_RollbackRestoresAllocation() => VerifyMySqlRollbackRestoresAllocation(); +} + +/// +/// Tests the stream partition storage layer via against PostgreSQL. +/// +[TestCategory("PostgreSql"), TestCategory("Functional"), TestCategory("AdoNet"), TestCategory("Streaming")] +[TestProvider("PostgreSql")] +[TestSuite("Functional")] +public class PostgreSqlAdoNetStreamPartitionTests : AdoNetStreamPartitionTests +{ + public PostgreSqlAdoNetStreamPartitionTests() : base(AdoNetInvariants.InvariantNamePostgreSql) + { + NpgsqlConnection.ClearAllPools(); + } + + [Fact] + public Task AppendStreamMessage_RollbackRestoresAllocation() => VerifyPostgreSqlRollbackRestoresAllocation(); +} + +/// +/// Tests the stream partition storage layer via : transactional +/// append with rollback-safe allocation, exclusive ordered reads, epoch-fenced monotonic checkpoints, +/// bounded retention/cleanup with hard-ceiling diagnostics, partition independence, and explicit schema +/// version/query-key enforcement. +/// +[TestCategory("AdoNet"), TestCategory("Streaming")] +[TestSuite("Functional")] +[TestArea("Streaming")] +public abstract class AdoNetStreamPartitionTests(string invariant) : IAsyncLifetime +{ + private const string TestDatabaseName = "OrleansStreamTest"; + + private IRelationalStorage _storage = null!; + private CancelableRelationalQueries _queries = null!; + + public async ValueTask InitializeAsync() + { + var testing = await RelationalStorageForTesting.SetupInstance(invariant, TestDatabaseName); + Assert.SkipWhen(IsNullOrEmpty(testing.CurrentConnectionString), $"Database '{TestDatabaseName}' not initialized"); + + _storage = RelationalStorage.CreateInstance(invariant, testing.CurrentConnectionString); + _queries = new(await RelationalOrleansQueries.CreateInstance(invariant, testing.CurrentConnectionString)); + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + + #region Helpers + + // A large range keeps identifiers effectively unique across the partition-independence tests, + // which (unlike the single-partition tests) need two distinct queue ids within the same test. + private static string RandomServiceId(int max = 1_000_000) => $"ServiceId{Random.Shared.Next(max)}"; + + private static string RandomProviderId(int max = 1_000_000) => $"ProviderId{Random.Shared.Next(max)}"; + + private static string RandomQueueId(int max = 1_000_000) => $"QueueId{Random.Shared.Next(max)}"; + + private static byte[] RandomPayload(int size = 128) + { + var payload = new byte[size]; + Random.Shared.NextBytes(payload); + return payload; + } + + private static (byte[] StreamIdBytes, int StreamNamespaceLength) RandomStreamKey() + { + var streamId = StreamId.Create($"ns-{Guid.NewGuid():N}", Guid.NewGuid()); + return (streamId.FullKey.ToArray(), streamId.Namespace.Length); + } + + private Task AppendAsync(string serviceId, string providerId, string queueId, byte[]? payload = null) + { + var (streamIdBytes, nsLength) = RandomStreamKey(); + return _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, streamIdBytes, nsLength, payload ?? RandomPayload()); + } + + private Task AgePartitionMessagesAsync(string serviceId, string providerId, string queueId) => + _storage.ExecuteAsync( + """ + UPDATE OrleansStreamMessage + SET CreatedOn = @CreatedOn + WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId AND QueueId = @QueueId + """, + command => + { + AddParameter(command, "CreatedOn", DateTime.UtcNow.AddDays(-2)); + AddParameter(command, "ServiceId", serviceId); + AddParameter(command, "ProviderId", providerId); + AddParameter(command, "QueueId", queueId); + }); + + private Task AgeCheckpointedMessagesAsync(string serviceId, string providerId, string queueId) => + _storage.ExecuteAsync( + """ + UPDATE OrleansStreamMessage + SET CheckpointedOn = @CheckpointedOn + WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId AND QueueId = @QueueId + """, + command => + { + AddParameter(command, "CheckpointedOn", DateTime.UtcNow.AddDays(-2)); + AddParameter(command, "ServiceId", serviceId); + AddParameter(command, "ProviderId", providerId); + AddParameter(command, "QueueId", queueId); + }); + + private Task MakeCleanupDueAsync(string serviceId, string providerId, string queueId) => + _storage.ExecuteAsync( + """ + UPDATE OrleansStreamPartition + SET CleanupOn = @CleanupOn + WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId AND QueueId = @QueueId + """, + command => + { + AddParameter(command, "CleanupOn", DateTime.UtcNow.AddSeconds(-1)); + AddParameter(command, "ServiceId", serviceId); + AddParameter(command, "ProviderId", providerId); + AddParameter(command, "QueueId", queueId); + }); + + private static void AddParameter(IDbCommand command, string name, object value) + { + var parameter = command.CreateParameter(); + parameter.ParameterName = name; + parameter.Value = value; + command.Parameters.Add(parameter); + } + + #endregion Helpers + + #region Append: sequential allocation and partition independence + + [Fact] + public async Task AppendStreamMessage_AllocatesSequentialMessageIdsPerPartition() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + var first = await AppendAsync(serviceId, providerId, queueId); + var second = await AppendAsync(serviceId, providerId, queueId); + var third = await AppendAsync(serviceId, providerId, queueId); + + Assert.Equal(1, first.MessageId); + Assert.Equal(2, second.MessageId); + Assert.Equal(3, third.MessageId); + } + + [Fact] + public async Task AppendStreamMessage_ConcurrentAppendsAreGapFree() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + const int count = 32; + + var results = await Task.WhenAll(Enumerable.Range(0, count).Select(_ => AppendAsync(serviceId, providerId, queueId))); + + Assert.Equal(Enumerable.Range(1, count).Select(static value => (long)value), results.Select(static result => result.MessageId).Order()); + } + + [Fact] + public async Task AppendStreamMessage_PartitionsAreIndependent() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueIdA = RandomQueueId(); + var queueIdB = RandomQueueId(); + + await AppendAsync(serviceId, providerId, queueIdA); + var secondA = await AppendAsync(serviceId, providerId, queueIdA); + var firstB = await AppendAsync(serviceId, providerId, queueIdB); + + Assert.Equal(2, secondA.MessageId); + Assert.Equal(1, firstB.MessageId); + + var boundsA = await _queries.GetStreamPartitionBoundsAsync(serviceId, providerId, queueIdA); + var boundsB = await _queries.GetStreamPartitionBoundsAsync(serviceId, providerId, queueIdB); + + Assert.Equal(2, boundsA!.TailMessageId); + Assert.Equal(1, boundsB!.TailMessageId); + Assert.Equal(1, boundsA.EarliestMessageId); + Assert.Equal(1, boundsB.EarliestMessageId); + } + + [Fact] + public async Task AppendStreamMessage_ValidatesStreamNamespaceBoundary() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + var payload = RandomPayload(); + var (streamIdBytes, _) = RandomStreamKey(); + + // An empty stream key (no bytes at all) is never a valid canonical StreamId. + await Assert.ThrowsAsync(() => + _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, [], 0, payload)); + + // A negative boundary cannot separate namespace from key. + await Assert.ThrowsAsync(() => + _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, streamIdBytes, -1, payload)); + + // A boundary consuming the entire key would leave an empty stream key, which is invalid. + await Assert.ThrowsAsync(() => + _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, streamIdBytes, streamIdBytes.Length, payload)); + + await Assert.ThrowsAsync(() => + _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, null!, 0, payload)); + + await Assert.ThrowsAsync(() => + _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, streamIdBytes, 0, null!)); + + // Valid boundaries must NOT throw: a zero-length namespace (entire key, no namespace) and a + // namespace that consumes every byte except the last (a single-byte key) are both legal, and + // each still allocates the next sequential message identifier as normal. + var zeroNamespaceAck = await _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, streamIdBytes, 0, payload); + Assert.Equal(1, zeroNamespaceAck.MessageId); + var maxNamespaceAck = await _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, streamIdBytes, streamIdBytes.Length - 1, payload); + Assert.Equal(2, maxNamespaceAck.MessageId); + } + + [Fact] + public async Task AppendStreamMessage_PersistsCanonicalStreamIdFullKeyAndNamespaceBoundary() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + var streamId = StreamId.Create("orders", Guid.NewGuid()); + var payload = RandomPayload(); + + var ack = await _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, streamId.FullKey.ToArray(), streamId.Namespace.Length, payload); + var messages = await _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: 0, maxCount: 10); + var stored = Assert.Single(messages); + + Assert.Equal(ack.MessageId, stored.MessageId); + Assert.True(streamId.FullKey.Span.SequenceEqual(stored.StreamIdBytes)); + Assert.Equal(streamId.Namespace.Length, stored.StreamNamespaceLength); + Assert.Equal(streamId, stored.StreamId); + Assert.True(streamId.Namespace.Span.SequenceEqual(stored.StreamId.Namespace.Span)); + Assert.True(streamId.Key.Span.SequenceEqual(stored.StreamId.Key.Span)); + Assert.Equal(payload, stored.Payload); + } + + #endregion Append + + #region Acquire: ownership, epoch, and checkpoint initialization + + [Fact] + public async Task AcquireStreamPartition_InitializesCheckpointBeforeEarliestWhenNotStartingFromNow() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + await AppendAsync(serviceId, providerId, queueId); + await AppendAsync(serviceId, providerId, queueId); + + var state = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: false); + + Assert.Equal(1, state.OwnerEpoch); + Assert.Equal(0, state.Checkpoint); // one before the earliest retained message (id 1) + Assert.Equal(1, state.EarliestMessageId); + Assert.Equal(2, state.TailMessageId); + } + + [Fact] + public async Task AcquireStreamPartition_InitializesCheckpointAtTailWhenStartingFromNow() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + await AppendAsync(serviceId, providerId, queueId); + var last = await AppendAsync(serviceId, providerId, queueId); + + var state = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: true); + + Assert.Equal(last.MessageId, state.Checkpoint); + Assert.Equal(last.MessageId, state.TailMessageId); + } + + [Fact] + public async Task AcquireStreamPartition_OnNeverAppendedPartitionHasNoBounds() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + var state = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: false); + + Assert.Equal(1, state.OwnerEpoch); + Assert.Equal(0, state.Checkpoint); + Assert.Null(state.EarliestMessageId); + Assert.Null(state.TailMessageId); + } + + [Fact] + public async Task AcquireStreamPartition_ReacquisitionIncrementsOwnerEpochAndPreservesCheckpoint() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + await AppendAsync(serviceId, providerId, queueId); + + var first = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: false); + await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, first.OwnerEpoch, 1); + + var second = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: false); + + Assert.Equal(first.OwnerEpoch + 1, second.OwnerEpoch); + Assert.Equal(1, second.Checkpoint); // preserved from the prior owner, not re-initialized + } + + [Fact] + public async Task AcquireStreamPartition_OwnerEpochAndCheckpointArePerPartition() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueIdA = RandomQueueId(); + var queueIdB = RandomQueueId(); + + await AppendAsync(serviceId, providerId, queueIdA); + await AppendAsync(serviceId, providerId, queueIdB); + + var stateA1 = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueIdA, startFromNow: false); + await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueIdA, startFromNow: false); + var stateB1 = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueIdB, startFromNow: false); + + var boundsA = await _queries.GetStreamPartitionBoundsAsync(serviceId, providerId, queueIdA); + var boundsB = await _queries.GetStreamPartitionBoundsAsync(serviceId, providerId, queueIdB); + + Assert.Equal(stateA1.OwnerEpoch + 1, boundsA!.OwnerEpoch); + Assert.Equal(stateB1.OwnerEpoch, boundsB!.OwnerEpoch); + } + + #endregion Acquire + + #region Read: exclusive ordered ranges + + [Fact] + public async Task ReadStreamMessages_ReturnsExclusiveOrderedRangeRespectingMaxCount() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + var acks = new List(); + for (var i = 0; i < 5; i++) + { + acks.Add(await AppendAsync(serviceId, providerId, queueId)); + } + + var page = await _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: acks[1].MessageId, maxCount: 2); + + Assert.Equal([acks[2].MessageId, acks[3].MessageId], page.Select(m => m.MessageId)); + Assert.Equal(page.OrderBy(m => m.MessageId).Select(m => m.MessageId), page.Select(m => m.MessageId)); + } + + [Fact] + public async Task ReadStreamMessages_EmptyWhenAfterMessageIdIsAtOrBeyondTail() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + var only = await AppendAsync(serviceId, providerId, queueId); + + var atTail = await _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: only.MessageId, maxCount: 10); + var beyondTail = await _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: only.MessageId + 100, maxCount: 10); + + Assert.Empty(atTail); + Assert.Empty(beyondTail); + } + + [Fact] + public async Task ReadStreamMessages_ValidatesArgumentBounds() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + await Assert.ThrowsAsync(() => + _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: -1, maxCount: 1)); + + await Assert.ThrowsAsync(() => + _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: 0, maxCount: 0)); + + await Assert.ThrowsAsync(() => + _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: 0, maxCount: -5)); + + // Valid boundaries must NOT throw: afterMessageId = 0 (read from the very start) and + // maxCount = 1 (the smallest legal page size) are both legal and return the expected message. + var appended = await AppendAsync(serviceId, providerId, queueId); + var page = await _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: 0, maxCount: 1); + var onlyMessage = Assert.Single(page); + Assert.Equal(appended.MessageId, onlyMessage.MessageId); + } + + #endregion Read + + #region Checkpoint: monotonicity, non-regression, and epoch fencing + + [Fact] + public async Task AdvanceStreamCheckpoint_AdvancesMonotonicallyAndRejectsRegression() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + await AppendAsync(serviceId, providerId, queueId); + await AppendAsync(serviceId, providerId, queueId); + await AppendAsync(serviceId, providerId, queueId); + + var state = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: false); + + var advanceTo2 = await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, state.OwnerEpoch, 2); + Assert.NotNull(advanceTo2); + Assert.True(advanceTo2!.Updated); + Assert.Equal(2, advanceTo2.Checkpoint); + + var regressTo1 = await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, state.OwnerEpoch, 1); + Assert.NotNull(regressTo1); + Assert.False(regressTo1!.Updated); + Assert.Equal(2, regressTo1.Checkpoint); // unchanged by the rejected regression + + var sameValue = await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, state.OwnerEpoch, 2); + Assert.NotNull(sameValue); + Assert.False(sameValue!.Updated); // strictly-forward only: re-affirming the same value is not an advance + Assert.Equal(2, sameValue.Checkpoint); + + var advanceTo3 = await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, state.OwnerEpoch, 3); + Assert.True(advanceTo3!.Updated); + Assert.Equal(3, advanceTo3.Checkpoint); + } + + [Fact] + public async Task AdvanceStreamCheckpoint_RejectsCheckpointAtOrBeyondTail() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + var only = await AppendAsync(serviceId, providerId, queueId); + var state = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: false); + + // The checkpoint marks the last fully-processed message; it cannot reach or exceed the + // not-yet-allocated next message identifier. + var result = await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, state.OwnerEpoch, only.MessageId + 1); + + Assert.NotNull(result); + Assert.False(result!.Updated); + } + + [Fact] + public async Task AdvanceStreamCheckpoint_FencesStaleOwnerEpochAfterReacquisition() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + await AppendAsync(serviceId, providerId, queueId); + await AppendAsync(serviceId, providerId, queueId); + + var firstOwner = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: false); + var secondOwner = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: false); + Assert.True(secondOwner.OwnerEpoch > firstOwner.OwnerEpoch); + + // The former owner's epoch must be fenced: its checkpoint attempt must not apply. + var staleAttempt = await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, firstOwner.OwnerEpoch, 2); + Assert.NotNull(staleAttempt); + Assert.False(staleAttempt!.Updated); + + // The current owner's epoch must still be able to advance the checkpoint. + var currentAttempt = await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, secondOwner.OwnerEpoch, 2); + Assert.NotNull(currentAttempt); + Assert.True(currentAttempt!.Updated); + Assert.Equal(2, currentAttempt.Checkpoint); + } + + [Fact] + public async Task RecoverableStreamUpdate_ReturnsPersistedStateWhenExpectedVersionConflicts() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + await AppendAsync(serviceId, providerId, queueId); + await AppendAsync(serviceId, providerId, queueId); + + IStreamCheckpointStore store = new AdoNetRecoverableStream( + serviceId, + providerId, + queueId, + new AdoNetStreamOptions(), + _queries.Inner, + NullLogger.Instance); + var loaded = await store.Load(CancellationToken.None); + Assert.Equal("0", loaded.Checkpoint); + Assert.Equal("1", loaded.Version); + + var conflict = await store.Update("1", expectedVersion: "2", CancellationToken.None); + + Assert.Equal(loaded.Checkpoint, conflict.Checkpoint); + Assert.Equal(loaded.Version, conflict.Version); + + var updated = await store.Update("1", conflict.Version, CancellationToken.None); + Assert.Equal("1", updated.Checkpoint); + Assert.Equal(loaded.Version, updated.Version); + } + + [Fact] + public async Task RecoverableStreamUpdate_ThrowsWhenPartitionOwnershipIsLost() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + await AppendAsync(serviceId, providerId, queueId); + await AppendAsync(serviceId, providerId, queueId); + + IStreamCheckpointStore store = new AdoNetRecoverableStream( + serviceId, + providerId, + queueId, + new AdoNetStreamOptions(), + _queries.Inner, + NullLogger.Instance); + var loaded = await store.Load(TestContext.Current.CancellationToken); + var newOwner = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: false); + Assert.NotEqual(long.Parse(loaded.Version, CultureInfo.InvariantCulture), newOwner.OwnerEpoch); + + var exception = await Assert.ThrowsAsync( + () => store.Update("1", loaded.Version, TestContext.Current.CancellationToken).AsTask()); + + Assert.Contains("ownership was lost", exception.Message); + } + + [Fact] + public async Task AdvanceStreamCheckpoint_ReturnsNullForUnknownPartition() + { + var result = await _queries.AdvanceStreamCheckpointAsync(RandomServiceId(), RandomProviderId(), RandomQueueId(), ownerEpoch: 1, checkpoint: 1); + + Assert.Null(result); + } + + [Fact] + public async Task AdvanceStreamCheckpoint_ValidatesArgumentBounds() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + await Assert.ThrowsAsync(() => + _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, ownerEpoch: 0, checkpoint: 1)); + + await Assert.ThrowsAsync(() => + _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, ownerEpoch: -1, checkpoint: 1)); + + await Assert.ThrowsAsync(() => + _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, ownerEpoch: 1, checkpoint: -1)); + + // Valid boundaries must NOT throw: ownerEpoch = 1 (the smallest legal epoch) and + // checkpoint = 0 (the smallest legal checkpoint) are both legal argument values, even + // though there is no matching partition row here (which is reported via a null result, + // not an exception). + var result = await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, ownerEpoch: 1, checkpoint: 0); + Assert.Null(result); + } + + #endregion Checkpoint + + #region Bounds: partition state reporting + + [Fact] + public async Task GetStreamPartitionBounds_ReflectsCurrentState() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + Assert.Null(await _queries.GetStreamPartitionBoundsAsync(serviceId, providerId, queueId)); + + await AppendAsync(serviceId, providerId, queueId); + var second = await AppendAsync(serviceId, providerId, queueId); + var state = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: true); + await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, state.OwnerEpoch, second.MessageId); + + var bounds = await _queries.GetStreamPartitionBoundsAsync(serviceId, providerId, queueId); + + Assert.NotNull(bounds); + Assert.Equal(state.OwnerEpoch, bounds!.OwnerEpoch); + Assert.Equal(second.MessageId + 1, bounds.NextMessageId); + Assert.Equal(second.MessageId, bounds.Checkpoint); + Assert.Equal(1, bounds.EarliestMessageId); + Assert.Equal(second.MessageId, bounds.TailMessageId); + } + + #endregion Bounds + + #region Cleanup: retention, hard ceiling diagnostics, batching, and throttling + + [Fact] + public async Task CleanupStreamMessages_RemovesCheckpointedMessagesAfterRetentionElapses() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + var acks = new List(); + for (var i = 0; i < 3; i++) + { + acks.Add(await AppendAsync(serviceId, providerId, queueId)); + } + + var state = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: true); + await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, state.OwnerEpoch, acks[^1].MessageId); + + await AgeCheckpointedMessagesAsync(serviceId, providerId, queueId); + + var result = await _queries.CleanupStreamMessagesAsync( + serviceId, providerId, queueId, + retentionPeriodSeconds: 1, + maximumRetentionPeriodSeconds: null, + cleanupIntervalSeconds: 60, + cleanupBatchSize: 100); + + Assert.True(result.Ran); + Assert.Equal(3, result.DeletedCount); + Assert.Equal(acks[^1].MessageId, result.DeletedThroughMessageId); + Assert.Equal(0, result.HardDeletedCount); + Assert.Null(result.HardDeletedFromMessageId); + Assert.Null(result.HardDeletedThroughMessageId); + Assert.Null(result.EarliestMessageId); + Assert.Null(result.TailMessageId); + + var remaining = await _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: 0, maxCount: 100); + Assert.Empty(remaining); + } + + [Fact] + public async Task CleanupStreamMessages_RetentionStartsWhenCheckpointAdvances() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + var appended = await AppendAsync(serviceId, providerId, queueId); + await AgePartitionMessagesAsync(serviceId, providerId, queueId); + var state = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: false); + + await _queries.AdvanceStreamCheckpointAsync( + serviceId, + providerId, + queueId, + state.OwnerEpoch, + appended.MessageId); + await MakeCleanupDueAsync(serviceId, providerId, queueId); + var cleanup = await _queries.CleanupStreamMessagesAsync( + serviceId, + providerId, + queueId, + retentionPeriodSeconds: 60, + maximumRetentionPeriodSeconds: null, + cleanupIntervalSeconds: 60, + cleanupBatchSize: 100); + + Assert.True(cleanup.Ran); + Assert.Equal(0, cleanup.DeletedCount); + Assert.Single(await _queries.ReadStreamMessagesAsync( + serviceId, + providerId, + queueId, + afterMessageId: 0, + maxCount: 100)); + } + + [Fact] + public async Task CleanupStreamMessages_AppliesHardCeilingAndReportsDiagnostics() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + var acks = new List(); + for (var i = 0; i < 3; i++) + { + acks.Add(await AppendAsync(serviceId, providerId, queueId)); + } + + // No checkpoint is ever established, so these messages are ahead of the (null) checkpoint; + // only the hard retention ceiling can force their removal, and that removal must be + // reported distinctly from a normal, checkpoint-driven cleanup. + await AgePartitionMessagesAsync(serviceId, providerId, queueId); + + var result = await _queries.CleanupStreamMessagesAsync( + serviceId, providerId, queueId, + retentionPeriodSeconds: 60, + maximumRetentionPeriodSeconds: 120, + cleanupIntervalSeconds: 60, + cleanupBatchSize: 100); + + Assert.True(result.Ran); + Assert.Equal(3, result.DeletedCount); + Assert.Equal(3, result.HardDeletedCount); + Assert.Equal(acks[0].MessageId, result.HardDeletedFromMessageId); + Assert.Equal(acks[^1].MessageId, result.HardDeletedThroughMessageId); + Assert.Null(result.Checkpoint); + } + + [Fact] + public async Task FullyPurgedUncheckpointedHistory_RemainsDetectableAsRetentionGap() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + var initial = await _queries.AcquireStreamPartitionAsync( + serviceId, + providerId, + queueId, + startFromNow: false); + Assert.Equal(0, initial.Checkpoint); + Assert.Equal(1, initial.NextMessageId); + + for (var i = 0; i < 3; i++) + { + await AppendAsync(serviceId, providerId, queueId); + } + + await AgePartitionMessagesAsync(serviceId, providerId, queueId); + await MakeCleanupDueAsync(serviceId, providerId, queueId); + var cleanup = await _queries.CleanupStreamMessagesAsync( + serviceId, + providerId, + queueId, + retentionPeriodSeconds: 60, + maximumRetentionPeriodSeconds: 120, + cleanupIntervalSeconds: 60, + cleanupBatchSize: 100); + Assert.Equal(3, cleanup.HardDeletedCount); + + var bounds = await _queries.GetStreamPartitionBoundsAsync(serviceId, providerId, queueId); + Assert.NotNull(bounds); + Assert.Equal(0, bounds.Checkpoint); + Assert.Equal(4, bounds.NextMessageId); + Assert.Null(bounds.EarliestMessageId); + Assert.Null(bounds.TailMessageId); + Assert.True(AdoNetRecoverableStream.HasRetentionGap(bounds)); + } + + [Fact] + public async Task CleanupStreamMessages_RespectsBatchSizeAcrossMultipleRuns() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + var acks = new List(); + for (var i = 0; i < 5; i++) + { + acks.Add(await AppendAsync(serviceId, providerId, queueId)); + } + + var state = await _queries.AcquireStreamPartitionAsync(serviceId, providerId, queueId, startFromNow: true); + await _queries.AdvanceStreamCheckpointAsync(serviceId, providerId, queueId, state.OwnerEpoch, acks[^1].MessageId); + + await AgeCheckpointedMessagesAsync(serviceId, providerId, queueId); + + var first = await _queries.CleanupStreamMessagesAsync(serviceId, providerId, queueId, 1, null, cleanupIntervalSeconds: 1, cleanupBatchSize: 2); + Assert.True(first.Ran); + Assert.Equal(2, first.DeletedCount); + + await MakeCleanupDueAsync(serviceId, providerId, queueId); + + var second = await _queries.CleanupStreamMessagesAsync(serviceId, providerId, queueId, 1, null, cleanupIntervalSeconds: 1, cleanupBatchSize: 2); + Assert.True(second.Ran); + Assert.Equal(2, second.DeletedCount); + + await MakeCleanupDueAsync(serviceId, providerId, queueId); + + var third = await _queries.CleanupStreamMessagesAsync(serviceId, providerId, queueId, 1, null, cleanupIntervalSeconds: 1, cleanupBatchSize: 2); + Assert.True(third.Ran); + Assert.Equal(1, third.DeletedCount); // the final, partial batch + + var remaining = await _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: 0, maxCount: 100); + Assert.Empty(remaining); + } + + [Fact] + public async Task CleanupStreamMessages_ThrottlesRepeatedRunsWithinInterval() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + await AppendAsync(serviceId, providerId, queueId); + + var first = await _queries.CleanupStreamMessagesAsync( + serviceId, providerId, queueId, + retentionPeriodSeconds: 1, maximumRetentionPeriodSeconds: null, cleanupIntervalSeconds: 60, cleanupBatchSize: 100); + Assert.True(first.Ran); + + // Immediately repeating the call must be throttled by CleanupInterval and not run again. + var second = await _queries.CleanupStreamMessagesAsync( + serviceId, providerId, queueId, + retentionPeriodSeconds: 1, maximumRetentionPeriodSeconds: null, cleanupIntervalSeconds: 60, cleanupBatchSize: 100); + Assert.False(second.Ran); + Assert.Equal(0, second.DeletedCount); + } + + [Fact] + public async Task CleanupStreamMessages_ValidatesArgumentBounds() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + await Assert.ThrowsAsync(() => + _queries.CleanupStreamMessagesAsync(serviceId, providerId, queueId, retentionPeriodSeconds: 0, maximumRetentionPeriodSeconds: null, cleanupIntervalSeconds: 1, cleanupBatchSize: 1)); + + await Assert.ThrowsAsync(() => + _queries.CleanupStreamMessagesAsync(serviceId, providerId, queueId, retentionPeriodSeconds: 1, maximumRetentionPeriodSeconds: null, cleanupIntervalSeconds: 0, cleanupBatchSize: 1)); + + await Assert.ThrowsAsync(() => + _queries.CleanupStreamMessagesAsync(serviceId, providerId, queueId, retentionPeriodSeconds: 1, maximumRetentionPeriodSeconds: null, cleanupIntervalSeconds: 1, cleanupBatchSize: 0)); + + await Assert.ThrowsAsync(() => + _queries.CleanupStreamMessagesAsync(serviceId, providerId, queueId, retentionPeriodSeconds: 10, maximumRetentionPeriodSeconds: 5, cleanupIntervalSeconds: 1, cleanupBatchSize: 1)); + + // Valid boundaries must NOT throw: retentionPeriodSeconds/cleanupIntervalSeconds/cleanupBatchSize + // of exactly 1 (the smallest legal values), and a maximumRetentionPeriodSeconds exactly equal to + // retentionPeriodSeconds (the ceiling may legitimately coincide with the normal retention period). + await AppendAsync(serviceId, providerId, queueId); + var result = await _queries.CleanupStreamMessagesAsync(serviceId, providerId, queueId, retentionPeriodSeconds: 1, maximumRetentionPeriodSeconds: 1, cleanupIntervalSeconds: 1, cleanupBatchSize: 1); + Assert.True(result.Ran); + } + + #endregion Cleanup + + #region Explicit schema mismatch + + /// + /// An old or partially-migrated schema that is missing stream partition query keys must fail + /// explicitly and name the missing keys, rather than fail lazily or silently on first use. + /// + [Fact] + public async Task CreateInstance_MissingStreamPartitionQueryKeys_FailsExplicitly() + { + await _storage.ExecuteAsync( + "DELETE FROM OrleansQuery WHERE QueryKey IN ('AppendStreamMessageKey', 'StreamSchemaVersionKey')", + command => { }, + cancellationToken: TestContext.Current.CancellationToken); + + var exception = await Assert.ThrowsAsync(() => + RelationalOrleansQueries.CreateInstance(invariant, _storage.ConnectionString)); + + Assert.Contains("AppendStreamMessageKey", exception.Message, StringComparison.Ordinal); + Assert.Contains("StreamSchemaVersionKey", exception.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task CreateInstance_MixedLegacyAndStreamPartitionQueryKeys_FailsExplicitly() + { + await _storage.ExecuteAsync( + "INSERT INTO OrleansQuery (QueryKey, QueryText) VALUES ('QueueStreamMessageKey', 'legacy')", + command => { }, + cancellationToken: TestContext.Current.CancellationToken); + + var exception = await Assert.ThrowsAsync(() => + RelationalOrleansQueries.CreateInstance(invariant, _storage.ConnectionString)); + + Assert.Contains("QueueStreamMessageKey", exception.Message, StringComparison.Ordinal); + Assert.Contains("no in-place migration", exception.Message, StringComparison.OrdinalIgnoreCase); + } + + #endregion Explicit schema mismatch + + #region SQL Server: raw-connection concurrency and rollback semantics + + /// + /// Holds an append transaction open on one connection, proves a concurrent append on the same + /// partition cannot proceed while it is in-flight (serialization/ordering), then rolls the first + /// transaction back and proves the allocated identifier was not burned: the next successful + /// append reuses it and no message row was left behind. + /// + protected async Task VerifySqlServerConcurrentAppendsAreSerializedAndRollbackDoesNotBurnIds() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + var (streamIdBytes, nsLength) = RandomStreamKey(); + var payload = RandomPayload(); + + await using var firstConnection = new SqlConnection(_storage.ConnectionString); + await firstConnection.OpenAsync(TestContext.Current.CancellationToken); + await using var firstTransaction = (SqlTransaction)await firstConnection.BeginTransactionAsync(TestContext.Current.CancellationToken); + await using var firstCommand = CreateAppendCommand(firstConnection, firstTransaction, serviceId, providerId, queueId, streamIdBytes, nsLength, payload); + var firstMessageId = await ReadMessageId(firstCommand); + Assert.Equal(1, firstMessageId); + + // A concurrent append on the same partition must not be able to proceed while the first + // transaction still holds the partition-row lock. + await using (var secondConnection = new SqlConnection(_storage.ConnectionString)) + { + await secondConnection.OpenAsync(TestContext.Current.CancellationToken); + await using var secondCommand = secondConnection.CreateCommand(); + secondCommand.CommandType = CommandType.Text; + secondCommand.CommandText = "SET LOCK_TIMEOUT 0; EXECUTE AppendStreamMessage @ServiceId, @ProviderId, @QueueId, @StreamIdBytes, @StreamNamespaceLength, @Payload;"; + secondCommand.Parameters.AddWithValue("ServiceId", serviceId); + secondCommand.Parameters.AddWithValue("ProviderId", providerId); + secondCommand.Parameters.AddWithValue("QueueId", queueId); + secondCommand.Parameters.AddWithValue("StreamIdBytes", streamIdBytes); + secondCommand.Parameters.AddWithValue("StreamNamespaceLength", nsLength); + secondCommand.Parameters.AddWithValue("Payload", payload); + + var exception = await Assert.ThrowsAsync( + () => secondCommand.ExecuteReaderAsync(TestContext.Current.CancellationToken)); + Assert.Equal(51000, exception.Number); + Assert.Contains("initialization lock", exception.Message, StringComparison.Ordinal); + } + + // Rolling back must not burn the allocated identifier: it is reused by the next append, + // and no message row for it was left behind. + await firstTransaction.RollbackAsync(TestContext.Current.CancellationToken); + + var afterRollback = await _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, streamIdBytes, nsLength, payload); + Assert.Equal(1, afterRollback.MessageId); + + var rows = await _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: 0, maxCount: 100); + var stored = Assert.Single(rows); + Assert.Equal(1, stored.MessageId); + } + + /// + /// While an append transaction is in-flight and uncommitted, a reader must observe only + /// previously-committed messages; once the transaction commits, the message becomes visible. + /// + protected async Task VerifySqlServerReadExcludesUncommittedInFlightAppend() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + + var committed = await AppendAsync(serviceId, providerId, queueId); + + var (streamIdBytes, nsLength) = RandomStreamKey(); + var payload = RandomPayload(); + + await using var connection = new SqlConnection(_storage.ConnectionString); + await connection.OpenAsync(TestContext.Current.CancellationToken); + await using var transaction = (SqlTransaction)await connection.BeginTransactionAsync(TestContext.Current.CancellationToken); + await using var command = CreateAppendCommand(connection, transaction, serviceId, providerId, queueId, streamIdBytes, nsLength, payload); + var inFlightMessageId = await ReadMessageId(command); + Assert.Equal(committed.MessageId + 1, inFlightMessageId); + + var duringAppend = await _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: 0, maxCount: 100); + Assert.Equal([committed.MessageId], duringAppend.Select(m => m.MessageId)); + + await transaction.CommitAsync(TestContext.Current.CancellationToken); + + var afterCommit = await _queries.ReadStreamMessagesAsync(serviceId, providerId, queueId, afterMessageId: 0, maxCount: 100); + Assert.Equal([committed.MessageId, inFlightMessageId], afterCommit.Select(m => m.MessageId)); + } + + protected async Task VerifySqlServerPartitionsAreIndependent() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var firstQueueId = RandomQueueId(); + var secondQueueId = RandomQueueId(); + var (streamIdBytes, nsLength) = RandomStreamKey(); + var payload = RandomPayload(); + + await using var connection = new SqlConnection(_storage.ConnectionString); + await connection.OpenAsync(TestContext.Current.CancellationToken); + await using var transaction = (SqlTransaction)await connection.BeginTransactionAsync(TestContext.Current.CancellationToken); + await using var command = CreateAppendCommand(connection, transaction, serviceId, providerId, firstQueueId, streamIdBytes, nsLength, payload); + Assert.Equal(1, await ReadMessageId(command)); + + var independent = await _queries.AppendStreamMessageAsync(serviceId, providerId, secondQueueId, streamIdBytes, nsLength, payload) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(1, independent.MessageId); + + await transaction.RollbackAsync(TestContext.Current.CancellationToken); + } + + protected async Task VerifyMySqlRollbackRestoresAllocation() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + var (streamIdBytes, nsLength) = RandomStreamKey(); + var payload = RandomPayload(); + + await using var connection = new MySqlConnection(_storage.ConnectionString); + await connection.OpenAsync(TestContext.Current.CancellationToken); + await using var transaction = await connection.BeginTransactionAsync(TestContext.Current.CancellationToken); + await using (var command = connection.CreateCommand()) + { + command.Transaction = transaction; + command.CommandText = "CALL AppendStreamMessage(@ServiceId, @ProviderId, @QueueId, @StreamIdBytes, @StreamNamespaceLength, @Payload, FALSE)"; + command.Parameters.AddWithValue("ServiceId", serviceId); + command.Parameters.AddWithValue("ProviderId", providerId); + command.Parameters.AddWithValue("QueueId", queueId); + command.Parameters.AddWithValue("StreamIdBytes", streamIdBytes); + command.Parameters.AddWithValue("StreamNamespaceLength", nsLength); + command.Parameters.AddWithValue("Payload", payload); + Assert.Equal(1, await ReadMessageId(command)); + } + + await transaction.RollbackAsync(TestContext.Current.CancellationToken); + + var afterRollback = await _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, streamIdBytes, nsLength, payload); + Assert.Equal(1, afterRollback.MessageId); + } + + protected async Task VerifyPostgreSqlRollbackRestoresAllocation() + { + var serviceId = RandomServiceId(); + var providerId = RandomProviderId(); + var queueId = RandomQueueId(); + var (streamIdBytes, nsLength) = RandomStreamKey(); + var payload = RandomPayload(); + + await using var connection = new NpgsqlConnection(_storage.ConnectionString); + await connection.OpenAsync(TestContext.Current.CancellationToken); + await using var transaction = await connection.BeginTransactionAsync(TestContext.Current.CancellationToken); + await using (var command = connection.CreateCommand()) + { + command.Transaction = transaction; + command.CommandText = "SELECT * FROM AppendStreamMessage(@ServiceId, @ProviderId, @QueueId, @StreamIdBytes, @StreamNamespaceLength, @Payload)"; + command.Parameters.AddWithValue("ServiceId", serviceId); + command.Parameters.AddWithValue("ProviderId", providerId); + command.Parameters.AddWithValue("QueueId", queueId); + command.Parameters.AddWithValue("StreamIdBytes", streamIdBytes); + command.Parameters.AddWithValue("StreamNamespaceLength", nsLength); + command.Parameters.AddWithValue("Payload", payload); + Assert.Equal(1, await ReadMessageId(command)); + } + + await transaction.RollbackAsync(TestContext.Current.CancellationToken); + + var afterRollback = await _queries.AppendStreamMessageAsync(serviceId, providerId, queueId, streamIdBytes, nsLength, payload); + Assert.Equal(1, afterRollback.MessageId); + } + + private static SqlCommand CreateAppendCommand( + SqlConnection connection, + SqlTransaction? transaction, + string serviceId, + string providerId, + string queueId, + byte[] streamIdBytes, + int streamNamespaceLength, + byte[] payload) + { + var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandType = CommandType.StoredProcedure; + command.CommandText = "AppendStreamMessage"; + command.Parameters.AddWithValue("ServiceId", serviceId); + command.Parameters.AddWithValue("ProviderId", providerId); + command.Parameters.AddWithValue("QueueId", queueId); + command.Parameters.AddWithValue("StreamIdBytes", streamIdBytes); + command.Parameters.AddWithValue("StreamNamespaceLength", streamNamespaceLength); + command.Parameters.AddWithValue("Payload", payload); + return command; + } + + private static async Task ReadMessageId(DbCommand command) + { + await using var reader = await command.ExecuteReaderAsync(TestContext.Current.CancellationToken); + Assert.True(await reader.ReadAsync(TestContext.Current.CancellationToken)); + return reader.GetInt64(reader.GetOrdinal(nameof(AdoNetStreamMessage.MessageId))); + } + + private sealed class CancelableRelationalQueries(RelationalOrleansQueries inner) + { + public RelationalOrleansQueries Inner { get; } = inner; + + public Task AppendStreamMessageAsync( + string serviceId, + string providerId, + string queueId, + byte[] streamIdBytes, + int streamNamespaceLength, + byte[] payload) + => Inner.AppendStreamMessageAsync( + serviceId, + providerId, + queueId, + streamIdBytes, + streamNamespaceLength, + payload); + + public Task AcquireStreamPartitionAsync( + string serviceId, + string providerId, + string queueId, + bool startFromNow) + => Inner.AcquireStreamPartitionAsync( + serviceId, + providerId, + queueId, + startFromNow, + TestContext.Current.CancellationToken); + + public Task> ReadStreamMessagesAsync( + string serviceId, + string providerId, + string queueId, + long afterMessageId, + int maxCount) + => Inner.ReadStreamMessagesAsync( + serviceId, + providerId, + queueId, + afterMessageId, + maxCount, + TestContext.Current.CancellationToken); + + public Task AdvanceStreamCheckpointAsync( + string serviceId, + string providerId, + string queueId, + long ownerEpoch, + long checkpoint) + => Inner.AdvanceStreamCheckpointAsync( + serviceId, + providerId, + queueId, + ownerEpoch, + checkpoint, + TestContext.Current.CancellationToken); + + public Task GetStreamPartitionBoundsAsync( + string serviceId, + string providerId, + string queueId) + => Inner.GetStreamPartitionBoundsAsync(serviceId, providerId, queueId); + + public Task CleanupStreamMessagesAsync( + string serviceId, + string providerId, + string queueId, + int retentionPeriodSeconds, + int? maximumRetentionPeriodSeconds, + int cleanupIntervalSeconds, + int cleanupBatchSize) + => Inner.CleanupStreamMessagesAsync( + serviceId, + providerId, + queueId, + retentionPeriodSeconds, + maximumRetentionPeriodSeconds, + cleanupIntervalSeconds, + cleanupBatchSize, + TestContext.Current.CancellationToken); + } + + #endregion SQL Server +} diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs new file mode 100644 index 00000000000..50624918403 --- /dev/null +++ b/test/Extensions/Orleans.AdoNet.Tests/Streaming/AdoNetStreamSchemaTests.cs @@ -0,0 +1,188 @@ +namespace Tester.AdoNet.Streaming; + +[TestCategory("AdoNet"), TestCategory("Streaming"), TestCategory("BVT")] +[TestProvider("None")] +[TestSuite("BVT")] +[TestArea("Streaming")] +public sealed class AdoNetStreamSchemaTests +{ + [Theory] + [InlineData("SQLServer")] + [InlineData("PostgreSQL")] + [InlineData("MySQL")] + public void SchemaDefinesVersionedStreamPartitions(string provider) + { + var script = ReadScript(provider); + + Assert.Contains("CREATE TABLE OrleansStreamPartition", script, StringComparison.Ordinal); + Assert.Contains("NextMessageId BIGINT NOT NULL", script, StringComparison.Ordinal); + Assert.Contains("Checkpoint", script, StringComparison.Ordinal); + Assert.Contains("OwnerEpoch BIGINT NOT NULL", script, StringComparison.Ordinal); + Assert.Contains("CleanupOn", script, StringComparison.Ordinal); + + Assert.Contains("CREATE TABLE OrleansStreamMessage", script, StringComparison.Ordinal); + Assert.Contains("StreamIdBytes", script, StringComparison.Ordinal); + Assert.Contains("StreamNamespaceLength INT NOT NULL", script, StringComparison.Ordinal); + Assert.Contains("CheckpointedOn", script, StringComparison.Ordinal); + Assert.Contains("CheckpointedOn = COALESCE", script, StringComparison.Ordinal); + Assert.Contains("CheckpointedOn IS NULL", script, StringComparison.Ordinal); + Assert.Contains("CheckpointedOn <", script, StringComparison.Ordinal); + Assert.Contains("CreatedOn <", script, StringComparison.Ordinal); + Assert.Contains("Payload", script, StringComparison.Ordinal); + + Assert.Contains("'StreamSchemaVersionKey', '2'", script, StringComparison.Ordinal); + Assert.Contains("'AppendStreamMessageKey'", script, StringComparison.Ordinal); + Assert.Contains("'AcquireStreamPartitionKey'", script, StringComparison.Ordinal); + Assert.Contains("'ReadStreamMessagesKey'", script, StringComparison.Ordinal); + Assert.Contains("'AdvanceStreamCheckpointKey'", script, StringComparison.Ordinal); + Assert.Contains("'GetStreamPartitionBoundsKey'", script, StringComparison.Ordinal); + Assert.Contains("'CleanupStreamMessagesKey'", script, StringComparison.Ordinal); + Assert.Contains("NextMessageId", GetStoredQuery(script, "GetStreamPartitionBoundsKey"), StringComparison.Ordinal); + + Assert.DoesNotContain("CREATE TABLE OrleansStreamDeadLetter", script, StringComparison.Ordinal); + Assert.DoesNotContain("CREATE TABLE OrleansStreamControl", script, StringComparison.Ordinal); + Assert.Contains("no in-place migration", script, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void SqlServerAppendUsesUpdateOutputWhileHoldingPartitionLock() + { + var script = ReadScript("SQLServer"); + + Assert.Contains("UPDATE OrleansStreamPartition WITH (UPDLOCK, ROWLOCK)", script, StringComparison.Ordinal); + Assert.Contains("OUTPUT Inserted.NextMessageId - 1", script, StringComparison.Ordinal); + Assert.Contains("@LockOwner = 'Transaction'", script, StringComparison.Ordinal); + } + + [Fact] + public void PostgreSqlAppendUsesUpdateReturning() + { + var script = ReadScript("PostgreSQL"); + + Assert.Contains("UPDATE OrleansStreamPartition AS P", script, StringComparison.Ordinal); + Assert.Contains("RETURNING P.NextMessageId - 1 INTO _MessageId", script, StringComparison.Ordinal); + Assert.Contains("FOR UPDATE", script, StringComparison.Ordinal); + } + + [Fact] + public void MySqlAppendUsesSelectForUpdateThenUpdate() + { + var script = ReadScript("MySQL"); + + Assert.Contains("FOR UPDATE", script, StringComparison.Ordinal); + Assert.Contains("NextMessageId = _MessageId + 1", script, StringComparison.Ordinal); + Assert.Contains("IN _ManageTransaction BOOLEAN", script, StringComparison.Ordinal); + Assert.Contains("@Payload, TRUE)", script, StringComparison.Ordinal); + Assert.DoesNotContain("@@session.in_transaction", script, StringComparison.Ordinal); + } + + [Fact] + public void MySqlSchemaUsesUnicodePartitionIdentifiers() + { + var script = ReadScript("MySQL"); + + Assert.Contains("ServiceId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL", script, StringComparison.Ordinal); + Assert.Contains("IN _ServiceId VARCHAR(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin", script, StringComparison.Ordinal); + Assert.DoesNotContain("ServiceId NVARCHAR(150)", script, StringComparison.Ordinal); + Assert.DoesNotContain("ProviderId NVARCHAR(150)", script, StringComparison.Ordinal); + Assert.DoesNotContain("QueueId NVARCHAR(150)", script, StringComparison.Ordinal); + } + + [Fact] + public void MySqlScriptDoesNotProduceWhitespaceOnlyBatches() + { + var batches = ReadScript("MySQL") + .Replace("END$$", "END;", StringComparison.Ordinal) + .Split(["DELIMITER $$", "DELIMITER ;"], StringSplitOptions.RemoveEmptyEntries); + + Assert.DoesNotContain(batches, string.IsNullOrWhiteSpace); + } + + [Theory] + [InlineData("SQLServer", "SELECT @LockedNextMessageId", "SET @Now = SYSUTCDATETIME()", "INSERT INTO OrleansStreamMessage")] + [InlineData("PostgreSQL", "RETURNING P.NextMessageId - 1 INTO _MessageId", "_Now := clock_timestamp()", "INSERT INTO OrleansStreamMessage")] + [InlineData("MySQL", "FOR UPDATE;", "SET _Now = UTC_TIMESTAMP(6)", "INSERT INTO OrleansStreamMessage")] + public void AppendSamplesMessageTimestampAfterPartitionLock( + string provider, + string lockMarker, + string timestampMarker, + string messageInsertMarker) + { + var script = ReadScript(provider); + + AssertOrder(script, lockMarker, timestampMarker, messageInsertMarker); + } + + [Theory] + [InlineData("SQLServer", "SELECT @LockedCheckpoint", "SET @Now = SYSUTCDATETIME()", "SET CheckpointedOn")] + [InlineData("PostgreSQL", "FOR UPDATE;", "_Now := clock_timestamp()", "SET CheckpointedOn")] + [InlineData("MySQL", "FOR UPDATE;", "SET _Now = UTC_TIMESTAMP(6)", "SET CheckpointedOn")] + public void CheckpointSamplesEligibilityTimestampAfterPartitionLock( + string provider, + string lockMarker, + string timestampMarker, + string eligibilityMarker) + { + var script = ReadScript(provider); + var checkpointProcedure = script[script.IndexOf("AdvanceStreamCheckpoint", StringComparison.Ordinal)..]; + + AssertOrder(checkpointProcedure, lockMarker, timestampMarker, eligibilityMarker); + } + + [Theory] + [InlineData("SQLServer", "AND (@LockedCheckpoint IS NULL OR MessageId > @LockedCheckpoint)")] + [InlineData("PostgreSQL", "AND (_PreviousCheckpoint IS NULL OR M.MessageId > _PreviousCheckpoint)")] + [InlineData("MySQL", "AND (_CurrentCheckpoint IS NULL OR MessageId > _CurrentCheckpoint)")] + public void CheckpointEligibilityUpdateStartsAfterPreviousCheckpoint(string provider, string lowerBound) + { + var checkpointProcedure = GetProcedure(ReadScript(provider), "AdvanceStreamCheckpoint", "CleanupStreamMessages"); + + Assert.Contains(lowerBound, checkpointProcedure, StringComparison.Ordinal); + } + + [Theory] + [InlineData("SQLServer", "READPAST")] + [InlineData("PostgreSQL", "SKIP LOCKED")] + [InlineData("MySQL", "SKIP LOCKED")] + public void CleanupWaitsForLeadingEligibleRows(string provider, string skipLockedMarker) + { + var script = ReadScript(provider); + var cleanupStart = script.IndexOf("CleanupStreamMessages", StringComparison.Ordinal); + Assert.True(cleanupStart >= 0); + var cleanupProcedure = script[cleanupStart..]; + + Assert.Contains("ORDER BY MessageId", cleanupProcedure, StringComparison.Ordinal); + Assert.DoesNotContain(skipLockedMarker, cleanupProcedure, StringComparison.Ordinal); + } + + private static string ReadScript(string provider) => + File.ReadAllText(Path.Combine(AppContext.BaseDirectory, $"{provider}-Streaming.sql")); + + private static string GetStoredQuery(string script, string queryKey) + { + var start = script.IndexOf($"('{queryKey}'", StringComparison.Ordinal); + Assert.True(start >= 0); + var end = script.IndexOfAny(['\r', '\n'], start); + return script[start..end]; + } + + private static string GetProcedure(string script, string startMarker, string endMarker) + { + var start = script.IndexOf(startMarker, StringComparison.Ordinal); + Assert.True(start >= 0); + var end = script.IndexOf(endMarker, start + startMarker.Length, StringComparison.Ordinal); + Assert.True(end > start); + return script[start..end]; + } + + private static void AssertOrder(string text, params string[] markers) + { + var previous = -1; + foreach (var marker in markers) + { + var current = text.IndexOf(marker, previous + 1, StringComparison.Ordinal); + Assert.True(current > previous, $"Expected '{marker}' after index {previous}."); + previous = current; + } + } +} diff --git a/test/Extensions/Orleans.AdoNet.Tests/Streaming/RelationalOrleansQueriesTests.cs b/test/Extensions/Orleans.AdoNet.Tests/Streaming/RelationalOrleansQueriesTests.cs deleted file mode 100644 index 389383f4d79..00000000000 --- a/test/Extensions/Orleans.AdoNet.Tests/Streaming/RelationalOrleansQueriesTests.cs +++ /dev/null @@ -1,1230 +0,0 @@ -using System.Collections.Concurrent; -using System.Data; -using Microsoft.Data.SqlClient; -using MySql.Data.MySqlClient; -using Npgsql; -using Orleans.Configuration; -using Orleans.Streaming.AdoNet; -using Orleans.Streaming.AdoNet.Storage; -using UnitTests.General; -using static System.String; - -namespace Tester.AdoNet.Streaming; - -/// -/// Tests the relational storage layer via against Sql Server. -/// -[TestCategory("SqlServer"), TestCategory("Functional"), TestCategory("AdoNet"), TestCategory("Streaming")] -[TestProvider("SqlServer")] -[TestSuite("Functional")] -public class SqlServerRelationalOrleansQueriesTests() : RelationalOrleansQueriesTests(AdoNetInvariants.InvariantNameSqlServer, 90) -{ - [Fact] - public Task RelationalOrleansQueries_SerializesQueueMessageCommits() => - VerifySqlServerQueueMessageCommitsAreSerialized(TestContext.Current.CancellationToken); -} - -/// -/// Tests the relational storage layer via against MySQL. -/// -[TestCategory("MySql"), TestCategory("Functional"), TestCategory("AdoNet"), TestCategory("Streaming")] -[TestProvider("MySql")] -[TestSuite("Functional")] -public class MySqlRelationalOrleansQueriesTests : RelationalOrleansQueriesTests -{ - public MySqlRelationalOrleansQueriesTests() : base(AdoNetInvariants.InvariantNameMySql, 20) - { - MySqlConnection.ClearAllPools(); - } - - [Fact] - public Task RelationalOrleansQueries_OrdersProviderResults() => - VerifyProviderResultsAreOrdered(TestContext.Current.CancellationToken); -} - -/// -/// Tests the relational storage layer via against PostgreSQL. -/// -[TestCategory("PostgreSql"), TestCategory("Functional"), TestCategory("AdoNet"), TestCategory("Streaming")] -[TestProvider("PostgreSql")] -[TestSuite("Functional")] -public class PostgreSqlRelationalOrleansQueriesTests : RelationalOrleansQueriesTests -{ - public PostgreSqlRelationalOrleansQueriesTests() : base(AdoNetInvariants.InvariantNamePostgreSql, 99) - { - NpgsqlConnection.ClearAllPools(); - } -} - -/// -/// Tests the relational storage layer via . -/// -[TestCategory("AdoNet"), TestCategory("Streaming")] -[TestSuite("Functional")] -[TestArea("Streaming")] -public abstract class RelationalOrleansQueriesTests(string invariant, int concurrency = 100) : IAsyncLifetime -{ - private const string TestDatabaseName = "OrleansStreamTest"; - - private IRelationalStorage _storage = null!; - private RelationalOrleansQueries _queries = null!; - - public async ValueTask InitializeAsync() - { - var testing = await RelationalStorageForTesting.SetupInstance( - invariant, - TestDatabaseName, - cancellationToken: TestContext.Current.CancellationToken); - Assert.SkipWhen(IsNullOrEmpty(testing.CurrentConnectionString), $"Database '{TestDatabaseName}' not initialized"); - - _storage = RelationalStorage.CreateInstance(invariant, testing.CurrentConnectionString); - - _queries = await RelationalOrleansQueries.CreateInstance(invariant, testing.CurrentConnectionString); - } - - private static string RandomServiceId(int max = 10) => $"ServiceId{Random.Shared.Next(max)}"; - - private static string RandomProviderId(int max = 10) => $"ProviderId{Random.Shared.Next(max)}"; - - private static string RandomQueueId(int max = 10) => $"QueueId{Random.Shared.Next(max)}"; - - private static int RandomExpiryTimeout(int max = 100) => Random.Shared.Next(max); - - private static byte[] RandomPayload(int size = 1_000_000) - { - var payload = new byte[size]; - Random.Shared.NextBytes(payload); - return payload; - } - - private async Task QueueMessagesAsync( - string serviceId, - string providerId, - string queueId, - byte[] payload, - int expiryTimeout, - int count, - CancellationToken cancellationToken) - { - using var semaphore = new SemaphoreSlim(concurrency); - return await Task.WhenAll(Enumerable.Range(0, count).Select(async _ => - { - return await ExecuteProviderOperationAsync( - semaphore, - () => _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, expiryTimeout), - cancellationToken); - })); - } - - private static async Task ExecuteProviderOperationAsync( - SemaphoreSlim semaphore, - Func> operationFactory, - CancellationToken cancellationToken) - { - await semaphore.WaitAsync(cancellationToken); - Task? operation = null; - try - { - cancellationToken.ThrowIfCancellationRequested(); - operation = operationFactory(); - return await operation.WaitAsync(cancellationToken); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - if (operation is not null) - { - if (operation.IsCompleted) - { - await ObserveLateCompletionAsync(operation); - } - else - { - _ = ObserveLateCompletionAsync(operation); - } - } - - throw; - } - finally - { - semaphore.Release(); - } - } - - private static async Task ObserveLateCompletionAsync(Task operation) - { - try - { - await operation; - } - catch - { - } - } - - public ValueTask DisposeAsync() => ValueTask.CompletedTask; - - protected async Task VerifyProviderResultsAreOrdered(CancellationToken cancellationToken) - { - const string reverseQuery = """ - SELECT ServiceId, ProviderId, QueueId, MessageId, Dequeued, VisibleOn, ExpiresOn, CreatedOn, ModifiedOn, Payload - FROM OrleansStreamMessage - WHERE ServiceId = @ServiceId AND ProviderId = @ProviderId AND QueueId = @QueueId - ORDER BY MessageId DESC - LIMIT @MaxCount - """; - - var serviceId = RandomServiceId(); - var providerId = RandomProviderId(); - var queueId = RandomQueueId(); - var payload = new byte[] { 0xFF }; - - await QueueMessagesAsync(serviceId, providerId, queueId, payload, 100, 100, cancellationToken); - - await _storage.ExecuteAsync( - "UPDATE OrleansQuery SET QueryText = @QueryText WHERE QueryKey = 'GetStreamMessagesKey'", - command => - { - var parameter = command.CreateParameter(); - parameter.ParameterName = "QueryText"; - parameter.Value = reverseQuery; - command.Parameters.Add(parameter); - }, - cancellationToken: cancellationToken); - - var queries = await RelationalOrleansQueries.CreateInstance(invariant, _storage.ConnectionString); - var messages = await queries.GetStreamMessagesAsync(serviceId, providerId, queueId, 100, 3, 10, 100, 10, 1000); - - Assert.Equal(messages.OrderBy(message => message.MessageId), messages); - } - - protected async Task VerifySqlServerQueueMessageCommitsAreSerialized(CancellationToken cancellationToken) - { - var serviceId = RandomServiceId(); - var providerId = RandomProviderId(); - var queueId = RandomQueueId(); - var payload = new byte[] { 0xFF }; - const int expiryTimeout = 100; - - await using var firstConnection = new SqlConnection(_storage.ConnectionString); - await firstConnection.OpenAsync(cancellationToken); - await using var firstTransaction = (SqlTransaction)await firstConnection.BeginTransactionAsync(cancellationToken); - await using var firstCommand = CreateQueueCommand(firstConnection, firstTransaction, serviceId, providerId, queueId, payload, expiryTimeout); - var firstMessageId = await ReadMessageId(firstCommand, cancellationToken); - - await using (var secondConnection = new SqlConnection(_storage.ConnectionString)) - { - await secondConnection.OpenAsync(cancellationToken); - await using var secondCommand = CreateQueueCommand(secondConnection, transaction: null, serviceId, providerId, queueId, payload, expiryTimeout); - secondCommand.CommandType = CommandType.Text; - secondCommand.CommandText = "SET LOCK_TIMEOUT 0; EXECUTE QueueStreamMessage @ServiceId, @ProviderId, @QueueId, @Payload, @ExpiryTimeout;"; - - var exception = await Assert.ThrowsAsync(() => secondCommand.ExecuteReaderAsync(cancellationToken)); - Assert.Equal(51000, exception.Number); - } - - await firstTransaction.CommitAsync(cancellationToken); - - var secondAck = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, expiryTimeout); - var messages = await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, 2, 3, 10, 100, 10, 1000); - - Assert.Equal([firstMessageId, secondAck.MessageId], messages.Select(message => message.MessageId)); - } - - private static SqlCommand CreateQueueCommand( - SqlConnection connection, - SqlTransaction? transaction, - string serviceId, - string providerId, - string queueId, - byte[] payload, - int expiryTimeout) - { - var command = connection.CreateCommand(); - command.Transaction = transaction; - command.CommandType = CommandType.StoredProcedure; - command.CommandText = "QueueStreamMessage"; - command.Parameters.AddWithValue("ServiceId", serviceId); - command.Parameters.AddWithValue("ProviderId", providerId); - command.Parameters.AddWithValue("QueueId", queueId); - command.Parameters.AddWithValue("Payload", payload); - command.Parameters.AddWithValue("ExpiryTimeout", expiryTimeout); - return command; - } - - private static async Task ReadMessageId(SqlCommand command, CancellationToken cancellationToken) - { - await using var reader = await command.ExecuteReaderAsync(cancellationToken); - Assert.True(await reader.ReadAsync(cancellationToken)); - return reader.GetInt64(reader.GetOrdinal(nameof(AdoNetStreamMessage.MessageId))); - } - - /// - /// Tests that a single message is queued. - /// - [Fact] - public async Task RelationalOrleansQueries_QueuesMessage() - { - // arrange - var serviceId = RandomServiceId(); - var providerId = RandomProviderId(); - var queueId = RandomQueueId(); - var expiryTimeout = RandomExpiryTimeout(); - var payload = RandomPayload(); - - // act - var before = DateTime.UtcNow.AddSeconds(-1); - var ack = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, expiryTimeout); - var after = DateTime.UtcNow.AddSeconds(1); - - // assert - ack - Assert.NotNull(ack); - Assert.Equal(serviceId, ack.ServiceId); - Assert.Equal(providerId, ack.ProviderId); - Assert.Equal(queueId, ack.QueueId); - Assert.Equal(1, ack.MessageId); - - // assert - storage - var messages = await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken); - var message = Assert.Single(messages); - Assert.Equal(serviceId, message.ServiceId); - Assert.Equal(providerId, message.ProviderId); - Assert.Equal(queueId, message.QueueId); - Assert.Equal(ack.MessageId, message.MessageId); - Assert.Equal(0, message.Dequeued); - Assert.True(message.VisibleOn >= before); - Assert.True(message.VisibleOn <= after); - Assert.True(message.ExpiresOn >= before.AddSeconds(expiryTimeout)); - Assert.True(message.ExpiresOn <= after.AddSeconds(expiryTimeout)); - Assert.Equal(message.VisibleOn, message.CreatedOn); - Assert.Equal(message.VisibleOn, message.ModifiedOn); - Assert.Equal(payload, message.Payload); - } - - /// - /// Tests that many messages are queued in parallel on the same queue. - /// - [TestSuite("Stress")] - [TestCategory("Stress")] - [Fact] - public async Task RelationalOrleansQueries_QueuesManyMessagesInParallel() - { - var cancellationToken = TestContext.Current.CancellationToken; - - // arrange - var serviceId = RandomServiceId(); - var providerId = RandomProviderId(); - var queueId = RandomQueueId(); - var expiryTimeout = RandomExpiryTimeout(); - var payload = RandomPayload(1000); - var count = 10000; - - // this keeps requests under the default connection pool limit to avoid flaky tests due to connection timeouts - using var semaphore = new SemaphoreSlim(concurrency); - - // act - var before = DateTime.UtcNow.AddSeconds(-1); - var acks = await Task.WhenAll(Enumerable - .Range(0, count) - .Select(i => Task.Run(async () => - { - return await ExecuteProviderOperationAsync( - semaphore, - () => _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, expiryTimeout), - cancellationToken); - }, cancellationToken)) - .ToList()); - var after = DateTime.UtcNow.AddSeconds(1); - - // assert - messages were inserted in sequence. - var ordered = acks - .OrderBy(x => x.ServiceId) - .ThenBy(x => x.ProviderId) - .ThenBy(x => x.QueueId) - .ThenBy(x => x.MessageId) - .ToList(); - var messageId = ordered[0].MessageId; - for (var i = 0; i < count; i++) - { - Assert.Equal(serviceId, ordered[i].ServiceId); - Assert.Equal(providerId, ordered[i].ProviderId); - Assert.Equal(queueId, ordered[i].QueueId); - Assert.Equal(messageId++, ordered[i].MessageId); - } - - // assert - messages were stored as expected - var stored = (await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - cancellationToken)) - .OrderBy(x => x.ServiceId) - .ThenBy(x => x.ProviderId) - .ThenBy(x => x.QueueId) - .ThenBy(x => x.MessageId) - .ToList(); - for (var i = 0; i < count; i++) - { - Assert.Equal(ordered[i].ServiceId, stored[i].ServiceId); - Assert.Equal(ordered[i].ProviderId, stored[i].ProviderId); - Assert.Equal(ordered[i].QueueId, stored[i].QueueId); - Assert.Equal(ordered[i].MessageId, stored[i].MessageId); - Assert.Equal(0, stored[i].Dequeued); - Assert.True(stored[i].VisibleOn >= before); - Assert.True(stored[i].VisibleOn <= after); - Assert.True(stored[i].ExpiresOn >= before.AddSeconds(expiryTimeout)); - Assert.True(stored[i].ExpiresOn <= after.AddSeconds(expiryTimeout)); - Assert.Equal(stored[i].VisibleOn, stored[i].CreatedOn); - Assert.Equal(stored[i].VisibleOn, stored[i].ModifiedOn); - Assert.Equal(payload, stored[i].Payload); - } - } - - /// - /// Tests that many messages are queued in parallel on many queues. - /// - [TestSuite("Stress")] - [TestCategory("Stress")] - [Fact] - public async Task RelationalOrleansQueries_QueuesManyMessagesInParallelOnManyQueues() - { - var cancellationToken = TestContext.Current.CancellationToken; - - // arrange - create up to 27 random partition keys with around 1000 random messages per partition in random order - var expiryTimeout = RandomExpiryTimeout(); - var count = 3 * 3 * 3 * 1000; - var partitions = Enumerable - .Range(0, count) - .Select(i => - ( - ServiceId: RandomServiceId(3), - ProviderId: RandomProviderId(3), - QueueId: RandomQueueId(3), - Payload: RandomPayload(1000) - )) - .ToList(); - - // this keeps requests under the default connection pool limit to avoid flaky tests due to connection timeouts - using var semaphore = new SemaphoreSlim(concurrency); - - // act - queue the random messages in parallel - var before = DateTime.UtcNow.AddSeconds(-1); - var results = await Task.WhenAll(partitions - .Select(p => Task.Run(async () => - { - var ack = await ExecuteProviderOperationAsync( - semaphore, - () => _queries.QueueStreamMessageAsync(p.ServiceId, p.ProviderId, p.QueueId, p.Payload, expiryTimeout), - cancellationToken); - return (Partition: p, Ack: ack); - }, cancellationToken)) - .ToList()); - var after = DateTime.UtcNow.AddSeconds(1); - - // assert - all messages were acknowledged - var messageIds = new SortedSet(); - foreach (var (partition, ack) in results) - { - Assert.Equal(ack.ServiceId, partition.ServiceId); - Assert.Equal(ack.ProviderId, partition.ProviderId); - Assert.Equal(ack.QueueId, partition.QueueId); - Assert.True(messageIds.Add(ack.MessageId), $"Duplicate {ack.MessageId}"); - } - - // assert - generated message ids are consistent - Assert.Equal(count, messageIds.Count); - Assert.Equal(1, messageIds.Min); - Assert.Equal(messageIds.Count, messageIds.Max); - - // assert - messages were stored as expected - var stored = (await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - cancellationToken)) - .ToDictionary(x => (x.ServiceId, x.ProviderId, x.QueueId, x.MessageId)); - - foreach (var (partition, ack) in results) - { - Assert.True(stored.TryGetValue((ack.ServiceId, ack.ProviderId, ack.QueueId, ack.MessageId), out var message), $"Message not found in storage"); - - Assert.Equal(0, message.Dequeued); - Assert.True(message.VisibleOn >= before); - Assert.True(message.VisibleOn <= after); - Assert.True(message.ExpiresOn >= before.AddSeconds(expiryTimeout)); - Assert.True(message.ExpiresOn <= after.AddSeconds(expiryTimeout)); - Assert.Equal(message.VisibleOn, message.CreatedOn); - Assert.Equal(message.VisibleOn, message.ModifiedOn); - Assert.Equal(partition.Payload, message.Payload); - - stored.Remove((ack.ServiceId, ack.ProviderId, ack.QueueId, ack.MessageId)); - } - } - - /// - /// Tests that a single message is dequeued correctly. - /// - [Fact] - public async Task RelationalOrleansQueries_DequeuesSingleMessage() - { - // arrange - await _storage.ExecuteAsync("DELETE FROM OrleansStreamMessage", TestContext.Current.CancellationToken); - var serviceId = RandomServiceId(); - var providerId = RandomProviderId(); - var queueId = RandomQueueId(); - var payload = RandomPayload(); - var expiryTimeout = 100; - var maxCount = 1; - var maxAttempts = 3; - var visibilityTimeout = 10; - var removalTimeout = 100; - var evictionInterval = 10; - var evictionBatchSize = 1000; - - // arrange - enqueue a message - var beforeQueueing = DateTime.UtcNow.AddSeconds(-1); - var ack = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, expiryTimeout); - var afterQueueing = DateTime.UtcNow.AddSeconds(1); - - // act - dequeue a message - var beforeDequeuing = DateTime.UtcNow.AddSeconds(-1); - var message = Assert.Single(await _queries.GetStreamMessagesAsync( - serviceId, - providerId, - queueId, - maxCount, - maxAttempts, - visibilityTimeout, - removalTimeout, - evictionInterval, - evictionBatchSize)); - var afterDequeuing = DateTime.UtcNow.AddSeconds(1); - - // assert - the message is the same - Assert.Equal(ack.ServiceId, message.ServiceId); - Assert.Equal(ack.ProviderId, message.ProviderId); - Assert.Equal(ack.QueueId, message.QueueId); - Assert.Equal(ack.MessageId, message.MessageId); - Assert.Equal(1, message.Dequeued); - Assert.True(message.VisibleOn >= beforeDequeuing.AddSeconds(visibilityTimeout)); - Assert.True(message.VisibleOn <= afterDequeuing.AddSeconds(visibilityTimeout)); - Assert.True(message.ExpiresOn >= beforeQueueing.AddSeconds(expiryTimeout)); - Assert.True(message.ExpiresOn <= afterQueueing.AddSeconds(expiryTimeout)); - Assert.True(message.CreatedOn >= beforeQueueing); - Assert.True(message.CreatedOn <= afterQueueing); - Assert.True(message.ModifiedOn >= beforeDequeuing); - Assert.True(message.ModifiedOn <= afterDequeuing); - Assert.Equal(payload, message.Payload); - - // assert - the stored message changed - var stored = Assert.Single(await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken)); - Assert.Equal(message.ServiceId, stored.ServiceId); - Assert.Equal(message.ProviderId, stored.ProviderId); - Assert.Equal(message.QueueId, stored.QueueId); - Assert.Equal(message.MessageId, stored.MessageId); - Assert.Equal(message.Dequeued, stored.Dequeued); - Assert.Equal(message.VisibleOn, stored.VisibleOn); - Assert.Equal(message.ExpiresOn, stored.ExpiresOn); - Assert.Equal(message.CreatedOn, stored.CreatedOn); - Assert.Equal(message.ModifiedOn, stored.ModifiedOn); - Assert.Equal(message.Payload, stored.Payload); - } - - /// - /// Tests that messages are dequeued in a batch. - /// - [Fact] - public async Task RelationalOrleansQueries_DequeuesMessageBatches() - { - // arrange - var serviceId = RandomServiceId(); - var providerId = RandomProviderId(); - var queueId = RandomQueueId(); - var payload = new byte[] { 0xFF }; - var expiryTimeout = 100; - var maxCount = 60; - var maxAttempts = 3; - var visibilityTimeout = 10; - var removalTimeout = 100; - var evictionInterval = 10; - var evictionBatchSize = 1000; - var total = 100; - - // arrange - enqueue messages concurrently - var beforeQueueing = DateTime.UtcNow.AddSeconds(-1); - var acks = await QueueMessagesAsync( - serviceId, - providerId, - queueId, - payload, - expiryTimeout, - total, - TestContext.Current.CancellationToken); - var afterQueueing = DateTime.UtcNow.AddSeconds(1); - - // act - dequeue all messages - var beforeDequeuing = DateTime.UtcNow.AddSeconds(-1); - var first = await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize); - var second = await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize); - var third = await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize); - var afterDequeuing = DateTime.UtcNow.AddSeconds(1); - - // assert - batch counts - Assert.Equal(maxCount, first.Count); - Assert.Equal(total - maxCount, second.Count); - Assert.Empty(third); - - var messages = first.Concat(second).Concat(third).ToList(); - Assert.Equal(messages.OrderBy(message => message.MessageId), messages); - - // assert - dequeued messages are consistent with acks - var ackLookup = acks.ToDictionary(x => (x.ServiceId, x.ProviderId, x.QueueId, x.MessageId)); - foreach (var message in messages) - { - Assert.True(ackLookup.TryGetValue((message.ServiceId, message.ProviderId, message.QueueId, message.MessageId), out var ack), "Ack not found"); - Assert.Equal(ack.ServiceId, message.ServiceId); - Assert.Equal(ack.ProviderId, message.ProviderId); - Assert.Equal(ack.QueueId, message.QueueId); - Assert.Equal(ack.MessageId, message.MessageId); - Assert.Equal(1, message.Dequeued); - Assert.True(message.VisibleOn >= beforeDequeuing.AddSeconds(visibilityTimeout)); - Assert.True(message.VisibleOn <= afterDequeuing.AddSeconds(visibilityTimeout)); - Assert.True(message.ExpiresOn >= beforeQueueing.AddSeconds(expiryTimeout)); - Assert.True(message.ExpiresOn <= afterQueueing.AddSeconds(expiryTimeout)); - Assert.True(message.CreatedOn >= beforeQueueing); - Assert.True(message.CreatedOn <= afterQueueing); - Assert.True(message.ModifiedOn >= beforeDequeuing); - Assert.True(message.ModifiedOn <= afterDequeuing); - Assert.Equal(payload, message.Payload); - - ackLookup.Remove((message.ServiceId, message.ProviderId, message.QueueId, message.MessageId)); - } - - // assert - stored messages are consistent with dequeued messages - var messageLookup = messages.ToDictionary(x => (x.ServiceId, x.ProviderId, x.QueueId, x.MessageId)); - var stored = await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken); - foreach (var item in stored) - { - Assert.True(messageLookup.TryGetValue((item.ServiceId, item.ProviderId, item.QueueId, item.MessageId), out var message), "Message not found"); - - Assert.Equal(message.ServiceId, item.ServiceId); - Assert.Equal(message.ProviderId, item.ProviderId); - Assert.Equal(message.QueueId, item.QueueId); - Assert.Equal(message.MessageId, item.MessageId); - Assert.Equal(message.Dequeued, item.Dequeued); - Assert.Equal(message.VisibleOn, item.VisibleOn); - Assert.Equal(message.ExpiresOn, item.ExpiresOn); - Assert.Equal(message.CreatedOn, item.CreatedOn); - Assert.Equal(message.ModifiedOn, item.ModifiedOn); - Assert.Equal(message.Payload, item.Payload); - } - } - - /// - /// Tests that a single message is re-dequeued after visibility timeout until max attempts. - /// - [Fact] - public async Task RelationalOrleansQueries_DequeuesSingleMessageAgainAfterVisibilityTimeout() - { - // arrange - var serviceId = RandomServiceId(); - var providerId = RandomProviderId(); - var queueId = RandomQueueId(); - var payload = RandomPayload(); - var expiryTimeout = 100; - var maxCount = 1; - var maxAttempts = 3; - var visibilityTimeout = 0; - var removalTimeout = 100; - var evictionInterval = 100; - var evictionBatchSize = 0; - - // arrange - enqueue a message - var beforeQueueing = DateTime.UtcNow.AddSeconds(-1); - var ack = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, expiryTimeout); - var afterQueueing = DateTime.UtcNow.AddSeconds(1); - - // act - dequeue messages until max attempts plus one - var beforeDequeuing = DateTime.UtcNow.AddSeconds(-1); - var results = new List>(); - for (var i = 0; i < maxAttempts + 1; i++) - { - results.Add(await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize)); - } - var afterDequeuing = DateTime.UtcNow.AddSeconds(1); - - // assert - batches are as expected - for (var i = 0; i < maxAttempts; i++) - { - var message = Assert.Single(results[i]); - - Assert.Equal(ack.ServiceId, message.ServiceId); - Assert.Equal(ack.ProviderId, message.ProviderId); - Assert.Equal(ack.QueueId, message.QueueId); - Assert.Equal(ack.MessageId, message.MessageId); - Assert.Equal(i + 1, message.Dequeued); - Assert.True(message.VisibleOn >= beforeDequeuing.AddSeconds(visibilityTimeout)); - Assert.True(message.VisibleOn <= afterDequeuing.AddSeconds(visibilityTimeout)); - Assert.True(message.ExpiresOn >= beforeQueueing.AddSeconds(expiryTimeout)); - Assert.True(message.ExpiresOn <= afterQueueing.AddSeconds(expiryTimeout)); - Assert.True(message.CreatedOn >= beforeQueueing); - Assert.True(message.CreatedOn <= afterQueueing); - Assert.True(message.ModifiedOn >= beforeDequeuing); - Assert.True(message.ModifiedOn <= afterDequeuing); - Assert.Equal(payload, message.Payload); - } - - // assert - final batch is empty - Assert.Empty(results[maxAttempts]); - - // assert - final stored message is consistent with final dequeued message - var stored = Assert.Single(await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken)); - var final = Assert.Single(results[maxAttempts - 1]); - Assert.Equal(final.ServiceId, stored.ServiceId); - Assert.Equal(final.ProviderId, stored.ProviderId); - Assert.Equal(final.QueueId, stored.QueueId); - Assert.Equal(final.MessageId, stored.MessageId); - Assert.Equal(final.Dequeued, stored.Dequeued); - Assert.Equal(final.VisibleOn, stored.VisibleOn); - Assert.Equal(final.ExpiresOn, stored.ExpiresOn); - Assert.Equal(final.CreatedOn, stored.CreatedOn); - Assert.Equal(final.ModifiedOn, stored.ModifiedOn); - Assert.Equal(final.Payload, stored.Payload); - } - - /// - /// Tests that a single message is not dequeued again before the visibility timeout. - /// - [Fact] - public async Task RelationalOrleansQueries_DoesNotDequeueSingleMessageBeforeVisibilityTimeout() - { - // arrange - var serviceId = RandomServiceId(); - var providerId = RandomProviderId(); - var queueId = RandomQueueId(); - var payload = RandomPayload(); - var expiryTimeout = 100; - var maxCount = 3; - var maxAttempts = 3; - var visibilityTimeout = 10; - var removalTimeout = 100; - var evictionInterval = 10; - var evictionBatchSize = 1000; - - // arrange - enqueue a message - var ack = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, expiryTimeout); - - // act - dequeue messages - var first = Assert.Single(await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize)); - var second = await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize); - - // assert - first dequeued message is consistent with ack - Assert.Equal(ack.ServiceId, first.ServiceId); - Assert.Equal(ack.ProviderId, first.ProviderId); - Assert.Equal(ack.QueueId, first.QueueId); - Assert.Equal(ack.MessageId, first.MessageId); - - // assert - stored message is consistent with first message - var stored = Assert.Single(await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken)); - Assert.Equal(first.ServiceId, stored.ServiceId); - Assert.Equal(first.ProviderId, stored.ProviderId); - Assert.Equal(first.QueueId, stored.QueueId); - Assert.Equal(first.MessageId, stored.MessageId); - Assert.Equal(first.Dequeued, stored.Dequeued); - Assert.Equal(first.VisibleOn, stored.VisibleOn); - Assert.Equal(first.ExpiresOn, stored.ExpiresOn); - Assert.Equal(first.CreatedOn, stored.CreatedOn); - Assert.Equal(first.ModifiedOn, stored.ModifiedOn); - Assert.Equal(first.Payload, stored.Payload); - - // assert - message not dequeued again - Assert.Empty(second); - } - - /// - /// Tests that a message can be released for immediate redelivery using its dequeue receipt. - /// - [Fact] - public async Task RelationalOrleansQueries_ReleasesMessageUsingDequeueReceipt() - { - var serviceId = $"Service-{Guid.NewGuid()}"; - var providerId = $"Provider-{Guid.NewGuid()}"; - var queueId = $"Queue-{Guid.NewGuid()}"; - var payload = RandomPayload(); - var expiryTimeout = 100; - var maxCount = 1; - var maxAttempts = 5; - var visibilityTimeout = 100; - var removalTimeout = 100; - var evictionInterval = 100; - var evictionBatchSize = 0; - - var ack = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, expiryTimeout); - var first = Assert.Single(await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize)); - var firstReceipt = new AdoNetStreamConfirmation(first.MessageId, first.Dequeued); - - var released = Assert.Single(await _queries.ReleaseStreamMessagesAsync(serviceId, providerId, queueId, [firstReceipt])); - Assert.Equal(ack.MessageId, released.MessageId); - - var second = Assert.Single(await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize)); - Assert.Equal(first.Dequeued + 1, second.Dequeued); - - Assert.Empty(await _queries.ReleaseStreamMessagesAsync(serviceId, providerId, queueId, [firstReceipt])); - Assert.Empty(await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize)); - - var secondReceipt = new AdoNetStreamConfirmation(second.MessageId, second.Dequeued); - Assert.Single(await _queries.ReleaseStreamMessagesAsync(serviceId, providerId, queueId, [secondReceipt])); - var third = Assert.Single(await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize)); - await _queries.ConfirmStreamMessagesAsync(serviceId, providerId, queueId, [new AdoNetStreamConfirmation(third.MessageId, third.Dequeued)]); - } - - /// - /// Tests that a single message is not dequeued again after expiry - /// - [Fact] - public async Task RelationalOrleansQueries_DoesNotDequeueSingleMessageAfterExpiry() - { - // arrange - var serviceId = RandomServiceId(); - var providerId = RandomProviderId(); - var queueId = RandomQueueId(); - var payload = RandomPayload(); - var expiryTimeout = 0; - var maxCount = 3; - var maxAttempts = 3; - var visibilityTimeout = 0; - var removalTimeout = 100; - var evictionInterval = 10; - var evictionBatchSize = 0; - - // arrange - enqueue a message - var before = DateTime.UtcNow.AddSeconds(-1); - var ack = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, expiryTimeout); - var after = DateTime.UtcNow.AddSeconds(1); - - // act - dequeue messages - var messages = await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize); - - // assert - no messages dequeued - Assert.Empty(messages); - - // assert - stored message are as expected - var stored = Assert.Single(await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken)); - Assert.Equal(ack.ServiceId, stored.ServiceId); - Assert.Equal(ack.ProviderId, stored.ProviderId); - Assert.Equal(ack.QueueId, stored.QueueId); - Assert.Equal(ack.MessageId, stored.MessageId); - Assert.Equal(0, stored.Dequeued); - Assert.True(stored.VisibleOn >= before); - Assert.True(stored.VisibleOn <= after); - Assert.True(stored.ExpiresOn >= before); - Assert.True(stored.ExpiresOn <= after); - Assert.True(stored.CreatedOn >= before); - Assert.True(stored.CreatedOn <= after); - Assert.True(stored.ModifiedOn >= before); - Assert.True(stored.ModifiedOn <= after); - Assert.Equal(payload, stored.Payload); - } - - /// - /// Tests that messages can be confirmed. - /// - [Fact] - public async Task RelationalOrleansQueries_ConfirmsMessages() - { - // arrange - var serviceId = RandomServiceId(); - var providerId = RandomProviderId(); - var queueId = RandomQueueId(); - var payload = RandomPayload(); - var expiryTimeout = 100; - var maxCount = 10; - var maxAttempts = 3; - var visibilityTimeout = 10; - var removalTimeout = 100; - var evictionInterval = 10; - var evictionBatchSize = 1000; - - // arrange - enqueue many messages - var acks = await Task.WhenAll(Enumerable - .Range(0, maxCount) - .Select(i => _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, expiryTimeout)) - .ToList()); - - // arrange - dequeue all messages - var messages = await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize); - - // act - confirm all messages - var items = messages.Select(x => new AdoNetStreamConfirmation(x.MessageId, x.Dequeued)).ToList(); - var results = await _queries.ConfirmStreamMessagesAsync(serviceId, providerId, queueId, items); - - // assert - confirmations are as expected - Assert.Equal(maxCount, acks.Length); - Assert.Equal(maxCount, messages.Count); - Assert.Equal(maxCount, results.Count); - - var lookup = acks.Select(x => (x.ServiceId, x.ProviderId, x.QueueId, x.MessageId)).ToHashSet(); - foreach (var result in results) - { - Assert.True(lookup.Remove((result.ServiceId, result.ProviderId, result.QueueId, result.MessageId)), "Unexpected Confirmation"); - } - - // assert - no data remains in storage - var stored = await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken); - Assert.Empty(stored); - } - - /// - /// Tests that messages are not confirmed if the receipt is incorrect. - /// - [Fact] - public async Task RelationalOrleansQueries_DoesNotConfirmMessagesWithWrongReceipt() - { - // arrange - var serviceId = RandomServiceId(); - var providerId = RandomProviderId(); - var queueId = RandomQueueId(); - var payload = RandomPayload(); - var expiryTimeout = 100; - var maxCount = 10; - var maxAttempts = 3; - var visibilityTimeout = 10; - var removalTimeout = 100; - var evictionInterval = 10; - var evictionBatchSize = 1000; - - // arrange - enqueue many messages - var acks = await Task.WhenAll(Enumerable - .Range(0, maxCount) - .Select(i => _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, expiryTimeout)) - .ToList()); - - // arrange - dequeue all messages - var messages = await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize); - - // act - confirm all messages in a faulty way - var faulty = messages.Select(x => new AdoNetStreamConfirmation(x.MessageId, x.Dequeued - 1)).ToList(); - var results = await _queries.ConfirmStreamMessagesAsync(serviceId, providerId, queueId, faulty); - - // assert - confirmations are as expected - Assert.Equal(maxCount, acks.Length); - Assert.Equal(maxCount, messages.Count); - Assert.Empty(results); - - // assert - data remains in storage - var stored = await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken); - Assert.Equal(maxCount, stored.Count()); - } - - /// - /// Chaos tests that some messages can be confirmed while others are not. - /// - [Fact] - public async Task RelationalOrleansQueries_ConfirmsSomeMessagesAndNotOthers() - { - // arrange - var serviceId = RandomServiceId(); - var providerId = RandomProviderId(); - var queueId = RandomQueueId(); - var payload = RandomPayload(1000); - var expiryTimeout = 100; - var maxCount = 100; - var maxAttempts = 3; - var visibilityTimeout = 10; - var removalTimeout = 100; - var evictionInterval = 10; - var evictionBatchSize = 1000; - var partial = 30; - - // arrange - enqueue many messages - var acks = await Task.WhenAll(Enumerable - .Range(0, maxCount) - .Select(i => _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, expiryTimeout)) - .ToList()); - - // arrange - dequeue all the messages - var messages = await _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize); - - // act - confirm some of the messages at random - var completed = Randomize(messages).Take(partial).Select(x => new AdoNetStreamConfirmation(x.MessageId, x.Dequeued)).ToList(); - var confirmed = await _queries.ConfirmStreamMessagesAsync(serviceId, providerId, queueId, completed); - - // assert - counts are as expected - Assert.Equal(maxCount, acks.Length); - Assert.Equal(maxCount, messages.Count); - Assert.Equal(partial, confirmed.Count); - - // assert - confirmed messages are as expected - var lookup = acks.ToDictionary(x => (x.ServiceId, x.ProviderId, x.QueueId, x.MessageId)); - var stored = (await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken)) - .ToDictionary(x => (x.ServiceId, x.ProviderId, x.QueueId, x.MessageId)); - foreach (var item in confirmed) - { - Assert.True(lookup.Remove((item.ServiceId, item.ProviderId, item.QueueId, item.MessageId)), "Unexpected Confirmation"); - Assert.False(stored.TryGetValue((item.ServiceId, item.ProviderId, item.QueueId, item.MessageId), out _), "Message still in storage"); - } - - // assert - unconfirmed messages remain in storage - Assert.Equal(maxCount - partial, stored.Count); - Assert.Equal(lookup.Keys.Order(), stored.Keys.Order()); - } - - /// - /// Chaos tests that queuing, dequeuing, confirmation and eviction work in parallel in a complex random scenario. - /// This looks for concurrent brittleness, especially proneness to database deadlocks, rather than a specific condition. - /// If this test faults due to deadlocks then there is likely some issue with the implementation that needs investigation. - /// - /// - /// At early dev time, this test consistently induced deadlocks until the underlying queries were perfected. - /// This is an expensive test to run but can protect against query regression. - /// For MySQL in particular, this test also detected deadlocks with the driver connection pool itself, which required a package upgrade. - /// See: https://bugs.mysql.com/bug.php?id=114272 - /// - [TestSuite("Stress")] - [TestCategory("Stress")] - [Fact] - public async Task RelationalOrleansQueries_ChaosTest() - { - var cancellationToken = TestContext.Current.CancellationToken; - - // arrange - generate test data - var total = 10000; - var serviceIds = Enumerable.Range(0, 3).Select(x => $"ServiceId{x}").ToList(); - var providerIds = Enumerable.Range(0, 3).Select(x => $"ProviderId{x}").ToList(); - var queueIds = Enumerable.Range(0, 3).Select(x => $"QueueId{x}").ToList(); - var payload = RandomPayload(1000); - var maxCount = 10; - var maxAttempts = 3; - var visibilityTimeout = 1; - var removalTimeout = 1; - var evictionInterval = 1; - var evictionBatchSize = 1000; - - // this keeps requests under the default connection pool limit to avoid flaky tests due to connection timeouts - using var semaphore = new SemaphoreSlim(concurrency); - - // act - chaos enqueue, dequeue, confirm - // the tasks below are not expected to result in a planned outcome but are expected to result in a consistent one - var acks = new ConcurrentBag(); - var dequeued1 = new ConcurrentBag(); - var dequeued2 = new ConcurrentBag(); - var confirmed = new ConcurrentBag(); - await Task.WhenAll(Enumerable - .Range(0, total) - .Select(async i => - { - // spin up a random enqueuing task - var enqueue = Task.Run(async () => - { - var serviceId = serviceIds[Random.Shared.Next(serviceIds.Count)]; - var providerId = providerIds[Random.Shared.Next(providerIds.Count)]; - var queueId = queueIds[Random.Shared.Next(queueIds.Count)]; - - var ack = await ExecuteProviderOperationAsync( - semaphore, - () => _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, visibilityTimeout), - cancellationToken); - - acks.Add(ack); - }, cancellationToken); - - // spin up a random dequeuing task that does not confirm - var dequeue = Task.Run(async () => - { - var serviceId = serviceIds[Random.Shared.Next(serviceIds.Count)]; - var providerId = providerIds[Random.Shared.Next(providerIds.Count)]; - var queueId = queueIds[Random.Shared.Next(queueIds.Count)]; - - var messages = await ExecuteProviderOperationAsync( - semaphore, - () => _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize), - cancellationToken); - - foreach (var item in messages) - { - dequeued1.Add(item); - } - }, cancellationToken); - - // spin a random dequeuing task that also confirms - var confirm = Task.Run(async () => - { - var serviceId = serviceIds[Random.Shared.Next(serviceIds.Count)]; - var providerId = providerIds[Random.Shared.Next(providerIds.Count)]; - var queueId = queueIds[Random.Shared.Next(queueIds.Count)]; - - var messages = await ExecuteProviderOperationAsync( - semaphore, - () => _queries.GetStreamMessagesAsync(serviceId, providerId, queueId, maxCount, maxAttempts, visibilityTimeout, removalTimeout, evictionInterval, evictionBatchSize), - cancellationToken); - - foreach (var item in messages) - { - dequeued2.Add(item); - } - - var confirmation = await ExecuteProviderOperationAsync( - semaphore, - () => _queries.ConfirmStreamMessagesAsync( - serviceId, - providerId, - queueId, - messages.Select(x => new AdoNetStreamConfirmation(x.MessageId, x.Dequeued)).ToList()), - cancellationToken); - - foreach (var item in confirmation) - { - confirmed.Add(item); - } - }, cancellationToken); - - // wait for all to complete - await Task.WhenAll(enqueue, dequeue, confirm); - }) - .ToList()); - - // assert - all messages were enqueued - Assert.Equal(total, acks.Count); - - // assert - some messages were dequeued (rng dependant, remove assert if flaky) - Assert.NotEmpty(dequeued1); - Assert.NotEmpty(dequeued2); - - // assert - some messages were confirmed (rng dependant, remove assert if flaky) - Assert.NotEmpty(confirmed); - - // assert - some messages were left behind (rng dependant, remove assert if flaky) - var stored = await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - cancellationToken); - Assert.NotEmpty(stored); - - // assert - confirmed messages were not left behind - Assert.Empty(confirmed.IntersectBy(stored.Select(x => x.MessageId), x => x.MessageId)); - - // assert - confirmed messages all match acks - Assert.Empty(confirmed.ExceptBy(acks.Select(x => x.MessageId), x => x.MessageId)); - } - - /// - /// Tests that a poisoned message can be moved to dead letters. - /// - [Fact] - public async Task RelationalOrleansQueries_MovesPoisonedMessageToDeadLetters() - { - // arrange - var serviceId = "ServiceId"; - var providerId = "ProviderId"; - var streamOptions = new AdoNetStreamOptions(); - var cacheOptions = new SimpleQueueCacheOptions(); - - // arrange - queue an expired message - var queueId = "QueueId"; - var payload = new byte[] { 0xFF }; - - var ack = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, streamOptions.ExpiryTimeout.TotalSecondsCeiling()); - - // arrange - dequeue the message and make immediately available - await _queries.GetStreamMessagesAsync(ack.ServiceId, ack.ProviderId, ack.QueueId, cacheOptions.CacheSize, streamOptions.MaxAttempts, 0, streamOptions.DeadLetterEvictionTimeout.TotalSecondsCeiling(), streamOptions.EvictionInterval.TotalSecondsCeiling(), streamOptions.EvictionBatchSize); - Assert.Empty(await _storage.ReadAsync( - "SELECT * FROM OrleansStreamDeadLetter", - TestContext.Current.CancellationToken)); - - // act - clean up with max attempts of one so the message above is flagged - await _queries.FailStreamMessageAsync(ack.ServiceId, ack.ProviderId, ack.QueueId, ack.MessageId, 1, streamOptions.DeadLetterEvictionTimeout.TotalSecondsCeiling()); - - // assert - message no longer in the message table - Assert.Empty(await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken)); - - // assert - message was moved - var dead = Assert.Single(await _storage.ReadAsync( - "SELECT * FROM OrleansStreamDeadLetter", - TestContext.Current.CancellationToken)); - Assert.Equal(serviceId, dead.ServiceId); - Assert.Equal(providerId, dead.ProviderId); - Assert.Equal(queueId, dead.QueueId); - Assert.Equal(ack.MessageId, dead.MessageId); - Assert.Equal(1, dead.Dequeued); - Assert.Equal(dead.CreatedOn.Add(streamOptions.ExpiryTimeout.SecondsCeiling()), dead.ExpiresOn); - Assert.Equal(dead.ModifiedOn, dead.VisibleOn); - Assert.Equal(dead.DeadOn.Add(streamOptions.DeadLetterEvictionTimeout.SecondsCeiling()), dead.RemoveOn); - Assert.Equal(payload, dead.Payload); - } - - /// - /// Tests that a healthy message is not moved to dead letters. - /// - [Fact] - public async Task RelationalOrleansQueries_DoesNotMoveHealthyMessageToDeadLetters() - { - // arrange - var serviceId = "ServiceId"; - var providerId = "ProviderId"; - var queueId = "QueueId"; - var streamOptions = new AdoNetStreamOptions(); - var cacheOptions = new SimpleQueueCacheOptions(); - - // arrange - queue a normal message - var payload = new byte[] { 0xFF }; - var ack = await _queries.QueueStreamMessageAsync(serviceId, providerId, queueId, payload, streamOptions.ExpiryTimeout.TotalSecondsCeiling()); - - // arrange - dequeue the message - await _queries.GetStreamMessagesAsync(ack.ServiceId, ack.ProviderId, ack.QueueId, cacheOptions.CacheSize, streamOptions.MaxAttempts, streamOptions.VisibilityTimeout.TotalSecondsCeiling(), streamOptions.DeadLetterEvictionTimeout.TotalSecondsCeiling(), streamOptions.EvictionInterval.TotalSecondsCeiling(), streamOptions.EvictionBatchSize); - - // act - fail the message - await _queries.FailStreamMessageAsync(ack.ServiceId, ack.ProviderId, ack.QueueId, ack.MessageId, streamOptions.MaxAttempts, streamOptions.DeadLetterEvictionTimeout.TotalSecondsCeiling()); - - // assert - the message is still in the table and was made visible again - var saved = Assert.Single(await _storage.ReadAsync( - "SELECT * FROM OrleansStreamMessage", - TestContext.Current.CancellationToken)); - Assert.Equal(ack.ServiceId, saved.ServiceId); - Assert.Equal(ack.ProviderId, saved.ProviderId); - Assert.Equal(ack.QueueId, saved.QueueId); - Assert.Equal(ack.MessageId, saved.MessageId); - Assert.Equal(1, saved.Dequeued); - Assert.Equal(saved.ModifiedOn, saved.VisibleOn); - - // assert - no message arrived at dead letters - Assert.Empty(await _storage.ReadAsync( - "SELECT * FROM OrleansStreamDeadLetter", - TestContext.Current.CancellationToken)); - } - - private static List Randomize(IEnumerable source) - { - var list = new List(source.TryGetNonEnumeratedCount(out var count) ? count : 0); - - foreach (var item in source) - { - var index = Random.Shared.Next(list.Count + 1); - if (index == list.Count) - { - list.Add(item); - } - else - { - list.Add(list[index]); - list[index] = item; - } - } - - return list; - } -} \ No newline at end of file diff --git a/test/Extensions/Orleans.Redis.Tests/Streaming/RedisStreamBatchContainerTests.cs b/test/Extensions/Orleans.Redis.Tests/Streaming/RedisStreamBatchContainerTests.cs index 1a96e4081f4..a79178f63d9 100644 --- a/test/Extensions/Orleans.Redis.Tests/Streaming/RedisStreamBatchContainerTests.cs +++ b/test/Extensions/Orleans.Redis.Tests/Streaming/RedisStreamBatchContainerTests.cs @@ -58,4 +58,18 @@ public void RedisStreamSequenceToken_CompareTo_UsesRedisSequenceNumberBeforeEven Assert.True(sameEntryNextEvent.CompareTo(nextEntrySameMillisecond) < 0); Assert.True(nextEntrySameMillisecond.CompareTo(nextMillisecond) < 0); } + + [Fact] + public void CreateSequenceTokenForEventPreservesRedisPosition() + { + var batchToken = new RedisStreamSequenceToken("100-7", 100, 7, 0); + + var eventToken = batchToken.CreateSequenceTokenForEvent(2); + + Assert.Equal("100-7", eventToken.EntryId); + Assert.Equal(100, eventToken.SequenceNumber); + Assert.Equal(7, eventToken.RedisSequenceNumber); + Assert.Equal(2, eventToken.EventIndex); + Assert.True(batchToken.CompareTo(eventToken) < 0); + } } diff --git a/test/Extensions/Orleans.Redis.Tests/Streaming/RedisStreamSequenceTokenTests.cs b/test/Extensions/Orleans.Redis.Tests/Streaming/RedisStreamSequenceTokenTests.cs new file mode 100644 index 00000000000..22d07986d34 --- /dev/null +++ b/test/Extensions/Orleans.Redis.Tests/Streaming/RedisStreamSequenceTokenTests.cs @@ -0,0 +1,45 @@ +using Orleans.Providers.Streams.Common; +using Orleans.Streaming.Redis; +using Orleans.Streams; +using Xunit; + +namespace Tester.Redis.Streaming; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Streaming")] +public sealed class RedisStreamSequenceTokenTests +{ + [Fact] + public void RedisTokensUseOneSymmetricEqualityAndOrderingContract() + { + StreamSequenceToken first = new RedisStreamSequenceToken("10-1", 10, 1, 0); + StreamSequenceToken equal = new RedisStreamSequenceToken("10-1", 10, 1, 0); + StreamSequenceToken differentEntry = new RedisStreamSequenceToken("010-1", 10, 1, 0); + StreamSequenceToken baseToken = new EventSequenceTokenV2(10, 0); + + Assert.True(first.Equals(equal)); + Assert.True(equal.Equals(first)); + Assert.Equal(0, first.CompareTo(equal)); + Assert.Equal(first.GetHashCode(), equal.GetHashCode()); + Assert.NotEqual(0, first.CompareTo(differentEntry)); + Assert.Equal(-Math.Sign(first.CompareTo(differentEntry)), Math.Sign(differentEntry.CompareTo(first))); + Assert.False(first.Equals(differentEntry)); + Assert.False(differentEntry.Equals(first)); + Assert.False(first.Equals(baseToken)); + Assert.False(baseToken.Equals(first)); + Assert.Throws(() => first.CompareTo(baseToken)); + Assert.Throws(() => baseToken.CompareTo(first)); + Assert.Single(new Dictionary + { + [first] = "first", + [equal] = "equal", + }); + Assert.Equal(2, new Dictionary + { + [first] = "first", + [differentEntry] = "different", + }.Count); + Assert.Equal(2, new SortedSet { first, differentEntry }.Count); + } +} diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/AzureTableStreamQueueCheckpointerContractTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/AzureTableStreamQueueCheckpointerContractTests.cs new file mode 100644 index 00000000000..cbf3ce824f0 --- /dev/null +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/AzureTableStreamQueueCheckpointerContractTests.cs @@ -0,0 +1,42 @@ +using Orleans.Streams; +using TestExtensions; +using UnitTests.StreamingTests; + +namespace ServiceBus.Tests.CheckpointerTests; + +[TestSuite("BVT")] +[TestProvider("EventHub")] +[TestArea("Streaming")] +[TestCategory("EventHub"), TestCategory("Streaming")] +public sealed class AzureTableStreamQueueCheckpointerContractTests : StreamQueueCheckpointerTests +{ + protected override OffsetRegressionPolicy RegressionPolicy => OffsetRegressionPolicy.Ignore; + + protected override Task> CreateCheckpointer( + ControllableCheckpointStore store) + => Task.FromResult>( + new AzureTableStreamQueueCheckpointer( + new TestStore(store), + PersistInterval, + StreamCheckpointComparers.Numeric)); + + private sealed class TestStore(ControllableCheckpointStore store) : IStreamCheckpointStore + { + public async ValueTask Load(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var checkpoint = await store.Load().ConfigureAwait(false); + return new(checkpoint, checkpoint); + } + + public async ValueTask Update( + string checkpoint, + string expectedVersion, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var persisted = await store.Write(checkpoint).ConfigureAwait(false); + return new(persisted, persisted); + } + } +} diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs index d5e208af096..eaa80ce8f07 100644 --- a/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/CheckpointerTests/EventHubCheckpointerTests.cs @@ -386,7 +386,10 @@ public async Task Update_WhenOffsetDoesNotAdvance_DoesNotChangeCheckpoint(string var checkpointer = CreateUninitializedCheckpointer(); SetPersistedOffset(checkpointer, "20"); - checkpointer.Update(candidate, new DateTime(2026, 1, 2, 3, 4, 5, DateTimeKind.Utc)); + checkpointer.Update( + candidate, + new DateTime(2026, 1, 2, 3, 4, 5, DateTimeKind.Utc), + TestContext.Current.CancellationToken); await checkpointer.FlushAsync(TestContext.Current.CancellationToken); Assert.True(checkpointer.CheckpointExists); @@ -401,11 +404,13 @@ public void Update_WhenOffsetAdvances_TracksLatestCheckpoint() var checkpointer = CreateUninitializedCheckpointer(); SetPersistedOffset(checkpointer, "20"); - checkpointer.Update("21", new DateTime(2026, 1, 2, 3, 4, 5, DateTimeKind.Utc)); + checkpointer.Update( + "21", + new DateTime(2026, 1, 2, 3, 4, 5, DateTimeKind.Utc), + TestContext.Current.CancellationToken); Assert.True(checkpointer.CheckpointExists); Assert.Equal("21", GetLatestOffset(checkpointer)); - Assert.Equal("21", GetEntityOffset(checkpointer)); } [TestSuite("BVT")] @@ -413,12 +418,15 @@ public void Update_WhenOffsetAdvances_TracksLatestCheckpoint() public void Update_WithNoComparer_TracksOpaqueCheckpoint() { var checkpointer = CreateUninitializedCheckpointer(useNumericComparer: false); + ThrottleSaves(checkpointer); - checkpointer.Update("opaque-checkpoint", new DateTime(2026, 1, 2, 3, 4, 5, DateTimeKind.Utc)); + checkpointer.Update( + "opaque-checkpoint", + new DateTime(2026, 1, 2, 3, 4, 5, DateTimeKind.Utc), + TestContext.Current.CancellationToken); Assert.True(checkpointer.CheckpointExists); Assert.Equal("opaque-checkpoint", GetLatestOffset(checkpointer)); - Assert.Equal("opaque-checkpoint", GetEntityOffset(checkpointer)); } [TestSuite("BVT")] @@ -441,7 +449,7 @@ public void EventHubCheckpointEntity_PreservesLegacyAzureTableSchema() streamProviderName: "provider/name", partition: "partition?1", serviceId: "service#id"); - var entity = GetField(checkpointer, "_entity"); + var entity = GetEntity(checkpointer); var entityType = entity.GetType(); Assert.Equal("EventHubCheckpoints_provider_name_service_id", GetEntityPartitionKey(checkpointer)); @@ -664,53 +672,68 @@ private static AzureTableStreamQueueCheckpointer CreateUninitializedCheckpointer private static void SetPersistedOffset(AzureTableStreamQueueCheckpointer checkpointer, string offset) { - SetField(checkpointer, "_latestCheckpoint", offset); - SetField(checkpointer, "_persistedCheckpoint", offset); + var inner = GetField(checkpointer, "_inner"); + SetField(inner, "_latestCheckpoint", offset); + SetField(inner, "_persistedState", new StreamCheckpointStoreState(offset, offset)); + SetField(inner, "_throttleSavesUntilUtc", DateTime.MaxValue); SetEntityOffset(checkpointer, offset); } private static string GetLatestOffset(AzureTableStreamQueueCheckpointer checkpointer) - => (string)GetField(checkpointer, "_latestCheckpoint"); + => (string)GetField(GetField(checkpointer, "_inner"), "_latestCheckpoint"); private static string GetEntityOffset(AzureTableStreamQueueCheckpointer checkpointer) - => (string)GetField(checkpointer, "_entity") + => (string)GetEntity(checkpointer) .GetType() .GetProperty("Offset")! - .GetValue(GetField(checkpointer, "_entity"))!; + .GetValue(GetEntity(checkpointer))!; private static string GetEntityPartitionKey(AzureTableStreamQueueCheckpointer checkpointer) - => (string)GetField(checkpointer, "_entity") + => (string)GetEntity(checkpointer) .GetType() .GetProperty("PartitionKey")! - .GetValue(GetField(checkpointer, "_entity"))!; + .GetValue(GetEntity(checkpointer))!; private static string GetEntityRowKey(AzureTableStreamQueueCheckpointer checkpointer) - => (string)GetField(checkpointer, "_entity") + => (string)GetEntity(checkpointer) .GetType() .GetProperty("RowKey")! - .GetValue(GetField(checkpointer, "_entity"))!; + .GetValue(GetEntity(checkpointer))!; private static void SetEntityOffset(AzureTableStreamQueueCheckpointer checkpointer, string offset) { - var entity = GetField(checkpointer, "_entity"); + var entity = GetEntity(checkpointer); entity.GetType().GetProperty("Offset")!.SetValue(entity, offset); } - private static object GetField(AzureTableStreamQueueCheckpointer checkpointer, string name) + private static object GetEntity(AzureTableStreamQueueCheckpointer checkpointer) { - var field = typeof(AzureTableStreamQueueCheckpointer).GetField( + var store = GetField(checkpointer, "_store"); + var property = store.GetType().GetProperty( + "Entity", + BindingFlags.Instance | BindingFlags.Public); + Assert.NotNull(property); + return property.GetValue(store)!; + } + + private static void ThrottleSaves(AzureTableStreamQueueCheckpointer checkpointer) + => SetField(GetField(checkpointer, "_inner"), "_throttleSavesUntilUtc", DateTime.MaxValue); + + private static object GetField(object instance, string name) + { + var field = instance.GetType().GetField( name, BindingFlags.Instance | BindingFlags.NonPublic); Assert.NotNull(field); - return field.GetValue(checkpointer)!; + return field.GetValue(instance)!; } - private static void SetField(AzureTableStreamQueueCheckpointer checkpointer, string name, object value) + private static void SetField(object instance, string name, object value) { - var field = typeof(AzureTableStreamQueueCheckpointer).GetField( + var field = instance.GetType().GetField( name, BindingFlags.Instance | BindingFlags.NonPublic); Assert.NotNull(field); - field.SetValue(checkpointer, value); + field.SetValue(instance, value); } } diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/EventHubSequenceTokenTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/EventHubSequenceTokenTests.cs new file mode 100644 index 00000000000..54e70fcf445 --- /dev/null +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/EventHubSequenceTokenTests.cs @@ -0,0 +1,81 @@ +using Orleans.Providers.Streams.Common; +using Orleans.Streaming.EventHubs; +using Orleans.Streams; +using Xunit; + +namespace UnitTests.Streaming; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Streaming")] +public sealed class EventHubSequenceTokenTests +{ + [Fact] + public void EventHubVersionsRemainSymmetricAndRejectBaseTokens() + { + StreamSequenceToken v1 = new EventHubSequenceToken("100", 10, 2); + StreamSequenceToken v2 = new EventHubSequenceTokenV2("100", 10, 2); + StreamSequenceToken baseToken = new EventSequenceToken(10, 2); + + Assert.True(v1.Equals(v2)); + Assert.True(v2.Equals(v1)); + Assert.Equal(0, v1.CompareTo(v2)); + Assert.Equal(0, v2.CompareTo(v1)); + Assert.Equal(v1.GetHashCode(), v2.GetHashCode()); + Assert.False(v1.Equals(baseToken)); + Assert.False(baseToken.Equals(v1)); + Assert.Throws(() => v1.CompareTo(baseToken)); + Assert.Throws(() => baseToken.CompareTo(v1)); + } + + [Fact] + public void EventHubOrderingUsesSequenceBeforeEventIndexAcrossVersions() + { + StreamSequenceToken olderSequence = new EventHubSequenceToken("9", 9, 99); + StreamSequenceToken newerSequence = new EventHubSequenceTokenV2("10", 10, 0); + StreamSequenceToken earlierEvent = new EventHubSequenceTokenV2("10", 10, 1); + StreamSequenceToken laterEvent = new EventHubSequenceToken("10", 10, 2); + + Assert.True(olderSequence.CompareTo(newerSequence) < 0); + Assert.True(newerSequence.CompareTo(olderSequence) > 0); + Assert.True(earlierEvent.CompareTo(laterEvent) < 0); + Assert.True(laterEvent.CompareTo(earlierEvent) > 0); + } + + [Fact] + public void SameCustomEventHubTokenSubtypeRemainsComparable() + { + StreamSequenceToken first = new CustomEventHubSequenceToken("10", 10, 2); + StreamSequenceToken second = new CustomEventHubSequenceToken("other-offset", 10, 2); + StreamSequenceToken builtIn = new EventHubSequenceToken("10", 10, 2); + + Assert.True(first.Equals(second)); + Assert.True(second.Equals(first)); + Assert.Equal(0, first.CompareTo(second)); + Assert.Equal(first.GetHashCode(), second.GetHashCode()); + Assert.False(first.Equals(builtIn)); + Assert.False(builtIn.Equals(first)); + Assert.Throws(() => first.CompareTo(builtIn)); + Assert.Throws(() => builtIn.CompareTo(first)); + } + + [Fact] + public void CreateSequenceTokenForEventPreservesConcreteTypeAndOffset() + { + var batchToken = new EventHubSequenceTokenV2("offset-10", 10, 0); + + var eventToken = Assert.IsType( + batchToken.CreateSequenceTokenForEvent(2)); + + Assert.Equal("offset-10", eventToken.EventHubOffset); + Assert.Equal(10, eventToken.SequenceNumber); + Assert.Equal(2, eventToken.EventIndex); + Assert.True(batchToken.CompareTo(eventToken) < 0); + } + + private sealed class CustomEventHubSequenceToken( + string eventHubOffset, + long sequenceNumber, + int eventIndex) + : EventHubSequenceToken(eventHubOffset, sequenceNumber, eventIndex); +} diff --git a/test/Extensions/Orleans.Streaming.EventHubs.Tests/EvictionStrategyTests/EHPurgeLogicTests.cs b/test/Extensions/Orleans.Streaming.EventHubs.Tests/EvictionStrategyTests/EHPurgeLogicTests.cs index cf5c1b9b330..0c001e87b2c 100644 --- a/test/Extensions/Orleans.Streaming.EventHubs.Tests/EvictionStrategyTests/EHPurgeLogicTests.cs +++ b/test/Extensions/Orleans.Streaming.EventHubs.Tests/EvictionStrategyTests/EHPurgeLogicTests.cs @@ -161,13 +161,12 @@ public async Task EventHubQueueCache_EvictionStrategy_Behavior() //perform purge - //after purge, inUseBuffers should be purged and return to the pool, except for the current buffer + //after purge, inUseBuffers should be purged and return to the pool var expectedPurgedBuffers = new List(); this.evictionStrategyList.ForEach(strategy => { var purgedBufferList = strategy.InUseBuffers.ToArray(); - //last one in purgedBufferList should be current buffer, which shouldn't be purged - for (int i = 0; i < purgedBufferList.Count() - 1; i++) + for (int i = 0; i < purgedBufferList.Count(); i++) expectedPurgedBuffers.Add(purgedBufferList[i]); }); @@ -175,8 +174,8 @@ public async Task EventHubQueueCache_EvictionStrategy_Behavior() this.receiver1.TryPurgeFromCache(out ignore); this.receiver2.TryPurgeFromCache(out ignore); - //Each cache should have all buffers purged, except for current buffer - this.evictionStrategyList.ForEach(strategy => Assert.Single(strategy.InUseBuffers)); + //Each cache should have all buffers purged + this.evictionStrategyList.ForEach(strategy => Assert.Empty(strategy.InUseBuffers)); var oldBuffersInCaches = new List(); this.evictionStrategyList.ForEach(strategy => { foreach (var inUseBuffer in strategy.InUseBuffers) diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/DynamoDBStreamQueueCheckpointerTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/DynamoDBStreamQueueCheckpointerTests.cs index ba67802a1d4..df568ea1ef9 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/DynamoDBStreamQueueCheckpointerTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/DynamoDBStreamQueueCheckpointerTests.cs @@ -34,20 +34,21 @@ protected override Task> CreateCheckpointer( private sealed class TestCheckpointStore(ControllableCheckpointStore store) : IDynamoDBStreamCheckpointStore { - public ValueTask Load(CancellationToken cancellationToken) + public async ValueTask Load(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - return new(store.Load()); + var checkpoint = await store.Load().ConfigureAwait(false); + return new(checkpoint, checkpoint); } - public async ValueTask Update( + public async ValueTask Update( string checkpoint, - string expectedCheckpoint, + string expectedVersion, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); await store.Write(checkpoint).ConfigureAwait(false); - return checkpoint; + return new(checkpoint, checkpoint); } } } @@ -74,9 +75,11 @@ public async Task UpdatePersistsArbitrarySizeSequenceNumberAsString() }); var store = CreateStore(client); - await store.Update(checkpoint, string.Empty, TestContext.Current.CancellationToken); + var state = await store.Update(checkpoint, string.Empty, TestContext.Current.CancellationToken); Assert.NotNull(write); + Assert.Equal(checkpoint, state.Checkpoint); + Assert.Equal("1", state.Version); Assert.Equal(checkpoint, write.Item[DynamoDBStreamCheckpointStore.CheckpointAttribute].S); Assert.Equal("1", write.Item[DynamoDBStreamCheckpointStore.VersionAttribute].N); Assert.Equal( @@ -99,12 +102,14 @@ public async Task ConditionalConflictWithNewerCheckpointDoesNotOverwriteIt() new ConditionalCheckFailedException("stale checkpoint"))); var store = CreateStore(client); - Assert.Equal("20", await store.Update("10", string.Empty, TestContext.Current.CancellationToken)); + var result = await store.Update("10", string.Empty, TestContext.Current.CancellationToken); + Assert.Equal("20", result.Checkpoint); + Assert.Equal("7", result.Version); await client.Received(1).PutItemAsync( Arg.Any(), Arg.Any()); - Assert.Equal("20", await store.Load(TestContext.Current.CancellationToken)); + Assert.Equal("20", (await store.Load(TestContext.Current.CancellationToken)).Checkpoint); } [Fact] @@ -131,8 +136,12 @@ public async Task ConditionalConflictWithOlderCheckpointAllowsRetryUsingItsVersi }); var store = CreateStore(client); - Assert.Equal("20", await store.Update("30", string.Empty, TestContext.Current.CancellationToken)); - Assert.Equal("30", await store.Update("30", "20", TestContext.Current.CancellationToken)); + var conflicted = await store.Update("30", string.Empty, TestContext.Current.CancellationToken); + Assert.Equal("20", conflicted.Checkpoint); + Assert.Equal("7", conflicted.Version); + var updated = await store.Update("30", "7", TestContext.Current.CancellationToken); + Assert.Equal("30", updated.Checkpoint); + Assert.Equal("8", updated.Version); Assert.Equal(2, writes.Count); Assert.Equal("#version = :expectedVersion", writes[1].ConditionExpression); @@ -142,20 +151,21 @@ public async Task ConditionalConflictWithOlderCheckpointAllowsRetryUsingItsVersi writes[1].ExpressionAttributeNames["#version"]); Assert.Equal("7", writes[1].ExpressionAttributeValues[":expectedVersion"].N); Assert.Equal("8", writes[1].Item[DynamoDBStreamCheckpointStore.VersionAttribute].N); - Assert.Equal("30", await store.Load(TestContext.Current.CancellationToken)); + Assert.Equal("30", (await store.Load(TestContext.Current.CancellationToken)).Checkpoint); } [Fact] - public async Task ExpectedCheckpointMismatchReturnsPersistedCheckpointWithoutWriting() + public async Task ExpectedVersionMismatchReturnsPersistedStateWithoutWriting() { var client = Substitute.For(); client.GetItemAsync(Arg.Any(), Arg.Any()) .Returns(Task.FromResult(CreateReadResponse("20", 7))); var store = CreateStore(client); - var result = await store.Update("30", "10", TestContext.Current.CancellationToken); + var result = await store.Update("30", "6", TestContext.Current.CancellationToken); - Assert.Equal("20", result); + Assert.Equal("20", result.Checkpoint); + Assert.Equal("7", result.Version); await client.DidNotReceive().PutItemAsync( Arg.Any(), Arg.Any()); diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisAdapterTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisAdapterTests.cs index 2899673b8ec..08e299d3481 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisAdapterTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisAdapterTests.cs @@ -105,6 +105,7 @@ private async Task SendAndReceiveFromQueueAdapter( int receivedBatches = 0; var streamsPerQueue = new ConcurrentDictionary>(); + var firstTokens = new ConcurrentDictionary<(QueueId QueueId, StreamId StreamId), StreamSequenceToken>(); // send events List events = CreateEvents(NumMessagesPerBatch); @@ -123,11 +124,18 @@ await Task.WhenAll(Enumerable.Range(0, NumBatches) foreach (var (queueId, receiver) in receivers) { var messages = (await receiver.GetQueueMessagesAsync(10, cancellationToken)).ToArray(); - foreach (var message in messages.Cast()) + foreach (var notification in messages) { + using var cursor = caches[queueId].GetCacheCursor( + notification.StreamId, + notification.SequenceToken); + Assert.True(cursor.MoveNext()); + var message = Assert.IsType(cursor.GetCurrent(out var exception)); + Assert.Null(exception); output.WriteLine($"Queue {queueId} received message on stream {message.StreamId}"); Assert.Equal(NumMessagesPerBatch / 2, message.GetEvents().Count()); Assert.Equal(NumMessagesPerBatch / 2, message.GetEvents().Count()); + firstTokens.TryAdd((queueId, message.StreamId), message.SequenceToken); streamsPerQueue.AddOrUpdate( queueId, @@ -150,7 +158,6 @@ await Task.WhenAll(Enumerable.Range(0, NumBatches) Assert.Equal(NumBatches, receivedBatches); // check to see if all the events are in the cache and we can enumerate through them - StreamSequenceToken firstInCache = new EventSequenceTokenV2(0); foreach (KeyValuePair> kvp in streamsPerQueue) { var receiver = receivers[kvp.Key]; @@ -158,6 +165,7 @@ await Task.WhenAll(Enumerable.Range(0, NumBatches) foreach (StreamId streamGuid in kvp.Value) { + var firstInCache = firstTokens[(kvp.Key, streamGuid)]; // read all messages in cache for stream IQueueCacheCursor cursor = qCache.GetCacheCursor(streamGuid, firstInCache); int messageCount = 0; diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisBatchContainerTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisBatchContainerTests.cs index e70e7ea9de2..3222da5bfd0 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisBatchContainerTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisBatchContainerTests.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.DependencyInjection; +using Orleans.Providers.Streams.Common; using Orleans.Runtime; using Orleans.Serialization; using Orleans.Streaming.Kinesis; @@ -40,5 +41,130 @@ public void GetEventsFiltersByRequestedType() Assert.Equal([1, 3], batch.GetEvents().Select(item => item.Item1)); Assert.Equal(["two"], batch.GetEvents().Select(item => item.Item1)); + + Assert.False(batch.ImportRequestContext()); + } + + [Fact] + public void GetEventsAssignsDistinctEventIndexPerEventWithinSameRecord() + { + var streamId = StreamId.Create("test", Guid.NewGuid()); + var payload = KinesisBatchContainer.ToKinesisPayload( + serializer, + streamId, + new object[] { "first", "second", "third" }, + requestContext: null); + var record = new KinesisRecord + { + Data = new MemoryStream(payload), + SequenceNumber = "999999999999999999999", + }; + var batch = KinesisBatchContainer.FromKinesisRecord(serializer, record, sequenceId: 0); + + var tokens = batch.GetEvents() + .Select(item => (Value: item.Item1, Token: (KinesisSequenceToken)item.Item2)) + .ToArray(); + + Assert.Equal(["first", "second", "third"], tokens.Select(t => t.Value)); + Assert.Equal([0, 1, 2], tokens.Select(t => t.Token.EventIndex)); + + Assert.True(tokens[0].Token.CompareTo(tokens[1].Token) < 0); + Assert.True(tokens[1].Token.CompareTo(tokens[2].Token) < 0); + Assert.All(tokens, t => Assert.Equal(record.SequenceNumber, t.Token.ShardSequence)); + } + + [Fact] + public void OldPayloadShapeWithRequestContextDecodesUnchanged() + { + const string legacyPayload = "ICAABQkDHUgFGWxlZ2FjeS1ldmVudOAhAQNBH2xlZ2FjeS10cmFjZS1pZEgFEXRyYWNlLTQy4CFAYWxlZ2FjeS1uYW1lc3BhY2UxMTExMTExMTIyMjIzMzMzNDQ0NDU1NTU1NTU1NTU1NQEhYSSIghng4A=="; + var streamId = StreamId.Create("legacy-namespace", Guid.Parse("11111111-2222-3333-4444-555555555555")); + var requestContext = new Dictionary { ["legacy-trace-id"] = "trace-42" }; + var payload = KinesisBatchContainer.ToKinesisPayload( + serializer, + streamId, + new object[] { 7, "legacy-event" }, + requestContext); + Assert.Equal(legacyPayload, Convert.ToBase64String(payload)); + + var record = new KinesisRecord + { + Data = new MemoryStream(Convert.FromBase64String(legacyPayload)), + SequenceNumber = "123456789012345678901234567890", + }; + + var batch = KinesisBatchContainer.FromKinesisRecord(serializer, record, sequenceId: 0); + + Assert.Equal(streamId, batch.StreamId); + Assert.Equal([7], batch.GetEvents().Select(item => item.Item1)); + Assert.Equal(["legacy-event"], batch.GetEvents().Select(item => item.Item1)); + + Assert.True(batch.ImportRequestContext()); + try + { + Assert.Equal("trace-42", RequestContext.Get("legacy-trace-id")); + } + finally + { + RequestContext.Clear(); + } + } + + [Fact] + public void CompareToOrdersByDurableShardSequenceNotReceiverLocalOrdinal() + { + var readFirstButNewer = KinesisBatchContainer.FromKinesisRecord( + serializer, + new KinesisRecord { Data = new MemoryStream(), SequenceNumber = "200000000000000000000000000000" }, + sequenceId: 0); + + var readSecondButOlder = KinesisBatchContainer.FromKinesisRecord( + serializer, + new KinesisRecord { Data = new MemoryStream(), SequenceNumber = "1" }, + sequenceId: 1); + + Assert.True(readFirstButNewer.CompareTo(readSecondButOlder) > 0); + Assert.True(readSecondButOlder.CompareTo(readFirstButNewer) < 0); + } + + [Fact] + public void RecoverableDataAdapter_PreservesRawPayloadAndExternalOffsetOrdering() + { + var streamId = StreamId.Create("test", Guid.NewGuid()); + var payload = KinesisBatchContainer.ToKinesisPayload( + serializer, + streamId, + new[] { "event" }, + requestContext: null); + var record = new KinesisRecord + { + Data = new MemoryStream(payload), + SequenceNumber = "123456789012345678901234567890", + }; + var queueMessage = new KinesisCacheRecord(record, sequenceNumber: 7); + var adapter = new KinesisRecoverableStreamDataAdapter(serializer); + + var position = adapter.GetStreamPosition(queueMessage); + var rawPayload = queueMessage.RawPayload; + record.Data = null!; + var cached = adapter.FromQueueMessage( + position, + queueMessage, + DateTime.UtcNow, + size => new byte[size]); + + Assert.Equal(streamId, cached.StreamId); + Assert.Same(rawPayload, queueMessage.RawPayload); + Assert.Equal(record.SequenceNumber, adapter.GetOffset(ref cached)); + Assert.True(adapter.Compare( + ref cached, + new KinesisSequenceToken("123456789012345678901234567889", 1000, 0)) > 0); + Assert.Equal(0, adapter.Compare( + ref cached, + new KinesisSequenceToken("000123456789012345678901234567890", 1000, 0))); + + var batch = Assert.IsType(adapter.GetBatchContainer(ref cached)); + Assert.Equal(streamId, batch.StreamId); + Assert.Equal(["event"], batch.GetEvents().Select(item => item.Item1)); + Assert.Equal(record.SequenceNumber, ((KinesisSequenceToken)batch.SequenceToken).ShardSequence); } } diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs index e539f5a364e..c01807adf32 100644 --- a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisRuntimeTests.cs @@ -1,8 +1,11 @@ using Amazon.Kinesis; using Amazon.Kinesis.Model; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Time.Testing; using NSubstitute; +using Orleans.Configuration; +using Orleans.Serialization; using Orleans.Streaming.Kinesis; using Orleans.Streams; using TestExtensions; @@ -93,6 +96,97 @@ public void ExplicitRegionOverridesServiceUrl() Assert.Equal("eu-west-1", KinesisAdapterFactory.GetRegionName(options)); } + [Fact] + public void QueueAdapterFactoryIsRewindable() + { + var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService>(); + using var factory = new KinesisAdapterFactory( + "Kinesis", + new KinesisStreamOptions(), + new SimpleQueueCacheOptions(), + serializer, + checkpointerFactory: null, + NullLoggerFactory.Instance); + + Assert.True(factory.IsRewindable); + Assert.Equal(StreamProviderDirection.ReadWrite, factory.Direction); + } + + [Fact] + public async Task PooledReceiver_ReadsAndDisposesLifecycleCancellationOnShutdown() + { + var client = Substitute.For(); + client.GetShardIteratorAsync( + Arg.Any(), + Arg.Any()) + .Returns(new GetShardIteratorResponse { ShardIterator = "iterator" }); + client.GetRecordsAsync(Arg.Any(), Arg.Any()) + .Returns(new GetRecordsResponse + { + NextShardIterator = "iterator", + Records = [], + }); + var checkpointer = Substitute.For>(); + checkpointer.Load(Arg.Any()).Returns(string.Empty); + var checkpointerFactory = Substitute.For(); + checkpointerFactory.Create("shard-1", Arg.Any()).Returns(checkpointer); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService>(); + var timeProvider = new FakeTimeProvider(); + var topologyMonitor = new KinesisShardTopologyMonitor( + client, + "stream", + ["shard-1"], + TimeSpan.FromMinutes(1), + timeProvider, + NullLogger.Instance); + var receiver = new KinesisPooledAdapterReceiver( + client, + "stream", + "shard-1", + checkpointerFactory, + new SimpleQueueCacheOptions(), + serializer, + NullLoggerFactory.Instance, + topologyMonitor, + TimeSpan.Zero, + timeProvider); + var lifecycleCancellationToken = receiver.LifecycleCancellationToken; + await receiver.Initialize(TimeSpan.FromSeconds(5)); + + Assert.Empty(await receiver.GetQueueMessagesAsync(10, CancellationToken.None)); + + await client.Received(1).GetRecordsAsync( + Arg.Any(), + Arg.Any()); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + + Assert.True(lifecycleCancellationToken.IsCancellationRequested); + Assert.Throws(() => _ = receiver.LifecycleCancellationToken); + } + + [Fact] + public async Task InitialShardIteratorUsesTrimHorizonWhenNoCheckpointExists() + { + var client = Substitute.For(); + var checkpointer = Substitute.For>(); + checkpointer.Load(Arg.Any()).Returns(string.Empty); + var checkpointerFactory = Substitute.For(); + checkpointerFactory.Create("shard-1", Arg.Any()).Returns(checkpointer); + client.GetShardIteratorAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(new GetShardIteratorResponse { ShardIterator = "iterator-1" })); + var receiver = CreateReceiver(client, checkpointerFactory, new FakeTimeProvider()); + + await receiver.Initialize(TimeSpan.FromSeconds(5)); + + await client.Received(1).GetShardIteratorAsync( + Arg.Is(request => + request.ShardIteratorType == ShardIteratorType.TRIM_HORIZON + && request.StartingSequenceNumber == null), + Arg.Any()); + } + [Fact] public async Task TopologyMonitorLatchesWhenShardSetChanges() { @@ -149,6 +243,99 @@ await client.Received(1).GetShardIteratorAsync( Arg.Any()); } + [Fact] + public async Task ReceiverAssignsMonotonicallyIncreasingLocalOrdinalsAcrossReads() + { + var client = Substitute.For(); + var checkpointer = Substitute.For>(); + checkpointer.Load(Arg.Any()).Returns(string.Empty); + var checkpointerFactory = Substitute.For(); + checkpointerFactory.Create("shard-1", Arg.Any()).Returns(checkpointer); + client.GetShardIteratorAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(new GetShardIteratorResponse { ShardIterator = "iterator-1" })); + client.GetRecordsAsync(Arg.Any(), Arg.Any()) + .Returns( + Task.FromResult(new GetRecordsResponse + { + NextShardIterator = "iterator-2", + Records = [ + new Amazon.Kinesis.Model.Record { SequenceNumber = "10", Data = new MemoryStream() }, + new Amazon.Kinesis.Model.Record { SequenceNumber = "20", Data = new MemoryStream() }, + ], + }), + Task.FromResult(new GetRecordsResponse + { + NextShardIterator = "iterator-3", + Records = [new Amazon.Kinesis.Model.Record { SequenceNumber = "30", Data = new MemoryStream() }], + })); + var timeProvider = new FakeTimeProvider { AutoAdvanceAmount = TimeSpan.FromMilliseconds(200) }; + var receiver = CreateReceiver(client, checkpointerFactory, timeProvider); + await receiver.Initialize(TimeSpan.FromSeconds(5)); + + var firstBatch = (await receiver.GetQueueMessagesAsync( + 10, + TestContext.Current.CancellationToken)).Cast().ToArray(); + var secondBatch = (await receiver.GetQueueMessagesAsync( + 10, + TestContext.Current.CancellationToken)).Cast().ToArray(); + + Assert.Equal([0L, 1L], firstBatch.Select(container => container.Token.SequenceNumber)); + Assert.Equal([2L], secondBatch.Select(container => container.Token.SequenceNumber)); + } + + [Fact] + public async Task MessagesDeliveredCommitsNumericallyHighestShardSequence() + { + var client = Substitute.For(); + var checkpointer = Substitute.For>(); + checkpointer.Load(Arg.Any()).Returns(string.Empty); + var checkpointerFactory = Substitute.For(); + checkpointerFactory.Create("shard-1", Arg.Any()).Returns(checkpointer); + client.GetShardIteratorAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(new GetShardIteratorResponse { ShardIterator = "iterator-1" })); + var receiver = CreateReceiver(client, checkpointerFactory, new FakeTimeProvider()); + await receiver.Initialize(TimeSpan.FromSeconds(5)); + + var hugeButReadFirst = KinesisBatchContainer.FromKinesisRecord( + null!, + new Amazon.Kinesis.Model.Record { SequenceNumber = "170141183460469231731687303715884105727", Data = new MemoryStream() }, + sequenceId: 0); + var smallButReadSecond = KinesisBatchContainer.FromKinesisRecord( + null!, + new Amazon.Kinesis.Model.Record { SequenceNumber = "42", Data = new MemoryStream() }, + sequenceId: 1); + + await receiver.MessagesDeliveredAsync( + [hugeButReadFirst, smallButReadSecond], + TestContext.Current.CancellationToken); + + checkpointer.Received(1).Update( + "170141183460469231731687303715884105727", + Arg.Any(), + Arg.Any()); + checkpointer.DidNotReceive().Update("42", Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task MessagesDeliveredWithEmptyListDoesNotUpdateCheckpointOrThrow() + { + var client = Substitute.For(); + var checkpointer = Substitute.For>(); + checkpointer.Load(Arg.Any()).Returns(string.Empty); + var checkpointerFactory = Substitute.For(); + checkpointerFactory.Create("shard-1", Arg.Any()).Returns(checkpointer); + client.GetShardIteratorAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(new GetShardIteratorResponse { ShardIterator = "iterator-1" })); + var receiver = CreateReceiver(client, checkpointerFactory, new FakeTimeProvider()); + await receiver.Initialize(TimeSpan.FromSeconds(5)); + + await receiver.MessagesDeliveredAsync( + Array.Empty(), + TestContext.Current.CancellationToken); + + checkpointer.DidNotReceive().Update(Arg.Any(), Arg.Any(), Arg.Any()); + } + [Fact] public async Task ReceiverLimitsGetRecordsToFiveCallsPerSecond() { @@ -283,6 +470,80 @@ await client.Received(1).GetRecordsAsync( TestContext.Current.CancellationToken); } + [Fact] + public async Task ConcurrentInitializationRetriesWhenOwningCallerCancels() + { + var client = Substitute.For(); + var checkpointer = Substitute.For>(); + checkpointer.Load(Arg.Any()).Returns(string.Empty); + var checkpointerFactory = Substitute.For(); + var firstAttemptStarted = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + var createCount = 0; + checkpointerFactory.Create("shard-1", Arg.Any()) + .Returns(async call => + { + var token = call.Arg(); + if (Interlocked.Increment(ref createCount) == 1) + { + firstAttemptStarted.SetResult(token); + await Task.Delay(Timeout.InfiniteTimeSpan, token); + } + + return checkpointer; + }); + client.GetShardIteratorAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(new GetShardIteratorResponse { ShardIterator = "iterator-1" })); + client.GetRecordsAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(new GetRecordsResponse + { + NextShardIterator = "iterator-1", + Records = [], + })); + var receiver = CreateReceiver(client, checkpointerFactory, new FakeTimeProvider()); + using var ownerCancellation = new CancellationTokenSource(); + + var owner = receiver.GetQueueMessagesAsync(10, ownerCancellation.Token); + var unaffected = receiver.GetQueueMessagesAsync(10, CancellationToken.None); + var firstAttemptToken = await firstAttemptStarted.Task; + ownerCancellation.Cancel(); + + await Assert.ThrowsAnyAsync(() => owner); + Assert.Empty(await unaffected); + + Assert.True(firstAttemptToken.IsCancellationRequested); + Assert.Equal(2, createCount); + await client.Received(1).GetShardIteratorAsync( + Arg.Any(), + Arg.Any()); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task ConcurrentInitializationPropagatesProviderCancellationWhenOwnerIsActive() + { + var client = Substitute.For(); + var checkpointerFactory = Substitute.For(); + var createCount = 0; + checkpointerFactory.Create("shard-1", Arg.Any()) + .Returns(_ => + { + Interlocked.Increment(ref createCount); + return Task.FromException>( + new OperationCanceledException("provider canceled independently")); + }); + var receiver = CreateReceiver(client, checkpointerFactory, new FakeTimeProvider()); + + await Assert.ThrowsAnyAsync( + () => receiver.GetQueueMessagesAsync(10, CancellationToken.None)); + + Assert.Equal(1, createCount); + await client.DidNotReceive().GetShardIteratorAsync( + Arg.Any(), + Arg.Any()); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + } + [Fact] public async Task ReceiverReadForwardsCancellationToken() { diff --git a/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisSequenceTokenTests.cs b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisSequenceTokenTests.cs new file mode 100644 index 00000000000..aeff22373a6 --- /dev/null +++ b/test/Extensions/Orleans.Streaming.Kinesis.Tests/KinesisSequenceTokenTests.cs @@ -0,0 +1,185 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Newtonsoft.Json; +using Orleans.Serialization; +using Orleans.Providers.Streams.Common; +using Orleans.Streams; +using Orleans.Streaming.Kinesis; +using TestExtensions; +using Xunit; + +namespace Orleans.Streaming.Kinesis.Tests; + +[TestSuite("BVT")] +[TestArea("Streaming")] +[TestProvider("Kinesis")] +[TestCategory("AWS"), TestCategory("Kinesis")] +[Collection(TestEnvironmentFixture.DefaultCollection)] +public sealed class KinesisSequenceTokenTests +{ + // Representative of real Kinesis shard sequence numbers (up to 128 bits), which are far beyond + // long.MaxValue (~9.22e18, 19 digits). + private const string HugeShardSequence = "170141183460469231731687303715884105727"; + private const string SlightlyLargerShardSequence = "170141183460469231731687303715884105728"; + + private readonly Serializer serializer; + + public KinesisSequenceTokenTests(TestEnvironmentFixture fixture) + { + serializer = fixture.Services.GetRequiredService>(); + } + + [Fact] + public void CompareToOrdersByShardSequenceMagnitudeBeyondInt64Range() + { + var older = new KinesisSequenceToken(HugeShardSequence, sequenceNumber: 0, eventIndex: 0); + var newer = new KinesisSequenceToken(SlightlyLargerShardSequence, sequenceNumber: 0, eventIndex: 0); + + Assert.True(older.CompareTo(newer) < 0); + Assert.True(newer.CompareTo(older) > 0); + Assert.Equal(0, older.CompareTo(older)); + Assert.False(older.Equals(newer)); + } + + [Theory] + [InlineData("9", "10")] + [InlineData("99", "100")] + [InlineData("999999999999999999", "1000000000000000000")] + public void CompareToUsesNumericNotLexicographicOrdering(string smaller, string larger) + { + var smallerToken = new KinesisSequenceToken(smaller, sequenceNumber: 0, eventIndex: 0); + var largerToken = new KinesisSequenceToken(larger, sequenceNumber: 0, eventIndex: 0); + + Assert.True(smallerToken.CompareTo(largerToken) < 0); + Assert.True(string.CompareOrdinal(smaller, larger) > 0); + } + + [Fact] + public void CompareToBreaksTiesOnEventIndexWhenShardSequenceMatches() + { + var first = new KinesisSequenceToken(HugeShardSequence, sequenceNumber: 5, eventIndex: 0); + var second = new KinesisSequenceToken(HugeShardSequence, sequenceNumber: 5, eventIndex: 1); + + Assert.True(first.CompareTo(second) < 0); + Assert.True(second.CompareTo(first) > 0); + Assert.False(first.Equals(second)); + + var differentReceiverOrdinal = new KinesisSequenceToken(HugeShardSequence, sequenceNumber: 999, eventIndex: 0); + Assert.Equal(0, first.CompareTo(differentReceiverOrdinal)); + Assert.True(first.Equals(differentReceiverOrdinal)); + } + + [Fact] + public void EqualsIgnoresShardSequenceStringFormattingButRespectsNumericValue() + { + var zeroPadded = new KinesisSequenceToken("007", sequenceNumber: 1, eventIndex: 2); + var unpadded = new KinesisSequenceToken("7", sequenceNumber: 999, eventIndex: 2); + + Assert.True(zeroPadded.Equals(unpadded)); + Assert.True(zeroPadded.Equals((object)unpadded)); + Assert.Equal(zeroPadded.GetHashCode(), unpadded.GetHashCode()); + } + + [Fact] + public void EquivalentTokensCoalesceInHashAndSortedCollections() + { + StreamSequenceToken zeroPadded = new KinesisSequenceToken("007", sequenceNumber: 1, eventIndex: 2); + StreamSequenceToken unpadded = new KinesisSequenceToken("7", sequenceNumber: 999, eventIndex: 2); + + Assert.Single(new HashSet { zeroPadded, unpadded }); + Assert.Single(new SortedSet { unpadded, zeroPadded }); + } + + [Fact] + public void KinesisAndBaseTokensAreIncompatibleInBothDirections() + { + StreamSequenceToken kinesis = new KinesisSequenceToken("7", sequenceNumber: 1, eventIndex: 2); + StreamSequenceToken baseToken = new EventSequenceTokenV2(1, 2); + + Assert.False(kinesis.Equals(baseToken)); + Assert.False(baseToken.Equals(kinesis)); + Assert.Throws(() => kinesis.CompareTo(baseToken)); + Assert.Throws(() => baseToken.CompareTo(kinesis)); + Assert.Equal(2, new Dictionary + { + [kinesis] = "kinesis", + [baseToken] = "base", + }.Count); + Assert.Throws( + () => new SortedSet { kinesis, baseToken }); + Assert.Throws( + () => new SortedSet { baseToken, kinesis }); + } + + [Fact] + public void CreateSequenceTokenForEventPreservesKinesisPosition() + { + var batchToken = new KinesisSequenceToken(HugeShardSequence, sequenceNumber: 42, eventIndex: 0); + + var eventToken = Assert.IsType(batchToken.CreateSequenceTokenForEvent(3)); + + Assert.Equal(HugeShardSequence, eventToken.ShardSequence); + Assert.Equal(42, eventToken.SequenceNumber); + Assert.Equal(3, eventToken.EventIndex); + Assert.True(batchToken.CompareTo(eventToken) < 0); + } + + [Fact] + public void RepeatedOrderingOperationsDoNotReparseShardSequence() + { + var older = new KinesisSequenceToken(HugeShardSequence, sequenceNumber: 0, eventIndex: 0); + var newer = new KinesisSequenceToken(SlightlyLargerShardSequence, sequenceNumber: 0, eventIndex: 0); + + _ = older.CompareTo(newer); + _ = older.GetHashCode(); + _ = newer.GetHashCode(); + var allocatedBefore = GC.GetAllocatedBytesForCurrentThread(); + var comparisonTotal = 0; + var hash = 0; + + for (var i = 0; i < 1_000; i++) + { + comparisonTotal += older.CompareTo(newer); + hash ^= older.GetHashCode(); + hash ^= newer.GetHashCode(); + } + + var allocated = GC.GetAllocatedBytesForCurrentThread() - allocatedBefore; + Assert.Equal(-1_000, comparisonTotal); + Assert.Equal(0, allocated); + GC.KeepAlive(hash); + } + + [Fact] + public void BinarySerializationRoundTripPreservesFieldsAndOrderingAfterRestart() + { + var original = new KinesisSequenceToken(HugeShardSequence, sequenceNumber: 42, eventIndex: 3); + + var bytes = serializer.SerializeToArray(original); + var restored = serializer.Deserialize(bytes); + + Assert.NotNull(restored); + Assert.NotSame(original, restored); + Assert.Equal(HugeShardSequence, restored.ShardSequence); + Assert.Equal(42, restored.SequenceNumber); + Assert.Equal(3, restored.EventIndex); + Assert.True(original.Equals(restored)); + Assert.Equal(0, original.CompareTo(restored)); + + var newer = new KinesisSequenceToken(SlightlyLargerShardSequence, sequenceNumber: 0, eventIndex: 0); + Assert.True(restored.CompareTo(newer) < 0); + } + + [Fact] + public void LegacyJsonDeserializationPreservesTokenFieldsAndOrdering() + { + var json = $$"""{"ShardSequence":"{{HugeShardSequence}}","SequenceNumber":7,"EventIndex":2}"""; + var restored = JsonConvert.DeserializeObject(json)!; + + Assert.NotNull(restored); + Assert.Equal(HugeShardSequence, restored.ShardSequence); + Assert.Equal(7, restored.SequenceNumber); + Assert.Equal(2, restored.EventIndex); + Assert.True(restored.CompareTo(new KinesisSequenceToken(SlightlyLargerShardSequence, 0, 0)) < 0); + } +} diff --git a/test/Orleans.Streaming.Tests/Checkpointers/ReusableStreamQueueCheckpointerTests.cs b/test/Orleans.Streaming.Tests/Checkpointers/ReusableStreamQueueCheckpointerTests.cs new file mode 100644 index 00000000000..a03ebb10216 --- /dev/null +++ b/test/Orleans.Streaming.Tests/Checkpointers/ReusableStreamQueueCheckpointerTests.cs @@ -0,0 +1,225 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Orleans.Streams; +using TestExtensions; +using Xunit; + +namespace UnitTests.StreamingTests; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestCategory("BVT")] +public sealed class ReusableStreamQueueCheckpointerTests : StreamQueueCheckpointerTests +{ + protected override OffsetRegressionPolicy RegressionPolicy => OffsetRegressionPolicy.Ignore; + + protected override Task> CreateCheckpointer( + ControllableCheckpointStore store) + => Task.FromResult>( + new StreamQueueCheckpointer( + new TestCheckpointStore(store), + new StreamQueueCheckpointerOptions + { + CheckpointComparer = StreamCheckpointComparers.Numeric, + PersistInterval = PersistInterval, + })); + + [Fact] + public async Task ConditionalConflict_RetriesWithReturnedVersion() + { + var store = new ConflictingCheckpointStore(); + var checkpointer = new StreamQueueCheckpointer( + store, + new StreamQueueCheckpointerOptions + { + CheckpointComparer = StreamCheckpointComparers.Numeric, + }); + Assert.Equal("10", await checkpointer.Load(CancellationToken.None)); + + checkpointer.Update("30", DateTime.UtcNow, CancellationToken.None); + await checkpointer.FlushAsync(CancellationToken.None); + + Assert.Equal(["version-1", "version-2"], store.ExpectedVersions); + Assert.Equal("30", (await store.Load(CancellationToken.None)).Checkpoint); + } + + [Fact] + public async Task ConditionalConflict_WithEmptyPersistedCheckpoint_RetriesFirstCheckpoint() + { + var store = new EmptyCheckpointConflictStore(); + var checkpointer = new StreamQueueCheckpointer( + store, + new StreamQueueCheckpointerOptions + { + CheckpointComparer = StreamCheckpointComparers.Numeric, + }); + Assert.Equal(string.Empty, await checkpointer.Load(CancellationToken.None)); + + checkpointer.Update("10", DateTime.UtcNow, CancellationToken.None); + await checkpointer.FlushAsync(CancellationToken.None); + + Assert.Equal(["version-1", "version-2"], store.ExpectedVersions); + Assert.Equal("10", (await store.Load(CancellationToken.None)).Checkpoint); + } + + [Fact] + public async Task SamePendingCheckpoint_RetriesAfterFailedWriteAtPersistInterval() + { + var store = new ControllableCheckpointStore("10"); + var checkpointer = await CreateCheckpointer(store); + Assert.Equal("10", await checkpointer.Load(CancellationToken.None)); + store.FailNextWrite(new InvalidOperationException("checkpoint write failed")); + + checkpointer.Update("20", DateTime.UtcNow, CancellationToken.None); + await store.WaitForWriteAttempts(1); + checkpointer.Update("20", DateTime.UtcNow + PersistInterval, CancellationToken.None); + await store.WaitForCompletedWrites(1); + + Assert.Equal(["20", "20"], store.WriteAttempts); + Assert.Equal(["20"], store.CompletedWrites); + Assert.Equal("20", store.PersistedCheckpoint); + } + + [Fact] + public async Task ConditionalConflict_WithoutComparerAdoptsAuthoritativeCheckpoint() + { + var store = new BlockingAuthoritativeConflictStore(); + var checkpointer = new StreamQueueCheckpointer( + store, + new StreamQueueCheckpointerOptions { CheckpointComparer = null }); + Assert.Equal("10", await checkpointer.Load(CancellationToken.None)); + + checkpointer.Update("20", DateTime.UtcNow, CancellationToken.None); + await store.FirstUpdateStarted.Task; + checkpointer.Update("30", DateTime.UtcNow, CancellationToken.None); + store.ReleaseFirstUpdate.SetResult(); + await checkpointer.FlushAsync(CancellationToken.None); + + Assert.Equal(["20"], store.Attempts); + Assert.Equal("40", (await store.Load(CancellationToken.None)).Checkpoint); + } + + private sealed class TestCheckpointStore(ControllableCheckpointStore store) : IStreamCheckpointStore + { + public async ValueTask Load(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var checkpoint = await store.Load().ConfigureAwait(false); + return new(checkpoint, checkpoint); + } + + public async ValueTask Update( + string checkpoint, + string expectedVersion, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var persistedCheckpoint = await store.Write(checkpoint).ConfigureAwait(false); + return new(persistedCheckpoint, persistedCheckpoint); + } + } + + private sealed class ConflictingCheckpointStore : IStreamCheckpointStore + { + private StreamCheckpointStoreState state = new("10", "version-1"); + private bool conflict = true; + + public List ExpectedVersions { get; } = []; + + public ValueTask Load(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return ValueTask.FromResult(state); + } + + public ValueTask Update( + string checkpoint, + string expectedVersion, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + ExpectedVersions.Add(expectedVersion); + if (conflict) + { + conflict = false; + state = new("20", "version-2"); + } + else + { + Assert.Equal(state.Version, expectedVersion); + state = new(checkpoint, "version-3"); + } + + return ValueTask.FromResult(state); + } + } + + private sealed class EmptyCheckpointConflictStore : IStreamCheckpointStore + { + private StreamCheckpointStoreState state = new(string.Empty, "version-1"); + private bool conflict = true; + + public List ExpectedVersions { get; } = []; + + public ValueTask Load(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return ValueTask.FromResult(state); + } + + public ValueTask Update( + string checkpoint, + string expectedVersion, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + ExpectedVersions.Add(expectedVersion); + if (conflict) + { + conflict = false; + state = new(string.Empty, "version-2"); + } + else + { + Assert.Equal(state.Version, expectedVersion); + state = new(checkpoint, "version-3"); + } + + return ValueTask.FromResult(state); + } + } + + private sealed class BlockingAuthoritativeConflictStore : IStreamCheckpointStore + { + private StreamCheckpointStoreState state = new("10", "version-1"); + + public TaskCompletionSource FirstUpdateStarted { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource ReleaseFirstUpdate { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public List Attempts { get; } = []; + + public ValueTask Load(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return ValueTask.FromResult(state); + } + + public async ValueTask Update( + string checkpoint, + string expectedVersion, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Attempts.Add(checkpoint); + FirstUpdateStarted.TrySetResult(); + await ReleaseFirstUpdate.Task.WaitAsync(cancellationToken); + state = new("40", "version-2"); + return state; + } + } +} diff --git a/test/Orleans.Streaming.Tests/Checkpointers/StreamQueueCheckpointerTests.cs b/test/Orleans.Streaming.Tests/Checkpointers/StreamQueueCheckpointerTests.cs index 26f132be3e7..032005ed111 100644 --- a/test/Orleans.Streaming.Tests/Checkpointers/StreamQueueCheckpointerTests.cs +++ b/test/Orleans.Streaming.Tests/Checkpointers/StreamQueueCheckpointerTests.cs @@ -74,6 +74,20 @@ public async Task Update_PersistsCheckpoint() Assert.Equal(["20"], store.CompletedWrites); } + [Theory] + [InlineData("9", "10")] + [InlineData("99", "100")] + public async Task Update_AcrossNumericBoundary_PersistsCheckpoint(string persisted, string candidate) + { + var (checkpointer, store) = await CreateLoadedSubject(persisted); + + checkpointer.Update(candidate, TestTimeUtc, CancellationToken.None); + await checkpointer.FlushAsync(CancellationToken.None); + + Assert.Equal(candidate, store.PersistedCheckpoint); + Assert.Equal([candidate], store.CompletedWrites); + } + [Fact] public async Task Update_WithinPersistInterval_ThrottlesWriteUntilFlush() { diff --git a/test/Orleans.Streaming.Tests/OrleansRuntime/Streams/EncodedOffsetPooledQueueCacheTests.cs b/test/Orleans.Streaming.Tests/OrleansRuntime/Streams/EncodedOffsetPooledQueueCacheTests.cs new file mode 100644 index 00000000000..2c364bc6104 --- /dev/null +++ b/test/Orleans.Streaming.Tests/OrleansRuntime/Streams/EncodedOffsetPooledQueueCacheTests.cs @@ -0,0 +1,198 @@ +using System.Globalization; +using Microsoft.Extensions.Logging.Abstractions; +using Orleans.Providers.Streams.Common; +using Orleans.Runtime; +using Orleans.Streams; +using TestExtensions; +using Xunit; + +namespace UnitTests.StreamingTests; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Streaming")] +[TestCategory("BVT")] +public sealed class EncodedOffsetPooledQueueCacheTests +{ + [Fact] + public void CachedMessageBlock_AdapterAwareSearchUsesEncodedOffset() + { + var adapter = new EncodedOffsetDataAdapter(); + var block = new CachedMessageBlock(3); + block.Add(CreateMessage(default, "001")); + block.Add(CreateMessage(default, "003")); + block.Add(CreateMessage(default, "005")); + + Assert.Equal(1, block.GetIndexOfFirstMessageLessThanOrEqualTo(new EncodedOffsetToken("003"), adapter)); + Assert.Equal(1, block.GetIndexOfFirstMessageLessThanOrEqualTo(new EncodedOffsetToken("004"), adapter)); + Assert.Equal(2, block.GetIndexOfFirstMessageLessThanOrEqualTo(new EncodedOffsetToken("005"), adapter)); + Assert.True(adapter.CompareCallCount >= 4); + } + + [Fact] + public void Cursor_AfterNewestWaitsUntilExternalOffsetArrives() + { + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var adapter = new EncodedOffsetDataAdapter(); + var cache = CreateCache(adapter); + Add(cache, streamId, "010", "020"); + var cursor = cache.GetCursor(streamId, new EncodedOffsetToken("030")); + + Assert.False(cache.TryGetNextMessage(cursor, out _)); + Assert.Equal(0, adapter.GetBatchContainerCallCount); + + Add(cache, streamId, "030"); + + Assert.True(cache.TryGetNextMessage(cursor, out var batch)); + Assert.Equal("030", Assert.IsType(batch.SequenceToken).Offset); + Assert.False(cache.TryGetNextMessage(cursor, out _)); + Assert.True(adapter.CompareCallCount > 0); + Assert.Equal(1, adapter.GetBatchContainerCallCount); + } + + [Fact] + public void Cursor_UsesExternalOffsetAcrossMessageBlocks() + { + const int defaultBlockSize = 16 * 1024; + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var adapter = new EncodedOffsetDataAdapter(); + var cache = CreateCache(adapter); + var messages = Enumerable.Range(0, defaultBlockSize + 2) + .Select(index => CreateMessage( + streamId, + index.ToString("D5", CultureInfo.InvariantCulture))) + .ToList(); + cache.Add(messages, DateTime.UnixEpoch); + var requested = (defaultBlockSize - 1).ToString("D5", CultureInfo.InvariantCulture); + + var cursor = cache.GetCursor(streamId, new EncodedOffsetToken(requested)); + + Assert.True(cache.TryGetNextMessage(cursor, out var first)); + Assert.Equal(requested, Assert.IsType(first.SequenceToken).Offset); + Assert.True(cache.TryGetNextMessage(cursor, out var second)); + Assert.Equal( + defaultBlockSize.ToString("D5", CultureInfo.InvariantCulture), + Assert.IsType(second.SequenceToken).Offset); + Assert.True(adapter.CompareCallCount >= 4); + } + + [Fact] + public void Cursor_WhenExternalPositionWasPurgedThrowsCacheMiss() + { + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var adapter = new EncodedOffsetDataAdapter(); + var cache = CreateCache(adapter); + Add(cache, streamId, "010", "020"); + var cursor = cache.GetCursor(streamId, new EncodedOffsetToken("010")); + cache.RemoveOldestMessage(); + + var exception = Assert.Throws( + () => cache.TryGetNextMessage(cursor, out _)); + + Assert.Equal(new EncodedOffsetToken("010").ToString(), exception.Requested); + Assert.Equal(new EncodedOffsetToken("020").ToString(), exception.Low); + Assert.Equal(new EncodedOffsetToken("020").ToString(), exception.High); + Assert.Equal(0, adapter.GetBatchContainerCallCount); + } + + private static PooledQueueCache CreateCache(EncodedOffsetDataAdapter adapter) + => new(adapter, NullLogger.Instance, cacheMonitor: null, cacheMonitorWriteInterval: null); + + private static void Add( + PooledQueueCache cache, + StreamId streamId, + params string[] offsets) + => cache.Add( + offsets.Select(offset => CreateMessage(streamId, offset)).ToList(), + DateTime.UnixEpoch); + + private static CachedMessage CreateMessage(StreamId streamId, string offset) + { + var bytes = new byte[SegmentBuilder.CalculateAppendSize(offset)]; + var segment = new ArraySegment(bytes); + var writeOffset = 0; + SegmentBuilder.Append(segment, ref writeOffset, offset); + return new CachedMessage + { + StreamId = streamId, + SequenceNumber = EncodedOffsetToken.SharedSequenceNumber, + EventIndex = 0, + EnqueueTimeUtc = DateTime.UnixEpoch, + DequeueTimeUtc = DateTime.UnixEpoch, + Segment = segment, + }; + } + + private sealed class EncodedOffsetDataAdapter : ICacheDataAdapter + { + public int CompareCallCount { get; private set; } + public int GetBatchContainerCallCount { get; private set; } + + public IBatchContainer GetBatchContainer(ref CachedMessage cachedMessage) + { + GetBatchContainerCallCount++; + return new TestBatchContainer( + cachedMessage.StreamId, + GetSequenceToken(ref cachedMessage)); + } + + public StreamSequenceToken GetSequenceToken(ref CachedMessage cachedMessage) + => new EncodedOffsetToken(ReadOffset(ref cachedMessage)); + + public int Compare(ref CachedMessage cachedMessage, StreamSequenceToken token) + { + CompareCallCount++; + var numericComparison = cachedMessage.Compare(token); + if (numericComparison != 0) + { + return numericComparison; + } + + return string.CompareOrdinal( + ReadOffset(ref cachedMessage), + Assert.IsType(token).Offset); + } + + private static string ReadOffset(ref CachedMessage cachedMessage) + { + var readOffset = 0; + return SegmentBuilder.ReadNextString(cachedMessage.Segment, ref readOffset)!; + } + } + + private sealed class EncodedOffsetToken(string offset) : StreamSequenceToken + { + public const long SharedSequenceNumber = 42; + + public string Offset { get; } = offset; + + public override long SequenceNumber { get; protected set; } = SharedSequenceNumber; + + public override int EventIndex { get; protected set; } + + public override bool Equals(StreamSequenceToken? other) + => other is EncodedOffsetToken token && string.Equals(Offset, token.Offset, StringComparison.Ordinal); + + public override int CompareTo(StreamSequenceToken? other) + => other is null + ? 1 + : string.CompareOrdinal(Offset, Assert.IsType(other).Offset); + + public override int GetHashCode() => StringComparer.Ordinal.GetHashCode(Offset); + + public override string ToString() => $"EncodedOffset({Offset})"; + } + + private sealed class TestBatchContainer( + StreamId streamId, + StreamSequenceToken sequenceToken) : IBatchContainer + { + public StreamId StreamId { get; } = streamId; + + public StreamSequenceToken SequenceToken { get; } = sequenceToken; + + public IEnumerable> GetEvents() => []; + + public bool ImportRequestContext() => false; + } +} diff --git a/test/Orleans.Streaming.Tests/OrleansRuntime/Streams/PooledCacheBufferOwnershipTests.cs b/test/Orleans.Streaming.Tests/OrleansRuntime/Streams/PooledCacheBufferOwnershipTests.cs new file mode 100644 index 00000000000..08e283d0b83 --- /dev/null +++ b/test/Orleans.Streaming.Tests/OrleansRuntime/Streams/PooledCacheBufferOwnershipTests.cs @@ -0,0 +1,113 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Orleans.Providers; +using Orleans.Providers.Streams.Common; +using Orleans.Providers.Streams.Generator; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Streams; +using Xunit; + +namespace UnitTests.OrleansRuntime.Streams; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Streaming")] +public sealed class PooledCacheBufferOwnershipTests +{ + [Fact] + public void MemoryCache_EmptyPurgeAllocatesFreshBufferForNextMessage() + { + var pool = new TrackingBufferPool(); + var serializer = new TestMemorySerializer(); + var cache = new MemoryPooledCache( + pool, + new AlwaysPurgePredicate(), + NullLogger.Instance, + serializer, + cacheMonitor: null, + monitorWriteInterval: null, + purgeMetadataInterval: null); + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + + cache.AddToCache([CreateMemoryBatch(streamId, 1, serializer)]); + Assert.False(cache.TryPurgeFromCache(out _)); + Assert.Equal(1, pool.FreeCount); + + cache.AddToCache([CreateMemoryBatch(streamId, 2, serializer)]); + + Assert.Equal(2, pool.AllocateCount); + } + + [Fact] + public void GeneratorCache_EmptyPurgeAllocatesFreshBufferForNextMessage() + { + var pool = new TrackingBufferPool(); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var cache = new GeneratorPooledCache( + pool, + NullLogger.Instance, + services.GetRequiredService(), + cacheMonitor: null, + monitorWriteInterval: null, + new AlwaysPurgePredicate()); + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + + cache.AddToCache([new GeneratedBatchContainer(streamId, 1, new EventSequenceTokenV2(1))]); + Assert.False(cache.TryPurgeFromCache(out _)); + Assert.Equal(1, pool.FreeCount); + + cache.AddToCache([new GeneratedBatchContainer(streamId, 2, new EventSequenceTokenV2(2))]); + + Assert.Equal(2, pool.AllocateCount); + } + + private static MemoryBatchContainer CreateMemoryBatch( + StreamId streamId, + long sequenceNumber, + TestMemorySerializer serializer) + => new( + new MemoryMessageData + { + StreamId = streamId, + SequenceNumber = sequenceNumber, + EnqueueTimeUtc = DateTime.UtcNow, + Payload = new byte[] { 1 }, + }, + serializer); + + private sealed class AlwaysPurgePredicate : TimePurgePredicate + { + public AlwaysPurgePredicate() + : base(TimeSpan.Zero, TimeSpan.Zero) + { + } + + public override bool ShouldPurgeFromTime(TimeSpan timeInCache, TimeSpan relativeAge) => true; + } + + private sealed class TrackingBufferPool : IObjectPool + { + public int AllocateCount { get; private set; } + + public int FreeCount { get; private set; } + + public FixedSizeBuffer Allocate() + { + AllocateCount++; + return new FixedSizeBuffer(4 * 1024) { Pool = this }; + } + + public void Free(FixedSizeBuffer resource) + { + FreeCount++; + } + } + + private sealed class TestMemorySerializer : IMemoryMessageBodySerializer + { + public ArraySegment Serialize(MemoryMessageBody body) => new byte[] { 1 }; + + public MemoryMessageBody Deserialize(ArraySegment bodyBytes) => new([], requestContext: null); + } +} diff --git a/test/Orleans.Streaming.Tests/StreamingTests/ClientStreamTestRunner.cs b/test/Orleans.Streaming.Tests/StreamingTests/ClientStreamTestRunner.cs index 5e37b83ef75..701a501a825 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/ClientStreamTestRunner.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/ClientStreamTestRunner.cs @@ -52,7 +52,6 @@ public async Task StreamConsumerOnDroppedClientTest( bool waitForRetryTimeouts = false, CancellationToken cancellationToken = default) { - var hasDeliveryFailureCounter = getDeliveryFailureCount is not null; getDeliveryFailureCount ??= DefaultDeliveryFailureCount; Guid streamGuid = Guid.NewGuid(); @@ -76,7 +75,7 @@ public async Task StreamConsumerOnDroppedClientTest( await ProduceEventsToClient(streamProviderName, streamGuid, streamNamespace, 10, eventCount, cancellationToken); // Wait for the dropped client's subscription to be removed after delivery fails. - if (waitForRetryTimeouts && hasDeliveryFailureCounter) + if (waitForRetryTimeouts) { using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(_timeout); diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs index 05a9072aaa2..a7dacedfe4b 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PersistentStreamPullingAgentTests.cs @@ -71,6 +71,12 @@ public async Task ReadFromQueue_DoesNotWaitForColdStreamRegistration() await receiver.Received(2).GetQueueMessagesAsync(Arg.Any(), Arg.Any()); } + private sealed class FilterByDataStreamFilter : IStreamFilter + { + public bool ShouldDeliver(StreamId streamId, object item, string? filterData) + => !string.Equals(filterData, "reject", StringComparison.Ordinal); + } + [TestSuite("BVT")] [TestProvider("None")] [TestArea("Streaming")] @@ -325,7 +331,8 @@ private static PersistentStreamPullingAgent CreateAgent( IQueueAdapterReceiver? receiver = null, IQueueAdapterCache? queueAdapterCache = null, TimeProvider? timeProvider = null, - StreamPullingAgentOptions? options = null) + StreamPullingAgentOptions? options = null, + IStreamFilter? streamFilter = null) { var siloAddress = SiloAddress.New(IPAddress.Loopback, 11111, 1); var localSiloDetails = Substitute.For(); @@ -362,7 +369,7 @@ private static PersistentStreamPullingAgent CreateAgent( SystemTargetGrainId.Create(SystemTargetGrainId.CreateGrainType("persistent-stream-pulling-agent-test"), siloAddress), "provider", pubSub!, - new NoOpStreamFilter(), + streamFilter ?? new NoOpStreamFilter(), queueId, options ?? new StreamPullingAgentOptions(), queueAdapter, @@ -376,8 +383,12 @@ private static PersistentStreamPullingAgent CreateAgent( private sealed class RecordingQueueCache : IQueueCache { + private TaskCompletionSource deliveryProgressUpdated = new(TaskCreationOptions.RunContinuationsAsynchronously); + public int DeliveryProgressCallCount { get; private set; } public List DeliveryProgressTokens { get; } = new(); + public List DeliveryProgressUtcTimes { get; } = new(); + public Task DeliveryProgressUpdated => deliveryProgressUpdated.Task; public int GetMaxAddCount() => 1000; @@ -402,23 +413,32 @@ public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToke { DeliveryProgressCallCount++; DeliveryProgressTokens.Add(earliestSubscriptionToken); + DeliveryProgressUtcTimes.Add(utcNow); + deliveryProgressUpdated.TrySetResult(true); } public void ClearDeliveryProgress() { DeliveryProgressCallCount = 0; DeliveryProgressTokens.Clear(); + DeliveryProgressUtcTimes.Clear(); + deliveryProgressUpdated = new(TaskCreationOptions.RunContinuationsAsynchronously); } } - private sealed class ScriptedQueueCache : IQueueCache + private sealed class ScriptedQueueCache(int maxCacheSize = 1000) : IQueueCache { private readonly List messages = new(); + private TaskCompletionSource deliveryProgressUpdated = new(TaskCreationOptions.RunContinuationsAsynchronously); + private StreamSequenceToken? purgedThrough; public int DeliveryProgressCallCount { get; private set; } public List DeliveryProgressTokens { get; } = new(); + public Task DeliveryProgressUpdated => deliveryProgressUpdated.Task; - public int GetMaxAddCount() => 1000; + public int GetMaxAddCount() + => Math.Max(0, maxCacheSize - messages.Count(message => + purgedThrough is null || message.SequenceToken.Newer(purgedThrough))); public void AddToCache(IList messages) { @@ -436,25 +456,40 @@ public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? return new ScriptedQueueCursor(messages, streamId, token); } - public bool IsUnderPressure() => false; + public bool IsUnderPressure() => GetMaxAddCount() == 0; public void UpdateDeliveryProgress(StreamSequenceToken? earliestSubscriptionToken, DateTime utcNow) { DeliveryProgressCallCount++; DeliveryProgressTokens.Add(earliestSubscriptionToken); + if (earliestSubscriptionToken is not null + && (purgedThrough is null || earliestSubscriptionToken.Newer(purgedThrough))) + { + purgedThrough = earliestSubscriptionToken; + } + deliveryProgressUpdated.TrySetResult(true); } public void ClearDeliveryProgress() { DeliveryProgressCallCount = 0; DeliveryProgressTokens.Clear(); + deliveryProgressUpdated = new(TaskCreationOptions.RunContinuationsAsynchronously); } } - private sealed class ScriptedQueueCursor(List messages, StreamId streamId, StreamSequenceToken? token) : IQueueCacheCursor + private sealed class ScriptedQueueCursor( + List messages, + StreamId streamId, + StreamSequenceToken? token) : IQueueCacheCursor, IQueueCacheCursorProgress { private int index = -1; private IBatchContainer? current; + private StreamSequenceToken? pendingSequenceToken; + private bool hasPendingDelivery; + private StreamSequenceToken? deliveredThroughToken; + + public StreamSequenceToken? SafeSequenceToken { get; private set; } public void Dispose() { @@ -471,14 +506,46 @@ public bool MoveNext() for (index++; index < messages.Count; index++) { var candidate = messages[index]; - if (candidate.StreamId.Equals(streamId) && (token is null || candidate.SequenceToken.Newer(token))) + if (token is not null && candidate.SequenceToken.CompareTo(token) < 0) + { + continue; + } + + if (candidate.StreamId.Equals(streamId)) { + if (deliveredThroughToken is not null + && candidate.SequenceToken.CompareTo(deliveredThroughToken) <= 0) + { + if (hasPendingDelivery) + { + pendingSequenceToken = candidate.SequenceToken; + } + else + { + SafeSequenceToken = candidate.SequenceToken; + } + + continue; + } + + hasPendingDelivery = true; + pendingSequenceToken = candidate.SequenceToken; current = candidate; return true; } + + if (hasPendingDelivery) + { + pendingSequenceToken = candidate.SequenceToken; + } + else + { + SafeSequenceToken = candidate.SequenceToken; + } } current = null; + index = messages.Count - 1; return false; } @@ -489,6 +556,21 @@ public void Refresh(StreamSequenceToken token) public void RecordDeliveryFailure() { } + + public void SetDeliveredThrough(StreamSequenceToken deliveredToken) + => deliveredThroughToken = deliveredToken; + + public void RecordDeliverySuccess() + { + if (!hasPendingDelivery) + { + return; + } + + SafeSequenceToken = pendingSequenceToken; + pendingSequenceToken = null; + hasPendingDelivery = false; + } } private sealed class PurgeablePooledQueueCache : IQueueCache @@ -576,6 +658,26 @@ private sealed class TestBatchContainer(StreamId streamId, StreamSequenceToken t public bool ImportRequestContext() => false; } + private sealed class ProviderSequenceToken(int providerOrder, long sequenceNumber) : StreamSequenceToken + { + public int ProviderOrder { get; } = providerOrder; + public override long SequenceNumber { get; protected set; } = sequenceNumber; + public override int EventIndex { get; protected set; } + + public override int CompareTo(StreamSequenceToken? other) + => other is ProviderSequenceToken token + ? ProviderOrder.CompareTo(token.ProviderOrder) + : throw new ArgumentException($"Cannot compare {GetType()} with {other?.GetType()}.", nameof(other)); + + public override bool Equals(StreamSequenceToken? other) + => other is ProviderSequenceToken token && ProviderOrder == token.ProviderOrder && SequenceNumber == token.SequenceNumber; + } + + private sealed class RejectingStreamFilter : IStreamFilter + { + public bool ShouldDeliver(StreamId streamId, object item, string? filterData) => false; + } + private sealed class RecordingConsumer : IStreamConsumerExtension { private readonly TaskCompletionSource releaseDelivery = new(TaskCreationOptions.RunContinuationsAsynchronously); @@ -608,6 +710,8 @@ private sealed class RecordingConsumer : IStreamConsumerExtension private sealed class RewindConsumer(StreamHandshakeToken rewindToken) : IStreamConsumerExtension { + private int returnedRewindToken; + public TaskCompletionSource Delivered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public Task DeliverImmutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, StreamSequenceToken currentToken, StreamHandshakeToken? handshakeToken) @@ -623,7 +727,8 @@ private sealed class RewindConsumer(StreamHandshakeToken rewindToken) : IStreamC public Task DeliverBatch(GuidId subscriptionId, QualifiedStreamId streamId, IBatchContainer item, StreamHandshakeToken? handshakeToken) { Delivered.TrySetResult(true); - return Task.FromResult(rewindToken); + return Task.FromResult( + Interlocked.Exchange(ref returnedRewindToken, 1) == 0 ? rewindToken : null); } public Task CompleteStream(GuidId subscriptionId) => Task.CompletedTask; @@ -633,6 +738,29 @@ private sealed class RewindConsumer(StreamHandshakeToken rewindToken) : IStreamC public Task GetSequenceToken(GuidId subscriptionId) => Task.FromResult(rewindToken); } + private sealed class ImmediateConsumer : IStreamConsumerExtension + { + public TaskCompletionSource Delivered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task DeliverImmutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, StreamSequenceToken currentToken, StreamHandshakeToken? handshakeToken) + => throw new NotSupportedException(); + + public Task DeliverMutable(GuidId subscriptionId, QualifiedStreamId streamId, object item, StreamSequenceToken currentToken, StreamHandshakeToken? handshakeToken) + => throw new NotSupportedException(); + + public Task DeliverBatch(GuidId subscriptionId, QualifiedStreamId streamId, IBatchContainer item, StreamHandshakeToken? handshakeToken) + { + Delivered.TrySetResult(true); + return Task.FromResult(null); + } + + public Task CompleteStream(GuidId subscriptionId) => Task.CompletedTask; + + public Task ErrorInStream(GuidId subscriptionId, Exception exc) => Task.CompletedTask; + + public Task GetSequenceToken(GuidId subscriptionId) => Task.FromResult(null); + } + [TestSuite("BVT")] [TestProvider("None")] [TestArea("Streaming")] @@ -694,7 +822,7 @@ public async Task ReadFromQueue_RefreshesIdleCursorAfterItsTokenMetadataIsPurged [TestProvider("None")] [TestArea("Streaming")] [Fact, TestCategory("BVT"), TestCategory("Streaming")] - public async Task Shutdown_UsesReturnedHandshakeTokenForDeliveryProgress() + public async Task Shutdown_AdvancesAfterConsumerAcceptsRetryFollowingHandshakeToken() { var pubSub = Substitute.For(); pubSub.RegisterProducer(default, default) @@ -735,6 +863,7 @@ public async Task Shutdown_UsesReturnedHandshakeTokenForDeliveryProgress() consumerData.IsRegistered = true; consumerData.LastToken = rewindToken; consumerData.LastProcessedToken = previousToken; + consumerData.LastSafePartitionToken = previousToken; consumerData.Cursor = queueCache.GetCacheCursor(qualifiedStreamId, previousToken); queueCache.ClearDeliveryProgress(); @@ -744,7 +873,7 @@ public async Task Shutdown_UsesReturnedHandshakeTokenForDeliveryProgress() queueCache.ClearDeliveryProgress(); await testAccessor.Shutdown(); - Assert.Equal(previousToken, Assert.Single(queueCache.DeliveryProgressTokens)); + Assert.Equal(attemptedToken, Assert.Single(queueCache.DeliveryProgressTokens)); } private static Task InitializeAgent(PersistentStreamPullingAgent agent) => agent.RunOrQueueTask(() => agent.Initialize()); @@ -863,6 +992,49 @@ public async Task Shutdown_WaitsForInFlightPumpWork() await pumpTask; } + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task Shutdown_DoesNotCheckpointBatchReturnedAfterShutdownStarts() + { + var queueReadStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var queueReadReleased = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) + .Returns(async _ => + { + queueReadStarted.TrySetResult(); + return await queueReadReleased.Task; + }); + receiver.Shutdown(Arg.Any()).Returns(Task.CompletedTask); + var queueCache = Substitute.For(); + queueCache.GetMaxAddCount().Returns(1); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(queueId).Returns(queueCache); + var agent = CreateAgent(pubSub: null, queueId, receiver, queueAdapterCache); + var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + + var pumpTask = testAccessor.RunQueuePump(queueId, TestContext.Current.CancellationToken); + await queueReadStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + var shutdownTask = testAccessor.Shutdown(); + queueReadReleased.SetResult( + [ + new GeneratedBatchContainer(streamId, 1, new EventSequenceTokenV2(1)), + ]); + + await shutdownTask; + await pumpTask; + + queueCache.DidNotReceive().AddToCache(Arg.Any>()); + queueCache.DidNotReceive().UpdateDeliveryProgress( + Arg.Any(), + Arg.Any()); + } + [TestSuite("BVT")] [TestProvider("None")] [TestArea("Streaming")] @@ -1145,5 +1317,761 @@ public async Task Shutdown_PushesFinalDeliveryProgress() // Shutdown should push a final delivery progress snapshot before tearing down. Assert.Single(queueCache.DeliveryProgressTokens); } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public void EventSequenceTokens_CompareAndEqualAcrossVersions() + { + StreamSequenceToken v1 = new EventSequenceToken(10, 2); + StreamSequenceToken v2 = new EventSequenceTokenV2(10, 2); + + Assert.Equal(0, v1.CompareTo(v2)); + Assert.Equal(0, v2.CompareTo(v1)); + Assert.Equal(v1, v2); + Assert.Equal(v2, v1); + Assert.Equal(v1.GetHashCode(), v2.GetHashCode()); + } + + [Fact] + public void EventSequenceTokens_CoalesceInHashAndSortedCollectionsAcrossVersions() + { + StreamSequenceToken v1 = new EventSequenceToken(10, 2); + StreamSequenceToken v2 = new EventSequenceTokenV2(10, 2); + + Assert.Single(new HashSet { v1, v2 }); + Assert.Single(new SortedSet { v2, v1 }); + } + + [Fact] + public void EventSequenceTokens_RejectDerivedTokensSymmetrically() + { + StreamSequenceToken baseToken = new EventSequenceTokenV2(10, 2); + StreamSequenceToken derivedToken = new DerivedEventSequenceToken(10, 2); + + Assert.False(baseToken.Equals(derivedToken)); + Assert.False(derivedToken.Equals(baseToken)); + Assert.Throws(() => baseToken.CompareTo(derivedToken)); + Assert.Throws(() => derivedToken.CompareTo(baseToken)); + Assert.Equal(2, new HashSet { baseToken, derivedToken }.Count); + Assert.Throws( + () => new SortedSet { baseToken, derivedToken }); + Assert.Throws( + () => new SortedSet { derivedToken, baseToken }); + + StreamSequenceToken v1Base = new EventSequenceToken(10, 2); + StreamSequenceToken v1Derived = new DerivedEventSequenceTokenV1(10, 2); + Assert.False(v1Base.Equals(v1Derived)); + Assert.False(v1Derived.Equals(v1Base)); + Assert.Throws(() => v1Base.CompareTo(v1Derived)); + Assert.Throws(() => v1Derived.CompareTo(v1Base)); + + StreamSequenceToken v2DerivedPeer = new DerivedEventSequenceToken(10, 2); + Assert.True(derivedToken.Equals(v2DerivedPeer)); + Assert.True(v2DerivedPeer.Equals(derivedToken)); + Assert.Equal(0, derivedToken.CompareTo(v2DerivedPeer)); + Assert.Equal(derivedToken.GetHashCode(), v2DerivedPeer.GetHashCode()); + + StreamSequenceToken v1DerivedPeer = new DerivedEventSequenceTokenV1(10, 2); + Assert.True(v1Derived.Equals(v1DerivedPeer)); + Assert.True(v1DerivedPeer.Equals(v1Derived)); + Assert.Equal(0, v1Derived.CompareTo(v1DerivedPeer)); + Assert.Equal(v1Derived.GetHashCode(), v1DerivedPeer.GetHashCode()); + + var v2EventToken = Assert.IsType( + ((DerivedEventSequenceToken)derivedToken).CreateSequenceTokenForEvent(3)); + Assert.Equal(10, v2EventToken.SequenceNumber); + Assert.Equal(3, v2EventToken.EventIndex); + + var v1EventToken = Assert.IsType( + ((DerivedEventSequenceTokenV1)v1Derived).CreateSequenceTokenForEvent(3)); + Assert.Equal(10, v1EventToken.SequenceNumber); + Assert.Equal(3, v1EventToken.EventIndex); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task DeliveryProgress_ReportsPeriodicallyUsingProviderTokenComparison() + { + var timeProvider = new FakeTimeProvider(DateTimeOffset.UtcNow); + var options = new StreamPullingAgentOptions(); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var receiver = Substitute.For(); + var queueCache = new RecordingQueueCache(); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var agent = CreateAgent(pubSub, queueId, receiver, queueAdapterCache, timeProvider, options); + var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + + await InitializeAgent(agent); + await testAccessor.RegisterStream(streamId, new EventSequenceTokenV2(1), timeProvider.GetUtcNow().UtcDateTime); + var streamData = (await testAccessor.GetPubSubCache()).Single().Value; + var providerEarliest = new ProviderSequenceToken(providerOrder: 95, sequenceNumber: 200); + var providerNewest = new ProviderSequenceToken(providerOrder: 200, sequenceNumber: 1); + + var earliestConsumer = streamData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + streamId, + streamConsumer: null!, + filterData: null, + now: timeProvider.GetUtcNow().UtcDateTime); + earliestConsumer.IsRegistered = true; + earliestConsumer.LastProcessedToken = providerEarliest; + + var newestConsumer = streamData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + streamId, + streamConsumer: null!, + filterData: null, + now: timeProvider.GetUtcNow().UtcDateTime); + newestConsumer.IsRegistered = true; + newestConsumer.LastProcessedToken = providerNewest; + queueCache.ClearDeliveryProgress(); + + timeProvider.Advance(options.DeliveryProgressUpdateInterval - TimeSpan.FromTicks(1)); + await testAccessor.GetPubSubCache(); + Assert.Empty(queueCache.DeliveryProgressTokens); + + timeProvider.Advance(TimeSpan.FromTicks(1)); + await queueCache.DeliveryProgressUpdated.WaitAsync( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + + Assert.Equal(providerEarliest, Assert.Single(queueCache.DeliveryProgressTokens)); + Assert.Equal(timeProvider.GetUtcNow().UtcDateTime, Assert.Single(queueCache.DeliveryProgressUtcTimes)); + + await testAccessor.Shutdown(); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Theory, TestCategory("BVT"), TestCategory("Streaming")] + [InlineData(SubscriptionStartTokenSource.Handshake, 10)] + [InlineData(SubscriptionStartTokenSource.Cache, 20)] + [InlineData(SubscriptionStartTokenSource.Pending, 30)] + public async Task DeliveryProgress_InclusiveStartRemainsUnsafeBeforeFirstBatch( + SubscriptionStartTokenSource tokenSource, + long expectedSequenceNumber) + { + var timeProvider = new FakeTimeProvider(DateTimeOffset.UtcNow); + var options = new StreamPullingAgentOptions(); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var queueCache = new RecordingQueueCache(); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var agent = CreateAgent( + pubSub, + queueId, + receiver: null, + queueAdapterCache, + timeProvider, + options); + var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await testAccessor.RegisterStream(streamId, new EventSequenceTokenV2(1), timeProvider.GetUtcNow().UtcDateTime); + var streamData = (await testAccessor.GetPubSubCache()).Single().Value; + var handshakeToken = new EventSequenceTokenV2(10); + var consumer = new StartingConsumer( + tokenSource == SubscriptionStartTokenSource.Handshake + ? StreamHandshakeToken.CreateStartToken(handshakeToken) + : null); + var consumerData = streamData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + streamId, + consumer, + filterData: null, + now: timeProvider.GetUtcNow().UtcDateTime); + var cacheToken = tokenSource is SubscriptionStartTokenSource.Handshake or SubscriptionStartTokenSource.Cache + ? new EventSequenceTokenV2(20) + : null; + consumerData.PendingStartToken = new EventSequenceTokenV2(30); + + Assert.True(await testAccessor.DoHandshakeWithConsumer(consumerData, cacheToken)); + consumerData.IsRegistered = true; + Assert.Equal(expectedSequenceNumber, consumerData.CursorStartToken?.SequenceNumber); + Assert.Null(consumerData.LastProcessedToken); + Assert.Null(consumerData.LastSafePartitionToken); + queueCache.ClearDeliveryProgress(); + + timeProvider.Advance(options.DeliveryProgressUpdateInterval); + await testAccessor.GetPubSubCache(); + + Assert.Empty(queueCache.DeliveryProgressTokens); + await testAccessor.Shutdown(); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task Handshake_DeliveryTokenStartsAfterConfirmedRecord() + { + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var qualifiedStreamId = new QualifiedStreamId("provider", streamId); + var queueCache = new ScriptedQueueCache(); + queueCache.AddToCache( + [ + new TestBatchContainer(streamId, new EventSequenceTokenV2(10)), + new TestBatchContainer(streamId, new EventSequenceTokenV2(11)), + ]); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>( + [new TestBatchContainer(streamId, new EventSequenceTokenV2(12))])); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var agent = CreateAgent(pubSub, queueId, receiver, queueAdapterCache); + var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await testAccessor.RegisterStream(qualifiedStreamId, new EventSequenceTokenV2(10), DateTime.UtcNow); + var streamData = (await testAccessor.GetPubSubCache()).Single().Value; + var consumer = new StartingConsumer( + StreamHandshakeToken.CreateDeliveyToken(new EventSequenceTokenV2(10))); + var consumerData = streamData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + qualifiedStreamId, + consumer, + filterData: null, + now: DateTime.UtcNow); + + Assert.True(await testAccessor.DoHandshakeWithConsumer(consumerData, cacheToken: null)); + consumerData.IsRegistered = true; + Assert.True(await testAccessor.ReadFromQueue(queueId, receiver, 1)); + await testAccessor.GetPubSubCache(); + + Assert.Equal([11L, 12L], consumer.DeliveredTokens.Select(token => token.SequenceNumber)); + await testAccessor.Shutdown(); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task ReadFromQueue_WakesIdleCursorAtFirstPartitionRecord() + { + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var qualifiedStreamId = new QualifiedStreamId("provider", streamId); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>( + [ + new TestBatchContainer(streamId, new EventSequenceTokenV2(2)), + new TestBatchContainer(streamId, new EventSequenceTokenV2(3)), + ])); + var queueCache = new PurgeablePooledQueueCache(); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var agent = CreateAgent(pubSub, queueId, receiver, queueAdapterCache); + var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await testAccessor.RegisterStream(qualifiedStreamId, new EventSequenceTokenV2(1), DateTime.UtcNow); + var streamData = (await testAccessor.GetPubSubCache()).Single().Value; + var consumer = new StartingConsumer(startToken: null); + var consumerData = streamData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + qualifiedStreamId, + consumer, + filterData: null, + now: DateTime.UtcNow); + consumerData.IsRegistered = true; + consumerData.Cursor = queueCache.GetCacheCursor(streamId, token: null); + Assert.False(consumerData.Cursor.MoveNext()); + + Assert.True(await testAccessor.ReadFromQueue(queueId, receiver, 2)); + await testAccessor.GetPubSubCache(); + + Assert.Equal([2L, 3L], consumer.DeliveredTokens.Select(token => token.SequenceNumber)); + await testAccessor.Shutdown(); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task DeliveryProgress_DeliveryTokenSeedsProcessedProgress() + { + var timeProvider = new FakeTimeProvider(DateTimeOffset.UtcNow); + var options = new StreamPullingAgentOptions(); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var queueCache = new RecordingQueueCache(); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); + var streamId = new QualifiedStreamId("provider", StreamId.Create("namespace", Guid.NewGuid())); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var agent = CreateAgent(pubSub, queueId, receiver: null, queueAdapterCache, timeProvider, options); + var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await testAccessor.RegisterStream(streamId, new EventSequenceTokenV2(1), timeProvider.GetUtcNow().UtcDateTime); + var streamData = (await testAccessor.GetPubSubCache()).Single().Value; + var deliveredToken = new EventSequenceTokenV2(10); + var consumerData = streamData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + streamId, + new StartingConsumer(StreamHandshakeToken.CreateDeliveyToken(deliveredToken)), + filterData: null, + now: timeProvider.GetUtcNow().UtcDateTime); + + Assert.True(await testAccessor.DoHandshakeWithConsumer(consumerData, cacheToken: null)); + consumerData.IsRegistered = true; + Assert.Equal(deliveredToken, consumerData.LastProcessedToken); + Assert.Null(consumerData.LastSafePartitionToken); + queueCache.ClearDeliveryProgress(); + + timeProvider.Advance(options.DeliveryProgressUpdateInterval); + await queueCache.DeliveryProgressUpdated.WaitAsync( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + + Assert.Equal(deliveredToken, Assert.Single(queueCache.DeliveryProgressTokens)); + await testAccessor.Shutdown(); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task DeliveryProgress_DeliveryTokenWaitsForPartitionScanInProgressAwareCache() + { + var timeProvider = new FakeTimeProvider(DateTimeOffset.UtcNow); + var options = new StreamPullingAgentOptions(); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamA = StreamId.Create("namespace", Guid.NewGuid()); + var streamB = StreamId.Create("namespace", Guid.NewGuid()); + var qualifiedA = new QualifiedStreamId("provider", streamA); + var qualifiedB = new QualifiedStreamId("provider", streamB); + var queueCache = new ScriptedQueueCache(); + queueCache.AddToCache( + [ + .. Enumerable.Range(2, 8) + .Select(sequence => (IBatchContainer)new TestBatchContainer( + streamB, + new EventSequenceTokenV2(sequence))), + new TestBatchContainer(streamA, new EventSequenceTokenV2(10)), + ]); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var agent = CreateAgent(pubSub, queueId, receiver: null, queueAdapterCache, timeProvider, options); + var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + await InitializeAgent(agent); + await testAccessor.RegisterStream(qualifiedA, new EventSequenceTokenV2(2), timeProvider.GetUtcNow().UtcDateTime); + await testAccessor.RegisterStream(qualifiedB, new EventSequenceTokenV2(2), timeProvider.GetUtcNow().UtcDateTime); + var streamData = (await testAccessor.GetPubSubCache())[qualifiedA]; + var consumerData = streamData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + qualifiedA, + new StartingConsumer(StreamHandshakeToken.CreateDeliveyToken(new EventSequenceTokenV2(10))), + filterData: null, + now: timeProvider.GetUtcNow().UtcDateTime); + + Assert.True(await testAccessor.DoHandshakeWithConsumer( + consumerData, + cacheToken: new EventSequenceTokenV2(2))); + consumerData.IsRegistered = true; + Assert.Equal(10, consumerData.LastProcessedToken?.SequenceNumber); + Assert.Null(consumerData.LastSafePartitionToken); + queueCache.ClearDeliveryProgress(); + + timeProvider.Advance(options.DeliveryProgressUpdateInterval); + await testAccessor.GetPubSubCache(); + Assert.Empty(queueCache.DeliveryProgressTokens); + + await testAccessor.RunConsumerCursor(consumerData); + Assert.Equal(10, consumerData.LastSafePartitionToken?.SequenceNumber); + timeProvider.Advance(options.DeliveryProgressUpdateInterval); + await queueCache.DeliveryProgressUpdated.WaitAsync( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + Assert.Equal(10, Assert.Single(queueCache.DeliveryProgressTokens)?.SequenceNumber); + await testAccessor.Shutdown(); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task DeliveryProgress_QuietSubscriptionAdvancesAcrossBusyStreamBeyondCapacity() + { + var timeProvider = new FakeTimeProvider(DateTimeOffset.UtcNow); + var options = new StreamPullingAgentOptions(); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var quietStreamId = StreamId.Create("namespace", Guid.NewGuid()); + var busyStreamId = StreamId.Create("namespace", Guid.NewGuid()); + var quietQualifiedId = new QualifiedStreamId("provider", quietStreamId); + var busyQualifiedId = new QualifiedStreamId("provider", busyStreamId); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) + .Returns( + Task.FromResult>( + [ + new TestBatchContainer(busyStreamId, new EventSequenceTokenV2(2)), + new TestBatchContainer(busyStreamId, new EventSequenceTokenV2(3)), + ]), + Task.FromResult>( + [ + new TestBatchContainer(busyStreamId, new EventSequenceTokenV2(4)), + new TestBatchContainer(busyStreamId, new EventSequenceTokenV2(5)), + ]), + Task.FromResult>( + [ + new TestBatchContainer(busyStreamId, new EventSequenceTokenV2(6)), + new TestBatchContainer(busyStreamId, new EventSequenceTokenV2(7)), + ])); + var queueCache = new ScriptedQueueCache(maxCacheSize: 3); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var agent = CreateAgent( + pubSub, + queueId, + receiver, + queueAdapterCache, + timeProvider, + options, + new FilterByDataStreamFilter()); + var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + + await InitializeAgent(agent); + await testAccessor.RegisterStream(quietQualifiedId, new EventSequenceTokenV2(1), timeProvider.GetUtcNow().UtcDateTime); + await testAccessor.RegisterStream(busyQualifiedId, new EventSequenceTokenV2(2), timeProvider.GetUtcNow().UtcDateTime); + queueCache.AddToCache([new TestBatchContainer(quietStreamId, new EventSequenceTokenV2(1))]); + var cache = await testAccessor.GetPubSubCache(); + var quietData = cache[quietQualifiedId]; + var quietConsumer = quietData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + quietQualifiedId, + new ImmediateConsumer(), + filterData: null, + now: timeProvider.GetUtcNow().UtcDateTime); + quietConsumer.IsRegistered = true; + quietConsumer.Cursor = queueCache.GetCacheCursor(quietStreamId, new EventSequenceTokenV2(1)); + var busyData = cache[busyQualifiedId]; + var busyConsumer = busyData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + busyQualifiedId, + new ImmediateConsumer(), + filterData: null, + now: timeProvider.GetUtcNow().UtcDateTime); + busyConsumer.IsRegistered = true; + busyConsumer.Cursor = queueCache.GetCacheCursor(busyStreamId, new EventSequenceTokenV2(2)); + var filteredConsumer = busyData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + busyQualifiedId, + streamConsumer: null!, + filterData: "reject", + now: timeProvider.GetUtcNow().UtcDateTime); + filteredConsumer.IsRegistered = true; + filteredConsumer.Cursor = queueCache.GetCacheCursor(busyStreamId, new EventSequenceTokenV2(2)); + + foreach (var expectedCheckpoint in new long[] { 3, 5, 7 }) + { + Assert.True(await testAccessor.ReadFromQueue(queueId, receiver, 2)); + await testAccessor.GetPubSubCache(); + timeProvider.Advance(options.DeliveryProgressUpdateInterval); + await queueCache.DeliveryProgressUpdated.WaitAsync( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + + var checkpoint = Assert.Single(queueCache.DeliveryProgressTokens); + Assert.NotNull(checkpoint); + Assert.Equal(expectedCheckpoint, checkpoint.SequenceNumber); + Assert.False(queueCache.IsUnderPressure()); + queueCache.ClearDeliveryProgress(); + } + + Assert.Equal(7, quietConsumer.LastSafePartitionToken?.SequenceNumber); + Assert.Equal(7, busyConsumer.LastProcessedToken?.SequenceNumber); + Assert.Equal(7, filteredConsumer.LastProcessedToken?.SequenceNumber); + await testAccessor.Shutdown(); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task DeliveryProgress_PendingRegistrationBlocksPeriodicUpdate() + { + var registration = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default).ReturnsForAnyArgs(_ => registration.Task); + var timeProvider = new FakeTimeProvider(DateTimeOffset.UtcNow); + var options = new StreamPullingAgentOptions(); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) + .Returns( + Task.FromResult>([new TestBatchContainer(streamId, new EventSequenceTokenV2(1))]), + Task.FromResult>([])); + var queueCache = new RecordingQueueCache(); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); + var agent = CreateAgent(pubSub, queueId, receiver, queueAdapterCache, timeProvider, options); + var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + + await InitializeAgent(agent); + await testAccessor.RunQueuePump(queueId, CancellationToken.None); + var streamData = (await testAccessor.GetPubSubCache()).Single().Value; + Assert.NotNull(streamData.RegistrationTask); + queueCache.ClearDeliveryProgress(); + + timeProvider.Advance(options.DeliveryProgressUpdateInterval); + await testAccessor.GetPubSubCache(); + Assert.Empty(queueCache.DeliveryProgressTokens); + + registration.SetResult(new HashSet()); + await streamData.RegistrationTask!; + await testAccessor.Shutdown(); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task DeliveryProgress_IncludesFilteredBatches() + { + var timeProvider = new FakeTimeProvider(DateTimeOffset.UtcNow); + var options = new StreamPullingAgentOptions(); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var qualifiedStreamId = new QualifiedStreamId("provider", streamId); + var previousToken = new EventSequenceTokenV2(1); + var filteredToken = new EventSequenceTokenV2(2); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>([new TestBatchContainer(streamId, filteredToken)])); + var queueCache = new ScriptedQueueCache(); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var agent = CreateAgent( + pubSub, + queueId, + receiver, + queueAdapterCache, + timeProvider, + options, + new RejectingStreamFilter()); + var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + + await InitializeAgent(agent); + await testAccessor.RegisterStream(qualifiedStreamId, previousToken, timeProvider.GetUtcNow().UtcDateTime); + var streamData = (await testAccessor.GetPubSubCache()).Single().Value; + var consumerData = streamData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + qualifiedStreamId, + streamConsumer: null!, + filterData: "reject", + now: timeProvider.GetUtcNow().UtcDateTime); + consumerData.IsRegistered = true; + consumerData.LastProcessedToken = previousToken; + consumerData.LastSafePartitionToken = previousToken; + consumerData.Cursor = queueCache.GetCacheCursor(streamId, previousToken); + queueCache.ClearDeliveryProgress(); + + Assert.True(await testAccessor.ReadFromQueue(queueId, receiver, 1)); + Assert.Equal(filteredToken, consumerData.LastProcessedToken); + + timeProvider.Advance(options.DeliveryProgressUpdateInterval); + await queueCache.DeliveryProgressUpdated.WaitAsync( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + Assert.Equal(filteredToken, Assert.Single(queueCache.DeliveryProgressTokens)); + + await testAccessor.Shutdown(); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task DeliveryProgress_DoesNotAdvancePastSlowSubscriber() + { + var timeProvider = new FakeTimeProvider(DateTimeOffset.UtcNow); + var options = new StreamPullingAgentOptions(); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var qualifiedStreamId = new QualifiedStreamId("provider", streamId); + var previousToken = new EventSequenceTokenV2(1); + var currentToken = new EventSequenceTokenV2(2); + var receiver = Substitute.For(); + receiver.GetQueueMessagesAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>([new TestBatchContainer(streamId, currentToken)])); + receiver.Shutdown(Arg.Any()).Returns(Task.CompletedTask); + var queueCache = new ScriptedQueueCache(); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); + var pubSub = Substitute.For(); + pubSub.RegisterProducer(default, default) + .ReturnsForAnyArgs(Task.FromResult>(new HashSet())); + var agent = CreateAgent(pubSub, queueId, receiver, queueAdapterCache, timeProvider, options); + var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + + await InitializeAgent(agent); + await testAccessor.RegisterStream(qualifiedStreamId, previousToken, timeProvider.GetUtcNow().UtcDateTime); + var streamData = (await testAccessor.GetPubSubCache()).Single().Value; + var slowConsumer = new RecordingConsumer(); + var slowConsumerData = streamData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + qualifiedStreamId, + slowConsumer, + filterData: null, + now: timeProvider.GetUtcNow().UtcDateTime); + slowConsumerData.IsRegistered = true; + slowConsumerData.LastProcessedToken = previousToken; + slowConsumerData.LastSafePartitionToken = previousToken; + slowConsumerData.Cursor = queueCache.GetCacheCursor(streamId, previousToken); + + var fastConsumer = new ImmediateConsumer(); + var fastConsumerData = streamData.AddConsumer( + GuidId.GetGuidId(Guid.NewGuid()), + qualifiedStreamId, + fastConsumer, + filterData: null, + now: timeProvider.GetUtcNow().UtcDateTime); + fastConsumerData.IsRegistered = true; + fastConsumerData.LastProcessedToken = previousToken; + fastConsumerData.LastSafePartitionToken = previousToken; + fastConsumerData.Cursor = queueCache.GetCacheCursor(streamId, previousToken); + queueCache.ClearDeliveryProgress(); + + Assert.True(await testAccessor.ReadFromQueue(queueId, receiver, 1)); + await slowConsumer.Delivered.Task.WaitAsync( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + await fastConsumer.Delivered.Task.WaitAsync( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + await testAccessor.GetPubSubCache(); + Assert.Equal(previousToken, slowConsumerData.LastProcessedToken); + Assert.Equal(currentToken, fastConsumerData.LastProcessedToken); + + timeProvider.Advance(options.DeliveryProgressUpdateInterval); + await queueCache.DeliveryProgressUpdated.WaitAsync( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + Assert.Equal(previousToken, Assert.Single(queueCache.DeliveryProgressTokens)); + + slowConsumer.ReleaseDelivery(); + await testAccessor.GetPubSubCache(); + await testAccessor.Shutdown(); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public async Task Shutdown_StopsPeriodicDeliveryProgressUpdates() + { + var timeProvider = new FakeTimeProvider(DateTimeOffset.UtcNow); + var options = new StreamPullingAgentOptions(); + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var receiver = Substitute.For(); + receiver.Shutdown(Arg.Any()).Returns(Task.CompletedTask); + var queueCache = new RecordingQueueCache(); + var queueAdapterCache = Substitute.For(); + queueAdapterCache.CreateQueueCache(Arg.Any()).Returns(queueCache); + var agent = CreateAgent(pubSub: null, queueId, receiver, queueAdapterCache, timeProvider, options); + var testAccessor = (PersistentStreamPullingAgent.ITestAccessor)agent; + + await InitializeAgent(agent); + await testAccessor.Shutdown(); + queueCache.ClearDeliveryProgress(); + + timeProvider.Advance(options.DeliveryProgressUpdateInterval); + + Assert.Empty(queueCache.DeliveryProgressTokens); + } + + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Streaming")] + [Fact, TestCategory("BVT"), TestCategory("Streaming")] + public void DeliveryProgress_RejectsNonPositiveUpdateInterval() + { + var queueId = QueueId.GetQueueId("queue", 0u, 0u); + var options = new StreamPullingAgentOptions { DeliveryProgressUpdateInterval = TimeSpan.Zero }; + + var exception = Assert.Throws(() => CreateAgent(pubSub: null, queueId, options: options)); + + Assert.Equal(nameof(options.DeliveryProgressUpdateInterval), exception.ParamName); + } + + public enum SubscriptionStartTokenSource + { + Handshake, + Cache, + Pending, + } + + private sealed class DerivedEventSequenceToken(long sequenceNumber, int eventIndex) + : EventSequenceTokenV2(sequenceNumber, eventIndex); + + private sealed class DerivedEventSequenceTokenV1(long sequenceNumber, int eventIndex) + : EventSequenceToken(sequenceNumber, eventIndex); + + private sealed class StartingConsumer(StreamHandshakeToken? startToken) : IStreamConsumerExtension + { + public List DeliveredTokens { get; } = []; + + public Task DeliverImmutable( + GuidId subscriptionId, + QualifiedStreamId streamId, + object item, + StreamSequenceToken currentToken, + StreamHandshakeToken? handshakeToken) => throw new NotSupportedException(); + + public Task DeliverMutable( + GuidId subscriptionId, + QualifiedStreamId streamId, + object item, + StreamSequenceToken currentToken, + StreamHandshakeToken? handshakeToken) => throw new NotSupportedException(); + + public Task DeliverBatch( + GuidId subscriptionId, + QualifiedStreamId streamId, + IBatchContainer item, + StreamHandshakeToken? handshakeToken) + { + DeliveredTokens.Add(item.SequenceToken); + return Task.FromResult(null); + } + + public Task CompleteStream(GuidId subscriptionId) => Task.CompletedTask; + + public Task ErrorInStream(GuidId subscriptionId, Exception exc) => Task.CompletedTask; + + public Task GetSequenceToken(GuidId subscriptionId) + => Task.FromResult(startToken); + } } } diff --git a/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.cs index 4119ca81a56..e4a81f28079 100644 --- a/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.cs +++ b/test/Orleans.Streaming.Tests/StreamingTests/PubSubRendezvousGrainTests.cs @@ -104,6 +104,27 @@ await Assert.ThrowsAsync( Assert.Equal(0, consumers); } + [Fact, TestCategory("BVT"), TestCategory("Streaming"), TestCategory("PubSub")] + public async Task UnregisterLastConsumerEmitsDiagnosticAfterClearingState() + { + var streamId = new QualifiedStreamId("ProviderName", StreamId.Create("StreamNamespace", Guid.NewGuid())); + var subscriptionId = GuidId.GetGuidId(Guid.NewGuid()); + var pubSubGrain = this.fixture.GrainFactory.GetGrain(streamId.ToString()); + using var observer = StreamingDiagnosticObserver.Create(); + + await pubSubGrain.RegisterConsumer(subscriptionId, streamId, default, null!); + await pubSubGrain.UnregisterConsumer(subscriptionId, streamId); + + var unregistered = await observer.WaitForSubscriptionUnregisteredAsync( + streamId.StreamId, + subscriptionId.Guid, + streamId.ProviderName, + TestContext.Current.CancellationToken); + Assert.Equal(streamId.StreamId, unregistered.StreamId); + Assert.Equal(subscriptionId.Guid, unregistered.SubscriptionId); + Assert.Equal(0, await pubSubGrain.ConsumerCount(streamId)); + } + /// /// This test fails because the producer must be grain reference which is not implied by the IStreamProducerExtension in the producer management calls. /// TODO: Fix rendezvous implementation. diff --git a/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs new file mode 100644 index 00000000000..46339c1d69c --- /dev/null +++ b/test/Orleans.Streaming.Tests/StreamingTests/RecoverableStreamReceiverTests.cs @@ -0,0 +1,1161 @@ +using System.Globalization; +using Microsoft.Extensions.Logging.Abstractions; +using Orleans.Providers.Streams.Common; +using Orleans.Runtime; +using Orleans.Streams; +using TestExtensions; +using Xunit; + +namespace UnitTests.StreamingTests; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Streaming")] +[TestCategory("BVT")] +public sealed class RecoverableStreamReceiverTests +{ + [Fact] + public async Task Receiver_ResumesAfterCheckpointAndPersistsDeliveryProgress() + { + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var source = new TestSource( + [ + new TestQueueMessage(streamId, 11, "payload"), + ]); + var adapter = new TestDataAdapter(); + var bufferPool = new ObjectPool(() => new FixedSizeBuffer(4 * 1024)); + var cache = new RecoverableStreamQueueCache( + 100, + bufferPool, + adapter, + new NoOpEvictionStrategy(), + NullLogger.Instance); + var checkpointer = new TestCheckpointer("10"); + var receiver = new RecoverableStreamReceiver( + source, + adapter, + cache, + checkpointer, + startFromNow: true); + + await receiver.Initialize(TimeSpan.FromSeconds(5)); + + Assert.Equal("10", source.StartPosition.Checkpoint); + Assert.True(source.StartPosition.StartFromNow); + var notifications = await receiver.GetQueueMessagesAsync(100, CancellationToken.None); + var notification = Assert.Single(notifications); + Assert.Equal(streamId, notification.StreamId); + Assert.Equal(11, notification.SequenceToken.SequenceNumber); + + using var cursor = receiver.GetCacheCursor(streamId, notification.SequenceToken); + Assert.True(cursor.MoveNext()); + var batch = Assert.IsType(cursor.GetCurrent(out var exception)); + Assert.Null(exception); + Assert.Equal("payload", batch.Payload); + Assert.True(adapter.CompareCallCount > 0); + Assert.False(cursor.MoveNext()); + Assert.Same(batch, cursor.GetCurrent(out exception)); + Assert.Null(exception); + + receiver.UpdateDeliveryProgress(new EventSequenceTokenV2(11), DateTime.UtcNow); + Assert.Equal("11", checkpointer.LastUpdatedCheckpoint); + + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + Assert.Equal(1, checkpointer.FlushCount); + Assert.True(source.IsShutdown); + } + + private sealed class BlockingInitializationCheckpointer : IStreamQueueCheckpointer + { + public TaskCompletionSource FirstLoadStarted { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource FirstLoadCancellationObserved { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource AllowFirstLoadToComplete { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public CancellationToken FirstLoadCancellation { get; private set; } + + public int LoadCount { get; private set; } + + public bool CheckpointExists => false; + + public Task Load() => Load(CancellationToken.None); + + public async Task Load(CancellationToken cancellationToken) + { + LoadCount++; + if (LoadCount == 1) + { + FirstLoadCancellation = cancellationToken; + FirstLoadStarted.TrySetResult(); + using var registration = cancellationToken.Register( + static state => ((TaskCompletionSource)state!).TrySetResult(), + FirstLoadCancellationObserved); + await FirstLoadCancellationObserved.Task; + await AllowFirstLoadToComplete.Task; + cancellationToken.ThrowIfCancellationRequested(); + } + + return string.Empty; + } + + public void Update(string offset, DateTime utcNow) { } + + public void Update(string offset, DateTime utcNow, CancellationToken cancellationToken) + => cancellationToken.ThrowIfCancellationRequested(); + + public Task FlushAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.CompletedTask; + } + } + + private sealed class IndependentlyCanceledCheckpointer : IStreamQueueCheckpointer + { + public int LoadCount { get; private set; } + + public bool CheckpointExists => false; + + public Task Load() => Load(CancellationToken.None); + + public async Task Load(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + LoadCount++; + await Task.Yield(); + throw new OperationCanceledException(new CancellationToken(canceled: true)); + } + + public void Update(string offset, DateTime utcNow) { } + + public void Update(string offset, DateTime utcNow, CancellationToken cancellationToken) + => cancellationToken.ThrowIfCancellationRequested(); + + public Task FlushAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.CompletedTask; + } + } + + [Fact] + public async Task Receiver_RestartRedeliversInclusiveBatchWhenCheckpointDidNotAdvance() + { + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var messages = new[] + { + new TestQueueMessage(streamId, 9, "old-9"), + new TestQueueMessage(streamId, 10, "old-10"), + new TestQueueMessage(streamId, 11, "payload"), + }; + var store = new TestCheckpointStore("10"); + var firstSource = new ReplaySource(messages); + var firstReceiver = CreateReceiver( + firstSource, + new StreamQueueCheckpointer( + store, + new StreamQueueCheckpointerOptions + { + CheckpointComparer = StreamCheckpointComparers.Numeric, + PersistInterval = TimeSpan.FromSeconds(1), + })); + await firstReceiver.Initialize(TimeSpan.FromSeconds(5)); + var firstNotification = Assert.Single(await firstReceiver.GetQueueMessagesAsync(10, CancellationToken.None)); + using (var cursor = firstReceiver.GetCacheCursor(streamId, firstNotification.SequenceToken)) + { + Assert.True(cursor.MoveNext()); + Assert.Equal(11, cursor.GetCurrent(out _)!.SequenceToken.SequenceNumber); + // The inclusive first batch was observed but never confirmed as delivered. + } + + await firstReceiver.Shutdown(TimeSpan.FromSeconds(5)); + Assert.Equal("10", store.State.Checkpoint); + + var secondSource = new ReplaySource(messages); + var secondReceiver = CreateReceiver( + secondSource, + new StreamQueueCheckpointer( + store, + new StreamQueueCheckpointerOptions + { + CheckpointComparer = StreamCheckpointComparers.Numeric, + PersistInterval = TimeSpan.FromSeconds(1), + })); + await secondReceiver.Initialize(TimeSpan.FromSeconds(5)); + + var redelivered = Assert.Single(await secondReceiver.GetQueueMessagesAsync(10, CancellationToken.None)); + Assert.Equal(11, redelivered.SequenceToken.SequenceNumber); + await secondReceiver.Shutdown(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task Receiver_QuietScanCheckpointRestartsAfterBusyTail() + { + var quietStream = StreamId.Create("namespace", Guid.NewGuid()); + var busyStream = StreamId.Create("namespace", Guid.NewGuid()); + var initialMessages = new[] + { + new TestQueueMessage(quietStream, 1, "quiet"), + new TestQueueMessage(busyStream, 2, "busy-2"), + new TestQueueMessage(busyStream, 3, "busy-3"), + }; + var store = new TestCheckpointStore(string.Empty); + var receiver = CreateReceiver( + new ReplaySource(initialMessages), + new StreamQueueCheckpointer( + store, + new StreamQueueCheckpointerOptions + { + CheckpointComparer = StreamCheckpointComparers.Numeric, + PersistInterval = TimeSpan.FromSeconds(1), + })); + await receiver.Initialize(TimeSpan.FromSeconds(5)); + var notifications = await receiver.GetQueueMessagesAsync(10, CancellationToken.None); + using var cursor = receiver.GetCacheCursor(quietStream, notifications[0].SequenceToken); + var progress = Assert.IsAssignableFrom(cursor); + Assert.True(cursor.MoveNext()); + progress.RecordDeliverySuccess(); + Assert.False(cursor.MoveNext()); + Assert.Equal(3, progress.SafeSequenceToken?.SequenceNumber); + + receiver.UpdateDeliveryProgress(progress.SafeSequenceToken, DateTime.UtcNow); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + Assert.Equal("3", store.State.Checkpoint); + + var restarted = CreateReceiver( + new ReplaySource( + [ + .. initialMessages, + new TestQueueMessage(busyStream, 4, "busy-4"), + ]), + new StreamQueueCheckpointer( + store, + new StreamQueueCheckpointerOptions + { + CheckpointComparer = StreamCheckpointComparers.Numeric, + PersistInterval = TimeSpan.FromSeconds(1), + })); + await restarted.Initialize(TimeSpan.FromSeconds(5)); + + var next = Assert.Single(await restarted.GetQueueMessagesAsync(10, CancellationToken.None)); + Assert.Equal(4, next.SequenceToken.SequenceNumber); + await restarted.Shutdown(TimeSpan.FromSeconds(5)); + } + + [Fact] + public void Registry_ReturnsSameReceiverAndCacheInstanceForQueue() + { + var created = 0; + var registry = new QueueAdapterReceiverRegistry(_ => + { + created++; + return new TestCombinedReceiver(); + }); + var queue = QueueId.GetQueueId("queue", 0, 0); + + IQueueAdapterReceiver receiver = registry.GetOrCreate(queue); + IQueueCache cache = registry.GetOrCreate(queue); + + Assert.Same(receiver, cache); + Assert.Equal(1, created); + Assert.Single(registry.Receivers); + } + + [Fact] + public async Task DeliveryProgress_WithNoSubscribers_AdvancesToNewestCachedRecord() + { + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var source = new TestSource([new TestQueueMessage(streamId, 11, "payload")]); + var adapter = new TestDataAdapter(); + var bufferPool = new TrackingBufferPool(); + var cache = new RecoverableStreamQueueCache( + 100, + bufferPool, + adapter, + new ChronologicalEvictionStrategy( + NullLogger.Instance, + new TimePurgePredicate(TimeSpan.MaxValue, TimeSpan.MaxValue), + cacheMonitor: null, + monitorWriteInterval: null), + NullLogger.Instance); + var checkpointer = new TestCheckpointer("10"); + var receiver = new RecoverableStreamReceiver( + source, + adapter, + cache, + checkpointer, + startFromNow: false); + await receiver.Initialize(TimeSpan.FromSeconds(5)); + _ = await receiver.GetQueueMessagesAsync(100, CancellationToken.None); + + receiver.UpdateDeliveryProgress(earliestSubscriptionToken: null, DateTime.UtcNow); + + Assert.Equal("11", checkpointer.LastUpdatedCheckpoint); + Assert.Equal(0, cache.ItemCount); + Assert.Equal(1, bufferPool.FreeCount); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + } + + [Fact] + public void Cache_CursorCreatedWhileEmptyReadsFirstLaterRecord() + { + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var adapter = new TestDataAdapter(); + var cache = new RecoverableStreamQueueCache( + 100, + new ObjectPool(() => new FixedSizeBuffer(4 * 1024)), + adapter, + new NoOpEvictionStrategy(), + NullLogger.Instance); + var initial = cache.Add([new TestQueueMessage(streamId, 10, "initial")], DateTime.UnixEpoch); + cache.UpdateDeliveryProgress(initial[0].SequenceToken, DateTime.UtcNow); + Assert.Equal(0, cache.ItemCount); + + using var cursor = cache.GetCacheCursor(streamId, token: null); + Assert.False(cursor.MoveNext()); + + _ = cache.Add([new TestQueueMessage(streamId, 11, "payload")], DateTime.UnixEpoch); + + Assert.True(cursor.MoveNext()); + Assert.Equal("payload", Assert.IsType(cursor.GetCurrent(out _)).Payload); + } + + [Fact] + public void CachePressure_BlocksUnsafeTimePurgeUntilDeliveryProgressAdvances() + { + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var adapter = new TestDataAdapter(); + var evictionStrategy = new NoOpEvictionStrategy(); + var cache = new RecoverableStreamQueueCache( + 100, + new ObjectPool(() => new FixedSizeBuffer(4 * 1024)), + adapter, + evictionStrategy, + NullLogger.Instance, + flowController: new FixedFlowController(0)); + var positions = cache.Add( + [new TestQueueMessage(streamId, 11, "payload")], + DateTime.UnixEpoch); + + Assert.True(cache.IsUnderPressure()); + Assert.False(cache.TryPurgeFromCache(out _)); + Assert.Equal(0, evictionStrategy.PerformPurgeCount); + Assert.Equal(1, cache.ItemCount); + + cache.UpdateDeliveryProgress(positions[0].SequenceToken, DateTime.UtcNow); + + Assert.Equal(0, cache.ItemCount); + } + + [Fact] + public void Cache_UnlimitedFlowControlUsesConfiguredReadLimit() + { + var cache = new RecoverableStreamQueueCache( + 100, + new ObjectPool(() => new FixedSizeBuffer(4 * 1024)), + new TestDataAdapter(), + new NoOpEvictionStrategy(), + NullLogger.Instance, + flowController: new FixedFlowController(QueueAdapterConstants.UNLIMITED_GET_QUEUE_MSG)); + + Assert.Equal(100, cache.GetMaxAddCount()); + Assert.False(cache.IsUnderPressure()); + } + + [Fact] + public void Cache_DisposeDrainsMessagesReturnsBuffersAndAllocatesFreshBufferIfReused() + { + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var bufferPool = new TrackingBufferPool(); + var cache = new RecoverableStreamQueueCache( + 100, + bufferPool, + new TestDataAdapter(), + new ChronologicalEvictionStrategy( + NullLogger.Instance, + new TimePurgePredicate(TimeSpan.MaxValue, TimeSpan.MaxValue), + cacheMonitor: null, + monitorWriteInterval: null), + NullLogger.Instance); + _ = cache.Add([new TestQueueMessage(streamId, 11, "payload")], DateTime.UnixEpoch); + + cache.Dispose(); + + Assert.Equal(0, cache.ItemCount); + Assert.Equal(1, bufferPool.AllocateCount); + Assert.Equal(1, bufferPool.FreeCount); + + _ = cache.Add([new TestQueueMessage(streamId, 12, "next")], DateTime.UnixEpoch); + + Assert.Equal(1, cache.ItemCount); + Assert.Equal(2, bufferPool.AllocateCount); + cache.Dispose(); + Assert.Equal(2, bufferPool.FreeCount); + } + + [Fact] + public void Cache_FailedPackingReturnsUncommittedPooledBuffers() + { + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var bufferPool = new TrackingBufferPool(); + var cache = new RecoverableStreamQueueCache( + 100, + bufferPool, + new ThrowingDataAdapter(), + new ChronologicalEvictionStrategy( + NullLogger.Instance, + new TimePurgePredicate(TimeSpan.MaxValue, TimeSpan.MaxValue), + cacheMonitor: null, + monitorWriteInterval: null), + NullLogger.Instance); + var message = new TestQueueMessage(streamId, 1, "payload"); + + Assert.Throws(() => cache.Add([message], DateTime.UnixEpoch)); + Assert.Throws(() => cache.Add([message], DateTime.UnixEpoch)); + + Assert.Equal(0, cache.ItemCount); + Assert.Equal(2, bufferPool.AllocateCount); + Assert.Equal(2, bufferPool.FreeCount); + } + + [Fact] + public void Cache_ReusesCurrentBufferAcrossAddsAndRollsBackFailedPacking() + { + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var first = new TestQueueMessage(streamId, 1, "a"); + var failed = new TestQueueMessage(streamId, 2, new string('b', 10)); + var final = new TestQueueMessage(streamId, 3, new string('c', 10)); + var bufferPool = new TrackingBufferPool( + GetPackedSize(first) + GetPackedSize(failed) + GetPackedSize(final) - 1); + var adapter = new TestDataAdapter(); + var cache = new RecoverableStreamQueueCache( + 100, + bufferPool, + adapter, + new NoOpEvictionStrategy(), + NullLogger.Instance); + _ = cache.Add([first], DateTime.UnixEpoch); + adapter.ThrowAfterPackingSequenceNumber = 2; + + Assert.Throws(() => cache.Add([failed], DateTime.UnixEpoch)); + + adapter.ThrowAfterPackingSequenceNumber = null; + _ = cache.Add([final], DateTime.UnixEpoch); + + Assert.Equal(2, cache.ItemCount); + Assert.Equal(1, bufferPool.AllocateCount); + + static int GetPackedSize(TestQueueMessage message) => + SegmentBuilder.CalculateAppendSize(message.SequenceNumber.ToString(CultureInfo.InvariantCulture)) + + SegmentBuilder.CalculateAppendSize(message.Payload); + } + + [Fact] + public void Cache_AddsRawRecordsInOrderAndDecodesLazily() + { + var streamA = StreamId.Create("namespace", Guid.NewGuid()); + var streamB = StreamId.Create("namespace", Guid.NewGuid()); + var messages = new[] + { + new TestQueueMessage(streamA, 10, "first"), + new TestQueueMessage(streamB, 11, "second"), + new TestQueueMessage(streamA, 12, "third"), + }; + var adapter = new TestDataAdapter(); + var cache = new RecoverableStreamQueueCache( + 100, + new ObjectPool(() => new FixedSizeBuffer(4 * 1024)), + adapter, + new NoOpEvictionStrategy(), + NullLogger.Instance); + + var positions = cache.Add(messages, DateTime.UnixEpoch); + + Assert.Equal([10, 11, 12], positions.Select(position => position.SequenceToken.SequenceNumber)); + Assert.Equal(3, adapter.PositionCallCount); + Assert.Equal(3, adapter.FromQueueMessageCallCount); + Assert.Equal(0, adapter.GetBatchContainerCallCount); + + using var cursor = cache.GetCacheCursor(streamA, positions[0].SequenceToken); + Assert.True(cursor.MoveNext()); + Assert.Equal("first", Assert.IsType(cursor.GetCurrent(out _)).Payload); + Assert.Equal(1, adapter.GetBatchContainerCallCount); + Assert.True(cursor.MoveNext()); + Assert.Equal("third", Assert.IsType(cursor.GetCurrent(out _)).Payload); + Assert.Equal(2, adapter.GetBatchContainerCallCount); + Assert.False(cursor.MoveNext()); + Assert.Equal(2, adapter.GetBatchContainerCallCount); + } + + [Fact] + public void Cache_CursorProgressRequiresDeliveryAndIncludesUnrelatedScans() + { + var streamA = StreamId.Create("namespace", Guid.NewGuid()); + var streamB = StreamId.Create("namespace", Guid.NewGuid()); + var cache = new RecoverableStreamQueueCache( + 100, + new ObjectPool(() => new FixedSizeBuffer(4 * 1024)), + new TestDataAdapter(), + new NoOpEvictionStrategy(), + NullLogger.Instance); + var positions = cache.Add( + [ + new TestQueueMessage(streamA, 1, "a-1"), + new TestQueueMessage(streamB, 2, "b-2"), + new TestQueueMessage(streamB, 3, "b-3"), + new TestQueueMessage(streamA, 4, "a-4"), + ], + DateTime.UnixEpoch); + using var cursor = cache.GetCacheCursor(streamA, positions[0].SequenceToken); + var progress = Assert.IsAssignableFrom(cursor); + + Assert.True(cursor.MoveNext()); + Assert.Null(progress.SafeSequenceToken); + + progress.RecordDeliverySuccess(); + Assert.Equal(1, progress.SafeSequenceToken?.SequenceNumber); + + Assert.True(cursor.MoveNext()); + Assert.Equal(3, progress.SafeSequenceToken?.SequenceNumber); + + progress.RecordDeliverySuccess(); + Assert.Equal(4, progress.SafeSequenceToken?.SequenceNumber); + } + + [Fact] + public void Cache_BatchedMatchesRemainPendingUntilWholeDeliverySucceeds() + { + var streamA = StreamId.Create("namespace", Guid.NewGuid()); + var streamB = StreamId.Create("namespace", Guid.NewGuid()); + var cache = new RecoverableStreamQueueCache( + 100, + new ObjectPool(() => new FixedSizeBuffer(4 * 1024)), + new TestDataAdapter(), + new NoOpEvictionStrategy(), + NullLogger.Instance); + var positions = cache.Add( + [ + new TestQueueMessage(streamA, 1, "a-1"), + new TestQueueMessage(streamB, 2, "b-2"), + new TestQueueMessage(streamA, 3, "a-3"), + ], + DateTime.UnixEpoch); + using var cursor = cache.GetCacheCursor(streamA, positions[0].SequenceToken); + var progress = Assert.IsAssignableFrom(cursor); + + Assert.True(cursor.MoveNext()); + Assert.True(cursor.MoveNext()); + Assert.Null(progress.SafeSequenceToken); + + progress.RecordDeliverySuccess(); + Assert.Equal(3, progress.SafeSequenceToken?.SequenceNumber); + } + + [Fact] + public void Cache_DeliveredThroughScansIntermediatePartitionRecordsWithoutRedelivery() + { + var streamA = StreamId.Create("namespace", Guid.NewGuid()); + var streamB = StreamId.Create("namespace", Guid.NewGuid()); + var cache = new RecoverableStreamQueueCache( + 100, + new ObjectPool(() => new FixedSizeBuffer(4 * 1024)), + new TestDataAdapter(), + new NoOpEvictionStrategy(), + NullLogger.Instance); + var positions = cache.Add( + [ + new TestQueueMessage(streamB, 2, "b-2"), + new TestQueueMessage(streamB, 3, "b-3"), + new TestQueueMessage(streamA, 10, "a-10"), + new TestQueueMessage(streamA, 11, "a-11"), + ], + DateTime.UnixEpoch); + using var cursor = cache.GetCacheCursor(streamA, positions[0].SequenceToken); + var progress = Assert.IsAssignableFrom(cursor); + progress.SetDeliveredThrough(new EventSequenceTokenV2(10)); + + Assert.True(cursor.MoveNext()); + Assert.Equal(11, cursor.GetCurrent(out _)!.SequenceToken.SequenceNumber); + Assert.Equal(10, progress.SafeSequenceToken?.SequenceNumber); + } + + [Fact] + public void Cache_DeliveryFailureRewindsToFirstPendingRecord() + { + var streamA = StreamId.Create("namespace", Guid.NewGuid()); + var streamB = StreamId.Create("namespace", Guid.NewGuid()); + var cache = new RecoverableStreamQueueCache( + 100, + new ObjectPool(() => new FixedSizeBuffer(4 * 1024)), + new TestDataAdapter(), + new NoOpEvictionStrategy(), + NullLogger.Instance); + var positions = cache.Add( + [ + new TestQueueMessage(streamA, 1, "a-1"), + new TestQueueMessage(streamB, 2, "b-2"), + new TestQueueMessage(streamA, 3, "a-3"), + ], + DateTime.UnixEpoch); + using var cursor = cache.GetCacheCursor(streamA, positions[0].SequenceToken); + var progress = Assert.IsAssignableFrom(cursor); + Assert.True(cursor.MoveNext()); + Assert.Equal(1, cursor.GetCurrent(out _)!.SequenceToken.SequenceNumber); + Assert.True(cursor.MoveNext()); + Assert.Equal(3, cursor.GetCurrent(out _)!.SequenceToken.SequenceNumber); + + cursor.RecordDeliveryFailure(); + + Assert.Null(progress.SafeSequenceToken); + Assert.True(cursor.MoveNext()); + Assert.Equal(1, cursor.GetCurrent(out _)!.SequenceToken.SequenceNumber); + progress.RecordDeliverySuccess(); + Assert.True(cursor.MoveNext()); + Assert.Equal(3, cursor.GetCurrent(out _)!.SequenceToken.SequenceNumber); + } + + [Fact] + public async Task Receiver_MidInitializationCancellationReachesLoadAndAllowsRetry() + { + var source = new TestSource([]); + var checkpointer = new BlockingInitializationCheckpointer(); + var receiver = CreateReceiver(source, checkpointer); + using var cancellation = new CancellationTokenSource(); + + var initialization = receiver.Initialize(cancellation.Token); + await checkpointer.FirstLoadStarted.Task; + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync(() => initialization); + Assert.True(checkpointer.FirstLoadCancellation.IsCancellationRequested); + + var retry = receiver.GetQueueMessagesAsync(10, CancellationToken.None); + Assert.Equal(1, checkpointer.LoadCount); + checkpointer.AllowFirstLoadToComplete.TrySetResult(); + + Assert.Empty(await retry); + Assert.Equal(2, checkpointer.LoadCount); + Assert.Equal(1, source.InitializeCount); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task Receiver_IndependentInitializationCancellationIsNotRetried() + { + var source = new TestSource([]); + var checkpointer = new IndependentlyCanceledCheckpointer(); + var receiver = CreateReceiver(source, checkpointer); + + await Assert.ThrowsAnyAsync( + () => receiver.GetQueueMessagesAsync(10, TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + + Assert.Equal(1, checkpointer.LoadCount); + Assert.Equal(0, source.InitializeCount); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + } + + [Fact] + public void Cache_QuietCursorAdvancesAcrossUnrelatedRecords() + { + var quietStream = StreamId.Create("namespace", Guid.NewGuid()); + var busyStream = StreamId.Create("namespace", Guid.NewGuid()); + var cache = new RecoverableStreamQueueCache( + 100, + new ObjectPool(() => new FixedSizeBuffer(4 * 1024)), + new TestDataAdapter(), + new NoOpEvictionStrategy(), + NullLogger.Instance); + var positions = cache.Add( + [ + new TestQueueMessage(quietStream, 1, "quiet"), + new TestQueueMessage(busyStream, 2, "busy-2"), + new TestQueueMessage(busyStream, 3, "busy-3"), + new TestQueueMessage(busyStream, 4, "busy-4"), + ], + DateTime.UnixEpoch); + using var cursor = cache.GetCacheCursor(quietStream, positions[0].SequenceToken); + var progress = Assert.IsAssignableFrom(cursor); + Assert.True(cursor.MoveNext()); + progress.RecordDeliverySuccess(); + + Assert.False(cursor.MoveNext()); + + Assert.Equal(4, progress.SafeSequenceToken?.SequenceNumber); + } + + [Fact] + public async Task Receiver_RetriesInitializationOnNextRead() + { + var streamId = StreamId.Create("namespace", Guid.NewGuid()); + var source = new TestSource( + [new TestQueueMessage(streamId, 1, "payload")], + initializationFailures: 1); + var receiver = CreateReceiver(source, new TestCheckpointer(string.Empty)); + + await Assert.ThrowsAsync( + () => receiver.GetQueueMessagesAsync(10, CancellationToken.None)); + var messages = await receiver.GetQueueMessagesAsync(10, CancellationToken.None); + + Assert.Single(messages); + Assert.Equal(2, source.InitializeCount); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task Receiver_PreCanceledReadDoesNotInitialize() + { + var source = new TestSource([]); + var receiver = CreateReceiver(source, new TestCheckpointer(string.Empty)); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync( + () => receiver.GetQueueMessagesAsync(10, cancellation.Token)); + + Assert.Equal(0, source.InitializeCount); + await receiver.Shutdown(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task Shutdown_WhenFlushFails_StillShutsDownSource() + { + var source = new TestSource([]); + var expected = new InvalidOperationException("flush failed"); + var receiver = CreateReceiver( + source, + new TestCheckpointer(string.Empty) { FlushException = expected }); + await receiver.Initialize(TimeSpan.FromSeconds(5)); + + var actual = await Assert.ThrowsAsync( + () => receiver.Shutdown(TimeSpan.FromSeconds(5))); + + Assert.Same(expected, actual); + Assert.True(source.IsShutdown); + } + + [Fact] + public async Task Registry_ConcurrentRequestsReturnSameWinningInstance() + { + const int participantCount = 3; + var queue = QueueId.GetQueueId("queue", 0, 0); + var factoryStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFactory = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var created = 0; + var registry = new QueueAdapterReceiverRegistry(_ => + { + Interlocked.Increment(ref created); + factoryStarted.TrySetResult(); + releaseFactory.Task.GetAwaiter().GetResult(); + return new TestCombinedReceiver(); + }); + + var requests = Enumerable.Range(0, participantCount) + .Select(_ => Task.Run(() => registry.GetOrCreate(queue))) + .ToArray(); + await factoryStarted.Task; + releaseFactory.SetResult(); + var results = await Task.WhenAll(requests); + + Assert.All(results, result => Assert.Same(results[0], result)); + Assert.Equal(1, created); + Assert.Same(results[0], Assert.Single(registry.Receivers).Value); + } + + [Fact] + public void Registry_FactoryFailureAllowsLaterCreationRetry() + { + var queue = QueueId.GetQueueId("queue", 0, 0); + var attempts = 0; + var registry = new QueueAdapterReceiverRegistry(_ => + { + if (Interlocked.Increment(ref attempts) == 1) + { + throw new InvalidOperationException("creation failed"); + } + + return new TestCombinedReceiver(); + }); + + Assert.Throws(() => registry.GetOrCreate(queue)); + Assert.Empty(registry.Receivers); + + var receiver = registry.GetOrCreate(queue); + + Assert.Equal(2, attempts); + Assert.Same(receiver, Assert.Single(registry.Receivers).Value); + } + + [Fact] + public void Registry_RemoveAllowsFreshReceiverForReassignedQueue() + { + var queue = QueueId.GetQueueId("queue", 0, 0); + var registry = new QueueAdapterReceiverRegistry( + _ => new TestCombinedReceiver()); + var first = registry.GetOrCreate(queue); + + Assert.True(registry.Remove(queue, first)); + var second = registry.GetOrCreate(queue); + + Assert.NotSame(first, second); + Assert.Same(second, Assert.Single(registry.Receivers).Value); + } + + private static RecoverableStreamReceiver CreateReceiver( + IRecoverableStreamSource source, + IStreamQueueCheckpointer checkpointer) + { + var adapter = new TestDataAdapter(); + var cache = new RecoverableStreamQueueCache( + 100, + new ObjectPool(() => new FixedSizeBuffer(4 * 1024)), + adapter, + new NoOpEvictionStrategy(), + NullLogger.Instance); + return new(source, adapter, cache, checkpointer, startFromNow: false); + } + + private sealed record TestQueueMessage(StreamId StreamId, long SequenceNumber, string Payload); + + private sealed class TestDataAdapter : IRecoverableStreamDataAdapter + { + public int CompareCallCount { get; private set; } + public int PositionCallCount { get; private set; } + public int FromQueueMessageCallCount { get; private set; } + public int GetBatchContainerCallCount { get; private set; } + public long? ThrowAfterPackingSequenceNumber { get; set; } + + public StreamPosition GetStreamPosition(TestQueueMessage queueMessage) + { + PositionCallCount++; + return new(queueMessage.StreamId, new EventSequenceTokenV2(queueMessage.SequenceNumber)); + } + + public CachedMessage FromQueueMessage( + StreamPosition streamPosition, + TestQueueMessage queueMessage, + DateTime dequeueTimeUtc, + Func> getSegment) + { + FromQueueMessageCallCount++; + var size = SegmentBuilder.CalculateAppendSize(queueMessage.SequenceNumber.ToString(CultureInfo.InvariantCulture)) + + SegmentBuilder.CalculateAppendSize(queueMessage.Payload); + var segment = getSegment(size); + var offset = 0; + SegmentBuilder.Append(segment, ref offset, queueMessage.SequenceNumber.ToString(CultureInfo.InvariantCulture)); + SegmentBuilder.Append(segment, ref offset, queueMessage.Payload); + if (queueMessage.SequenceNumber == ThrowAfterPackingSequenceNumber) + { + throw new InvalidOperationException("packing failed"); + } + + return new CachedMessage + { + StreamId = streamPosition.StreamId, + SequenceNumber = queueMessage.SequenceNumber, + EventIndex = streamPosition.SequenceToken.EventIndex, + EnqueueTimeUtc = dequeueTimeUtc, + DequeueTimeUtc = dequeueTimeUtc, + Segment = segment, + }; + } + + public IBatchContainer GetBatchContainer(ref CachedMessage cachedMessage) + { + GetBatchContainerCallCount++; + var offset = 0; + _ = SegmentBuilder.ReadNextString(cachedMessage.Segment, ref offset); + var payload = SegmentBuilder.ReadNextString(cachedMessage.Segment, ref offset); + return new TestBatchContainer( + cachedMessage.StreamId, + GetSequenceToken(ref cachedMessage), + payload!); + } + + public StreamSequenceToken GetSequenceToken(ref CachedMessage cachedMessage) + => new EventSequenceTokenV2(cachedMessage.SequenceNumber, cachedMessage.EventIndex); + + public int Compare(ref CachedMessage cachedMessage, StreamSequenceToken token) + { + CompareCallCount++; + return cachedMessage.SequenceNumber != token.SequenceNumber + ? cachedMessage.SequenceNumber.CompareTo(token.SequenceNumber) + : cachedMessage.EventIndex.CompareTo(token.EventIndex); + } + + public string GetOffset(ref CachedMessage cachedMessage) + { + var offset = 0; + return SegmentBuilder.ReadNextString(cachedMessage.Segment, ref offset)!; + } + + public bool TryGetOffset(StreamSequenceToken token, out string offset) + { + offset = token.SequenceNumber.ToString(CultureInfo.InvariantCulture); + return true; + } + } + + private sealed class ThrowingDataAdapter : IRecoverableStreamDataAdapter + { + public StreamPosition GetStreamPosition(TestQueueMessage queueMessage) + => new(queueMessage.StreamId, new EventSequenceTokenV2(queueMessage.SequenceNumber)); + + public CachedMessage FromQueueMessage( + StreamPosition streamPosition, + TestQueueMessage queueMessage, + DateTime dequeueTimeUtc, + Func> getSegment) + { + _ = getSegment(16); + throw new InvalidOperationException("packing failed"); + } + + public IBatchContainer GetBatchContainer(ref CachedMessage cachedMessage) + => throw new NotSupportedException(); + + public StreamSequenceToken GetSequenceToken(ref CachedMessage cachedMessage) + => new EventSequenceTokenV2(cachedMessage.SequenceNumber); + + public int Compare(ref CachedMessage cachedMessage, StreamSequenceToken token) + => cachedMessage.SequenceNumber.CompareTo(token.SequenceNumber); + + public string GetOffset(ref CachedMessage cachedMessage) + => cachedMessage.SequenceNumber.ToString(CultureInfo.InvariantCulture); + + public bool TryGetOffset(StreamSequenceToken token, out string offset) + { + offset = token.SequenceNumber.ToString(CultureInfo.InvariantCulture); + return true; + } + } + + private sealed class TestBatchContainer( + StreamId streamId, + StreamSequenceToken sequenceToken, + string payload) : IBatchContainer + { + public StreamId StreamId { get; } = streamId; + + public StreamSequenceToken SequenceToken { get; } = sequenceToken; + + public string Payload { get; } = payload; + + public IEnumerable> GetEvents() => []; + + public bool ImportRequestContext() => false; + } + + private sealed class TestSource( + IReadOnlyList messages, + int initializationFailures = 0) : IRecoverableStreamSource + { + private bool read; + private int remainingInitializationFailures = initializationFailures; + + public RecoverableStreamStartPosition StartPosition { get; private set; } + + public bool IsShutdown { get; private set; } + public int InitializeCount { get; private set; } + + public Task Initialize(RecoverableStreamStartPosition position, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + InitializeCount++; + if (remainingInitializationFailures-- > 0) + { + throw new InvalidOperationException("initialization failed"); + } + + StartPosition = position; + return Task.CompletedTask; + } + + public Task> Read(int maxCount, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (read) + { + return Task.FromResult>([]); + } + + read = true; + return Task.FromResult(messages); + } + + public Task Shutdown(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + IsShutdown = true; + return Task.CompletedTask; + } + } + + private sealed class TestCheckpointer(string checkpoint) : IStreamQueueCheckpointer + { + public bool CheckpointExists => !string.IsNullOrEmpty(checkpoint); + + public string? LastUpdatedCheckpoint { get; private set; } + + public int FlushCount { get; private set; } + public Exception? FlushException { get; init; } + + public Task Load() => Task.FromResult(checkpoint); + + public Task Load(CancellationToken cancellationToken) + => cancellationToken.IsCancellationRequested + ? Task.FromCanceled(cancellationToken) + : Task.FromResult(checkpoint); + + public void Update(string offset, DateTime utcNow) => LastUpdatedCheckpoint = offset; + + public void Update(string offset, DateTime utcNow, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + LastUpdatedCheckpoint = offset; + } + + public Task FlushAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + FlushCount++; + if (FlushException is not null) + { + return Task.FromException(FlushException); + } + + return Task.CompletedTask; + } + } + + private sealed class TestCheckpointStore(string checkpoint) : IStreamCheckpointStore + { + public StreamCheckpointStoreState State { get; private set; } = new(checkpoint, "1"); + + public ValueTask Load(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return ValueTask.FromResult(State); + } + + public ValueTask Update( + string checkpoint, + string expectedVersion, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Assert.Equal(State.Version, expectedVersion); + State = new(checkpoint, (int.Parse(State.Version) + 1).ToString(CultureInfo.InvariantCulture)); + return ValueTask.FromResult(State); + } + } + + private sealed class ReplaySource(IReadOnlyList messages) + : IRecoverableStreamSource + { + private long checkpoint; + private bool read; + + public Task Initialize(RecoverableStreamStartPosition position, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + checkpoint = string.IsNullOrEmpty(position.Checkpoint) + ? 0 + : long.Parse(position.Checkpoint, CultureInfo.InvariantCulture); + return Task.CompletedTask; + } + + public Task> Read(int maxCount, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (read) + { + return Task.FromResult>([]); + } + + read = true; + return Task.FromResult>( + messages.Where(message => message.SequenceNumber > checkpoint).Take(maxCount).ToList()); + } + + public Task Shutdown(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.CompletedTask; + } + } + + private sealed class NoOpEvictionStrategy : IEvictionStrategy + { + public int PerformPurgeCount { get; private set; } + + public IPurgeObservable PurgeObservable { private get; set; } = null!; + + public Action? OnPurged { get; set; } + + public void PerformPurge(DateTime utcNow) + { + PerformPurgeCount++; + } + + public void OnBlockAllocated(FixedSizeBuffer newBlock) + { + } + } + + private sealed class FixedFlowController(int maxAddCount) : IQueueFlowController + { + public int GetMaxAddCount() => maxAddCount; + } + + private sealed class TrackingBufferPool(int bufferSize = 4 * 1024) : IObjectPool + { + public int AllocateCount { get; private set; } + + public int FreeCount { get; private set; } + + public FixedSizeBuffer Allocate() + { + AllocateCount++; + return new(bufferSize) { Pool = this }; + } + + public void Free(FixedSizeBuffer resource) + { + FreeCount++; + } + } + + private sealed class TestCombinedReceiver : IQueueAdapterReceiver, IQueueCache + { + public Task Initialize(TimeSpan timeout) => Task.CompletedTask; + + public Task> GetQueueMessagesAsync(int maxCount) + => Task.FromResult>([]); + + public Task MessagesDeliveredAsync(IList messages) => Task.CompletedTask; + + public Task Shutdown(TimeSpan timeout) => Task.CompletedTask; + + public int GetMaxAddCount() => 1; + + public void AddToCache(IList messages) + { + } + + public bool TryPurgeFromCache(out IList purgedItems) + { + purgedItems = null!; + return false; + } + + public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken? token) + => throw new NotSupportedException(); + + public bool IsUnderPressure() => false; + } +}