feat: implement lock lost handling and exceptions

LitRedis.Core/Exceptions/AcquireLockFailedException.cs

@@ -4,6 +4,6 @@ namespace LitRedis.Core.Exceptions;
 
 public class AcquireLockFailedException : Exception
 {
-    public AcquireLockFailedException(TimeSpan waitTime)
-        : base($"Failed to acquire lock after waiting {waitTime}") { }
+    public AcquireLockFailedException(string key, TimeSpan waitTime)
+        : base($"Failed to acquire lock for key '{key}' after waiting {waitTime}") { }
 }

LitRedis.Core/Exceptions/LockLostException.cs

@@ -0,0 +1,14 @@
+using System;
+
+namespace LitRedis.Core.Exceptions;
+
+public class LockLostException : Exception
+{
+    public LockLostException()
+        : base("The distributed lock was lost before the operation completed.") { }
+
+    public LockLostException(string key)
+        : base($"The distributed lock for key '{key}' was lost before the operation completed.") { }
+
+    public LockLostException(string message, Exception innerException) : base(message, innerException) { }
+}

LitRedis.Core/Implementations/LitRedisDistributedLockService.cs

@@ -35,44 +35,82 @@ public async Task<LitRedisDistributedLockModel> AcquireLockAsync(
 
             if (await _litRedisDistributedLock.TakeLockAsync(lockKey, token, requestLockModel.LockIncrease, cancellationToken))
             {
-                var stopped = false;
+                var lockLostCts = new CancellationTokenSource();
+                var stopped = 0;
 
-                _ = Task.Run(async () =>
-                {
-                    while (!stopped && !cancellationToken.IsCancellationRequested)
+                LitRedisDistributedLockModel model = null;
+
+                model = LitRedisDistributedLockModel.Success(
+                    async () =>
                     {
                         try
                         {
-                            await _litRedisDistributedLock.ExtendLockAsync(lockKey, token, requestLockModel.LockIncrease, cancellationToken);
+                            // stop the keep alive thread and release the lock
+                            Interlocked.Exchange(ref stopped, 1);
+
+                            if (!model.IsLost)
+                            {
+                                await _litRedisDistributedLock.ReleaseLockAsync(lockKey, token, CancellationToken.None);
+                            }
+
                         }
                         catch (Exception ex)
                         {
-                            _logger.LogError(ex, "Extended Distributed Lock Error");
+                            _logger.LogError(ex, "Error releasing lock");
                         }
+                    },
+                    lockLostCts,
+                    lockKey
+                );
+
+                _ = Task.Run(async () =>
+                {
+                    var consecutiveFailures = 0;
 
+                    while (Volatile.Read(ref stopped) == 0 && !cancellationToken.IsCancellationRequested)
+                    {
                         try
                         {
                             await Task.Delay(requestLockModel.RenewLockInterval, cancellationToken);
                         }
                         catch (TaskCanceledException) { }
-                    }
-                }, cancellationToken);
 
-                return LitRedisDistributedLockModel.Success(
-                    async () =>
-                    {
+                        if (Volatile.Read(ref stopped) != 0 || cancellationToken.IsCancellationRequested)
+                        {
+                            break;
+                        }
+
                         try
                         {
-                            //stop the keep alive thread and release the lock
-                            stopped = true;
-                            await _litRedisDistributedLock.ReleaseLockAsync(lockKey, token, cancellationToken);
+                            var extended = await _litRedisDistributedLock.ExtendLockAsync(lockKey, token, requestLockModel.LockIncrease, cancellationToken);
+                            if (!extended)
+                            {
+                                _logger.LogWarning("Failed to extend distributed lock for key {LockKey}. Lock may have been lost.", lockKey);
+                                model.MarkLost();
+                                break;
+                            }
+
+                            consecutiveFailures = 0;
+                        }
+                        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+                        {
+                            break;
                         }
                         catch (Exception ex)
                         {
-                            _logger.LogError(ex, "Error releasing lock");
+                            consecutiveFailures++;
+                            _logger.LogError(ex, "Extended Distributed Lock Error ({ConsecutiveFailures}/{MaxRetries})", consecutiveFailures, requestLockModel.MaxExtendRetries);
+
+                            if (consecutiveFailures >= requestLockModel.MaxExtendRetries)
+                            {
+                                model.MarkLost();
+                                break;
+                            }
                         }
                     }
-                );
+                }, cancellationToken);
+
+                return model;
             }
 
             try

LitRedis.Core/Models/LitRedisDistributedLockModel.cs

@@ -1,12 +1,15 @@
 using System;
+using System.Threading;
 using System.Threading.Tasks;
 using LitRedis.Core.Exceptions;
 
 namespace LitRedis.Core.Models;
 
 public class LitRedisDistributedLockModel : IDisposable, IAsyncDisposable
 {
-    private bool _disposed;
+    private int _disposed;
+    private int _lostFlag;
+    private CancellationTokenSource _lockLostCts;
 
     private Func<Task> ReleaseAction { get; set; }
 
@@ -16,19 +19,70 @@ public class LitRedisDistributedLockModel : IDisposable, IAsyncDisposable
 
     public TimeSpan? WaitTime { get; set; }
 
-    public LitRedisDistributedLockModel(bool succeeded, Func<Task> releaseAction, bool wasCancelled, TimeSpan? waitTime)
+    /// <summary>
+    /// Gets the current status of the distributed lock.
+    /// </summary>
+    public LitRedisLockStatus Status { get; internal set; } = LitRedisLockStatus.NotAcquired;
+
+    /// <summary>
+    /// Returns true if the lock has been lost. Thread-safe.
+    /// </summary>
+    public bool IsLost => Volatile.Read(ref _lostFlag) == 1;
+
+    /// <summary>
+    /// Cancelled if the background renewal loop fails to extend the lock, indicating the lock may have been lost.
+    /// </summary>
+    public CancellationToken LockLostToken => _lockLostCts?.Token ?? CancellationToken.None;
+
+    /// <summary>
+    /// Gets the key that was locked.
+    /// </summary>
+    public string Key { get; }
+
+    public LitRedisDistributedLockModel(bool succeeded, Func<Task> releaseAction, bool wasCancelled, TimeSpan? waitTime, CancellationTokenSource lockLostCts = null, string key = null)
     {
         Succeeded = succeeded;
         ReleaseAction = releaseAction;
         WasCancelled = wasCancelled;
         WaitTime = waitTime;
+        _lockLostCts = lockLostCts;
+        Key = key;
     }
 
-    public static LitRedisDistributedLockModel Success(Func<Task> releaseAction = null)
-        => new(true, releaseAction, false, null);
+    public static LitRedisDistributedLockModel Success(Func<Task> releaseAction = null, CancellationTokenSource lockLostCts = null, string key = null)
+        => new(true, releaseAction, false, null, lockLostCts, key)
+        {
+            Status = LitRedisLockStatus.Acquired,
+        };
 
     public static LitRedisDistributedLockModel Failure(bool wasCancelled = false, TimeSpan? waitTime = null)
-        => new(false, null, wasCancelled, waitTime);
+        => new(false, null, wasCancelled, waitTime) { Status = LitRedisLockStatus.NotAcquired };
+
+    /// <summary>
+    /// Marks the lock as lost and signals <see cref="LockLostToken"/>.
+    /// </summary>
+    internal void MarkLost()
+    {
+        if (Interlocked.CompareExchange(ref _lostFlag, 1, 0) != 0)
+        {
+            return;
+        }
+
+        Status = LitRedisLockStatus.Lost;
+
+        // Signal the cancellation token to notify callers that the lock was lost.
+        // Do not null out the field here so callers can still observe the cancelled
+        // token via the LockLostToken property. Disposal will occur in DisposeAsync.
+        var cts = Volatile.Read(ref _lockLostCts);
+        try
+        {
+            cts?.Cancel();
+        }
+        catch (ObjectDisposedException)
+        {
+            // CTS was already disposed; nothing to signal.
+        }
+    }
 
     private async Task ReleaseAsync()
     {
@@ -42,21 +96,33 @@ private async Task ReleaseAsync()
 
     public async ValueTask DisposeAsync()
     {
-        if (_disposed)
+        if (Interlocked.CompareExchange(ref _disposed, 1, 0) != 0)
         {
             return;
         }
 
         await ReleaseAsync();
 
-        _disposed = true;
+        var cts = Interlocked.Exchange(ref _lockLostCts, null);
+        cts?.Dispose();
     }
 
     public void ThrowIfFailedToAcquire()
     {
         if (!Succeeded)
         {
-            throw new AcquireLockFailedException(WaitTime.GetValueOrDefault());
+            throw new AcquireLockFailedException(Key, WaitTime.GetValueOrDefault());
+        }
+    }
+
+    /// <summary>
+    /// Throws a <see cref="LockLostException"/> if the background renewal loop has reported that the lock was lost.
+    /// </summary>
+    public void ThrowOnLockLost()
+    {
+        if (IsLost)
+        {
+            throw new LockLostException(Key);
         }
     }
 

LitRedis.Core/Models/LitRedisLockStatus.cs

@@ -0,0 +1,8 @@
+namespace LitRedis.Core.Models;
+
+public enum LitRedisLockStatus
+{
+    NotAcquired = 0,
+    Acquired = 1,
+    Lost = 2,
+}

LitRedis.Core/Models/RequestLockModel.cs

@@ -22,13 +22,30 @@ public class RequestLockModel
     /// <summary>
     /// Gets or sets a value indicating how long to extend the lock each time it is renewed.
     /// </summary>
-    public TimeSpan LockIncrease { get; set; } = TimeSpan.FromSeconds(10);
+    public TimeSpan LockIncrease { get; set; } = TimeSpan.FromSeconds(30);
 
     /// <summary>
-    ///  Sets a value indicating how long to extend the lock each time it is renewed.
+    /// Gets or sets the maximum number of consecutive exceptions from <c>ExtendLockAsync</c>
+    /// before the lock is considered lost. Defaults to 3. Set to 0 to mark lost on the first exception.
+    /// </summary>
+    public int MaxExtendRetries { get; set; } = 3;
+
+    /// <summary>
+    /// Sets the maximum number of consecutive exceptions before the lock is considered lost.
+    /// </summary>
+    /// <param name="maxExtendRetries">Maximum consecutive exceptions allowed before marking the lock lost.</param>
+    /// <returns>The request lock model</returns>
+    public RequestLockModel WithMaxExtendRetries(int maxExtendRetries)
+    {
+        MaxExtendRetries = maxExtendRetries;
+        return this;
+    }
+
+    /// <summary>
+    /// Sets a value indicating how long to extend the lock each time it is renewed.
     /// </summary>
     /// <param name="lockIncrease">How long to increase the lock for.</param>
-    /// <returns></returns>
+    /// <returns>The request lock model</returns>
     public RequestLockModel WithLockIncrease(TimeSpan lockIncrease)
     {
         LockIncrease = lockIncrease;

LitRedis.Sample/SampleHostedService.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Threading;
 using System.Threading.Tasks;
+using LitRedis.Core.Exceptions;
 using LitRedis.Core.Interfaces;
 using LitRedis.Core.Models;
 using Microsoft.Extensions.Caching.Distributed;
@@ -29,6 +30,8 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
 
             await DoLockNoWaitSample(stoppingToken);
 
+            await DoLockLostSample(stoppingToken);
+
             while (!stoppingToken.IsCancellationRequested)
             {
                 var tasks = new[]
@@ -60,6 +63,7 @@ private async Task DoLockNoWaitSample(CancellationToken stoppingToken)
                 }
             }, stoppingToken);
 
+            await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken);
 
             var dontGetIt = Task.Run(async () =>
             {
@@ -84,6 +88,55 @@ private async Task DoLockNoWaitSample(CancellationToken stoppingToken)
             await Task.WhenAll(getIt, dontGetIt);
         }
 
+        private async Task DoLockLostSample(CancellationToken stoppingToken)
+        {
+            try
+            {
+                var myId = Guid.NewGuid();
+
+                // We will always lose the lock if the renewal interval is longer than the lock increase
+                var model = RequestLockModel
+                    .WithKey("lit-sample-lock-lost")
+                    .WithLockIncrease(TimeSpan.FromSeconds(2))
+                    .WithRenewLockInterval(TimeSpan.FromSeconds(5));
+
+                await using var locker = await redisDistributedLockService.AcquireLockAsync(model, stoppingToken);
+
+                if (!locker.Succeeded)
+                {
+                    logger.LogInformation("Never got the lock {@Id}", myId);
+                    return;
+                }
+
+                logger.LogInformation("Got the lock, status is {@Status} {@Id}", locker.Status, myId);
+
+                // Option 1: link the LockLostToken into your work so it cancels if the lock is lost
+                using var workCts = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken, locker.LockLostToken);
+
+                try
+                {
+                    await Task.Delay(TimeSpan.FromSeconds(10), workCts.Token);
+                }
+                catch (OperationCanceledException) when (locker.Status == LitRedisLockStatus.Lost)
+                {
+                    logger.LogWarning("Work was cancelled because the lock was lost {@Id}", myId);
+                }
+
+                // Option 2: poll explicitly when reaching a critical section
+                locker.ThrowOnLockLost();
+
+                logger.LogInformation("Finished work while still holding the lock {@Id}", myId);
+            }
+            catch (LockLostException ex)
+            {
+                logger.LogCritical(ex, "Lock was lost mid-operation");
+            }
+            catch (Exception ex)
+            {
+                logger.LogCritical(ex, "Error running lock lost sample");
+            }
+        }
+
         private async Task DoLockSample(CancellationToken stoppingToken)
         {
             try