Generic handler for distributed cache exceptions

src/OrchardCore/OrchardCore.Infrastructure/Documents/DocumentManager.cs

@@ -4,9 +4,11 @@
 using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Options;
+using OrchardCore.Caching;
 using OrchardCore.Data.Documents;
 using OrchardCore.Documents.Options;
 using OrchardCore.Environment.Shell.Scope;
+using OrchardCore.Modules;
 
 namespace OrchardCore.Documents
 {
@@ -25,14 +27,17 @@ namespace OrchardCore.Documents
         public DocumentManager(
             IDistributedCache distributedCache,
             IMemoryCache memoryCache,
-            IOptionsMonitor<DocumentOptions> options)
+            IOptionsMonitor<DocumentOptions> options,
+            IClock clock)
         {
             _distributedCache = distributedCache;
             _memoryCache = memoryCache;
             _options = options.Get(typeof(TDocument).FullName);
 
             if (!(_distributedCache is MemoryDistributedCache))
             {
+                //TODO: Discuss if this would be better handled with DI assuming someone can provide a clean services.AddDecorator(..) implementation 
+                _distributedCache = new DistributedCacheCircuitBreaker(_distributedCache, clock);
                 _isDistributed = true;
             }
 

src/OrchardCore/OrchardCore.Infrastructure/Documents/DocumentManagerOfT.cs

@@ -3,6 +3,7 @@
 using Microsoft.Extensions.Options;
 using OrchardCore.Data.Documents;
 using OrchardCore.Documents.Options;
+using OrchardCore.Modules;
 
 namespace OrchardCore.Documents
 {
@@ -15,8 +16,9 @@ public class DocumentManager<TDocumentStore, TDocument> : DocumentManager<TDocum
         public DocumentManager(
             IDistributedCache distributedCache,
             IMemoryCache memoryCache,
-            IOptionsMonitor<DocumentOptions> options)
-            : base(distributedCache, memoryCache, options)
+            IOptionsMonitor<DocumentOptions> options,
+            IClock clock)
+            : base(distributedCache, memoryCache, options, clock)
         {
             DocumentStoreServiceType = typeof(TDocumentStore);
         }

src/OrchardCore/OrchardCore.Infrastructure/Documents/VolatileDocumentManager.cs

@@ -6,6 +6,7 @@
 using OrchardCore.Data.Documents;
 using OrchardCore.Documents.Options;
 using OrchardCore.Locking.Distributed;
+using OrchardCore.Modules;
 
 namespace OrchardCore.Documents
 {
@@ -26,8 +27,9 @@ public VolatileDocumentManager(
             IDistributedCache distributedCache,
             IDistributedLock distributedLock,
             IMemoryCache memoryCache,
-            IOptionsMonitor<DocumentOptions> options)
-            : base(distributedCache, memoryCache, options)
+            IOptionsMonitor<DocumentOptions> options,
+            IClock clock)
+            : base(distributedCache, memoryCache, options, clock)
         {
             _isVolatile = true;
             _distributedLock = distributedLock;

src/OrchardCore/OrchardCore/Caching/Distributed/DistributedCacheCircuitBreaker.cs

@@ -0,0 +1,218 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Caching.Distributed;
+using OrchardCore.Modules;
+
+namespace OrchardCore.Caching
+{
+    /// <summary>
+    /// Wrapper to ensure the system stays available during Redis outages
+    /// </summary>
+    public class DistributedCacheCircuitBreaker : IDistributedCache
+    {
+        private readonly IDistributedCache _underlyingCache;
+        private readonly IClock _clock;
+
+        private static DateTime _retryAt;
+        private static bool _cacheUnavailable = false;
+        private static readonly object _cacheUnavailableLock = new();
+
+        public DistributedCacheCircuitBreaker(IDistributedCache cache, IClock clock)
+        {
+            _underlyingCache = cache;
+            _clock = clock;
+        }
+
+        public byte[] Get(string key)
+        {
+            CheckDistributedErrorRetry();
+
+            if (_cacheUnavailable)
+            {
+                return null;
+            }
+
+            try
+            {
+                return _underlyingCache.Get(key);
+            }
+            catch (Exception ex) when (!ex.IsFatal())
+            {
+                RedisCacheFailed();
+                return null;
+            }
+        }
+
+        public async Task<byte[]> GetAsync(string key, CancellationToken token = default)
+        {
+            CheckDistributedErrorRetry();
+
+            if (_cacheUnavailable)
+            {
+                return null;
+            }
+
+            try
+            {
+                return await _underlyingCache.GetAsync(key, token);
+            }
+            catch (Exception ex) when (!ex.IsFatal())
+            {
+                RedisCacheFailed();
+
+                return null;
+            }
+        }
+
+        public void Refresh(string key)
+        {
+            CheckDistributedErrorRetry();
+
+            if (_cacheUnavailable)
+            {
+                return;
+            }
+
+            try
+            {
+                _underlyingCache.Refresh(key);
+            }
+            catch (Exception ex) when (!ex.IsFatal())
+            {
+                RedisCacheFailed();
+            }
+        }
+
+        public async Task RefreshAsync(string key, CancellationToken token = default)
+        {
+            CheckDistributedErrorRetry();
+
+            if (_cacheUnavailable)
+            {
+                return;
+            }
+
+            try
+            {
+                await _underlyingCache.RefreshAsync(key, token);
+            }
+            catch (Exception ex) when (!ex.IsFatal())
+            {
+                RedisCacheFailed();
+            }
+        }
+
+        public void Remove(string key)
+        {
+            CheckDistributedErrorRetry();
+
+            if (_cacheUnavailable)
+            {
+                return;
+            }
+
+            try
+            {
+                _underlyingCache.Remove(key);
+            }
+            catch (Exception ex) when (!ex.IsFatal())
+            {
+                RedisCacheFailed();
+            }
+        }
+
+        public async Task RemoveAsync(string key, CancellationToken token = default)
+        {
+            CheckDistributedErrorRetry();
+
+            if (_cacheUnavailable)
+            {
+                return;
+            }
+
+            try
+            {
+                await _underlyingCache.RemoveAsync(key, token);
+            }
+            catch (Exception ex) when (!ex.IsFatal())
+            {
+                RedisCacheFailed();
+            }
+        }
+
+        public void Set(string key, byte[] value, DistributedCacheEntryOptions options)
+        {
+            CheckDistributedErrorRetry();
+
+            if (_cacheUnavailable)
+            {
+                return;
+            }
+
+            try
+            {
+                _underlyingCache.Set(key, value, options);
+            }
+            catch (Exception ex) when (!ex.IsFatal())
+            {
+                RedisCacheFailed();
+            }
+        }
+
+        public async Task SetAsync(string key, byte[] value, DistributedCacheEntryOptions options, CancellationToken token = default)
+        {
+            CheckDistributedErrorRetry();
+
+            if (_cacheUnavailable)
+            {
+                return;
+            }
+
+            try
+            {
+                await _underlyingCache.SetAsync(key, value, options, token);
+            }
+            catch (Exception ex) when (!ex.IsFatal())
+            {
+                RedisCacheFailed();
+            }
+        }
+
+        private void CheckDistributedErrorRetry()
+        {
+            if (!_cacheUnavailable)
+            {
+                return;
+            }
+
+            lock (_cacheUnavailableLock)
+            {
+                // Recheck value as it may have been changed before we could lock
+                if (_cacheUnavailable && _clock.UtcNow > _retryAt)
+                {
+                    _cacheUnavailable = false;
+                }
+            }
+        }
+        private void RedisCacheFailed()
+        {
+            if (_cacheUnavailable)
+            {
+                return;
+            }
+
+            lock (_cacheUnavailableLock)
+            {
+                if (_cacheUnavailable)
+                {
+                    return;
+                }
+
+                // TODO: move to configuration or should this be an increasing number like RedisLock?
+                _retryAt = _clock.UtcNow.AddSeconds(15);
+                _cacheUnavailable = true;
+            }
+        }
+    }
+}