Cache Failover and Shared options (#10338)

src/OrchardCore.Modules/OrchardCore.DynamicCache/Services/DefaultDynamicCacheService.cs

@@ -5,7 +5,9 @@
 using System.Text;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Caching.Distributed;
+using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using OrchardCore.Abstractions.Pooling;
 using OrchardCore.DynamicCache.Models;
@@ -15,11 +17,17 @@ namespace OrchardCore.DynamicCache.Services
 {
     public class DefaultDynamicCacheService : IDynamicCacheService
     {
+        public const string FailoverKey = "OrchardCore_DynamicCache_FailoverKey";
+        public static TimeSpan DefaultFailoverRetryLatency = TimeSpan.FromSeconds(30);
+
         private readonly PoolingJsonSerializer _serializer;
         private readonly ICacheContextManager _cacheContextManager;
         private readonly IDynamicCache _dynamicCache;
+        private readonly IMemoryCache _memoryCache;
         private readonly IServiceProvider _serviceProvider;
+        private readonly DynamicCacheOptions _dynamicCacheOptions;
         private readonly CacheOptions _cacheOptions;
+        private readonly ILogger _logger;
 
         private readonly Dictionary<string, string> _localCache = new Dictionary<string, string>();
         private ITagCache _tagcache;
@@ -28,14 +36,21 @@ public DefaultDynamicCacheService(
             ArrayPool<char> _arrayPool,
             ICacheContextManager cacheContextManager,
             IDynamicCache dynamicCache,
+            IMemoryCache memoryCache,
             IServiceProvider serviceProvider,
-            IOptions<CacheOptions> options)
+            IOptions<DynamicCacheOptions> dynamicCacheOptions,
+            IOptions<CacheOptions> options,
+            ILogger<DefaultDynamicCacheService> logger)
         {
             _serializer = new PoolingJsonSerializer(_arrayPool);
             _cacheContextManager = cacheContextManager;
             _dynamicCache = dynamicCache;
+            _memoryCache = memoryCache;
             _serviceProvider = serviceProvider;
+            _dynamicCacheOptions = dynamicCacheOptions.Value;
+            _dynamicCacheOptions.FailoverRetryLatency ??= DefaultFailoverRetryLatency;
             _cacheOptions = options.Value;
+            _logger = logger;
         }
 
         public async Task<string> GetCachedValueAsync(CacheContext context)
@@ -84,6 +99,12 @@ public Task TagRemovedAsync(string tag, IEnumerable<string> keys)
 
         private async Task SetCachedValueAsync(string cacheKey, string value, CacheContext context)
         {
+            var failover = _memoryCache.Get<bool>(FailoverKey);
+            if (failover)
+            {
+                return;
+            }
+
             var bytes = Encoding.UTF8.GetBytes(value);
 
             var options = new DistributedCacheEntryOptions
@@ -99,7 +120,21 @@ private async Task SetCachedValueAsync(string cacheKey, string value, CacheConte
                 options.SlidingExpiration = new TimeSpan(0, 1, 0);
             }
 
-            await _dynamicCache.SetAsync(cacheKey, bytes, options);
+            try
+            {
+                await _dynamicCache.SetAsync(cacheKey, bytes, options);
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(e, "Failed to write the '{CacheKey}' to the dynamic cache", cacheKey);
+
+                _memoryCache.Set(FailoverKey, true, new MemoryCacheEntryOptions()
+                {
+                    AbsoluteExpirationRelativeToNow = _dynamicCacheOptions.FailoverRetryLatency
+                });
+
+                return;
+            }
 
             // Lazy load to prevent cyclic dependency
             _tagcache ??= _serviceProvider.GetRequiredService<ITagCache>();
@@ -133,13 +168,33 @@ private async Task<string> GetCachedStringAsync(string cacheKey)
                 return content;
             }
 
-            var bytes = await _dynamicCache.GetAsync(cacheKey);
-            if (bytes == null)
+            var failover = _memoryCache.Get<bool>(FailoverKey);
+            if (failover)
             {
                 return null;
             }
 
-            return Encoding.UTF8.GetString(bytes);
+            try
+            {
+                var bytes = await _dynamicCache.GetAsync(cacheKey);
+                if (bytes == null)
+                {
+                    return null;
+                }
+
+                return Encoding.UTF8.GetString(bytes);
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(e, "Failed to read the '{CacheKey}' from the dynamic cache", cacheKey);
+
+                _memoryCache.Set(FailoverKey, true, new MemoryCacheEntryOptions()
+                {
+                    AbsoluteExpirationRelativeToNow = _dynamicCacheOptions.FailoverRetryLatency
+                });
+            }
+
+            return null;
         }
 
         private async Task<CacheContext> GetCachedContextAsync(string cacheKey)

src/OrchardCore.Modules/OrchardCore.DynamicCache/Startup.cs

@@ -6,6 +6,7 @@
 using OrchardCore.DynamicCache.Services;
 using OrchardCore.DynamicCache.TagHelpers;
 using OrchardCore.Environment.Cache;
+using OrchardCore.Environment.Shell.Configuration;
 using OrchardCore.Modules;
 
 namespace OrchardCore.DynamicCache
@@ -15,6 +16,13 @@ namespace OrchardCore.DynamicCache
     /// </summary>
     public class Startup : StartupBase
     {
+        private readonly IShellConfiguration _shellConfiguration;
+
+        public Startup(IShellConfiguration shellConfiguration)
+        {
+            _shellConfiguration = shellConfiguration;
+        }
+
         public override void ConfigureServices(IServiceCollection services)
         {
             services.AddScoped<IDynamicCacheService, DefaultDynamicCacheService>();
@@ -27,7 +35,9 @@ public override void ConfigureServices(IServiceCollection services)
             services.AddSingleton<DynamicCacheTagHelperService>();
             services.AddTagHelpers<DynamicCacheTagHelper>();
             services.AddTagHelpers<CacheDependencyTagHelper>();
+
             services.AddTransient<IConfigureOptions<CacheOptions>, CacheOptionsConfiguration>();
+            services.Configure<DynamicCacheOptions>(_shellConfiguration.GetSection("OrchardCore_DynamicCache"));
         }
     }
 }

src/OrchardCore.Modules/OrchardCore.Redis/Services/RedisBus.cs

@@ -31,6 +31,11 @@ public async Task SubscribeAsync(string channel, Action<string, string> handler)
             if (_redis.Connection == null)
             {
                 await _redis.ConnectAsync();
+                if (_redis.Connection == null)
+                {
+                    _logger.LogError("Unable to subscribe to the channel '{ChannelName}'.", _channelPrefix + channel);
+                    return;
+                }
             }
 
             try
@@ -51,7 +56,7 @@ await subscriber.SubscribeAsync(_channelPrefix + channel, (redisChannel, redisVa
             }
             catch (Exception e)
             {
-                _logger.LogError(e, "'Unable to subscribe to the channel {ChannelName}'.", _channelPrefix + channel);
+                _logger.LogError(e, "Unable to subscribe to the channel '{ChannelName}'.", _channelPrefix + channel);
             }
         }
 
@@ -60,6 +65,11 @@ public async Task PublishAsync(string channel, string message)
             if (_redis.Connection == null)
             {
                 await _redis.ConnectAsync();
+                if (_redis.Connection == null)
+                {
+                    _logger.LogError("Unable to publish to the channel '{ChannelName}'.", _channelPrefix + channel);
+                    return;
+                }
             }
 
             try
@@ -68,7 +78,7 @@ public async Task PublishAsync(string channel, string message)
             }
             catch (Exception e)
             {
-                _logger.LogError(e, "'Unable to publish to the channel {ChannelName}'.", _channelPrefix + channel);
+                _logger.LogError(e, "Unable to publish to the channel '{ChannelName}'.", _channelPrefix + channel);
             }
         }
     }

src/OrchardCore.Modules/OrchardCore.Redis/Services/RedisLock.cs

@@ -81,6 +81,11 @@ public async Task<bool> IsLockAcquiredAsync(string key)
             if (_redis.Database == null)
             {
                 await _redis.ConnectAsync();
+                if (_redis.Database == null)
+                {
+                    _logger.LogError("Fails to check whether the named lock '{LockName}' is already acquired.", _prefix + key);
+                    return false;
+                }
             }
 
             try
@@ -100,6 +105,11 @@ private async Task<bool> LockAsync(string key, TimeSpan expiry)
             if (_redis.Database == null)
             {
                 await _redis.ConnectAsync();
+                if (_redis.Database == null)
+                {
+                    _logger.LogError("Fails to acquire the named lock '{LockName}'.", _prefix + key);
+                    return false;
+                }
             }
 
             try
@@ -116,11 +126,6 @@ private async Task<bool> LockAsync(string key, TimeSpan expiry)
 
         private async ValueTask ReleaseAsync(string key)
         {
-            if (_redis.Database == null)
-            {
-                await _redis.ConnectAsync();
-            }
-
             try
             {
                 await _redis.Database.LockReleaseAsync(_prefix + key, _hostName);

src/OrchardCore.Modules/OrchardCore.Redis/Services/RedisTagCache.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
@@ -32,11 +33,23 @@ public async Task TagAsync(string key, params string[] tags)
             if (_redis.Database == null)
             {
                 await _redis.ConnectAsync();
+                if (_redis.Database == null)
+                {
+                    _logger.LogError("Fails to add the '{KeyName}' to the {PrefixName} tags.", key, _prefix);
+                    return;
+                }
             }
 
-            foreach (var tag in tags)
+            try
             {
-                await _redis.Database.SetAddAsync(_prefix + tag, key);
+                foreach (var tag in tags)
+                {
+                    await _redis.Database.SetAddAsync(_prefix + tag, key);
+                }
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(e, "Fails to add the '{KeyName}' to the {PrefixName} tags.", key, _prefix);
             }
         }
 
@@ -45,37 +58,63 @@ public async Task<IEnumerable<string>> GetTaggedItemsAsync(string tag)
             if (_redis.Database == null)
             {
                 await _redis.ConnectAsync();
+                if (_redis.Database == null)
+                {
+                    _logger.LogError("Fails to get '{TagName}' items.", _prefix + tag);
+                    return Enumerable.Empty<string>();
+                }
             }
 
-            var values = await _redis.Database.SetMembersAsync(_prefix + tag);
+            try
+            {
+                var values = await _redis.Database.SetMembersAsync(_prefix + tag);
+
+                if (values == null || values.Length == 0)
+                {
+                    return Enumerable.Empty<string>();
+                }
 
-            if (values == null || values.Length == 0)
+                return values.Select(v => (string)v).ToArray();
+            }
+            catch (Exception e)
             {
-                return Enumerable.Empty<string>();
+                _logger.LogError(e, "Fails to get '{TagName}' items.", _prefix + tag);
             }
 
-            return values.Select(v => (string)v).ToArray();
+            return Enumerable.Empty<string>();
         }
 
         public async Task RemoveTagAsync(string tag)
         {
             if (_redis.Database == null)
             {
                 await _redis.ConnectAsync();
+                if (_redis.Database == null)
+                {
+                    _logger.LogError("Fails to remove the '{TagName}'.", _prefix + tag);
+                    return;
+                }
             }
 
-            var values = await _redis.Database.SetMembersAsync(_prefix + tag);
-
-            if (values == null || values.Length == 0)
+            try
             {
-                return;
-            }
+                var values = await _redis.Database.SetMembersAsync(_prefix + tag);
+
+                if (values == null || values.Length == 0)
+                {
+                    return;
+                }
 
-            var set = values.Select(v => (string)v).ToArray();
+                var set = values.Select(v => (string)v).ToArray();
 
-            await _redis.Database.KeyDeleteAsync(_prefix + tag);
+                await _redis.Database.KeyDeleteAsync(_prefix + tag);
 
-            await _tagRemovedEventHandlers.InvokeAsync(x => x.TagRemovedAsync(tag, set), _logger);
+                await _tagRemovedEventHandlers.InvokeAsync(x => x.TagRemovedAsync(tag, set), _logger);
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(e, "Fails to remove the '{TagName}'.", _prefix + tag);
+            }
         }
     }
 }

src/OrchardCore/OrchardCore.Data/Documents/FileDocumentStore.cs

@@ -67,9 +67,7 @@ public FileDocumentStore(IOptions<ShellOptions> shellOptions, ShellSettings shel
         /// <inheritdoc />
         public Task UpdateAsync<T>(T document, Func<T, Task> updateCache, bool checkConcurrency = false)
         {
-            var documentStore = ShellScope.Services.GetRequiredService<IDocumentStore>();
-
-            documentStore.AfterCommitSuccess<T>(async () =>
+            DocumentStore.AfterCommitSuccess<T>(async () =>
             {
                 await SaveDocumentAsync(document);
                 ShellScope.Set(typeof(T), null);
@@ -79,9 +77,9 @@ public Task UpdateAsync<T>(T document, Func<T, Task> updateCache, bool checkConc
             return Task.CompletedTask;
         }
 
-        public Task CancelAsync() => throw new NotImplementedException();
-        public void AfterCommitSuccess<T>(DocumentStoreCommitSuccessDelegate afterCommitSuccess) => throw new NotImplementedException();
-        public void AfterCommitFailure<T>(DocumentStoreCommitFailureDelegate afterCommitFailure) => throw new NotImplementedException();
+        public Task CancelAsync() => DocumentStore.CancelAsync();
+        public void AfterCommitSuccess<T>(DocumentStoreCommitSuccessDelegate afterCommitSuccess) => DocumentStore.AfterCommitSuccess<T>(afterCommitSuccess);
+        public void AfterCommitFailure<T>(DocumentStoreCommitFailureDelegate afterCommitFailure) => DocumentStore.AfterCommitFailure<T>(afterCommitFailure);
         public Task CommitAsync() => throw new NotImplementedException();
 
         private async Task<T> GetDocumentAsync<T>()
@@ -146,5 +144,7 @@ private async Task SaveDocumentAsync<T>(T document)
                 _semaphore.Release();
             }
         }
+
+        private static IDocumentStore DocumentStore => ShellScope.Services.GetRequiredService<IDocumentStore>();
     }
 }

src/OrchardCore/OrchardCore.DynamicCache.Abstractions/DynamicCacheOptions.cs

@@ -0,0 +1,9 @@
+using System;
+
+namespace OrchardCore.DynamicCache
+{
+    public class DynamicCacheOptions
+    {
+        public TimeSpan? FailoverRetryLatency { get; set; }
+    }
+}

src/OrchardCore/OrchardCore.Infrastructure.Abstractions/Documents/Options/DocumentNamedOptions.cs

@@ -0,0 +1,14 @@
+namespace OrchardCore.Documents.Options
+{
+    public interface IDocumentNamedOptions
+    {
+        string CacheKey { get; set; }
+        string CacheIdKey { get; set; }
+    }
+
+    public class DocumentNamedOptions : DocumentOptionsBase, IDocumentNamedOptions
+    {
+        public string CacheKey { get; set; }
+        public string CacheIdKey { get; set; }
+    }
+}